def versions(self):
""" NyFile versions
"""
extfile = self.data.pop('_ext_file')
versions = self.data.pop('versions', [])
if versions:
versions = versions.objectValues()
for version in versions:
if version.is_broken():
continue
sfile = StringIO(version.data)
self.logger.debug('\t VERSION FILENAME: %s',
'/'.join(version.filename))
sfile.filename = version.filename[-1]
sfile.headers = {'content-type': version.content_type}
yield sfile
if not extfile.is_broken():
sfile = StringIO(extfile.data)
self.logger.debug('\t FILENAME: %s', '/'.join(extfile.filename))
sfile.filename = extfile.filename[-1]
sfile.headers = {'content-type': extfile.content_type}
yield sfile
def versions(self):
""" Exports NyExFile versions
"""
languages = self.data.get('_languages', ())
if len(languages) > 1:
self.logger.debug("\t Object %s has %r languages: %r",
self.data.get('id', 'ERROR'), len(languages),
languages)
if not languages:
languages = [x['id'] for x in self.data['_objects']]
for language in languages:
nyfile = self.data.pop(language, None)
if not nyfile:
continue
extfile = getattr(nyfile, '_ext_file', None)
if not extfile:
continue
versions = getattr(nyfile, 'versions', [])
if versions:
versions = versions.objectValues()
for version in versions:
filename = '/'.join(version.filename)
if not filename:
continue
if version.is_broken():
self.logger.warn('\t BROKEN VERSION: %s (%s)',
version.getId(), filename)
continue
sfile = StringIO(version.data)
self.logger.debug('\t VERSION FILENAME: %s', filename)
sfile.filename = version.filename[-1]
sfile.headers = {'content-type': version.content_type}
sfile.releasedate = self.data['releasedate']
yield sfile
filename = '/'.join(extfile.filename)
if not filename:
continue
if extfile.is_broken():
self.logger.warn('\t BROKEN EXTFILE: %s (%s)', language,
filename)
continue
sfile = StringIO(extfile.data)
self.logger.debug('\t FILENAME: %s', filename)
sfile.filename = extfile.filename[-1]
sfile.headers = {'content-type': extfile.content_type}
sfile.releasedate = self.data['releasedate']
yield sfile
def versions(self):
""" Exports NyExFile versions
"""
languages = self.data.get('_languages', ())
if len(languages) > 1:
self.logger.debug("\t Object %s has %r languages: %r",
self.data.get('id', 'ERROR'), len(languages),
languages)
if not languages:
languages = [x['id'] for x in self.data['_objects']]
for language in languages:
nyfile = self.data.pop(language, None)
if not nyfile:
continue
extfile = getattr(nyfile, '_ext_file', None)
if not extfile:
continue
versions = getattr(nyfile, 'versions', [])
if versions:
versions = versions.objectValues()
for version in versions:
filename = '/'.join(version.filename)
if not filename:
continue
if version.is_broken():
self.logger.warn('\t BROKEN VERSION: %s (%s)',
version.getId(), filename)
continue
sfile = StringIO(version.data)
self.logger.debug('\t VERSION FILENAME: %s', filename)
sfile.filename = version.filename[-1]
sfile.headers = {'content-type': version.content_type}
sfile.releasedate = self.data['releasedate']
yield sfile
filename = '/'.join(extfile.filename)
if not filename:
continue
if extfile.is_broken():
self.logger.warn('\t BROKEN EXTFILE: %s (%s)', language,
filename)
continue
sfile = StringIO(extfile.data)
self.logger.debug('\t FILENAME: %s', filename)
sfile.filename = extfile.filename[-1]
sfile.headers = {'content-type': extfile.content_type}
sfile.releasedate = self.data['releasedate']
yield sfile
def test_add_with_file(self):
myfile = StringIO('hello data!')
myfile.filename = 'my.jpg'
myfile.headers = {'content-type': 'image/jpeg'}
tzinfo = self.portal.get_tzinfo()
now_pre = datetime.now(tzinfo)
self.add_bfile(id='mybfile', title='My bfile', uploaded_file=myfile)
now_post = datetime.now(tzinfo)
mybfile = self.portal.myfolder.mybfile
self.assertTrue(mybfile.id, 'mybfile')
self.assertTrue(mybfile.title, 'My bfile')
vstore = mybfile._versions_i18n[mybfile.get_selected_language()]
self.assertEqual(len(vstore), 1)
bf = vstore[0].__of__(mybfile)
path1 = mybfile.current_version.getPhysicalPath()
path2 = bf.getPhysicalPath()
self.assertTrue(path1 == path2)
ver = mybfile.current_version
self.assertTrue(
now_pre <= pytz.utc.localize(ver.timestamp) <= now_post)
self.assertEqual(ver.open().read(), 'hello data!')
self.assertEqual(ver.filename, 'my.jpg')
self.assertEqual(ver.size, 11)
self.assertEqual(ver.content_type, 'image/jpeg')
def test_add_with_file(self):
myfile = StringIO('hello data!')
myfile.filename = 'my.jpg'
myfile.headers = {'content-type': 'image/jpeg'}
tzinfo = self.portal.get_tzinfo()
now_pre = datetime.now(tzinfo)
self.add_bfile(id='mybfile', title='My bfile', uploaded_file=myfile)
now_post = datetime.now(tzinfo)
mybfile = self.portal.myfolder.mybfile
self.assertTrue(mybfile.id, 'mybfile')
self.assertTrue(mybfile.title, 'My bfile')
vstore = mybfile._versions_i18n[mybfile.get_selected_language()]
self.assertEqual(len(vstore), 1)
bf = vstore[0].__of__(mybfile)
path1 = mybfile.current_version.getPhysicalPath()
path2 = bf.getPhysicalPath()
self.assertTrue(path1 == path2)
ver = mybfile.current_version
self.assertTrue(now_pre <= pytz.utc.localize(ver.timestamp) <= now_post)
self.assertEqual(ver.open().read(), 'hello data!')
self.assertEqual(ver.filename, 'my.jpg')
self.assertEqual(ver.size, 11)
self.assertEqual(ver.content_type, 'image/jpeg')
def test_view_zip(self):
""" Custom view of zipfile """
from zipfile import ZipFile
zfile = open(os.path.join(os.path.dirname(__file__), "fixtures", "one_file_zip.zip"), "rb")
upload_file = StringIO(zfile.read())
upload_file.filename = "test.zip"
upload_file.headers = {"content-type": "application/zip"}
namelist = ZipFile(zfile).namelist()
zfile.close()
addNyBFile(
self.portal.myfolder,
submitted=1,
contributor="contributor",
id="test",
title="test",
uploaded_file=upload_file,
)
transaction.commit()
# The result is a simple html with the contents of the zipfile (filepaths)
self.browser.go("http://localhost/portal/myfolder/test/download/en/1/test.zip?action=view")
html_content = self.browser.get_html()
for filepath in namelist:
self.assertTrue(filepath in html_content)
zfile.close()
def test_send_data(self):
data = 'some test data'
f = StringIO(data)
f.filename = 'my.txt'
f.headers = {'content-type': 'text/plain'}
bf = make_blobfile(f, somerandomkw='thevalue')
ok_headers = {'Content-Length': 14,
'Content-Type': 'text/plain',
'Content-Disposition': ("attachment;filename*="
"UTF-8''my.txt")}
response = MockResponse()
ret = bf.send_data(response)
self.assertEqual(response.headers, ok_headers)
self.assertEqual(ret, 'some test data')
ok_headers['Content-Disposition'] = "attachment"
ret = bf.send_data(response, set_filename=False)
self.assertEqual(response.headers, ok_headers)
self.assertEqual(ret, 'some test data')
self.assertEqual(response.headers.get('X-Sendfile'), None)
request = MockRequest()
request.set_header('X-NaayaEnableSendfile', 'on')
ret = bf.send_data(response, REQUEST=request)
self.assertEqual(ret, "[body should be replaced by front-end server]")
self.assertEqual(response.headers.get('X-Sendfile'),
bf._current_filename())
def test_send_data(self):
data = 'some test data'
f = StringIO(data)
f.filename = 'my.txt'
f.headers = {'content-type': 'text/plain'}
bf = make_blobfile(f, somerandomkw='thevalue')
ok_headers = {
'Content-Length': 14,
'Content-Type': 'text/plain',
'Content-Disposition': ("attachment;filename*="
"UTF-8''my.txt")
}
response = MockResponse()
ret = bf.send_data(response)
self.assertEqual(response.headers, ok_headers)
self.assertEqual(ret, 'some test data')
ok_headers['Content-Disposition'] = "attachment"
ret = bf.send_data(response, set_filename=False)
self.assertEqual(response.headers, ok_headers)
self.assertEqual(ret, 'some test data')
self.assertEqual(response.headers.get('X-Sendfile'), None)
request = MockRequest()
request.set_header('X-NaayaEnableSendfile', 'on')
ret = bf.send_data(response, REQUEST=request)
self.assertEqual(ret, "[body should be replaced by front-end server]")
self.assertEqual(response.headers.get('X-Sendfile'),
bf._current_filename())
def handle_upload(self, id, attached_file):
if not attached_file:
return
if id in self.objectIds():
self.manage_delObjects([id, ])
attached_file.seek(0)
data = attached_file.read()
sfile = StringIO(data)
sfile.filename = attached_file.filename[-1]
content_type = mimetypes.guess_type(attached_file.filename)[0]
if content_type is None:
content_type = getattr(attached_file, 'headers', {}).get(
'content-type', 'application/octet-stream')
sfile.headers = {'content-type': content_type}
if id in self.objectIds():
self.manage_delObjects([id, ])
bf = make_blobfile(sfile,
title=attached_file.filename,
removed=False,
timestamp=datetime.utcnow(),
contributor='')
bf.content_type = content_type
bf.filename = id
attached_file.seek(0)
bf.size = len(attached_file.read())
setattr(self, id, bf)
def test_view_zip(self):
""" Custom view of zipfile """
from zipfile import ZipFile
zfile = open(
os.path.join(os.path.dirname(__file__), 'fixtures',
'one_file_zip.zip'), 'rb')
upload_file = StringIO(zfile.read())
upload_file.filename = 'test.zip'
upload_file.headers = {'content-type': 'application/zip'}
namelist = ZipFile(zfile).namelist()
zfile.close()
addNyBFile(self.portal.myfolder,
submitted=1,
contributor='contributor',
id='test',
title="test",
uploaded_file=upload_file)
transaction.commit()
#The result is a simple html with the contents of the zipfile (filepaths)
self.browser.go(
'http://localhost/portal/myfolder/test/download/1/test.zip?action=view'
)
html_content = self.browser.get_html()
for filepath in namelist:
self.assertTrue(filepath in html_content)
zfile.close()
def test_exceptions(self):
"""Different Exceptions"""
file_contents = 'simple contents'
uploaded_file = StringIO(file_contents)
uploaded_file.filename = 'test.txt'
uploaded_file.headers = {'content-type': 'text/plain; charset=utf-8'}
addNyLocalizedBFile(self.portal.myfolder,
submitted=1,
contributor='contributor',
id='test',
title="test",
uploaded_file=uploaded_file,
approved=True)
transaction.commit()
self.browser.go(
'http://localhost/portal/myfolder/test/download/0/test.txt?action=view'
)
self.failUnless('An error was encountered' in self.browser.get_html())
self.failUnless('NotFound' in self.browser.get_html())
self.browser.go(
'http://localhost/portal/de/myfolder/test/download/1/test.txt?action=view'
)
self.failUnless('An error was encountered' in self.browser.get_html())
self.failUnless('NotFound' in self.browser.get_html())
self.browser.go(
'http://localhost/portal/fr/myfolder/test/download/1/test.txt?action=view'
)
self.failUnless('An error was encountered' in self.browser.get_html())
self.failUnless('NotFound' in self.browser.get_html())
def getDummyFileForContent(name, ctype):
# make output file like and add an headers dict, so the contenttype
# is properly set in the headers
output = StringIO()
output.headers = {'content-type': ctype}
file_ = File(name, name, output)
alsoProvides(file_, ICookedFile)
return file_
def getDummyFileForContent(name, ctype):
# make output file like and add an headers dict, so the contenttype
# is properly set in the headers
output = StringIO()
output.headers = {'content-type': ctype}
file_ = File(name, name, output)
alsoProvides(file_, ICookedFile)
return file_
def _update(self, portal):
brains = portal.portal_catalog(meta_type=['Naaya Survey Answer'])
for brain in brains:
try:
obj = brain.getObject()
except:
self.log.warning(
"Couldn't get obj based on catalog information %s",
brain.getURL())
continue
for upload in obj.objectValues('ExtFile'):
if upload is None:
continue
if 'blob' in getattr(upload, 'meta_type', "").lower():
continue
if upload.is_broken():
self.log.warning(
"\t BROKEN EXTFILE: Couldn't migrate extfile for "
"%s because of broken file", obj.absolute_url()
)
continue
if upload.get_size():
data = upload.index_html()
sfile = StringIO(data)
self.log.debug('\t VERSION FILENAME: %s',
'/'.join(upload.filename))
sfile.filename = upload.filename[-1]
sfile.headers = {
'content-type': upload.content_type}
bf = make_blobfile(sfile,
title=upload.title,
removed=False,
timestamp=datetime.utcnow(),
contributor='')
bf.filename = upload.id
bf.content_type = upload.content_type
bf.size = len(data)
upload._delete('/'.join(upload.filename))
setattr(obj, upload.id, bf)
self.log.info("Migrated %s bytes to bfile: %s" %
(len(data), obj.absolute_url()))
obj._p_changed = True
self.log.info('Migration done')
return True
def versions(self):
""" NyFile versions
"""
versions = self.data.pop('versions', [])
if versions:
versions = versions.objectValues()
for version in versions:
if version.is_broken():
self.logger.warning("\t BROKEN EXTFILE: %s",
self.context.absolute_url())
continue
sfile = StringIO(version.data)
self.logger.debug('\t VERSION FILENAME: %s',
'/'.join(version.filename))
sfile.filename = version.filename[-1]
sfile.headers = {'content-type': version.content_type}
yield sfile
extfile = self.data.pop('_ext_file', None)
if extfile is not None:
if not extfile.is_broken():
sfile = StringIO(extfile.data)
self.logger.debug('\t FILENAME: %s',
'/'.join(extfile.filename))
sfile.filename = extfile.filename[-1]
sfile.headers = {'content-type': extfile.content_type}
yield sfile
else:
self.logger.warning("\t BROKEN EXTFILE: %s",
self.context.absolute_url())
bfile = self.data.pop('_bfile', None)
if bfile is not None:
filename = bfile.filename or bfile.id
content = bfile.raw_data()
if len(content) > 0:
sfile = StringIO()
sfile.write(content)
self.logger.debug('\t FILENAME: %s', filename)
sfile.filename = filename
sfile.headers = {'content-type': bfile.content_type}
yield sfile
def versions(self):
""" NyFile versions
"""
versions = self.data.pop('versions', [])
if versions:
versions = versions.objectValues()
for version in versions:
if version.is_broken():
self.logger.warning("\t BROKEN EXTFILE: %s",
self.context.absolute_url())
continue
sfile = StringIO(version.data)
self.logger.debug('\t VERSION FILENAME: %s',
'/'.join(version.filename))
sfile.filename = version.filename[-1]
sfile.headers = {'content-type': version.content_type}
yield sfile
extfile = self.data.pop('_ext_file', None)
if extfile is not None:
if not extfile.is_broken():
sfile = StringIO(extfile.data)
self.logger.debug('\t FILENAME: %s', '/'.join(extfile.filename))
sfile.filename = extfile.filename[-1]
sfile.headers = {'content-type': extfile.content_type}
yield sfile
else:
self.logger.warning("\t BROKEN EXTFILE: %s",
self.context.absolute_url())
bfile = self.data.pop('_bfile', None)
if bfile is not None:
filename = bfile.filename or bfile.id
content = bfile.raw_data()
if len(content) > 0:
sfile = StringIO()
sfile.write(content)
self.logger.debug('\t FILENAME: %s', filename)
sfile.filename = filename
sfile.headers = {'content-type': bfile.content_type}
yield sfile
def test_add_with_file_with_fake_content_type(self):
myfile = StringIO('hello data!')
myfile.filename = 'my.pdf'
myfile.headers = {'content-type': 'image/jpeg'}
self.add_localizedbfile(id='mylocalizedbfile', title='My Localized bfile', uploaded_file=myfile, lang='en')
mylocalizedbfile = self.portal.myfolder.mylocalizedbfile
ver = mylocalizedbfile.current_version
self.assertEqual(ver.content_type, 'application/pdf')
def test_add_with_file_with_fake_content_type(self):
myfile = StringIO('hello data!')
myfile.filename = 'my.pdf'
myfile.headers = {'content-type': 'image/jpeg'}
self.add_bfile(id='mybfile', title='My bfile', uploaded_file=myfile)
mybfile = self.portal.myfolder.mybfile
ver = mybfile.current_version
self.assertEqual(ver.content_type, 'application/pdf')
def test_refresh(self, validator, fetch):
validator.side_effect = lambda pk: setattr(
FileUpload.objects.get(pk=pk), 'validation',
json.dumps({'success': True, 'messages': []}))
content = StringIO(MANIFEST)
content.headers = {'Content-Type': 'application/x-web-app-manifest+json'}
fetch.return_value = content
AddonUser.objects.create(addon_id=337141, user_id=self.user.pk)
res = self.client.post(self.url)
eq_(res.status_code, 204)
assert fetch.called
def deferredGetContent(self, deferred, name, skin=None):
""" uploads data of a resource to deferred """
# "deferred" was previosly created by a getDummyFileForContent
# call in the __bobo_traverse__ method. As the name suggests,
# the file is merely a traversable dummy with appropriate
# headers and name. Now as soon as REQUEST.traverse
# finishes and gets to the part where it calls the tuples
# register using post_traverse (that's actually happening
# right now) we can be sure, that all necessary security
# stuff has taken place (e.g. authentication).
kw = {'skin':skin,'name':name}
data = None
duration = self.cache_duration # duration in seconds
if not self.getDebugMode() and self.isCacheable(name):
if self.ZCacheable_isCachingEnabled():
data = self.ZCacheable_get(keywords=kw)
if data is None:
# This is the part were we would fail if
# we would just return the ressource
# without using the post_traverse hook:
# self.__getitem__ leads (indirectly) to
# a restrictedTraverse call which performs
# security checks. So if a tool (or its ressource)
# is not "View"able by anonymous - we'd
# get an Unauthorized exception.
data = self.__getitem__(name)
self.ZCacheable_set(data, keywords=kw)
else:
data = self.__getitem__(name)
duration = 0
output, contenttype = data
seconds = float(duration)*24.0*3600.0
response = self.REQUEST.RESPONSE
response.setHeader('Expires',rfc1123_date((DateTime() + duration).timeTime()))
response.setHeader('Cache-Control', 'max-age=%d' % int(seconds))
if isinstance(output, unicode):
portal_props = getToolByName(self, 'portal_properties')
site_props = portal_props.site_properties
charset = site_props.getProperty('default_charset', 'utf-8')
output = output.encode(charset)
if 'charset=' not in contenttype:
contenttype += ';charset=' + charset
out = StringIO(output)
out.headers = {'content-type': contenttype}
# At this point we are ready to provide some content
# for our dummy and since it's just a File instance,
# we can "upload" (a quite delusive method name) the
# data and that's it.
deferred.manage_upload(out)
def test_factory_ie6_filename(self):
data = 'some test data'
f = StringIO(data)
f.filename = r'C:\\Documents and Settings\\uzer\\Desktop\\data.txt'
f.headers = {'content-type': 'text/plain'}
bf = make_blobfile(f, somerandomkw='thevalue')
self.assertEqual(bf.open().read(), data)
self.assertEqual(bf.content_type, 'text/plain')
self.assertEqual(bf.filename, 'data.txt')
self.assertEqual(bf.somerandomkw, 'thevalue')
def open(self, url, data=None):
self.requested.append(url)
w = int(re.search(r'width=(\d+)', url, re.IGNORECASE).group(1))
h = int(re.search(r'height=(\d+)', url, re.IGNORECASE).group(1))
format = re.search(r'format=image(/|%2F)(\w+)', url, re.IGNORECASE).group(2)
transparent = re.search(r'transparent=(\w+)', url, re.IGNORECASE)
transparent = True if transparent and transparent.group(1).lower() == 'true' else False
result = StringIO()
create_debug_img((int(w), int(h)), transparent).save(result, format=format)
result.seek(0)
result.headers = {'Content-type': 'image/'+format}
return result
def test_factory(self):
data = 'some test data'
f = StringIO(data)
f.filename = 'my.txt'
f.headers = {'content-type': 'text/plain'}
bf = make_blobfile(f, somerandomkw='thevalue')
self.assertEqual(bf.open().read(), data)
self.assertEqual(bf.content_type, 'text/plain')
self.assertEqual(bf.filename, 'my.txt')
self.assertEqual(bf.somerandomkw, 'thevalue')
def test_factory_ie6_filename(self):
data = 'some test data'
f = StringIO(data)
f.filename = r'C:\\Documents and Settings\\uzer\\Desktop\\data.txt'
f.headers = {'content-type': 'text/plain'}
bf = make_blobfile(f, somerandomkw='thevalue')
self.assertEqual(bf.open().read(), data)
self.assertEqual(bf.content_type, 'text/plain')
self.assertEqual(bf.filename, 'data.txt')
self.assertEqual(bf.somerandomkw, 'thevalue')
def test_factory(self):
data = 'some test data'
f = StringIO(data)
f.filename = 'my.txt'
f.headers = {'content-type': 'text/plain'}
bf = make_blobfile(f, somerandomkw='thevalue')
self.assertEqual(bf.open().read(), data)
self.assertEqual(bf.content_type, 'text/plain')
self.assertEqual(bf.filename, 'my.txt')
self.assertEqual(bf.somerandomkw, 'thevalue')
def open(self, url, data=None):
self.requested.append(url)
w = int(re.search(r'width=(\d+)', url, re.IGNORECASE).group(1))
h = int(re.search(r'height=(\d+)', url, re.IGNORECASE).group(1))
format = re.search(r'format=image(/|%2F)(\w+)', url, re.IGNORECASE).group(2)
transparent = re.search(r'transparent=(\w+)', url, re.IGNORECASE)
transparent = True if transparent and transparent.group(1).lower() == 'true' else False
result = StringIO()
create_debug_img((int(w), int(h)), transparent).save(result, format=format)
result.seek(0)
result.headers = {'Content-type': 'image/'+format}
return result
def test_update_content(self):
from Products.Reportek.blob import OfsBlobFile
content = 'hello blobby world!\n'
myfile = OfsBlobFile('myfile')
upload_file = StringIO(content)
upload_file.headers = {'Content-Type': 'text/plain'}
myfile.manage_edit(Mock(form={'file': upload_file}), Mock())
with myfile.data_file.open() as f:
self.assertEqual(f.read(), content)
self.assertEqual(myfile.data_file.content_type, 'text/plain')
def test_add_with_file(self):
myfile = StringIO('hello data!')
myfile.filename = 'my.jpg'
myfile.headers = {'content-type': 'image/jpeg'}
now_pre = datetime.utcnow()
self.add_localizedbfile(id='mylocalizedbfile', title='My Localized bfile', uploaded_file=myfile, lang='en')
now_post = datetime.utcnow()
mylocalizedbfile = self.portal.myfolder.mylocalizedbfile
language = 'en'
self.assertTrue(mylocalizedbfile.id, 'mylocalizedbfile')
self.assertTrue(mylocalizedbfile.title, 'My Localized bfile')
#no french version for this file
self.assertFalse(mylocalizedbfile._versions.has_key('fr'))
self.assertEqual(len(mylocalizedbfile._versions), 1)
self.assertTrue(mylocalizedbfile.current_version is mylocalizedbfile._versions[language][0])
ver = mylocalizedbfile.current_version
self.assertTrue(now_pre <= ver.timestamp <= now_post)
self.assertEqual(ver.open().read(), 'hello data!')
self.assertEqual(ver.filename, 'my.jpg')
self.assertEqual(ver.size, 11)
self.assertEqual(ver.content_type, 'image/jpeg')
#test french version
myfrfile = StringIO('hello data from France!')
myfrfile.filename = 'my-fr.jpg'
myfrfile.headers = {'content-type': 'image/jpeg'}
mylocalizedbfile._save_file(myfrfile, 'fr',
contributor='contributor')
self.assertEqual(len(mylocalizedbfile._versions), 2)
self.assertTrue(mylocalizedbfile._versions.has_key('fr'))
fr_file = mylocalizedbfile._versions['fr'][0]
self.assertEqual(fr_file.open().read(), 'hello data from France!')
self.assertEqual(fr_file.filename, 'my-fr.jpg')
self.assertEqual(fr_file.content_type, 'image/jpeg')
def get_content(self, folder, filename):
path = join(folder, filename)
with open(path) as f:
content = f.read()
sfile = StringIO(content)
if filename.endswith('.undo'):
filename = filename[:-5]
sfile.filename = filename
try:
ct = mimetypes.guess_type(filename)[0]
if ct:
sfile.headers = {'content-type': ct}
except:
self.log.warning("Could not guess content type for %r", filename)
return sfile
def test_view(self):
"""Same as download, but without the attachment header"""
file_contents = 'simple contents'
uploaded_file = StringIO(file_contents)
uploaded_file.filename = 'test.txt'
uploaded_file.headers = {'content-type': 'text/plain; charset=utf-8'}
addNyLocalizedBFile(self.portal.myfolder, submitted=1,
contributor='contributor', id='test', title="test",
uploaded_file=uploaded_file, approved=True)
transaction.commit()
#View the file
self.browser.go('http://localhost/portal/myfolder/test/download/1/test.txt?action=view')
self.assertEqual(file_contents, self.browser.get_html())
def versions(self):
""" NyFile versions
"""
extfile = self.data.pop('_ext_file')
versions = self.data.pop('versions', [])
if versions:
versions = versions.objectValues()
for version in versions:
if version.is_broken():
continue
sfile = StringIO(version.data)
self.logger.debug('\t VERSION FILENAME: %s', '/'.join(version.filename))
sfile.filename = version.filename[-1]
sfile.headers = {'content-type': version.content_type}
yield sfile
if not extfile.is_broken():
sfile = StringIO(extfile.data)
self.logger.debug('\t FILENAME: %s', '/'.join(extfile.filename))
sfile.filename = extfile.filename[-1]
sfile.headers = {'content-type': extfile.content_type}
yield sfile
def get_content(self, folder, filename):
path = join(folder, filename)
with open(path) as f:
content = f.read()
sfile = StringIO(content)
if filename.endswith('.undo'):
filename = filename[:-5]
sfile.filename = filename
try:
ct = mimetypes.guess_type(filename)[0]
if ct:
sfile.headers = {'content-type': ct}
except:
self.log.warning("Could not guess content type for %r", filename)
return sfile
def test_view(self):
"""Same as download, but without the attachment header"""
file_contents = 'simple contents'
uploaded_file = StringIO(file_contents)
uploaded_file.filename = 'test.txt'
uploaded_file.headers = {'content-type': 'text/plain; charset=utf-8'}
addNyBFile(self.portal.myfolder, submitted=1,
contributor='contributor', id='test', title="test",
uploaded_file=uploaded_file, approved=True)
transaction.commit()
#View the file
self.browser.go('http://localhost/portal/myfolder/test/download/en/1/test.txt?action=view')
self.assertEqual(file_contents, self.browser.get_html())
def test_random_action(self):
"""Radom action"""
file_contents = 'simple contents'
uploaded_file = StringIO(file_contents)
uploaded_file.filename = 'test.txt'
uploaded_file.headers = {'content-type': 'text/plain; charset=utf-8'}
addNyLocalizedBFile(self.portal.myfolder, submitted=1,
contributor='contributor', id='test', title="test",
uploaded_file=uploaded_file, approved=True)
transaction.commit()
self.browser.go('http://localhost/portal/de/myfolder/test/download/1/test.txt?action=shop')
self.failUnless('An error was encountered' in self.browser.get_html())
self.failUnless('NotFound' in self.browser.get_html())
def test_direct_download(self):
""" In folder listing we should have a download link that allows users
to download the last version directly
"""
file_contents = 'simple contents'
uploaded_file = StringIO(file_contents)
uploaded_file.filename = 'test.txt'
uploaded_file.headers = {'content-type': 'text/plain; charset=utf-8'}
addNyLocalizedBFile(self.portal.myfolder, id='mylocalizedbfile_download', title='My Localized file',
uploaded_file=uploaded_file, submitted=1,
contributor='admin')
transaction.commit()
self.browser_do_login('admin', '')
self.browser.go('http://localhost/portal/myfolder')
self.assertTrue(
self.portal.myfolder.mylocalizedbfile_download.current_version_download_url() in
self.browser.get_html())
def test_direct_download(self):
""" In folder listing we should have a download link that allows users
to download the last version directly
"""
file_contents = 'simple contents'
uploaded_file = StringIO(file_contents)
uploaded_file.filename = 'test.txt'
uploaded_file.headers = {'content-type': 'text/plain; charset=utf-8'}
addNyBFile(self.portal.myfolder, id='mybfile_download', title='My file',
uploaded_file=uploaded_file, submitted=1,
contributor='admin')
transaction.commit()
self.browser_do_login('admin', '')
self.browser.go('http://localhost/portal/myfolder')
self.assertTrue(
self.portal.myfolder.mybfile_download.current_version_download_url() in
self.browser.get_html())
def test_view(self):
"""Same as download, but without the attachment header"""
file_contents = "simple contents"
uploaded_file = StringIO(file_contents)
uploaded_file.filename = "test.txt"
uploaded_file.headers = {"content-type": "text/plain; charset=utf-8"}
addNyBFile(
self.portal.myfolder,
submitted=1,
contributor="contributor",
id="test",
title="test",
uploaded_file=uploaded_file,
approved=True,
)
transaction.commit()
# View the file
self.browser.go("http://localhost/portal/myfolder/test/download/en/1/test.txt?action=view")
self.assertEqual(file_contents, self.browser.get_html())
def test_add_with_file(self):
myfile = StringIO('hello data!')
myfile.filename = 'my.jpg'
myfile.headers = {'content-type': 'image/jpeg'}
tzinfo = self.portal.get_tzinfo()
now_pre = datetime.now(tzinfo)
self.add_bfile(id='mybfile', title='My bfile', uploaded_file=myfile)
now_post = datetime.now(tzinfo)
mybfile = self.portal.myfolder.mybfile
self.assertTrue(mybfile.id, 'mybfile')
self.assertTrue(mybfile.title, 'My bfile')
self.assertEqual(len(mybfile._versions), 1)
self.assertTrue(mybfile.current_version is mybfile._versions[0])
ver = mybfile.current_version
self.assertTrue(now_pre <= ver.timestamp <= now_post)
self.assertEqual(ver.open().read(), 'hello data!')
self.assertEqual(ver.filename, 'my.jpg')
self.assertEqual(ver.size, 11)
self.assertEqual(ver.content_type, 'image/jpeg')
def test_direct_download(self):
""" In folder listing we should have a download link that allows users
to download the last version directly
"""
file_contents = "simple contents"
uploaded_file = StringIO(file_contents)
uploaded_file.filename = "test.txt"
uploaded_file.headers = {"content-type": "text/plain; charset=utf-8"}
addNyBFile(
self.portal.myfolder,
id="mybfile_download",
title="My file",
uploaded_file=uploaded_file,
submitted=1,
contributor="admin",
)
transaction.commit()
self.browser_do_login("admin", "")
self.browser.go("http://localhost/portal/myfolder")
self.assertTrue(self.portal.myfolder.mybfile_download.current_version_download_url() in self.browser.get_html())
def test_add_with_file(self):
myfile = StringIO('hello data!')
myfile.filename = 'my.jpg'
myfile.headers = {'content-type': 'image/jpeg'}
tzinfo = self.portal.get_tzinfo()
now_pre = datetime.now(tzinfo)
self.add_bfile(id='mybfile', title='My bfile', uploaded_file=myfile)
now_post = datetime.now(tzinfo)
mybfile = self.portal.myfolder.mybfile
self.assertTrue(mybfile.id, 'mybfile')
self.assertTrue(mybfile.title, 'My bfile')
self.assertEqual(len(mybfile._versions), 1)
self.assertTrue(mybfile.current_version.getPhysicalPath() ==
mybfile._versions[0].__of__(mybfile).getPhysicalPath())
ver = mybfile.current_version
self.assertTrue(now_pre <= ver.timestamp <= now_post)
self.assertEqual(ver.open().read(), 'hello data!')
self.assertEqual(ver.filename, 'my.jpg')
self.assertEqual(ver.size, 11)
self.assertEqual(ver.content_type, 'image/jpeg')
def test_view_zip(self):
""" Custom view of zipfile """
from zipfile import ZipFile
zfile = open(os.path.join(os.path.dirname(__file__), 'fixtures',
'one_file_zip.zip'), 'rb')
upload_file = StringIO(zfile.read())
upload_file.filename = 'test.zip'
upload_file.headers = {'content-type': 'application/zip'}
namelist = ZipFile(zfile).namelist()
zfile.close()
addNyLocalizedBFile(self.portal.myfolder, submitted=1,
contributor='contributor', id='test', title="test",
uploaded_file=upload_file)
transaction.commit()
#The result is a simple html with the contents of the zipfile (filepaths)
self.browser.go('http://localhost/portal/myfolder/test/download/1/test.zip?action=view')
html_content = self.browser.get_html()
for filepath in namelist:
self.assertTrue(filepath in html_content)
zfile.close()
def getDummyFileForContent(name, ctype):
# make output file like and add an headers dict, so the contenttype
# is properly set in the headers
output = StringIO()
output.headers = {'content-type': ctype}
return File(name, name, output)
def make_file(self, filename, content_type, data):
f = StringIO(data)
f.filename = filename
f.headers = {'content-type': content_type}
return f
def _update(self, portal):
brains = portal.portal_catalog(meta_type=['Naaya Survey Answer'])
for brain in brains:
try:
obj = brain.getObject()
except:
self.log.warning(
"Couldn't get obj based on catalog information %s",
brain.getURL())
continue
if not hasattr(obj, 'w_assessment-upload'):
continue
upload = getattr(obj, 'w_assessment-upload', None)
if upload is None:
continue
if 'blob' in getattr(upload, 'meta_type', "").lower():
continue
if upload.is_broken():
self.log.warning(
"\t BROKEN EXTFILE: Couldn't migrate extfile for "
"%s because of broken file", obj.absolute_url())
continue
if upload.get_size():
data = upload.index_html()
sfile = StringIO(data)
self.log.debug('\t VERSION FILENAME: %s',
'/'.join(upload.filename))
sfile.filename = upload.filename[-1]
sfile.headers = {'content-type': upload.content_type}
bf = make_blobfile(sfile,
title=upload.title,
removed=False,
timestamp=datetime.utcnow(),
contributor='')
bf.filename = upload.id
bf.content_type = upload.content_type
bf.size = len(data)
upload._delete('/'.join(upload.filename))
setattr(obj, 'w_assessment-upload', bf)
self.log.info("Migrated %s bytes to bfile: %s" %
(len(data), obj.absolute_url()))
obj._p_changed = True
brains = portal.portal_catalog(meta_type=['Naaya Mega Survey'])
for brain in brains:
obj = brain.getObject()
self.log.info("Migrating object %s", obj.absolute_url())
for attach in obj.objectValues('Naaya Survey Attachment'):
self.log.info("Migrating survey attachment %s",
attach.absolute_url())
for fobj in attach.objectValues():
if not hasattr(fobj, '_ext_file'):
continue
if fobj._ext_file.is_broken():
self.log.warning(
"\t BROKEN EXTFILE: Couldn't migrate extfile for "
"%s because of broken file", fobj.absolute_url())
continue
if fobj._ext_file.get_size():
data = fobj._ext_file.index_html()
blob = fobj._bfile.open_write()
blob.write(data)
blob.seek(0)
blob.close()
fobj._bfile.filename = fobj._ext_file.id
fobj._bfile.content_type = fobj._ext_file.content_type
fobj._bfile.size = len(data)
fobj._p_changed = True
fobj._bfile._p_changed = True
self.log.info("Migrated %s bytes to bfile: %s" %
(len(data), fobj.absolute_url()))
fobj._ext_file._delete('/'.join(
fobj._ext_file.filename))
del fobj._ext_file
self.log.info("Removed _ext_file for %s" %
fobj.absolute_url())
self.log.info('Migrated to bfile: %s' %
fobj.absolute_url())
self.log.info('Migration done')
return True
def handler(doc):
if 'headers' not in doc:
logger.info('skipping blank rss feed %(_id)r', doc)
return
# I need the binary attachment to send via feedparser
content = open_schema_attachment(doc, "response")
# we trick feedparser by giving it a file-like object which also
# supplies a headers object.
f = StringIO(content)
headers = {}
# turn back to single items.
for name, vals in doc['headers'].iteritems():
headers[name] = ';'.join(vals)
# must provide a content-location for relative urls.
if 'content-location' not in headers:
headers['content-location'] = doc['uri']
class FakeHeaders:
get = headers.get
dict = headers
f.headers = FakeHeaders()
info = feedparser.parse(f)
# nuke and convert some of the channel info.
info_values = info.copy()
del info_values['entries']
info_values = make_feedparser_jsonable(info_values)
entries = info.entries
logger.info("feed %r has %d items total", doc['uri'], len(entries))
# for now each item, see what items are new.
entries_by_rdkey = {}
keys = []
for entry in entries:
guid = getattr(entry, 'guid', None) or getattr(entry, 'link', None)
if not guid:
logger.info("can't work out an rd_key for rss entry %s", entry)
continue
rd_key = ['rss-entry', guid]
entries_by_rdkey[tuple(rd_key)] = entry
keys.append(['rd.core.content', 'key-schema_id', [rd_key, 'rd.raw.rss-entry']])
# query for existing items
existing_by_rdkey = {}
result = open_view(keys=keys, reduce=False, include_docs=True)
for row in result['rows']:
existing_by_rdkey[tuple(row['value']['rd_key'])] = row['doc']
num = 0
for rd_key, entry in entries_by_rdkey.iteritems():
try:
existing = existing_by_rdkey[rd_key]
# XXX - should we check the content is the same???
logger.debug('rss item %r already exists - skipping', rd_key)
except KeyError:
rd_key = list(rd_key)
logger.info('creating rss-entry %r', rd_key)
items = make_feedparser_jsonable(entry)
items['channel'] = info_values
emit_schema('rd.raw.rss-entry', items, rd_key=rd_key)
num += 1
logger.info('created %d rss items from %r', num, doc['_id'])
def make_file(self, filename, content_type, data):
f = StringIO(data)
f.filename = filename
f.headers = {"content-type": content_type}
return f
def _update(self, portal):
brains = portal.portal_catalog(meta_type=['Naaya Survey Answer'])
for brain in brains:
try:
obj = brain.getObject()
except:
self.log.warning("Couldn't get obj based on catalog information %s", brain.getURL())
continue
if not hasattr(obj, 'w_assessment-upload'):
continue
upload = getattr(obj, 'w_assessment-upload', None)
if upload is None:
continue
if 'blob' in getattr(upload, 'meta_type', "").lower():
continue
if upload.is_broken():
self.log.warning(
"\t BROKEN EXTFILE: Couldn't migrate extfile for "
"%s because of broken file", obj.absolute_url()
)
continue
if upload.get_size():
data = upload.index_html()
sfile = StringIO(data)
self.log.debug('\t VERSION FILENAME: %s',
'/'.join(upload.filename))
sfile.filename = upload.filename[-1]
sfile.headers = {'content-type': upload.content_type}
bf = make_blobfile(sfile,
title=upload.title,
removed=False,
timestamp=datetime.utcnow(),
contributor='')
bf.filename = upload.id
bf.content_type = upload.content_type
bf.size = len(data)
upload._delete('/'.join(upload.filename))
setattr(obj, 'w_assessment-upload', bf)
self.log.info("Migrated %s bytes to bfile: %s" %
(len(data), obj.absolute_url()))
obj._p_changed = True
brains = portal.portal_catalog(meta_type=['Naaya Mega Survey'])
for brain in brains:
obj = brain.getObject()
self.log.info("Migrating object %s", obj.absolute_url())
for attach in obj.objectValues('Naaya Survey Attachment'):
self.log.info("Migrating survey attachment %s",
attach.absolute_url())
for fobj in attach.objectValues():
if not hasattr(fobj, '_ext_file'):
continue
if fobj._ext_file.is_broken():
self.log.warning(
"\t BROKEN EXTFILE: Couldn't migrate extfile for "
"%s because of broken file", fobj.absolute_url()
)
continue
if fobj._ext_file.get_size():
data = fobj._ext_file.index_html()
blob = fobj._bfile.open_write()
blob.write(data)
blob.seek(0)
blob.close()
fobj._bfile.filename = fobj._ext_file.id
fobj._bfile.content_type = fobj._ext_file.content_type
fobj._bfile.size = len(data)
fobj._p_changed = True
fobj._bfile._p_changed = True
self.log.info("Migrated %s bytes to bfile: %s" %
(len(data), fobj.absolute_url()))
fobj._ext_file._delete('/'.join(fobj._ext_file.filename))
del fobj._ext_file
self.log.info("Removed _ext_file for %s" %
fobj.absolute_url())
self.log.info('Migrated to bfile: %s' %
fobj.absolute_url())
self.log.info('Migration done')
return True
