def test_collection(self):
with open(self.test_collection_file, 'w') as f:
for data in ZipStream(self.files):
f.write(data)
with ZipFile(self.test_collection_file, 'r') as f:
self.assertIsNone(f.testzip())
def get(self, rtip_id):
tip_export = yield get_tip_export(self.current_user.user_id, rtip_id, self.request.language)
self.set_header('X-Download-Options', 'noopen')
self.set_header('Content-Type', 'application/octet-stream')
self.set_header('Content-Disposition', 'attachment; filename=\"%s.zip\"' % tip_export['tip']['sequence_number'])
self.zip_stream = iter(ZipStream(tip_export['files']))
def zip_chunk():
chunk = []
chunk_size = 0
for data in self.zip_stream:
if len(data):
chunk_size += len(data)
chunk.append(data)
if chunk_size >= GLSettings.file_chunk_size:
return ''.join(chunk)
return ''.join(chunk)
chunk = yield threads.deferToThread(zip_chunk)
while len(chunk):
self.write(chunk)
self.flush()
yield deferred_sleep(0.01)
chunk = yield threads.deferToThread(zip_chunk)
def test_zipstream(self):
output = StringIO.StringIO()
for data in ZipStream(self.files):
output.write(data)
with ZipFile(output, 'r') as f:
self.assertIsNone(f.testzip())
def test_collection(self):
for compression in ['zipstored', 'zipdeflated']:
with open(self.test_collection_file, 'w') as f:
opts = get_compression_opts(compression)
for data in ZipStream(self.files, opts['compression_type']):
f.write(data)
with ZipFile(self.test_collection_file, 'r') as f:
self.assertIsNone(f.testzip())
def get(self, rtip_id):
tip_export = yield get_tip_export(self.request.tid,
self.current_user.user_id, rtip_id,
self.request.language)
if tip_export['crypto_tip_prv_key']:
tip_export['tip'] = yield deferToThread(
decrypt_tip, self.current_user.cc,
tip_export['crypto_tip_prv_key'], tip_export['tip'])
for file_dict in tip_export['tip']['rfiles'] + tip_export['tip'][
'wbfiles']:
if file_dict.get('status',
'') == 'encrypted' or file_dict.get('forged'):
continue
tip_prv_key = GCE.asymmetric_decrypt(
self.current_user.cc, tip_export['crypto_tip_prv_key'])
file_dict['fo'] = GCE.streaming_encryption_open(
'DECRYPT', tip_prv_key, file_dict['path'])
del file_dict['path']
for file_dict in tip_export['tip']['rfiles']:
file_dict['name'] = 'files/' + file_dict['name']
if file_dict.get('status', '') == 'encrypted':
file_dict['name'] += '.pgp'
for file_dict in tip_export['tip']['wbfiles']:
file_dict[
'name'] = 'files_attached_from_recipients/' + file_dict['name']
tip_export['comments'] = tip_export['tip']['comments']
tip_export['messages'] = tip_export['tip']['messages']
files = tip_export['tip']['rfiles'] + tip_export['tip']['wbfiles']
del tip_export['tip']['rfiles'], tip_export['tip']['wbfiles']
export_template = Templating().format_template(
tip_export['notification']['export_template'],
tip_export).encode()
export_template = msdos_encode(export_template.decode()).encode()
files.append({
'fo': BytesIO(export_template),
'name': 'data.txt',
'forged': True
})
self.request.setHeader(b'X-Download-Options', b'noopen')
self.request.setHeader(b'Content-Type', b'application/octet-stream')
self.request.setHeader(b'Content-Disposition',
b'attachment; filename="submission.zip"')
self.zip_stream = iter(ZipStream(files))
yield ZipStreamProducer(self, self.zip_stream).start()
def get(self, rtip_id):
tip_export = yield get_tip_export(self.current_user.user_id, rtip_id, self.request.language)
self.request.setHeader('X-Download-Options', 'noopen')
self.request.setHeader('Content-Type', 'application/octet-stream')
self.request.setHeader('Content-Disposition', 'attachment; filename=\"%s.zip\"' % tip_export['tip']['sequence_number'])
self.zip_stream = iter(ZipStream(tip_export['files']))
yield ZipStreamProducer(self, self.zip_stream).start()
def get(self, rtip_id):
tip_export = yield get_tip_export(self.request.tid,
self.current_user.user_id, rtip_id,
self.request.language)
self.request.setHeader(b'X-Download-Options', b'noopen')
self.request.setHeader(b'Content-Type', b'application/octet-stream')
self.request.setHeader(b'Content-Disposition',
b'attachment; filename="submission.zip"')
self.zip_stream = iter(ZipStream(tip_export['files']))
yield ZipStreamProducer(self, self.zip_stream).start()
def post(self, rtip_id, compression):
files_dict = yield download_all_files(self.current_user.user_id,
rtip_id)
if compression is None:
compression = 'zipdeflated'
opts = get_compression_opts(compression)
node_dict = yield admin.admin_serialize_node(self.request.language)
receiver_dict = yield get_receiver_from_rtip(rtip_id,
self.request.language)
rtip_dict = yield get_rtip_info(rtip_id, self.request.language)
collection_tip_dict = yield get_collection_info(rtip_id)
context_dict = yield admin.get_context(rtip_dict['context_id'], 'en')
notif_dict = yield admin.notification.get_notification(
self.request.language)
mock_event = Event(
type=u'zip_collection',
trigger='Download',
node_info=node_dict,
receiver_info=receiver_dict,
context_info=context_dict,
tip_info=rtip_dict,
subevent_info=collection_tip_dict,
do_mail=False,
)
formatted_coll = Templating().format_template(
notif_dict['zip_description'], mock_event).encode('utf-8')
files_dict.append({
'buf': formatted_coll,
'name': "COLLECTION_INFO.txt"
})
self.set_status(200)
self.set_header('X-Download-Options', 'noopen')
self.set_header('Content-Type', 'application/octet-stream')
self.set_header('Content-Disposition',
'attachment; filename=\"%s\"' % opts['filename'])
if compression in ['zipstored', 'zipdeflated']:
for data in ZipStream(files_dict, opts['compression_type']):
self.write(data)
self.finish()
def post(self, rtip_id):
tip_export = yield get_tip_export(self.current_user.user_id, rtip_id)
self.set_status(200)
self.set_header('X-Download-Options', 'noopen')
self.set_header('Content-Type', 'application/octet-stream')
self.set_header(
'Content-Disposition', 'attachment; filename=\"%s.zip\"' %
tip_export['tip']['sequence_number'])
for data in ZipStream(tip_export['files']):
self.write(data)
self.finish()
def test_zipstream(self):
output = StringIO.StringIO()
for data in ZipStream(self.files):
output.write(data)
with ZipFile(output, 'r') as f:
self.assertIsNone(f.testzip())
with ZipFile(output, 'r') as f:
infolist = f.infolist()
self.assertTrue(len(infolist), 2)
for ff in infolist:
if ff.filename == self.unicode_seq:
self.assertTrue(ff.file_size == len(self.unicode_seq))
else:
self.assertTrue(ff.file_size == os.stat(os.path.abspath(__file__)).st_size)
def get(self, rtip_id):
tip_export = yield get_tip_export(self.request.tid,
self.current_user.user_id,
rtip_id,
self.request.language)
if tip_export['crypto_tip_prv_key']:
for file_dict in tip_export['files']:
if file_dict['forged']:
continue
tip_prv_key = GCE.asymmetric_decrypt(self.current_user.cc, tip_export['crypto_tip_prv_key'])
file_dict['fo'] = GCE.streaming_encryption_open('DECRYPT', tip_prv_key, file_dict['path'])
del file_dict['path']
self.request.setHeader(b'X-Download-Options', b'noopen')
self.request.setHeader(b'Content-Type', b'application/octet-stream')
self.request.setHeader(b'Content-Disposition', b'attachment; filename="submission.zip"')
self.zip_stream = iter(ZipStream(tip_export['files']))
yield ZipStreamProducer(self, self.zip_stream).start()
def post(self, rtip_id, compression):
files_dict = yield download_all_files(self.current_user.user_id,
rtip_id)
if not files_dict:
raise errors.DownloadLimitExceeded
if compression is None:
compression = 'zipstored'
opts = get_compression_opts(compression)
node_dict = yield admin.admin_serialize_node()
receiver_dict = yield get_receiver_from_rtip(rtip_id)
rtip_dict = yield get_rtip_info(rtip_id)
collection_tip_dict = yield get_collection_info(rtip_id)
context_dict = yield admin.get_context(rtip_dict['context_id'])
notif_dict = yield admin.get_notification()
mock_event = Event(type=u'zip_collection',
trigger='Download',
notification_settings=notif_dict,
node_info=node_dict,
receiver_info=receiver_dict,
context_info=context_dict,
plugin=None,
trigger_info=collection_tip_dict,
trigger_parent=rtip_dict)
formatted_coll = Templating().format_template(
notif_dict['zip_description'], mock_event).encode('utf-8')
# log.debug("Generating collection content with: %s" % formatted_coll)
files_dict.append({
'buf': formatted_coll,
'name': "COLLECTION_INFO.txt"
})
self.set_status(200)
self.set_header('X-Download-Options', 'noopen')
self.set_header('Content-Type', 'application/octet-stream')
self.set_header('Content-Disposition',
'attachment; filename=\"%s\"' % opts['filename'])
if compression in ['zipstored', 'zipdeflated']:
for data in ZipStream(files_dict, opts['compression_type']):
self.write(data)
elif compression in ['tar', 'targz', 'tarbz2']:
collectionstreamer = CollectionStreamer(self)
tar = tarfile.open("collection." + compression,
'w|' + opts['compression_type'],
collectionstreamer)
for f in files_dict:
if 'path' in f:
try:
tar.add(f['path'], f['name'])
except (OSError, IOError) as excpd:
log.err(
"OSError while adding %s to files collection: %s" %
(f['path'], excpd))
elif 'buf' in f:
tarinfo = tarfile.TarInfo(f['name'])
tarinfo.size = len(f['buf'])
tar.addfile(tarinfo, StringIO.StringIO(f['buf']))
tar.close()
self.finish()