def test_collection(self): with open(self.test_collection_file, 'w') as f: for data in ZipStream(self.files): f.write(data) with ZipFile(self.test_collection_file, 'r') as f: self.assertIsNone(f.testzip())
def get(self, rtip_id): tip_export = yield get_tip_export(self.current_user.user_id, rtip_id, self.request.language) self.set_header('X-Download-Options', 'noopen') self.set_header('Content-Type', 'application/octet-stream') self.set_header('Content-Disposition', 'attachment; filename=\"%s.zip\"' % tip_export['tip']['sequence_number']) self.zip_stream = iter(ZipStream(tip_export['files'])) def zip_chunk(): chunk = [] chunk_size = 0 for data in self.zip_stream: if len(data): chunk_size += len(data) chunk.append(data) if chunk_size >= GLSettings.file_chunk_size: return ''.join(chunk) return ''.join(chunk) chunk = yield threads.deferToThread(zip_chunk) while len(chunk): self.write(chunk) self.flush() yield deferred_sleep(0.01) chunk = yield threads.deferToThread(zip_chunk)
def test_zipstream(self): output = StringIO.StringIO() for data in ZipStream(self.files): output.write(data) with ZipFile(output, 'r') as f: self.assertIsNone(f.testzip())
def test_collection(self): for compression in ['zipstored', 'zipdeflated']: with open(self.test_collection_file, 'w') as f: opts = get_compression_opts(compression) for data in ZipStream(self.files, opts['compression_type']): f.write(data) with ZipFile(self.test_collection_file, 'r') as f: self.assertIsNone(f.testzip())
def get(self, rtip_id): tip_export = yield get_tip_export(self.request.tid, self.current_user.user_id, rtip_id, self.request.language) if tip_export['crypto_tip_prv_key']: tip_export['tip'] = yield deferToThread( decrypt_tip, self.current_user.cc, tip_export['crypto_tip_prv_key'], tip_export['tip']) for file_dict in tip_export['tip']['rfiles'] + tip_export['tip'][ 'wbfiles']: if file_dict.get('status', '') == 'encrypted' or file_dict.get('forged'): continue tip_prv_key = GCE.asymmetric_decrypt( self.current_user.cc, tip_export['crypto_tip_prv_key']) file_dict['fo'] = GCE.streaming_encryption_open( 'DECRYPT', tip_prv_key, file_dict['path']) del file_dict['path'] for file_dict in tip_export['tip']['rfiles']: file_dict['name'] = 'files/' + file_dict['name'] if file_dict.get('status', '') == 'encrypted': file_dict['name'] += '.pgp' for file_dict in tip_export['tip']['wbfiles']: file_dict[ 'name'] = 'files_attached_from_recipients/' + file_dict['name'] tip_export['comments'] = tip_export['tip']['comments'] tip_export['messages'] = tip_export['tip']['messages'] files = tip_export['tip']['rfiles'] + tip_export['tip']['wbfiles'] del tip_export['tip']['rfiles'], tip_export['tip']['wbfiles'] export_template = Templating().format_template( tip_export['notification']['export_template'], tip_export).encode() export_template = msdos_encode(export_template.decode()).encode() files.append({ 'fo': BytesIO(export_template), 'name': 'data.txt', 'forged': True }) self.request.setHeader(b'X-Download-Options', b'noopen') self.request.setHeader(b'Content-Type', b'application/octet-stream') self.request.setHeader(b'Content-Disposition', b'attachment; filename="submission.zip"') self.zip_stream = iter(ZipStream(files)) yield ZipStreamProducer(self, self.zip_stream).start()
def get(self, rtip_id): tip_export = yield get_tip_export(self.current_user.user_id, rtip_id, self.request.language) self.request.setHeader('X-Download-Options', 'noopen') self.request.setHeader('Content-Type', 'application/octet-stream') self.request.setHeader('Content-Disposition', 'attachment; filename=\"%s.zip\"' % tip_export['tip']['sequence_number']) self.zip_stream = iter(ZipStream(tip_export['files'])) yield ZipStreamProducer(self, self.zip_stream).start()
def get(self, rtip_id): tip_export = yield get_tip_export(self.request.tid, self.current_user.user_id, rtip_id, self.request.language) self.request.setHeader(b'X-Download-Options', b'noopen') self.request.setHeader(b'Content-Type', b'application/octet-stream') self.request.setHeader(b'Content-Disposition', b'attachment; filename="submission.zip"') self.zip_stream = iter(ZipStream(tip_export['files'])) yield ZipStreamProducer(self, self.zip_stream).start()
def post(self, rtip_id, compression): files_dict = yield download_all_files(self.current_user.user_id, rtip_id) if compression is None: compression = 'zipdeflated' opts = get_compression_opts(compression) node_dict = yield admin.admin_serialize_node(self.request.language) receiver_dict = yield get_receiver_from_rtip(rtip_id, self.request.language) rtip_dict = yield get_rtip_info(rtip_id, self.request.language) collection_tip_dict = yield get_collection_info(rtip_id) context_dict = yield admin.get_context(rtip_dict['context_id'], 'en') notif_dict = yield admin.notification.get_notification( self.request.language) mock_event = Event( type=u'zip_collection', trigger='Download', node_info=node_dict, receiver_info=receiver_dict, context_info=context_dict, tip_info=rtip_dict, subevent_info=collection_tip_dict, do_mail=False, ) formatted_coll = Templating().format_template( notif_dict['zip_description'], mock_event).encode('utf-8') files_dict.append({ 'buf': formatted_coll, 'name': "COLLECTION_INFO.txt" }) self.set_status(200) self.set_header('X-Download-Options', 'noopen') self.set_header('Content-Type', 'application/octet-stream') self.set_header('Content-Disposition', 'attachment; filename=\"%s\"' % opts['filename']) if compression in ['zipstored', 'zipdeflated']: for data in ZipStream(files_dict, opts['compression_type']): self.write(data) self.finish()
def post(self, rtip_id): tip_export = yield get_tip_export(self.current_user.user_id, rtip_id) self.set_status(200) self.set_header('X-Download-Options', 'noopen') self.set_header('Content-Type', 'application/octet-stream') self.set_header( 'Content-Disposition', 'attachment; filename=\"%s.zip\"' % tip_export['tip']['sequence_number']) for data in ZipStream(tip_export['files']): self.write(data) self.finish()
def test_zipstream(self): output = StringIO.StringIO() for data in ZipStream(self.files): output.write(data) with ZipFile(output, 'r') as f: self.assertIsNone(f.testzip()) with ZipFile(output, 'r') as f: infolist = f.infolist() self.assertTrue(len(infolist), 2) for ff in infolist: if ff.filename == self.unicode_seq: self.assertTrue(ff.file_size == len(self.unicode_seq)) else: self.assertTrue(ff.file_size == os.stat(os.path.abspath(__file__)).st_size)
def get(self, rtip_id): tip_export = yield get_tip_export(self.request.tid, self.current_user.user_id, rtip_id, self.request.language) if tip_export['crypto_tip_prv_key']: for file_dict in tip_export['files']: if file_dict['forged']: continue tip_prv_key = GCE.asymmetric_decrypt(self.current_user.cc, tip_export['crypto_tip_prv_key']) file_dict['fo'] = GCE.streaming_encryption_open('DECRYPT', tip_prv_key, file_dict['path']) del file_dict['path'] self.request.setHeader(b'X-Download-Options', b'noopen') self.request.setHeader(b'Content-Type', b'application/octet-stream') self.request.setHeader(b'Content-Disposition', b'attachment; filename="submission.zip"') self.zip_stream = iter(ZipStream(tip_export['files'])) yield ZipStreamProducer(self, self.zip_stream).start()
def post(self, rtip_id, compression): files_dict = yield download_all_files(self.current_user.user_id, rtip_id) if not files_dict: raise errors.DownloadLimitExceeded if compression is None: compression = 'zipstored' opts = get_compression_opts(compression) node_dict = yield admin.admin_serialize_node() receiver_dict = yield get_receiver_from_rtip(rtip_id) rtip_dict = yield get_rtip_info(rtip_id) collection_tip_dict = yield get_collection_info(rtip_id) context_dict = yield admin.get_context(rtip_dict['context_id']) notif_dict = yield admin.get_notification() mock_event = Event(type=u'zip_collection', trigger='Download', notification_settings=notif_dict, node_info=node_dict, receiver_info=receiver_dict, context_info=context_dict, plugin=None, trigger_info=collection_tip_dict, trigger_parent=rtip_dict) formatted_coll = Templating().format_template( notif_dict['zip_description'], mock_event).encode('utf-8') # log.debug("Generating collection content with: %s" % formatted_coll) files_dict.append({ 'buf': formatted_coll, 'name': "COLLECTION_INFO.txt" }) self.set_status(200) self.set_header('X-Download-Options', 'noopen') self.set_header('Content-Type', 'application/octet-stream') self.set_header('Content-Disposition', 'attachment; filename=\"%s\"' % opts['filename']) if compression in ['zipstored', 'zipdeflated']: for data in ZipStream(files_dict, opts['compression_type']): self.write(data) elif compression in ['tar', 'targz', 'tarbz2']: collectionstreamer = CollectionStreamer(self) tar = tarfile.open("collection." + compression, 'w|' + opts['compression_type'], collectionstreamer) for f in files_dict: if 'path' in f: try: tar.add(f['path'], f['name']) except (OSError, IOError) as excpd: log.err( "OSError while adding %s to files collection: %s" % (f['path'], excpd)) elif 'buf' in f: tarinfo = tarfile.TarInfo(f['name']) tarinfo.size = len(f['buf']) tar.addfile(tarinfo, StringIO.StringIO(f['buf'])) tar.close() self.finish()