def save_uploadfile_in_zipfile( upload_file, upload_filename, dest_zipfile_name, dest_filename_in_zip=None): nzf = ZipFile(dest_zipfile_name, mode='w', **ZIP_WRITE_OPTS) if upload_filename[-3:].lower() == u'zip': # 'zipfile, check and rename content' zf = ZipFile(StringIO(upload_file.read()), 'r') file_names = zf.namelist() for filename in file_names: #write file to new a = get_new_filename(filename, dest_filename_in_zip) nzf.writestr(a, zf.read(filename)) zf.close() else: # 'normal file' nzf.writestr( get_new_filename(upload_filename, dest_filename_in_zip), upload_file.read()) nzf.close() # Remove comment line from .asc and .inc files after uploading remove_comments_from_asc_files(os.path.dirname(dest_zipfile_name))
def MyUnzipFile(in_file_path, target_root_path): """Unzip file. >>> unzip_file("d:\\zout.zip", "d:\\")""" zin = ZipFile(in_file_path, "r") zin.extractall(target_root_path) zin.close()
class UnZip(): """Unzip the given file into a temporary directory.""" def __init__(self, input_file): self.input_file = input_file def unzip(self): self.zip_file = ZipFile(self.input_file, 'r') self.tempdir = tempfile.mkdtemp() self.zip_file.extractall(self.tempdir) return self.tempdir def cleanup(self): self.zip_file.close() for root, dirs, files in os.walk(self.tempdir, topdown=False): for name in files: os.remove(os.path.join(root, name)) for name in dirs: os.rmdir(os.path.join(root, name)) os.rmdir(self.tempdir) def __enter__(self): return self.unzip() def __exit__(self, *args): self.cleanup() return False
def export_gtfs(self, gtfs_file): """Export a GTFS file as feed Keyword arguments: gtfs_file - A path or file-like object for the GTFS feed This function will close the file in order to finalize it. """ z = ZipFile(gtfs_file, 'w') gtfs_order = ( ('agency.txt', Agency), ('calendar.txt', Service), ('calendar_dates.txt', ServiceDate), ('fare_attributes.txt', Fare), ('fare_rules.txt', FareRule), ('feed_info.txt', FeedInfo), ('frequencies.txt', Frequency), ('routes.txt', Route), ('shapes.txt', ShapePoint), ('stop_times.txt', StopTime), ('stops.txt', Stop), ('transfers.txt', Transfer), ('trips.txt', Trip), ) for filename, exporter in gtfs_order: content = exporter.objects.in_feed(self).export_txt() if content: z.writestr(filename, content) z.close()
def main(): # test if PyXML is installed try: import _xmlplus.parsers.expat pyxml = '--includes _xmlplus.parsers.expat' except ImportError: pyxml = '' # create exe status = os.system('python setup.py py2exe %s >& build.log' % pyxml) if status != 0: raise RuntimeError, 'Error creating EXE' # create distribution import pyste version = pyste.__VERSION__ zip = ZipFile('pyste-%s.zip' % version, 'w', ZIP_DEFLATED) # include the base files dist_dir = 'dist/pyste' for basefile in os.listdir(dist_dir): zip.write(os.path.join(dist_dir, basefile), os.path.join('pyste', basefile)) # include documentation for doc_file in findfiles('../doc', '*.*'): dest_name = os.path.join('pyste/doc', doc_file[3:]) zip.write(doc_file, dest_name) zip.write('../index.html', 'pyste/doc/index.html') zip.close() # cleanup os.remove('build.log') shutil.rmtree('build') shutil.rmtree('dist')
def __save_zip(self, file): """Save a Zip ODF from the available parts. """ # Parts were loaded by "save" parts = self.__parts compression = ZIP_DEFLATED try: filezip = ZipFile(file, 'w', compression=compression) except RuntimeError: # No zlib module compression = ZIP_STORED filezip = ZipFile(file, 'w', compression=compression) # Parts to save, except manifest at the end part_names = parts.keys() part_names.remove(ODF_MANIFEST) # "Pretty-save" parts in some order # mimetype requires to be first and uncompressed filezip.compression = ZIP_STORED filezip.writestr('mimetype', parts['mimetype']) filezip.compression = compression part_names.remove('mimetype') # XML parts for path in ODF_CONTENT, ODF_META, ODF_SETTINGS, ODF_STYLES: filezip.writestr(path, parts[path]) part_names.remove(path) # Everything else for path in part_names: data = parts[path] if data is None: # Deleted continue filezip.writestr(path, data) # Manifest filezip.writestr(ODF_MANIFEST, parts[ODF_MANIFEST]) filezip.close()
def build(worker_dir, sha, repo_url, destination, concurrency): """Download and build sources in a temporary directory then move exe to destination""" tmp_dir = tempfile.mkdtemp() os.chdir(tmp_dir) with open('sf.gz', 'wb+') as f: f.write(requests.get(github_api(repo_url) + '/zipball/' + sha).content) zip_file = ZipFile('sf.gz') zip_file.extractall() zip_file.close() for name in zip_file.namelist(): if name.endswith('/src/'): src_dir = name os.chdir(src_dir) custom_make = os.path.join(worker_dir, 'custom_make.txt') if os.path.exists(custom_make): with open(custom_make, 'r') as m: make_cmd = m.read().strip() subprocess.check_call(make_cmd, shell=True) else: subprocess.check_call(MAKE_CMD + ' -j %s' % (concurrency), shell=True) shutil.move('stockfish'+ EXE_SUFFIX, destination) os.chdir(worker_dir) shutil.rmtree(tmp_dir)
def doTest(self, expected_ext, files, *modules, **kw): z = ZipFile(TEMP_ZIP, "w") try: for name, (mtime, data) in files.items(): zinfo = ZipInfo(name, time.localtime(mtime)) zinfo.compress_type = self.compression z.writestr(zinfo, data) z.close() stuff = kw.get("stuff", None) if stuff is not None: # Prepend 'stuff' to the start of the zipfile f = open(TEMP_ZIP, "rb") data = f.read() f.close() f = open(TEMP_ZIP, "wb") f.write(stuff) f.write(data) f.close() sys.path.insert(0, TEMP_ZIP) mod = __import__(".".join(modules), globals(), locals(), ["__dummy__"]) if expected_ext: file = mod.get_file() self.assertEquals(file, os.path.join(TEMP_ZIP, *modules) + expected_ext) finally: z.close() os.remove(TEMP_ZIP)
def _render_zip(self, req, filename, repos, diff): """ZIP archive with all the added and/or modified files.""" new_rev = diff.new_rev req.send_response(200) req.send_header('Content-Type', 'application/zip') req.send_header('Content-Disposition', 'attachment;' 'filename=%s.zip' % filename) from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED buf = StringIO() zipfile = ZipFile(buf, 'w', ZIP_DEFLATED) for old_node, new_node, kind, change in repos.get_changes(**diff): if kind == Node.FILE and change != Changeset.DELETE: assert new_node zipinfo = ZipInfo() zipinfo.filename = new_node.path.encode('utf-8') # Note: unicode filenames are not supported by zipfile. # UTF-8 is not supported by all Zip tools either, # but as some does, I think UTF-8 is the best option here. zipinfo.date_time = time.gmtime(new_node.last_modified)[:6] zipinfo.compress_type = ZIP_DEFLATED zipfile.writestr(zipinfo, new_node.get_content().read()) zipfile.close() buf.seek(0, 2) # be sure to be at the end req.send_header("Content-Length", buf.tell()) req.end_headers() req.write(buf.getvalue())
def acquire_resource(self, target_path, format_dict): """ Downloads the zip file and extracts the files listed in :meth:`zip_file_contents` to the target path. """ import cStringIO as StringIO from zipfile import ZipFile target_dir = os.path.dirname(target_path) if not os.path.isdir(target_dir): os.makedirs(target_dir) url = self.url(format_dict) shapefile_online = self._urlopen(url) zfh = ZipFile(StringIO.StringIO(shapefile_online.read()), 'r') for member_path in self.zip_file_contents(format_dict): ext = os.path.splitext(member_path)[1] target = os.path.splitext(target_path)[0] + ext member = zfh.getinfo(member_path) with open(target, 'wb') as fh: fh.write(zfh.open(member).read()) shapefile_online.close() zfh.close() return target_path
def aqcuire_all_resources(self, format_dict): import cStringIO as StringIO from zipfile import ZipFile # Download archive. url = self.url(format_dict) shapefile_online = self._urlopen(url) zfh = ZipFile(StringIO.StringIO(shapefile_online.read()), 'r') shapefile_online.close() # Iterate through all scales and levels and extract relevant files. modified_format_dict = dict(format_dict) scales = ('c', 'l', 'i', 'h', 'f') levels = (1, 2, 3, 4) for scale, level in itertools.product(scales, levels): modified_format_dict.update({'scale': scale, 'level': level}) target_path = self.target_path(modified_format_dict) target_dir = os.path.dirname(target_path) if not os.path.isdir(target_dir): os.makedirs(target_dir) for member_path in self.zip_file_contents(modified_format_dict): ext = os.path.splitext(member_path)[1] target = os.path.splitext(target_path)[0] + ext member = zfh.getinfo(member_path) with open(target, 'wb') as fh: fh.write(zfh.open(member).read()) zfh.close()
def do_export(self, REQUEST=None): """ """ if REQUEST and not self.getParentNode().checkPermissionView(): raise Unauthorized errors = [] my_container = self.getParentNode() objects_to_archive = self.gather_objects(my_container) file_like_object = StringIO() zip_file = ZipFile(file_like_object, 'w') archive_files = [] try: for obj in objects_to_archive: added_path = None if self.is_exportable(obj): added_path = self.add_object_to_zip(obj, zip_file) if added_path: archive_files.append((obj.title_or_id(), getattr(obj, 'meta_label', obj.meta_type), added_path)) self.add_index(zip_file, archive_files) zip_file.close() except Exception, e: errors.append(e)
def save_pickle_in_cfile(self, local_fname, networkref): """ Creates a pickled version of the graph and stores it in the cfile Parameters ---------- local_fname: string The filename used in the Pickle folder to store networkref: NetworkX Graph instance The NetworkX graph to pickle """ logger.info('Write a generated graph pickle to the connectome file.') picklefilepath = os.path.join(tempfile.gettempdir(),local_fname) from networkx import write_gpickle # add nodekeys, edgekeys, graphid to helpernode 'n0' before storage helperdict = {'nodekeys': networkref.nodekeys.copy(), \ 'edgekeys': networkref.edgekeys.copy(), \ 'graphid' : networkref.networkid } networkref.graph.add_node('n0') networkref.graph.node['n0'] = helperdict write_gpickle(networkref.graph, picklefilepath) networkref.graph.remove_node('n0') from zipfile import ZipFile, ZIP_DEFLATED tmpzipfile = ZipFile(self.data.fullpathtofile, 'a', ZIP_DEFLATED) # store it in the zip file tmpzipfile.write(picklefilepath, 'Pickle/' + local_fname) tmpzipfile.close() # remove pickle file from system logger.debug('Unlink: %s' % picklefilepath) os.unlink(picklefilepath)
def layer_type(filename): """Finds out if a filename is a Feature or a Vector returns a gsconfig resource_type string that can be either 'featureType' or 'coverage' """ base_name, extension = os.path.splitext(filename) shp_exts = ['.shp',] cov_exts = ['.tif', '.tiff', '.geotiff', '.geotif'] csv_exts = ['.csv'] kml_exts = ['.kml'] if extension.lower() == '.zip': zf = ZipFile(filename) # ZipFile doesn't support with statement in 2.6, so don't do it try: for n in zf.namelist(): b, e = os.path.splitext(n.lower()) if e in shp_exts or e in cov_exts or e in csv_exts: base_name, extension = b,e finally: zf.close() if extension.lower() in shp_exts + csv_exts + kml_exts: return FeatureType.resource_type elif extension.lower() in cov_exts: return Coverage.resource_type else: msg = ('Saving of extension [%s] is not implemented' % extension) raise GeoNodeException(msg)
def pack(self, output_dir, devel=False, force=False, keep_temp=False): self.output_dir = os.path.realpath(output_dir) if not os.path.exists(self.output_dir): os.makedirs(self.output_dir) self.force = force self.keep_temp = keep_temp paths = [] self.merge_modules = {} # create runtime package p = self._create_msi_installer(PackageType.RUNTIME) paths.append(p) # create devel package if devel and not isinstance(self.package, App): p = self._create_msi_installer(PackageType.DEVEL) paths.append(p) # create zip with merge modules self.package.set_mode(PackageType.RUNTIME) zipf = ZipFile(os.path.join(self.output_dir, '%s-merge-modules.zip' % self._package_name()), 'w') for p in self.merge_modules[PackageType.RUNTIME]: zipf.write(p) zipf.close() if not keep_temp: for msms in self.merge_modules.values(): for p in msms: os.remove(p) return paths
def __exportStyle(self, dirstylename, filename,cfgxml): """ Exporta un estilo a un archivo ZIP """ if not filename.lower().endswith('.zip'): filename += '.zip' sfile=os.path.basename(filename) log.debug("Export style %s" % dirstylename) try: zippedFile = ZipFile(filename, "w") try: if cfgxml!='': for contFile in dirstylename.files(): if contFile.basename()!= 'config.xml': zippedFile.write(unicode(contFile.normpath()), contFile.basename(), ZIP_DEFLATED) else: for contFile in dirstylename.files(): zippedFile.write(unicode(contFile.normpath()), contFile.basename(), ZIP_DEFLATED) finally: if cfgxml!='': zippedFile.writestr('config.xml', cfgxml) zippedFile.close() self.alert(_(u'Correct'), _(u'Style exported correctly: %s') % sfile) except IOError: self.alert(_(u'Error'), _(u'Could not export style : %s') % filename.basename()) self.action = ""
def zip_layer_folder(dir_path, layer_name): """ Create a zip archive with the content of the folder located at `dir_path` and name it with `layer_name`. Parameters ---------- dir_path: str The path to the temporary folder in which are located the files to be zipped. layer_name: str The name of the concerned layer (will be used as file name for the zip archive). Returns ------- raw_content: str The zip archive archive_name: str The name of the archive (used later in the header of the response). """ filenames = os.listdir(dir_path) zip_stream = BytesIO() myZip = ZipFile(zip_stream, "w", compression=ZIP_DEFLATED) for filename in filenames: if not filename.endswith('.geojson'): f_name = path_join(dir_path, filename) myZip.write(f_name, filename, ZIP_DEFLATED) myZip.close() zip_stream.seek(0) return zip_stream.read(), ''.join([layer_name, ".zip"])
def upload_analyzer(self): """Upload analyzer to guest. @return: operation status. """ zip_data = StringIO() zip_file = ZipFile(zip_data, "w", ZIP_DEFLATED) root = os.path.join("analyzer", self.platform) root_len = len(os.path.abspath(root)) if not os.path.exists(root): log.error("No valid analyzer found at path: %s" % root) return False for root, dirs, files in os.walk(root): archive_root = os.path.abspath(root)[root_len:] for name in files: path = os.path.join(root, name) archive_name = os.path.join(archive_root, name) zip_file.write(path, archive_name, ZIP_DEFLATED) zip_file.close() data = xmlrpclib.Binary(zip_data.getvalue()) zip_data.close() log.debug("Uploading analyzer to guest (ip=%s)" % self.ip) self.server.add_analyzer(data)
def get_results(self): """Get analysis results. @return: data. """ root = self._get_root(container="cuckoo", create=False) if not os.path.exists(root): return False zip_data = StringIO() zip_file = ZipFile(zip_data, "w", ZIP_DEFLATED) root_len = len(os.path.abspath(root)) for root, dirs, files in os.walk(root): archive_root = os.path.abspath(root)[root_len:] for name in files: path = os.path.join(root, name) archive_name = os.path.join(archive_root, name) zip_file.write(path, archive_name, ZIP_DEFLATED) zip_file.close() data = xmlrpclib.Binary(zip_data.getvalue()) zip_data.close() return data
def parse_template(template_name): """Resolve template name into absolute path to the template and boolean if absolute path is temporary directory. """ if template_name.startswith('http'): if '#' in template_name: url, subpath = template_name.rsplit('#', 1) else: url = template_name subpath = '' with tempfile.NamedTemporaryFile() as tmpfile: urlretrieve(url, tmpfile.name) if not is_zipfile(tmpfile.name): raise ConfigurationError("Not a zip file: %s" % tmpfile) zf = ZipFile(tmpfile) try: path = tempfile.mkdtemp() zf.extractall(path) return os.path.join(path, subpath), True finally: zf.close() registry = TemplatesRegistry() if registry.has_template(template_name): path = registry.path_of_template(template_name) elif ':' in template_name: path = resolve_dotted_path(template_name) else: path = os.path.realpath(template_name) if not os.path.isdir(path): raise ConfigurationError('Template directory does not exist: %s' % path) return path, False
class WikiArchiver(BaseWikiManager): def _serialize_page(self, page): pdict = dict(id=page.id, name=page.name, type=page.type, created=page.created, modified=page.modified) return pdict def create_new_zipfile(self): fields = ['id', 'name', 'type', 'created', 'modified'] self.zipfileobj = BytesIO() self.csvfileobj = StringIO() self.zipfile = ZipFile(self.zipfileobj, 'w') self.csvfile = csv.DictWriter(self.csvfileobj, fields) def archive_pages(self): for page in self.query().all(): pdict = self._serialize_page(page) self.csvfile.writerow(pdict) filename = 'tutwiki-page-%04d.txt' % page.id self.zipfile.writestr(filename, bytes(page.content)) csvfilename = 'tutwiki-dbinfo.csv' self.zipfile.writestr(csvfilename, self.csvfileobj.getvalue()) self.zipfile.close() #self.zipfileobj.seek(0) #return self.zipfileobj.read() return self.zipfileobj.getvalue()
def createDevEnv(baseDir, type): fileBuffer = StringIO() createBuild(baseDir, type=type, outFile=fileBuffer, devenv=True, releaseBuild=True) from zipfile import ZipFile zip = ZipFile(StringIO(fileBuffer.getvalue()), 'r') zip.extractall(os.path.join(baseDir, 'devenv')) zip.close() print 'Development environment created, waiting for connections from active extensions...' metadata = readMetadata(baseDir, type) connections = [0] import SocketServer, time, thread class ConnectionHandler(SocketServer.BaseRequestHandler): def handle(self): connections[0] += 1 self.request.sendall('HTTP/1.0 OK\nConnection: close\n\n%s' % metadata.get('general', 'basename')) server = SocketServer.TCPServer(('localhost', 43816), ConnectionHandler) def shutdown_server(server): time.sleep(10) server.shutdown() thread.start_new_thread(shutdown_server, (server,)) server.serve_forever() if connections[0] == 0: print 'Warning: No incoming connections, extension probably not active in the browser yet' else: print 'Handled %i connection(s)' % connections[0]
def preloadFont(cls, font, directory=DEFAULT_DIR): """ Load font file into memory. This can be overriden with a superclass to create different font sources. """ fontPath = os.path.join(directory, font + ".flf") if not os.path.exists(fontPath): fontPath = os.path.join(directory, font + ".tlf") if not os.path.exists(fontPath): raise pyfiglet.FontNotFound("%s doesn't exist" % font) if is_zipfile(fontPath): z = None try: z = ZipFile(fontPath, "r") data = z.read(z.getinfo(z.infolist()[0].filename)) z.close() return data.decode("utf-8", "replace") if ST3 else data except Exception as e: if z is not None: z.close() raise pyfiglet.FontError("couldn't read %s: %s" % (fontPath, e)) else: try: with open(fontPath, "rb") as f: data = f.read() return data.decode("utf-8", "replace") if ST3 else data except Exception as e: raise pyfiglet.FontError("couldn't open %s: %s" % (fontPath, e)) raise pyfiglet.FontNotFound(font)
def compileToZip(self): """ Compile the exam as a .zip file """ def cleanpath(path): if path=='': return '' dirname, basename = os.path.split(path) dirname=cleanpath(dirname) if basename!='.': dirname = os.path.join(dirname,basename) return dirname f = ZipFile(self.options.output,'w') for (dst,src) in self.files.items(): dst = ZipInfo(cleanpath(dst)) dst.external_attr = 0o644<<16 dst.date_time = datetime.datetime.today().timetuple() if isinstance(src,basestring): f.writestr(dst,open(src,'rb').read()) else: f.writestr(dst,src.read()) print("Exam created in %s" % os.path.relpath(self.options.output)) f.close()
def get_info(in_stream): """ Return the version and submitter strings from zipfile byte stream. """ arch = ZipFile(in_stream, 'r') try: return unpack_info(arch.read('__INFO__')) finally: arch.close()
def getZipForLanguages(self, languages, path=None): self.verifyPaths() # strip out any duplicates languages = list(set(languages)) write_to = StringIO() if path is None else path zip_file = ZipFile(write_to, 'w') zip_file.write(self.getRainbowPath(), 'rainbow.js', zipfile.ZIP_DEFLATED) # include minimized version even when downloading the dev version zip_file.write(self.getRainbowPath().replace('.js', '.min.js'), 'rainbow.min.js', zipfile.ZIP_DEFLATED) # include themes as well if self.theme_path: files = glob.glob(self.theme_path + '/*.css') for file_name in files: zip_file.write(file_name, os.path.join('themes', os.path.basename(file_name)), zipfile.ZIP_DEFLATED) for language in languages: zip_file.write(self.getPathForLanguage(language), os.path.join('language', language + '.js'), zipfile.ZIP_DEFLATED) zip_file.close() return write_to
def testZipImporterMethodsInSubDirectory(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep files = {packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} z = ZipFile(TEMP_ZIP, "w") try: for name, (mtime, data) in files.items(): zinfo = ZipInfo(name, time.localtime(mtime)) zinfo.compress_type = self.compression z.writestr(zinfo, data) z.close() zi = zipimport.zipimporter(TEMP_ZIP + os.sep + packdir) self.assertEquals(zi.archive, TEMP_ZIP) self.assertEquals(zi.prefix, packdir) self.assertEquals(zi.is_package(TESTPACK2), True) zi.load_module(TESTPACK2) self.assertEquals(zi.is_package(TESTPACK2 + os.sep + '__init__'), False) self.assertEquals(zi.is_package(TESTPACK2 + os.sep + TESTMOD), False) mod_name = TESTPACK2 + os.sep + TESTMOD mod = __import__(module_path_to_dotted_name(mod_name)) self.assertEquals(zi.get_source(TESTPACK2), None) self.assertEquals(zi.get_source(mod_name), None) finally: z.close() os.remove(TEMP_ZIP)
def test_replace_metadata(self): old_zip_file = get_test_zipfile('LotsOfFiles') metadata = models.parse_zipfile_metadata(old_zip_file) old_zip_file.seek(0) extension = models.Extension.objects.create_from_metadata(metadata, creator=self.user) version = models.ExtensionVersion(extension=extension, source=File(old_zip_file)) version.parse_metadata_json(metadata) new_zip = version.get_zipfile('r') old_zip = ZipFile(File(old_zip_file), 'r') self.assertEqual(len(old_zip.infolist()), len(new_zip.infolist())) self.assertEqual(new_zip.read("metadata.json"), version.make_metadata_json_string()) for old_info in old_zip.infolist(): if old_info.filename == "metadata.json": continue new_info = new_zip.getinfo(old_info.filename) self.assertEqual(old_zip.read(old_info), new_zip.read(new_info)) self.assertEqual(old_info.date_time, new_info.date_time) old_zip.close() new_zip.close()
def logs(request): db = sqlite3.connect(r'C:\Users\Administrator\Desktop\Raspi\mysite\newdb.db') cursor = db.cursor() cursor.execute('SELECT * FROM connhistory') data = cursor.fetchall() with open("data.csv", 'wb')as csvfile: writer = csv.writer(csvfile) for row in data: writer.writerow(list(row)) with open("data.csv", 'rb')as csvfile: data = csvfile.read() in_memory = StringIO() zip = ZipFile(in_memory, "a") zip.writestr("data.csv",data) zip.close() response = HttpResponse(content_type="application/zip;") response["Content-Disposition"] = "attachment; filename=zipfiles.zip" in_memory.seek(0) response.write(in_memory.read()) return response
def export_zip(self, paths): stringio = StringIO() archive = ZipFile(stringio, mode='w') def _add_resource(resource): for filename in resource.get_files_to_archive(True): if filename.endswith('.metadata'): continue path = Path(self.handler.key).get_pathto(filename) archive.writestr(str(path), resource.handler.to_str()) for path in paths: child = self.get_resource(path, soft=True) if child is None: continue # A Folder => we add its content if isinstance(child, Folder): for subchild in child.traverse_resources(): if subchild is None or isinstance(subchild, Folder): continue _add_resource(subchild) else: _add_resource(child) archive.close() return stringio.getvalue()
exzip = exname + '.zip' try: os.mkdir(exname) print(f" - Temporary dir created: {exname}") shutil.copy(expdforig, os.path.join(exname, expdf)) print(f" - Copied exercise pdf file: {expdforig}") shutil.copytree("code", os.path.join(exname, "code")) print(" - Copied the code folder") print(" - Removing unnecessary files:") pdfpyfiles = set( glob.glob(exname + '/**/*.pdf', recursive=True) + glob.glob(exname + '/**/*.py', recursive=True)) allfiles = set(glob.glob(exname + '/**/*.*', recursive=True)) for f in allfiles.difference(pdfpyfiles): removeFileOrFolder(f) hiddenfilesorPaths = glob.glob(exname + '/**/.*') + glob.glob(exname + '/**/__*__') for f in hiddenfilesorPaths: removeFileOrFolder(f) zipf = ZipFile(exzip, 'w', ZIP_DEFLATED) zipdir(exname, zipf) zipf.close() print(f" - .zip file for upload created: {exzip}") shutil.rmtree(exname) print(" - Temporary dir removed") except: raise ( f"Error occured while creating submission .zip: {sys.exc_info()[0]}" )
with tempfile.NamedTemporaryFile(suffix='.zip') as tmp: # the file transfer can take a long time; by default # cherrypy limits responses to 300s; we increase it to 1h cherrypy.response.timeout = 3600 try: # create the zipfile archive = ZipFile(tmp.name, 'w') # add all the recording files for recording in selected_recordings: # ignore missing files if recording.fileExists(): archive.write(recording.abspath, recording.path) finally: # safer this way archive.close() # create a suitable name for the export if selected_date: served_filename = ('Recordings_%s.zip' % formatDateIso(selected_date)) elif filter_start == filter_finish: served_filename = ('Recordings_%s.zip' % formatDateIso(filter_start)) else: served_filename = ('Recordings_%s_to_%s.zip' % (formatDateIso(filter_start), formatDateIso(filter_finish))) # use cherrypy utility to push the file for download. This # also means that we don't have to move the config file into
for root, dirs, files in os.walk(PRJ_PATH): is_exclude_dir = root.startswith(tuple(exclude_dirs)) if (isNameMatch(os.path.basename(root), exclude) or is_exclude_dir) and root != PRJ_PATH: if not is_exclude_dir: exclude_dirs.append(root) continue else: for name in files: if not isNameMatch(name, exclude): fullname = os.path.join(root, name) arcname = fullname[len(PRJ_PAR_DIR):] if len(minify['dirs']) > 0 and root.startswith( tuple(minify['dirs'])) and fnmatch.fnmatch( name, '*.js') and not isNameMatch( name, minify['exclude']): fullname = minifyJs(fullname, arcname) z.write(fullname, arcname) print(arcname[1:], 'was added to the package\n') z.close() print('Package was created') if os.path.exists(TMP_DIR): print('\nDeleting temporary files...') shutil.rmtree(TMP_DIR, ignore_errors=False, onerror=None) print('Ok') writeToChangelogFile() print('Done!') input('\nPress "Enter" to exit')
End year: 2018 ''' # download the data from the source logger.info('Downloading raw data') download = glob.glob(os.path.join(os.path.expanduser("~"), 'Downloads', 'historical_emissions.zip'))[0] # move this file into your data directory raw_data_file = os.path.join(data_dir, os.path.basename(download)) shutil.move(download, raw_data_file) # unzip historical emissions data raw_data_file_unzipped = raw_data_file.split('.')[0] zip_ref = ZipFile(raw_data_file, 'r') zip_ref.extractall(raw_data_file_unzipped) zip_ref.close() ''' Process data ''' # read in historical emissions csv data to pandas dataframe filename = os.path.join(raw_data_file_unzipped, 'historical_emissions.csv') # filename = 'data/historical_emissions/historical_emissions.csv' df = pd.read_csv(filename) # add iso_a3 to the dataframe df_edit = pd.merge(df, df_carto[['iso_a3','name']], left_on = 'Country',right_on = 'name', how = 'left') # deal with exceptions df_edit.loc[df_edit['Country'] == 'United States','iso_a3'] = 'USA' df_edit.loc[df_edit['Country'] == 'Tanzania','iso_a3'] = 'TZA'
with open(DATA_FILE, "wb") as file: shutil.copyfileobj(data.raw, file) del data train_file = os.path.join(DATA_DIR, LING10_TRAIN_LARGE, "train_set.txt") test_file = os.path.join(DATA_DIR, LING10_TRAIN_LARGE, "test_set.txt") chars_map_file = os.path.join(DATA_DIR, LING10_TRAIN_LARGE, "chars.json") #If any of the content doesn't exist, extract the zip file if not os.path.exists(train_file) or not os.path.exists( test_file) or not os.path.exists(chars_map_file): extractor = ZipFile(DATA_FILE) extractor.extractall(DATA_DIR) extractor.close() #READ THE DATA FILES train_data = open(train_file, encoding="utf-8").read() test_data = open(test_file, encoding="utf-8").read() char_map_data = open(chars_map_file) #Load the char map char_map = json.load(char_map_data) char_to_idx = char_map["char_to_idx"] #Split the train and test into lines train_data = train_data.splitlines() test_data = test_data.splitlines()
""" Created on 2019年1月18日 @author: Irony @site: https://pyqt5.com https://github.com/892768447 @email: [email protected] @file: Test.buildzip @description: """ import os from pathlib import Path import shutil from zipfile import ZipFile, ZIP_DEFLATED __Author__ = "Irony" __Copyright__ = "Copyright (c) 2019" os.chdir('../') os.makedirs('dist', exist_ok=True) # 压缩文件到zip zipfp = ZipFile(os.path.abspath('release/Library.zip'), 'w', ZIP_DEFLATED) for file in Path('Library').rglob('*.pyc'): print('add file: %s' % file) zipfp.write(os.path.abspath(str(file)), str(file)[len('Library'):]) zipfp.close() shutil.rmtree('Library', ignore_errors=True)
def auto(quality, email, self): internet_date = urlopen('http://just-the-time.appspot.com/') result = internet_date.read().strip() result_str = result.decode('utf-8') result_str = result_str.replace(' ', '-') result_set = result_str.split('-') final_date = result_set[2] + result_set[1] + result_set[0] cwd = os.getcwd() DayList = [ 'MON', 'TUE', 'WED', 'THU', 'FRI', 'SAT', 'SUN', ] day_is = DayList[datetime.date(int(result_set[0]), int(result_set[1]), int(result_set[2])).weekday()] i = 0 j = 1 c = 0 download_Counter = 0 extension = "" percentage_Download = 0 static_url = 'http://epaper.livehindustan.com/epaperimages/' + final_date \ + '/' + final_date static_url = static_url + '-ng1r-del-' extension = ".jpg" while c != 2: QApplication.processEvents() url = static_url + str(i) + str(j) + extension h = requests.head(url, allow_redirects=True) header = h.headers content_type = header.get('content-type') if (quality): if 'image/jpeg' in content_type.lower(): urllib.request.urlretrieve( url, 'local-filename' + str(i * 10 + j) + '.jpeg') j = j + 1 c = 0 download_Counter = download_Counter + 1 percentage_Download = (download_Counter / 26) * 100 self.pbar.setValue(percentage_Download) print(str(int(percentage_Download)) + "% Downloaded.") else: c = c + 1 i = i + 1 j = 0 print('Downloads Completed.') self.pbar.setValue(100) file_name = day_is.upper() + result_set[0] + result_set[1] + result_set[2] with open(file_name + ".pdf", "wb") as f: f.write( img2pdf.convert( [i for i in os.listdir(cwd) if i.endswith(".jpeg")])) for i in os.listdir(cwd): if i.endswith('.jpeg'): os.remove(i) comp_file = file_name + '.pdf' while not os.path.exists(comp_file): time.sleep(1) if os.path.isfile(comp_file): ZipFile(file_name + '.zip', 'w').write(comp_file) ZipFile.close(ZipFile) else: raise ValueError("%s isn't a file!" % comp_file) fromaddr = 'Enter an email adress to send.' toaddr = email cwd = os.getcwd() msg = MIMEMultipart() msg['From'] = fromaddr msg['To'] = toaddr msg['Subject'] = 'NEWSPAPER for' + day_is body = 'Download the file and uncompress it. Read and Enjoy.' msg.attach(MIMEText(body, 'plain')) outfilename = file_name + '.zip' file_to_open = os.path.join(cwd, outfilename) attachment = open(file_to_open, 'rb') part = MIMEBase('application', 'octet-stream') part.set_payload(attachment.read()) encoders.encode_base64(part) part.add_header('Content-Disposition', 'attachment; filename= %s' % outfilename) msg.attach(part) server = smtplib.SMTP('smtp.gmail.com', 587) server.starttls() server.login(fromaddr, 'enter_password') text = msg.as_string() server.sendmail(fromaddr, toaddr, text) server.quit()
input() exit() if result[0] == True: newName = result[1][:-4] + lib.getDate() oldFDBPath = result[1] newFDBPath = newName + ".fdb" zipName = newName + ".zip" lib.copy(oldFDBPath, newFDBPath) zipObject = ZipFile(zipName, 'w', compression=zipfile.ZIP_BZIP2, compresslevel=9) print("compressing file...") zipObject.write(newFDBPath) zipObject.close() lib.restoreLowDash(oldFDBPath) print("local database is ready for work") print("creating auxiliary copy...") lib.deleteCopy(newFDBPath) print("auxiliary copy deleted") print("sending file...") response = (str(lib.sendFile(zipName, user, password))) print(response) if response != "<Response [200]>": print( "something went wrong, but now a database backup exists in the current dir :(" ) else: print("file sent")
def keyholemarkup2x(file,output='df', out_filename = None): """ Takes Keyhole Markup Language Zipped (KMZ) or KML file as input. The output is a pandas dataframe, geopandas geodataframe, csv, geojson, or shapefile. All core functionality from: http://programmingadvent.blogspot.com/2013/06/kmzkml-file-parsing-with-python.html Parameters ---------- file : {string} The string path to your KMZ or . output : {string} Defines the type of output. Valid selections include: - shapefile - 'shp', 'shapefile', or 'ESRI Shapefile' Returns ------- self : object """ r = re.compile(r'(?<=\.)km+[lz]?',re.I) try: extension = r.search(file).group(0) #(re.findall(r'(?<=\.)[\w]+',file))[-1] except IOError as e: logging.error("I/O error {0}".format(e)) if (extension.lower()=='kml') is True: buffer = file elif (extension.lower()=='kmz') is True: kmz = ZipFile(file, 'r') vmatch = np.vectorize(lambda x:bool(r.search(x))) A = np.array(kmz.namelist()) sel = vmatch(A) buffer = kmz.open(A[sel][0],'r') else: raise ValueError('Incorrect file format entered. Please provide the ' 'path to a valid KML or KMZ file.') parser = xml.sax.make_parser() handler = PlacemarkHandler() parser.setContentHandler(handler) parser.parse(buffer) try: kmz.close() except: pass df = pd.DataFrame(handler.mapping).T names = list(map(lambda x: x.lower(),df.columns)) if 'description' in names: extradata = df.apply(PlacemarkHandler.htmlizer,axis=1) df = df.join(extradata) output = output.lower() if output=='df' or output=='dataframe' or output == None: result = df elif output=='csv': out_filename = (out_filename or file[:-4]) + ".csv" df.to_csv(out_filename,encoding='utf-8',sep="\t") result = ("Successfully converted {0} to CSV and output to" " disk at {1}".format(file,out_filename)) elif output=='gpd' or output == 'gdf' or output=='geoframe' or output == 'geodataframe': try: import shapely from shapely.geometry import Polygon,LineString,Point except ImportError as e: raise ImportError('This operation requires shapely. {0}'.format(e)) try: import fiona except ImportError as e: raise ImportError('This operation requires fiona. {0}'.format(e)) try: import geopandas as gpd except ImportError as e: raise ImportError('This operation requires geopandas. {0}'.format(e)) geos = gpd.GeoDataFrame(df.apply(PlacemarkHandler.spatializer,axis=1)) result = gpd.GeoDataFrame(pd.concat([df,geos],axis=1)) elif output=='geojson' or output=='json': try: import shapely from shapely.geometry import Polygon,LineString,Point except ImportError as e: raise ImportError('This operation requires shapely. {0}'.format(e)) try: import fiona except ImportError as e: raise ImportError('This operation requires fiona. {0}'.format(e)) try: import geopandas as gpd except ImportError as e: raise ImportError('This operation requires geopandas. {0}'.format(e)) try: import geojson except ImportError as e: raise ImportError('This operation requires geojson. {0}'.format(e)) geos = gpd.GeoDataFrame(df.apply(PlacemarkHandler.spatializer,axis=1)) gdf = gpd.GeoDataFrame(pd.concat([df,geos],axis=1)) out_filename = file[:-3] + "geojson" gdf.to_file(out_filename,driver='GeoJSON') validation = geojson.is_valid(geojson.load(open(out_filename)))['valid'] if validation == 'yes': result = ("Successfully converted {0} to GeoJSON and output to" " disk at {1}".format(file,out_filename)) else: raise ValueError('The geojson conversion did not create a ' 'valid geojson object. Try to clean your ' 'data or try another file.') elif output=='shapefile' or output=='shp' or output =='esri shapefile': try: import shapely from shapely.geometry import Polygon,LineString,Point except ImportError as e: raise ImportError('This operation requires shapely. {0}'.format(e)) try: import fiona except ImportError as e: raise ImportError('This operation requires fiona. {0}'.format(e)) try: import geopandas as gpd except ImportError as e: raise ImportError('This operation requires geopandas. {0}'.format(e)) try: import shapefile except ImportError as e: raise ImportError('This operation requires pyshp. {0}'.format(e)) geos = gpd.GeoDataFrame(df.apply(PlacemarkHandler.spatializer,axis=1)) gdf = gpd.GeoDataFrame(pd.concat([df,geos],axis=1)) out_filename = (out_filename or file[:-4]) + ".shp" gdf.to_file(out_filename,driver='ESRI Shapefile') sf = shapefile.Reader(out_filename) import shapefile sf = shapefile.Reader(out_filename) if len(sf.shapes())>0: validation = "yes" else: validation = "no" if validation == 'yes': result = ("Successfully converted {0} to Shapefile and output to" " disk at {1}".format(file,out_filename)) else: raise ValueError('The Shapefile conversion did not create a ' 'valid shapefile object. Try to clean your ' 'data or try another file.') else: raise ValueError('The conversion returned no data; check if' ' you entered a correct output file type. ' 'Valid output types are geojson, shapefile,' ' csv, geodataframe, and/or pandas dataframe.') return result
def do_downloads(self,checkfolder=True,output=None): for x in list.children[2:]: if x.checked: if not hasattr(self,"progress"): self.progress = progress() root.add_child(self.progress) self.progress.height = 20 self.progress.width = 400 self.progress.rpos[1] = list.rpos[1]+list.height+20 self.progress.progress = 0 print self.dl_url+"/"+x.file serv = urllib2.urlopen(self.dl_url+x.file) size = int(serv.info()["Content-Length"]) read = 0 bytes = 0 cli = open(x.filename,"wb") s = time.time() bps = 0 while not Engine.quit_threads: r = serv.read(1024) if not r: break cli.write(r) read += len(r) bytes += len(r) self.progress.progress = read/float(size) if time.time()-s>1: bps = bytes/(time.time()-s) s = time.time() bytes = 0 self.progress.text = "%sKB/%sKB - %s KB/s"%(read/1000.0,size/1000.0,bps/1000.0) if output: self.progress.rpos = [0,0] self.progress.width = 256 self.progress.draw(output[0]) output[1]() for evt in pygame.event.get(): if evt.type == pygame.QUIT: raise SystemExit serv.close() cli.close() if os.path.exists(self.path+"/"+x.text): if not os.path.isdir(self.path+"/"+x.text): os.remove(self.path+"/"+x.text) else: removeall(self.path+"/"+x.text) if not os.path.exists(self.path+"/"+x.text): os.mkdir(self.path+"/"+x.text) try: z = ZipFile(x.filename,"r") except: print "File corrupt" return for name in z.namelist(): txt = z.read(name) if "/" in name: try: os.makedirs(self.path+"/"+x.text+"/"+name.rsplit("/",1)[0]) except: pass if not name.endswith("/"): f = open(self.path+"/"+x.text+"/"+name,"wb") f.write(txt) f.close() z.close() os.remove(x.filename) root.children.remove(self.progress) list.children.remove(x) if len(list.children)<=2: list.status_box.text = "No more new downloads." del self.progress
def table_data_export(request: HttpRequest) -> HttpResponse: """ View that handles generating and returning zip files containing csv files full of records. :param request: The incoming HTTP request. :return: The HTTP response to return to the user. """ table_data_file_name = "/tmp/tabledata.zip" if request.method == 'POST': map_data = request.POST.getlist('ids[]') # sort based on record type map_data.sort(key=lambda x: x[0]) # Lists to hold core ids of each record # I'm not proud of this who_items = [] sota_items = [] profile_items = [] org_items = [] cap_items = [] guide_items = [] case_items = [] expe_items = [] bluejustice_items = [] # Loop through each item, get the type, and add the core id to the relevant list for item in map_data: record = item.split('&') type = record[0] issf_core_id = record[1] if type == 'SSF Governance': cap_items.append(issf_core_id) elif type == 'SSF Guidelines': guide_items.append(issf_core_id) elif type == 'SSF Organization': org_items.append(issf_core_id) elif type == 'SSF Profile': profile_items.append(issf_core_id) elif type == 'State-of-the-Art in SSF Research': sota_items.append(issf_core_id) elif type == 'Who\'s Who in SSF': who_items.append(issf_core_id) elif type == 'Case Study': case_items.append(issf_core_id) elif type == 'SSF Experiences': expe_items.append(issf_core_id) elif type == 'SSF Blue Justice': bluejustice_items.append(issf_core_id) # Generate the zipfile containing all the records zipfile = ZipFile(table_data_file_name, 'w') cap_records = SSFCapacityNeed.objects.filter(issf_core_id__in=cap_items).values( 'issf_core_id', 'contributor_id__first_name', 'contributor_id__last_name', 'contribution_date', 'geographic_scope_type', 'capacity_need_title', 'capacity_need_description', 'capacity_need_category', 'capacity_need_type' ) guide_records = SSFGuidelines.objects.filter(issf_core_id__in=guide_items).values( 'issf_core_id', 'contributor_id__first_name', 'contributor_id__last_name', 'contribution_date', 'geographic_scope_type', 'title', 'location', 'start_day', 'start_month', 'start_year', 'end_day', 'end_month', 'end_year', 'organizer', 'purpose', 'link', 'activity_type', 'activity_coverage', 'ongoing' ) org_records = SSFOrganization.objects.filter(issf_core_id__in=org_items).values( 'issf_core_id', 'contributor_id__first_name', 'contributor_id__last_name', 'contribution_date', 'geographic_scope_type', 'organization_name', 'mission', 'address1', 'address2', 'prov_state', 'country__short_name', 'postal_code', 'city_town', 'year_established', 'ssf_defined', 'ssf_definition', 'organization_type_union', 'organization_type_support', 'organization_type_coop', 'organization_type_flag', 'organization_type_other', 'organization_type_other_text', 'motivation_voice', 'motivation_market', 'motivation_sustainability', 'motivation_economics', 'motivation_rights', 'motivation_collaboration', 'motivation_other', 'motivation_other_text', 'activities_capacity', 'activities_sustainability', 'activities_networking', 'activities_marketing', 'activities_collaboration', 'activities_other', 'activities_other_text', 'network_types_state', 'network_types_ssfos', 'network_types_community', 'network_types_society', 'network_types_ngos', 'network_types_other', 'network_types_other_text', 'achievements', 'success_factors', 'obstacles', 'organization_point' ) pro_records = SSFProfile.objects.filter(issf_core_id__in=profile_items).values( 'issf_core_id', 'contributor_id__first_name', 'contributor_id__last_name', 'contribution_date', 'geographic_scope_type', 'ssf_name', 'ssf_defined', 'ssf_definition', 'data_day', 'data_month', 'data_year', 'data_end_day', 'data_end_month', 'data_end_year', 'comments', 'sources', 'percent' ) sota_records = SSFKnowledge.objects.filter(issf_core_id__in=sota_items).values( 'issf_core_id', 'contribution_date', 'contributor__first_name', 'contributor__last_name', 'geographic_scope_type', 'publication_type__publication_type', 'other_publication_type', 'level1_title', 'level2_title', 'year', 'nonenglish_language__language_name', 'nonenglish_title', 'ssf_defined', 'ssf_definition', 'lsf_considered', 'fishery_type_details', 'gear_type_details', 'ecosystem_type_details', 'demographics_na', 'demographics_age', 'demographics_education', 'demographics_ethnicity', 'demographics_gender', 'demographics_health', 'demographics_income', 'demographics_religion', 'demographics_unspecified', 'demographics_other', 'demographics_other_text', 'demographic_details', 'employment_na', 'employment_full_time', 'employment_part_time', 'employment_seasonal', 'employment_unspecified', 'employment_details', 'stage_na', 'stage_pre_harvest', 'stage_harvest', 'stage_post_harvest', 'stage_unspecified', 'market_details', 'governance_details', 'management_details', 'research_method', 'method_specify_qualitative', 'method_specify_quantitative', 'method_specify_mixed', 'aim_purpose_question', 'theme_issue_details', 'solutions_offered', 'solution_details', 'explicit_implications_recommendations', 'implication_details', 'comments' ) case_records = SSFCaseStudies.objects.filter(issf_core_id__in=case_items).values( 'issf_core_id', 'contributor_id__first_name', 'contributor_id__last_name', 'contribution_date', 'geographic_scope_type', 'name', 'role', 'description_area', 'description_fishery', 'description_issues', 'issues_challenges', 'stakeholders', 'transdisciplinary', 'background_context', 'activities_innovation' ) expe_records = SSFExperiences.objects.filter(issf_core_id__in=expe_items).values( 'issf_core_id', 'contributor_id__first_name', 'contributor_id__last_name', 'contribution_date', 'geographic_scope_type', 'title', 'name', 'description' ) bluejustice_records = SSFBlueJustice.objects.filter(issf_core_id__in=bluejustice_items).values( 'issf_core_id', 'contributor_id__first_name', 'contributor_id__last_name', 'contribution_date', 'geographic_scope_type', 'name', 'email', 'affiliation', 'country', 'role', 'img_url', 'photo_location', 'date_of_photo', 'photographer', 'title', 'video_url', 'vimeo_video_url', 'description', 'contributor', 'ssf_name', 'ssf_location', 'ssf_country', 'ssf_main_species', 'ssf_type_aquaculture', 'ssf_type_recreational', 'ssf_type_commercial', 'ssf_type_subsistence', 'ssf_type_indigenous', 'ssf_type_other', 'ecosystem_type_marine', 'ecosystem_type_freshwater', 'ecosystem_type_brackish', 'ecosystem_detailed_archipelago', 'ecosystem_detailed_beach', 'ecosystem_detailed_coastal', 'ecosystem_detailed_coral_reef', 'ecosystem_detailed_deep_sea', 'ecosystem_detailed_estuary', 'ecosystem_detailed_fjord', 'ecosystem_detailed_intertidal', 'ecosystem_detailed_lagoon', 'ecosystem_detailed_lake', 'ecosystem_detailed_mangrove', 'ecosystem_detailed_open_ocean', 'ecosystem_detailed_river', 'ecosystem_detailed_salt_marsh', 'ecosystem_detailed_other', 'ssf_terms_artisanal', 'ssf_terms_coastal', 'ssf_terms_indigenous', 'ssf_terms_inland', 'ssf_terms_inshore', 'ssf_terms_small_boat', 'ssf_terms_small_scale', 'ssf_terms_subsistence', 'ssf_terms_traditional', 'ssf_terms_others', 'ssf_terms_fisheries', 'ssf_terms_fisheries_definiton', 'main_gears_dredge', 'main_gears_lift_net', 'main_gears_cast_net', 'main_gears_poison', 'main_gears_gillnet', 'main_gears_recreational_fishing_gears', 'main_gears_gleaning', 'main_gears_seine_net', 'main_gears_harpoon', 'main_gears_surrounding_net', 'main_gears_harvesting_machines', 'main_gears_traps', 'main_gears_hook_line', 'main_gears_trawls', 'main_gears_others', 'main_vessel_type', 'main_vessel_number', 'main_vessel_engine', 'ss_fishers_numbers', 'ss_fishers_full_time', 'ss_fishers_women', 'total_number_households', 'households_participation_percentage', 'background_about_ssf', 'justice_in_context', 'types_of_justice_distributive', 'types_of_justice_social', 'types_of_justice_economic', 'types_of_justice_market', 'types_of_justice_infrastructure', 'types_of_justice_regulatory', 'types_of_justice_procedural', 'types_of_justice_environmental', 'types_of_justice_others', 'dealing_with_justice', 'covid_19_related' ) who_records = SSFPerson.objects.filter(issf_core_id__in=who_items).values( 'issf_core_id', 'contributor_id__first_name', 'contributor_id__last_name', 'contribution_date', 'geographic_scope_type', 'number_publications', 'education_level', 'research_method', 'issues_addressed', 'url', 'other_education_level', 'affiliation', 'address1', 'address2', 'city_town', 'prov_state', 'country__short_name', 'postal_code', 'is_researcher', 'person_point' ) write_file_csv('capacity.csv', cap_records, zipfile) write_file_csv('guidelines.csv', guide_records, zipfile) write_file_csv('organization.csv', org_records, zipfile) write_file_csv('profile.csv', pro_records, zipfile) write_file_csv('state_of_the_art.csv', sota_records, zipfile) write_file_csv('case_studies.csv', case_records, zipfile) write_file_csv('experiences.csv', expe_records, zipfile) write_file_csv('bluejustice.csv', bluejustice_records, zipfile) write_file_csv('whos_who.csv', who_records, zipfile) main_attrs = MainAttributeView.objects.filter(issf_core_id__in=profile_items).values( 'issf_core_id', 'attribute__question_number', 'attribute__attribute_label', 'value', 'attribute__units_label', 'attribute_value__value_label', 'other_value', 'additional', 'additional_value__value_label' ) write_file_csv('main_attributes.csv', main_attrs, zipfile) author_records = KnowledgeAuthorSimple.objects.filter(knowledge_core__in=sota_items).values() write_file_csv('authors.csv', author_records, zipfile) all_ids = cap_items + guide_items + org_items + profile_items + sota_items + who_items + expe_items + case_items theme_issue_records = CommonThemeIssueView.objects.filter(issf_core_id__in=all_ids).values( 'issf_core_id', 'selected_theme_issue_id', 'theme_issue_value__theme_issue_label', 'theme_issue_value__theme_issue__theme_issue_category', 'other_theme_issue' ) characteristic_records = CommonAttributeView.objects.filter(issf_core_id__in=all_ids).values( 'issf_core_id', 'selected_attribute_id', 'attribute__attribute_category', 'attribute__attribute_label', 'attribute__units_label', 'attribute__additional_field', 'attribute_value__value_label', 'other_value' ) write_file_csv('themes_issues.csv', theme_issue_records, zipfile) write_file_csv('characteristics.csv', characteristic_records, zipfile) geog_scope_local_records = GeographicScopeLocalArea.objects.filter(issf_core_id__in=all_ids).values( 'geographic_scope_local_area_id', 'issf_core_id', 'local_area_name', 'local_area_alternate_name', 'country__short_name', 'local_area_setting', 'local_area_setting_other', 'local_area_point' ) geog_scope_regional_records = Geographic_Scope_Region.objects.filter(issf_core_id__in=all_ids).values( 'geographic_scope_region_id', 'issf_core_id', 'region__region_name', 'region_name_other' ) geog_scope_subnational_records = GeographicScopeSubnation.objects.filter(issf_core_id__in=all_ids).values( 'geographic_scope_subnation_id', 'issf_core_id', 'subnation_name', 'country__short_name', 'subnation_type', 'subnation_type_other', 'subnation_point' ) geog_scope_national_records = GeographicScopeNation.objects.filter(issf_core_id__in=all_ids).values( 'geographic_scope_nation_id', 'issf_core_id', 'country__short_name' ) species_records = Species.objects.filter(issf_core_id__in=all_ids).defer('species_id').values() write_file_csv('geog_scope_local.csv', geog_scope_local_records, zipfile) write_file_csv('geog_scope_regional.csv', geog_scope_regional_records, zipfile) write_file_csv('geog_scope_subnational.csv', geog_scope_subnational_records, zipfile) write_file_csv('geog_scope_national.csv', geog_scope_national_records, zipfile) write_file_csv('species.csv', species_records, zipfile) zipfile.write('/issf/export/README within exported zip files.txt', 'README.txt') zipfile.close() return HttpResponse("Created tabledata.zip") else: # Return the already generated zipfile if os.path.isfile(table_data_file_name): zipfile = open(table_data_file_name, 'rb') response = HttpResponse(zipfile, content_type='application/x-zip-compressed') response['Content-Disposition'] = 'attachment; filename="tabledata.zip"' else: response = HttpResponse("No tabledata.zip") return response
def create_offline_file(job, offline_job): offline_job.status = 'Running' offline_job.save() try: # Temp dir string temp_dir = job.random_id # Create the directory os.mkdir(temp_dir) # Temp dir string temp_dir_string = temp_dir + '/' # Create an Array for IDs and the component data component_data = {} # Iterate over all diff data for array for component in job.sorted_component_list(): if component.diff_html: component_data['diff-' + str(component.id)] = component.diff_html # Iterate over all metadata for the array for component in Component.objects.filter(component_type__org__in=[ job.sorted_orgs()[0], job.sorted_orgs()[1] ]): component_data['component-' + str(component.id)] = component.content # Create JSON file component_json = open(temp_dir_string + 'components.json', 'w+') component_json.write('var component_data = ' + json.dumps(component_data) + ';') component_json.close() # Create html file compare_result = open(temp_dir_string + 'compare_results_offline.html', 'w+') # Build the html using the template contentxt t = loader.get_template('compare_results_offline.html') c = Context({ 'org_left_username': job.sorted_orgs()[0].username, 'org_right_username': job.sorted_orgs()[1].username, 'html_rows': ''.join( list(job.sorted_component_list().values_list('row_html', flat=True))) }) # Write template contents to file compare_result.write(t.render(c)) compare_result.close() # Create zip file for all content zip_file = ZipFile(temp_dir_string + 'compare_results.zip', 'w') # Add JSON files zip_file.write(temp_dir_string + 'components.json', 'data/components.json') # Add html file zip_file.write(temp_dir_string + 'compare_results_offline.html', 'compare_results_offline.html') # Add all static files for root, dirs, files in os.walk('staticfiles'): for file in files: zip_file.write(os.path.join(root, file)) # Close the file zip_file.close() # Re-open the file zip_file = open(temp_dir_string + 'compare_results.zip') # Save file to model job.zip_file.save(temp_dir + '.zip', File(zip_file)) job.save() # Close the file again zip_file.close() # Remove the files and directories for f in glob.glob(temp_dir_string + '*'): os.remove(f) os.rmdir(temp_dir) # Update status to finished offline_job.status = 'Finished' except Exception as error: offline_job.status = 'Error' offline_job.error = error offline_job.error_stacktrace = traceback.format_exc() offline_job.save()
def run(self): f = ZipFile(self.outfile, 'w', ZIP_DEFLATED) f.write(self.infile) f.close() print('Finished child_process zip of:', self.infile)
def profile_csv(request: HttpRequest) -> HttpResponse: """ URL that generates and returns a zipfile of csv files for GCPC. Now unused. :param request: The incoming HTTP request. :return: The HTTP response to return to the user. """ profile_records = SSFProfile.objects.all().values( 'issf_core_id', 'contributor_id__first_name', 'contributor_id__last_name', 'contribution_date', 'geographic_scope_type', 'ssf_name', 'ssf_defined', 'ssf_definition', 'data_day', 'data_month', 'data_year', 'data_end_day', 'data_end_month', 'data_end_year', 'comments', 'sources', 'percent' ) zipfile = ZipFile('profile_data.zip', 'w') main_attrs = MainAttributeView.objects.all().values( 'issf_core_id', 'attribute__question_number', 'attribute__attribute_label', 'value', 'attribute__units_label', 'attribute_value__value_label', 'other_value', 'additional', 'additional_value__value_label' ) write_file_csv('profile.csv', profile_records, zipfile) write_file_csv('main_attributes.csv', main_attrs, zipfile) geog_scope_local_records = GeographicScopeLocalArea.objects.filter(issf_core__core_record_type='SSF Profile').values( 'geographic_scope_local_area_id', 'issf_core_id', 'local_area_name', 'local_area_alternate_name', 'country__short_name', 'local_area_setting', 'local_area_setting_other', 'local_area_point' ) geog_scope_regional_records = Geographic_Scope_Region.objects.filter(issf_core__core_record_type='SSF Profile').values( 'geographic_scope_region_id', 'issf_core_id', 'region__region_name', 'region_name_other' ) geog_scope_subnational_records = GeographicScopeSubnation.objects.filter(issf_core__core_record_type='SSF Profile').values( 'geographic_scope_subnation_id', 'issf_core_id', 'subnation_name', 'country__short_name', 'subnation_type', 'subnation_type_other', 'subnation_point' ) geog_scope_national_records = GeographicScopeNation.objects.filter(issf_core__core_record_type='SSF Profile').values( 'geographic_scope_nation_id', 'issf_core_id', 'country__short_name' ) write_file_csv('geog_scope_local.csv', geog_scope_local_records, zipfile) write_file_csv('geog_scope_regional.csv', geog_scope_regional_records, zipfile) write_file_csv('geog_scope_subnational.csv', geog_scope_subnational_records, zipfile) write_file_csv('geog_scope_national.csv', geog_scope_national_records, zipfile) zipfile.close() zipfile = open('profile_data.zip', 'rb') response = HttpResponse(zipfile, content_type='application/x-zip-compressed') response['Content-Disposition'] = 'attachment; filename="profile_data.zip"' return response
def download_image(self): ARCHIVE_TYPES = ['zip'] suffix = re.compile(r'^.*?[.](?P<ext>tar\.gz|tar\.bz2|\w+)$').match( self.image_name).group('ext') # Check if file exists already: if path.isfile(self.image_name): if self.checksum_verification(): logger.info('Local image found, skipping download: %s', self.local_file_path) if suffix not in ARCHIVE_TYPES: return True else: os.remove(self.local_file_path) if not path.isfile(self.image_name): # Download file to cli-tool-client try: request.urlretrieve(self.raw_image_url, self.local_file_path) except URLError: logger.exception('Failed download of image using urllib') return False self.checksum_verification() # Unzips image when suffix is zip or tar.gz and then changes image name to extracted one. # For EC2 and SCVMM images is zip used and for GCE is tar.gz used. archive_path = self.image_name if suffix not in ARCHIVE_TYPES: return True else: if suffix == 'zip': try: archive = ZipFile(archive_path) zipinfo = archive.infolist() self._unzipped_file = zipinfo[0].filename except Exception: logger.exception( "Getting information of {} archive failed.".format( self.image_name)) return False if path.isfile(self.image_name): try: os.remove(self.image_name) except Exception: logger.exception( "Deleting previously unpacked file {} failed.". format(self.image_name)) return False logger.info( f"Image archived - unpacking as : {self._unzipped_file}") try: archive.extractall() archive.close() # remove the archive os.remove(archive_path) return True except Exception: logger.exception(f"{suffix} archive unpacked failed.") return False
import os from zipfile import ZipFile directory = '/Users/Albaba/Library/Mobile Documents/com~apple~CloudDocs/Coding Workplaces/Repos/Web Programming/midterm' # create a ZipFile object zipObj = ZipFile('text_files.zip', 'w') new_dir = directory + '/text_files' for filename in os.listdir(directory): if filename.endswith(".php") or filename.endswith(".css"): #print(filename) #zipObj.write(os.path.join(directory, filename)) else: continue #close zip zipObj.close()
def compile_templates(self, target, extensions=None, filter_func=None, zip='deflated', log_function=None, ignore_errors=True, py_compile=False): """Finds all the templates the loader can find, compiles them and stores them in `target`. If `zip` is `None`, instead of in a zipfile, the templates will be stored in a directory. By default a deflate zip algorithm is used. To switch to the stored algorithm, `zip` can be set to ``'stored'``. `extensions` and `filter_func` are passed to :meth:`list_templates`. Each template returned will be compiled to the target folder or zipfile. By default template compilation errors are ignored. In case a log function is provided, errors are logged. If you want template syntax errors to abort the compilation you can set `ignore_errors` to `False` and you will get an exception on syntax errors. If `py_compile` is set to `True` .pyc files will be written to the target instead of standard .py files. This flag does not do anything on pypy and Python 3 where pyc files are not picked up by itself and don't give much benefit. .. versionadded:: 2.4 """ from jinja2.loaders import ModuleLoader if log_function is None: log_function = lambda x: None if py_compile: if not PY2 or PYPY: from warnings import warn warn(Warning('py_compile has no effect on pypy or Python 3')) py_compile = False else: import imp import marshal py_header = imp.get_magic() + \ '\xff\xff\xff\xff'.encode('iso-8859-15') # Python 3.3 added a source filesize to the header if sys.version_info >= (3, 3): py_header += '\x00\x00\x00\x00'.encode('iso-8859-15') def write_file(filename, data, mode): if zip: info = ZipInfo(filename) info.external_attr = 0o755 << 16 zip_file.writestr(info, data) else: f = open(os.path.join(target, filename), mode) try: f.write(data) finally: f.close() if zip is not None: from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED, ZIP_STORED zip_file = ZipFile(target, 'w', dict(deflated=ZIP_DEFLATED, stored=ZIP_STORED)[zip]) log_function('Compiling into Zip archive "%s"' % target) else: if not os.path.isdir(target): os.makedirs(target) log_function('Compiling into folder "%s"' % target) try: for name in self.list_templates(extensions, filter_func): source, filename, _ = self.loader.get_source(self, name) try: code = self.compile(source, name, filename, True, True) except TemplateSyntaxError as e: if not ignore_errors: raise log_function('Could not compile "%s": %s' % (name, e)) continue filename = ModuleLoader.get_module_filename(name) if py_compile: c = self._compile(code, encode_filename(filename)) write_file(filename + 'c', py_header + marshal.dumps(c), 'wb') log_function('Byte-compiled "%s" as %s' % (name, filename + 'c')) else: write_file(filename, code, 'w') log_function('Compiled "%s" as %s' % (name, filename)) finally: if zip: zip_file.close() log_function('Finished compiling templates')
'X': '#636363', 'Y': '#00ecff', 'Z': '#d9d9d9', } if not os.path.exists('dist'): os.mkdir('dist') with open('manifest.json') as f: manifest_template = f.read() with open('addon.js') as f: addon_template = f.read() with Image.open('icon.png') as icon: for letter, color in letters.items(): addon = addon_template.replace('{color}', color).replace('{letter}', letter) manifest = manifest_template.replace('{letter}', letter) filename = 'dist/%s-in-status-bar.zip' % letter.lower() zip_file = ZipFile(filename, mode='w') zip_file.writestr('manifest.json', manifest) zip_file.writestr('addon.js', addon) rgb = [ord(c) for c in color[1:].decode('hex')] icon.putpalette([0, 0, 0, rgb[0], rgb[1], rgb[2]]) icon_contents = StringIO() icon.save(icon_contents, format='PNG') icon_contents.seek(0) zip_file.writestr('icon.png', icon_contents.read()) zip_file.close()
def pack_content(path, is_peti): """Pack any custom content into the map. Filelist format: "[control char]filename[\t packname]" Filename is the name of the actual file. If given packname is the name to save it into the packfile as. If the first character of the filename is '#', the file will be added to the soundscript manifest too. """ files = set() # Files to pack. soundscripts = set() # Soundscripts need to be added to the manifest too.. rem_soundscripts = set( ) # Soundscripts to exclude, so we can override the sounds. particles = set() additional_files = set() # .vvd files etc which also are needed. preload_files = set() # Files we want to force preloading try: pack_list = open(path[:-4] + '.filelist.txt') except (IOError, FileNotFoundError): pass # Assume no files if missing.. # There might still be things to inject. else: with pack_list: for line in pack_list: line = line.strip().lower() if not line or line.startswith('//'): continue # Skip blanks or comments if line[:8] == 'precache': preload_files.add(line) continue if line[:2] == '-#': rem_soundscripts.add(line[2:]) continue if line[:1] == '#': line = line[1:] soundscripts.add(line) # We need to add particle systems to a manifest. if line.startswith('particles/'): particles.add(line) if line[-4:] == '.mdl': additional_files.update( {line[:-4] + ext for ext in MDL_ADDITIONAL_EXT}) files.add(line) # Remove guessed files not in the original list. additional_files -= files # Only generate a soundscript for PeTI maps.. if is_peti: music_data = CONF.find_key('MusicScript', []) if music_data.value: generate_music_script(music_data, files) # Add the new script to the manifest file.. soundscripts.add('scripts/BEE2_generated_music.txt') # We still generate these in hammer-mode - it's still useful there. # If no files are packed, no manifest will be added either. gen_sound_manifest(soundscripts, rem_soundscripts) gen_part_manifest(particles) gen_auto_script(preload_files, is_peti) inject_names = list(inject_files()) # Abort packing if no packfiles exist, and no injected files exist either. if not files and not inject_names: LOGGER.info('No files to pack!') return LOGGER.info('Files to pack:') for file in sorted(files): # \t seperates the original and in-pack name if used. LOGGER.info(' # "' + file.replace('\t', '" as "') + '"') if additional_files and LOGGER.isEnabledFor(logging.DEBUG): LOGGER.info('Potential additional files:') for file in sorted(additional_files): LOGGER.debug(' # "' + file + '"') LOGGER.info('Injected files:') for _, file in inject_names: LOGGER.info(' # "' + file + '"') LOGGER.info("Packing Files!") bsp_file = BSP(path) LOGGER.debug(' - Header read') bsp_file.read_header() # Manipulate the zip entirely in memory zip_data = BytesIO() zip_data.write(bsp_file.get_lump(BSP_LUMPS.PAKFILE)) zipfile = ZipFile(zip_data, mode='a') LOGGER.debug(' - Existing zip read') zip_write = get_zip_writer(zipfile) for file in files: pack_file(zip_write, file) for file in additional_files: pack_file(zip_write, file, suppress_error=True) for filename, arcname in inject_names: LOGGER.info('Injecting "{}" into packfile.', arcname) zip_write(filename, arcname) LOGGER.debug(' - Added files') zipfile.close() # Finalise the zip modification # Copy the zipfile into the BSP file, and adjust the headers bsp_file.replace_lump( path, BSP_LUMPS.PAKFILE, zip_data.getvalue(), # Get the binary data we need ) LOGGER.debug(' - BSP written!') LOGGER.info("Packing complete!")
class AudioPositions(Positions): def __init__(self, zip_filename, position_file_name=None, is_time=True, fps=48000,audio_metadata_extractor= None): import re self.zip_filename = zip_filename self.fps = fps self.audio_metadata_extractor = self.audio_metadata_extractor if audio_metadata_extractor is None else audio_metadata_extractor self.dir = os.path.dirname(os.path.abspath(self.zip_filename)) file_type_matcher = re.compile( '.*\.(' + '|'.join([ft[1][ft[1].rfind('.') + 1:] for ft in audiofiletypes]) + ')') self.myzip = ZipFile(zip_filename, 'r') self.names = [name for name in self.myzip.namelist() if len(file_type_matcher.findall(name.lower())) > 0 and \ os.path.basename(name) == name] self.file_meta = {} if position_file_name is not None: Positions.__init__(self,Positions.read(position_file_name, is_time=is_time, fps=48000).positions,fps=48000) for position in self.positions: if position[2] not in self.names: raise ValueError('Missing {} from audio zip file'.format(position[2])) else: meta = self.audio_metadata_extractor(self.names[0]) self.fps = int(meta['sample_rate']) positions = [] positions.append( Positions.to_segment(0, is_time=True, fps=self.fps) + (self.names[0],)) last = 0 for name_pos in range(len(self.names)-1): # get duration of last for start of next if name_pos > 0: meta = self.audio_metadata_extractor(self.names[name_pos]) last = float(meta['duration'])*1000.0 + last if int(meta['sample_rate']) != self.fps: raise ValueError('Mismatched sample rate {} from audio zip file {}'.format(self.fps,int(meta['sample_rate']) )) positions.append(Positions.to_segment(last, is_time=True, fps=int(meta['sample_rate'])) + (self.names[name_pos+1],)) Positions.__init__(self,positions, fps=fps) def audio_metadata_extractor(self, filename): place = self.get_file(filename) meta = get_meta_from_video(place, show_streams=True, media_types=['audio'])[0] return [x for x in meta if len(x) > 0][0] def get_file(self, name): path = os.path.join(self.dir,name) if os.path.exists(path): return path if os.path.exists(name): return name return self.myzip.extract(name, self.dir) def _get_duration(self, position_file): if os.path.basename(position_file) not in self.file_meta: meta = self.audio_metadata_extractor(position_file) self.file_meta[os.path.basename(position_file)] = meta else: meta = self.file_meta[os.path.basename(position_file)] return float(meta['duration'])*1000.0 def isOpened(self): #TODO: check names, what else return True def release(self): import shutil shutil.rmtree(self.dir) self.myzip.close()
def unzip(filepath, dest): zipfile = ZipFile(filepath) zipfile.extractall(dest) zipfile.close()
class StaticEventCreator(object): """Define process which generates a static (offline) version of an Indico event.""" def __init__(self, rh, event): self._rh = rh self.event = event self._display_tz = self.event.display_tzinfo.zone self._zip_file = None self._content_dir = _normalize_path(u'OfflineWebsite-{}'.format(event.title)) self._web_dir = os.path.join(get_root_path('indico'), 'web') self._static_dir = os.path.join(self._web_dir, 'static') def create(self): """Trigger the creation of a ZIP file containing the site.""" temp_file = NamedTemporaryFile(suffix='indico.tmp', dir=config.TEMP_DIR) self._zip_file = ZipFile(temp_file.name, 'w', allowZip64=True) with collect_static_files() as used_assets: # create the home page html html = self._create_home().encode('utf-8') # Mathjax plugins can only be known in runtime self._copy_folder(os.path.join(self._content_dir, 'static', 'dist', 'js', 'mathjax'), os.path.join(self._static_dir, 'dist', 'js', 'mathjax')) # Materials and additional pages self._copy_all_material() self._create_other_pages() # Create index.html file (main page for the event) index_path = os.path.join(self._content_dir, 'index.html') self._zip_file.writestr(index_path, html) self._write_generated_js() # Copy static assets to ZIP file self._copy_static_files(used_assets) self._copy_plugin_files(used_assets) if config.CUSTOMIZATION_DIR: self._copy_customization_files(used_assets) temp_file.delete = False chmod_umask(temp_file.name) self._zip_file.close() return temp_file.name def _write_generated_js(self): global_js = generate_global_file().encode('utf-8') user_js = generate_user_file().encode('utf-8') i18n_js = u"window.TRANSLATIONS = {};".format(generate_i18n_file(session.lang)).encode('utf-8') gen_path = os.path.join(self._content_dir, 'assets') self._zip_file.writestr(os.path.join(gen_path, 'js-vars', 'global.js'), global_js) self._zip_file.writestr(os.path.join(gen_path, 'js-vars', 'user.js'), user_js) self._zip_file.writestr(os.path.join(gen_path, 'i18n', session.lang + '.js'), i18n_js) def _copy_static_files(self, used_assets): # add favicon used_assets.add('static/images/indico.ico') # assets css_files = {url for url in used_assets if re.match('static/dist/.*\.css$', url)} for file_path in css_files: with open(os.path.join(self._web_dir, file_path)) as f: rewritten_css, used_urls, __ = rewrite_css_urls(self.event, f.read()) used_assets |= used_urls self._zip_file.writestr(os.path.join(self._content_dir, file_path), rewritten_css) for file_path in used_assets - css_files: if not re.match('^static/(images|fonts|dist)/(?!js/ckeditor/)', file_path): continue self._copy_file(os.path.join(self._content_dir, file_path), os.path.join(self._web_dir, file_path)) def _copy_plugin_files(self, used_assets): css_files = {url for url in used_assets if re.match('static/plugins/.*\.css$', url)} for file_path in css_files: plugin_name, path = re.match(r'static/plugins/([^/]+)/(.+.css)', file_path).groups() plugin = plugin_engine.get_plugin(plugin_name) with open(os.path.join(plugin.root_path, 'static', path)) as f: rewritten_css, used_urls, __ = rewrite_css_urls(self.event, f.read()) used_assets |= used_urls self._zip_file.writestr(os.path.join(self._content_dir, file_path), rewritten_css) for file_path in used_assets - css_files: match = re.match(r'static/plugins/([^/]+)/(.+)', file_path) if not match: continue plugin_name, path = match.groups() plugin = plugin_engine.get_plugin(plugin_name) self._copy_file(os.path.join(self._content_dir, file_path), os.path.join(plugin.root_path, 'static', path)) def _strip_custom_prefix(self, url): # strip the 'static/custom/' prefix from the given url/path return '/'.join(url.split('/')[2:]) def _copy_customization_files(self, used_assets): css_files = {url for url in used_assets if re.match('static/custom/.*\.css$', url)} for file_path in css_files: with open(os.path.join(config.CUSTOMIZATION_DIR, self._strip_custom_prefix(file_path))) as f: rewritten_css, used_urls, __ = rewrite_css_urls(self.event, f.read()) used_assets |= used_urls self._zip_file.writestr(os.path.join(self._content_dir, file_path), rewritten_css) for file_path in used_assets - css_files: if not file_path.startswith('static/custom/'): continue self._copy_file(os.path.join(self._content_dir, file_path), os.path.join(config.CUSTOMIZATION_DIR, self._strip_custom_prefix(file_path))) def _create_home(self): return WPStaticSimpleEventDisplay(self._rh, self.event, self.event.theme).display() def _create_other_pages(self): pass def _copy_all_material(self): self._add_material(self.event, '') for contrib in self.event.contributions: if not contrib.can_access(None): continue self._add_material(contrib, "%s-contribution" % contrib.friendly_id) for sc in contrib.subcontributions: self._add_material(sc, "%s-subcontribution" % sc.friendly_id) for session_ in self.event.sessions: if not session.can_access(None): continue self._add_material(session_, "%s-session" % session_.friendly_id) def _add_material(self, target, type_): for folder in AttachmentFolder.get_for_linked_object(target, preload_event=True): for attachment in folder.attachments: if not attachment.can_access(None): continue if attachment.type == AttachmentType.file: dst_path = posixpath.join(self._content_dir, "material", type_, "{}-{}".format(attachment.id, attachment.file.filename)) with attachment.file.get_local_path() as file_path: self._copy_file(dst_path, file_path) def _copy_file(self, dest, src): """Copy a file from a source path to a destination inside the ZIP.""" self._zip_file.write(src, dest) def _copy_folder(self, dest, src): for root, subfolders, files in os.walk(src): dst_dirpath = os.path.join(dest, os.path.relpath(root, src)) for filename in files: src_filepath = os.path.join(src, root, filename) self._zip_file.write(src_filepath, os.path.join(dst_dirpath, filename))
def create_workflow(drops, cwl_filename, buffer): """ Create a CWL workflow from a given Physical Graph Template A CWL workflow consists of multiple files. A single file describing the workflow, and multiple files each describing one step in the workflow. All the files are combined into one zip file, so that a single file can be downloaded by the user. NOTE: CWL only supports workflow steps that are bash shell applications Non-BashShellApp nodes are unable to be implemented in CWL """ # search the drops for non-BashShellApp drops, # if found, the graph cannot be translated into CWL for index, node in enumerate(drops): dataType = node.get('dt', '') if dataType not in SUPPORTED_CATEGORIES: raise Exception('Node {0} has an unsupported category: {1}'.format( index, dataType)) # create list for command line tool description files step_files = [] # create the workflow cwl_workflow = cwlgen.Workflow('', label='', doc='', cwl_version='v1.0') # create files dictionary files = {} # look for input and output files in the pg_spec for index, node in enumerate(drops): command = node.get('command', None) dataType = node.get('dt', None) outputId = node.get('oid', None) outputs = node.get('outputs', []) if len(outputs) > 0: files[outputs[0]] = "step" + str(index) + "/output_file_0" # add steps to the workflow for index, node in enumerate(drops): dataType = node.get('dt', '') if dataType == 'BashShellApp': name = node.get('nm', '') inputs = node.get('inputs', []) outputs = node.get('outputs', []) # create command line tool description filename = "step" + str(index) + ".cwl" contents = create_command_line_tool(node) # add contents of command line tool description to list of step files step_files.append({"filename": filename, "contents": contents}) # create step step = cwlgen.WorkflowStep("step" + str(index), run=filename) # add input to step for index, input in enumerate(inputs): step.inputs.append( cwlgen.WorkflowStepInput('input_file_' + str(index), source=files[input])) # add output to step for index, output in enumerate(outputs): step.out.append( cwlgen.WorkflowStepOutput('output_file_' + str(index))) # add step to workflow cwl_workflow.steps.append(step) # put workflow and command line tool description files all together in a zip zipObj = ZipFile(buffer, 'w') for step_file in step_files: zipObj.writestr(step_file["filename"], six.b(step_file["contents"])) zipObj.writestr(cwl_filename, six.b(cwl_workflow.export_string())) zipObj.close()
import urllib from zipfile import ZipFile import os tagname = "1.0" zipname = "markdown-presenter-" + tagname + ".zip" zippath = "./" + zipname url = "https://github.com/jsakamoto/MarkdownPresenter/releases/download/v." + tagname + "/" + zipname # download zip archive of Markdown Presenter. urllib.urlretrieve(url, zippath) # ectract the zip. zfile = ZipFile(zippath) zfile.extractall(".") zfile.close() # clean up zip. os.remove(zippath) # launch default web browser to open Markdown Presenter # after one shot timer to wait for warming up HTTP daemon. def launch(): time.sleep(1) webbrowser.open("http://localhost:8000/Presenter.html") thread.start_new_thread(launch, ()) # start mini HTTP daemon.
def save(self, delete_zip_import=True, *args, **kwargs): """ If a zip file is uploaded, extract any images from it and add them to the gallery, before removing the zip file. """ super().save(*args, **kwargs) if self.zip_import: zip_file = ZipFile(self.zip_import) for name in zip_file.namelist(): data = zip_file.read(name) try: from PIL import Image image = Image.open(BytesIO(data)) image.load() image = Image.open(BytesIO(data)) image.verify() except ImportError: pass except: # noqa continue name = os.path.split(name)[1] # In python3, name is a string. Convert it to bytes. if not isinstance(name, bytes): try: name = name.encode("cp437") except UnicodeEncodeError: # File name includes characters that aren't in cp437, # which isn't supported by most zip tooling. They will # not appear correctly. tempname = name # Decode byte-name. if isinstance(name, bytes): encoding = charsetdetect(name)["encoding"] tempname = name.decode(encoding) # A gallery with a slug of "/" tries to extract files # to / on disk; see os.path.join docs. slug = self.slug if self.slug != "/" else "" path = os.path.join(GALLERIES_UPLOAD_DIR, slug, tempname) try: saved_path = default_storage.save(path, ContentFile(data)) except UnicodeEncodeError: from warnings import warn warn("A file was saved that contains unicode " "characters in its path, but somehow the current " "locale does not support utf-8. You may need to set " "'LC_ALL' to a correct value, eg: 'en_US.UTF-8'.") # The native() call is needed here around str because # os.path.join() in Python 2.x (in posixpath.py) # mixes byte-strings with unicode strings without # explicit conversion, which raises a TypeError as it # would on Python 3. path = os.path.join(GALLERIES_UPLOAD_DIR, slug, str(name, errors="ignore")) saved_path = default_storage.save(path, ContentFile(data)) self.images.create(file=saved_path) if delete_zip_import: zip_file.close() self.zip_import.delete(save=True)
def provision_group_purchase_branding(sln_group_purchase_settings, main_branding, language): if not sln_group_purchase_settings.branding_hash: logging.info("Storing GROUP PURCHASE branding") stream = ZipFile(StringIO(main_branding.blob)) try: new_zip_stream = StringIO() zip_ = ZipFile(new_zip_stream, 'w', compression=ZIP_DEFLATED) try: path = os.path.join(os.path.dirname(solutions.__file__), 'common', 'templates', 'brandings/app_jquery.tmpl.js') zip_.writestr("jquery.tmpl.min.js", file_get_contents(path)) path = os.path.join(os.path.dirname(solutions.__file__), 'common', 'templates', 'brandings/moment-with-locales.min.js') zip_.writestr("moment-with-locales.min.js", file_get_contents(path)) zip_.writestr( "app-translations.js", JINJA_ENVIRONMENT.get_template( "brandings/app_group_purchases_translations.js"). render({ 'language': language }).encode("utf-8")) path = os.path.join(os.path.dirname(solutions.__file__), 'common', 'templates', 'brandings/app_group_purchases.js') zip_.writestr("app.js", file_get_contents(path).encode("utf-8")) for file_name in set(stream.namelist()): str_ = stream.read(file_name) if file_name == 'branding.html': html = str_ # Remove previously added dimensions: html = re.sub( "<meta\\s+property=\\\"rt:dimensions\\\"\\s+content=\\\"\\[\\d+,\\d+,\\d+,\\d+\\]\\\"\\s*/>", "", html) html = re.sub( '<head>', """<head> <link href="jquery/jquery.mobile.inline-png-1.4.2.min.css" rel="stylesheet" media="screen"> <style type="text/css"> #group_purchases-empty{text-align: center;} div.groupPurchase{padding: 10px 10px 25px 10px;} img.groupPurchasePicture { width: 100%; margin-top: 10px; } div.backgoundLight{background: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAB3RJTUUH3gEYDyEEzIMX+AAAACZpVFh0Q29tbWVudAAAAAAAQ3JlYXRlZCB3aXRoIEdJTVAgb24gYSBNYWOV5F9bAAAADUlEQVQI12NgYGCQBAAAHgAaOwrXiAAAAABJRU5ErkJggg==");} div.backgoundDark{background: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAB3RJTUUH3gEYDyIY868YdAAAACZpVFh0Q29tbWVudAAAAAAAQ3JlYXRlZCB3aXRoIEdJTVAgb24gYSBNYWOV5F9bAAAADUlEQVQI12P4//+/JAAJFQMXEGL3cQAAAABJRU5ErkJggg==");} h2.title { margin: 0;} .subscribed { font-weight: bold; } </style> <script src="jquery/jquery-1.11.0.min.js"></script> <script src="jquery/jquery.mobile-1.4.2.min.js"></script> <script src="jquery.tmpl.min.js"></script> <script src="moment-with-locales.min.js"></script> <script src="rogerthat/rogerthat-1.0.js" type="text/javascript"></script> <script src="rogerthat/rogerthat.api-1.0.js" type="text/javascript"></script> <script src="app-translations.js" type="text/javascript"></script> <script src="app.js" type="text/javascript"></script>""", html) html = re.sub( '<nuntiuz_message/>', """<div data-role="popup" id="gp-popup" class="ui-content"> <a href="#" data-rel="back" data-role="button" data-theme="a" data-icon="delete" data-iconpos="notext" class="closePopupOverlay ui-btn-right">Close</a> <div id="gp-popup-content"></div> </div> <div id="menu"></div>""", html) zip_.writestr('app.html', html) else: zip_.writestr(file_name, str_) finally: zip_.close() branding_content = new_zip_stream.getvalue() new_zip_stream.close() sln_group_purchase_settings.branding_hash = put_branding( u"Group Purchase App", base64.b64encode(branding_content)).id sln_group_purchase_settings.put() except: logging.error("Failure while parsing group purchase app branding", exc_info=1) raise finally: stream.close()
def get_or_create_model_shapefile_def(model_loc, output_dir, generate, srid, check_source_file_last_modified=None): ''' input: model_loc: name of directory or zipfile of model (recognize .zip) output_dir: output directory of shapefiles generate: dict with sources to generate srid return: generated files source date ''' log.debug('!!!get_or_create_model_shapefile_def') #check source if not os.path.isfile(model_loc) and not os.path.isdir( model_loc.encode('utf8')): log.warning('source file is missing. ' + model_loc) return (False, None, None, ) if model_loc[-3:].lower() == 'zip': log.debug('read model from zipfile: {0}'.format(model_loc)) zip_file = ZipFile(model_loc, "r") sobek_gr = Stream(zip_file.read('network.gr').replace('\r\n', '\n')) sobek_struc = Stream(zip_file.read('network.st')) source_file_last_modified = datetime.datetime.fromtimestamp( os.stat(model_loc.encode('utf8'))[8]) zip_file.close() else: log.debug('read model from directory: {0}'.format(model_loc)) sobek_gr = open(os.path.join(model_loc, 'network.gr')) sobek_struc = open(os.path.join(model_loc, 'network.st')) source_file_last_modified = datetime.datetime.fromtimestamp( os.stat(os.path.join(model_loc.encode('utf8'), 'network.gr'))[8]) source_up_to_date = False log.debug( 'in db origin date is {0}'.format(check_source_file_last_modified)) log.debug('file date is {0}'.format(source_file_last_modified)) if (check_source_file_last_modified and ( check_source_file_last_modified >= source_file_last_modified)): log.info('source file is still up to date') source_up_to_date = True #check output log.debug('output dir for shapefile is ' + output_dir) if not os.path.isdir(output_dir): log.info('create output dir' + output_dir) os.makedirs(output_dir) shapefile_name = {} for type in ['nodes', 'branches', 'structures']: shapefile_name[type] = os.path.join(output_dir, type + '.shp') if generate[type] and os.path.isfile(shapefile_name[type]): if source_up_to_date: log.info('output for ' + type + ' exist and is up to date.') generate[type] = False else: log.info('output for ' + type + ' exist and is not up to date, ' \ 'or recreation is needed. Remove previous output.') os.remove(shapefile_name[type]) os.remove(shapefile_name[type].replace('.shp', '.shx')) os.remove(shapefile_name[type].replace('.shp', '.dbf')) os.remove(shapefile_name[type].replace('.shp', '.prj')) node_array = [] branch_array = [] node = {} node_dict = {} #start with reading source files if generate['nodes'] or generate['branches'] or generate['structures']: log.debug('read locations of nodes and branches') pool = sobek.File(sobek_gr) #original SRS oSRS = ogr.osr.SpatialReference() # warning oSRS.ImportFromEPSG(28992) gives wrong string # '+proj=sterea +lat_0=52.15616055555555 +lon_0=5.387638888888889 + # k=0.9999079 +x_0=155000 +y_0=463000 +ellps=bessel +units=m +no_defs ' oSRS.ImportFromProj4( "+proj=sterea +lat_0=52.15616055555555 +" \ "lon_0=5.38763888888889 +k=0.999908 +x_0=155000 +" \ "y_0=463000 +ellps=bessel +towgs84=565.237,50.0087," \ "465.658,-0.406857,0.350733,-1.87035,4.0812 +units=m +no_defs") #target SRS tSRS = ogr.osr.SpatialReference() tSRS.ImportFromEPSG(900913) poCT = ogr.osr.CoordinateTransformation(oSRS, tSRS) for candidate_grid in pool['GRID']: branch_id = candidate_grid['id'][0] table = candidate_grid['gr gr'][-1:][0] for row_no in range(table.rows()): node_id = table[row_no, 3] #check if node is not already in dict if not node_id in node.keys(): node[node_id] = True pnt = poCT.TransformPoint( table[row_no, 5], table[row_no, 6], 0.) node_array.append({'id': node_id, 'type': 0, 'x': int(pnt[0]), 'y': int(pnt[1])}) node_dict[node_id] = {'id': node_id, 'type': 0, 'x': int(pnt[0]), 'y': int(pnt[1])} sub_branch_id = table[row_no, 4] if (row_no != range(table.rows()) and sub_branch_id != ''): branch_array.append({'id': sub_branch_id, 'branch_id': branch_id, 'dist_from': table[row_no, 0], 'dist_to': table[row_no + 1, 0], 'type': 0, 'to': table[row_no + 1, 3], 'from': table[row_no, 3]}) del pool #start generating shapefiles if generate['nodes']: log.info('generate shapefile for nodes') #make shapefile for nodes # Create the file object, a layer, and an attribute t_srs = osr.SpatialReference() t_srs.SetFromUserInput('epsg:900913') drv = ogr.GetDriverByName('ESRI Shapefile') ds = drv.CreateDataSource(str(shapefile_name['nodes'])) layer = ds.CreateLayer( ds.GetName(), geom_type=ogr.wkbPoint, srs=t_srs) layer.CreateField(ogr.FieldDefn('id', ogr.OFTString)) layer.CreateField(ogr.FieldDefn('type', ogr.OFTInteger)) # Could loop the following over some number of features geom = ogr.Geometry(type=ogr.wkbPoint) fid = 0 for node in node_array: geom.SetPoint(0, node['x'], node['y']) feat = ogr.Feature(feature_def=layer.GetLayerDefn()) feat.SetGeometry(geom) feat.SetFID(fid) feat.SetField('id', node['id']) feat.SetField('type', node['type']) layer.CreateFeature(feat) fid = fid + 1 log.debug('nr of nodes {0}'.format(fid)) layer.SyncToDisk() # Clean up ds.Destroy() if generate['branches']: log.info('generate shapefile for breaches') #make shapefile for branches # Create the file object, a layer, and an attribute t_srs = osr.SpatialReference() t_srs.SetFromUserInput('epsg:900913') drv = ogr.GetDriverByName('ESRI Shapefile') dsb = drv.CreateDataSource(str(shapefile_name['branches'])) layerb = dsb.CreateLayer( dsb.GetName(), geom_type=ogr.wkbLineString, srs=t_srs) layerb.CreateField(ogr.FieldDefn('id', ogr.OFTString)) layerb.CreateField(ogr.FieldDefn('type', ogr.OFTInteger)) layerb.CreateField(ogr.FieldDefn('from', ogr.OFTString)) layerb.CreateField(ogr.FieldDefn('to', ogr.OFTInteger)) fid = 0 for branch in branch_array: line = ogr.Geometry(type=ogr.wkbLineString) line.AddPoint(node_dict[branch['from']]['x'], node_dict[branch['from']]['y']) line.AddPoint(node_dict[branch['to']]['x'], node_dict[branch['to']]['y']) feat = ogr.Feature(feature_def=layerb.GetLayerDefn()) feat.SetGeometry(line) feat.SetFID(fid) feat.SetField('id', branch['id']) feat.SetField('type', branch['type']) feat.SetField('from', branch['from']) feat.SetField('to', branch['to']) layerb.CreateFeature(feat) fid = fid + 1 log.debug('nr of branches {0}'.format(fid)) layerb.SyncToDisk() # Clean up dsb.Destroy() if generate['structures']: log.debug('read locations of structures') #read extra sources for structures pool = sobek.File(sobek_struc) struc_array = [] for candidate in pool['STRU']: struc = {} struc['id'] = candidate['id'][0] if 'nm' in candidate.keys(): struc['name'] = candidate['nm'][0] else: struc['name'] = '-' struc['branch_id'] = candidate['ci'][0] struc['dist'] = candidate['lc'][0] struc['type'] = -1 location_found = False for branch in branch_array: if ((branch['branch_id'] == struc['branch_id']) and ( struc['dist'] >= branch['dist_from']) and ( struc['dist'] < branch['dist_to'])): struc['x'] = (node_dict[branch['from']]['x'] + node_dict[branch['to']]['x']) / 2 struc['y'] = (node_dict[branch['from']]['y'] + node_dict[branch['to']]['y']) / 2 location_found = True break if location_found == False: log.warning('structure location not found for {0}'.format(struc)) struc['x'] = 0 struc['y'] = 0 if ((struc['id'][:4] == 'lkb_') or ( struc['id'][:6] == 'c_lkb_') or ( struc['name'][:4] == 'lkb_')): struc['origin'] = 1 else: struc['origin'] = 0 struc_array.append(struc) #make shapefile for structures t_srs = osr.SpatialReference() t_srs.SetFromUserInput('epsg:900913') drv = ogr.GetDriverByName('ESRI Shapefile') log.info('create shapefile ' + shapefile_name['structures']) ds = drv.CreateDataSource(str(shapefile_name['structures'])) layer = ds.CreateLayer(ds.GetName(), geom_type=ogr.wkbPoint, srs=t_srs) layer.CreateField(ogr.FieldDefn('id', ogr.OFTString)) layer.CreateField(ogr.FieldDefn('type', ogr.OFTInteger)) layer.CreateField(ogr.FieldDefn('origin', ogr.OFTInteger)) # Could loop the following over some number of features geom = ogr.Geometry(type=ogr.wkbPoint) fid = 0 for node in struc_array: geom.SetPoint(0, node['x'], node['y']) feat = ogr.Feature(feature_def=layer.GetLayerDefn()) feat.SetGeometry(geom) feat.SetFID(fid) feat.SetField('id', node['id']) feat.SetField('type', node['type']) feat.SetField('origin', node['origin']) layer.CreateFeature(feat) feat.Destroy() fid = fid + 1 log.debug('nr of structures {0}'.format(fid)) layer.SyncToDisk() # Clean up ds.Destroy() del struc_array del pool del node_array del branch_array del node del node_dict return (True, shapefile_name, source_file_last_modified, )
def scrape(self, chamber, session): # Unfortunately, you now have to request access to FTP. # This method of retrieving votes needs to be be changed or # fall back to traditional web scraping. if session == '2009': # 2009 files have a different delimiter and naming scheme. vote_data_url = 'ftp://www.ncleg.net/Bill_Status/Vote Data 2009.zip' naming_scheme = '{session}{file_label}.txt' delimiter = ";" else: vote_data_url = 'ftp://www.ncleg.net/Bill_Status/Votes%s.zip' % session naming_scheme = '{file_label}_{session}.txt' delimiter = "\t" fname, resp = self.urlretrieve(vote_data_url) # fname = "/Users/brian/Downloads/Vote Data 2009.zip" zf = ZipFile(fname) chamber_code = 'H' if chamber == 'lower' else 'S' # Members_YYYY.txt: tab separated # 0: id (unique only in chamber) # 1: H or S # 2: member name # 3-5: county, district, party # 6: mmUserId member_file = zf.open(naming_scheme.format(file_label='Members', session=session)) members = {} for line in member_file.readlines(): data = line.split(delimiter) if data[1] == chamber_code: members[data[0]] = data[2] # Votes_YYYY.txt # 0: sequence number # 1: chamber (S/H) # 2: date # 3: prefix # 4: bill_id # 5: yes votes # 6: no votes # 7: excused absences # 8: excused votes # 9: didn't votes # 10: total yes+no # 11: sponsor # 12: reading info # 13: info # 20: PASSED/FAILED # 21: legislative day vote_file = zf.open(naming_scheme.format(file_label='Votes', session=session)) bill_chambers = {'H':'lower', 'S':'upper'} votes = {} for line in vote_file.readlines(): data = line.split(delimiter) if len(data) < 24: self.warning('line too short %s', data) continue if data[1] == chamber_code: date = datetime.datetime.strptime(data[2][:16], '%Y-%m-%d %H:%M') if data[3][0] not in bill_chambers: # skip votes that aren't on bills self.log('skipping vote %s' % data[0]) continue votes[data[0]] = Vote(chamber, date, data[13], 'PASS' in data[20], int(data[5]), int(data[6]), int(data[7])+int(data[8])+int(data[9]), bill_chamber=bill_chambers[data[3][0]], bill_id=data[3]+data[4], session=session) member_vote_file = zf.open(naming_scheme.format(file_label='MemberVotes', session=session)) # 0: member id # 1: chamber (S/H) # 2: vote id # 3: vote chamber (always same as 1) # 4: vote (Y,N,E,X) # 5: pair ID (member) # 6: pair order # If a vote is paired then it should be counted as an 'other' for line in member_vote_file.readlines(): data = line.split(delimiter) if data[1] == chamber_code: try: member_voting = members[data[0]] except KeyError: self.debug('Member %s not found.' % data[0]) continue try: vote = votes[data[2]] except KeyError: self.debug('Vote %s not found.' % data[2]) continue # -1 votes are Lt. Gov, not included in count, so we add them if data[4] == 'Y' and not data[5]: if data[0] == '-1': vote['yes_count'] += 1 vote.yes(member_voting) elif data[4] == 'N' and not data[5]: if data[0] == '-1': vote['no_count'] += 1 vote.no(member_voting) else: # for some reason other_count is high for paired votes if data[5]: vote['other_count'] -= 1 # is either E: excused, X: no vote, or paired (doesn't count) vote.other(member_voting) for vote in votes.itervalues(): #vote.validate() vote.add_source(vote_data_url) self.save_vote(vote) # remove file zf.close() os.remove(fname)
def get_or_create_value_presentation_source( scenario, pt, get_animation_info, check_timestamp_original_sourcefile): ''' create PresentationSource objects for given Flooding.Scenario and PresentationType input: scenario: Flooding.Scenario object pt: PresentationType object check_timestamp_original_sourcefile output: source object if there are no Results to fill the presentationLayer, this function throws an exception In case of an error, an exeption is thrown. ''' log.debug('!!!get_or_create_value_presentation_source') dest_dir = settings.EXTERNAL_RESULT_MOUNTED_DIR presentation_dir = settings.EXTERNAL_PRESENTATION_MOUNTED_DIR try: animation = {} result = scenario.result_set.get(resulttype__presentationtype=pt) link_type, _ = SourceLinkType.objects.get_or_create( name='flooding_scenario_value') source_link, new = SourceLink.objects.get_or_create( link_id=str(scenario.id), sourcelinktype=link_type, type='fl_rlt_' + str(result.id)) source, new = source_link.presentationsource.get_or_create( type=PresentationSource.SOURCE_TYPE_ZIPPED_HISFILE) if new or source.file_location is None: new = True elif not os.path.isfile( os.path.join(presentation_dir, rel_path(source.file_location))): new = True #source resultloc = result.resultloc.replace('\\', '/') source_file_name = os.path.join(dest_dir, resultloc) if check_timestamp_original_sourcefile and not new: log.debug('check timestamps') if os.path.isfile(source_file_name): source_file_last_modified = datetime.datetime.fromtimestamp( os.stat(source_file_name.encode('utf8'))[8]) #adding timestamps where this was not filled before if source.t_origin == None or source.t_source == None: source.t_origin = source_file_last_modified source.t_source = datetime.datetime.now() source.save() elif (source_file_last_modified > source.t_origin): log.info('source file has changed, recreate') new = True output_dir_name = os.path.join( 'flooding', 'scenario', str(scenario.id)) filename = source_file_name.replace('\\', '/').split('/')[-1] output_file_name = os.path.join(output_dir_name, filename) if new: log.debug('copy file') #output output_path = os.path.join(presentation_dir, output_dir_name) if not os.path.isdir(output_path): os.makedirs(output_path) destination_file_name = os.path.join( presentation_dir, output_file_name) log.debug('source file is {0}'.format(source_file_name)) log.debug('destination is {0}'.format(destination_file_name)) try: if filename[-3:].lower() == 'zip': copyfile(source_file_name, destination_file_name) else: dest = ZipFile(destination_file_name.encode('utf8')[:-3] + '.zip', mode="w", compression=ZIP_DEFLATED) dest.writestr( filename, file(source_file_name, 'rb').read()) dest.close() source.file_location = output_file_name source.t_original = datetime.datetime.fromtimestamp( os.stat(source_file_name.encode('utf8'))[8]) source.t_source = datetime.datetime.now() source.save() get_animation_info = True except IOError as e: source.delete() raise IOError(e) if get_animation_info: log.debug('get animation information') zip_name = os.path.join(presentation_dir, output_file_name) input_file = ZipFile(zip_name, "r") his = HISFile( Stream(input_file.read(input_file.filelist[0].filename))) input_file.close() delta = (his.get_datetime_of_timestep(1) - his.get_datetime_of_timestep(0)) animation['delta_timestep'] = ( delta.days + float(delta.seconds) / (24 * 60 * 60)) animation['firstnr'] = 0 animation['lastnr'] = his.size() - 1 animation['startnr'] = 0 except Exception as e: log.error('error generation value source') log.error(','.join(map(str, e.args))) return False, None, None, None return True, source, animation, new
def writeStoryImpl(self, out): ## Python 2.5 ZipFile is rather more primative than later ## versions. It can operate on a file, or on a StringIO, but ## not on an open stream. OTOH, I suspect we would have had ## problems with closing and opening again to change the ## compression type anyway. zipio = StringIO.StringIO() ## mimetype must be first file and uncompressed. Python 2.5 ## ZipFile can't change compression type file-by-file, so we ## have to close and re-open outputepub = ZipFile(zipio, 'w', compression=ZIP_STORED) outputepub.debug = 3 outputepub.writestr('mimetype', 'application/epub+zip') outputepub.close() ## Re-open file for content. outputepub = ZipFile(zipio, 'a', compression=ZIP_DEFLATED) outputepub.debug = 3 ## Create META-INF/container.xml file. The only thing it does is ## point to content.opf containerdom = getDOMImplementation().createDocument( None, "container", None) containertop = containerdom.documentElement containertop.setAttribute("version", "1.0") containertop.setAttribute( "xmlns", "urn:oasis:names:tc:opendocument:xmlns:container") rootfiles = containerdom.createElement("rootfiles") containertop.appendChild(rootfiles) rootfiles.appendChild( newTag( containerdom, "rootfile", { "full-path": "content.opf", "media-type": "application/oebps-package+xml" })) outputepub.writestr("META-INF/container.xml", containerdom.toxml(encoding='utf-8')) containerdom.unlink() del containerdom ## Epub has two metadata files with real data. We're putting ## them in content.opf (pointed to by META-INF/container.xml) ## and toc.ncx (pointed to by content.opf) ## content.opf contains metadata, a 'manifest' list of all ## other included files, and another 'spine' list of the items in the ## file uniqueid = 'fanficfare-uid:%s-u%s-s%s' % ( self.getMetadata('site'), self.story.getList('authorId')[0], self.getMetadata('storyId')) contentdom = getDOMImplementation().createDocument( None, "package", None) package = contentdom.documentElement package.setAttribute("version", "2.0") package.setAttribute("xmlns", "http://www.idpf.org/2007/opf") package.setAttribute("unique-identifier", "fanficfare-uid") metadata = newTag(contentdom, "metadata", attrs={ "xmlns:dc": "http://purl.org/dc/elements/1.1/", "xmlns:opf": "http://www.idpf.org/2007/opf" }) package.appendChild(metadata) metadata.appendChild( newTag(contentdom, "dc:identifier", text=uniqueid, attrs={"id": "fanficfare-uid"})) if self.getMetadata('title'): metadata.appendChild( newTag(contentdom, "dc:title", text=self.getMetadata('title'))) if self.getMetadata('author'): if self.story.isList('author'): for auth in self.story.getList('author'): metadata.appendChild( newTag(contentdom, "dc:creator", attrs={"opf:role": "aut"}, text=auth)) else: metadata.appendChild( newTag(contentdom, "dc:creator", attrs={"opf:role": "aut"}, text=self.getMetadata('author'))) metadata.appendChild( newTag(contentdom, "dc:contributor", text="FanFicFare [https://github.com/JimmXinu/FanFicFare]", attrs={"opf:role": "bkp"})) metadata.appendChild(newTag(contentdom, "dc:rights", text="")) if self.story.getMetadata('langcode'): metadata.appendChild( newTag(contentdom, "dc:language", text=self.story.getMetadata('langcode'))) else: metadata.appendChild(newTag(contentdom, "dc:language", text='en')) # published, created, updated, calibre # Leave calling self.story.getMetadataRaw directly in case date format changes. if self.story.getMetadataRaw('datePublished'): metadata.appendChild( newTag(contentdom, "dc:date", attrs={"opf:event": "publication"}, text=self.story.getMetadataRaw( 'datePublished').strftime("%Y-%m-%d"))) if self.story.getMetadataRaw('dateCreated'): metadata.appendChild( newTag(contentdom, "dc:date", attrs={"opf:event": "creation"}, text=self.story.getMetadataRaw('dateCreated').strftime( "%Y-%m-%d"))) if self.story.getMetadataRaw('dateUpdated'): metadata.appendChild( newTag(contentdom, "dc:date", attrs={"opf:event": "modification"}, text=self.story.getMetadataRaw('dateUpdated').strftime( "%Y-%m-%d"))) metadata.appendChild( newTag(contentdom, "meta", attrs={ "name": "calibre:timestamp", "content": self.story.getMetadataRaw('dateUpdated').strftime( "%Y-%m-%dT%H:%M:%S") })) series = self.story.getMetadataRaw('series') if series and self.getConfig('calibre_series_meta'): series_index = "0.0" if '[' in series: logger.debug(series) ## assumed "series [series_index]" series_index = series[series.index(' [') + 2:-1] series = series[:series.index(' [')] ## calibre always outputs a series_index and it's ## always a float with 1 or 2 decimals. FFF usually ## has either an integer or no index. (injected ## calibre series is the only float at this time) series_index = "%.2f" % float(series_index) metadata.appendChild( newTag(contentdom, "meta", attrs={ "name": "calibre:series", "content": series })) metadata.appendChild( newTag(contentdom, "meta", attrs={ "name": "calibre:series_index", "content": series_index })) if self.getMetadata('description'): metadata.appendChild( newTag(contentdom, "dc:description", text=self.getMetadata('description'))) for subject in self.story.getSubjectTags(): metadata.appendChild(newTag(contentdom, "dc:subject", text=subject)) if self.getMetadata('site'): metadata.appendChild( newTag(contentdom, "dc:publisher", text=self.getMetadata('site'))) if self.getMetadata('storyUrl'): metadata.appendChild( newTag(contentdom, "dc:identifier", attrs={"opf:scheme": "URL"}, text=self.getMetadata('storyUrl'))) metadata.appendChild( newTag(contentdom, "dc:source", text=self.getMetadata('storyUrl'))) ## end of metadata, create manifest. items = [] # list of (id, href, type, title) tuples(all strings) itemrefs = [] # list of strings -- idrefs from .opfs' spines items.append(("ncx", "toc.ncx", "application/x-dtbncx+xml", None)) ## we'll generate the toc.ncx file, ## but it needs to be in the items manifest. guide = None coverIO = None coverimgid = "image0000" if not self.story.cover and self.story.oldcover: logger.debug( "writer_epub: no new cover, has old cover, write image.") (oldcoverhtmlhref, oldcoverhtmltype, oldcoverhtmldata, oldcoverimghref, oldcoverimgtype, oldcoverimgdata) = self.story.oldcover outputepub.writestr(oldcoverhtmlhref, oldcoverhtmldata) outputepub.writestr(oldcoverimghref, oldcoverimgdata) coverimgid = "image0" items.append((coverimgid, oldcoverimghref, oldcoverimgtype, None)) items.append(("cover", oldcoverhtmlhref, oldcoverhtmltype, None)) itemrefs.append("cover") metadata.appendChild( newTag(contentdom, "meta", { "content": "image0", "name": "cover" })) guide = newTag(contentdom, "guide") guide.appendChild( newTag(contentdom, "reference", attrs={ "type": "cover", "title": "Cover", "href": oldcoverhtmlhref })) if self.getConfig('include_images'): imgcount = 0 for imgmap in self.story.getImgUrls(): imgfile = "OEBPS/" + imgmap['newsrc'] outputepub.writestr(imgfile, imgmap['data']) items.append( ("image%04d" % imgcount, imgfile, imgmap['mime'], None)) imgcount += 1 if 'cover' in imgfile: # make sure coverimgid is set to the cover, not # just the first image. coverimgid = items[-1][0] items.append(("style", "OEBPS/stylesheet.css", "text/css", None)) if self.story.cover: # Note that the id of the cover xhmtl *must* be 'cover' # for it to work on Nook. items.append( ("cover", "OEBPS/cover.xhtml", "application/xhtml+xml", None)) itemrefs.append("cover") # # <meta name="cover" content="cover.jpg"/> metadata.appendChild( newTag(contentdom, "meta", { "content": coverimgid, "name": "cover" })) # cover stuff for later: # at end of <package>: # <guide> # <reference type="cover" title="Cover" href="Text/cover.xhtml"/> # </guide> guide = newTag(contentdom, "guide") guide.appendChild( newTag(contentdom, "reference", attrs={ "type": "cover", "title": "Cover", "href": "OEBPS/cover.xhtml" })) if self.hasConfig("cover_content"): COVER = string.Template(self.getConfig("cover_content")) else: COVER = self.EPUB_COVER coverIO = StringIO.StringIO() coverIO.write( COVER.substitute( dict(self.story.getAllMetadata().items() + {'coverimg': self.story.cover}.items()))) if self.getConfig("include_titlepage"): items.append(("title_page", "OEBPS/title_page.xhtml", "application/xhtml+xml", "Title Page")) itemrefs.append("title_page") if len(self.story.getChapters()) > 1 and self.getConfig( "include_tocpage") and not self.metaonly: items.append(("toc_page", "OEBPS/toc_page.xhtml", "application/xhtml+xml", "Table of Contents")) itemrefs.append("toc_page") ## save where to insert logpage. logpage_indices = (len(items), len(itemrefs)) dologpage = ( self.getConfig("include_logpage") == "smart" and \ (self.story.logfile or self.story.getMetadataRaw("status") == "In-Progress") ) \ or self.getConfig("include_logpage") == "true" ## collect chapter urls and file names for internalize_text_links option. chapurlmap = {} for index, chap in enumerate(self.story.getChapters(fortoc=True)): if chap['html']: i = index + 1 items.append(("file%s" % chap['index04'], "OEBPS/file%s.xhtml" % chap['index04'], "application/xhtml+xml", chap['title'])) itemrefs.append("file%s" % chap['index04']) chapurlmap[chap['url']] = "file%s.xhtml" % chap[ 'index04'] # url -> relative epub file name. if dologpage: if self.getConfig("logpage_at_end") == "true": ## insert logpage after chapters. logpage_indices = (len(items), len(itemrefs)) items.insert(logpage_indices[0], ("log_page", "OEBPS/log_page.xhtml", "application/xhtml+xml", "Update Log")) itemrefs.insert(logpage_indices[1], "log_page") manifest = contentdom.createElement("manifest") package.appendChild(manifest) for item in items: (id, href, type, title) = item manifest.appendChild( newTag(contentdom, "item", attrs={ 'id': id, 'href': href, 'media-type': type })) spine = newTag(contentdom, "spine", attrs={"toc": "ncx"}) package.appendChild(spine) for itemref in itemrefs: spine.appendChild( newTag(contentdom, "itemref", attrs={ "idref": itemref, "linear": "yes" })) # guide only exists if there's a cover. if guide: package.appendChild(guide) # write content.opf to zip. contentxml = contentdom.toxml(encoding='utf-8') # tweak for brain damaged Nook STR. Nook insists on name before content. contentxml = contentxml.replace( '<meta content="%s" name="cover"/>' % coverimgid, '<meta name="cover" content="%s"/>' % coverimgid) outputepub.writestr("content.opf", contentxml) contentdom.unlink() del contentdom ## create toc.ncx file tocncxdom = getDOMImplementation().createDocument(None, "ncx", None) ncx = tocncxdom.documentElement ncx.setAttribute("version", "2005-1") ncx.setAttribute("xmlns", "http://www.daisy.org/z3986/2005/ncx/") head = tocncxdom.createElement("head") ncx.appendChild(head) head.appendChild( newTag(tocncxdom, "meta", attrs={ "name": "dtb:uid", "content": uniqueid })) head.appendChild( newTag(tocncxdom, "meta", attrs={ "name": "dtb:depth", "content": "1" })) head.appendChild( newTag(tocncxdom, "meta", attrs={ "name": "dtb:totalPageCount", "content": "0" })) head.appendChild( newTag(tocncxdom, "meta", attrs={ "name": "dtb:maxPageNumber", "content": "0" })) docTitle = tocncxdom.createElement("docTitle") docTitle.appendChild( newTag(tocncxdom, "text", text=self.getMetadata('title'))) ncx.appendChild(docTitle) tocnavMap = tocncxdom.createElement("navMap") ncx.appendChild(tocnavMap) # <navPoint id="<id>" playOrder="<risingnumberfrom0>"> # <navLabel> # <text><chapter title></text> # </navLabel> # <content src="<chapterfile>"/> # </navPoint> index = 0 for item in items: (id, href, type, title) = item # only items to be skipped, cover.xhtml, images, toc.ncx, stylesheet.css, should have no title. if title: navPoint = newTag(tocncxdom, "navPoint", attrs={ 'id': id, 'playOrder': unicode(index) }) tocnavMap.appendChild(navPoint) navLabel = newTag(tocncxdom, "navLabel") navPoint.appendChild(navLabel) ## the xml library will re-escape as needed. navLabel.appendChild( newTag(tocncxdom, "text", text=stripHTML(title))) navPoint.appendChild( newTag(tocncxdom, "content", attrs={"src": href})) index = index + 1 # write toc.ncx to zip file outputepub.writestr("toc.ncx", tocncxdom.toxml(encoding='utf-8')) tocncxdom.unlink() del tocncxdom # write stylesheet.css file. outputepub.writestr( "OEBPS/stylesheet.css", self.EPUB_CSS.substitute(self.story.getAllMetadata())) # write title page. if self.getConfig("titlepage_use_table"): TITLE_PAGE_START = self.EPUB_TABLE_TITLE_PAGE_START TITLE_ENTRY = self.EPUB_TABLE_TITLE_ENTRY WIDE_TITLE_ENTRY = self.EPUB_TABLE_TITLE_WIDE_ENTRY NO_TITLE_ENTRY = self.EPUB_TABLE_NO_TITLE_ENTRY TITLE_PAGE_END = self.EPUB_TABLE_TITLE_PAGE_END else: TITLE_PAGE_START = self.EPUB_TITLE_PAGE_START TITLE_ENTRY = self.EPUB_TITLE_ENTRY WIDE_TITLE_ENTRY = self.EPUB_TITLE_ENTRY # same, only wide in tables. NO_TITLE_ENTRY = self.EPUB_NO_TITLE_ENTRY TITLE_PAGE_END = self.EPUB_TITLE_PAGE_END if coverIO: outputepub.writestr("OEBPS/cover.xhtml", coverIO.getvalue()) coverIO.close() titlepageIO = StringIO.StringIO() self.writeTitlePage(out=titlepageIO, START=TITLE_PAGE_START, ENTRY=TITLE_ENTRY, WIDE_ENTRY=WIDE_TITLE_ENTRY, END=TITLE_PAGE_END, NO_TITLE_ENTRY=NO_TITLE_ENTRY) if titlepageIO.getvalue(): # will be false if no title page. outputepub.writestr("OEBPS/title_page.xhtml", titlepageIO.getvalue()) titlepageIO.close() # write toc page. tocpageIO = StringIO.StringIO() self.writeTOCPage(tocpageIO, self.EPUB_TOC_PAGE_START, self.EPUB_TOC_ENTRY, self.EPUB_TOC_PAGE_END) if tocpageIO.getvalue(): # will be false if no toc page. outputepub.writestr("OEBPS/toc_page.xhtml", tocpageIO.getvalue()) tocpageIO.close() if dologpage: # write log page. logpageIO = StringIO.StringIO() self.writeLogPage(logpageIO) outputepub.writestr("OEBPS/log_page.xhtml", logpageIO.getvalue()) logpageIO.close() if self.hasConfig('chapter_start'): CHAPTER_START = string.Template(self.getConfig("chapter_start")) else: CHAPTER_START = self.EPUB_CHAPTER_START if self.hasConfig('chapter_end'): CHAPTER_END = string.Template(self.getConfig("chapter_end")) else: CHAPTER_END = self.EPUB_CHAPTER_END for index, chap in enumerate( self.story.getChapters()): # (url,title,html) if chap['html']: chap_data = chap['html'] if self.getConfig('internalize_text_links'): soup = bs4.BeautifulSoup(chap['html'], 'html5lib') changed = False for alink in soup.find_all('a'): if alink.has_attr( 'href') and alink['href'] in chapurlmap: alink['href'] = chapurlmap[alink['href']] changed = True if changed: chap_data = unicode(soup) # Don't want html, head or body tags in # chapter html--bs4 insists on adding them. chap_data = re.sub(r"</?(html|head|body)[^>]*>\r?\n?", "", chap_data) #logger.debug('Writing chapter text for: %s' % chap.title) chap['url'] = removeEntities(chap['url']) chap['chapter'] = removeEntities(chap['chapter']) chap['title'] = removeEntities(chap['title']) chap['origchapter'] = removeEntities(chap['origtitle']) chap['tocchapter'] = removeEntities(chap['toctitle']) # escape double quotes in all vals. for k, v in chap.items(): if isinstance(v, basestring): chap[k] = v.replace('"', '"') fullhtml = CHAPTER_START.substitute(chap) + \ chap_data.strip() + \ CHAPTER_END.substitute(chap) # strip to avoid ever growning numbers of newlines. # ffnet(& maybe others) gives the whole chapter text # as one line. This causes problems for nook(at # least) when the chapter size starts getting big # (200k+) fullhtml = re.sub(r'(</p>|<br ?/>)\n*', r'\1\n', fullhtml) outputepub.writestr("OEBPS/file%s.xhtml" % chap['index04'], fullhtml.encode('utf-8')) del fullhtml if self.story.calibrebookmark: outputepub.writestr("META-INF/calibre_bookmarks.txt", self.story.calibrebookmark) # declares all the files created by Windows. otherwise, when # it runs in appengine, windows unzips the files as 000 perms. for zf in outputepub.filelist: zf.create_system = 0 outputepub.close() out.write(zipio.getvalue()) zipio.close()