def import_view(request):
"""
Gets the existing declared parsers for the current project.
This view handles only the file based import parsers.
"""
choices = []
choices_url = []
render_dict = {}
choices, choices_url, classes = discover_available_parsers()
form = ImportDatasetFormWithFile(choices, prefix="with-file")
form_without_file = ImportDatasetForm(
choices_url, prefix="without-file")
if request.method == 'POST':
if 'upload-file' in request.POST:
form = ImportDatasetFormWithFile(
choices, request.POST, request.FILES, prefix="with-file")
if form.is_valid():
print(request.FILES)
uploaded = request.FILES['with-file-zipfile']
destination_dir, destination_file = create_tmp_destination(
uploaded.name)
with open(destination_file, 'w+') as f:
f.write(uploaded.file.read())
zfile = ZipFile(f)
for name in zfile.namelist():
try:
zfile.extract(
name, os.path.dirname(os.path.realpath(f.name)))
if name.endswith('shp'):
parser = classes[int(form['parser'].value())]
import_datas.delay(
'/'.join((destination_dir, name)),
parser.__name__, parser.__module__
)
continue
except Exception:
raise
if 'import-web' in request.POST:
form_without_file = ImportDatasetForm(
choices_url, request.POST, prefix="without-file")
if form_without_file.is_valid():
parser = classes[int(form_without_file['parser'].value())]
import_datas_from_web.delay(
parser.__name__, parser.__module__
)
# Hide second form if parser has no web based imports.
render_dict['form'] = form
if choices_url:
render_dict['form_without_file'] = form_without_file
return render(request, 'common/import_dataset.html', render_dict)
def _loadShpList(self):
"""
Loads (extracts) a batch of shapefiles from an archive. Only those that match
the given pattern are extracted.
"""
self.shpList = None
shpFiles = list()
tmpFiles = list()
shpmatch = re.compile('\.shp$')
searchRe = re.compile(self.pattern)
# Iterate through the list of archives.
for i in range(self.zipIndex, len(self.zipList)):
f = self.zipList[i]
try:
# Open the zip file and read the entries.
z = ZipFile(f)
entries = z.namelist()
# Iterate through the entries. If an entry matches the search string, extract
# it to the tmp folder.
for e in entries:
if searchRe.search(e):
z.extract(e, self.tmpDir)
# Add to the list of extracted files.
tmpFiles.append(e)
# If the entry is a shapefile, add it to the shp list.
if shpmatch.search(e):
shpFiles.append(e)
except Exception, e:
print "Failed to open {0}: {1}.".format(f, e.__str__())
finally:
def extract_subfiles(source_path, dest_path, verbose=False):
if os.path.isdir(source_path):
for dirpath, dirnames, filenames in os.walk(source_path):
relpath = os.path.relpath(dirpath, source_path)
new_dir_path = os.path.join(dest_path, relpath)
if not os.path.isdir(new_dir_path):
os.mkdir(new_dir_path)
for filename in filenames:
try:
source_file_path = os.path.join(dirpath, filename)
relpath = os.path.relpath(source_file_path, source_path)
dest_file_path = os.path.join(dest_path, relpath)
print dest_file_path
if dest_file_path.endswith('.cod'):
zip = ZipFile(source_file_path)
for info in zip.infolist():
if verbose:
print ' %s (extracted)' % info.filename
dest_unzip_path = os.path.split(dest_file_path)[0]
if not os.path.realpath(os.path.join(dest_unzip_path, info.filename)).startswith(os.path.realpath(dest_unzip_path)):
raise(Exception('Security exception: zip file %s attempted to extract to a non-local location' % info.filename))
zip.extract(info, path = dest_unzip_path)
else:
shutil.copyfile(source_file_path, dest_file_path)
except Exception, e:
if str(e) == 'File is not a zip file':
# this is a cod file or some other file
shutil.copyfile(source_file_path, dest_file_path)
else:
if verbose:
print >>sys.stderr, 'Error:',
print >>sys.stderr, str(e)
raise(e)
def main():
parser = argparse.ArgumentParser('Update the icomoon icon font from the provided archive')
parser.add_argument('archive', help='Path to .zip file generated by icomoon')
args = parser.parse_args()
script_dir = os.path.dirname(os.path.abspath(__file__))
vendor_style_dir = script_dir + '/../h/static/styles/vendor'
icon_font_archive = ZipFile(args.archive)
icon_font_archive.extract('selection.json', vendor_style_dir + '/fonts')
icon_font_archive.extract('fonts/h.woff', vendor_style_dir)
css_input_file = icon_font_archive.open('style.css')
css_output_file = open(vendor_style_dir + '/icomoon.css', 'w')
for line in css_input_file:
if "format('woff')" in line:
# inline the WOFF format file
woff_content = icon_font_archive.open('fonts/h.woff').read()
woff_src_line = """
/* WARNING - the URL below is inlined
* because the CSS asset pipeline is not correctly rebasing
* URLs when concatenating files together.
*
* See issue #2571
*/
src:url('data:application/font-woff;base64,%s') format('woff');
"""
css_output_file.write(woff_src_line % b64encode(woff_content))
elif "url(" in line:
# skip non-WOFF format fonts
pass
else:
css_output_file.write(line)
def download_custom_reports(domain, report_id, report_name):
"""Downloads custom report data csv zip export from commcare.
The extracted csv is placed in data/DOMAIN/latest/Pregnancy Visit.csv.
:domain - commcare domain
:report_id - custom report id for report
"""
if not report_id:
raise Exception(u"Please ensure the custom report id"
u" is configured in CUSTOM_REPORTS settings.")
url_path = "/a/%(domain)s/reports/export/custom/%(id)s/download/"\
"?format=csv" % {'domain': domain, 'id': report_id}
f = download_from_commcare(url_path)
try:
z = ZipFile(f, 'r')
except BadZipfile:
print f.read()
else:
zdst = os.path.join('data', domain, 'latest')
dst = os.path.join(zdst, report_name)
if z.NameToInfo.keys():
filename = z.NameToInfo.keys()[0]
src = os.path.join(zdst, filename)
z.extract(z.NameToInfo[filename], zdst)
if os.path.exists(src):
os.rename(src, dst)
print u"Successfully downloaded %s" % report_name
def main(opts):
for filename in opts.files:
zip_file = ZipFile(filename)
info_list = zip_file.infolist()
for entry in info_list:
entry.filename = decode_filename(entry.filename, opts.codepage)
zip_file.extract(entry, path=opts.dest)
def install(self, sid, data, fname):
try:
z = ZipFile(io.BytesIO(data))
except zipfile.BadZipfile:
showWarning(_("The download was corrupt. Please try again."))
return
name = os.path.splitext(fname)[0]
# previously installed?
meta = self.addonMeta(sid)
base = self.addonsFolder(sid)
if os.path.exists(base):
self.backupUserFiles(sid)
self.deleteAddon(sid)
os.mkdir(base)
self.restoreUserFiles(sid)
# extract
for n in z.namelist():
if n.endswith("/"):
# folder; ignore
continue
path = os.path.join(base, n)
# skip existing user files
if os.path.exists(path) and n.startswith("user_files/"):
continue
z.extract(n, base)
# update metadata
meta['name'] = name
meta['mod'] = intTime()
self.writeAddonMeta(sid, meta)
def unzip(filename, match_dir=False, destdir=None):
"""
Extract all files from a zip archive
filename: The path to the zip file
match_dir: If True all files in the zip must be contained in a subdirectory
named after the archive file with extension removed
destdir: Extract the zip into this directory, default current directory
return: If match_dir is True then returns the subdirectory (including
destdir), otherwise returns destdir or '.'
"""
if not destdir:
destdir = '.'
z = ZipFile(filename)
unzipped = '.'
if match_dir:
if not filename.endswith('.zip'):
raise FileException('Expected .zip file extension', filename)
unzipped = os.path.basename(filename)[:-4]
check_extracted_paths(z.namelist(), unzipped)
else:
check_extracted_paths(z.namelist())
# File permissions, see
# http://stackoverflow.com/a/6297838
# http://stackoverflow.com/a/3015466
for info in z.infolist():
log.debug('Extracting %s to %s', info.filename, destdir)
z.extract(info, destdir)
os.chmod(os.path.join(destdir, info.filename),
info.external_attr >> 16 & 4095)
return os.path.join(destdir, unzipped)
def extract(zip):
"""Extrait le fichier '/word/document.xml'
d'une archive word dans le répertoire courant
"""
myzip = ZipFile(zip, 'a')
myzip.extract('word/document.xml', './../')
def fetch_fs_peptide(data_home=None, download_if_missing=True):
"""Loader for the Fs peptide dataset
Parameters
----------
data_home : optional, default: None
Specify another download and cache folder for the datasets. By default
all mixtape data is stored in '~/mixtape_data' subfolders.
download_if_missing: optional, True by default
If False, raise a IOError if the data is not locally available
instead of trying to download the data from the source site.
"""
data_home = get_data_home(data_home=data_home)
if not exists(data_home):
makedirs(data_home)
data_dir = join(data_home, TARGET_DIRECTORY)
if not exists(data_dir):
print('downloading fs peptide from %s to %s' % (DATA_URL, data_home))
fhandle = urlopen(DATA_URL)
buf = BytesIO(fhandle.read())
zip_file = ZipFile(buf)
makedirs(data_dir)
for name in zip_file.namelist():
zip_file.extract(name, path=data_dir)
top = md.load(join(data_dir, 'fs_peptide.pdb'))
trajectories = []
for fn in sorted(glob(join(data_dir, 'trajectory*.xtc'))):
print('loading %s...' % basename(fn))
trajectories.append(md.load(fn, top=top))
return Bunch(trajectories=trajectories, DESCR=__doc__)
def unzip(filename):
z = ZipFile(filename)
names = z.namelist()
for path in names:
if path.startswith('__MACOSX/'):
continue
base, name = os.path.split(path)
if name.startswith('._') and\
'%s/' % name.replace('._', '', 1) in names:
continue
double = os.path.join('__MACOSX', base, '._' + name)
if double in names:
print '=> %s.bin' % path
info = z.getinfo(path)
bin = MacBinary(name)
bin.data = z.open(path, 'r').read()
bin.res = z.open(double, 'r').read()
modified = datetime.datetime(*info.date_time)
bin.modified = time.mktime(modified.timetuple())
bin.created = time.time()
if not os.path.exists(base):
os.makedirs(base)
with open('%s.bin' % path.rstrip('\r'), 'wb') as f:
f.write(bin.encode())
else:
print '-> %s' % path
z.extract(path)
def download(url, dirname, filename):
if int(__addon__.getSetting("sub_language1")) == 0:
return {}
cache_path = os.path.join(CACHE_DIR, "temp.zip")
try:
urlretrieve(url, cache_path)
z = ZipFile(cache_path)
except:
return None
path = None
for f in z.namelist():
ext = os.path.splitext(f)[1]
if not ext in _subtitles_formats:
continue
path = os.path.join(dirname, filename + ext)
z.extract(f, CACHE_DIR)
if os.path.isfile(path):
os.unlink(path)
os.rename(os.path.join(CACHE_DIR, f), path)
break
z.close()
os.unlink(cache_path)
return path
def census2000(summary_file_num, filenum):
base_url = 'http://www2.census.gov/census_2000/datasets/'
path = os.path.join(
'Summary_File_%s' % summary_file_num, state_name.replace(' ', '_'),
'%s000%02d_uf%s.zip' % (state_abbr, filenum, summary_file_num)
)
reader = remote_fallback_reader(
'../data/census_2000/datasets',
base_url,
path
)
z = ZipFile(reader)
n = z.namelist()
z.extract(n[0], '/tmp/')
filename = '/tmp/%s000%02d.uf%s' % (state_abbr, filenum, summary_file_num)
# count the number of columns, to determine how many columns to fill
firstline = open(filename).readline()
cols = ['fileid', 'stusab', 'chariter', 'cifsn', 'logrecno']
cols.extend(map(
lambda c: 'col%s' % str(c+1),
range(firstline.count(',')+1-5) # subtract 5 to account for the 5 geo header columns
))
cursor.copy_from(open(filename), 'census_row', sep=',',
columns=cols)
os.unlink(filename)
def extract_from_zip(filename):
zp = ZipFile(filename, 'r')
namelist = zp.namelist()
desired_files = [x for x in namelist if 'Award' in x or 'Summary' in x]
for file in desired_files:
if not os.path.exists(file):
zp.extract(file)
def addCBZFiles(self, filenames):
directories = []
tempDir = tempfile.gettempdir()
filenames.sort()
filenamesListed = []
for i in xrange(0, self.listWidgetFiles.count()):
filenamesListed.append(self.listWidgetFiles.item(i).text())
for filename in filenames:
folderName = os.path.splitext(basename(str(filename)))[0]
path = tempDir + "/" + folderName + "/"
cbzFile = ZipFile(str(filename))
for f in cbzFile.namelist():
if f.endswith('/'):
try:
os.makedirs(path + f)
except:
pass # the dir exists so we are going to extract the images only.
else:
cbzFile.extract(f, path)
if os.path.isdir(unicode(path)): # Add the directories
directories.append(path)
self.addImageDirs(directories) # Add the files
def acs(file_num, geo_type_dir):
base_url = 'http://www2.census.gov/acs2010_5yr/summaryfile/2006-2010_ACSSF_By_State_By_Sequence_Table_Subset/'
path = os.path.join(
state_name.replace(' ', ''), geo_type_dir,
'20105%s%04d000.zip' % (state_abbr, file_num)
)
reader = remote_fallback_reader(
'../data/acs2006_2010_5yr/summaryfile/2006-2010_ACSSF_By_State_By_Sequence_Table_Subset',
base_url,
path
)
z = ZipFile(reader)
n = z.namelist()
z.extractall('/tmp/')
files = ['e20105%s%04d000.txt' % (state_abbr, file_num),
'm20105%s%04d000.txt' % (state_abbr, file_num),]
for f in files:
z.extract(f, '/tmp/')
# count the number of columns, to determine how many columns to fill
firstline = open('/tmp/' + f).readline()
if not firstline:
# some files are empty, so just continue to the next
os.unlink('/tmp/' + f)
continue
cols = ['fileid', 'filetype', 'stusab', 'chariter', 'cifsn', 'logrecno']
cols.extend(map(
lambda c: 'col%s' % str(c+1),
range(firstline.count(',')+1-6) # subtract 6 to account for the 6 geo header columns
))
cursor.copy_from(open('/tmp/%s' % f), 'census_row', sep=',',
columns=cols)
os.unlink('/tmp/' + f)
def unzip(zippath, outdir):
"""
Unzip the contents of zippath to outdir. Example:
unzip("zelda30tribute.zip", "./assets")
"""
f = open(zippath, 'rb')
z = ZipFile(f)
ziprootdir = commondir(z.namelist())
fileno = 0
for asset in assets:
fileno += 1
outpath = joinpaths(outdir, asset)
inpath = joinpaths(ziprootdir, asset)
print("Extracting ({}/{}): {} -> {}"
.format(fileno, len(assets), inpath, outpath))
z.extract(inpath, outdir)
if ziprootdir:
# If the assets are stored within a top folder:
srcpath = joinpaths(asset_dir, ziprootdir)
dstpath = asset_dir
print("Replacing top folder: {} -> {}".format(srcpath, dstpath))
topfiles = os.listdir(srcpath)
for topfile in topfiles:
# Remove if it happens to already be there:
try:
shutil.rmtree(joinpaths(dstpath, topfile))
except OSError as e:
if e.errno != errno.ENOENT:
raise
shutil.move(joinpaths(srcpath, topfile), dstpath)
shutil.rmtree(srcpath)
def base_scrape_alexa(url, browser_args, out_name, add_http):
b = Browser(name, **browser_args)
r = b.go(url)
zip_file = StringIO(r.content)
z = ZipFile(zip_file, 'r')
csv_file = 'top-1m.csv'
z.extract(csv_file, out_dir)
file_path = path.join(out_dir, csv_file)
lines = strip_open(file_path)
out_path = path.join(out_dir, out_name)
if add_http:
os.remove(file_path)
with open(out_path, 'w+') as f:
for line in lines:
print >> f, 'http://' + line.split(',')[1]
else:
try:
os.rename(file_path, out_path)
except Exception as e:
pass
return len(lines)
def install(self, sid, data, fname):
try:
z = ZipFile(io.BytesIO(data))
except zipfile.BadZipfile:
showWarning(_("The download was corrupt. Please try again."))
return
name = os.path.splitext(fname)[0]
# previously installed?
meta = self.addonMeta(sid)
base = self.addonsFolder(sid)
if os.path.exists(base):
self.deleteAddon(sid)
# extract
os.mkdir(base)
for n in z.namelist():
if n.endswith("/"):
# folder; ignore
continue
# write
z.extract(n, base)
# update metadata
meta['name'] = name
meta['mod'] = intTime()
self.writeAddonMeta(sid, meta)
def upgrade(self, version, force = False):
if not mkdir(self.path):
return version
if self.updater == "bukkitdev":
uver, urlh = self.__bukkitdev_info()
elif self.updater == "bukkitdl":
uver, urlh = self.__bukkitdl_info()
elif self.updater == "github":
uver, urlh = self.__github_info()
elif self.updater == "jenkins":
uver, urlh = self.__jenkins_info()
else:
log.error("%s: package upgrade failed: invalid updater `%s'",
self.package, self.updater)
return version
if not urlh:
log.error("%s: package upgrade failed", self.package)
return version
out = os.path.join(self.path, "%s.%s" % (self.package, self.type))
if uver and uver == version and not force:
log.info("%s: package already up-to-date", self.package)
return version
if not self.dryrun and not download(urlh, out):
return version
log.info("%s: package upgraded: %s -> %s", self.package, version, uver)
if self.dryrun or (self.type != "zip"):
return uver
if len(self.extract) < 1:
return uver
zf = ZipFile(out, "r")
nl = zf.namelist()
for path in self.extract:
if not path.endswith("/") and path in nl:
zf.extract(path, self.path)
continue
for zpath in nl:
if zpath.endswith("/"):
continue
if not zpath.startswith(path):
continue
zf.extract(zpath, self.path)
zf.close()
unlink(out)
return uver
def extract(upload, check_file):
directory = mkdtemp()
f = os.path.join(cfg["UPLOAD_FOLDER"], upload)
z = ZipFile(f, 'r')
z.extract(check_file, directory)
z.close()
return directory
def get_subtitle(url, path):
in_data = urllib2.urlopen(url)
temp_file = NamedTemporaryFile()
temp_file.write(in_data.read())
in_data.close()
temp_file.seek(0)
if is_zipfile(temp_file.name):
zip_file = ZipFile(temp_file)
for name in zip_file.namelist():
# don't unzip stub __MACOSX folders
if '.srt' in name and '__MACOSX' not in name:
logger.info(' '.join(['Unpacking zipped subtitle', name, 'to', os.path.dirname(path)]))
zip_file.extract(name, os.path.dirname(path))
zip_file.close()
elif is_rarfile(temp_file.name):
rar_path = path + '.rar'
logger.info('Saving rared subtitle as %s' % rar_path)
with open(rar_path, 'w') as out_file:
out_file.write(temp_file.read())
try:
import subprocess
#extract all .srt in the rared file
ret_code = subprocess.call(['unrar', 'e', '-n*srt', rar_path])
if ret_code == 0:
logger.info('Unpacking rared subtitle to %s' % os.path.dirname(path))
os.remove(rar_path)
except OSError:
logger.info('Unpacking rared subtitle failed.'
'Please, install unrar to automate this step.')
temp_file.close()
def main():
print "initializing"
ap.env.overwriteOutput = True
ap.env.workspace = WORKSPACE
dirlist = os.listdir(ASCZIPDIR)
print dirlist
for z in dirlist:
if z[-8:]==".asc.zip":
zf = ZipFile("%s%s" % (ASCZIPDIR,z))
z0=zf.namelist()[0]
print "extracting %s" % z0
zf.extract(z0,UNZIPDIR)
dirlist = os.listdir(UNZIPDIR)
print dirlist
for z in dirlist:
if z[-4:]==".asc":
targetfile = "%s%s/%s" % (CWD, WORKSPACE, z.split('.')[0])
srcfile = "%s%s%s" % (CWD,UNZIPDIR,z)
print "converting %s - %s" % (srcfile, targetfile)
if not ap.Exists(targetfile):
ap.ASCIIToRaster_conversion(srcfile, targetfile, "FLOAT")
ap.DefineProjection_management(targetfile, ap.SpatialReference("WGS 1984"))
else:
print "%s exists" % targetfile
print "complete"
def _create_resource(self, data_set, site, file_):
""" Creates a new resource or file associated with its data set
:param data_set:
:param site:
:param file_:
"""
#content of the zip file
zip_file_name = file_.filename
zip_path = os.path.join(config.DATA_SETS_DIR, zip_file_name)
file_.save(zip_path)
sourcezip = ZipFile(zip_path)
i = 0
while i < len(sourcezip.namelist()):
zip_entry_path = os.path.join(os.path.abspath(os.path.dirname(zip_path)), sourcezip.namelist()[i])
sourcezip.extract(sourcezip.namelist()[i], config.DATA_SETS_DIR)
url = self.parser.get_file_name().replace(".zip","") + "_" + str(i)
site.action.resource_create(package_id=data_set, upload=open(zip_entry_path),
name=sourcezip.namelist()[i], url=url)
i += 1
#xml content
xml_file_name = self.parser.get_dataset().id + ".xml"
path = os.path.join(config.DATA_SETS_DIR, xml_file_name)
with open(path, "w") as ttmp:
ttmp.write(self._content.encode(encoding="utf-8"))
url = xml_file_name
site.action.resource_create(package_id=data_set, upload=open(path),
name=xml_file_name, url=url)
def extract_archive(name):
with open(name, 'rb') as file_handle:
zipfile = ZipFile(file_handle)
for name in zipfile.namelist():
out_path = os.getcwd()
zipfile.extract(name, out_path)
zipfile.close()
def get_dmf():
"""Downloads DMF, convert to CSV for SQL"""
URL = """http://www.fda.gov/drugs/developmentapprovalprocess/
formssubmissionrequirements/drugmasterfilesdmfs/default.htm"""
bs = BeautifulSoup(requests.get(URL).content)
dmf = [link for link in bs.findAll('a')
if 'DMF zip file' in link.renderContents()]
ob_link = dmf.pop()
ob_link = ob_link.get('href')
download_link = urlparse.urljoin(URL, ob_link)
def dlfile(url):
f = requests.get(url)
with open(os.path.basename(url), "wb") as local_file:
local_file.write(f.content)
return local_file.name
dlfile(download_link)
zipfiles = dlfile(download_link)
zf = ZipFile(zipfiles)
zip_contents = zf.namelist()
match = [target for target in zip_contents if target.endswith('.xls')]
filename = str(match[0])
zf.extract(match.pop())
master = (pd.read_excel(filename, sheetname='Sheet1',
skiprows=1, index_col='DMF#', encoding='UTF-8'))
nameoutput = 'dmf_to_load_' + strftime('%Y%m%d')
master.to_csv(nameoutput + '.csv', encoding='UTF-8', sep='|')
def download_flights(start_date, end_date):
pkl_path = 'data/processed/flights/flights{0}.{1}-{2}.{3}.pkl'.format(start_date.year, start_date.month,
end_date.year,
end_date.month)
if Path(pkl_path).exists():
return pd.read_pickle(pkl_path)
files, failed = download_range(r'http://tsdata.bts.gov/PREZIP/On_Time_On_Time_Performance_%Y_%-m.zip',
'data/raw/flights', start_date, end_date)
dest_path = Path('data/unpacked/flights')
dest_path.mkdir(parents=True, exist_ok=True)
frames = []
for zip_file in files:
zf = ZipFile(str(zip_file))
csv_name = zip_file.with_suffix('.csv').name
csv_file = dest_path / csv_name
if not csv_file.exists():
print("Extracting", csv_name)
zf.extract(csv_name, str(dest_path))
print("...done")
else:
print(csv_name, "already extracted")
df = flightdata.read_csv(str(csv_file))
frames.append(df)
all_flights = merge_frames(frames, ignore_index=True)
all_flights.to_pickle(pkl_path)
return all_flights
def dump_files(zipfile: ZipFile):
"""Dump packed files to a location.
"""
dump_folder = CONF['packfile_dump', '']
if not dump_folder:
return
dump_folder = os.path.abspath(dump_folder)
# Delete files in the folder, but don't delete the folder itself.
try:
files = os.listdir(dump_folder)
except FileNotFoundError:
return
for name in files:
name = os.path.join(dump_folder, name)
if os.path.isdir(name):
try:
shutil.rmtree(name)
except OSError:
# It's possible to fail here, if the window is open elsewhere.
# If so, just skip removal and fill the folder.
pass
else:
os.remove(name)
for zipinfo in zipfile.infolist():
zipfile.extract(zipinfo, dump_folder)
def open_zip(self, infiles):
metadata_extracted = False
metadata = None
data_files = []
for infile in infiles:
try:
zip = ZipFile(infile, 'r')
for efile in zip.namelist():
if not metadata_extracted and efile.startswith('Stationsmetadaten'):
zip.extract(efile, path=self.workdir)
metadata = os.path.join(self.workdir, efile)
metadata_extracted = True
logger.debug("metadata: %s", metadata)
if efile.startswith('produkt'):
zip.extract(efile, path=self.workdir)
data_files.append(
os.path.join(self.workdir, efile)
)
logger.debug("data: %s", os.path.join(self.workdir, efile))
except BadZipfile as e:
logger.error("could not open: {}".format(infile))
raise e
return metadata, data_files
def SRTM3_retrieve(lon, lat, data_dir=None):
if data_dir is None:
dname = os.path.dirname
# be more clever in the data directory so that users can define a setting.
data_dir = os.path.join(dname(dname(__file__)), "data", "SRTM3")
x = "%s%03d" % ("E" if lon > 0 else "W", abs(int(lon)))
y = "%s%02d" % ("N" if lat > 0 else "S", abs(int(lat)))
filename = "{y}{x}.hgt".format(x=x, y=y)
filepath = os.path.join(data_dir, filename)
if not os.path.exists(filepath):
# download the zip file
file_url = _SRTM3_FILE_LOOKUP.get(u"{y}{x}".format(x=x, y=y), None)
# no file exists in the SRTM dataset for these coordinates
if file_url is None:
return None
import urllib2
import cStringIO as StringIO
from zipfile import ZipFile
if not os.path.exists(data_dir):
os.makedirs(data_dir)
srtm_online = urllib2.urlopen(file_url)
zfh = ZipFile(StringIO.StringIO(srtm_online.read()), "r")
zfh.extract(filename, data_dir)
return filepath
def get_symbols(exchange="polo"):
if exchange == "polo":
polo = ccxt.poloniex()
polo.load_markets()
return polo.symbols
elif exchange == "stock":
url = "https://www.quandl.com/api/v3/databases/WIKI/codes.json?api_key=%s" % QUANDL_APIKEY
res = urlopen(url)
zipfile = ZipFile(BytesIO(res.read()))
zipfile.extract(zipfile.namelist()[0])
df = pd.read_csv(zipfile.namelist()[0], header=None)
return df[0].values
else:
raise NotImplementedError()
def _read_one_data(self, url, params):
results = []
# retrying _MAX_RETRY_COUNT
for _ in range(1, _MAX_RETRY_COUNT):
try:
url = self.url.format(**params)
tree = ElemTree.fromstring(
requests.get(url).text.encode('utf-8'))
break
except requests.HTTPError:
sleep(_SLEEP_TIME)
else:
raise Exception
for el in tree.findall('.//' + self._namespace + 'entry'):
updated = el.find(self._namespace + 'updated').text
updated = parser.parse(updated, ignoretz=True)
if self.start <= updated <= self.end:
id = el.find(self._namespace + 'id').text
title = el.find(self._namespace + 'title').text
docid = el.find(self._namespace + 'docid').text
url = el.find(self._namespace +
'link[@type="application/zip"]').attrib['href']
is_yuho = any(
[word in title for word in ['有価証券報告書', '四半期報告書']])
is_quarterly = '四半期報告書' in title
if self.fetch_xbrl:
r = requests.get(url)
if r.ok:
z = ZipFile(BytesIO(r.content))
for info in z.infolist():
if is_yuho and '.xbrl' in info.filename and 'AuditDoc' not in info.filename:
self.xbrl_filename = info.filename.split(
'/')[-1]
z.extract(info.filename)
results.append({
'id': id,
'title': title,
'docid': docid,
'url': url,
'updated': updated,
'is_yuho': is_yuho,
'is_quarterly': is_quarterly,
})
sleep(_SLEEP_TIME)
return results
def rename_and_prepare(base_file):
"""ensure the file(s) have a proper name @hack this should be done
in a nicer way, but needs fixing now To fix longer term: if
geonode computes a name, the uploader should respect it As it
is/was, geonode will compute a name based on the zipfile but the
importer will use names as it unpacks the zipfile. Renaming all
the various pieces seems a burden on the client
Additionally, if a SLD file is present, extract this.
"""
name, ext = os.path.splitext(os.path.basename(base_file))
dirname = os.path.dirname(base_file)
if ext == ".zip":
zf = ZipFile(base_file, 'r')
rename = False
main_file = None
for f in zf.namelist():
name, ext = os.path.splitext(os.path.basename(f))
if _clean_string(name) != name:
rename = True
# @todo other files - need to unify extension handling somewhere
if ext.lower() == '.shp':
main_file = f
elif ext.lower() == '.tif':
main_file = f
elif ext.lower() == '.csv':
main_file = f
# if an sld is there, extract so it can be found
if ext.lower() == '.sld':
zf.extract(f, dirname)
if not main_file: raise Exception(
'Could not locate a shapefile or tif file')
if rename:
# dang, have to unpack and rename
zf.extractall(dirname)
zf.close()
if rename:
os.unlink(base_file)
base_file = os.path.join(dirname, main_file)
for f in os.listdir(dirname):
safe = _clean_string(f)
if safe != f:
os.rename(os.path.join(dirname, f), os.path.join(dirname, safe))
return os.path.join(
dirname,
_clean_string(os.path.basename(base_file))
)
def make_runtime(capsule, output, licfile=None, platform=None):
myzip = ZipFile(capsule, 'r')
if 'pytransform.key' in myzip.namelist():
myzip.extract('pytransform.key', output)
else:
myzip.extract('pyshield.key', output)
myzip.extract('pyshield.lic', output)
myzip.extract('product.key', output)
if licfile is None:
myzip.extract('license.lic', output)
else:
shutil.copy2(licfile, os.path.join(output, 'license.lic'))
if platform is None:
libname = dll_name + dll_ext
libfile = os.path.join(PYARMOR_PATH, libname)
if not os.path.exists(libfile):
sysname = pytransform.format_platname()
libpath = os.path.join(PYARMOR_PATH, 'platforms')
libfile = os.path.join(libpath, sysname, libname)
shutil.copy2(libfile, output)
else:
path = os.path.join(PYARMOR_PATH, 'platforms', platform)
for x in os.listdir(path):
shutil.copy2(os.path.join(path, x), output)
shutil.copy2(os.path.join(PYARMOR_PATH, 'pytransform.py'), output)
def unzip(self, infile, path):
"""
Given a zip file (infile) and path to unzip to (path), unzip the file and return the full path.
"""
path = path.strip("/")
zip = ZipFile(infile)
names = zip.namelist()
# # When passed PKWare zips with Type 9 compression, we can read the zip header, but
# # are unable to do the extraction. If for some reason we have a problem reading
# # headers, we can use the system unzip to do so, as commented below:
# status = not subprocess.call(["which", "unzip"])
# if status:
# out = subprocess.run(["unzip", "-Z1", infile], stdout=subprocess.PIPE)
# names = [ i.decode() for i in out.stdout.split()]
extractions = []
for name in names:
if name.startswith("npidata_pfile_") and \
name.endswith(".csv") and \
not "FileHeader" in name:
extractions.append(name)
if len(extractions) != 1:
print("Did not find exactly one file in {infile}. Exiting.")
sys.exit(1)
csv_file = extractions[0]
try:
zip.extract(csv_file, path)
except NotImplementedError as e:
print(
"Python does not support Type 9 compression, trying system unzip..."
)
if not subprocess.run(["which", "unzip"]).returncode:
out = subprocess.run(["unzip", infile, csv_file, "-d", path],
stdout=subprocess.PIPE)
if out.returncode:
print("Can't unzip this file. Local unzip failed.")
raise
else:
print(
"Can't unzip this file. Type 9 compression not supported, and no local unzip."
)
raise
return f"{path}/{csv_file}"
def import_file(uploaded, parser):
destination_dir, destination_file = create_tmp_destination(uploaded.name)
with open(destination_file, 'w+') as f:
f.write(uploaded.file.read())
zfile = ZipFile(f)
for name in zfile.namelist():
try:
zfile.extract(name, os.path.dirname(os.path.realpath(f.name)))
except Exception:
raise
if name.endswith('shp'):
import_datas.delay(parser.__name__, '/'.join(
(destination_dir, name)), parser.__module__)
def extractFile(filepath):
filename, file_extension = os.path.splitext(filepath)
if file_extension == '.fz':
os.rename(filepath, "output.fz")
else:
with open(filepath, 'rb') as file:
z = ZipFile(file)
for name in z.namelist():
if '.fzp' not in name and '.fz' in name:
z.extract(name)
os.rename(name, "output.fz")
break
def classify_zip():
if request.method == 'POST':
file = request.files['file']
if not file:
return "No file sent."
if not file.filename.split(".")[-1] == 'zip':
return ".zip is the only compression format currently supported"
filename = secure_filename(file.filename)
zip_file_path = os.path.join(app.config['UPLOAD_FOLDER'], filename)
file.save(zip_file_path)
zip_file = ZipFile(zip_file_path)
zip_file_list = list_zip_files(zip_file_path)
all_images = filter_image_files(zip_file_list, ALLOWED_EXTENSIONS)
if len(all_images) == 0:
return "No image files detected in the zip file"
#loop through images
start = 0
increment = 500
all_images_len = len(all_images)
while start < all_images_len:
end = start + increment
if end > len(all_images):
end = len(all_images)
#extract filenames
curr_file_list = all_images[start:end]
for filename in curr_file_list:
zip_file.extract(filename, path=app.config['UPLOAD_FOLDER'])
curr_file_list = [
os.path.join(app.config['UPLOAD_FOLDER'], x)
for x in curr_file_list
]
predictions = predict_multiple(GRAPH, curr_file_list, LABELS)
#remove files
for curr_file in curr_file_list:
os.remove(curr_file)
return make_response(json.dumps(predictions, cls=NumpyEncoder))
start = end + 1
def fetch_data(self, i):
url = self._data["images"][i]["rawUrl"]
r = self._client.get(url)
zip = ZipFile(BytesIO(r.content))
images = []
with tempfile.TemporaryDirectory() as tmpdir:
for name in zip.namelist():
if name.endswith(".tif"):
zip.extract(name, tmpdir)
images.append(Image.open(tmpdir + "/" + name))
zip.close()
arrays = [np.array(image) for image in images]
combined_image = np.dstack(arrays)
return combined_image
def __call__(self, data_bytes):
zf = ZipFile(io.BytesIO(data_bytes))
uncompress_size = sum((file.file_size for file in zf.infolist()))
with tqdm(desc=self.label,
total=uncompress_size,
unit='B',
unit_scale=True,
unit_divisor=1024) as bar:
for f in zf.infolist():
filename = f.filename
zf.extract(f, self.workspace_out_dir)
bar.set_postfix(file=filename[-10:], refresh=False)
bar.update(f.file_size)
def unzip_fp(f):
with tmp_dir() as tmp:
zf = ZipFile(f, mode='r')
members = zf.namelist()
members = [
m for m in members if (('__MACOS' not in m) and (
not m.startswith('/')) and (not '..' in m))
]
for member in members:
zf.extract(member, path=tmp)
yield tmp
def make_runtime(capsule, output, licfile=None, platforms=None, package=False):
if package:
output = os.path.join(output, 'pytransform')
if not os.path.exists(output):
os.makedirs(output)
logging.info('Generating runtime files to %s', output)
myzip = ZipFile(capsule, 'r')
if 'pytransform.key' in myzip.namelist():
logging.info('Extract pytransform.key')
myzip.extract('pytransform.key', output)
else:
logging.info('Extract pyshield.key, pyshield.lic, product.key')
myzip.extract('pyshield.key', output)
myzip.extract('pyshield.lic', output)
myzip.extract('product.key', output)
if licfile is None:
logging.info('Extract license.lic')
myzip.extract('license.lic', output)
else:
logging.info('Copying %s as license file', licfile)
shutil.copy2(licfile, os.path.join(output, 'license.lic'))
if not platforms:
libfile = pytransform._pytransform._name
if not os.path.exists(libfile):
libname = dll_name + dll_ext
libfile = os.path.join(PYARMOR_PATH, libname)
if not os.path.exists(libfile):
pname = pytransform.format_platform()
libpath = os.path.join(PYARMOR_PATH, 'platforms')
libfile = os.path.join(libpath, pname, libname)
logging.info('Copying %s', libfile)
shutil.copy2(libfile, output)
elif len(platforms) == 1:
filename = _build_platforms(platforms)[0]
logging.info('Copying %s', filename)
shutil.copy2(filename, output)
else:
libpath = os.path.join(output, pytransform.plat_path)
logging.info('Create library path to support multiple platforms: %s',
libpath)
if not os.path.exists(libpath):
os.mkdir(libpath)
filenames = _build_platforms(platforms)
for platid, filename in list(zip(platforms, filenames)):
logging.info('Copying %s', filename)
path = os.path.join(libpath, *platid.split('.'))
logging.info('To %s', path)
if not os.path.exists(path):
os.makedirs(path)
shutil.copy2(filename, path)
filename = os.path.join(PYARMOR_PATH, 'pytransform.py')
shutil.copy2(filename,
os.path.join(output, '__init__.py') if package else output)
logging.info('Generate runtime files OK')
def install_forge(name, version, forge_version):
'''安装forge'''
version_path = os.path.join(g.config['cur_gamepath'], 'versions')
game_path = os.path.join(version_path, name)
config = json.load(open(os.path.join(game_path, f'{name}.json')))
installerpath = game_path + f'\\installer.jar'
# 获取forge的配置
zip = ZipFile(installerpath)
zip.extract('version.json', game_path)
zip.extract('install_profile.json', game_path)
forge_config = json.load(open(os.path.join(game_path,
f'version.json')))
install_profile = json.load(
open(os.path.join(game_path, f'install_profile.json')))
thread_count = threading.active_count()
for i in install_profile['libraries']:
Game.analysis_library(zip, i)
for i in forge_config['libraries']:
Game.analysis_library(zip, i)
while threading.active_count() > thread_count:
g.logapi.debug(f'剩余线程{threading.active_count()}({thread_count})')
time.sleep(1)
g.logapi.debug(f'剩余线程{threading.active_count()}({thread_count})')
zip.close()
# forge-client
lib_path = os.path.join(g.config['cur_gamepath'], f'libraries')
userdev_path = game_path + f'\\userdev.jar'
client_path = os.path.join(
lib_path,
f'net/minecraftforge/forge/{version}-{forge_version}/forge-{version}-{forge_version}-client.jar'
)
# 获取需要的文件
file_subset_list = []
with ZipFile(userdev_path) as userdev:
for i in userdev.namelist():
if 'patches' in i and '.' in i:
file_subset_list.append(i)
Game.stream_conents(userdev_path, client_path, file_subset_list,
lambda path: '/'.join(path.split('/')[1:]))
# 拼接
Game.splicing(config, forge_config)
json.dump(config,
open(os.path.join(game_path, f'{name}.json'), mode='w'))
def download_and_unzip(buffer, a, b, path):
unzipped = []
from zipfile import ZipFile, BadZipfile
try:
zip_file = ZipFile(buffer)
except BadZipFile: # Often happens with GEE API
print("bad_zip")
return None
files = zip_file.namelist()
for i in range(a, b):
zip_file.extract(files[i], path + "/tiff/")
#print("{} downloaded and unzippped".format(files[i]))
unzipped.append(files[i])
return unzipped
def classify_zip():
"""Classify all images from a zip file"""
if request.method != "POST":
return
file = request.files["file"]
if not file:
return "No file sent."
if file.filename.split(".")[-1] != "zip":
return ".zip is the only compression format currently supported"
filename = secure_filename(file.filename)
zip_file_path = os.path.join(app.config["UPLOAD_FOLDER"], filename)
file.save(zip_file_path)
zip_file = ZipFile(zip_file_path)
zip_file_list = list_zip_files(zip_file_path)
all_images = filter_image_files(zip_file_list, ALLOWED_EXTENSIONS)
if len(all_images) == 0:
return "No image files detected in the zip file"
# loop through images
start = 0
increment = 500
all_images_len = len(all_images)
while start < all_images_len:
end = start + increment
if end > len(all_images):
end = len(all_images)
# extract filenames
curr_file_list = all_images[start:end]
for filename in curr_file_list:
zip_file.extract(filename, path=app.config["UPLOAD_FOLDER"])
curr_file_list = [
os.path.join(app.config["UPLOAD_FOLDER"], x)
for x in curr_file_list
]
predictions = predict_multiple(curr_file_list)
# remove files
for curr_file in curr_file_list:
os.remove(curr_file)
return make_response(jsonify(predictions))
def _parse_connectome_surfaces(self, connectome_surface, warning_message, should_center):
"""
Parse data from a CSurface object and save it in our internal Surface DataTypes
"""
surfaces, processed_files = [], []
parser = GIFTIParser(self.storage_path, self.operation_id)
for c_surface in connectome_surface:
if c_surface.src in processed_files:
continue
try:
# create a meaningful but unique temporary path to extract
tmpdir = os.path.join(gettempdir(), c_surface.parent_cfile.get_unique_cff_name())
self.log.debug("Extracting %s[%s] into %s ..." % (c_surface.src, c_surface.name, tmpdir))
_zipfile = ZipFile(c_surface.parent_cfile.src, 'r', ZIP_DEFLATED)
gifti_file_1 = _zipfile.extract(c_surface.src, tmpdir)
gifti_file_2 = None
surface_name, pair_surface = self._find_pair_file(c_surface, connectome_surface)
if pair_surface:
self.log.debug("Extracting pair %s[%s] into %s ..." % (pair_surface.src, pair_surface.name, tmpdir))
gifti_file_2 = _zipfile.extract(pair_surface.src, tmpdir)
surface_type = self._guess_surface_type(c_surface.src.lower())
self.logger.info("We will import surface %s as type %s" % (c_surface.src, surface_type))
surface = parser.parse(gifti_file_1, gifti_file_2, surface_type, should_center)
surface.user_tag_1 = surface_name
validation_result = surface.validate()
if validation_result.warnings:
warning_message += validation_result.summary() + "\n"
surfaces.append(surface)
if pair_surface:
processed_files.append(pair_surface.src)
processed_files.append(c_surface.src)
if os.path.exists(tmpdir):
shutil.rmtree(tmpdir)
except ParseException:
self.logger.exception("Could not import a Surface entity.")
warning_message += "Problem when importing Surfaces!! \n"
except OSError:
self.log.exception("Could not clean up temporary file(s).")
return surfaces
def shortcut_download(dataset, compression_type='tar.gz'):
"""Download and unpack pre-processed dataset"""
if compression_type not in ['tar.gz', 'zip']:
print('Warning! Wrong compression format. Changing to tar.gz')
compression_type = 'tar.gz'
if dataset == 'reddit_casual' and compression_type == 'zip':
print(
'Warning! Zip format is not supported for reddit casual dataset due to file size. Changing to tar.gz'
)
compression_type = 'tar.gz'
if not os.path.exists(datasets_dir):
os.makedirs(datasets_dir)
compressed_url = f'https://affect.media.mit.edu/neural_chat/datasets/{dataset}_preprocessed.{compression_type}'
compressed_file_dir = datasets_dir.joinpath(dataset)
compressed_file_path = datasets_dir.joinpath(
f'{dataset}_preprocessed.{compression_type}')
# Prepare Dialog data
if not os.path.exists(compressed_file_dir):
print(f'Downloading {compressed_url} to {compressed_file_path}')
urlretrieve(compressed_url, compressed_file_path)
print(f'Successfully downloaded {compressed_file_path}')
if compression_type == 'tar.gz':
tar_ref = tarfile.open(compressed_file_path, 'r:gz')
for member in tar_ref.getmembers():
try:
tar_ref.extract(member, path=datasets_dir)
print(f'Extracting {member.name}: OK')
except Exception as e:
print(f'Extracting {member.name}: ERROR - {e}')
tar_ref.close()
elif compression_type == 'zip':
zip_ref = ZipFile(compressed_file_path, mode='r')
for member in zip_ref.infolist():
try:
zip_ref.extract(member, path=datasets_dir)
print(f'Extracting {member}: OK')
except Exception as e:
print(f'Extracting {member}: ERROR - {e}')
zip_ref.close()
print(f'Successfully extracted {compressed_file_path}')
else:
print('Directory already exists. Aborting download.')
def grab_ensdf_decay(build_dir=""):
"""
Grabs the ENSDF decay data files
if not already present.
Parameters
----------
build_dir : str
Major directory to place html files in. 'ENSDF/' will be appended.
"""
# Add ENSDF to build_dir
build_dir = os.path.join(build_dir, 'ENSDF')
try:
os.makedirs(build_dir)
except OSError:
pass
# Grab ENSDF files and unzip them.
iaea_base_url = 'http://www.nndc.bnl.gov/ensarchivals/distributions/dist19/'
cf_base_url = 'https://github.com/pyne/data/raw/master/'
ensdf_zip = ['ensdf_191004_099.zip',
'ensdf_191004_199.zip',
'ensdf_191004_300.zip', ]
for f in ensdf_zip:
fpath = os.path.join(build_dir, f)
if f not in os.listdir(build_dir):
print(" grabbing {0} and placing it in {1}".format(f, fpath))
conn = urllib.urlopen(iaea_base_url + f)
with open(fpath, 'wb') as f:
f.write(conn.read())
if os.path.getsize(fpath) < 1048576:
print(" could not get {0} from NNDC; trying mirror".format(f))
os.remove(fpath)
conn = urllib.urlopen(cf_base_url + f)
with open(fpath, 'wb') as f:
f.write(conn.read())
# not using ZipFile context manager (with statement for Python 2.6)
try:
zf = ZipFile(fpath)
for name in zf.namelist():
if not os.path.exists(os.path.join(build_dir, name)):
print(" extracting {0} from {1}".format(name, fpath))
zf.extract(name, build_dir)
finally:
zf.close()
def oem_logo_check():
filecheck = FileCheck()
LOGOSCALE = {
"7寸": ["102ML", "700ML", "750ML", "2070", "2070D", "2070S"],
"4.3寸": ["2043T", "2043E", "2043E-N"],
"3.5寸": ["2035T"]
}
logo_md5s = []
filename = "logo.NTB"
path_oem_logos = r"E:\Redmine2019\LEVIOEM测试\logo_OEM"
path_levi_oem = r"\\192.168.11.20\hmi软件镜像\nuc972(V1.0 ~ V1.4)\组态_LEVI\OEM"
filelist = filecheck.lookfolder(path_levi_oem) # 获取文件绝对路径
print(filelist)
path_temp = r"E:\Redmine2019\LEVIOEM测试\temp"
oem_logo_dic = dict()
for fullpath in filelist: # 穷举OEM目录下所有文件
path, name = os.path.split(fullpath) # 分离“productfile.osf”
if name == "productfile.osf": # 取productfile.osf文件
osf = ZipFile(fullpath)
fileb = osf.open(filename, 'rU')
md5 = filecheck.calc_md5(fileb)
p, oeminfostr = os.path.split(
path) # 分离“NUC972_2070S_OEM泉州科源_6.4.18_2019-01-23”
oeminfo = oeminfostr.split("_")
if len(oeminfo) >= 4: # 具有标准命名格式的才进行处理
hmitype = oeminfo[1] # 获取2070S
oemcomp = ''.join(oeminfo[2:-2]) # 获取OEM泉州科源
if oemcomp not in oem_logo_dic.keys(): # 相同OEM厂家的归类
oem_logo_dic[oemcomp] = {"7寸": [], "4.3寸": [], "3.5寸": []}
for scale in LOGOSCALE.keys(): # 按不同尺寸区分
if hmitype in LOGOSCALE[scale]: # 型号归类
if not oem_logo_dic[oemcomp][scale]: # 若未存MD5值
oem_logo_dic[oemcomp][scale].append(md5)
depath = os.path.join(
path_oem_logos, oemcomp,
scale) # 创建目录logos\OEM泉州科源\7寸
if not os.path.exists(depath):
os.makedirs(depath)
osf.extract(filename, depath)
else: # 已存MD5，但值不一致，说明文件不同。
if md5 not in oem_logo_dic[oemcomp][scale]:
print("OEM LOGO 文件冲突：{}".format(
oeminfostr)) # 同厂家同尺寸屏但LOGO文件不一致
d = json.dumps(oem_logo_dic)
print(d)
with open(
os.path.join(path_oem_logos, "{} md5 dict.json".format(filename)),
"w") as f:
f.write(d)
def agregar_theme(request):
if request.method == "POST":
form = ThemeForm(request.POST, request.FILES)
if form.is_valid():
ZipFile(request.FILES.get('comprimido'), 'r')
nombre_carpeta = ""
error_template = False
error_static = False
error_imagen = False
error_json = False
mensaje = ""
unziped = ZipFile(request.FILES.get('comprimido'), 'r')
print unziped.namelist()
for file_path in unziped.namelist():
split_directorio = file_path.split("/")
if split_directorio[0] != 'template' and split_directorio[
0] != 'static':
if not nombre_carpeta:
nombre_carpeta = split_directorio[0]
if split_directorio[1] == 'template' or split_directorio[
0] == 'template':
archivo = unziped.extract(
file_path, "templates/" + request.user.username + "/" +
request.POST.get('theme_titulo'))
if not error_template:
error_template = True
if split_directorio[1] == 'static' or split_directorio[
0] == 'static':
# print "entro"
archivo = unziped.extract(
file_path, "static/" + request.user.username + "/" +
request.POST.get('theme_titulo'))
if not error_static:
error_static = True
if error_static and error_template:
theme = form.save()
theme.carpeta_titulo = nombre_carpeta
theme.save()
mensaje = "Se Creo el theme Satisfactoriamente"
return redirect(reverse_lazy("configuracion:home"))
else:
mensaje = "Hubo error posiblemente por que no tiene la carpeta template y static"
# file_content = unziped.printdir()
# print "----"
# print file_path
# print unziped.extract(file_path, "carpeta nueva/template")
else:
form = ThemeForm()
return render(request, 'core/agregar_theme.html', locals())
def cache(self):
if not exists(self.data_home):
makedirs(self.data_home)
if not exists(self.data_dir):
print('downloading met-enk from %s to %s' %
(DATA_URL, self.data_dir))
fhandle = urlopen(DATA_URL)
buf = BytesIO(fhandle.read())
zip_file = ZipFile(buf)
makedirs(self.data_dir)
for name in zip_file.namelist():
zip_file.extract(name, path=self.data_dir)
self.cached = True
def unzip(self):
try:
from zipfile import ZipFile
except:
raise ImportError("没有zipFile库")
file = ZipFile(r"{}".format(self.file))
if not os.path.exists(self.path):
os.mkdir(self.path)
for file_name in file.namelist():
filename = file_name.encode("cp437").decode("gbk")
file.extract(file_name, path=self.path)
os.chdir(self.path)
os.rename(file_name, filename)
def _extract_certificate(file_name):
"""Extract the cert of an APK and return the extracted file name."""
apk_zip = ZipFile(file_name)
cert_file = None
cert_file_name_regexp = re.compile(r'^META-INF\/.*\.[RD]SA$')
for zipped_file in apk_zip.filelist:
if cert_file_name_regexp.match(zipped_file.filename):
cert_file = zipped_file
break
else:
raise CryptoVerificationError(
file_name, message='No certificate found in APK. Is it signed?')
apk_zip.extract(cert_file, SETTINGS['temp_dir'])
return os.path.join(SETTINGS['temp_dir'], cert_file.filename)
def download_and_extract(self, lang):
print('Downloading corpus')
if path.exists(f'data/{lang}.txt'):
return
url = f'{self.base_url}/{lang}-eng.zip'
request = Request(url, None, {'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.7) '
'Gecko/2009021910 Firefox/3.0.7'})
archive = urlopen(request)
fd = open(f'/tmp/{lang}.zip', 'wb')
fd.write(archive.read())
fd.close()
zf = ZipFile(f'/tmp/{lang}.zip')
zf.extract(member=f'{lang.split("-")[0]}.txt', path=f'data/')
return self
def unzip():
print("Unzipping data...")
zipFile = ZipFile(zipFolder, "r")
uncompressedSize = sum(file.file_size for file in zipFile.infolist())
extractedSize = 0
pbar = ProgressBar(widgets=[Percentage(), Bar()], maxval=100).start()
start = datetime.datetime.now()
for file in zipFile.infolist():
extractedSize += file.file_size
percent = extractedSize * 100 / uncompressedSize
pbar.update(percent)
zipFile.extract(file, path=zipFolder.rsplit(".", 1)[0])
print("Unzipped in {} s.".format((datetime.datetime.now() - start).seconds))
zipFile.close()
def extract(_p_dir, file='RomDataBu/df_kl.h5.zip'):
"""
解压数据
"""
data_example_zip = os.path.join(_p_dir, file)
try:
from zipfile import ZipFile
zip_h5 = ZipFile(data_example_zip, 'r')
unzip_dir = os.path.join(_p_dir, 'RomDataBu/')
for h5 in zip_h5.namelist():
zip_h5.extract(h5, unzip_dir)
zip_h5.close()
except Exception as e:
print('example env failed! e={}'.format(e))
def __extract_cbz(self, file):
""" extract first image in cbz file, cache it in local cache folder"""
try:
archive_zip = ZipFile(file, 'r')
extract_path = self.cache + path.basename(file)
if self.mode is 'all':
archive_zip.extractall(path=extract_path)
else:
first_file = archive_zip.namelist()[0]
archive_zip.extract(member=first_file, path=extract_path)
except BadZipFile as e:
raise e
finally:
archive_zip.close()
return extract_path
def extract_overrides(self, pack_archive: zipfile.ZipFile):
p = os.path
# TODO : Slight chance of 'override' folder being a flexibly named dir
# : that is hard referenced within the addon info json from the api
for i in pack_archive.namelist():
if i.startswith('overrides'):
pack_archive.extract(i, self.out_dir)
o_folder = p.join(self.out_dir, 'overrides')
# o_temp = p.join(self.out_dir, '_overrides_temp')
# TODO : Might be bad idea if overrides folder contains a folder named overrides (:3)
copy_tree(o_folder, self.out_dir)
remove_tree(o_folder)
def unzip(zip_filename, dest_folder, loading_bar=tqdm):
"""
Unzipping archive
"""
zip_file = ZipFile(zip_filename)
all_files = zip_file.infolist()
uncompressed_size = sum((file.file_size for file in zip_file.infolist()))
files = zip_file
# Uncompressing
with loading_bar(total=uncompressed_size) as pbar:
for file in all_files:
zip_file.extract(file, dest_folder)
pbar.update(file.file_size)
def install(self, data, fname):
if fname.endswith(".py"):
# .py files go directly into the addon folder
path = os.path.join(self.addonsFolder(), fname)
open(path, "w").write(data)
return
# .zip file
z = ZipFile(StringIO(data))
base = self.addonsFolder()
for n in z.namelist():
if n.endswith("/"):
# folder; ignore
continue
# write
z.extract(n, base)
