def procesarZIP(archivo):
try:
z = ZipFile(archivo)
except:
return
nombre = z.filename.split('.')[0]
ruta_temporal = '/tmp/%s/' % nombre
z.extractall(ruta_temporal)
for a in z.filelist:
if a.filename[-3:] == "dbf":
archivo_dbf = "%s%s" % (ruta_temporal, a.filename)
puntos = shp2dataframe(archivo_dbf)
# print "%s %s" % (nombre, puntos['REND'].mean())
# ya quito los nulos
no_nulos = filter(None, puntos.REND.values)
promedio = promedioRendimiento(no_nulos)
if a.filename[-3:] == "shp":
archivo_shp = "%s%s" % (ruta_temporal, a.filename)
centro_x, centro_y = obtenerCentroide(archivo_shp)
print "%s,%s,%s,%s" % (nombre.split('/')[-1], promedio, centro_x, centro_y)
borrarArbol(ruta_temporal)
def MyUnzipFile(in_file_path, target_root_path):
"""Unzip file.
>>> unzip_file("d:\\zout.zip", "d:\\")"""
zin = ZipFile(in_file_path, "r")
zin.extractall(target_root_path)
zin.close()
def build(worker_dir, sha, repo_url, destination, concurrency):
"""Download and build sources in a temporary directory then move exe to destination"""
tmp_dir = tempfile.mkdtemp()
os.chdir(tmp_dir)
with open('sf.gz', 'wb+') as f:
f.write(requests.get(github_api(repo_url) + '/zipball/' + sha).content)
zip_file = ZipFile('sf.gz')
zip_file.extractall()
zip_file.close()
for name in zip_file.namelist():
if name.endswith('/src/'):
src_dir = name
os.chdir(src_dir)
custom_make = os.path.join(worker_dir, 'custom_make.txt')
if os.path.exists(custom_make):
with open(custom_make, 'r') as m:
make_cmd = m.read().strip()
subprocess.check_call(make_cmd, shell=True)
else:
subprocess.check_call(MAKE_CMD + ' -j %s' % (concurrency), shell=True)
shutil.move('stockfish'+ EXE_SUFFIX, destination)
os.chdir(worker_dir)
shutil.rmtree(tmp_dir)
def download_demo(demo_name):
"Downloads and extracts the demo zip file"
print(green("- Downloading"), end="")
response = urllib2.urlopen(DEMOS[demo_name]["url"])
# Extract real name of zipfile
content_disposition = response.headers.getheader("content-disposition")
real_name = os.path.splitext(content_disposition.split("; ")[1].split("=")[1])[0]
fname = "{0}.zip".format(demo_name)
# Download zipfile
with open(fname, "wb") as tmpfile:
while True:
packet = response.read(2 ** 16)
if not packet:
print(green("done"))
break
tmpfile.write(packet)
sys.stdout.write(green("."))
sys.stdout.flush()
response.close()
print(green("- Extracting"))
# Extract zipfile
zipfile = ZipFile(fname)
zipfile.extractall()
# Clean up and rename to the correct demo name
os.remove(fname)
os.rename(real_name, demo_name)
class UnZip():
"""Unzip the given file into a temporary directory."""
def __init__(self, input_file):
self.input_file = input_file
def unzip(self):
self.zip_file = ZipFile(self.input_file, 'r')
self.tempdir = tempfile.mkdtemp()
self.zip_file.extractall(self.tempdir)
return self.tempdir
def cleanup(self):
self.zip_file.close()
for root, dirs, files in os.walk(self.tempdir, topdown=False):
for name in files:
os.remove(os.path.join(root, name))
for name in dirs:
os.rmdir(os.path.join(root, name))
os.rmdir(self.tempdir)
def __enter__(self):
return self.unzip()
def __exit__(self, *args):
self.cleanup()
return False
def parse_template(template_name):
"""Resolve template name into absolute path to the template
and boolean if absolute path is temporary directory.
"""
if template_name.startswith('http'):
if '#' in template_name:
url, subpath = template_name.rsplit('#', 1)
else:
url = template_name
subpath = ''
with tempfile.NamedTemporaryFile() as tmpfile:
urlretrieve(url, tmpfile.name)
if not is_zipfile(tmpfile.name):
raise ConfigurationError("Not a zip file: %s" % tmpfile)
zf = ZipFile(tmpfile)
try:
path = tempfile.mkdtemp()
zf.extractall(path)
return os.path.join(path, subpath), True
finally:
zf.close()
registry = TemplatesRegistry()
if registry.has_template(template_name):
path = registry.path_of_template(template_name)
elif ':' in template_name:
path = resolve_dotted_path(template_name)
else:
path = os.path.realpath(template_name)
if not os.path.isdir(path):
raise ConfigurationError('Template directory does not exist: %s' % path)
return path, False
def download_file(url, name, root_destination='~/data/', zipfile=False,
replace=False):
"""Download a file from dropbox, google drive, or a URL.
This will download a file and store it in a '~/data/` folder,
creating directories if need be. It will also work for zip
files, in which case it will unzip all of the files to the
desired location.
Parameters
----------
url : string
The url of the file to download. This may be a dropbox
or google drive "share link", or a regular URL. If it
is a share link, then it should point to a single file and
not a folder. To download folders, zip them first.
name : string
The name / path of the file for the downloaded file, or
the folder to zip the data into if the file is a zipfile.
root_destination : string
The root folder where data will be downloaded.
zipfile : bool
Whether the URL points to a zip file. If yes, it will be
unzipped to root_destination + name.
replace : bool
If True and the URL points to a single file, overwrite the
old file if possible.
"""
# Make sure we have directories to dump files
home = op.expanduser('~')
tmpfile = home + '/tmp/tmp'
if not op.isdir(home + '/data/'):
print('Creating data folder...')
os.makedirs(home + '/data/')
if not op.isdir(home + '/tmp/'):
print('Creating tmp folder...')
os.makedirs(home + '/tmp/')
download_path = _convert_url_to_downloadable(url)
# Now save to the new destination
out_path = root_destination.replace('~', home) + name
if not op.isdir(op.dirname(out_path)):
print('Creating path {} for output data'.format(out_path))
os.makedirs(op.dirname(out_path))
if zipfile is True:
_fetch_file(download_path, tmpfile)
myzip = ZipFile(tmpfile)
myzip.extractall(out_path)
os.remove(tmpfile)
else:
if len(name) == 0:
raise ValueError('Cannot overwrite the root data directory')
if replace is False and op.exists(out_path):
raise ValueError('Path {} exists, use `replace=True` to '
'overwrite'.format(out_path))
_fetch_file(download_path, out_path)
print('Successfully moved file to {}'.format(out_path))
def _prepare():
""" Create two dirs, one with one dataset and one with two datasets
"""
need_internet()
global _prepared
if _prepared and os.path.isfile(_prepared[2]):
return _prepared
# Prepare sources
fname1 = get_remote_file('images/dicom_sample1.zip')
fname2 = get_remote_file('images/dicom_sample2.zip')
dname1 = os.path.join(test_dir, 'dicom_sample1')
dname2 = os.path.join(test_dir, 'dicom_sample2')
# Extract zipfiles
z = ZipFile(fname1)
z.extractall(dname1)
z.extractall(dname2)
z = ZipFile(fname2)
z.extractall(dname2)
# Get arbitrary file names
fname1 = os.path.join(dname1, os.listdir(dname1)[0])
fname2 = os.path.join(dname2, os.listdir(dname2)[0])
# Cache and return
_prepared = dname1, dname2, fname1, fname2
return dname1, dname2, fname1, fname2
def createDevEnv(baseDir, type):
fileBuffer = StringIO()
createBuild(baseDir, type=type, outFile=fileBuffer, devenv=True, releaseBuild=True)
from zipfile import ZipFile
zip = ZipFile(StringIO(fileBuffer.getvalue()), 'r')
zip.extractall(os.path.join(baseDir, 'devenv'))
zip.close()
print 'Development environment created, waiting for connections from active extensions...'
metadata = readMetadata(baseDir, type)
connections = [0]
import SocketServer, time, thread
class ConnectionHandler(SocketServer.BaseRequestHandler):
def handle(self):
connections[0] += 1
self.request.sendall('HTTP/1.0 OK\nConnection: close\n\n%s' % metadata.get('general', 'basename'))
server = SocketServer.TCPServer(('localhost', 43816), ConnectionHandler)
def shutdown_server(server):
time.sleep(10)
server.shutdown()
thread.start_new_thread(shutdown_server, (server,))
server.serve_forever()
if connections[0] == 0:
print 'Warning: No incoming connections, extension probably not active in the browser yet'
else:
print 'Handled %i connection(s)' % connections[0]
def _download_xbrl_file(info_dicts,p):
no = p
directory_path = os.getcwd()+'/xbrl_files/'
while(no < len(info_dicts)):
info_dict = info_dicts[no]
no += proc
# 証券CDごとのディレクトリ作成
company_path = directory_path + info_dict['cd'] + '/'
ir_path = company_path + info_dict['id']
make_directory(company_path)
# 証券コード×IR情報ごとのディレクトリ作成
if os.path.exists(ir_path):
continue
make_directory(ir_path)
print('Process('+str(p + 1) + '):downloading:' + info_dict['update']+'_'+info_dict['title'])
url = info_dict['url']
r = requests.get(url)
if r.ok:
#Requestによりファイルを取得して、Unzipする
r = requests.get(url)
z = ZipFile(io.BytesIO(r.content))
z.extractall(ir_path) # unzip the file and save files to path.
def extract_zip_file(self, transformation, submitted_file):
log.info('Extracting zip file %s' % transformation.filename)
#create transformations directory if it does not exist
transformations_dir = self.get_transformations_dir()
if not os.path.isdir(transformations_dir):
log.info('Creating transformations directory')
os.mkdir(transformations_dir)
#delete package directory if it exists
package_dir = os.path.join(transformations_dir, transformation.package_id)
if os.path.isdir(package_dir):
log.info('Deleting package directory %s' % package_dir)
shutil.rmtree(package_dir)
#create directory again and change to it
log.info('Creating package directory %s' % package_dir)
os.mkdir(package_dir)
os.chdir(package_dir)
zipfile = ZipFile(submitted_file)
log.info('Extracting file %s to directory %s' % (transformation.filename, package_dir))
zipfile.extractall()
#obtain data
submitted_file.seek(0)
transformation.data = submitted_file.read()
transformation.timestamp = datetime.now()
log.info('File %s extracted' % transformation.filename)
return package_dir
def update(self, update):
dpath = self.xplanedir + '/Resources/Downloads'
installpath = self.xplanedir + '/Resources/plugins/PythonScripts'
# Broadcast message to all plugins
XPLMSendMessageToPlugin(XPLM_NO_PLUGIN_ID, 0x8000000 | 8090 , long(1))
PI_SendMessageToScript(self, None, 0x8000000 | 8090, 1)
sleep(1)
if not os.path.exists(dpath):
os.mkdir(dpath)
if update['update_type'] == 'direct' and update['update_filename']:
urllib.urlretrieve(update['update_url'], dpath + '/' + update['update_filename'])
copy(dpath + '/' + update['update_filename'], installpath + '/' + update['update_filename'])
print dpath + '/' + update['update_filename'], installpath + '/' + update['update_filename']
elif update['update_type'] == 'zip':
zipfile = dpath + '/._xpjpcUPDATE.zip'
# Download update
urllib.urlretrieve(update['update_url'], zipfile)
zip = ZipFile(zipfile, 'r')
# Check zip file
if not zip.testzip():
# Unzip
unzipdir = dpath + '/' + zip.namelist()[0]
zip.extractall(dpath)
zip.close()
# Move files
self.tcopy(unzipdir, installpath)
rmtree(unzipdir)
os.remove(zipfile)
def create_app(name, engine):
"""
Create a Skeleton application (needs internet connection to github)
"""
try:
if engine.lower() == "sqlalchemy":
url = urlopen(SQLA_REPO_URL)
dirname = "Flask-AppBuilder-Skeleton-master"
elif engine.lower() == "mongoengine":
url = urlopen(MONGOENGIE_REPO_URL)
dirname = "Flask-AppBuilder-Skeleton-me-master"
zipfile = ZipFile(BytesIO(url.read()))
zipfile.extractall()
os.rename(dirname, name)
click.echo(click.style("Downloaded the skeleton app, good coding!", fg="green"))
return True
except Exception as e:
click.echo(click.style("Something went wrong {0}".format(e), fg="red"))
if engine.lower() == "sqlalchemy":
click.echo(
click.style(
"Try downloading from {0}".format(SQLA_REPO_URL), fg="green"
)
)
elif engine.lower() == "mongoengine":
click.echo(
click.style(
"Try downloading from {0}".format(MONGOENGIE_REPO_URL), fg="green"
)
)
return False
def _dl_ui():
url = urlopen(self._dist_url)
z = ZipFile(StringIO(url.read()))
names = [i for i in z.namelist() if '.DS_exStore' not in i and '__MACOSX' not in i]
z.extractall(self.update_dir, members=names)
log.info("Downloaded files for UI commit " + str(self.git_version).replace("\n", ""))
return self.branch
def handle_uploaded_file(f, extract_path, request, user):
zipfilepath = settings.COURSE_UPLOAD_DIR + f.name
with open(zipfilepath, 'wb+') as destination:
for chunk in f.chunks():
destination.write(chunk)
try:
zip_file = ZipFile(zipfilepath)
zip_file.extractall(path=extract_path)
except BadZipfile:
messages.error(request, _("Invalid zip file"), extra_tags="danger")
return False, 500
mod_name = ''
print(os.listdir(extract_path))
for dir in os.listdir(extract_path)[:1]:
mod_name = dir
# check there is at least a sub dir
if mod_name == '':
messages.info(request, _("Invalid course zip file"), extra_tags="danger")
return False, 400
try:
course, response = process_course(extract_path, f, mod_name, request, user)
except Exception as e:
messages.error(request, e.message, extra_tags="danger")
return False, 500
finally:
# remove the temp upload files
shutil.rmtree(extract_path, ignore_errors=True)
return course, 200
def getWP():
print 'downloading wordpress........'
open(cwd + '/temp/wp.zip', 'wb').write(urllib.urlopen('http://fr.wordpress.org/wordpress-3.5-fr_FR.zip').read())
zip = ZipFile(cwd + '/temp/wp.zip')
print 'done'
print 'Extracting wordpress...'
zip.extractall(cwd + '/temp/')
print 'done'
print 'Moving the wordpress installation to C:/wamp/www/' + projectName
# - Copy the WP installation
shutil.copytree(cwd+ '/temp/wordpress/', wampPath + projectName)
# - remove the junk
#copytree(cwd + '/temp/wordpress/', wampPath + projectName)
print 'done'
print 'Getting ready for the first run.. Opening your web browser...'
new = 2 # open in a new tab, if possible
# open a public URL, in this case, the webbrowser docs
url = "localhost/" + projectName + "/wp-admin/setup-config.php"
webbrowser.open(url,new=new)
def get_source(options):
""" Downloads pyload source from bitbucket tip or given rev"""
if options.rev: options.url = "https://bitbucket.org/spoob/pyload/get/%s.zip" % options.rev
pyload = path("pyload")
if len(pyload.listdir()) and not options.clean:
return
elif pyload.exists():
pyload.rmtree()
urlretrieve(options.src, "pyload_src.zip")
zip = ZipFile("pyload_src.zip")
zip.extractall()
path("pyload_src.zip").remove()
folder = [x for x in path(".").dirs() if x.name.startswith("spoob-pyload-")][0]
folder.move(pyload)
change_mode(pyload, 0644)
change_mode(pyload, 0755, folder=True)
for file in pyload.files():
if file.name.endswith(".py"):
file.chmod(0755)
(pyload / ".hgtags").remove()
(pyload / ".gitignore").remove()
#(pyload / "docs").rmtree()
f = open(pyload / "__init__.py", "wb")
f.close()
def setUpClass(cls):
# We zip the GRASSData folder to be nice to GitHub, unzip it
cls.grassDBasePath = os.path.abspath('./tests/data/GRASSData')
grassDBaseZip = "%s.zip" % (cls.grassDBasePath,)
if not os.access(grassDBaseZip, os.R_OK):
raise IOError(errno.EACCES, "Unable to read GRASS data zip %s" %
grassDBaseZip)
grassDBaseDir = os.path.split(grassDBaseZip)[0]
if not os.access(grassDBaseDir, os.W_OK):
raise IOError(errno.EACCES, "Unable to write to GRASS data parent dir %s" %
grassDBaseDir)
zip = ZipFile(grassDBaseZip, 'r')
extractDir = os.path.split(cls.grassDBasePath)[0]
zip.extractall(path=extractDir)
gisbase = os.environ['GISBASE']
grassConfig = GRASSConfig(gisbase=gisbase, dbase=cls.grassDBasePath, location='DR5_5m', mapset='taehee')
cls.grassdatalookup = GrassDataLookup(grass_config=grassConfig)
cls.inPatchID = 309999
cls.inZoneID = 73
cls.inHillID = 73
cls.easting = 349325.089515
cls.northing = 4350341.816966
cls.patchMap = "patch_5m"
cls.zoneMap = "hillslope"
cls.hillslopeMap = "hillslope"
def testPNGFileToConvertOdpToHTML(self):
"""Test run_generate method from odp with png to html.
This test if returns good png files"""
generate_result = self.proxy.run_generate('test_png.odp',
encodestring(
open(join('data', 'test_png.odp')).read()),
None, 'html',
'application/vnd.oasis.opendocument.presentation')
response_code, response_dict, response_message = generate_result
self.assertEquals(response_code, 200)
self.assertEquals(type(response_dict), DictType)
self.assertNotEquals(response_dict['data'], '')
self.assertEquals(response_dict['mime'], 'application/zip')
output_url = join(self.tmp_url, "zip.zip")
open(output_url, 'w').write(decodestring(response_dict['data']))
self.assertTrue(is_zipfile(output_url))
zipfile = ZipFile(output_url)
try:
png_path = join(self.tmp_url, "img0.png")
zipfile.extractall(self.tmp_url)
content_type = self._getFileType(encodestring(open(png_path).read()))
self.assertEquals(content_type, 'image/png')
m = magic.Magic()
self.assertTrue("8-bit/color RGB" in m.from_file(png_path))
finally:
zipfile.close()
if exists(output_url):
remove(output_url)
def automerge(params):
print "Resourcepack Auto-Merge Begin"
os.mkdir('myresourcepack')
pack1 = ZipFile(params[0])
pack2 = ZipFile(params[1])
print "Extracting "+pack1.filename+"..."
pack1.extractall('myresourcepack/')
print "Extracting "+pack2.filename+"..."
pack2.extractall('myresourcepack/')
print "Creating pack.mcmeta file..."
packMeta1 = getPackMeta(pack1)
packMeta2 = getPackMeta(pack2)
resultMeta = mergePackMeta(packMeta1,packMeta2)
with open('myresourcepack/pack.mcmeta','w') as f:
f.write(json.dumps(resultMeta))
print "pack.mcmeta created"
print "Creating final zip..."
# Begin: Thanks to http://stackoverflow.com/a/3612455
target_dir = 'myresourcepack'
zip = ZipFile('myresourcepack.zip', 'w')
rootlen = len(target_dir) + 1
for base, dirs, files in os.walk(target_dir):
for file in files:
fn = os.path.join(base, file)
zip.write(fn, fn[rootlen:])
# End stackoverflow thing
print "Auto-Merge done!"
def _unpack(archive_file):
"""Unpacks and extracts files from an archive
This function will unpack and extra files from the file archive_file. It
will return the directory to which the files were unpacked.
An AttributeError is raised when the archive is not supported (when the
name does not end with '.zip' or '.tar.gz')
Returns string.
"""
logger.info("Unpacking archive '{0}'".format(archive_file))
if archive_file.endswith('.zip'):
archive = ZipFile(archive_file)
rootdir = archive.namelist()[0]
elif archive_file.endswith('.tar.gz'):
archive = tarfile.open(archive_file)
rootdir = archive.getnames()[0]
else:
raise AttributeError("Unsupported archive. Can't unpack.")
logger.info("Archive root folder is '{0}'".format(rootdir))
try:
shutil.rmtree(rootdir)
except OSError:
pass
logger.info("Extracting to '{0}'".format(rootdir))
archive.extractall()
archive.close()
return rootdir
def load(self):
"""Creates one Temporary Document and write data into it.
Return the url for the document.
"""
# creates only the url of the file.
file_path = tempfile.NamedTemporaryFile(suffix=".%s" % self.source_format,
dir=self.directory_name).name
# stores the data in temporary file
open(file_path, 'wb').write(self.original_data)
# If is a zipfile is need extract all files from whitin the compressed file
if is_zipfile(file_path):
zipfile = ZipFile(file_path)
zip_filename_list = zipfile.namelist()
if 'mimetype' not in zip_filename_list and \
'[Content_Types].xml' not in zip_filename_list:
zipfile_path = file_path
zipfile.extractall(path=self.directory_name)
zipfile.close()
filename_list = listdir(self.directory_name)
if 'index.html' in filename_list:
file_path = join(self.directory_name, 'index.html')
else:
mimetype_list = ['text/html', 'application/xhtml+xml']
for filename in filename_list:
if mimetypes.guess_type(filename)[0] in mimetype_list:
file_path = join(self.directory_name, filename)
break
if zipfile_path != file_path:
remove(zipfile_path)
return file_path
def archive_download(self, url, cache_path, checksum):
rc = 0
# Download
arcfile = os.path.join(cache_path, os.path.basename(url))
tdir = os.path.dirname(arcfile)
filetool.directory(tdir)
tfp = open(arcfile, "wb")
#(fname, urlinfo) = urllib.urlretrieve(url, arcfile)
urlobj = urllib.urlopen(url)
assert urlobj.getcode() == 200, "Could not the download contrib archive: %s" % url
hashobj = self.copy_and_hash(urlobj.fp, tfp)
assert hashobj.hexdigest()==checksum, "Checksum of archive does not validate (should be: %s): %s" % (checksum, arcfile)
urlobj.close()
tfp.close()
# Extract
if url.endswith('.zip'):
zipf = ZipFile(arcfile, 'r')
zipf.extractall(tdir)
zipf.close()
else: # .tar, .tgz, .tar.gz, .tar.bz2
tar = tarfile.open(arcfile)
tar.extractall(tdir)
tar.close
# Eliminate archive top-dir
_, archive_dirs, _ = os.walk(tdir).next()
assert archive_dirs, "The downloaded archive is not in single top-dir format: %s" % arcfile
archive_top = os.path.join(tdir, archive_dirs[0]) # just take the first dir entry
for item in os.listdir(archive_top):
shutil.move(os.path.join(archive_top, item), tdir)
os.rmdir(archive_top)
os.unlink(arcfile)
return rc
def download_and_unzip(url, pth='./'):
if not os.path.exists(pth):
print('Creating the directory: {}'.format(pth))
os.makedirs(pth)
print('Attempting to download the file: ', url)
file_name = os.path.join(pth, url.split('/')[-1])
try:
f, header = urlretrieve(url, file_name)
except:
msg = 'Cannot download file: {}'.format(url)
raise Exception(msg)
# Unzip the file, and delete zip file if successful.
if 'zip' in os.path.basename(file_name) or 'exe' in os.path.basename(file_name):
z = ZipFile(file_name)
try:
print('Extracting the zipfile...')
z.extractall(pth)
except:
p = 'Could not unzip the file. Stopping.'
raise Exception(p)
z.close()
elif 'tar' in os.path.basename(file_name):
ar = tarfile.open(file_name)
ar.extractall(path=pth)
ar.close()
print('Deleting the zipfile...')
os.remove(file_name)
print('Done downloading and extracting...')
def acs(file_num, geo_type_dir):
base_url = 'http://www2.census.gov/acs2010_5yr/summaryfile/2006-2010_ACSSF_By_State_By_Sequence_Table_Subset/'
path = os.path.join(
state_name.replace(' ', ''), geo_type_dir,
'20105%s%04d000.zip' % (state_abbr, file_num)
)
reader = remote_fallback_reader(
'../data/acs2006_2010_5yr/summaryfile/2006-2010_ACSSF_By_State_By_Sequence_Table_Subset',
base_url,
path
)
z = ZipFile(reader)
n = z.namelist()
z.extractall('/tmp/')
files = ['e20105%s%04d000.txt' % (state_abbr, file_num),
'm20105%s%04d000.txt' % (state_abbr, file_num),]
for f in files:
z.extract(f, '/tmp/')
# count the number of columns, to determine how many columns to fill
firstline = open('/tmp/' + f).readline()
if not firstline:
# some files are empty, so just continue to the next
os.unlink('/tmp/' + f)
continue
cols = ['fileid', 'filetype', 'stusab', 'chariter', 'cifsn', 'logrecno']
cols.extend(map(
lambda c: 'col%s' % str(c+1),
range(firstline.count(',')+1-6) # subtract 6 to account for the 6 geo header columns
))
cursor.copy_from(open('/tmp/%s' % f), 'census_row', sep=',',
columns=cols)
os.unlink('/tmp/' + f)
def main(args):
manifest = args[1]
print("creating work directory")
workdir = tempfile.mkdtemp(prefix="work-dir", dir=".")
print("extracting manifest")
zip_file = ZipFile(manifest, "r")
zip_file.extractall(workdir)
zip_file = ZipFile(os.path.join(workdir, "consumer_export.zip"), "r")
zip_file.extractall(workdir)
print("reading products")
products = glob.glob(os.path.join(workdir, "export", "products",
"*.json"))
content = []
for product in products:
print product
content.extend(load_content(product))
# These credentials should be good for a normal Katello/SAM deployment:
conn = psycopg2.connect("dbname=canadianTenPin user=postgres")
scan_content(conn, content)
conn.commit()
print("cleaning up")
shutil.rmtree(workdir)
print("all done.")
def _copyZipDir(self, zipFilePath, inDirPath, path):
'''
Copy a directory from a ZIP archive to a filesystem directory
@param zipFilePath: string
The path of the ZIP archive
@param inDirPath: string
The path to the file in the ZIP archive
@param path: string
Destination path where the ZIP directory is copied
'''
# make sure the ZIP file path is normalized and uses the OS separator
zipFilePath = normOSPath(zipFilePath)
# make sure the ZIP file path is normalized and uses the ZIP separator
inDirPath = normZipPath(inDirPath)
zipFile = ZipFile(zipFilePath)
if isdir(path):
if self._isSyncFile(zipFilePath, path):
return
rmtree(path)
entries = [ent for ent in zipFile.namelist() if ent.startswith(inDirPath)]
tmpDir = TemporaryDirectory()
zipFile.extractall(tmpDir.name, entries)
tmpDirPath = join(tmpDir.name, normOSPath(inDirPath))
os.makedirs(path)
for entry in os.listdir(tmpDirPath):
move(join(tmpDirPath, entry), path)
def ImportTfZip( config = None, ConfigSet = "ImportTfZip" ):
'''
Imports events and config from file (or stdin)
@param config: The config object to use. If None, then we try to use defaults or stdin
@param ConfigSet: The section of the configuration to use
@raise KeyError: If supplied ConfigSet is not in config
@return: (weeksums, config)
'''
# fetch config values
ZipDataDir = config.get( ConfigSet, "ZipDataDir" ).rstrip("/\\")
ZipFilename = config.get( ConfigSet, "ZipFile" )
zf = ZipFile( ZipFilename )
zf.extractall( ZipDataDir )
ImportTfSection = "ImportTf"
SheetName = config.get( ImportTfSection, "Sheetname" )
AllEvents = []
for DataFile in zf.namelist():
config.set( ImportTfSection, "InFile", "%s/%s"%(ZipDataDir, DataFile) )
config.set( ImportTfSection, "CsvFile", "%s/%s.%s.%s"%(ZipDataDir, DataFile,SheetName,u'csv') )
m = re.findall( "[0-9]{4}", DataFile )
if m:
config.set( ImportTfSection, "Year", m[0] )
try:
events, config = ImportTf( config, ImportTfSection )
AllEvents += events
except ValueError, e :
# no events
sys.stderr.write( "%s\n"% e.message )
def unzip_snap_mp4(abspath, quiet=False):
zipped_snap = ZipFile(abspath)
# unzip /path/to/zipfile.mp4 to /path/to/zipfile
unzip_dir = os.path.splitext(abspath)[0]
zipped_snap.extractall(unzip_dir)
# move /path/to/zipfile.mp4 to /path/to/zipfile.zip
os.rename(abspath, unzip_dir + '.zip')
for f in os.listdir(unzip_dir):
# mv /path/to/zipfile/media~* /path/to/zipfile.mp4
if f.split('~')[0] == 'media':
os.rename(os.path.join(unzip_dir, f), unzip_dir + '.mp4')
# mv /path/to/zipfile/overlay~* /path/to/zipfile_overlay.png
elif f.split('~')[0] == 'overlay':
os.rename(os.path.join(unzip_dir, f),
unzip_dir + '_overlay.png')
try:
os.rmdir(unzip_dir)
except OSError:
print('Something other than a video or overlay was in {0}. \
Cannot remove directory, not empty.'
.format(unzip_dir + '.zip'))
if not quiet:
print('Unzipped {0}'.format(abspath))
def extract_then_load(pk, zfilepath, test_method=None):
strip_excess = False
stripz_output = None
messages = []
#: open zip file and get paths
arcpy.AddMessage('uploaded {}.'.format(zfilepath))
zfile = ZipFile(zfilepath)
zfilenames = zfile.namelist()
zfile_exts = [name.split('.')[1] for name in zfilenames]
zfile_name = zfilenames[0].split('.')[0]
zfile_folder = join(arcpy.env.scratchFolder, zfile_name)
shapefile = join(zfile_folder, zfile_name + '.shp')
arcpy.AddMessage('verify that all files are present')
#: verify that all files are present
for ext in required_files:
if ext not in zfile_exts:
raise Exception('Missing .{} file'.format(ext))
zfile.extractall(zfile_folder)
arcpy.AddMessage('validating geometry')
#: validate geometry
checkgeom_output = 'in_memory/checkgeometry'
arcpy.CheckGeometry_management(shapefile, checkgeom_output)
if int(arcpy.GetCount_management(checkgeom_output).getOutput(0)) > 0:
with arcpy.da.SearchCursor(checkgeom_output, ['PROBLEM']) as scur:
arcpy.AddError('Geometry Error: {}'.format(scur.next()[0]))
raise Exception('Geometry Error: {}'.format(scur.next()[0]))
arcpy.AddMessage('validating geometry type')
#: validate geometry type for category
described = arcpy.Describe(shapefile)
if described.shapeType != 'Polygon':
arcpy.AddError(
'Incorrect shape type of {}. Fire perimeters are polygons.'.format(
described.shapeType))
raise Exception(
'Incorrect shape type of {}. Fire perimeters are polygons.'.format(
described.shapeType))
if described.hasZ or described.hasM:
strip_excess = True
arcpy.AddMessage('reprojecting if necessary')
#: reproject if necessary
reprojected_fc = None
input_sr = described.spatialReference
if input_sr.name != utm.name:
#: Project doesn't support the in_memory workspace
arcpy.AddMessage('Reprojected data from {} to {}'.format(
input_sr.factoryCode, utm.factoryCode))
messages.append('Reprojected data from {} to {}'.format(
input_sr.factoryCode, utm.factoryCode))
reprojected_fc = '{}/project'.format(arcpy.env.scratchGDB)
shapefile = arcpy.Project_management(shapefile, reprojected_fc, utm)
arcpy.AddMessage('Removing m and z if necessary')
if strip_excess and input_sr.name == utm.name:
arcpy.AddMessage('Removing m and z')
stripz_output = 'in_memory/stripz'
shapefile = arcpy.management.CopyFeatures(shapefile, stripz_output)
arcpy.AddMessage('unioning all shapes')
#: union all shapes in shapefile
mergedGeometry = None
features = 0
with arcpy.da.SearchCursor(shapefile, ['SHAPE@']) as scur:
for shape, in scur:
features = features + 1
if mergedGeometry is None:
mergedGeometry = shape
continue
mergedGeometry = mergedGeometry.union(shape)
if features == 0:
arcpy.AddError('Shapefile is empty')
raise Exception('Shapefile is empty')
if features > 1:
arcpy.AddMessage('Unioned {} features into one.'.format(features))
messages.append('Unioned {} features into one.'.format(features))
arcpy.AddMessage('cleaning up temp data')
#: delete temp data
if reprojected_fc is not None and arcpy.Exists(reprojected_fc):
arcpy.Delete_management(reprojected_fc)
if stripz_output is not None and arcpy.Exists(stripz_output):
arcpy.Delete_management(stripz_output)
if arcpy.Exists(zfile_folder):
arcpy.Delete_management(zfile_folder)
arcpy.AddMessage('inserting geometry')
#: insert geometry into database
db_method = store_geometry_for
if test_method is not None:
db_method = test_method
status, message = db_method(pk, mergedGeometry.WKT)
arcpy.AddMessage('db response {}, {}'.format(status, message))
if message is not None:
messages.append(message)
return (status, messages)
def execute_task(self, task):
# Note that this task is not thread safe: at most one can be executed at a time
# Create temp folder
with tempfile.TemporaryDirectory() as tmpdir:
# Unzip
try:
with task.zip_file.open('rb') as fp:
submission = ZipFile(fp)
submission.extractall(tmpdir)
except:
return self.TaskResult.error('Failed to unzip provided file',
traceback.format_exc())
# Rewrite big files with a warning (GitHub accepts up to 100MiB, but we will use a much lower limit of 1MiB
# to make sure the full repo won't too big either)
for root, dirs, files in os.walk(tmpdir):
for name in files:
file_path = os.path.join(root, name)
size_MiB = os.path.getsize(file_path) / 1024 / 1024
if size_MiB > 1:
logger.warning(
f'File {file_path} is too big ({size_MiB:.1f}MiB)')
with open(file_path, 'w') as fp:
fp.write(
f'!!! The original file was too large !!!\nPlease find the complete ZIP on the platform'
)
# Prepare commands
competitor = task.competitor
environment = competitor.environment
submitter = competitor.submitter
destination = f'data/publish_repo/{environment.slug}/{competitor.name}'
commit_message = f'Publish {competitor.name} v{task.version_number} for {environment.name} by {submitter.username}'
cmds = [
['git', '-C', 'data/publish_repo', 'fetch'],
[
'git', '-C', 'data/publish_repo', 'reset', '--hard',
'origin/master'
],
['rm', '-rf', destination],
['mkdir', '-p', destination],
['cp', '-r', tmpdir + '/.', destination],
['git', '-C', 'data/publish_repo', 'add', '-A'],
[
'git', '-C', 'data/publish_repo', 'commit',
'--allow-empty', '-m', commit_message
],
['git', '-C', 'data/publish_repo', 'push'],
]
# Run them sequentially
logs = ''
for cmd in cmds:
status, extra_logs = run_shell(cmd)
logs += f'$ {cmd}\n{extra_logs}'
if status != 0:
return self.TaskResult.error(
f'Operation failed with status {status}', logs)
_, commit = run_shell(
['git', '-C', 'data/publish_repo', 'rev-parse', 'HEAD'])
task.publish_url = f'{PUBLISHER_WEB_URL}/blob/{commit.strip()}/{environment.slug}/{competitor.name}/player.py'
return self.TaskResult.success(logs)
def POST_AUTH(self, courseid, taskid): # pylint: disable=arguments-differ
""" Edit a task """
if not id_checker(taskid) or not id_checker(courseid):
raise Exception("Invalid course/task id")
course, _ = self.get_course_and_check_rights(courseid,
allow_all_staff=False)
data = web.input(task_file={})
# Delete task ?
if "delete" in data:
self.task_factory.delete_task(courseid, taskid)
if data.get("wipe", False):
self.wipe_task(courseid, taskid)
raise web.seeother("/admin/" + courseid + "/tasks")
# Else, parse content
try:
try:
task_zip = data.get("task_file").file
except:
task_zip = None
del data["task_file"]
problems = self.dict_from_prefix("problem", data)
limits = self.dict_from_prefix("limits", data)
data = {
key: val
for key, val in data.items() if not key.startswith("problem")
and not key.startswith("limits")
}
del data["@action"]
if data["@filetype"] not in self.task_factory.get_available_task_file_extensions(
):
return json.dumps({
"status":
"error",
"message":
"Invalid file type: {}".format(str(data["@filetype"]))
})
file_ext = data["@filetype"]
del data["@filetype"]
if problems is None:
return json.dumps({
"status":
"error",
"message":
"You cannot create a task without subproblems"
})
# Order the problems (this line also deletes @order from the result)
data["problems"] = OrderedDict([
(key, self.parse_problem(val))
for key, val in sorted(iter(problems.items()),
key=lambda x: int(x[1]['@order']))
])
data["limits"] = limits
if "hard_time" in data["limits"] and data["limits"][
"hard_time"] == "":
del data["limits"]["hard_time"]
# Weight
try:
data["weight"] = float(data["weight"])
except:
return json.dumps({
"status":
"error",
"message":
"Grade weight must be a floating-point number"
})
# Groups
if "groups" in data:
data["groups"] = True if data["groups"] == "true" else False
# Submision storage
if "store_all" in data:
try:
stored_submissions = data["stored_submissions"]
data["stored_submissions"] = 0 if data[
"store_all"] == "true" else int(stored_submissions)
except:
return json.dumps({
"status":
"error",
"message":
"The number of stored submission must be positive!"
})
if data["store_all"] == "false" and data[
"stored_submissions"] <= 0:
return json.dumps({
"status":
"error",
"message":
"The number of stored submission must be positive!"
})
del data['store_all']
# Submission limits
if "submission_limit" in data:
if data["submission_limit"] == "none":
result = {"amount": -1, "period": -1}
elif data["submission_limit"] == "hard":
try:
result = {
"amount": int(data["submission_limit_hard"]),
"period": -1
}
except:
return json.dumps({
"status":
"error",
"message":
"Invalid submission limit!"
})
else:
try:
result = {
"amount": int(data["submission_limit_soft_0"]),
"period": int(data["submission_limit_soft_1"])
}
except:
return json.dumps({
"status":
"error",
"message":
"Invalid submission limit!"
})
del data["submission_limit_hard"]
del data["submission_limit_soft_0"]
del data["submission_limit_soft_1"]
data["submission_limit"] = result
# Accessible
if data["accessible"] == "custom":
data["accessible"] = "{}/{}".format(data["accessible_start"],
data["accessible_end"])
elif data["accessible"] == "true":
data["accessible"] = True
else:
data["accessible"] = False
del data["accessible_start"]
del data["accessible_end"]
# Checkboxes
if data.get("responseIsHTML"):
data["responseIsHTML"] = True
# Network grading
data["network_grading"] = "network_grading" in data
except Exception as message:
return json.dumps({
"status":
"error",
"message":
"Your browser returned an invalid form ({})".format(
str(message))
})
# Get the course
try:
course = self.course_factory.get_course(courseid)
except:
return json.dumps({
"status":
"error",
"message":
"Error while reading course's informations"
})
# Get original data
try:
orig_data = self.task_factory.get_task_descriptor_content(
courseid, taskid)
data["order"] = orig_data["order"]
except:
pass
directory_path = self.task_factory.get_directory_path(courseid, taskid)
try:
WebAppTask(course, taskid, data, directory_path,
self.plugin_manager)
except Exception as message:
return json.dumps({
"status":
"error",
"message":
"Invalid data: {}".format(str(message))
})
if not os.path.exists(directory_path):
os.mkdir(directory_path)
if task_zip:
try:
zipfile = ZipFile(task_zip)
except Exception as message:
return json.dumps({
"status":
"error",
"message":
"Cannot read zip file. Files were not modified"
})
try:
zipfile.extractall(directory_path)
except Exception as message:
return json.dumps({
"status":
"error",
"message":
"There was a problem while extracting the zip archive. Some files may have been modified"
})
self.task_factory.delete_all_possible_task_files(courseid, taskid)
self.task_factory.update_task_descriptor_content(
courseid, taskid, data, force_extension=file_ext)
return json.dumps({"status": "ok"})
def zdsplode(name, verbose=False):
start_dir = os.path.abspath(os.getcwd())
# Match on the filename
# [base].[ext]
# where ext is one of zip, tar, gz, tgz, tar.gz, or tar.bz2
m = re.match(
r'^(?P<base>.*?)[.](?P<ext>zip|tar|tgz|tar\.gz|tar\.bz2|tar\.bz|gz)$',
name)
if not m:
# Not a compressed file that we're going to try to extract
return
if verbose:
print('Extracting {}'.format(name))
base, ext = m.groups()
try:
if ext == 'zip':
cfile = ZipFile(name)
elif ext == 'gz':
cfile = gzip.open(name, 'r')
else:
cfile = tarfile.open(name, 'r:*')
except (IOError, tarfile.ReadError, BadZipfile):
print('Error reading file for extraction {}'.format(name))
return
try:
# extract to a dir of its own to start with.
extract_dir = datetime.datetime.now().isoformat()
if ext == 'gz':
os.makedirs(extract_dir)
f = open(os.path.join(extract_dir, base), 'wb')
chunk = 1024 * 8
buff = cfile.read(chunk)
while buff:
f.write(buff)
buff = cfile.read(chunk)
f.close()
else:
cfile.extractall(extract_dir)
except OSError:
print('Error extracting {}'.format(name))
return
finally:
cfile.close()
# If there's no directory at all, then it was probably an empty archive
if not os.path.isdir(extract_dir):
return
try:
extract_files = os.listdir(extract_dir)
if len(extract_files) == 1 and extract_files[0] == base:
# If there's only one file/dir in the dir, and that file/dir
# matches the base name of the archive, move the file/dir back one
# into the parent dir and remove the extract directory.
# The classic tar.gz -> dir and txt.gz -> file cases.
shutil.move(os.path.join(extract_dir, extract_files[0]), start_dir)
shutil.rmtree(extract_dir)
# Set the name of the extracted dir for recursive decompression
extract_dir = extract_files[0]
else:
# If there's more than one file in the dir, or if that file/dir
# doesn't match the base name of the archive rename the extract dir
# to the basename of the archive.
# The 'barfing files all over pwd' case, the 'archive contains
# var/log/blah/blah' case, and the 'archive contains a single,
# differently named file' case.
shutil.move(os.path.join(extract_dir), base)
# Set the name of the extracted dir for recursive decompression
extract_dir = base
except shutil.Error as e:
print('Error arranging directories:')
print(e)
return
# See if there's anything left to do
if not os.path.isdir(extract_dir):
return
# Get a list of files for recursive decompression
sub_files = os.listdir(extract_dir)
# Extract anything compressed that this archive had in it.
os.chdir(extract_dir)
for sub_file in sub_files:
zdsplode(sub_file)
os.chdir(start_dir)
def restore(self, name: dict):
if not name:
print(setColorText("Name empty!", Colors.YELLOW))
return
bkName = name.get("bkname", "HyperUBot")
bkSource = name.get("source")
bkName += ".hbotbk"
if not os.path.exists(bkSource):
print(
setColorText(f"{bkSource}: No such file or directory",
Colors.YELLOW))
return
try:
contents = None
userbot = [
"userbot/", "userbot/__init__.py", "userbot/__main__.py"
]
bkZIP = ZipFile(bkSource, "r")
contents = bkZIP.namelist()
lpfile = None
for x in contents:
if x == "list.paths":
lpfile = bkZIP.read(x)
break
list_paths = self.__getListPaths(lpfile)
result = 0
for y in contents:
for uname in userbot:
if uname == y:
result += 1
if not self.__comparePaths(contents, list_paths) or \
not result == len(userbot):
print(
setColorText(f"{bkSource}: Invalid archive format!",
Colors.RED))
return
try:
print("Removing current environment...")
self._remove(self._list_dirs("."),
[os.path.join(".", RECOVERY_NAME)])
except Exception as e:
print(
setColorText(f"Failed to remove current environment: {e}",
Colors.RED))
return
print("Restoring backup archive...")
bkZIP.extractall(
path=".",
members=[z for z in contents if not z == "list.paths"])
bkZIP.close()
print(setColorText("Restore completed.", Colors.GREEN))
except BadZipFile as bze:
print(setColorText(f"Bad zip archive: {bze}", Colors.RED))
except LargeZipFile as lze:
print(
setColorText(f"Zip archive too large (>64): {lze}",
Colors.RED))
except Exception as e:
print(
setColorText(f"Failed to restore backup archive: {e}",
Colors.RED))
return
def processGuseStatus(self, wfidsDict, RemoteAPIPassword, gUSEurl):
""" Process the status of the workflow, if running call processDetailsinfo, if finished download results using the RemoteAPI
Arguments:
wfidsDict -- dict of folderNumber:workflow ID to loop through
RemoteAPIPassword -- the RemoteAPI password as defined on the gUSE server
gUSEurl -- the URL of the gUSE server
Process the status for each workflow. Status is what is returend by RemoteApiDetailsInfo (called returnVal there)
Based on this value, if the workflow is running call processDetailsInfo, if it is finished call the RemoteApi 'download' method.
Once finished remove id from wfidsDict, this method runs until this list is empty
"""
# create group_id so all insersions and analysis can be tracked
group_id = datetime.datetime.now().strftime("%Y%m%d-%H%M%S%Z-") + str(random.random())
# folders to write the result file to (in the beginning the same as number of workflows)
while True:
# loop through all folder numbers of active workflows
for folderNumber in wfidsDict.keys():
# get the workflow ID for that folder number (folder number will be seen in 'instance_X')
wfid = wfidsDict[folderNumber]
# call RemoteAPI m='detailsinfo' for the workflow with wfid
try:
wfstatus = self.RemoteApiDetailsInfo(gUSEurl, RemoteAPIPassword, wfid)
except Exception as e:
print e
wfstatus = -1
# if there is an exception make wfstatus = -1, then wait for 15 seconds and continue
# this was a workaround to prevent clogging the gUSE server
if wfstatus == -1:
sleep(15)
continue;
# get a new timestamp of when the workflow finished according to the Raccoon2 UI (small delay)
self.timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S-%f")
# if there were no errors with RemoteAPI m='detailsinfo', process the result it returned
wfstate = self.processDetailsinfo(wfstatus, str(folderNumber), self.app.engine.guseNumberOfInstances.get())
# value returned: 0 - suspended, or not valid data.
if wfstate == 0:
# remove the key-value (folderNumber-wfid) pair for that folder number from the dict and start over looping through the rest
del wfidsDict[folderNumber]
continue
# value returned: 1 - finished, or error.
elif wfstate == 1:
# download the results into this file
resultFileName = self.resultFolderName + os.sep + "instance_" + str(folderNumber) + os.sep + "gUSE-cloud-vina-results.zip"
# call RemoteAPI m='download'
self.RemoteApiDownload(gUSEurl, RemoteAPIPassword, wfid, resultFileName)
# open the returned zip file and extract only the Vina output files which are stored in "output.zip"
# store the extracted zip in a temporary folder which will be deleted at the end
z = ZipFile(self.resultFolderName + os.sep + "instance_" + str(folderNumber) + os.sep + 'gUSE-cloud-vina-results.zip', 'r')
temporary_folder = "res_" + self.timestamp
os.mkdir(temporary_folder)
z.extractall(temporary_folder, filter(lambda f: f.endswith('output.zip'), z.namelist()))
output_zip_name_path = ""
for name in z.namelist():
if name.endswith("output.zip"):
output_zip_name_path = name
z.close()
# in case nothing is downloaded - *could be done more efficiently
if output_zip_name_path == "":
print "Error in downloading results or in the job"
# if no error
else:
# open the "output.zip" where the results are stored
z = ZipFile(temporary_folder + os.sep + output_zip_name_path)
# this is the new folder where the results will be extracted into, with the new timestamp
results_folder = self.resultFolderName + os.sep + "instance_" + str(folderNumber) + os.sep + "gUSE-cloud-vina-results-" + self.timestamp
os.mkdir(results_folder)
# extract only files ending with .pdbqt_log.txt or .pdbqt
z.extractall(results_folder, filter(lambda f: f.endswith(('.pdbqt_log.txt', '.pdbqt')), z.namelist()))
z.close()
# remove "temporary_folder" and all its empty and non-empty sub-folders
# os.remove(temporary_folder + os.sep + output_zip_name_path)
# last_separator = output_zip_name_path.rfind(os.sep)
# os.removedirs(temporary_folder + os.sep + output_zip_name_path[0:last_separator])
# maybe remove all input files including results.zip too (*leave for now*)
#os.remove("../ligands.zip")
#os.remove("../receptors.zip")
#os.remove("../conf.txt")
#os.remove("../output_names.txt")
#os.remove("../certs.zip")
# once downloaded, remove the element from the dict and continue the loop
del wfidsDict[folderNumber]
continue
# start processing the next workflow in 5 seconds
sleep(5)
if not wfidsDict:
# print when all workflows have been processed
currentTime = datetime.datetime.now()
timeDifference = currentTime - self.startTime
self.queue.put("VS experiment finished: " + str(timeDifference))
print "VS experiment finished: " + str(timeDifference)
break
else:
# restart getting details for all workflows in 15 seconds
sleep(15)
import shutil
import os
try:
from urllib.request import urlretrieve
except ImportError:
from urllib import urlretrieve
class ProgressReporter:
def __init__(self):
self.progress = None
def __call__(self, block_count, block_size, total):
progress = int((block_count * block_size * 100) / total)
progress = int(progress / 20) * 20
if progress != self.progress:
print("{}%".format(progress))
self.progress = progress
url = "http://www.steinberg.net/sdk_downloads/vstsdk366_27_06_2016_build_61.zip" # noqa
urlretrieve(url, "Steinberg.zip", ProgressReporter())
zip = ZipFile('Steinberg.zip')
zip.extractall()
shutil.rmtree("Steinberg", ignore_errors=True)
os.rename("VST3 SDK", "Steinberg")
shutil.rmtree("Steinberg.zip", ignore_errors=True)
def POST_AUTH(self, courseid, taskid): # pylint: disable=arguments-differ
""" Edit a task """
if not id_checker(taskid) or not id_checker(courseid):
raise Exception("Invalid course/problem id")
course, _ = self.get_course_and_check_rights(courseid,
allow_all_staff=False)
data = web.input(problem_file={}, task_file={})
previous_courseid = courseid
# Task exists?
try:
self.task_factory.get_task(course, taskid)
except:
# If doesn't, use bank instead
courseid = self.bank_name
course = self.course_factory.get_course(courseid)
# Delete task ?
if "delete" in data:
try:
self.task_factory.delete_task(courseid, taskid)
except:
raise web.seeother("/admin/" + previous_courseid + "/tasks")
if data.get("wipe", False):
self.wipe_task(courseid, taskid)
raise web.seeother("/admin/" + previous_courseid + "/tasks")
# Else, parse content
try:
task_zip = data.get("task_file").file
except:
task_zip = None
del data["task_file"]
try:
problem_file = data.get('problem_file')
except Exception as e:
problem_file = None
del data["problem_file"]
problems = self.dict_from_prefix("problem", data)
limits = self.dict_from_prefix("limits", data)
data = {
key: val
for key, val in data.items()
if not key.startswith("problem") and not key.startswith("limits")
}
del data["@action"]
if data["@filetype"] not in self.task_factory.get_available_task_file_extensions(
):
return json.dumps({
"status":
"error",
"message":
"Invalid file type: {}".format(str(data["@filetype"]))
})
file_ext = data["@filetype"]
del data["@filetype"]
if problems is None:
problems = {
'1': {
"type": "code-file",
"header": "",
"allowed_exts": ".py"
}
}
#return json.dumps({"status": "error", "message": "You cannot create a task without subproblems"})
# Order the problems (this line also deletes @order from the result)
data["problems"] = OrderedDict([(key, self.parse_problem(val))
for key, val in problems.items()])
data["limits"] = limits
data["limits"]["time"] = 30
if "hard_time" in data["limits"] and data["limits"]["hard_time"] == "":
del data["limits"]["hard_time"]
if problem_file is not None:
self.upload_pfile(
courseid, taskid, "public/" + taskid + ".pdf",
problem_file.file
if not isinstance(problem_file, str) else problem_file)
run = os.path.join(self.task_factory._tasks_directory, "run", "run")
if os.path.exists(run):
with open(run) as f:
self.upload_pfile(courseid, taskid, "run", f)
wanted_path = self.verify_path(courseid, taskid, "run", True)
os.system("sed -i -e 's/REPLACEWITHTIME/" +
str(data["limits"]["real_time"]) + "/g' " + wanted_path)
# Difficulty
try:
data["difficulty"] = int(data["difficulty"])
if not (data["difficulty"] > 0 and data["difficulty"] <= 5):
return json.dumps({
"status":
"error",
"message":
"Difficulty level must be between 1 and 10"
})
except:
return json.dumps({
"status":
"error",
"message":
"Difficulty level must be an integer number"
})
# Name
if len(data["name"]) == 0:
return json.dumps({
"status": "error",
"message": "Field 'name' must have non-empty."
})
# Weight
try:
data["weight"] = 1.0
except:
return json.dumps({
"status":
"error",
"message":
"Grade weight must be a floating-point number"
})
try:
data["authenticity_percentage"] = float(
data["authenticity_percentage"])
except:
return json.dumps({
"status":
"error",
"message":
"Authenticity percentage must be a floating-point number"
})
# Groups
if "groups" in data:
data["groups"] = True if data["groups"] == "true" else False
# Submission storage
if "store_all" in data:
try:
stored_submissions = data["stored_submissions"]
data["stored_submissions"] = 0 if data[
"store_all"] == "true" else int(stored_submissions)
except:
return json.dumps({
"status":
"error",
"message":
"The number of stored submission must be positive!"
})
if data["store_all"] == "false" and data["stored_submissions"] <= 0:
return json.dumps({
"status":
"error",
"message":
"The number of stored submission must be positive!"
})
del data['store_all']
# Submission limits
if "submission_limit" in data:
if data["submission_limit"] == "none":
result = {"amount": -1, "period": -1}
elif data["submission_limit"] == "hard":
try:
result = {
"amount": int(data["submission_limit_hard"]),
"period": -1
}
except:
return json.dumps({
"status": "error",
"message": "Invalid submission limit!"
})
else:
try:
result = {
"amount": int(data["submission_limit_soft_0"]),
"period": int(data["submission_limit_soft_1"])
}
except:
return json.dumps({
"status": "error",
"message": "Invalid submission limit!"
})
del data["submission_limit_hard"]
del data["submission_limit_soft_0"]
del data["submission_limit_soft_1"]
data["submission_limit"] = result
data["accessible"] = True
# Checkboxes
if data.get("responseIsHTML"):
data["responseIsHTML"] = True
# Network grading
data["network_grading"] = "network_grading" in data
data["code_analysis"] = "code_analysis" in data
if data["code_analysis"] and os.path.exists(run):
os.system("sed -i -e 's/#STATIC//g' " + wanted_path)
# Get the course
try:
course = self.course_factory.get_course(courseid)
except:
return json.dumps({
"status":
"error",
"message":
"Error while reading course's informations"
})
# Get original data
try:
orig_data = self.task_factory.get_task_descriptor_content(
courseid, taskid)
data["order"] = orig_data["order"]
except:
pass
directory_path = self.task_factory.get_directory_path(courseid, taskid)
try:
WebAppTask(course, taskid, data, directory_path,
self.plugin_manager)
except Exception as message:
return json.dumps({
"status":
"error",
"message":
"Invalid data: {}".format(str(message))
})
if not os.path.exists(directory_path):
os.mkdir(directory_path)
if task_zip:
try:
zipfile = ZipFile(task_zip)
except Exception as message:
return json.dumps({
"status":
"error",
"message":
"Cannot read zip file. Files were not modified"
})
try:
zipfile.extractall(directory_path)
except Exception as message:
return json.dumps({
"status":
"error",
"message":
"There was a problem while extracting the zip archive. Some files may have been modified"
})
self.task_factory.delete_all_possible_task_files(courseid, taskid)
self.task_factory.update_task_descriptor_content(
courseid, taskid, data, force_extension=file_ext)
return json.dumps({"status": "ok"})
def extract_data():
zf = ZipFile('covid-world-vaccination-progress.zip')
zf.extractall()
zf.close()
def unzip(zip_uri, is_url, clone_to_dir='.', no_input=False, password=None):
"""Download and unpack a zipfile at a given URI.
This will download the zipfile to the cookiecutter repository,
and unpack into a temporary directory.
:param zip_uri: The URI for the zipfile.
:param is_url: Is the zip URI a URL or a file?
:param clone_to_dir: The cookiecutter repository directory
to put the archive into.
:param no_input: Suppress any prompts
:param password: The password to use when unpacking the repository.
"""
# Ensure that clone_to_dir exists
clone_to_dir = os.path.expanduser(clone_to_dir)
make_sure_path_exists(clone_to_dir)
if is_url:
# Build the name of the cached zipfile,
# and prompt to delete if it already exists.
identifier = zip_uri.rsplit('/', 1)[1]
zip_path = os.path.join(clone_to_dir, identifier)
if os.path.exists(zip_path):
download = prompt_and_delete(zip_path, no_input=no_input)
else:
download = True
if download:
# (Re) download the zipfile
r = requests.get(zip_uri, stream=True)
with open(zip_path, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
else:
# Just use the local zipfile as-is.
zip_path = os.path.abspath(zip_uri)
# Now unpack the repository. The zipfile will be unpacked
# into a temporary directory
try:
zip_file = ZipFile(zip_path)
if len(zip_file.namelist()) == 0:
raise InvalidZipRepository(
'Zip repository {} is empty'.format(zip_uri))
# The first record in the zipfile should be the directory entry for
# the archive. If it isn't a directory, there's a problem.
first_filename = zip_file.namelist()[0]
if not first_filename.endswith('/'):
raise InvalidZipRepository('Zip repository {} does not include '
'a top-level directory'.format(zip_uri))
# Construct the final target directory
project_name = first_filename[:-1]
unzip_base = tempfile.mkdtemp()
unzip_path = os.path.join(unzip_base, project_name)
# Extract the zip file into the temporary directory
try:
zip_file.extractall(path=unzip_base)
except RuntimeError:
# File is password protected; try to get a password from the
# environment; if that doesn't work, ask the user.
if password is not None:
try:
zip_file.extractall(path=unzip_base,
pwd=password.encode('utf-8'))
except RuntimeError:
raise InvalidZipRepository(
'Invalid password provided for protected repository')
elif no_input:
raise InvalidZipRepository(
'Unable to unlock password protected repository')
else:
retry = 0
while retry is not None:
try:
password = read_repo_password('Repo password')
zip_file.extractall(path=unzip_base,
pwd=password.encode('utf-8'))
retry = None
except RuntimeError:
retry += 1
if retry == 3:
raise InvalidZipRepository(
'Invalid password provided '
'for protected repository')
except BadZipFile:
raise InvalidZipRepository(
'Zip repository {} is not a valid zip archive:'.format(zip_uri))
return unzip_path
def lambda_handler(event, context):
logger.info('Event %s', event)
OAUTH_token = event['context']['git-token']
OutputBucket = event['context']['output-bucket']
# temp_archive = '/tmp/archive.zip'
# Identify git host flavour
hostflavour = 'generic'
if 'X-Hub-Signature' in event['params']['header'].keys():
hostflavour = 'githubent'
elif 'X-Gitlab-Event' in event['params']['header'].keys():
hostflavour = 'gitlab'
elif 'User-Agent' in event['params']['header'].keys():
if event['params']['header']['User-Agent'].startswith(
'Bitbucket-Webhooks'):
hostflavour = 'bitbucket'
elif event['params']['header']['User-Agent'].startswith(
'GitHub-Hookshot'):
hostflavour = 'github'
headers = {}
branch = 'master'
if hostflavour == 'githubent':
archive_url = event['body-json']['repository']['archive_url']
owner = event['body-json']['repository']['owner']['name']
name = event['body-json']['repository']['name']
# replace the code archive download and branch reference placeholders
archive_url = archive_url.replace('{archive_format}',
'zipball').replace(
'{/ref}', '/master')
# add access token information to archive url
archive_url = archive_url + '?access_token=' + OAUTH_token
elif hostflavour == 'github':
archive_url = event['body-json']['repository']['archive_url']
owner = event['body-json']['repository']['owner']['login']
name = event['body-json']['repository']['name']
# replace the code archive download and branch reference placeholders
branch_name = event['body-json']['ref'].replace('refs/heads/', '')
archive_url = archive_url.replace('{archive_format}',
'zipball').replace(
'{/ref}', '/' + branch_name)
# add access token information to archive url
archive_url = archive_url + '?access_token=' + OAUTH_token
elif hostflavour == 'gitlab':
# https://gitlab.com/jaymcconnell/gitlab-test-30/repository/archive.zip?ref=master
archive_url = event['body-json']['project']['http_url'].replace(
'.git', '/repository/archive.zip?ref=master'
) + '&private_token=' + OAUTH_token
owner = event['body-json']['project']['namespace']
name = event['body-json']['project']['name']
elif hostflavour == 'bitbucket':
branch = event['body-json']['push']['changes'][0]['new']['name']
archive_url = event['body-json']['repository']['links']['html'][
'href'] + '/get/' + branch + '.zip'
owner = event['body-json']['repository']['owner']['username']
name = event['body-json']['repository']['name']
r = requests.post('https://bitbucket.org/site/oauth2/access_token',
data={'grant_type': 'client_credentials'},
auth=(event['context']['oauth-key'],
event['context']['oauth-secret']))
if 'error' in r.json().keys():
logger.error('Could not get OAuth token. %s: %s' %
(r.json()['error'], r.json()['error_description']))
raise Exception('Failed to get OAuth token')
headers['Authorization'] = 'Bearer ' + r.json()['access_token']
s3_archive_file = "%s/%s/%s/%s.zip" % (owner, name, branch, name)
# download the code archive via archive url
logger.info('Downloading archive from %s' % archive_url)
r = requests.get(archive_url, verify=verify, headers=headers)
f = StringIO(r.content)
zip = ZipFile(f)
path = '/tmp/code'
zipped_code = '/tmp/zipped_code'
try:
shutil.rmtree(path)
os.remove(zipped_code + '.zip')
except:
pass
finally:
os.makedirs(path)
# Write to /tmp dir without any common preffixes
zip.extractall(path, get_members(zip))
# Create zip from /tmp dir without any common preffixes
shutil.make_archive(zipped_code, 'zip', path)
logger.info("Uploading zip to S3://%s/%s" %
(OutputBucket, s3_archive_file))
s3_client.upload_file(zipped_code + '.zip', OutputBucket, s3_archive_file)
logger.info('Upload Complete')
def configureDB_faculty(requests):
response = {"configureDB_faculty": "active"}
if requests.method == "GET":
return render(requests, "Administrator/uploadcsv.html", response)
bootstrapFile = {}
try:
file = requests.FILES.get("file", False)
action = requests.POST.get("action")
start_date = requests.POST.get("start_date")
end_date = requests.POST.get("end_date")
if not utilities.validateDate(
start_date) or not utilities.validateDate(end_date):
raise Exception("Incorrect date format, should be YYYY-MM-DD")
bootstrapFile['start_date'] = start_date
bootstrapFile['end_date'] = end_date
if action != None:
bootstrap.clear_Database()
if file.name.endswith('.zip'):
unzipped = ZipFile(file)
unzipped.extractall(os.path.abspath('bootstrap_files'))
bootstrapFile['file_type'] = 'zip'
for fileName in unzipped.namelist():
if fileName.lower() == 'faculty_information.xlsx':
bootstrapFile['faculty'] = os.path.abspath(
'bootstrap_files/' + fileName)
elif fileName.lower() == 'course_information.xlsx':
bootstrapFile['course'] = os.path.abspath(
'bootstrap_files/' + fileName)
if 'faculty' not in bootstrapFile.keys(
) or 'course' not in bootstrapFile.keys():
raise Exception(
"Invalid file information within .zip file. Please upload faculty or course information only."
)
elif file.name.endswith('.xlsx'):
if file.name.lower() == 'faculty_information.xlsx':
bootstrapFile['file_path'] = file.temporary_file_path()
bootstrapFile['file_type'] = 'excel'
bootstrapFile['file_information'] = 'faculty'
elif file.name.lower() == 'course_information.xlsx':
bootstrapFile['file_path'] = file.temporary_file_path()
bootstrapFile['file_type'] = 'excel'
bootstrapFile['file_information'] = 'course'
else:
raise Exception(
"Invalid file information. Please upload faculty or course information only."
)
else:
raise Exception(
"Invalid file type. Please upload .xlsx or .zip only")
# If file is .xlsx or .zip then proceed with processing
response['results'] = bootstrap.bootstrap_Faculty(bootstrapFile)
except Exception as e:
# Uncomment for debugging - to print stack trace wihtout halting the process
traceback.print_exc()
response['error_message'] = e.args[0]
return render(requests, "Administrator/uploadcsv.html", response)
response['message'] = 'Successful Upload'
return render(requests, "Administrator/uploadcsv.html", response)
def on_message(self, message):
ms = json.loads(message)
if list(ms.keys())[0] == "Start":
"""Начало загрузки"""
self.name = ms["Start"]["Name"]
try:
self.myfile = open("archiv/"+ms["Start"]["Name"], "wb")
self.write_message(json.dumps({"Process": "MoreData"}))
except IOError:
os.mkdir("archiv")
self.myfile = open("archiv/"+ms["Start"]["Name"], "wb")
self.write_message(json.dumps({"Process": "MoreData"}))
if list(ms.keys())[0] == "Upload":
"""Процесс загрузки"""
da = ms["Upload"]["Data"]
da = da.split(",")[1]
file_bytes = io.BytesIO(base64.b64decode(da)).read()
self.myfile.write(file_bytes)
self.write_message(json.dumps({"Process": "MoreData"}))
if list(ms.keys())[0] == "Done":
"""Конец загрузки"""
self.myfile.close()
self.write_message(json.dumps({"Process": "Process"}))
if list(ms.keys())[0] == "Process":
HD = DATA()
lss = ["gz", "tar", "xz"]
if self.name.split(".")[-1] in lss:
tar = tarfile.open("archiv/"+self.name, "r")
tar.extractall("data/"+self.name, members=self.track_progress(tar))
tar.close()
HD.parseIMG(self.name)
if self.name.split(".")[-1] == "zip":
zip = ZipFile("archiv/"+self.name)
zip.extractall("data/"+self.name)
HD.parseIMG(self.name)
#print (HD.file, ms["type"] == 1)
if int(ms["type"]) == 4:
for ixx, fx in enumerate(list(HD.file.keys())[:]):
file_n = HD.file[fx][0]
try:
answ_H = HD.file[fx][1]
except IndexError:
answ_H = ""
iop = cv2.imread(file_n)
#print ("POST V")
answ_V = ViPost(file_n, str(ixx), ms["task"], int(ms["type"]))#""#gg(iop, ms["task"], int(ms["type"]))
#print (answ_V["text"],"POST L")
#answ_L = cutimg(iop, str(ixx), ms["task"], int(ms["type"]))
#print (answ_L["text"]) #answ_V[0]
#print (file_n, ms["task"], ms["type"]) #int(ms["type"]))
post = {"file": file_n,
"answ": "", "answ_H": answ_H, "_type":ms["type"],
"answ_L" : "", "answ_V": answ_V["text"], "_task":ms["task"]}
#"answ": "", "answ_V": "", "task":ms["task"], "type":ms["type"]}
#"answ_L" : answ_L["text"], "answ_V": "1", "task":ms["task"]}
I = new.posts.insert_one(post).inserted_id
self.ss = new.posts.find()
self.write_message(json.dumps({"Process": "loaddone"}))
if int(ms["type"]) == 3:
for ixx, fx in enumerate(list(HD.file.keys())[:]):
file_n = HD.file[fx][0]
try:
answ_H = HD.file[fx][1]
except IndexError:
answ_H = ""
iop = cv2.imread(file_n)
#print ("POST V")
answ_V = ViPost(file_n, str(ixx), ms["task"], int(ms["type"]))
PP = cutimg(file_n, iop, ixx, [int(x) for x in answ_V["text"]], ms["task"], 3)
I = new.posts.insert_one(PP).inserted_id #print (PP)
self.ss = new.posts.find()
self.write_message(json.dumps({"Process": "loaddone"}))
else:
im_w = len(list(HD.file.keys()))
w = 9
w_num = int(im_w/w)
num = 0
#print ("ErrorS", im_w, w_num, w)
for ix in range(0, w_num):
#print (len(list(HD.file.keys())[ix*w:(ix+1)*w]))
t_dict = {}
t_dict["_type"] = ms["type"]
t_dict["_task"] = ms["task"]
for kix in list(HD.file.keys())[ix*w:(ix+1)*w]:
#print (HD.file[kix],ix)
file_n = HD.file[kix][0]
post = {"file": file_n,
"answ": "", "answ_H": "",
"answ_L" : "", "answ_V": "1"}
t_dict[str(ix)+"_"+str(kix)] = post
#print (len(list(t_dict.keys())))
I = new.posts.insert_one(t_dict).inserted_id
self.ss = new.posts.find()
self.write_message(json.dumps({"Process": "loaddone"}))
if list(ms.keys())[0] == "SeeAllData":
if ms["SeeAllData"] == "ProcessNext":
try:
posttoCH = new.posts.find_one(ObjectId(ms["idxx"]))
if posttoCH["_type"] == 1:
for ik, k in enumerate(list(posttoCH.keys())[3:]):
posttoCH[k]["answ_V"] = ms["answ_v"][ik]
new.posts.update_one({"_id" : ObjectId(ms["idxx"])},
{"$set": posttoCH}, upsert=True)
else:
self.tempID = ObjectId(ms["idxx"])
posttoCH = new.posts.find_one(ObjectId(ms["idxx"]))
posttoCH["answ_V"] = deansw(ms["answ_v"])
new.posts.update_one({"_id" : ObjectId(ms["idxx"])},
{"$set": posttoCH}, upsert=True)
#posttoCH = new.posts.find_one(ObjectId(ms["idxx"]))
except KeyError:
pass
if new.posts.count() != 0:
try:
obj = self.ss.next()
except StopIteration:
self.ss.rewind()
obj = self.ss.next()
#print (obj["task"])
if obj["_type"] == 1:
k_answ = []
l_data = []
ixxx = 0
#print (obj["_type"])
for k in list(obj.keys())[3:]:
n_file = obj[k]['file']
ixxx += 1
if int(obj[k]['answ_V']) == 1:
k_answ.append(ixxx)
iop = cv2.imread(n_file)
print (n_file)#, iop.shape)
l_data.append(iop)
iop = newimg(np.array(l_data),3)
idxx = obj["_id"]
answ_v = k_answ
answ = k_answ
task = obj["_task"]
#LeoPost(fname, str(idxx))
s = base64.b64encode(imgbite(iop))
obj = {"image":s.decode('ascii'),
"answ_V": answ_v,
"answ": answ,
"task": task,
"type":3,
"_id": str(idxx),
"name": ""}
#print (obj["type"])
self.write_message(json.dumps(obj))
else:
try:
obj = self.ss.next()
except StopIteration:
self.ss.rewind()
obj = self.ss.next()
idxx = obj["_id"]
fname = obj["file"]
answ_v = obj["answ_V"]
answ = obj["answ"]
task = obj["_task"]
tp = obj["_type"]
#print (fname, answ_v, answ, idxx, task, tp)
iop = cv2.imread(fname)
#LeoPost(fname, str(idxx))
s = base64.b64encode(imgbite(iop))
#print (s)
obj = {"image":s.decode('ascii'),
"answ_V": answ_v,
"answ": answ,
"task": task,
"type":tp,
"_id": str(idxx),
"name": fname}
#print (obj["type"])
self.write_message(json.dumps(obj))
import urllib.request as urllib
import time
from zipfile import ZipFile
import os
tagname = "1.3.1"
zipname = "markdown-presenter-" + tagname + ".zip"
zippath = "./" + zipname
url = "https://github.com/jsakamoto/MarkdownPresenter/releases/download/v." + tagname + "/" + zipname
# download zip archive of Markdown Presenter.
urllib.urlretrieve(url, zippath)
# ectract the zip.
zfile = ZipFile(zippath)
zfile.extractall(".")
zfile.close()
# clean up zip.
os.remove(zippath)
# launch default web browser to open Markdown Presenter
# after one shot timer to wait for warming up HTTP daemon.
def launch():
time.sleep(1)
webbrowser.open("http://localhost:8000/Presenter.html")
thread.start_new_thread(launch, ())
def unzip_files():
for zipf in zipfiles:
zip_ref = ZipFile(zipf, 'r')
zip_ref.extractall(zipf.rstrip('.zip.ZIP'))
zip_ref.close()
def unzip(fileName):
print "Extracting: " + fileName,
zip = ZipFile(fileName)
zip.extractall()
zip.close()
print " [DONE]"
def POST_AUTH(self, courseid, taskid): # pylint: disable=arguments-differ
""" Edit a task """
if not id_checker(taskid) or not id_checker(courseid):
raise Exception("Invalid course/task id")
course, __ = self.get_course_and_check_rights(courseid,
allow_all_staff=False)
data = web.input(task_file={})
# Delete task ?
if "delete" in data:
self.task_factory.delete_task(courseid, taskid)
if data.get("wipe", False):
self.wipe_task(courseid, taskid)
raise web.seeother(self.app.get_homepath() + "/admin/" + courseid +
"/tasks")
# Else, parse content
try:
try:
task_zip = data.get("task_file").file
except:
task_zip = None
del data["task_file"]
problems = self.dict_from_prefix("problem", data)
limits = self.dict_from_prefix("limits", data)
#Tags
tags = self.dict_from_prefix("tags", data)
if tags is None:
tags = {}
tags = OrderedDict(sorted(tags.items(),
key=lambda item: item[0])) # Sort by key
# Repair tags
for k in tags:
tags[k]["visible"] = (
"visible" in tags[k]
) # Since unckecked checkboxes are not present here, we manually add them to avoid later errors
tags[k]["type"] = int(tags[k]["type"])
if not "id" in tags[k]:
tags[k][
"id"] = "" # Since textinput is disabled when the tag is organisational, the id field is missing. We add it to avoid Keys Errors
if tags[k]["type"] == 2:
tags[k]["id"] = "" # Force no id if organisational tag
# Remove uncompleted tags (tags with no name or no id)
for k in list(tags.keys()):
if (tags[k]["id"] == ""
and tags[k]["type"] != 2) or tags[k]["name"] == "":
del tags[k]
# Find duplicate ids. Return an error if some tags use the same id.
for k in tags:
if tags[k][
"type"] != 2: # Ignore organisational tags since they have no id.
count = 0
id = str(tags[k]["id"])
if (" " in id):
return json.dumps({
"status":
"error",
"message":
_("You can not use spaces in the tag id field.")
})
if not id_checker(id):
return json.dumps({
"status":
"error",
"message":
_("Invalid tag id: {}").format(id)
})
for k2 in tags:
if tags[k2]["type"] != 2 and tags[k2]["id"] == id:
count = count + 1
if count > 1:
return json.dumps({
"status":
"error",
"message":
_("Some tags have the same id! The id of a tag must be unique."
)
})
data = {
key: val
for key, val in data.items() if not key.startswith("problem")
and not key.startswith("limits") and not key.startswith("tags")
and not key.startswith("/")
}
del data["@action"]
# Determines the task filetype
if data["@filetype"] not in self.task_factory.get_available_task_file_extensions(
):
return json.dumps({
"status":
"error",
"message":
_("Invalid file type: {}").format(str(data["@filetype"]))
})
file_ext = data["@filetype"]
del data["@filetype"]
# Parse and order the problems (also deletes @order from the result)
if problems is None:
data["problems"] = OrderedDict([])
else:
data["problems"] = OrderedDict([
(key, self.parse_problem(val))
for key, val in sorted(iter(problems.items()),
key=lambda x: int(x[1]['@order']))
])
# Task limits
data["limits"] = limits
data["tags"] = OrderedDict(
sorted(tags.items(), key=lambda x: x[1]['type']))
if "hard_time" in data["limits"] and data["limits"][
"hard_time"] == "":
del data["limits"]["hard_time"]
# Weight
try:
data["weight"] = float(data["weight"])
except:
return json.dumps({
"status":
"error",
"message":
_("Grade weight must be a floating-point number")
})
# Groups
if "groups" in data:
data["groups"] = True if data["groups"] == "true" else False
# Submision storage
if "store_all" in data:
try:
stored_submissions = data["stored_submissions"]
data["stored_submissions"] = 0 if data[
"store_all"] == "true" else int(stored_submissions)
except:
return json.dumps({
"status":
"error",
"message":
_("The number of stored submission must be positive!")
})
if data["store_all"] == "false" and data[
"stored_submissions"] <= 0:
return json.dumps({
"status":
"error",
"message":
_("The number of stored submission must be positive!")
})
del data['store_all']
# Submission limits
if "submission_limit" in data:
if data["submission_limit"] == "none":
result = {"amount": -1, "period": -1}
elif data["submission_limit"] == "hard":
try:
result = {
"amount": int(data["submission_limit_hard"]),
"period": -1
}
except:
return json.dumps({
"status":
"error",
"message":
_("Invalid submission limit!")
})
else:
try:
result = {
"amount": int(data["submission_limit_soft_0"]),
"period": int(data["submission_limit_soft_1"])
}
except:
return json.dumps({
"status":
"error",
"message":
_("Invalid submission limit!")
})
del data["submission_limit_hard"]
del data["submission_limit_soft_0"]
del data["submission_limit_soft_1"]
data["submission_limit"] = result
# Accessible
if data["accessible"] == "custom":
data["accessible"] = "{}/{}/{}".format(
data["accessible_start"], data["accessible_soft_end"],
data["accessible_end"])
elif data["accessible"] == "true":
data["accessible"] = True
else:
data["accessible"] = False
del data["accessible_start"]
del data["accessible_end"]
del data["accessible_soft_end"]
# Checkboxes
if data.get("responseIsHTML"):
data["responseIsHTML"] = True
# Network grading
data["network_grading"] = "network_grading" in data
except Exception as message:
return json.dumps({
"status":
"error",
"message":
_("Your browser returned an invalid form ({})").format(message)
})
# Get the course
try:
course = self.course_factory.get_course(courseid)
except:
return json.dumps({
"status":
"error",
"message":
_("Error while reading course's informations")
})
# Get original data
try:
orig_data = self.task_factory.get_task_descriptor_content(
courseid, taskid)
data["order"] = orig_data["order"]
except:
pass
task_fs = self.task_factory.get_task_fs(courseid, taskid)
task_fs.ensure_exists()
# Call plugins and return the first error
plugin_results = self.plugin_manager.call_hook('task_editor_submit',
course=course,
taskid=taskid,
task_data=data,
task_fs=task_fs)
# Retrieve the first non-null element
error = next(filter(None, plugin_results), None)
if error is not None:
return error
try:
WebAppTask(course, taskid, data, task_fs, None,
self.plugin_manager,
self.task_factory.get_problem_types())
except Exception as message:
return json.dumps({
"status":
"error",
"message":
_("Invalid data: {}").format(str(message))
})
if task_zip:
try:
zipfile = ZipFile(task_zip)
except Exception:
return json.dumps({
"status":
"error",
"message":
_("Cannot read zip file. Files were not modified")
})
with tempfile.TemporaryDirectory() as tmpdirname:
try:
zipfile.extractall(tmpdirname)
except Exception:
return json.dumps({
"status":
"error",
"message":
_("There was a problem while extracting the zip archive. Some files may have been modified"
)
})
task_fs.copy_to(tmpdirname)
self.task_factory.delete_all_possible_task_files(courseid, taskid)
self.task_factory.update_task_descriptor_content(
courseid, taskid, data, force_extension=file_ext)
course.update_all_tags_cache()
return json.dumps({"status": "ok"})
def test_001_build_android(self):
Tns.build_android(attributes={"--path": self.app_name})
assert File.pattern_exists(self.platforms_android, "*.aar")
assert not File.pattern_exists(self.platforms_android, "*.plist")
assert not File.pattern_exists(self.platforms_android, "*.android.js")
assert not File.pattern_exists(self.platforms_android, "*.ios.js")
# Configs are respected
assert 'debug' in File.read(
os.path.join(self.app_name, TnsAsserts.PLATFORM_ANDROID_APP_PATH,
'config.json'))
# And new platform specific file and verify next build is ok (test for issue #2697)
src = os.path.join(self.app_name, 'app', 'app.js')
dest_1 = os.path.join(self.app_name, 'app', 'new.android.js')
dest_2 = os.path.join(self.app_name, 'app', 'new.ios.js')
File.copy(src=src, dest=dest_1)
File.copy(src=src, dest=dest_2)
# Verify incremental native build
before_build = datetime.datetime.now()
output = Tns.build_android(attributes={"--path": self.app_name})
after_build = datetime.datetime.now()
assert "Gradle build..." in output, "Gradle build not called."
assert output.count(
"Gradle build...") is 1, "Only one gradle build is triggered."
assert (after_build - before_build).total_seconds(
) < 20, "Incremental build takes more then 20 sec."
# Verify platform specific files
assert File.pattern_exists(self.platforms_android, "*.aar")
assert not File.pattern_exists(self.platforms_android, "*.plist")
assert not File.pattern_exists(self.platforms_android, "*.android.js")
assert not File.pattern_exists(self.platforms_android, "*.ios.js")
# Verify apk does not contain aar files
archive = ZipFile(
os.path.join(self.app_name,
TnsAsserts.PLATFORM_ANDROID_APK_DEBUG_PATH,
self.debug_apk))
archive.extractall(self.app_name + "/temp")
archive.close()
# Ceanup META-INF folder. It contains com.android.support.... files which are expected to be there due to
# https://github.com/NativeScript/nativescript-cli/pull/3923
Folder.cleanup(os.path.join(self.app_name, "temp", "META-INF"))
assert not File.pattern_exists(self.app_name + "/temp", "*.aar")
assert not File.pattern_exists(self.app_name + "/temp", "*.plist")
assert not File.pattern_exists(self.app_name + "/temp", "*.android.*")
assert not File.pattern_exists(self.app_name + "/temp", "*.ios.*")
Folder.cleanup(self.app_name + "/temp")
# Verify incremental native build
before_build = datetime.datetime.now()
output = Tns.build_android(attributes={"--path": self.app_name})
after_build = datetime.datetime.now()
assert "Gradle build..." in output, "Gradle build not called."
assert output.count(
"Gradle build...") is 1, "Only one gradle build is triggered."
assert (after_build - before_build).total_seconds(
) < 20, "Incremental build takes more then 20 sec."
# Verify clean build force native project rebuild
before_build = datetime.datetime.now()
output = Tns.build_android(attributes={
"--path": self.app_name,
"--clean": ""
})
after_build = datetime.datetime.now()
build_time = (after_build - before_build).total_seconds()
assert "Gradle clean..." in output, "Gradle clean is not called."
assert "Gradle build..." in output, "Gradle build is not called."
assert output.count(
"Gradle build...") is 1, "More than 1 gradle build is triggered."
assert build_time > 10, "Clean build takes less then 15 sec."
assert build_time < 90, "Clean build takes more than 90 sec."
# within this directory, create files to store raw and processed data
data_dir = util_files.prep_dirs(dataset_name)
'''
Download data and save to your data directory
'''
# insert the url used to download the data from the source website
url = 'https://gain.nd.edu/assets/323406/resources_2019_19_01_21h59_1_1_.zip' #check
# download the data from the source
raw_data_file = os.path.join(data_dir, os.path.basename(url))
urllib.request.urlretrieve(url, raw_data_file)
#unzip source data
raw_data_file_unzipped = raw_data_file.split('.')[0]
zip_ref = ZipFile(raw_data_file, 'r')
zip_ref.extractall(raw_data_file_unzipped)
zip_ref.close()
'''
Process data
'''
#read in climate change vulnerability data to pandas dataframe
filename = os.path.join(raw_data_file_unzipped, 'resources', 'vulnerability',
'vulnerability.csv')
vulnerability_df = pd.read_csv(filename)
#read in climate change readiness data to pandas dataframe
filename = os.path.join(raw_data_file_unzipped, 'resources', 'readiness',
'readiness.csv')
readiness_df = pd.read_csv(filename)
#read in nd-gain score data to pandas dataframe
class Installer:
def __init__(self):
self.window = Tk()
self.window.iconbitmap('assets/install.ico')
self.window.title('Installer')
self.window.resizable(False, False)
self.window.geometry('550x300')
self.window.configure(background='white')
load = Image.open("assets/ins.png")
space = Label(self.window, bg='white')
space.pack(side=BOTTOM, pady=2)
load = load.resize((180, 180), Image.ANTIALIAS)
img = ImageTk.PhotoImage(load)
image = Label(self.window, text='kalfoi', image=img, bg='white')
image.image = img
image.pack(side=LEFT, padx=20)
space = Label(self.window, bg='white')
space.pack(pady=20)
tit = Label(self.window,
text='Instalador ',
bg='white',
font="sans 25 ",
fg='grey')
tit.pack()
self.op = Label(self.window,
text='Selecione as opções desejadas\t \n',
bg='white',
font="sans 13 ",
fg='grey')
self.op.pack()
self.v = IntVar()
c = Checkbutton(self.window,
text="Criar ícone na área de trabalho\t\t\t\t",
bg='white',
onvalue=1,
offvalue=0,
variable=self.v)
c.var = self.v
c.pack()
self.w = IntVar()
d = Checkbutton(self.window,
text="Adicionar ao menu Iniciar \t\t\t\t\t",
bg='white',
onvalue=1,
offvalue=0,
variable=self.w)
d.var = self.w
d.pack()
self.bt = Button(self.window,
text='Instalar',
height=2,
width=10,
cursor='hand2',
command=self.load)
self.bt.pack(side=RIGHT, padx=30)
self.widgets = [c, d]
def load(self):
self.bt['cursor'] = 'watch'
self.window['cursor'] = 'watch'
self.window.after(10, self.start)
def start(self):
user_path = '\\'.join([i for i in os.getcwd().split('\\')[:3]]) + '\\'
self.zip = ZipFile('Tubs.zip')
self.zip.extractall(user_path)
self.path = user_path + 'Tubs\\'
lista = [
(self.v.get(), winshell.desktop()),
(self.w.get(),
os.path.abspath(
user_path +
"\\AppData\\Roaming\\Microsoft\\Windows\\Start Menu\\Programs"
))
]
for i in lista:
if i[0]:
link_filepath = os.path.join(i[1], "Gerador.lnk")
with winshell.shortcut(link_filepath) as link:
link.path = self.path + 'interface.exe'
link.description = "Gerador de Certificados"
link.working_directory = self.path
self.window['cursor'] = 'arrow'
self.bt['cursor'] = 'hand2'
for i in self.widgets:
i.destroy()
self.op['text'] = 'Instalação Finalizada\t\t '
self.bt['text'] = 'Encerrar'
self.bt['command'] = self.window.destroy
logging.info("Downloading data...")
response = requests.get(
'http://fme.discomap.eea.europa.eu/fmedatadownload/MarineLitter/MLWPivotExport.fmw'
'?CommunityCode=&FromDate=2010-01-01&ToDate=2022-12-31'
'&opt_showresult=false&opt_servicemode=sync')
downloadlink = re.search(r"<a\s+(?:[^>]*?\s+)?href=([\"'])(.*?)\1>",
response.content.decode()).group(2)
logging.info("Saving data...")
zipfile = requests.get(downloadlink)
open(f'{here}/{ZIP_FILE}', 'wb').write(zipfile.content)
logging.info("Uzipping data...")
zipObject = ZipFile(f'{here}/{ZIP_FILE}', 'r')
zipObject.extractall(path=here)
logging.info("Loading data...")
# Data to initialize database with
data = pd.read_csv(f'{here}/CSV_1/MLW_PivotExport/MLW_Data.csv',
encoding="ISO-8859-1")
# Delete database file if it exists currently
if os.path.exists(f'{here}/{DB_FILE}'):
os.remove(f'{here}/{DB_FILE}')
# Create the database
db.create_all()
# populate the database
conn = sqlite3.connect(f'{here}/{DB_FILE}')
from keras.preprocessing.image import ImageDataGenerator
from zipfile import ZipFile
import os
!wget --no-check-certificate \
https://storage.googleapis.com/laurencemoroney-blog.appspot.com/horse-or-human.zip \
-O /tmp/horse-or-human.zip
!wget --no-check-certificate \
https://storage.googleapis.com/laurencemoroney-blog.appspot.com/validation-horse-or-human.zip \
-O /tmp/validation-horse-or-human.zip
# Extracting the files (This was written in Google Colab)
file_name = '/tmp/horse-or-human.zip'
zip_ref = ZipFile(file_name, 'r')
zip_ref.extractall('/tmp/horse-or-human')
file_name = '/tmp/validation-horse-or-human.zip'
zip_ref = ZipFile(file_name, 'r')
zip_ref.extractall('/tmp/validation-horse-or-human')
train_horse_dir = os.path.join('/tmp/horse-or-human/horses')
train_human_dir = os.path.join('/tmp/horse-or-human/humans')
validation_horse_dir = os.path.join('/tmp/validation-horse-or-human/horses')
validation_human_dir = os.path.join('/tmp/validation-horse-or-human/humans')
# Setting up ImageDataGenerators. You can implement Augmentation here, to reduce overfitting.
train_datagen = ImageDataGenerator(rescale=1/255)
train_generator = train_datagen.flow_from_directory(
directory='/tmp/horse-or-human',
target_size=(300,300),
def extract_zip(what, where):
print("extracting {}...".format(what))
zip_file = ZipFile(what)
zip_file.extractall(path=where)
zip_file.close()
import tarfile
import os
import platform
target = platform.system() + "-" + platform.architecture()[0]
arduino_url = "https://downloads.arduino.cc/arduino-cli"
images = {
"Windows-64bit": "arduino-cli_latest_Windows_64bit.zip",
"Windows-32bit": "arduino-cli_latest_Windows_32bit.zip",
"Linux-64bit": "arduino-cli_latest_Linux_64bit.tar.gz"
}
destdir = "tools/bin"
try:
os.mkdir("tools")
except:
pass
filename = images[target]
r = urlopen(arduino_url + "/" + images[target])
data = BytesIO(r.read())
if (filename.endswith(".tar.gz") or filename.endswith(".tgz")):
tar = tarfile.open(fileobj=data, mode="r")
content = tar.extractall(path=destdir)
elif (filename.endswith(".zip")):
zipfile = ZipFile(data)
zipfile.extractall(path=destdir)
def extract(zipFilename, dm_extraction_dir):
zipTest = ZipFile(zipFilename)
zipTest.extractall(dm_extraction_dir)
def scan(self, codeURL, runtime):
if not self.config:
return # invalid config
zippath = self.downloads.joinpath('lambda.zip')
zippath.write_bytes(requests.get(codeURL).content)
if not is_zipfile(zippath):
return # invalid zip
zf = ZipFile(zippath)
# Unzip Lambda source code
for _ in zf.namelist():
zf.extractall(self.downloads, members=[_])
# Configure sonar-project.properties
if runtime.startswith('python'):
language = 'py'
elif runtime.startswith('node'):
language = 'js'
elif runtime.startswith('java'):
language = 'java'
else:
return # unsupported language
Path(self.downloads, 'sonar-project.properties').write_text(
SONAR_PROJECT_PROPERTIES.format(self.config['url'],
self.config['login'],
self.config['password'], language))
# Run sonar-scanner
cwd = Path('.').resolve()
cd(self.downloads)
sh(shsplit(self.config['command']), stdout=DEVNULL, stderr=DEVNULL)
cd(cwd)
rmtree(self.downloads, ignore_errors=True)
self.downloads.mkdir(parents=True, exist_ok=True)
# Get results
curl = requests.Session()
curl.auth = (self.config['login'], self.config['password'])
while True:
sleep(3)
task = json.loads(
curl.get(
f'{self.config["url"]}/api/ce/activity').text)['tasks'][0]
if task['status'] in ['SUCCESS', 'FAIL']:
break
issues = json.loads(
curl.get(
f'{self.config["url"]}/api/issues/search?project=lambdaguard').
text)['issues']
curl.post(f'{self.config["url"]}/api/projects/delete',
data={'project': 'lambdaguard'})
for issue in issues:
if issue['status'] != 'OPEN':
continue
where = issue['component'].split(':', 1)[1]
yield {
'level':
'high',
'text':
f'{issue["message"]}\n{where} on line {issue["textRange"]["startLine"]}.'
}
test_list_url = "data/multi-woz/testListFile.json"
data_url = "data/multi-woz/data.json"
domains = ["restaurant", "taxi", "train", "attraction", "hotel"]
data_train = {}
data_test = {}
data_val = {}
if not os.path.isfile(word_vectors_url):
if not os.path.exists("data"):
os.makedirs("word-vectors")
print("Downloading and unzipping the pre-trained word embeddings")
resp = urlopen(vectors_url)
zip_ref = ZipFile(BytesIO(resp.read()))
zip_ref.extractall("word-vectors")
zip_ref.close()
if os.path.isfile("data/train.json"):
exit()
print("Preprocessing the data and creating the ontology")
if not os.path.exists("data"):
os.makedirs("data")
os.makedirs("data/multi-woz")
if not os.path.exists(data_url):
print("Downloading and unzipping the MultiWoz dataset")
answer = int(
input(
def POST(self, courseid, taskid):
""" Edit a task """
if not id_checker(taskid) or not id_checker(courseid):
raise Exception("Invalid course/task id")
course, _ = self.get_course_and_check_rights(courseid, allow_all_staff=False)
# Parse content
try:
data = web.input(task_file={})
try:
task_zip = data.get("task_file").file
except:
task_zip = None
del data["task_file"]
problems = self.dict_from_prefix("problem", data)
limits = self.dict_from_prefix("limits", data)
data = {key: val for key, val in data.iteritems() if not key.startswith("problem") and not key.startswith("limits")}
del data["@action"]
if data["@filetype"] not in self.task_factory.get_available_task_file_extensions():
return json.dumps({"status": "error", "message": "Invalid file type: {}".format(str(data["@filetype"]))})
file_ext = data["@filetype"]
del data["@filetype"]
if problems is None:
return json.dumps({"status": "error", "message": "You cannot create a task without subproblems"})
# Order the problems (this line also deletes @order from the result)
data["problems"] = OrderedDict([(key, self.parse_problem(val))
for key, val in sorted(problems.iteritems(), key=lambda x: int(x[1]['@order']))])
data["limits"] = limits
if "hard_time" in data["limits"] and data["limits"]["hard_time"] == "":
del data["limits"]["hard_time"]
# Weight
try:
data["weight"] = float(data["weight"])
except:
return json.dumps({"status": "error", "message": "Grade weight must be a floating-point number"})
# Groups
if "groups" in data:
data["groups"] = True if data["groups"] == "true" else False
# Accessible
if data["accessible"] == "custom":
data["accessible"] = "{}/{}".format(data["accessible_start"], data["accessible_end"])
elif data["accessible"] == "true":
data["accessible"] = True
else:
data["accessible"] = False
del data["accessible_start"]
del data["accessible_end"]
# Checkboxes
if data.get("responseIsHTML"):
data["responseIsHTML"] = True
# Network grading
data["network_grading"] = "network_grading" in data
except Exception as message:
return json.dumps({"status": "error", "message": "Your browser returned an invalid form ({})".format(str(message))})
# Get the course
try:
course = self.course_factory.get_course(courseid)
except:
return json.dumps({"status": "error", "message": "Error while reading course's informations"})
# Get original data
try:
orig_data = self.task_factory.get_task_descriptor_content(courseid, taskid)
data["order"] = orig_data["order"]
except:
pass
directory_path = self.task_factory.get_directory_path(courseid, taskid)
try:
WebAppTask(course, taskid, data, directory_path)
except Exception as message:
return json.dumps({"status": "error", "message": "Invalid data: {}".format(str(message))})
if not os.path.exists(directory_path):
os.mkdir(directory_path)
if task_zip:
try:
zipfile = ZipFile(task_zip)
except Exception as message:
return json.dumps({"status": "error", "message": "Cannot read zip file. Files were not modified"})
try:
zipfile.extractall(directory_path)
except Exception as message:
return json.dumps(
{"status": "error", "message": "There was a problem while extracting the zip archive. Some files may have been modified"})
self.task_factory.delete_all_possible_task_files(courseid, taskid)
self.task_factory.update_task_descriptor_content(courseid, taskid, data, force_extension=file_ext)
return json.dumps({"status": "ok"})
def unzip(self):
percent = 0.0
self.task = Task.objects.get(pk=self.task.id)
self.task.status = 'PROGRESS: UNZIP'
meta = {
'action': 'unzip',
'file': 'Unzip %s' % self.source,
'percent': percent,
'items_unzipped': 0
}
self.task.meta.update(meta)
self.task.save()
zip_path = settings.DOWNLOAD_PATH
unzip_path = settings.SOURCE_PATH
final_path = check_create_folder('%s/%s' % (unzip_path, self.source))
tmp_path = check_create_folder('%s/%s/tmp' % (unzip_path, self.source))
tmp_path2 = check_create_folder('%s/%s/tmp2' %
(unzip_path, self.source))
total_weight = len(self.files)
for zipped in self.files:
#Initial Unzip Round
# operation weigth 30%
weight = 0.3
zip = ZipFile('%s/%s/%s' % (zip_path, self.source, zipped), 'r')
zip.extractall(path=tmp_path)
zip.close()
percent += (weight / total_weight) * 100
self.task.meta['percent'] = percent
self.task.save()
# Second round of unzipping of files inside the unzip file
# operation weight 70%
weight = 0.7
new_zip_files = glob.glob(tmp_path + '/*/*.zip')
total_files = len(new_zip_files)
file_counter = 0
for zipped in new_zip_files:
file_counter += 1
zip = ZipFile(zipped, 'r')
zip.extractall(path=tmp_path2)
zip.close()
percent += (file_counter / total_files) * (
(weight / total_weight) * 100)
if file_counter % 20 == 0:
self.task.meta['items_unzipped'] = file_counter
self.task.meta['percent'] = percent
self.task.save()
# copy xml files to the correct place
unzipped_files = glob.glob(tmp_path2 + '/*.xml')
for item in unzipped_files:
shutil.copy(item, final_path)
# delete tmp files
try:
shutil.rmtree(tmp_path)
# shutil.rmtree(tmp_path2)
except OSError as exc:
if exc.errno != errno.ENOENT:
raise
self.task.meta['percent'] = 100
self.task.save()
return True
if not os.path.exists(APPENGINE_TARGET_DIR):
print('Downloading the AppEngine SDK...')
#First try and get it from the 'featured' folder
sdk_file = urlopen(FEATURED_SDK_REPO + APPENGINE_SDK_FILENAME)
if sdk_file.getcode() == 404:
#Failing that, 'deprecated'
sdk_file = urlopen(DEPRECATED_SDK_REPO + APPENGINE_SDK_FILENAME)
#Handle other errors
if sdk_file.getcode() >= 299:
raise Exception('App Engine SDK could not be found. {} returned code {}.'.format(sdk_file.geturl(), sdk_file.getcode()))
zipfile = ZipFile(StringIO(sdk_file.read()))
zipfile.extractall(TARGET_DIR)
#Make sure the dev_appserver and appcfg are executable
for module in ("dev_appserver.py", "appcfg.py"):
app = os.path.join(APPENGINE_TARGET_DIR, module)
st = os.stat(app)
os.chmod(app, st.st_mode | stat.S_IEXEC)
else:
print('Not updating SDK as it exists. Remove {} and re-run to get the latest SDK'.format(APPENGINE_TARGET_DIR))
print("Running pip...")
args = ["pip", "install", "--no-deps", "-r", REQUIREMENTS_FILE, "-t", TARGET_DIR, "-I"]
p = subprocess.Popen(args)
p.wait()
print("Installing Django {}".format(DJANGO_VERSION))
def extractAll(zip):
"""
Extrait tous les fichiers d'une archive word dans un répertoire temporaire.
"""
myzip = ZipFile(zip, 'a')
myzip.extractall('./../tmp/')
