def fetch_data(links):
# clean everything out
delete_folder(temp_dir)
delete_file(output_file_name)
# download the links
for i,link in enumerate(links):
# extension = os.path.splitext(link)[1]
extension = 'jpg'
urllib.urlretrieve(link, '%s/%s.%s' % (temp_dir, i, extension))
# zip the images
shutil.make_archive(output_file_name, 'zip', temp_dir)
# make watson requests
r = requests.post(url, files={'images_file.zip': open(output_file_name + '.zip', 'rb')}, auth=(BLUEMIX_USERNAME, BLUEMIX_PASSWORD))
# massage the data back
if (r.status_code == 200):
data = json.loads(r.text)
for i, link in enumerate(links):
j = get_index(data['images'], i)
data['images'][j]['image'] = link
return data
# transform them back to links
abort(403)
def action_callback_destroy(args, config):
"""
1. Disable the site
2. Reload Apache
3. Make an archive of the site
4. Remove VHost
5. Remove directories
"""
# 1. Disable the site
__disable_site(args, config)
# 2. Reload Apache
__reload_apache(args, config)
# 3. Make an archive of the site
backup_folder = os.path.join(args.config, 'backup')
if not os.path.exists(backup_folder):
os.mkdir(backup_folder, 0755)
__fix_permission(backup_folder, 0755, login_uid, login_gid)
archive_name = os.path.join(backup_folder, '-'.join([args.domain, time.strftime('%Y%m%dT%H%M%S'), str(uuid.uuid1())]))
shutil.make_archive(archive_name, 'gztar', siteroot)
__fix_permission(archive_name, 0644, login_uid, login_gid)
# 4. Remove hosts entry
__remove_hosts_entry(args, config)
# 5. Remove directories
__clean_filesystem(args, config)
def main():
# Make a copy or duplicate of an existing file
if path.exists("textfile.txt"):
# Get the path of the file in the current directory
src = path.realpath("textfile.txt")
# Separate the path part from the filename
head, tail = path.split(src)
print "path: " + head
print "file: " + tail
# Now, make a backup file
dst = src + ".bak"
# Then use the shell to make a copy of the file
shutil.copy(src,dst)
# If you want to copy over perms, modification times, and other data
shutil.copystat(src,dst)
# Rename the original file
os.rename("textfile.txt", "newfile.txt")
# Put things intp a ZIP archive
root_dir,tail = path.split(src)
shutil.make_archive("archive", "zip", root_dir)
# More control over ZIP files
with ZipFile("testzip.zip", "w") as newzip:
newzip.write("newfile.txt")
newzip.write("textfile.txt.bak")
def zip_dict(dst_fname):
dir_fpath, basename = os.path.split(dst_fname)
root_dir, dir_fname = os.path.split(dir_fpath)
# uzado de zip ne funkciigas, rompas Colordict - t.e. la lasta komencas blinki senfine
#fmt = "zip"
fmt = "gztar"
# se vortara datumo estas ĝuste en arkivo, ne en ia dosierujo, do CSS/figuroj estas ne trovataj
save_without_folder = False # True #
if save_without_folder:
fname = shutil.make_archive(o_p.join(revo_dicts_fpath, dir_fname), fmt, dir_fpath)
else:
fname = shutil.make_archive(o_p.join(revo_dicts_fpath, dir_fname), fmt, root_dir, base_dir=dir_fname)
ifo_fname = os.path.splitext(dst_fname)[0] + ".ifo"
with open(ifo_fname) as ifo_f:
properties = {}
for line in ifo_f:
lst = line.split("=")
if len(lst) >= 2:
key, value = lst[0].strip(), lst[1].strip()
if key and value:
properties[key] = value
words_cnt = int(properties.get("wordcount"))
synwordcount = properties.get("synwordcount")
if synwordcount:
words_cnt += int(synwordcount)
fname = os.path.basename(fname)
# du spacetoj fine estas por Markdown liniavanco
print("http://new.bombono.org/download/revo/%(fname)s\t%(words_cnt)s " % locals())
def fetch_sync_gateway_logs(prefix, is_perf_run=False):
print("\n")
print("Pulling logs")
# fetch logs from sync_gateway instances
status = run_ansible_playbook("fetch-sync-gateway-logs.yml", stop_on_fail=False)
if status != 0:
log.error("Error pulling logs")
# zip logs and timestamp
if os.path.isdir("/tmp/sg_logs"):
date_time = time.strftime("%Y-%m-%d-%H-%M-%S")
if is_perf_run:
name = "/tmp/{}-sglogs".format(prefix)
else:
name = "/tmp/{}-{}-sglogs".format(prefix, date_time)
shutil.make_archive(name, "zip", "/tmp/sg_logs")
shutil.rmtree("/tmp/sg_logs")
print("sync_gateway logs copied here {}\n".format(name))
zip_file_path = "{}.zip".format(name)
if is_perf_run:
# Move perf logs to performance_results
shutil.copy(zip_file_path, "performance_results/{}/".format(prefix))
print("\n")
return zip_file_path
def config(output='/opt/canopsis/var/backups'):
logger = logging.getLogger()
logger.debug('Config Backup start')
logger.debug('Output: %s' % output)
logger.debug('Create output dir if not exists')
if not os.path.exists(output):
os.makedirs(output)
logger.debug('Create temp dir')
archive_name = 'backup_config'
tmp_dir = mkdtemp(prefix='/opt/canopsis/tmp/')
logger.debug('Create file with installed packages')
export_output = Popen('pkgmgr export', shell=True, stdout=PIPE)
export_output.wait()
f = open('/opt/canopsis/etc/.packages', 'w')
f.writelines(export_output.stdout.read())
f.close()
logger.debug('Copy config files into tmp folder')
shutil.copytree('/opt/canopsis/etc', '%s/%s' % (tmp_dir, archive_name))
logger.debug('Create archive into %s' % output)
logger.debug('Archive name: %s' % archive_name)
shutil.make_archive('%s/%s' % (output, archive_name),
'zip',
tmp_dir)
logger.debug('Remove temp dir')
shutil.rmtree(tmp_dir)
logger.debug('Config Backup finished')
def mongo(host='localhost', output='/opt/canopsis/var/backups'):
logger = logging.getLogger()
logger.debug('Mongo Backup start')
logger.debug('Host : %s' % host)
logger.debug('Output: %s' % output)
logger.debug('Create temp dir')
archive_name = 'backup_mongodb'
tmp_dir = mkdtemp(prefix='/opt/canopsis/tmp/')
os.makedirs('%s/%s' % (tmp_dir, archive_name))
logger.debug('Create output dir if not exists')
if not os.path.exists(output):
os.makedirs(output)
logger.debug('Launch mongodump')
mongodump_cmd = '/opt/canopsis/bin/mongodump --host %s --out %s/%s' % (host, tmp_dir, archive_name)
logger.debug('Command: %s' % mongodump_cmd)
dump_output = Popen(mongodump_cmd, shell=True)
dump_output.wait()
logger.debug('Create archive into %s' % output)
shutil.make_archive('%s/%s' % (output, archive_name),
'zip',
tmp_dir)
logger.debug('Remove temp dir')
shutil.rmtree(tmp_dir)
logger.debug('Mongo Backup finished')
def create_zip_files(self, dirs):
"""Create zip files."""
logmsg.header('Creating zip files...', self.logger)
# Create output directory to hold our zips
output_dir = 'Zips'
if not os.path.exists(output_dir):
os.mkdir(output_dir)
num_zips = 0
# Create a progress bar
pbar = ProgressBar(term_width=80, maxval=len(dirs)).start()
for d in dirs:
self.logger.debug('Zipping: "%s"' % d)
parent_dir = os.path.join(os.path.dirname(d).split(os.path.sep)[-1], '')
if parent_dir == self.input_dir:
parent_dir = ''
output_file = os.path.join(output_dir, parent_dir, os.path.basename(d))
shutil.make_archive(output_file, format="zip", root_dir=d)
num_zips += 1
# Update progress bar
pbar.update(num_zips)
# Ensure progress bar is finished
pbar.finish()
time_elapsed = "(Time Elapsed: {0})".format(secs_to_mins(pbar.seconds_elapsed))
logmsg.success('Created {0} zip files {1}'.format(num_zips,
time_elapsed), self.logger)
def build_windows():
"""Build windows executables/setups."""
utils.print_title("Updating 3rdparty content")
update_3rdparty.run(ace=False, pdfjs=True, fancy_dmg=False)
utils.print_title("Building Windows binaries")
parts = str(sys.version_info.major), str(sys.version_info.minor)
ver = ''.join(parts)
dot_ver = '.'.join(parts)
# Get python path from registry if possible
try:
reg64_key = winreg.OpenKeyEx(winreg.HKEY_LOCAL_MACHINE,
r'SOFTWARE\Python\PythonCore'
r'\{}\InstallPath'.format(dot_ver))
python_x64 = winreg.QueryValueEx(reg64_key, 'ExecutablePath')[0]
except FileNotFoundError:
python_x64 = r'C:\Python{}\python.exe'.format(ver)
out_pyinstaller = os.path.join('dist', 'qutebrowser')
out_64 = os.path.join('dist',
'qutebrowser-{}-x64'.format(qutebrowser.__version__))
artifacts = []
from scripts.dev import gen_versioninfo
utils.print_title("Updating VersionInfo file")
gen_versioninfo.main()
utils.print_title("Running pyinstaller 64bit")
_maybe_remove(out_64)
call_tox('pyinstaller', '-r', python=python_x64)
shutil.move(out_pyinstaller, out_64)
utils.print_title("Running 64bit smoke test")
smoke_test(os.path.join(out_64, 'qutebrowser.exe'))
utils.print_title("Building installers")
subprocess.run(['makensis.exe',
'/DX64',
'/DVERSION={}'.format(qutebrowser.__version__),
'misc/qutebrowser.nsi'], check=True)
name_64 = 'qutebrowser-{}-amd64.exe'.format(qutebrowser.__version__)
artifacts += [
(os.path.join('dist', name_64),
'application/vnd.microsoft.portable-executable',
'Windows 64bit installer'),
]
utils.print_title("Zipping 64bit standalone...")
name = 'qutebrowser-{}-windows-standalone-amd64'.format(
qutebrowser.__version__)
shutil.make_archive(name, 'zip', 'dist', os.path.basename(out_64))
artifacts.append(('{}.zip'.format(name),
'application/zip',
'Windows 64bit standalone'))
return artifacts
def make_zip(self):
"""Try to make a ZIP file."""
try:
# iterate over lists of folders to backup
for folder_to_backup in self.origins:
percentage = int(self.origins.index(folder_to_backup) /
len(self.origins) * 100)
self.setLabelText(self.template.format(
folder_to_backup[:99], self.destination.lower()[:99],
self._date, datetime.now().isoformat()[:-7],
self.seconds_time_to_human_str(time.time() - self._time),
len(self.origins) - self.origins.index(folder_to_backup),
percentage))
self.setValue(percentage)
QApplication.processEvents() # Forces the UI to Update
log.info("Folder to backup: {}.".format(folder_to_backup))
shutil.make_archive(folder_to_backup, "zip",
folder_to_backup, logger=log)
self.move_zip(folder_to_backup + ".zip")
except Exception as reason:
log.warning(reason)
else:
log.info("Copia de Seguridad Backup Termino bien.")
finally:
log.info("Finished BackUp from {} to {}.".format(
self.origins, self.destination))
def Craeate_addon_from_github(URL,local_repo_folder):
archive_suffix="/archive/master.zip"
print(URL)
addonname=URL.strip('/').split('/')[-1]
if not os.path.exists(local_repo_folder+os.sep+addonname):
print("Making folder for addon in repo: ",addonname)
os.makedirs(local_repo_folder+os.sep+addonname)
download_file(URL+archive_suffix,local_repo_folder+os.sep+addonname+os.sep+"master.zip")
try:
xml_frm_file,ziptype=zipfilehandler(local_repo_folder+os.sep+addonname+os.sep+"master.zip")
except Exception as e:
print("cannot create a zip from githuburl ",URL)
return
root = ET.fromstring(xml_frm_file)
for element in root.iter("addon"):
addon_name=element.attrib['id']
addon_version=element.attrib['version']
try:
currntzip=zipfile.ZipFile(local_repo_folder+os.sep+addonname+os.sep+"master.zip")
currntzip.extractall(local_repo_folder+os.sep+addonname+os.sep)
currntzip.close()
shutil.move(local_repo_folder+os.sep+addonname+os.sep+addon_name+"-master",local_repo_folder+os.sep+addonname+os.sep+addon_name)
os.remove(local_repo_folder+os.sep+addonname+os.sep+"master.zip")
shutil.make_archive(local_repo_folder+os.sep+addon_name+os.sep+addon_name+"-"+addon_version,'zip',local_repo_folder+os.sep+addon_name,addon_name)
shutil.rmtree(local_repo_folder+os.sep+addonname+os.sep+addon_name)
except Exception as e:
print("could not save fil ",addonname)
def generate():
output_dir = "/tmp/" + str(random.randint(1, sys.maxint))
output_file_name = "/tmp/" + str(random.randint(1, sys.maxint))
try:
data = {
"app_name" : request.form["app_name"],
"language" : request.form["language"],
"company_name" : request.form["company_name"],
"app_package_name_prefix" : request.form["app_package_name_prefix"],
"app_class_prefix": request.form["app_class_prefix"],
"application_id" : request.form["application_id"],
"compile_sdk_version" : int(request.form["compile_sdk_version"]),
"build_tools_version" : request.form["build_tools_version"],
"min_sdk_version" : int(request.form["min_sdk_version"]),
"target_sdk_version" : int(request.form["target_sdk_version"]),
"crashlytics_api_key" : request.form["crashlytics_api_key"],
"output_dir" : output_dir
}
grow.grow(data)
shutil.make_archive(output_file_name, 'zip', output_dir)
return send_file(output_file_name + ".zip",
mimetype="application/zip",
as_attachment=True,
attachment_filename=request.form["app_class_prefix"] + ".zip")
finally:
if os.path.exists(output_dir):
shutil.rmtree(output_dir)
if os.path.exists(output_file_name):
os.remove(output_file_name)
def zip_dicom(directory):
"""
Function that zip a directory.
:param directory: path to the directory to zip
:type directory: str
:return: archive -> path to the created zip file
:rtype: str
"""
archive = directory + '.zip'
if (os.listdir(directory) == []):
sys.exit(
"The directory " + directory + " is empty and will not be zipped.")
else:
shutil.make_archive(directory, 'zip', directory)
if (os.path.exists(archive)):
shutil.rmtree(directory)
return archive
else:
sys.exit(archive + " could not be created.")
def make_archive(self, path, root_dir=None, use_git=False, prefix=None):
"""Creates a tar.gz archive.
Args:
path (str): Path to the archive to create without extension.
root_dir (str): Root directory from which the archive should be
created.
"""
if prefix:
prefix += '/'
if use_git:
if root_dir:
raise ConfigurationError("archiving with root dir with git not implemented")
cmd = ['git', 'archive', '-o', path + '.tar.gz']
cmd.append('--prefix=' + prefix)
cmd.extend(['-9', 'HEAD'])
self.run_cmd(cmd)
else:
# TODO: Check that root_dir is a subdirectory of the workspace
# (this all does not work if it is the workspace itself).
if not os.path.isabs(root_dir):
root_dir = os.path.join(self._cwd.cwd, root_dir)
org_dir = root_dir
root_dir, base_dir = os.path.split(root_dir)
# TODO: Instead of renaming the directory twice, we could use
# tarfile directly to create the archive.
if prefix:
base_dir = prefix
shutil.move(org_dir, os.path.join(root_dir, prefix))
if not base_dir:
base_dir = '.'
shutil.make_archive(path, 'gztar', root_dir, base_dir)
if prefix:
shutil.move(os.path.join(root_dir, prefix), org_dir)
def ziptex(self):
localskeletond = os.path.join(WD,'skeleton')
try:
shutil.rmtree(localskeletond)
except OSError:
pass
shutil.copytree(lspskeletond, localskeletond)
os.chdir(localskeletond)
localcommands = codecs.open('localcommands.tex','a', encoding='utf-8')
localpackages = codecs.open('localpackages.tex','a', encoding='utf-8')
localcounters = codecs.open('localcounters.tex','a', encoding='utf-8')
content = codecs.open('chapters/filename.tex','w', encoding='utf-8')
contentorig = codecs.open('chapters/filenameorig.tex','w', encoding='utf-8')
localcommands.write(self.commands)
localcommands.write(self.environments)
localcommands.close()
localpackages.write(self.packages)
localpackages.close()
localcounters.write(self.counters)
localcounters.close()
content.write(self.modtext)
content.close()
contentorig.write(self.text)
contentorig.close()
os.chdir(WD)
self.zipfn = str(uuid.uuid4())
shutil.make_archive(self.zipfn, 'zip', localskeletond)
shutil.move(self.zipfn+'.zip',wwwdir)
def display_data(self, trans, data, preview=False, filename=None,
to_ext=None, size=None, offset=None, **kwd):
"""Documented as an old display method, but still gets called via tests etc
This allows us to format the data shown in the central pane via the "eye" icon.
"""
if not preview:
trans.response.set_content_type(data.get_mime())
trans.log_event("Display dataset id: %s" % str(data.id))
# the target directory name
neo4j_dir_name = '/dataset_{}_files/neo4jdb'.format(data.dataset.id)
dir_name = str(os.path.dirname(trans.app.object_store.get_filename(data.dataset))) + neo4j_dir_name
# generate unique filename for this dataset
fname = ''.join(c in FILENAME_VALID_CHARS and c or '_' for c in data.name)[0:150]
# zip the target directory (dir_name) using the fname
shutil.make_archive(fname, 'zip', dir_name)
download_zip = fname + '.zip'
# setup headers for the download
trans.response.headers['Content-Length'] = int(os.stat(download_zip).st_size)
# force octet-stream so Safari doesn't append mime extensions to filename
trans.response.set_content_type("application/octet-stream")
trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % \
(data.hid, download_zip, "zip")
return open(download_zip)
def config():
logger = logging.getLogger()
logger.debug('Config Backup start:')
archive_name = 'backup_config'
tmp_dir = '%s/%s' % (backup_path, archive_name)
if os.path.exists(tmp_dir):
logger.debug(' + Remove old temp dir')
shutil.rmtree(tmp_dir)
logger.debug(' + List all packages')
lines = []
for package in ubik_api.db.get_installed():
lines.append(package.name)
lines.append('\n')
lines = lines[:-1]
f = open('%s/etc/.packages' % home_path, 'w')
f.writelines(lines)
f.close()
logger.debug(' + Copy config files')
shutil.copytree('%s/etc' % home_path, '%s/' % tmp_dir)
logger.debug(' + Make archive')
shutil.make_archive('%s/%s' % (backup_path, archive_name), 'zip', tmp_dir)
logger.debug(' + Remove temp dir')
shutil.rmtree(tmp_dir)
logger.debug('Config Backup finished')
def run(self):
update_version_py()
# unless we update this, the sdist command will keep using the old
version = get_version()
update_ansible_version(version)
dest_dir = "ansible/playbooks/files"
extra_files = ["ansible/irma-ansible/irma-ansible.py"]
for subproj in ["common", "frontend", "brain", "probe"]:
subproj_archive = "{}-{}".format(subproj, version)
subproj_archive_path = os.path.join(dest_dir, subproj_archive)
print("Packaging {}.zip".format(subproj_archive))
extra_files.append(subproj_archive_path + ".zip")
shutil.make_archive(subproj_archive_path, "zip", root_dir=subproj)
webui_archive = "core-webui-{}".format(version)
webui_archive_path = os.path.join(dest_dir, webui_archive)
print("Packaging {}.zip".format(webui_archive))
# webui dist is generated in
# root_folder/frontend/web/dist
if os.path.exists("frontend/web/dist"):
extra_files.append(webui_archive_path + ".zip")
shutil.make_archive(webui_archive_path, "zip",
base_dir="frontend/web/dist")
self._create_manifest_git()
self._append_manifest(extra_files)
self.distribution.metadata.version = version
_sdist.run(self)
return
def s_gen(file_path, num_samples):
# If the file path is None, use the current directory
if file_path is None:
file_path = os.getcwd()
output('Looking for .aif files in: {}'.format(file_path))
files = get_files(file_path)
if len(files) == 0:
output_err('Could not find an files to generate samples from!')
output_err('Try running the program from a directory that contains .aif files.')
output_err('Exiting ...')
sys.exit(1)
output('Generating samples from: {}'.format(files))
output("Creating 'samples' directory")
sample_folder = 'samples'
os.mkdir(sample_folder)
for f in files:
generate_samples(sample_folder, f, num_samples)
output('Compressing samples into {}.zip'.format(sample_folder))
# Zip the sample directory
shutil.make_archive(sample_folder, 'zip', sample_folder)
output('Deleting temporary sample directory')
# Delete sample folder
shutil.rmtree(sample_folder)
def ZipTests(self):
self.__maintain_folders()
folder = 'Test Set - '+self.folder_name
os.chdir(self.folder_name)
if(os.path.isdir(folder)):
shutil.rmtree(folder,ignore_errors=True)
if(os.path.isfile(folder+'.zip')):
os.remove(folder+'.zip')
os.mkdir(folder)
folder_escaped = pipes.quote(folder)
if(self.ip_count==0):
shutil.rmtree(folder,ignore_errors=True)
os.chdir('..')
raise Exception('Empty input and output folders')
elif(self.ip_count!=0 and not os.path.isfile('output/output'+str(self.ip_count-1)+'.txt')):
shutil.rmtree(folder,ignore_errors=True)
os.chdir('..')
raise Exception('Outputs missing for some input files')
else:
x1 = os.system('cp -a input '+folder_escaped)
x2 = os.system('cp -a output '+folder_escaped)
if(x1 or x2):
shutil.rmtree(folder,ignore_errors=True)
os.chdir('..')
raise Exception('Cannot finish operation: Input/Output folder missing')
else:
shutil.make_archive(folder,'zip',folder)
shutil.rmtree(folder,ignore_errors=True)
os.chdir('..')
def stash_db_and_deliveries(home=False):
"""
Dump the database and deliveries into the
``db_and_deliveries_stash/``-directory.
:param home:
Use ``home=yes`` to stash to ``~/.devilry_db_and_deliveries_stash``
instead of ``<this dir>/db_and_deliveries_stash/``
"""
stashdir = _get_stashdir(home)
if exists(stashdir):
rmtree(stashdir)
mkdir(stashdir)
# DB
dbdumpfile = join(stashdir, "dbdump.sql")
backup_db(dbdumpfile)
_gzip_file(dbdumpfile)
# Delivery files
import logging
logging.basicConfig(level=logging.INFO)
log = logging.getLogger("files.zip")
make_archive(join(stashdir, "files"), "zip", logger=log, base_dir="deliverystorehier")
def build_release():
print "build_release"
# files in release
FILES = ["bg.js", "load.js", "manifest.json", "css.ini",
"maintab.html", "maintab.js", "maintab.css",
"options.html", "options.js", "options.css",
"config.js", "gear19.png", "gear128.png",
"ext/beautify-css.min.js",
"ext/beautify-html.min.js",
"ext/beautify.min.js",
"ext/googlecode.css",
"ext/highlight.pack.js",
"ext/zepto.min.js",
]
# temp dir
dir = tempfile.mkdtemp(suffix="")
print "created", dir
# copy files
os.mkdir(dir+"/ext")
for x in FILES:
shutil.copyfile(x, dir+"/"+x)
# make zip
print "compressing files"
for i, x in enumerate(os.listdir(dir)):
print " ", i+1, x
shutil.make_archive("./release", "zip", dir)
print "release.zip created"
print "deleting", dir
shutil.rmtree(dir)
def zip_directory(manga, chapter_num, url):
directory = application.config['UPLOAD_FOLDER'] + url + "/"
filename = make_zip_filename(manga, chapter_num)
# To make sure there is no recursive compression
shutil.make_archive(directory + "../" + filename, "zip", directory)
shutil.move(directory + "../" + filename + ".zip",
directory + filename + ".zip")
def test_make_tarball(self):
# creating something to tar
root_dir, base_dir = self._create_files('')
tmpdir2 = self.mkdtemp()
# force shutil to create the directory
os.rmdir(tmpdir2)
# working with relative paths
work_dir = os.path.dirname(tmpdir2)
rel_base_name = os.path.join(os.path.basename(tmpdir2), 'archive')
with support.change_cwd(work_dir):
base_name = os.path.abspath(rel_base_name)
tarball = make_archive(rel_base_name, 'gztar', root_dir, '.')
# check if the compressed tarball was created
self.assertEqual(tarball, base_name + '.tar.gz')
self.assertTrue(os.path.isfile(tarball))
self.assertTrue(tarfile.is_tarfile(tarball))
with tarfile.open(tarball, 'r:gz') as tf:
self.assertEqual(sorted(tf.getnames()),
['.', './file1', './file2',
'./sub', './sub/file3', './sub2'])
# trying an uncompressed one
with support.change_cwd(work_dir):
tarball = make_archive(rel_base_name, 'tar', root_dir, '.')
self.assertEqual(tarball, base_name + '.tar')
self.assertTrue(os.path.isfile(tarball))
self.assertTrue(tarfile.is_tarfile(tarball))
with tarfile.open(tarball, 'r') as tf:
self.assertEqual(sorted(tf.getnames()),
['.', './file1', './file2',
'./sub', './sub/file3', './sub2'])
def test_tarfile_vs_tar(self):
root_dir, base_dir = self._create_files()
base_name = os.path.join(self.mkdtemp(), 'archive')
tarball = make_archive(base_name, 'gztar', root_dir, base_dir)
# check if the compressed tarball was created
self.assertEqual(tarball, base_name + '.tar.gz')
self.assertTrue(os.path.isfile(tarball))
# now create another tarball using `tar`
tarball2 = os.path.join(root_dir, 'archive2.tar')
tar_cmd = ['tar', '-cf', 'archive2.tar', base_dir]
subprocess.check_call(tar_cmd, cwd=root_dir)
self.assertTrue(os.path.isfile(tarball2))
# let's compare both tarballs
self.assertEqual(self._tarinfo(tarball), self._tarinfo(tarball2))
# trying an uncompressed one
tarball = make_archive(base_name, 'tar', root_dir, base_dir)
self.assertEqual(tarball, base_name + '.tar')
self.assertTrue(os.path.isfile(tarball))
# now for a dry_run
tarball = make_archive(base_name, 'tar', root_dir, base_dir,
dry_run=True)
self.assertEqual(tarball, base_name + '.tar')
self.assertTrue(os.path.isfile(tarball))
def main():
parse_command_line()
version = get_version()
dir_name = APP_NAME + '-' + version
if not stable_build:
dir_name += '-canary'
if debug_build:
dir_name += '-dbg'
print(dir_name)
out_dir = os.path.join(BUILD_DIR, dir_name)
archive_path = out_dir + '.zip'
delete(out_dir, archive_path)
copy_files(SOURCE_DIR, out_dir, FILES_TO_COPY)
background_js_files = process_manifest(out_dir, version)
compile_js(os.path.join(out_dir, 'js', 'background.js'),
background_js_files,
BACKGROUND_COMPILATION_LEVEL,
BACKGROUND_EXTERNS)
js_files = process_index(out_dir)
compile_js(os.path.join(out_dir, TARGET_JS),
js_files,
COMPILATION_LEVEL,
JS_EXTERNS)
print('Archiving', archive_path)
shutil.make_archive(out_dir, 'zip',
root_dir=os.path.abspath(BUILD_DIR),
base_dir=dir_name,
verbose=True)
def domove(self, current):
#directory
if self.dir_list[current][2] == True:
#clobber
if self.dir_list[current][3] == 'yes':
shutil.make_archive(save_root + current + ".zip", "zip", self.dir_list[current][0] )
else:
#noclobber
#get date and time
date = datetime.datetime.now()
date = date.isoformat().split('.')[0:-1][0].replace(':',"")
shutil.make_archive(save_root + current + date + ".zip", "zip", self.dir_list[current][0] )
else:
#single file
if self.dir_list[current][3] == 'yes':
#clobber
zf = zipfile.ZipFile(save_root + current + '.zip', 'w')
zf.write(self.dir_list[current][0])
zf.close()
else:
#append to zipfile
date = datetime.datetime.now()
date = date.isoformat().split('.')[0:-1][0].replace(':',"")
zf = zipfile.ZipFile(save_root + current + date+ '.zip', 'w')
zf.write(self.dir_list[current][0])
zf.close()
return
pass
def test_make_archive_owner_group(self):
# testing make_archive with owner and group, with various combinations
# this works even if there's not gid/uid support
if UID_GID_SUPPORT:
group = grp.getgrgid(0)[0]
owner = pwd.getpwuid(0)[0]
else:
group = owner = 'root'
base_dir, root_dir, base_name = self._create_files()
base_name = os.path.join(self.mkdtemp() , 'archive')
res = make_archive(base_name, 'zip', root_dir, base_dir, owner=owner,
group=group)
self.assertTrue(os.path.exists(res))
res = make_archive(base_name, 'zip', root_dir, base_dir)
self.assertTrue(os.path.exists(res))
res = make_archive(base_name, 'tar', root_dir, base_dir,
owner=owner, group=group)
self.assertTrue(os.path.exists(res))
res = make_archive(base_name, 'tar', root_dir, base_dir,
owner='kjhkjhkjg', group='oihohoh')
self.assertTrue(os.path.exists(res))
def _backup_linear_log(self, fs_now, current_work_dir, backup_format, backup_work_dir, backup_history):
delete_previous_backups = False
# Aren't we going to exceed the limit?
max_backups = min(backup_history, MAX_BACKUPS)
backup_contents = []
for item in os.listdir(backup_work_dir):
item = os.path.join(backup_work_dir, item)
# We tally files only and assume each file must be one of ours so we can safely drop it if need be
if os.path.isfile(item):
backup_contents.append(item)
len_backups = len(backup_contents)
# It's the first backup or we're past the limit so we need a fresh prefix
if not len_backups or len_backups >= max_backups:
next_prefix = _first_prefix
else:
next_prefix = str(len_backups).zfill(len(_first_prefix))
# Also, we need to delete previous backups if we're starting anew
if len_backups >= max_backups:
delete_previous_backups = True
backup_name = '{}-{}'.format(next_prefix, fs_now)
backup_path = os.path.join(backup_work_dir, backup_name)
shutil.make_archive(backup_path, backup_format, current_work_dir, verbose=True, logger=self.logger)
if delete_previous_backups:
self._delete(backup_contents)
def main():
# make a duplicate of an existing file
if path.exists("textfile.txt"):
# get the path to the file in the current directory
src = path.realpath("textfile.txt");
# separate the path part from the filename
head, tail = path.split(src)
print "path: " + head
print "file: " + tail
# let's make a backup copy by appending "bak" to the name
dst = src + ".bak"
# now use the shell to make a copy of the file
shutil.copy(src,dst)
# copy over the permissions, modification times, and other info
shutil.copystat(src, dst)
# rename the original file
os.rename("textfile.txt", "newfile.txt")
# now put things into a ZIP archive
root_dir,tail = path.split(src)
shutil.make_archive("archive", "zip", root_dir)
# more fine-grained control over ZIP files
with ZipFile("testzip.zip","w") as newzip:
newzip.write("newfile.txt")
newzip.write("textfile.txt.bak")
def runner(dbase, run_directory, orphan_directory=None, hold_directory=None):
"""
High-throughput calculation runner.
Parameters
----------
dbase : iprPy.Database
The database to interact with.
run_directory : str
The path to the directory where the calculation instances to run are
located.
orphan_directory : str, optional
The path for the orphan directory where incomplete calculations are
moved. If None (default) then will use 'orphan' at the same level as
the run_directory.
hold_directory : str, optional
The path for the hold directory where tar archives that failed to be
uploaded are moved to. If None (default) then will use 'hold' at the
same level as the run_directory.
"""
# Get path to Python executable running this script
py_exe = sys.executable
if py_exe is None:
py_exe = 'python'
# Get absolute path to run_directory
run_directory = os.path.abspath(run_directory)
# Get original working directory
original_dir = os.getcwd()
# Define runner log file
d = datetime.datetime.now()
pid = os.getpid()
runner_log_dir = Settings().runner_log_directory
if not os.path.isdir(runner_log_dir):
os.makedirs(runner_log_dir)
log_file = os.path.join(
runner_log_dir, '%04i-%02i-%02i-%02i-%02i-%06i-%i.log' %
(d.year, d.month, d.day, d.minute, d.second, d.microsecond, pid))
# Set default orphan_directory
if orphan_directory is None:
orphan_directory = os.path.join(os.path.dirname(run_directory),
'orphan')
# Set default orphan_directory
if hold_directory is None:
hold_directory = os.path.join(os.path.dirname(run_directory), 'hold')
# Start runner log file
with open(log_file, 'a') as log:
# Change to the run directory
os.chdir(run_directory)
# Initialize bidfailcount counter
bidfailcount = 0
# Announce the runner's pid
print(f'Runner started with pid {pid}', flush=True)
# flist is the running list of calculations
flist = os.listdir(run_directory)
while len(flist) > 0:
# Pick a random calculation from the list
index = random.randint(0, len(flist) - 1)
sim = flist[index]
# Submit a bid and check if it succeeded
if bid(sim):
# Reset bidfailcount
bidfailcount = 0
# Move to simulation directory
os.chdir(sim)
log.write('%s\n' % sim)
# Check that the calculation has calc_*.py, calc_*.in and
# record in the database
try:
record = dbase.get_record(name=sim)
calc_py = get_file('calc_*.py')
calc_in = get_file('calc_*.in')
# Pass ConnectionErrors forward killing runner
except requests.ConnectionError as e:
raise requests.ConnectionError(e)
# If not complete, zip and move to the orphan directory
except:
log.write(
'Incomplete simulation: moved to orphan directory\n\n')
os.chdir(run_directory)
if not os.path.isdir(orphan_directory):
os.makedirs(orphan_directory)
shutil.make_archive(os.path.join(orphan_directory, sim),
'gztar',
root_dir=run_directory,
base_dir=sim)
removecalc(os.path.join(run_directory, sim))
flist = os.listdir(run_directory)
continue
# Check if any files in the calculation folder are incomplete
# records
error_flag = False
ready_flag = True
for fname in glob.iglob('*'):
parent_sim, ext = os.path.splitext(os.path.basename(fname))
if ext in ('.json', '.xml'):
parent = DM(fname)
try:
status = parent.find('status')
# Check parent record in database to see if it has completed
if status == 'not calculated':
parent_record = dbase.get_record(
name=parent_sim)
try:
status = parent_record.content.find(
'status')
# Mark flag if still incomplete
if status == 'not calculated':
ready_flag = False
break
# Skip if parent calculation failed
elif status == 'error':
with open(os.path.basename(fname),
'w') as f:
parent_record.content.json(
fp=f, indent=4)
error_flag = True
error_message = 'parent calculation issued an error'
break
# Ignore if unknown status
else:
raise ValueError('unknown status')
# Copy parent record to calculation folder if it is now complete
except:
with open(os.path.basename(fname),
'w') as f:
parent_record.content.json(fp=f,
indent=4)
log.write(
'parent %s copied to sim folder\n' %
parent_sim)
# skip if parent calculation failed
elif status == 'error':
error_flag = True
error_message = 'parent calculation issued an error'
break
except:
continue
# Handle calculations that have unfinished parents
if not ready_flag:
bid_files = glob.glob('*.bid')
os.chdir(run_directory)
for bid_file in bid_files:
os.remove(os.path.join(sim, bid_file))
flist = [parent_sim]
log.write('parent %s not ready\n\n' % parent_sim)
continue
# Run the calculation
try:
assert not error_flag, error_message
run = subprocess.Popen([py_exe, calc_py, calc_in, sim],
stderr=subprocess.PIPE)
error_message = run.stderr.read()
try:
error_message = error_message.decode('UTF-8')
except:
pass
# Load results.json
try:
model = DM('results.json')
# Throw errors if no results.json
except:
error_flag = True
assert not error_flag, error_message
log.write('sim calculated successfully\n')
# Catch any errors and build results.json
except:
model = record.content
keys = list(model.keys())
record_type = keys[0]
model[record_type]['status'] = 'error'
model[record_type]['error'] = str(sys.exc_info()[1])
with open('results.json', 'w') as f:
model.json(fp=f, indent=4)
log.write('error: %s\n' % model[record_type]['error'])
# Update record
tries = 0
while tries < 10:
try:
dbase.update_record(content=model, name=sim)
break
except:
tries += 1
if tries == 10:
os.chdir(run_directory)
log.write('failed to update record\n')
else:
# Archive calculation and add to database or hold_directory
try:
dbase.add_tar(root_dir=run_directory, name=sim)
except:
log.write('failed to upload archive\n')
if not os.path.isdir(hold_directory):
os.makedirs(hold_directory)
shutil.move(sim + '.tar.gz', hold_directory)
os.chdir(run_directory)
removecalc(os.path.join(run_directory, sim))
log.write('\n')
# Else if bid(sim) failed
else:
bidfailcount += 1
# Stop unproductive worker after 10 consecutive bid fails
if bidfailcount > 10:
print("Didn't find an open simulation", flush=True)
break
# Pause for 10 seconds before trying again
time.sleep(10)
# Regenerate flist and flush log file
flist = os.listdir(run_directory)
log.flush()
os.fsync(log.fileno())
print('No simulations left to run', flush=True)
os.chdir(original_dir)
def upload_site(site_dir, s3, to_bucket, to_key):
with tempfile.NamedTemporaryFile() as tmp_file:
site_zip_file = shutil.make_archive(tmp_file.name, 'zip', site_dir)
s3.upload_file(site_zip_file, to_bucket, to_key)
def zip_folders(folders):
print('Zipping...')
for folder in folders:
shutil.make_archive('{}'.format(folder), 'zip', '{}'.format(folder))
shutil.rmtree('{}'.format(folder))
"""
import shutil, tempfile
# Config
ignore_patterns = (
"__pycache__",
"*.pyc",
".git",
".gitignore",
".vscode",
"startup.blend1",
"TODO.md",
"dev",
"docs",
"verification",
)
output_filename = "blenderfds"
# Make
with tempfile.TemporaryDirectory() as tmpdirname:
shutil.copytree(
"..",
tmpdirname + "/blenderfds",
symlinks=False,
ignore=shutil.ignore_patterns(*ignore_patterns),
)
shutil.make_archive(output_filename, "zip", tmpdirname)
print(f"Done: {output_filename}.zip")
answer_dict["task1"][language][word] = task_1_answer
answer_dict["task2"][language][word] = task_2_answer
##############################################
# Evaluate the answer and save it ############
##############################################
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(answer_dict)
for task in answer_dict:
for language in answer_dict[task]:
task_path = f"{paths['answer_path_main']}{task}/"
os.makedirs(task_path, exist_ok=True)
with open(f"{task_path}{language}.txt", 'w', encoding='utf-8') as f_out:
for word in answer_dict[task][language]:
answer = int(answer_dict[task][language][word]) if task == "task1" else \
float(answer_dict[task][language][word])
if language == "english":
for t_word in english_target_words:
if word == t_word[:-3]:
word = t_word
f_out.write('\t'.join((word, str(answer) + '\n')))
shutil.make_archive(paths['out_zip_path_main'], 'zip', paths['in_zip_path_main'])
print("done")
if match2:
path = rename_to + match2.group(
1) + "/" + match2.group(2) + "/"
if match and match2:
new_filename = new_line + ".pdf"
os.makedirs(os.path.dirname(path), exist_ok=True)
shutil.copy(fullpath, path + new_filename)
def two_d(addLeading):
addLeading = str(addLeading)
if (len(addLeading) < 2):
addLeading = "0" + addLeading
return addLeading
if __name__ == '__main__':
extract_text()
currentMonth = datetime.now().month - 1
currentYear = datetime.now().year
#currentMonth = two_d(currentMonth)
zip_dir = str(currentYear) + "/" + str(currentMonth)
print(zip_dir)
zip_dir = rename_to + zip_dir
zip_file = str(currentMonth) + ".zip"
cwd = os.getcwd()
print(cwd)
shutil.make_archive(str(currentMonth), 'zip', zip_dir)
shutil.move(zip_file, rename_to)
shutil.move(os.path.join(cwd, zip_file), os.path.join(rename_to, zip_file))
def perform():
if request.method == 'POST':
try:
temdir = tempfile.gettempdir()
template_file = request.files['template']
template_file_name = secure_filename(template_file.filename)
template_file.save(os.path.join(temdir, template_file_name))
font_size = request.form['fontsize']
name_file = request.files['csv']
name_file_name = secure_filename(name_file.filename)
name_file.save(os.path.join(temdir, name_file_name))
# font color conversion to rgb format
color_code_hex = request.form['fontcolor']
value = color_code_hex.lstrip('#')
lv = len(value)
color_code_rgb = tuple(
int(value[i:i + lv // 3], 16) for i in range(0, lv, lv // 3))
# converion from rgb to bgr for cv2 processing
color_code_bgr = color_code_rgb[::-1]
font = cv2.FONT_HERSHEY_COMPLEX
fontScale = int(font_size)
color = color_code_bgr
thickness = 5
# form namesList on file type of names file
fileFormat = name_file_name.split('.')[-1] # get the .format
if fileFormat == 'txt':
names = open(f'/tmp/{name_file_name}') # names uploaded
namesList = [name[:-1] for name in names]
elif fileFormat == 'csv':
import pandas as pd
df = pd.read_csv(f'/tmp/{name_file_name}')
namesList = df[
'Name'] # set the default value for name column in csv file
else:
raise Exception(
f'Names file format not supported: {fileFormat}')
for name in namesList:
text = name.upper()
img = cv2.imread(f'/tmp/{template_file_name}')
cert_len = img.shape[1]
cert_mid = cert_len // 2
txtsize = cv2.getTextSize(text, font, fontScale, thickness)
txt_len = txtsize[0][0]
if (txt_len % 2 == 0):
mid = txt_len // 2
else:
mid = (txt_len // 2) + 1
org = (cert_mid - mid, 560)
img1 = cv2.putText(img, text, org, font, fontScale, color,
thickness, cv2.LINE_AA)
path = r"/tmp/" #path to save the certificates
cv2.imwrite(os.path.join(path, text + ".png"), img1)
# compressing to zip to upload
shutil.make_archive('./static/certificates', 'zip', '/tmp')
return render_template(
'home.html', c='certificates.zip',
con='block') # returning cerificates in zip to download
except Exception as e:
print(e)
return render_template('home.html', certificates='', con='none')
def execution():
execution_instance = {}
if 'config_uuid' in request.form:
config_uuid = request.form['config_uuid']
config_entry = persistence['configuration'][config_uuid]
execution_instance['config'] = config_entry
# Create UUID for execution
execution_uuid = str(uuid.uuid4())
execution_instance['uuid'] = execution_uuid
# Execution folder will be below configuration folder
execution_path = os.path.join(storage_path, config_uuid,
execution_uuid)
# '-n' cli mode (mandatory)
# '-e' generate report dashboard
# '-o' output folder for report dashboard
# '-j' run log file name
# '-l' test sample results filename
jmeter_cli_call = [
jmeter_executable, '-n', '-e', '-o ' +
os.path.join(execution_path, test_results_dashboard_folder_name),
'-j ' + os.path.join(execution_path, test_run_log_filename),
'-l ' + os.path.join(execution_path, test_sample_results_filename)
]
# Possible extensions
# * -r -R remote (server mode)
# * -H -P Proxy
# * many more ( jmeter -? )
# * Parameter string for -Dpropkey=propvalue
if 'host' in config_entry:
jmeter_target_host = config_entry['host']
current_app.logger.info(f'Setting host to {jmeter_target_host}')
jmeter_cli_call.append('-JHOST=' + jmeter_target_host)
if 'port' in config_entry:
jmeter_target_port = config_entry['port']
current_app.logger.info(f'Setting port to {jmeter_target_port}')
jmeter_cli_call.append('-JPORT=' + jmeter_target_port)
if 'test_plan_path' in config_entry:
os.mkdir(execution_path)
jmeter_cli_call.append(
'-t ' +
os.path.join(storage_path, config_entry['test_plan_path']))
if 'properties_path' in config_entry:
jmeter_cli_call.append(
'-p ' + os.path.join(config_entry['properties_path']))
else:
return "Configuration does not contain a test plan.", status.HTTP_404_NOT_FOUND
execution_instance['cli_call'] = jmeter_cli_call
current_app.logger.info(f'JMeter CLI call: {str(jmeter_cli_call)}')
execution_start = datetime.datetime.now()
os.system(' '.join(jmeter_cli_call))
execution_end = datetime.datetime.now()
execution_instance['execution_start'] = execution_start
execution_instance['execution_end'] = execution_end
with tempfile.NamedTemporaryFile() as tf:
tmp_zip_file = shutil.make_archive(tf.name, 'zip', execution_path)
shutil.copy2(tmp_zip_file,
os.path.join(execution_path, result_zip_file_name))
persistence['execution'][execution_uuid] = execution_instance
return jsonify(execution_instance), status.HTTP_201_CREATED
else:
return "No configuration with that ID found.", jsonify(
persistence), status.HTTP_404_NOT_FOUND
sys.stdout.write("Au revoir ! \n")
sys.exit(0)
sys.stdout.write("saisissez q pour quitter ou Ctr c \n")
while True:
sys.stdout.write("Chemin du fichier à sauvegarder: \n")
src = input()
sys.stdout.write("Répertoire où stocker le fichier: \n")
dst = input()
signal.signal(signal.SIGINT, leave)
if src == q or dst == q:
leave()
isHere = False
shutil.make_archive('folder_test', 'gztar', 'folder_test')
if not os.path.exists(dst):
os.makedirs(dst) # On vérifie la présence d 'un fichier
try:
if isHere == False:
sys.stdout.write("Vérification du fichier data \n")
for file in os.listdir(dst):
if file == "folder_test.tar.gz":
ishere = True
sys.stdout.write("Comparaison des deux sauvegardes\n")
with gzip.open(dst) as file:
old_save = f.read()
with gzip.open(src + ".tar.gz") as file:
new_save = f.read()
if new_save != old_save:
def copy_issues(issues_directory, upload_directory, tag_name):
"""Copies issue files (only top 5 smallest files from each folder) into the upload_directory
Args:
issues_directory (string): Issues directory
upload_directory (string): Upload directory
tag_name (string): Tag name for zip file
Returns:
[type]: [description]
"""
# Create upload_directory
if not os.path.isdir(upload_directory):
os.makedirs(upload_directory)
# Create temp directory to copy all issues to upload. We don't want to create a sub-folder
# inside issues_directory because that will also get included twice.
with TempDir() as prep_directory:
def sorter_by_size(pair):
""" Sorts the pair (file_name, file_size) tuple in ascending order of file_size
Args:
pair ([(string, int)]): List of tuple of file_name, file_size
"""
pair.sort(key=lambda x: x[1], reverse=False)
return pair
summary_of_summary = []
for file_path, dirs, files in walk(issues_directory, topdown=True):
filename_with_size = []
# Credit: https://stackoverflow.com/a/19859907
dirs[:] = [d for d in dirs]
for name in files:
if not name.lower().endswith(".g.cs"):
continue
curr_file_path = path.join(file_path, name)
size = getsize(curr_file_path)
filename_with_size.append((curr_file_path, size))
if len(filename_with_size) == 0:
continue
summary_file = path.join(file_path, "summary.txt")
summary_of_summary.append("**** " + file_path)
with open(summary_file, 'r') as sf:
summary_of_summary.append(sf.read())
filename_with_size.append(
(summary_file, 0)) # always copy summary.txt
# Copy atmost 5 files from each bucket
sorted_files = [
f[0] for f in sorter_by_size(filename_with_size)[:6]
] # sorter_by_size(filename_with_size)[:6]
print('### Copying below files from {0} to {1}:'.format(
issues_directory, prep_directory))
print('')
print(os.linesep.join(sorted_files))
for src_file in sorted_files:
dst_file = src_file.replace(issues_directory, prep_directory)
dst_directory = path.dirname(dst_file)
if not os.path.exists(dst_directory):
os.makedirs(dst_directory)
try:
shutil.copy2(src_file, dst_file)
except PermissionError as pe_error:
print('Ignoring PermissionError: {0}'.format(pe_error))
issues_summary_file_name = "issues-summary-{}.txt".format(tag_name)
print("Creating {} in {}".format(issues_summary_file_name,
upload_directory))
issues_summary_file = os.path.join(upload_directory,
issues_summary_file_name)
with open(issues_summary_file, 'w') as sf:
sf.write(os.linesep.join(summary_of_summary))
# Also copy the issues-summary inside zip folder
dst_issue_summary_file = os.path.join(prep_directory,
issues_summary_file_name)
try:
shutil.copy2(issues_summary_file, dst_issue_summary_file)
except PermissionError as pe_error:
print('Ignoring PermissionError: {0}'.format(pe_error))
# Zip compress the files we will upload
zip_path = os.path.join(prep_directory, "AllIssues-" + tag_name)
print("Creating archive: " + zip_path)
shutil.make_archive(zip_path, 'zip', prep_directory)
zip_path += ".zip"
dst_zip_path = os.path.join(upload_directory,
"AllIssues-" + tag_name + ".zip")
print("Copying {} to {}".format(zip_path, dst_zip_path))
try:
shutil.copy2(zip_path, dst_zip_path)
except PermissionError as pe_error:
print('Ignoring PermissionError: {0}'.format(pe_error))
src_antigen_log = os.path.join(issues_directory,
get_antigen_filename(tag_name))
dst_antigen_log = os.path.join(upload_directory,
get_antigen_filename(tag_name))
print("Copying {} to {}".format(src_antigen_log, dst_antigen_log))
try:
shutil.copy2(src_antigen_log, dst_antigen_log)
except PermissionError as pe_error:
print('Ignoring PermissionError: {0}'.format(pe_error))
import os
import shutil
def copy_folder(src, dst):
try:
shutil.copytree(src, dst)
except OSError as exc:
if exc.errno == errno.ENOTDIR:
shutil.copy(src, dst)
else:
raise
tempPath = '../release/ARCMF.VR'
if not os.path.exists(tempPath):
os.makedirs(tempPath)
missionPath = "{}/ARCMF.VR".format(tempPath)
copy_folder("../ARCMF.VR", missionPath)
shutil.make_archive('../release/ARCMF', 'zip', tempPath)
shutil.rmtree(tempPath)
# Copy configuration
configurations = [
n for n in listdir(f'{SRCDIR}/configurations') if not n.startswith('_')
]
for configuration in configurations:
configuration_name = f'infra-quickstart-{configuration}'
configuration_src = f'{SRCDIR}/configurations/{configuration}'
configuration_dist = f'{DISTDIR}/{configuration_name}'
archive_name = f'{configuration_dist}-{version}'
manifests_src = f'{SRCDIR}/manifests'
manifests_dist = f'{configuration_dist}/manifests'
cicd_src = f'{SRCDIR}/ci-cd'
cicd_dist = f'{configuration_dist}/ci-cd'
copytree(configuration_src, configuration_dist)
copytree(manifests_src, manifests_dist)
copytree(cicd_src, cicd_dist)
# Replace templated variable with version in clusters.tf
jinja = Environment(loader=FileSystemLoader(configuration_dist))
template = jinja.get_template('clusters.tf')
data = template.render(version=version)
with open(f'{configuration_dist}/clusters.tf', 'w') as f:
f.write(data)
# always include newline at end of file
f.write('\n')
make_archive(archive_name, 'zip', f'{DISTDIR}', configuration_name)
shutil.copyfile(currentPath, tempPath)
print(">>> All merges completed. Generating necessary files. <<<")
## Simple stuffs, just getting all the pack names and put them in pack.mcmeta
Packs = getRelevantPacks("..\\..\\pack.mcmeta")
## Strips the path so only the pack name left
Packs = [
directory.replace("\\data\\minecraft\\..\\..\\pack.mcmeta", "")
for directory in Packs
]
Directory = "temp\\centrosome\\"
ensure_dir(Directory)
with open(Directory + "pack.mcmeta", "w+") as file:
Json = {"pack": {"pack_format": 5, "description": Packs}}
Json.update(
{"description": "Merged Packs using Cocoon's Datapack Merger."})
file.write(json.dumps(Json, sort_keys=False, indent=4))
shutil.make_archive("centrosome", "zip", Directory)
print(">>>>> Datapacks Merged & Packed <<<<<")
#shutil.rmtree('temp', ignore_errors=True)
print(">>>>>> Deleted Temp Folders <<<<<<")
input("Press Enter to Close Terminal.")
exit(0)
temp_directory = common.generate_temp_path()
os.makedirs(temp_directory)
kosmos_version = get_kosmos_version(args)
auto_build = False
if hasattr(args, 'auto'):
auto_build = args.auto
version_messages = init_version_messages(args, kosmos_version)
build_messages = modules.build(temp_directory, kosmos_version,
args.command, auto_build)
common.delete_path(args.output)
if build_messages is not None:
version_messages += build_messages
if args.command == common.Command.SDSetup:
shutil.move(temp_directory, args.output)
else:
shutil.make_archive(
os.path.splitext(args.output)[0], 'zip', temp_directory)
common.delete_path(os.path.join(os.getcwd(), 'tmp'))
for message in version_messages:
print(message)
else:
common.delete_path(os.path.join(os.getcwd(), 'tmp'))
cur_file = open(str(costume['baseLayerID'])+os.path.splitext(costume['baseLayerMD5'])[1],"wb")
cur_file.write(cur_file_data)
cur_file.close()
if "sounds" in json_obj.keys():
sound_id = 0
for sound in json_obj['sounds']:
cur_file_data = requests.get(getMD5(sound['md5'])).content
cur_file = open(str(sound_id)+".wav","wb")
cur_file.write(cur_file_data)
cur_file.close()
sound_id+=1
with open("sprite.json","w") as temp:
json.dump(json_obj,temp,indent=4)
if not nozipsprites:
os.chdir(backpack_folder)
make_archive("Sprite - " + item['name'],'zip',"Sprite - " + item['name'])
os.rename("Sprite - " + item['name'] + '.zip',"Sprite - " + item['name'] + '.sprite2')
rmtree("Sprite - " + item['name'])
#os.system("zip -r 'Sprite - " + item['name'] + ".zip'
elif item['type'] == "image":
if verbose:
print("Costume - Name:",item['name'])
if not overwrite:
if user_input:
os.chdir(user_mkdir("Costume - " + item['name']))
elif normal:
os.chdir(safe_mkdir("Costume - " + item['name']))
else:
os.chdir(just_mkdir("Costume - " + item['name']))
with open(item['name']+os.path.splitext(item['md5'])[1],"wb") as temp:
known_lines.append(char_line)
append_to_file(temp_file,line)
if time.strptime(line_date['Date'],'%Y-%m-%d %H:%M:%S')>time.strptime(range_stop,'%Y-%m-%d %H:%M:%S'):
break
if line_date['Error']!=None and start_found==True:
log_file_to_save.write(line)
if char_line not in known_lines:
known_lines.append(char_line)
append_to_file(temp_file, line)
# Save statistics and close file
log_stat_info['NumberOfLines']=len(known_lines)
statistics_list.append(log_stat_info)
log_file_to_save.close()
not_relevant_lines=['### '+item['Log']+' ###' for item in statistics_list if item['NumberOfLines']==0]
statistics_list=[item for item in statistics_list if item['NumberOfLines']!=0]
print_in_color('Statisctics - number of lines per log', 'yellow')
print_list(statistics_list)
print_in_color('Skipped logs (LogTool was not able to detect timestamps in log content), are:', 'yellow')
print_list(skipped_logs)
append_to_file(result_file,'### Statistics: Number of lines per file:')
write_list_to_file(result_file,statistics_list,False)
for line in open(temp_file, 'r').readlines():
if line.strip() not in not_relevant_lines:
append_to_file(result_file,line)
os.remove(temp_file)
exec_command_line_command('gzip '+result_file)
shutil.make_archive(result_dir, 'zip', result_dir)
print('Execution time:'+str(time.time()-start_time))
print('SUCCESS!!!')
targetName = 'pros.exe'
else:
targetName = 'pros'
setup(name='pros-cli',
version=open('version').read().strip(),
packages=modules,
url='https://github.com/purduesigbots/pros-cli',
license='MPL-2.0',
author='Purdue ACM Sigbots',
author_email='*****@*****.**',
description='Command Line Interface for managing PROS projects',
options={"build_exe": build_exe_options},
install_requires=install_reqs,
executables=[Executable('proscli/main.py', targetName=targetName)])
if sys.argv[1] == 'build_exe':
import py_compile
import distutils.util
build_dir = './build/exe.{}-{}.{}'.format(distutils.util.get_platform(),
sys.version_info[0],
sys.version_info[1])
py_compile.compile('./prosconductor/providers/githubreleases.py',
cfile='{}/githubreleases.pyc'.format(build_dir))
import shutil
import platform
shutil.make_archive(
'pros_cli-{}-win-{}'.format(
open('version').read().strip(),
platform.architecture()[0]), 'zip', build_dir, '.')
def main(cluster, runtime, mail, username, password):
for script in ['test-functional.py', 'test-restapi-get.py']:
if not os.path.isfile(script):
print '{} is not in current work directory {}'.format(script, os.getcwd())
return
CPU_COMMAND = "cores=$((`grep -c ^processor /proc/cpuinfo`-1)) && for i in `seq 1 $cores`; do while : ; do : ; done & done && sleep {} && for i in `seq 1 $cores`; do kill %$i; done".format(30)
# 3 test batches currently
commands = [
[
'test-functional.py {} --category diag-pingpong-tournament --timeout 2000 --continuous {} --username {} --password {}'.format(cluster, runtime, username, password),
],
[
'test-functional.py {} --category diag-pingpong-parallel --timeout 2000 --continuous {} --username {} --password {}'.format(cluster, runtime, username, password),
],
[
'test-functional.py {} --category clusrun --command "echo -n test" --result "test" --timeout 200 --continuous {} --username {} --password {}'.format(cluster, runtime, username, password),
],
[
'test-functional.py {} --category clusrun --command "ping localhost" --cancel 10 --timeout 200 --continuous {} --username {} --password {}'.format(cluster, runtime, username, password),
'test-functional.py {} --category clusrun --command "ping localhost" --cancel 30 --timeout 200 --continuous {} --username {} --password {}'.format(cluster, runtime, username, password),
'test-functional.py {} --category clusrun --command "ping localhost" --cancel 60 --timeout 200 --continuous {} --username {} --password {}'.format(cluster, runtime, username, password),
'test-functional.py {} --category clusrun --command "sleep 10" --timeout 200 --continuous {} --username {} --password {}'.format(cluster, runtime, username, password),
'test-functional.py {} --category clusrun --command "sleep 30" --timeout 200 --continuous {} --username {} --password {}'.format(cluster, runtime, username, password),
'test-functional.py {} --category clusrun --command "sleep 60" --timeout 200 --continuous {} --username {} --password {}'.format(cluster, runtime, username, password),
'test-functional.py {} --category clusrun --command "echo -n test" --result "test" --timeout 200 --continuous {} --username {} --password {}'.format(cluster, runtime, username, password),
'test-functional.py {} --category clusrun --command "whoami" --result "root\\n" --timeout 200 --continuous {} --username {} --password {}'.format(cluster, runtime, username, password),
'test-functional.py {} --category clusrun --command "hostname" --timeout 200 --continuous {} --username {} --password {}'.format(cluster, runtime, username, password),
'test-functional.py {} --category diag-pingpong-tournament --timeout 2000 --continuous {} --username {} --password {}'.format(cluster, runtime, username, password),
'test-functional.py {} --category diag-pingpong-parallel --timeout 2000 --continuous {} --username {} --password {}'.format(cluster, runtime, username, password),
'test-functional.py {} --category diag-pingpong-tournament --cancel 10 --timeout 200 --continuous {} --username {} --password {}'.format(cluster, runtime, username, password),
'test-functional.py {} --category diag-pingpong-parallel --cancel 30 --timeout 200 --continuous {} --username {} --password {}'.format(cluster, runtime, username, password),
'test-restapi-get.py {} --continuous {}'.format(cluster, runtime, username, password)
],
]
startTime = formatdate(localtime=True)
# adapt to different OS
runOnWindows = ''
runOnLinux = 'python '
if os.name == 'nt':
prefix = runOnWindows
else:
prefix = runOnLinux
commands = [[prefix + command for command in batch] for batch in commands]
# create log directory
logDir = '{}/test_logs/{}'.format(os.getcwd(), time.time())
os.makedirs(logDir)
logs = [['batch{}-thread{}.log'.format(j, i) for i in range(len(commands[j]))] for j in range(len(commands))]
# start and wait test threads per batch
for i in range(len(commands)):
threads = [subprocess.Popen(commands[i][j], shell = True, stdout = open('{}/{}'.format(logDir, logs[i][j]), 'w'), stderr = subprocess.STDOUT) for j in range(len(commands[i]))]
wait = [thread.wait() for thread in threads]
# get the results from logs
results = {}
for log in os.listdir(logDir):
with open('{}/{}'.format(logDir, log), 'r') as f:
result = f.readlines()[-1]
results[log] = result
endTime = formatdate(localtime=True)
mailBody = '<h4>Time:</h4>' + '<b>{}</b>    -    <b>{}</b>'.format(startTime, endTime) \
+ '<h4><br/>Results:</h4>' + '<br/><br/>'.join(['<br/>'.join(['{}: {}'.format(log, results[log]) for log in batch]) for batch in logs]) \
+ '<h4><br/>Details:</h4>' + '<br/><br/>'.join(['<br/>'.join(["<b>Log file</b>: {}".format(logs[i][j]), \
"<b>Command</b>: {}".format(commands[i][j]), \
"<b>Result</b>: {}".format(results[logs[i][j]])]) for i in range(len(commands)) for j in range(len(commands[i]))])
if mail:
# send notification mail
shutil.make_archive(logDir, 'zip', logDir)
with open(logDir+'.zip', 'rb') as f:
attachment = MIMEApplication(f.read())
attachment['Content-Disposition'] = 'attachment; filename="logs.zip"'
sender = mail['Sender']
to = mail['To']
cc = mail['Cc']
receivers = to + cc
message = MIMEMultipart()
message['From'] = Header(socket.gethostname(), 'utf-8')
message['To'] = COMMASPACE.join(to)
message['Cc'] = COMMASPACE.join(cc)
message['Subject'] = 'Continuous functional test result for cluster {}'.format(cluster)
message.attach(MIMEText(mailBody, 'html'))
message.attach(attachment)
smtp = smtplib.SMTP(mail['SmtpServer'])
smtp.starttls()
smtp.ehlo()
smtp.login(mail['UserName'], mail['Password'])
smtp.sendmail(sender, receivers, message.as_string())
else:
with open(logDir+'.html', 'w') as f:
f.write(mailBody)
def _create_zip(self, src, dest, name):
"""Zips the Go package in src named 'name' into dest."""
shutil.make_archive(os.path.join(dest, name), 'zip', root_dir=src)
def update_pf(Data, pc, tc):
pf, projects = model.retrieve_pf()
pf[4] = " <title>" + Data[0] + "</title>"
pf[32] = " <a class=\"navbar-brand js-scroll\" href=\"#page-top\">" + Data[
0] + "</a>"
pf[68] = " <h1 class=\"intro-title mb-4\">" + Data[0] + "</h1>"
pf[69] = " <p class=\"intro-subtitle\"><span class=\"text-slider-items\">" + Data[
1] + "," + Data[2] + "," + Data[3] + "," + Data[
4] + "</span><strong class=\"text-slider\"></strong></p>"
pf[91] = " <p><span class=\"title-s\">Name: </span> <span>" + Data[
0] + "</span></p>"
pf[92] = " <p><span class=\"title-s\">Profile: </span> <span>" + Data[
1] + "</span></p>"
pf[93] = " <p><span class=\"title-s\">Email: </span> <span>" + Data[
5] + "</span></p>"
pf[94] = " <p><span class=\"title-s\">Phone: </span> <span>" + Data[
6] + "</span></p>"
pf[100] = " <span>" + Data[
7] + "</span> <span class=\"pull-right\">" + Data[8] + "%</span>"
pf[105] = " <span>" + Data[
9] + "</span> <span class=\"pull-right\">" + Data[10] + "%</span>"
pf[110] = " <span>" + Data[
11] + "</span> <span class=\"pull-right\">" + Data[12] + "%</span>"
pf[115] = " <span>" + Data[
13] + "</span> <span class=\"pull-right\">" + Data[14] + "%</span>"
pf[130] = Data[15]
pf[163] = " <h2 class=\"s-title\">" + Data[16] + "</h2>"
pf[160] = " <span class=\"ico-circle\"><i class=" + Data[
17] + "></i></span>"
pf[165] = Data[18]
pf[176] = " <h2 class=\"s-title\">" + Data[19] + "</h2>"
pf[178] = " <span class=\"ico-circle\"><i class=" + Data[
20] + "></i></span>"
pf[173] = Data[21]
pf[189] = " <h2 class=\"s-title\">" + Data[22] + "</h2>"
pf[186] = " <span class=\"ico-circle\"><i class=" + Data[
23] + "></i></span>"
pf[191] = Data[24]
pf[202] = " <h2 class=\"s-title\">" + Data[25] + "</h2>"
pf[199] = " <span class=\"ico-circle\"><i class=" + Data[
26] + "></i></span>"
pf[204] = Data[27]
pf[215] = " <h2 class=\"s-title\">" + Data[28] + "</h2>"
pf[212] = " <span class=\"ico-circle\"><i class=" + Data[
29] + "></i></span>"
pf[217] = Data[30]
pf[228] = " <h2 class=\"s-title\">" + Data[31] + "</h2>"
pf[225] = " <span class=\"ico-circle\"><i class=" + Data[
32] + "></i></span>"
pf[230] = Data[33]
pf[249] = " <p class=\"counter\">" + Data[34] + "</p>"
pf[260] = " <p class=\"counter\">" + Data[35] + "</p>"
pf[271] = " <p class=\"counter\">" + Data[36] + "</p>"
pf[521] = " <li><a href=" + Data[
-4] + "><span class=\"ico-circle\"><i class=\"ion-social-linkedin\"></i></span></a></li>"
pf[522] = " <li><a href=" + Data[
-3] + "><span class=\"ico-circle\"><i class=\"ion-social-github\"></i></span></a></li>"
pf[523] = " <li><a href=" + Data[
-2] + "><span class=\"ico-circle\"><i class=\"ion-android-globe\"></i></span></a></li>"
pf[524] = " <li><a href=" + Data[
-1] + "><span class=\"ico-circle\"><i class=\"ion-social-instagram\"></i></span></a></li>"
current = 37
updated_indx = 305
for i in range(int(pc)):
if (i < 6):
pf[updated_indx] = " <h2 class=\"w-title\">" + Data[
current] + "</h2>"
pf[updated_indx +
2] = " <a href=" "><span class=\"w-ctegory\">" + Data[
current + 1] + "</span> / <span class=\"w-date\">" + Data[
current + 2] + "</span></a>"
if (i < 5):
updated_indx += 21
elif (i == 5):
updated_indx += 11
else:
pf[updated_indx] += "\n<div class=\"col-md-4\"><div class=\"work-box\"><a href=\"img/work-5.jpg\" data-lightbox=\"gallery-mf\"><div class=\"work-img\"><img src=\"img/work-5.jpg\" alt=\"\" class=\"img-fluid\"></div><div class=\"work-content\"><div class=\"row\"><div class=\"col-sm-8\"><h2 class=\"w-title\">" + Data[
current] + "</h2><div class=\"w-more\"><span class=\"w-ctegory\">" + Data[
current + 1] + "</span> / <span class=\"w-date\">" + Ddata[
current +
2] + "</span></div></div><div class=\"col-sm-4\"></div></div></div></a></div></div>\n"
current += 3
testi_indx = 439
for i in range(int(tc)):
if (i == 0):
pf[testi_indx] = " <span class=\"author\">" + Data[
current] + "</span>"
pf[testi_indx + 4] = Data[current + 1]
testi_indx += 12
else:
pf[testi_indx] += "\n<div class=\"container\"><div class=\"row\"><div class=\"col-md-12\"><div id=\"testimonial-mf\" class=\"owl-carousel owl-theme\"><div class=\"testimonial-box\"><div class=\"author-test\"><img src=\"img/testimonial-2.jpg\" alt=\"\" class=\"rounded-circle b-shadow-a\"><span class=\"author\">" + Data[
current] + "</span></div><div class=\"content-test\"><p class=\"description lead\">" + Data[
current +
1] + "</p><span class=\"comit\"><i class=\"fa fa-quote-right\"></i></span></div></div></div></div></div></div>\n"
current += 2
pf = pf[:updated_indx + 10] + pf[422:]
s = " "
pf = s.join(pf)
copy_tree("layouts/template1", "cache")
os.remove("cache/index.html")
with open("cache/index.html", "w") as f:
f.write(pf)
temp = gen.gen()
shutil.make_archive(temp, 'zip', "cache/")
return temp
# The Python Standard Library also supports creating TAR and ZIP archives using the high-level methods
# in the shutil module. The archiving utilities in shutil allow you to create, read, and extract ZIP and TAR archives.
# These utilities rely on the lower level tarfile and zipfile modules.
#
# Working With Archives Using shutil.make_archive()
# shutil.make_archive() takes at least two arguments: the name of the archive and an archive format.
# By default, it compresses all the files in the current directory into the archive format specified
# in the format argument. You can pass in an optional root_dir argument to compress files in
# a different directory. .make_archive() supports the zip, tar, bztar, and gztar archive formats.
#
# This is how to create a TAR archive using shutil:
import shutil
# shutil.make_archive(base_name, format, root_dir)
shutil.make_archive('data/backup', 'tar', 'data/')
# This copies everything in data/ and creates an archive called backup.tar in the filesystem and returns its name.
# To extract the archive, call .unpack_archive():
shutil.unpack_archive('backup.tar', 'extract_dir/')
# Calling .unpack_archive() and passing in an archive name and destination directory extracts the contents
# of backup.tar into extract_dir/. ZIP archives can be created and extracted in the same way.
def create_output_workflow(workflow_name):
make_archive(workflow_name, 'zip', OUTPUT_PATH)
base = os.path.splitext(f'{workflow_name}.zip')[0]
os.rename(f'{workflow_name}.zip', base + '.knwf')
rmtree(INPUT_PATH)
rmtree(OUTPUT_PATH)
import requests
import zipfile
import io
import pathlib
from io import BytesIO
import base64
import shutil
import os
# Endpoint of API defined
url = f'http://thehandler.hopto.org:4000/trainForMe'
save_path = './recorded_gestures'
#zipping
print('Zipping recorded data as data.zip')
output_filename = 'data'
shutil.make_archive(output_filename, 'zip', save_path)
print('Data zipping complete')
"""
# encoding the zip file into base64 format
with open("data.zip", "rb") as f: #take a look
bytes = f.read()
encoded = base64.b64encode(bytes)
# defining request body of the API call
payload = {
"ZipFile": encoded.decode('ascii')
}
headers = {
def main(argv=None):
parser = get_parser()
arguments = parser.parse_args(argv)
verbose = arguments.v
set_global_loglevel(verbose=verbose)
# See if there's a configuration file and import those options
if arguments.config is not None:
print('configuring')
with open(arguments.config, 'r') as conf:
_, ext = os.path.splitext(arguments.config)
if ext == '.json':
config = json.load(conf)
if ext == '.yml' or ext == '.yaml':
config = yaml.load(conf, Loader=yaml.Loader)
# Warn people if they're overriding their config file
if len(argv) > 2:
warnings.warn(UserWarning('Using the `-config|-c` flag with additional arguments is discouraged'))
# Check for unsupported arguments
orig_keys = set(vars(arguments).keys())
config_keys = set(config.keys())
if orig_keys != config_keys:
for k in config_keys.difference(orig_keys):
del config[k] # Remove the unknown key
warnings.warn(UserWarning(
'Unknown key "{}" found in your configuration file, ignoring.'.format(k)))
# Update the default to match the config
parser.set_defaults(**config)
# Reparse the arguments
arguments = parser.parse_args(argv)
# Set up email notifications if desired
do_email = arguments.email_to is not None
if do_email:
email_to = arguments.email_to
if arguments.email_from is not None:
email_from = arguments.email_from
else:
email_from = arguments.email_to
smtp_host, smtp_port = arguments.email_host.split(":")
smtp_port = int(smtp_port)
email_pass = getpass('Please input your email password:\n')
def send_notification(subject, message):
send_email(email_to, email_from,
subject=subject,
message=message,
passwd=email_pass,
smtp_host=smtp_host,
smtp_port=smtp_port)
while True:
send_test = input('Would you like to send a test email to validate your settings? [Y/n]:\n')
if send_test.lower() in ['', 'y', 'n']:
break
else:
print('Please input y or n')
if send_test.lower() in ['', 'y']:
send_notification('sct_run_batch: test notification', 'Looks good')
# Set up output directories and create them if they don't already exist
path_output = os.path.abspath(os.path.expanduser(arguments.path_output))
path_results = os.path.join(path_output, 'results')
path_data_processed = os.path.join(path_output, 'data_processed')
path_log = os.path.join(path_output, 'log')
path_qc = os.path.join(path_output, 'qc')
path_segmanual = os.path.abspath(os.path.expanduser(arguments.path_segmanual))
script = os.path.abspath(os.path.expanduser(arguments.script))
path_data = os.path.abspath(os.path.expanduser(arguments.path_data))
for pth in [path_output, path_results, path_data_processed, path_log, path_qc]:
os.makedirs(pth, exist_ok=True)
# Check that the script can be found
if not os.path.exists(script):
raise FileNotFoundError('Couldn\'t find the script script at {}'.format(script))
# Setup overall log
batch_log = open(os.path.join(path_log, arguments.batch_log), 'w')
# Duplicate init_sct message to batch_log
print('\n--\nSpinal Cord Toolbox ({})\n'.format(__version__), file=batch_log, flush=True)
# Tee IO to batch_log and std(out/err)
orig_stdout = sys.stdout
orig_stderr = sys.stderr
sys.stdout = Tee(batch_log, orig_stdout)
sys.stderr = Tee(batch_log, orig_stderr)
def reset_streams():
sys.stdout = orig_stdout
sys.stderr = orig_stderr
# Display OS
print("INFO SYSTEM")
print("-----------")
platform_running = sys.platform
if platform_running.find('darwin') != -1:
os_running = 'osx'
elif platform_running.find('linux') != -1:
os_running = 'linux'
print('OS: ' + os_running + ' (' + platform.platform() + ')')
# Display number of CPU cores
print('CPU cores: Available: {} | Threads used by ITK Programs: {}'.format(multiprocessing.cpu_count(), arguments.itk_threads))
# Display RAM available
print("RAM: Total {} MB | Available {} MB | Used {} MB".format(
int(psutil.virtual_memory().total / 1024 / 1024),
int(psutil.virtual_memory().available / 1024 / 1024),
int(psutil.virtual_memory().used / 1024 / 1024),
))
# Log the current arguments (in yaml because it's cleaner)
print('\nINPUT ARGUMENTS')
print("---------------")
print(yaml.dump(vars(arguments)))
# Display script version info
print("SCRIPT")
print("------")
print("git commit: {}".format(__get_commit(path_to_git_folder=os.path.dirname(script))))
print("git origin: {}".format(__get_git_origin(path_to_git_folder=os.path.dirname(script))))
print("Copying script to output folder...")
try:
# Copy the script and record the new location
script_copy = os.path.abspath(shutil.copy(script, arguments.path_output))
print("{} -> {}".format(script, script_copy))
script = script_copy
except shutil.SameFileError:
print("Input and output folder are the same. Skipping copy.")
pass
except IsADirectoryError:
print("Input folder is a directory (not a file). Skipping copy.")
pass
print("Setting execute permissions for script file {} ...".format(arguments.script))
script_stat = os.stat(script)
os.chmod(script, script_stat.st_mode | S_IEXEC)
# Display data version info
print("\nDATA")
print("----")
print("git commit: {}".format(__get_commit(path_to_git_folder=path_data)))
print("git origin: {}\n".format(__get_git_origin(path_to_git_folder=path_data)))
# Find subjects and process inclusion/exclusions
subject_dirs = [f for f in os.listdir(path_data) if f.startswith(arguments.subject_prefix)]
# Handle inclusion lists
assert not ((arguments.include is not None) and (arguments.include_list is not None)),\
'Only one of `include` and `include-list` can be used'
if arguments.include is not None:
subject_dirs = [f for f in subject_dirs if re.search(arguments.include, f) is not None]
if arguments.include_list is not None:
# TODO decide if we should warn users if one of their inclusions isn't around
subject_dirs = [f for f in subject_dirs if f in arguments.include_list]
# Handle exclusions
assert not ((arguments.exclude is not None) and (arguments.exclude_list is not None)),\
'Only one of `exclude` and `exclude-list` can be used'
if arguments.exclude is not None:
subject_dirs = [f for f in subject_dirs if re.search(arguments.exclude, f) is None]
if arguments.exclude_list is not None:
subject_dirs = [f for f in subject_dirs if f not in arguments.exclude_list]
# Determine the number of jobs we can run simulataneously
if arguments.jobs < 1:
jobs = multiprocessing.cpu_count() + arguments.jobs
else:
jobs = arguments.jobs
print("RUNNING")
print("-------")
print("Processing {} subjects in parallel. (Worker processes used: {}).".format(len(subject_dirs), jobs))
# Run the jobs, recording start and end times
start = datetime.datetime.now()
# Trap errors to send an email if a script fails.
try:
with multiprocessing.Pool(jobs) as p:
run_single_dir = functools.partial(run_single,
script=script,
script_args=arguments.script_args,
path_segmanual=path_segmanual,
path_data=path_data,
path_data_processed=path_data_processed,
path_results=path_results,
path_log=path_log,
path_qc=path_qc,
itk_threads=arguments.itk_threads,
continue_on_error=arguments.continue_on_error)
results = list(p.imap(run_single_dir, subject_dirs))
except Exception as e:
if do_email:
message = ('Oh no there has been the following error in your pipeline:\n\n'
'{}'.format(e))
try:
# I consider the multiprocessing error more significant than a potential email error, this
# ensures that the multiprocessing error is signalled.
send_notification('sct_run_batch errored', message)
except Exception:
raise e
raise e
else:
raise e
end = datetime.datetime.now()
# Check for failed subjects
fails = [sd for (sd, ret) in zip(subject_dirs, results) if ret.returncode != 0]
if len(fails) == 0:
status_message = '\nHooray! your batch completed successfully :-)\n'
else:
status_message = ('\nYour batch completed but some subjects may have not completed '
'successfully, please consult the logs for:\n'
'{}\n'.format('\n'.join(fails)))
print(status_message)
# Display timing
duration = end - start
timing_message = ('Started: {} | Ended: {} | Duration: {}\n'.format(
start.strftime('%Hh%Mm%Ss'),
end.strftime('%Hh%Mm%Ss'),
(datetime.datetime.utcfromtimestamp(0) + duration).strftime('%Hh%Mm%Ss')))
print(timing_message)
if do_email:
send_notification('sct_run_batch: Run completed',
status_message + timing_message)
open_cmd = 'open' if sys.platform == 'darwin' else 'xdg-open'
print('To open the Quality Control (QC) report on a web-browser, run the following:\n'
'{} {}/index.html'.format(open_cmd, path_qc))
if arguments.zip:
file_zip = 'sct_run_batch_{}'.format(time.strftime('%Y%m%d%H%M%S'))
path_tmp = os.path.join(tempfile.mkdtemp(), file_zip)
os.makedirs(os.path.join(path_tmp, file_zip))
for folder in [path_log, path_qc, path_results]:
shutil.copytree(folder, os.path.join(path_tmp, file_zip, os.path.split(folder)[-1]))
shutil.make_archive(os.path.join(path_output, file_zip), 'zip', path_tmp)
shutil.rmtree(path_tmp)
print("\nOutput zip archive: {}.zip".format(os.path.join(path_output, file_zip)))
reset_streams()
batch_log.close()
# Rename the GUI executables.
if project == "GUI":
linuxOutputFile = outputDirectory + "/Nexus-LU-Launcher"
windowsOutputFile = outputDirectory + "/Nexus-LU-Launcher.exe"
if os.path.exists(linuxOutputFile):
os.remove(linuxOutputFile)
elif os.path.exists(windowsOutputFile):
os.remove(windowsOutputFile)
if os.path.exists(outputDirectory + "/NLUL.GUI"):
os.rename(outputDirectory + "/NLUL.GUI", linuxOutputFile)
elif os.path.exists(outputDirectory + "/NLUL.GUI.exe"):
os.rename(outputDirectory + "/NLUL.GUI.exe", windowsOutputFile)
# Create the archive.
shutil.make_archive(
"bin/NLUL-" + project + "-" + platform[0], "zip",
"NLUL." + project + "/bin/Release/" + dotNetVersion + "/" +
platform[1] + "/publish")
# Clear the existing macOS GUI release.
if os.path.exists("bin/NLUL-GUI-macOS-x64.zip"):
print("Clearing the macOS x64 NLUL-GUI release.")
os.remove("bin/NLUL-GUI-macOS-x64.zip")
# Package the macOS release.
print("Packaging macOS release.")
dotNetVersion = os.listdir("NLUL.GUI/bin/Release/")[0]
for macOsBuild in MACOS_PACKAGE_BUILDS:
shutil.copytree(
"NLUL.GUI/bin/Release/" + dotNetVersion + "/" + macOsBuild[1] +
"/publish", "bin/NLUL-GUI-" + macOsBuild[0] +
"/Nexus LU Launcher.app/Contents/MacOS")
def lambda_handler(event, context):
params = None
logger.info('Event %s', event)
OAUTH_token = event['context']['git-token']
OutputBucket = event['context']['output-bucket']
# temp_archive = '/tmp/archive.zip'
# Identify git host flavour
hostflavour = 'generic'
if 'X-Hub-Signature' in event['params']['header'].keys():
hostflavour = 'githubent'
elif 'X-Gitlab-Event' in event['params']['header'].keys():
hostflavour = 'gitlab'
elif 'User-Agent' in event['params']['header'].keys():
if event['params']['header']['User-Agent'].startswith(
'Bitbucket-Webhooks'):
hostflavour = 'bitbucket'
elif event['params']['header']['User-Agent'].startswith(
'GitHub-Hookshot'):
hostflavour = 'github'
elif 'Bitbucket-' in event['params']['header']['User-Agent']:
hostflavour = 'bitbucket-server'
elif event['body-json']['publisherId'] == 'tfs':
hostflavour = 'tfs'
headers = {}
branch = 'master'
if hostflavour == 'githubent':
archive_url = event['body-json']['repository']['archive_url']
owner = event['body-json']['repository']['owner']['name']
name = event['body-json']['repository']['name']
# replace the code archive download and branch reference placeholders
archive_url = archive_url.replace('{archive_format}',
'zipball').replace(
'{/ref}', '/master')
# add access token information to archive url
archive_url = archive_url + '?access_token=' + OAUTH_token
elif hostflavour == 'github':
archive_url = event['body-json']['repository']['archive_url']
owner = event['body-json']['repository']['owner']['login']
name = event['body-json']['repository']['name']
# replace the code archive download and branch reference placeholders
branch_name = event['body-json']['ref'].replace('refs/heads/', '')
archive_url = archive_url.replace('{archive_format}',
'zipball').replace(
'{/ref}', '/' + branch_name)
# add access token information to archive url
archive_url = archive_url + '?access_token=' + OAUTH_token
elif hostflavour == 'gitlab':
#https://gitlab.com/jaymcconnell/gitlab-test-30/repository/archive.zip?ref=master
archive_root = event['body-json']['project']['http_url'].strip('.git')
project_id = event['body-json']['project_id']
branch = event['body-json']['ref'].replace('refs/heads/', '')
archive_url = "https://gitlab.com/api/v4/projects/{}/repository/archive.zip".format(
project_id)
params = {'private_token': OAUTH_token, 'sha': branch}
owner = event['body-json']['project']['namespace']
name = event['body-json']['project']['name']
elif hostflavour == 'bitbucket':
branch = event['body-json']['push']['changes'][0]['new']['name']
archive_url = event['body-json']['repository']['links']['html'][
'href'] + '/get/' + branch + '.zip'
owner = event['body-json']['repository']['owner']['username']
name = event['body-json']['repository']['name']
r = requests.post('https://bitbucket.org/site/oauth2/access_token',
data={'grant_type': 'client_credentials'},
auth=(event['context']['oauth-key'],
event['context']['oauth-secret']))
if 'error' in r.json().keys():
logger.error('Could not get OAuth token. %s: %s' %
(r.json()['error'], r.json()['error_description']))
raise Exception('Failed to get OAuth token')
headers['Authorization'] = 'Bearer ' + r.json()['access_token']
elif hostflavour == 'tfs':
archive_url = event['body-json']['resourceContainers']['account'][
'baseUrl'] + 'DefaultCollection/' + event['body-json'][
'resourceContainers']['project'][
'id'] + '/_apis/git/repositories/' + event['body-json'][
'resource']['repository']['id'] + '/items'
owner = event['body-json']['resource']['pushedBy']['displayName']
name = event['body-json']['resource']['repository']['name']
pat_in_base64 = base64.encodestring(':%s' %
event['context']['git-token'])
headers['Authorization'] = 'Basic %s' % pat_in_base64
headers['Authorization'] = headers['Authorization'].replace('\n', '')
headers['Accept'] = 'application/zip'
elif hostflavour == 'bitbucket-server':
clone_urls = event['body-json']['repository']['links']['clone']
http_clone_url = None
for clone_url in clone_urls:
if clone_url.get('name') == "http":
http_clone_url = clone_url.get('href')
if http_clone_url is None:
raise Exception(
"Could not find http clone url from the webhook payload")
if len(event['body-json']['changes']) != 1:
raise Exception("Could not handle the number of changes")
change = event['body-json']['changes'][0]
url_parts = urlparse(http_clone_url)
owner = event['body-json']['repository']['project']['name']
name = event['body-json']['repository']['name']
archive_url = "{scheme}://{netloc}/rest/api/latest/projects/{project}/repos/{repo}/archive?at={hash}&format=zip".format(
scheme=url_parts.scheme,
netloc=url_parts.netloc
if os.environ.get("SCM_HOSTNAME_OVERRIDE", '') == '' else
os.environ.get("SCM_HOSTNAME_OVERRIDE"),
project=owner,
repo=name,
hash=change['toHash'],
)
branch = change['refId'].replace('refs/heads/', '')
secret = base64.b64encode(":".join(
[event['context']['oauth-key'], event['context']['oauth-secret']]))
headers['Authorization'] = 'Basic ' + secret
# download the code archive via archive url
logger.info('Downloading archive from %s' % archive_url)
r = requests.get(archive_url,
verify=verify,
headers=headers,
params=params)
f = StringIO(r.content)
zip = ZipFile(f)
path = '/tmp/code'
zipped_code = '/tmp/zipped_code'
try:
shutil.rmtree(path)
os.remove(zipped_code + '.zip')
except:
pass
finally:
os.makedirs(path)
# Write to /tmp dir without any common preffixes
zip.extractall(path, get_members(zip))
# Create zip from /tmp dir without any common preffixes
s3_archive_file = "%s/%s/%s/%s.zip" % (owner, name, branch, name)
shutil.make_archive(zipped_code, 'zip', path)
logger.info("Uploading zip to S3://%s/%s" %
(OutputBucket, s3_archive_file))
s3_client.upload_file(zipped_code + '.zip', OutputBucket, s3_archive_file)
logger.info('Upload Complete')
def get_analysis_zip(self):
shutil.make_archive(self.analysis_output_folder, 'zip', self.analysis_output_folder)
return self.analysis_output_folder + ".zip"
required.remove('opencv-python')
# PyQt
# Naming conflict with PyPI package.
# `pyqt` package should be installed via conda instead
# cf. https://github.com/ContinuumIO/anaconda-issues/issues/1554
if PY3 and 'CONDA_PREFIX' in os.environ:
required.remove('pyqt5; python_version >= "3"')
# compress psychojs to a zip file for packaging
# only takes 0.5s but could skip if you prefer
if ('-noJS' in argv) or not exists('psychojs') or ('clean' in argv):
pass
else:
import shutil
shutil.make_archive(join('psychopy', 'psychojs'),
'zip', 'psychojs')
# regenerate __init__.py only if we're in the source repos (not in a source
# zip file)
try:
from packaging import createInitFile # won't exist in a sdist.zip
writeNewInit = True
except ImportError:
writeNewInit = False
if writeNewInit:
# determine what type of dist is being created
# (install and bdist might do compiliing and then build platform is needed)
for arg in argv:
if arg.startswith('bdist') or arg.startswith('install'):
dist = 'bdist'
def get_pictures_zip(self):
shutil.make_archive(self.pictures_output_folder, 'zip', self.pictures_output_folder)
return self.pictures_output_folder + ".zip"
f1.close()
f2 = open(os.getcwd() + "/a2/main.clj.html", "w+")
f2.write(clojureSite)
f2.close()
f3 = open(os.getcwd() + "/a3/Main.scala.html", "w+")
f3.write(scalaSite)
f3.close()
f4 = open(os.getcwd() + "/a4/main.pl.html", "w+")
f4.write(prologSite)
f4.close()
f5 = open(os.getcwd() + "/a5/Main.py.html", "w+")
f5.write(pythonSite)
f5.close()
assignments = parser()
generateHTML(assignments)
email = "*****@*****.**"
shutil.make_archive('csc344', 'zip', os.getcwd())
msg = MIMEMultipart()
attach = open("csc344.zip", "rb")
msg['From'] = email
msg['To'] = email
part = MIMEApplication(attach.read(), Name="csc344.zip")
msg.attach(part)
part.add_header('Content-Disposition', "attachment; filename= " + "csc344.zip")
server = smtplib.SMTP_SSL('smtp.aol.com', 465)
server.login(email, "laxbro4life123")
server.sendmail(email, email, msg.as_string())
server.close()
