def unpack(sproj, branch, outdir): subprocess.check_call(['git', 'clone', '-b', branch, 'https://github.com/mesonbuild/%s.git' % sproj, outdir]) usfile = os.path.join(outdir, 'upstream.wrap') assert(os.path.isfile(usfile)) config = configparser.ConfigParser() config.read(usfile) us_url = config['wrap-file']['source_url'] us = urllib.request.urlopen(us_url).read() h = hashlib.sha256() h.update(us) dig = h.hexdigest() should = config['wrap-file']['source_hash'] if dig != should: print('Incorrect hash on download.') print(' expected:', dig) print(' obtained:', should) return 1 spdir = os.path.split(outdir)[0] ofilename = os.path.join(spdir, config['wrap-file']['source_filename']) with open(ofilename, 'wb') as ofile: ofile.write(us) if 'lead_directory_missing' in config['wrap-file']: os.mkdir(outdir) shutil.unpack_archive(ofilename, outdir) else: shutil.unpack_archive(ofilename, spdir) extdir = os.path.join(spdir, config['wrap-file']['directory']) assert(os.path.isdir(extdir)) shutil.move(os.path.join(outdir, '.git'), extdir) subprocess.check_call(['git', 'reset', '--hard'], cwd=extdir) shutil.rmtree(outdir) shutil.move(extdir, outdir) shutil.rmtree(os.path.join(outdir, '.git')) os.unlink(ofilename)
def openFile(fileName): """ Otevøe soubor fileName vhodným kompresním programem a vrátí otevøený stream. @result {Stream | None} """ fileExtension = fileName[fileName.rfind(".") + 1:].lower() if fileExtension in UNCOMPRESSED_EXTENSIONS: return open(fileName) else: formats = shutil.get_archive_formats() for dict in formats: if fileExtension == dict[0]: tempDir = "tempDir" if os.path.exists(tempDir): shutil.rmtree(tempDir) shutil.unpack_archive(fileName, tempDir) onlyfiles = [ f for f in os.listdir(tempDir) if os.path.isfile(os.path.join(tempDir,f)) ] if onlyfiles == []: return None else: return open(tempDir + '\\' + onlyfiles[0]) pass return None
def restore(self, backup_file, message=str(), delay=0): """ Restores the backup of the world from the given *backup_file*. If the backup archive contains the configuration and the server, they are restored too. Raises: Exception A lot of things can go wrong here, so catch *Exception* if you want to be sure, you catch everything. """ # Extract the backup in a temporary directory and copy then all things # into the EMSM directories. with tempfile.TemporaryDirectory() as temp_dir: shutil.unpack_archive( filename = backup_file, extract_dir = temp_dir ) # Stop the world. was_online = self.world.is_online() if was_online: self.world.send_command("say {}".format(message)) time.sleep(delay) self.world.kill_processes() # Restore the world. manifest = self._get_manifest(temp_dir) self._restore_world(manifest, temp_dir) self._restore_world_configuration(manifest, temp_dir) self._restore_server(manifest, temp_dir) if was_online: self.world.start() return None
def restore(self, backup_file, message=str(), delay=0): """ Restores the backup of the world from the given *backup_file*. If the backup archive contains the server executable it will be restored too if necessairy. Exceptions: * WorldStartFailed * WorldStopFailed * ... shutil.unpack_archive() exceptions ... """ # Extract the backup in a temporary directory and copy then all things # into the EMSM directories. with tempfile.TemporaryDirectory() as temp_dir: shutil.unpack_archive( filename = backup_file, extract_dir = temp_dir ) # Stop the world. was_online = self._world.is_online() if was_online: self._world.send_command("say {}".format(message)) time.sleep(delay) self._world.kill_processes() # Restore the world. self._restore_world(temp_dir) self._restore_world_conf(temp_dir) # Restart the world if it was online before restoring. if was_online: self._world.start() return None
def extract_package(self, package): if sys.version_info < (3, 5): try: import lzma del lzma try: shutil.register_unpack_format('xztar', ['.tar.xz', '.txz'], shutil._unpack_tarfile, [], "xz'ed tar-file") except shutil.RegistryError: pass except ImportError: pass target_dir = os.path.join(self.subdir_root, package.get('directory')) if os.path.isdir(target_dir): return extract_dir = self.subdir_root # Some upstreams ship packages that do not have a leading directory. # Create one for them. try: package.get('lead_directory_missing') os.mkdir(target_dir) extract_dir = target_dir except KeyError: pass shutil.unpack_archive(os.path.join(self.cachedir, package.get('source_filename')), extract_dir) if package.has_patch(): shutil.unpack_archive(os.path.join(self.cachedir, package.get('patch_filename')), self.subdir_root)
def download(self): furlo = FBURLopener({}) try: tmpfile, msg = furlo.retrieve(self.url, reporthook=self.rhook) print() except HTTPError as ex: urlcleanup() sys.exit(ex) except URLError as ex: urlcleanup() sys.exit(ex) if os.path.exists(self.dlpath) and filecmp.cmp(self.dlpath, tmpfile): print('You already have the newest version of ' + self.plugin) done = True else: shutil.copyfile(tmpfile, self.dlpath) print(self.plugin + ' downloaded.') done = False urlcleanup() if done or self.format == 'jar': return try: shutil.unpack_archive(self.dlpath, self.dest_dir, self.format) except ValueError as ex: sys.exit('Error: ' + str(ex))
def restore(self, backup_file, message=str(), delay=0): """ Restores the backup backup_file. Raises: WorldError """ with tempfile.TemporaryDirectory() as temp_dir: shutil.unpack_archive( filename = backup_file, extract_dir = temp_dir ) # Stop the world. was_online = self.world.is_online() if was_online: self.world.send_command("say {}".format(message)) time.sleep(delay) self.world.kill_processes() # Delete the world and restore the backup. for i in range(5): # XXX: Fixes an error with server.log.lck # and shutil.rmtree(...). try: shutil.rmtree(self.world.directory) break except OSError: time.sleep(0.05) shutil.copytree(temp_dir, self.world.directory) if was_online: self.world.start() return None
def get_treebank_perseus_latin_tar(self): """Fetch Persus's Latin treebank files""" orig_files_dir_treebank_perseus_latin = \ os.path.join(self.orig_files_dir, 'treebank_perseus_latin') pg_url = 'https://raw.githubusercontent.com/kylepjohnson/' \ 'treebank_perseus_latin/master/treebank_perseus_latin.tar.gz' session = requests.Session() session.mount(pg_url, SSLAdapter(ssl.PROTOCOL_TLSv1)) pg_tar = session.get(pg_url, stream=True) treebank_perseus_latin_file_name = urlsplit(pg_url).path.split('/')[-1] treebank_perseus_latin_file_path = \ os.path.join(orig_files_dir_treebank_perseus_latin, treebank_perseus_latin_file_name) try: with open(treebank_perseus_latin_file_path, 'wb') as new_file: new_file.write(pg_tar.content) logging.info('Finished writing %s.', treebank_perseus_latin_file_name) except IOError: logging.error('Failed to write file %s', treebank_perseus_latin_file_name) try: shutil.unpack_archive(treebank_perseus_latin_file_path, self.compiled_files_dir) logging.info('Finished unpacking %s', treebank_perseus_latin_file_name) except IOError: logging.info('Failed to unpack %s.', treebank_perseus_latin_file_name)
def get_treebank_perseus_latin_tar(self): """Fetch Perseus's Latin treebank files""" compiled_files_dir_treebank_perseus_latin = os.path.join(self.compiled_files_dir, 'treebank_perseus_latin') if os.path.isdir(compiled_files_dir_treebank_perseus_latin) is True: pass else: os.mkdir(compiled_files_dir_treebank_perseus_latin) logging.info('Made new directory at "%s"', compiled_files_dir_treebank_perseus_latin) orig_files_dir_treebank_perseus_latin = \ os.path.join(self.orig_files_dir, 'treebank_perseus_latin') pg_url = 'https://raw.githubusercontent.com/cltk/latin_treebank_perseus/master/latin_treebank_perseus.tar.gz' session = requests.Session() session.mount(pg_url, SSLAdapter(ssl.PROTOCOL_TLSv1)) pg_tar = session.get(pg_url, stream=True) treebank_perseus_latin_file_name = urlsplit(pg_url).path.split('/')[-1] treebank_perseus_latin_file_path = \ os.path.join(orig_files_dir_treebank_perseus_latin, treebank_perseus_latin_file_name) try: with open(treebank_perseus_latin_file_path, 'wb') as new_file: new_file.write(pg_tar.content) logging.info('Finished writing %s.', treebank_perseus_latin_file_name) except IOError: logging.error('Failed to write file %s', treebank_perseus_latin_file_name) try: shutil.unpack_archive(treebank_perseus_latin_file_path, compiled_files_dir_treebank_perseus_latin) logging.info('Finished unpacking %s', treebank_perseus_latin_file_name) except IOError: logging.info('Failed to unpack %s.', treebank_perseus_latin_file_name)
def get_lacus_curtius_latin_tar(self): """Fetch lacus_curtius_latin_tar""" orig_files_dir_lacus_curtius_latin = \ os.path.join(self.orig_files_dir, 'lacus_curtius_latin') lc_url = 'https://raw.githubusercontent.com/cltk/' \ 'latin_corpus_lacus_curtius/master/lacus_curtius.tar.gz' session = requests.Session() session.mount(lc_url, SSLAdapter(ssl.PROTOCOL_TLSv1)) ll_tar = session.get(lc_url, stream=True) lacus_curtius_latin_file_name = urlsplit(lc_url).path.split('/')[-1] lacus_curtius_latin_file_path = \ os.path.join(orig_files_dir_lacus_curtius_latin, lacus_curtius_latin_file_name) try: with open(lacus_curtius_latin_file_path, 'wb') as new_file: new_file.write(ll_tar.content) logging.info('Finished writing %s.', lacus_curtius_latin_file_name) except IOError: logging.error('Failed to write file %s', lacus_curtius_latin_file_name) try: shutil.unpack_archive(lacus_curtius_latin_file_path, self.compiled_files_dir) logging.info('Finished unpacking %s', lacus_curtius_latin_file_name) except IOError: logging.info('Failed to unpack %s.', lacus_curtius_latin_file_name)
def get_sentence_tokens_greek_tar(self): """Fetch algorithm for Greek sentence tokenization""" orig_files_dir_tokens_greek = \ os.path.join(self.orig_files_dir, 'sentence_tokens_greek') # make compiled files dir for tokens_greek compiled_files_dir_tokens_greek = \ os.path.join(self.compiled_files_dir, 'sentence_tokens_greek') if os.path.isdir(compiled_files_dir_tokens_greek) is True: pass else: os.mkdir(compiled_files_dir_tokens_greek) pg_url = 'https://raw.githubusercontent.com/cltk/' \ 'cltk_greek_sentence_tokenizer/master/greek.tar.gz' session = requests.Session() session.mount(pg_url, SSLAdapter(ssl.PROTOCOL_TLSv1)) pg_tar = session.get(pg_url, stream=True) tokens_greek_file_name = urlsplit(pg_url).path.split('/')[-1] tokens_greek_file_path = os.path.join(orig_files_dir_tokens_greek, tokens_greek_file_name) try: with open(tokens_greek_file_path, 'wb') as new_file: new_file.write(pg_tar.content) logging.info('Finished writing %s.', tokens_greek_file_name) try: shutil.unpack_archive(tokens_greek_file_path, compiled_files_dir_tokens_greek) logging.info('Finished unpacking %s.', tokens_greek_file_name) except IOError: logging.info('Failed to unpack %s.', tokens_greek_file_name) except IOError: logging.error('Failed to write file %s', tokens_greek_file_name)
def get_pos_latin_tar(self): """Fetch Latin part-of-speech files""" orig_files_dir_pos_latin = os.path.join(self.orig_files_dir, 'pos_latin') pg_url = 'https://raw.githubusercontent.com/cltk/pos_latin/' \ 'master/pos_latin.tar.gz' session = requests.Session() session.mount(pg_url, SSLAdapter(ssl.PROTOCOL_TLSv1)) pg_tar = session.get(pg_url, stream=True) pos_latin_file_name = urlsplit(pg_url).path.split('/')[-1] pos_latin_file_path = os.path.join(orig_files_dir_pos_latin, pos_latin_file_name) try: with open(pos_latin_file_path, 'wb') as new_file: new_file.write(pg_tar.content) logging.info('Finished writing %s.', pos_latin_file_name) except IOError: logging.error('Failed to write file %s', pos_latin_file_name) compiled_files_dir_pos_latin = os.path.join(self.compiled_files_dir, 'pos_latin') if os.path.isdir(compiled_files_dir_pos_latin) is True: pass else: os.mkdir(compiled_files_dir_pos_latin) logging.info('Made new directory "pos_latin" at "%s"', compiled_files_dir_pos_latin) try: shutil.unpack_archive(pos_latin_file_path, compiled_files_dir_pos_latin) logging.info('Finished unpacking %s', pos_latin_file_name) except IOError: logging.info('Failed to unpack %s.', pos_latin_file_name)
def read_gtfs(path): """ Given a path (string or pathlib object) to a (zipped) GTFS feed, read the feed and return its corresponding Feed instance. NOTES: - Ignore files that are not valid GTFS; see https://developers.google.com/transit/gtfs/reference/. - Ensure that all ID fields that could be string ('stop_id', 'route_id', etc.) are parsed as strings and not as numbers. """ path = Path(path) # Unzip feed into temporary directory tmp_dir = tempfile.TemporaryDirectory() shutil.unpack_archive(str(path), tmp_dir.name, 'zip') # Read valid GTFS files into Pandas data frames feed_dict = {} dtype = {field: str for field in Feed.str_fields} # ensure some string types for p in Path(tmp_dir.name).iterdir(): name = p.stem if name in Feed.gtfs_tables: feed_dict[name] = pd.read_csv(p, dtype=dtype) # Delete temporary directory tmp_dir.cleanup() return Feed(**feed_dict)
def get_cltk_latin_linguistic_data_tar(self): """Get CLTK's ML taggers, tokenizers, etc.""" orig_files_dir_ling_latin = \ os.path.join(self.orig_files_dir, 'cltk_latin_linguistic_data') latin_dir = os.path.join(self.cltk_data, 'latin') if os.path.isdir(latin_dir) is True: pass else: os.mkdir(latin_dir) latin_dir_ling = os.path.join(latin_dir, 'cltk_linguistic_data') if os.path.isdir(latin_dir_ling) is True: pass else: os.mkdir(latin_dir_ling) pg_url = 'https://raw.githubusercontent.com/cltk/cltk_latin_linguistic_data/master/latin.tar.gz' session = requests.Session() session.mount(pg_url, SSLAdapter(ssl.PROTOCOL_TLSv1)) pg_tar = session.get(pg_url, stream=True) ling_latin_file_name = urlsplit(pg_url).path.split('/')[-1] tar_latin_file_path = os.path.join(orig_files_dir_ling_latin, ling_latin_file_name) try: with open(tar_latin_file_path, 'wb') as new_file: new_file.write(pg_tar.content) logging.info('Finished writing %s.', ling_latin_file_name) try: shutil.unpack_archive(tar_latin_file_path, latin_dir_ling) logging.info('Finished unpacking %s.', ling_latin_file_name) except IOError: logging.info('Failed to unpack %s.', ling_latin_file_name) except IOError: logging.error('Failed to write file %s', ling_latin_file_name)
def get_file_content( params, target_directory ): directory_content = params.get( 'directory_content', [] ) for content in directory_content: target_path = os.path.join( target_directory, content.get( 'subdir', '' ) ) try: os.makedirs( target_path ) except OSError: pass if content.get( 'file_source', {}).get( 'file_source_selector', None ) == 'URL': ( filename, headers ) = urlretrieve( content.get( 'file_source', {}).get( 'file_URL', None ) ) try: bname = headers['Content-Disposition'] except KeyError: bname = os.path.basename( urllib2.urlparse.urlsplit( content.get( 'file_source', {}).get( 'file_URL', None ) ).path ) else: filename = content.get( 'file_source', {}).get( 'file_history', None ) bname = os.path.basename( filename ) file_action = content.get( 'file_action', {}).get( 'file_action_selector', None ) if file_action == 'unpack': unpack_archive( filename, target_path ) else: filename_override = content.get( 'file_action', {}).get( 'filename_override', None ) if filename_override: target_path = os.path.join( target_path, filename_override ) else: target_path = os.path.join( target_path, bname ) shutil.copyfile( filename, target_path ) return len( directory_content )
def telecharger_envoi(self, archive): """Intégration d'une archive au sein d'un dossier """ tmp = cfg.TMP / motaleatoire(6) tmp.mkdir() tmp_archive = tmp / archive.filename archive.save(str(tmp_archive)) try: os.chdir(str(tmp)) shutil.unpack_archive( str(tmp_archive), extract_dir=str(tmp), format='zip', ) tmp_archive.unlink() dossier = [dss for dss in tmp.iterdir()] if len(dossier) != 1: return self.afficher(markdown(_( "L'archive doit contenir un et un seul dossier, " "dans lequel doivent se trouver :\n\n" "- les fichiers du projet ;\n" "- éventuellement, le dossier `.git` contenant " "les données de gestion de version." ))) dossier = dossier[0] shutil.copytree(str(dossier), str(self.dossier)) if not (self.dossier / '.git').is_dir(): self.depot.initialiser() b.redirect(i18n_path('/' + self.chemin)) except shutil.ReadError as err: f.traiter_erreur(err) return self.afficher(_("Ceci n'est pas une archive zip.")) finally: os.chdir(str(cfg.PWD)) shutil.rmtree(str(tmp))
def import_yaml(): ''' import user pre-defined files from ~/xpdUser/Import Files can be compreesed or .yml, once imported, bt.list() should show updated acquire object list ''' src_dir = glbl.import_dir dst_dir = glbl.yaml_dir f_list = os.listdir(src_dir) if len(f_list) == 0: print('INFO: There is no pre-defined user objects in {}'.format(src_dir)) return # two possibilites: .yml or compressed files; shutil should handle all compressed cases moved_f_list = [] for f in f_list: full_path = os.path.join(src_dir, f) (root, ext) = os.path.splitext(f) if ext == '.yml': shutil.copy(full_path, dst_dir) moved_f_list.append(f) # FIXME - do we want user confirmation? os.remove(full_path) else: try: shutil.unpack_archive(full_path, dst_dir) moved_f_list.append(f) # FIXME - do we want user confirmation? os.remove(full_path) except ReadError: print('Unrecongnized file type {} is found inside {}'.format(f, src_dir)) pass return moved_f_list
def fetch_data(): """ What it expects: ---------------- That the DATA_DIR has been created What it does: ------------- Downloads the zip file from the Guardian Saves it to DATA_DIR Unzips it into DATA_DIR What it returns: ---------------- Nothing """ resp = requests.get(SRC_URL) print("Downloading", SRC_URL) # save the zip file to disk with open(ZIPPED_DATA_FILENAME, 'wb') as f: f.write(resp.content) # unzip the zip file print("Unzipping", ZIPPED_DATA_FILENAME) unpack_archive(ZIPPED_DATA_FILENAME, extract_dir=DATA_DIR)
def test_unpack_registery(self): formats = get_unpack_formats() def _boo(filename, extract_dir, extra): self.assertEquals(extra, 1) self.assertEquals(filename, 'stuff.boo') self.assertEquals(extract_dir, 'xx') register_unpack_format('Boo', ['.boo', '.b2'], _boo, [('extra', 1)]) unpack_archive('stuff.boo', 'xx') # trying to register a .boo unpacker again self.assertRaises(RegistryError, register_unpack_format, 'Boo2', ['.boo'], _boo) # should work now unregister_unpack_format('Boo') register_unpack_format('Boo2', ['.boo'], _boo) self.assertIn(('Boo2', ['.boo'], ''), get_unpack_formats()) self.assertNotIn(('Boo', ['.boo'], ''), get_unpack_formats()) # let's leave a clean state unregister_unpack_format('Boo2') self.assertEquals(get_unpack_formats(), formats)
def aurbuild(package_name): os.chdir(BUILDFOLDER) # Get the filename from the URL. url = "https://aur.archlinux.org/cgit/aur.git/snapshot/{0}.tar.gz".format(package_name) fileinfo = urllib.parse.urlparse(url) filename = urllib.parse.unquote(os.path.basename(fileinfo.path)) # Get the file. urllib.request.urlretrieve(url, filename) # Extract the file. shutil.unpack_archive(filename, BUILDFOLDER) AURFOLDER=BUILDFOLDER+"/"+package_name subprocess.run(""" chmod a+rwx -R {0} cd {0} su {2} -s /bin/bash -c 'makepkg --noconfirm -A -s' pacman -U --noconfirm ./{1}-*.pkg.tar.xz """.format(AURFOLDER, package_name, USERNAMEVAR), shell=True) # Cleanup os.chdir(BUILDFOLDER) if os.path.isdir(AURFOLDER): shutil.rmtree(AURFOLDER) os.remove(BUILDFOLDER+"/"+filename) return
def setUpClass(cls): super(TestImporterAFMAsc, cls).setUpClass() testdata = os.path.join(os.path.dirname(__file__), '..', '..', 'testdata') filepath = os.path.join(testdata, 'afm.asc.zip') cls.tmpdir = tempfile.mkdtemp() shutil.unpack_archive(filepath, cls.tmpdir, 'zip')
def apply_patch(self): path = self.get_file_internal('patch') try: shutil.unpack_archive(path, self.subdir_root) except Exception: with tempfile.TemporaryDirectory() as workdir: shutil.unpack_archive(path, workdir) self.copy_tree(workdir, self.subdir_root)
def main(): print('Downloading', SOURCE_URL) resp = requests.get(SOURCE_URL) with open(DATA_ZIP_PATH, 'wb') as zip_file: zip_file.write(resp.content) unpack_archive(DATA_ZIP_PATH, extract_dir=GENDER_DIR) baby_filenames = glob(path.join(GENDER_DIR, '*.txt')) print('There are {} txt files'.format(len(baby_filenames)))
def process_topography(in_path, out_path, high_definition=False): """ Convert each SRTM HGT topography file in the directory ``in_path`` to a SPLAT! Data File (SDF) file in the directory ``out_path``, creating the directory if it does not exist. If ``high_definition``, then assume the input data is high definition. INPUT: - ``in_path``: string or Path object specifying a directory - ``out_path``: string or Path object specifying a directory - ``high_definition``: boolean OUTPUT: None. NOTES: - Calls SPLAT!'s ``srtm2sdf`` or ``srtm2sdf-hd`` (if ``high_definition``) command to do the work - Raises a ``subprocess.CalledProcessError`` if SPLAT! fails to convert a file - Each SRTM1 or SRTM3 file must have a name of the form <SRTM tile ID>[.something].hgt.zip or <SRTM tile ID>[.something].hgt, e.g. S36E173.SRTMGL3.hgt.zip """ in_path = Path(in_path) out_path = Path(out_path) if not out_path.exists(): out_path.mkdir(parents=True) splat = 'srtm2sdf' if high_definition: splat += '-hd' sdf_pattern = re.compile(r"[\d\w\-\:]+\.sdf") for f in in_path.iterdir(): if not (f.name.endswith('.hgt') or f.name.endswith('.hgt.zip')): continue # Unzip if necessary is_zip = False if f.name.endswith('.zip'): is_zip = True shutil.unpack_archive(str(f), str(f.parent)) tile_id = f.name.split('.')[0] f = f.parent/'{!s}.hgt'.format(tile_id) # Convert to SDF cp = subprocess.run([splat, f.name], cwd=str(f.parent), stdout=subprocess.PIPE, universal_newlines=True, check=True) # Get name of output file, which SPLAT! created and which differs # from the original name, and move the output to the out path m = sdf_pattern.search(cp.stdout) name = m.group(0) src = in_path/name tgt = out_path/name shutil.move(str(src), str(tgt)) # Clean up if is_zip: f.unlink()
def pre_benchmark_run(self) -> None: """ When benchmarking, it is better to have a bigger workload. We use transmission for this purpose """ shutil.unpack_archive( os.path.join(get_global_conf().get("install", "source_directory"), "cppcheck-152/cppcheck-1.52.tar.gz"), "/tmp/cppcheck-152" ) self.cmd = " ".join(self.cmd.split(" ")[:-1]) + " /tmp/cppcheck-152/cppcheck-1.52"
def install_s3_object(s3_bucket, filename, destination): s3 = boto3.resource('s3') with tempfile.TemporaryDirectory() as tmpdir: saved_file = os.path.join(tmpdir, filename) s3.Object(s3_bucket, filename).download_file(saved_file) if filename.endswith('.tar'): shutil.unpack_archive(saved_file, destination) else: shutil.copy(saved_file, destination)
def pre_benchmark_run(self) -> None: """ For benchmarking, we need to work on bigger files. We use transmission for this purpose """ shutil.unpack_archive( os.path.join(get_global_conf().get("install", "source_directory"), "cppcheck-148/cppcheck-1.48.tar.gz"), "/tmp/cppcheck-148" ) self.cmd = " ".join(self.cmd.split(" ")[:-1]) + " /tmp/cppcheck-148/cppcheck-1.48"
def untar(path, fname, deleteTar=True): """Unpacks the given archive file to the same directory, then (by default) deletes the archive file. """ print('unpacking ' + fname) fullpath = os.path.join(path, fname) shutil.unpack_archive(fullpath, path) if deleteTar: os.remove(fullpath)
def test_shallow_clone(): with TemporaryDirectory() as tmpdir: archive_filename = resource_filename(__name__, 'data/test_repository.tar') unpack_archive(archive_filename, tmpdir) with shallow_clone(os.path.join(tmpdir, 'test_repository')) as working_copy_path: assert 'known_file' in os.listdir(working_copy_path) with open(os.path.join(working_copy_path, 'known_file')) as known_file: assert known_file.read() == 'Hello, World!\n'
def unpackFile( downloaddir, filename, workdir ): """unpack file specified by 'filename' from 'downloaddir' into 'workdir'""" EmergeDebug.debug("unpacking this file: %s" % filename) ( shortname, ext ) = os.path.splitext( filename ) if re.match( "(.*\.tar.*$|.*\.tgz$)", filename ): shutil.unpack_archive(os.path.join(downloaddir, filename),workdir) elif ext == "": return True return un7zip( os.path.join( downloaddir, filename ), workdir, ext )
#!/usr/bin/python3 """ Just a routine practice: unzip a .zip file. """ import shutil output_dir = '/tmp/os_logs' zip_filename = 'os_logs.zip' shutil.unpack_archive(zip_filename, output_dir, 'zip')
def _setupSSHDImpl(public_key, tunnel, ngrok_token, ngrok_region, is_VNC): #apt-get update #apt-get upgrade my_apt = _MyApt() #Following packages are useless because nvidia kernel modules are already loaded and I cannot remove or update it. #Uninstall them because upgrading them take long time. my_apt.deleteInstalledPkg("nvidia-dkms", "nvidia-kernel-common", "nvidia-kernel-source") my_apt.commit() my_apt.update_upgrade() my_apt.commit() subprocess.run(["unminimize"], input="y\n", check=True, universal_newlines=True) my_apt.installPkg("openssh-server") my_apt.commit() my_apt.close() #Reset host keys for i in pathlib.Path("/etc/ssh").glob("ssh_host_*_key"): i.unlink() subprocess.run(["ssh-keygen", "-A"], check=True) #Prevent ssh session disconnection. with open("/etc/ssh/sshd_config", "a") as f: f.write("\n\n# Options added by remocolab\n") f.write("ClientAliveInterval 120\n") if public_key != None: f.write("PasswordAuthentication no\n") msg = "" msg += "ECDSA key fingerprint of host:\n" ret = subprocess.run( ["ssh-keygen", "-lvf", "/etc/ssh/ssh_host_ecdsa_key.pub"], stdout=subprocess.PIPE, check=True, universal_newlines=True) msg += ret.stdout + "\n" root_password = secrets.token_urlsafe() user_password = secrets.token_urlsafe() user_name = "colab" msg += "✂️" * 24 + "\n" msg += f"root password: {root_password}\n" msg += f"{user_name} password: {user_password}\n" msg += "✂️" * 24 + "\n" subprocess.run(["useradd", "-s", "/bin/bash", "-m", user_name]) subprocess.run(["adduser", user_name, "sudo"], check=True) subprocess.run(["chpasswd"], input=f"root:{root_password}", universal_newlines=True) subprocess.run(["chpasswd"], input=f"{user_name}:{user_password}", universal_newlines=True) subprocess.run(["service", "ssh", "restart"]) _set_public_key(user_name, public_key) ssh_common_options = "-o UserKnownHostsFile=/dev/null -o VisualHostKey=yes" if tunnel == "ngrok": pyngrok_config = pyngrok.conf.PyngrokConfig(auth_token=ngrok_token, region=ngrok_region) url = pyngrok.ngrok.connect(port=22, proto="tcp", pyngrok_config=pyngrok_config) m = re.match("tcp://(.+):(\d+)", url) hostname = m.group(1) port = m.group(2) ssh_common_options += f" -p {port}" elif tunnel == "argotunnel": _download( "https://bin.equinox.io/c/VdrWdbjqyF/cloudflared-stable-linux-amd64.deb", "cloudflared.deb") my_apt.installDebPackage("cloudflared.deb") print('DEB LINUX') shutil.unpack_archive("cloudflared.deb") cfd_proc = subprocess.Popen([ "./cloudflared", "tunnel", "--url", "ssh://*****:*****@{hostname}\n" else: msg += "Command to connect to the ssh server:\n" msg += "✂️" * 24 + "\n" msg += f"ssh {ssh_common_options} {user_name}@{hostname}\n" msg += "✂️" * 24 + "\n" return msg
print('') print('###############|| Post processing ||###############') print('') for top in topologies: model_name = top['name'] + get_extensions(top['framework'])[0] weights_name = top['name'] + get_extensions(top['framework'])[1] output = os.path.join(args.output_dir, top['output']) path_to_model = os.path.join(output, model_name) path_to_weights = os.path.join(output, weights_name) for path, subdirs, files in os.walk(output): for name in files: fname = os.path.join(path, name) if fname.endswith('.tar.gz'): print('========= Extracting files from %s.tar.gz' % (top['name'])) shutil.unpack_archive(fname, path) if {'model_path_prefix', 'weights_path_prefix'} <= top.keys(): downloaded_model = os.path.join(output, top['model_path_prefix']) downloaded_weights = os.path.join(output, top['weights_path_prefix']) if (os.path.exists(downloaded_model) and os.path.exists(downloaded_weights)): print( '========= Moving %s and %s to %s after untarring the archive =========' % (model_name, weights_name, output)) shutil.move(downloaded_model, path_to_model) shutil.move(downloaded_weights, path_to_weights) elif 'model_path_prefix' in top: downloaded_model = os.path.join(output, top['model_path_prefix']) if os.path.exists(downloaded_model): print( '========= Moving %s to %s after untarring the archive ========='
from pathlib import Path import src.utils repo_dir = Path(__file__).parent.resolve() assert repo_dir.name == 'hicetnunc-dataset' cache_dir = repo_dir / 'cache' export_cache_dir = repo_dir / 'export_cache' export_cache_archive_file = repo_dir / 'cache.zip' export_dataset_archive_file = repo_dir / 'dataset.zip' if not (cache_dir / 'ipfs0').exists(): if export_cache_archive_file.exists(): print('Extracting cache.zip...', end=' ') sys.stdout.flush() shutil.unpack_archive(export_cache_archive_file, cache_dir, 'zip') print('done\n') else: print('You should download cache.zip and put into repository root') sys.exit() ipfs0_dir = cache_dir / 'ipfs0' ipfs0_dir.mkdir(exist_ok=True) ipfs1_dir = cache_dir / 'ipfs1' ipfs1_dir.mkdir(exist_ok=True) previews_dir = cache_dir / 'previews' previews_dir.mkdir(exist_ok=True) previews_dimensions_cache_file = previews_dir / 'dimensions.json' parsed_transactions_dir = cache_dir / 'parsed_transactions' parsed_transactions_dir.mkdir(exist_ok=True)
for url in urls: name = url[-14:-7] os.chdir(path) dir = os.path.join(f"{name}") if not os.path.exists(dir): os.mkdir(dir) urllib.request.urlretrieve(url, path + f"/{name}/{name}.v4.tar") os.chdir(path + f"/{name}") shutil.unpack_archive(f"{name}.v4.tar", extract_dir=path + f"/{name}") print("Files successfully downloaded") ##################################### # Extract the files os.chdir(path) folders = [folder for folder in os.listdir() if "F1" in folder] dir = os.path.join("Images") if not os.path.exists(dir): os.mkdir(dir)
def unzip_batch(batch, ligand_dir): targz = str(ligand_dir / batch) + ".tar.gz" output_dir = str(ligand_dir) shutil.unpack_archive(targz, output_dir, format='gztar')
try: open(filepath, "r").read() except UnicodeDecodeError: stream = open(filepath, "rb").read() if len(stream) > 4: if stream[0:4] == b"PK\x03\x04": print( "[ddcrypt] file: zip archive data [sig:PK\\x03\\x04]" ) filenoformat = filename if filename.endswith(".zip") or filename.endswith( ".apk"): filenoformat = filename[0:len(filename) - 4] destdir = destdir + filenoformat + "/" print("[ddcrypt] extracting raw resources...") shutil.unpack_archive(filepath, destdir, "zip") strings = "" try: androidmanifest = open( destdir + "AndroidManifest.xml", "r").read() strings = androidmanifest userdir = destdir + "assets/user" if os.path.isdir(userdir): isfilejs = False for f in os.listdir(userdir): if f.lower().endswith(".js"): isfilejs = True break if not isfilejs: print(
def trigger_training_job(): # Define Vars < Change the vars>. # In a production situation, don't put secrets in source code, but as secret variables, # see https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch#secret-variables workspace="<Name of your workspace>" subscription_id="<Subscription id>" resource_grp="<Name of your resource group where aml service is created>" domain = "westeurope.azuredatabricks.net" # change location in case databricks instance is not in westeurope DBR_PAT_TOKEN = bytes("<<your Databricks Personal Access Token>>", encoding='utf-8') # adding b' notebookRemote = "/3_IncomeNotebookDevops" experiment_name = "experiment_model_release" model_name_run = datetime.datetime.now().strftime("%Y%m%d%H%M%S")+ "_dbrmod.mml" # in case you want to change the name, keep the .mml extension model_name = "databricksmodel.mml" # in case you want to change the name, keep the .mml extension # # Step 1: Create job and attach it to cluster # # In this steps, secret are added as parameters (spn_tenant, spn_clientid, spn_clientsecret) # Never do this in a production situation, but use secret scope backed by key vault instead # See https://docs.azuredatabricks.net/user-guide/secrets/secret-scopes.html#azure-key-vault-backed-scopes response = requests.post( 'https://%s/api/2.0/jobs/create' % domain, headers={'Authorization': b"Bearer " + DBR_PAT_TOKEN}, json={ "name": "Run AzureDevopsNotebook Job", "new_cluster": { "spark_version": "4.0.x-scala2.11", "node_type_id": "Standard_D3_v2", "spark_env_vars": { 'PYSPARK_PYTHON': '/databricks/python3/bin/python3', }, "autoscale": { "min_workers": 1, "max_workers": 2 } }, "libraries": [ { "pypi": { "package": "azureml-sdk[databricks]" } } ], "notebook_task": { "notebook_path": notebookRemote, "base_parameters": [{"key":"subscription_id", "value":subscription_id}, {"key":"resource_group", "value":resource_grp}, {"key":"workspace_name","value":workspace}, {"key":"model_name", "value":model_name_run} ] } } ) if response.status_code != 200: print("Error launching cluster: %s: %s" % (response.json()["error_code"], response.json()["message"])) exit(2) # # Step 2: Start job # databricks_job_id = response.json()['job_id'] response = requests.post( 'https://%s/api/2.0/jobs/run-now' % domain, headers={'Authorization': b"Bearer " + DBR_PAT_TOKEN}, json={ "job_id": + databricks_job_id } ) if response.status_code != 200: print("Error launching cluster: %s: %s" % (response.json()["error_code"], response.json()["message"])) exit(3) print(response.json()['run_id']) # # Step 3: Wait until job is finished # databricks_run_id = response.json()['run_id'] scriptRun = 1 count = 0 while scriptRun == 1: response = requests.get( 'https://%s/api/2.0/jobs/runs/get?run_id=%s' % (domain, databricks_run_id), headers={'Authorization': b"Bearer " + DBR_PAT_TOKEN}, ) state = response.json()['state'] life_cycle_state = state['life_cycle_state'] print(state) if life_cycle_state in ["TERMINATED", "SKIPPED", "INTERNAL_ERROR"]: result_state = state['result_state'] if result_state == "SUCCESS": print("run ok") scriptRun = 0 #exit(0) else: exit(4) elif count > 180: print("time out occurred after 30 minutes") exit(5) else: count += 1 time.sleep(30) # wait 30 seconds before next status update # # Step 4: Retrieve model from dbfs # mdl, ext = model_name_run.split(".") model_zip_run = mdl + ".zip" response = requests.get( 'https://%s/api/2.0/dbfs/read?path=/%s' % (domain, model_zip_run), headers={'Authorization': b"Bearer " + DBR_PAT_TOKEN} ) if response.status_code != 200: print("Error copying dbfs results: %s: %s" % (response.json()["error_code"], response.json()["message"])) exit(6) model_output = base64.b64decode(response.json()['data']) # download model in deploy folder os.chdir("deploy") with open(model_zip_run, "wb") as outfile: outfile.write(model_output) print("Downloaded model {} to Project root directory".format(model_name)) # # Step 5: Put model to Azure ML Service # cli_auth = AzureCliAuthentication() ws = Workspace(workspace_name = workspace, subscription_id = subscription_id, resource_group = resource_grp, auth=cli_auth) ws.get_details() # start a training run by defining an experiment myexperiment = Experiment(ws, experiment_name) run = myexperiment.start_logging() run.upload_file("outputs/" + model_zip_run, model_zip_run) run.complete() run_id = run.id print ("run id:", run_id) # unzip file to model_name_run shutil.unpack_archive(model_zip_run, model_name_run) model = Model.register( model_path=model_name_run, # this points to a local file model_name=model_name, # this is the name the model is registered as tags={"area": "spar", "type": "regression", "run_id": run_id}, description="Medium blog test model", workspace=ws, ) print("Model registered: {} \nModel Description: {} \nModel Version: {}".format(model.name, model.description, model.version)) # Step 6. Finally, writing the registered model details to conf/model.json model_json = {} model_json["model_name"] = model.name model_json["model_version"] = model.version model_json["run_id"] = run_id model_json["model_name_run"] = model_name_run with open("../conf/model.json", "w") as outfile: json.dump(model_json, outfile)
def add_node_protocol(self): ''' Add the node to the network ''' timeout = False print("Add Node protocol started !") print("Asking Leader Information from Standard IP's ") #send message "add_node" to standard IP's send_msg_add_node(self.config_table, self.PORT) with self.AN_condition: timeout = self.AN_condition.wait(timeout = self.add_node_timeout) # blocking - Waiting for reply from sponsor node if timeout is False: print("No reply from any node... Sending request again") send_msg_add_node(self.config_table, self.PORT) with self.AN_condition: timeout = self.AN_condition.wait(timeout = self.add_node_timeout) # blocking - Waiting for reply from sponsor node if timeout is False: print("Still no response. Exiting...") os._exit(0) message = self.thread_msg_qs[self.main_thread_tid].get() self.sponsor_host = message._source_host self.sponsor_port = message.get_data('port') # Port on which sponsor server is listening self.ldr_ip = message.get_data('ldr_ip') self.ldr_port = message.get_data('ldr_port') self.network_dict = message.get_data('network_dict') self.ldr_id = message.get_data('id') self.network_dict[message._msg_id[0]]=(self.sponsor_host,self.sponsor_port,1) self.pause_heartbeat = False print("network_dict: ",self.network_dict) print("Sponsor_host is : ",self.sponsor_host," Sponsor port is : ",self.sponsor_port) print("Message recieved: Leader ip :", self.ldr_ip, "Leader port :",self.ldr_port) try: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.connect((self.ldr_ip,self.ldr_port)) send_msg(s, Message(Msg_type['AN_assign_id'], data_dict = {'port': self.PORT })) # Contacting leader to give a new id except: exception_handler(self.ldr_ip,self.ldr_port) with self.AN_condition: timeout = self.AN_condition.wait(timeout = self.add_node_timeout) # blocking - Waiting for leader to assign ID if timeout is False: print("Leader not responding. Exiting...") os._exit(0) message = self.thread_msg_qs[self.main_thread_tid].get() self.node_id = message.get_data('id') print("Message recieved: New id assigned :", self.node_id) #Get file system from sponsored node try: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.connect((self.sponsor_host, self.sponsor_port)) send_msg(s, Message(Msg_type['AN_FS_data_req'], data_dict = {'port': self.PORT })) s.close() except: print("Not able to connect to sponsor node not responding. Exiting...") os._exit(0) with self.AN_condition: timeout = self.AN_condition.wait(timeout = 5*self.add_node_timeout) if timeout is False: print("Sponsor node not responding. Exiting...") os._exit(0) message = self.thread_msg_qs[self.main_thread_tid].get() self.file_system_name = message.get_data('name') file_system_size = message.get_data('size') self.meta_data = message.get_data('meta_data') print("File system name is ",self.file_system_name," size is ",file_system_size) file_pointer = open(self.file_system_name+".zip",'wb') current_size = 0 chunk_no = 0 try: while True: # with self.AN_condition: # self.AN_condition.wait() if self.thread_msg_qs[self.main_thread_tid].empty() is False: message = self.thread_msg_qs[self.main_thread_tid].get() current_size+=file_pointer.write(message.get_data('data')) # current_size+= sys.getsizeof(message.get_data('data')) print("Size of chunk_no ",chunk_no," is ",sys.getsizeof(message.get_data('data'))) print("File size transferred ",current_size) if current_size >= file_system_size: break except: print("Connection broken. File System transfer failed. Exiting...") os._exit(0) file_pointer.close() if os.path.exists("./"+self.file_system_name): shutil.rmtree("./"+self.file_system_name) os.makedirs("./"+self.file_system_name) shutil.unpack_archive("./"+self.file_system_name + ".zip",'./'+self.file_system_name) # zipfilePath = ("./"+self.file_system_name + ".zip") # zip1 = zipfile.ZipFile(zipfilePath) # zip1.extractall(".") # zip1.close() # self.inputs.remove(self.file_system_port) # self.file_system_port.close() self.add_node = True print("Node added successfully !") print("DEUBG_MSG: metadata",self.meta_data)
async def extractor(download_path): try: shutil.unpack_archive(str(download_path), str(directory_path)) except Exception as error: shutil.rmtree(directory_path, onerror=on_error) raise DirectoryNotExtractedError(directory_path) from error
def convert_contribution_to_pack(self): """Create a Pack in the content repo from the contents of a contribution zipfile""" try: packs_dir = os.path.join(get_content_path(), 'Packs') metadata_dict = {} with zipfile.ZipFile(self.contribution) as zipped_contrib: with zipped_contrib.open('metadata.json') as metadata_file: click.echo( f'Pulling relevant information from {metadata_file.name}', color=LOG_COLORS.NATIVE) metadata = json.loads(metadata_file.read()) # a name passed on the cmd line should take precedence over one pulled # from contribution metadata pack_display_name = self.name or metadata.get( 'name', 'ContributionPack') # Strip 'Pack' suffix from pack display name if present pack_display_name = pack_display_name.strip() if pack_display_name.casefold().endswith('pack') > len( pack_display_name) > 4: stripped_pack_display_name = pack_display_name[: -4].strip( ) pack_display_name = stripped_pack_display_name or pack_display_name pack_name = self.dir_name or self.format_pack_dir_name( metadata.get('name', 'ContributionPack')) # a description passed on the cmd line should take precedence over one pulled # from contribution metadata metadata_dict[ 'description'] = self.description or metadata.get( 'description') metadata_dict['name'] = pack_display_name metadata_dict['author'] = self.author or metadata.get( 'author', '') metadata_dict['support'] = 'community' metadata_dict['url'] = metadata.get( 'supportDetails', {}).get('url', MARKETPLACE_LIVE_DISCUSSIONS) metadata_dict['categories'] = metadata.get( 'categories') if metadata.get('categories') else [] metadata_dict['tags'] = metadata.get( 'tags') if metadata.get('tags') else [] metadata_dict['useCases'] = metadata.get( 'useCases') if metadata.get('useCases') else [] metadata_dict['keywords'] = metadata.get( 'keywords') if metadata.get('keywords') else [] while os.path.exists(os.path.join(packs_dir, pack_name)): click.echo( f'Modifying pack name because pack {pack_name} already exists in the content repo', color=LOG_COLORS.NATIVE) if len(pack_name) >= 2 and pack_name[-2].lower( ) == 'v' and pack_name[-1].isdigit(): # increment by one pack_name = pack_name[:-1] + str(int(pack_name[-1]) + 1) else: pack_name += 'V2' click.echo(f'New pack name is "{pack_name}"', color=LOG_COLORS.NATIVE) pack_dir = os.path.join(packs_dir, pack_name) os.mkdir(pack_dir) shutil.unpack_archive(filename=self.contribution, extract_dir=pack_dir) pack_subdirectories = get_child_directories(pack_dir) for pack_subdir in pack_subdirectories: basename = os.path.basename(pack_subdir) if basename in ENTITY_TYPE_TO_DIR: dst_name = ENTITY_TYPE_TO_DIR.get(basename) src_path = os.path.join(pack_dir, basename) dst_path = os.path.join(pack_dir, dst_name) if os.path.exists(dst_path): # move src folder files to dst folder content_item_dir = dst_path for _, _, files in os.walk(src_path, topdown=False): for name in files: src_file_path = os.path.join(src_path, name) shutil.move(src_file_path, dst_path) else: # replace dst folder with src folder content_item_dir = shutil.move(src_path, dst_path) if basename in {SCRIPT, AUTOMATION, INTEGRATION}: self.content_item_to_package_format(content_item_dir, del_unified=True) # create pack's base files self.full_output_path = pack_dir self.create_pack_base_files() metadata_dict = Initiator.create_metadata(fill_manually=False, data=metadata_dict) metadata_path = os.path.join(self.full_output_path, 'pack_metadata.json') with open(metadata_path, 'w') as pack_metadata_file: json.dump(metadata_dict, pack_metadata_file, indent=4) # remove metadata.json file os.remove(os.path.join(pack_dir, 'metadata.json')) click.echo( f'Executing \'format\' on the restructured contribution zip files at "{pack_dir}"' ) from_version = '6.0.0' format_manager(input=pack_dir, from_version=from_version, no_validate=True, update_docker=True, assume_yes=True) except Exception as e: click.echo( f'Creating a Pack from the contribution zip failed with error: {e}\n {traceback.format_exc()}', color=LOG_COLORS.RED) finally: if self.contrib_conversion_errs: click.echo( 'The following errors occurred while converting unified content YAMLs to package structure:' ) click.echo( textwrap.indent('\n'.join(self.contrib_conversion_errs), '\t'))
def load_experiment(args): '''load experiment data''' package_path = os.path.expanduser(args.path) if not os.path.exists(args.path): print_error('file path %s does not exist!' % args.path) exit(1) if args.searchSpacePath and os.path.isdir(args.searchSpacePath): print_error( 'search space path should be a full path with filename, not a directory!' ) exit(1) temp_root_dir = generate_temp_dir() shutil.unpack_archive(package_path, temp_root_dir) print_normal('Loading...') # Step1. Validation if not os.path.exists(args.codeDir): print_error('Invalid: codeDir path does not exist!') exit(1) if args.logDir: if not os.path.exists(args.logDir): print_error('Invalid: logDir path does not exist!') exit(1) experiment_temp_dir = os.path.join(temp_root_dir, 'experiment') if not os.path.exists(os.path.join(experiment_temp_dir, 'db')): print_error('Invalid archive file: db file does not exist!') shutil.rmtree(temp_root_dir) exit(1) nnictl_temp_dir = os.path.join(temp_root_dir, 'nnictl') if not os.path.exists(os.path.join(nnictl_temp_dir, '.experiment')): print_error( 'Invalid archive file: nnictl metadata file does not exist!') shutil.rmtree(temp_root_dir) exit(1) try: with open(os.path.join(nnictl_temp_dir, '.experiment'), 'r') as file: experiment_metadata = json.load(file) except ValueError as err: print_error('Invalid nnictl metadata file: %s' % err) shutil.rmtree(temp_root_dir) exit(1) experiments_config = Experiments() experiments_dict = experiments_config.get_all_experiments() experiment_id = experiment_metadata.get('id') if experiment_id in experiments_dict: print_error('Invalid: experiment id already exist!') shutil.rmtree(temp_root_dir) exit(1) if not os.path.exists(os.path.join(nnictl_temp_dir, experiment_id)): print_error('Invalid: experiment metadata does not exist!') shutil.rmtree(temp_root_dir) exit(1) # Step2. Copy nnictl metadata src_path = os.path.join(nnictl_temp_dir, experiment_id) dest_path = os.path.join(NNI_HOME_DIR, experiment_id) if os.path.exists(dest_path): shutil.rmtree(dest_path) shutil.copytree(src_path, dest_path) # Step3. Copy experiment data os.rename(os.path.join(temp_root_dir, 'experiment'), os.path.join(temp_root_dir, experiment_id)) src_path = os.path.join(os.path.join(temp_root_dir, experiment_id)) experiment_config = Config(experiment_id, temp_root_dir).get_config() if args.logDir: logDir = args.logDir experiment_config['logDir'] = logDir else: if experiment_config.get('logDir'): logDir = experiment_config['logDir'] else: logDir = NNI_HOME_DIR dest_path = os.path.join(logDir, experiment_id) if os.path.exists(dest_path): shutil.rmtree(dest_path) shutil.copytree(src_path, dest_path) # Step4. Copy code dir codeDir = os.path.expanduser(args.codeDir) if not os.path.isabs(codeDir): codeDir = os.path.join(os.getcwd(), codeDir) print_normal('Expand codeDir to %s' % codeDir) experiment_config['trial']['codeDir'] = codeDir archive_code_dir = os.path.join(temp_root_dir, 'code') if os.path.exists(archive_code_dir): file_list = os.listdir(archive_code_dir) for file_name in file_list: src_path = os.path.join(archive_code_dir, file_name) target_path = os.path.join(codeDir, file_name) if os.path.exists(target_path): print_error('Copy %s failed, %s exist!' % (file_name, target_path)) continue if os.path.isdir(src_path): shutil.copytree(src_path, target_path) else: shutil.copy(src_path, target_path) # Step5. Create experiment metadata experiments_config.add_experiment( experiment_id, experiment_metadata.get('port'), experiment_metadata.get('startTime'), experiment_metadata.get('platform'), experiment_metadata.get('experimentName'), experiment_metadata.get('endTime'), experiment_metadata.get('status'), experiment_metadata.get('tag'), experiment_metadata.get('pid'), experiment_metadata.get('webUrl'), logDir) print_normal('Load experiment %s succsss!' % experiment_id) # Step6. Cleanup temp data shutil.rmtree(temp_root_dir)
def install(self): if not self.path.is_dir(): github_runner = self.get_runner_app() self.path.parent.mkdir(exist_ok=True, parents=True) shutil.unpack_archive(str(github_runner.absolute()), str(self.path.absolute()))
shutil.copystat() # copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2, # ignore_dangling_symlinks=False) 复制一个文件夹及其内容到另一个文件夹下,ignore过滤条件,可以多个 # 注意,必须要指定一个文件夹,即把a文件夹的内容,复制到父目录的b里面(b一定不能事先存在) shutil.copytree('a', '../b', ignore=shutil.ignore_patterns('*.bat', '*.py')) # 获取一个路径的磁盘占用,返回一个三元组(total,used,free) print(shutil.disk_usage('D:\\')) # 显示支持的打包格式/解压包的格式 print(shutil.get_archive_formats()) print(shutil.get_unpack_formats()) # 打包文件 (包名,格式,要打包的路径) shutil.make_archive('a_zip', 'zip', 'a') # 移动文件或文件夹到另一路径: (源文件路径,目标路径) shutil.move('a', '../') # 注册一个新的打包方式? shutil.register_archive_format() shutil.register_unpack_format() # 删除文件夹及其内容,选择是否忽略错误(默认false,没有要删除的文件夹会报错) shutil.rmtree('a', ignore_errors=True) # (文件名,[,解压路径,解压方式]) unpack_archive(filename, extract_dir=None, format=None) shutil.unpack_archive('a.zip', 'a') # shutil.unregister_archive_format() # shutil.unregister_unpack_format()
def unzip_file(a, b): shutil.unpack_archive(a, extract_dir=b)
parser.add_argument('--ffmpeg_path',type=str,default='ffmpeg') parser.add_argument('--sample_rate',type=int,default=8000) parser.add_argument('--use_abs_path',default=False,action='store_true',help='Use absolute paths in resulting manifest') args = parser.parse_args() progress_function = lambda gen : gen try: import tqdm progress_function = tqdm.tqdm except: print('tqdm not available, will not show progress') if not os.path.exists(args.extracted_dir): os.makedirs(args.extracted_dir,exist_ok=True) print('Unpacking archive') shutil.unpack_archive(args.zip_file,args.extracted_dir) def convert_to_wav(src,dest,ffmpeg_path,sr): if os.path.exists(dest): return conversion_command = r'%s -hide_banner -loglevel warning -i %s -ar %d %s' % (ffmpeg_path,src,sr,dest) subprocess.check_output(conversion_command.split(' ')) def get_wav_filename(src): return os.path.join(args.target_dir,os.path.splitext(os.path.basename(src))[0] + '.wav') def convert_librispeech_audio(src): dest = get_wav_filename(src) convert_to_wav(src,dest,args.ffmpeg_path,args.sample_rate)
# destibnation folder destination_folder = Path('data') # set up some filenames zip_file = destination_folder.joinpath(zipfile) shape_file = zip_file.with_name(zipfile.replace('zip', 'shp')) # download zip if need to if not Path(zip_file).exists(): r = requests.get(tm_borders_url) with open(zip_file, 'wb') as fp: fp.write(r.content) # extract shp from zip if need to if force or not Path(shape_file).exists(): shutil.unpack_archive(zip_file.as_posix(), extract_dir=destination_folder) # Read the LAI dataset for a given country and year # read in the LAI data for given country code from geog0111.process_timeseries import process_timeseries ''' Note, the saved npz file can be quite large e.g. 8.1 G for France. You can override saving it by setting save = False but if it is saved, it will be faster to access data the next time you need it. If you have a slow network, you might set download=False ''' save = True
def main() -> None: import argparse parser = argparse.ArgumentParser( description="bootstrapping utility for hadrian.", epilog=USAGE, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('-d', '--deps', type=Path, help='bootstrap dependency file (plan-bootstrap.json)') parser.add_argument('-w', '--with-compiler', type=Path, help='path to GHC') parser.add_argument('-s', '--bootstrap-sources', type=Path, help='Path to prefetched bootstrap sources tarball') subparsers = parser.add_subparsers(dest="command") parser_list = subparsers.add_parser( 'list-sources', help='list all sources required to download') parser_list.add_argument('-o', '--output', type=Path, default='fetch_plan.json') parser_fetch = subparsers.add_parser( 'fetch', help='fetch all required sources from hackage (for offline builds)') parser_fetch.add_argument('-o', '--output', type=Path, default='bootstrap-sources') parser_fetch.add_argument( '-p', '--fetch-plan', type=Path, default=None, help= "A json document that lists the urls required for download (optional)") args = parser.parse_args() ghc = None if args.deps is None: if args.bootstrap_sources is None: # find appropriate plan in the same directory as the script ghc = find_ghc(args.with_compiler) args.deps = Path( sys.path[0] ) / f"plan-bootstrap-{ghc.version.replace('.','_')}.json" print(f"defaulting bootstrap plan to {args.deps}") # We have a tarball with all the required information, unpack it and use for further elif args.bootstrap_sources is not None and args.command != 'list-sources': print(f'Unpacking {args.bootstrap_sources} to {TARBALLS}') shutil.unpack_archive(args.bootstrap_sources.resolve(), TARBALLS, 'gztar') args.deps = TARBALLS / 'plan-bootstrap.json' print( f"using plan-bootstrap.json ({args.deps}) from {args.bootstrap_sources}" ) else: print( "We need a bootstrap plan (plan-bootstrap.json) or a tarball containing bootstrap information" ) print("Perhaps pick an appropriate one from: ") for child in Path(sys.path[0]).iterdir: if child.match('plan-bootstrap-*.json'): print(child) sys.exit(1) info = read_bootstrap_info(args.deps) print( dedent(""" DO NOT use this script if you have another recent cabal-install available. This script is intended only for bootstrapping hadrian on new architectures. """)) if (args.command == 'fetch'): if args.fetch_plan is not None: plan = { path: FetchInfo(p["url"], p["sha256"]) for path, p in json.load(args.fetch_plan.open()).items() } else: plan = gen_fetch_plan(info) if ghc is None: ghc = find_ghc(args.with_compiler) # In temporary directory, create a directory which we will archive tmpdir = TMPDIR.resolve() tmpdir.mkdir(parents=True, exist_ok=True) rootdir = Path(tempfile.mkdtemp(dir=tmpdir)) fetch_from_plan(plan, rootdir) shutil.copyfile(args.deps, rootdir / 'plan-bootstrap.json') fmt = 'gztar' if platform.system() == 'Windows': fmt = 'zip' archivename = shutil.make_archive(args.output, fmt, root_dir=rootdir) print(f""" Bootstrap sources saved to {archivename} Use `bootstrap.py -w {ghc.ghc_path} -s {archivename}` to continue """) elif (args.command == 'list-sources'): ghc = find_ghc(args.with_compiler) plan = gen_fetch_plan(info) with open(args.output, 'w') as out: json.dump({path: val._asdict() for path, val in plan.items()}, out) print(f"Required hackage sources saved to {args.output}") tarfmt = "\n./" print(f""" Download the files listed in {args.output}, copying {args.deps} to plan-bootstrap.json, and save them to a tarball ($TARBALL) The contents of $TARBALL should look like: ./ ./plan-bootstrap.json ./{tarfmt.join(path for path in plan)} To generate $TARBALL, e.g. files in sources-tarball, `TARBALL=sources-tarball.tar.gz; pushd sources-tarball; tar -zcf ../$TARBALL .; popd` Then use `bootstrap.py -w {ghc.ghc_path} -s $TARBALL` to continue Alternatively, you could use `bootstrap.py -w {ghc.ghc_path} -d {args.deps} fetch -o sources-tarball` to download and generate the tarball, skipping this step """) elif (args.command == None): if ghc is None: ghc = find_ghc(args.with_compiler) print( f'Bootstrapping hadrian with GHC {ghc.version} at {ghc.ghc_path}...' ) if args.bootstrap_sources is None: plan = gen_fetch_plan(info) fetch_from_plan(plan, TARBALLS) bootstrap(info, ghc) hadrian_path = (BINDIR / 'hadrian').resolve() archive = make_archive(hadrian_path) print( dedent(f''' Bootstrapping finished! The resulting hadrian executable can be found at {hadrian_path} It has been archived for distribution in {archive} You can use this executable to build GHC. ''')) else: print(f"No such command: {args.command}")
def main(parsec_source): BUILD_DIR.mkdir(parents=True, exist_ok=True) # Retrieve parsec version global_dict = {} exec((parsec_source / "parsec/_version.py").read_text(), global_dict) parsec_version = global_dict.get("__version__") print(f"### Detected Parsec version {parsec_version} ###") # Fetch CPython distrib if not CPYTHON_DISTRIB_ARCHIVE.is_file(): print(f"### Fetch CPython {CPYTHON_VERSION} build ###") req = urlopen(CPYTHON_DISTRIB_URL) CPYTHON_DISTRIB_ARCHIVE.write_bytes(req.read()) # Bootstrap tools virtualenv if not TOOLS_VENV_DIR.is_dir(): print(f"### Create tool virtualenv ###") run(f"python -m venv {TOOLS_VENV_DIR}") run(f"{ TOOLS_VENV_DIR / 'Scripts/python' } -m pip install wheel") if not WHEELS_DIR.is_dir(): print(f"### Generate wheels from Parsec&dependencies ###") run( f"{ TOOLS_VENV_DIR / 'Scripts/python' } -m pip wheel {parsec_source}[core] --wheel-dir {WHEELS_DIR}" ) # Now we actually generate the build target target_dir = BUILD_DIR / f"parsec-{parsec_version}-{get_archslug()}" if target_dir.exists(): raise SystemExit(f"{target_dir} already exists, exiting...") # Extract CPython distrib print(f"### Extracting CPython embedded distribution ###") shutil.unpack_archive(str(CPYTHON_DISTRIB_ARCHIVE), extract_dir=str(target_dir)) # Bootstrap build virtualenv build_venv_dir = target_dir / "build_venv" print(f"### Installing wheels in temporary virtualenv ###") run(f"python -m venv {build_venv_dir}") wheels = " ".join(map(str, WHEELS_DIR.glob("*.whl"))) run(f"{ build_venv_dir / 'Scripts/python' } -m pip install {wheels}") # Move build virtualenv's site-packages to the build and patch imports print(f"### Move site-packages to embedded distribution ###") shutil.move(build_venv_dir / "Lib/site-packages", target_dir / "site-packages") shutil.rmtree(build_venv_dir) pth_file, = target_dir.glob("*._pth") pth_file.write_text(pth_file.read_text() + "site-packages\n") # Include LICENSE file (target_dir / "LICENSE.txt").write_text((parsec_source / "LICENSE").read_text()) # Build parsec.exe resource_rc = BUILD_DIR / "resource.rc" resource_res = BUILD_DIR / "resource.res" versioninfo = (*re.match(r"^.*([0-9]+)\.([0-9]+)\.([0-9]+)", parsec_version).groups(), "0") escaped_parsec_ico = str(Path("parsec.ico").resolve()).replace("\\", "\\\\") escaped_parsec_manifest = str(Path("parsec.manifest").resolve()).replace("\\", "\\\\") resource_rc.write_text( f""" #include <windows.h> 1 RT_MANIFEST "{escaped_parsec_manifest}" 2 ICON "{escaped_parsec_ico}" VS_VERSION_INFO VERSIONINFO FILEVERSION {','.join(versioninfo)} PRODUCTVERSION {','.join(versioninfo)} FILEFLAGSMASK 0x3fL FILEFLAGS 0x0L FILEOS VOS__WINDOWS32 FILETYPE VFT_APP FILESUBTYPE 0x0L BEGIN BLOCK "StringFileInfo" BEGIN BLOCK "000004b0" BEGIN VALUE "CompanyName", "Scille SAS\\0" VALUE "FileDescription", "Parsec Secure Cloud Storage\\0" VALUE "FileVersion", "{parsec_version}\\0" VALUE "InternalName", "Parsec GUI Bootstrapper\\0" VALUE "LegalCopyright", "Parsec Cloud (https://parsec.cloud) Copyright (c) AGPLv3 2019 Scille SAS\\0" VALUE "OriginalFilename", "parsec.exe\\0" VALUE "ProductName", "Parsec\\0" VALUE "ProductVersion", "{parsec_version}\\0" END END BLOCK "VarFileInfo" BEGIN VALUE "Translation", 0x0, 1200 END END """ ) run(f"rc.exe /i. /fo {resource_res} {resource_rc}") # Must make sure /Fo option ends with a "\", otherwise it is not considered as a folder... run(f"cl.exe parsec-bootstrap.c /c /I { CPYTHON_DIR / 'include' } /Fo{BUILD_DIR}\\") run( f"link.exe { BUILD_DIR / 'parsec-bootstrap.obj' } {resource_res} " f"/LIBPATH:{ CPYTHON_DIR / 'libs' } /OUT:{ target_dir / 'parsec.exe' } " f"/subsystem:windows /entry:mainCRTStartup" ) # Create build info file for NSIS installer (BUILD_DIR / "BUILD.tmp").write_text( f'target = "{target_dir}"\n' f'parsec_version = "{parsec_version}"\n' f'python_version = "{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"\n' f'platform = "{get_archslug()}"\n' ) # Create the install and uninstall file list for NSIS installer target_files = [] def _recursive_collect_target_files(curr_dir): subdirs = [] for entry in curr_dir.iterdir(): if entry.is_dir(): subdirs.append(entry) else: target_files.append((False, entry.relative_to(target_dir))) for subdir in subdirs: target_files.append((True, subdir.relative_to(target_dir))) _recursive_collect_target_files(subdir) _recursive_collect_target_files(target_dir) install_files_lines = ["; Files to install", 'SetOutPath "$INSTDIR\\"'] curr_dir = Path(".") for target_is_dir, target_file in target_files: if target_is_dir: install_files_lines.append(f'SetOutPath "$INSTDIR\\{target_file}"') curr_dir = target_file else: assert curr_dir == target_file.parent install_files_lines.append(f'File "${{PARSEC_FREEZE_BUILD_DIR}}\\{target_file}"') (BUILD_DIR / "install_files.nsh").write_text("\n".join(install_files_lines)) uninstall_files_lines = ["; Files to uninstall"] for target_is_dir, target_file in reversed(target_files): if target_is_dir: uninstall_files_lines.append(f'RMDir "$INSTDIR\\{target_file}"') else: uninstall_files_lines.append(f'Delete "$INSTDIR\\{target_file}"') (BUILD_DIR / "uninstall_files.nsh").write_text("\n".join(uninstall_files_lines))
def update_the_challenge(): """ Updates the local copy of The Challenge. """ # Check if the user is online if not check_internet_connection(): print("You are not connected to the internet. Try again later.") sys.exit() # Get the latest version most_recent_version, most_recent_files = get_most_recent_version_and_files( ) # Get local version local_version = __version__ # Check if local version is smaller than the GitHub version if version.parse(local_version) < version.parse(most_recent_version): print( f"There is a new version, {most_recent_version}, available.\n(Installed Version: {__version__})\n" ) while True: print("Do you want to update to the new version?") want_to_update = input("[Y]es or [N]o: ").upper() if want_to_update not in ["Y", "N"]: print("Please enter either 'Y' or 'N'.\n") elif want_to_update == "N": print("Keeping local version. Quitting now.") sys.exit() else: print("Starting update process...") break else: print( "You are already on the latest version. Quitting the program now.") sys.exit() print() # Get the latest distribution file distribution_url = "" for file in most_recent_files: if file["path"].find("The-Challenge-Production-Server_") != -1: distribution_url = file["url"] break # Download the latest distribution if distribution_url != "": print("Downloading latest distribution...") download_request = requests.get(distribution_url) download_request.raise_for_status() print("Done!") print("Writing distribution package contents to file...") with open("./The-Challenge_Latest-Dist.tar.gz", "wb+") as f: f.write(b64decode(json.loads(download_request.text)["content"])) f.close() print("Done!") else: print("Can't get the latest distribution. Try again later.") sys.exit() # Extract the latest distribution print("Extracting contents of latest distribution...") shutil.unpack_archive("The-Challenge_Latest-Dist.tar.gz", "./extracted") os.remove("./The-Challenge_Latest-Dist.tar.gz") print("Done!") # Recursively try to find the wheel file in the extracted folder try: latest_wheel_file = [ f for f in glob.glob("./extracted/" + "**/*.whl", recursive=True) ][0] # Once found install it using pip os.system(f"pip install -U {latest_wheel_file}") print("The update was completed successfully.") except IndexError: print("The latest distribution file cannot be found. Quitting now...") sys.exit() # Clean up shutil.rmtree("./extracted") # Offer to automatically restart the service print("\n!!! IMPORTANT !!!") print( "Only answer 'Y' to the following prompt if you (a) are on Ubuntu; (b) have a systemd service that " "hosts The Challenge's server; and (c) are an administrator that can use the 'sudo' command." ) print("!!! IMPORTANT !!!\n") while True: print("Would you like to restart the systemd service?") confirm_systemd_name = input("[Y]es or [N]o: ").upper() if confirm_systemd_name not in ["Y", "N"]: print("Please enter either 'Y' or 'N'.\n") elif confirm_systemd_name == "N": print("Quitting now.") sys.exit() else: break # Ask user to input the systemd service name while True: print("Please enter the systemd service name.") systemd_service_name = input("?> ") if systemd_service_name == "": print("Please enter the name.") else: print( "\nPlease confirm that you want to restart the systemd service named:" ) print(f"'{systemd_service_name}'") while True: confirm_systemd_name = input("[Y]es or [N]o: ").upper() if confirm_systemd_name not in ["Y", "N"]: print("Please enter either 'Y' or 'N'.\n") print("Please confirm the systemd service name.") elif confirm_systemd_name == "N": print( "Disregarding current input of the systemd service name." ) break else: os.system(f"sudo systemctl restart {systemd_service_name}") print("The systemd service has been restarted. Quitting.") sys.exit()
# download with wget sftp_cmd = "sshpass -p 'burnt' sftp fire@" + url + " " + cfg.get( args.source, 'local_path') sftp_status = os.system(sftp_cmd) # TODO: check time for wget process # check wget_status if sftp_status == 0: log.info('download finished: ' + localFilename) break else: # if wget_status =! 0 is due a some error log.info("attempt " + str(attempt) + ': error downloading: ' + url) sleep(120) ##### extract shutil.unpack_archive(localFilename, cfg.get(args.source, 'local_path')) ##### dissolve geometries import geopandas as gpd from shapely.geometry.multipolygon import MultiPolygon burned_area_file = localFilename.replace(".shapefiles.tar.gz", ".shp") f_in = gpd.read_file(burned_area_file) f_dissolve = gpd.geoseries.GeoSeries( MultiPolygon([geom for geom in f_in.unary_union.geoms])) f_dissolve.crs = f_in.crs df_clip = gpd.clip(f_dissolve, gpd.read_file("../shapes/Colombia.shp")) for f in glob.glob(burned_area_file.replace(".shp", "*")): os.remove(f) df_clip.to_file(burned_area_file)
4. Write to sql """ import shutil import os import glob from simpledbf import Dbf5 import sqlite3 import pandas as pd all_zip_files = glob.glob("data/customs_ru_raw/*.zip") # file = all_zip_files[0] for file in all_zip_files: # Unzip shutil.unpack_archive(file, "data/temp", "zip") # Read dbf to DataFrame df = Dbf5("data/temp/DATTSVT.dbf", codec="CP866").to_dataframe() # Delete already readed file os.remove("data/temp/DATTSVT.dbf") # Rename columns df = df.rename( columns={ "Stoim": "usd", "Netto": "kg", "Kol": "qty", "period": "year_month", "edizm": "uom",
def _setupSSHDImpl(ngrok_token, ngrok_region, is_VNC): #apt-get update #apt-get upgrade my_apt = _MyApt() #Following packages are useless because nvidia kernel modules are already loaded and I cannot remove or update it. #Uninstall them because upgrading them take long time. my_apt.deleteInstalledPkg("nvidia-dkms", "nvidia-kernel-common", "nvidia-kernel-source") my_apt.commit() my_apt.update_upgrade() my_apt.commit() subprocess.run(["unminimize"], input="y\n", check=True, universal_newlines=True) my_apt.installPkg("openssh-server") my_apt.commit() my_apt.close() #Reset host keys for i in pathlib.Path("/etc/ssh").glob("ssh_host_*_key"): i.unlink() subprocess.run(["ssh-keygen", "-A"], check=True) #Prevent ssh session disconnection. with open("/etc/ssh/sshd_config", "a") as f: f.write("\n\nClientAliveInterval 120\n") msg = "" msg += "ECDSA key fingerprint of host:\n" ret = subprocess.run( ["ssh-keygen", "-lvf", "/etc/ssh/ssh_host_ecdsa_key.pub"], stdout=subprocess.PIPE, check=True, universal_newlines=True) msg += ret.stdout + "\n" _download( "https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip", "ngrok.zip") shutil.unpack_archive("ngrok.zip") pathlib.Path("ngrok").chmod(stat.S_IXUSR) root_password = secrets.token_urlsafe() user_password = secrets.token_urlsafe() user_name = "colab" msg += "✂️" * 24 + "\n" msg += f"root password: {root_password}\n" msg += f"{user_name} password: {user_password}\n" msg += "✂️" * 24 + "\n" subprocess.run(["useradd", "-s", "/bin/bash", "-m", user_name]) subprocess.run(["adduser", user_name, "sudo"], check=True) subprocess.run(["chpasswd"], input=f"root:{root_password}", universal_newlines=True) subprocess.run(["chpasswd"], input=f"{user_name}:{user_password}", universal_newlines=True) subprocess.run(["service", "ssh", "restart"]) if not pathlib.Path('/root/.ngrok2/ngrok.yml').exists(): subprocess.run(["./ngrok", "authtoken", ngrok_token]) ngrok_proc = subprocess.Popen( ["./ngrok", "tcp", "-region", ngrok_region, "22"]) time.sleep(2) if ngrok_proc.poll() != None: raise RuntimeError("Failed to run ngrok. Return code:" + str(ngrok_proc.returncode) + "\nSee runtime log for more info.") with urllib.request.urlopen( "http://*****:*****@{hostname}\n" else: msg += "Command to connect to the ssh server:\n" msg += "✂️" * 24 + "\n" msg += f"ssh {ssh_common_options} -p {port} {user_name}@{hostname}\n" msg += "✂️" * 24 + "\n" return msg
def unpack_archive(path): shutil.unpack_archive(path)
import zipfile import shutil f = open("file_one.txt", "w+") f.write("This is the first file") f.close() f = open("file_two.txt", "w+") f.write("This is the second file") f.close() # compress files comp_file = zipfile.ZipFile("files.zip", "w") comp_file.write("file_one.txt", compress_type=zipfile.ZIP_DEFLATED) comp_file.write("file_two.txt", compress_type=zipfile.ZIP_DEFLATED) comp_file.close() # decompress files zip_obj = zipfile.ZipFile("files.zip", "r") zip_obj.extractall("extracted_files") # Creating a zip archive zip_path = "E:\\Workspace\\Complete-Python-3-Bootcamp\\12-Advanced Python Modules" out_file = "example" shutil.make_archive(out_file, 'zip', zip_path) # Extracting a zip archive shutil.unpack_archive("example.zip", "final_unzip", 'zip')
for match in matches: url = match filename = url[url.rfind("/")+1:] file_path = path.join(models_dir, filename) dir_name = filename.split(".")[0] dir_path = path.join(models_dir, dir_name) # if the archived file not exist, download it if not os.path.isfile(file_path) and not os.path.isdir(dir_path): print("downloading ... {}".format(filename)) urllib.request.urlretrieve(url, filename=file_path) # if the dir already existed, no need to unarchive again if not os.path.isdir(dir_path): print("unarchiving ... {}".format(filename)) shutil.unpack_archive(file_path, dir_path) # if remove is flagged and archive exist, remove if after unarchiving if os.path.isfile(file_path): os.remove(file_path) # https://github.com/tensorflow/models/tree/master/research/slim#exporting-the-inference-graph for f in os.listdir(dir_path): if f.endswith(".ckpt"): ckpt = f break if ckpt = "inception_resnet_v2_2016_08_30.ckpt": ckpt = "inception_resnet_v2.ckpt" pb = ckpt.split(".")[0] + ".pb" cmd = "python " + os.environ["GOPATH"]+"/src/github.com/tensorflow/models/research/slim/export_inference_graph.py --alsologtostderr --model_name=" + ckpt.split(".")[ 0] + " --output_file=" + path.join(dir_path, pb)
def _prepare_task(self, identifier, description): """ Prepare a working directory before starting the solution. :param identifier: Verification task identifier. :param description: Dictionary with task description. :raise SchedulerException: If a task cannot be scheduled or preparation failed. """ # Prepare working directory task_work_dir = os.path.join(self.work_dir, "tasks", identifier) task_data_dir = os.path.join(task_work_dir, "data") job_id = description['job id'] self.logger.debug( "Make directory for the task to solve {!r}".format(task_data_dir)) os.makedirs(task_data_dir.encode("utf8"), exist_ok=True) # This method can be called several times to adjust resource limitations but we should avoid extra downloads # from the server if identifier not in self.__tasks: archive = os.path.join(task_work_dir, "task.zip") self.logger.debug( "Pull from the verification gateway archive {!r}".format( archive)) ret = self.server.pull_task(identifier, archive) if not ret: self.logger.info( "Seems that the task data cannot be downloaded because of a respected reason, " "so we have nothing to do there") os._exit(1) self.logger.debug("Unpack archive {!r} to {!r}".format( archive, task_data_dir)) shutil.unpack_archive(archive, task_data_dir) # Update description description.update(self.__get_credentials(job_id)) # TODO: Add more exceptions handling to make code more reliable with open(os.path.join( os.path.join(self.work_dir, "tasks", identifier), "task.json"), "w", encoding="utf8") as fp: json.dump(description, fp, ensure_ascii=False, sort_keys=True, indent=4) # Prepare command to submit self.logger.debug( "Prepare arguments of the task {!r}".format(identifier)) task_data_dir = os.path.join(self.work_dir, "tasks", identifier, "data") try: assert description["priority"] in ["LOW", "IDLE"] run = Run(task_data_dir, description) except Exception as err: raise schedulers.SchedulerException( 'Cannot prepare task description on base of given benchmark.xml: {}' .format(err)) self.__track_task(job_id, run, identifier) return True
def unpack(zipfile: str): filename = datapath(zipfile) rootdir = datapath(zipfile[:-4]) shutil.unpack_archive(filename, extract_dir=rootdir, format="zip")
def _process_task_result(self, identifier, future, description): """ Process result and send results to the server. :param identifier: Task identifier string. :param future: Future object. :return: status of the task after solution: FINISHED. :raise SchedulerException: in case of ERROR status. """ run = self.__tasks[identifier] self.__drop_task(identifier) task_work_dir = os.path.join(self.work_dir, "tasks", identifier) solution_file = os.path.join(task_work_dir, "solution.zip") self.logger.debug( "Save solution to the disk as {}".format(solution_file)) try: result = future.result() except Exception as err: error_msg = "Task {} has been finished but no data has been received: {}".format( identifier, err) self.logger.warning(error_msg) raise schedulers.SchedulerException(error_msg) # Save result with open(solution_file, 'wb') as sa: sa.write(result) # Unpack results task_solution_dir = os.path.join(task_work_dir, "solution") self.logger.debug( "Make directory for the solution to extract {0}".format( task_solution_dir)) os.makedirs(task_solution_dir.encode("utf8"), exist_ok=True) self.logger.debug("Extract results from {} to {}".format( solution_file, task_solution_dir)) shutil.unpack_archive(solution_file, task_solution_dir) # Process results and convert RunExec output to result description # TODO: what will happen if there will be several input files? # Simulate BenchExec behaviour when one input file is provided. os.makedirs(os.path.join(task_solution_dir, "output", "benchmark.logfiles").encode("utf8"), exist_ok=True) shutil.move( os.path.join(task_solution_dir, 'output.log'), os.path.join( task_solution_dir, "output", "benchmark.logfiles", "{}.log".format(os.path.basename(run.run.sourcefiles[0])))) try: solution_identifier, solution_description = self.__extract_description( task_solution_dir) self.logger.debug( "Successfully extracted solution {} for task {}".format( solution_identifier, identifier)) except Exception as err: self.logger.warning( "Cannot extract results from a solution: {}".format(err)) raise err # Make fake BenchExec XML report self.__make_fake_benchexec( solution_description, os.path.join(task_work_dir, 'solution', 'output', "benchmark.results.xml")) # Add actual restrictions solution_description['resource limits'] = description[ "resource limits"] # Make archive solution_archive = os.path.join(task_work_dir, "solution") self.logger.debug( "Make archive {} with a solution of the task {}.zip".format( solution_archive, identifier)) shutil.make_archive(solution_archive, 'zip', task_solution_dir) solution_archive += ".zip" # Push result self.logger.debug( "Upload solution archive {} of the task {} to the verification gateway" .format(solution_archive, identifier)) try: utils.submit_task_results(self.logger, self.server, self.scheduler_type(), identifier, solution_description, os.path.join(task_work_dir, "solution")) except Exception as err: error_msg = "Cannot submit solution results of task {}: {}".format( identifier, err) self.logger.warning(error_msg) raise schedulers.SchedulerException(error_msg) if "keep working directory" not in self.conf["scheduler"] or \ not self.conf["scheduler"]["keep working directory"]: self.logger.debug("Clean task working directory {} for {}".format( task_work_dir, identifier)) shutil.rmtree(task_work_dir) self.logger.debug( "Task {} has been processed successfully".format(identifier)) return "FINISHED"
####################################################################################### # In the name of God, the Compassionate, the Merciful # Pyabr (c) 2020 Mani Jamali. GNU General Public License v3.0 # # Programmer & Creator: Mani Jamali <*****@*****.**> # Telegram or Gap channel: @pyabr # Telegram or Gap group: @pyabr_community # Git source: github.com/manijamali2003/pyabr # ####################################################################################### import shutil, os, sys if not os.path.isdir("build-packs"): os.mkdir("build-packs") if not os.path.isdir("wheel/src"): os.mkdir("wheel/src") shutil.unpack_archive('wheel/setup.zip', 'wheel/setup', 'zip') # Unpack setup wheel package ## Copy all files and dirs in wheel/setup/src ## list = os.listdir('.') list.remove('.git') list.remove('.idea') list.remove('wheel') list.remove('latest') for i in list: if os.path.isdir(i): shutil.copytree(i, 'wheel/src/' + i) else: shutil.copyfile(i, 'wheel/src/' + i)