def command_create(working_dir: str, project_name, *argv): print('working-dir:' + working_dir) pkg_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) project_path = os.path.abspath(os.path.join(working_dir, project_name)) print('Creating restfx project "%s"' % project_name) if os.path.exists(project_path): print('[ERROR] Project path "%s" exists.' % project_path) sys.exit(1) # 示例文件 sample_file = os.path.abspath( os.path.join(pkg_root, 'internal_assets', 'sample.tar.gz')) print('Creating project structure') import tarfile tarfile = tarfile.open(sample_file) tarfile.extractall(project_path) tarfile.close() command_genid(project_path) print("""Created ! It is time to have fun with restfx. """.format(project_name=project_name))
def extractfile(future_path): if future_path.endswith('.zip'): print("Unzipping to : "+future_path+" patience will be needed.") zipfile.extractall(future_path, None, None) if future_path.endswith('.tgz'): print("Extracting file to : "+future_path+" patience will be needed.") tarfile.extractall(future_path, None) else: print("Format seems wrong....") return
def extract_tar_files(tar_file, out_directory): """ extracts all files in a tar archive """ tar_name = ntpath.basename(tar_file) un_tar_path = os.path.join(tar_file, tar_name) tarfile.TarFile(tar_file) tarfile.extractall(out_directory) return data_interpreter(un_tar_path, out_directory)
def _analyze_tarfile_for_import(tarfile, project, schema, tmpdir): def read_sp_manifest_file(path): # Must use forward slashes, not os.path.sep. fn_manifest = _tarfile_path_join(path, project.Job.FN_MANIFEST) try: with closing(tarfile.extractfile(fn_manifest)) as file: if sys.version_info < (3, 6): return json.loads(file.read().decode()) else: return json.loads(file.read()) except KeyError: pass if schema is None: schema_function = read_sp_manifest_file elif callable(schema): schema_function = _with_consistency_check(schema, read_sp_manifest_file) elif isinstance(schema, str): schema_function = _with_consistency_check( _make_path_based_schema_function(schema), read_sp_manifest_file) else: raise TypeError( "The schema variable must be None, callable, or a string.") mappings = dict() skip_subdirs = set() dirs = [member.name for member in tarfile.getmembers() if member.isdir()] for name in sorted(dirs): if os.path.dirname( name) in skip_subdirs: # skip all sub-dirs of identified dirs skip_subdirs.add(name) continue sp = schema_function(name) if sp is not None: job = project.open_job(sp) if os.path.exists(job.workspace()): raise DestinationExistsError(job) mappings[name] = job skip_subdirs.add(name) # Check uniqueness if len(set(mappings.values())) != len(mappings): raise StatepointParsingError( "The jobs identified with the given schema function are not unique!" ) tarfile.extractall(path=tmpdir) for path, job in mappings.items(): src = os.path.join(tmpdir, path) assert os.path.isdir(tmpdir) assert os.path.isdir(src) yield src, _CopyFromTarFileExecutor(src, job)
def extract_tar(target, current, zip=False): tar = None if not zip: import tarfile tar = tarfile.open(target) else: import zipfile tar = zipfile.ZipFile(target) try: tar.extractall(path=current) except: return False return True
def extract_arc(self, target, current, zip=False, rar=False): tar = None if not zip and not rar: import tarfile tar = tarfile.open(target) elif zip: import zipfile tar = zipfile.ZipFile(target) elif rar: import rarfile tar = rarfile.RarFile(target) try: tar.extractall(path=current) except Exception, e: return False
def download_peaks2maps_model(data_dir=None, overwrite=False, verbose=1): """ Download the trained Peaks2Maps model from OHBM 2018. """ url = "https://zenodo.org/record/1257721/files/ohbm2018_model.tar.xz?download=1" temp_dataset_name = 'peaks2maps_model_ohbm2018__temp' temp_data_dir = _get_dataset_dir(temp_dataset_name, data_dir=data_dir, verbose=verbose) dataset_name = 'peaks2maps_model_ohbm2018' data_dir = temp_data_dir.replace(temp_dataset_name, dataset_name) desc_file = op.join(data_dir, 'description.txt') if op.isfile(desc_file) and overwrite is False: shutil.rmtree(temp_data_dir) return data_dir LGR.info('Downloading the model (this is a one-off operation)...') # Streaming, so we can iterate over the response. r = requests.get(url, stream=True) f = BytesIO() # Total size in bytes. total_size = int(r.headers.get('content-length', 0)) block_size = 1024 * 1024 wrote = 0 for data in tqdm(r.iter_content(block_size), total=math.ceil(total_size // block_size), unit='MB', unit_scale=True): wrote = wrote + len(data) f.write(data) if total_size != 0 and wrote != total_size: raise Exception("Download interrupted") f.seek(0) LGR.info('Uncompressing the model to {}...'.format(temp_data_dir)) tarfile = TarFile(fileobj=LZMAFile(f), mode="r") tarfile.extractall(temp_data_dir) os.rename(op.join(temp_data_dir, 'ohbm2018_model'), data_dir) shutil.rmtree(temp_data_dir) with open(desc_file, 'w') as fo: fo.write('The trained Peaks2Maps model from OHBM 2018.') if verbose > 0: print('\nDataset moved to {}\n'.format(data_dir)) return data_dir
def _run_setup_py(tarfile, setup_filename, data): tempdir = tempfile.mkdtemp() setuptools.sandbox.DirectorySandbox(tempdir).run(lambda: tarfile.extractall(tempdir)) setup_filename = os.path.join(tempdir, setup_filename) distutils.core._setup_stop_after = "config" setuptools.sandbox.run_setup(setup_filename, "") dist = distutils.core._setup_distribution shutil.rmtree(tempdir) if dist.ext_modules: data["is_extension"] = True if dist.scripts: data["scripts"] = dist.scripts if dist.test_suite: data["test_suite"] = dist.test_suite if dist.install_requires: data["install_requires"] = dist.install_requires if dist.extras_require: data["extras_require"] = dist.extras_require if dist.data_files: data["data_files"] = dist.data_files if dist.entry_points: data["entry_points"] = dist.entry_points
exit() try: if console == False: from appJar import gui app = gui("Music Player") except ImportError: LogErr() bcast("AppJar could not be imported") bcast("This is required for the GUI to work") option = raw_input("Install? (yes/no): ") if option.lower() == "yes" or option.lower() == "y": try: bcast("Installing AppJar...") urllib.urlretrieve(url + "/appJar.tar.gz", "AppJar.tar.gz") tarfile.open("appJar.tar.gz") tarfile.extractall() tarfile.close() os.remove("appJar.tar.gz") from AppJar import gui app = gui("Music Player") except: LogErr() bcast("Unable to download and install AppJar") bcast("The GUI will now be disabled") console = True elif option.lower() == "no" or option.lower() == "n": bcast("GUI will now be disabled.") bcast("You will be prompted again on next run") else: bcast("Invalid option") bcast("AppJar will not be installed")
def _analyze_tarfile_for_import(tarfile, project, schema, tmpdir): """Validate paths in tarfile. Parameters ---------- tarfile : :class:`tarfile.TarFile` tarfile to analyze. project : :class:`~signac.Project` The project to import the data into. schema : str or callable An optional schema function, which is either a string or a function that accepts a path as its first and only argument and returns the corresponding state point as dict. (Default value = None). tmpdir : :class:`tempfile.TemporaryDirectory` Temporary directory, an instance of ``TemporaryDirectory``. Yields ------ src : str Source path. copy_executor : callable A callable that uses a provided function to copy to a destination. Raises ------ TypeError If the schema given is not None, callable, or a string. :class:`~signac.errors.DestinationExistsError` If a job is already initialized. :class:`~signac.errors.StatepointParsingError` If the jobs identified with the given schema function are not unique. AssertionError If ``tmpdir`` given is not a directory. """ def read_sp_manifest_file(path): """Read state point from the manifest file. Parameters ---------- path : str Path to manifest file. Returns ------- dict state point. """ # Must use forward slashes, not os.path.sep. fn_manifest = _tarfile_path_join(path, project.Job.FN_MANIFEST) try: with closing(tarfile.extractfile(fn_manifest)) as file: return json.loads(file.read()) except KeyError: pass if schema is None: schema_function = read_sp_manifest_file elif callable(schema): schema_function = _with_consistency_check(schema, read_sp_manifest_file) elif isinstance(schema, str): schema_function = _with_consistency_check( _make_path_based_schema_function(schema), read_sp_manifest_file ) else: raise TypeError("The schema variable must be None, callable, or a string.") mappings = {} skip_subdirs = set() dirs = [member.name for member in tarfile.getmembers() if member.isdir()] for name in sorted(dirs): if ( os.path.dirname(name) in skip_subdirs ): # skip all sub-dirs of identified dirs skip_subdirs.add(name) continue sp = schema_function(name) if sp is not None: job = project.open_job(sp) if os.path.exists(job.workspace()): raise DestinationExistsError(job) mappings[name] = job skip_subdirs.add(name) # Check uniqueness if len(set(mappings.values())) != len(mappings): raise StatepointParsingError( "The jobs identified with the given schema function are not unique!" ) tarfile.extractall(path=tmpdir) for path, job in mappings.items(): assert os.path.isdir(tmpdir) src = os.path.join(tmpdir, path) assert os.path.isdir(src) copy_executor = _CopyFromTarFileExecutor(src, job) yield src, copy_executor
import pathlib import tarfile import os print("start setting ORCA") tmp_orca_dir = pathlib.Path("/tmp/orca_tmp", "r") tarfile = tarfile.open(pathlib.Path.home() / "ORCA-303.tbz") if not (tmp_orca_dir / "orca_3_0_3_linux_x86-64/ORCA").is_file(): print("ORCA has not copied on remote node") tarfile.extractall(str(tmp_orca_dir)) os.system("export orca_dir=" + str(tmp_orca_dir) + "/orca_3_0_3_linux_x86-64/") if (tmp_orca_dir / "orca_3_0_3_linux_x86-64/ORCA").is_file(): print("ORCA was copied") tarfile.close() print("")
print "Data must be stored on CASTOR ..." else: print "Data must be stored on AFS ..." if (output_mode != "ROOTMODE"): print "Getting files for " + crabdir + " and putting in " + ntupdir else: print "Getting files for " + input_root + " and putting in " + ntupdir if (output_mode != "ROOTMODE"): # untar if required files = os.listdir(crabdir + "/res") for file in files: if re.search("tgz", file): tarfile.open(file, "r:gz") tarfile.extractall(crabdir + "/res") # hadd all the files together if (output_mode != "ROOTMODE"): hadd = Popen("hadd -f " + ntupdir + "/" + crabdir + ".root " + crabdir + "/res/*.root", shell=True) hadd.wait() else: result = commands.getoutput("nsls " + input_root + "/ > " + crabdir + "/list.txt") result = commands.getoutput("awk '{print \"rfio:" + input_root + "/\" $1}' " + crabdir + "/list.txt > " + crabdir + "/list2.txt") hadd = Popen("hadd -f " + crabdir + ".root @" + crabdir + "/list2.txt", shell=True)
def uncompress_tgz(self, path): tarfile = tarfiles.open(path) tarfile.extractall(self.path) tarfile.close()
from connector.settings import ELFINDER_ROOT, ELFINDER_URL, ELFINDER_THUMB
CORD19_FILES: List[Tuple[str, bool]] = [ ("comm_use_subset.tar.gz", True), ("biorxiv_medrxiv.tar.gz", True), ("noncomm_use_subset.tar.gz", True), ("custom_license.tar.gz", True), (f"metadata.csv", True)] # Borrowed from https://stackoverflow.com/questions/16694907/download-large-file-in-python-with-requests def download_file(fname: str) -> None: with requests.get(CORD19_BASE + fname, stream=True) as r: r.raise_for_status() with open(os.path.join(SOURCE_DIR, fname), 'wb') as f: for chunk in r.iter_content(chunk_size=8192): if chunk: # filter out keep-alive new chunks f.write(chunk) for file, download in CORD19_FILES: if download: print(f"Downloading: {file}") download_file(file) else: print(f"Skipped: {file}") for file, _ in CORD19_FILES: if file.endswith(".tar.gz"): tarfile = tarfile.open(os.path.join(SOURCE_DIR, file), "r:gz") tarfile.extractall(SOURCE_DIR)
def create_metafile(self, datafile): info = collections.defaultdict(lambda: "") # Extract datafile in temporary directory and scan for dependencies. tmpdir = self.mktemp(directory=True) if self.payload_compression == "xz": tarfile = tar.InnerTarFileXz.open(datafile) else: tarfile = tar.InnerTarFile.open(datafile) tarfile.extractall(path=tmpdir) tarfile.close() # Run the dependency tracker. self.pkg.track_dependencies(self.builder, tmpdir) # Generic package information including Pakfire information. info.update({ "pakfire_version" : PAKFIRE_VERSION, "uuid" : self.pkg.uuid, "type" : "binary", }) # Include distribution information. info.update(self.pakfire.distro.info) info.update(self.pkg.info) # Update package information for string formatting. info.update({ "groups" : " ".join(self.pkg.groups), "prerequires" : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \ for d in self.pkg.prerequires]), "requires" : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \ for d in self.pkg.requires]), "provides" : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \ for d in self.pkg.provides]), "conflicts" : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \ for d in self.pkg.conflicts]), "obsoletes" : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \ for d in self.pkg.obsoletes]), "recommends" : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \ for d in self.pkg.recommends]), "suggests" : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \ for d in self.pkg.suggests]), }) # Format description. description = [PACKAGE_INFO_DESCRIPTION_LINE % l \ for l in util.text_wrap(self.pkg.description, length=80)] info["description"] = "\n".join(description) # Build information. info.update({ # Package it built right now. "build_time" : int(time.time()), "build_id" : uuid.uuid4(), }) # Installed size (equals size of the uncompressed tarball). info.update({ "inst_size" : self.getsize(datafile), }) metafile = self.mktemp() f = open(metafile, "w") f.write(PACKAGE_INFO % info) f.close() return metafile
print "Data must be stored on CASTOR ..." else : print "Data must be stored on AFS ..." if (output_mode!="ROOTMODE"): print "Getting files for "+crabdir+" and putting in "+ntupdir else: print "Getting files for "+input_root+" and putting in "+ntupdir if (output_mode!="ROOTMODE"): # untar if required files=os.listdir(crabdir+"/res") for file in files: if re.search("tgz", file): tarfile.open(file, "r:gz") tarfile.extractall(crabdir+"/res") # hadd all the files together if (output_mode!="ROOTMODE"): hadd=Popen("hadd -f "+ntupdir+"/"+crabdir+".root "+crabdir+"/res/*.root", shell=True) hadd.wait() else: result=commands.getoutput("nsls "+input_root+"/ > "+crabdir+"/list.txt") result=commands.getoutput("awk '{print \"rfio:"+input_root+"/\" $1}' "+crabdir+"/list.txt > "+crabdir+"/list2.txt"); hadd=Popen("hadd -f "+crabdir+".root @"+crabdir+"/list2.txt", shell=True) hadd.wait() print "" print "Move "+crabdir+".root to "+output_root+"/" result=commands.getoutput("rfcp "+crabdir+".root "+output_root) result=commands.getoutput("mv "+crabdir+".root "+ntupdir+"/") print "------------------------------------------------------------"
def provided_members_archive_handler(filename): tar = tarfile.open(filename) tarfile.extractall(path=tempfile.mkdtemp(), members=tar) tar.close()