def download_package(destination, product, version, compiler):
remove_existing_package(destination, product, version)
label = get_release_label()
file_name = "{0}-{1}-{2}-{3}.tar.gz".format(product, version, compiler, label)
url_path="/{0}/{1}-{2}/{0}-{1}-{2}-{3}.tar.gz".format(product, version, compiler, label)
download_path = HOST + url_path
print "URL {0}".format(download_path)
print "Downloading {0} to {1}".format(file_name, destination)
# --no-clobber avoids downloading the file if a file with the name already exists
sh.wget(download_path, directory_prefix=destination, no_clobber=True)
print "Extracting {0}".format(file_name)
sh.tar(z=True, x=True, f=os.path.join(destination, file_name), directory=destination)
sh.rm(os.path.join(destination, file_name))
if product == "kudu":
# The Kudu tarball is actually a renamed parcel. Rename the contents to match the
# naming convention.
kudu_dirs = glob.glob("{0}/KUDU*{1}*".format(destination, version))
if not kudu_dirs:
raise Exception("Could not find contents of Kudu tarball")
if len(kudu_dirs) > 1:
raise Exception("Found too many Kudu folders: %s" % (kudu_dirs, ))
new_dir = "{0}/{1}-{2}".format(destination, product, version)
if os.path.exists(new_dir):
shutil.rmtree(new_dir)
os.rename(kudu_dirs[0], new_dir)
write_version_file(destination, product, version, compiler, label)
def setup(self):
self.work_dir = tempfile.mkdtemp("carpyncho_loader_test")
self.teardown()
if not os.path.exists(self.EXAMPLE_DATA_PATH):
os.makedirs(self.EXAMPLE_DATA_PATH)
sh.tar(j=True,
x=True,
f=self.TAR_DATA_PATH,
directory=self.EXAMPLE_DATA_PATH)
shutil.copytree(self.EXAMPLE_DATA_PATH, self.work_dir)
self.input_path = os.path.join(self.work_dir, "example_data")
self.data_path = os.path.join(self.work_dir, "stored")
self.test_cache = os.path.join(self.work_dir, "test_cache")
dirs = {
"INPUT_PATH": self.input_path,
"DATA_PATH": self.data_path,
"RAW_TILES_DIR": os.path.join(self.data_path, "raw_tiles"),
"NPY_TILES_DIR": os.path.join(self.data_path, "npy_tiles"),
"RAW_PAWPRINTS_DIR": os.path.join(self.data_path, "raw_pawprints"),
"NPY_PAWPRINTS_DIR": os.path.join(self.data_path, "npy_pawprints"),
"MATCHS_DIR": os.path.join(self.data_path, "matchs"),
"LC_DIR": os.path.join(self.data_path, "light_curves"),
"SAMPLES_DIR": os.path.join(self.data_path, "samples")
}
for k, d in dirs.items():
if not os.path.exists(d):
os.makedirs(d)
self.patch("corral.conf.settings.{}".format(k), d)
self.runned = self.run_another_tests(self.run_before)
def download_package(destination, product, version, compiler):
remove_existing_package(destination, product, version)
label = get_release_label()
file_name = "{0}-{1}-{2}-{3}.tar.gz".format(product, version, compiler, label)
url_path="/{0}/{1}-{2}/{0}-{1}-{2}-{3}.tar.gz".format(product, version, compiler, label)
download_path = HOST + url_path
print "URL {0}".format(download_path)
print "Downloading {0} to {1}".format(file_name, destination)
# --no-clobber avoids downloading the file if a file with the name already exists
sh.wget(download_path, directory_prefix=destination, no_clobber=True)
print "Extracting {0}".format(file_name)
sh.tar(z=True, x=True, f=os.path.join(destination, file_name), directory=destination)
sh.rm(os.path.join(destination, file_name))
if product == "kudu":
# The Kudu tarball is actually a renamed parcel. Rename the contents to match the
# naming convention.
kudu_dirs = glob.glob("{0}/KUDU*{1}*".format(destination, version))
if not kudu_dirs:
raise Exception("Could not find contents of Kudu tarball")
if len(kudu_dirs) > 1:
raise Exception("Found too many Kudu folders: %s" % (kudu_dirs, ))
new_dir = "{0}/{1}-{2}".format(destination, product, version)
if os.path.exists(new_dir):
shutil.rmtree(new_dir)
os.rename(kudu_dirs[0], new_dir)
print "renamed shit"
write_version_file(destination, product, version, compiler, label)
def fetch(self, path):
mkdir('-p', self._own_workdir)
# download sources
archive_filename = str(Path(self._workdir).joinpath('kernel.tar.gz'))
if not Path(archive_filename).is_file():
wget('-O{}'.format(archive_filename),
path,
_out=sys.stdout,
_err=sys.stderr)
# extract sources
extracted_dir = str(Path(self._own_workdir).joinpath('_extracted'))
mkdir('-p', extracted_dir)
tar('-xf',
archive_filename,
'-C',
extracted_dir,
_out=sys.stdout,
_err=sys.stderr)
kernel_src_dir = [
d for d in Path(extracted_dir).iterdir() if d.is_dir()
]
if (len(kernel_src_dir) != 1):
raise RuntimeError('Linux Kernel sources failed to extract')
self._dir = kernel_src_dir[0]
def file_backup_restore(process, action):
if action == 'backup':
with sh.pushd('/opt/anaconda'):
sh.tar(
"--exclude=storage/pgdata",
"--exclude=storage/object/anaconda-repository",
"-czvf",
process.storage_backup_name,
"storage"
)
sh.mv(
f'/opt/anaconda/{process.storage_backup_name}',
f'{process.backup_directory}/'
)
elif action == 'restore':
sh.cp(
f'{process.backup_directory}/{process.storage_backup_name}',
'/opt/anaconda'
)
with sh.pushd('/opt/anaconda'):
sh.tar(
'-xzvf',
f'/opt/anaconda/{process.storage_backup_name}'
)
sh.rm(f'{process.storage_backup_name}')
def initialize():
# noinspection PyUnresolvedReferences
from sh import wget, tar, rm, shasum
if not os.path.exists(prefix):
os.makedirs(prefix)
if (not os.path.exists(dirs['inputs'])) or (not os.path.exists(
dirs['intermediates'])):
try:
if not os.path.exists(prefix):
logger.info("Creating {DIR}".format(DIR=prefix))
os.makedirs(prefix)
logger.info("Downloading data from {URL} to {DIR}".format(
URL=data_url, DIR=prefix))
tar(wget(data_url, "-qO-", _piped=True), "xz", _cwd=prefix)
logger.info("Checking checksums of downloaded files")
for line in shasum("-c", _cwd=prefix, _in=checksums, _iter=True):
logger.info(line)
except Exception as e:
logger.info("Error: {}".format(e.message))
logger.info("Deleting {DIR}".format(DIR=dirs['inputs']))
rm(dirs['inputs'], '-rf')
logger.info("Deleting {DIR}".format(DIR=dirs['intermediates']))
rm(dirs['intermediates'], '-rf')
raise
# make sure all those directories exist
for d in (dirs['outputs'], dirs['plots']):
if not os.path.exists(d):
logger.info("Creating {DIR}".format(DIR=d))
os.makedirs(d)
def cleanup_and_restore_files(process):
timestamp = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H%M%S')
# Compress and timestamp the existing files minus the repos
with sh.pushd('/opt/anaconda'):
sh.tar(
"--exclude=storage/object/anaconda-repository",
"-czvf",
f"git_pgdata.snapshot_{timestamp}.tar.gz",
"storage"
)
# Cleanup directories as things will get restored
sh.rm('-Rf', '/opt/anaconda/storage/git')
sh.rm('-Rf', '/opt/anaconda/storage/pgdata')
sh.rm('-Rf', '/opt/anaconda/storage/object/anaconda-objects')
sh.rm('-Rf', '/opt/anaconda/storage/object/anaconda-projects')
# Restore the files
file_backup_restore(process, 'restore')
# Recreate the postgres directory and set permissions
sh.mkdir(process.postgres_system_backup)
sh.chown('999:root', f'{process.postgres_system_backup}')
sh.chmod('700', f'{process.postgres_system_backup}')
return
def packaging_lib(libmace_output_dir, project_name):
print("* Package libs for %s" % project_name)
tar_package_name = "libmace_%s.tar.gz" % project_name
project_dir = "%s/%s" % (libmace_output_dir, project_name)
tar_package_path = "%s/%s" % (project_dir, tar_package_name)
if os.path.exists(tar_package_path):
sh.rm("-rf", tar_package_path)
print("Start packaging '%s' libs into %s" % (project_name,
tar_package_path))
which_sys = platform.system()
if which_sys == "Linux":
sh.tar(
"cvzf",
"%s" % tar_package_path,
glob.glob("%s/*" % project_dir),
"--exclude",
"%s/_tmp" % project_dir,
_fg=True)
elif which_sys == "Darwin":
sh.tar(
"--exclude",
"%s/_tmp" % project_dir,
"-cvzf",
"%s" % tar_package_path,
glob.glob("%s/*" % project_dir),
_fg=True)
print("Packaging Done!\n")
return tar_package_path
def main():
tar_files = sys.argv[1]
sample_ratio = float(sys.argv[2])
np.random.seed(1234)
tar_files = glob.glob(tar_files)
print(tar_files)
work_dir = os.path.dirname(tar_files[0])
os.system(f'mkdir -p {work_dir}/binary_files/')
for tar_file in tar_files:
print(f'extracting {tar_file}')
tar('-C', f'{work_dir}/binary_files/', '-xvf', tar_file)
all_files = glob.glob(f'{work_dir}/binary_files/*.jsonl')
all_files.sort()
print(f'{len(all_files)} in total')
sampled_files = np.random.choice(all_files,
replace=False,
size=int(sample_ratio * len(all_files)))
print(f'{len(sampled_files)} sampled files')
os.chdir(work_dir)
with open(f'sampled_binaries.txt', 'w') as f:
for fname in sampled_files:
fname = os.path.basename(fname)
f.write(fname + '\n')
print('creating tar file')
os.chdir('binary_files/')
tar('-cf', f'../sampled_binaries_{sample_ratio}.tar', '-T',
'../sampled_binaries.txt')
def extract_reviews(input_file_name, output_file_name):
# extracts to folder "sorted_data"
sh.tar("xvf", input_file_name)
reviews = []
raw_dir = "sorted_data"
categories = [
name for name in os.listdir(raw_dir)
if os.path.isdir(os.path.join(raw_dir, name))
]
for category in categories:
positive_file_name = os.path.join(raw_dir, category, "positive.review")
negative_file_name = os.path.join(raw_dir, category, "negative.review")
positive_reviews = get_reviews(positive_file_name,
label=':)',
categroy=category)
negative_reviews = get_reviews(negative_file_name,
label=':(',
categroy=category)
reviews.extend(positive_reviews)
reviews.extend(negative_reviews)
# This folder is really big, and we still have the compressed version of this anyway so there's
# no need to keep it around.
sh.rm("-rf", raw_dir)
with open(output_file_name, 'wb') as output_file:
json.dump(reviews, output_file)
output_file.write(u"\n")
def sh_asterisk():
'''
如果遇到要传递参数为 * 的命令，应该怎么做
https://stackoverflow.com/questions/32923189/how-to-pass-an-asterisk-to-module-sh-in-python
'''
# 等价 linux 命令 tar czf 1.tar.gz *
sh.tar("czf", "1.tar.gz", sh.glob("*"))
def copy_proguard_mapping(flavor, version_name):
folder_path = 'releases'
sh.mkdir("-p", folder_path)
output_file = '%s/wikipedia-%s.mapping.tar.gz' % (folder_path, version_name)
input_file = 'wikipedia/build/outputs/mapping/%s/release/mapping.txt' % flavor
sh.tar('czf', output_file, input_file)
print ' proguard mapping: %s' % output_file
def initialize():
# noinspection PyUnresolvedReferences
from sh import wget, tar, rm, shasum
if not os.path.exists(prefix):
os.makedirs(prefix)
if (not os.path.exists(dirs['inputs'])) or (not os.path.exists(dirs['intermediates'])):
try:
if not os.path.exists(prefix):
logger.info("Creating {DIR}".format(DIR=prefix))
os.makedirs(prefix)
logger.info("Downloading data from {URL} to {DIR}".format(URL=data_url, DIR=prefix))
tar(wget(data_url, "-qO-", _piped=True), "xz", _cwd=prefix)
logger.info("Checking checksums of downloaded files")
for line in shasum("-c", _cwd=prefix, _in=checksums, _iter=True):
logger.info(line)
except Exception as e:
logger.info("Error: {}".format(e.message))
logger.info("Deleting {DIR}".format(DIR=dirs['inputs']))
rm(dirs['inputs'], '-rf')
logger.info("Deleting {DIR}".format(DIR=dirs['intermediates']))
rm(dirs['intermediates'], '-rf')
raise
# make sure all those directories exist
for d in (dirs['outputs'], dirs['plots']):
if not os.path.exists(d):
logger.info("Creating {DIR}".format(DIR=d))
os.makedirs(d)
def build_stress(stress_revision, name=None):
# Build a stress revision
try:
git_id = sh.git('--git-dir={home}/fab/cassandra.git'.format(home=HOME),
'rev-parse', stress_revision).strip()
except sh.ErrorReturnCode:
raise AssertionError(
'Invalid stress_revision: {}'.format(stress_revision))
path = os.path.join(CASSANDRA_STRESS_PATH, git_id)
if not os.path.exists(path):
logger.info("Building cassandra-stress '{}' in '{}'.".format(
stress_revision, path))
os.makedirs(path)
sh.tar(
sh.git("--git-dir={home}/fab/cassandra.git".format(home=HOME),
"archive", git_id), 'x', '-C', path)
antcmd('-Dbasedir={}'.format(path),
'-f',
'{}/build.xml'.format(path),
'realclean',
'jar',
_env={
"JAVA_TOOL_OPTIONS": "-Dfile.encoding=UTF8",
"JAVA_HOME": JAVA_HOME
})
name = name if name else stress_revision
return {name: git_id}
def add_files(self):
files = ["pytorch_model.bin", "tf_model.h5", "config.json"]
files.extend(self.ORIGINAL_FILES)
files += ["README.md", "model_card"]
with cd(self.git_path):
sh.git("add", *files, _fg=True)
if "nbest_predictions.json" in self.TASK_INFO[self.task]["eval_files"]:
with cd(self.git_path / "eval"):
sh.tar("-cvzf", "nbest_predictions.json.tgz",
"nbest_predictions.json")
sh.rm("nbest_predictions.json")
to_copy = self.get_copy_list()
for files, dest in to_copy:
with cd(dest):
for f in files:
if f == "nbest_predictions.json":
f += ".tgz"
sh.git("add", f, _fg=True)
assert ("checkpoint_dir" not in self.checkpoint_info)
self.checkpoint_info["checkpoint_path"] = self.checkpoint_path
with (self.git_path / "model_info.json").open("w") as f:
f.write(pretty_json(self.checkpoint_info))
with cd(self.git_path):
sh.git("add", "model_info.json")
def fetch(self, output):
dl_path = self.get_latest_tarball()
raw_path, _ = urllib.urlretrieve(dl_path)
# The downloaded file is a gzip'd tarball.
extract_path = self._make_temp_directory("rootfetch-apple-extracted")
sh.tar("-xzv", "-f", raw_path, "-C", extract_path, strip_components=1)
# We now have a directory with all the apple files. We need to find the
# roots directory, parse out all the different formats, then generate a
# single file that has PEMs in it.
certificates_path = os.path.join(extract_path, "certificates", "roots")
for f in os.listdir(certificates_path):
full_path = os.path.join(certificates_path, f)
if not os.path.isfile(full_path):
continue
# Skip hidden files, such as .cvsignore
if f.startswith('.'):
continue
pem = self.make_pem(full_path)
output.write("# ")
output.write(f)
output.write("\n")
output.write("\n".join(pem))
output.write("\n\n")
def zip_it(args, sourcedir, destinationdir):
tail = sourcedir.split('/')[-1:][0]
dest_dir = destinationdir + tail
dest_file = dest_dir + ".tgz"
if args.zipname:
dest_file = args.zipname + ".tgz"
#print('tail = ' + tail + ' dest_file=' + dest_file)
if args.zip or args.zipdel:
sh.tar("-cvzf", dest_file, "-C", destinationdir, tail)
logging.info("Created " + dest_file)
if args.zipdel:
sh.rm("-rf", dest_dir)
if args.useremail:
print("Encrypting " + dest_file + "...")
gpg_file = dest_file + ".gpg"
gpg_home = os.getenv("HOME") + "/.gpghome"
gpg = gnupg.GPG(gnupghome=gpg_home)
with open(dest_file, 'rb') as f:
status = gpg.encrypt_file(file=f,
recipients=[args.useremail],
output=gpg_file)
print('encryption ok: ', status.ok)
#print('encryption status: ', status.status)
#print('encryption stderr: ', status.stderr)
print('~' * 50)
def _extract_keyring(self):
''' extract the parabola keyring '''
cache = next(iter(self._repos['libre'].pkgentries_cache.values()))
keyring_pkgentries = cache['parabola-keyring']
keyring_pkgentry = keyring_pkgentries[0]
keyring_pkgfile = next(iter(keyring_pkgentry.pkgfiles.values()))[0]
src = keyring_pkgfile.path
dst = self._keyring_dir
if not os.path.isdir(
dst) or os.path.getmtime(dst) <= os.path.getmtime(src):
os.makedirs(dst, exist_ok=True)
shutil.rmtree(dst)
os.makedirs(dst, exist_ok=True)
sh.tar('xf', src, _cwd=dst)
keyring_file = os.path.join(dst, 'usr', 'share', 'pacman', 'keyrings',
'parabola.gpg')
self._keyring = GPG_PACMAN.scan_keys(keyring_file)
for key in self._keyring:
key['packages'] = []
self._key_cache[key['keyid']] = key
for key_id, subkey in key['subkey_info'].items():
if 's' in subkey['cap']:
subkey['packages'] = []
self._key_cache[key_id] = subkey
self._key_cache[key_id]['master_key'] = key['keyid']
def archive(self, ref, destination, *archive_args):
"""
Archive the specified GIT ref to the specified destination
Any extra args are passed to the sh command directly so you can
add extra flags for `git archive` should you desire.
"""
tmp_dir = tempfile.mkdtemp()
tar_file = None
try:
# create our git interface via the sh module
# see:
# github.com/stackstrap/stackstrap/blob/master/scripts/loose_ssh.sh
git = sh.git.bake(_cwd=tmp_dir, _env={'GIT_SSH': "loose_ssh.sh"})
self.log.debug("Cloning %s to %s" % (self.url, tmp_dir))
git('clone', '--recursive', self.url, tmp_dir)
self.log.debug("Archiving '{ref}' to {destination}".format(
ref=ref, destination=destination))
settings.mkdir_p(destination)
(fd, tar_file) = tempfile.mkstemp()
git.archive('origin/%s' % ref, *archive_args, _out=tar_file)
sh.tar("xpf", tar_file, _cwd=destination)
finally:
if tar_file:
os.remove(tar_file)
if os.path.isdir(tmp_dir):
shutil.rmtree(tmp_dir)
def packaging_lib(libmace_output_dir, project_name):
print("* Package libs for %s" % project_name)
tar_package_name = "libmace_%s.tar.gz" % project_name
project_dir = "%s/%s" % (libmace_output_dir, project_name)
tar_package_path = "%s/%s" % (project_dir, tar_package_name)
if os.path.exists(tar_package_path):
sh.rm("-rf", tar_package_path)
print("Start packaging '%s' libs into %s" % (project_name,
tar_package_path))
which_sys = platform.system()
if which_sys == "Linux":
sh.tar(
"cvzf",
"%s" % tar_package_path,
glob.glob("%s/*" % project_dir),
"--exclude",
"%s/_tmp" % project_dir,
_fg=True)
elif which_sys == "Darwin":
sh.tar(
"--exclude",
"%s/_tmp" % project_dir,
"-cvzf",
"%s" % tar_package_path,
glob.glob("%s/*" % project_dir),
_fg=True)
print("Packaging Done!\n")
return tar_package_path
def _load_pkgentries(self):
''' extract and then load the entries in the db.tar.xz '''
arches_dir = os.path.join(self._pkgfiles_dir, 'os')
for arch in os.scandir(arches_dir):
if arch.name not in CONFIG.parabola.arches:
continue
repo_file = os.path.join(arch.path, '%s.db' % self._name)
if os.path.exists(repo_file):
mtime = os.path.getmtime(repo_file)
dst = os.path.join(self._pkgentries_dir, 'os', arch.name)
if os.path.isdir(dst) and os.path.getmtime(dst) > mtime:
continue
os.makedirs(dst, exist_ok=True)
shutil.rmtree(dst)
os.makedirs(dst, exist_ok=True)
sh.tar('xf', repo_file, _cwd=dst)
i = 0
arches_dir = os.path.join(self._pkgentries_dir, 'os')
for arch in os.scandir(arches_dir):
if arch.name not in CONFIG.parabola.arches:
continue
for pkgentry_dir in os.scandir(arch.path):
pkgentry = PkgEntry(self, pkgentry_dir.path, arch.name)
i += 1
if sys.stdout.isatty():
sys.stdout.write(' %s pkgentries: %i\r' % (self._name, i))
sys.stdout.flush()
self._pkgentries.append(pkgentry)
if pkgentry.arch not in self._pkgentries_cache:
self._pkgentries_cache[pkgentry.arch] = {}
if pkgentry.pkgname not in self._pkgentries_cache[
pkgentry.arch]:
self._pkgentries_cache[pkgentry.arch][
pkgentry.pkgname] = []
self._pkgentries_cache[pkgentry.arch][pkgentry.pkgname].append(
pkgentry)
for provides in pkgentry.provides.union([pkgentry.pkgname]):
if pkgentry.arch not in self._provides_cache:
self._provides_cache[pkgentry.arch] = {}
splits = ['==', '>=', '<=', '>', '<', '=']
for split in splits:
if split in provides:
provides = provides.split(split)[0]
break
if provides not in self._provides_cache[pkgentry.arch]:
self._provides_cache[pkgentry.arch][provides] = []
self._provides_cache[pkgentry.arch][provides].append(
pkgentry)
def wget_and_unpack_package(download_path, file_name, destination, wget_no_clobber):
print "URL {0}".format(download_path)
print "Downloading {0} to {1}".format(file_name, destination)
# --no-clobber avoids downloading the file if a file with the name already exists
sh.wget(download_path, directory_prefix=destination, no_clobber=wget_no_clobber)
print "Extracting {0}".format(file_name)
sh.tar(z=True, x=True, f=os.path.join(destination, file_name), directory=destination)
sh.rm(os.path.join(destination, file_name))
def _archive(includes):
tar_args = [git.archive('--format=tar','HEAD'),'czf','archive.tar.gz']
for pttrn in includes:
tar_args.append('--include')
tar_args.append(pttrn)
import debug
tar_args.append('@-')
tar(*tar_args)
def unzip_single_event(event_directory):
"""
unzip the sac files in a directory
"""
current_path = str(sh.pwd())[:-1] # pylint: disable=not-callable
sh.cd(event_directory)
sh.tar("xvf", "SAC.tar.gz")
sh.cd(current_path)
def wget_and_unpack_package(download_path, file_name, destination, wget_no_clobber):
print "URL {0}".format(download_path)
print "Downloading {0} to {1}".format(file_name, destination)
# --no-clobber avoids downloading the file if a file with the name already exists
sh.wget(download_path, directory_prefix=destination, no_clobber=wget_no_clobber)
print "Extracting {0}".format(file_name)
sh.tar(z=True, x=True, f=os.path.join(destination, file_name), directory=destination)
sh.rm(os.path.join(destination, file_name))
def copy_proguard_mapping(flavor, target):
folder_path = 'releases'
sh.mkdir("-p", folder_path)
output_file = '%s/wikipedia-%s.mapping.tar.gz' % (folder_path,
get_release_name(target))
input_file = 'wikipedia/build/outputs/mapping/%s/release/mapping.txt' % flavor
sh.tar('czf', output_file, input_file)
print ' proguard mapping: %s' % output_file
def input_dir(self):
if not os.path.exists(self.test_data_dir):
if not os.path.exists(self.test_data_tarball):
sh.wget('-P', self.test_dir, data_tarball_url)
sh.tar('zxvf', self.test_data_tarball, '-C', self.test_dir)
return os.path.join(self.test_data_dir, 'input')
def archive_repository(self, destination, *archive_args):
try:
(fd, tar_file) = tempfile.mkstemp()
self.git.archive("remotes/origin/%s" % self.git_ref, *archive_args, _out=tar_file)
sh.tar("xf", tar_file, _cwd=destination)
finally:
if tar_file:
os.remove(tar_file)
def install_cmake( build_dir, prefix ):
cmake_archive='cmake-2.8.11.2'
sh.cd( build_dir )
sh.wget( '-nc', 'http://www.cmake.org/files/v2.8/%s.tar.gz' % cmake_archive )
sh.tar( 'xvzf', '%s.tar.gz' % cmake_archive )
sh.cd( cmake_archive )
subprocess.check_call( [ './configure', '--prefix', PREFIX ], shell = True )
sh.make( '-j4' )
sh.make.install()
def install_cmake(build_dir, prefix):
cmake_archive = 'cmake-2.8.11.2'
sh.cd(build_dir)
sh.wget('-nc', 'http://www.cmake.org/files/v2.8/%s.tar.gz' % cmake_archive)
sh.tar('xvzf', '%s.tar.gz' % cmake_archive)
sh.cd(cmake_archive)
subprocess.check_call(['./configure', '--prefix', PREFIX], shell=True)
sh.make('-j4')
sh.make.install()
def unpack(plugin):
fname="../{pname}_{uversion}*.orig-{name}.tar*".format(pname=c.package, uversion=UPSTREAM_VERSION, name=plugin["Plugin"])
fname = glob.glob(fname)[0]
try:
os.mkdir(plugin["Plugin"])
except FileExistsError:
shutil.rmtree(plugin["Plugin"])
os.mkdir(plugin["Plugin"])
sh.tar(["-C", plugin["Plugin"], "--strip-components=1", "-axf", fname])
def test_make_tarball(self):
"""
Can we make tarballs for all versions in the git repository.
"""
experiment_repository = models.ExperimentRepository.objects.create(**self.setup_fields)
for _hash, tstamp in experiment_repository.get_all_commits():
experiment_details, tarball_path = utils.make_tarball(experiment_repository.path, _hash)
########
# Test #
########
# Do the experiment details look right?
# Are the experiments mentioned in the settings file in the
# experiment_details dir?
experiments_settings = configobj.ConfigObj(
os.path.join(experiment_repository.path, conf.repository_settings_filename)
)
for class_name in experiments_settings["experiments"]:
self.assertTrue(experiment_details.has_key(class_name))
########
# Test #
########
# Are the contents of the tarball as they should be?
tmpdir = tempfile.mkdtemp()
if conf.tarball_compression_method == "bz2":
tar_cmd = "-xjf"
elif conf.tarball_compression_method == "gz":
tar_cmd = "-xzf"
sh.tar(tar_cmd, tarball_path, "-C", tmpdir)
checksum_list = self.mock_repository.commit_dictionary[_hash]
self.assertTrue(sys.check_directory_checksums(checksum_list, tmpdir))
# Delete the exported directory.
shutil.rmtree(tmpdir)
########
# Test #
########
# Check the contents of the tarball; does it import.
tarball_model = utils.ExperimentArchiveTarball(tarball_path)
self.assertTrue(tarball_model.integrity_check())
self.assertTrue(tarball_model.import_check())
# Delete the tarball
os.unlink(tarball_path)
models.ExperimentRepository.objects.all().delete()
def setup_test_input_dir():
data_tarball_url = 'http://s3-ap-southeast-2.amazonaws.com/dp-drop/esmgrids/test/test_data.tar.gz'
test_data_tarball = os.path.join(test_dir, 'test_data.tar.gz')
if not os.path.exists(test_data_dir):
if not os.path.exists(test_data_tarball):
sh.wget('-P', test_dir, data_tarball_url)
sh.tar('zxvf', test_data_tarball, '-C', test_dir)
return os.path.join(test_data_dir, 'input')
def setUp(self, *args, **kwargs):
super(ChefPluginSoloTest, self).setUp(*args, **kwargs)
self.blueprint_dir = self.copy_blueprint('chef-plugin')
# Get resources
with self.blueprint_dir:
for res in 'cookbooks', 'data_bags', 'environments', 'roles':
sh.tar('czf', res + '.tar.gz', res)
def archive_repository(self, destination, *archive_args):
try:
(fd, tar_file) = tempfile.mkstemp()
self.git.archive("remotes/origin/%s" % self.git_ref,
*archive_args,
_out=tar_file)
sh.tar("xf", tar_file, _cwd=destination)
finally:
if tar_file:
os.remove(tar_file)
def unpack(plugin):
fname = "../{pname}_{uversion}*.orig-{name}.tar*".format(
pname=c.package, uversion=UPSTREAM_VERSION, name=plugin["Plugin"])
fname = glob.glob(fname)[0]
try:
os.mkdir(plugin["Plugin"])
except FileExistsError:
shutil.rmtree(plugin["Plugin"])
os.mkdir(plugin["Plugin"])
sh.tar(["-C", plugin["Plugin"], "--strip-components=1", "-axf", fname])
def format(self, project: Project):
"""
Take the original code from archive, extract it to a temp directory,
run the formatter, and then copy the formatter result to the project
root
"""
with tempfile.TemporaryDirectory() as temp_dir:
sh.tar("-C", temp_dir, "-x", "-f", str(project.archive_path))
self.do_format(temp_dir, project)
sh.rsync("-a", temp_dir + "/", str(project.root))
def setUp(self, *args, **kwargs):
super(ChefPluginSoloTest, self).setUp(*args, **kwargs)
self.blueprint_dir = self.copy_blueprint('chef-plugin')
# Get resources
with self.blueprint_dir:
for res in 'cookbooks', 'data_bags', 'environments', 'roles':
sh.tar('czf', res+'.tar.gz', res)
def run(self, repos, nightly):
self.log("Tarballing it...")
cd(self._basedir)
version = get_version(repos, nightly)
import platform
bits = platform.architecture()[0][:2]
bundle_name = "Bitmask-linux%s-%s" % (bits, version)
mv("Bitmask", bundle_name)
tar("cjf", bundle_name + ".tar.bz2", bundle_name)
self.log("Done")
def extract_tar_archive(self, to_directory='/opt'):
if self.backup_directory != '/opt/anaconda_backup':
temp_path = pathlib.Path(self.backup_directory)
to_directory = temp_path.parent
if not tarfile.is_tarfile(self.restore_file):
raise exceptions.NotValidTarfile(
'tar archive file is not a valid tar file')
# Ensure the backup is extracted to the right place
sh.tar('-xzvf', self.restore_file, '-C', to_directory)
def get_heroku_client_path() -> str:
"""
Get the path to the heroku executable client, download a new one if it
doesnt exist.
"""
print("Locating heroku...")
# Install Heroku CLI
os_name = None
bit_architecture = None
# Get the platform we are working on
if sys.platform == "darwin": # Mac OS X
os_name = "darwin"
elif sys.platform.startswith("linux"): # Linux
os_name = "linux"
else:
os_name = "windows"
# Find our architecture
bit_architecture_info = platform.architecture()[0]
if "64bit" in bit_architecture_info:
bit_architecture = "x64"
else:
bit_architecture = "x86"
# Find existing heroku files to use
existing_heroku_directory_names = glob.glob(
os.path.join(HEROKU_TMP_DIR, "heroku-cli-*")
)
if len(existing_heroku_directory_names) == 0:
print("Getting heroku")
if os.path.exists(os.path.join(HEROKU_TMP_DIR, "heroku.tar.gz")):
os.remove(os.path.join(HEROKU_TMP_DIR, "heroku.tar.gz"))
# Get the heroku client and unzip
tar_path = os.path.join(HEROKU_TMP_DIR, "heroku.tar.gz")
sh.wget(
shlex.split(
"{}-{}-{}.tar.gz -O {}".format(
HEROKU_CLIENT_URL, os_name, bit_architecture, tar_path
)
)
)
sh.tar(shlex.split(f"-xvzf {tar_path} -C {HEROKU_TMP_DIR}"))
# Clean up the tar
if os.path.exists(tar_path):
os.remove(tar_path)
heroku_directory_name = os.path.basename(
glob.glob(os.path.join(HEROKU_TMP_DIR, "heroku-cli-*"))[0]
)
heroku_directory_path = os.path.join(HEROKU_TMP_DIR, heroku_directory_name)
return os.path.join(heroku_directory_path, "bin", "heroku")
def script():
"""Run 'cfy logs get' and extract output to CONFIGURATION_DIR/logs"""
logs_dir = cosmo.dir / 'logs'
if logs_dir.exists():
shutil.rmtree(logs_dir, ignore_errors=True)
logs_dir.mkdir()
logs_tar = logs_dir / 'logs.tar.gz'
with logs_dir:
cfy.logs.get(destination_path=logs_tar).wait()
tar('xf', logs_tar, strip_components=1)
logs_tar.remove()
def archiveDownload(url, destination, archiveType):
logging.info('Now downloading archive file from URL %s to %s' % (url, destination))
filename = wget.download(url)
if archiveType == 'zip':
logging.info('Unzipping zip file from: ', filename)
sh.unzip(filename)
elif archiveType == 'tar.gz':
logging.info('Untarring tar.gz file from: ', filename)
sh.tar('-xvzf', filename )
logging.info('Removing archive file.')
sh.rm(filename)
return
def download_package(name, destination, compiler=""):
label = map_release_label()
if len(compiler) > 0:
compiler = "-" + compiler
url = "{0}/{1}/label={2}/artifact/toolchain/build/{3}{4}.tar.gz".format(HOST, BUILD, label, name, compiler)
# Download the file
print "Downloading {0}".format(name)
sh.wget(url, directory_prefix=destination, no_clobber=True)
# Extract
print "Extracting {0}".format(name)
sh.tar(z=True, x=True, f="{0}/{1}{2}.tar.gz".format(destination, name, compiler), directory=destination)
sh.rm("{0}/{1}{2}.tar.gz".format(destination, name, compiler))
def handle(self, file_name, dest):
dest = os.path.abspath(dest)
from sh import unzip
if not os.path.exists(dest):
os.makedirs(dest)
# Make moves
tar(file_name, '-d', dest)
if len(os.listdir(dest)) is 1:
# somewhat properly packaged tarball
dest = os.path.join(dest, os.listdir(dest).pop())
return dest
def download_package(destination, product, version, compiler):
label = get_release_label()
file_name = "{0}-{1}-{2}-{3}.tar.gz".format(product, version, compiler, label)
url_path="/{0}/{1}-{2}/{0}-{1}-{2}-{3}.tar.gz".format(product, version, compiler, label)
download_path = HOST + url_path
print "URL {0}".format(download_path)
print "Downloading {0} to {1}".format(file_name, destination)
# --no-clobber avoids downloading the file if a file with the name already exists
sh.wget(download_path, directory_prefix=destination, no_clobber=True)
print "Extracting {0}".format(file_name)
sh.tar(z=True, x=True, f=os.path.join(destination, file_name), directory=destination)
sh.rm(os.path.join(destination, file_name))
def tgz_handle(self, file_name, dest):
dest = os.path.abspath(dest)
from sh import tar
ops_flags = "zxvf"
if not os.path.exists(dest):
os.makedirs(dest)
# Make moves
tar(ops_flags, file_name, "-C", dest)
if len(os.listdir(dest)) is 1:
# somewhat properly packaged tarball
dest = os.path.join(dest, os.listdir(dest).pop())
return dest
def gitCheckout(version):
directory = versionDirectory(version)
if os.path.exists(directory):
# already checked out
# (since version is always a commit ID, the code can never have changed)
return False
else:
os.makedirs(directory)
# Make sure we're inside the git repository first
os.chdir(config.PROJECT_ROOT)
# how to checkout branch to new working directory, see:
# http://blog.jessitron.com/2013/10/git-checkout-multiple-branches-at-same.html
sh.tar(sh.git.archive(version), "-xC", directory)
return True
def setup_tomcat(self, args):
try:
log("Downloading Tomcat ...")
result = wget(dict['TOMCAT_DOWNLOAD_URL'])
except:
log("Error getting Tomcat from : " + dict['TOMCAT_DOWNLOAD_URL'])
try:
log("Extracting Tomcat ...")
result = tar("xvzf " , dict['TOMCAT_VERSION']+ ".tar.gz")
except:
log("Error extracting Tomcat ..." + dict['TOMCAT_VERSION']+ ".tar.gz")
setup_airavata_server(args)
try:
log("Copying the Airavata war files to Tomcat's webapp directory ...")
result = cp(dict['AIRAVATA_VERSION']+ "/*.war " , dict['TOMCAT_VERSION']+ "/webapps", "-v")
except:
log("Error copying the Airavata war files to Tomcat's webapp directory ...")
try :
log("Granting executeable permissions to the script")
result = chmod("a+x" , dict['TOMCAT_VERSION']+ "/*.sh")
except:
log("Error granting executable permissions to " + dict['TOMCAT_VERSION']+ "/*.sh")
def git_export(project, _hash):
'''
Export commit `_hash` of git project `project` into a temporary directory.
Return the name of the temporary directory.
The process requires tar-ing and untar-ing.
'''
tmpdir = tempfile.mkdtemp()
tarball_name = os.path.join(tmpdir, 'exported_git_project.tar')
assert os.path.exists(project) and os.path.isdir(project)
git = sh.git.bake(_cwd=project)
git('archive', _hash, '-o', tarball_name)
export_tmpdir = tempfile.mkdtemp()
sh.tar('-xf', tarball_name, '-C', export_tmpdir)
shutil.rmtree(tmpdir) # Delete the tmpdir.
return export_tmpdir
def postClone(self, cloned_files, target_dir, version):
"""
Extracts the compressed archives.
.. versionadded:: 0.3.0
"""
f = cloned_files[0]
if self.newer("hg18", version):
# GZIP and TAR the file and save to the target directory
sh.tar("-xzf", f, "-C", target_dir)
else:
# Rename to ".zip"
sh.mv(f, f.replace("tar.gz", "zip"))
# GunZIP the file (and remove the archive)
sh.gunzip(f)
def packaging_lib(libmace_output_dir, project_name):
print("* Package libs for %s" % project_name)
tar_package_name = "libmace_%s.tar.gz" % project_name
project_dir = "%s/%s" % (libmace_output_dir, project_name)
tar_package_path = "%s/%s" % (project_dir, tar_package_name)
if os.path.exists(tar_package_path):
sh.rm("-rf", tar_package_path)
print("Start packaging '%s' libs into %s" % (project_name,
tar_package_path))
sh.tar(
"cvzf",
"%s" % tar_package_path,
glob.glob("%s/*" % project_dir),
"--exclude",
"%s/_tmp" % project_dir,
_fg=True)
print("Packaging Done!\n")
def place(self, cache, path):
data = self.cache(cache)
opts = []
if self.url.fragment is not None:
slashes = self.url.fragment.count('/')
# If the fragment is `a/b/d/`, then we get three slashes and pass
# three to `--strip-components`. The effect is similar to using
# `rsync` or `cp -a` with trailing slashes.
if slashes > 0:
opts += ['--strip-components', str(slashes)]
# Append fragment to arguments list, so only this file/dir is
# extracted.
opts += ['--', self.url.fragment]
mkdir('-p', path.dirname)
if self.url.fragment is None or self.url.fragment.endswith('/'):
tar('-xf', str(data), '-C', str(path), *opts)
else:
tar('-xf', str(data), '--to-stdout', *opts, _out=str(path))
def untar_bundle(self, bundle_path):
"""Unpack tarred bundle
:param bundle_path: path to bundle to unzip
:returns unpacked_bundle: path to unpacked bundle
"""
unpacked_bundle = sh.tar("xfz", bundle_path)
unpacked_bundle = path(self.config["unpacked_bundle"])
if not unpacked_bundle.exists():
raise RuntimeError("Could not find unpacked bundle \"{}\"".
format(unpacked_bundle))
return unpacked_bundle
def setup_stress(stress_revision):
stress_path = None
try:
git_id = sh.git('--git-dir={home}/fab/cassandra.git'
.format(home=HOME), 'rev-parse', stress_revision).strip()
except sh.ErrorReturnCode:
raise AssertionError('Invalid stress_revision: {}'.format(stress_revision))
path = os.path.join(CASSANDRA_STRESS_PATH, git_id)
if not os.path.exists(path):
logger.info("Building cassandra-stress '{}' in '{}'.".format(stress_revision, path))
os.makedirs(path)
sh.tar(
sh.git("--git-dir={home}/fab/cassandra.git".format(home=HOME), "archive", git_id),
'x', '-C', path
)
antcmd('-Dbasedir={}'.format(path), '-f', '{}/build.xml'.format(path), 'realclean', 'jar')
stress_path = os.path.join(path, 'tools/bin/cassandra-stress')
return stress_path
def retrieve(self):
if self.do_retrieve:
sh.rm("-fr", self.dirname)
os.mkdir(self.dirname)
self.pushd(self.dirname)
retrieved = FileBuildConfiguration.download(self.url)
if ".tar.gz" in retrieved:
sh.tar("xvzf", retrieved)
if ".tar.bz2" in retrieved:
sh.tar("xjvf", retrieved)
if ".zip" in retrieved:
sh.unzip(retrieved)
else:
self.pushd(self.dirname)
# Either one directory *OR* one directory + a README.
if(len(os.listdir(".")) <= 3):
# we can assume that we need to chdir before we can build, so set that to the local build path
for curr in os.listdir("."):
if(os.path.isdir(curr)):
self.buildpath = curr
if not getattr(self, 'buildpath'):
self.buildpath = "."
self.popd()
def _install_ckeditor(stdout=None):
"""Callback to install necessary library for the IMCE module"""
arg = 'xf'
if stdout:
arg += 'v'
http.dl(CKEDITOR_URI, 'ckeditor.tar.gz')
output = tar('xf', 'ckeditor.tar.gz')
if stdout and output:
stdout.write(str(output).strip() + '\n')
rm('ckeditor.tar.gz')
def build_stress(stress_revision, name=None):
# Build a stress revision
try:
git_id = sh.git('--git-dir={home}/fab/cassandra.git'
.format(home=HOME), 'rev-parse', stress_revision).strip()
except sh.ErrorReturnCode:
raise AssertionError('Invalid stress_revision: {}'.format(stress_revision))
path = os.path.join(CASSANDRA_STRESS_PATH, git_id)
if not os.path.exists(path):
logger.info("Building cassandra-stress '{}' in '{}'.".format(stress_revision, path))
os.makedirs(path)
sh.tar(
sh.git("--git-dir={home}/fab/cassandra.git".format(home=HOME), "archive", git_id),
'x', '-C', path
)
antcmd('-Dbasedir={}'.format(path), '-f', '{}/build.xml'.format(path),
'realclean', 'jar', _env={"JAVA_TOOL_OPTIONS": "-Dfile.encoding=UTF8",
"JAVA_HOME": JAVA_HOME})
name = name if name else stress_revision
return {name: git_id}
def archive(self, ref, destination, *archive_args):
"""
Archive the specified GIT ref to the specified destination
Any extra args are passed to the sh command directly so you can
add extra flags for `git archive` should you desire.
"""
tmp_dir = tempfile.mkdtemp()
tar_file = None
try:
# create our git interface via the sh module
# see:
# github.com/stackstrap/stackstrap/blob/master/scripts/loose_ssh.sh
git = sh.git.bake(_cwd=tmp_dir, _env={
'GIT_SSH': "loose_ssh.sh"
})
self.log.debug("Cloning %s to %s" % (self.url, tmp_dir))
git('clone', '--recursive', self.url, tmp_dir)
self.log.debug("Archiving '{ref}' to {destination}".format(
ref=ref,
destination=destination
))
settings.mkdir_p(destination)
(fd, tar_file) = tempfile.mkstemp()
git.archive('origin/%s' % ref, *archive_args, _out=tar_file)
sh.tar("xpf", tar_file, _cwd=destination)
finally:
if tar_file:
os.remove(tar_file)
if os.path.isdir(tmp_dir):
shutil.rmtree(tmp_dir)
def _get_code(self, nmpi_job, job_desc):
"""
Obtain the code and place it in the working directory.
If the experiment description is the URL of a Git repository, try to clone it.
If it is the URL of a zip or .tar.gz archive, download and unpack it.
Otherwise, the content of "code" is the code: write it to a file.
"""
url_candidate = urlparse(nmpi_job['code'])
logger.debug("Get code: %s %s", url_candidate.netloc, url_candidate.path)
if url_candidate.scheme and url_candidate.path.endswith((".tar.gz", ".zip", ".tgz")):
self._create_working_directory(job_desc.working_directory)
target = os.path.join(job_desc.working_directory, os.path.basename(url_candidate.path))
#urlretrieve(nmpi_job['code'], target) # not working via KIP https proxy
curl(nmpi_job['code'], '-o', target)
logger.info("Retrieved file from {} to local target {}".format(nmpi_job['code'], target))
if url_candidate.path.endswith((".tar.gz", ".tgz")):
tar("xzf", target, directory=job_desc.working_directory)
elif url_candidate.path.endswith(".zip"):
try:
# -o for auto-overwrite
unzip('-o', target, d=job_desc.working_directory)
except:
logger.error("Could not unzip file {}".format(target))
else:
try:
# Check the "code" field for a git url (clone it into the workdir) or a script (create a file into the workdir)
# URL: use git clone
git.clone('--recursive', nmpi_job['code'], job_desc.working_directory)
logger.info("Cloned repository {}".format(nmpi_job['code']))
except (sh.ErrorReturnCode_128, sh.ErrorReturnCode):
# SCRIPT: create file (in the current directory)
logger.info("The code field appears to be a script.")
self._create_working_directory(job_desc.working_directory)
with codecs.open(job_desc.arguments[0], 'w', encoding='utf8') as job_main_script:
job_main_script.write(nmpi_job['code'])
