def extract_archive(path): # Use libarchive to do extraction. curdir = os.getcwd() os.chdir(app.config['UPLOAD_FOLDER']) libarchive.extract_file(path) os.chdir(curdir) return find_legendsxml(app.config['UPLOAD_FOLDER'])
def run_test(flag, filename):
archive_path = os.path.join(data_dir, 'flags.tar')
try:
extract_file(archive_path)
with pytest.raises(ArchiveError):
extract_file(archive_path, flag)
finally:
if os.path.exists(filename):
os.remove(filename)
def run_test(flag, filename):
archive_path = os.path.join(data_dir, 'flags.tar')
try:
extract_file(archive_path)
with pytest.raises(ArchiveError):
extract_file(archive_path, flag)
finally:
if os.path.exists(filename):
os.remove(filename)
def _tar_xf(tarball, dir_path):
flags = libarchive.extract.EXTRACT_TIME | \
libarchive.extract.EXTRACT_PERM | \
libarchive.extract.EXTRACT_SECURE_NODOTDOT | \
libarchive.extract.EXTRACT_SECURE_SYMLINKS | \
libarchive.extract.EXTRACT_SECURE_NOABSOLUTEPATHS
if not os.path.isabs(tarball):
tarball = os.path.join(os.getcwd(), tarball)
with utils.tmp_chdir(dir_path):
libarchive.extract_file(tarball, flags)
def deb_files(self, deb_file):
try:
archive_type = ArchiveType("AR")
except Exception as e:
return deb_file
if len(deb_file) > archive_type.maxSize:
return deb_file
tmp_dir = tempfile.mkdtemp()
# first: save the stream to a local file
tmp_file = tempfile.NamedTemporaryFile()
tmp_file.write(deb_file)
tmp_file.seek(0)
# chdir to the tmp_dir which the new ar file resides
# and extract it so work on the 'copy' of the stream
with in_dir(tmp_dir):
libarchive.extract_file(tmp_file.name)
file2inject = "data.tar.gz"
infoz = {"type": "TAR", "format": "ustar", "filter": "gzip"}
if os.path.exists(os.path.join(tmp_dir, "data.tar.xz")):
file2inject = "data.tar.xz"
infoz = {"type": "LZMA", "format": "gnutar", "filter": "xz"}
# recreate the injected archive
with open(os.path.join(tmp_dir, file2inject), "r+b") as f:
bfz = f.read()
f.seek(0)
f.write(self.archive_files(bfz, infoz, include_dirs=True))
f.flush()
blk = []
def write_data(data):
blk.append(data[:])
return len(data[:])
with libarchive.custom_writer(write_data, "ar_bsd") as archive:
archive.add_files(os.path.join(tmp_dir, "debian-binary"))
archive.add_files(os.path.join(tmp_dir, "control.tar.gz"))
archive.add_files(os.path.join(tmp_dir, file2inject))
buf = b"".join(blk)
# clean up
shutil.rmtree(tmp_dir, ignore_errors=True)
tmp_file.close()
return buf
def deb_files(self, deb_file):
try:
archive_type = ArchiveType('AR')
except Exception as e:
return deb_file
if len(deb_file) > archive_type.maxSize:
return deb_file
tmp_dir = tempfile.mkdtemp()
# first: save the stream to a local file
tmp_file = tempfile.NamedTemporaryFile()
tmp_file.write(deb_file)
tmp_file.seek(0)
# chdir to the tmp_dir which the new ar file resides
# and extract it so work on the 'copy' of the stream
with in_dir(tmp_dir):
libarchive.extract_file(tmp_file.name)
file2inject = 'data.tar.gz'
infoz = {'type': 'TAR', 'format': 'ustar', 'filter': 'gzip'}
if os.path.exists(os.path.join(tmp_dir, 'data.tar.xz')):
file2inject = 'data.tar.xz'
infoz = {'type': 'LZMA', 'format': 'gnutar', 'filter': 'xz'}
# recreate the injected archive
with open(os.path.join(tmp_dir, file2inject), 'r+b') as f:
bfz = f.read()
f.seek(0)
f.write(self.archive_files(bfz, infoz, include_dirs=True))
f.flush()
blk = []
def write_data(data):
blk.append(data[:])
return len(data[:])
with libarchive.custom_writer(write_data, 'ar_bsd') as archive:
archive.add_files(os.path.join(tmp_dir, 'debian-binary'))
archive.add_files(os.path.join(tmp_dir, 'control.tar.gz'))
archive.add_files(os.path.join(tmp_dir, file2inject))
buf = b''.join(blk)
# clean up
shutil.rmtree(tmp_dir, ignore_errors=True)
tmp_file.close()
return buf
def _tar_xf(tarball, dir_path):
flags = libarchive.extract.EXTRACT_TIME | \
libarchive.extract.EXTRACT_PERM | \
libarchive.extract.EXTRACT_SECURE_NODOTDOT | \
libarchive.extract.EXTRACT_SECURE_SYMLINKS | \
libarchive.extract.EXTRACT_SECURE_NOABSOLUTEPATHS | \
libarchive.extract.EXTRACT_SPARSE | \
libarchive.extract.EXTRACT_UNLINK
if not os.path.isabs(tarball):
tarball = os.path.join(os.getcwd(), tarball)
with utils.tmp_chdir(dir_path):
try:
libarchive.extract_file(tarball, flags)
except libarchive.ArchiveError as e:
raise InvalidArchiveError(tarball, str(e))
def extractall(self, dst):
if self.archive.endswith('.zip'):
assert zipfile.is_zipfile(self.archive)
zf = zipfile.ZipFile(self.archive, 'r')
zf.extractall(dst)
elif self.archive.endswith('.tar.bz2'):
assert tarfile.is_tarfile(self.archive)
tf = tarfile.open(self.archive, 'r')
tf.extractall(dst)
else:
self.archive.endswith('.7z')
savedir = os.getcwd()
os.makedirs(dst)
os.chdir(dst)
libarchive.extract_file(self.archive)
os.chdir(savedir)
def unzip(filename, to_location, format=None):
"""Unzip the corresponding, assumed to exist, zipped DEM into the DEM directory."""
logging.info(f'Unzipping: "{filename}"')
logging.info(f' to: "{to_location}"')
if format is None:
if filename.endswith('.zip'):
format = 'zip'
elif filename.endswith('.gz'):
format = 'zip'
elif filename.endswith('.7z'):
format = '7z'
elif filename.endswith('.bz2'):
format = 'bz2'
else:
raise RuntimeError(
f'Cannot detect the zip format of file: {filename}')
logging.info(f' as fmt: "{format}"')
if format == 'zip':
import zipfile
try:
with zipfile.ZipFile(filename, 'r') as zip_ref:
zip_ref.extractall(to_location)
except zipfile.BadZipFile as err:
logging.error('Failed to unzip: "{}"'.format(filename))
logging.error(
'Likely this is the result of a previous job failing, partial download, internet connection issues, or other failed download. Try removing the file, which will result in it being re-downloaded.'
)
raise err
elif format == '7z':
import libarchive
cwd = os.getcwd()
try:
os.chdir(to_location)
libarchive.extract_file(filename)
except Exception as err:
os.chdir(cwd)
raise err
else:
os.chdir(cwd)
else:
raise NotImplementedError(
'Unzipping file of format {format} is not yet implemented.')
return to_location
def unZip(data):
print ("unz")
paths = data["inputs"][0]
print (paths)
filePath = joinFolder(paths)
outpath = os.path.splitext(filePath)[0]
#outpath = joinFolder(os.path.dirname(filePath), os.path.basename(filePath))
folderPath = os.path.dirname(filePath)
if (filePath.endswith(".zip")):
fh = open(filePath, 'rb')
z = zipfile.ZipFile(fh)
for name in z.namelist():
z.extract(name, outpath)
fh.close()
elif (filePath.endswith(".tar.gz")):
tar = tarfile.open(filePath, "r:gz")
tar.extractall(outpath)
tar.close()
elif (filePath.endswith(".tar")):
tar = tarfile.open(filePath, "r:")
tar.extractall(outpath)
tar.close()
elif (filePath.endswith(".rar")):
print ("at this")
with rarfile.RarFile(filePath) as opened_rar:
opened_rar.extractall(folderPath)
#for f in rf.infolist():
#
#tar = tarfile.open(filePath, "r:")
#tar.extractall(outpath)
#tar.close()
elif (filePath.endswith(".7z")):
cwd = os.getcwd()
print ("at7z")
os.chdir(folderPath)
libarchive.extract_file(filePath)
os.chdir(cwd)
else:
pass
def test_files(tmpdir):
archive_path = tmpdir.strpath+'/test.tar.gz'
# Collect information on what should be in the archive
tree = treestat('libarchive')
# Create an archive of our libarchive/ directory
with libarchive.file_writer(archive_path, 'ustar', 'gzip') as archive:
archive.add_files('libarchive/')
# Read the archive and check that the data is correct
with libarchive.file_reader(archive_path) as archive:
check_archive(archive, tree)
# Extract the archive in tmpdir and check that the data is intact
with in_dir(tmpdir.strpath):
flags = EXTRACT_OWNER | EXTRACT_PERM | EXTRACT_TIME
libarchive.extract_file(archive_path, flags)
tree2 = treestat('libarchive')
assert tree2 == tree
def test_files(tmpdir):
archive_path = tmpdir.strpath+'/test.tar.gz'
# Collect information on what should be in the archive
tree = treestat('libarchive')
# Create an archive of our libarchive/ directory
with libarchive.file_writer(archive_path, 'ustar', 'gzip') as archive:
archive.add_files('libarchive/')
# Read the archive and check that the data is correct
with libarchive.file_reader(archive_path) as archive:
check_archive(archive, tree)
# Extract the archive in tmpdir and check that the data is intact
with in_dir(tmpdir.strpath):
flags = EXTRACT_OWNER | EXTRACT_PERM | EXTRACT_TIME
libarchive.extract_file(archive_path, flags)
tree2 = treestat('libarchive')
assert tree2 == tree
def decompress(filepath, sandbox):
"""
Decompress filepath on a temporary directory in sandbox.
"""
new_filepath = tempfile.mkdtemp(dir=sandbox)
previous_workdir = os.getcwd()
os.chdir(new_filepath)
try:
libarchive.extract_file(filepath)
os.chdir(previous_workdir)
return Decompressed(source=filepath, path=new_filepath)
except libarchive.exception.ArchiveError as e:
os.chdir(previous_workdir)
# More than 2 files (. and ..) in new_filepath, implies there
# was a partial decompression
partial_decompression = len(os.listdir(new_filepath)) > 2
return DecompressionFailed(source=filepath,
path=new_filepath,
partial=partial_decompression,
error=e)
account = sys.argv[1]
except IndexError:
print("Usage:\n unpack_archive.py <account>")
sys.exit(1)
archive = glob.glob(os.path.join(BASE_PATH, f"archives/Archive_{account}_*"))[0]
# check if archive has been unpacked already
date_file = os.path.join(BASE_PATH, f"date_{account}")
try:
with open(date_file, "r") as f:
old_date = f.readline().strip()
except FileNotFoundError:
old_date = "0000-00-00"
new_date = archive.split("_")[-1].split(".")[0]
if not old_date < new_date:
print(f"No new archive found for {account}")
sys.exit(0)
target_dir = os.path.join(BASE_PATH, f"unpacked_{account}")
try:
os.chdir(target_dir)
except FileNotFoundError:
os.mkdir(target_dir)
os.chdir(target_dir)
print(f"unpacking: {archive} to {target_dir}")
libarchive.extract_file(archive)
with open(date_file, "w") as f:
f.write(new_date + "\n")
import libarchive from sys import argv libarchive.extract_file(argv[1])
def extract(path):
return libarchive.extract_file(path)
def fix_package_deps(pkgs_dir: Path, filename: str, channel: str, tmpdir: str) -> Path:
"""Possibly fix package dependencies in pkgs/main .conda file.
For driver see:
https://github.com/ContinuumIO/anaconda-issues/issues/11920
:param pkgs_dir: directory with downloaded conda packages (e.g. ~/miniconda3/pkgs)
:param filename: filename of package (e.g. 'numpy-1.18.5-py37h1da2735_0.conda')
:param tmpdir: temporary directory for doing conda package munging
:returns pkg_file: path to existing or fixed conda package
"""
# Check if package file (*.tar.bz2 or *.conda) is a conda zip archive
# and that is comes from pkgs/main. Only those might need fixing.
pkg_file = pkgs_dir / filename
if (pkg_file.suffix != '.conda'
or not channel.startswith('https://repo.anaconda.com/pkgs/main')):
return pkg_file
# Unzip pkg_file in a temp dir
tmp_pkgs_dir = Path(tmpdir)
pkg_dir = tmp_pkgs_dir / pkg_file.with_suffix('').name
if pkg_dir.exists():
shutil.rmtree(pkg_dir)
pkg_dir.mkdir()
with zipfile.ZipFile(pkg_file, 'r') as pkg_zip:
pkg_zip.extractall(pkg_dir)
info_tar = pkg_dir / f'info-{pkg_dir.name}.tar.zst'
with chdir(pkg_dir):
libarchive.extract_file(info_tar.name)
pkg_info_file = pkg_dir / 'info' / 'index.json'
with open(pkg_info_file) as fh:
pkg_info = json.load(fh)
pkg_depends = pkg_info['depends']
# If the package dependencies are the same as upstream then no change req'd
upstream_repodata_url = f'{channel}/repodata.json.bz2'
upstream_repodata = get_upstream_repodata(upstream_repodata_url)
try:
upstream_depends = upstream_repodata['packages.conda'][filename]['depends']
except KeyError:
print(f' WARNING: package {filename} apparently came from defaults but '
f'no entry in upstream_repodata was found => assuming dependencies OK')
upstream_depends = pkg_depends
if pkg_depends == upstream_depends:
return pkg_file
print('Fixing depends for the following diffs')
print('\n'.join(line.strip() for line in difflib.ndiff(pkg_depends, upstream_depends)
if re.match(r'\S', line)))
pkg_info['depends'] = upstream_depends
with open(pkg_info_file, 'w') as fh:
json.dump(pkg_info, fh, indent=4)
print(f'Unlinking {info_tar} and making new version')
info_tar.unlink()
with chdir(pkg_dir):
with libarchive.file_writer(info_tar.name, 'ustar', 'zstd') as archive:
archive.add_files('info')
try:
shutil.rmtree(pkg_dir / 'info')
except OSError as exc:
# Happened on Windows, just wait a bit and try again
print(f'Failed, trying again: {exc}')
import time
time.sleep(1)
shutil.rmtree(pkg_dir / 'info')
print(f'Making new zip file {pkg_file}')
pkg_file = tmp_pkgs_dir / filename
shutil.make_archive(str(pkg_file), format='zip', root_dir=pkg_dir, base_dir='.')
pkg_file.with_suffix('.conda.zip').rename(pkg_file)
return pkg_file
def main():
is_github_url = False
this_dir = os.getcwd()
# Unpack
config = Config()
cran_metadata = {}
# Some packages are missing on some systems. Need to mark them so they get skipped.
to_be_packaged = set()
with TemporaryDirectory() as merged_td:
for platform, details in sources.items():
with TemporaryDirectory() as td:
os.chdir(td)
libdir = None
# libarchive cannot handle the .exe, just skip it. Means we cannot figure out packages that are not available
# for Windows.
if platform == 'win_no':
details['cached_as'], sha256 = cache_file(
config.src_cache, details['url'], details['fn'],
details['sha'])
libarchive.extract_file(details['cached_as'])
libdir = os.path.join(td, details['library'])
library = os.listdir(libdir)
print(library)
details['to_be_packaged'] = set(library) - set(
R_BASE_PACKAGE_NAMES)
elif platform == 'linux':
details['cached_as'], sha256 = cache_file(
config.src_cache, details['url'], details['fn'],
details['sha'])
libarchive.extract_file(details['cached_as'])
import glob
for filename in glob.iglob('**/*.rpm', recursive=True):
print(filename)
libarchive.extract_file(filename)
libdir = os.path.join(td, details['library'])
library = os.listdir(libdir)
print(library)
details['to_be_packaged'] = set(library) - set(
R_BASE_PACKAGE_NAMES)
elif platform == 'mac':
details['cached_as'], sha256 = cache_file(
config.src_cache, details['url'], details['fn'],
details['sha'])
os.system("bsdtar -xf {}".format(details['cached_as']))
payloads = glob.glob('./**/Payload', recursive=True)
print(payloads)
for payload in payloads:
libarchive.extract_file(payload)
libdir = os.path.join(td, details['library'])
library = os.listdir(libdir)
print(library)
details['to_be_packaged'] = set(library) - set(
R_BASE_PACKAGE_NAMES)
if libdir:
distutils.dir_util.copy_tree(libdir, merged_td)
os.chdir(merged_td)
libdir = merged_td
# Fudge until we can unpack the Windows installer .exe on Linux
sources['win']['to_be_packaged'] = sources['linux']['to_be_packaged']
# Get the superset of all packages (note we will no longer have the DESCRIPTION?!)
for platform, details in sources.items():
if 'to_be_packaged' in details:
to_be_packaged.update(details['to_be_packaged'])
package_dicts = {}
for package in sorted(list(to_be_packaged)):
p = os.path.join(libdir, package, "DESCRIPTION")
with open(p) as cran_description:
description_text = cran_description.read()
d = dict_from_cran_lines(
remove_package_line_continuations(
description_text.splitlines()))
d['orig_description'] = description_text
package = d['Package'].lower()
cran_metadata[package] = d
# Make sure package always uses the CRAN capitalization
package = cran_metadata[package.lower()]['Package']
package_dicts[package.lower()] = {}
package_dicts[package.lower()]['osx'] = True if package in sources[
'mac']['to_be_packaged'] else False
package_dicts[package.lower()]['win'] = True if package in sources[
'win']['to_be_packaged'] else False
package_dicts[
package.lower()]['linux'] = True if package in sources[
'linux']['to_be_packaged'] else False
for package in sorted(list(to_be_packaged)):
cran_package = cran_metadata[package.lower()]
package_dicts[package.lower()].update({
'cran_packagename':
package,
'packagename':
'r-' + package.lower(),
'patches':
'',
'build_number':
0,
'build_depends':
'',
'run_depends':
'',
# CRAN doesn't seem to have this metadata :(
'home_comment':
'#',
'homeurl':
'',
'summary_comment':
'#',
'summary':
'',
})
d = package_dicts[package.lower()]
d['url_key'] = 'url:'
d['fn_key'] = 'fn:'
d['git_url_key'] = ''
d['git_tag_key'] = ''
d['git_url'] = ''
d['git_tag'] = ''
d['hash_entry'] = ''
d['build'], d['skip'] = build_and_skip_olw[(
package_dicts[package.lower()]['osx'],
package_dicts[package.lower()]['linux'],
package_dicts[package.lower()]['win'])]
d['cran_version'] = cran_package['Version']
# Conda versions cannot have -. Conda (verlib) will treat _ as a .
d['conda_version'] = d['cran_version'].replace('-', '_')
patches = []
script_env = []
extra_recipe_maintainers = []
build_number = 0
if not len(patches):
patches.append("# patches:\n")
patches.append(" # List any patch files here\n")
patches.append(" # - fix.patch")
if len(extra_recipe_maintainers):
extra_recipe_maintainers[1:].sort()
extra_recipe_maintainers.insert(0, "extra:\n ")
d['build_number'] = build_number
cached_path = None
d['cran_metadata'] = '\n'.join(
['# %s' % l for l in cran_package['orig_lines'] if l])
# XXX: We should maybe normalize these
d['license'] = cran_package.get("License", "None")
d['license_family'] = guess_license_family(
d['license'], allowed_license_families)
if 'License_is_FOSS' in cran_package:
d['license'] += ' (FOSS)'
if cran_package.get('License_restricts_use') == 'yes':
d['license'] += ' (Restricts use)'
if "URL" in cran_package:
d['home_comment'] = ''
d['homeurl'] = ' ' + yaml_quote_string(cran_package['URL'])
else:
# use CRAN page as homepage if nothing has been specified
d['home_comment'] = ''
d['homeurl'] = ' https://mran.microsoft.com/package/{}'.format(
package)
if 'Description' in cran_package:
d['summary_comment'] = ''
d['summary'] = ' ' + yaml_quote_string(
cran_package['Description'], indent=6)
if "Suggests" in cran_package:
d['suggests'] = " # Suggests: %s" % cran_package['Suggests']
else:
d['suggests'] = ''
if package.lower() == 'revoutilsmath':
d['always_include_files'] = " always_include_files:\n" \
" - lib/R/lib/libRblas.so # [linux]"
else:
d['always_include_files'] = ''
# Every package depends on at least R.
# I'm not sure what the difference between depends and imports is.
depends = [
s.strip() for s in cran_package.get('Depends', '').split(',')
if s.strip()
]
imports = [
s.strip() for s in cran_package.get('Imports', '').split(',')
if s.strip()
]
links = [
s.strip() for s in cran_package.get("LinkingTo", '').split(',')
if s.strip()
]
dep_dict = {}
seen = set()
for s in list(chain(imports, depends, links)):
match = VERSION_DEPENDENCY_REGEX.match(s)
if not match:
sys.exit(
"Could not parse version from dependency of %s: %s" %
(package, s))
name = match.group('name')
if name in seen:
continue
seen.add(name)
archs = match.group('archs')
relop = match.group('relop') or ''
ver = match.group('version') or ''
ver = ver.replace('-', '_')
# If there is a relop there should be a version
assert not relop or ver
if archs:
sys.exit(
"Don't know how to handle archs from dependency of "
"package %s: %s" % (package, s))
dep_dict[name] = '{relop}{version}'.format(relop=relop,
version=ver)
if 'R' not in dep_dict:
dep_dict['R'] = ''
need_git = is_github_url
if cran_package.get("NeedsCompilation", 'no') == 'yes' and False:
with tarfile.open(cached_path) as tf:
need_f = any([
f.name.lower().endswith(('.f', '.f90', '.f77'))
for f in tf
])
# Fortran builds use CC to perform the link (they do not call the linker directly).
need_c = True if need_f else \
any([f.name.lower().endswith('.c') for f in tf])
need_cxx = any([
f.name.lower().endswith(
('.cxx', '.cpp', '.cc', '.c++')) for f in tf
])
need_autotools = any(
[f.name.lower().endswith('/configure') for f in tf])
need_make = True if any((need_autotools, need_f, need_cxx, need_c)) else \
any([f.name.lower().endswith(('/makefile', '/makevars'))
for f in tf])
else:
need_c = need_cxx = need_f = need_autotools = need_make = False
for dep_type in ['build', 'run']:
deps = []
# Put non-R dependencies first.
if dep_type == 'build':
if need_c:
deps.append(
"{indent}{{{{ compiler('c') }}}} # [not win]"
.format(indent=INDENT))
if need_cxx:
deps.append(
"{indent}{{{{ compiler('cxx') }}}} # [not win]"
.format(indent=INDENT))
if need_f:
deps.append(
"{indent}{{{{ compiler('fortran') }}}} # [not win]"
.format(indent=INDENT))
if need_c or need_cxx or need_f:
deps.append(
"{indent}{{{{native}}}}toolchain # [win]".
format(indent=INDENT))
if need_autotools or need_make or need_git:
deps.append(
"{indent}{{{{posix}}}}filesystem # [win]".
format(indent=INDENT))
if need_git:
deps.append(
"{indent}{{{{posix}}}}git".format(indent=INDENT))
if need_autotools:
deps.append(
"{indent}{{{{posix}}}}sed # [win]".
format(indent=INDENT))
deps.append(
"{indent}{{{{posix}}}}grep # [win]".
format(indent=INDENT))
deps.append("{indent}{{{{posix}}}}autoconf".format(
indent=INDENT))
deps.append("{indent}{{{{posix}}}}automake".format(
indent=INDENT))
deps.append("{indent}{{{{posix}}}}pkg-config".format(
indent=INDENT))
if need_make:
deps.append(
"{indent}{{{{posix}}}}make".format(indent=INDENT))
elif dep_type == 'run':
if need_c or need_cxx or need_f:
deps.append(
"{indent}{{{{native}}}}gcc-libs # [win]".
format(indent=INDENT))
for name in sorted(dep_dict):
if name in R_BASE_PACKAGE_NAMES:
continue
if name == 'R':
# Put R first
# Regardless of build or run, and whether this is a recommended package or not,
# it can only depend on 'r-base' since anything else can and will cause cycles
# in the dependency graph. The cran metadata lists all dependencies anyway, even
# those packages that are in the recommended group.
# r_name = 'r-base ' + VERSION
# We don't include any R version restrictions because we
# always build R packages against an exact R version
# deps.insert(0, '{indent}{r_name}'.format(indent=INDENT, r_name=r_name))
# Bit of a hack since I added overlinking checking.
r_name = 'mro-base ' + VERSION
deps.insert(
0, '{indent}{r_name}'.format(indent=INDENT,
r_name=r_name))
else:
conda_name = 'r-' + name.lower()
if dep_dict[name]:
deps.append('{indent}{name} {version}'.format(
name=conda_name,
version=dep_dict[name],
indent=INDENT))
else:
deps.append('{indent}{name}'.format(
name=conda_name, indent=INDENT))
d['dep_dict'] = dep_dict # We need this for (1)
# Make pin_subpackage from the deps. Done separately so the above is the same as conda-build's
# CRAN skeleton (so it is easy to refactor CRAN skeleton so it can be reused here later).
for i, dep in enumerate(deps):
groups = re.match('(\n.* - )([\w-]+) ?([>=\w0-9._]+)?',
dep, re.MULTILINE)
indent = groups.group(1)
name = groups.group(2)
pinning = groups.group(3)
if pinning:
if '>=' in pinning:
deps[
i] = "{}{{{{ pin_subpackage('{}', min_pin='{}', max_pin=None) }}}}".format(
indent, name, pinning.replace('>=', ''))
else:
if name == 'mro-base':
# We end up with filenames with 'r34*' in them unless we specify the version here.
# TODO :: Ask @msarahan about this.
deps[i] = "{}{} {{{{ version }}}}".format(
indent, name)
else:
deps[
i] = "{}{{{{ pin_subpackage('{}', min_pin='{}', max_pin='{}') }}}}".format(
indent, name, pinning, pinning)
else:
deps[
i] = "{}{{{{ pin_subpackage('{}', min_pin='x.x.x.x.x.x', max_pin='x.x.x.x.x.x') }}}}".format(
indent, name)
# Add missing conda package dependencies.
if dep_type == 'run':
if d['packagename'] in extra_deps:
for extra_dep in extra_deps[d['packagename']]:
print("extra_dep is {}".format(extra_dep))
deps.append(extra_dep)
print(deps)
d['%s_depends' % dep_type] = ''.join(deps)
template = {
'version': VERSION,
'win_url': sources['win']['url'],
'win_fn': sources['win']['fn'],
'win_sha': sources['win']['sha'],
'linux_url': sources['linux']['url'],
'linux_fn': sources['linux']['fn'],
'linux_sha': sources['linux']['sha'],
'mac_url': sources['mac']['url'],
'mac_fn': sources['mac']['fn'],
'mac_sha': sources['mac']['sha']
}
with open(os.path.join(this_dir, 'meta.yaml'), 'w') as meta_yaml:
meta_yaml.write(HEADER.format(**template))
meta_yaml.write(BASE_PACKAGE)
for package in package_dicts:
d = package_dicts[package]
# Normalize the metadata values
d = {
k: unicodedata.normalize("NFKD", text_type(v)).encode(
'ascii', 'ignore').decode()
for k, v in iteritems(d)
}
meta_yaml.write(PACKAGE.format(**d))
meta_yaml.write(MRO_BASICS_METAPACKAGE)
meta_subs = []
for package in package_dicts:
meta_subs.append(' - {}{}'.format(
package_dicts[package]['packagename'],
package_dicts[package]['build']))
meta_yaml.write('\n'.join(sorted(meta_subs)))
with open(os.path.join(this_dir, 'build.sh'), 'w') as build_sh:
build_sh.write(BUILD_SH.format(**template))
with open(os.path.join(this_dir, 'install-mro-base.sh'),
'w') as install_mro_base:
install_mro_base.write(INSTALL_MRO_BASE_HEADER.format(**template))
for excluded in sorted(to_be_packaged, key=lambda s: s.lower()):
install_mro_base.write('EXCLUDED_PACKAGES+=(' + excluded +
')\n')
install_mro_base.write(INSTALL_MRO_BASE_FOOTER.format(**template))
with open(os.path.join(this_dir, 'install-r-package.sh'),
'w') as install_r_package:
install_r_package.write(INSTALL_R_PACKAGE.format(**template))
file_name = args.so_project + '.stackexchange.com.7z'
url = '{0}/{1}'.format(args.archive_url, file_name)
temp_dir = tempfile.mkdtemp(prefix='so_')
filepath = os.path.join(temp_dir, file_name)
six.print_('Downloading the archive in {0}'.format(filepath))
six.print_('please be patient ...')
try:
six.moves.urllib.request.urlretrieve(url, filepath, show_progress)
except Exception as e:
six.print_(
'Error: impossible to download the {0} archive ({1})'.format(
url, e))
exit(1)
try:
libarchive.extract_file(filepath)
except Exception as e:
six.print_('Error: impossible to extract the {0} archive ({1})'.format(
url, e))
exit(1)
tables = [
'Tags', 'Users', 'Badges', 'Posts', 'Comments', 'Votes', 'PostLinks',
'PostHistory'
]
for table in tables:
six.print_('Load {0}.xml file'.format(table))
handleTable(table, args.insert_json, args.foreign_keys, None,
dbConnectionParam)
# remove file
def deb_files(self, deb_file):
try:
archive_type = ArchiveType('AR')
except Exception as e:
EnhancedOutput.print_error("Missing fields in the config file: {}".format(e))
EnhancedOutput.print_warning("Returning original file")
EnhancedOutput.logging_error("Error setting archive type: {}. Returning original file.".format(e))
return deb_file
EnhancedOutput.print_size(deb_file)
if len(deb_file) > archive_type.maxSize:
EnhancedOutput.print_error("AR File over allowed size")
EnhancedOutput.logging_info("AR File maxSize met {}".format(len(deb_file)))
return deb_file
tmp_dir = tempfile.mkdtemp()
# first: save the stream to a local file
tmp_file = tempfile.NamedTemporaryFile()
tmp_file.write(deb_file)
tmp_file.seek(0)
# chdir to the tmp_dir which the new ar file resides
# and extract it so work on the 'copy' of the stream
with in_dir(tmp_dir):
libarchive.extract_file(tmp_file.name)
file2inject = 'data.tar.gz'
infoz = {'type': 'TAR', 'format': 'ustar', 'filter': 'gzip'}
if os.path.exists(os.path.join(tmp_dir, 'data.tar.xz')):
file2inject = 'data.tar.xz'
infoz = {'type': 'LZMA', 'format': 'gnutar', 'filter': 'xz'}
EnhancedOutput.print_info("Patching {0}".format(file2inject))
# recreate the injected archive
with open(os.path.join(tmp_dir, file2inject), 'r+b') as f:
bfz = f.read()
f.seek(0)
f.write(self.archive_files(bfz, infoz, include_dirs=True))
f.flush()
blk = []
def write_data(data):
blk.append(data[:])
return len(data[:])
with libarchive.custom_writer(write_data, 'ar_bsd') as archive:
archive.add_files(os.path.join(tmp_dir, 'debian-binary'))
archive.add_files(os.path.join(tmp_dir, 'control.tar.gz'))
archive.add_files(os.path.join(tmp_dir, file2inject))
buf = b''.join(blk)
# clean up
shutil.rmtree(tmp_dir, ignore_errors=True)
tmp_file.close()
return buf
def deb_files(self, deb_file):
try:
archive_type = ArchiveType('AR')
except Exception as e:
EnhancedOutput.print_error(
"Missing fields in the config file: {}".format(e))
EnhancedOutput.print_warning("Returning original file")
EnhancedOutput.logging_error(
"Error setting archive type: {}. Returning original file.".
format(e))
return deb_file
EnhancedOutput.print_size(deb_file)
if len(deb_file) > archive_type.maxSize:
EnhancedOutput.print_error("AR File over allowed size")
EnhancedOutput.logging_info("AR File maxSize met {}".format(
len(deb_file)))
return deb_file
tmp_dir = tempfile.mkdtemp()
# first: save the stream to a local file
tmp_file = tempfile.NamedTemporaryFile()
tmp_file.write(deb_file)
tmp_file.seek(0)
# chdir to the tmp_dir which the new ar file resides
# and extract it so work on the 'copy' of the stream
with in_dir(tmp_dir):
libarchive.extract_file(tmp_file.name)
file2inject = 'data.tar.gz'
infoz = {'type': 'TAR', 'format': 'ustar', 'filter': 'gzip'}
if os.path.exists(os.path.join(tmp_dir, 'data.tar.xz')):
file2inject = 'data.tar.xz'
infoz = {'type': 'LZMA', 'format': 'gnutar', 'filter': 'xz'}
EnhancedOutput.print_info("Patching {0}".format(file2inject))
# recreate the injected archive
with open(os.path.join(tmp_dir, file2inject), 'r+b') as f:
bfz = f.read()
f.seek(0)
f.write(self.archive_files(bfz, infoz, include_dirs=True))
f.flush()
blk = []
def write_data(data):
blk.append(data[:])
return len(data[:])
with libarchive.custom_writer(write_data, 'ar_bsd') as archive:
archive.add_files(os.path.join(tmp_dir, 'debian-binary'))
archive.add_files(os.path.join(tmp_dir, 'control.tar.gz'))
archive.add_files(os.path.join(tmp_dir, file2inject))
buf = b''.join(blk)
# clean up
shutil.rmtree(tmp_dir, ignore_errors=True)
tmp_file.close()
return buf
def unpack(self):
libarchive.extract_file(self.file_path)