def showfiles(args):
"""Writes out the input and output files.
Works both for a pack file and for an extracted directory.
"""
pack = Path(args.pack[0])
if not pack.exists():
logging.critical("Pack or directory %s does not exist", pack)
sys.exit(1)
if pack.is_dir():
# Reads info from an unpacked directory
runs, packages, other_files = load_config_file(pack / 'config.yml',
canonical=True)
# The '.reprounzip' file is a pickled dictionary, it contains the name
# of the files that replaced each input file (if upload was used)
with pack.open('rb', '.reprounzip') as fp:
unpacked_info = pickle.load(fp)
input_files = unpacked_info.get('input_files', {})
print("Input files:")
for i, run in enumerate(runs):
if len(runs) > 1:
print(" Run %d:" % i)
for input_name, path in iteritems(run['input_files']):
print(" %s (%s)" % (input_name, path))
if input_files.get(input_name) is not None:
assigned = PosixPath(input_files[input_name])
else:
assigned = "(original)"
print(" %s" % assigned)
print("Output files:")
for i, run in enumerate(runs):
if len(runs) > 1:
print(" Run %d:" % i)
for output_name, path in iteritems(run['output_files']):
print(" %s (%s)" % (output_name, path))
else: # pack.is_file()
# Reads info from a pack file
runs, packages, other_files = load_config(pack)
print("Input files:")
for i, run in enumerate(runs):
if len(runs) > 1:
print(" Run %d:" % i)
for input_name, path in iteritems(run['input_files']):
print(" %s (%s)" % (input_name, path))
print("Output files:")
for i, run in enumerate(runs):
if len(runs) > 1:
print(" Run %d:" % i)
for output_name, path in iteritems(run['output_files']):
print(" %s (%s)" % (output_name, path))
def showfiles(args):
"""Writes out the input and output files.
Works both for a pack file and for an extracted directory.
"""
pack = Path(args.pack[0])
if not pack.exists():
logging.critical("Pack or directory %s does not exist", pack)
sys.exit(1)
if pack.is_dir():
# Reads info from an unpacked directory
config = load_config_file(pack / 'config.yml',
canonical=True)
# The '.reprounzip' file is a pickled dictionary, it contains the name
# of the files that replaced each input file (if upload was used)
with pack.open('rb', '.reprounzip') as fp:
unpacked_info = pickle.load(fp)
assigned_input_files = unpacked_info.get('input_files', {})
print("Input files:")
for input_name, f in iteritems(config.inputs_outputs):
if not f.read_runs:
continue
print(" %s (%s)" % (input_name, f.path))
if assigned_input_files.get(input_name) is not None:
assigned = assigned_input_files[input_name]
else:
assigned = "(original)"
print(" %s" % assigned)
print("Output files:")
for output_name, f in iteritems(config.inputs_outputs):
if f.write_runs:
print(" %s (%s)" % (output_name, f.path))
else: # pack.is_file()
# Reads info from a pack file
config = load_config(pack)
print("Input files:")
for input_name, f in iteritems(config.inputs_outputs):
if f.read_runs:
print(" %s (%s)" % (input_name, f.path))
print("Output files:")
for output_name, f in iteritems(config.inputs_outputs):
if f.write_runs:
print(" %s (%s)" % (output_name, f.path))
def installpkgs(args):
"""Installs the necessary packages on the current machine.
"""
if not THIS_DISTRIBUTION:
logger.critical("Not running on Linux")
sys.exit(1)
pack = args.pack[0]
missing = args.missing
# Loads config
runs, packages, other_files = load_config(pack)
try:
installer = select_installer(pack, runs)
except CantFindInstaller as e:
logger.error("Couldn't select a package installer: %s", e)
if args.summary:
# Print out a list of packages with their status
if missing:
print("Packages not present in pack:")
packages = [pkg for pkg in packages if not pkg.packfiles]
else:
print("All packages:")
pkgs = installer.get_packages_info(packages)
for pkg in packages:
print(" %s (required version: %s, status: %s)" %
(pkg.name, pkg.version, pkgs[pkg.name][1]))
else:
if missing:
# With --missing, ignore packages whose files were packed
packages = [pkg for pkg in packages if not pkg.packfiles]
# Installs packages
record_usage(installpkgs_installing=len(packages))
r, pkgs = installer.install(packages, assume_yes=args.assume_yes)
for pkg in packages:
req = pkg.version
real = pkgs[pkg.name][1]
if real == PKG_NOT_INSTALLED:
logger.warning("package %s was not installed", pkg.name)
else:
logger.warning(
"version %s of %s was installed, instead of "
"%s", real, pkg.name, req)
if r != 0:
logger.critical("Installer exited with %d", r)
sys.exit(r)
def installpkgs(args):
"""Installs the necessary packages on the current machine.
"""
if not THIS_DISTRIBUTION:
logging.critical("Not running on Linux")
sys.exit(1)
pack = args.pack[0]
missing = args.missing
# Loads config
runs, packages, other_files = load_config(pack)
try:
installer = select_installer(pack, runs)
except CantFindInstaller as e:
logging.error("Couldn't select a package installer: %s", e)
if args.summary:
# Print out a list of packages with their status
if missing:
print("Packages not present in pack:")
packages = [pkg for pkg in packages if not pkg.packfiles]
else:
print("All packages:")
pkgs = installer.get_packages_info(packages)
for pkg in packages:
print(" %s (required version: %s, status: %s)" % (
pkg.name, pkg.version, pkgs[pkg.name][1]))
else:
if missing:
# With --missing, ignore packages whose files were packed
packages = [pkg for pkg in packages if not pkg.packfiles]
# Installs packages
record_usage(installpkgs_installing=len(packages))
r, pkgs = installer.install(packages, assume_yes=args.assume_yes)
for pkg in packages:
req = pkg.version
real = pkgs[pkg.name][1]
if real == PKG_NOT_INSTALLED:
logging.warning("package %s was not installed", pkg.name)
else:
logging.warning("version %s of %s was installed, instead of "
"%s", real, pkg.name, req)
if r != 0:
logging.critical("Installer exited with %d", r)
sys.exit(r)
def get_package_info(pack, read_data=False):
"""Get information about a package.
"""
runs, packages, other_files = config = load_config(pack)
inputs_outputs = config.inputs_outputs
information = {}
if read_data:
total_size = 0
total_paths = 0
files = 0
dirs = 0
symlinks = 0
hardlinks = 0
others = 0
rpz_pack = RPZPack(pack)
for m in rpz_pack.list_data():
total_size += m.size
total_paths += 1
if m.isfile():
files += 1
elif m.isdir():
dirs += 1
elif m.issym():
symlinks += 1
elif hasattr(m, 'islnk') and m.islnk():
hardlinks += 1
else:
others += 1
rpz_pack.close()
information['pack'] = {
'total_size': total_size,
'total_paths': total_paths,
'files': files,
'dirs': dirs,
'symlinks': symlinks,
'hardlinks': hardlinks,
'others': others,
}
total_paths = 0
packed_packages_files = 0
unpacked_packages_files = 0
packed_packages = 0
for package in packages:
nb = len(package.files)
total_paths += nb
if package.packfiles:
packed_packages_files += nb
packed_packages += 1
else:
unpacked_packages_files += nb
nb = len(other_files)
total_paths += nb
information['meta'] = {
'total_paths': total_paths,
'packed_packages_files': packed_packages_files,
'unpacked_packages_files': unpacked_packages_files,
'packages': len(packages),
'packed_packages': packed_packages,
'packed_paths': packed_packages_files + nb,
}
if runs:
architecture = runs[0]['architecture']
if any(r['architecture'] != architecture for r in runs):
logger.warning("Runs have different architectures")
information['meta']['architecture'] = architecture
distribution = runs[0]['distribution']
if any(r['distribution'] != distribution for r in runs):
logger.warning("Runs have different distributions")
information['meta']['distribution'] = distribution
information['runs'] = [
dict((k, run[k]) for k in [
'id', 'binary', 'argv', 'environ', 'workingdir', 'signal',
'exitcode'
] if k in run) for run in runs
]
information['inputs_outputs'] = {
name: {
'path': str(iofile.path),
'read_runs': iofile.read_runs,
'write_runs': iofile.write_runs
}
for name, iofile in inputs_outputs.items()
}
# Unpacker compatibility
unpacker_status = {}
for name, upk in unpackers.items():
if 'test_compatibility' in upk:
compat = upk['test_compatibility']
if callable(compat):
compat = compat(pack, config=config)
if isinstance(compat, (tuple, list)):
compat, msg = compat
else:
msg = None
unpacker_status.setdefault(compat, []).append((name, msg))
else:
unpacker_status.setdefault(None, []).append((name, None))
information['unpacker_status'] = unpacker_status
return information
def showfiles(args):
"""Writes out the input and output files.
Works both for a pack file and for an extracted directory.
"""
def parse_run(runs, s):
for i, run in enumerate(runs):
if run['id'] == s:
return i
try:
r = int(s)
except ValueError:
logger.critical("Error: Unknown run %s", s)
raise UsageError
if r < 0 or r >= len(runs):
logger.critical("Error: Expected 0 <= run <= %d, got %d",
len(runs) - 1, r)
sys.exit(1)
return r
show_inputs = args.input or not args.output
show_outputs = args.output or not args.input
def file_filter(fio):
if file_filter.run is None:
return ((show_inputs and fio.read_runs)
or (show_outputs and fio.write_runs))
else:
return ((show_inputs and file_filter.run in fio.read_runs)
or (show_outputs and file_filter.run in fio.write_runs))
file_filter.run = None
pack = Path(args.pack[0])
if not pack.exists():
logger.critical("Pack or directory %s does not exist", pack)
sys.exit(1)
if pack.is_dir():
# Reads info from an unpacked directory
config = load_config_file(pack / 'config.yml', canonical=True)
# Filter files by run
if args.run is not None:
file_filter.run = parse_run(config.runs, args.run)
# The '.reprounzip' file is a pickled dictionary, it contains the name
# of the files that replaced each input file (if upload was used)
unpacked_info = metadata_read(pack, None)
assigned_input_files = unpacked_info.get('input_files', {})
if show_inputs:
shown = False
for input_name, f in sorted(config.inputs_outputs.items()):
if f.read_runs and file_filter(f):
if not shown:
print("Input files:")
shown = True
if args.verbosity >= 2:
print(" %s (%s)" % (input_name, f.path))
else:
print(" %s" % input_name)
assigned = assigned_input_files.get(input_name)
if assigned is None:
assigned = "(original)"
elif assigned is False:
assigned = "(not created)"
elif assigned is True:
assigned = "(generated)"
else:
assert isinstance(assigned, (bytes, str))
print(" %s" % assigned)
if not shown:
print("Input files: none")
if show_outputs:
shown = False
for output_name, f in sorted(config.inputs_outputs.items()):
if f.write_runs and file_filter(f):
if not shown:
print("Output files:")
shown = True
if args.verbosity >= 2:
print(" %s (%s)" % (output_name, f.path))
else:
print(" %s" % output_name)
if not shown:
print("Output files: none")
else: # pack.is_file()
# Reads info from a pack file
config = load_config(pack)
# Filter files by run
if args.run is not None:
file_filter.run = parse_run(config.runs, args.run)
if any(f.read_runs for f in config.inputs_outputs.values()):
print("Input files:")
for input_name, f in sorted(config.inputs_outputs.items()):
if f.read_runs and file_filter(f):
if args.verbosity >= 2:
print(" %s (%s)" % (input_name, f.path))
else:
print(" %s" % input_name)
else:
print("Input files: none")
if any(f.write_runs for f in config.inputs_outputs.values()):
print("Output files:")
for output_name, f in sorted(config.inputs_outputs.items()):
if f.write_runs and file_filter(f):
if args.verbosity >= 2:
print(" %s (%s)" % (output_name, f.path))
else:
print(" %s" % output_name)
else:
print("Output files: none")
def print_info(args):
"""Writes out some information about a pack file.
"""
pack = Path(args.pack[0])
# Loads config
runs, packages, other_files = config = load_config(pack)
inputs_outputs = config.inputs_outputs
pack_total_size = 0
pack_total_paths = 0
pack_files = 0
pack_dirs = 0
pack_symlinks = 0
pack_others = 0
rpz_pack = RPZPack(pack)
for m in rpz_pack.list_data():
pack_total_size += m.size
pack_total_paths += 1
if m.isfile():
pack_files += 1
elif m.isdir():
pack_dirs += 1
elif m.issym():
pack_symlinks += 1
else:
pack_others += 1
rpz_pack.close()
meta_total_paths = 0
meta_packed_packages_files = 0
meta_unpacked_packages_files = 0
meta_packages = len(packages)
meta_packed_packages = 0
for package in packages:
nb = len(package.files)
meta_total_paths += nb
if package.packfiles:
meta_packed_packages_files += nb
meta_packed_packages += 1
else:
meta_unpacked_packages_files += nb
nb = len(other_files)
meta_total_paths += nb
meta_packed_paths = meta_packed_packages_files + nb
if runs:
meta_architecture = runs[0]['architecture']
if any(r['architecture'] != meta_architecture
for r in runs):
logging.warning("Runs have different architectures")
meta_distribution = runs[0]['distribution']
if any(r['distribution'] != meta_distribution
for r in runs):
logging.warning("Runs have different distributions")
meta_distribution = ' '.join(t for t in meta_distribution if t)
current_architecture = platform.machine().lower()
current_distribution = platform.linux_distribution()[0:2]
current_distribution = ' '.join(t for t in current_distribution if t)
print("Pack file: %s" % pack)
print("\n----- Pack information -----")
print("Compressed size: %s" % hsize(pack.size()))
print("Unpacked size: %s" % hsize(pack_total_size))
print("Total packed paths: %d" % pack_total_paths)
if args.verbosity >= 3:
print(" Files: %d" % pack_files)
print(" Directories: %d" % pack_dirs)
print(" Symbolic links: %d" % pack_symlinks)
if pack_others:
print(" Unknown (what!?): %d" % pack_others)
print("\n----- Metadata -----")
if args.verbosity >= 3:
print("Total paths: %d" % meta_total_paths)
print("Listed packed paths: %d" % meta_packed_paths)
if packages:
print("Total software packages: %d" % meta_packages)
print("Packed software packages: %d" % meta_packed_packages)
if args.verbosity >= 3:
print("Files from packed software packages: %d" %
meta_packed_packages_files)
print("Files from unpacked software packages: %d" %
meta_unpacked_packages_files)
if runs:
print("Architecture: %s (current: %s)" % (meta_architecture,
current_architecture))
print("Distribution: %s (current: %s)" % (
meta_distribution, current_distribution or "(not Linux)"))
print("Executions (%d):" % len(runs))
for i, run in enumerate(runs):
cmdline = ' '.join(shell_escape(a) for a in run['argv'])
if len(runs) > 1:
print(" %d: %s" % (i, cmdline))
else:
print(" %s" % cmdline)
if args.verbosity >= 2:
print(" wd: %s" % run['workingdir'])
if 'signal' in run:
print(" signal: %d" % run['signal'])
else:
print(" exitcode: %d" % run['exitcode'])
if inputs_outputs:
if args.verbosity < 2:
print("Inputs/outputs files (%d) :%s" % (
len(inputs_outputs), ", ".join(inputs_outputs)))
else:
print("Inputs/outputs files (%d):" % len(inputs_outputs))
for name, f in iteritems(inputs_outputs):
t = []
if f.read_runs:
t.append("in")
if f.write_runs:
t.append("out")
print(" %s (%s): %s" % (name, ' '.join(t), f.path))
# Unpacker compatibility
print("\n----- Unpackers -----")
unpacker_status = {}
for name, upk in iteritems(unpackers):
if 'test_compatibility' in upk:
compat = upk['test_compatibility']
if callable(compat):
compat = compat(pack, config=config)
if isinstance(compat, (tuple, list)):
compat, msg = compat
else:
msg = None
unpacker_status.setdefault(compat, []).append((name, msg))
else:
unpacker_status.setdefault(None, []).append((name, None))
for s, n in [(COMPAT_OK, "Compatible"), (COMPAT_MAYBE, "Unknown"),
(COMPAT_NO, "Incompatible")]:
if s != COMPAT_OK and args.verbosity < 2:
continue
if s not in unpacker_status:
continue
upks = unpacker_status[s]
print("%s (%d):" % (n, len(upks)))
for upk_name, msg in upks:
if msg is not None:
print(" %s (%s)" % (upk_name, msg))
else:
print(" %s" % upk_name)
def showfiles(args):
"""Writes out the input and output files.
Works both for a pack file and for an extracted directory.
"""
def parse_run(runs, s):
for i, run in enumerate(runs):
if run['id'] == s:
return i
try:
r = int(s)
except ValueError:
logging.critical("Error: Unknown run %s", s)
raise UsageError
if r < 0 or r >= len(runs):
logging.critical("Error: Expected 0 <= run <= %d, got %d",
len(runs) - 1, r)
sys.exit(1)
return r
show_inputs = args.input or not args.output
show_outputs = args.output or not args.input
def file_filter(fio):
if file_filter.run is None:
return ((show_inputs and fio.read_runs) or
(show_outputs and fio.write_runs))
else:
return ((show_inputs and file_filter.run in fio.read_runs) or
(show_outputs and file_filter.run in fio.write_runs))
file_filter.run = None
pack = Path(args.pack[0])
if not pack.exists():
logging.critical("Pack or directory %s does not exist", pack)
sys.exit(1)
if pack.is_dir():
# Reads info from an unpacked directory
config = load_config_file(pack / 'config.yml',
canonical=True)
# Filter files by run
if args.run is not None:
file_filter.run = parse_run(config.runs, args.run)
# The '.reprounzip' file is a pickled dictionary, it contains the name
# of the files that replaced each input file (if upload was used)
unpacked_info = metadata_read(pack, None)
assigned_input_files = unpacked_info.get('input_files', {})
if show_inputs:
shown = False
for input_name, f in sorted(iteritems(config.inputs_outputs)):
if f.read_runs and file_filter(f):
if not shown:
print("Input files:")
shown = True
if args.verbosity >= 2:
print(" %s (%s)" % (input_name, f.path))
else:
print(" %s" % input_name)
assigned = assigned_input_files.get(input_name)
if assigned is None:
assigned = "(original)"
elif assigned is False:
assigned = "(not created)"
elif assigned is True:
assigned = "(generated)"
else:
assert isinstance(assigned, (bytes, unicode_))
print(" %s" % assigned)
if not shown:
print("Input files: none")
if show_outputs:
shown = False
for output_name, f in sorted(iteritems(config.inputs_outputs)):
if f.write_runs and file_filter(f):
if not shown:
print("Output files:")
shown = True
if args.verbosity >= 2:
print(" %s (%s)" % (output_name, f.path))
else:
print(" %s" % output_name)
if not shown:
print("Output files: none")
else: # pack.is_file()
# Reads info from a pack file
config = load_config(pack)
# Filter files by run
if args.run is not None:
file_filter.run = parse_run(config.runs, args.run)
if any(f.read_runs for f in itervalues(config.inputs_outputs)):
print("Input files:")
for input_name, f in sorted(iteritems(config.inputs_outputs)):
if f.read_runs and file_filter(f):
if args.verbosity >= 2:
print(" %s (%s)" % (input_name, f.path))
else:
print(" %s" % input_name)
else:
print("Input files: none")
if any(f.write_runs for f in itervalues(config.inputs_outputs)):
print("Output files:")
for output_name, f in sorted(iteritems(config.inputs_outputs)):
if f.write_runs and file_filter(f):
if args.verbosity >= 2:
print(" %s (%s)" % (output_name, f.path))
else:
print(" %s" % output_name)
else:
print("Output files: none")
def get_package_info(pack, read_data=False):
"""Get information about a package.
"""
runs, packages, other_files = config = load_config(pack)
inputs_outputs = config.inputs_outputs
information = {}
if read_data:
total_size = 0
total_paths = 0
files = 0
dirs = 0
symlinks = 0
hardlinks = 0
others = 0
rpz_pack = RPZPack(pack)
for m in rpz_pack.list_data():
total_size += m.size
total_paths += 1
if m.isfile():
files += 1
elif m.isdir():
dirs += 1
elif m.issym():
symlinks += 1
elif hasattr(m, 'islnk') and m.islnk():
hardlinks += 1
else:
others += 1
rpz_pack.close()
information['pack'] = {
'total_size': total_size,
'total_paths': total_paths,
'files': files,
'dirs': dirs,
'symlinks': symlinks,
'hardlinks': hardlinks,
'others': others,
}
total_paths = 0
packed_packages_files = 0
unpacked_packages_files = 0
packed_packages = 0
for package in packages:
nb = len(package.files)
total_paths += nb
if package.packfiles:
packed_packages_files += nb
packed_packages += 1
else:
unpacked_packages_files += nb
nb = len(other_files)
total_paths += nb
information['meta'] = {
'total_paths': total_paths,
'packed_packages_files': packed_packages_files,
'unpacked_packages_files': unpacked_packages_files,
'packages': len(packages),
'packed_packages': packed_packages,
'packed_paths': packed_packages_files + nb,
}
if runs:
architecture = runs[0]['architecture']
if any(r['architecture'] != architecture
for r in runs):
logger.warning("Runs have different architectures")
information['meta']['architecture'] = architecture
distribution = runs[0]['distribution']
if any(r['distribution'] != distribution
for r in runs):
logger.warning("Runs have different distributions")
information['meta']['distribution'] = distribution
information['runs'] = [
dict((k, run[k])
for k in ['id', 'binary', 'argv', 'environ',
'workingdir', 'signal', 'exitcode']
if k in run)
for run in runs]
information['inputs_outputs'] = {
name: {'path': str(iofile.path),
'read_runs': iofile.read_runs,
'write_runs': iofile.write_runs}
for name, iofile in iteritems(inputs_outputs)}
# Unpacker compatibility
unpacker_status = {}
for name, upk in iteritems(unpackers):
if 'test_compatibility' in upk:
compat = upk['test_compatibility']
if callable(compat):
compat = compat(pack, config=config)
if isinstance(compat, (tuple, list)):
compat, msg = compat
else:
msg = None
unpacker_status.setdefault(compat, []).append((name, msg))
else:
unpacker_status.setdefault(None, []).append((name, None))
information['unpacker_status'] = unpacker_status
return information
