Beispiel #1
0
def find_all_links(filename, include_target=False):
    """Dereferences symlinks from a path.

    If include_target is True, this also returns the real path of the final
    target.

    Example:
        /
            a -> b
            b
                g -> c
                c -> ../a/d
                d
                    e -> /f
            f
    >>> find_all_links('/a/g/e', True)
    ['/a', '/b/c', '/b/g', '/b/d/e', '/f']
    """
    files = set()
    filename = Path(filename)
    assert filename.absolute()
    path = find_all_links_recursive(filename, files)
    files = list(files)
    if include_target:
        files.append(path)
    return files
Beispiel #2
0
    def search_for_files(self, files):
        # Make a set of all the requested files
        requested = dict((f.path, f) for f in self.filter_files(files))
        found = {}  # {path: pkgname}

        # Request a few files at a time so we don't hit the command-line size
        # limit
        iter_batch = iter(requested)
        while True:
            batch = list(itertools.islice(iter_batch, MAX_ARGV))
            if not batch:
                break

            proc = subprocess.Popen(['dpkg-query', '-S'] +
                                    [path.path for path in batch],
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
            out, err = proc.communicate()
            for l in out.splitlines():
                pkgname, path = l.split(b': ', 1)
                path = Path(path.strip())
                # 8-bit safe encoding, because this might be a localized error
                # message (that we don't care about)
                pkgname = pkgname.decode('iso-8859-1')
                if ', ' in pkgname:  # Multiple packages
                    found[path] = None
                    continue
                pkgname = pkgname.split(':', 1)[0]  # Remove :arch
                if path in requested:
                    if ' ' not in pkgname:
                        # If we had assigned it to a package already, undo
                        if path in found:
                            found[path] = None
                        # Else assign to the package
                        else:
                            found[path] = pkgname

        # Remaining files are not from packages
        self.unknown_files.update(
            f for f in files
            if f.path in requested and found.get(f.path) is None)

        nb_pkg_files = 0

        for path, pkgname in iteritems(found):
            if pkgname is None:
                continue
            if pkgname in self.packages:
                package = self.packages[pkgname]
            else:
                package = self._create_package(pkgname)
                self.packages[pkgname] = package
            package.add_file(requested.pop(path))
            nb_pkg_files += 1

        logger.info("%d packages with %d files, and %d other files",
                    len(self.packages),
                    nb_pkg_files,
                    len(self.unknown_files))
Beispiel #3
0
def showfiles(args):
    """Writes out the input and output files.

    Works both for a pack file and for an extracted directory.
    """
    pack = Path(args.pack[0])

    if not pack.exists():
        logging.critical("Pack or directory %s does not exist", pack)
        sys.exit(1)

    if pack.is_dir():
        # Reads info from an unpacked directory
        runs, packages, other_files = load_config_file(pack / 'config.yml',
                                                       canonical=True)
        # The '.reprounzip' file is a pickled dictionary, it contains the name
        # of the files that replaced each input file (if upload was used)
        with pack.open('rb', '.reprounzip') as fp:
            unpacked_info = pickle.load(fp)
        input_files = unpacked_info.get('input_files', {})

        print("Input files:")
        for i, run in enumerate(runs):
            if len(runs) > 1:
                print("  Run %d:" % i)
            for input_name, path in iteritems(run['input_files']):
                print("    %s (%s)" % (input_name, path))
                if input_files.get(input_name) is not None:
                    assigned = PosixPath(input_files[input_name])
                else:
                    assigned = "(original)"
                print("      %s" % assigned)

        print("Output files:")
        for i, run in enumerate(runs):
            if len(runs) > 1:
                print("  Run %d:" % i)
            for output_name, path in iteritems(run['output_files']):
                print("    %s (%s)" % (output_name, path))

    else:  # pack.is_file()
        # Reads info from a pack file
        runs, packages, other_files = load_config(pack)

        print("Input files:")
        for i, run in enumerate(runs):
            if len(runs) > 1:
                print("  Run %d:" % i)
            for input_name, path in iteritems(run['input_files']):
                print("    %s (%s)" % (input_name, path))

        print("Output files:")
        for i, run in enumerate(runs):
            if len(runs) > 1:
                print("  Run %d:" % i)
            for output_name, path in iteritems(run['output_files']):
                print("    %s (%s)" % (output_name, path))
Beispiel #4
0
def vagrant_destroy_dir(args):
    """Destroys the directory.
    """
    target = Path(args.target[0])
    read_dict(target)

    signals.pre_destroy(target=target)
    target.rmtree()
    signals.post_destroy(target=target)
Beispiel #5
0
def docker_destroy_dir(args):
    """Destroys the directory.
    """
    target = Path(args.target[0])
    read_dict(target / '.reprounzip')

    logging.info("Removing directory %s...", target)
    signals.pre_destroy(target=target)
    target.rmtree()
    signals.post_destroy(target=target)
Beispiel #6
0
def showfiles(args):
    """Writes out the input and output files.

    Works both for a pack file and for an extracted directory.
    """
    pack = Path(args.pack[0])

    if not pack.exists():
        logging.critical("Pack or directory %s does not exist", pack)
        sys.exit(1)

    if pack.is_dir():
        # Reads info from an unpacked directory
        config = load_config_file(pack / 'config.yml',
                                  canonical=True)
        # The '.reprounzip' file is a pickled dictionary, it contains the name
        # of the files that replaced each input file (if upload was used)
        with pack.open('rb', '.reprounzip') as fp:
            unpacked_info = pickle.load(fp)
        assigned_input_files = unpacked_info.get('input_files', {})

        print("Input files:")
        for input_name, f in iteritems(config.inputs_outputs):
            if not f.read_runs:
                continue
            print("    %s (%s)" % (input_name, f.path))
            if assigned_input_files.get(input_name) is not None:
                assigned = assigned_input_files[input_name]
            else:
                assigned = "(original)"
            print("      %s" % assigned)

        print("Output files:")
        for output_name, f in iteritems(config.inputs_outputs):
            if f.write_runs:
                print("    %s (%s)" % (output_name, f.path))

    else:  # pack.is_file()
        # Reads info from a pack file
        config = load_config(pack)

        print("Input files:")
        for input_name, f in iteritems(config.inputs_outputs):
            if f.read_runs:
                print("    %s (%s)" % (input_name, f.path))

        print("Output files:")
        for output_name, f in iteritems(config.inputs_outputs):
            if f.write_runs:
                print("    %s (%s)" % (output_name, f.path))
Beispiel #7
0
def load_config(pack):
    """Utility method loading the YAML configuration from inside a pack file.

    Decompresses the config.yml file from the tarball to a temporary file then
    loads it. Note that decompressing a single file is inefficient, thus
    calling this method can be slow.
    """
    tmp = Path.tempdir(prefix='reprozip_')
    try:
        # Loads info from package
        tar = tarfile.open(str(pack), 'r:*')
        f = tar.extractfile('METADATA/version')
        version = f.read()
        f.close()
        if version != b'REPROZIP VERSION 1\n':
            logging.critical("Unknown pack format")
            sys.exit(1)
        tar.extract('METADATA/config.yml', path=str(tmp))
        tar.close()
        configfile = tmp / 'METADATA/config.yml'
        ret = reprounzip.common.load_config(configfile, canonical=True)
    finally:
        tmp.rmtree()

    return ret
Beispiel #8
0
 def _extract_file(self, member, target):
     member = copy.copy(member)
     member.name = str(target.components[-1])
     self.tar.extract(member,
                      path=str(Path.cwd() / target.parent))
     target.chmod(0o644)
     assert target.is_file()
Beispiel #9
0
def graph(args):
    """graph subcommand.

    Reads in the trace sqlite3 database and writes out a graph in GraphViz DOT
    format.
    """
    if args.pack is not None:
        tmp = Path.tempdir(prefix='reprounzip_')
        try:
            tar = tarfile.open(args.pack, 'r:*')
            f = tar.extractfile('METADATA/version')
            version = f.read()
            f.close()
            if version != b'REPROZIP VERSION 1\n':
                logging.critical("Unknown pack format")
                sys.exit(1)
            try:
                tar.extract('METADATA/config.yml', path=str(tmp))
                tar.extract('METADATA/trace.sqlite3', path=str(tmp))
            except KeyError as e:
                logging.critical("Error extracting from pack: %s", e.args[0])
            generate(Path(args.target[0]),
                     tmp / 'METADATA',
                     args.all_forks)
        finally:
            tmp.rmtree()
    else:
        generate(Path(args.target[0]), Path(args.dir), args.all_forks)
Beispiel #10
0
def trace(binary, argv, directory, append, verbosity=1):
    """Main function for the trace subcommand.
    """
    cwd = Path.cwd()
    if (any(cwd.lies_under(c) for c in magic_dirs + system_dirs) and
            not cwd.lies_under('/usr/local')):
        logging.warning(
            "You are running this experiment from a system directory! "
            "Autodetection of non-system files will probably not work as "
            "intended")

    # Trace directory
    if not append:
        if directory.exists():
            logging.info("Removing existing directory %s", directory)
            directory.rmtree()
        directory.mkdir(parents=True)
    else:
        if not directory.exists():
            logging.warning("--continue was specified but %s does not exist "
                            "-- creating", directory)
            directory.mkdir(parents=True)

    # Runs the trace
    database = directory / 'trace.sqlite3'
    logging.info("Running program")
    # Might raise _pytracer.Error
    c = _pytracer.execute(binary, argv, database.path, verbosity)
    if c != 0:
        if c & 0x0100:
            logging.warning("Program appears to have been terminated by "
                            "signal %d", c & 0xFF)
        else:
            logging.warning("Program exited with non-zero code %d", c)
    logging.info("Program completed")
Beispiel #11
0
    def test_cwd(self):
        """Tests cwd, in_dir."""
        cwd = os.getcwd()

        if os.name == 'nt' and isinstance(cwd, bytes):
            cwd = cwd.decode('mbcs')
        elif os.name != 'nt' and isinstance(cwd, unicode):
            cwd = cwd.encode(sys.getfilesystemencoding())
        self.assertEqual(Path.cwd().path, cwd)
        tmp = Path.tempdir()
        with tmp.in_dir():
            self.assertEqual(Path.cwd(), tmp)
        self.assertNotEqual(Path.cwd(), tmp)
        self.assertTrue(tmp.exists())
        tmp.rmdir()
        self.assertFalse(tmp.exists())
Beispiel #12
0
def setup_usage_report(name, version):
    """Sets up the usagestats module.
    """
    global _usage_report

    # Unpack CA certificate
    fd, certificate_file = Path.tempfile(prefix='rpz_stats_ca_', suffix='.pem')
    with certificate_file.open('wb') as fp:
        fp.write(usage_report_ca)
    os.close(fd)
    atexit.register(os.remove, certificate_file.path)

    _usage_report = usagestats.Stats(
            '~/.reprozip/usage_stats',
            usagestats.Prompt(enable='%s usage_report --enable' % name,
                              disable='%s usage_report --disable' % name),
            os.environ.get('REPROZIP_USAGE_URL',
                           'https://reprozip-stats.poly.edu/'),
            version='%s %s' % (name, version),
            unique_user_id=True,
            env_var='REPROZIP_USAGE_STATS',
            ssl_verify=certificate_file.path)
    try:
        os.getcwd().encode('ascii')
    except (UnicodeEncodeError, UnicodeDecodeError):
        record_usage(cwd_ascii=False)
    else:
        record_usage(cwd_ascii=True)
Beispiel #13
0
 def wrapper(*args, **kwargs):
     tmp = Path.tempdir(prefix='reprozip_tests_')
     try:
         with tmp.in_dir():
             return f(*args, **kwargs)
     finally:
         tmp.rmtree(ignore_errors=True)
Beispiel #14
0
    def __init__(self, pack):
        self.pack = Path(pack)

        self.tar = tarfile.open(str(self.pack), 'r:*')
        f = self.tar.extractfile('METADATA/version')
        version = f.read()
        f.close()
        if version.startswith(b'REPROZIP VERSION '):
            try:
                version = int(version[17:].rstrip())
            except ValueError:
                version = None
            if version in (1, 2):
                self.version = version
                self.data_prefix = PosixPath(b'DATA')
            else:
                raise ValueError(
                    "Unknown format version %r (maybe you should upgrade "
                    "reprounzip? I only know versions 1 and 2" % version)
        else:
            raise ValueError("File doesn't appear to be a RPZ pack")

        if self.version == 1:
            self.data = self.tar
        elif version == 2:
            self.data = tarfile.open(
                fileobj=self.tar.extractfile('DATA.tar.gz'),
                mode='r:*')
        else:
            assert False
Beispiel #15
0
def testrun(args):
    """testrun subcommand.

    Runs the command with the tracer using a temporary sqlite3 database, then
    reads it and dumps it out.

    Not really useful, except for debugging.
    """
    fd, database = Path.tempfile(prefix='reprozip_', suffix='.sqlite3')
    os.close(fd)
    try:
        if args.arg0 is not None:
            argv = [args.arg0] + args.cmdline[1:]
        else:
            argv = args.cmdline
        logging.debug("Starting tracer, binary=%r, argv=%r",
                      args.cmdline[0], argv)
        c = _pytracer.execute(args.cmdline[0], argv, database.path,
                              args.verbosity)
        print("\n\n-----------------------------------------------------------"
              "--------------------")
        print_db(database)
        if c != 0:
            if c & 0x0100:
                print("\nWarning: program appears to have been terminated by "
                      "signal %d" % (c & 0xFF))
            else:
                print("\nWarning: program exited with non-zero code %d" % c)
    finally:
        database.remove()
Beispiel #16
0
    def prepare_upload(self, files):
        if 'current_image' not in self.unpacked_info:
            sys.stderr.write("Image doesn't exist yet, have you run "
                             "setup/build?\n")
            sys.exit(1)

        self.build_directory = Path.tempdir(prefix='reprozip_build_')
        self.docker_copy = []
Beispiel #17
0
 def _get_package_for_file(self, filename):
     p = subprocess.Popen(['dpkg', '-S', filename.path],
                          stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE)
     out, err = p.communicate()
     for l in out.splitlines():
         pkgname, f = l.split(b': ', 1)
         f = Path(f.strip())
         # 8-bit safe encoding, because this might be a localized error
         # message (that we don't care about)
         pkgname = (pkgname.decode('iso-8859-1')
                           .split(':', 1)[0])    # Removes :arch
         self.package_files[f] = pkgname
         if f == filename:
             if ' ' not in pkgname:
                 return pkgname
     return None
Beispiel #18
0
 def with_trace(self):
     """Context manager that extracts the trace database to a temporary file.
     """
     fd, tmp = Path.tempfile(prefix='reprounzip_')
     os.close(fd)
     self.extract_trace(tmp)
     yield tmp
     tmp.remove()
Beispiel #19
0
def setup_logging(tag, verbosity):
    """Sets up the logging module.
    """
    levels = [logging.CRITICAL, logging.WARNING, logging.INFO, logging.DEBUG]
    console_level = levels[min(verbosity, 3)]
    file_level = logging.INFO
    min_level = min(console_level, file_level)

    # Create formatter, with same format as C extension
    fmt = "[%s] %%(asctime)s %%(levelname)s: %%(message)s" % tag
    formatter = LoggingDateFormatter(fmt)

    # Console logger
    handler = logging.StreamHandler()
    handler.setLevel(console_level)
    handler.setFormatter(formatter)

    # Set up logger
    logger = logging.root
    logger.setLevel(min_level)
    logger.addHandler(handler)

    # File logger
    dotrpz = Path('~/.reprozip').expand_user()
    try:
        if not dotrpz.is_dir():
            dotrpz.mkdir()
        filehandler = logging.handlers.RotatingFileHandler(str(dotrpz / 'log'),
                                                           mode='a',
                                                           delay=False,
                                                           maxBytes=400000,
                                                           backupCount=5)
    except (IOError, OSError):
        logging.warning("Couldn't create log file %s", dotrpz / 'log')
    else:
        filehandler.setFormatter(formatter)
        filehandler.setLevel(file_level)
        logger.addHandler(filehandler)

        filehandler.emit(logging.root.makeRecord(
            __name__.split('.', 1)[0],
            logging.INFO,
            "(log start)", 0,
            "Log opened %s %s",
            (datetime.now().strftime("%Y-%m-%d"), sys.argv),
            None))
Beispiel #20
0
def get_reprozip_ca_certificate():
    """Gets the ReproZip CA certificate filename.
    """
    fd, certificate_file = Path.tempfile(prefix='rpz_stats_ca_', suffix='.pem')
    with certificate_file.open('wb') as fp:
        fp.write(usage_report_ca)
    os.close(fd)
    atexit.register(os.remove, certificate_file.path)
    return certificate_file
Beispiel #21
0
 def download_and_print(self, remote_path):
     # Download to temporary file
     fd, temp = Path.tempfile(prefix='reprozip_output_')
     os.close(fd)
     self.download(remote_path, temp)
     # Output to stdout
     with temp.open('rb') as fp:
         copyfile(fp, stdout_bytes)
     temp.remove()
Beispiel #22
0
def directory_destroy(args):
    """Destroys the directory.
    """
    target = Path(args.target[0])
    metadata_read(target, 'directory')

    logger.info("Removing directory %s...", target)
    signals.pre_destroy(target=target)
    rmtree_fixed(target)
    signals.post_destroy(target=target)
Beispiel #23
0
def directory_destroy(args):
    """Destroys the directory.
    """
    target = Path(args.target[0])
    read_dict(target / '.reprounzip', 'directory')

    logging.info("Removing directory %s...", target)
    signals.pre_destroy(target=target)
    rmtree_fixed(target)
    signals.post_destroy(target=target)
Beispiel #24
0
def vagrant_destroy_vm(args):
    """Destroys the VM through Vagrant.
    """
    target = Path(args.target[0])
    read_dict(target)

    retcode = subprocess.call(['vagrant', 'destroy', '-f'], cwd=target.path)
    if retcode != 0:
        logging.critical("vagrant destroy failed with code %d, ignoring...",
                         retcode)
Beispiel #25
0
    def extract_config(self, target):
        """Extracts the config to the specified path.

        It is up to the caller to remove that file once done.
        """
        member = copy.copy(self.tar.getmember('METADATA/config.yml'))
        member.name = str(target.components[-1])
        self.tar.extract(member, path=str(Path.cwd() / target.parent))
        target.chmod(0o644)
        assert target.is_file()
Beispiel #26
0
    def extract_config(self, target):
        """Extracts the config to the specified path.

        It is up to the caller to remove that file once done.
        """
        member = copy.copy(self.tar.getmember('METADATA/config.yml'))
        member.name = str(target.components[-1])
        self.tar.extract(member,
                         path=str(Path.cwd() / target.parent))
        assert target.is_file()
Beispiel #27
0
def reset(args):
    """reset subcommand.

    Just regenerates the configuration (config.yml) from the trace
    (trace.sqlite3).
    """
    reprozip.tracer.trace.write_configuration(Path(args.dir),
                                              args.identify_packages,
                                              args.find_inputs_outputs,
                                              overwrite=True)
Beispiel #28
0
def trace(binary, argv, directory, append, verbosity=1):
    """Main function for the trace subcommand.
    """
    cwd = Path.cwd()
    if (any(cwd.lies_under(c) for c in magic_dirs + system_dirs)
            and not cwd.lies_under('/usr/local')):
        logging.warning(
            "You are running this experiment from a system directory! "
            "Autodetection of non-system files will probably not work as "
            "intended")

    # Trace directory
    if directory.exists():
        if append is None:
            r = tty_prompt(
                "Trace directory %s exists\n"
                "(a)ppend run to the trace, (d)elete it or (s)top? [a/d/s] " %
                directory, 'aAdDsS')
            if r is None:
                logging.critical(
                    "Trace directory %s exists\n"
                    "Please use either --continue or --overwrite\n", directory)
                sys.exit(125)
            elif r in 'sS':
                sys.exit(125)
            elif r in 'dD':
                directory.rmtree()
                directory.mkdir()
            logging.warning(
                "You can use --overwrite to replace the existing trace "
                "(or --continue to append\nwithout prompt)")
        elif append is False:
            logging.info("Removing existing trace directory %s", directory)
            directory.rmtree()
            directory.mkdir(parents=True)
    else:
        if append is True:
            logging.warning("--continue was set but trace doesn't exist yet")
        directory.mkdir()

    # Runs the trace
    database = directory / 'trace.sqlite3'
    logging.info("Running program")
    # Might raise _pytracer.Error
    c = _pytracer.execute(binary, argv, database.path, verbosity)
    if c != 0:
        if c & 0x0100:
            logging.warning(
                "Program appears to have been terminated by "
                "signal %d", c & 0xFF)
        else:
            logging.warning("Program exited with non-zero code %d", c)
    logging.info("Program completed")

    return c
Beispiel #29
0
def main():
    parser = argparse.ArgumentParser(
        description="Adds __future__ imports to Python files")
    parser.add_argument('-v',
                        '--verbose',
                        action='count',
                        dest='verbosity',
                        default=1)
    parser.add_argument('-e',
                        '--enable',
                        action='append',
                        help="Future import to enable")
    parser.add_argument('file',
                        nargs=argparse.ONE_OR_MORE,
                        help="File or directory in which to replace")
    args = parser.parse_args()
    levels = [logging.CRITICAL, logging.WARNING, logging.INFO, logging.DEBUG]
    logging.basicConfig(level=levels[args.verbosity])

    if not args.enable:
        logging.critical("Nothing to do")
        sys.exit(1)

    enable = set(to_bytes(feature) for feature in args.enable)
    unrecognized = enable - FUTURES
    if unrecognized:
        logging.critical("Error: unknown futures %s" % ', '.join(unrecognized))
        sys.exit(1)

    for target in args.file:
        target = Path(target)
        if target.is_file():
            if not target.name.endswith('.py'):
                logging.warning("File %s doesn't end with .py, processing "
                                "anyway..." % target)
            process_file(target, enable)
        elif target.is_dir():
            logging.info("Processing %s recursively..." % target)
            for filename in target.recursedir('*.py'):
                process_file(filename, enable)
        else:
            logging.warning("Skipping %s..." % target)
Beispiel #30
0
 def add_data(self, filename):
     if filename in self.seen:
         return
     path = Path('/')
     for c in filename.components[1:]:
         path = path / c
         if path in self.seen:
             continue
         logging.debug("%s -> %s", path, data_path(path))
         self.tar.add(str(path), str(data_path(path)), recursive=False)
         self.seen.add(path)
Beispiel #31
0
def graph(args):
    """graph subcommand.

    Reads in the trace sqlite3 database and writes out a graph in GraphViz DOT
    format or JSON.
    """
    def call_generate(args, config, trace):
        generate(Path(args.target[0]), config, trace, args.all_forks,
                 args.format, args.packages, args.processes, args.otherfiles,
                 args.regex_filter, args.regex_replace, args.aggregate)

    if args.pack is not None:
        rpz_pack = RPZPack(args.pack)
        with rpz_pack.with_config() as config:
            with rpz_pack.with_trace() as trace:
                call_generate(args, config, trace)
    else:
        call_generate(args,
                      Path(args.dir) / 'config.yml',
                      Path(args.dir) / 'trace.sqlite3')
Beispiel #32
0
def docker_setup(args):
    """Does both create and build.

    Removes the directory if building fails.
    """
    docker_setup_create(args)
    try:
        docker_setup_build(args)
    except:
        Path(args.target[0]).rmtree(ignore_errors=True)
        raise
Beispiel #33
0
 def make_paths(cls, obj):
     if isinstance(obj, set):
         return set(cls.make_paths(e) for e in obj)
     elif isinstance(obj, list):
         return [cls.make_paths(e) for e in obj]
     elif isinstance(obj, AbstractPath):
         return obj
     elif isinstance(obj, (bytes, unicode_)):
         return Path(obj)
     else:
         assert False
Beispiel #34
0
 def test_tempfile(self):
     """Tests tempfile."""
     fd, f = Path.tempfile()
     os.close(fd)
     try:
         self.assertTrue(f.exists())
         self.assertTrue(f.is_file())
         self.assertTrue(f.is_absolute)
     finally:
         f.remove()
         self.assertFalse(f.exists())
Beispiel #35
0
 def test_tempfile(self):
     """Tests tempfile."""
     fd, f = Path.tempfile()
     os.close(fd)
     try:
         self.assertTrue(f.exists())
         self.assertTrue(f.is_file())
         self.assertTrue(f.is_absolute)
     finally:
         f.remove()
         self.assertFalse(f.exists())
Beispiel #36
0
 def download(self, remote_path, local_path):
     # Docker copies to a file in the specified directory, cannot just take
     # a file name (#4272)
     tmpdir = Path.tempdir(prefix='reprozip_docker_output_')
     try:
         subprocess.check_call(['docker', 'cp',
                                self.container + b':' + remote_path.path,
                                tmpdir.path])
         (tmpdir / remote_path.name).copyfile(local_path)
     finally:
         tmpdir.rmtree()
Beispiel #37
0
    def extract_trace(self, target):
        """Extracts the trace database to the specified path.

        It is up to the caller to remove that file once done.
        """
        target = Path(target)
        if self.version == 1:
            member = self.tar.getmember('METADATA/trace.sqlite3')
        elif self.version == 2:
            try:
                member = self.tar.getmember('METADATA/trace.sqlite3.gz')
            except KeyError:
                member = self.tar.getmember('METADATA/trace.sqlite3')
        else:
            assert False
        member = copy.copy(member)
        member.name = str(target.components[-1])
        self.tar.extract(member,
                         path=str(Path.cwd() / target.parent))
        assert target.is_file()
Beispiel #38
0
def print_info(args):
    """Writes out some information about a pack file.
    """
    pack = Path(args.pack[0])

    info = get_package_info(pack, read_data=args.json or args.verbosity >= 2)
    if args.json:
        json.dump(info, sys.stdout, indent=2)
        sys.stdout.write('\n')
    else:
        _print_package_info(pack, info, args.verbosity)
Beispiel #39
0
 def upload_file(self, local_path, input_path):
     stem, ext = local_path.stem, local_path.ext
     name = local_path.name
     nb = 0
     while (self.build_directory / name).exists():
         nb += 1
         name = stem + ('_%d' % nb).encode('ascii') + ext
     name = Path(name)
     local_path.copyfile(self.build_directory / name)
     logging.info("Copied file %s to %s", local_path, name)
     self.docker_copy.append((name, input_path))
Beispiel #40
0
def chroot_destroy(args):
    """Destroys the directory, unmounting first if necessary.
    """
    target = Path(args.target[0])

    chroot_unmount(target)

    logger.info("Removing directory %s...", target)
    signals.pre_destroy(target=target)
    rmtree_fixed(target)
    signals.post_destroy(target=target)
Beispiel #41
0
def do_vistrails(target):
    """Create a VisTrails workflow that runs the experiment.

    This is called from signals after an experiment has been setup by any
    unpacker.
    """
    record_usage(do_vistrails=True)
    unpacker = signals.unpacker
    dot_vistrails = Path('~/.vistrails').expand_user()

    runs, packages, other_files = load_config(target / 'config.yml',
                                              canonical=True)
    for i, run in enumerate(runs):
        module_name = write_cltools_module(run, dot_vistrails)

        # Writes VisTrails workflow
        bundle = target / 'vistrails.vt'
        logging.info("Writing VisTrails workflow %s...", bundle)
        vtdir = Path.tempdir(prefix='reprounzip_vistrails_')
        try:
            with vtdir.open('w', 'vistrail', encoding='utf-8',
                            newline='\n') as fp:
                vistrail = VISTRAILS_TEMPLATE
                cmdline = ' '.join(shell_escape(arg) for arg in run['argv'])
                vistrail = vistrail.format(date='2014-11-12 15:31:18',
                                           unpacker=unpacker,
                                           directory=escape_xml(
                                               str(target.absolute())),
                                           cmdline=escape_xml(cmdline),
                                           module_name=module_name,
                                           run=i)
                fp.write(vistrail)

            with bundle.open('wb') as fp:
                z = zipfile.ZipFile(fp, 'w')
                with vtdir.in_dir():
                    for path in Path('.').recursedir():
                        z.write(str(path))
                z.close()
        finally:
            vtdir.rmtree()
Beispiel #42
0
def chroot_run(args):
    """Runs the command in the chroot.
    """
    target = Path(args.target[0])
    read_dict(target / '.reprounzip', 'chroot')
    cmdline = args.cmdline

    # Loads config
    runs, packages, other_files = load_config_file(target / 'config.yml', True)

    selected_runs = get_runs(runs, args.run, cmdline)

    root = target / 'root'

    # X11 handler
    x11 = X11Handler(args.x11, ('local', socket.gethostname()),
                     args.x11_display)

    cmds = []
    for run_number in selected_runs:
        run = runs[run_number]
        cmd = 'cd %s && ' % shell_escape(run['workingdir'])
        cmd += '/usr/bin/env -i '
        environ = x11.fix_env(run['environ'])
        cmd += ' '.join('%s=%s' % (k, shell_escape(v))
                        for k, v in iteritems(environ))
        cmd += ' '
        # FIXME : Use exec -a or something if binary != argv[0]
        if cmdline is None:
            argv = [run['binary']] + run['argv'][1:]
        else:
            argv = cmdline
        cmd += ' '.join(shell_escape(a) for a in argv)
        userspec = '%s:%s' % (run.get('uid', 1000), run.get('gid', 1000))
        cmd = 'chroot --userspec=%s %s /bin/sh -c %s' % (
            userspec, shell_escape(unicode_(root)), shell_escape(cmd))
        cmds.append(cmd)
    cmds = [
        'chroot %s /bin/sh -c %s' %
        (shell_escape(unicode_(root)), shell_escape(c)) for c in x11.init_cmds
    ] + cmds
    cmds = ' && '.join(cmds)

    # Starts forwarding
    forwarders = []
    for portnum, connector in x11.port_forward:
        fwd = LocalForwarder(connector, portnum)
        forwarders.append(fwd)

    signals.pre_run(target=target)
    retcode = interruptible_call(cmds, shell=True)
    stderr.write("\n*** Command finished, status: %d\n" % retcode)
    signals.post_run(target=target, retcode=retcode)
Beispiel #43
0
def docker_upload(args):
    """Replaces an input file in the container.
    """
    target = Path(args.target[0])
    files = args.file
    unpacked_info = read_dict(target)
    input_files = unpacked_info.setdefault('input_files', {})

    try:
        ContainerUploader(target, input_files, files, unpacked_info)
    finally:
        write_dict(target, unpacked_info)
Beispiel #44
0
 def download_and_print(self, remote_path):
     # Download to temporary file
     fd, temp = Path.tempfile(prefix='reprozip_output_')
     os.close(fd)
     download_status = self.download(remote_path, temp)
     if download_status is not None and not download_status:
         return False
     # Output to stdout
     with temp.open('rb') as fp:
         copyfile(fp, stdout_bytes)
     temp.remove()
     return True
Beispiel #45
0
 def download_and_print(self, remote_path):
     # Download to temporary file
     fd, temp = Path.tempfile(prefix='reprozip_output_')
     os.close(fd)
     download_status = self.download(remote_path, temp)
     if download_status is not None and not download_status:
         return False
     # Output to stdout
     with temp.open('rb') as fp:
         copyfile(fp, stdout_bytes)
     temp.remove()
     return True
Beispiel #46
0
 def setUpClass(cls):
     """Builds a test hierarchy."""
     cls.tmp = Path.tempdir()
     cls.tmp.open('w', 'file').close()
     cls.tmp.open('w', u'r\xE9mi\'s file').close()
     d = cls.tmp.mkdir(u'r\xE9pertoire')
     d.open('w', 'file').close()
     d.mkdir('nested')
     if issubclass(Path, PosixPath):
         (d / 'last').symlink('..')
     else:
         d.open('w', 'last').close()
Beispiel #47
0
 def setUpClass(cls):
     """Builds a test hierarchy."""
     cls.tmp = Path.tempdir()
     cls.tmp.open('w', 'file').close()
     cls.tmp.open('w', u'r\xE9mi\'s file').close()
     d = cls.tmp.mkdir(u'r\xE9pertoire')
     d.open('w', 'file').close()
     d.mkdir('nested')
     if issubclass(Path, PosixPath):
         (d / 'last').symlink('..')
     else:
         d.open('w', 'last').close()
Beispiel #48
0
    def download(self, job_id, files, **kwargs):
        """Downloads files from server.
        """
        check_jobid(job_id)

        if not files:
            return
        if isinstance(files, string_types):
            files = [files]
        directory = False
        recursive = kwargs.pop('recursive', True)

        if 'destination' in kwargs and 'directory' in kwargs:
            raise TypeError("Only use one of 'destination' or 'directory'")
        elif 'destination' in kwargs:
            destination = Path(kwargs.pop('destination'))
            if len(files) != 1:
                raise ValueError("'destination' specified but multiple files "
                                 "given; did you mean to use 'directory'?")
        elif 'directory' in kwargs:
            destination = Path(kwargs.pop('directory'))
            directory = True
        if kwargs:
            raise TypeError("Got unexpected keyword arguments")

        # Might raise JobNotFound
        status, target, result = self.status(job_id)

        scp_client = self.get_scp_client()
        for filename in files:
            logger.info("Downloading %s", target / filename)
            if directory:
                scp_client.get(str(target / filename),
                               str(destination / filename),
                               recursive=recursive)
            else:
                scp_client.get(str(target / filename),
                               str(destination),
                               recursive=recursive)
Beispiel #49
0
def upload(args):
    """Replaces an input file in the directory.
    """
    target = Path(args.target[0])
    files = args.file
    unpacked_info = read_dict(target / '.reprounzip', args.type)
    input_files = unpacked_info.setdefault('input_files', {})

    try:
        LocalUploader(target, input_files, files, args.type,
                      args.type == 'chroot' and args.restore_owner)
    finally:
        write_dict(target / '.reprounzip', unpacked_info, args.type)
Beispiel #50
0
def find_all_links_recursive(filename, files):
    path = Path('/')
    for c in filename.components[1:]:
        # At this point, path is a canonical path, and all links in it have
        # been resolved

        # We add the next path component
        path = path / c

        # That component is possibly a link
        if path.is_link():
            # Adds the link itself
            files.add(path)

            target = path.read_link(absolute=True)
            # Here, target might contain a number of symlinks
            if target not in files:
                # Recurse on this new path
                find_all_links_recursive(target, files)
            # Restores the invariant; realpath might resolve several links here
            path = path.resolve()
    return path
Beispiel #51
0
 def post(self):
     self._notebook_file = Path(self.get_body_argument('file'))
     name = self._notebook_file.unicodename
     if name.endswith('.ipynb'):
         name = name[:-6]
     name = u'%s_%s.rpz' % (name, datetime.now().strftime('%Y%m%d-%H%M%S'))
     self._pack_file = self._notebook_file.parent / name
     self.nbapp.log.info("reprozip: tracing request from client: file=%r",
                         self._notebook_file)
     self._tempdir = Path.tempdir()
     self.nbapp.log.info("reprozip: created temp directory %r",
                         self._tempdir)
     proc = Subprocess([
         sys.executable, '-c',
         'from reprozip_jupyter.main import main; main()', 'trace',
         '--dont-save-notebook', '-d', self._tempdir.path,
         self._notebook_file.path
     ],
                       stdin=subprocess.PIPE)
     proc.stdin.close()
     proc.set_exit_callback(self._trace_done)
     self.nbapp.log.info("reprozip: started tracing...")
Beispiel #52
0
def upload(args):
    """Replaces an input file in the directory.
    """
    target = Path(args.target[0])
    files = args.file
    unpacked_info = metadata_read(target, args.type)
    input_files = unpacked_info.setdefault('input_files', {})

    try:
        LocalUploader(target, input_files, files,
                      args.type, args.type == 'chroot' and args.restore_owner)
    finally:
        metadata_write(target, unpacked_info, args.type)
Beispiel #53
0
def vagrant_upload(args):
    """Replaces an input file in the VM.
    """
    target = Path(args.target[0])
    files = args.file
    unpacked_info = read_dict(target / '.reprounzip')
    input_files = unpacked_info.setdefault('input_files', {})
    use_chroot = unpacked_info['use_chroot']

    try:
        SSHUploader(target, input_files, files, use_chroot)
    finally:
        write_dict(target / '.reprounzip', unpacked_info)
Beispiel #54
0
def find_all_links_recursive(filename, files):
    path = Path('/')
    for c in filename.components[1:]:
        # At this point, path is a canonical path, and all links in it have
        # been resolved

        # We add the next path component
        path = path / c

        # That component is possibly a link
        if path.is_link():
            # Adds the link itself
            files.add(path)

            target = path.read_link(absolute=True)
            # Here, target might contain a number of symlinks
            if target not in files:
                # Recurse on this new path
                find_all_links_recursive(target, files)
            # Restores the invariant; realpath might resolve several links here
            path = path.resolve()
    return path
Beispiel #55
0
def setup_logging(tag, verbosity):
    """Sets up the logging module.
    """
    levels = [logging.CRITICAL, logging.WARNING, logging.INFO, logging.DEBUG]
    console_level = levels[min(verbosity, 3)]
    file_level = logging.INFO
    min_level = min(console_level, file_level)

    # Create formatter, with same format as C extension
    fmt = "[%s] %%(asctime)s %%(levelname)s: %%(message)s" % tag
    formatter = LoggingDateFormatter(fmt)

    # Console logger
    handler = logging.StreamHandler()
    handler.setLevel(console_level)
    handler.setFormatter(formatter)

    # Set up logger
    logger = logging.root
    logger.setLevel(min_level)
    logger.addHandler(handler)

    # File logger
    dotrpz = Path('~/.reprozip').expand_user()
    try:
        if not dotrpz.is_dir():
            dotrpz.mkdir()
        filehandler = logging.handlers.RotatingFileHandler(str(dotrpz / 'log'),
                                                           mode='a',
                                                           delay=False,
                                                           maxBytes=400000,
                                                           backupCount=5)
    except (IOError, OSError):
        logging.warning("Couldn't create log file %s", dotrpz / 'log')
    else:
        filehandler.setFormatter(formatter)
        filehandler.setLevel(file_level)
        logger.addHandler(filehandler)
Beispiel #56
0
 def __init__(self, path):
     path = Path(path)
     size = None
     if path.exists():
         if path.is_link():
             self.comment = "Link to %s" % path.read_link(absolute=True)
         elif path.is_dir():
             self.comment = "Directory"
         else:
             size = path.size()
             self.comment = hsize(size)
     File.__init__(self, path, size)
Beispiel #57
0
def combine(args):
    """combine subcommand.

    Reads in multiple trace databases and combines them into one.

    The runs from the original traces are appended ('run_id' field gets
    translated to avoid conflicts).
    """
    traces = []
    for tracepath in args.traces:
        if tracepath == '-':
            tracepath = Path(args.dir) / 'trace.sqlite3'
        else:
            tracepath = Path(tracepath)
            if tracepath.is_dir():
                tracepath = tracepath / 'trace.sqlite3'
        traces.append(tracepath)

    reprozip.traceutils.combine_traces(traces, Path(args.dir))
    reprozip.tracer.trace.write_configuration(Path(args.dir),
                                              args.identify_packages,
                                              args.find_inputs_outputs,
                                              overwrite=True)
Beispiel #58
0
 def download(self, remote_path, local_path):
     # Docker copies to a file in the specified directory, cannot just take
     # a file name (#4272)
     tmpdir = Path.tempdir(prefix='reprozip_docker_output_')
     try:
         ret = subprocess.call(['docker', 'cp',
                               self.container + b':' + remote_path.path,
                               tmpdir.path])
         if ret != 0:
             logging.critical("Can't get output file: %s", remote_path)
             sys.exit(1)
         (tmpdir / remote_path.name).copyfile(local_path)
     finally:
         tmpdir.rmtree()