Ejemplo n.º 1
0
    def move(self):
        try:
            log.debug("Moving '%s' to '%s'", self.config['move_from_remote'], self.config['move_to_remote'])

            # build cmd
            cmd = "%s %s %s %s --config=%s" % (cmd_quote(self.rclone_binary_path),
                                               'move',
                                               cmd_quote(self.config['move_from_remote']),
                                               cmd_quote(self.config['move_to_remote']),
                                               cmd_quote(self.rclone_config_path))
            extras = self.__extras2string()
            if len(extras) > 2:
                cmd += ' %s' % extras
            excludes = self.__excludes2string()
            if len(excludes) > 2:
                cmd += ' %s' % excludes
            if self.plex.get('enabled'):
                r = re.compile(r"https?://(www\.)?")
                rc_url = r.sub('', self.plex['rclone']['url']).strip().strip('/')
                cmd += ' --rc --rc-addr=%s' % cmd_quote(rc_url)
            if self.dry_run:
                cmd += ' --dry-run'

            # exec
            log.debug("Using: %s", cmd)
            process.execute(cmd, logs=True)
            return True

        except Exception:
            log.exception("Exception occurred while moving '%s' to '%s':", self.config['move_from_remote'],
                          self.config['move_to_remote'])
        return False
Ejemplo n.º 2
0
def virt_costumize_script(image, script_file, args_str='', log_file=None,
                          verbose=False):
    # NOTE: cannot specify a constant destination file :(
    # --copy-in {script_file}:/root/custumie.sh did not worked
    # LIBGUESTFS_BACKEND=direct , file permissionsss
    base_name = os.path.basename(script_file)
    cmd = '/root/' + base_name + ' ' + args_str
    if verbose:
        verb = '--verbose'
    else:
        verb = ''
    (r, log) = localsh.run_log(("LIBGUESTFS_BACKEND=direct "
                                "virt-customize {verbose} --add {image} "
                                "--memsize 1024 "
                                "--copy-in {script_file}:/root/ "
                                "--chmod 755:/root/{base_name} "
                                "--run-command {cmd} "
                                "--selinux-relabel ").format(
        image=cmd_quote(image), script_file=cmd_quote(script_file),
        base_name=cmd_quote(base_name),
        cmd=cmd_quote(cmd), verbose=verb))
    print(log)
    if log_file:
        f = open(log_file, "w")
        f.write(log)
        f.close()
    if r:
        raise Exception("virt_costumize Failed")
Ejemplo n.º 3
0
    def sync(self, **kwargs):
        if not self.instance_id:
            log.error("Sync was called, but no instance_id was found, aborting...")
            return False, None, None
        kwargs.update(self.kwargs)

        # create RcloneSyncer object
        rclone = RcloneSyncer(self.sync_from_config, self.sync_to_config, **kwargs)

        # start sync
        log.info("Starting sync for instance: %r", self.instance_id)
        resp, delayed_check, delayed_trigger = rclone.sync(self._wrap_command)
        log.info("Finished syncing for instance: %r", self.instance_id)

        # copy rclone.conf back from instance (in-case refresh tokens were used) (Copy seems not to be working atm)
        # cmd = "%s --region=%s cp %s:/root/.config/rclone/rclone.conf %s" % (
        #     cmd_quote(self.tool_path), cmd_quote(self.region), cmd_quote(self.instance_id),
        #     cmd_quote(os.path.dirname(kwargs['rclone_config'])))

        # Use exec cat > rclone config until cp is resolved
        cmd = "%s --region=%s exec %s cat /root/.config/rclone/rclone.conf > %s" % (
            cmd_quote(self.tool_path), cmd_quote(self.region), cmd_quote(self.instance_id),
            cmd_quote(kwargs['rclone_config']))
        log.debug("Using: %s", cmd)

        log.debug("Copying rclone config from instance %r to: %r", self.instance_id, kwargs['rclone_config'])
        config_resp = process.popen(cmd, shell=True)
        if config_resp is None or len(config_resp) >= 2:
            log.error("Unexpected response while copying rclone config from instance: %s", config_resp)
        else:
            log.info("Copied rclone.conf from instance")

        return resp, delayed_check, delayed_trigger
Ejemplo n.º 4
0
    def upload(self, callback):
        try:
            log.debug("Uploading '%s' to '%s'", self.config['upload_folder'],
                      self.config['upload_remote'])
            # build cmd
            cmd = "rclone move %s %s" % (
                cmd_quote(self.config['upload_folder']),
                cmd_quote(self.config['upload_remote']))

            extras = self.__extras2string()
            if len(extras) > 2:
                cmd += ' %s' % extras
            excludes = self.__excludes2string()
            if len(excludes) > 2:
                cmd += ' %s' % excludes
            if self.dry_run:
                cmd += ' --dry-run'

            # exec
            log.debug("Using: %s", cmd)
            process.execute(cmd, callback)
            return True
        except Exception:
            log.exception(
                "Exception occurred while uploading '%s' to remote: %s",
                self.config['upload_folder'], self.name)
        return False
Ejemplo n.º 5
0
    def get_reproduction_cmd(self, verb, jid):
        """Get a command line to reproduce this stage with the proper environment."""

        # Define the base env command
        get_env_cmd = 'catkin {} --get-env {}'.format(verb, jid)

        # Add additional env args
        env_overrides_formatted = ' '.join([
            '{}={}'.format(k, cmd_quote(v))
            for k, v in self.env_overrides.items()
        ])

        # Define the actual command to reproduce
        cmd_str = ' '.join([cmd_quote(t) for t in self.async_execute_process_kwargs['cmd']])

        # Define the command to run the subcommand
        env_cmd = 'catkin env -si {} {}'.format(
            env_overrides_formatted,
            cmd_str)

        # Return the full command
        return 'cd {}; {} | {}; cd -'.format(
            self.async_execute_process_kwargs['cwd'],
            get_env_cmd,
            env_cmd)
Ejemplo n.º 6
0
def show_sections(config):
    if os.name == "nt":
        final_cmd = '""%s" --list"' % config["PLEX_SCANNER"]
    else:
        cmd = "export LD_LIBRARY_PATH=" + config["PLEX_LD_LIBRARY_PATH"] + ";"
        if not config["USE_DOCKER"]:
            cmd += "export PLEX_MEDIA_SERVER_APPLICATION_SUPPORT_DIR=" + config["PLEX_SUPPORT_DIR"] + ";"
        cmd += config["PLEX_SCANNER"] + " --list"

        if config["USE_DOCKER"]:
            final_cmd = "docker exec -u %s -it %s bash -c %s" % (
                cmd_quote(config["PLEX_USER"]),
                cmd_quote(config["DOCKER_NAME"]),
                cmd_quote(cmd),
            )
        elif config["USE_SUDO"]:
            final_cmd = 'sudo -u %s bash -c "%s"' % (config["PLEX_USER"], cmd)
        else:
            final_cmd = cmd
    logger.info("Using Plex Scanner")
    print("\n")
    print("Plex Sections:")
    print("==============")
    logger.debug(final_cmd)
    os.system(final_cmd)
Ejemplo n.º 7
0
 def _delete(self, filename):
     commandline = "lftp -c \"source %s; cd %s; rm %s\"" % (cmd_quote(
         self.tempname), cmd_quote(self.remote_path), cmd_quote(filename))
     log.Debug("CMD: %s" % commandline)
     _, l, e = self.subprocess_popen(commandline)
     log.Debug("STDERR:\n" "%s" % (e))
     log.Debug("STDOUT:\n" "%s" % (l))
Ejemplo n.º 8
0
    def get_reproduction_cmd(self, verb, jid):
        """Get a command line to reproduce this stage with the proper environment."""

        # Define the base env command
        get_env_cmd = 'catkin {} --get-env {}'.format(verb, jid)

        # Add additional env args
        env_overrides_formatted = ' '.join([
            '{}={}'.format(k, cmd_quote(v))
            for k, v in self.env_overrides.items()
        ])

        # Define the actual command to reproduce
        cmd_str = ' '.join([cmd_quote(t) for t in self.async_execute_process_kwargs['cmd']])

        # Define the command to run the subcommand
        env_cmd = 'catkin env -si {} {}'.format(
            env_overrides_formatted,
            cmd_str)

        # Return the full command
        return 'cd {}; {} | {}; cd -'.format(
            self.async_execute_process_kwargs['cwd'],
            get_env_cmd,
            env_cmd)
Ejemplo n.º 9
0
    def move(self):
        try:
            log.debug("Moving '%s' to '%s'", self.config['move_from_remote'], self.config['move_to_remote'])

            # build cmd
            cmd = "%s %s %s %s --config=%s" % (cmd_quote(self.rclone_binary_path),
                                               'move',
                                               cmd_quote(self.config['move_from_remote']),
                                               cmd_quote(self.config['move_to_remote']),
                                               cmd_quote(self.rclone_config_path))
            extras = self.__extras2string()
            if len(extras) > 2:
                cmd += ' %s' % extras
            excludes = self.__excludes2string()
            if len(excludes) > 2:
                cmd += ' %s' % excludes
            if self.dry_run:
                cmd += ' --dry-run'
            if self.use_rc:
                cmd += ' --rc --rc-addr=%s' % cmd_quote('localhost:7949')

            # exec
            log.debug("Using: %s", cmd)
            process.execute(cmd, logs=True)
            return True

        except Exception:
            log.exception("Exception occurred while moving '%s' to '%s':", self.config['move_from_remote'],
                          self.config['move_to_remote'])
        return False
Ejemplo n.º 10
0
def build_cmd(config, section, scan_path, scan_op):
    # build plex scanner command
    logger.info("Building Plex Scan Command")
    if os.name == 'nt':
        final_cmd = '""%s" --scan --refresh --section %s --directory "%s""' \
                    % (config['PLEX_SCANNER'], str(section), scan_path)
    else:
        cmd = 'export LD_LIBRARY_PATH=' + config['PLEX_LD_LIBRARY_PATH'] + ';'
        if not config['USE_DOCKER']:
            cmd += 'export PLEX_MEDIA_SERVER_APPLICATION_SUPPORT_DIR=' + config[
                'PLEX_SUPPORT_DIR'] + ';'
        if scan_op == 'scan':
            cmd += config[
                'PLEX_SCANNER'] + ' --scan --refresh --section ' + str(
                    section) + ' --directory ' + cmd_quote(scan_path)
        elif scan_op == 'analyze':
            media_id = get_media_id(config, scan_path)
            cmd += config['PLEX_SCANNER'] + ' --analyze -o ' + str(media_id)
        elif scan_op == 'deep':
            media_id = get_media_id(config, scan_path)
            cmd += config['PLEX_SCANNER'] + ' --analyze-deeply -o ' + str(
                media_id)

        if config['USE_DOCKER']:
            final_cmd = 'docker exec -i %s bash -c %s' % (cmd_quote(
                config['DOCKER_NAME']), cmd_quote(cmd))
        elif config['USE_SUDO']:
            final_cmd = 'sudo -u %s bash -c %s' % (config['PLEX_USER'],
                                                   cmd_quote(cmd))
        else:
            final_cmd = cmd

    return final_cmd
Ejemplo n.º 11
0
 def _get(self, remote_filename, local_path):
     commandline = "lftp -c \"source %s; get %s -o %s\"" % (
         cmd_quote(self.tempname), cmd_quote(self.remote_path) +
         remote_filename, cmd_quote(local_path.name))
     log.Debug("CMD: %s" % commandline)
     _, l, e = self.subprocess_popen(commandline)
     log.Debug("STDERR:\n" "%s" % (e))
     log.Debug("STDOUT:\n" "%s" % (l))
Ejemplo n.º 12
0
    def setup(self, **kwargs):
        if not self.instance_id:
            log.error("Setup was called, but no instance_id was found, aborting...")
            return False
        if 'rclone_config' not in kwargs:
            log.error("Setup was called, but no rclone_config was found, aborting...")
            self.destroy()
            return False

        # install unzip
        cmd_exec = "apt-get -qq update && apt-get -y -qq install unzip && which unzip"
        cmd = "%s --region=%s exec %s %s" % (
            cmd_quote(self.tool_path), cmd_quote(self.region), cmd_quote(self.instance_id), cmd_quote(cmd_exec))
        log.debug("Using: %s", cmd)

        log.debug("Installing rclone to instance: %r", self.instance_id)
        resp = process.popen(cmd)
        if not resp or '/usr/bin/unzip' not in resp.lower():
            log.error("Unexpected response while installing unzip: %s", resp)
            self.destroy()
            return False
        log.info("Installed unzip")

        # install rclone to instance
        cmd_exec = "cd ~ && curl -sO https://downloads.rclone.org/rclone-current-linux-amd64.zip && " \
                   "unzip -oq rclone-current-linux-amd64.zip && cd rclone-*-linux-amd64 && " \
                   "cp -rf rclone /usr/bin/ && cd ~ && rm -rf rclone-* && chown root:root /usr/bin/rclone && " \
                   "chmod 755 /usr/bin/rclone && mkdir -p /root/.config/rclone && which rclone"
        cmd = "%s --region=%s exec %s %s" % (
            cmd_quote(self.tool_path), cmd_quote(self.region), cmd_quote(self.instance_id), cmd_quote(cmd_exec))
        log.debug("Using: %s", cmd)

        log.debug("Installing rclone to instance: %r", self.instance_id)
        resp = process.popen(cmd)
        if not resp or '/usr/bin/rclone' not in resp.lower():
            log.error("Unexpected response while installing rclone: %s", resp)
            self.destroy()
            return False
        log.info("Installed rclone")

        # copy rclone.conf to instance
        cmd = "%s --region=%s cp %s %s:/root/.config/rclone/" % (
            cmd_quote(self.tool_path), cmd_quote(self.region), cmd_quote(kwargs['rclone_config']),
            cmd_quote(self.instance_id))
        log.debug("Using: %s", cmd)

        log.debug("Copying rclone config %r to instance: %r", kwargs['rclone_config'], self.instance_id)
        resp = process.popen(cmd)
        if resp is None or len(resp) >= 2:
            log.error("Unexpected response while copying rclone config: %s", resp)
            self.destroy()
            return False
        log.info("Copied across rclone.conf")

        log.info("Successfully setup instance: %r", self.instance_id)
        return True
Ejemplo n.º 13
0
 def _put(self, source_path, remote_filename):
     commandline = "lftp -c \"source %s; mkdir -p %s; put %s -o %s\"" % (
         self.tempname, cmd_quote(
             self.remote_path), cmd_quote(source_path.name),
         cmd_quote(self.remote_path) + remote_filename)
     log.Debug("CMD: %s" % commandline)
     s, l, e = self.subprocess_popen(commandline)
     log.Debug("STATUS: %s" % s)
     log.Debug("STDERR:\n" "%s" % (e))
     log.Debug("STDOUT:\n" "%s" % (l))
Ejemplo n.º 14
0
 def _get(self, remote_filename, local_path):
     if isinstance(remote_filename, b"".__class__):
         remote_filename = util.fsdecode(remote_filename)
     commandline = u"lftp -c \"source %s; get %s -o %s\"" % (
         cmd_quote(self.tempname), cmd_quote(self.remote_path) +
         remote_filename, cmd_quote(local_path.uc_name))
     log.Debug(u"CMD: %s" % commandline)
     _, l, e = self.subprocess_popen(commandline)
     log.Debug(u"STDERR:\n" u"%s" % (e))
     log.Debug(u"STDOUT:\n" u"%s" % (l))
Ejemplo n.º 15
0
def get_jobs(user="******", queue="*"):
    """Get list of dicts of job info"""
    user = cmd_quote(user)
    queue = cmd_quote(queue)

    if FAKE:
        jobs_text = open_file("test/jobs.txt")
    else:
        jobs_text = run_command(["qstat", "-u", user, "-q", queue])
    jobs = process_jobs_xml(jobs_text)
    return jobs
Ejemplo n.º 16
0
    def upload_files(self, file_defs):
        print "Uploading files ... "

        for curr_file_def in file_defs:
            full_local_filepath, dest_filename, full_remote_dir_path, num_files, overwrite = curr_file_def

            # Find all the files that match our full_local_filepath (which may contain pattern)
            local_files = sorted(glob.glob(full_local_filepath))

            try:
                if len(local_files) > 0:
                    # Ensure the remote dir exits
                    # TODO: How expensive are calls to mkdir if dir already exists,
                    #       better to check dir exists first?
                    subprocess.check_call("ssh " + remote_account + " 'mkdir -p " +
                                          full_remote_dir_path + "'", shell=True)
                    curr_file_num = 1
                    for curr_file in local_files:
                        if (num_files is not 0) and (curr_file_num > num_files):
                            break

                        # Deal with the case where we want to alter the destination filename
                        curr_src_full_filename = os.path.basename(curr_file)
                        if dest_filename is "":
                            full_remote_filepath = cmd_quote(os.path.join(full_remote_dir_path,
                                                                          curr_src_full_filename))
                        else:
                            filename, extension = os.path.splitext(curr_src_full_filename)
                            full_remote_filepath = cmd_quote(os.path.join(full_remote_dir_path,
                                                                          dest_filename + extension))

                        # Deal with the case where we do not want to overwrite the dest file
                        file_num = 2
                        if overwrite is False:
                            full_remote_filename = os.path.basename(full_remote_filepath)
                            while subprocess.call(['ssh', remote_account, 'test -e ' +
                                    full_remote_filepath]) == 0:
                                filename_no_ext, filename_ext = os.path.splitext(full_remote_filename)

                                full_remote_filepath = cmd_quote(os.path.join(full_remote_dir_path,
                                                                              filename_no_ext + "_" + str(
                                                                                  file_num) + filename_ext))
                                file_num += 1

                        subprocess.check_call("scp " + curr_file + " " +
                                              remote_account + ":" + full_remote_filepath, shell=True)

                        curr_file_num += 1

            except subprocess.CalledProcessError as e:
                print "Error uploading files: ", e.returncode
                raise

        print "... upload finished."
Ejemplo n.º 17
0
 def _put(self, source_path, remote_filename):
     if isinstance(remote_filename, b"".__class__):
         remote_filename = util.fsdecode(remote_filename)
     commandline = u"lftp -c \"source %s; mkdir -p %s; put %s -o %s\"" % (
         self.tempname, cmd_quote(
             self.remote_path), cmd_quote(source_path.uc_name),
         cmd_quote(self.remote_path) + util.fsdecode(remote_filename))
     log.Debug(u"CMD: %s" % commandline)
     s, l, e = self.subprocess_popen(commandline)
     log.Debug(u"STATUS: %s" % s)
     log.Debug(u"STDERR:\n" u"%s" % (e))
     log.Debug(u"STDOUT:\n" u"%s" % (l))
Ejemplo n.º 18
0
 def _delete(self, filename):
     commandline = "lftp -c \"source %s; cd %s; rm %s\"" % (
         cmd_quote(self.tempname),
         cmd_quote(self.remote_path),
         cmd_quote(filename)
     )
     log.Debug("CMD: %s" % commandline)
     _, l, e = self.subprocess_popen(commandline)
     log.Debug("STDERR:\n"
               "%s" % (e))
     log.Debug("STDOUT:\n"
               "%s" % (l))
Ejemplo n.º 19
0
    def upload_files(self, file_defs):
        print "Uploading files ... "

        for curr_file_def in file_defs:
            full_local_filepath, dest_filename, full_remote_dir_path, num_files, overwrite = curr_file_def

            # Find all the files that match our full_local_filepath (which may contain pattern)
            local_files = sorted(glob.glob(full_local_filepath))

            try:
                if len(local_files) > 0:
                    # Ensure the remote dir exits
                    # TODO: How expensive are calls to mkdir if dir already exists, 
                    #       better to check dir exists first?
                    subprocess.check_call("ssh " + remote_account + " 'mkdir -p " + 
                                          full_remote_dir_path + "'", shell=True)
                    curr_file_num = 1
                    for curr_file in local_files:
                        if (num_files is not 0) and (curr_file_num > num_files):
                            break

                        # Deal with the case where we want to alter the destination filename
                        curr_src_full_filename = os.path.basename(curr_file)
                        if dest_filename is "":
                            full_remote_filepath = cmd_quote(os.path.join(full_remote_dir_path, 
                                                       curr_src_full_filename))
                        else:
                            filename, extension = os.path.splitext(curr_src_full_filename)
                            full_remote_filepath = cmd_quote(os.path.join(full_remote_dir_path, 
                                                       dest_filename + extension))
                        
                        # Deal with the case where we do not want to overwrite the dest file
                        file_num = 2
                        if overwrite is False:
                            full_remote_filename = os.path.basename(full_remote_filepath)
                            while subprocess.call(['ssh', remote_account, 'test -e ' + 
                                                   full_remote_filepath]) == 0:
                                filename_no_ext, filename_ext = os.path.splitext(full_remote_filename)

                                full_remote_filepath = cmd_quote(os.path.join(full_remote_dir_path, 
                                            filename_no_ext + "_" + str(file_num) + filename_ext))
                                file_num += 1

                        subprocess.check_call("scp " + curr_file + " " + 
                                 remote_account + ":" + full_remote_filepath, shell=True)

                        curr_file_num += 1

            except subprocess.CalledProcessError as e:
                print "Error uploading files: ", e.returncode
                raise

        print "... upload finished."
Ejemplo n.º 20
0
    def __init__(self, instance, plugin):
        super(DaemonDevice, self).__init__(instance, plugin)

        self.namePattern = k_psSearch_daemon(
            processname=self.props['processName'],
            args=self.props['startArgs'])
        self.onCmd = k_daemonStartCmd(
            processname=cmd_quote(self.props['processName']),
            apppath=cmd_quote(self.props['applicationPath']),
            args=cmd_quote(self.props['startArgs']))
        self.defaultOffCmd = k_daemonStopCmd(
            processname=cmd_quote(self.props['processName']))
Ejemplo n.º 21
0
 def _get(self, remote_filename, local_path):
     commandline = "lftp -c \"source %s; get %s -o %s\"" % (
         cmd_quote(self.tempname),
         cmd_quote(self.remote_path) + remote_filename,
         cmd_quote(local_path.name)
     )
     log.Debug("CMD: %s" % commandline)
     _, l, e = self.subprocess_popen(commandline)
     log.Debug("STDERR:\n"
               "%s" % (e))
     log.Debug("STDOUT:\n"
               "%s" % (l))
Ejemplo n.º 22
0
def analyze_item(config, scan_path):
    if not os.path.exists(config['PLEX_DATABASE_PATH']):
        logger.warning(
            "Could not analyze of '%s' because Plex database could not be found.",
            scan_path)
        return
    # get files metadata_item_id
    metadata_item_ids = get_file_metadata_ids(config, scan_path)
    if metadata_item_ids is None or not len(metadata_item_ids):
        logger.warning(
            "Aborting analysis of '%s' because could not find any 'metadata_item_id' for it.",
            scan_path)
        return
    metadata_item_id = ','.join(str(x) for x in metadata_item_ids)

    # build Plex analyze command
    analyze_type = 'analyze-deeply' if config['PLEX_ANALYZE_TYPE'].lower(
    ) == 'deep' else 'analyze'
    if os.name == 'nt':
        final_cmd = '"%s" --%s --item %s' % (config['PLEX_SCANNER'],
                                             analyze_type, metadata_item_id)
    else:
        cmd = 'export LD_LIBRARY_PATH=' + config['PLEX_LD_LIBRARY_PATH'] + ';'
        if not config['USE_DOCKER']:
            cmd += 'export PLEX_MEDIA_SERVER_APPLICATION_SUPPORT_DIR=' + config[
                'PLEX_SUPPORT_DIR'] + ';'
        cmd += config[
            'PLEX_SCANNER'] + ' --' + analyze_type + ' --item ' + metadata_item_id

        if config['USE_DOCKER']:
            final_cmd = 'docker exec -u %s -i %s bash -c %s' % \
                        (cmd_quote(config['PLEX_USER']), cmd_quote(config['DOCKER_NAME']), cmd_quote(cmd))
        elif config['USE_SUDO']:
            final_cmd = 'sudo -u %s bash -c %s' % (config['PLEX_USER'],
                                                   cmd_quote(cmd))
        else:
            final_cmd = cmd

    # begin analysis
    logger.debug(
        "Starting %s analysis of 'metadata_item': %s",
        'deep' if config['PLEX_ANALYZE_TYPE'].lower() == 'deep' else 'basic',
        metadata_item_id)
    logger.debug(final_cmd)
    if os.name == 'nt':
        utils.run_command(final_cmd)
    else:
        utils.run_command(final_cmd.encode("utf-8"))
    logger.info(
        "Finished %s analysis of 'metadata_item': %s",
        'deep' if config['PLEX_ANALYZE_TYPE'].lower() == 'deep' else 'basic',
        metadata_item_id)
Ejemplo n.º 23
0
 def _put(self, source_path, remote_filename):
     commandline = "lftp -c \"source %s; mkdir -p %s; put %s -o %s\"" % (
         self.tempname,
         cmd_quote(self.remote_path),
         cmd_quote(source_path.name),
         cmd_quote(self.remote_path) + remote_filename
     )
     log.Debug("CMD: %s" % commandline)
     s, l, e = self.subprocess_popen(commandline)
     log.Debug("STATUS: %s" % s)
     log.Debug("STDERR:\n"
               "%s" % (e))
     log.Debug("STDOUT:\n"
               "%s" % (l))
Ejemplo n.º 24
0
def analyze_item(config, scan_path):
    if not os.path.exists(config["PLEX_DATABASE_PATH"]):
        logger.warning("Could not analyze of '%s' because Plex database could not be found.", scan_path)
        return
    # get files metadata_item_id
    metadata_item_ids = get_file_metadata_ids(config, scan_path)
    if metadata_item_ids is None or not metadata_item_ids:
        logger.warning(
            "Aborting analysis of '%s' because could not find any 'metadata_item_id' for it.",
            scan_path,
        )
        return
    metadata_item_id = ",".join(str(x) for x in metadata_item_ids)

    # build Plex analyze command
    analyze_type = "analyze-deeply" if config["PLEX_ANALYZE_TYPE"].lower() == "deep" else "analyze"
    if os.name == "nt":
        final_cmd = '"%s" --%s --item %s' % (config["PLEX_SCANNER"], analyze_type, metadata_item_id)
    else:
        cmd = "export LD_LIBRARY_PATH=" + config["PLEX_LD_LIBRARY_PATH"] + ";"
        if not config["USE_DOCKER"]:
            cmd += "export PLEX_MEDIA_SERVER_APPLICATION_SUPPORT_DIR=" + config["PLEX_SUPPORT_DIR"] + ";"
        cmd += config["PLEX_SCANNER"] + " --" + analyze_type + " --item " + metadata_item_id

        if config["USE_DOCKER"]:
            final_cmd = "docker exec -u %s -i %s bash -c %s" % (
                cmd_quote(config["PLEX_USER"]),
                cmd_quote(config["DOCKER_NAME"]),
                cmd_quote(cmd),
            )
        elif config["USE_SUDO"]:
            final_cmd = "sudo -u %s bash -c %s" % (config["PLEX_USER"], cmd_quote(cmd))
        else:
            final_cmd = cmd

    # begin analysis
    logger.debug(
        "Starting %s analysis of 'metadata_item': %s",
        "deep" if config["PLEX_ANALYZE_TYPE"].lower() == "deep" else "basic",
        metadata_item_id,
    )
    logger.debug(final_cmd)
    utils.run_command(final_cmd.encode("utf-8"))
    logger.info(
        "Finished %s analysis of 'metadata_item': %s",
        "deep" if config["PLEX_ANALYZE_TYPE"].lower() == "deep" else "basic",
        metadata_item_id,
    )
Ejemplo n.º 25
0
 def __excludes2string(self):
     return ' '.join(
         "--exclude=%s" % (
             cmd_quote(glob.escape(value) if value.startswith(os.path.sep) else value) if isinstance(value,
                                                                                                     str) else value)
         for value in
         self.config['rclone_excludes']).replace('=None', '').strip()
Ejemplo n.º 26
0
def bindings_sanity_check(bindings, data_dir, check_mode):
    """Performs sanity-checking on the bindings list and related arguments.
    Return: <bindings> <error>"""

    # bindings is a list of the following:
    # {
    #   device: [REQUIRED]
    #   encryption_password: a password
    #   encryption_key: /data_dir/filename
    #   encryption_key_src: /path/to/file/on/controlnode
    #   slot: 1 (default)
    #   state: present (default) | absent
    #   password_temporary: no (default)
    #   threshold: 1 (default)
    #   servers: [] (default)
    # }
    if not bindings:
        return None, {"msg": "No devices set"}

    for idx, binding in enumerate(bindings):
        # Set up the state.
        if "state" not in binding:
            bindings[idx]["state"] = "present"
        else:
            if binding["state"] not in ["present", "absent"]:
                errmsg = "state must be present or absent"
                return None, {"msg": errmsg}

        # Make sure we have the required information for the binding.
        if "device" not in binding:
            errmsg = "Each binding must have a device set"
            return None, {"msg": errmsg}

        # When running under check mode, encryption_key is not used, which means we
        # also do not need to have data_dir defined.
        if ("encryption_key" in binding
                or "encryption_key_src" in binding) and not check_mode:
            if not data_dir:
                return None, {"msg": "data_dir needs to be defined"}

            if "encryption_key" in binding:
                basefile = os.path.basename(binding["encryption_key"])
            else:
                basefile = os.path.basename(binding["encryption_key_src"])
            keyfile = os.path.join(data_dir, basefile)
            bindings[idx]["encryption_key"] = cmd_quote(keyfile)

        # The defaults for the remaining binding attributes.
        binding_defaults = {
            "slot": 1,
            "threshold": 1,
            "password_temporary": False,
            "servers": [],
        }

        for attr in binding_defaults:
            if attr not in bindings[idx]:
                bindings[idx][attr] = binding_defaults[attr]

    return bindings, None
Ejemplo n.º 27
0
def execute_command(
    cmd,
    cwd=None,
    capture=False,
    env=None
):  # type: (t.List[str], t.Optional[str], bool, t.Optional[t.Dict[str, str]]) -> None
    """Execute the specified command."""
    log('Execute command: %s' % ' '.join(cmd_quote(c) for c in cmd),
        verbosity=1)

    cmd_bytes = [to_bytes(c) for c in cmd]

    if capture:
        stdout = subprocess.PIPE
        stderr = subprocess.PIPE
    else:
        stdout = None
        stderr = None

    cwd_bytes = to_optional_bytes(cwd)
    process = subprocess.Popen(cmd_bytes,
                               cwd=cwd_bytes,
                               stdin=devnull(),
                               stdout=stdout,
                               stderr=stderr,
                               env=env)  # pylint: disable=consider-using-with
    stdout_bytes, stderr_bytes = process.communicate()
    stdout_text = to_optional_text(stdout_bytes) or u''
    stderr_text = to_optional_text(stderr_bytes) or u''

    if process.returncode != 0:
        raise SubprocessError(cmd, process.returncode, stdout_text,
                              stderr_text)
Ejemplo n.º 28
0
    def _list(self):
        # Do a long listing to avoid connection reset
        # remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/')).rstrip()
        remote_dir = urllib.parse.unquote(self.parsed_url.path)
        # print remote_dir
        quoted_path = cmd_quote(self.remote_path)
        # failing to cd into the folder might be because it was not created already
        commandline = u"lftp -c \"source %s; ( cd %s && ls ) || ( mkdir -p %s && cd %s && ls )\"" % (
            cmd_quote(self.tempname), quoted_path, quoted_path, quoted_path)
        log.Debug(u"CMD: %s" % commandline)
        _, l, e = self.subprocess_popen(commandline)
        log.Debug(u"STDERR:\n" u"%s" % (e))
        log.Debug(u"STDOUT:\n" u"%s" % (l))

        # Look for our files as the last element of a long list line
        return [x.split()[-1] for x in l.split(b'\n') if x]
Ejemplo n.º 29
0
def ffmpeg_pipe_stream(stream_url):
    global PLEX_FFMPEG_PATH, PLEX_BUFFER_SIZE

    pipe_cmd = "%s -re -i %s " \
               "-codec copy " \
               "-nostats " \
               "-loglevel 0 " \
               "-bsf:v h264_mp4toannexb " \
               "-f mpegts " \
               "-tune zerolatency " \
               "pipe:1" % (PLEX_FFMPEG_PATH, cmd_quote(stream_url))

    p = subprocess.Popen(shlex.split(pipe_cmd),
                         stdout=subprocess.PIPE,
                         bufsize=-1)
    try:
        pipes = [p.stdout]
        while pipes:
            ready, _, _ = select(pipes, [], [])
            for pipe in ready:
                data = pipe.read(PLEX_BUFFER_SIZE << 10)
                if data:
                    yield data
                else:
                    pipes.remove(pipe)
    except Exception:
        pass
    except GeneratorExit:
        pass

    try:
        p.terminate()
    except Exception:
        pass
    return
Ejemplo n.º 30
0
 def do_rabbit_addusers(cname):
     self = facility.get_component(cname)
     pwd = cmd_quote(util.get_keymgr()(self.name, 'openstack'))
     localsh.run("""rabbitmqctl add_user openstack {passwd} ||
                 rabbitmqctl change_password openstack {passwd} &&
                 rabbitmqctl set_permissions -p / openstack ".*" ".*" ".*"
                 """.format(passwd=pwd))
Ejemplo n.º 31
0
    def __init__(self, cmd, status=0, stdout=None, stderr=None, runtime=None):
        """
        :type cmd: list[str]
        :type status: int
        :type stdout: str | None
        :type stderr: str | None
        :type runtime: float | None
        """
        message = 'Command "%s" returned exit status %s.\n' % (' '.join(
            cmd_quote(c) for c in cmd), status)

        if stderr:
            message += '>>> Standard Error\n'
            message += '%s%s\n' % (stderr.strip(), Display.clear)

        if stdout:
            message += '>>> Standard Output\n'
            message += '%s%s\n' % (stdout.strip(), Display.clear)

        message = message.strip()

        super(SubprocessError, self).__init__(message)

        self.cmd = cmd
        self.message = message
        self.status = status
        self.stdout = stdout
        self.stderr = stderr
        self.runtime = runtime
Ejemplo n.º 32
0
    def delete_key(self):
        """Delete the server's public key from remote host.

        For example::

            wrapper = RemoteWrapper(hostname, username)
            with wrapper.connect(password, public_key_filename):
                wrapper.delete_key()

        """
        if self._public_key_filename is None:
            self._public_key_filename = os.path.expanduser('~/.ssh/id_rsa.pub')
        if self._client is None:
            raise ValueError(
                'Wrapper must be connected before delete_key is called')

        with open(self._public_key_filename, 'rt') as f:
            key = f.read().strip()

        self.chdir('/tmp')

        filename = 'django-remote-submission-{}'.format(uuid.uuid4)
        with self.open(filename, 'wt') as f:
            program = textwrap.dedent('''\
            sed -i.bak -e /{key}/d $HOME/.ssh/authorized_keys
            '''.format(key=cmd_quote(key.replace('/', '\/'))))

            f.write(program)

        args = [
            'bash',
            '/tmp/' + filename,
        ]

        self.exec_command(args, '/')
Ejemplo n.º 33
0
def get_size(path, excludes=None):
    try:
        cmd = "du -s --block-size=1G"
        if excludes:
            for item in excludes:
                cmd += ' --exclude=%s' % cmd_quote(item)
        cmd += ' %s | cut -f1' % cmd_quote(path)
        log.debug("Using: %s", cmd)
        # get size
        proc = os.popen(cmd)
        data = proc.read().strip("\n")
        proc.close()
        return int(data) if data.isdigit() else 0
    except Exception:
        log.exception("Exception getting size of %r: ", path)
    return 0
Ejemplo n.º 34
0
def format_env_dict(environ):
    """Format an environment dict for printing to console similarly to `typeset` builtin."""

    return '\n'.join([
        'typeset -x {}={}'.format(k, cmd_quote(v))
        for k, v in environ.items()
    ])
Ejemplo n.º 35
0
def format_env_dict(environ):
    """Format an environment dict for printing to console similarly to `typeset` builtin."""

    return '\n'.join([
        'typeset -x {}={}'.format(k, cmd_quote(v))
        for k, v in environ.items()
    ])
Ejemplo n.º 36
0
 def __extras2string(self):
     return ' '.join(
         "%s=%s" %
         (key, cmd_quote(value) if isinstance(value, str) else value)
         for (key,
              value) in self.rclone_extras.items()).replace('=None',
                                                            '').strip()
Ejemplo n.º 37
0
def format_usage(program_name, description, commands=None, options=()):
    """
    Construct the usage text.

    Parameters
    ----------
        program_name : str
            Usually the name of the python file that contains the experiment.
        description : str
            description of this experiment (usually the docstring).
        commands : dict[str, func]
            Dictionary of supported commands.
            Each entry should be a tuple of (name, function).
        options : list[sacred.commandline_options.CommandLineOption]
            A list of all supported commandline options.

    Returns
    -------
        str
            The complete formatted usage text for this experiment.
            It adheres to the structure required by ``docopt``.

    """
    usage = USAGE_TEMPLATE.format(
        program_name=cmd_quote(program_name),
        description=description.strip() if description else '',
        options=_format_options_usage(options),
        arguments=_format_arguments_usage(options),
        commands=_format_command_usage(commands)
    )
    return usage
Ejemplo n.º 38
0
 def __parse_argument(self, argument):
     long_option_match = self.long_option_pattern.match(argument)
     short_option_match = self.short_option_pattern.match(argument)
     if long_option_match:
         groupdict = long_option_match.groupdict()
         return '--{option}={value}'.format(
             option=groupdict['option'],
             value=cmd_quote(groupdict['value']),
         )
     elif short_option_match:
         groupdict = short_option_match.groupdict()
         return '-{option}={value}'.format(
             option=groupdict['option'],
             value=cmd_quote(groupdict['value']),
         )
     else:
         return cmd_quote(argument)
Ejemplo n.º 39
0
def print_creds(environment_overrides):
    exports = []
    for var, value in environment_overrides.items():
        if value is not None:
            exports.append("export {}={}".format(var, cmd_quote(value)))
        else:
            exports.append("unset {}".format(var))

    print("\n".join(exports))
Ejemplo n.º 40
0
    def _list(self):
        # Do a long listing to avoid connection reset
        # remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/')).rstrip()
        remote_dir = urllib.unquote(self.parsed_url.path)
        # print remote_dir
        quoted_path = cmd_quote(self.remote_path)
        # failing to cd into the folder might be because it was not created already
        commandline = "lftp -c \"source %s; ( cd %s && ls ) || ( mkdir -p %s && cd %s && ls )\"" % (
            cmd_quote(self.tempname),
            quoted_path, quoted_path, quoted_path
        )
        log.Debug("CMD: %s" % commandline)
        _, l, e = self.subprocess_popen(commandline)
        log.Debug("STDERR:\n"
                  "%s" % (e))
        log.Debug("STDOUT:\n"
                  "%s" % (l))

        # Look for our files as the last element of a long list line
        return [x.split()[-1] for x in l.split('\n') if x]
Ejemplo n.º 41
0
def convert(fname, pages=None):
    try:
        # attempt to find a pdftotext binary
        cmd = "where" if platform.system() == "Windows" else "which"
        cmd = subprocess.check_output(cmd + ' pdftotext', shell=True)
        cmd = cmd.decode('utf8').strip()
        if not cmd:
            raise EnvironmentError("pdftotext not found")
        
        return subprocess.check_output(' '.join([cmd, cmd_quote(fname), '-']), shell=True)
    except (EnvironmentError, subprocess.CalledProcessError):
        #logging.warning("pdftotext not found, defaulting to pdfminer.")
        return convert_miner(fname, pages=pages)
Ejemplo n.º 42
0
def expand_one_verb_alias(sysargs, verb_aliases, used_aliases):
    """Iterate through sysargs looking for expandable verb aliases.

    When a verb alias is found, sysargs is modified to effectively expand the alias.
    The alias is removed from verb_aliases and added to used_aliases.
    After finding and expanding an alias, this function returns True.
    If no alias is found to be expanded, this function returns False.
    """
    cmd = os.path.basename(sys.argv[0])
    for index, arg in enumerate(sysargs):
        if arg.startswith('-'):
            # Not a verb, continue through the arguments
            continue
        if arg in used_aliases:
            print(fmt(
                "@!@{gf}==>@| Expanding alias '@!@{yf}" + arg +
                "@|' was previously expanded, ignoring this time to prevent infinite recursion."
            ))
        if arg in verb_aliases:
            before = [] if index == 0 else sysargs[:index - 1]
            after = [] if index == len(sysargs) else sysargs[index + 1:]
            sysargs[:] = before + verb_aliases[arg] + after
            print(fmt(
                "@!@{gf}==>@| Expanding alias "
                "'@!@{yf}{alias}@|' "
                "from '@{yf}{before} @!{alias}@{boldoff}{after}@|' "
                "to '@{yf}{before} @!{expansion}@{boldoff}{after}@|'"
            ).format(
                alias=arg,
                expansion=' '.join([cmd_quote(aarg) for aarg in verb_aliases[arg]]),
                before=' '.join([cmd] + before),
                after=(' '.join([''] + after) if after else '')
            ))
            # Prevent the alias from being used again, to prevent infinite recursion
            used_aliases.append(arg)
            del verb_aliases[arg]
            # Return True since one has been found
            return True
        # Return False since no verb alias was found
        return False
Ejemplo n.º 43
0
def command(args, extra_args):  # noqa: C901 FIXME!!!
    stack = "parallelcluster-" + args.cluster_name
    config = cfnconfig.ParallelClusterConfig(args)
    if args.command in config.aliases:
        config_command = config.aliases[args.command]
    else:
        config_command = "ssh {CFN_USER}@{MASTER_IP} {ARGS}"

    cfn = boto3.client(
        "cloudformation",
        region_name=config.region,
        aws_access_key_id=config.aws_access_key_id,
        aws_secret_access_key=config.aws_secret_access_key,
    )
    try:
        stack_result = cfn.describe_stacks(StackName=stack).get("Stacks")[0]
        status = stack_result.get("StackStatus")
        valid_status = ["CREATE_COMPLETE", "UPDATE_COMPLETE", "UPDATE_ROLLBACK_COMPLETE"]
        invalid_status = ["DELETE_COMPLETE", "DELETE_IN_PROGRESS"]

        if status in invalid_status:
            LOGGER.info("Stack status: %s. Cannot SSH while in %s", status, " or ".join(invalid_status))
            sys.exit(1)
        elif status in valid_status:
            outputs = stack_result.get("Outputs")
            username = _get_output_value(outputs, "ClusterUser")
            ip = (
                _get_output_value(outputs, "MasterPublicIP")
                if _get_output_value(outputs, "MasterPublicIP")
                else _get_output_value(outputs, "MasterPrivateIP")
            )

            if not username:
                LOGGER.info("Failed to get cluster %s username.", args.cluster_name)
                sys.exit(1)

            if not ip:
                LOGGER.info("Failed to get cluster %s ip.", args.cluster_name)
                sys.exit(1)
        else:
            # Stack is in CREATING, CREATED_FAILED, or ROLLBACK_COMPLETE but MasterServer is running
            ip = _get_master_server_ip(stack, config)
            template = cfn.get_template(StackName=stack)
            mappings = template.get("TemplateBody").get("Mappings").get("OSFeatures")
            base_os = _get_param_value(stack_result.get("Parameters"), "BaseOS")
            username = mappings.get(base_os).get("User")

        try:
            from shlex import quote as cmd_quote
        except ImportError:
            from pipes import quote as cmd_quote

        # build command
        cmd = config_command.format(
            CFN_USER=username, MASTER_IP=ip, ARGS=" ".join(cmd_quote(str(e)) for e in extra_args)
        )

        # run command
        if not args.dryrun:
            os.system(cmd)
        else:
            LOGGER.info(cmd)
    except ClientError as e:
        LOGGER.critical(e.response.get("Error").get("Message"))
        sys.stdout.flush()
        sys.exit(1)
    except KeyboardInterrupt:
        LOGGER.info("\nExiting...")
        sys.exit(0)
Ejemplo n.º 44
0
def get_resultspace_environment(result_space_path, base_env=None, quiet=False, cached=True, strict=True):
    """Get the environemt variables which result from sourcing another catkin
    workspace's setup files as the string output of `cmake -E environment`.
    This cmake command is used to be as portable as possible.

    :param result_space_path: path to a Catkin result-space whose environment should be loaded, ``str``
    :type result_space_path: str
    :param quiet: don't throw exceptions, ``bool``
    :type quiet: bool
    :param cached: use the cached environment
    :type cached: bool
    :param strict: require the ``.catkin`` file exists in the resultspace
    :type strict: bool

    :returns: a dictionary of environment variables and their values
    """

    # Set bae environment to the current environment
    if base_env is None:
        base_env = dict(os.environ)

    # Get the MD5 checksums for the current env hooks
    # TODO: the env hooks path should be defined somewhere
    env_hooks_path = os.path.join(result_space_path, 'etc', 'catkin', 'profile.d')
    if os.path.exists(env_hooks_path):
        env_hooks = [
            md5(open(os.path.join(env_hooks_path, path)).read().encode('utf-8')).hexdigest()
            for path in os.listdir(env_hooks_path)]
    else:
        env_hooks = []

    # Check the cache first, if desired
    if cached and result_space_path in _resultspace_env_cache:
        (cached_base_env, cached_env_hooks, result_env) = _resultspace_env_cache.get(result_space_path)
        if env_hooks == cached_env_hooks and cached_base_env == base_env:
            return dict(result_env)

    # Check to make sure result_space_path is a valid directory
    if not os.path.isdir(result_space_path):
        if quiet:
            return dict()
        raise IOError(
            "Cannot load environment from resultspace \"%s\" because it does not "
            "exist." % result_space_path
        )

    # Check to make sure result_space_path contains a `.catkin` file
    # TODO: `.catkin` should be defined somewhere as an atom in catkin_pkg
    if strict and not os.path.exists(os.path.join(result_space_path, '.catkin')):
        if quiet:
            return dict()
        raise IOError(
            "Cannot load environment from resultspace \"%s\" because it does not "
            "appear to be a catkin-generated resultspace (missing .catkin marker "
            "file)." % result_space_path
        )

    # Determine the shell to use to source the setup file
    shell_path = os.environ.get('SHELL', None)
    if shell_path is None:
        shell_path = DEFAULT_SHELL
        if not os.path.isfile(shell_path):
            raise RuntimeError(
                "Cannot determine shell executable. "
                "The 'SHELL' environment variable is not set and "
                "the default '{0}' does not exist.".format(shell_path)
            )
    (_, shell_name) = os.path.split(shell_path)

    # Use fallback shell if using a non-standard shell
    if shell_name not in ['bash', 'zsh']:
        shell_name = 'bash'

    # Check to make sure result_space_path contains the appropriate setup file
    setup_file_path = os.path.join(result_space_path, 'env.sh')
    if not os.path.exists(setup_file_path):
        if quiet:
            return dict()
        raise IOError(
            "Cannot load environment from resultspace \"%s\" because the "
            "required setup file \"%s\" does not exist." % (result_space_path, setup_file_path)
        )

    # Construct a command list which sources the setup file and prints the env to stdout
    norc_flags = {
        'bash': '--norc',
        'zsh': '-f'
    }

    command = ' '.join([
        cmd_quote(setup_file_path),
        shell_path,
        norc_flags[shell_name],
        '-c',
        '"typeset -px"'
    ])

    # Define some "blacklisted" environment variables which shouldn't be copied
    blacklisted_keys = ('_', 'PWD')
    env_dict = {}

    try:
        # Run the command synchronously to get the resultspace environmnet
        if 0:
            # NOTE: This sometimes fails to get all output (returns prematurely)
            lines = ''
            for ret in execute_process(command, cwd=os.getcwd(), env=base_env, emulate_tty=False, shell=True):
                if type(ret) is bytes:
                    ret = ret.decode()
                if isinstance(ret, string_type):
                    lines += ret
        else:
            p = subprocess.Popen(command, cwd=os.getcwd(), env=base_env, shell=True, stdout=subprocess.PIPE)
            lines, _ = p.communicate()

        # Extract the environment variables
        env_dict = {
            k: v
            for k, v in parse_env_str(lines).items()
            if k not in blacklisted_keys
        }

        # Check to make sure we got some kind of environment
        if len(env_dict) > 0:
            # Cache the result
            _resultspace_env_cache[result_space_path] = (base_env, env_hooks, env_dict)
        else:
            print("WARNING: Sourced environment from `{}` has no environment variables. Something is wrong.".format(
                setup_file_path))

    except IOError as err:
        print("WARNING: Failed to extract environment from resultspace: {}: {}".format(
            result_space_path, str(err)),
            file=sys.stderr)

    return dict(env_dict)
Ejemplo n.º 45
0
def catkin_main(sysargs):
    # Initialize config
    try:
        initialize_config()
    except RuntimeError as exc:
        sys.exit("Failed to initialize config: {0}".format(exc))

    # Create a top level parser
    parser = argparse.ArgumentParser(
        description="catkin command", formatter_class=argparse.RawDescriptionHelpFormatter)
    add = parser.add_argument
    add('-a', '--list-aliases', action="store_true", default=False,
        help="Lists the current verb aliases and then quits, all other arguments are ignored")
    add('--test-colors', action='store_true', default=False,
        help="Prints a color test pattern to the screen and then quits, all other arguments are ignored")
    add('--version', action='store_true', default=False,
        help="Prints the catkin_tools version.")
    color_control_group = parser.add_mutually_exclusive_group()
    add = color_control_group.add_argument
    add('--force-color', action='store_true', default=False,
        help='Forces catkin to output in color, even when the terminal does not appear to support it.')
    add('--no-color', action='store_true', default=False,
        help='Forces catkin to not use color in the output, regardless of the detect terminal type.')

    # Deprecated, moved to `catkin locate --shell-verbs
    add('--locate-extra-shell-verbs', action='store_true', help=argparse.SUPPRESS)

    # Generate a list of verbs available
    verbs = list_verbs()

    # Create the subparsers for each verb and collect the argument preprocessors
    argument_preprocessors = create_subparsers(parser, verbs)

    # Get verb aliases
    verb_aliases = get_verb_aliases()

    # Setup sysargs
    sysargs = sys.argv[1:] if sysargs is None else sysargs

    # Get colors config
    no_color = False
    force_color = os.environ.get('CATKIN_TOOLS_FORCE_COLOR', False)
    for arg in sysargs:
        if arg == '--no-color':
            no_color = True
        if arg == '--force-color':
            force_color = True

    if no_color or not force_color and not is_tty(sys.stdout):
        set_color(False)

    # Check for version
    if '--version' in sysargs:
        print('catkin_tools {} (C) 2014-{} Open Source Robotics Foundation'.format(
            pkg_resources.get_distribution('catkin_tools').version,
            date.today().year)
        )
        print('catkin_tools is released under the Apache License,'
              ' Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0)')
        print('---')
        print('Using Python {}'.format(''.join(sys.version.split('\n'))))
        sys.exit(0)

    # Deprecated option
    if '--locate-extra-shell-verbs' in sysargs:
        print('Please use `catkin locate --shell-verbs` instead of `catkin --locate-extra-shell-verbs`',
              file=sys.stderr)
        sys.exit(0)

    # Check for --test-colors
    for arg in sysargs:
        if arg == '--test-colors':
            test_colors()
            sys.exit(0)
        if not arg.startswith('-'):
            break

    # Check for --list-aliases
    for arg in sysargs:
        if arg == '--list-aliases' or arg == '-a':
            for alias in sorted(list(verb_aliases.keys())):
                print("{0}: {1}".format(alias, ' '.join([cmd_quote(aarg) for aarg in verb_aliases[alias]])))
            sys.exit(0)
        if not arg.startswith('-'):
            break

    # Do verb alias expansion
    sysargs = expand_verb_aliases(sysargs, verb_aliases)

    # Determine the verb, splitting arguments into pre and post verb
    verb = None
    pre_verb_args = []
    post_verb_args = []
    for index, arg in enumerate(sysargs):
        # If the arg does not start with a `-` then it is a positional argument
        # The first positional argument must be the verb
        if not arg.startswith('-'):
            verb = arg
            post_verb_args = sysargs[index + 1:]
            break
        # If the `-h` or `--help` option comes before the verb, parse_args
        if arg in ['-h', '--help']:
            parser.parse_args(sysargs)
        # Otherwise it is a pre-verb option
        pre_verb_args.append(arg)

    # Error on no verb provided
    if verb is None:
        print(parser.format_usage())
        sys.exit("Error: No verb provided.")
    # Error on unknown verb provided
    if verb not in verbs:
        print(parser.format_usage())
        sys.exit("Error: Unknown verb '{0}' provided.".format(verb))

    # First allow the verb's argument preprocessor to strip any args
    # and return any "extra" information it wants as a dict
    processed_post_verb_args, extras = argument_preprocessors[verb](post_verb_args)
    # Then allow argparse to process the left over post-verb arguments along
    # with the pre-verb arguments and the verb itself
    args = parser.parse_args(pre_verb_args + [verb] + processed_post_verb_args)
    # Extend the argparse result with the extras from the preprocessor
    for key, value in extras.items():
        setattr(args, key, value)

    # Finally call the subparser's main function with the processed args
    # and the extras which the preprocessor may have returned
    sys.exit(args.main(args) or 0)
Ejemplo n.º 46
0
    def prime_docker_container(self, runtime, func_arn, env_vars, lambda_cwd):
        """
        Prepares a persistent docker container for a specific function.
        :param runtime: Lamda runtime environment. python2.7, nodejs6.10, etc.
        :param func_arn: The ARN of the lambda function.
        :param env_vars: The environment variables for the lambda.
        :param lambda_cwd: The local directory containing the code for the lambda function.
        :return: ContainerInfo class containing the container name and default entry point.
        """
        with self.docker_container_lock:
            # Get the container name and id.
            container_name = self.get_container_name(func_arn)

            LOG.debug('Priming docker container: %s' % container_name)

            status = self.get_docker_container_status(func_arn)
            # Container is not running or doesn't exist.
            if status < 1:
                # Make sure the container does not exist in any form/state.
                self.destroy_docker_container(func_arn)

                env_vars_str = ' '.join(['-e {}={}'.format(k, cmd_quote(v)) for (k, v) in env_vars])

                # Create and start the container
                LOG.debug('Creating container: %s' % container_name)
                cmd = (
                    'docker create'
                    ' --name "%s"'
                    ' --entrypoint /bin/bash'  # Load bash when it starts.
                    ' --interactive'  # Keeps the container running bash.
                    ' -e AWS_LAMBDA_EVENT_BODY="$AWS_LAMBDA_EVENT_BODY"'
                    ' -e HOSTNAME="$HOSTNAME"'
                    ' -e LOCALSTACK_HOSTNAME="$LOCALSTACK_HOSTNAME"'
                    '  %s'  # env_vars
                    ' lambci/lambda:%s'
                ) % (container_name, env_vars_str, runtime)
                LOG.debug(cmd)
                run(cmd, stderr=subprocess.PIPE, outfile=subprocess.PIPE)

                LOG.debug('Copying files to container "%s" from "%s".' % (container_name, lambda_cwd))
                cmd = (
                    'docker cp'
                    ' "%s/." "%s:/var/task"'
                ) % (lambda_cwd, container_name)
                LOG.debug(cmd)
                run(cmd, stderr=subprocess.PIPE, outfile=subprocess.PIPE)

                LOG.debug('Starting container: %s' % container_name)
                cmd = 'docker start %s' % (container_name)
                LOG.debug(cmd)
                run(cmd, stderr=subprocess.PIPE, outfile=subprocess.PIPE)
                # give the container some time to start up
                time.sleep(1)

            # Get the entry point for the image.
            LOG.debug('Getting the entrypoint for image: lambci/lambda:%s' % runtime)
            cmd = (
                'docker image inspect'
                ' --format="{{ .ContainerConfig.Entrypoint }}"'
                ' lambci/lambda:%s'
            ) % (runtime)

            LOG.debug(cmd)
            run_result = run(cmd, async=False, stderr=subprocess.PIPE, outfile=subprocess.PIPE)

            entry_point = run_result.strip('[]\n\r ')

            LOG.debug('Using entrypoint "%s" for container "%s".' % (entry_point, container_name))
            return ContainerInfo(container_name, entry_point)
Ejemplo n.º 47
0
    def __init__(self, parsed_url):
        duplicity.backend.Backend.__init__(self, parsed_url)

        # we expect an output
        try:
            p = os.popen("lftp --version")
            fout = p.read()
            ret = p.close()
        except Exception:
            pass
        # there is no output if lftp not found
        if not fout:
            log.FatalError("LFTP not found:  Please install LFTP.",
                           log.ErrorCode.ftps_lftp_missing)

        # version is the second word of the second part of the first line
        version = fout.split('\n')[0].split(' | ')[1].split()[1]
        log.Notice("LFTP version is %s" % version)

        self.parsed_url = parsed_url

        self.scheme = duplicity.backend.strip_prefix(parsed_url.scheme, 'lftp').lower()
        self.scheme = re.sub('^webdav', 'http', self.scheme)
        self.url_string = self.scheme + '://' + parsed_url.hostname
        if parsed_url.port:
            self.url_string += ":%s" % parsed_url.port

        self.remote_path = re.sub('^/', '', parsed_url.path)

        # Fix up an empty remote path
        if len(self.remote_path) == 0:
            self.remote_path = '/'

        # Use an explicit directory name.
        if self.remote_path[-1] != '/':
            self.remote_path += '/'

        self.authflag = ''
        if self.parsed_url.username:
            self.username = self.parsed_url.username
            self.password = self.get_password()
            self.authflag = "-u '%s,%s'" % (self.username, self.password)

        if globals.ftp_connection == 'regular':
            self.conn_opt = 'off'
        else:
            self.conn_opt = 'on'

        # check for cacert file if https
        self.cacert_file = globals.ssl_cacert_file
        if self.scheme == 'https' and not globals.ssl_no_check_certificate:
            cacert_candidates = ["~/.duplicity/cacert.pem",
                                 "~/duplicity_cacert.pem",
                                 "/etc/duplicity/cacert.pem"]
            # look for a default cacert file
            if not self.cacert_file:
                for path in cacert_candidates:
                    path = os.path.expanduser(path)
                    if (os.path.isfile(path)):
                        self.cacert_file = path
                        break

        # save config into a reusable temp file
        self.tempfile, self.tempname = tempdir.default().mkstemp()
        os.write(self.tempfile, "set ssl:verify-certificate " +
                 ("false" if globals.ssl_no_check_certificate else "true") + "\n")
        if self.cacert_file:
            os.write(self.tempfile, "set ssl:ca-file " + cmd_quote(self.cacert_file) + "\n")
        if globals.ssl_cacert_path:
            os.write(self.tempfile, "set ssl:ca-path " + cmd_quote(globals.ssl_cacert_path) + "\n")
        if self.parsed_url.scheme == 'ftps':
            os.write(self.tempfile, "set ftp:ssl-allow true\n")
            os.write(self.tempfile, "set ftp:ssl-protect-data true\n")
            os.write(self.tempfile, "set ftp:ssl-protect-list true\n")
        elif self.parsed_url.scheme == 'ftpes':
            os.write(self.tempfile, "set ftp:ssl-force on\n")
            os.write(self.tempfile, "set ftp:ssl-protect-data on\n")
            os.write(self.tempfile, "set ftp:ssl-protect-list on\n")
        else:
            os.write(self.tempfile, "set ftp:ssl-allow false\n")
        os.write(self.tempfile, "set http:use-propfind true\n")
        os.write(self.tempfile, "set net:timeout %s\n" % globals.timeout)
        os.write(self.tempfile, "set net:max-retries %s\n" % globals.num_retries)
        os.write(self.tempfile, "set ftp:passive-mode %s\n" % self.conn_opt)
        if log.getverbosity() >= log.DEBUG:
            os.write(self.tempfile, "debug\n")
        if self.parsed_url.scheme == 'ftpes':
            os.write(self.tempfile, "open %s %s\n" % (self.authflag, self.url_string.replace('ftpes', 'ftp')))
        else:
            os.write(self.tempfile, "open %s %s\n" % (self.authflag, self.url_string))
        os.close(self.tempfile)
        # print settings in debug mode
        if log.getverbosity() >= log.DEBUG:
            f = open(self.tempname, 'r')
            log.Debug("SETTINGS: \n"
                      "%s" % f.read())