def _bowtie2_index_command(bowtie2_path, reference): (reference_root, _) = os.path.splitext(reference) return '{bowtie2}-build {reference_in} {bt2_index_base}'.format(**{ 'bowtie2': shlex.quote(bowtie2_path), 'reference_in': shlex.quote(reference), 'bt2_index_base': shlex.quote(reference_root) })
def sync_to_worker_if_possible(self): """Syncs the local logdir on driver to worker if possible. Requires ray cluster to be started with the autoscaler. Also requires rsync to be installed. """ if self.worker_ip == self.local_ip: return ssh_key = get_ssh_key() ssh_user = get_ssh_user() global _log_sync_warned if ssh_key is None or ssh_user is None: if not _log_sync_warned: logger.error("Log sync requires cluster to be setup with " "`ray up`.") _log_sync_warned = True return if not distutils.spawn.find_executable("rsync"): logger.error("Log sync requires rsync to be installed.") return source = '{}/'.format(self.local_dir) target = '{}@{}:{}/'.format(ssh_user, self.worker_ip, self.local_dir) final_cmd = (("""rsync -savz -e "ssh -i {} -o ConnectTimeout=120s """ """-o StrictHostKeyChecking=no" {} {}""").format( quote(ssh_key), quote(source), quote(target))) logger.info("Syncing results to %s", str(self.worker_ip)) sync_process = subprocess.Popen( final_cmd, shell=True, stdout=self.logfile) sync_process.wait()
def svg2png(svg_filepath, png_filepath, dpi): """ Convert svg to png with inkscape. Delete svg file. :param str svg_filepath: SVG file path :param str png_filepath: PNG file path :param int dpi: DotPerInch for resolution :return: """ quoted_svg_filepath = shlex.quote(svg_filepath) quoted_png_filepath = shlex.quote(png_filepath) command = "inkscape -d {dpi} -z -f {svg_filepath} -e {png_filepath}".format( svg_filepath=quoted_svg_filepath, png_filepath=quoted_png_filepath, dpi=dpi) # execute command status = subprocess.call(shlex.split(command)) if status != 0: exit(status) # remove temporary svg file if os.path.exists(svg_filepath): try: os.unlink(svg_filepath) except IOError: pass
def _novoalign_command(path, args, ncpu, reference, sample_name, read1, read2=None): """ Args: path (str): path to aligner executable args (str): raw arguments to be passed to the aligner ncpu: number of alignment threads to launch reference: (str): reference filename sample_name (str): read1 (str): absolute path to read1 fastq[.gz|.bz2] read2 (str): absolute path to read2 fastq[.gz|.bz2] Returns: string: command to execute bowtie2 aligner """ import os aligner_command = '{novoalign} -d {dbname} -f {read1} {read2} {paired_string} -c {ncpu} -o SAM {bam_string} {novoalign_args}'.format(**{ 'novoalign': path, 'dbname': shlex.quote(reference + '.idx'), 'read1': shlex.quote(read1), 'read2': shlex.quote(read2) if read2 else '', 'paired_string': '-i PE 500,100' if read2 else '', 'ncpu': shlex.quote(str(ncpu)), 'bam_string': shlex.quote('@RG\\tID:{sample_name}\\tSM:{sample_name}'.format(sample_name=sample_name)), 'novoalign_args': ' '.join(map(shlex.quote, shlex.split(args))) }) return aligner_command
def png_add_mask_and_drop_shadow(source_filepath, mask_filepath, destination_filepath, shadow_offset=10): """ Resize source png at 205x280 (75dpi), then cut out with 75 dpi mask, then add drop shadow :param str source_filepath: Path to source png file :param str mask_filepath: Path to mask png file :param str destination_filepath: Path to save result file :param int shadow_offset: Offset of shadow in pixels :return: """ quoted_source_filepath = shlex.quote(source_filepath) quoted_mask_filepath = shlex.quote(mask_filepath) quoted_destination_filepath = shlex.quote(destination_filepath) command = "convert \( {source_filepath} -resize 205x280 {mask_filepath} -alpha Off -compose copyopacity -composite \) \ -background black \( +clone -shadow 60x{offset}+{offset}+{offset} \) +swap \ -compose Over -composite +repage \ {destination_filepath}".format( source_filepath=quoted_source_filepath, mask_filepath=quoted_mask_filepath, destination_filepath=quoted_destination_filepath, offset=shadow_offset) # execute command status = subprocess.call(shlex.split(command)) if status != 0: exit(status)
def write_symlinks(changes): for obj in changes: new = obj["new_filename"] old = obj["filenames"] for f in old: print("rm "+shlex.quote(f)) print("ln -s "+shlex.quote(os.path.abspath(new))+" "+shlex.quote(f))
def bash_complete(self, prefix, line, begidx, endidx): """Attempts BASH completion.""" splt = line.split() cmd = splt[0] func = self.bash_complete_funcs.get(cmd, None) fnme = self.bash_complete_files.get(cmd, None) if func is None or fnme is None: return set() idx = n = 0 for n, tok in enumerate(splt): if tok == prefix: idx = line.find(prefix, idx) if idx >= begidx: break prev = tok if len(prefix) == 0: prefix = '""' n += 1 else: prefix = shlex.quote(prefix) script = BASH_COMPLETE_SCRIPT.format( filename=fnme, line=' '.join(shlex.quote(p) for p in splt), comp_line=shlex.quote(line), n=n, func=func, cmd=cmd, end=endidx + 1, prefix=prefix, prev=shlex.quote(prev)) try: out = subprocess.check_output( ['bash'], input=script, universal_newlines=True, stderr=subprocess.PIPE, env=builtins.__xonsh_env__.detype()) except subprocess.CalledProcessError: out = '' rtn = set(out.splitlines()) return rtn
def xcheck_host_envar(conf, name, wafname=None): wafname = wafname or name chost, chost_envar = get_chost_stuff(conf) specific = None if chost: specific = os.environ.get('%s_%s' % (chost_envar, name), None) if specific: value = Utils.to_list(specific) conf.env[wafname] += value conf.msg('Will use cross-compilation %s from %s_%s' \ % (name, chost_envar, name), " ".join(quote(x) for x in value)) return envar = os.environ.get('HOST_%s' % name, None) if envar is None: return value = Utils.to_list(envar) if envar != '' else [envar] conf.env[wafname] = value conf.msg('Will use cross-compilation %s from HOST_%s' \ % (name, name), " ".join(quote(x) for x in value))
def update_platform(workspace_id: int, platform_id: int, platform_data) -> dict: """ Update the platform entry :param workspace_id: :param platform_id: :return: The updated platform definition """ platform_name = shlex.quote(platform_data['name']) platform_url = shlex.quote(platform_data['url']) session = db_session() workspace = session.query(Workspace).filter(Workspace.id == workspace_id).first() if workspace is None: raise NotFound("workspace with id {} could not be found".format(workspace_id)) platform = session.query(Platform). \ filter(Platform.workspace == workspace). \ filter(Platform.id == platform_id). \ first() if platform is None: raise NotFound("Platform with id {} could not be found".format(platform_id)) if platform_name != platform.name: existing_platforms = session.query(Platform). \ filter(Platform.workspace == workspace). \ filter(Platform.name == platform_data['name']). \ all() if len(existing_platforms) > 0: raise NameConflict("Platform with name {} already exists".format(platform_data['name'])) platform.name = platform_name platform.url = platform_url update_workspace_descriptor(platform.workspace) session.commit() return platform.as_dict()
def complete_from_bash(prefix, line, begidx, endidx, ctx): """Completes based on results from BASH completion.""" update_bash_completion() splt = line.split() cmd = splt[0] func = BASH_COMPLETE_FUNCS.get(cmd, None) fnme = BASH_COMPLETE_FILES.get(cmd, None) if func is None or fnme is None: return set() idx = n = 0 for n, tok in enumerate(splt): if tok == prefix: idx = line.find(prefix, idx) if idx >= begidx: break prev = tok if len(prefix) == 0: prefix = '""' n += 1 else: prefix = shlex.quote(prefix) script = BASH_COMPLETE_SCRIPT.format( filename=fnme, line=' '.join(shlex.quote(p) for p in splt), comp_line=shlex.quote(line), n=n, func=func, cmd=cmd, end=endidx + 1, prefix=prefix, prev=shlex.quote(prev)) try: out = subprocess.check_output( [xp.bash_command()], input=script, universal_newlines=True, stderr=subprocess.PIPE, env=builtins.__xonsh_env__.detype()) except (subprocess.CalledProcessError, FileNotFoundError): out = '' rtn = set(out.splitlines()) return rtn
def file(self, root, filename): self.file_count += 1 fullpath = os.path.join(root, filename) byte_size = os.lstat(fullpath).st_size self.size_total += byte_size self.ctx.current_file = fullpath local_vars = { '_': os.path.basename(filename), 'p': fullpath, 'ap': os.path.abspath(fullpath), 'apq': shlex.quote(os.path.abspath(fullpath)), 'pq': shlex.quote(fullpath), 'q': shlex.quote(filename), } fmt = string.Formatter() for (literal_text, field_name, format_spec, _) in fmt.parse(self.fmt_str): if literal_text is not None: sys.stdout.write(literal_text) if field_name is not None: value = eval(field_name, self.global_vars, local_vars) # pylint: disable=W0123 sys.stdout.write(format(value, format_spec))
def send_notify(self, uid, title, body): icon,name,surname = self._get_icon(uid) summary = title + ", " + name + ' ' + surname summary = shlex.quote(summary) body = shlex.quote(self._sanitize(body)) os.system("notify-send -i {icon} {summary} {body}".format( icon = icon, summary = summary, body = body))
def get_command(self, file, **options): # on darwin open returns immediately resulting in the temp # file removal while app is opening command = "open -a /Applications/Preview.app" command = "(%s %s; sleep 20; rm -f %s)&" % (command, quote(file), quote(file)) return command
async def async_exec(*args, display=False): """Execute, return code & log.""" argsp = [] for arg in args: if os.path.isfile(arg): argsp.append("\\\n {}".format(shlex.quote(arg))) else: argsp.append(shlex.quote(arg)) printc('cyan', *argsp) try: kwargs = {'loop': LOOP, 'stdout': asyncio.subprocess.PIPE, 'stderr': asyncio.subprocess.STDOUT} if display: kwargs['stderr'] = asyncio.subprocess.PIPE proc = await asyncio.create_subprocess_exec(*args, **kwargs) except FileNotFoundError as err: printc(FAIL, "Could not execute {}. Did you install test requirements?" .format(args[0])) raise err if not display: # Readin stdout into log stdout, _ = await proc.communicate() else: # read child's stdout/stderr concurrently (capture and display) stdout, _ = await asyncio.gather( read_stream(proc.stdout, sys.stdout.write), read_stream(proc.stderr, sys.stderr.write)) exit_code = await proc.wait() stdout = stdout.decode('utf-8') return exit_code, stdout
def run_configure_script(self): "runs configure-landscape, returns output (LDS hostname)" ldscreds = self.config.getopt('landscapecreds') args = {"bin": self.lscape_configure_bin, "admin_email": shlex.quote(ldscreds['admin_email']), "admin_name": shlex.quote(ldscreds['admin_name']), "sys_email": shlex.quote(ldscreds['system_email']), "maas_host": shlex.quote( self.config.getopt('maascreds')['api_host'])} cmd = ("{bin} --admin-email {admin_email} " "--admin-name {admin_name} " "--system-email {sys_email} " "--maas-host {maas_host}".format(**args)) log.debug("Running landscape configure: {}".format(cmd)) out = utils.get_command_output(cmd, timeout=None) if out['status']: log.error("Problem with configuring Landscape: {}.".format(out)) raise Exception("Error configuring Landscape.") return out['output'].strip()
def send_command(self, server_cmd): """ Sends the given command to all screen sessions with the world's screen name. :raises WorldIsOfflineError: if the world is offline. .. warning:: There is no guarantee, that the server reacted to the command. """ pids = self.pids() # Break if the world is offline. if not pids: raise WorldIsOfflineError(self) # Translate the server command for *cross-server* support. server_cmd = self._server.translate_command(server_cmd) # Quote the command. # The '\n' simulates pressing the ENTER key in a screen session. server_cmd += "\n\n" server_cmd = shlex.quote(server_cmd) # Send the command to the server. for pid in pids: sys_cmd = "screen -S {0}.{1} -p 0 -X stuff {2}"\ .format(pid, shlex.quote(self.screen_name()), server_cmd) sys_cmd = shlex.split(sys_cmd) subprocess.call(sys_cmd) return None
def start(self, collection, docker, ping, database_name): options = self.options """Launches a cAdvisor container on the instance.""" volumes = { '/': {'bind': '/rootfs', 'ro': True}, '/var/run': {'bind': '/var/run', 'ro': False}, '/sys': {'bind': '/sys', 'ro': True}, '/var/lib/docker': {'bind': '/var/lib/docker', 'ro': True} } logger.debug("cAdvisor: Writing stats to %s" % database_name) command_args = " ".join([ "-storage_driver=influxdb", "-log_dir=/", "-storage_driver_db=%s" % quote(database_name), "-storage_driver_host=%s:%d" % (quote(options.host), options.port), "-storage_driver_user=%s" % quote(options.user), "-storage_driver_password=%s" % quote(options.password), "-storage_driver_secure=%d" % options.secure, # TODO: Calculate based on the run time. "-storage_driver_buffer_duration=5s" ]) yield docker.run_containers(collection, self.info.name, None, command_args, volumes, ports={8080: 8080}) yield self.wait(collection, ping)
def create_catalogue(workspace_id: int, catalogue_data): """ Creates a catalgoue in the given workspace. A catalogue is defined by its name and url. These are given as json data :param workspace_id: Workspace ID of the target workspace, where the catalogue should get created. :return: Catalogue descriptor """ catalogue_name = shlex.quote(catalogue_data['name']) catalogue_url = shlex.quote(catalogue_data['url']) session = db_session() workspace = session.query(Workspace).filter(Workspace.id == workspace_id).first() if workspace is None: raise NotFound("workspace with id {} could not be found".format(workspace_id)) existing_catalogues = session.query(Catalogue). \ filter(Catalogue.workspace == workspace). \ filter(Catalogue.name == catalogue_data['name']). \ all() if len(existing_catalogues) > 0: raise NameConflict("catalogue with name {} already exists".format(catalogue_data['name'])) catalogue = Catalogue(name=catalogue_name, url=catalogue_url, workspace=workspace) session.add(catalogue) session.commit() update_workspace_descriptor(catalogue.workspace) return catalogue.as_dict()
def update_catalogue(workspace_id, catalogue_id, catalogue_data): """ Updates a specific catalogue by its id. The catalogue applies the given name and url, that are in the json parameter. :param workspace_id: The Workspace ID :param catalogue_id: The Catalogue ID :return: The updated Catalogue descriptor """ catalogue_name = shlex.quote(catalogue_data['name']) catalogue_url = shlex.quote(catalogue_data['url']) session = db_session() workspace = session.query(Workspace).filter(Workspace.id == workspace_id).first() if workspace is None: raise NotFound("workspace with id {} could not be found".format(workspace_id)) catalogue = session.query(Catalogue). \ filter(Catalogue.workspace == workspace). \ filter(Catalogue.id == catalogue_id). \ first() if catalogue is None: raise NotFound("catalogue with id {} could not be found".format(catalogue_id)) if catalogue_name != catalogue.name: existing_catalogues = session.query(catalogue). \ filter(catalogue.workspace == workspace). \ filter(catalogue.name == catalogue_data['name']). \ all() if len(existing_catalogues) > 0: raise NameConflict("catalogue with name {} already exists".format(catalogue_data['name'])) catalogue.name = catalogue_name catalogue.url = catalogue_url session.commit() update_workspace_descriptor(catalogue.workspace) return catalogue.as_dict()
def runExtensionCommand(self, command, filefilter, defaultext): import shlex of = ('%of' in command) html = ('%html' in command) if of: if defaultext and not filefilter: filefilter = '*'+defaultext fileName = QFileDialog.getSaveFileName(self, self.tr('Export document'), '', filefilter)[0] if not fileName: return if defaultext and not QFileInfo(fileName).suffix(): fileName += defaultext else: fileName = 'out' + defaultext basename = '.%s.retext-temp' % self.currentTab.getBaseName() if html: tmpname = basename+'.html' self.saveHtml(tmpname) else: tmpname = basename + self.currentTab.getActiveMarkupClass().default_extension self.currentTab.writeTextToFile(tmpname) command = command.replace('%of', shlex.quote(fileName)) command = command.replace('%html' if html else '%if', shlex.quote(tmpname)) try: Popen(str(command), shell=True).wait() except Exception as error: errorstr = str(error) QMessageBox.warning(self, '', self.tr('Failed to execute the command:') + '\n' + errorstr) QFile(tmpname).remove()
def clone(self, remote_address, target_directory=None, bare=False, recursive=True, branch=None, no_checkout=False, add_as_origin=False): if target_directory is not None: target_directory = Path(target_directory) self._set_wkdir(target_directory.dirname) cmd = [ "clone", "--bare" if bare else "", "--recursive" if recursive else "", "--branch {}".format(shlex.quote(branch)) if branch is not None else "", "--no-checkout" if no_checkout else "", remote_address, ## shlex.quote(Path(remote_address).nice_full_path), shlex.quote(Path(target_directory).basename) if target_directory is not None else "" ] if target_directory is None: self._local = Path(os.path.join(self._wkdir.full_path, Path(remote_address).basename)) else: self._local = Path(target_directory) self._set_cmd(cmd)._run() if add_as_origin: self._set_cmd([ "remote", 'add', 'origin', remote_address ])._run() return self
def install( self, ssh_client: paramiko.client.SSHClient, cluster: FlintrockCluster): print("[{h}] Installing Spark...".format( h=ssh_client.get_transport().getpeername()[0])) try: if self.version: with ssh_client.open_sftp() as sftp: sftp.put( localpath=os.path.join(SCRIPTS_DIR, 'install-spark.sh'), remotepath='/tmp/install-spark.sh') sftp.chmod(path='/tmp/install-spark.sh', mode=0o755) url = self.download_source.format(v=self.version) ssh_check_output( client=ssh_client, command=""" set -e /tmp/install-spark.sh {url} rm -f /tmp/install-spark.sh """.format(url=shlex.quote(url))) else: ssh_check_output( client=ssh_client, command=""" set -e sudo yum install -y git sudo yum install -y java-devel """) ssh_check_output( client=ssh_client, command=""" set -e git clone {repo} spark cd spark git reset --hard {commit} if [ -e "make-distribution.sh" ]; then ./make-distribution.sh -Phadoop-2.6 else ./dev/make-distribution.sh -Phadoop-2.6 fi """.format( repo=shlex.quote(self.git_repository), commit=shlex.quote(self.git_commit))) ssh_check_output( client=ssh_client, command=""" set -e for f in $(find spark/bin -type f -executable -not -name '*.cmd'); do sudo ln -s "$(pwd)/$f" "/usr/local/bin/$(basename $f)" done echo "export SPARK_HOME='$(pwd)/spark'" >> .bashrc """) except Exception as e: # TODO: This should be a more specific exception. print("Error: Failed to install Spark.", file=sys.stderr) print(e, file=sys.stderr) raise
def xcheck_host_prog(conf, name, tool, wafname=None): wafname = wafname or name chost, chost_envar = get_chost_stuff(conf) specific = None if chost: specific = os.environ.get('%s_%s' % (chost_envar, name)) if specific: value = Utils.to_list(specific) conf.env[wafname] += value conf.msg('Will use cross-compilation %s from %s_%s' % (name, chost_envar, name), " ".join(quote(x) for x in value)) return else: envar = os.environ.get('HOST_%s' % name) if envar is not None: value = Utils.to_list(envar) conf.env[wafname] = value conf.msg('Will use cross-compilation %s from HOST_%s' % (name, name), " ".join(quote(x) for x in value)) return if conf.env[wafname]: return value = None if chost: value = '%s-%s' % (chost, tool) if value: conf.env[wafname] = value conf.msg('Will use cross-compilation %s from CHOST' % wafname, value)
def run(self): from subprocess import call from pathlib import Path import os.path import inspect import shlex tm_path = shlex.quote(self.input()['tm'].fn) lm_path = shlex.quote(self.input()['lm']['blm'].fn) home = os.path.expanduser('~') dir_name = Path(inspect.stack()[-1][1]).absolute().parent current_dir_from_home = dir_name.relative_to(home) print(current_dir_from_home) toktagger_cmd = 'cd {cdir}; source {venv}; ./toktagger.py -t {tm} -l {lm} -f /'.format( cdir='~/' + shlex.quote(str(current_dir_from_home)), venv=shlex.quote('venv/bin/activate'), tm=tm_path, lm=lm_path) print(toktagger_cmd) parallel_cmd = 'parallel {params} -k --block-size {blocksize} --pipe {cmd}'.format( params=self.parallel_params, blocksize=self.parallel_blocksize, cmd=shlex.quote(toktagger_cmd)) cmd = shlex.split(parallel_cmd) print('running... ', parallel_cmd) with self.input()['input'].open( 'r') as in_file, self.output().open('w') as out_file: retcode = call(cmd, stdin=in_file, stdout=out_file) assert retcode == 0
def disp_kv(key, val): """ display a shell-escaped version of value ``val`` of ``key``, using terminal 'dim' attribute for read-only variables. """ return (self.dim(shlex.quote(val)) if key in self.server.readonly_env else shlex.quote(val))
def daemux_start(transitions, session="pmjq", shell='sh'): """Instantiate the transitions, each in its own tmux window""" for t in transitions: t = normalize(t) commands = [] # Template "directories" deals with watch-able templates # that use a list as input for dirs_key in [x for x in ["inputs", "outputs", "errors"] if x in t]: commands.append("watch -n1 "+shlex.quote( COMMAND_TEMPLATES['directories'].format( dirs=' '.join( map(lambda d: os.path.dirname(smart_unquote(d)), t[dirs_key]))))) # Template "stderr" deals with the log files if "stderr" in t: commands.append("watch -n1 "+shlex.quote( COMMAND_TEMPLATES['stderr'].format( stderr=os.path.dirname( smart_unquote(t['stderr']))))) # The command if shell == "sh": commands.append(pmjq_command(t)) elif shell == 'fish': commands.append(pmjq_command(t, redirect='^')) # The other templates can be used as is for k in [k for k in COMMAND_TEMPLATES if k not in ['directories', 'stderr', 'cmd']]: if k in t: commands.append(COMMAND_TEMPLATES[k].format(**t)) for i, cmd in enumerate(commands): daemux.start(cmd, session=session, window=t['id'], pane=i, layout='tiled')
def keygen(self, filename='', passphrase=''): """ Generate a public/private key pair and store them in ``filename``, encrypted by ``passphrase`` :param str filename: File name to store the private key in. The file name for the public key will be derived from this name by suffixing it with ``.pub`` :param str passphrase: The passphrase used for encrypting the private key. Please note this passphrase will only be accepted if it's longer than 4 characters. In case the passphrase being empty or too short, ssh-keygen will ask for a passphrase using the system's ssh-askpass mechanism. """ self._arguments.extend([ '-q', '-t', self._algorithm, '-b', str(self._keylength), '-O', 'clear', '-O', 'permit-pty', '-C', shlex.quote('{user}@{host}'.format(user=os.getlogin(), host=os.uname().nodename)), '-f', shlex.quote(filename) ]) if passphrase and len(passphrase) > 4: self._arguments.extend([ '-N', shlex.quote(passphrase) ]) self._execute()
def __str__(self): ret = '' for attr in 'cmd', 'ret_code', 'out', 'err': value = getattr(self, attr, None) if value is not None and str(value).strip(): mesg = '' if attr == 'cmd' and self.cmd_kwargs.get('stdin_files'): mesg += 'cat' for file_path in self.cmd_kwargs.get('stdin_files'): mesg += ' ' + quote(file_path) mesg += ' | ' if attr == 'cmd' and isinstance(value, list): mesg += ' '.join(quote(item) for item in value) else: mesg = str(value).strip() if attr == 'cmd' and self.cmd_kwargs.get('stdin_str'): mesg += ' <<<%s' % quote(self.cmd_kwargs.get('stdin_str')) if len(mesg.splitlines()) > 1: fmt = self.JOB_LOG_FMT_M else: fmt = self.JOB_LOG_FMT_1 if not mesg.endswith('\n'): mesg += '\n' ret += fmt % { 'cmd_key': self.cmd_key, 'attr': attr, 'mesg': mesg} return ret.rstrip()
def _snap_command(path, args, ncpu, reference, output_folder, sample_name, read1, read2=None): """ Args: path (str): path to aligner executable args (str): raw arguments to be passed to the aligner ncpu: number of alignment threads to launch reference: (str): reference filename output_folder (str): directory for aligner output sample_name (str): read1 (str): absolute path to read1 fastq[.gz|.bz2] read2 (str): absolute path to read2 fastq[.gz|.bz2] Returns: string: command to execute bowtie2 aligner """ import os aligner_command = '{snap} {single_or_paired} {ref_dir} {read1} {read2} -t {ncpu} -b {snap_args} -o sam -'.format(**{ 'snap': path, 'single_or_paired': 'paired' if read2 else 'single', 'ref_dir': shlex.quote(os.path.join(output_folder, 'reference', 'snap')), 'read1': shlex.quote(read1), 'read2': shlex.quote(read2) if read2 else '', 'ncpu': shlex.quote(str(ncpu)), 'snap_args': ' '.join(map(shlex.quote, shlex.split(args))) }) return aligner_command
def configure( self, ssh_client: paramiko.client.SSHClient, cluster: FlintrockCluster): template_paths = [ 'spark/conf/spark-env.sh', 'spark/conf/slaves', ] ssh_check_output( client=ssh_client, command="mkdir -p spark/conf", ) for template_path in template_paths: ssh_check_output( client=ssh_client, command=""" echo {f} > {p} """.format( f=shlex.quote( get_formatted_template( path=os.path.join(THIS_DIR, "templates", template_path), mapping=generate_template_mapping( cluster=cluster, spark_executor_instances=self.spark_executor_instances, hadoop_version=self.hadoop_version, spark_version=self.version or self.git_commit, ))), p=shlex.quote(template_path)))
def mpi_run(settings, nics, env, command, stdout=None, stderr=None): """ Runs mpi_run. Args: settings: Settings for running MPI. Note: settings.num_proc and settings.hosts must not be None. nics: Interfaces to include by MPI. env: Environment dictionary to use for running command. command: Command and arguments to run as a list of string. stdout: Stdout of the mpi process. Only used when settings.run_func_mode is True. stderr: Stderr of the mpi process. Only used when settings.run_func_mode is True. """ mpi_impl_flags, impl_binding_args = _get_mpi_implementation_flags( settings.tcp_flag) if mpi_impl_flags is None: raise Exception(_MPI_NOT_FOUND_ERROR_MSG) ssh_port_arg = '-mca plm_rsh_args \"-p {ssh_port}\"'.format( ssh_port=settings.ssh_port) if settings.ssh_port else '' # if user does not specify any hosts, mpirun by default uses local host. # There is no need to specify localhost. hosts_arg = '-H {hosts}'.format(hosts=settings.hosts) tcp_intf_arg = '-mca btl_tcp_if_include {nics}'.format( nics=','.join(nics)) if nics else '' nccl_socket_intf_arg = '-x NCCL_SOCKET_IFNAME={nics}'.format( nics=','.join(nics)) if nics else '' # On large cluster runs (e.g. Summit), we need extra settings to work around OpenMPI issues if settings.num_hosts and settings.num_hosts >= _LARGE_CLUSTER_THRESHOLD: mpi_impl_flags.append('-mca plm_rsh_no_tree_spawn true') mpi_impl_flags.append('-mca plm_rsh_num_concurrent {}'.format( settings.num_hosts)) binding_args = settings.binding_args if settings.binding_args else ' '.join( impl_binding_args) # Pass all the env variables to the mpirun command. mpirun_command = ( 'mpirun --allow-run-as-root --tag-output ' '-np {num_proc} {hosts_arg} ' '{binding_args} ' '{mpi_args} ' '{ssh_port_arg} ' '{tcp_intf_arg} ' '{nccl_socket_intf_arg} ' '{output_filename_arg} ' '{env} {extra_mpi_args} {command}' # expect a lot of environment variables .format(num_proc=settings.num_proc, hosts_arg=hosts_arg, binding_args=binding_args, mpi_args=' '.join(mpi_impl_flags), tcp_intf_arg=tcp_intf_arg, nccl_socket_intf_arg=nccl_socket_intf_arg, ssh_port_arg=ssh_port_arg, output_filename_arg='--output-filename ' + settings.output_filename if settings.output_filename else '', env=' '.join('-x %s' % key for key in sorted(env.keys()) if env_util.is_exportable(key)), extra_mpi_args=settings.extra_mpi_args if settings.extra_mpi_args else '', command=' '.join(quote(par) for par in command))) if settings.verbose >= 2: print(mpirun_command) # we need the driver's PATH in env to run mpirun, # env for mpirun is different to env encoded in mpirun_command if 'PATH' not in env and 'PATH' in os.environ: env = copy.copy(env) # copy env so we do not leak env modifications env['PATH'] = os.environ['PATH'] # Execute the mpirun command. if settings.run_func_mode: exit_code = safe_shell_exec.execute(mpirun_command, env=env, stdout=stdout, stderr=stderr) if exit_code != 0: raise RuntimeError( "mpirun failed with exit code {exit_code}".format( exit_code=exit_code)) else: os.execve('/bin/sh', ['/bin/sh', '-c', mpirun_command], env)
def connect(ssh_cmd, rhostport, python, stderr, options): portl = [] if re.sub(r'.*@', '', rhostport or '').count(':') > 1: if rhostport.count(']') or rhostport.count('['): result = rhostport.split(']') rhost = result[0].strip('[') if len(result) > 1: result[1] = result[1].strip(':') if result[1] != '': portl = ['-p', str(int(result[1]))] # can't disambiguate IPv6 colons and a port number. pass the hostname # through. else: rhost = rhostport else: # IPv4 l = (rhostport or '').rsplit(':', 1) rhost = l[0] if len(l) > 1: portl = ['-p', str(int(l[1]))] if rhost == '-': rhost = None z = zlib.compressobj(1) content = readfile('sshuttle.assembler') optdata = ''.join("%s=%r\n" % (k, v) for (k, v) in list(options.items())) optdata = optdata.encode("UTF8") content2 = (empackage(z, 'sshuttle') + empackage(z, 'sshuttle.cmdline_options', optdata) + empackage(z, 'sshuttle.helpers') + empackage(z, 'sshuttle.ssnet') + empackage(z, 'sshuttle.hostwatch') + empackage(z, 'sshuttle.server') + b"\n") pyscript = r""" import sys, os; verbosity=%d; sys.stdin = os.fdopen(0, "rb"); exec(compile(sys.stdin.read(%d), "assembler.py", "exec")) """ % (helpers.verbose or 0, len(content)) pyscript = re.sub(r'\s+', ' ', pyscript.strip()) if not rhost: # ignore the --python argument when running locally; we already know # which python version works. argv = [sys.executable, '-c', pyscript] else: if ssh_cmd: sshl = shlex.split(ssh_cmd) else: sshl = ['ssh'] if python: pycmd = "'%s' -c '%s'" % (python, pyscript) else: pycmd = ("P=python3; $P -V 2>%s || P=python; " "exec \"$P\" -c %s") % (os.devnull, quote(pyscript)) pycmd = ("exec /bin/sh -c %s" % quote(pycmd)) argv = (sshl + portl + [rhost, '--', pycmd]) (s1, s2) = socket.socketpair() def setup(): # runs in the child process s2.close() s1a, s1b = os.dup(s1.fileno()), os.dup(s1.fileno()) s1.close() debug2('executing: %r\n' % argv) p = ssubprocess.Popen(argv, stdin=s1a, stdout=s1b, preexec_fn=setup, close_fds=True, stderr=stderr) os.close(s1a) os.close(s1b) s2.sendall(content) s2.sendall(content2) return p, s2
def connect(ssh_cmd, rhostport, python, stderr, options): username, password, port, host = parse_hostport(rhostport) if username: rhost = "{}@{}".format(username, host) else: rhost = host z = zlib.compressobj(1) content = get_module_source('sshuttle.assembler') optdata = ''.join("%s=%r\n" % (k, v) for (k, v) in list(options.items())) optdata = optdata.encode("UTF8") content2 = (empackage(z, 'sshuttle') + empackage(z, 'sshuttle.cmdline_options', optdata) + empackage(z, 'sshuttle.helpers') + empackage(z, 'sshuttle.ssnet') + empackage(z, 'sshuttle.hostwatch') + empackage(z, 'sshuttle.server') + b"\n") pyscript = r""" import sys, os; verbosity=%d; sys.stdin = os.fdopen(0, "rb"); exec(compile(sys.stdin.read(%d), "assembler.py", "exec")) """ % (helpers.verbose or 0, len(content)) pyscript = re.sub(r'\s+', ' ', pyscript.strip()) if not rhost: # ignore the --python argument when running locally; we already know # which python version works. argv = [sys.executable, '-c', pyscript] else: if ssh_cmd: sshl = shlex.split(ssh_cmd) else: sshl = ['ssh'] if port is not None: portl = ["-p", str(port)] else: portl = [] if python: pycmd = "'%s' -c '%s'" % (python, pyscript) else: pycmd = ("P=python3; $P -V 2>%s || P=python; " "exec \"$P\" -c %s") % (os.devnull, quote(pyscript)) pycmd = ("/bin/sh -c {}".format(quote(pycmd))) if password is not None: os.environ['SSHPASS'] = str(password) argv = (["sshpass", "-e"] + sshl + portl + [rhost, '--', pycmd]) else: argv = (sshl + portl + [rhost, '--', pycmd]) (s1, s2) = socket.socketpair() def setup(): # runs in the child process s2.close() s1a, s1b = os.dup(s1.fileno()), os.dup(s1.fileno()) s1.close() debug2('executing: %r\n' % argv) p = ssubprocess.Popen(argv, stdin=s1a, stdout=s1b, preexec_fn=setup, close_fds=True, stderr=stderr) os.close(s1a) os.close(s1b) s2.sendall(content) s2.sendall(content2) return p, s2
import shlex import subprocess import tempfile import asm_processor dir_path = os.path.dirname(os.path.realpath(__file__)) prelude = os.path.join(dir_path, "prelude.s") all_args = sys.argv[1:] sep1 = all_args.index('--') sep2 = all_args.index('--', sep1 + 1) compiler = all_args[:sep1] assembler = all_args[sep1 + 1:sep2] assembler_sh = ' '.join(shlex.quote(x) for x in assembler) compile_args = all_args[sep2 + 1:] in_file = compile_args[-1] out_ind = compile_args.index('-o') out_file = compile_args[out_ind + 1] del compile_args[-1] del compile_args[out_ind + 1] del compile_args[out_ind] in_dir = os.path.split(os.path.realpath(in_file))[0] opt_flags = [ x for x in compile_args if x in ['-g3', '-g', '-O1', '-O2', '-framepointer'] ]
signal.signal(signal.SIGINT, lambda x, y: True) # Set the prompt args.append('--init-command') prompt_cmd = '''functions --copy fish_prompt original_fish_prompt function fish_prompt echo -n '[gst-{}] '(original_fish_prompt) end'''.format(gst_version) args.append(prompt_cmd) elif args[0].endswith('zsh'): tmpdir = tempfile.TemporaryDirectory() # Let the GC remove the tmp file tmprc = open(os.path.join(tmpdir.name, '.zshrc'), 'w') zshrc = os.path.expanduser('~/.zshrc') if os.path.exists(zshrc): with open(zshrc, 'r') as src: shutil.copyfileobj(src, tmprc) tmprc.write( '\nexport PROMPT="[gst-{}] $PROMPT"'.format(gst_version)) tmprc.flush() env['ZDOTDIR'] = tmpdir.name try: if options.only_environment: for name, value in env.items(): print('{}={}'.format(name, shlex.quote(value))) print('export {}'.format(name)) else: exit(subprocess.call(args, close_fds=False, env=env)) except subprocess.CalledProcessError as e: exit(e.returncode)
def command_string(self) -> str: return ' '.join(shlex.quote(c) for c in self.command)
def sublime(paths): quoted_paths = [shlex.quote(p) for p in paths] return App(command=_cmd(["subl", "-n"] + quoted_paths), xClass="Subl")
def path(self) -> str: return quote('/dev/md/' + self.name)
def path(self) -> str: # If LV or VG name has an hyphen '-', LVM doubles it in the path return quote('/dev/mapper/' + self.vg_name.replace('-', '--') + '-' + self.lv_name.replace('-', '--'))
def run_workflow_using_multiple_service_tables( input_datas, path_to_knime_executable, path_to_knime_workflow, input_service_table_node_ids, output_service_table_node_ids, *, live_passthru_stdout_stderr=False, output_as_pandas_dataframes=True, input_json_filename_pattern="input_%d.json", output_json_filename_pattern="output_%d.json", ): """Executes the requested KNIME workflow, feeding the supplied data to the Container Input (Table) nodes in that workflow and returning the output from the workflow's Container Output (Table) nodes.""" abspath_to_knime_workflow = Path(path_to_knime_workflow).absolute() if not Path(path_to_knime_executable).exists(): raise ValueError(f"Executable not found: {path_to_knime_executable}") with tempfile.TemporaryDirectory() as temp_dir: logging.debug(f"using temp dir: {temp_dir}") option_flags_input_service_table_nodes = [] for node_id, data in zip(input_service_table_node_ids, input_datas): input_json_filename = input_json_filename_pattern % node_id input_json_filepath = Path(temp_dir, input_json_filename) # Support pandas DataFrame-like inputs. try: data = convert_dataframe_to_knime_friendly_dict(data) except AttributeError: pass with open(input_json_filepath, "w") as input_json_fh: json.dump(data, input_json_fh) option_flags_input_service_table_nodes.append( f'-option={node_id},inputPathOrUrl,"{input_json_filepath}",String' ) option_flags_output_service_table_nodes = [] expected_output_json_files = [] for node_id in output_service_table_node_ids: output_json_filename = output_json_filename_pattern % node_id output_json_filepath = Path(temp_dir, output_json_filename) option_flags_input_service_table_nodes.append( f'-option={node_id},outputPathOrUrl,"{output_json_filepath}",String', ) expected_output_json_files.append(output_json_filepath) data_dir = Path(temp_dir, "knime_data") # shlex.quote handles executable paths containing spaces, etc. # On Windows, cmd shell requires double-quotes, hence replace() shell_command = " ".join([ shlex.quote(path_to_knime_executable).replace("'", '"'), "-nosplash", "-debug", "--launcher.suppressErrors", "-application org.knime.product.KNIME_BATCH_APPLICATION", f"-data {data_dir}", f'-workflowDir="{abspath_to_knime_workflow}"', " ".join(option_flags_input_service_table_nodes), " ".join(option_flags_output_service_table_nodes), ]) logging.info(f"knime invocation: {shell_command}") result = subprocess.run( shell_command, shell=True, stdout=subprocess.PIPE if not live_passthru_stdout_stderr else None, stderr=subprocess.PIPE if not live_passthru_stdout_stderr else None, ) logging.info(f"exit code from KNIME execution: {result.returncode}") knime_outputs = [] try: for output_json_filepath in expected_output_json_files: with open(output_json_filepath) as output_json_fh: single_node_knime_output = json.load(output_json_fh) knime_outputs.append(single_node_knime_output) except FileNotFoundError: logging.error(f"captured stdout: {result.stdout}") logging.error(f"captured stderr: {result.stderr}") raise ChildProcessError("Output from KNIME not found") if output_as_pandas_dataframes: try: import pandas as pd for i, output in enumerate(knime_outputs): df_columns = list( k for d in output['table-spec'] for k, v in d.items() ) knime_outputs[i] = pd.DataFrame( output['table-data'], columns=df_columns ) except ImportError: logging.warning("requested output as DataFrame not possible") except Exception as e: logging.error("error while converting KNIME output to DataFrame") raise e if result.returncode != 0: logging.info(f"captured stdout: {result.stdout}") logging.info(f"captured stderr: {result.stderr}") return knime_outputs
def path(self) -> str: prefix = 'PARTUUID=' if self.partition else 'UUID=' return '"$(findfs ' + quote(prefix + self.uuid) + ')"'
def path(self) -> str: return quote(self.mountpoint)
def get_or_create_head_node(config, config_file, no_restart, restart_only, yes, override_cluster_name): """Create the cluster head node, which in turn creates the workers.""" provider = get_node_provider(config["provider"], config["cluster_name"]) try: head_node_tags = { TAG_RAY_NODE_TYPE: "head", } nodes = provider.non_terminated_nodes(head_node_tags) if len(nodes) > 0: head_node = nodes[0] else: head_node = None if not head_node: confirm("This will create a new cluster", yes) elif not no_restart: confirm("This will restart cluster services", yes) launch_hash = hash_launch_conf(config["head_node"], config["auth"]) if head_node is None or provider.node_tags(head_node).get( TAG_RAY_LAUNCH_CONFIG) != launch_hash: if head_node is not None: confirm("Head node config out-of-date. It will be terminated", yes) logger.info( "get_or_create_head_node: " "Terminating outdated head node {}".format(head_node)) provider.terminate_node(head_node) logger.info("get_or_create_head_node: Launching new head node...") head_node_tags[TAG_RAY_LAUNCH_CONFIG] = launch_hash head_node_tags[TAG_RAY_NODE_NAME] = "ray-{}-head".format( config["cluster_name"]) provider.create_node(config["head_node"], head_node_tags, 1) nodes = provider.non_terminated_nodes(head_node_tags) assert len(nodes) == 1, "Failed to create head node." head_node = nodes[0] # TODO(ekl) right now we always update the head node even if the hash # matches. We could prompt the user for what they want to do here. runtime_hash = hash_runtime_conf(config["file_mounts"], config) logger.info("get_or_create_head_node: Updating files on head node...") # Rewrite the auth config so that the head node can update the workers remote_key_path = "~/ray_bootstrap_key.pem" remote_config = copy.deepcopy(config) remote_config["auth"]["ssh_private_key"] = remote_key_path # Adjust for new file locations new_mounts = {} for remote_path in config["file_mounts"]: new_mounts[remote_path] = remote_path remote_config["file_mounts"] = new_mounts remote_config["no_restart"] = no_restart # Now inject the rewritten config and SSH key into the head node remote_config_file = tempfile.NamedTemporaryFile( "w", prefix="ray-bootstrap-") remote_config_file.write(json.dumps(remote_config)) remote_config_file.flush() config["file_mounts"].update({ remote_key_path: config["auth"]["ssh_private_key"], "~/ray_bootstrap_config.yaml": remote_config_file.name }) if restart_only: init_commands = config["head_start_ray_commands"] elif no_restart: init_commands = config["head_setup_commands"] else: init_commands = (config["head_setup_commands"] + config["head_start_ray_commands"]) updater = NodeUpdaterThread( node_id=head_node, provider_config=config["provider"], provider=provider, auth_config=config["auth"], cluster_name=config["cluster_name"], file_mounts=config["file_mounts"], initialization_commands=config["initialization_commands"], setup_commands=init_commands, runtime_hash=runtime_hash, ) updater.start() updater.join() # Refresh the node cache so we see the external ip if available provider.non_terminated_nodes(head_node_tags) if config.get("provider", {}).get("use_internal_ips", False) is True: head_node_ip = provider.internal_ip(head_node) else: head_node_ip = provider.external_ip(head_node) if updater.exitcode != 0: logger.error("get_or_create_head_node: " "Updating {} failed".format(head_node_ip)) sys.exit(1) logger.info( "get_or_create_head_node: " "Head node up-to-date, IP address is: {}".format(head_node_ip)) monitor_str = "tail -n 100 -f /tmp/ray/session_*/logs/monitor*" use_docker = bool(config["docker"]["container_name"]) if override_cluster_name: modifiers = " --cluster-name={}".format( quote(override_cluster_name)) else: modifiers = "" print("To monitor auto-scaling activity, you can run:\n\n" " ray exec {} {}{}{}\n".format( config_file, "--docker " if use_docker else " ", quote(monitor_str), modifiers)) print("To open a console on the cluster:\n\n" " ray attach {}{}\n".format(config_file, modifiers)) print("To ssh manually to the cluster, run:\n\n" " ssh -i {} {}@{}\n".format(config["auth"]["ssh_private_key"], config["auth"]["ssh_user"], head_node_ip)) finally: provider.cleanup()
def path(self) -> str: prefix = 'PARTLABEL=' if self.partition else 'LABEL=' return '"$(findfs ' + quote(prefix + self.label) + ')"'
def copy(self, src, dest): dest = shlex.quote(dest) src = shlex.quote(src) self.execute(f"cp {src} {dest}")
def path(self) -> str: return quote(self.datapath)
def build(options: BuildOptions): try: subprocess.check_call(['docker', '--version']) except Exception: print( 'cibuildwheel: Docker not found. Docker is required to run Linux builds. ' 'If you\'re building on Travis CI, add `services: [docker]` to your .travis.yml.' 'If you\'re building on Circle CI in Linux, add a `setup_remote_docker` step to your .circleci/config.yml', file=sys.stderr) exit(2) python_configurations = get_python_configurations(options.build_selector) platforms = [ ('cp', 'manylinux_x86_64', options.manylinux_images['x86_64']), ('cp', 'manylinux_i686', options.manylinux_images['i686']), ('cp', 'manylinux_aarch64', options.manylinux_images['aarch64']), ('cp', 'manylinux_ppc64le', options.manylinux_images['ppc64le']), ('cp', 'manylinux_s390x', options.manylinux_images['s390x']), ('pp', 'manylinux_x86_64', options.manylinux_images['pypy_x86_64']), ] for implementation, platform_tag, docker_image in platforms: platform_configs = [ c for c in python_configurations if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag) ] if not platform_configs: continue container_name = 'cibuildwheel-{}'.format(uuid.uuid4()) try: call([ 'docker', 'create', '--env', 'CIBUILDWHEEL', '--name', container_name, '-i', '-v', '/:/host', # ignored on CircleCI docker_image, '/bin/bash' ]) call([ 'docker', 'cp', os.path.abspath(options.project_dir) + '/.', container_name + ':/project' ]) call(['docker', 'start', container_name]) for config in platform_configs: if options.dependency_constraints: constraints_file = options.dependency_constraints.get_for_python_version( config.version) # `docker cp` causes 'no space left on device' error when # a container is running and the host filesystem is # mounted. https://github.com/moby/moby/issues/38995 # Use `docker exec` instead. with open(constraints_file, 'rb') as f: call( [ 'docker', 'exec', '-i', container_name, 'sh', '-c', 'cat > /constraints.txt' ], input=f.read(), ) call(['docker', 'exec', '-i', container_name, '/bin/bash'], universal_newlines=True, input=''' # give xtrace output an extra level of indent inside docker PS4=' + ' set -o errexit set -o xtrace mkdir -p /output cd /project PYBIN="{config_python_bin}" export PATH="$PYBIN:$PATH" {environment_exports} # check the active python and pip are in PYBIN if [ "$(which pip)" != "$PYBIN/pip" ]; then echo "cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it." exit 1 fi if [ "$(which python)" != "$PYBIN/python" ]; then echo "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it." exit 1 fi if [ ! -z {before_build} ]; then sh -c {before_build} fi # Build the wheel rm -rf /tmp/built_wheel mkdir /tmp/built_wheel pip wheel . -w /tmp/built_wheel --no-deps {build_verbosity_flag} built_wheel=(/tmp/built_wheel/*.whl) # repair the wheel rm -rf /tmp/repaired_wheels mkdir /tmp/repaired_wheels # NOTE: 'built_wheel' here is a bash array of glob matches; "$built_wheel" returns # the first element if [[ "$built_wheel" == *none-any.whl ]] || [ -z {repair_command} ]; then # pure Python wheel or empty repair command mv "$built_wheel" /tmp/repaired_wheels else sh -c {repair_command} repair_command "$built_wheel" fi repaired_wheels=(/tmp/repaired_wheels/*.whl) if [ ! -z {test_command} ]; then # Set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. pip install {dependency_install_flags} virtualenv venv_dir=`mktemp -d`/venv python -m virtualenv --no-download "$venv_dir" export __CIBW_VIRTUALENV_PATH__=$venv_dir # run the tests in a subshell to keep that `activate` # script from polluting the env ( source "$venv_dir/bin/activate" echo "Running tests using `which python`" if [ ! -z {before_test} ]; then sh -c {before_test} fi # Install the wheel we just built # Note: If auditwheel produced two wheels, it's because the earlier produced wheel # conforms to multiple manylinux standards. These multiple versions of the wheel are # functionally the same, differing only in name, wheel metadata, and possibly include # different external shared libraries. so it doesn't matter which one we run the tests on. # Let's just pick the first one. pip install "${{repaired_wheels[0]}}"{test_extras} # Install any requirements to run the tests if [ ! -z "{test_requires}" ]; then pip install {test_requires} fi # Run the tests from a different directory pushd $HOME sh -c {test_command} popd ) # exit if tests failed (needed for older bash versions) if [ $? -ne 0 ]; then exit 1; fi # clean up rm -rf "$venv_dir" fi # we're all done here; move it to output mv "${{repaired_wheels[@]}}" /output for repaired_wheel in "${{repaired_wheels[@]}}"; do chown {uid}:{gid} "/output/$(basename "$repaired_wheel")" done '''.format( config_python_bin=config.path + '/bin', test_requires=' '.join(options.test_requires), test_extras=options.test_extras, test_command=shlex.quote( prepare_command(options.test_command, project='/project') if options. test_command else ''), before_build=shlex.quote( prepare_command(options.before_build, project='/project') if options. before_build else ''), build_verbosity_flag=' '.join( get_build_verbosity_extra_flags( options.build_verbosity)), repair_command=shlex.quote( prepare_command(options.repair_command, wheel='"$1"', dest_dir='/tmp/repaired_wheels') if options.repair_command else ''), environment_exports='\n'.join( options.environment.as_shell_commands()), uid=os.getuid(), gid=os.getgid(), before_test=shlex.quote( prepare_command(options.before_test, project='/project') if options. before_test else ''), dependency_install_flags='-c /constraints.txt' if options.dependency_constraints else '', )) # copy the output back into the host call([ 'docker', 'cp', container_name + ':/output/.', os.path.abspath(options.output_dir) ]) except subprocess.CalledProcessError as error: troubleshoot(options.project_dir, error) exit(1) finally: # Still gets executed, even when 'exit(1)' gets called call(['docker', 'rm', '--force', '-v', container_name])
def hardlink(self, src, dest): dest = shlex.quote(dest) src = shlex.quote(src) self.execute(f"ln {src} {dest}")
def create_remote_file(self, vm, filename, content): self.loop.run_until_complete( vm.run_for_stdio('cat > {}'.format(shlex.quote(filename)), user='******', input=content.encode('utf-8')))
def __repr__(self): """Create representation of Task.""" return '<Task({}): command={}, config={}>'.format( self.taskname, repr(' '.join([quote(opt) for opt in self._build_command()])), repr(self.config))
def main(): args = _ARGPARSER.parse_args() arch = architectures.get(args.arch) is_native = (args.arch == uname.machine) qemu = qemu_helpers.Qemu(arch.qemuname) qemu.probe() need_initramfs = args.force_initramfs or qemu.cannot_overmount_virtfs config = mkinitramfs.Config() kimg,dtb,modfiles,moddir = find_kernel_and_mods(arch, args) config.modfiles = modfiles if config.modfiles: need_initramfs = True qemuargs = [qemu.qemubin] kernelargs = [] # Put the '-name' flag first so it's easily visible in ps, top, etc. if args.name: qemuargs.extend(['-name', args.name]) kernelargs.append('virtme_hostname=%s' % args.name) # Set up virtfs export_virtfs(qemu, arch, qemuargs, args.root, '/dev/root', readonly=(not args.rw)) guest_tools_path = guest_tools.find_guest_tools() if guest_tools_path is None: raise ValueError("couldn't find guest tools -- virtme is installed incorrectly") export_virtfs(qemu, arch, qemuargs, guest_tools_path, 'virtme.guesttools') initcmds = ['mkdir -p /run/virtme/guesttools', '/bin/mount -n -t 9p -o ro,version=9p2000.L,trans=virtio,access=any virtme.guesttools /run/virtme/guesttools', 'exec /run/virtme/guesttools/virtme-init'] # Map modules if moddir is not None: export_virtfs(qemu, arch, qemuargs, moddir, 'virtme.moddir') # Set up mounts mount_index = 0 for dirtype, dirarg in itertools.chain((('rwdir', i) for i in args.rwdir), (('rodir', i) for i in args.rodir)): m = _RWDIR_RE.match(dirarg) if not m: arg_fail('invalid --%s parameter %r' % (dirtype, dirarg)) if m.group(2) is not None: guestpath = m.group(1) hostpath = m.group(2) else: hostpath = m.group(1) guestpath = os.path.relpath(hostpath, args.root) if guestpath.startswith('..'): arg_fail('%r is not inside the root' % hostpath) idx = mount_index mount_index += 1 tag = 'virtme.initmount%d' % idx export_virtfs(qemu, arch, qemuargs, hostpath, tag, readonly=(dirtype != 'rwdir')) kernelargs.append('virtme_initmount%d=%s' % (idx, guestpath)) # Turn on KVM if available if is_native: qemuargs.extend(['-machine', 'accel=kvm:tcg']) # Add architecture-specific options qemuargs.extend(arch.qemuargs(is_native)) # Set up / override baseline devices qemuargs.extend(['-parallel', 'none']) qemuargs.extend(['-net', 'none']) if not args.graphics and not args.script_sh and not args.script_exec: # It would be nice to use virtconsole, but it's terminally broken # in current kernels. Nonetheless, I'm configuring the console # manually to make it easier to tweak in the future. qemuargs.extend(['-echr', '1']) qemuargs.extend(['-serial', 'none']) qemuargs.extend(['-chardev', 'stdio,id=console,signal=off,mux=on']) # We should be using the new-style -device serialdev,chardev=xyz, # but many architecture-specific serial devices don't support that. qemuargs.extend(['-serial', 'chardev:console']) qemuargs.extend(['-mon', 'chardev=console']) kernelargs.extend(arch.earlyconsole_args()) kernelargs.extend(arch.serial_console_args()) qemuargs.extend(arch.qemu_nodisplay_args()) # PS/2 probing is slow; give the kernel a hint to speed it up. kernelargs.extend(['psmouse.proto=exps']) # Fix the terminal defaults (and set iutf8 because that's a better # default nowadays). I don't know of any way to keep this up to date # after startup, though. try: terminal_size = os.get_terminal_size() kernelargs.extend(['virtme_stty_con=rows %d cols %d iutf8' % (terminal_size.lines, terminal_size.columns)]) except OSError as e: # don't die if running with a non-TTY stdout if e.errno != errno.ENOTTY: raise # Propagate the terminal type if 'TERM' in os.environ: kernelargs.extend(['TERM=%s' % os.environ['TERM']]) if args.balloon: qemuargs.extend(['-balloon', 'virtio']) if args.disk: qemuargs.extend(['-device', '%s,id=scsi' % arch.virtio_dev_type('scsi')]) for i,d in enumerate(args.disk): namefile = d.split('=', 1) if len(namefile) != 2: arg_fail('invalid argument to --disk') name,fn = namefile if '=' in fn or ',' in fn: arg_fail("--disk filenames cannot contain '=' or ','") if '=' in fn or ',' in name: arg_fail("--disk device names cannot contain '=' or ','") driveid = 'disk%d' % i qemuargs.extend(['-drive', 'if=none,id=%s,file=%s' % (driveid, fn), '-device', 'scsi-hd,drive=%s,vendor=virtme,product=disk,serial=%s' % (driveid, name)]) has_script = False def do_script(shellcmd, use_exec=False): if args.graphics: arg_fail('scripts and --graphics are mutually exclusive') nonlocal has_script nonlocal need_initramfs if has_script: arg_fail('conflicting script options') has_script = True need_initramfs = True # TODO: Fix this # Turn off default I/O qemuargs.extend(arch.qemu_nodisplay_args()) # Send kernel logs to stderr qemuargs.extend(['-serial', 'none']) qemuargs.extend(['-chardev', 'file,id=console,path=/proc/self/fd/2']) # We should be using the new-style -device serialdev,chardev=xyz, # but many architecture-specific serial devices don't support that. qemuargs.extend(['-serial', 'chardev:console']) # Set up a virtserialport for script I/O qemuargs.extend(['-chardev', 'stdio,id=stdio,signal=on,mux=off']) qemuargs.extend(['-device', arch.virtio_dev_type('serial')]) qemuargs.extend(['-device', 'virtserialport,name=virtme.scriptio,chardev=stdio']) # Scripts shouldn't reboot qemuargs.extend(['-no-reboot']) # Ask virtme-init to run the script config.virtme_data[b'script'] = """#!/bin/sh {prefix}{shellcmd} """.format(shellcmd=shellcmd, prefix="exec " if use_exec else "").encode('ascii') # Nasty issue: QEMU will set O_NONBLOCK on fds 0, 1, and 2. # This isn't inherently bad, but it can cause a problem if # another process is reading from 1 or writing to 0, which is # exactly what happens if you're using a terminal and you # redirect some, but not all, of the tty fds. Work around it # by giving QEMU private copies of the open object if either # of them is a terminal. for oldfd,mode in ((0,os.O_RDONLY), (1,os.O_WRONLY), (2,os.O_WRONLY)): if os.isatty(oldfd): try: newfd = os.open('/proc/self/fd/%d' % oldfd, mode) except OSError: pass else: os.dup2(newfd, oldfd) os.close(newfd) if args.script_sh is not None: do_script(args.script_sh) if args.script_exec is not None: do_script(shlex.quote(args.script_exec), use_exec=True) if args.net: qemuargs.extend(['-net', 'nic,model=virtio']) qemuargs.extend(['-net', 'user']) kernelargs.extend(['virtme.dhcp']) if args.pwd: rel_pwd = os.path.relpath(os.getcwd(), args.root) if rel_pwd.startswith('..'): print('current working directory is not contained in the root') return 1 kernelargs.append('virtme_chdir=%s' % rel_pwd) if need_initramfs: if args.busybox is not None: config.busybox = args.busybox else: config.busybox = mkinitramfs.find_busybox(args.root, is_native) if config.busybox is None: print('virtme-run: initramfs is needed, and no busybox was found', file=sys.stderr) return 1 # Set up the initramfs (warning: hack ahead) tmpfd,tmpname = tempfile.mkstemp('irfs') os.unlink(tmpname) tmpfile = os.fdopen(tmpfd, 'r+b') mkinitramfs.mkinitramfs(tmpfile, config) tmpfile.flush() fcntl.fcntl(tmpfd, fcntl.F_SETFD, 0) initrdpath = '/proc/self/fd/%d' % tmpfile.fileno() else: # No initramfs! Warning: this is slower than using an initramfs # because the kernel will wait for device probing to finish. # Sigh. kernelargs.extend([ 'rootfstype=9p', 'rootflags=version=9p2000.L,trans=virtio,access=any', 'raid=noautodetect', 'rw' if args.rw else 'ro', ]) initrdpath = None initcmds.insert(0, 'mount -t tmpfs run /run') # Now that we're done setting up kernelargs, append user-specified args # and then initargs kernelargs.extend(args.kopt) # Unknown options get turned into arguments to init, which is annoying # because we're explicitly passing '--' to set the arguments directly. # Fortunately, 'init=' will clear any arguments parsed so far, so make # sure that 'init=' appears directly before '--'. kernelargs.append('init=/bin/sh') kernelargs.append('--') kernelargs.extend(['-c', ';'.join(initcmds)]) if args.xen is None: # Load a normal kernel qemuargs.extend(['-kernel', kimg]) if kernelargs: qemuargs.extend(['-append', ' '.join(quote_karg(a) for a in kernelargs)]) if initrdpath is not None: qemuargs.extend(['-initrd', initrdpath]) if dtb is not None: qemuargs.extend(['-dtb', dtb]) else: # Use multiboot syntax to load Xen qemuargs.extend(['-kernel', args.xen]) qemuargs.extend(['-initrd', '%s %s%s' % ( kimg, ' '.join(quote_karg(a).replace(',', ',,') for a in kernelargs), (',%s' % initrdpath) if initrdpath is not None else '')]) # Handle --qemu-opt(s) qemuargs.extend(args.qemu_opt) if args.qemu_opts is not None: qemuargs.extend(args.qemu_opts) if args.show_command: print(' '.join(shlex.quote(a) for a in qemuargs)) # Go! if not args.dry_run: os.execv(qemu.qemubin, qemuargs)
def call(args, input=None, universal_newlines=False): print('+ ' + ' '.join(shlex.quote(a) for a in args)) subprocess.run(args, input=input, universal_newlines=universal_newlines, check=True)
def run_on_master( cluster, system_paasta_config, cmd_parts, timeout=None, err_code=-1, graceful_exit=False, stdin=None, ): """Find connectable master for :cluster: and :system_paasta_config: args and invoke command from :cmd_parts:, wrapping it in ssh call. :returns (exit code, output) :param cluster: cluster to find master in :param system_paasta_config: system configuration to lookup master data :param cmd_parts: passed into paasta_tools.utils._run as command along with ssh bits :param timeout: see paasta_tools.utils._run documentation (default: None) :param err_code: code to return along with error message when something goes wrong (default: -1) :param graceful_exit: wrap command in a bash script that waits for input and kills the original command; trap SIGINT and send newline into stdin """ try: master = connectable_master(cluster, system_paasta_config) except NoMasterError as e: return (err_code, str(e)) if graceful_exit: # signals don't travel over ssh, kill process when anything lands on stdin instead cmd_parts.append( # send process to background and capture it's pid '& p=$!; ' + # wait for stdin with timeout in a loop, exit when original process finished 'while ! read -t1; do ! kill -0 $p 2>/dev/null && kill $$; done; ' + # kill original process if loop finished (something on stdin) 'kill $p; wait', ) stdin = subprocess.PIPE stdin_interrupt = True popen_kwargs = {'preexec_fn': os.setsid} else: stdin_interrupt = False popen_kwargs = {} cmd_parts = [ 'ssh', '-q', '-t', '-t', '-A', master, "sudo /bin/bash -c %s" % quote(' '.join(cmd_parts)) ] log.debug("Running %s" % ' '.join(cmd_parts)) return _run( cmd_parts, timeout=timeout, stream=True, stdin=stdin, stdin_interrupt=stdin_interrupt, popen_kwargs=popen_kwargs, )
def get_or_create_head_node(config, config_file, no_restart, restart_only, yes, override_cluster_name, _provider=None, _runner=subprocess): """Create the cluster head node, which in turn creates the workers.""" provider = (_provider or get_node_provider(config["provider"], config["cluster_name"])) config = copy.deepcopy(config) raw_config_file = config_file # used for printing to the user config_file = os.path.abspath(config_file) try: head_node_tags = { TAG_RAY_NODE_KIND: NODE_KIND_HEAD, } nodes = provider.non_terminated_nodes(head_node_tags) if len(nodes) > 0: head_node = nodes[0] else: head_node = None if not head_node: cli_logger.confirm(yes, "No head node found. " "Launching a new cluster.", _abort=True) cli_logger.old_confirm("This will create a new cluster", yes) elif not no_restart: cli_logger.old_confirm("This will restart cluster services", yes) if head_node: if restart_only: cli_logger.confirm( yes, "Updating cluster configuration and " "restarting the cluster Ray runtime. " "Setup commands will not be run due to `{}`.\n", cf.bold("--restart-only"), _abort=True) elif no_restart: cli_logger.print( "Cluster Ray runtime will not be restarted due " "to `{}`.", cf.bold("--no-restart")) cli_logger.confirm(yes, "Updating cluster configuration and " "running setup commands.", _abort=True) else: cli_logger.print( "Updating cluster configuration and running full setup.") cli_logger.confirm( yes, cf.bold("Cluster Ray runtime will be restarted."), _abort=True) cli_logger.newline() # TODO(ekl) this logic is duplicated in node_launcher.py (keep in sync) head_node_config = copy.deepcopy(config["head_node"]) if "head_node_type" in config: head_node_tags[TAG_RAY_USER_NODE_TYPE] = config["head_node_type"] head_node_config.update(config["available_node_types"][ config["head_node_type"]]["node_config"]) launch_hash = hash_launch_conf(head_node_config, config["auth"]) if head_node is None or provider.node_tags(head_node).get( TAG_RAY_LAUNCH_CONFIG) != launch_hash: with cli_logger.group("Acquiring an up-to-date head node"): if head_node is not None: cli_logger.print( "Currently running head node is out-of-date with " "cluster configuration") cli_logger.print( "hash is {}, expected {}", cf.bold( provider.node_tags(head_node).get( TAG_RAY_LAUNCH_CONFIG)), cf.bold(launch_hash)) cli_logger.confirm(yes, "Relaunching it.", _abort=True) cli_logger.old_confirm( "Head node config out-of-date. It will be terminated", yes) cli_logger.old_info( logger, "get_or_create_head_node: " "Shutting down outdated head node {}", head_node) provider.terminate_node(head_node) cli_logger.print("Terminated head node {}", head_node) cli_logger.old_info( logger, "get_or_create_head_node: Launching new head node...") head_node_tags[TAG_RAY_LAUNCH_CONFIG] = launch_hash head_node_tags[TAG_RAY_NODE_NAME] = "ray-{}-head".format( config["cluster_name"]) provider.create_node(head_node_config, head_node_tags, 1) cli_logger.print("Launched a new head node") start = time.time() head_node = None with cli_logger.timed("Fetching the new head node"): while True: if time.time() - start > 50: cli_logger.abort( "Head node fetch timed out.") # todo: msg raise RuntimeError("Failed to create head node.") nodes = provider.non_terminated_nodes(head_node_tags) if len(nodes) == 1: head_node = nodes[0] break time.sleep(1) cli_logger.newline() with cli_logger.group( "Setting up head node", _numbered=("<>", 1, 1), # cf.bold(provider.node_tags(head_node)[TAG_RAY_NODE_NAME]), _tags=dict()): # add id, ARN to tags? # TODO(ekl) right now we always update the head node even if the # hash matches. # We could prompt the user for what they want to do here. # No need to pass in cluster_sync_files because we use this # hash to set up the head node (runtime_hash, file_mounts_contents_hash) = hash_runtime_conf( config["file_mounts"], None, config) cli_logger.old_info( logger, "get_or_create_head_node: Updating files on head node...") # Rewrite the auth config so that the head # node can update the workers remote_config = copy.deepcopy(config) # drop proxy options if they exist, otherwise # head node won't be able to connect to workers remote_config["auth"].pop("ssh_proxy_command", None) if "ssh_private_key" in config["auth"]: remote_key_path = "~/ray_bootstrap_key.pem" remote_config["auth"]["ssh_private_key"] = remote_key_path # Adjust for new file locations new_mounts = {} for remote_path in config["file_mounts"]: new_mounts[remote_path] = remote_path remote_config["file_mounts"] = new_mounts remote_config["no_restart"] = no_restart # Now inject the rewritten config and SSH key into the head node remote_config_file = tempfile.NamedTemporaryFile( "w", prefix="ray-bootstrap-") remote_config_file.write(json.dumps(remote_config)) remote_config_file.flush() config["file_mounts"].update( {"~/ray_bootstrap_config.yaml": remote_config_file.name}) if "ssh_private_key" in config["auth"]: config["file_mounts"].update({ remote_key_path: config["auth"]["ssh_private_key"], }) cli_logger.print("Prepared bootstrap config") if restart_only: init_commands = [] ray_start_commands = config["head_start_ray_commands"] elif no_restart: init_commands = config["head_setup_commands"] ray_start_commands = [] else: init_commands = config["head_setup_commands"] ray_start_commands = config["head_start_ray_commands"] if not no_restart: warn_about_bad_start_command(ray_start_commands) updater = NodeUpdaterThread( node_id=head_node, provider_config=config["provider"], provider=provider, auth_config=config["auth"], cluster_name=config["cluster_name"], file_mounts=config["file_mounts"], initialization_commands=config["initialization_commands"], setup_commands=init_commands, ray_start_commands=ray_start_commands, process_runner=_runner, runtime_hash=runtime_hash, file_mounts_contents_hash=file_mounts_contents_hash, docker_config=config.get("docker")) updater.start() updater.join() # Refresh the node cache so we see the external ip if available provider.non_terminated_nodes(head_node_tags) if config.get("provider", {}).get("use_internal_ips", False) is True: head_node_ip = provider.internal_ip(head_node) else: head_node_ip = provider.external_ip(head_node) if updater.exitcode != 0: # todo: this does not follow the mockup and is not good enough cli_logger.abort("Failed to setup head node.") cli_logger.old_error( logger, "get_or_create_head_node: " "Updating {} failed", head_node_ip) sys.exit(1) cli_logger.old_info( logger, "get_or_create_head_node: " "Head node up-to-date, IP address is: {}", head_node_ip) monitor_str = "tail -n 100 -f /tmp/ray/session_*/logs/monitor*" if override_cluster_name: modifiers = " --cluster-name={}".format( quote(override_cluster_name)) else: modifiers = "" if cli_logger.old_style: print("To monitor autoscaling activity, you can run:\n\n" " ray exec {} {}{}\n".format(config_file, quote(monitor_str), modifiers)) print("To open a console on the cluster:\n\n" " ray attach {}{}\n".format(config_file, modifiers)) print("To get a remote shell to the cluster manually, run:\n\n" " {}\n".format( updater.cmd_runner.remote_shell_command_str())) cli_logger.newline() with cli_logger.group("Useful commands"): cli_logger.print("Monitor autoscaling with") cli_logger.print(cf.bold(" ray exec {}{} {}"), raw_config_file, modifiers, quote(monitor_str)) cli_logger.print("Connect to a terminal on the cluster head") cli_logger.print(cf.bold(" ray attach {}{}"), raw_config_file, modifiers) finally: provider.cleanup()
async def wrap_notarization_with_sudo(config, key_config, all_paths, path_attr="zip_path"): """Wrap the notarization requests with sudo. Apple creates a lockfile per user for notarization. To notarize concurrently, we use sudo against a set of accounts (``config['local_notarization_accounts']``). Args: config (dict): the running config key_config (dict): the config for this signing key all_paths (list): the list of ``App`` objects path_attr (str, optional): the attribute that the zip path is under. Defaults to ``zip_path`` Raises: IScriptError: on failure Returns: dict: uuid to log path """ futures = [] accounts = config["local_notarization_accounts"] counter = 0 uuids = {} for app in all_paths: app.check_required_attrs([path_attr, "parent_dir"]) while counter < len(all_paths): futures = [] for account in accounts: app = all_paths[counter] app.notarization_log_path = f"{app.parent_dir}-notarization.log" bundle_id = get_bundle_id(key_config["base_bundle_id"], counter=str(counter)) zip_path = getattr(app, path_attr) # XXX potentially run the notarization + get_uuid_from_log in a # helper function per app, so we can retry them individually on # error. That would also let us record the path per UUID, # should we need that complexity later. # Not doing that now, so notarization errors are more visible. base_cmdln = " ".join([ "xcrun", "altool", "--notarize-app", "-f", zip_path, "--primary-bundle-id", '"{}"'.format(bundle_id), "-u", key_config["apple_notarization_account"], "--asc-provider", key_config["apple_asc_provider"], "--password", ]) cmd = [ "sudo", "su", account, "-c", base_cmdln + " {}".format( shlex.quote(key_config["apple_notarization_password"])) ] log_cmd = ["sudo", "su", account, "-c", base_cmdln + " ********"] futures.append( asyncio.ensure_future( retry_async( run_command, args=[cmd], kwargs={ "log_path": app.notarization_log_path, "log_cmd": log_cmd, "exception": IScriptError }, retry_exceptions=(IScriptError, ), attempts=10, ))) counter += 1 if counter >= len(all_paths): break await raise_future_exceptions(futures) for app in all_paths: uuids[get_uuid_from_log( app.notarization_log_path)] = app.notarization_log_path return uuids
def _safe_load_file(self, file_path): '''Takes a file path and loads it in a safe way evading posible injections''' return os.path.expanduser(shlex.quote(file_path))
def insert_sosflow(self, sosd_path, sos_analysis_path, run_path, ppn): """Insert a new component at start of list to launch sosflow daemon. Should be called only once.""" assert self.run_components[0].name != 'sosflow' # sos_args must be calculated before adding sosflow as a RunComponent, # as get_total_nodes() needs to return only application nodes and not # any nodes required by sosflow. num_listeners = self._get_total_sosflow_component_nodes() # return if no components are setup to use sosflow. That is, # sosflow=False in `codes` for all components if num_listeners == 0: return # From Kevin Huck, U of Oregon max_listeners_per_aggregator = 64 num_aggregators = math.ceil(num_listeners / max_listeners_per_aggregator) # Add sos aggregators to be run # common aggregator parameters sos_args = [ '-l', str(num_listeners), '-a', str(num_aggregators), '-w', shlex.quote(run_path) ] sos_cmd = ' '.join([sosd_path] + sos_args) sos_fork_cmd = sos_cmd + ' -k @LISTENER_RANK@ -r listener' # now add each aggregator, starting with the analysis aggregator listener_node_offset = 0 for i in range(num_aggregators): sosd_args = sos_args + [ '-k', str(i), '-r', 'aggregator', ] rc_name = 'sosflow_aggregator_' + str(i) rc_exe_path = sosd_path # If sos analysis is enabled, the first aggregator should be # the sos analysis script instead of a plain sosd aggregator. if i == 0 and self.sosflow_analysis: rc_name = "sosflow_analysis" rc_exe_path = sos_analysis_path sosd_args = [sosd_path] + sosd_args self.node_layout.add_node({rc_name: 1}) rc = RunComponent(rc_name, rc_exe_path, sosd_args, nprocs=1, sleep_after=5, working_dir=self.run_path) rc.env['sos_cmd'] = sos_cmd rc.env['SOS_FORK_COMMAND'] = sos_fork_cmd rc.env['SOS_CMD_PORT'] = '22500' rc.env['SOS_EVPATH_MEETUP'] = run_path rc.env['TAU_SOS'] = '1' self.run_components.insert(i, rc) listener_node_offset += 1 # add env vars to each run, including sosflow daemon # NOTE: not sure how many if any are required for sosd, but # should not hurt to have them, and simplifies the offset # calculation for rc in self.run_components: # ignore component if not setup to use sosflow if not rc.linked_with_sosflow: continue # TODO: is this actually used directly? rc.env['sos_cmd'] = sos_cmd rc.env['SOS_FORK_COMMAND'] = sos_fork_cmd code_node = self.node_layout.get_node_containing_code(rc.name) # TODO: we don't yet know how SOSFLOW will support apps that # do node shairng, so for now require that there is no # sharing. assert len(code_node) == 1 code_procs_per_node = code_node[rc.name] code_nodes = int(math.ceil(rc.nprocs / code_procs_per_node)) # Set the TCP port that the listener will listen to, # and the port that clients will attempt to connect to. rc.env['SOS_CMD_PORT'] = '22500' # Set the directory where the SOS listeners and aggregators # will use to establish EVPath links to each other rc.env['SOS_EVPATH_MEETUP'] = run_path # Tell TAU that it should connect to SOS # and send TAU data to SOS when adios_close(), # adios_advance_step() calls are made, # and when the application terminates. rc.env['TAU_SOS'] = '1' # Tell SOS how many application ranks per node there are # How do you get this information? # TODO: This will change when we have the ability to set a # different number of procs per node rc.env['SOS_APP_RANKS_PER_NODE'] = str(code_procs_per_node) # Tell SOS what 'rank' it's listeners should start with # the aggregator was 'rank' 0, so this node's listener will be 1 # This offset is the node count where this fob component starts rc.env['SOS_LISTENER_RANK_OFFSET'] = str(listener_node_offset) # TODO: this assumes node exclusive. To support node sharing # with custom layouts, will need to know layout here and # calculate actual node usage. This potentially duplicates # functionality needed in workflow, should eventual converge # so they are using the same model. listener_node_offset += code_nodes
def archive_file_scan(path, extensions, launch_prefix=""): ext = os.path.splitext(path)[1][1:].lower() # print(path) # print(ext) try: if ext == 'rar': matches = 0 count = 0 line = launch_prefix + "unrar lb -p- " + shlex.quote( path) + " " + shlex.quote(os.path.dirname(path)) + os.sep result = subprocess.run(shlex.split(line), stdout=subprocess.PIPE) file_list = result.stdout.decode("utf-8", 'ignore').split("\n") # print(file_list) for fi in file_list: for ty in extensions: if fi[len(ty) * -1:].lower() == ty: matches += 1 break elif is_ignorable_file(fi): count -= 1 break elif is_music_related(fi): matches += 5 count += 1 if count > 200: #print("RAR archive has many files") #print(" --- " + path) return 0 if matches == 0: #print("RAR archive does not appear to contain audio files") #print(" --- " + path) return 0 if count == 0: #print("Archive has no files") #print(" --- " + path) return 0 elif ext == '7z': matches = 0 count = 0 line = launch_prefix + "7z l " + shlex.quote( path) # + " " + shlex.quote(os.path.dirname(path)) + os.sep result = subprocess.run(shlex.split(line), stdout=subprocess.PIPE) file_list = result.stdout.decode("utf-8", 'ignore').split("\n") # print(file_list) for fi in file_list: if '....A' not in fi: continue for ty in extensions: if fi[len(ty) * -1:].lower() == ty: matches += 1 break elif is_ignorable_file(fi): count -= 1 break elif is_music_related(fi): matches += 5 count += 1 if count > 200: #print("7z archive has many files") #print(" --- " + path) return 0 if matches == 0: #print("7z archive does not appear to contain audio files") #print(" --- " + path) return 0 if count == 0: #print("7z archive has no files") #print(" --- " + path) return 0 elif ext == "zip": zip_ref = zipfile.ZipFile(path, 'r') matches = 0 count = 0 #print(zip_ref.namelist()) for fi in zip_ref.namelist(): for ty in extensions: if fi[len(ty) * -1:].lower() == ty: matches += 1 break elif is_ignorable_file(fi): count -= 1 break elif is_music_related(fi): matches += 5 count += 1 if count == 0: #print("Archive has no files") #print(" --- " + path) return 0 if count > 300: #print("Zip archive has many files") #print(" --- " + path) return 0 if matches == 0: #print("Zip archive does not appear to contain audio files") #print(" --- " + path) return 0 else: return 0 except: print("Archive test error") return 0 if count == 0: return 0 ratio = matches / count if count < 5 and matches > 0: ratio = 100 return ratio
def main(): arg_parser = argparse.ArgumentParser() arg_parser.add_argument( '--llvm_version', help='Version number MAJOR.MINOR of the LLVM toolchain', required=True) arg_parser.add_argument( '--disassembler', help='Path to disassembler, or just "binja", if installed.', required=True) arg_parser.add_argument( '--workspace_dir', help='Directory in which intermediate and final files are placed', required=True) arg_parser.add_argument('--binary', help='Path to the binary to be lifted', required=True) arg_parser.add_argument('--clang', help='Path to clang, if not using remill-clang', required=False, default="") arg_parser.add_argument('--dry_run', help='Should the actual commands be executed?', default=False, required=False) arg_parser.add_argument('--legacy_mode', help='Are we producing legacy mode bitcode?', default=False, required=False, action='store_true') arg_parser.add_argument( '--extra_args', help= 'A space-delimited list of any extra arguments to pass to the lifter.', default="", required=False) args, command_args = arg_parser.parse_known_args() # Set up the workspace. args.workspace_dir = os.path.realpath(args.workspace_dir) bin_dir = os.path.join(args.workspace_dir, 'bin') lib_dir = os.path.join(args.workspace_dir, 'lib') obj_dir = os.path.join(args.workspace_dir, 'obj') lifted_obj_dir = os.path.join(args.workspace_dir, 'lifted_obj') cfg_dir = os.path.join(args.workspace_dir, 'cfg') bc_dir = os.path.join(args.workspace_dir, 'bc') log_dir = os.path.join(args.workspace_dir, 'log') print("mkdir -p {}".format(args.workspace_dir)) print("mkdir -p {}".format(bin_dir)) print("mkdir -p {}".format(lifted_obj_dir)) print("mkdir -p {}".format(lib_dir)) print("mkdir -p {}".format(obj_dir)) print("mkdir -p {}".format(cfg_dir)) print("mkdir -p {}".format(bc_dir)) print("mkdir -p {}".format(log_dir)) make_directory(args.workspace_dir) make_directory(bin_dir) make_directory(bin_dir) make_directory(lifted_obj_dir) make_directory(lib_dir) make_directory(obj_dir) make_directory(cfg_dir) make_directory(bc_dir) make_directory(log_dir) path = os.path.realpath(args.binary) path_hash = hash_file(path) binary = os.path.join(obj_dir, path_hash) lifted_binary = os.path.join(lifted_obj_dir, path_hash) cfg = os.path.join(cfg_dir, "{}.cfg".format(path_hash)) bitcode = os.path.join(bc_dir, "{}.bc".format(path_hash)) log = os.path.join(log_dir, "{}.log".format(path_hash)) # Copy the binary into the workspace's object directory. print("cp {} {}".format(path, binary)) print("chmod a+x {}".format(binary)) if not os.path.isfile(binary): shutil.copyfile(args.binary, binary) make_executable(binary) # Copy the shared libraries into the workspace's object directory, and then # add symbolic links from the workspace's library directory into the object # directory. libs = [] for name, path in binary_libraries(binary): path_hash = hash_file(path) library = os.path.join(obj_dir, "{}.so".format(path_hash)) if not os.path.isfile(library): shutil.copyfile(path, library) make_executable(library) sym_name = os.path.join(lib_dir, name) if os.path.exists(sym_name): os.remove(sym_name) try: os.symlink(library, sym_name) except: pass print("cp {} {}".format(path, library)) print("chmod a+x {}".format(library)) print("rm {}".format(sym_name)) print("ln {} {}".format(library, sym_name)) libs.append(sym_name) os_name = 'linux' binary_name = os.path.basename(args.binary) address_size, arch, is_pie = binary_info(binary) # Disassembler Seetings da = '' if ('binja' == args.disassembler ) or 'binaryninja' == 'binja' == args.disassembler: da = 'binja' else: ida_version = { "x86_avx": "idal", "amd64_avx": "idal64", "aarch64": "idal64" }[arch] da = quote(os.path.join(args.disassembler, ida_version)) # Disassemble the binary. disass_args = [ 'mcsema-disass', '--arch', arch, '--os', os_name, '--binary', quote(binary), '--output', quote(cfg), '--entrypoint', 'main', '--disassembler', da, '--log_file', quote(log) ] if is_pie: disass_args.append("--pie-mode") disass_args.extend(command_args) print(" ".join(disass_args)) ret = subprocess.call(disass_args) if ret: return ret # Lift the binary. mcsema_lift_args = [ 'mcsema-lift-{}'.format(args.llvm_version), '--arch', arch, '--os', os_name, '--cfg', cfg, '--output', bitcode ] if args.extra_args != "": for arg in args.extra_args.split(' '): mcsema_lift_args.append(arg) if args.legacy_mode: mcsema_lift_args.append('--legacy_mode') print(" ".join(mcsema_lift_args)) ret = subprocess.call(mcsema_lift_args) if ret: return ret # Not compiling a binary. if args.legacy_mode: return 0 # Build up the command-line invocation to clang. clang_args = [] if (args.clang != ""): clang_args = [os.path.join(args.clang)] else: clang_args = [ os.path.join('remill-clang-{}'.format(args.llvm_version)) ] clang_args += [ '-rdynamic', is_pie and '-fPIC' or '', is_pie and '-pie' or '', '-o', lifted_binary, bitcode, '/usr/local/lib/libmcsema_rt{}-{}.a'.format(address_size, args.llvm_version), '-lm' ] for lib in libs: clang_args.append(lib) # Compile back to an executable. print(" ".join(clang_args)) ret = subprocess.call(clang_args) if ret: return ret # Create two scripts to run the original and native. run_native = os.path.join(bin_dir, binary_name) with open(run_native, "w") as f: f.write("""#!/usr/bin/env bash LD_LIBRARY_PATH={} {} "$@" """.format(lib_dir, binary)) run_lifted = os.path.join(bin_dir, "{}.lifted".format(binary_name)) with open(run_lifted, "w") as f: f.write("""#!/usr/bin/env bash LD_LIBRARY_PATH={} {} "$@" """.format(lib_dir, lifted_binary)) make_executable(run_native) make_executable(run_lifted) return 0
def _get_uncommented_code(path: pathlib.Path, *, iquotes_options: str, compiler: str) -> bytes: command = """{} {} -fpreprocessed -dD -E {}""".format( compiler, iquotes_options, shlex.quote(str(path))) return subprocess.check_output(command, shell=True)