def set_ds_test_env(test_name, hosts, test_env): TEMP_FILE = test_env["TEST_DIR"] + test_name hosts.all.shell(cmd="mkdir -p {0}".format(test_env["TEST_DIR"])) results = hosts.all.shell(cmd='hlq') for result in results.contacted.values(): hlq = result.get("stdout") if(len(hlq) > 8): hlq = hlq[:8] test_env["DS_NAME"] = hlq + "." + test_name.upper() + "." + test_env["DS_TYPE"] try: hosts.all.zos_data_set(name=test_env["DS_NAME"], type=test_env["DS_TYPE"]) hosts.all.shell(cmd="echo \"{0}\" > {1}".format(test_env["TEST_CONT"], TEMP_FILE)) if test_env["DS_TYPE"] in ["PDS", "PDSE"]: test_env["DS_NAME"] = test_env["DS_NAME"] + "(MEM)" hosts.all.zos_data_set(name=test_env["DS_NAME"], state="present", type="member") cmdStr = "cp -CM {0} \"//'{1}'\"".format(quote(TEMP_FILE), test_env["DS_NAME"]) else: cmdStr = "cp {0} \"//'{1}'\" ".format(quote(TEMP_FILE), test_env["DS_NAME"]) if test_env["ENCODING"] != "IBM-1047": hosts.all.zos_encode(src=TEMP_FILE, dest=test_env["DS_NAME"], from_encoding="IBM-1047", to_encoding=test_env["ENCODING"]) # cmdStr = "/u/behnam/tools/cp_withencoding/bin/cp2 {0} {1} {2}".format(test_env["ENCODING"], quote(TEMP_FILE), quote(test_env["DS_NAME"])) # hosts.all.shell(cmd=cmdStr) else: hosts.all.shell(cmd=cmdStr) hosts.all.shell(cmd="rm -rf " + test_env["TEST_DIR"]) cmdStr = "cat \"//'{0}'\" | wc -l ".format(test_env["DS_NAME"]) results = hosts.all.shell(cmd=cmdStr) pprint(vars(results)) for result in results.contacted.values(): assert int(result.get("stdout")) != 0 except Exception: clean_ds_test_env(test_env["DS_NAME"], hosts) assert 1 == 0, "Failed to set the test env"
def print_discovered_devices(self, devices): self._stream.write(_SHELL_VAR_PREFIX + 'DEVICES=%u\n' % len(devices)) for device_n, device in zip(itertools.count(), devices): self._stream.write(_SHELL_VAR_PREFIX + 'DEVICE_%u_NAME=%s\n' % (device_n, shellescape.quote(device['name']))) self._stream.write( _SHELL_VAR_PREFIX + 'DEVICE_%u_ADDRESS=%s\n' % (device_n, shellescape.quote(device['address']))) self._stream.flush()
def encrypt_file(self, inpath, force_nocompress=False, force_compress=False, armored=False, checksum=False): """public method for single file encryption with optional compression, ASCII armored formatting, and file hash digest generation""" if armored: if force_compress: command_stub = self.command_maxcompress_armored elif force_nocompress: command_stub = self.command_nocompress_armored else: if self._is_compress_filetype(inpath): command_stub = self.command_default_armored else: command_stub = self.command_nocompress_armored else: if force_compress: command_stub = self.command_maxcompress elif force_nocompress: command_stub = self.command_nocompress else: if self._is_compress_filetype(inpath): command_stub = self.command_default else: command_stub = self.command_nocompress encrypted_outpath = self._create_outfilepath(inpath) system_command = command_stub + encrypted_outpath + " --passphrase " + quote( self.passphrase) + " --symmetric " + quote(inpath) try: response = muterun(system_command) # check returned status code if response.exitcode == 0: stdout(encrypted_outpath + " was generated from " + inpath) if checksum: # add a SHA256 hash digest of the encrypted file - requested by user --hash flag in command from crypto.library import hash encrypted_file_hash = hash.generate_hash(encrypted_outpath) if len(encrypted_file_hash) == 64: stdout("SHA256 hash digest for " + encrypted_outpath + " :") stdout(encrypted_file_hash) else: stdout( "Unable to generate a SHA256 hash digest for the file " + encrypted_outpath) else: stderr(response.stderr, 0) stderr("Encryption failed") sys.exit(1) except Exception as e: stderr( "There was a problem with the execution of gpg. Encryption failed. Error: [" + str(e) + "]") sys.exit(1)
def print_discovered_devices(self, devices): self._stream.write(_SHELL_VAR_PREFIX + 'DEVICES=%u\n' % len(devices)) for device_n, device in zip(itertools.count(), devices): self._stream.write( _SHELL_VAR_PREFIX + 'DEVICE_%u_NAME=%s\n' % ( device_n, shellescape.quote(device['name']))) self._stream.write( _SHELL_VAR_PREFIX + 'DEVICE_%u_ADDRESS=%s\n' % ( device_n, shellescape.quote(device['address']))) self._stream.flush()
def print_temperatures(self, value): for var_name in ('current_temp', 'manual_temp', 'target_temp_l', 'target_temp_h', 'offset_temp'): val_str = '%f' % value[var_name] self._stream.write(_SHELL_VAR_PREFIX + '%s=%s\n' % (var_name.upper(), shellescape.quote(val_str))) for var_name in ('window_open_detection', 'window_open_minutes'): val_str = '%u' % value[var_name] self._stream.write(_SHELL_VAR_PREFIX + '%s=%s\n' % (var_name.upper(), shellescape.quote(val_str))) self._stream.flush()
def envDump(shell, log): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc """Dump environment to a .fuzzmanagerconf file.""" # Platform and OS detection for the spec, part of which is in: # https://wiki.mozilla.org/Security/CrashSignatures fmconf_platform = "x86" if shell.build_opts.enable32 else "x86-64" fmconf_os = None if platform.system() == "Linux": fmconf_os = "linux" elif platform.system() == "Darwin": fmconf_os = "macosx" elif platform.system() == "Windows": fmconf_os = "windows" with io.open(str(log), "a", encoding="utf-8", errors="replace") as f: f.write("# Information about shell:\n# \n") f.write("# Create another shell in shell-cache like this one:\n") f.write('# %s -u -m %s -b "%s" -r %s\n# \n' % ( # Perhaps this can go into a separate function. See ensureBuild in bot.py "python" + re.search("python.*[2-3]", os.__file__).group(0).replace( "/", "").split("python")[-1], "funfuzz.js.compile_shell", shell.build_opts.build_options_str, shell.get_hg_hash())) f.write("# Full environment is:\n") f.write("# %s\n# \n" % str(shell.get_env_full())) f.write( "# Full configuration command with needed environment variables is:\n" ) f.write("# %s %s\n# \n" % (" ".join(quote(str(x)) for x in shell.get_env_added()), " ".join( quote(str(x)) for x in shell.get_cfg_cmd_excl_env()))) # .fuzzmanagerconf details f.write("\n") f.write("[Main]\n") f.write("platform = %s\n" % fmconf_platform) f.write("product = %s\n" % shell.get_repo_name()) f.write("product_version = %s\n" % shell.get_hg_hash()) f.write("os = %s\n" % fmconf_os) f.write("\n") f.write("[Metadata]\n") f.write("buildFlags = %s\n" % shell.build_opts.build_options_str) f.write("majorVersion = %s\n" % shell.get_version().split(".")[0]) f.write("pathPrefix = %s/\n" % shell.get_repo_dir()) f.write("version = %s\n" % shell.get_version())
def print_holidays(self, value): for holiday_n, holiday in zip(itertools.count(), value): for var_name in 'start', 'end': var_val = ('' if holiday[var_name] is None else holiday[var_name].isoformat()) self._stream.write( _SHELL_VAR_PREFIX + 'HOLIDAY_%u_%s=%s\n' % (holiday_n, var_name.upper(), shellescape.quote(var_val))) self._stream.write( _SHELL_VAR_PREFIX + 'HOLIDAY_%u_TEMP=%s\n' % (holiday_n, shellescape.quote('' if holiday['temp'] is None else '%f' % holiday['temp']))) self._stream.flush()
def print_holidays(self, value): for holiday_n, holiday in zip(itertools.count(), value): for var_name in 'start', 'end': var_val = ('' if holiday[var_name] is None else holiday[var_name].isoformat()) self._stream.write( _SHELL_VAR_PREFIX + 'HOLIDAY_%u_%s=%s\n' % ( holiday_n, var_name.upper(), shellescape.quote(var_val))) self._stream.write( _SHELL_VAR_PREFIX + 'HOLIDAY_%u_TEMP=%s\n' % ( holiday_n, shellescape.quote( '' if holiday['temp'] is None else '%f' % holiday['temp']))) self._stream.flush()
def _is_compress_filetype(self, inpath): """private method that performs magic number and size check on file to determine whether to compress the file""" # check for common file type suffixes in order to avoid the need for file reads to check magic number for binary vs. text file if self._is_common_binary(inpath): return False elif self._is_common_text(inpath): return True else: # files > 10kB get checked for compression (arbitrary decision to skip compression on small files) the_file_size = file_size(inpath) if the_file_size > 10240: if the_file_size > 512000: # seems to be a break point at ~ 500kb where file compression offset by additional file read, so limit tests to files > 500kB try: system_command = "file --mime-type -b " + quote(inpath) response = muterun(system_command) if response.stdout[0:5] == "text/": # check for a text file mime type return True # appropriate size, appropriate file mime type else: return False # appropriate size, inappropriate file mime type except Exception: return False else: return True # if file size is < 500kB, skip the additional file read and just go with compression else: return False # below minimum size to consider compression, do not compress
def normalize_shell_variable(myvar): newNormalizedVar = quote(myvar) # myvar.replace('\'', '\\\'') # newNormalizedVar = normalizedVar # newNormalizedVar = '\'' + normalizedVar + '\'' return newNormalizedVar
def run_lithium(lithArgs, logPrefix, targetTime): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc # pylint: disable=missing-return-type-doc,missing-type-doc """Run Lithium as a subprocess: reduce to the smallest file that has at least the same unhappiness level. Returns a tuple of (lithlogfn, LITH_*, details). """ deletableLithTemp = None # pylint: disable=invalid-name if targetTime: # FIXME: this could be based on whether bot has a remoteHost # pylint: disable=fixme # loop is being used by bot deletableLithTemp = tempfile.mkdtemp(prefix="fuzzbot-lithium") # pylint: disable=invalid-name lithArgs = ["--maxruntime=" + str(targetTime), "--tempdir=" + deletableLithTemp] + lithArgs else: # loop is being run standalone lithtmp = logPrefix.parent / (logPrefix.stem + "-lith-tmp") Path.mkdir(lithtmp) lithArgs = ["--tempdir=" + str(lithtmp)] + lithArgs lithlogfn = (logPrefix.parent / (logPrefix.stem + "-lith-out")).with_suffix(".txt") print("Preparing to run Lithium, log file %s" % lithlogfn) print(" ".join(quote(str(x)) for x in runlithiumpy + lithArgs)) with io.open(str(lithlogfn), "w", encoding="utf-8", errors="replace") as f: subprocess.run(runlithiumpy + lithArgs, stderr=subprocess.STDOUT, stdout=f) print("Done running Lithium") if deletableLithTemp: shutil.rmtree(deletableLithTemp) r = readLithiumResult(lithlogfn) # pylint: disable=invalid-name subprocess.run(["gzip", "-f", str(lithlogfn)], check=True) return r
def reactionStop(request): ctx = {} if request.POST: # 拿到計算所需的參數 stop = request.POST['stop'] tmp = os.popen('docker stop ' + quote(stop)).readlines() tmp_for_all = os.popen('docker ps -a').readlines() try: if stop == tmp[0].strip(): # 成功停止 ctx['statusStop'] = 'Success' ctx['id'] = '' ctx['name'] = '' ctx['statusCMD'] = '' ctx['output'] = '' ctx['allContainer'] = ''.join(tmp_for_all) else: ctx['statusStop'] = 'Fail' ctx['status'] = '' ctx['output'] = 'No output' ctx['allContainer'] = '\n' + ''.join(tmp_for_all) except: ctx['statusStop'] = 'Fail' ctx['status'] = '' ctx['output'] = 'No output' ctx['allContainer'] = '\n' + ''.join(tmp_for_all) return render(request, "reaction.html", ctx)
def react(request): ctx = {} if request.POST: # 拿到計算所需的參數 command = request.POST['command'] tmp_for_all = os.popen('docker ps -a').readlines() # name = request.POST['name'] # 抓最新的執行 name = ((os.popen('docker ps -l').readlines()[1].strip()).split())[-1] idNow = ((os.popen('docker ps -l').readlines()[1].strip()).split())[0] # os.system('docker start -i '+ tmp_ctx['name']) cmd = 'docker exec -i ' + name + ' ' + quote(command) try: tmp = os.popen(cmd).readlines() ctx['id'] = idNow ctx['name'] = name ctx['statusCMD'] = 'Success' ctx['output'] = ''.join(tmp) ctx['allContainer'] = ''.join(tmp_for_all) except: ctx['status'] = 'Fail' ctx['output'] = 'No output' ctx['allContainer'] = '\n' + ''.join(tmp_for_all) print(ctx) else: print('No Request') return render(request, "reaction.html", ctx)
def main(): title = argv[2] if len(argv) > 2 else os.path.splitext( os.path.basename(argv[1]))[0] tmpdir = os.path.dirname(os.path.abspath(__file__)) + '/tmp' command = command_generator(quote(os.path.abspath(argv[1]))) if os.path.isdir(tmpdir): shutil.rmtree(tmpdir) os.mkdir(tmpdir) os.chdir(tmpdir) print('ffmpeg %s' % command) os.system('ffmpeg %s' % command) i, lines = 0, open('out.m3u8', 'r').read() executor = ThreadPoolExecutor(max_workers=10) futures = { executor.submit(upload_yuque, chunk): chunk for chunk in glob.glob('*.ts') } for future in as_completed(futures): lines = lines.replace(futures[future], future.result()) i += 1 print('[%s/%s] Uploaded %s to %s' % (i, len(futures), futures[future], future.result())) print('This video has been published to: %s' % publish(lines, title))
def store_jcl(localhost, jcl_filename): with open("tests/functional/files/{0}".format(jcl_filename)) as f: jcl_file_content = f.read() localhost.file(path=TEMP_PATH, state="directory") localhost.shell(cmd="echo {0} > {1}/SAMPLE".format( quote(JOB_CARD_CONTENTS + jcl_file_content), TEMP_PATH))
def lith_reduce(strategy): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc # pylint: disable=missing-return-type-doc,missing-type-doc """Lithium reduction commands accepting various strategies. Args: strategy (str): Intended strategy to use Returns: (tuple): The finished Lithium run result and details """ reductionCount[0] += 1 # Remove empty elements full_lith_args = [x for x in (strategy + lithArgs) if x] print(" ".join( quote(str(x)) for x in [sys.executable, "-u", "-m", "lithium"] + full_lith_args)) desc = "-chars" if strategy == "--char" else "-lines" (lith_result, lith_details) = run_lithium( # pylint: disable=invalid-name full_lith_args, (logPrefix.parent / ("%s-%s%s" % (logPrefix.stem, reductionCount[0], desc))), targetTime) if lith_result == LITH_FINISHED: shutil.copy2(str(infilename), str(backup_file)) return lith_result, lith_details
def process(self): progress = Progress('hashing') __LOG__.debug( 'hashing %d files (%s)', len(self._queue), sizeof_fmt(self.bytes_to_hash) ) progress.start(self, maxval=self.bytes_to_hash) for (file_path, file_size, dest) in self._queue: __LOG__.debug( 'hashing %s (%s)...', quote(file_path), sizeof_fmt(file_size) ) try: hash_values = hashfile(file_path) except Exception as err: __LOG__.exception('hashing of %s failed: ', file_path) self.errors.append(err) else: dest.update(hash_values) self.processed.append(dest) self.bytes_processed = self.bytes_processed + file_size progress.update(self, val=self.bytes_processed) progress.finish(self) __LOG__.debug('%d files hashed', len(self.processed)) return self.processed
def main(): video = quote(os.path.abspath(sys.argv[1])) title = sys.argv[2] if len(sys.argv) > 2 else os.path.splitext( os.path.basename(sys.argv[1]))[0] stime = float(sys.argv[3]) if len(sys.argv) > 3 else segment_time(video) tmpdir = os.path.dirname(os.path.abspath(__file__)) + '/tmp' if os.path.isdir(tmpdir): shutil.rmtree(tmpdir) os.mkdir(tmpdir) os.chdir(tmpdir) # os.system('ffmpeg -i %s -codec copy -map 0 -f segment -segment_list out.m3u8 -segment_list_flags +live -segment_time 5 out%%03d.ts' % video) # os.system('ffmpeg -i %s -vcodec copy -acodec aac -hls_list_size 0 -hls_segment_size 3000000 -f hls out.m3u8' % video) os.system( 'ffmpeg -i %s -vcodec copy -acodec aac -map 0 -f segment -segment_list out.m3u8 -segment_time %d out%%03d.ts' % (video, stime)) i, lines = 0, open('out.m3u8', 'r').read() executor = ThreadPoolExecutor(max_workers=10) futures = { executor.submit(upload_yuque, chunk): chunk for chunk in glob.glob('*.ts') } for future in as_completed(futures): lines = lines.replace(futures[future], future.result()) i += 1 print('[%s/%s] Uploaded %s to %s' % (i, len(futures), futures[future], future.result())) print('This video has been published to: %s' % publish(lines, title))
def _compute_binding( self, processed_token: Any, context: MutableMapping[Text, Any], bindings_map: MutableMapping[int, MutableSequence[Any]], is_shell_command: bool = False, full_js: bool = False, expression_lib: Optional[MutableSequence[Text]] = None ) -> MutableMapping[int, MutableSequence[Any]]: # Obtain token value value = _get_value(processed_token, self.item_separator) # If token value is null or an empty array, skip the command token if value is None: return bindings_map # Otherwise else: # Obtain prefix if present if self.prefix is not None: if isinstance(value, bool): value = [self.prefix if value else ''] elif self.separate: if isinstance(value, MutableSequence): value = [ self.prefix, " ".join([str(v) for v in value]) ] else: value = [self.prefix, value] elif isinstance(value, MutableSequence): value = [self.prefix].extend(value) else: value = [self.prefix + str(value)] # If value is a boolean with no prefix, skip it if isinstance(value, bool): return bindings_map # Ensure value is a list if not isinstance(value, MutableSequence): value = [value] # Process shell escape if is_shell_command and self.shell_quote: value = [shellescape.quote(v) for v in value] # Obtain token position if isinstance(self.position, str) and not self.position.isnumeric(): context['self'] = processed_token position = eval_expression(expression=self.position, context=context, full_js=full_js, expression_lib=expression_lib) try: position = int(position) if position is not None else 0 except ValueError: position = 0 else: position = int(self.position) if position not in bindings_map: bindings_map[position] = [] # Place value in proper position bindings_map[position].append(value) return bindings_map
def validate(ctx, service_name): service = Service(yml=Config(filename=ctx.obj['CONFIG_FILE'], env_file=ctx.obj['ENV_FILE']).get_service(service_name)) host, name, user, passwd, port = _get_db_parameters(service) cmd = "/usr/bin/mysql --host={} --user={} --password={} --port={} --execute='select version(), current_date;'" cmd = cmd.format(host, user, quote(passwd), port) success, output = service.run_remote_script([cmd]) print success print output
def escapePathForShell(path): """ Escapes a filesystem path for use as a command-line argument """ if platform.system() == 'Windows': return '"{}"'.format(path.replace('"', '""')) else: return shellescape.quote(path)
def print_temperatures(self, value): for var_name in ('current_temp', 'manual_temp', 'target_temp_l', 'target_temp_h', 'offset_temp'): val_str = '%f' % value[var_name] self._stream.write( _SHELL_VAR_PREFIX + '%s=%s\n' % ( var_name.upper(), shellescape.quote(val_str))) for var_name in ('window_open_detection', 'window_open_minutes'): val_str = '%u' % value[var_name] self._stream.write( _SHELL_VAR_PREFIX + '%s=%s\n' % ( var_name.upper(), shellescape.quote(val_str))) self._stream.flush()
def append_file(self, file_name, data): """Appends a block of data to a file through the shell. Args: file_name: The name of the file to write to. data: The string of data to write. """ return self.run('echo %s >> %s' % (shellescape.quote(data), file_name))
def command(self): args = self.command_format(self.properties) sanitized_args = [] for arg in args: if type(arg) is str: sanitized_args.append(shellescape.quote(arg)) else: sanitized_args.append(arg) cmd = self.CMD.format(*sanitized_args) return cmd
def add_package_modifier(self, name, modifiers): for modifier in modifiers: slug = re.sub(r'\W+', '_', modifier.split(None, 1)[0]) self.engine.run_cmd( self.build_container, "chroot-{arch}-docker -c \"test -f /etc/portage/package.{name} && " "echo {modifier} >>/etc/portage/package.{name} || " "mkdir -p /etc/portage/package.{name} && " "echo {modifier} >>/etc/portage/package.{name}/{slug}\"".format( arch=self.arch, modifier=quote(modifier), slug=slug, name=name))
def escape_shell(inp): """ Shell-escapes input param :param inp: :return: """ try: return shellescape.quote(inp) except: pass quote(inp)
def print_days(self, value): for day_n, day in zip(itertools.count(), value): for period_n, period in zip(itertools.count(), day): for var_name in 'start', 'end': var_val = ('' if period[var_name] is None else period[var_name].isoformat()) self._stream.write( _SHELL_VAR_PREFIX + 'DAY_%u_PERIOD_%u_%s=%s\n' % ( day_n, period_n, var_name.upper(), shellescape.quote(var_val))) self._stream.flush()
def pinpoint(itest, logPrefix, jsEngine, engineFlags, infilename, # pylint: disable=invalid-name,missing-param-doc bisectRepo, build_options_str, targetTime, suspiciousLevel): # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc,too-many-arguments,too-many-locals """Run Lithium and autobisectjs. itest must be an array of the form [module, ...] where module is an interestingness module. The module's "interesting" function must accept [...] + [jsEngine] + engineFlags + infilename (If it's not prepared to accept engineFlags, engineFlags must be empty.) """ lithArgs = itest + [str(jsEngine)] + engineFlags + [str(infilename)] # pylint: disable=invalid-name (lithResult, lithDetails) = reduction_strat( # pylint: disable=invalid-name logPrefix, infilename, lithArgs, targetTime, suspiciousLevel) print() print("Done running Lithium on the part in between DDBEGIN and DDEND. To reproduce, run:") print(" ".join(quote(str(x)) for x in [sys.executable, "-u", "-m", "lithium", "--strategy=check-only"] + lithArgs)) print() # pylint: disable=literal-comparison if (bisectRepo is not "none" and targetTime >= 3 * 60 * 60 and build_options_str is not None and testJsShellOrXpcshell(jsEngine) != "xpcshell"): autobisectCmd = ( # pylint: disable=invalid-name [sys.executable, "-u", "-m", "funfuzz.autobisectjs"] + ["-b", build_options_str] + ["-p", " ".join(engineFlags + [str(infilename)])] + ["-i"] + [str(x) for x in itest] ) print(" ".join(quote(str(x)) for x in autobisectCmd)) autobisect_log = (logPrefix.parent / (logPrefix.stem + "-autobisect")).with_suffix(".txt") with io.open(str(autobisect_log), "w", encoding="utf-8", errors="replace") as f: subprocess.run(autobisectCmd, stderr=subprocess.STDOUT, stdout=f) print("Done running autobisectjs. Log: %s" % autobisect_log) with io.open(str(autobisect_log), "r", encoding="utf-8", errors="replace") as f: lines = f.readlines() autobisect_log_trunc = file_manipulation.truncateMid(lines, 50, ["..."]) else: autobisect_log_trunc = [] return (lithResult, lithDetails, autobisect_log_trunc)
def menu_activate_cb(self, menu, locations): # self.debug( 'type of location: ' + str( type(locations) ) ) for f in locations: try: path = f.get_location().get_path() escaped_path = shellescape.quote(path) command = "nodejs /composer/node/ocdownload/ocdownload.js " + escaped_path self.debug('menu_activate_cb runs ' + command) exit_code = os.system(command) self.debug('exit code: ' + str(exit_code)) except Exception as e: self.debug('Exception: ' + str(e))
def test_job_submit_USS(ansible_zos_module): hosts = ansible_zos_module hosts.all.file(path=TEMP_PATH, state="directory") hosts.all.shell(cmd="echo {0} > {1}/SAMPLE".format( quote(JCL_FILE_CONTENTS), TEMP_PATH)) results = hosts.all.zos_job_submit(src="{0}/SAMPLE".format(TEMP_PATH), location="USS", wait=True, volume=None) hosts.all.file(path=TEMP_PATH, state="absent") for result in results.contacted.values(): assert result.get("jobs")[0].get("ret_code").get("msg_code") == "0000" assert result.get("jobs")[0].get("ret_code").get("code") == 0 assert result.get("changed") is True
def test_job_submit_USS(ansible_zos_module): hosts = ansible_zos_module hosts.all.file(path=TEMP_PATH, state='directory') hosts.all.shell( cmd='echo {} > {}/SAMPLE'.format(quote(JCL_FILE_CONTENTS), TEMP_PATH)) results = hosts.all.zos_job_submit(src='{}/SAMPLE'.format(TEMP_PATH), location="USS", wait=True, volume=None) hosts.all.file(path=TEMP_PATH, state='absent') for result in results.contacted.values(): assert result.get('jobs')[0].get('ret_code').get('msg_code') == '0000' assert result.get('jobs')[0].get('ret_code').get('code') == 0 assert result.get('changed') == True
def test_zos_job_output_job_exists(ansible_zos_module): hosts = ansible_zos_module hosts.all.file(path=TEMP_PATH, state="directory") hosts.all.shell( cmd="echo {0} > {1}/SAMPLE".format(quote(JCL_FILE_CONTENTS), TEMP_PATH) ) hosts.all.zos_job_submit( src="{0}/SAMPLE".format(TEMP_PATH), location="USS", wait=True, volume=None ) hosts.all.file(path=TEMP_PATH, state="absent") results = hosts.all.zos_job_output(job_name="SAMPLE") for result in results.contacted.values(): assert result.get("changed") is False assert result.get("jobs") is not None
def test_zos_job_output_job_exists(ansible_zos_module): hosts = ansible_zos_module hosts.all.file(path=TEMP_PATH, state='directory') hosts.all.shell( cmd='echo {} > {}/SAMPLE'.format(quote(JCL_FILE_CONTENTS), TEMP_PATH)) hosts.all.zos_job_submit(src='{}/SAMPLE'.format(TEMP_PATH), location="USS", wait=True, volume=None) hosts.all.file(path=TEMP_PATH, state='absent') results = hosts.all.zos_job_output(job_name='SAMPLE') for result in results.contacted.values(): assert result.get('changed') is False assert result.get('zos_job_output') is not None
def testBinary(shellPath, args, useValgrind): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc # pylint: disable=missing-return-type-doc,missing-type-doc """Test the given shell with the given args.""" test_cmd = (constructVgCmdList() if useValgrind else []) + [str(shellPath)] + args sps.vdump("The testing command is: " + " ".join(quote(str(x)) for x in test_cmd)) test_cmd_result = subprocess.run( test_cmd, cwd=os.getcwdu() if sys.version_info.major == 2 else os.getcwd(), # pylint: disable=no-member env=env_with_path(str(shellPath.parent)), stderr=subprocess.STDOUT, stdout=subprocess.PIPE, timeout=999) out, return_code = test_cmd_result.stdout.decode("utf-8", errors="replace"), test_cmd_result.returncode sps.vdump("The exit code is: " + str(return_code)) return out, return_code
def search_file(self, search_string, file_name): """Searches through a file for a string. Args: search_string: The string or pattern to look for. file_name: The name of the file to search. Returns: True if the string or pattern was found, False otherwise. """ try: self.run('grep %s %s' % (shellescape.quote(search_string), file_name)) return True except job.Error: return False
def encrypt_file(self, inpath, force_nocompress=False, force_compress=False, armored=False, checksum=False): """public method for single file encryption with optional compression, ASCII armored formatting, and file hash digest generation""" if armored: if force_compress: command_stub = self.command_maxcompress_armored elif force_nocompress: command_stub = self.command_nocompress_armored else: if self._is_compress_filetype(inpath): command_stub = self.command_default_armored else: command_stub = self.command_nocompress_armored else: if force_compress: command_stub = self.command_maxcompress elif force_nocompress: command_stub = self.command_nocompress else: if self._is_compress_filetype(inpath): command_stub = self.command_default else: command_stub = self.command_nocompress encrypted_outpath = self._create_outfilepath(inpath) system_command = command_stub + encrypted_outpath + " --passphrase " + quote(self.passphrase) + " --symmetric " + quote(inpath) try: response = muterun(system_command) # check returned status code if response.exitcode == 0: stdout(encrypted_outpath + " was generated from " + inpath) if checksum: # add a SHA256 hash digest of the encrypted file - requested by user --hash flag in command from crypto.library import hash encrypted_file_hash = hash.generate_hash(encrypted_outpath) if len(encrypted_file_hash) == 64: stdout("SHA256 hash digest for " + encrypted_outpath + " :") stdout(encrypted_file_hash) else: stdout("Unable to generate a SHA256 hash digest for the file " + encrypted_outpath) else: stderr(response.stderr, 0) stderr("Encryption failed") sys.exit(1) except Exception as e: stderr("There was a problem with the execution of gpg. Encryption failed. Error: [" + str(e) + "]") sys.exit(1)
def _step0_scan_db(self): progress = Progress('scanning db') __LOG__.debug('scanning db...') db_entries_count = self.file_registry.count() old_files_deleted = 0 mtime_changed = 0 size_changed = 0 index = 0 progress.start(self, maxval=db_entries_count) for db_file in self.file_registry.find_all(): index = index + 1 progress.update(self, val=index) abs_path = join(self.base_dir, db_file.path) try: changed = False stat = os.stat(abs_path) if db_file.mtime != int(stat.st_mtime): mtime_changed = mtime_changed + 1 changed = True if db_file.size != stat.st_size: size_changed = size_changed + 1 changed = True if changed: db_file.mtime = int(stat.st_mtime) db_file.size = stat.st_size self.hash_queue.append(abs_path, stat.st_size, db_file) self.visited_files.append(s(db_file.path)) except OSError as err: if err.errno == errno.ENOENT: __LOG__.debug('deleting %s', quote(db_file.path)) self.file_registry.delete(db_file) old_files_deleted = old_files_deleted + 1 else: raise # pragma: no cover progress.finish(self) __LOG__.debug('mtime of %d files changed', mtime_changed) __LOG__.debug('size of %d files changed', size_changed) __LOG__.debug('%d old files deleted', old_files_deleted) return len(self.visited_files)
def _step1_scan_fs(self): progress = Progress('scanning fs') __LOG__.debug('scanning %s', quote(self.base_dir)) new_files_found = 0 progress.start(self) for entry in files_of_dir(self.base_dir, self.is_excluded): progress.update(self) rel_path = os.path.relpath(entry.path, self.base_dir) if rel_path not in self.visited_files: new_files_found = new_files_found + 1 db_file = model.File( path=u(rel_path), mtime=int(entry.stats.st_mtime), size=entry.stats.st_size, ) self.hash_queue.append( entry.path, entry.stats.st_size, db_file ) progress.finish(self) __LOG__.debug('%d new files found', new_files_found) return new_files_found
def format_path(path): return quote(path).encode('utf-8')
def format_path(path): return quote(path)
def main(): import os import sys from time import sleep import getpass import tarfile from Naked.commandline import Command from Naked.toolshed.shell import execute, muterun from Naked.toolshed.system import dir_exists, file_exists, list_all_files, make_path, stdout, stderr, is_dir from shellescape import quote # ------------------------------------------------------------------------------------------ # [ Instantiate command line object ] # used for all subsequent conditional logic in the CLI application # ------------------------------------------------------------------------------------------ c = Command(sys.argv[0], sys.argv[1:]) # ------------------------------------------------------------------------------------------ # [ VALIDATION LOGIC ] - early validation of appropriate command syntax # Test that user entered at least one argument to the executable, print usage if not # ------------------------------------------------------------------------------------------ if not c.command_suite_validates(): from crypto.settings import usage as crypto_usage print(crypto_usage) sys.exit(1) # ------------------------------------------------------------------------------------------ # [ HELP, VERSION, USAGE LOGIC ] # Naked framework provides default help, usage, and version commands for all applications # --> settings for user messages are assigned in the lib/crypto/settings.py file # ------------------------------------------------------------------------------------------ if c.help(): # User requested crypto help information from crypto.settings import help as crypto_help print(crypto_help) sys.exit(0) elif c.usage(): # User requested crypto usage information from crypto.settings import usage as crypto_usage print(crypto_usage) sys.exit(0) elif c.version(): # User requested crypto version information from crypto.settings import app_name, major_version, minor_version, patch_version version_display_string = app_name + " " + major_version + "." + minor_version + "." + patch_version print(version_display_string) sys.exit(0) # ------------------------------------------------------------------------------------------ # [ APPLICATION LOGIC ] # # ------------------------------------------------------------------------------------------ elif c.argc > 1: # code for multi-file processing and commands that include options use_standard_output = False # print to stdout flag use_file_overwrite = False # overwrite existing file untar_archives = True # untar decrypted tar archives, true by default # set user option flags if c.option("--stdout") or c.option("-s"): use_standard_output = True if c.option("--overwrite") or c.option("-o"): use_file_overwrite = True if c.option("--nountar"): untar_archives = False directory_list = [] # directory paths included in the user entered paths from the command line file_list = ( [] ) # file paths included in the user entered paths from the command line (and inside directories entered) for argument in c.argv: if file_exists(argument): # user included a file, add it to the file_list for decryption if argument.endswith(".crypt"): file_list.append(argument) # add .crypt files to the list of files for decryption elif argument.endswith(".gpg"): file_list.append(argument) elif argument.endswith(".asc"): file_list.append(argument) elif argument.endswith(".pgp"): file_list.append(argument) else: # cannot identify as an encrypted file, give it a shot anyways but warn user file_list.append(argument) stdout( "Could not confirm that '" + argument + "' is encrypted based upon the file type. Attempting decryption. Keep your fingers crossed..." ) elif dir_exists(argument): # user included a directory, add it to the directory_list directory_list.append(argument) else: if argument[0] == "-": pass # if it is an option, do nothing else: stderr( "'" + argument + "' does not appear to be an existing file or directory. Aborting decryption attempt for this request." ) # unroll the contained directory files into the file_list IF they are encrypted file types if len(directory_list) > 0: for directory in directory_list: directory_file_list = list_all_files(directory) for contained_file in directory_file_list: if contained_file.endswith(".crypt"): file_list.append( make_path(directory, contained_file) ) # include the file with a filepath 'directory path/contained_file path' elif contained_file.endswith(".gpg"): file_list.append(make_path(directory, contained_file)) elif contained_file.endswith("asc"): file_list.append(make_path(directory, contained_file)) elif contained_file.endswith(".pgp"): file_list.append(make_path(directory, contained_file)) # confirm that there are files for decryption, if not abort if len(file_list) == 0: stderr("Could not identify files for decryption") sys.exit(1) # get passphrase used to symmetrically decrypt the file passphrase = getpass.getpass("Please enter your passphrase: ") if len(passphrase) == 0: # confirm that user entered a passphrase stderr("You did not enter a passphrase. Please repeat your command and try again.") sys.exit(1) passphrase_confirm = getpass.getpass("Please enter your passphrase again: ") if passphrase == passphrase_confirm: # begin decryption of each requested file. the directory path was already added to the file path above for encrypted_file in file_list: # create the decrypted file name decrypted_filename = "" if encrypted_file.endswith(".crypt"): decrypted_filename = encrypted_file[0:-6] elif ( encrypted_file.endswith(".gpg") or encrypted_file.endswith(".asc") or encrypted_file.endswith(".pgp") ): decrypted_filename = encrypted_file[0:-4] else: decrypted_filename = ( encrypted_file + ".decrypt" ) # if it was a file without a known encrypted file type, add the .decrypt suffix # determine whether file overwrite will take place with the decrypted file skip_file = False # flag that indicates this file should not be encrypted created_tmp_files = False if not use_standard_output: # if not writing a file, no need to check for overwrite if file_exists(decrypted_filename): if ( use_file_overwrite ): # rename the existing file to temp file which will be erased or replaced (on decryption failures) below tmp_filename = decrypted_filename + ".tmp" os.rename(decrypted_filename, tmp_filename) created_tmp_files = True else: stdout( "The file path '" + decrypted_filename + "' already exists. This file was not decrypted." ) skip_file = True # begin decryption if not skip_file: if ( use_standard_output ): # using --quiet flag to suppress stdout messages from gpg, just want the file data in stdout stream system_command = ( "gpg --batch --quiet --passphrase " + quote(passphrase) + " -d " + quote(encrypted_file) ) successful_execution = execute( system_command ) # use naked execute function to directly push to stdout, rather than return stdout if not successful_execution: stderr("Unable to decrypt file '" + encrypted_file + "'", 0) if created_tmp_files: # restore the moved tmp file to original if decrypt failed tmp_filename = decrypted_filename + ".tmp" if file_exists(tmp_filename): os.rename(tmp_filename, decrypted_filename) else: # decryption successful but we are in stdout flag so do not include any other output from decrypto pass else: system_command = ( "gpg --batch -o " + quote(decrypted_filename) + " --passphrase " + quote(passphrase) + " -d " + quote(encrypted_file) ) response = muterun(system_command) if response.exitcode == 0: stdout("'" + encrypted_file + "' decrypted to '" + decrypted_filename + "'") else: # failed decryption if created_tmp_files: # restore the moved tmp file to original if decrypt failed tmp_filename = decrypted_filename + ".tmp" if file_exists(tmp_filename): os.rename(tmp_filename, decrypted_filename) # report the error stderr(response.stderr) stderr("Decryption failed for " + encrypted_file) # cleanup: remove the tmp file if created_tmp_files: tmp_filename = decrypted_filename + ".tmp" if file_exists(tmp_filename): os.remove(tmp_filename) # untar/extract any detected archive file(s) if untar_archives is True: if decrypted_filename.endswith(".tar") and tarfile.is_tarfile(decrypted_filename): untar_path_tuple = os.path.split(decrypted_filename) untar_path = untar_path_tuple[0] if use_file_overwrite: with tarfile.open(decrypted_filename) as tar: if len(untar_path) > 0: tar.extractall( path=untar_path ) # use dir path from the decrypted_filename if not CWD stdout( "'" + decrypted_filename + "' unpacked in the directory path '" + untar_path + "'" ) else: tar.extractall() # else use CWD stdout("'" + decrypted_filename + "' unpacked in the current working directory") else: with tarfile.TarFile(decrypted_filename, "r", errorlevel=1) as tar: for tarinfo in tar: t_file = tarinfo.name if len(untar_path) > 0: t_file_path = os.path.join(untar_path, t_file) else: t_file_path = t_file if not os.path.exists(t_file_path): try: if len(untar_path) > 0: tar.extract(t_file, path=untar_path) # write to the appropriate dir else: tar.extract(t_file) # write to CWD except IOError as e: stderr("Failed to unpack the file '" + t_file_path + "' [" + str(e) + "]") elif is_dir(t_file_path): pass # do nothing if it exists and is a directory, no need to warn else: # it is a file and it already exists, provide user error message stderr( "Failed to unpack the file '" + t_file_path + "'. File already exists. Use the --overwrite flag to replace existing files." ) # remove the decrypted tar archive file os.remove(decrypted_filename) # overwrite the entered passphrases after file decryption is complete for all files passphrase = "" passphrase_confirm = "" # add a short pause to hinder brute force pexpect style password attacks with decrypto sleep(0.2) # 200ms pause else: # passphrases did not match passphrase = "" passphrase_confirm = "" stderr("The passphrases did not match. Please enter your command again.") sys.exit(1) elif c.argc == 1: # simple single file or directory processing with default settings path = c.arg0 if file_exists(path): # SINGLE FILE check_existing_file = False # check for a file with the name of new decrypted filename in the directory if path.endswith(".crypt"): decrypted_filename = path[0:-6] # remove the .crypt suffix check_existing_file = True elif path.endswith(".gpg") or path.endswith(".pgp") or path.endswith(".asc"): decrypted_filename = path[0:-4] check_existing_file = True else: decrypted_filename = ( path + ".decrypt" ) # if there is not a standard file type, then add a .decrypt suffix to the decrypted file name stdout( "Could not confirm that the requested file is encrypted based upon the file type. Attempting decryption. Keep your fingers crossed..." ) # confirm that the decrypted path does not already exist, if so abort with warning message to user if check_existing_file is True: if file_exists(decrypted_filename): stderr( "Your file will be decrypted to '" + decrypted_filename + "' and this file path already exists. Please move the file or use the --overwrite option with your command if you intend to replace the current file." ) sys.exit(1) # get passphrase used to symmetrically decrypt the file passphrase = getpass.getpass("Please enter your passphrase: ") if len(passphrase) == 0: # confirm that user entered a passphrase stderr("You did not enter a passphrase. Please repeat your command and try again.") sys.exit(1) passphrase_confirm = getpass.getpass("Please enter your passphrase again: ") # confirm that the passphrases match if passphrase == passphrase_confirm: system_command = ( "gpg --batch -o " + quote(decrypted_filename) + " --passphrase " + quote(passphrase) + " -d " + quote(path) ) response = muterun(system_command) if response.exitcode == 0: # unpack tar archive generated from the decryption, if present if decrypted_filename.endswith(".tar") and tarfile.is_tarfile(decrypted_filename): untar_path_tuple = os.path.split(decrypted_filename) untar_path = untar_path_tuple[0] with tarfile.TarFile(decrypted_filename, "r", errorlevel=1) as tar: for tarinfo in tar: t_file = tarinfo.name if len(untar_path) > 0: t_file_path = os.path.join(untar_path, t_file) else: t_file_path = t_file if not os.path.exists(t_file_path): try: if len(untar_path) > 0: tar.extract(t_file, path=untar_path) # write to the appropriate dir else: tar.extract(t_file) # write to CWD except IOError as e: stderr("Failed to unpack the file '" + t_file_path + "' [" + str(e) + "]") elif is_dir(t_file_path): pass # do nothing if it exists and is a directory, no need to warn else: # it is a file and it already exists, provide user error message stderr( "Failed to unpack the file '" + t_file_path + "'. File already exists. Use the --overwrite flag to replace existing files." ) # remove the decrypted tar archive os.remove(decrypted_filename) stdout("Decryption complete") # overwrite user entered passphrases passphrase = "" passphrase_confirm = "" sys.exit(0) else: stderr(response.stderr) stderr("Decryption failed") # overwrite user entered passphrases passphrase = "" passphrase_confirm = "" # add a short pause to hinder brute force pexpect style password attacks with decrypto sleep(0.2) # 200ms pause sys.exit(1) else: stderr("The passphrases did not match. Please enter your command again.") sys.exit(1) elif dir_exists(path): # SINGLE DIRECTORY dirty_directory_file_list = list_all_files(path) directory_file_list = [ x for x in dirty_directory_file_list if (x.endswith(".crypt") or x.endswith(".gpg") or x.endswith(".pgp") or x.endswith(".asc")) ] # if there are no encrypted files found, warn and abort if len(directory_file_list) == 0: stderr("There are no encrypted files in the directory") sys.exit(1) # prompt for the passphrase passphrase = getpass.getpass("Please enter your passphrase: ") if len(passphrase) == 0: # confirm that user entered a passphrase stderr("You did not enter a passphrase. Please repeat your command and try again.") sys.exit(1) passphrase_confirm = getpass.getpass("Please enter your passphrase again: ") if passphrase == passphrase_confirm: # decrypt all of the encypted files in the directory for filepath in directory_file_list: absolute_filepath = make_path( path, filepath ) # combine the directory path and file name into absolute path # remove file suffix from the decrypted file path that writes to disk if absolute_filepath.endswith(".crypt"): decrypted_filepath = absolute_filepath[0:-6] # remove the .crypt suffix elif ( absolute_filepath.endswith(".gpg") or absolute_filepath.endswith(".pgp") or absolute_filepath.endswith(".asc") ): decrypted_filepath = absolute_filepath[0:-4] # confirm that the file does not already exist if file_exists(decrypted_filepath): stdout( "The file path '" + decrypted_filepath + "' already exists. This file was not decrypted." ) else: system_command = ( "gpg --batch -o " + quote(decrypted_filepath) + " --passphrase " + quote(passphrase) + " -d " + quote(absolute_filepath) ) response = muterun(system_command) if response.exitcode == 0: stdout("'" + absolute_filepath + "' decrypted to '" + decrypted_filepath + "'") else: stderr(response.stderr) stderr("Decryption failed for " + absolute_filepath) # overwrite user entered passphrases passphrase = "" passphrase_confirm = "" # add a short pause to hinder brute force pexpect style password attacks with decrypto sleep(0.2) # 200ms pause else: # overwrite user entered passphrases passphrase = "" passphrase_confirm = "" stderr("The passphrases did not match. Please enter your command again.") sys.exit(1) else: # error message, not a file or directory. user entry error stderr("The path that you entered does not appear to be an existing file or directory. Please try again.") sys.exit(1) # ------------------------------------------------------------------------------------------ # [ DEFAULT MESSAGE FOR MATCH FAILURE ] # Message to provide to the user when all above conditional logic fails to meet a true condition # ------------------------------------------------------------------------------------------ else: print("Could not complete your request. Please try again.") sys.exit(1)
def maybe_quote(arg): return shellescape.quote(arg) if needs_shell_quoting(arg) else arg
def _escape(self, value): """Escape given value unless it is safe.""" if isinstance(value, SafeString): return value return shellescape.quote(value)
def job(self, joborder, input_basedir, output_callback, **kwargs): builder = self._init_job(joborder, input_basedir, **kwargs) if self.tool["baseCommand"]: for n, b in enumerate(aslist(self.tool["baseCommand"])): builder.bindings.append({ "position": [-1000000, n], "valueFrom": b }) if self.tool.get("arguments"): for i, a in enumerate(self.tool["arguments"]): if isinstance(a, dict): a = copy.copy(a) if a.get("position"): a["position"] = [a["position"], i] else: a["position"] = [0, i] a["do_eval"] = a["valueFrom"] a["valueFrom"] = None builder.bindings.append(a) else: builder.bindings.append({ "position": [0, i], "valueFrom": a }) builder.bindings.sort(key=lambda a: a["position"]) reffiles = set((f["path"] for f in builder.files)) j = self.makeJobRunner() j.joborder = builder.job j.stdin = None j.stdout = None j.successCodes = self.tool.get("successCodes") j.temporaryFailCodes = self.tool.get("temporaryFailCodes") j.permanentFailCodes = self.tool.get("permanentFailCodes") j.requirements = self.requirements j.hints = self.hints j.name = uniquename(kwargs.get("name", str(id(j)))) _logger.debug("[job %s] initializing from %s%s", j.name, self.tool.get("id", ""), " as part of %s" % kwargs["part_of"] if "part_of" in kwargs else "") _logger.debug("[job %s] %s", j.name, json.dumps(joborder, indent=4)) builder.pathmapper = None if self.tool.get("stdin"): j.stdin = builder.do_eval(self.tool["stdin"]) if isinstance(j.stdin, dict) and "ref" in j.stdin: j.stdin = builder.job[j.stdin["ref"][1:]]["path"] reffiles.add(j.stdin) if self.tool.get("stdout"): j.stdout = builder.do_eval(self.tool["stdout"]) if os.path.isabs(j.stdout) or ".." in j.stdout: raise validate.ValidationException("stdout must be a relative path") builder.pathmapper = self.makePathMapper(reffiles, input_basedir, **kwargs) builder.requirements = j.requirements for f in builder.files: f["path"] = builder.pathmapper.mapper(f["path"])[1] _logger.debug("[job %s] command line bindings is %s", j.name, json.dumps(builder.bindings, indent=4)) _logger.debug("[job %s] path mappings is %s", j.name, json.dumps({p: builder.pathmapper.mapper(p) for p in builder.pathmapper.files()}, indent=4)) dockerReq, _ = self.get_requirement("DockerRequirement") if dockerReq and kwargs.get("use_container"): out_prefix = kwargs.get("tmp_outdir_prefix") j.outdir = kwargs.get("outdir") or tempfile.mkdtemp(prefix=out_prefix) tmpdir_prefix = kwargs.get('tmpdir_prefix') j.tmpdir = kwargs.get("tmpdir") or tempfile.mkdtemp(prefix=tmpdir_prefix) else: j.outdir = builder.outdir j.tmpdir = builder.tmpdir createFiles, _ = self.get_requirement("CreateFileRequirement") j.generatefiles = {} if createFiles: for t in createFiles["fileDef"]: j.generatefiles[builder.do_eval(t["filename"])] = copy.deepcopy(builder.do_eval(t["fileContent"])) j.environment = {} evr, _ = self.get_requirement("EnvVarRequirement") if evr: for t in evr["envDef"]: j.environment[t["envName"]] = builder.do_eval(t["envValue"]) shellcmd, _ = self.get_requirement("ShellCommandRequirement") if shellcmd: cmd = [] for b in builder.bindings: arg = builder.generate_arg(b) if b.get("shellQuote", True): arg = [shellescape.quote(a) for a in aslist(arg)] cmd.extend(aslist(arg)) j.command_line = ["/bin/sh", "-c", " ".join(cmd)] else: j.command_line = flatten(map(builder.generate_arg, builder.bindings)) j.pathmapper = builder.pathmapper j.collect_outputs = functools.partial(self.collect_output_ports, self.tool["outputs"], builder) j.output_callback = output_callback yield j
def job(self, joborder, output_callback, **kwargs): # type: (Dict[str,str], str, Callable[..., Any], **Any) -> Generator[Union[CommandLineJob, CallbackJob], None, None] jobname = uniquename(kwargs.get("name", shortname(self.tool.get("id", "job")))) if kwargs.get("cachedir"): cacheargs = kwargs.copy() cacheargs["outdir"] = "/out" cacheargs["tmpdir"] = "/tmp" cachebuilder = self._init_job(joborder, **cacheargs) cachebuilder.pathmapper = PathMapper(set((f["path"] for f in cachebuilder.files)), kwargs["basedir"]) cmdline = flatten(map(cachebuilder.generate_arg, cachebuilder.bindings)) (docker_req, docker_is_req) = self.get_requirement("DockerRequirement") if docker_req and kwargs.get("use_container") is not False: dockerimg = docker_req.get("dockerImageId") or docker_req.get("dockerPull") cmdline = ["docker", "run", dockerimg] + cmdline keydict = {"cmdline": cmdline} for _,f in cachebuilder.pathmapper.items(): st = os.stat(f[0]) keydict[f[0]] = [st.st_size, int(st.st_mtime * 1000)] interesting = {"DockerRequirement", "EnvVarRequirement", "CreateFileRequirement", "ShellCommandRequirement"} for rh in (self.requirements, self.hints): for r in reversed(rh): if r["class"] in interesting and r["class"] not in keydict: keydict[r["class"]] = r keydictstr = json.dumps(keydict, separators=(',',':'), sort_keys=True) cachekey = hashlib.md5(keydictstr).hexdigest() _logger.debug("[job %s] keydictstr is %s -> %s", jobname, keydictstr, cachekey) jobcache = os.path.join(kwargs["cachedir"], cachekey) jobcachepending = jobcache + ".pending" if os.path.isdir(jobcache) and not os.path.isfile(jobcachepending): if docker_req and kwargs.get("use_container") is not False: cachebuilder.outdir = kwargs.get("docker_outdir") or "/var/spool/cwl" else: cachebuilder.outdir = jobcache _logger.info("[job %s] Using cached output in %s", jobname, jobcache) yield CallbackJob(self, output_callback, cachebuilder, jobcache) return else: _logger.info("[job %s] Output of job will be cached in %s", jobname, jobcache) shutil.rmtree(jobcache, True) os.makedirs(jobcache) kwargs["outdir"] = jobcache open(jobcachepending, "w").close() def rm_pending_output_callback(output_callback, jobcachepending, outputs, processStatus): if processStatus == "success": os.remove(jobcachepending) output_callback(outputs, processStatus) output_callback = cast( Callable[..., Any], # known bug in mypy # https://github.com/python/mypy/issues/797 partial(rm_pending_output_callback, output_callback, jobcachepending)) builder = self._init_job(joborder, **kwargs) reffiles = set((f["path"] for f in builder.files)) j = self.makeJobRunner() j.builder = builder j.joborder = builder.job j.stdin = None j.stdout = None j.successCodes = self.tool.get("successCodes") j.temporaryFailCodes = self.tool.get("temporaryFailCodes") j.permanentFailCodes = self.tool.get("permanentFailCodes") j.requirements = self.requirements j.hints = self.hints j.name = jobname _logger.debug(u"[job %s] initializing from %s%s", j.name, self.tool.get("id", ""), u" as part of %s" % kwargs["part_of"] if "part_of" in kwargs else "") _logger.debug(u"[job %s] %s", j.name, json.dumps(joborder, indent=4)) builder.pathmapper = None if self.tool.get("stdin"): j.stdin = builder.do_eval(self.tool["stdin"]) reffiles.add(j.stdin) if self.tool.get("stdout"): j.stdout = builder.do_eval(self.tool["stdout"]) if os.path.isabs(j.stdout) or ".." in j.stdout: raise validate.ValidationException("stdout must be a relative path") builder.pathmapper = self.makePathMapper(reffiles, **kwargs) builder.requirements = j.requirements # map files to assigned path inside a container. We need to also explicitly # walk over input as implicit reassignment doesn't reach everything in builder.bindings def _check_adjust(f): # type: (Dict[str,Any]) -> Dict[str,Any] if not f.get("containerfs"): f["path"] = builder.pathmapper.mapper(f["path"])[1] f["containerfs"] = True return f _logger.debug(u"[job %s] path mappings is %s", j.name, json.dumps({p: builder.pathmapper.mapper(p) for p in builder.pathmapper.files()}, indent=4)) adjustFileObjs(builder.files, _check_adjust) adjustFileObjs(builder.bindings, _check_adjust) _logger.debug(u"[job %s] command line bindings is %s", j.name, json.dumps(builder.bindings, indent=4)) dockerReq = self.get_requirement("DockerRequirement")[0] if dockerReq and kwargs.get("use_container"): out_prefix = kwargs.get("tmp_outdir_prefix") j.outdir = kwargs.get("outdir") or tempfile.mkdtemp(prefix=out_prefix) tmpdir_prefix = kwargs.get('tmpdir_prefix') j.tmpdir = kwargs.get("tmpdir") or tempfile.mkdtemp(prefix=tmpdir_prefix) else: j.outdir = builder.outdir j.tmpdir = builder.tmpdir createFiles = self.get_requirement("CreateFileRequirement")[0] j.generatefiles = {} if createFiles: for t in createFiles["fileDef"]: j.generatefiles[builder.do_eval(t["filename"])] = copy.deepcopy(builder.do_eval(t["fileContent"])) j.environment = {} evr = self.get_requirement("EnvVarRequirement")[0] if evr: for t in evr["envDef"]: j.environment[t["envName"]] = builder.do_eval(t["envValue"]) shellcmd = self.get_requirement("ShellCommandRequirement")[0] if shellcmd: cmd = [] # type: List[str] for b in builder.bindings: arg = builder.generate_arg(b) if b.get("shellQuote", True): arg = [shellescape.quote(a) for a in aslist(arg)] cmd.extend(aslist(arg)) j.command_line = ["/bin/sh", "-c", " ".join(cmd)] else: j.command_line = flatten(map(builder.generate_arg, builder.bindings)) j.pathmapper = builder.pathmapper j.collect_outputs = partial( self.collect_output_ports, self.tool["outputs"], builder) j.output_callback = output_callback yield j
def run(self, dry_run=False, pull_image=True, rm_container=True, rm_tmpdir=True, move_outputs=True, **kwargs): #_logger.info("[job %s] starting with outdir %s", id(self), self.outdir) if not os.path.exists(self.outdir): os.makedirs(self.outdir) #with open(os.path.join(outdir, "cwl.input.json"), "w") as fp: # json.dump(self.joborder, fp) runtime = [] env = {"TMPDIR": self.tmpdir} (docker_req, docker_is_req) = get_feature(self, "DockerRequirement") for f in self.pathmapper.files(): if not os.path.exists(self.pathmapper.mapper(f)[0]): raise WorkflowException("Required input file %s not found" % self.pathmapper.mapper(f)[0]) img_id = None if docker_req and kwargs.get("use_container") is not False: env = os.environ img_id = docker.get_from_requirements(docker_req, docker_is_req, pull_image) if docker_is_req and img_id is None: raise WorkflowException("Docker is required for running this tool.") if img_id: runtime = ["docker", "run", "-i"] for src in self.pathmapper.files(): vol = self.pathmapper.mapper(src) runtime.append("--volume=%s:%s:ro" % vol) runtime.append("--volume=%s:%s:rw" % (os.path.abspath(self.outdir), "/tmp/job_output")) runtime.append("--volume=%s:%s:rw" % (os.path.abspath(self.tmpdir), "/tmp/job_tmp")) runtime.append("--workdir=%s" % ("/tmp/job_output")) runtime.append("--user=%s" % (os.geteuid())) if rm_container: runtime.append("--rm") runtime.append("--env=TMPDIR=/tmp/job_tmp") for t,v in self.environment.items(): runtime.append("--env=%s=%s" % (t, v)) runtime.append(img_id) else: env = self.environment if not os.path.exists(self.tmpdir): os.makedirs(self.tmpdir) env["TMPDIR"] = self.tmpdir stdin = None stdout = None _logger.info("[job %s] exec %s%s%s", id(self), " ".join([shellescape.quote(arg) if needs_shell_quoting(arg) else arg for arg in (runtime + self.command_line)]), ' < %s' % (self.stdin) if self.stdin else '', ' > %s' % os.path.join(self.outdir, self.stdout) if self.stdout else '') if dry_run: return (self.outdir, {}) outputs = {} try: for t in self.generatefiles: if isinstance(self.generatefiles[t], dict): os.symlink(self.generatefiles[t]["path"], os.path.join(self.outdir, t)) else: with open(os.path.join(self.outdir, t), "w") as f: f.write(self.generatefiles[t]) if self.stdin: stdin = open(self.pathmapper.mapper(self.stdin)[0], "rb") else: stdin = subprocess.PIPE if self.stdout: absout = os.path.join(self.outdir, self.stdout) dn = os.path.dirname(absout) if dn and not os.path.exists(dn): os.makedirs(dn) stdout = open(absout, "wb") else: stdout = sys.stderr sp = subprocess.Popen(runtime + self.command_line, shell=False, close_fds=True, stdin=stdin, stdout=stdout, env=env, cwd=self.outdir) if stdin == subprocess.PIPE: sp.stdin.close() rcode = sp.wait() if stdin != subprocess.PIPE: stdin.close() if stdout is not sys.stderr: stdout.close() if self.successCodes and rcode in self.successCodes: processStatus = "success" elif self.temporaryFailCodes and rcode in self.temporaryFailCodes: processStatus = "temporaryFail" elif self.permanentFailCodes and rcode in self.permanentFailCodes: processStatus = "permanentFail" elif rcode == 0: processStatus = "success" else: processStatus = "permanentFail" for t in self.generatefiles: if isinstance(self.generatefiles[t], dict): os.remove(os.path.join(self.outdir, t)) os.symlink(self.pathmapper.reversemap(self.generatefiles[t]["path"])[1], os.path.join(self.outdir, t)) outputs = self.collect_outputs(self.outdir) except Exception as e: _logger.exception("Exception while running job") processStatus = "permanentFail" if processStatus != "success": _logger.warn("[job %s] completed %s", id(self), processStatus) else: _logger.debug("[job %s] completed %s", id(self), processStatus) _logger.debug("[job %s] %s", id(self), json.dumps(outputs, indent=4)) self.output_callback(outputs, processStatus) if rm_tmpdir: _logger.debug("[job %s] Removing temporary directory %s", id(self), self.tmpdir) shutil.rmtree(self.tmpdir, True) if move_outputs and empty_subtree(self.outdir): _logger.debug("[job %s] Removing empty output directory %s", id(self), self.tmpdir) shutil.rmtree(self.outdir, True)
def _execute(self, runtime, env, rm_tmpdir=True, move_outputs="move"): # type: (List[Text], MutableMapping[Text, Text], bool, Text) -> None scr, _ = get_feature(self, "ShellCommandRequirement") shouldquote = None # type: Callable[[Any], Any] if scr: shouldquote = lambda x: False else: shouldquote = needs_shell_quoting_re.search _logger.info(u"[job %s] %s$ %s%s%s%s", self.name, self.outdir, " \\\n ".join([shellescape.quote(Text(arg)) if shouldquote(Text(arg)) else Text(arg) for arg in (runtime + self.command_line)]), u' < %s' % self.stdin if self.stdin else '', u' > %s' % os.path.join(self.outdir, self.stdout) if self.stdout else '', u' 2> %s' % os.path.join(self.outdir, self.stderr) if self.stderr else '') outputs = {} # type: Dict[Text,Text] try: stdin_path = None if self.stdin: stdin_path = self.pathmapper.reversemap(self.stdin)[1] stderr_path = None if self.stderr: abserr = os.path.join(self.outdir, self.stderr) dnerr = os.path.dirname(abserr) if dnerr and not os.path.exists(dnerr): os.makedirs(dnerr) stderr_path = abserr stdout_path = None if self.stdout: absout = os.path.join(self.outdir, self.stdout) dn = os.path.dirname(absout) if dn and not os.path.exists(dn): os.makedirs(dn) stdout_path = absout commands = [Text(x) for x in (runtime + self.command_line)] job_script_contents = None # type: Text builder = getattr(self, "builder", None) # type: Builder if builder is not None: job_script_contents = builder.build_job_script(commands) rcode = _job_popen( commands, stdin_path=stdin_path, stdout_path=stdout_path, stderr_path=stderr_path, env=env, cwd=self.outdir, job_script_contents=job_script_contents, ) if self.successCodes and rcode in self.successCodes: processStatus = "success" elif self.temporaryFailCodes and rcode in self.temporaryFailCodes: processStatus = "temporaryFail" elif self.permanentFailCodes and rcode in self.permanentFailCodes: processStatus = "permanentFail" elif rcode == 0: processStatus = "success" else: processStatus = "permanentFail" if self.generatefiles["listing"]: relink_initialworkdir(self.generatemapper, self.outdir, self.builder.outdir, inplace_update=self.inplace_update) outputs = self.collect_outputs(self.outdir) outputs = bytes2str_in_dicts(outputs) # type: ignore except OSError as e: if e.errno == 2: if runtime: _logger.error(u"'%s' not found", runtime[0]) else: _logger.error(u"'%s' not found", self.command_line[0]) else: _logger.exception("Exception while running job") processStatus = "permanentFail" except WorkflowException as e: _logger.error(u"[job %s] Job error:\n%s" % (self.name, e)) processStatus = "permanentFail" except Exception as e: _logger.exception("Exception while running job") processStatus = "permanentFail" if processStatus != "success": _logger.warning(u"[job %s] completed %s", self.name, processStatus) else: _logger.info(u"[job %s] completed %s", self.name, processStatus) if _logger.isEnabledFor(logging.DEBUG): _logger.debug(u"[job %s] %s", self.name, json.dumps(outputs, indent=4)) self.output_callback(outputs, processStatus) if self.stagedir and os.path.exists(self.stagedir): _logger.debug(u"[job %s] Removing input staging directory %s", self.name, self.stagedir) shutil.rmtree(self.stagedir, True) if rm_tmpdir: _logger.debug(u"[job %s] Removing temporary directory %s", self.name, self.tmpdir) shutil.rmtree(self.tmpdir, True)
def _execute(self, runtime, # type: List[Text] env, # type: MutableMapping[Text, Text] runtimeContext, # type: RuntimeContext monitor_function=None, # type: Optional[Callable] ): # type: (...) -> None scr, _ = self.get_requirement("ShellCommandRequirement") shouldquote = needs_shell_quoting_re.search # type: Callable[[Any], Any] if scr is not None: shouldquote = lambda x: False _logger.info(u"[job %s] %s$ %s%s%s%s", self.name, self.outdir, " \\\n ".join([shellescape.quote(Text(arg)) if shouldquote(Text(arg)) else Text(arg) for arg in (runtime + self.command_line)]), u' < %s' % self.stdin if self.stdin else '', u' > %s' % os.path.join(self.outdir, self.stdout) if self.stdout else '', u' 2> %s' % os.path.join(self.outdir, self.stderr) if self.stderr else '') if self.joborder is not None and runtimeContext.research_obj is not None: job_order = self.joborder assert runtimeContext.process_run_id is not None assert runtimeContext.prov_obj is not None runtimeContext.prov_obj.used_artefacts( job_order, runtimeContext.process_run_id, str(self.name)) outputs = {} # type: Dict[Text,Text] try: stdin_path = None if self.stdin is not None: rmap = self.pathmapper.reversemap(self.stdin) if rmap is None: raise WorkflowException( "{} missing from pathmapper".format(self.stdin)) else: stdin_path = rmap[1] stderr_path = None if self.stderr is not None: abserr = os.path.join(self.outdir, self.stderr) dnerr = os.path.dirname(abserr) if dnerr and not os.path.exists(dnerr): os.makedirs(dnerr) stderr_path = abserr stdout_path = None if self.stdout is not None: absout = os.path.join(self.outdir, self.stdout) dnout = os.path.dirname(absout) if dnout and not os.path.exists(dnout): os.makedirs(dnout) stdout_path = absout commands = [Text(x) for x in runtime + self.command_line] if runtimeContext.secret_store is not None: commands = runtimeContext.secret_store.retrieve(commands) env = runtimeContext.secret_store.retrieve(env) job_script_contents = None # type: Optional[Text] builder = getattr(self, "builder", None) # type: Builder if builder is not None: job_script_contents = builder.build_job_script(commands) rcode = _job_popen( commands, stdin_path=stdin_path, stdout_path=stdout_path, stderr_path=stderr_path, env=env, cwd=self.outdir, job_dir=tempfile.mkdtemp(prefix=getdefault(runtimeContext.tmp_outdir_prefix, DEFAULT_TMP_PREFIX)), job_script_contents=job_script_contents, timelimit=self.timelimit, name=self.name, monitor_function=monitor_function ) if rcode in self.successCodes: processStatus = "success" elif rcode in self.temporaryFailCodes: processStatus = "temporaryFail" elif rcode in self.permanentFailCodes: processStatus = "permanentFail" elif rcode == 0: processStatus = "success" else: processStatus = "permanentFail" if 'listing' in self.generatefiles: assert self.generatemapper is not None relink_initialworkdir( self.generatemapper, self.outdir, self.builder.outdir, inplace_update=self.inplace_update) outputs = self.collect_outputs(self.outdir) outputs = bytes2str_in_dicts(outputs) # type: ignore except OSError as e: if e.errno == 2: if runtime: _logger.error(u"'%s' not found: %s", runtime[0], e) else: _logger.error(u"'%s' not found: %s", self.command_line[0], e) else: _logger.exception(u"Exception while running job") processStatus = "permanentFail" except WorkflowException as err: _logger.error(u"[job %s] Job error:\n%s", self.name, err) processStatus = "permanentFail" except Exception as e: _logger.exception(u"Exception while running job") processStatus = "permanentFail" if runtimeContext.research_obj is not None \ and self.prov_obj is not None \ and runtimeContext.process_run_id is not None: # creating entities for the outputs produced by each step (in the provenance document) self.prov_obj.record_process_end(str(self.name), runtimeContext.process_run_id, outputs, datetime.datetime.now()) if processStatus != "success": _logger.warning(u"[job %s] completed %s", self.name, processStatus) else: _logger.info(u"[job %s] completed %s", self.name, processStatus) if _logger.isEnabledFor(logging.DEBUG): _logger.debug(u"[job %s] %s", self.name, json_dumps(outputs, indent=4)) if self.generatemapper is not None and runtimeContext.secret_store is not None: # Delete any runtime-generated files containing secrets. for _, p in self.generatemapper.items(): if p.type == "CreateFile": if runtimeContext.secret_store.has_secret(p.resolved): host_outdir = self.outdir container_outdir = self.builder.outdir host_outdir_tgt = p.target if p.target.startswith(container_outdir + "/"): host_outdir_tgt = os.path.join( host_outdir, p.target[len(container_outdir) + 1:]) os.remove(host_outdir_tgt) if runtimeContext.workflow_eval_lock is None: raise WorkflowException("runtimeContext.workflow_eval_lock must not be None") with runtimeContext.workflow_eval_lock: self.output_callback(outputs, processStatus) if self.stagedir is not None and os.path.exists(self.stagedir): _logger.debug(u"[job %s] Removing input staging directory %s", self.name, self.stagedir) shutil.rmtree(self.stagedir, True) if runtimeContext.rm_tmpdir: _logger.debug(u"[job %s] Removing temporary directory %s", self.name, self.tmpdir) shutil.rmtree(self.tmpdir, True)
def run(self, dry_run=False, pull_image=True, rm_container=True, rm_tmpdir=True, move_outputs=True, **kwargs): if not os.path.exists(self.outdir): os.makedirs(self.outdir) #with open(os.path.join(outdir, "cwl.input.json"), "w") as fp: # json.dump(self.joborder, fp) runtime = [] env = {"TMPDIR": self.tmpdir} (docker_req, docker_is_req) = get_feature(self, "DockerRequirement") for f in self.pathmapper.files(): if not os.path.isfile(self.pathmapper.mapper(f)[0]): raise WorkflowException("Required input file %s not found or is not a regular file." % self.pathmapper.mapper(f)[0]) img_id = None if docker_req and kwargs.get("use_container") is not False: env = os.environ img_id = docker.get_from_requirements(docker_req, docker_is_req, pull_image) if docker_is_req and img_id is None: raise WorkflowException("Docker is required for running this tool.") if img_id: runtime = ["docker", "run", "-i"] for src in self.pathmapper.files(): vol = self.pathmapper.mapper(src) runtime.append("--volume=%s:%s:ro" % vol) runtime.append("--volume=%s:%s:rw" % (os.path.abspath(self.outdir), "/var/spool/cwl")) runtime.append("--volume=%s:%s:rw" % (os.path.abspath(self.tmpdir), "/tmp")) runtime.append("--workdir=%s" % ("/var/spool/cwl")) runtime.append("--read-only=true") runtime.append("--net=none") euid = docker_vm_uid() or os.geteuid() runtime.append("--user=%s" % (euid)) if rm_container: runtime.append("--rm") runtime.append("--env=TMPDIR=/tmp") for t,v in self.environment.items(): runtime.append("--env=%s=%s" % (t, v)) runtime.append(img_id) else: env = self.environment if not os.path.exists(self.tmpdir): os.makedirs(self.tmpdir) env["TMPDIR"] = self.tmpdir vars_to_preserve = kwargs.get("preserve_environment") if vars_to_preserve is not None: for key, value in os.environ.items(): if key in vars_to_preserve and key not in env: env[key] = value stdin = None stdout = None scr, _ = get_feature(self, "ShellCommandRequirement") if scr: shouldquote = lambda x: False else: shouldquote = needs_shell_quoting_re.search _logger.info("[job %s] %s$ %s%s%s", self.name, self.outdir, " ".join([shellescape.quote(str(arg)) if shouldquote(str(arg)) else str(arg) for arg in (runtime + self.command_line)]), ' < %s' % (self.stdin) if self.stdin else '', ' > %s' % os.path.join(self.outdir, self.stdout) if self.stdout else '') if dry_run: return (self.outdir, {}) outputs = {} try: for t in self.generatefiles: if isinstance(self.generatefiles[t], dict): src = self.generatefiles[t]["path"] dst = os.path.join(self.outdir, t) if os.path.dirname(self.pathmapper.reversemap(src)[1]) != self.outdir: _logger.debug("symlinking %s to %s", dst, src) os.symlink(src, dst) else: with open(os.path.join(self.outdir, t), "w") as f: f.write(self.generatefiles[t]) if self.stdin: stdin = open(self.pathmapper.mapper(self.stdin)[0], "rb") else: stdin = subprocess.PIPE if self.stdout: absout = os.path.join(self.outdir, self.stdout) dn = os.path.dirname(absout) if dn and not os.path.exists(dn): os.makedirs(dn) stdout = open(absout, "wb") else: stdout = sys.stderr sp = subprocess.Popen([str(x) for x in runtime + self.command_line], shell=False, close_fds=True, stdin=stdin, stdout=stdout, env=env, cwd=self.outdir) if stdin == subprocess.PIPE: sp.stdin.close() rcode = sp.wait() if stdin != subprocess.PIPE: stdin.close() if stdout is not sys.stderr: stdout.close() if self.successCodes and rcode in self.successCodes: processStatus = "success" elif self.temporaryFailCodes and rcode in self.temporaryFailCodes: processStatus = "temporaryFail" elif self.permanentFailCodes and rcode in self.permanentFailCodes: processStatus = "permanentFail" elif rcode == 0: processStatus = "success" else: processStatus = "permanentFail" for t in self.generatefiles: if isinstance(self.generatefiles[t], dict): src = self.generatefiles[t]["path"] dst = os.path.join(self.outdir, t) if os.path.dirname(self.pathmapper.reversemap(src)[1]) != self.outdir: os.remove(dst) os.symlink(self.pathmapper.reversemap(src)[1], dst) outputs = self.collect_outputs(self.outdir) except OSError as e: if e.errno == 2: if runtime: _logger.error("'%s' not found", runtime[0]) else: _logger.error("'%s' not found", self.command_line[0]) else: _logger.exception("Exception while running job") processStatus = "permanentFail" except WorkflowException as e: _logger.error("Error while running job: %s" % e) processStatus = "permanentFail" except Exception as e: _logger.exception("Exception while running job") processStatus = "permanentFail" if processStatus != "success": _logger.warn("[job %s] completed %s", self.name, processStatus) else: _logger.debug("[job %s] completed %s", self.name, processStatus) _logger.debug("[job %s] %s", self.name, json.dumps(outputs, indent=4)) self.output_callback(outputs, processStatus) if rm_tmpdir: _logger.debug("[job %s] Removing temporary directory %s", self.name, self.tmpdir) shutil.rmtree(self.tmpdir, True) if move_outputs and empty_subtree(self.outdir): _logger.debug("[job %s] Removing empty output directory %s", self.name, self.outdir) shutil.rmtree(self.outdir, True)
def job(self, job_order, # type: Dict[Text, Text] output_callbacks, # type: Callable[[Any, Any], Any] **kwargs # type: Any ): # type: (...) -> Generator[Union[JobBase, CallbackJob], None, None] jobname = uniquename(kwargs.get("name", shortname(self.tool.get("id", "job")))) if kwargs.get("cachedir"): cacheargs = kwargs.copy() cacheargs["outdir"] = "/out" cacheargs["tmpdir"] = "/tmp" cacheargs["stagedir"] = "/stage" cachebuilder = self._init_job(job_order, **cacheargs) cachebuilder.pathmapper = PathMapper(cachebuilder.files, kwargs["basedir"], cachebuilder.stagedir, separateDirs=False) _check_adjust = partial(check_adjust, cachebuilder) visit_class([cachebuilder.files, cachebuilder.bindings], ("File", "Directory"), _check_adjust) cmdline = flatten(list(map(cachebuilder.generate_arg, cachebuilder.bindings))) (docker_req, docker_is_req) = self.get_requirement("DockerRequirement") if docker_req and kwargs.get("use_container"): dockerimg = docker_req.get("dockerImageId") or docker_req.get("dockerPull") elif kwargs.get("default_container", None) is not None and kwargs.get("use_container"): dockerimg = kwargs.get("default_container") else: dockerimg = None if dockerimg: cmdline = ["docker", "run", dockerimg] + cmdline keydict = {u"cmdline": cmdline} if "stdout" in self.tool: keydict["stdout"] = self.tool["stdout"] for location, f in cachebuilder.pathmapper.items(): if f.type == "File": checksum = next((e['checksum'] for e in cachebuilder.files if 'location' in e and e['location'] == location and 'checksum' in e and e['checksum'] != 'sha1$hash'), None) st = os.stat(f.resolved) if checksum: keydict[f.resolved] = [st.st_size, checksum] else: keydict[f.resolved] = [st.st_size, int(st.st_mtime * 1000)] interesting = {"DockerRequirement", "EnvVarRequirement", "CreateFileRequirement", "ShellCommandRequirement"} for rh in (self.requirements, self.hints): for r in reversed(rh): if r["class"] in interesting and r["class"] not in keydict: keydict[r["class"]] = r keydictstr = json.dumps(keydict, separators=(',', ':'), sort_keys=True) cachekey = hashlib.md5(keydictstr.encode('utf-8')).hexdigest() _logger.debug("[job %s] keydictstr is %s -> %s", jobname, keydictstr, cachekey) jobcache = os.path.join(kwargs["cachedir"], cachekey) jobcachepending = jobcache + ".pending" if os.path.isdir(jobcache) and not os.path.isfile(jobcachepending): if docker_req and kwargs.get("use_container"): cachebuilder.outdir = kwargs.get("docker_outdir") or "/var/spool/cwl" else: cachebuilder.outdir = jobcache _logger.info("[job %s] Using cached output in %s", jobname, jobcache) yield CallbackJob(self, output_callbacks, cachebuilder, jobcache) return else: _logger.info("[job %s] Output of job will be cached in %s", jobname, jobcache) shutil.rmtree(jobcache, True) os.makedirs(jobcache) kwargs["outdir"] = jobcache open(jobcachepending, "w").close() def rm_pending_output_callback(output_callbacks, jobcachepending, outputs, processStatus): if processStatus == "success": os.remove(jobcachepending) output_callbacks(outputs, processStatus) output_callbacks = cast( Callable[..., Any], # known bug in mypy # https://github.com/python/mypy/issues/797 partial(rm_pending_output_callback, output_callbacks, jobcachepending)) builder = self._init_job(job_order, **kwargs) reffiles = copy.deepcopy(builder.files) j = self.makeJobRunner(**kwargs) j.builder = builder j.joborder = builder.job j.make_pathmapper = self.makePathMapper j.stdin = None j.stderr = None j.stdout = None j.successCodes = self.tool.get("successCodes") j.temporaryFailCodes = self.tool.get("temporaryFailCodes") j.permanentFailCodes = self.tool.get("permanentFailCodes") j.requirements = self.requirements j.hints = self.hints j.name = jobname debug = _logger.isEnabledFor(logging.DEBUG) if debug: _logger.debug(u"[job %s] initializing from %s%s", j.name, self.tool.get("id", ""), u" as part of %s" % kwargs["part_of"] if "part_of" in kwargs else "") _logger.debug(u"[job %s] %s", j.name, json.dumps(job_order, indent=4)) builder.pathmapper = None make_path_mapper_kwargs = kwargs if "stagedir" in make_path_mapper_kwargs: make_path_mapper_kwargs = make_path_mapper_kwargs.copy() del make_path_mapper_kwargs["stagedir"] builder.pathmapper = self.makePathMapper(reffiles, builder.stagedir, **make_path_mapper_kwargs) builder.requirements = j.requirements _check_adjust = partial(check_adjust, builder) visit_class([builder.files, builder.bindings], ("File", "Directory"), _check_adjust) initialWorkdir = self.get_requirement("InitialWorkDirRequirement")[0] j.generatefiles = {"class": "Directory", "listing": [], "basename": ""} if initialWorkdir: ls = [] # type: List[Dict[Text, Any]] if isinstance(initialWorkdir["listing"], (str, Text)): ls = builder.do_eval(initialWorkdir["listing"]) else: for t in initialWorkdir["listing"]: if "entry" in t: et = {u"entry": builder.do_eval(t["entry"])} if "entryname" in t: et["entryname"] = builder.do_eval(t["entryname"]) else: et["entryname"] = None et["writable"] = t.get("writable", False) ls.append(et) else: ls.append(builder.do_eval(t)) for i, t in enumerate(ls): if "entry" in t: if isinstance(t["entry"], string_types): ls[i] = { "class": "File", "basename": t["entryname"], "contents": t["entry"], "writable": t.get("writable") } else: if t.get("entryname") or t.get("writable"): t = copy.deepcopy(t) if t.get("entryname"): t["entry"]["basename"] = t["entryname"] t["entry"]["writable"] = t.get("writable") ls[i] = t["entry"] j.generatefiles[u"listing"] = ls for l in ls: self.updatePathmap(builder.outdir, builder.pathmapper, l) visit_class([builder.files, builder.bindings], ("File", "Directory"), _check_adjust) if debug: _logger.debug(u"[job %s] path mappings is %s", j.name, json.dumps({p: builder.pathmapper.mapper(p) for p in builder.pathmapper.files()}, indent=4)) if self.tool.get("stdin"): with SourceLine(self.tool, "stdin", validate.ValidationException, debug): j.stdin = builder.do_eval(self.tool["stdin"]) reffiles.append({"class": "File", "path": j.stdin}) if self.tool.get("stderr"): with SourceLine(self.tool, "stderr", validate.ValidationException, debug): j.stderr = builder.do_eval(self.tool["stderr"]) if os.path.isabs(j.stderr) or ".." in j.stderr: raise validate.ValidationException("stderr must be a relative path, got '%s'" % j.stderr) if self.tool.get("stdout"): with SourceLine(self.tool, "stdout", validate.ValidationException, debug): j.stdout = builder.do_eval(self.tool["stdout"]) if os.path.isabs(j.stdout) or ".." in j.stdout or not j.stdout: raise validate.ValidationException("stdout must be a relative path, got '%s'" % j.stdout) if debug: _logger.debug(u"[job %s] command line bindings is %s", j.name, json.dumps(builder.bindings, indent=4)) dockerReq = self.get_requirement("DockerRequirement")[0] if dockerReq and kwargs.get("use_container"): out_prefix = kwargs.get("tmp_outdir_prefix") j.outdir = kwargs.get("outdir") or tempfile.mkdtemp(prefix=out_prefix) tmpdir_prefix = kwargs.get('tmpdir_prefix') j.tmpdir = kwargs.get("tmpdir") or tempfile.mkdtemp(prefix=tmpdir_prefix) j.stagedir = tempfile.mkdtemp(prefix=tmpdir_prefix) else: j.outdir = builder.outdir j.tmpdir = builder.tmpdir j.stagedir = builder.stagedir inplaceUpdateReq = self.get_requirement("http://commonwl.org/cwltool#InplaceUpdateRequirement")[0] if inplaceUpdateReq: j.inplace_update = inplaceUpdateReq["inplaceUpdate"] normalizeFilesDirs(j.generatefiles) readers = {} muts = set() if builder.mutation_manager: def register_mut(f): muts.add(f["location"]) builder.mutation_manager.register_mutation(j.name, f) def register_reader(f): if f["location"] not in muts: builder.mutation_manager.register_reader(j.name, f) readers[f["location"]] = f for li in j.generatefiles["listing"]: li = cast(Dict[Text, Any], li) if li.get("writable") and j.inplace_update: adjustFileObjs(li, register_mut) adjustDirObjs(li, register_mut) else: adjustFileObjs(li, register_reader) adjustDirObjs(li, register_reader) adjustFileObjs(builder.files, register_reader) adjustFileObjs(builder.bindings, register_reader) adjustDirObjs(builder.files, register_reader) adjustDirObjs(builder.bindings, register_reader) j.environment = {} evr = self.get_requirement("EnvVarRequirement")[0] if evr: for t in evr["envDef"]: j.environment[t["envName"]] = builder.do_eval(t["envValue"]) shellcmd = self.get_requirement("ShellCommandRequirement")[0] if shellcmd: cmd = [] # type: List[Text] for b in builder.bindings: arg = builder.generate_arg(b) if b.get("shellQuote", True): arg = [shellescape.quote(a) for a in aslist(arg)] cmd.extend(aslist(arg)) j.command_line = ["/bin/sh", "-c", " ".join(cmd)] else: j.command_line = flatten(list(map(builder.generate_arg, builder.bindings))) j.pathmapper = builder.pathmapper j.collect_outputs = partial( self.collect_output_ports, self.tool["outputs"], builder, compute_checksum=kwargs.get("compute_checksum", True), jobname=jobname, readers=readers) j.output_callback = output_callbacks yield j
def job(self, joborder, output_callback, **kwargs): # type: (Dict[Text, Text], Callable[..., Any], **Any) -> Generator[Union[CommandLineJob, CallbackJob], None, None] jobname = uniquename(kwargs.get("name", shortname(self.tool.get("id", "job")))) if kwargs.get("cachedir"): cacheargs = kwargs.copy() cacheargs["outdir"] = "/out" cacheargs["tmpdir"] = "/tmp" cacheargs["stagedir"] = "/stage" cachebuilder = self._init_job(joborder, **cacheargs) cachebuilder.pathmapper = PathMapper(cachebuilder.files, kwargs["basedir"], cachebuilder.stagedir, separateDirs=False) _check_adjust = partial(check_adjust, cachebuilder) adjustFileObjs(cachebuilder.files, _check_adjust) adjustFileObjs(cachebuilder.bindings, _check_adjust) adjustDirObjs(cachebuilder.files, _check_adjust) adjustDirObjs(cachebuilder.bindings, _check_adjust) cmdline = flatten(map(cachebuilder.generate_arg, cachebuilder.bindings)) (docker_req, docker_is_req) = self.get_requirement("DockerRequirement") if docker_req and kwargs.get("use_container") is not False: dockerimg = docker_req.get("dockerImageId") or docker_req.get("dockerPull") cmdline = ["docker", "run", dockerimg] + cmdline keydict = {u"cmdline": cmdline} for _,f in cachebuilder.pathmapper.items(): if f.type == "File": st = os.stat(f.resolved) keydict[f.resolved] = [st.st_size, int(st.st_mtime * 1000)] interesting = {"DockerRequirement", "EnvVarRequirement", "CreateFileRequirement", "ShellCommandRequirement"} for rh in (self.requirements, self.hints): for r in reversed(rh): if r["class"] in interesting and r["class"] not in keydict: keydict[r["class"]] = r keydictstr = json.dumps(keydict, separators=(',',':'), sort_keys=True) cachekey = hashlib.md5(keydictstr).hexdigest() _logger.debug("[job %s] keydictstr is %s -> %s", jobname, keydictstr, cachekey) jobcache = os.path.join(kwargs["cachedir"], cachekey) jobcachepending = jobcache + ".pending" if os.path.isdir(jobcache) and not os.path.isfile(jobcachepending): if docker_req and kwargs.get("use_container") is not False: cachebuilder.outdir = kwargs.get("docker_outdir") or "/var/spool/cwl" else: cachebuilder.outdir = jobcache _logger.info("[job %s] Using cached output in %s", jobname, jobcache) yield CallbackJob(self, output_callback, cachebuilder, jobcache) return else: _logger.info("[job %s] Output of job will be cached in %s", jobname, jobcache) shutil.rmtree(jobcache, True) os.makedirs(jobcache) kwargs["outdir"] = jobcache open(jobcachepending, "w").close() def rm_pending_output_callback(output_callback, jobcachepending, outputs, processStatus): if processStatus == "success": os.remove(jobcachepending) output_callback(outputs, processStatus) output_callback = cast( Callable[..., Any], # known bug in mypy # https://github.com/python/mypy/issues/797 partial(rm_pending_output_callback, output_callback, jobcachepending)) builder = self._init_job(joborder, **kwargs) reffiles = copy.deepcopy(builder.files) j = self.makeJobRunner() j.builder = builder j.joborder = builder.job j.stdin = None j.stderr = None j.stdout = None j.successCodes = self.tool.get("successCodes") j.temporaryFailCodes = self.tool.get("temporaryFailCodes") j.permanentFailCodes = self.tool.get("permanentFailCodes") j.requirements = self.requirements j.hints = self.hints j.name = jobname _logger.debug(u"[job %s] initializing from %s%s", j.name, self.tool.get("id", ""), u" as part of %s" % kwargs["part_of"] if "part_of" in kwargs else "") _logger.debug(u"[job %s] %s", j.name, json.dumps(joborder, indent=4)) builder.pathmapper = None make_path_mapper_kwargs = kwargs if "stagedir" in make_path_mapper_kwargs: make_path_mapper_kwargs = make_path_mapper_kwargs.copy() del make_path_mapper_kwargs["stagedir"] builder.pathmapper = self.makePathMapper(reffiles, builder.stagedir, **make_path_mapper_kwargs) builder.requirements = j.requirements _logger.debug(u"[job %s] path mappings is %s", j.name, json.dumps({p: builder.pathmapper.mapper(p) for p in builder.pathmapper.files()}, indent=4)) _check_adjust = partial(check_adjust, builder) adjustFileObjs(builder.files, _check_adjust) adjustFileObjs(builder.bindings, _check_adjust) adjustDirObjs(builder.files, _check_adjust) adjustDirObjs(builder.bindings, _check_adjust) if self.tool.get("stdin"): j.stdin = builder.do_eval(self.tool["stdin"]) reffiles.append({"class": "File", "path": j.stdin}) if self.tool.get("stderr"): j.stderr = builder.do_eval(self.tool["stderr"]) if os.path.isabs(j.stderr) or ".." in j.stderr: raise validate.ValidationException("stderr must be a relative path") if self.tool.get("stdout"): j.stdout = builder.do_eval(self.tool["stdout"]) if os.path.isabs(j.stdout) or ".." in j.stdout or not j.stdout: raise validate.ValidationException("stdout must be a relative path") _logger.debug(u"[job %s] command line bindings is %s", j.name, json.dumps(builder.bindings, indent=4)) dockerReq = self.get_requirement("DockerRequirement")[0] if dockerReq and kwargs.get("use_container"): out_prefix = kwargs.get("tmp_outdir_prefix") j.outdir = kwargs.get("outdir") or tempfile.mkdtemp(prefix=out_prefix) tmpdir_prefix = kwargs.get('tmpdir_prefix') j.tmpdir = kwargs.get("tmpdir") or tempfile.mkdtemp(prefix=tmpdir_prefix) j.stagedir = tempfile.mkdtemp(prefix=tmpdir_prefix) else: j.outdir = builder.outdir j.tmpdir = builder.tmpdir j.stagedir = builder.stagedir initialWorkdir = self.get_requirement("InitialWorkDirRequirement")[0] j.generatefiles = {"class": "Directory", "listing": [], "basename": ""} if initialWorkdir: ls = [] # type: List[Dict[Text, Any]] if isinstance(initialWorkdir["listing"], (str, Text)): ls = builder.do_eval(initialWorkdir["listing"]) else: for t in initialWorkdir["listing"]: if "entry" in t: et = {u"entry": builder.do_eval(t["entry"])} if "entryname" in t: et["entryname"] = builder.do_eval(t["entryname"]) else: et["entryname"] = None et["writable"] = t.get("writable", False) ls.append(et) else: ls.append(builder.do_eval(t)) for i,t in enumerate(ls): if "entry" in t: if isinstance(t["entry"], basestring): ls[i] = { "class": "File", "basename": t["entryname"], "contents": t["entry"], "writable": t.get("writable") } else: if t["entryname"]: t = copy.deepcopy(t) t["entry"]["basename"] = t["entryname"] t["entry"]["writable"] = t.get("writable") ls[i] = t["entry"] j.generatefiles[u"listing"] = ls normalizeFilesDirs(j.generatefiles) j.environment = {} evr = self.get_requirement("EnvVarRequirement")[0] if evr: for t in evr["envDef"]: j.environment[t["envName"]] = builder.do_eval(t["envValue"]) shellcmd = self.get_requirement("ShellCommandRequirement")[0] if shellcmd: cmd = [] # type: List[Text] for b in builder.bindings: arg = builder.generate_arg(b) if b.get("shellQuote", True): arg = [shellescape.quote(a) for a in aslist(arg)] cmd.extend(aslist(arg)) j.command_line = ["/bin/sh", "-c", " ".join(cmd)] else: j.command_line = flatten(map(builder.generate_arg, builder.bindings)) j.pathmapper = builder.pathmapper j.collect_outputs = partial( self.collect_output_ports, self.tool["outputs"], builder, compute_checksum=kwargs.get("compute_checksum", True)) j.output_callback = output_callback yield j
def job(self, job_order, # type: Mapping[Text, Text] output_callbacks, # type: Callable[[Any, Any], Any] runtimeContext # RuntimeContext ): # type: (...) -> Generator[Union[JobBase, CallbackJob], None, None] workReuse, _ = self.get_requirement("WorkReuse") enableReuse = workReuse.get("enableReuse", True) if workReuse else True jobname = uniquename(runtimeContext.name or shortname(self.tool.get("id", "job"))) if runtimeContext.cachedir and enableReuse: cachecontext = runtimeContext.copy() cachecontext.outdir = "/out" cachecontext.tmpdir = "/tmp" cachecontext.stagedir = "/stage" cachebuilder = self._init_job(job_order, cachecontext) cachebuilder.pathmapper = PathMapper(cachebuilder.files, runtimeContext.basedir, cachebuilder.stagedir, separateDirs=False) _check_adjust = partial(check_adjust, cachebuilder) visit_class([cachebuilder.files, cachebuilder.bindings], ("File", "Directory"), _check_adjust) cmdline = flatten(list(map(cachebuilder.generate_arg, cachebuilder.bindings))) docker_req, _ = self.get_requirement("DockerRequirement") if docker_req is not None and runtimeContext.use_container: dockerimg = docker_req.get("dockerImageId") or docker_req.get("dockerPull") elif runtimeContext.default_container is not None and runtimeContext.use_container: dockerimg = runtimeContext.default_container else: dockerimg = None if dockerimg is not None: cmdline = ["docker", "run", dockerimg] + cmdline # not really run using docker, just for hashing purposes keydict = {u"cmdline": cmdline} for shortcut in ["stdout", "stderr"]: # later, add "stdin" if shortcut in self.tool: keydict[shortcut] = self.tool[shortcut] for location, fobj in cachebuilder.pathmapper.items(): if fobj.type == "File": checksum = next( (e['checksum'] for e in cachebuilder.files if 'location' in e and e['location'] == location and 'checksum' in e and e['checksum'] != 'sha1$hash'), None) fobj_stat = os.stat(fobj.resolved) if checksum is not None: keydict[fobj.resolved] = [fobj_stat.st_size, checksum] else: keydict[fobj.resolved] = [fobj_stat.st_size, int(fobj_stat.st_mtime * 1000)] interesting = {"DockerRequirement", "EnvVarRequirement", "CreateFileRequirement", "ShellCommandRequirement"} for rh in (self.original_requirements, self.original_hints): for r in reversed(rh): if r["class"] in interesting and r["class"] not in keydict: keydict[r["class"]] = r keydictstr = json_dumps(keydict, separators=(',', ':'), sort_keys=True) cachekey = hashlib.md5(keydictstr.encode('utf-8')).hexdigest() _logger.debug("[job %s] keydictstr is %s -> %s", jobname, keydictstr, cachekey) jobcache = os.path.join(runtimeContext.cachedir, cachekey) jobcachepending = "{}.{}.pending".format( jobcache, threading.current_thread().ident) if os.path.isdir(jobcache) and not os.path.isfile(jobcachepending): if docker_req and runtimeContext.use_container: cachebuilder.outdir = runtimeContext.docker_outdir or random_outdir() else: cachebuilder.outdir = jobcache _logger.info("[job %s] Using cached output in %s", jobname, jobcache) yield CallbackJob(self, output_callbacks, cachebuilder, jobcache) return else: _logger.info("[job %s] Output of job will be cached in %s", jobname, jobcache) shutil.rmtree(jobcache, True) os.makedirs(jobcache) runtimeContext = runtimeContext.copy() runtimeContext.outdir = jobcache open(jobcachepending, "w").close() def rm_pending_output_callback(output_callbacks, jobcachepending, outputs, processStatus): if processStatus == "success": os.remove(jobcachepending) output_callbacks(outputs, processStatus) output_callbacks = partial( rm_pending_output_callback, output_callbacks, jobcachepending) builder = self._init_job(job_order, runtimeContext) reffiles = copy.deepcopy(builder.files) j = self.make_job_runner(runtimeContext)( builder, builder.job, self.make_path_mapper, self.requirements, self.hints, jobname) j.prov_obj = self.prov_obj j.successCodes = self.tool.get("successCodes", []) j.temporaryFailCodes = self.tool.get("temporaryFailCodes", []) j.permanentFailCodes = self.tool.get("permanentFailCodes", []) debug = _logger.isEnabledFor(logging.DEBUG) if debug: _logger.debug(u"[job %s] initializing from %s%s", j.name, self.tool.get("id", ""), u" as part of %s" % runtimeContext.part_of if runtimeContext.part_of else "") _logger.debug(u"[job %s] %s", j.name, json_dumps(builder.job, indent=4)) builder.pathmapper = self.make_path_mapper( reffiles, builder.stagedir, runtimeContext, True) builder.requirements = j.requirements _check_adjust = partial(check_adjust, builder) visit_class([builder.files, builder.bindings], ("File", "Directory"), _check_adjust) initialWorkdir, _ = self.get_requirement("InitialWorkDirRequirement") if initialWorkdir is not None: ls = [] # type: List[Dict[Text, Any]] if isinstance(initialWorkdir["listing"], string_types): ls = builder.do_eval(initialWorkdir["listing"]) else: for t in initialWorkdir["listing"]: if isinstance(t, Mapping) and "entry" in t: entry_exp = builder.do_eval(t["entry"], strip_whitespace=False) for entry in aslist(entry_exp): et = {u"entry": entry} if "entryname" in t: et["entryname"] = builder.do_eval(t["entryname"]) else: et["entryname"] = None et["writable"] = t.get("writable", False) if et[u"entry"]: ls.append(et) else: initwd_item = builder.do_eval(t) if not initwd_item: continue if isinstance(initwd_item, MutableSequence): ls.extend(initwd_item) else: ls.append(initwd_item) for i, t in enumerate(ls): if "entry" in t: if isinstance(t["entry"], string_types): ls[i] = { "class": "File", "basename": t["entryname"], "contents": t["entry"], "writable": t.get("writable") } else: if t.get("entryname") or t.get("writable"): t = copy.deepcopy(t) if t.get("entryname"): t["entry"]["basename"] = t["entryname"] t["entry"]["writable"] = t.get("writable") ls[i] = t["entry"] j.generatefiles["listing"] = ls for l in ls: self.updatePathmap(builder.outdir, builder.pathmapper, l) visit_class([builder.files, builder.bindings], ("File", "Directory"), _check_adjust) if debug: _logger.debug(u"[job %s] path mappings is %s", j.name, json_dumps({p: builder.pathmapper.mapper(p) for p in builder.pathmapper.files()}, indent=4)) if self.tool.get("stdin"): with SourceLine(self.tool, "stdin", validate.ValidationException, debug): j.stdin = builder.do_eval(self.tool["stdin"]) assert j.stdin is not None reffiles.append({"class": "File", "path": j.stdin}) if self.tool.get("stderr"): with SourceLine(self.tool, "stderr", validate.ValidationException, debug): j.stderr = builder.do_eval(self.tool["stderr"]) assert j.stderr is not None if os.path.isabs(j.stderr) or ".." in j.stderr: raise validate.ValidationException( "stderr must be a relative path, got '%s'" % j.stderr) if self.tool.get("stdout"): with SourceLine(self.tool, "stdout", validate.ValidationException, debug): j.stdout = builder.do_eval(self.tool["stdout"]) assert j.stdout is not None if os.path.isabs(j.stdout) or ".." in j.stdout or not j.stdout: raise validate.ValidationException( "stdout must be a relative path, got '%s'" % j.stdout) if debug: _logger.debug(u"[job %s] command line bindings is %s", j.name, json_dumps(builder.bindings, indent=4)) dockerReq, _ = self.get_requirement("DockerRequirement") if dockerReq is not None and runtimeContext.use_container: out_dir, out_prefix = os.path.split( runtimeContext.tmp_outdir_prefix) j.outdir = runtimeContext.outdir or \ tempfile.mkdtemp(prefix=out_prefix, dir=out_dir) tmpdir_dir, tmpdir_prefix = os.path.split( runtimeContext.tmpdir_prefix) j.tmpdir = runtimeContext.tmpdir or \ tempfile.mkdtemp(prefix=tmpdir_prefix, dir=tmpdir_dir) j.stagedir = tempfile.mkdtemp(prefix=tmpdir_prefix, dir=tmpdir_dir) else: j.outdir = builder.outdir j.tmpdir = builder.tmpdir j.stagedir = builder.stagedir inplaceUpdateReq, _ = self.get_requirement("InplaceUpdateRequirement") if inplaceUpdateReq is not None: j.inplace_update = inplaceUpdateReq["inplaceUpdate"] normalizeFilesDirs(j.generatefiles) readers = {} # type: Dict[Text, Any] muts = set() # type: Set[Text] if builder.mutation_manager is not None: def register_mut(f): muts.add(f["location"]) builder.mutation_manager.register_mutation(j.name, f) def register_reader(f): if f["location"] not in muts: builder.mutation_manager.register_reader(j.name, f) readers[f["location"]] = copy.deepcopy(f) for li in j.generatefiles["listing"]: li = cast(Dict[Text, Any], li) if li.get("writable") and j.inplace_update: adjustFileObjs(li, register_mut) adjustDirObjs(li, register_mut) else: adjustFileObjs(li, register_reader) adjustDirObjs(li, register_reader) adjustFileObjs(builder.files, register_reader) adjustFileObjs(builder.bindings, register_reader) adjustDirObjs(builder.files, register_reader) adjustDirObjs(builder.bindings, register_reader) timelimit, _ = self.get_requirement("TimeLimit") if timelimit is not None: with SourceLine(timelimit, "timelimit", validate.ValidationException, debug): j.timelimit = builder.do_eval(timelimit["timelimit"]) if not isinstance(j.timelimit, int) or j.timelimit < 0: raise Exception("timelimit must be an integer >= 0, got: %s" % j.timelimit) networkaccess, _ = self.get_requirement("NetworkAccess") if networkaccess is not None: with SourceLine(networkaccess, "networkAccess", validate.ValidationException, debug): j.networkaccess = builder.do_eval(networkaccess["networkAccess"]) if not isinstance(j.networkaccess, bool): raise Exception("networkAccess must be a boolean, got: %s" % j.networkaccess) j.environment = {} evr, _ = self.get_requirement("EnvVarRequirement") if evr is not None: for t in evr["envDef"]: j.environment[t["envName"]] = builder.do_eval(t["envValue"]) shellcmd, _ = self.get_requirement("ShellCommandRequirement") if shellcmd is not None: cmd = [] # type: List[Text] for b in builder.bindings: arg = builder.generate_arg(b) if b.get("shellQuote", True): arg = [shellescape.quote(a) for a in aslist(arg)] cmd.extend(aslist(arg)) j.command_line = ["/bin/sh", "-c", " ".join(cmd)] else: j.command_line = flatten(list(map(builder.generate_arg, builder.bindings))) j.pathmapper = builder.pathmapper j.collect_outputs = partial( self.collect_output_ports, self.tool["outputs"], builder, compute_checksum=getdefault(runtimeContext.compute_checksum, True), jobname=jobname, readers=readers) j.output_callback = output_callbacks yield j
def run(self, dry_run=False, pull_image=True, rm_container=True, rm_tmpdir=True, move_outputs="move", **kwargs): # type: (bool, bool, bool, bool, bool, Text, **Any) -> Union[Tuple[Text, Dict[None, None]], None] if not os.path.exists(self.outdir): os.makedirs(self.outdir) #with open(os.path.join(outdir, "cwl.input.json"), "w") as fp: # json.dump(self.joborder, fp) runtime = [] # type: List[Text] (docker_req, docker_is_req) = get_feature(self, "DockerRequirement") for knownfile in self.pathmapper.files(): p = self.pathmapper.mapper(knownfile) if p.type == "File" and not os.path.isfile(p[0]): raise WorkflowException( u"Input file %s (at %s) not found or is not a regular " "file." % (knownfile, self.pathmapper.mapper(knownfile)[0])) img_id = None env = None # type: Union[MutableMapping[Text, Text], MutableMapping[str, str]] if docker_req and kwargs.get("use_container") is not False: env = os.environ img_id = docker.get_from_requirements(docker_req, docker_is_req, pull_image) if docker_is_req and img_id is None: raise WorkflowException("Docker is required for running this tool.") if img_id: runtime = ["docker", "run", "-i"] for src in self.pathmapper.files(): vol = self.pathmapper.mapper(src) if vol.type == "File": runtime.append(u"--volume=%s:%s:ro" % (vol.resolved, vol.target)) if vol.type == "CreateFile": createtmp = os.path.join(self.stagedir, os.path.basename(vol.target)) with open(createtmp, "w") as f: f.write(vol.resolved.encode("utf-8")) runtime.append(u"--volume=%s:%s:ro" % (createtmp, vol.target)) runtime.append(u"--volume=%s:%s:rw" % (os.path.realpath(self.outdir), "/var/spool/cwl")) runtime.append(u"--volume=%s:%s:rw" % (os.path.realpath(self.tmpdir), "/tmp")) runtime.append(u"--workdir=%s" % ("/var/spool/cwl")) runtime.append("--read-only=true") if (kwargs.get("enable_net", None) is None and kwargs.get("custom_net", None) is not None): runtime.append("--net=none") elif kwargs.get("custom_net", None) is not None: runtime.append("--net={0}".format(kwargs.get("custom_net"))) if self.stdout: runtime.append("--log-driver=none") euid = docker_vm_uid() or os.geteuid() runtime.append(u"--user=%s" % (euid)) if rm_container: runtime.append("--rm") runtime.append("--env=TMPDIR=/tmp") # spec currently says "HOME must be set to the designated output # directory." but spec might change to designated temp directory. # runtime.append("--env=HOME=/tmp") runtime.append("--env=HOME=/var/spool/cwl") for t,v in self.environment.items(): runtime.append(u"--env=%s=%s" % (t, v)) runtime.append(img_id) else: env = self.environment if not os.path.exists(self.tmpdir): os.makedirs(self.tmpdir) vars_to_preserve = kwargs.get("preserve_environment") if vars_to_preserve is not None: for key, value in os.environ.items(): if key in vars_to_preserve and key not in env: env[key] = value env["HOME"] = self.outdir env["TMPDIR"] = self.tmpdir stageFiles(self.pathmapper, os.symlink) stdin = None # type: Union[IO[Any], int] stderr = None # type: IO[Any] stdout = None # type: IO[Any] scr, _ = get_feature(self, "ShellCommandRequirement") if scr: shouldquote = lambda x: False else: shouldquote = needs_shell_quoting_re.search _logger.info(u"[job %s] %s$ %s%s%s%s", self.name, self.outdir, " \\\n ".join([shellescape.quote(Text(arg)) if shouldquote(Text(arg)) else Text(arg) for arg in (runtime + self.command_line)]), u' < %s' % self.stdin if self.stdin else '', u' > %s' % os.path.join(self.outdir, self.stdout) if self.stdout else '', u' 2> %s' % os.path.join(self.outdir, self.stderr) if self.stderr else '') if dry_run: return (self.outdir, {}) outputs = {} # type: Dict[Text,Text] try: if self.generatefiles["listing"]: generatemapper = PathMapper([self.generatefiles], self.outdir, self.outdir, separateDirs=False) _logger.debug(u"[job %s] initial work dir %s", self.name, json.dumps({p: generatemapper.mapper(p) for p in generatemapper.files()}, indent=4)) def linkoutdir(src, tgt): # Need to make the link to the staged file (may be inside # the container) for _, item in self.pathmapper.items(): if src == item.resolved: os.symlink(item.target, tgt) break stageFiles(generatemapper, linkoutdir) if self.stdin: stdin = open(self.pathmapper.reversemap(self.stdin)[1], "rb") else: stdin = subprocess.PIPE if self.stderr: abserr = os.path.join(self.outdir, self.stderr) dnerr = os.path.dirname(abserr) if dnerr and not os.path.exists(dnerr): os.makedirs(dnerr) stderr = open(abserr, "wb") else: stderr = sys.stderr if self.stdout: absout = os.path.join(self.outdir, self.stdout) dn = os.path.dirname(absout) if dn and not os.path.exists(dn): os.makedirs(dn) stdout = open(absout, "wb") else: stdout = sys.stderr sp = subprocess.Popen([Text(x).encode('utf-8') for x in runtime + self.command_line], shell=False, close_fds=True, stdin=stdin, stderr=stderr, stdout=stdout, env=env, cwd=self.outdir) if sp.stdin: sp.stdin.close() rcode = sp.wait() if isinstance(stdin, file): stdin.close() if stderr is not sys.stderr: stderr.close() if stdout is not sys.stderr: stdout.close() if self.successCodes and rcode in self.successCodes: processStatus = "success" elif self.temporaryFailCodes and rcode in self.temporaryFailCodes: processStatus = "temporaryFail" elif self.permanentFailCodes and rcode in self.permanentFailCodes: processStatus = "permanentFail" elif rcode == 0: processStatus = "success" else: processStatus = "permanentFail" if self.generatefiles["listing"]: def linkoutdir(src, tgt): # Need to make the link to the staged file (may be inside # the container) if os.path.islink(tgt): os.remove(tgt) os.symlink(src, tgt) stageFiles(generatemapper, linkoutdir, ignoreWritable=True) outputs = self.collect_outputs(self.outdir) except OSError as e: if e.errno == 2: if runtime: _logger.error(u"'%s' not found", runtime[0]) else: _logger.error(u"'%s' not found", self.command_line[0]) else: _logger.exception("Exception while running job") processStatus = "permanentFail" except WorkflowException as e: _logger.error(u"Error while running job: %s" % e) processStatus = "permanentFail" except Exception as e: _logger.exception("Exception while running job") processStatus = "permanentFail" if processStatus != "success": _logger.warn(u"[job %s] completed %s", self.name, processStatus) else: _logger.debug(u"[job %s] completed %s", self.name, processStatus) _logger.debug(u"[job %s] %s", self.name, json.dumps(outputs, indent=4)) self.output_callback(outputs, processStatus) if self.stagedir and os.path.exists(self.stagedir): _logger.debug(u"[job %s] Removing input staging directory %s", self.name, self.stagedir) shutil.rmtree(self.stagedir, True) if rm_tmpdir: _logger.debug(u"[job %s] Removing temporary directory %s", self.name, self.tmpdir) shutil.rmtree(self.tmpdir, True) if move_outputs == "move" and empty_subtree(self.outdir): _logger.debug(u"[job %s] Removing empty output directory %s", self.name, self.outdir) shutil.rmtree(self.outdir, True)
def _print_simple(self, name, value): self._stream.write( _SHELL_VAR_PREFIX + '%s=%s\n' % ( name.upper(), shellescape.quote(value))) self._stream.flush()