def remove_comments(tex, cwd): """ """ cmd = sarge.shell_format('latexpand --keep-includes {0} -o {0}_new', tex) RUN_CMD(cmd, cwd=cwd) cmd = sarge.shell_format('mv {0}_new {0}', tex) RUN_CMD(cmd, cwd=cwd)
def remove_cref(tex, cwd): """ """ sty_file = cwd+'/dynlearn.sty' # add poorman to cleveref import cref_regex1 = (r'usepackage\{cleveref\}', r'usepackage[poorman]{cleveref}') cref_regex2 = (r'usepackage\[(\w*)\]\{cleveref\}', r'usepackage[\g<1>,poorman]{cleveref}') with open(sty_file) as dynlearn: source = dynlearn.read() for exp, repl in [cref_regex1, cref_regex2]: source = re.sub(exp, repl, source) with open(sty_file, 'w') as dynlearn: dynlearn.write(source) # generate and run the sed script compile_tex(tex, cwd) sed_cmd = sarge.shell_format("sed -f {0}.sed {0}.tex > {0}.tex.temp", tex.split('.')[0]) sarge.run(sed_cmd, cwd=cwd) sarge.run(sarge.shell_format("mv {0}.temp {0}", tex), cwd=cwd) # remove the cleveref import cref_regex3 = r'\\usepackage\[.*\]\{cleveref\}' comment_lines(sty_file, [cref_regex3])
def play(playbook, verbosity=0): if verbosity > 0: verb = "-{0}".format('v' * verbosity) cmd = shell_format('ansible-playbook {0} {1}.yml', verb, playbook) else: cmd = shell_format('ansible-playbook {0}.yml', playbook) if run(cmd, cwd=ansible_root).returncode > 0: click.echo('Operation failed.') return False else: return True
def generate_password(self): """Generates an org password with the sfdx utility. """ if self.password_failed: self.logger.warn( "Skipping resetting password since last attempt failed") return # Set a random password so it's available via cci org info command = sarge.shell_format( "sfdx force:user:password:generate -u {0}", self.username) self.logger.info( "Generating scratch org user password with command {}".format( command)) p = sarge.Command( command, stdout=sarge.Capture(buffer_size=-1), stderr=sarge.Capture(buffer_size=-1), shell=True, ) p.run() stdout = [] for line in p.stdout: stdout.append(line) stderr = [] for line in p.stderr: stderr.append(line) if p.returncode: self.config["password_failed"] = True # Don't throw an exception because of failure creating the # password, just notify in a log message self.logger.warn("Failed to set password: \n{}\n{}".format( "\n".join(stdout), "\n".join(stderr)))
def check_google_dmarc(domain): """ Check a Google DMARC DNS TXT record Args: domain (str): DNS domain name Returns: int: 0 if OK, 1 on error | https://support.google.com/a/answer/2466580 | https://support.google.com/a/answer/2466563 """ dmarc_domain = "_dmarc." + domain cmd = sarge.shell_format("dig {0} txt +short", dmarc_domain) log.info('cmd', op='check_google_dmarc', cmd=cmd) proc = sarge.capture_both(cmd) output = proc.stdout.text.rstrip().split('\n') for line in output: log.debug('TXT', record=line) expected = u"\"v=DMARC1" # ... "\; p=none\; rua=mailto:" if line.startswith(expected): if 'p=' in line: return 0 errmsg = "%r != %r" % (output, expected) log.error('err', msg=errmsg) return 1
def check_google_spf(domain): """ Check a Google SPF DNS TXT record Args: domain (str): DNS domain name Returns: int: 0 if OK, 1 on error | https://support.google.com/a/answer/178723?hl=en """ cmd = sarge.shell_format("dig {0} txt +short", domain) log.info('cmd', op='check_google_spf', cmd=cmd) proc = sarge.capture_both(cmd) output = proc.stdout.text.rstrip().split('\n') for line in output: log.debug('TXT', record=line) expected = u"\"v=spf1 include:_spf.google.com ~all\"" if line == expected: return 0 errmsg = "%r != %r" % (output, expected) log.error('err', msg=errmsg) return 1
def check_google_mx(domain): """ Check Google MX DNS records Args: domain (str): DNS domain name Returns: int: 0 if OK, 1 on error | https://support.google.com/a/topic/2716885?hl=en&ref_topic=2426592 """ cmd = sarge.shell_format("dig {0} mx +short", domain) log.info('cmd', cmd=cmd) output = sarge.capture_both(cmd).stdout.text.rstrip() log.debug('MX', record=output) result = 0 check_domain1 = "aspmx.l.google.com." check_domain2 = "googlemail.com." lines = output.split('\n') if not lines: log.error('err', msg="No MX records found for %r" % domain) result += 1 for l in lines: l = l.lower() if not (l.endswith(check_domain1) or l.endswith(check_domain2)): result += 1 log.error('err', msg="%r does not end with %r or %r" % (l, check_domain1, check_domain2)) if result is None: result += 1 return result
def delete_org(self): """ Uses sfdx force:org:delete to delete the org """ if not self.created: self.logger.info( "Skipping org deletion: the scratch org has not been created") return command = sarge.shell_format("sfdx force:org:delete -p -u {0}", self.username) self.logger.info( "Deleting scratch org with command {}".format(command)) p = sarge.Command(command, stdout=sarge.Capture(buffer_size=-1), shell=True) p.run() org_info = None stdout = [] for line in p.stdout: stdout.append(line) if line.startswith("An error occurred deleting this org"): self.logger.error(line) else: self.logger.info(line) if p.returncode: message = "Failed to delete scratch org: \n{}".format( "".join(stdout)) raise ScratchOrgException(message) # Flag that this org has been created self.config["created"] = False self.config["username"] = None
def exposed_rmrf(self, dir_): """ classid 'rm -rf' """ logger.debug("Removing directory {0}".format(dir_)) return run_command_forget_output( shell_format('rm -rf {0}', dir_), cwd=self.cwd)
def remove_cref(tex, cwd): """ """ sty_file = cwd+'/dynlearn.sty' # add poorman to cleveref import cref_regex1 = (r'usepackage\{cleveref\}', r'usepackage[poorman]{cleveref}') cref_regex2 = (r'usepackage\[(\w*)\]\{cleveref\}', r'usepackage[\g<1>,poorman]{cleveref}') with open(sty_file) as dynlearn: source = dynlearn.read() for exp, repl in [cref_regex1, cref_regex2]: source = re.sub(exp, repl, source) with open(sty_file, 'w') as dynlearn: dynlearn.write(source) # generate and run the sed script compile_tex(tex, cwd) sed_cmd = sarge.shell_format("sed -f {0}.sed {0}.tex", tex.split('.')[0]) cleaned = RUN_CMD(sed_cmd, cwd=cwd, stdout=sarge.Capture()) with (pathlib.Path(cwd) / tex).open('w') as output: output.write(cleaned.stdout.text) # remove the cleveref import cref_regex3 = r'\\usepackage\[.*\]\{cleveref\}' comment_lines(sty_file, [cref_regex3])
def carto_css(mml, name): """ Takes MML string input and writes it to a Mapnik XML file. :param mml: an mml string, containing the proper CartoCSS styling and connection attributes required by the CartoCSS conversion program :param name: the unique name of the layer (standard method is to name it with its database schema and table name) :return mapfile: a cascadenik-ready document. """ from sarge import shell_format, run, Capture create_static_content_subdir('cartocss') mml_file = "{0}/cartocss/{1}.mml".format(settings.MEDIA_ROOT, name) xml_file = mml_file.replace(".mml", ".xml") f = open(mml_file, 'w+') f.write(mml) f.close() if not settings.FOOTPRINT_INIT: carto_css_command = shell_format("{0}/carto {1}".format(settings.BIN_DIR, mml_file)) logger.debug("Running carto: %s" % carto_css_command) carto_css_content = None try: carto_result = capture_both(carto_css_command) assert not any(carto_result.returncodes) carto_css_content = carto_result.stdout.text logger.debug("Carto xml content: %s" % carto_css_command) f = open(xml_file, 'w') f.write(carto_css_content) f.close() except AssertionError, e: logger.error("Failed to generate cartocss for {mml}. Exception: {message}. {carto_output}".format( mml=mml_file, message=carto_result.stderr.text, carto_output=carto_css_content)) raise e
def create_org(self): """ Uses sfdx force:org:create to create the org """ if not self.config_file: # FIXME: raise exception return if not self.scratch_org_type: self.config["scratch_org_type"] = "workspace" options = { "config_file": self.config_file, "devhub": " --targetdevhubusername {}".format(self.devhub) if self.devhub else "", "namespaced": " -n" if not self.namespaced else "", "days": " --durationdays {}".format(self.days) if self.days else "", "alias": sarge.shell_format(' -a "{0!s}"', self.sfdx_alias) if self.sfdx_alias else "", "extraargs": os.environ.get("SFDX_ORG_CREATE_ARGS", ""), } # This feels a little dirty, but the use cases for extra args would mostly # work best with env vars command = "sfdx force:org:create -f {config_file}{devhub}{namespaced}{days}{alias} {extraargs}".format( **options) self.logger.info( "Creating scratch org with command {}".format(command)) p = sarge.Command(command, stdout=sarge.Capture(buffer_size=-1), shell=True) p.run() re_obj = re.compile( "Successfully created scratch org: (.+), username: (.+)") stdout = [] for line in p.stdout: match = re_obj.search(line) if match: self.config["org_id"] = match.group(1) self.config["username"] = match.group(2) stdout.append(line) self.logger.info(line) self.config["date_created"] = datetime.datetime.now() if p.returncode: message = "{}: \n{}".format(FAILED_TO_CREATE_SCRATCH_ORG, "".join(stdout)) raise ScratchOrgException(message) if self.config.get("set_password"): self.generate_password() # Flag that this org has been created self.config["created"] = True
def exposed_mkdir(self, dir_): """ classic 'mkdir' """ logger.debug("Creating directory {0}".format(dir_)) formated = shell_format('mkdir {0}', dir_) logger.debug("Creating formated {0}".format(formated)) return run_command_forget_output(formated, cwd=self.cwd)
def test_get_git_config(self, Command): Command.return_value = p = mock.Mock( stdout=io.BytesIO(b"*****@*****.**"), stderr=io.BytesIO(b""), returncode=0) assert "*****@*****.**" == utils.get_git_config("user.email") assert (sarge.shell_format('git config --get "{0!s}"', "user.email") == Command.call_args[0][0]) p.run.assert_called_once()
def create_and_populate(tex): """ """ cwd = pathlib.Path().cwd() pdir = (cwd / DIRECTORY_NAME) if pdir.exists(): LOG.warn('Removing existing {} directory'.format(DIRECTORY_NAME)) cmd = sarge.shell_format("rm -rf {0}", DIRECTORY_NAME) sarge.run(cmd, cwd=str(cwd)) pdir.mkdir() files = TEMPLATE_FILES + find_bib_files(tex) + [tex] for filename in files: cmd = sarge.shell_format("cp {0} {1}", filename, DIRECTORY_NAME) sarge.run(cmd, cwd=str(cwd)) return str(pdir)
def _extract_srpm_contents(srpm_path, target_dir): """Extract an SRPM package using rpm2cpio / cpio into `target_dir`""" old_paths = set(target_dir.iterdir()) raw_command = 'rpm2cpio {} | cpio -idm > /dev/null 2>&1' command = sarge.shell_format(raw_command, srpm_path) log.debug("Extracting SRPM contents: %r", command) sarge.run(command, cwd=str(target_dir)) new_paths = sorted(set(target_dir.iterdir()) - old_paths) log.debug("Extracted files: %s", new_paths) return new_paths
def sfdx( command, username=None, log_note=None, access_token=None, args=None, env=None, capture_output=True, check_return=False, ): """Call an sfdx command and capture its output. Be sure to quote user input that is part of the command using `sarge.shell_format`. Returns a `sarge` Command instance with returncode, stdout, stderr """ command = "sfdx {}".format(command) if args is not None: for arg in args: command += " " + sarge.shell_quote(arg) if username: command += sarge.shell_format(" -u {0}", username) if log_note: logger.info("{} with command: {}".format(log_note, command)) # Avoid logging access token if access_token: command += sarge.shell_format(" -u {0}", access_token) p = sarge.Command( command, stdout=sarge.Capture(buffer_size=-1) if capture_output else None, stderr=sarge.Capture(buffer_size=-1) if capture_output else None, shell=True, env=env, ) p.run() if capture_output: p.stdout_text = io.TextIOWrapper(p.stdout, encoding=sys.stdout.encoding) p.stderr_text = io.TextIOWrapper(p.stderr, encoding=sys.stdout.encoding) if check_return and p.returncode: raise Exception(f"Command exited with return code {p.returncode}") return p
def set_default_org(self, name): """ set the default org for tasks and flows by name """ org = self.get_org(name) self.unset_default_org() org.config["default"] = True org.save() if org.created: sfdx( sarge.shell_format("force:config:set defaultusername={}", org.sfdx_alias))
def get_git_config(config_key): p = sarge.Command( sarge.shell_format('git config --get "{0!s}"', config_key), stderr=sarge.Capture(buffer_size=-1), stdout=sarge.Capture(buffer_size=-1), shell=True, ) p.run() config_value = io.TextIOWrapper(p.stdout).read().strip() return config_value if config_value and not p.returncode else None
def test_get_git_config(self, Command): Command.return_value = p = mock.Mock( stdout=io.BytesIO(b"*****@*****.**"), stderr=io.BytesIO(b""), returncode=0 ) self.assertEqual("*****@*****.**", utils.get_git_config("user.email")) self.assertEqual( sarge.shell_format('git config --get "{0!s}"', "user.email"), Command.call_args[0][0], ) p.run.assert_called_once()
def check_multiple_alts(chrom, start, end, alt, samples, reference_file=None): # FIXME: PyVCF can't be used as it loads the wrong coordinate with # fetch() if reference_file is None: return alt alt = alt.split(",") if len(alt) == 1: return alt[0] region = "{0}:{1}-{2}".format(chrom, start, end) sample_display = ",".join(samples) bcftools = ("/usr/bin/bcftools view {0} -r {1} -a -s {2}" " --exclude-uncalled -H") bcftools = sarge.shell_format(bcftools, reference_file, region, sample_display) command = sarge.capture_both(bcftools) mutation_table = pd.read_table(command.stdout, header=None, names=["CHROM", "POS", "ID", "REF", "ALT"], usecols=["CHROM", "POS", "ID", "REF", "ALT"]) if mutation_table.empty: # Try alternative approach: sometimes bcftools asserts when # launched from sarge import subprocess with open(os.devnull) as null: cmd = subprocess.Popen(bcftools, shell=True, stdout=subprocess.PIPE, stderr=null) mutation_table = pd.read_table(cmd.stdout, header=None, names=["CHROM", "POS", "ID", "REF", "ALT"], usecols=["CHROM", "POS", "ID", "REF", "ALT"]) seen_alt = mutation_table["ALT"].item() if len(seen_alt.split(",")) > 1: message = ("Detected more than one allele for sample pair {}: {}," " mutation position {}-{}, ref {}") print(message.format(", ".join(samples), seen_alt, chrom, start, mutation_table["REF"].item()), file=sys.stderr) # This is a false positive! Same locus but different ALTs return return seen_alt
def set_default_org(self, name): """ set the default org for tasks by name key """ org = self.get_org(name) self.unset_default_org() org.config["default"] = True self.set_org(org) if org.created: sfdx( sarge.shell_format( "force:config:set defaultusername={}", org.sfdx_alias ) )
def test_generate_output(self): os.makedirs('.git') os.makedirs('test_dir') with tempfile.TemporaryDirectory(prefix='test') as tmp_test,\ tempfile.TemporaryDirectory(prefix='d2g_') as tmp: generate_output(sarge.shell_format('cp -r test_dir {0}', tmp_test), tmp, []) self.assertTrue(os.path.exists(os.path.join(tmp_test, 'test_dir'))) self.assertFalse(os.path.exists(os.path.join(tmp_test, '.git')))
def whois(domain): """ Get whois information with whois Args: domain (str): DNS domain Returns: str: whois output """ cmd = sarge.shell_format('whois {0}', domain) log.info('cmd', cmd=cmd) output = sarge.capture_both(cmd) return output
def create_org(self): """ Uses sfdx force:org:create to create the org """ if not self.config_file: # FIXME: raise exception return if not self.scratch_org_type: self.config['scratch_org_type'] = 'workspace' options = { 'config_file': self.config_file, 'devhub': ' --targetdevhubusername {}'.format(self.devhub) if self.devhub else '', 'namespaced': ' -n' if not self.namespaced else '', 'days': ' --durationdays {}'.format(self.days) if self.days else '', 'alias': sarge.shell_format(' -a "{0!s}"', self.sfdx_alias) if self.sfdx_alias else '', 'extraargs': os.environ.get('SFDX_ORG_CREATE_ARGS', ''), } # This feels a little dirty, but the use cases for extra args would mostly # work best with env vars command = 'sfdx force:org:create -f {config_file}{devhub}{namespaced}{days}{alias} {extraargs}'.format(**options) self.logger.info( 'Creating scratch org with command {}'.format(command)) p = sarge.Command(command, stdout=sarge.Capture(buffer_size=-1), shell=True) p.run() re_obj = re.compile( 'Successfully created scratch org: (.+), username: (.+)') stdout = [] for line in p.stdout: match = re_obj.search(line) if match: self.config['org_id'] = match.group(1) self.config['username'] = match.group(2) stdout.append(line) self.logger.info(line) self.config['date_created'] = datetime.datetime.now() if p.returncode: message = '{}: \n{}'.format( FAILED_TO_CREATE_SCRATCH_ORG, ''.join(stdout), ) raise ScratchOrgException(message) if self.config.get('set_password'): self.generate_password() # Flag that this org has been created self.config['created'] = True
def dig_txt(domain): """ Get DNS TXT records with dig Args: domain (str): DNS domain Returns: str: dig output """ cmd = sarge.shell_format( "dig {0} txt +cmd +nocomments +question +noidentify +nostats", domain) log.info('cmd', cmd=cmd) output = sarge.capture_both(cmd) return output
def start_python_process(component_name, run_in_fg, py_path, program_dir, on_keyboard_interrupt=None, failed_to_start_err=-100, extra_options=None, stderr_path=None, stdin_data=None): """ Starts a new process from a given Python path, either in background or foreground (run_in_fg). """ options = { 'fg': run_in_fg, } if extra_options: options.update(extra_options) options = CLI_ARG_SEP.join('{}={}'.format(k, v) for k, v in options.items()) py_path_option = shell_format('-m {0}', py_path) program_dir_option = shell_format('{0}', program_dir) if program_dir else '' extra_cli_options = '{} {} {}'.format(py_path_option, program_dir_option, options) extra_cli_options = '{} '.format(py_path_option) if program_dir_option: extra_cli_options += '{} '.format(program_dir_option) extra_cli_options += '{}'.format(options) return start_process(component_name, get_executable(), run_in_fg, None, extra_cli_options, on_keyboard_interrupt, failed_to_start_err, extra_options, stderr_path, stdin_data)
def dig_dnskey(zone): """ Get DNSSEC DNS records with dig Args: zone (str): DNS zone Returns: str: dig output """ cmd = sarge.shell_format( "dig {0} +dnssec dnskey +cmd +nocomments +question +noidentify +nostats", zone) log.info('cmd', cmd=cmd) output = sarge.capture_both(cmd) return output
def create_and_populate(tex): """ """ cwd = pathlib.Path().cwd() pdir = (cwd / DIRECTORY_NAME) if pdir.exists(): LOG.warn('Removing existing {} directory'.format(DIRECTORY_NAME)) cmd = sarge.shell_format("rm -rf {0}", DIRECTORY_NAME) RUN_CMD(cmd, cwd=str(cwd)) pdir.mkdir() files = TEMPLATE_FILES + find_bib_files(tex) + [tex] for filename in files: if pathlib.Path(filename).exists(): cmd = sarge.shell_format("cp {0} {1}", filename, DIRECTORY_NAME) RUN_CMD(cmd, cwd=str(cwd)) # inline inputs cmd = sarge.shell_format('latexpand --keep-comments {0} -o {1}/{0}', tex, str(pdir)) RUN_CMD(cmd, cwd=str(cwd)) return str(pdir)
def callScript(self, panValue, tiltValue): self.panValue = max( self._settings.get(["pan", "minValue"]), min(self._settings.get(["pan", "maxValue"]), panValue)) self.tiltValue = max( self._settings.get(["tilt", "minValue"]), min(self._settings.get(["tilt", "maxValue"]), tiltValue)) # if there are anly pantilt handlers, loop through them, then return if len(self.pantiltHandlers) > 0: values = { 'pan': self.panValue, 'panMin': self._settings.get(["pan", "minValue"]), 'panMax': self._settings.get(["pan", "maxValue"]), 'tilt': self.tiltValue, 'tiltMin': self._settings.get(["tilt", "minValue"]), 'tiltMax': self._settings.get(["tilt", "maxValue"]) } for name, handler in self.pantiltHandlers.items(): handler(values) return script = self._settings.get(["pathToScript"]) if script == "": return self._logger.info("Performing command: {}".format(script)) try: # we run this with shell=True since we have to trust whatever # our admin configured as command and since we want to allow # shell-alike handling here... p = sarge.run(sarge.shell_format('{0} {1} {2}', script, panValue, tiltValue), stdout=sarge.Capture(), stderr=sarge.Capture(), shell=True, async=False) if p.returncode != 0: returncode = p.returncode stdout_text = p.stdout.text stderr_text = p.stderr.text error = "Command failed with return code {}:\nSTDOUT: {}\nSTDERR: {}".format( returncode, stdout_text, stderr_text) self._logger.warn(error) except Exception, e: error = "Command failed: {}".format(str(e)) self._logger.warn(error)
def nslookup(domain, nameserver=''): """ Get nslookup information with nslookup (resolve a domainname to an IP) Args: domain (str): DNS domain nameserver (str): DNS domain name server to query (default: ``''``) Returns: str: nslookup output """ if not domain.endswith('.'): domain = domain + '.' cmd = sarge.shell_format('nslookup {0} {1}', domain, nameserver) log.info('cmd', cmd=cmd) output = sarge.capture_both(cmd) return output
def ping(host, count=5, deadline=5): """ ICMP ECHO ping a host (with ``ping``) Args: host (str): hostname or IP count (int): number of packets to send (``-c``) deadline (int): timeout in seconds (``-w``) Returns: :py:class:`PingSummary`: parsed ping summary """ cmd = sarge.shell_format("""ping -q -c {0} -w {1} -i 0.2 {2}""", count, deadline, host) p = sarge.capture_stdout(cmd) return _parse_ping_output(TextIOWrapper(p.stdout))
def dig_mx(domain): """ Get MX DNS records with dig Args: domain (str): DNS domain Returns: str: dig output | https://en.wikipedia.org/wiki/MX_record """ cmd = sarge.shell_format( "dig {0} mx +cmd +nocomments +question +noidentify +nostats", domain) log.info('cmd', cmd=cmd) output = sarge.capture_both(cmd) return output
def ping(host, count=5, deadline=5): """ ICMP ECHO ping a host (with ``ping``) Args: host (str): hostname or IP count (int): number of packets to send (``-c``) deadline (int): timeout in seconds (``-w``) Returns: :py:class:`PingSummary`: parsed ping summary """ cmd = sarge.shell_format( """ping -q -c {0} -w {1} -i 0.2 {2}""", count, deadline, host) p = sarge.capture_stdout(cmd) return _parse_ping_output(TextIOWrapper(p.stdout))
def force_refresh_oauth_token(self): # Call force:org:display and parse output to get instance_url and # access_token command = sarge.shell_format('sfdx force:org:open -r -u {0}', self.username) self.logger.info( 'Refreshing OAuth token with command: {}'.format(command)) p = sarge.Command(command, stdout=sarge.Capture(buffer_size=-1), shell=True) p.run() stdout_list = [] for line in p.stdout: stdout_list.append(line.strip()) if p.returncode: self.logger.error('Return code: {}'.format(p.returncode)) for line in stdout_list: self.logger.error(line) message = 'Message: {}'.format('\n'.join(stdout_list)) raise ScratchOrgException(message)
def calculate_coverage(sample, bed_data): sample_name = extract_samplename(str(sample)) coverage = sarge.shell_format(COMMAND, sample, bed_data) coverage = sarge.capture_stdout(coverage) coverage_raw = pd.read_table( coverage.stdout, header=None, names=["chr", "start", "end", "name", "coverage"]) # FIXME: No idea why this happens (misplaced index) but we must fix it if pd.Series(["+", "-"]).isin(coverage_raw["name"]).any(): coverage_raw.reset_index(inplace=True) coverage_raw.columns = [ "chr", "start", "end", "name", "score", "strand", "coverage" ] # clean up bed results (Split at "_", get first result) # FIXME: Be more flexible coverage_raw["name"] = coverage_raw["name"].apply( lambda x: x.split("_")[0].strip()) coverage_table = coverage_raw[["coverage", "name" ]].groupby("name").agg(["mean", "std"]) mean_coverage = coverage_table["coverage", "mean"] mean_std = coverage_table["coverage", "std"] coverage_table.columns.set_names(["", ""], inplace=True) coverage_table.index.set_names(["Gene"], inplace=True) coverage_table["coverage", "mean"] = mean_coverage.round(3) coverage_table["coverage", "std"] = mean_std.round(3) global_coverage_mean = coverage_raw["coverage"].mean() global_coverage_std = coverage_raw["coverage"].std() return dict(name=sample_name, cov=coverage_table, mean=global_coverage_mean, std=global_coverage_std)
def sfdx(command, username=None, log_note=None): """Call an sfdx command and capture its output. Be sure to quote user input that is part of the command using `sarge.shell_format`. Returns a `sarge` Command instance with returncode, stdout, stderr """ command = "sfdx {}".format(command) if username: command += sarge.shell_format(" -u {0}", username) if log_note: logger.info("{} with command: {}".format(log_note, command)) p = sarge.Command( command, stdout=sarge.Capture(buffer_size=-1), stderr=sarge.Capture(buffer_size=-1), shell=True, ) p.run() return p
def moveto(path, pattern, write_changes=False): """ Move a pattern (glob) of files to a directory Args: path (str) -- directory path pattern (str) -- filename glob pattern Yields: sarge.run outputs """ # files = !ls $pattern log.debug('moveto()', path=path, pattern=pattern, write_changes=write_changes) files = match_file_pattern(pattern) path_ = pathlib.Path(path) path_.name in files and files.remove(path_.name) log.info('patternmatch', files=files) if not files: return # !mkdir $path if not os.path.exists(path): log.info('mkdir', path=path) os.makedirs(path) git_mv_opts = '-n' if write_changes: git_mv_opts = '' for f in files: cmd = sarge.shell_format( "git mv %s {0} {1}" % git_mv_opts, f, os.path.join(path, f)) log.info('cmd', cmd=cmd) yield sarge.capture_both(cmd)
def image_renamer(): fn = "figure_{:02}.{}" i = 1 file_path = yield while True: options, file_path = file_path.groups() file_path = find_img("{}/{}".format(cwd, file_path)) new_file_name = fn.format(i, file_path.split('.')[-1]) to_path = "{}/{}".format(dest_dir, new_file_name) cmd = sarge.shell_format("cp {0} {1}", file_path, to_path) sarge.run(cmd) i += 1 incl_gfx = "\\includegraphics{}{{{}}}" file_path = yield incl_gfx.format(options, new_file_name)
def carto_css(mml, name): """ Takes MML string input and writes it to a Mapnik XML file. :param mml: an mml string, containing the proper CartoCSS styling and connection attributes required by the CartoCSS conversion program :param name: the unique name of the layer (standard method is to name it with its database schema and table name) :return mapfile: a cascadenik-ready document. """ from sarge import shell_format, run, Capture create_static_content_subdir('cartocss') mml_file = "{0}/cartocss/{1}.mml".format(settings.MEDIA_ROOT, name) xml_file = mml_file.replace(".mml", ".xml") f = open(mml_file, 'w+') f.write(mml) f.close() if not settings.FOOTPRINT_INIT: carto_css_command = shell_format("{0}/carto {1}".format( settings.BIN_DIR, mml_file)) logger.debug("Running carto: %s" % carto_css_command) carto_css_content = None try: carto_result = capture_both(carto_css_command) assert not any(carto_result.returncodes) carto_css_content = carto_result.stdout.text logger.debug("Carto xml content: %s" % carto_css_command) f = open(xml_file, 'w') f.write(carto_css_content) f.close() except AssertionError, e: logger.error( "Failed to generate cartocss for {mml}. Exception: {message}. {carto_output}" .format(mml=mml_file, message=carto_result.stderr.text, carto_output=carto_css_content)) raise e
def callScript(self, panValue, tiltValue): self.panValue = max(self._settings.get(["pan", "minValue"]), min(self._settings.get(["pan", "maxValue"]), panValue)) self.tiltValue = max(self._settings.get(["tilt", "minValue"]), min(self._settings.get(["tilt", "maxValue"]), tiltValue)) # if there are anly pantilt handlers, loop through them, then return if len(self.pantiltHandlers) > 0: values = {'pan': self.panValue, 'panMin': self._settings.get(["pan", "minValue"]), 'panMax': self._settings.get(["pan", "maxValue"]), 'tilt': self.tiltValue, 'tiltMin': self._settings.get(["tilt", "minValue"]), 'tiltMax': self._settings.get(["tilt", "maxValue"])} for name, handler in self.pantiltHandlers.items(): handler(values) return script = self._settings.get(["pathToScript"]) if script == "": return self._logger.info("Performing command: {}".format(script)) try: # we run this with shell=True since we have to trust whatever # our admin configured as command and since we want to allow # shell-alike handling here... p = sarge.run(sarge.shell_format('{0} {1} {2}', script, panValue, tiltValue), stdout=sarge.Capture(), stderr=sarge.Capture(), shell=True, async=False) if p.returncode != 0: returncode = p.returncode stdout_text = p.stdout.text stderr_text = p.stderr.text error = "Command failed with return code {}:\nSTDOUT: {}\nSTDERR: {}".format(returncode, stdout_text, stderr_text) self._logger.warn(error) except Exception, e: error = "Command failed: {}".format(str(e)) self._logger.warn(error)
def check_google_dkim(domain, prefix=DEFAULT_GOOGLE_DKIM_PREFIX): """ Check a Google DKIM DNS TXT record Args: domain (str): DNS domain name prefix (str): DKIM ``s=`` selector ('DKIM prefix') Returns: int: 0 if OK, 1 on error | https://support.google.com/a/answer/174126 | https://admin.google.com/AdminHome?fral=1#AppDetails:service=email&flyout=dkim .. note:: This check function only finds "v=DKIM1" TXT records; it defaults to the default ``google`` prefix and **does not validate DKIM signatures**. | http://dkim.org/specs/rfc4871-dkimbase.html#rfc.section.3.6.2.1 | http://dkim.org/specs/rfc4871-dkimbase.html#rfc.section.A.3 """ dkim_record_name = "%s._domainkey.%s" % (prefix, domain) cmd = sarge.shell_format("dig {0} txt +short", dkim_record_name) log.info('cmd', op='check_google_dkim', cmd=cmd) proc = sarge.capture_both(cmd) output = proc.stdout.text.rstrip().split('\n') for line in output: log.debug('TXT', record=line) expected = u"\"v=DKIM1" # ... "\; p=none\; rua=mailto:" if line.startswith(expected): if 'k=' in line and 'p=' in line: return 0 errmsg = "%s is not a valid DKIM record" % (output) log.error('err', msg=errmsg) return 1
def scratch_info(self): if hasattr(self, "_scratch_info"): return self._scratch_info # Create the org if it hasn't already been created if not self.created: self.create_org() self.logger.info("Getting scratch org info from Salesforce DX") # Call force:org:display and parse output to get instance_url and # access_token command = sarge.shell_format("sfdx force:org:display -u {0} --json", self.username) p = sarge.Command( command, stderr=sarge.Capture(buffer_size=-1), stdout=sarge.Capture(buffer_size=-1), shell=True, ) p.run() org_info = None stderr_list = [line.strip() for line in p.stderr] stdout_list = [line.strip() for line in p.stdout] if p.returncode: self.logger.error("Return code: {}".format(p.returncode)) for line in stderr_list: self.logger.error(line) for line in stdout_list: self.logger.error(line) message = "\nstderr:\n{}".format("\n".join(stderr_list)) message += "\nstdout:\n{}".format("\n".join(stdout_list)) raise ScratchOrgException(message) else: json_txt = "".join(stdout_list) try: org_info = json.loads("".join(stdout_list)) except Exception as e: raise ScratchOrgException( "Failed to parse json from output. This can happen if " "your scratch org gets deleted.\n " "Exception: {}\n Output: {}".format( e.__class__.__name__, "".join(stdout_list))) org_id = org_info["result"]["accessToken"].split("!")[0] if "password" in org_info["result"] and org_info["result"]["password"]: password = org_info["result"]["password"] else: password = self.config.get("password") self._scratch_info = { "instance_url": org_info["result"]["instanceUrl"], "access_token": org_info["result"]["accessToken"], "org_id": org_id, "username": org_info["result"]["username"], "password": password, } self.config.update(self._scratch_info) self._scratch_info_date = datetime.datetime.utcnow() return self._scratch_info
def test_shlex_issue_31(self): cmd = "python -c 'print('\''ok'\'')'" list(shell_shlex(cmd, control='();>|&', posix=True)) shell_format("python -c {0}", "print('ok')") list(shell_shlex(cmd, control='();>|&', posix=True))
def test_formatter(self): self.assertEqual(shell_format('ls {0}', '*.py'), "ls '*.py'") self.assertEqual(shell_format('ls {0!s}', '*.py'), "ls *.py")
def run(command, *args, **kwargs): command = sarge.shell_format(command, *args) log.info('run process: %s', command) return sarge.run(command, stdout=Capture(), stderr=Capture(), **kwargs)
def scratch_info(self): if hasattr(self, '_scratch_info'): return self._scratch_info # Create the org if it hasn't already been created if not self.created: self.create_org() self.logger.info('Getting scratch org info from Salesforce DX') # Call force:org:display and parse output to get instance_url and # access_token command = sarge.shell_format('sfdx force:org:display -u {0} --json', self.username) p = sarge.Command( command, stderr=sarge.Capture(buffer_size=-1), stdout=sarge.Capture(buffer_size=-1), shell=True ) p.run() org_info = None stderr_list = [line.strip() for line in p.stderr] stdout_list = [line.strip() for line in p.stdout] if p.returncode: self.logger.error('Return code: {}'.format(p.returncode)) for line in stderr_list: self.logger.error(line) for line in stdout_list: self.logger.error(line) message = '\nstderr:\n{}'.format('\n'.join(stderr_list)) message += '\nstdout:\n{}'.format('\n'.join(stdout_list)) raise ScratchOrgException(message) else: json_txt = ''.join(stdout_list) try: org_info = json.loads(''.join(stdout_list)) except Exception as e: raise ScratchOrgException( 'Failed to parse json from output. This can happen if ' 'your scratch org gets deleted.\n ' 'Exception: {}\n Output: {}'.format( e.__class__.__name__, ''.join(stdout_list), ) ) org_id = org_info['result']['accessToken'].split('!')[0] if 'password' in org_info['result'] and org_info['result']['password']: password = org_info['result']['password'] else: password = self.config.get('password') self._scratch_info = { 'instance_url': org_info['result']['instanceUrl'], 'access_token': org_info['result']['accessToken'], 'org_id': org_id, 'username': org_info['result']['username'], 'password': password, } self.config.update(self._scratch_info) self._scratch_info_date = datetime.datetime.utcnow() return self._scratch_info
def create_org(self): """ Uses sfdx force:org:create to create the org """ if not self.config_file: raise ScratchOrgException( f"Scratch org config {self.name} is missing a config_file") if not self.scratch_org_type: self.config["scratch_org_type"] = "workspace" # If the scratch org definition itself contains an `adminEmail` entry, # we don't want to override it from our own configuration, which may # simply come from the user's Git config. with open(self.config_file, "r") as org_def: org_def_data = json.load(org_def) org_def_has_email = "adminEmail" in org_def_data devhub = self._choose_devhub() instance = self.instance or os.environ.get("SFDX_SIGNUP_INSTANCE") options = { "config_file": self.config_file, "devhub": f" --targetdevhubusername {devhub}" if devhub else "", "namespaced": " -n" if not self.namespaced else "", "days": f" --durationdays {self.days}" if self.days else "", "wait": " -w 120", "alias": sarge.shell_format(' -a "{0!s}"', self.sfdx_alias) if self.sfdx_alias else "", "email": sarge.shell_format(' adminEmail="{0!s}"', self.email_address) if self.email_address and not org_def_has_email else "", "default": " -s" if self.default else "", "instance": f" instance={instance}" if instance else "", "extraargs": os.environ.get("SFDX_ORG_CREATE_ARGS", ""), } # This feels a little dirty, but the use cases for extra args would mostly # work best with env vars command = "force:org:create -f {config_file}{devhub}{namespaced}{days}{alias}{default}{wait}{email}{instance} {extraargs}".format( **options) p = sfdx(command, username=None, log_note="Creating scratch org") stderr = [line.strip() for line in p.stderr_text] stdout = [line.strip() for line in p.stdout_text] if p.returncode: message = f"{FAILED_TO_CREATE_SCRATCH_ORG}: \n{nl.join(stdout)}\n{nl.join(stderr)}" raise ScratchOrgException(message) re_obj = re.compile( "Successfully created scratch org: (.+), username: (.+)") username = None for line in stdout: match = re_obj.search(line) if match: self.config["org_id"] = match.group(1) self.config["username"] = username = match.group(2) self.logger.info(line) for line in stderr: self.logger.error(line) if username is None: raise ScratchOrgException( "SFDX claimed to be successful but there was no username " "in the output...maybe there was a gack?") self.config["date_created"] = datetime.datetime.utcnow() if self.config.get("set_password"): self.generate_password() # Flag that this org has been created self.config["created"] = True
def get_free_space(location): p = run(shell_format('du -hs {0}', location), stdout=Capture()) return p.stdout.text.split()[0]
def settings2options(settingsdict): options = [ sarge.shell_format("--{} {}", key, val) for (key, val) in settingsdict.items() ] return " ".join(options)
def create_org(self): """ Uses sfdx force:org:create to create the org """ if not self.config_file: # FIXME: raise exception return if not self.scratch_org_type: self.config["scratch_org_type"] = "workspace" # If the scratch org definition itself contains an `adminEmail` entry, # we don't want to override it from our own configuration, which may # simply come from the user's Git config. with open(self.config_file, "r") as org_def: org_def_data = json.load(org_def) org_def_has_email = "adminEmail" in org_def_data options = { "config_file": self.config_file, "devhub": " --targetdevhubusername {}".format(self.devhub) if self.devhub else "", "namespaced": " -n" if not self.namespaced else "", "days": " --durationdays {}".format(self.days) if self.days else "", "alias": sarge.shell_format(' -a "{0!s}"', self.sfdx_alias) if self.sfdx_alias else "", "email": sarge.shell_format('adminEmail="{0!s}"', self.email_address) if self.email_address and not org_def_has_email else "", "default": " -s" if self.default else "", "extraargs": os.environ.get("SFDX_ORG_CREATE_ARGS", ""), } # This feels a little dirty, but the use cases for extra args would mostly # work best with env vars command = "force:org:create -f {config_file}{devhub}{namespaced}{days}{alias}{default} {email} {extraargs}".format( **options) p = sfdx(command, username=None, log_note="Creating scratch org") stderr = [line.strip() for line in p.stderr_text] stdout = [line.strip() for line in p.stdout_text] if p.returncode: message = "{}: \n{}\n{}".format(FAILED_TO_CREATE_SCRATCH_ORG, "\n".join(stdout), "\n".join(stderr)) raise ScratchOrgException(message) re_obj = re.compile( "Successfully created scratch org: (.+), username: (.+)") for line in stdout: match = re_obj.search(line) if match: self.config["org_id"] = match.group(1) self.config["username"] = match.group(2) self.logger.info(line) for line in stderr: self.logger.error(line) self.config["date_created"] = datetime.datetime.now() if self.config.get("set_password"): self.generate_password() # Flag that this org has been created self.config["created"] = True
def test_formatter(self): self.assertEqual(shell_format("ls {0}", "*.py"), "ls '*.py'") self.assertEqual(shell_format("ls {0!s}", "*.py"), "ls *.py")