def remove_comments(tex, cwd):
"""
"""
cmd = sarge.shell_format('latexpand --keep-includes {0} -o {0}_new', tex)
RUN_CMD(cmd, cwd=cwd)
cmd = sarge.shell_format('mv {0}_new {0}', tex)
RUN_CMD(cmd, cwd=cwd)
def remove_cref(tex, cwd):
"""
"""
sty_file = cwd+'/dynlearn.sty'
# add poorman to cleveref import
cref_regex1 = (r'usepackage\{cleveref\}', r'usepackage[poorman]{cleveref}')
cref_regex2 = (r'usepackage\[(\w*)\]\{cleveref\}', r'usepackage[\g<1>,poorman]{cleveref}')
with open(sty_file) as dynlearn:
source = dynlearn.read()
for exp, repl in [cref_regex1, cref_regex2]:
source = re.sub(exp, repl, source)
with open(sty_file, 'w') as dynlearn:
dynlearn.write(source)
# generate and run the sed script
compile_tex(tex, cwd)
sed_cmd = sarge.shell_format("sed -f {0}.sed {0}.tex > {0}.tex.temp", tex.split('.')[0])
sarge.run(sed_cmd, cwd=cwd)
sarge.run(sarge.shell_format("mv {0}.temp {0}", tex), cwd=cwd)
# remove the cleveref import
cref_regex3 = r'\\usepackage\[.*\]\{cleveref\}'
comment_lines(sty_file, [cref_regex3])
def play(playbook, verbosity=0):
if verbosity > 0:
verb = "-{0}".format('v' * verbosity)
cmd = shell_format('ansible-playbook {0} {1}.yml', verb, playbook)
else:
cmd = shell_format('ansible-playbook {0}.yml', playbook)
if run(cmd, cwd=ansible_root).returncode > 0:
click.echo('Operation failed.')
return False
else:
return True
def play(playbook, verbosity=0):
if verbosity > 0:
verb = "-{0}".format('v' * verbosity)
cmd = shell_format('ansible-playbook {0} {1}.yml', verb, playbook)
else:
cmd = shell_format('ansible-playbook {0}.yml', playbook)
if run(cmd, cwd=ansible_root).returncode > 0:
click.echo('Operation failed.')
return False
else:
return True
def generate_password(self):
"""Generates an org password with the sfdx utility. """
if self.password_failed:
self.logger.warn(
"Skipping resetting password since last attempt failed")
return
# Set a random password so it's available via cci org info
command = sarge.shell_format(
"sfdx force:user:password:generate -u {0}", self.username)
self.logger.info(
"Generating scratch org user password with command {}".format(
command))
p = sarge.Command(
command,
stdout=sarge.Capture(buffer_size=-1),
stderr=sarge.Capture(buffer_size=-1),
shell=True,
)
p.run()
stdout = []
for line in p.stdout:
stdout.append(line)
stderr = []
for line in p.stderr:
stderr.append(line)
if p.returncode:
self.config["password_failed"] = True
# Don't throw an exception because of failure creating the
# password, just notify in a log message
self.logger.warn("Failed to set password: \n{}\n{}".format(
"\n".join(stdout), "\n".join(stderr)))
def check_google_dmarc(domain):
"""
Check a Google DMARC DNS TXT record
Args:
domain (str): DNS domain name
Returns:
int: 0 if OK, 1 on error
| https://support.google.com/a/answer/2466580
| https://support.google.com/a/answer/2466563
"""
dmarc_domain = "_dmarc." + domain
cmd = sarge.shell_format("dig {0} txt +short", dmarc_domain)
log.info('cmd', op='check_google_dmarc', cmd=cmd)
proc = sarge.capture_both(cmd)
output = proc.stdout.text.rstrip().split('\n')
for line in output:
log.debug('TXT', record=line)
expected = u"\"v=DMARC1" # ... "\; p=none\; rua=mailto:"
if line.startswith(expected):
if 'p=' in line:
return 0
errmsg = "%r != %r" % (output, expected)
log.error('err', msg=errmsg)
return 1
def check_google_spf(domain):
"""
Check a Google SPF DNS TXT record
Args:
domain (str): DNS domain name
Returns:
int: 0 if OK, 1 on error
| https://support.google.com/a/answer/178723?hl=en
"""
cmd = sarge.shell_format("dig {0} txt +short", domain)
log.info('cmd', op='check_google_spf', cmd=cmd)
proc = sarge.capture_both(cmd)
output = proc.stdout.text.rstrip().split('\n')
for line in output:
log.debug('TXT', record=line)
expected = u"\"v=spf1 include:_spf.google.com ~all\""
if line == expected:
return 0
errmsg = "%r != %r" % (output, expected)
log.error('err', msg=errmsg)
return 1
def check_google_mx(domain):
"""
Check Google MX DNS records
Args:
domain (str): DNS domain name
Returns:
int: 0 if OK, 1 on error
| https://support.google.com/a/topic/2716885?hl=en&ref_topic=2426592
"""
cmd = sarge.shell_format("dig {0} mx +short", domain)
log.info('cmd', cmd=cmd)
output = sarge.capture_both(cmd).stdout.text.rstrip()
log.debug('MX', record=output)
result = 0
check_domain1 = "aspmx.l.google.com."
check_domain2 = "googlemail.com."
lines = output.split('\n')
if not lines:
log.error('err', msg="No MX records found for %r" % domain)
result += 1
for l in lines:
l = l.lower()
if not (l.endswith(check_domain1) or l.endswith(check_domain2)):
result += 1
log.error('err', msg="%r does not end with %r or %r" %
(l, check_domain1, check_domain2))
if result is None:
result += 1
return result
def delete_org(self):
""" Uses sfdx force:org:delete to delete the org """
if not self.created:
self.logger.info(
"Skipping org deletion: the scratch org has not been created")
return
command = sarge.shell_format("sfdx force:org:delete -p -u {0}",
self.username)
self.logger.info(
"Deleting scratch org with command {}".format(command))
p = sarge.Command(command,
stdout=sarge.Capture(buffer_size=-1),
shell=True)
p.run()
org_info = None
stdout = []
for line in p.stdout:
stdout.append(line)
if line.startswith("An error occurred deleting this org"):
self.logger.error(line)
else:
self.logger.info(line)
if p.returncode:
message = "Failed to delete scratch org: \n{}".format(
"".join(stdout))
raise ScratchOrgException(message)
# Flag that this org has been created
self.config["created"] = False
self.config["username"] = None
def exposed_rmrf(self, dir_):
"""
classid 'rm -rf'
"""
logger.debug("Removing directory {0}".format(dir_))
return run_command_forget_output(
shell_format('rm -rf {0}', dir_), cwd=self.cwd)
def remove_cref(tex, cwd):
"""
"""
sty_file = cwd+'/dynlearn.sty'
# add poorman to cleveref import
cref_regex1 = (r'usepackage\{cleveref\}', r'usepackage[poorman]{cleveref}')
cref_regex2 = (r'usepackage\[(\w*)\]\{cleveref\}', r'usepackage[\g<1>,poorman]{cleveref}')
with open(sty_file) as dynlearn:
source = dynlearn.read()
for exp, repl in [cref_regex1, cref_regex2]:
source = re.sub(exp, repl, source)
with open(sty_file, 'w') as dynlearn:
dynlearn.write(source)
# generate and run the sed script
compile_tex(tex, cwd)
sed_cmd = sarge.shell_format("sed -f {0}.sed {0}.tex", tex.split('.')[0])
cleaned = RUN_CMD(sed_cmd, cwd=cwd, stdout=sarge.Capture())
with (pathlib.Path(cwd) / tex).open('w') as output:
output.write(cleaned.stdout.text)
# remove the cleveref import
cref_regex3 = r'\\usepackage\[.*\]\{cleveref\}'
comment_lines(sty_file, [cref_regex3])
def carto_css(mml, name):
"""
Takes MML string input and writes it to a Mapnik XML file.
:param mml: an mml string, containing the proper CartoCSS styling and connection attributes required by the
CartoCSS conversion program
:param name: the unique name of the layer (standard method is to name it with its database schema and table name)
:return mapfile: a cascadenik-ready document.
"""
from sarge import shell_format, run, Capture
create_static_content_subdir('cartocss')
mml_file = "{0}/cartocss/{1}.mml".format(settings.MEDIA_ROOT, name)
xml_file = mml_file.replace(".mml", ".xml")
f = open(mml_file, 'w+')
f.write(mml)
f.close()
if not settings.FOOTPRINT_INIT:
carto_css_command = shell_format("{0}/carto {1}".format(settings.BIN_DIR, mml_file))
logger.debug("Running carto: %s" % carto_css_command)
carto_css_content = None
try:
carto_result = capture_both(carto_css_command)
assert not any(carto_result.returncodes)
carto_css_content = carto_result.stdout.text
logger.debug("Carto xml content: %s" % carto_css_command)
f = open(xml_file, 'w')
f.write(carto_css_content)
f.close()
except AssertionError, e:
logger.error("Failed to generate cartocss for {mml}. Exception: {message}. {carto_output}".format(
mml=mml_file, message=carto_result.stderr.text, carto_output=carto_css_content))
raise e
def create_org(self):
""" Uses sfdx force:org:create to create the org """
if not self.config_file:
# FIXME: raise exception
return
if not self.scratch_org_type:
self.config["scratch_org_type"] = "workspace"
options = {
"config_file":
self.config_file,
"devhub":
" --targetdevhubusername {}".format(self.devhub)
if self.devhub else "",
"namespaced":
" -n" if not self.namespaced else "",
"days":
" --durationdays {}".format(self.days) if self.days else "",
"alias":
sarge.shell_format(' -a "{0!s}"', self.sfdx_alias)
if self.sfdx_alias else "",
"extraargs":
os.environ.get("SFDX_ORG_CREATE_ARGS", ""),
}
# This feels a little dirty, but the use cases for extra args would mostly
# work best with env vars
command = "sfdx force:org:create -f {config_file}{devhub}{namespaced}{days}{alias} {extraargs}".format(
**options)
self.logger.info(
"Creating scratch org with command {}".format(command))
p = sarge.Command(command,
stdout=sarge.Capture(buffer_size=-1),
shell=True)
p.run()
re_obj = re.compile(
"Successfully created scratch org: (.+), username: (.+)")
stdout = []
for line in p.stdout:
match = re_obj.search(line)
if match:
self.config["org_id"] = match.group(1)
self.config["username"] = match.group(2)
stdout.append(line)
self.logger.info(line)
self.config["date_created"] = datetime.datetime.now()
if p.returncode:
message = "{}: \n{}".format(FAILED_TO_CREATE_SCRATCH_ORG,
"".join(stdout))
raise ScratchOrgException(message)
if self.config.get("set_password"):
self.generate_password()
# Flag that this org has been created
self.config["created"] = True
def exposed_mkdir(self, dir_):
"""
classic 'mkdir'
"""
logger.debug("Creating directory {0}".format(dir_))
formated = shell_format('mkdir {0}', dir_)
logger.debug("Creating formated {0}".format(formated))
return run_command_forget_output(formated, cwd=self.cwd)
def test_get_git_config(self, Command):
Command.return_value = p = mock.Mock(
stdout=io.BytesIO(b"*****@*****.**"),
stderr=io.BytesIO(b""),
returncode=0)
assert "*****@*****.**" == utils.get_git_config("user.email")
assert (sarge.shell_format('git config --get "{0!s}"',
"user.email") == Command.call_args[0][0])
p.run.assert_called_once()
def create_and_populate(tex):
"""
"""
cwd = pathlib.Path().cwd()
pdir = (cwd / DIRECTORY_NAME)
if pdir.exists():
LOG.warn('Removing existing {} directory'.format(DIRECTORY_NAME))
cmd = sarge.shell_format("rm -rf {0}", DIRECTORY_NAME)
sarge.run(cmd, cwd=str(cwd))
pdir.mkdir()
files = TEMPLATE_FILES + find_bib_files(tex) + [tex]
for filename in files:
cmd = sarge.shell_format("cp {0} {1}", filename, DIRECTORY_NAME)
sarge.run(cmd, cwd=str(cwd))
return str(pdir)
def _extract_srpm_contents(srpm_path, target_dir):
"""Extract an SRPM package using rpm2cpio / cpio into `target_dir`"""
old_paths = set(target_dir.iterdir())
raw_command = 'rpm2cpio {} | cpio -idm > /dev/null 2>&1'
command = sarge.shell_format(raw_command, srpm_path)
log.debug("Extracting SRPM contents: %r", command)
sarge.run(command, cwd=str(target_dir))
new_paths = sorted(set(target_dir.iterdir()) - old_paths)
log.debug("Extracted files: %s", new_paths)
return new_paths
def sfdx(
command,
username=None,
log_note=None,
access_token=None,
args=None,
env=None,
capture_output=True,
check_return=False,
):
"""Call an sfdx command and capture its output.
Be sure to quote user input that is part of the command using `sarge.shell_format`.
Returns a `sarge` Command instance with returncode, stdout, stderr
"""
command = "sfdx {}".format(command)
if args is not None:
for arg in args:
command += " " + sarge.shell_quote(arg)
if username:
command += sarge.shell_format(" -u {0}", username)
if log_note:
logger.info("{} with command: {}".format(log_note, command))
# Avoid logging access token
if access_token:
command += sarge.shell_format(" -u {0}", access_token)
p = sarge.Command(
command,
stdout=sarge.Capture(buffer_size=-1) if capture_output else None,
stderr=sarge.Capture(buffer_size=-1) if capture_output else None,
shell=True,
env=env,
)
p.run()
if capture_output:
p.stdout_text = io.TextIOWrapper(p.stdout,
encoding=sys.stdout.encoding)
p.stderr_text = io.TextIOWrapper(p.stderr,
encoding=sys.stdout.encoding)
if check_return and p.returncode:
raise Exception(f"Command exited with return code {p.returncode}")
return p
def set_default_org(self, name):
""" set the default org for tasks and flows by name """
org = self.get_org(name)
self.unset_default_org()
org.config["default"] = True
org.save()
if org.created:
sfdx(
sarge.shell_format("force:config:set defaultusername={}",
org.sfdx_alias))
def get_git_config(config_key):
p = sarge.Command(
sarge.shell_format('git config --get "{0!s}"', config_key),
stderr=sarge.Capture(buffer_size=-1),
stdout=sarge.Capture(buffer_size=-1),
shell=True,
)
p.run()
config_value = io.TextIOWrapper(p.stdout).read().strip()
return config_value if config_value and not p.returncode else None
def get_git_config(config_key):
p = sarge.Command(
sarge.shell_format('git config --get "{0!s}"', config_key),
stderr=sarge.Capture(buffer_size=-1),
stdout=sarge.Capture(buffer_size=-1),
shell=True,
)
p.run()
config_value = io.TextIOWrapper(p.stdout).read().strip()
return config_value if config_value and not p.returncode else None
def test_get_git_config(self, Command):
Command.return_value = p = mock.Mock(
stdout=io.BytesIO(b"*****@*****.**"), stderr=io.BytesIO(b""), returncode=0
)
self.assertEqual("*****@*****.**", utils.get_git_config("user.email"))
self.assertEqual(
sarge.shell_format('git config --get "{0!s}"', "user.email"),
Command.call_args[0][0],
)
p.run.assert_called_once()
def check_multiple_alts(chrom, start, end, alt, samples, reference_file=None):
# FIXME: PyVCF can't be used as it loads the wrong coordinate with
# fetch()
if reference_file is None:
return alt
alt = alt.split(",")
if len(alt) == 1:
return alt[0]
region = "{0}:{1}-{2}".format(chrom, start, end)
sample_display = ",".join(samples)
bcftools = ("/usr/bin/bcftools view {0} -r {1} -a -s {2}"
" --exclude-uncalled -H")
bcftools = sarge.shell_format(bcftools, reference_file, region,
sample_display)
command = sarge.capture_both(bcftools)
mutation_table = pd.read_table(command.stdout, header=None,
names=["CHROM", "POS", "ID", "REF", "ALT"],
usecols=["CHROM", "POS", "ID", "REF",
"ALT"])
if mutation_table.empty:
# Try alternative approach: sometimes bcftools asserts when
# launched from sarge
import subprocess
with open(os.devnull) as null:
cmd = subprocess.Popen(bcftools, shell=True,
stdout=subprocess.PIPE,
stderr=null)
mutation_table = pd.read_table(cmd.stdout, header=None,
names=["CHROM", "POS", "ID",
"REF", "ALT"],
usecols=["CHROM", "POS", "ID",
"REF", "ALT"])
seen_alt = mutation_table["ALT"].item()
if len(seen_alt.split(",")) > 1:
message = ("Detected more than one allele for sample pair {}: {},"
" mutation position {}-{}, ref {}")
print(message.format(", ".join(samples), seen_alt, chrom, start,
mutation_table["REF"].item()),
file=sys.stderr)
# This is a false positive! Same locus but different ALTs
return
return seen_alt
def set_default_org(self, name):
""" set the default org for tasks by name key """
org = self.get_org(name)
self.unset_default_org()
org.config["default"] = True
self.set_org(org)
if org.created:
sfdx(
sarge.shell_format(
"force:config:set defaultusername={}", org.sfdx_alias
)
)
def test_generate_output(self):
os.makedirs('.git')
os.makedirs('test_dir')
with tempfile.TemporaryDirectory(prefix='test') as tmp_test,\
tempfile.TemporaryDirectory(prefix='d2g_') as tmp:
generate_output(sarge.shell_format('cp -r test_dir {0}', tmp_test),
tmp, [])
self.assertTrue(os.path.exists(os.path.join(tmp_test, 'test_dir')))
self.assertFalse(os.path.exists(os.path.join(tmp_test, '.git')))
def whois(domain):
"""
Get whois information with whois
Args:
domain (str): DNS domain
Returns:
str: whois output
"""
cmd = sarge.shell_format('whois {0}', domain)
log.info('cmd', cmd=cmd)
output = sarge.capture_both(cmd)
return output
def create_org(self):
""" Uses sfdx force:org:create to create the org """
if not self.config_file:
# FIXME: raise exception
return
if not self.scratch_org_type:
self.config['scratch_org_type'] = 'workspace'
options = {
'config_file': self.config_file,
'devhub': ' --targetdevhubusername {}'.format(self.devhub) if self.devhub else '',
'namespaced': ' -n' if not self.namespaced else '',
'days': ' --durationdays {}'.format(self.days) if self.days else '',
'alias': sarge.shell_format(' -a "{0!s}"', self.sfdx_alias) if self.sfdx_alias else '',
'extraargs': os.environ.get('SFDX_ORG_CREATE_ARGS', ''),
}
# This feels a little dirty, but the use cases for extra args would mostly
# work best with env vars
command = 'sfdx force:org:create -f {config_file}{devhub}{namespaced}{days}{alias} {extraargs}'.format(**options)
self.logger.info(
'Creating scratch org with command {}'.format(command))
p = sarge.Command(command, stdout=sarge.Capture(buffer_size=-1), shell=True)
p.run()
re_obj = re.compile(
'Successfully created scratch org: (.+), username: (.+)')
stdout = []
for line in p.stdout:
match = re_obj.search(line)
if match:
self.config['org_id'] = match.group(1)
self.config['username'] = match.group(2)
stdout.append(line)
self.logger.info(line)
self.config['date_created'] = datetime.datetime.now()
if p.returncode:
message = '{}: \n{}'.format(
FAILED_TO_CREATE_SCRATCH_ORG,
''.join(stdout),
)
raise ScratchOrgException(message)
if self.config.get('set_password'):
self.generate_password()
# Flag that this org has been created
self.config['created'] = True
def dig_txt(domain):
"""
Get DNS TXT records with dig
Args:
domain (str): DNS domain
Returns:
str: dig output
"""
cmd = sarge.shell_format(
"dig {0} txt +cmd +nocomments +question +noidentify +nostats",
domain)
log.info('cmd', cmd=cmd)
output = sarge.capture_both(cmd)
return output
def start_python_process(component_name,
run_in_fg,
py_path,
program_dir,
on_keyboard_interrupt=None,
failed_to_start_err=-100,
extra_options=None,
stderr_path=None,
stdin_data=None):
""" Starts a new process from a given Python path, either in background or foreground (run_in_fg).
"""
options = {
'fg': run_in_fg,
}
if extra_options:
options.update(extra_options)
options = CLI_ARG_SEP.join('{}={}'.format(k, v)
for k, v in options.items())
py_path_option = shell_format('-m {0}', py_path)
program_dir_option = shell_format('{0}',
program_dir) if program_dir else ''
extra_cli_options = '{} {} {}'.format(py_path_option, program_dir_option,
options)
extra_cli_options = '{} '.format(py_path_option)
if program_dir_option:
extra_cli_options += '{} '.format(program_dir_option)
extra_cli_options += '{}'.format(options)
return start_process(component_name, get_executable(), run_in_fg, None,
extra_cli_options, on_keyboard_interrupt,
failed_to_start_err, extra_options, stderr_path,
stdin_data)
def dig_dnskey(zone):
"""
Get DNSSEC DNS records with dig
Args:
zone (str): DNS zone
Returns:
str: dig output
"""
cmd = sarge.shell_format(
"dig {0} +dnssec dnskey +cmd +nocomments +question +noidentify +nostats",
zone)
log.info('cmd', cmd=cmd)
output = sarge.capture_both(cmd)
return output
def create_and_populate(tex):
"""
"""
cwd = pathlib.Path().cwd()
pdir = (cwd / DIRECTORY_NAME)
if pdir.exists():
LOG.warn('Removing existing {} directory'.format(DIRECTORY_NAME))
cmd = sarge.shell_format("rm -rf {0}", DIRECTORY_NAME)
RUN_CMD(cmd, cwd=str(cwd))
pdir.mkdir()
files = TEMPLATE_FILES + find_bib_files(tex) + [tex]
for filename in files:
if pathlib.Path(filename).exists():
cmd = sarge.shell_format("cp {0} {1}", filename, DIRECTORY_NAME)
RUN_CMD(cmd, cwd=str(cwd))
# inline inputs
cmd = sarge.shell_format('latexpand --keep-comments {0} -o {1}/{0}', tex, str(pdir))
RUN_CMD(cmd, cwd=str(cwd))
return str(pdir)
def callScript(self, panValue, tiltValue):
self.panValue = max(
self._settings.get(["pan", "minValue"]),
min(self._settings.get(["pan", "maxValue"]), panValue))
self.tiltValue = max(
self._settings.get(["tilt", "minValue"]),
min(self._settings.get(["tilt", "maxValue"]), tiltValue))
# if there are anly pantilt handlers, loop through them, then return
if len(self.pantiltHandlers) > 0:
values = {
'pan': self.panValue,
'panMin': self._settings.get(["pan", "minValue"]),
'panMax': self._settings.get(["pan", "maxValue"]),
'tilt': self.tiltValue,
'tiltMin': self._settings.get(["tilt", "minValue"]),
'tiltMax': self._settings.get(["tilt", "maxValue"])
}
for name, handler in self.pantiltHandlers.items():
handler(values)
return
script = self._settings.get(["pathToScript"])
if script == "":
return
self._logger.info("Performing command: {}".format(script))
try:
# we run this with shell=True since we have to trust whatever
# our admin configured as command and since we want to allow
# shell-alike handling here...
p = sarge.run(sarge.shell_format('{0} {1} {2}', script, panValue,
tiltValue),
stdout=sarge.Capture(),
stderr=sarge.Capture(),
shell=True,
async=False)
if p.returncode != 0:
returncode = p.returncode
stdout_text = p.stdout.text
stderr_text = p.stderr.text
error = "Command failed with return code {}:\nSTDOUT: {}\nSTDERR: {}".format(
returncode, stdout_text, stderr_text)
self._logger.warn(error)
except Exception, e:
error = "Command failed: {}".format(str(e))
self._logger.warn(error)
def nslookup(domain, nameserver=''):
"""
Get nslookup information with nslookup (resolve a domainname to an IP)
Args:
domain (str): DNS domain
nameserver (str): DNS domain name server to query (default: ``''``)
Returns:
str: nslookup output
"""
if not domain.endswith('.'):
domain = domain + '.'
cmd = sarge.shell_format('nslookup {0} {1}', domain, nameserver)
log.info('cmd', cmd=cmd)
output = sarge.capture_both(cmd)
return output
def ping(host, count=5, deadline=5):
"""
ICMP ECHO ping a host (with ``ping``)
Args:
host (str): hostname or IP
count (int): number of packets to send (``-c``)
deadline (int): timeout in seconds (``-w``)
Returns:
:py:class:`PingSummary`: parsed ping summary
"""
cmd = sarge.shell_format("""ping -q -c {0} -w {1} -i 0.2 {2}""", count,
deadline, host)
p = sarge.capture_stdout(cmd)
return _parse_ping_output(TextIOWrapper(p.stdout))
def dig_mx(domain):
"""
Get MX DNS records with dig
Args:
domain (str): DNS domain
Returns:
str: dig output
| https://en.wikipedia.org/wiki/MX_record
"""
cmd = sarge.shell_format(
"dig {0} mx +cmd +nocomments +question +noidentify +nostats",
domain)
log.info('cmd', cmd=cmd)
output = sarge.capture_both(cmd)
return output
def ping(host, count=5, deadline=5):
"""
ICMP ECHO ping a host (with ``ping``)
Args:
host (str): hostname or IP
count (int): number of packets to send (``-c``)
deadline (int): timeout in seconds (``-w``)
Returns:
:py:class:`PingSummary`: parsed ping summary
"""
cmd = sarge.shell_format(
"""ping -q -c {0} -w {1} -i 0.2 {2}""",
count,
deadline,
host)
p = sarge.capture_stdout(cmd)
return _parse_ping_output(TextIOWrapper(p.stdout))
def force_refresh_oauth_token(self):
# Call force:org:display and parse output to get instance_url and
# access_token
command = sarge.shell_format('sfdx force:org:open -r -u {0}', self.username)
self.logger.info(
'Refreshing OAuth token with command: {}'.format(command))
p = sarge.Command(command, stdout=sarge.Capture(buffer_size=-1), shell=True)
p.run()
stdout_list = []
for line in p.stdout:
stdout_list.append(line.strip())
if p.returncode:
self.logger.error('Return code: {}'.format(p.returncode))
for line in stdout_list:
self.logger.error(line)
message = 'Message: {}'.format('\n'.join(stdout_list))
raise ScratchOrgException(message)
def calculate_coverage(sample, bed_data):
sample_name = extract_samplename(str(sample))
coverage = sarge.shell_format(COMMAND, sample, bed_data)
coverage = sarge.capture_stdout(coverage)
coverage_raw = pd.read_table(
coverage.stdout,
header=None,
names=["chr", "start", "end", "name", "coverage"])
# FIXME: No idea why this happens (misplaced index) but we must fix it
if pd.Series(["+", "-"]).isin(coverage_raw["name"]).any():
coverage_raw.reset_index(inplace=True)
coverage_raw.columns = [
"chr", "start", "end", "name", "score", "strand", "coverage"
]
# clean up bed results (Split at "_", get first result)
# FIXME: Be more flexible
coverage_raw["name"] = coverage_raw["name"].apply(
lambda x: x.split("_")[0].strip())
coverage_table = coverage_raw[["coverage", "name"
]].groupby("name").agg(["mean", "std"])
mean_coverage = coverage_table["coverage", "mean"]
mean_std = coverage_table["coverage", "std"]
coverage_table.columns.set_names(["", ""], inplace=True)
coverage_table.index.set_names(["Gene"], inplace=True)
coverage_table["coverage", "mean"] = mean_coverage.round(3)
coverage_table["coverage", "std"] = mean_std.round(3)
global_coverage_mean = coverage_raw["coverage"].mean()
global_coverage_std = coverage_raw["coverage"].std()
return dict(name=sample_name,
cov=coverage_table,
mean=global_coverage_mean,
std=global_coverage_std)
def sfdx(command, username=None, log_note=None):
"""Call an sfdx command and capture its output.
Be sure to quote user input that is part of the command using `sarge.shell_format`.
Returns a `sarge` Command instance with returncode, stdout, stderr
"""
command = "sfdx {}".format(command)
if username:
command += sarge.shell_format(" -u {0}", username)
if log_note:
logger.info("{} with command: {}".format(log_note, command))
p = sarge.Command(
command,
stdout=sarge.Capture(buffer_size=-1),
stderr=sarge.Capture(buffer_size=-1),
shell=True,
)
p.run()
return p
def moveto(path, pattern, write_changes=False):
"""
Move a pattern (glob) of files to a directory
Args:
path (str) -- directory path
pattern (str) -- filename glob pattern
Yields:
sarge.run outputs
"""
# files = !ls $pattern
log.debug('moveto()', path=path,
pattern=pattern, write_changes=write_changes)
files = match_file_pattern(pattern)
path_ = pathlib.Path(path)
path_.name in files and files.remove(path_.name)
log.info('patternmatch', files=files)
if not files:
return
# !mkdir $path
if not os.path.exists(path):
log.info('mkdir', path=path)
os.makedirs(path)
git_mv_opts = '-n'
if write_changes:
git_mv_opts = ''
for f in files:
cmd = sarge.shell_format(
"git mv %s {0} {1}" % git_mv_opts,
f, os.path.join(path, f))
log.info('cmd', cmd=cmd)
yield sarge.capture_both(cmd)
def image_renamer():
fn = "figure_{:02}.{}"
i = 1
file_path = yield
while True:
options, file_path = file_path.groups()
file_path = find_img("{}/{}".format(cwd, file_path))
new_file_name = fn.format(i, file_path.split('.')[-1])
to_path = "{}/{}".format(dest_dir, new_file_name)
cmd = sarge.shell_format("cp {0} {1}", file_path, to_path)
sarge.run(cmd)
i += 1
incl_gfx = "\\includegraphics{}{{{}}}"
file_path = yield incl_gfx.format(options, new_file_name)
def image_renamer():
fn = "figure_{:02}.{}"
i = 1
file_path = yield
while True:
options, file_path = file_path.groups()
file_path = find_img("{}/{}".format(cwd, file_path))
new_file_name = fn.format(i, file_path.split('.')[-1])
to_path = "{}/{}".format(dest_dir, new_file_name)
cmd = sarge.shell_format("cp {0} {1}", file_path, to_path)
sarge.run(cmd)
i += 1
incl_gfx = "\\includegraphics{}{{{}}}"
file_path = yield incl_gfx.format(options, new_file_name)
def carto_css(mml, name):
"""
Takes MML string input and writes it to a Mapnik XML file.
:param mml: an mml string, containing the proper CartoCSS styling and connection attributes required by the
CartoCSS conversion program
:param name: the unique name of the layer (standard method is to name it with its database schema and table name)
:return mapfile: a cascadenik-ready document.
"""
from sarge import shell_format, run, Capture
create_static_content_subdir('cartocss')
mml_file = "{0}/cartocss/{1}.mml".format(settings.MEDIA_ROOT, name)
xml_file = mml_file.replace(".mml", ".xml")
f = open(mml_file, 'w+')
f.write(mml)
f.close()
if not settings.FOOTPRINT_INIT:
carto_css_command = shell_format("{0}/carto {1}".format(
settings.BIN_DIR, mml_file))
logger.debug("Running carto: %s" % carto_css_command)
carto_css_content = None
try:
carto_result = capture_both(carto_css_command)
assert not any(carto_result.returncodes)
carto_css_content = carto_result.stdout.text
logger.debug("Carto xml content: %s" % carto_css_command)
f = open(xml_file, 'w')
f.write(carto_css_content)
f.close()
except AssertionError, e:
logger.error(
"Failed to generate cartocss for {mml}. Exception: {message}. {carto_output}"
.format(mml=mml_file,
message=carto_result.stderr.text,
carto_output=carto_css_content))
raise e
def callScript(self, panValue, tiltValue):
self.panValue = max(self._settings.get(["pan", "minValue"]), min(self._settings.get(["pan", "maxValue"]), panValue))
self.tiltValue = max(self._settings.get(["tilt", "minValue"]), min(self._settings.get(["tilt", "maxValue"]), tiltValue))
# if there are anly pantilt handlers, loop through them, then return
if len(self.pantiltHandlers) > 0:
values = {'pan': self.panValue, 'panMin': self._settings.get(["pan", "minValue"]),
'panMax': self._settings.get(["pan", "maxValue"]),
'tilt': self.tiltValue, 'tiltMin': self._settings.get(["tilt", "minValue"]),
'tiltMax': self._settings.get(["tilt", "maxValue"])}
for name, handler in self.pantiltHandlers.items():
handler(values)
return
script = self._settings.get(["pathToScript"])
if script == "":
return
self._logger.info("Performing command: {}".format(script))
try:
# we run this with shell=True since we have to trust whatever
# our admin configured as command and since we want to allow
# shell-alike handling here...
p = sarge.run(sarge.shell_format('{0} {1} {2}', script, panValue, tiltValue),
stdout=sarge.Capture(),
stderr=sarge.Capture(),
shell=True,
async=False)
if p.returncode != 0:
returncode = p.returncode
stdout_text = p.stdout.text
stderr_text = p.stderr.text
error = "Command failed with return code {}:\nSTDOUT: {}\nSTDERR: {}".format(returncode, stdout_text, stderr_text)
self._logger.warn(error)
except Exception, e:
error = "Command failed: {}".format(str(e))
self._logger.warn(error)
def check_google_dkim(domain, prefix=DEFAULT_GOOGLE_DKIM_PREFIX):
"""
Check a Google DKIM DNS TXT record
Args:
domain (str): DNS domain name
prefix (str): DKIM ``s=`` selector ('DKIM prefix')
Returns:
int: 0 if OK, 1 on error
| https://support.google.com/a/answer/174126
| https://admin.google.com/AdminHome?fral=1#AppDetails:service=email&flyout=dkim
.. note:: This check function only finds "v=DKIM1" TXT records;
it defaults to the default ``google`` prefix
and **does not validate DKIM signatures**.
| http://dkim.org/specs/rfc4871-dkimbase.html#rfc.section.3.6.2.1
| http://dkim.org/specs/rfc4871-dkimbase.html#rfc.section.A.3
"""
dkim_record_name = "%s._domainkey.%s" % (prefix, domain)
cmd = sarge.shell_format("dig {0} txt +short", dkim_record_name)
log.info('cmd', op='check_google_dkim', cmd=cmd)
proc = sarge.capture_both(cmd)
output = proc.stdout.text.rstrip().split('\n')
for line in output:
log.debug('TXT', record=line)
expected = u"\"v=DKIM1" # ... "\; p=none\; rua=mailto:"
if line.startswith(expected):
if 'k=' in line and 'p=' in line:
return 0
errmsg = "%s is not a valid DKIM record" % (output)
log.error('err', msg=errmsg)
return 1
def scratch_info(self):
if hasattr(self, "_scratch_info"):
return self._scratch_info
# Create the org if it hasn't already been created
if not self.created:
self.create_org()
self.logger.info("Getting scratch org info from Salesforce DX")
# Call force:org:display and parse output to get instance_url and
# access_token
command = sarge.shell_format("sfdx force:org:display -u {0} --json",
self.username)
p = sarge.Command(
command,
stderr=sarge.Capture(buffer_size=-1),
stdout=sarge.Capture(buffer_size=-1),
shell=True,
)
p.run()
org_info = None
stderr_list = [line.strip() for line in p.stderr]
stdout_list = [line.strip() for line in p.stdout]
if p.returncode:
self.logger.error("Return code: {}".format(p.returncode))
for line in stderr_list:
self.logger.error(line)
for line in stdout_list:
self.logger.error(line)
message = "\nstderr:\n{}".format("\n".join(stderr_list))
message += "\nstdout:\n{}".format("\n".join(stdout_list))
raise ScratchOrgException(message)
else:
json_txt = "".join(stdout_list)
try:
org_info = json.loads("".join(stdout_list))
except Exception as e:
raise ScratchOrgException(
"Failed to parse json from output. This can happen if "
"your scratch org gets deleted.\n "
"Exception: {}\n Output: {}".format(
e.__class__.__name__, "".join(stdout_list)))
org_id = org_info["result"]["accessToken"].split("!")[0]
if "password" in org_info["result"] and org_info["result"]["password"]:
password = org_info["result"]["password"]
else:
password = self.config.get("password")
self._scratch_info = {
"instance_url": org_info["result"]["instanceUrl"],
"access_token": org_info["result"]["accessToken"],
"org_id": org_id,
"username": org_info["result"]["username"],
"password": password,
}
self.config.update(self._scratch_info)
self._scratch_info_date = datetime.datetime.utcnow()
return self._scratch_info
def test_shlex_issue_31(self):
cmd = "python -c 'print('\''ok'\'')'"
list(shell_shlex(cmd, control='();>|&', posix=True))
shell_format("python -c {0}", "print('ok')")
list(shell_shlex(cmd, control='();>|&', posix=True))
def test_formatter(self):
self.assertEqual(shell_format('ls {0}', '*.py'), "ls '*.py'")
self.assertEqual(shell_format('ls {0!s}', '*.py'), "ls *.py")
def run(command, *args, **kwargs):
command = sarge.shell_format(command, *args)
log.info('run process: %s', command)
return sarge.run(command, stdout=Capture(), stderr=Capture(), **kwargs)
def scratch_info(self):
if hasattr(self, '_scratch_info'):
return self._scratch_info
# Create the org if it hasn't already been created
if not self.created:
self.create_org()
self.logger.info('Getting scratch org info from Salesforce DX')
# Call force:org:display and parse output to get instance_url and
# access_token
command = sarge.shell_format('sfdx force:org:display -u {0} --json', self.username)
p = sarge.Command(
command,
stderr=sarge.Capture(buffer_size=-1),
stdout=sarge.Capture(buffer_size=-1),
shell=True
)
p.run()
org_info = None
stderr_list = [line.strip() for line in p.stderr]
stdout_list = [line.strip() for line in p.stdout]
if p.returncode:
self.logger.error('Return code: {}'.format(p.returncode))
for line in stderr_list:
self.logger.error(line)
for line in stdout_list:
self.logger.error(line)
message = '\nstderr:\n{}'.format('\n'.join(stderr_list))
message += '\nstdout:\n{}'.format('\n'.join(stdout_list))
raise ScratchOrgException(message)
else:
json_txt = ''.join(stdout_list)
try:
org_info = json.loads(''.join(stdout_list))
except Exception as e:
raise ScratchOrgException(
'Failed to parse json from output. This can happen if '
'your scratch org gets deleted.\n '
'Exception: {}\n Output: {}'.format(
e.__class__.__name__,
''.join(stdout_list),
)
)
org_id = org_info['result']['accessToken'].split('!')[0]
if 'password' in org_info['result'] and org_info['result']['password']:
password = org_info['result']['password']
else:
password = self.config.get('password')
self._scratch_info = {
'instance_url': org_info['result']['instanceUrl'],
'access_token': org_info['result']['accessToken'],
'org_id': org_id,
'username': org_info['result']['username'],
'password': password,
}
self.config.update(self._scratch_info)
self._scratch_info_date = datetime.datetime.utcnow()
return self._scratch_info
def create_org(self):
""" Uses sfdx force:org:create to create the org """
if not self.config_file:
raise ScratchOrgException(
f"Scratch org config {self.name} is missing a config_file")
if not self.scratch_org_type:
self.config["scratch_org_type"] = "workspace"
# If the scratch org definition itself contains an `adminEmail` entry,
# we don't want to override it from our own configuration, which may
# simply come from the user's Git config.
with open(self.config_file, "r") as org_def:
org_def_data = json.load(org_def)
org_def_has_email = "adminEmail" in org_def_data
devhub = self._choose_devhub()
instance = self.instance or os.environ.get("SFDX_SIGNUP_INSTANCE")
options = {
"config_file":
self.config_file,
"devhub":
f" --targetdevhubusername {devhub}" if devhub else "",
"namespaced":
" -n" if not self.namespaced else "",
"days":
f" --durationdays {self.days}" if self.days else "",
"wait":
" -w 120",
"alias":
sarge.shell_format(' -a "{0!s}"', self.sfdx_alias)
if self.sfdx_alias else "",
"email":
sarge.shell_format(' adminEmail="{0!s}"', self.email_address)
if self.email_address and not org_def_has_email else "",
"default":
" -s" if self.default else "",
"instance":
f" instance={instance}" if instance else "",
"extraargs":
os.environ.get("SFDX_ORG_CREATE_ARGS", ""),
}
# This feels a little dirty, but the use cases for extra args would mostly
# work best with env vars
command = "force:org:create -f {config_file}{devhub}{namespaced}{days}{alias}{default}{wait}{email}{instance} {extraargs}".format(
**options)
p = sfdx(command, username=None, log_note="Creating scratch org")
stderr = [line.strip() for line in p.stderr_text]
stdout = [line.strip() for line in p.stdout_text]
if p.returncode:
message = f"{FAILED_TO_CREATE_SCRATCH_ORG}: \n{nl.join(stdout)}\n{nl.join(stderr)}"
raise ScratchOrgException(message)
re_obj = re.compile(
"Successfully created scratch org: (.+), username: (.+)")
username = None
for line in stdout:
match = re_obj.search(line)
if match:
self.config["org_id"] = match.group(1)
self.config["username"] = username = match.group(2)
self.logger.info(line)
for line in stderr:
self.logger.error(line)
if username is None:
raise ScratchOrgException(
"SFDX claimed to be successful but there was no username "
"in the output...maybe there was a gack?")
self.config["date_created"] = datetime.datetime.utcnow()
if self.config.get("set_password"):
self.generate_password()
# Flag that this org has been created
self.config["created"] = True
def get_free_space(location):
p = run(shell_format('du -hs {0}', location), stdout=Capture())
return p.stdout.text.split()[0]
def run(command, *args, **kwargs):
command = sarge.shell_format(command, *args)
log.info('run process: %s', command)
return sarge.run(command, stdout=Capture(), stderr=Capture(), **kwargs)
def settings2options(settingsdict):
options = [
sarge.shell_format("--{} {}", key, val)
for (key, val) in settingsdict.items()
]
return " ".join(options)
def create_org(self):
""" Uses sfdx force:org:create to create the org """
if not self.config_file:
# FIXME: raise exception
return
if not self.scratch_org_type:
self.config["scratch_org_type"] = "workspace"
# If the scratch org definition itself contains an `adminEmail` entry,
# we don't want to override it from our own configuration, which may
# simply come from the user's Git config.
with open(self.config_file, "r") as org_def:
org_def_data = json.load(org_def)
org_def_has_email = "adminEmail" in org_def_data
options = {
"config_file":
self.config_file,
"devhub":
" --targetdevhubusername {}".format(self.devhub)
if self.devhub else "",
"namespaced":
" -n" if not self.namespaced else "",
"days":
" --durationdays {}".format(self.days) if self.days else "",
"alias":
sarge.shell_format(' -a "{0!s}"', self.sfdx_alias)
if self.sfdx_alias else "",
"email":
sarge.shell_format('adminEmail="{0!s}"', self.email_address)
if self.email_address and not org_def_has_email else "",
"default":
" -s" if self.default else "",
"extraargs":
os.environ.get("SFDX_ORG_CREATE_ARGS", ""),
}
# This feels a little dirty, but the use cases for extra args would mostly
# work best with env vars
command = "force:org:create -f {config_file}{devhub}{namespaced}{days}{alias}{default} {email} {extraargs}".format(
**options)
p = sfdx(command, username=None, log_note="Creating scratch org")
stderr = [line.strip() for line in p.stderr_text]
stdout = [line.strip() for line in p.stdout_text]
if p.returncode:
message = "{}: \n{}\n{}".format(FAILED_TO_CREATE_SCRATCH_ORG,
"\n".join(stdout),
"\n".join(stderr))
raise ScratchOrgException(message)
re_obj = re.compile(
"Successfully created scratch org: (.+), username: (.+)")
for line in stdout:
match = re_obj.search(line)
if match:
self.config["org_id"] = match.group(1)
self.config["username"] = match.group(2)
self.logger.info(line)
for line in stderr:
self.logger.error(line)
self.config["date_created"] = datetime.datetime.now()
if self.config.get("set_password"):
self.generate_password()
# Flag that this org has been created
self.config["created"] = True
def test_formatter(self):
self.assertEqual(shell_format("ls {0}", "*.py"), "ls '*.py'")
self.assertEqual(shell_format("ls {0!s}", "*.py"), "ls *.py")
