def env(self):
# strategy: create a tempfile, let python dump its full env in a subprocess. and load the
# env file again afterwards
script = self.script
if script not in self._envs:
with tmp_file() as tmp:
tmp_path = os.path.realpath(tmp[1])
cmd = "bash -l -c 'source \"{0}\"; python -c \"" \
"import os,pickle;pickle.dump(os.environ,open(\\\"{1}\\\",\\\"w\\\"))\"'"
cmd = cmd.format(script, tmp_path)
returncode, out, _ = interruptable_popen(
cmd,
shell=True,
executable="/bin/bash",
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if returncode != 0:
raise Exception("bash sandbox env loading failed: " +
str(out))
with open(tmp_path, "r") as f:
env = six.moves.cPickle.load(f)
# add env variables defined in the config
env.update(self.get_config_env())
# add env variables defined by the task
env.update(self.get_task_env())
# cache
self._envs[script] = env
return self._envs[script]
def renew_arc_proxy(password="", lifetime="8 days", proxy_file=None):
"""
Renews the arc proxy using a password *password* and a default *lifetime* of 8 days, which is
internally parsed by :py:func:`law.util.parse_duration` where the default input unit is hours.
To ensure that the *password* it is not visible in any process listing, it is written to a
temporary file first and piped into the ``arcproxy`` command. When *proxy_file* is *None*, it
defaults to the result of :py:func:`get_arc_proxy_file`. Otherwise, when it evaluates to
*False*, ``arcproxy`` is invoked without a custom proxy file.
"""
# convert the lifetime to seconds
lifetime_seconds = int(parse_duration(lifetime, input_unit="h", unit="s"))
if proxy_file is None:
proxy_file = get_arc_proxy_file()
args = "--constraint=validityPeriod={}".format(lifetime_seconds)
if proxy_file:
proxy_file = os.path.expandvars(os.path.expanduser(proxy_file))
args += " --proxy={}".format(proxy_file)
with tmp_file() as (_, tmp):
with open(tmp, "w") as f:
f.write(password)
cmd = "arcproxy --passwordsource=key=file:{} {}".format(tmp, args)
code, out, _ = interruptable_popen(cmd,
shell=True,
executable="/bin/bash",
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if code != 0:
raise Exception("arcproxy failed: {}".format(out))
def env(self):
# strategy: create a tempfile, forward it to a container, let python dump its full env,
# close the container and load the env file
if self.image not in self._envs:
with tmp_file() as tmp:
tmp_path = os.path.realpath(tmp[1])
env_path = os.path.join("/tmp", str(hash(tmp_path))[-8:])
cmd = "docker run --rm -v {1}:{2} {0} python -c \"" \
"import os,pickle;pickle.dump(os.environ,open('{2}','w'))\""
cmd = cmd.format(self.image, tmp_path, env_path)
returncode, out, _ = interruptable_popen(
cmd,
shell=True,
executable="/bin/bash",
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if returncode != 0:
raise Exception("docker sandbox env loading failed: " +
str(out))
with open(tmp_path, "r") as f:
env = six.moves.cPickle.load(f)
# add env variables defined in the config
env.update(self.get_config_env())
# add env variables defined by the task
env.update(self.get_task_env())
# cache
self._envs[self.image] = env
return self._envs[self.image]
def env(self):
# strategy: create a tempfile, let python dump its full env in a subprocess and load the
# env file again afterwards
script = self.script
if script not in self._envs:
with tmp_file() as tmp:
tmp_path = os.path.realpath(tmp[1])
# get the bash command
bash_cmd = self._bash_cmd()
# build commands to setup the environment
setup_cmds = self._build_setup_cmds(self._get_env())
# build the python command that dumps the environment
py_cmd = "import os,pickle;" \
+ "pickle.dump(dict(os.environ),open('{}','wb'),protocol=2)".format(tmp_path)
# build the full command
cmd = quote_cmd(bash_cmd + [
"-c",
"; ".join(
flatten("source \"{}\" \"\"".format(
self.script), setup_cmds,
quote_cmd(["python", "-c", py_cmd]))),
])
# run it
returncode = interruptable_popen(cmd,
shell=True,
executable="/bin/bash")[0]
if returncode != 0:
raise Exception("bash sandbox env loading failed")
# load the environment from the tmp file
pickle_kwargs = {"encoding": "utf-8"} if six.PY3 else {}
with open(tmp_path, "rb") as f:
env = collections.OrderedDict(
six.moves.cPickle.load(f, **pickle_kwargs))
# cache it
self._envs[script] = env
return self._envs[script]
def env(self):
# strategy: create a tempfile, forward it to a container, let python dump its full env,
# close the container and load the env file
if self.image not in self._envs:
with tmp_file() as tmp:
tmp_path = os.path.realpath(tmp[1])
env_path = os.path.join("/tmp", str(hash(tmp_path))[-8:])
# build commands to setup the environment
setup_cmds = self._build_setup_cmds(self._get_env())
# arguments to configure the environment
args = ["-v", "{}:{}".format(tmp_path, env_path)
] + self.common_args()
# build the command
py_cmd = "import os,pickle;" \
+ "pickle.dump(dict(os.environ),open('{}','wb'),protocol=2)".format(env_path)
cmd = quote_cmd(["docker", "run"] + args + [
self.image,
"bash",
"-l",
"-c",
"; ".join(
flatten(setup_cmds, quote_cmd(["python", "-c", py_cmd
]))),
])
# run it
returncode = interruptable_popen(cmd,
shell=True,
executable="/bin/bash")[0]
if returncode != 0:
raise Exception("docker sandbox env loading failed")
# load the environment from the tmp file
with open(tmp_path, "rb") as f:
env = six.moves.cPickle.load(f)
# cache
self._envs[self.image] = env
return self._envs[self.image]
def renew_voms_proxy(passwd="", vo=None, lifetime="196:00"):
"""
Renews the voms proxy using a password *passwd*, an optional virtual organization name *vo*, and
a default *lifetime* of 8 days. The password is written to a temporary file first and piped into
the renewal commad to ensure it is not visible in the process list.
"""
with tmp_file() as (_, tmp):
with open(tmp, "w") as f:
f.write(passwd)
cmd = "cat '{}' | voms-proxy-init --valid '{}'".format(tmp, lifetime)
if vo:
cmd += " -voms '{}'".format(vo)
code, out, _ = interruptable_popen(cmd,
shell=True,
executable="/bin/bash",
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if code != 0:
raise Exception("proxy renewal failed: {}".format(out))
def renew_voms_proxy(password="", vo=None, lifetime="8 days", proxy_file=None):
"""
Renews the voms proxy using a password *password*, an optional virtual organization name *vo*,
and a default *lifetime* of 8 days, which is internally parsed by
:py:func:`law.util.parse_duration` where the default input unit is hours. To ensure that the
*password* is not visible in any process listing, it is written to a temporary file first and
piped into the ``voms-proxy-init`` command. When *proxy_file* is *None*, it defaults to the
result of :py:func:`get_voms_proxy_file`.
"""
# parse and format the lifetime
lifetime_seconds = max(parse_duration(lifetime, input_unit="h", unit="s"),
60.0)
lifetime = human_duration(seconds=lifetime_seconds, colon_format="h")
# cut the seconds part
normalized = ":".join((2 - lifetime.count(":")) * ["00"] + [""]) + lifetime
lifetime = ":".join(normalized.rsplit(":", 3)[-3:-1])
# when proxy_file is None, get the default
# when empty string, don't add a --out argument
if proxy_file is None:
proxy_file = get_voms_proxy_file()
with tmp_file() as (_, tmp):
with open(tmp, "w") as f:
f.write(password)
cmd = "cat '{}' | voms-proxy-init --valid '{}'".format(tmp, lifetime)
if vo:
cmd += " -voms '{}'".format(vo)
if proxy_file:
proxy_file = os.path.expandvars(os.path.expanduser(proxy_file))
cmd += " --out '{}'".format(proxy_file)
code, out, _ = interruptable_popen(cmd,
shell=True,
executable="/bin/bash",
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if code != 0:
raise Exception("voms-proxy-init failed: {}".format(out))