def upload(environ, key, source, dest=None, **kwargs):
"""Uploads the file from ``source`` to filetracker under ``environ[key]``
name.
``source``
Filename to upload.
``dest``
A filename, directory name or ``None``. In the two latter cases,
the file is named the same as in ``environ[key]``.
``**kwargs``
Passed directly to :meth:`filetracker.client.Client.put_file`.
See the note about ``environ['use_filetracker']`` in
:func:`sio.workers.ft.download`.
Returns the filetracker path to the saved file.
"""
if dest is None or key in environ:
dest = environ[key]
elif dest.endswith(os.sep):
dest = os.path.join(dest, os.path.split(source)[1])
if not _use_filetracker(dest, environ):
dest = os.path.join(_original_cwd, dest)
if not os.path.exists(dest) or not os.path.samefile(source, dest):
shutil.copy(source, dest)
else:
logger.debug("Uploading %s", dest)
perf_timer = util.PerfTimer()
dest = instance().put_file(dest, source, **kwargs)
logger.debug(" completed in %.2fs", perf_timer.elapsed)
environ[key] = dest
return dest
def download(environ, key, dest=None, skip_if_exists=False, **kwargs):
"""Downloads the file from ``environ[key]`` and saves it to ``dest``.
``dest``
A filename, directory name or ``None``. In the two latter cases,
the file is named the same as in ``environ[key]``.
``skip_if_exists``
If ``True`` and ``dest`` points to an existing file (not a directory
or ``None``), then the file is not downloaded.
``**kwargs``
Passed directly to :meth:`filetracker.client.Client.get_file`.
The value under ``environ['use_filetracker']`` affects downloading
in the followins way:
* if ``True``, nothing special happens
* if ``False``, the file is not downloaded from filetracker, but the
passed path is assumed to be a regular filesystem path
* if ``'auto'``, the file is assumed to be a local filename only if
it is a relative path (this is usually the case when developers play).
Returns the path to the saved file.
"""
if dest and skip_if_exists and os.path.exists(util.tempcwd(dest)):
return dest
source = environ[key]
if dest is None:
dest = os.path.split(source)[1]
elif dest.endswith(os.sep):
dest = os.path.join(dest, os.path.split(source)[1])
dest = util.tempcwd(dest)
if not _use_filetracker(source, environ):
source = os.path.join(_original_cwd, source)
if not os.path.exists(dest) or not os.path.samefile(source, dest):
shutil.copy(source, dest)
else:
kwargs.setdefault('add_to_cache', False)
logger.debug("Downloading %s", source)
perf_timer = util.PerfTimer()
instance().get_file(source, dest, **kwargs)
logger.debug(" completed in %.2fs", perf_timer.elapsed)
return dest
def execute_command(command,
env=None,
split_lines=False,
stdin=None,
stdout=None,
stderr=None,
forward_stderr=False,
capture_output=False,
output_limit=None,
real_time_limit=None,
ignore_errors=False,
extra_ignore_errors=(),
**kwargs):
"""Utility function to run arbitrary command.
``stdin``
Could be either file opened with ``open(fname, 'r')``
or None (then it is inherited from current process).
``stdout``, ``stderr``
Could be files opened with ``open(fname, 'w')``, sys.std*
or None - then it's suppressed.
``forward_stderr``
Forwards stderr to stdout.
``capture_output``
Returns program output in renv key ``stdout``.
``output_limit``
Limits returned output when ``capture_output=True`` (in bytes).
Returns renv: dictionary containing:
``real_time_used``
Wall clock time it took to execute the command (in ms).
``return_code``
Status code that program returned.
``real_time_killed``
Only when process was killed due to exceeding real time limit.
``stdout``
Only when ``capture_output=True``: output of the command
"""
# Using temporary file is way faster than using subproces.PIPE
# and it prevents deadlocks.
command = shellquote(command)
logger.debug('Executing: %s', command)
stdout = capture_output and tempfile.TemporaryFile() or stdout
# redirect output to /dev/null if None given
devnull = open(os.devnull, 'wb')
stdout = stdout or devnull
stderr = stderr or devnull
ret_env = {}
if env is not None:
for key, value in six.iteritems(env):
env[key] = str(value)
perf_timer = util.PerfTimer()
p = subprocess.Popen(
command,
stdin=stdin,
stdout=stdout,
stderr=forward_stderr and subprocess.STDOUT or stderr,
shell=True,
close_fds=True,
universal_newlines=True,
env=env,
cwd=tempcwd(),
preexec_fn=os.setpgrp,
)
kill_timer = None
if real_time_limit:
def oot_killer():
ret_env['real_time_killed'] = True
os.killpg(p.pid, signal.SIGKILL)
kill_timer = Timer(ms2s(real_time_limit), oot_killer)
kill_timer.start()
rc = p.wait()
ret_env['return_code'] = rc
if kill_timer:
kill_timer.cancel()
ret_env['real_time_used'] = s2ms(perf_timer.elapsed)
logger.debug(
'Command "%s" exited with code %d, took %.2fs',
str(command),
rc,
perf_timer.elapsed,
)
devnull.close()
if capture_output:
stdout.seek(0)
ret_env['stdout'] = stdout.read(output_limit or -1)
stdout.close()
if split_lines:
ret_env['stdout'] = ret_env['stdout'].split(b'\n')
if rc and not ignore_errors and rc not in extra_ignore_errors:
raise ExecError(
'Failed to execute command: %s. Returned with code %s\n' %
(command, rc))
return ret_env