def __init__(self, *args, **kwargs):
self.logger = kwargs.pop('logger', None)
self.logPipeOut = None
self.logPipeErr = None
if self.logger:
from vsi.tools.python import args_to_kwargs, command_list_to_string
from subprocess import Popen as Popen_orig
kwargs_view = args_to_kwargs(Popen_orig, args, kwargs)
self.logger.debug('Popen cmd: %s' % command_list_to_string(kwargs_view['args']))
if 'stderr' not in kwargs:
self.logPipeErr=LogPipe(self.logger, STDERR_LEVEL, STDERR_PREAMBLE);
kwargs['stderr'] = self.logPipeErr
if 'stdout' not in kwargs:
self.logPipeOut=LogPipe(self.logger, STDOUT_LEVEL, STDOUT_PREAMBLE)
kwargs['stdout'] = self.logPipeOut
super(Popen, self).__init__(*args, **kwargs)
#Start the loggers (which close the parent copy of write side of the pipe
if self.logPipeErr:
self.logPipeErr.start()
if self.logPipeOut:
self.logPipeOut.start()
def __init__(self, *args, **kwargs):
self.logger = kwargs.pop('logger', None)
self.logPipeOut = None
self.logPipeErr = None
if self.logger:
from vsi.tools.python import args_to_kwargs, command_list_to_string
from subprocess import Popen as Popen_orig
kwargs_view = args_to_kwargs(Popen_orig, args, kwargs)
self.logger.debug('Popen cmd: %s' %
command_list_to_string(kwargs_view['args']))
if 'stderr' not in kwargs:
self.logPipeErr = LogPipe(self.logger, STDERR_LEVEL,
STDERR_PREAMBLE)
kwargs['stderr'] = self.logPipeErr
if 'stdout' not in kwargs:
self.logPipeOut = LogPipe(self.logger, STDOUT_LEVEL,
STDOUT_PREAMBLE)
kwargs['stdout'] = self.logPipeOut
super(Popen, self).__init__(*args, **kwargs)
#Start the loggers (which close the parent copy of write side of the pipe
if self.logPipeErr:
self.logPipeErr.start()
if self.logPipeOut:
self.logPipeOut.start()
def test_arg_to_kwargs(self):
def a(x, y, z):
pass
def f(x, y, z=18):
pass
def b(x, *args):
pass
def c(x=15, **kwargs):
pass
def d(a, y=15, *args, **kwargs):
pass
def e(a, **kwargs):
pass
def g(x=11):
pass
def h(*args):
pass
def i(**kwargs):
pass
class A(object):
def __init__(self, a, b=15, *args, **kwargs):
pass
def fun(self, a, b=151, *args, **kwargs):
pass
def __call__(self, a, b=152, *args, **kwargs):
pass
@staticmethod
def stat(a, b=153, *args, **kwargs):
pass
@classmethod
def classy(cls, a, b=157, *args, **kwargs):
pass
aa = A(1)
tests = ((a, [1, 2,
3], {}), (f, [1, 2,
3], {}), (f, [1, 2], {}), (f, [1, 2], {
'z': 22
}), (b, [1], {}), (b, [1, 2,
3], {}), (c, [1], {
'w': 22
}), (c, [], {
'x': 11,
'w': 22
}), (c, [], {
'w': 22
}), (d, [11], {}),
(d, [11, 12], {}), (d, [11, 12, 13, 14], {}), (d, [11], {
'x': 15,
'y': 16
}), (d, [], {
'a': 10,
'x': 16
}), (d, [11, 12, 13, 14], {
'x': 15,
'z': 37
}), (e, [1], {
'x': 14
}), (e, [], {
'a': 2,
'x': 14
}), (g, [], {}), (g, [1], {}), (h, [], {}),
(h, [100, 202, 303], {}), (i, [], {}), (i, [], {
'a': 31,
'b': 29
}), (A, [11, 22, 33], {
'x': 14
}), (A, [11], {}), (aa.fun, [13, 23, 34], {
'x': 16
}), (aa.fun, [99], {}), (aa, [14, 24, 35], {
'x': 17
}), (aa, [98], {}), (aa.stat, [12, 33, 44], {
'y': 35
}), (aa.stat, [21], {}), (aa.classy, [22, 34, 45], {
'y': 53
}), (aa.classy, [27], {}), (d, [111, 222, 333], {
'xx': 92,
'args': 28
})) #This is valid python
answers = ({
'y': 2,
'x': 1,
'z': 3
}, {
'y': 2,
'x': 1,
'z': 3
}, {
'y': 2,
'x': 1,
'z': 18
}, {
'y': 2,
'x': 1,
'z': 22
}, {
'x': 1,
ARGS: ()
}, {
'x': 1,
ARGS: (2, 3)
}, {
'x': 1,
KWARGS: {
'w': 22
}
}, {
'x': 11,
KWARGS: {
'w': 22
}
}, {
'x': 15,
KWARGS: {
'w': 22
}
}, {
'a': 11,
'y': 15,
KWARGS: {},
ARGS: ()
}, {
'a': 11,
'y': 12,
KWARGS: {},
ARGS: ()
}, {
'a': 11,
'y': 12,
KWARGS: {},
ARGS: (13, 14)
}, {
'a': 11,
'y': 16,
KWARGS: {
'x': 15
},
ARGS: ()
}, {
'a': 10,
'y': 15,
KWARGS: {
'x': 16
},
ARGS: ()
}, {
'a': 11,
'y': 12,
KWARGS: {
'x': 15,
'z': 37
},
ARGS: (13, 14)
}, {
'a': 1,
KWARGS: {
'x': 14
}
}, {
'a': 2,
KWARGS: {
'x': 14
}
}, {
'x': 11
}, {
'x': 1
}, {
ARGS: ()
}, {
ARGS: (100, 202, 303)
}, {
KWARGS: {}
}, {
KWARGS: {
'a': 31,
'b': 29
}
}, {
'a': 11,
'b': 22,
KWARGS: {
'x': 14
},
ARGS: (33, )
}, {
'a': 11,
'b': 15,
KWARGS: {},
ARGS: ()
}, {
'a': 13,
'b': 23,
KWARGS: {
'x': 16
},
ARGS: (34, )
}, {
'a': 99,
'b': 151,
KWARGS: {},
ARGS: ()
}, {
'a': 14,
'b': 24,
KWARGS: {
'x': 17
},
ARGS: (35, )
}, {
'a': 98,
'b': 152,
KWARGS: {},
ARGS: ()
}, {
'a': 12,
'b': 33,
KWARGS: {
'y': 35
},
ARGS: (44, )
}, {
'a': 21,
'b': 153,
KWARGS: {},
ARGS: ()
}, {
'a': 22,
'b': 34,
KWARGS: {
'y': 53
},
ARGS: (45, )
}, {
'a': 27,
'b': 157,
KWARGS: {},
ARGS: ()
}, {
'a': 111,
'y': 222,
ARGS: (333, ),
KWARGS: {
'xx': 92,
'args': 28
}
})
for test, answer in zip(tests, answers):
self.assertEqual(args_to_kwargs(test[0], test[1], test[2]), answer)
self.assertEqual(args_to_kwargs_easy(test[0], *test[1], **test[2]),
answer)
tests = ((A, 'fun', [10, 21, 32], {
'x': 15
}), (A, 'fun', [100], {}), (A, 'stat', [12, 33, 44], {
'y': 35
}), (A, 'stat', [21], {}), (A, 'classy', [22, 34, 45], {
'y': 53
}), (A, 'classy', [27], {}))
answers = ({
'a': 10,
'b': 21,
KWARGS: {
'x': 15
},
ARGS: (32, )
}, {
'a': 100,
'b': 151,
KWARGS: {},
ARGS: ()
}, {
'a': 12,
'b': 33,
KWARGS: {
'y': 35
},
ARGS: (44, )
}, {
'a': 21,
'b': 153,
KWARGS: {},
ARGS: ()
}, {
'a': 22,
'b': 34,
KWARGS: {
'y': 53
},
ARGS: (45, )
}, {
'a': 27,
'b': 157,
KWARGS: {},
ARGS: ()
})
for test, answer in zip(tests, answers):
self.assertEqual(
args_to_kwargs_unbound(test[0], test[1], test[2], test[3]),
answer)
self.assertEqual(
args_to_kwargs_unbound_easy(test[0], test[1], *test[2],
**test[3]), answer)
if sys.version_info.major == 2:
value = getattr(test[0], test[1])
self.assertEqual(args_to_kwargs(value, test[2], test[3]),
answer)
self.assertEqual(
args_to_kwargs_easy(value, *test[2], **test[3]), answer)
def test_keyword_only_args(self):
from vsi.tools.python import (args_to_kwargs, args_to_kwargs_unbound,
args_to_kwargs_easy,
args_to_kwargs_unbound_easy, KWARGS,
ARGS)
def test1(a, b=15):
pass
def test2(a, b=15, **kwargs):
pass
def test5(**kwargs):
pass
def test3(a, b=15, *args, c, d=27, **kwargs):
pass
def test4(a, b=15, *args, c, d=27):
pass
tests = (
(test3, [1], {
'c': 3
}),
(test3, [1, 2], {
'c': 3
}),
(test3, [1, 2], {
'c': 3,
'd': 4
}),
(test3, [1, 2], {
'c': 3,
'e': 5
}),
(test4, [1], {
'c': 3
}),
(test4, [1, 2], {
'c': 3
}),
(test4, [1, 2], {
'c': 3,
'd': 4
}),
)
answers = (
{
'a': 1,
'b': 15,
'c': 3,
'd': 27,
KWARGS: {},
ARGS: ()
},
{
'a': 1,
'b': 2,
'c': 3,
'd': 27,
KWARGS: {},
ARGS: ()
},
{
'a': 1,
'b': 2,
'c': 3,
'd': 4,
KWARGS: {},
ARGS: ()
},
{
'a': 1,
'b': 2,
'c': 3,
'd': 27,
KWARGS: {
'e': 5
},
ARGS: ()
},
{
'a': 1,
'b': 15,
'c': 3,
'd': 27,
ARGS: ()
},
{
'a': 1,
'b': 2,
'c': 3,
'd': 27,
ARGS: ()
},
{
'a': 1,
'b': 2,
'c': 3,
'd': 4,
ARGS: ()
},
)
for test, answer in zip(tests, answers):
self.assertEqual(args_to_kwargs(test[0], test[1], test[2]), answer)
self.assertEqual(args_to_kwargs_easy(test[0], *test[1], **test[2]),
answer)
# Technically invalid tests
# Missing positional arg or kwonly arg
tests = (
(test3, [1, 3, 4, 5], {}),
(test3, [1], {}),
(test3, [], {}),
(test3, [], {
'c': 11
}),
(test4, [1, 3, 4, 5], {}),
(test4, [1], {}),
(test4, [], {}),
(test4, [], {
'c': 11
}),
(test4, [], {
'c': 3,
'e': 5
}),
)
answers = (
{
'a': 1,
'b': 3,
'd': 27,
KWARGS: {},
ARGS: (4, 5)
},
{
'a': 1,
'b': 15,
'd': 27,
KWARGS: {},
ARGS: ()
},
{
'b': 15,
'd': 27,
KWARGS: {},
ARGS: ()
},
{
'b': 15,
'c': 11,
'd': 27,
KWARGS: {},
ARGS: ()
},
{
'a': 1,
'b': 3,
'd': 27,
ARGS: (4, 5)
},
{
'a': 1,
'b': 15,
'd': 27,
ARGS: ()
},
{
'b': 15,
'd': 27,
ARGS: ()
},
{
'b': 15,
'c': 11,
'd': 27,
ARGS: ()
},
{
'b': 15,
'c': 3,
'd': 27,
'e': 5,
ARGS: ()
},
)
for test, answer in zip(tests, answers):
with self.assertLogs('vsi.tools.python', level='WARNING') as cm:
self.assertEqual(args_to_kwargs(test[0], test[1], test[2]),
answer)
self.assertTrue(
any('missing required arguments' in x for x in cm.output))
with self.assertLogs('vsi.tools.python', level='WARNING') as cm:
self.assertEqual(
args_to_kwargs_easy(test[0], *test[1], **test[2]), answer)
self.assertTrue(
any('missing required arguments' in x for x in cm.output))
# Too many kwargs
tests = (
(test1, [], {
'a': 11,
'b': 22,
'c': 33
}),
(test1, [21], {
'b': 22,
'c': 33
}),
(test4, [1, 2], {
'c': 3,
'e': 5
}),
(test4, [], {
'c': 3,
'e': 5
}),
)
answers = (
{
'a': 11,
'b': 22,
'c': 33
},
{
'a': 21,
'b': 22,
'c': 33
},
{
'a': 1,
'b': 2,
'c': 3,
'd': 27,
'e': 5,
ARGS: ()
},
{
'b': 15,
'c': 3,
'd': 27,
'e': 5,
ARGS: ()
},
)
for test, answer in zip(tests, answers):
with self.assertLogs('vsi.tools.python', level='WARNING') as cm:
self.assertEqual(args_to_kwargs(test[0], test[1], test[2]),
answer)
self.assertTrue(
any('Unspecified keyword argument' in x for x in cm.output))
with self.assertLogs('vsi.tools.python', level='WARNING') as cm:
self.assertEqual(
args_to_kwargs_easy(test[0], *test[1], **test[2]), answer)
self.assertTrue(
any('Unspecified keyword argument' in x for x in cm.output))
# Too many positional args
tests = (
(test1, [1, 2, 3], {}),
(test2, [1, 2, 3], {}),
(test5, [1], {}),
(test5, [1], {
'a': 15
}),
)
answers = (
{
'a': 1,
'b': 2
},
{
'a': 1,
'b': 2,
KWARGS: {}
},
{
KWARGS: {}
},
{
KWARGS: {
'a': 15
}
},
)
for test, answer in zip(tests, answers):
with self.assertLogs('vsi.tools.python', level='WARNING') as cm:
self.assertEqual(args_to_kwargs(test[0], test[1], test[2]),
answer)
self.assertTrue(
any('Too many positional arguments specified' in x
for x in cm.output))
with self.assertLogs('vsi.tools.python', level='WARNING') as cm:
self.assertEqual(
args_to_kwargs_easy(test[0], *test[1], **test[2]), answer)
self.assertTrue(
any('Too many positional arguments specified' in x
for x in cm.output))
def __call__(self, *args, **kwargs):
# this is only set when apply_async was called.
logger.debug4(
f"Running task: {self} with args {args} and kwargs {kwargs}")
if getattr(self.request, 'settings', None):
if not settings.configured:
# Cover a potential (unlikely) corner case where setting might not be
# configured yet
settings.configure({'processing_dir': gettempdir()})
# Create a settings context, so I can replace it with the task's settings
with settings:
# Calculate the exector's mapped version of the runner's settings
compute_volume_map, reverse_compute_volume_map, \
executor_volume_map, reverse_executor_volume_map = \
self._get_volume_mappings()
# Load the executor version of the runner's settings
settings._wrapped.clear()
settings._wrapped.update(
self.translate_paths(self.request.settings,
reverse_compute_volume_map,
executor_volume_map))
# This is needed here because I just loaded settings from a runner!
settings.terra.zone = 'task'
# Just in case processing dir doesn't exist
if not os.path.exists(settings.processing_dir):
logger.critical(
f'Dir "{settings.processing_dir}" is not accessible '
'by the executor, please make sure the worker has '
'access to this directory')
settings.processing_dir = gettempdir()
logger.warning(
'Using temporary directory: '
f'"{settings.processing_dir}" for the processing dir')
# Calculate the executor's mapped version of the arguments
func, is_method = unwrap_wraps(self)
# Note: This code won't handle decorators that add/remove args. Oh well
# Only have to do this if it's a bound method, but inspect would not
# detect this. This happens when you have multiple layers of decorating
if is_method and not inspect.ismethod(func):
mySelf = object()
kwargs = args_to_kwargs(func, (mySelf, ) + args, kwargs)
self_key = [
key for (key, value) in kwargs.items()
if value == mySelf
][0]
kwargs.pop(self_key)
else:
kwargs = args_to_kwargs(func, args, kwargs)
args_only = kwargs.pop(ARGS, ())
kwargs.update(kwargs.pop(KWARGS, ()))
kwargs = self.translate_paths(kwargs,
reverse_compute_volume_map,
executor_volume_map)
# Set up logger to talk to master controller
terra.logger._logs.reconfigure_logger(pre_run_task=True)
# Don't call func here, you'll miss any other decorators
return_value = self.run(*args_only, **kwargs)
# Calculate the runner mapped version of the executor's return value
return_value = self.translate_paths(
return_value, reverse_executor_volume_map,
compute_volume_map)
else:
# Must call (synchronous) apply or python __call__ with no volume
# mappings
# Use a flag for people who are somehow getting here with settings
# unconfigured
original_zone = None
if settings.configured:
original_zone = settings.terra.zone
settings.terra.zone = 'task'
try:
return_value = self.run(*args, **kwargs)
finally:
if original_zone is not None:
settings.terra.zone = original_zone
return return_value