def inner(*args, **kwargs):
sys_monitor = ResourceMonitor().start()
start = time.time()
result = fn(*args, **kwargs)
elapsed = time.time() - start
duration_str = '%.2f' % elapsed
sys_monitor.stop()
mem_str = 'mem: %s%%(peak:%s%%) cpu:%d%%' % \
(sys_monitor.current_memory, sys_monitor.peak_memory, int(sys_monitor.avg_cpu_load))
ret_str = variable_info(result)
stack_depth = get_stack_depth()
fn_str = function_name_with_class(fn)
msg = ' ' * stack_depth + \
'%s, elapsed: %s, returned: %s, sys %s' % \
(fn_str, duration_str, ret_str, mem_str)
if elapsed >= LOGGING_VERBOSITY.min_time:
simple_logger.log(LOGGING_VERBOSITY.level, msg)
return result
def inner(*args, **kwargs):
start = time.time()
result = fn(*args, **kwargs)
duration = time.time() - start
simple_logger.log(LOGGING_VERBOSITY.level, log_format, fn.__name__,
duration)
return result
def unpickle(self, remote_path, in_memory=True):
logger.log(self.log_level, f'S3: unpickling from {remote_path}')
if in_memory:
return pickle.loads(self.read(remote_path))
else:
with tempfile.NamedTemporaryFile(delete=True) as temp:
self._download_through_disk(remote_path, temp)
return pickle.load(temp)
def remote_to_local(self, remote_path, local_path, overwrite=True, in_memory=True):
if not os.path.exists(local_path) or overwrite:
os.makedirs(os.path.dirname(local_path), exist_ok=True)
logger.log(self.log_level, f'S3: copying from {remote_path} to {local_path}')
with open(local_path, 'wb') as local:
if in_memory:
local.write(self.read(remote_path))
else:
self._download_through_disk(remote_path, local)
def inner(*args, **kwargs):
sys_monitor = ResourceMonitor().start()
start = time.time()
result = fn(*args, **kwargs)
elapsed = time.time() - start
sys_monitor.stop()
if elapsed >= LOGGING_VERBOSITY.min_time:
msg = ' ' * get_stack_depth() + \
f'{function_name_with_class(fn)}, elapsed: {elapsed:.2f}, ' \
f'returned: {variable_info(result)}, sys mem: {sys_monitor.current_memory}%' \
f'(peak:{sys_monitor.peak_memory}%) cpu:{int(sys_monitor.avg_cpu_load)}%'
simple_logger.log(LOGGING_VERBOSITY.level, msg)
return result
def local_to_remote(self, local_path, remote_path, compress=True):
logger.log(self.log_level, f'S3: copying from {local_path} to {remote_path}')
with open(local_path, 'rb') as local:
self.write_binary(local.read(), remote_path, compress=compress)
def pickle(self, obj, remote_path, compress=True):
logger.log(self.log_level, f'S3: pickling to {remote_path}')
return self.write_binary(pickle.dumps(obj), remote_path, compress=compress)