def inner(*args, **kwargs): sys_monitor = ResourceMonitor().start() start = time.time() result = fn(*args, **kwargs) elapsed = time.time() - start duration_str = '%.2f' % elapsed sys_monitor.stop() mem_str = 'mem: %s%%(peak:%s%%) cpu:%d%%' % \ (sys_monitor.current_memory, sys_monitor.peak_memory, int(sys_monitor.avg_cpu_load)) ret_str = variable_info(result) stack_depth = get_stack_depth() fn_str = function_name_with_class(fn) msg = ' ' * stack_depth + \ '%s, elapsed: %s, returned: %s, sys %s' % \ (fn_str, duration_str, ret_str, mem_str) if elapsed >= LOGGING_VERBOSITY.min_time: simple_logger.log(LOGGING_VERBOSITY.level, msg) return result
def inner(*args, **kwargs): start = time.time() result = fn(*args, **kwargs) duration = time.time() - start simple_logger.log(LOGGING_VERBOSITY.level, log_format, fn.__name__, duration) return result
def unpickle(self, remote_path, in_memory=True): logger.log(self.log_level, f'S3: unpickling from {remote_path}') if in_memory: return pickle.loads(self.read(remote_path)) else: with tempfile.NamedTemporaryFile(delete=True) as temp: self._download_through_disk(remote_path, temp) return pickle.load(temp)
def remote_to_local(self, remote_path, local_path, overwrite=True, in_memory=True): if not os.path.exists(local_path) or overwrite: os.makedirs(os.path.dirname(local_path), exist_ok=True) logger.log(self.log_level, f'S3: copying from {remote_path} to {local_path}') with open(local_path, 'wb') as local: if in_memory: local.write(self.read(remote_path)) else: self._download_through_disk(remote_path, local)
def inner(*args, **kwargs): sys_monitor = ResourceMonitor().start() start = time.time() result = fn(*args, **kwargs) elapsed = time.time() - start sys_monitor.stop() if elapsed >= LOGGING_VERBOSITY.min_time: msg = ' ' * get_stack_depth() + \ f'{function_name_with_class(fn)}, elapsed: {elapsed:.2f}, ' \ f'returned: {variable_info(result)}, sys mem: {sys_monitor.current_memory}%' \ f'(peak:{sys_monitor.peak_memory}%) cpu:{int(sys_monitor.avg_cpu_load)}%' simple_logger.log(LOGGING_VERBOSITY.level, msg) return result
def local_to_remote(self, local_path, remote_path, compress=True): logger.log(self.log_level, f'S3: copying from {local_path} to {remote_path}') with open(local_path, 'rb') as local: self.write_binary(local.read(), remote_path, compress=compress)
def pickle(self, obj, remote_path, compress=True): logger.log(self.log_level, f'S3: pickling to {remote_path}') return self.write_binary(pickle.dumps(obj), remote_path, compress=compress)