def get_current_time(self): ''' Get the current time. Returns: A string with the current hours and minutes. ''' current_time = get_short_date_time(now()) # strip the date i = current_time.find(' ') current_hour_min = current_time[i+1:] # strip the milliseconds i = current_hour_min.find('.') current_hour_min = current_hour_min[:i] i = current_hour_min.find(':') current_hour = current_hour_min[:i] if len(current_hour) < 2: current_hour = f'0{current_hour}' current_min = current_hour_min[i+1:] if len(current_min) < 2: current_min = f'{current_min}0' return f'{current_hour}:{current_min}'
def get_newest_actions(prev_timestamp=None): ''' Wait for new actions. Return actions, or None if there aren't any. >>> actions = get_newest_actions() >>> 'timestamp' in actions True >>> 'data' in actions True ''' if time_actions_updated is None and action_updates is None: log('no new actions yet') actions = get_json_update(timestamp(now()), {}) else: if prev_timestamp: while time_actions_updated <= prev_timestamp or len( action_updates) == 0: sleep(1) log(f'got newest actions {time_actions_updated} > {prev_timestamp}' ) actions = get_json_update(time_actions_updated, action_updates) return actions
def get_deadline(timeout=None): ''' Return a datetime deadline from timeout. 'timeout' can be seconds or a timedelta. Default is timedelta.max. >>> from datetime import datetime >>> deadline = get_deadline() >>> deadline is None True >>> deadline = get_deadline(timedelta(seconds=1)) >>> type(deadline) is datetime True >>> deadline = get_deadline(1) >>> type(deadline) is datetime True >>> deadline = get_deadline(1.1) >>> type(deadline) is datetime True >>> deadline('bad timeout value') Traceback (most recent call last): ... TypeError: 'datetime.datetime' object is not callable ''' if timeout is None: deadline = None elif isinstance(timeout, timedelta): deadline = now() + timeout elif isinstance(timeout, (float, int)): deadline = now() + timedelta(seconds=timeout) else: raise ValueError( f'timeout must be one of (seconds, timedelta, None), not {type(timeout)}' ) # log(f'deadline: {deadline}') return deadline
def clear_action_updates(): ''' Clear all action_updates. >>> clear_action_updates() >>> action_updates {} ''' global action_updates, time_actions_updated action_updates.clear() time_actions_updated = now()
def set_action_update(key, value): ''' Set an update for Blockchain Backup's actions. >>> set_action_update('header-id', 'Test') 'Test' ''' global action_updates, time_actions_updated log(f'set {key} action: {value}') action_updates[key] = value time_actions_updated = now() return action_updates[key]
def get_page(self, request): # let's see what we know about the environment gen_utils.clear_action_updates() # last block in django database; may be different from last in blockchain last_block_updated = state.get_last_block_updated() # ready if blockchain-backup has processed some blockchain data bcb_run_already = last_block_updated > 0 bin_dir_ok = preferences.bin_dir_ok() data_dir = preferences.get_data_dir() backup_dir_ok, backup_dir_error = preferences.backup_dir_ok() backup_dir = preferences.get_backup_dir() last_bcb_version = state.get_latest_bcb_version() if bcb_run_already: data_dir_ok, __ = preferences.data_dir_ok() else: if data_dir and os.path.exists(data_dir): data_dir_ok, __ = preferences.data_dir_ok() else: data_dir_ok = False gen_utils.check_for_updates() params = { 'data_dir': data_dir, 'data_dir_ok': data_dir_ok, 'backup_dir': backup_dir, 'backup_dir_ok': backup_dir_ok, 'backup_dir_error': backup_dir_error, 'last_bcb_version': last_bcb_version, 'bcb_up_to_date': last_bcb_version >= CURRENT_VERSION, 'need_backup': get_next_backup_time() < now(), 'last_backed_up_time': state.get_last_backed_up_time(), } #log('params: {}'.format(params)) response = gen_utils.get_home_page_response(request, bcb_run_already, bin_dir_ok, params) return response
def check_for_updates(current_time=None, force=False, reason=None): ''' Check to see if updates are needed. >>> from blockchain_backup.bitcoin.tests import utils as test_utils >>> test_utils.init_database() >>> check_for_updates() True ''' updates_checked = False try: if current_time is None: current_time = now() next_updates_time = state.get_last_update_time() + timedelta(hours=24) if force or next_updates_time <= current_time: log('starting to check for the latest updates') # set the update time now so we don't restart the check too often state.set_last_update_time(current_time) command_args = [] command_args.append('python3') # get the path for check_for_updates.py, regardless of virtualenv, etc. check_program = os.path.realpath( os.path.abspath( os.path.join(os.path.dirname(blockchain_backup_file), 'config', 'check_for_updates.py'))) command_args.append(check_program) if reason is not None: command_args.append(reason) background(*command_args) updates_checked = True except: # 'bare except' because it catches more than "except Exception" log(format_exc()) return updates_checked
def measure(*args, **kwargs): ''' Profile a function decorated with @profile. Args: args: the function's positional args. kwargs: the function's keyword args. Returns: The function's return values. ''' if profile_enabled: try: f.__name__ except AttributeError: # static methods have no __name__ # see https://stackoverflow.com/questions/41921255/staticmethod-object-is-not-callable function = f.__func__ function_name = function.__name__ else: function = f function_name = f.__name__ f_name = f'{caller_module_name(ignore=[__file__])}.{function_name}' if f_name in functions: f_data = functions[f_name] f_data.count += 1 else: f_data = Simple() f_data.name = f_name f_data.count = 1 f_data.calls = [] f_data.total_time = timedelta() functions[f_name] = f_data metrics = Simple() metrics.start = now() # to_stderr(f'Call {f_name}') try: result = function(*args, **kwargs) except Exception as exc: to_stderr(exc) to_stderr(f'args: {args}') to_stderr(f'kwargs: {kwargs}') raise # if result is None: # to_stderr(f'Called {f_name}') # else: # to_stderr(f'Called {f_name}: {result}') metrics.end = now() metrics.elapsed_time = metrics.end - metrics.start f_data.total_time += metrics.elapsed_time f_data.calls.append(metrics) functions[f_name] = f_data else: result = f(*args, **kwargs) return result
Copyright 2021 DeNova Last modified: 2021-03-16 This file is open source, licensed under GPLv3 <http://www.gnu.org/licenses/>. ''' from datetime import timedelta from functools import wraps import sys from denova.python.dict import Simple from denova.python.times import now, timestamp, elapsed_time, log_elapsed_time, timedelta_to_human_readable from denova.python.utils import caller_module_name functions = {} start = now() # let other modules set some params profile_enabled = True verbose = False def profile(f): ''' Function decorator to profile functions. Args: f: The function to profile. ''' @wraps(f) def measure(*args, **kwargs): ''' Profile a function decorated with @profile.
def timed_out(): return timeout and (now() >= deadline)
def wait(event, timeout=None, sleep_time=1, event_args=None, event_kwargs=None): ''' Wait for an event. Retries event until success or timeout. Default is to ignore exceptions except when there is a timeout. 'event' is a function. event() succeeds if it does not raise an exception. Each call to event() continues until it succeeds or raises an exception. It is not interrupted if it times out. 'timeout' can be in seconds as an int or float, or a datetime.timedelta, or a datetime.datetime. Default is None, which means no timeout. If the timeout deadline passes while event() is running, event() is not interrupted. If event() times out while running and does not succeed, wait() raises the exception from event(). 'sleep_time' is in seconds. Default is one. 'event_args' is an list of positional args to event(). Default is None. 'event_kwargs' is an dict of keyword args to event(). Default is None. Returns result from event() if no timeout, or if timeout returns last exception. ''' def timed_out(): return timeout and (now() >= deadline) if timeout: if isinstance(timeout, int) or isinstance(timeout, float): deadline = now() + datetime.timedelta(seconds=timeout) elif isinstance(timeout, datetime.timedelta): deadline = now() + timeout elif isinstance(timeout, datetime.datetime): deadline = timeout else: raise TypeError( 'timeout must be an int, float, datetime.timedelta, or datetime.datetime' ) log.debug(f'wait() timeout: {timeout}, deadline: {deadline}') if event_args is None: event_args = [] if event_kwargs is None: event_kwargs = {} success = False while not success: try: result = event(*event_args, **event_kwargs) except KeyboardInterrupt: raise except: # 'bare except' because it catches more than "except Exception" if timed_out(): log.debug(f'wait() timed out with exception: {format_exc()}') raise else: log.debug( f'wait() ignored exception because not timed out:\n{format_exc()}' ) else: success = True if not timed_out(): time.sleep(sleep_time) return result
def unlock(lockname, nonce, pid, timeout=None, server_required=True): ''' >>> log('Unlock a process or thread that was previously locked.') >>> lockname = 'lock1' >>> __, nonce, pid = lock(lockname) >>> unlock(lockname, nonce, pid) True >>> log('A bad nonce should fail.') >>> lockname = 'lock1' >>> __, nonce, pid = lock(lockname) >>> try: ... unlock(lockname, 'bad nonce', pid) ... assert False, 'Unexpectedly passed bad nonce' ... except LockFailed: ... pass >>> unlock(lockname, nonce, pid) True ''' deadline = get_deadline(timeout) # log(f'unlock deadline: {deadline}') # DEBUG # we must be persistent in case the Safelock is busy is_locked = True last_warning = None while is_locked: try: is_locked = try_to_unlock(lockname, nonce, pid, server_required=server_required) except TimeoutError as te: log(str(te)) except LockFailed as lf: # we need a better way to handle serious errors if 'Wrong nonce' in str(lf): raise else: message = lf.args[0] if message != last_warning: last_warning = message log(message) except: # 'bare except' because it catches more than "except Exception" log(format_exc()) raise if is_locked: if deadline and now() > deadline: log.warning(f'unlock timed out: {lockname}') raise LockTimeout(warning_msg) sleep(0.1) # log(f'unlocked: {lockname}') # DEBUG # only returned for testing purposes return not is_locked
def lock(lockname, timeout=None, server_required=True): ''' Lock a process or thread to prevent concurrency issues. 'lockname' is the name of the lock. Every process or thread that calls "lock()" from the same source file and line number contends for the same lock. If you want many instances of a class to run at the same time, each instance's lockname for a particular call to lock() must use a different lockname. Example:: lockname = f'MyClass {self.instance_id()}' lock(lockname) You may still choose to include the source path and line number from denova.python.process.caller() in your lockname. If for some reason you use 'with locked()' with no name twice on the same line, the second 'with locked()' will fail. They both have the same default lockname with the same caller and line number. You're extremely unlikely to do that accidentally. >>> pid = os.getpid() >>> log(f'pid: {pid}') >>> log('test simple lock()/unlock()') >>> from denova.os.process import is_pid_active >>> lockname = 'lock1' >>> is_locked, nonce, pid = lock(lockname) >>> is_locked True >>> isinstance(nonce, str) True >>> is_pid_active(pid) True >>> unlock(lockname, nonce, pid) True >>> log('test relock') >>> lockname = 'lock1' >>> is_locked, nonce, __ = lock(lockname) >>> is_locked True >>> log('while locked, try to lock again should fail') >>> try: ... lock(lockname, timeout=timedelta(milliseconds=3)) ... except LockTimeout as lt: ... print(str(lt)) lock timed out: lock1 >>> log('now unlock it') >>> unlock(lockname, nonce, pid) True >>> log('try 2 locks') >>> lockname1 = 'lock1' >>> is_locked1, nonce1, pid1 = lock(lockname1) >>> is_locked1 True >>> lockname2 = 'lock2' >>> is_locked2, nonce2, pid2 = lock(lockname2) >>> is_locked2 True >>> nonce1 != nonce2 True >>> pid1 == pid2 True >>> unlock(lockname1, nonce1, pid1) True >>> unlock(lockname2, nonce2, pid2) True ''' nonce = None pid = os.getpid() deadline = get_deadline(timeout) if DEBUGGING: log(f'lock deadline: {deadline}') # DEBUG # we can probably factor this out into a general case loop_count = 0 is_locked = False last_warning = None while not is_locked: try: # only report every 10 secs # if (loop_count % 100) == 0: # log(f'call lock({lockname})') # DEBUG is_locked, nonce = try_to_lock(lockname, pid, server_required=server_required) except TimeoutError as te: log(str(te)) except LockFailed as lf: # we need a better way to handle serious errors if 'Wrong nonce' in str(lf): raise else: message = lf.args[0] if message != last_warning: last_warning = message log(message) except: # 'bare except' because it catches more than "except Exception" log(format_exc()) raise if not is_locked: if deadline and now() > deadline: warning_msg = f'lock timed out: {lockname}' log.warning(warning_msg) raise LockTimeout(warning_msg) sleep(0.1) loop_count = loop_count + 1 return is_locked, nonce, pid