def apply(self, step, *args, **kwargs): func, tag = step_loader.func_and_autotag(step, *args, **kwargs) log.debug(f'apply {self.path} {step} {func} {args} {kwargs}') if self._state == 'spun_up': log.debug(f'applying to unclean') return func(self, *args, **kwargs) elif self._state == 'loaded': log.debug(f'applying to clean') return self._cache_aware_apply(step, tag, func, args, kwargs) else: log.critical(f'apply to state={self._state}') raise RuntimeError(f'State machine error, apply to {self._state}')
def build(first_step, *args, fingertip_last_step=False, **kwargs): func, tag = step_loader.func_and_autotag(first_step, *args, **kwargs) # Could there already be a cached result? mpath = path.machines(tag) lock_path = path.machines('.' + tag + '-lock') log.info(f'acquiring lock for {tag}...') transient_hint = func.transient if hasattr(func, 'transient') else None if callable(transient_hint): transient_hint = supply_last_step_if_requested(transient_hint, fingertip_last_step) transient_hint = transient_hint(*args, **kwargs) transient = (transient_hint in ('always', True) or transient_hint == 'last' and fingertip_last_step) with lock.Lock(lock_path) if not transient else lock.NoLock(): if not os.path.exists(mpath) or needs_a_rebuild(mpath): log.info(f'building {tag}...') func = supply_last_step_if_requested(func, fingertip_last_step) first = func(*args, **kwargs) if first is None: assert transient, 'first step returned None' return if transient: log.info(f'succesfully built and discarded {tag}') first._finalize() # discard (not fast-dropped though) if transient_hint == 'last' and fingertip_last_step: fname = f'{datetime.datetime.utcnow().isoformat()}.txt' t = path.logs(fname, makedirs=True) with open(t, 'w') as f: f.write(first.log_contents) return t else: log.info(f'succesfully built and saved {tag}') first._finalize(link_as=mpath, name_hint=tag) if fingertip_last_step: return os.path.join(mpath, 'log.txt') m = clone_and_load(mpath) m.log = log.Sublogger('fingertip.<just built>', os.path.join(m.path, 'log.txt')) return m
def build(first_step, *args, **kwargs): func, tag = step_loader.func_and_autotag(first_step, *args, **kwargs) # Could there already be a cached result? mpath = path.machines(tag) lock_path = path.machines('.' + tag + '-lock') log.info(f'acquiring lock for {tag}...') do_lock = not hasattr(func, 'transient') with lock.MaybeLock(lock_path, lock=do_lock): if not os.path.exists(mpath) or needs_a_rebuild(mpath): log.info(f'building {tag}...') first = func(*args, **kwargs) if first is None: return first._finalize(link_as=mpath, name_hint=tag) log.info(f'succesfully built {tag}') m = clone_and_load(mpath) return m