def _sync_metadata(echo, metadata, datastore_root, attempt): if metadata.TYPE == 'local': def echo_none(*args, **kwargs): pass path = os.path.join( datastore_root, MetaflowDataStore.filename_with_attempt_prefix( 'metadata.tgz', attempt)) url = urlparse(path) bucket = url.netloc key = url.path.lstrip('/') s3, err = get_s3_client() try: s3.head_object(Bucket=bucket, Key=key) # If we are here, we can download the object with util.TempDir() as td: tar_file_path = os.path.join(td, 'metadata.tgz') with open(tar_file_path, 'wb') as f: s3.download_fileobj(bucket, key, f) with tarfile.open(tar_file_path, 'r:gz') as tar: tar.extractall(td) copy_tree( os.path.join(td, DATASTORE_LOCAL_DIR), LocalDataStore.get_datastore_root_from_config(echo_none), update=True) except err as e: # noqa F841 pass
def default_info(cls): from metaflow.datastore.local import LocalDataStore def print_clean(line, **kwargs): print(line) v = LocalDataStore.get_datastore_root_from_config(print_clean, create_on_absent=False) if v is None: return '<No %s directory found in current working tree>' % DATASTORE_LOCAL_DIR return os.path.dirname(v)
def get_latest_run_id(echo, flow_name): from metaflow.datastore.local import LocalDataStore local_root = LocalDataStore.datastore_root if local_root is None: v = LocalDataStore.get_datastore_root_from_config(echo, create_on_absent=False) LocalDataStore.datastore_root = local_root = v if local_root: path = os.path.join(local_root, flow_name, 'latest_run') if os.path.exists(path): with open(path) as f: return f.read() return None
def write_latest_run_id(obj, run_id): from metaflow.datastore.local import LocalDataStore if LocalDataStore.datastore_root is None: LocalDataStore.datastore_root = LocalDataStore.get_datastore_root_from_config(obj.echo) path = os.path.join(LocalDataStore.datastore_root, obj.flow.name) try: os.makedirs(path) except OSError as x: if x.errno != 17: # Directories exists in other casewhich is fine raise with open(os.path.join(path, 'latest_run'), 'w') as f: f.write(str(run_id))
def step_init(self, flow, graph, step, decos, environment, datastore, logger): if environment.TYPE != 'conda': raise InvalidEnvironmentException('The *@conda* decorator requires ' '--environment=conda') def _logger(line, **kwargs): logger(line) self.local_root = LocalDataStore.get_datastore_root_from_config(_logger) environment.set_local_root(self.local_root) self.architecture = self._architecture(decos) self.step = step self.flow = flow self.datastore = datastore self.base_attributes = self._get_base_attributes()
def _make_path( flow_name=None, run_id=None, step_name=None, task_id=None, pathspec=None, create_on_absent=True): from metaflow.datastore.local import LocalDataStore if LocalDataStore.datastore_root is None: def print_clean(line, **kwargs): print(line) LocalDataStore.datastore_root = LocalDataStore.get_datastore_root_from_config( print_clean, create_on_absent=create_on_absent) if LocalDataStore.datastore_root is None: return None return LocalDataStore.make_path(flow_name, run_id, step_name, task_id, pathspec)
def reset_local(): # Get the local data store path path = LocalDataStore.get_datastore_root_from_config( echo, create_on_absent=False) if path: if click.confirm("Confirm deleting your local datastore at:" +\ click.style("\"{0}\"".format(path), fg='cyan') + "? " +\ click.style("WARNING:", fg='red') +\ "This step cannot be undone."): shutil.rmtree(path) echo('Local datastore successfully reset.') else: # Nothing to do here since we didnt find an existing store. echo('Local datastore is already reset.')
def status(): from metaflow.client import get_metadata res = get_metadata() if res: res = res.split('@') else: raise click.ClickException( 'Unknown status: cannot find a Metadata provider') if res[0] == 'service': echo('Using Metadata provider at: ', nl=False) echo('"%s"\n' % res[1], fg='cyan') echo('To list available flows, type:\n') echo('1. python') echo('2. from metaflow import Metaflow') echo('3. list(Metaflow())') return from metaflow.client import namespace, metadata, Metaflow # Get the local data store path path = LocalDataStore.get_datastore_root_from_config( echo, create_on_absent=False) # Throw an exception if path is None: raise click.ClickException("Could not find " +\ click.style('"%s"' % DATASTORE_LOCAL_DIR, fg='red') +\ " in the current working tree.") stripped_path = os.path.dirname(path) namespace(None) metadata('local@%s' % stripped_path) echo('Working tree found at: ', nl=False) echo('"%s"\n' % stripped_path, fg='cyan') echo('Available flows:', fg='cyan', bold=True) for flow in Metaflow(): echo('* %s' % flow, fg='cyan')