def _make_path(
flow_name=None, run_id=None, step_name=None, task_id=None, pathspec=None,
create_on_absent=True):
from metaflow.datastore.local import LocalDataStore
if LocalDataStore.datastore_root is None:
def print_clean(line, **kwargs):
print(line)
LocalDataStore.datastore_root = LocalDataStore.get_datastore_root_from_config(
print_clean, create_on_absent=create_on_absent)
if LocalDataStore.datastore_root is None:
return None
return LocalDataStore.make_path(flow_name, run_id, step_name, task_id, pathspec)
def _sync_metadata(echo, metadata, datastore_root, attempt):
if metadata.TYPE == 'local':
def echo_none(*args, **kwargs):
pass
path = os.path.join(
datastore_root,
MetaflowDataStore.filename_with_attempt_prefix(
'metadata.tgz', attempt))
url = urlparse(path)
bucket = url.netloc
key = url.path.lstrip('/')
s3, err = get_s3_client()
try:
s3.head_object(Bucket=bucket, Key=key)
# If we are here, we can download the object
with util.TempDir() as td:
tar_file_path = os.path.join(td, 'metadata.tgz')
with open(tar_file_path, 'wb') as f:
s3.download_fileobj(bucket, key, f)
with tarfile.open(tar_file_path, 'r:gz') as tar:
tar.extractall(td)
copy_tree(
os.path.join(td, DATASTORE_LOCAL_DIR),
LocalDataStore.get_datastore_root_from_config(echo_none),
update=True)
except err as e: # noqa F841
pass
def default_info(cls):
from metaflow.datastore.local import LocalDataStore
def print_clean(line, **kwargs):
print(line)
v = LocalDataStore.get_datastore_root_from_config(print_clean, create_on_absent=False)
if v is None:
return '<No %s directory found in current working tree>' % DATASTORE_LOCAL_DIR
return os.path.dirname(v)
def get_latest_run_id(echo, flow_name):
from metaflow.datastore.local import LocalDataStore
local_root = LocalDataStore.datastore_root
if local_root is None:
v = LocalDataStore.get_datastore_root_from_config(echo, create_on_absent=False)
LocalDataStore.datastore_root = local_root = v
if local_root:
path = os.path.join(local_root, flow_name, 'latest_run')
if os.path.exists(path):
with open(path) as f:
return f.read()
return None
def write_latest_run_id(obj, run_id):
from metaflow.datastore.local import LocalDataStore
if LocalDataStore.datastore_root is None:
LocalDataStore.datastore_root = LocalDataStore.get_datastore_root_from_config(obj.echo)
path = os.path.join(LocalDataStore.datastore_root, obj.flow.name)
try:
os.makedirs(path)
except OSError as x:
if x.errno != 17:
# Directories exists in other casewhich is fine
raise
with open(os.path.join(path, 'latest_run'), 'w') as f:
f.write(str(run_id))
def step_init(self, flow, graph, step, decos, environment, datastore, logger):
if environment.TYPE != 'conda':
raise InvalidEnvironmentException('The *@conda* decorator requires '
'--environment=conda')
def _logger(line, **kwargs):
logger(line)
self.local_root = LocalDataStore.get_datastore_root_from_config(_logger)
environment.set_local_root(self.local_root)
self.architecture = self._architecture(decos)
self.step = step
self.flow = flow
self.datastore = datastore
self.base_attributes = self._get_base_attributes()
def reset_local():
# Get the local data store path
path = LocalDataStore.get_datastore_root_from_config(
echo, create_on_absent=False)
if path:
if click.confirm("Confirm deleting your local datastore at:" +\
click.style("\"{0}\"".format(path), fg='cyan') + "? " +\
click.style("WARNING:", fg='red') +\
"This step cannot be undone."):
shutil.rmtree(path)
echo('Local datastore successfully reset.')
else:
# Nothing to do here since we didnt find an existing store.
echo('Local datastore is already reset.')
def status():
from metaflow.client import get_metadata
res = get_metadata()
if res:
res = res.split('@')
else:
raise click.ClickException(
'Unknown status: cannot find a Metadata provider')
if res[0] == 'service':
echo('Using Metadata provider at: ', nl=False)
echo('"%s"\n' % res[1], fg='cyan')
echo('To list available flows, type:\n')
echo('1. python')
echo('2. from metaflow import Metaflow')
echo('3. list(Metaflow())')
return
from metaflow.client import namespace, metadata, Metaflow
# Get the local data store path
path = LocalDataStore.get_datastore_root_from_config(
echo, create_on_absent=False)
# Throw an exception
if path is None:
raise click.ClickException("Could not find " +\
click.style('"%s"' % DATASTORE_LOCAL_DIR,
fg='red') +\
" in the current working tree.")
stripped_path = os.path.dirname(path)
namespace(None)
metadata('local@%s' % stripped_path)
echo('Working tree found at: ', nl=False)
echo('"%s"\n' % stripped_path, fg='cyan')
echo('Available flows:', fg='cyan', bold=True)
for flow in Metaflow():
echo('* %s' % flow, fg='cyan')