def get_outputs_path():
"""
The outputs path generated by polyaxon based on the hierarchy of the experiment:
Experiment:
`user/project/group/experiment/files`
Jobs:
`user/project/jobs/files`
"""
ensure_in_custer()
return os.getenv('POLYAXON_RUN_OUTPUTS_PATH', None)
def get_data_paths():
"""
The data paths dictionary that you mounted for the job/experiment.
{'data1': '/data/1/', 'data-foo': '/data/foo'}
"""
ensure_in_custer()
data_path = os.getenv('POLYAXON_RUN_DATA_PATHS', None)
try:
return json.loads(data_path) if data_path else None
except (ValueError, TypeError):
print('Could get data paths, '
'please make sure this is running inside a polyaxon job.')
return None
def get_task_info():
"""Returns the task info: {"type": str, "index": int}."""
if settings.NO_OP:
return None
ensure_in_custer()
info = os.getenv('POLYAXON_TASK_INFO', None)
try:
return json.loads(info) if info else None
except (ValueError, TypeError):
print('Could get task info, '
'please make sure this is running inside a polyaxon job.')
return None
def get_declarations():
"""
Returns all the experiment declarations based on both:
* declarations section
* matrix section
"""
if settings.NO_OP:
return None
ensure_in_custer()
declarations = os.getenv('POLYAXON_DECLARATIONS', None)
try:
return json.loads(declarations) if declarations else None
except (ValueError, TypeError):
print('Could get declarations, '
'please make sure this is running inside a polyaxon job.')
return None
def get_outputs_refs_paths():
"""
The references outputs paths requested by the user,
the order follows the order specified by the user:
{
'jobs': [
`user/project/job12/files`,
],
'experiments': [
`user/project/group/experiment1/files`,
`user/project/experiment100/files`
]
}
"""
ensure_in_custer()
outputs_refs = os.getenv('POLYAXON_REFS_OUTPUTS_PATHS', None)
try:
return json.loads(outputs_refs) if outputs_refs else None
except (ValueError, TypeError):
print('Could get outputs refs paths, '
'please make sure this is running inside a polyaxon job.')
return None
def get_experiment_info():
"""
Returns information about the experiment:
* project_name
* experiment_group_name
* experiment_name
* project_uuid
* experiment_group_uuid
* experiment_uuid
"""
if settings.NO_OP:
return None
ensure_in_custer()
info = os.getenv('POLYAXON_EXPERIMENT_INFO', None)
try:
return json.loads(info) if info else None
except (ValueError, TypeError):
print('Could get experiment info, '
'please make sure this is running inside a polyaxon job.')
return None
def get_cluster_def():
"""Returns cluster definition created by polyaxon.
{
"master": ["plxjob-master0-8eefb7a1146f476ca66e3bee9b88c1de:2000"],
"worker": ["plxjob-worker1-8eefb7a1146f476ca66e3bee9b88c1de:2000",
"plxjob-worker2-8eefb7a1146f476ca66e3bee9b88c1de:2000"],
"ps": ["plxjob-ps3-8eefb7a1146f476ca66e3bee9b88c1de:2000"],
}
:return: dict
"""
if settings.NO_OP:
return None
ensure_in_custer()
cluster = os.getenv('POLYAXON_CLUSTER', None)
try:
return json.loads(cluster) if cluster else None
except (ValueError, TypeError):
print('Could get cluster definition, '
'please make sure this is running inside a polyaxon job.')
return None
def get_job_info():
"""Returns information about the job:
* project_name
* job_name
* project_uuid
* job_uuid
* role
* type
* app
"""
if settings.NO_OP:
return None
ensure_in_custer()
info = os.getenv('POLYAXON_JOB_INFO', None)
try:
return json.loads(info) if info else None
except (ValueError, TypeError):
print('Could get experiment info, '
'please make sure this is running inside a polyaxon job.')
return None
def get_tf_config(cls, envvar='TF_CONFIG'):
"""
Returns the TF_CONFIG defining the cluster and the current task.
if `envvar` is not null, it will set and env variable with `envvar`.
"""
if settings.NO_OP:
return None
ensure_in_custer()
cluster_def = cls.get_cluster_def()
task_info = cls.get_task_info()
tf_config = {
'cluster': cluster_def,
'task': task_info,
'model_dir': get_outputs_path(),
'environment': 'cloud'
}
if envvar:
os.environ[envvar] = json.dumps(tf_config)
return tf_config
def get_log_level():
"""If set on the polyaxonfile it will return the log level."""
ensure_in_custer()
return os.getenv('POLYAXON_LOG_LEVEL', None)