def test_layered_settings(self):
from openpaisdk import LayeredSettings, __flags__
__flags__.custom_predefined = [{
'name': 'test-key-1',
}, {
'name': 'test-key-2',
'action': 'append',
'default': []
}]
LayeredSettings.reset()
# ? add / update append key
for test_key in ['test-key-1', 'test-key-2']:
for i, layer in enumerate(LayeredSettings.layers):
LayeredSettings.update(layer.name, test_key, i)
if layer.act_append(test_key):
self.assertTrue(isinstance(layer.values[test_key], list),
msg=f"{layer.values}")
self.assertEqual(0, LayeredSettings.get('test-key-1'))
self.assertListEqual([0, 1, 2, 3], LayeredSettings.get('test-key-2'))
# ? delete
for test_key in ['test-key-1', 'test-key-2']:
for i, layer in enumerate(LayeredSettings.layers):
LayeredSettings.update(layer.name, test_key, None, delete=True)
# ? reset the predefined
__flags__.custom_predefined = []
LayeredSettings.reset()
def get_random_var_name(self):
import random
from openpaisdk import LayeredSettings
lst = [
x for x in LayeredSettings.keys()
if not LayeredSettings.act_append(x)
]
ret = lst[random.randint(0, len(lst) - 1)]
to_screen(f"random select {ret} in {lst}")
return ret
def __set_selected(self, ctx):
from openpaisdk import LayeredSettings
LayeredSettings.update('global_default', 'web-default-form',
ctx['form'])
LayeredSettings.update('global_default', 'web-default-image',
ctx['docker_image'])
LayeredSettings.update(
'global_default', 'web-default-resource',
','.join([str(ctx['gpu']),
str(ctx['cpu']),
str(ctx['memoryMB'])]))
def test_unknown_variable_defined(self):
from openpaisdk import LayeredSettings, __flags__
test_key, test_value = 'test-key-long-existing', randstr(10)
__flags__.custom_predefined = [
{
'name': test_key,
},
]
LayeredSettings.reset()
# ? add / update append key
LayeredSettings.update('local_default', test_key, test_value)
# ? reset the predefined
__flags__.custom_predefined = []
LayeredSettings.reset()
self.assertEqual(test_value, LayeredSettings.get(test_key))
# cannot delete or change the unknown variable
LayeredSettings.update('local_default', test_key, randstr(10))
LayeredSettings.reset()
self.assertEqual(test_value, LayeredSettings.get(test_key))
LayeredSettings.update('local_default', test_key, delete=True)
LayeredSettings.reset()
self.assertEqual(test_value, LayeredSettings.get(test_key))
def step2_setup(self):
from openpaisdk.notebook import NotebookConfiguration
from openpaisdk import LayeredSettings
NotebookConfiguration.set(**self.settings)
for key in self.settings.keys():
LayeredSettings.update('user_basic', key, -1)
def _read_defaults_helper():
from openpaisdk import LayeredSettings
from openpaisdk.job import JobResource
# add default settings
image_list = LayeredSettings.get('image-list')
if image_list is None or len(image_list) == 0:
# add default images here
default_images = [
'openpai/pytorch-py36-cu90',
'openpai/pytorch-py36-cpu',
'openpai/tensorflow-py36-cu90',
'openpai/tensorflow-py36-cpu',
]
for image in default_images:
LayeredSettings.update('global_default', 'image-list',
image)
image_list = LayeredSettings.get('image-list')
resource_list = JobResource.parse_list(
LayeredSettings.get('resource-list'))
if resource_list is None or len(resource_list) == 0:
# add default resource here
default_resources = [
'1,4,8g',
'1,8,16g',
'0,4,8g',
'2,8,16g',
'4,16,32g',
]
for resource in default_resources:
LayeredSettings.update('global_default', 'resource-list',
resource)
resource_list = JobResource.parse_list(
LayeredSettings.get('resource-list'))
return {
'image-list':
image_list,
'resource-list':
resource_list,
'web-default-form':
LayeredSettings.get('web-default-form'),
'web-default-image':
LayeredSettings.get('web-default-image'),
'web-default-resource':
LayeredSettings.get('web-default-resource'),
}
def __init__(self):
from openpaisdk import LayeredSettings, ClusterList
if LayeredSettings.get('container-sdk-branch') != 'master':
LayeredSettings.update('user_basic', 'container-sdk-branch',
'master')
self.cll = ClusterList().load()
def __submit_job_helper(self, ctx):
import tempfile
from openpaisdk import Job
import os
import sys
from openpaisdk.notebook import get_notebook_path
from openpaisdk import LayeredSettings
import yaml
# save settings
self.__set_selected(ctx)
# setting layers description
# layer name | from : priority
# user_advanced | NotebookConfiguration.set : 0
# user_basic | extension panel selection : 1
# local_default | deaults in .openpai/defaults.yaml : 2
# global_default | defaults in ~/.openpai/defaults.yaml : 3
# - | predefined in flags.py : 4
LayeredSettings.update("user_basic", "cluster-alias", ctx['cluster'])
LayeredSettings.update("user_basic", "virtual-cluster", ctx['vc'])
LayeredSettings.update("user_basic", "image", ctx['docker_image'])
LayeredSettings.update("user_basic", "cpu", ctx['cpu']),
LayeredSettings.update("user_basic", "gpu", ctx['gpu']),
LayeredSettings.update("user_basic", "memoryMB", ctx['memoryMB'])
cfgs = LayeredSettings.as_dict()
notebook_path = get_notebook_path()
_, _, sources = next(os.walk('.'))
if ctx['form'] == 'file':
jobname = 'python_' + tempfile.mkdtemp()[-8:]
mode = 'script'
elif ctx['form'] == 'notebook':
jobname = 'jupyter_' + tempfile.mkdtemp()[-8:]
mode = 'interactive'
else:
jobname = 'silent_' + tempfile.mkdtemp()[-8:]
mode = 'silent'
job = Job(jobname)\
.from_notebook(
nb_file=get_notebook_path(),
cluster={
'cluster_alias': cfgs['cluster-alias'],
'virtual_cluster': cfgs['virtual-cluster'],
'workspace': cfgs['workspace'],
},
mode=mode,
**{
'token': '',
'image': cfgs["image"],
'resources': {
'cpu': cfgs["cpu"],
'gpu': cfgs["gpu"],
'memoryMB': cfgs["memoryMB"],
'mem': cfgs['mem']
},
'sources': sources + cfgs["sources"],
'pip_installs': cfgs["pip-installs"],
}
)
ctx['job_config'] = yaml.dump(job.get_config(),
default_flow_style=False)
ctx['jobname'] = job.name
if ctx['type'] == 'quick':
ret = job.submit()
ctx['joblink'] = ret['job_link']
ctx['jobname'] = ret['job_name']
return ctx