def submitTaskWithFiles(self, clientCodeFile, args, files={}, job_id=None):
old_cwd = os.getcwd()
cwd = os.path.dirname(os.path.realpath(__file__))
os.chdir(cwd)
if job_id is not None:
experiment_name = self.project_name + "_" + str(job_id)
else:
experiment_name = self.project_name + "_" + str(uuid.uuid4())
tmpdir = tempfile.gettempdir()
args_file = os.path.join(tmpdir, experiment_name + "_args.pkl")
workspace_orig = os.getcwd()
ignore_arg = ''
ignore_filepath = os.path.join(workspace_orig, ".studioml_ignore")
if os.path.exists(ignore_filepath) and \
not os.path.isdir(ignore_filepath):
ignore_arg = "--exclude-from=%s" % ignore_filepath
workspace_new = fs_tracker.get_artifact_cache('workspace',
experiment_name)
rsync_cp(workspace_orig, workspace_new, ignore_arg, self.logger)
distpath = os.path.join(old_cwd, 'dist')
if os.path.exists(distpath):
self.logger.info('dist folder found at {}, ' +
'copying into workspace')
rsync_cp(distpath, os.path.join(workspace_new, 'dist'))
self.logger.info('Created workspace ' + workspace_new)
artifacts = self._create_artifacts(clientCodeFile, args_file,
workspace_new, files)
with open(args_file, 'wb') as f:
f.write(pickle.dumps(args, protocol=2))
experiment = create_experiment('completion_service_client.py',
[self.config['verbose']],
experiment_name=experiment_name,
project=self.project_name,
artifacts=artifacts,
resources_needed=self.resources_needed)
tic = time.time()
runner.submit_experiments([experiment],
config=self.config,
logger=self.logger,
cloud=self.cloud,
queue_name=self.queue_name)
self.submitted[experiment.key] = time.time()
os.chdir(old_cwd)
toc = time.time()
self.logger.info('Submitted experiment ' + experiment.key + ' in ' +
str(toc - tic) + ' s')
return experiment_name
def test_create_experiment(self):
_, experiment_name, filename, args = get_test_experiment()
experiment_project = 'create_experiment_project'
experiment = create_experiment(
filename, args, experiment_name, experiment_project)
packages = [p for p in freeze.freeze()]
self.assertTrue(experiment.key == experiment_name)
self.assertTrue(experiment.filename == filename)
self.assertTrue(experiment.args == args)
self.assertTrue(experiment.project == experiment_project)
self.assertTrue(sorted(experiment.pythonenv) == sorted(packages))
def test_create_experiment(self):
_, experiment_name, filename, args = get_test_experiment()
experiment_project = 'create_experiment_project'
experiment = create_experiment(
filename, args, experiment_name, experiment_project)
packages = [
p._key +
'==' +
p._version for p in pip.pip.get_installed_distributions(
local_only=True)]
self.assertTrue(experiment.key == experiment_name)
self.assertTrue(experiment.filename == filename)
self.assertTrue(experiment.args == args)
self.assertTrue(experiment.project == experiment_project)
self.assertTrue(experiment.pythonenv == packages)
def submitTaskWithFiles(self, clientCodeFile, args, files={}):
old_cwd = os.getcwd()
cwd = os.path.dirname(os.path.realpath(__file__))
os.chdir(cwd)
experiment_name = self.project_name + "_" + str(uuid.uuid4())
tmpdir = tempfile.gettempdir()
args_file = os.path.join(tmpdir, experiment_name + "_args.pkl")
workspace_orig = os.getcwd()
ignore_arg = ''
ignore_filepath = os.path.join(workspace_orig, ".studioml_ignore")
if os.path.exists(ignore_filepath) and \
not os.path.isdir(ignore_filepath):
ignore_arg = "--exclude-from=%s" % ignore_filepath
workspace_new = fs_tracker.get_artifact_cache(
'workspace', experiment_name)
rsync_cp(workspace_orig, workspace_new, ignore_arg, self.logger)
distpath = os.path.join(old_cwd, 'dist')
if os.path.exists(distpath):
self.logger.info('dist folder found at {}, ' +
'copying into workspace')
rsync_cp(distpath, os.path.join(workspace_new, 'dist'))
self.logger.info('Created workspace ' + workspace_new)
artifacts = {
'retval': {
'mutable': True
},
'clientscript': {
'mutable': False,
'local': clientCodeFile
},
'args': {
'mutable': False,
'local': args_file
},
'workspace': {
'mutable': False,
'local': workspace_new
}
}
for tag, name in six.iteritems(files):
artifacts[tag] = {}
url_schema = re.compile('^https{0,1}://')
s3_schema = re.compile('^s3://')
gcs_schema = re.compile('^gs://')
if url_schema.match(name):
artifacts[tag]['url'] = name
elif s3_schema.match(name) or gcs_schema.match(name):
artifacts[tag]['qualified'] = name
else:
artifacts[tag]['local'] = os.path.abspath(
os.path.expanduser(name))
artifacts[tag]['mutable'] = False
with open(args_file, 'wb') as f:
f.write(pickle.dumps(args, protocol=2))
experiment = create_experiment(
'completion_service_client.py',
[self.config['verbose']],
experiment_name=experiment_name,
project=self.project_name,
artifacts=artifacts,
resources_needed=self.resources_needed)
tic = time.time()
runner.submit_experiments(
[experiment],
config=self.config,
logger=self.logger,
cloud=self.cloud,
queue_name=self.queue_name)
self.submitted.add(experiment.key)
os.chdir(old_cwd)
toc = time.time()
self.logger.info('Submitted experiment ' + experiment.key +
' in ' + str(toc - tic) + ' s')
return experiment_name
def get_test_experiment():
filename = 'test.py'
args = ['a', 'b', 'c']
experiment_name = 'test_experiment_' + str(uuid.uuid4())
experiment = create_experiment(filename, args, experiment_name)
return experiment, experiment_name, filename, args