def list_files(client, resource_group, job_name, output_directory_id=STANDARD_OUTPUT_DIRECTORY_ID, path='.',
expiry=DEFAULT_URL_EXPIRY_MIN):
options = models.JobsListOutputFilesOptions(
outputdirectoryid=output_directory_id,
directory=path,
linkexpiryinminutes=expiry)
return list(client.list_output_files(resource_group, job_name, options))
def get_metric(self, job_name, resource_group, workspace_name, experiment_name, client):
files = client.jobs.list_output_files(resource_group, workspace_name, experiment_name, job_name,
models.JobsListOutputFilesOptions(outputdirectoryid=self.list_option))
val = float("inf")
for file in list(files):
if file.name == self.logfile:
text = ""
try:
r = requests.get(file.download_url, stream=True)
for chunk in r.iter_content(chunk_size=512 * 1024):
if chunk: # filter out keep-alive new chunks
text += chunk.decode(encoding='UTF-8')
except Exception as e:
print(e)
vals = re.findall(self.regex, text, re.DOTALL)
if self.metric is "last":
val = float(vals[len(vals) - 1])
elif self.metric is "mean":
val = sum([float(m) for m in vals])/len(vals)
elif self.metric is "min":
val = min([float(m) for m in vals])
elif self.metric is "max":
val = max([float(m) for m in vals])
break
return val
def assert_job_files_are(test, client, resource_group, job_name,
output_directory_id, expected):
"""Checks that the given task has expected output
:param AzureMgmtTestCase test: test instance.
:param BatchAIManagementClient client: client instance.
:param str resource_group: resource group name.
:param str job_name: job name.
:param str output_directory_id: output directory id.
:param dict(str, str) expected: expected content.
"""
paged = client.jobs.list_output_files(
resource_group, job_name,
models.JobsListOutputFilesOptions(output_directory_id))
files = paged.get(paged.current_page)
actual = dict()
for f in files:
v = requests.get(f.download_url).content
actual[f.name] = v if isinstance(v, six.string_types) else v.decode()
test.assertEquals(sorted(actual.keys()), sorted(expected.keys()))
for k, v in expected.items():
a = actual[k]
if isinstance(v, six.string_types):
test.assertEquals(a, v, k)
else:
test.assertRegexpMatches(actual.get(k), v, k)
def retrieve_outputs(config, job_name, output_model_name):
''' Get stdout, stderr, retrained model, and label-to-index dict '''
client = get_client(config)
status_files = client.jobs.list_output_files(
resource_group_name=config.bait_resource_group_name,
job_name=job_name,
jobs_list_output_files_options=tm.JobsListOutputFilesOptions(
'stdOuterr'))
for file in list(status_files):
download_from_file_share(file.download_url,
os.path.join('outputs', file.name))
output_files = client.jobs.list_output_files(
resource_group_name=config.bait_resource_group_name,
job_name=job_name,
jobs_list_output_files_options=tm.JobsListOutputFilesOptions('MODEL'))
for file in list(output_files):
transfer_fileshare_to_blob(config, file.download_url,
output_model_name)
client.jobs.delete(resource_group_name=config.bait_resource_group_name,
job_name=job_name)
return
def assert_job_files_in_path_are(test, client, resource_group, job_name,
output_directory_id, path, expected):
"""Checks that the given task has expected output
:param AzureMgmtTestCase test: test instance.
:param BatchAIManagementClient client: client instance.
:param str resource_group: resource group name.
:param str job_name: job name.
:param str output_directory_id: output directory id.
:param str path: a path inside of output directory.
:param dict(str, str or None) expected: expected content, directories must have None value.
"""
files = client.jobs.list_output_files(
resource_group, Helpers.DEFAULT_WORKSPACE_NAME,
Helpers.DEFAULT_EXPERIMENT_NAME, job_name,
models.JobsListOutputFilesOptions(
outputdirectoryid=output_directory_id, directory=path))
actual = dict()
execution_log_found = False
for f in files:
if (output_directory_id == Helpers.STANDARD_OUTPUT_DIRECTORY_ID
and f.name.startswith('execution')
and f.name.endswith('.log')):
execution_log_found = True
continue
actual[f.name] = None
if f.file_type == models.FileType.file:
v = requests.get(f.download_url).content
actual[f.name] = v if isinstance(
v, six.string_types) else v.decode()
test.assertEquals(sorted(actual.keys()), sorted(expected.keys()))
for k, v in expected.items():
a = actual[k]
if a is None and v is None:
# both are directories
continue
if v is None:
test.fail(
'Expected {0} to be a directory, got a file'.format(k))
if a is None:
test.fail(
'Expected {0} to be a file, got a directory'.format(k))
if isinstance(v, six.string_types):
test.assertEquals(v, a,
k + "expected {0} got {1}".format(v, a))
else:
test.assertRegexpMatches(actual.get(k), v, k)
if output_directory_id == Helpers.STANDARD_OUTPUT_DIRECTORY_ID and not execution_log_found:
test.fail("No execution log was generated for the job.")
def tail(self):
if not self.url:
files = self.client.jobs.list_output_files(
self.resource_group, self.workspace_name, self.experiment_name, self.job_name,
models.JobsListOutputFilesOptions(outputdirectoryid=self.output_directory_id))
if not files:
return
else:
for f in list(files):
if f.name == self.file_name:
self.url = f.download_url
if self.url:
r = requests.get(self.url, headers={
'Range': 'bytes={0}-'.format(self.downloaded)})
if int(r.status_code / 100) == 2:
self.downloaded += len(r.content)
print(r.content.decode(), end='')
def download_files(job_name, output_id, output_folder=None):
""" Downloads the files from the output of the job locally
Parameters
----------
job_name: [str] The name of the job e.g run_cntk, run_pytorch
output_id: [str] The id of the output you want to download the files from e.g stdOuterr, notebooks
"""
if output_folder:
logger.info('Downloading files to {}'.format(output_folder))
files = client.jobs.list_output_files(
config.group_name, job_name,
models.JobsListOutputFilesOptions(output_id))
for file in files:
logger.info('Downloading {}'.format(file.name))
file_name = path.join(output_folder,
file.name) if output_folder else file.name
ut.download_file(file.download_url, file_name)
print("All files Downloaded")
def list_files(client, resource_group, job_name, directory):
options = models.JobsListOutputFilesOptions(directory)
return list(client.list_output_files(resource_group, job_name, options))
id='MODEL',
path_prefix='$AZ_BATCHAI_MOUNT_ROOT/{0}'.format(relative_mount_point),
path_suffix="Models")],
# Container configuration
container_settings=models.ContainerSettings(
image_source_registry=models.ImageSourceRegistry(image='microsoft/cntk:2.1-gpu-python3.5-cuda8.0-cudnn6.0')),
# Toolkit specific settings
cntk_settings = models.CNTKsettings(
python_script_file_path='$AZ_BATCHAI_INPUT_SAMPLE/ConvNet_MNIST.py',
command_line_args='$AZ_BATCHAI_INPUT_SAMPLE $AZ_BATCHAI_OUTPUT_MODEL')
)
# Create the job
client.jobs.create(resource_group_name, job_name, parameters).result()
## MONITOR JOB
job = client.jobs.get(resource_group_name, job_name)
print('Job state: {0} '.format(job.execution_state.name))
files = client.jobs.list_output_files(resource_group_name, job_name, models.JobsListOutputFilesOptions(outputdirectoryid="stdouterr"))
for file in list(files):
print('file: {0}, download url: {1}'.format(file.name, file.download_url))
client.jobs.delete(resource_group_name, job_name)
client.clusters.delete(resource_group_name, cluster_name)
