def MakeContinueFunction(job_id):
"""Returns a function to decide if log fetcher should continue polling.
Args:
job_id: String id of job.
Returns:
A one-argument function decides if log fetcher should continue.
"""
jobs_client = jobs.JobsClient()
project_id = properties.VALUES.core.project.Get(required=True)
job_ref = resources.REGISTRY.Create(
'ml.projects.jobs', jobsId=job_id, projectsId=project_id)
def ShouldContinue(periods_without_logs):
"""Returns whether to continue polling the logs.
Returns False only once we've checked the job and it is finished; we only
check whether the job is finished once we've gone >1 interval without
getting any new logs.
Args:
periods_without_logs: integer number of empty polls.
Returns:
True if we haven't tried polling more than once or if job is not finished.
"""
if periods_without_logs <= 1:
return True
return jobs_client.Get(job_ref).endTime is None
return ShouldContinue
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
Returns:
Some value that we want to have printed later.
"""
return jobs.JobsClient().List()
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
Returns:
Some value that we want to have printed later.
"""
job_ref = resources.REGISTRY.Parse(args.job, collection='ml.projects.jobs')
return jobs.JobsClient().Cancel(job_ref)
def Run(self, args):
return jobs_util.SubmitPrediction(
jobs.JobsClient('v1'), args.job,
model_dir=args.model_dir,
model=args.model,
version=args.version,
input_paths=args.input_paths,
data_format=args.data_format,
output_path=args.output_path,
region=args.region,
runtime_version=args.runtime_version,
max_worker_count=args.max_worker_count)
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
Returns:
Some value that we want to have printed later.
"""
project_ref = resources.REGISTRY.Parse(
properties.VALUES.core.project.Get(required=True),
collection='ml.projects')
return jobs.JobsClient().List(project_ref)
def Run(self, args):
job = jobs_util.SubmitTraining(jobs.JobsClient('v1'),
args.job,
job_dir=args.job_dir,
staging_bucket=args.staging_bucket,
packages=args.packages,
package_path=args.package_path,
scale_tier=args.scale_tier,
config=args.config,
module_name=args.module_name,
runtime_version=args.runtime_version,
async_=args. async,
user_args=args.user_args)
# If the job itself failed, we will return a failure status.
if not args. async and job.state is not job.StateValueValuesEnum.SUCCEEDED:
self.exit_code = 1
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
Returns:
Some value that we want to have printed later.
"""
job = jobs.BuildBatchPredictionJob(job_name=args.job,
model_name=args.model,
version_name=args.version,
input_paths=args.input_paths,
data_format=args.data_format,
output_path=args.output_path,
region=args.region)
return jobs.JobsClient().Create(job)
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
Returns:
Some value that we want to have printed later.
"""
project_ref = resources.REGISTRY.Parse(
properties.VALUES.core.project.Get(required=True),
collection='ml.projects')
job = jobs.BuildBatchPredictionJob(
job_name=args.job,
model_name=args.model,
version_name=args.version,
input_paths=args.input_paths,
data_format=args.data_format,
output_path=args.output_path,
region=args.region,
runtime_version=args.runtime_version)
return jobs.JobsClient().Create(project_ref, job)
def Run(self, args):
job_ref = resources.REGISTRY.Parse(args.job,
collection='ml.projects.jobs')
job = jobs.JobsClient().Get(job_ref)
self.job = job # Hack to make the Epilog() method work.
return job
def Run(self, args):
return jobs_util.List(jobs.JobsClient('v1'))
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
Returns:
Some value that we want to have printed later.
"""
region = properties.VALUES.compute.region.Get(required=True)
staging_location = jobs_prep.GetStagingLocation(
staging_bucket=args.staging_bucket,
job_id=args.job,
job_dir=args.job_dir)
try:
uris = jobs_prep.UploadPythonPackages(
packages=args.packages,
package_path=args.package_path,
staging_location=staging_location)
except jobs_prep.NoStagingLocationError:
raise flags.ArgumentError(
'If local packages are provided, the `--staging-bucket` or '
'`--job-dir` flag must be given.')
log.debug('Using {0} as trainer uris'.format(uris))
scale_tier_enum = (jobs.GetMessagesModule(
).GoogleCloudMlV1beta1TrainingInput.ScaleTierValueValuesEnum)
scale_tier = scale_tier_enum(
args.scale_tier) if args.scale_tier else None
job = jobs.BuildTrainingJob(
path=args.config,
module_name=args.module_name,
job_name=args.job,
trainer_uri=uris,
region=region,
job_dir=args.job_dir.ToUrl() if args.job_dir else None,
scale_tier=scale_tier,
user_args=args.user_args,
runtime_version=args.runtime_version)
jobs_client = jobs.JobsClient()
project_ref = resources.REGISTRY.Parse(
properties.VALUES.core.project.Get(required=True),
collection='ml.projects')
job = jobs_client.Create(project_ref, job)
log.status.Print('Job [{}] submitted successfully.'.format(job.jobId))
if args. async:
log.status.Print(_FOLLOW_UP_MESSAGE.format(job_id=job.jobId))
return job
log_fetcher = stream.LogFetcher(
filters=log_utils.LogFilters(job.jobId),
polling_interval=_POLLING_INTERVAL,
continue_func=log_utils.MakeContinueFunction(job.jobId))
printer = resource_printer.Printer(log_utils.LOG_FORMAT, out=log.err)
def _CtrlCHandler(signal, frame):
del signal, frame # Unused
raise KeyboardInterrupt
with execution_utils.CtrlCSection(_CtrlCHandler):
try:
printer.Print(log_utils.SplitMultiline(
log_fetcher.YieldLogs()))
except KeyboardInterrupt:
log.status.Print('Received keyboard interrupt.')
log.status.Print(_FOLLOW_UP_MESSAGE.format(job_id=job.jobId))
job_ref = resources.REGISTRY.Parse(job.jobId,
collection='ml.projects.jobs')
job = jobs_client.Get(job_ref)
# If the job itself failed, we will return a failure status.
if job.state is not job.StateValueValuesEnum.SUCCEEDED:
self.exit_code = 1
return job
def Run(self, args):
return jobs_util.Cancel(jobs.JobsClient('v1'), args.job)
def Run(self, args):
job = jobs_util.Describe(jobs.JobsClient('v1'), args.job)
self.job = job # Hack to make the Epilog method work
return job
