def MakeContinueFunction(job_id): """Returns a function to decide if log fetcher should continue polling. Args: job_id: String id of job. Returns: A one-argument function decides if log fetcher should continue. """ jobs_client = jobs.JobsClient() project_id = properties.VALUES.core.project.Get(required=True) job_ref = resources.REGISTRY.Create( 'ml.projects.jobs', jobsId=job_id, projectsId=project_id) def ShouldContinue(periods_without_logs): """Returns whether to continue polling the logs. Returns False only once we've checked the job and it is finished; we only check whether the job is finished once we've gone >1 interval without getting any new logs. Args: periods_without_logs: integer number of empty polls. Returns: True if we haven't tried polling more than once or if job is not finished. """ if periods_without_logs <= 1: return True return jobs_client.Get(job_ref).endTime is None return ShouldContinue
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace. All the arguments that were provided to this command invocation. Returns: Some value that we want to have printed later. """ return jobs.JobsClient().List()
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace. All the arguments that were provided to this command invocation. Returns: Some value that we want to have printed later. """ job_ref = resources.REGISTRY.Parse(args.job, collection='ml.projects.jobs') return jobs.JobsClient().Cancel(job_ref)
def Run(self, args): return jobs_util.SubmitPrediction( jobs.JobsClient('v1'), args.job, model_dir=args.model_dir, model=args.model, version=args.version, input_paths=args.input_paths, data_format=args.data_format, output_path=args.output_path, region=args.region, runtime_version=args.runtime_version, max_worker_count=args.max_worker_count)
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace. All the arguments that were provided to this command invocation. Returns: Some value that we want to have printed later. """ project_ref = resources.REGISTRY.Parse( properties.VALUES.core.project.Get(required=True), collection='ml.projects') return jobs.JobsClient().List(project_ref)
def Run(self, args): job = jobs_util.SubmitTraining(jobs.JobsClient('v1'), args.job, job_dir=args.job_dir, staging_bucket=args.staging_bucket, packages=args.packages, package_path=args.package_path, scale_tier=args.scale_tier, config=args.config, module_name=args.module_name, runtime_version=args.runtime_version, async_=args. async, user_args=args.user_args) # If the job itself failed, we will return a failure status. if not args. async and job.state is not job.StateValueValuesEnum.SUCCEEDED: self.exit_code = 1
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace. All the arguments that were provided to this command invocation. Returns: Some value that we want to have printed later. """ job = jobs.BuildBatchPredictionJob(job_name=args.job, model_name=args.model, version_name=args.version, input_paths=args.input_paths, data_format=args.data_format, output_path=args.output_path, region=args.region) return jobs.JobsClient().Create(job)
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace. All the arguments that were provided to this command invocation. Returns: Some value that we want to have printed later. """ project_ref = resources.REGISTRY.Parse( properties.VALUES.core.project.Get(required=True), collection='ml.projects') job = jobs.BuildBatchPredictionJob( job_name=args.job, model_name=args.model, version_name=args.version, input_paths=args.input_paths, data_format=args.data_format, output_path=args.output_path, region=args.region, runtime_version=args.runtime_version) return jobs.JobsClient().Create(project_ref, job)
def Run(self, args): job_ref = resources.REGISTRY.Parse(args.job, collection='ml.projects.jobs') job = jobs.JobsClient().Get(job_ref) self.job = job # Hack to make the Epilog() method work. return job
def Run(self, args): return jobs_util.List(jobs.JobsClient('v1'))
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace. All the arguments that were provided to this command invocation. Returns: Some value that we want to have printed later. """ region = properties.VALUES.compute.region.Get(required=True) staging_location = jobs_prep.GetStagingLocation( staging_bucket=args.staging_bucket, job_id=args.job, job_dir=args.job_dir) try: uris = jobs_prep.UploadPythonPackages( packages=args.packages, package_path=args.package_path, staging_location=staging_location) except jobs_prep.NoStagingLocationError: raise flags.ArgumentError( 'If local packages are provided, the `--staging-bucket` or ' '`--job-dir` flag must be given.') log.debug('Using {0} as trainer uris'.format(uris)) scale_tier_enum = (jobs.GetMessagesModule( ).GoogleCloudMlV1beta1TrainingInput.ScaleTierValueValuesEnum) scale_tier = scale_tier_enum( args.scale_tier) if args.scale_tier else None job = jobs.BuildTrainingJob( path=args.config, module_name=args.module_name, job_name=args.job, trainer_uri=uris, region=region, job_dir=args.job_dir.ToUrl() if args.job_dir else None, scale_tier=scale_tier, user_args=args.user_args, runtime_version=args.runtime_version) jobs_client = jobs.JobsClient() project_ref = resources.REGISTRY.Parse( properties.VALUES.core.project.Get(required=True), collection='ml.projects') job = jobs_client.Create(project_ref, job) log.status.Print('Job [{}] submitted successfully.'.format(job.jobId)) if args. async: log.status.Print(_FOLLOW_UP_MESSAGE.format(job_id=job.jobId)) return job log_fetcher = stream.LogFetcher( filters=log_utils.LogFilters(job.jobId), polling_interval=_POLLING_INTERVAL, continue_func=log_utils.MakeContinueFunction(job.jobId)) printer = resource_printer.Printer(log_utils.LOG_FORMAT, out=log.err) def _CtrlCHandler(signal, frame): del signal, frame # Unused raise KeyboardInterrupt with execution_utils.CtrlCSection(_CtrlCHandler): try: printer.Print(log_utils.SplitMultiline( log_fetcher.YieldLogs())) except KeyboardInterrupt: log.status.Print('Received keyboard interrupt.') log.status.Print(_FOLLOW_UP_MESSAGE.format(job_id=job.jobId)) job_ref = resources.REGISTRY.Parse(job.jobId, collection='ml.projects.jobs') job = jobs_client.Get(job_ref) # If the job itself failed, we will return a failure status. if job.state is not job.StateValueValuesEnum.SUCCEEDED: self.exit_code = 1 return job
def Run(self, args): return jobs_util.Cancel(jobs.JobsClient('v1'), args.job)
def Run(self, args): job = jobs_util.Describe(jobs.JobsClient('v1'), args.job) self.job = job # Hack to make the Epilog method work return job