def main(argv=None):
parser = argparse.ArgumentParser(description='Submit PySpark Job')
parser.add_argument('--region',
type=str,
help='The region where the cluster launches.')
parser.add_argument('--jobflow_id',
type=str,
help='The name of the cluster to run job.')
parser.add_argument('--job_name', type=str, help='The name of spark job.')
parser.add_argument('--py_file',
type=str,
help='A path to a pyspark file run during the step')
parser.add_argument('--input', type=str, help='File path of the dataset.')
parser.add_argument('--output',
type=str,
help='Output path of the result files.')
parser.add_argument('--output_file',
type=str,
help='S3 URI of the training job results.')
args = parser.parse_args()
logging.getLogger().setLevel(logging.INFO)
client = _utils.get_client(args.region)
logging.info('Submitting job to %s...', args.jobflow_id)
spark_args = [args.input, args.output]
step_id = _utils.submit_pyspark_job(client, args.jobflow_id, args.job_name,
args.py_file, spark_args)
logging.info('Job request submitted. Waiting for completion...')
_utils.wait_for_job(client, args.jobflow_id, step_id)
Path('/output.txt').write_text(unicode(step_id))
Path(args.output_file).parent.mkdir(parents=True, exist_ok=True)
Path(args.output_file).write_text(unicode(args.output))
logging.info('Job completed.')
def main(argv=None):
parser = argparse.ArgumentParser(description='ML Analyzer')
parser.add_argument('--project', type=str, help='Google Cloud project ID to use.')
parser.add_argument('--region', type=str, help='Which zone to run the analyzer.')
parser.add_argument('--cluster', type=str, help='The name of the cluster to run job.')
parser.add_argument('--output', type=str, help='GCS path to use for output.')
parser.add_argument('--train', type=str, help='GCS path of the training csv file.')
parser.add_argument('--schema', type=str, help='GCS path of the json schema file.')
parser.add_argument('--output-dir-uri-output-path',
type=str,
default='/output.txt',
help='Local output path for the file containing the output dir URI.')
args = parser.parse_args()
code_path = os.path.dirname(os.path.realpath(__file__))
runfile_source = os.path.join(code_path, 'analyze_run.py')
dest_files = _utils.copy_resources_to_gcs([runfile_source], args.output)
try:
api = _utils.get_client()
print('Submitting job...')
spark_args = ['--output', args.output, '--train', args.train, '--schema', args.schema]
job_id = _utils.submit_pyspark_job(
api, args.project, args.region, args.cluster, dest_files[0], spark_args)
print('Job request submitted. Waiting for completion...')
_utils.wait_for_job(api, args.project, args.region, job_id)
Path(args.output_dir_uri_output_path).parent.mkdir(parents=True, exist_ok=True)
Path(args.output_dir_uri_output_path).write_text(args.output)
print('Job completed.')
finally:
_utils.remove_resources_from_gcs(dest_files)
def main(argv=None):
parser = argparse.ArgumentParser(description='ML Analyzer')
parser.add_argument('--project',
type=str,
help='Google Cloud project ID to use.')
parser.add_argument('--region',
type=str,
help='Which zone to run the analyzer.')
parser.add_argument('--cluster',
type=str,
help='The name of the cluster to run job.')
parser.add_argument('--output',
type=str,
help='GCS path to use for output.')
parser.add_argument('--train',
type=str,
help='GCS path of the training csv file.')
parser.add_argument('--schema',
type=str,
help='GCS path of the json schema file.')
args = parser.parse_args()
code_path = os.path.dirname(os.path.realpath(__file__))
dirname = os.path.basename(__file__).split('.')[0]
runfile_source = os.path.join(code_path, dirname, 'run.py')
dest_files = _utils.copy_resources_to_gcs([runfile_source], args.output)
try:
api = _utils.get_client()
print('Submitting job...')
spark_args = [
'--output', args.output, '--train', args.train, '--schema',
args.schema
]
job_id = _utils.submit_pyspark_job(api, args.project, args.region,
args.cluster, dest_files[0],
spark_args)
print('Job request submitted. Waiting for completion...')
_utils.wait_for_job(api, args.project, args.region, job_id)
with open('/output.txt', 'w') as f:
f.write(args.output)
print('Job completed.')
finally:
_utils.remove_resources_from_gcs(dest_files)
def main(argv=None):
parser = argparse.ArgumentParser(description='ML Transfomer')
parser.add_argument('--project',
type=str,
help='Google Cloud project ID to use.')
parser.add_argument('--region',
type=str,
help='Which zone to run the analyzer.')
parser.add_argument('--cluster',
type=str,
help='The name of the cluster to run job.')
parser.add_argument('--output',
type=str,
help='GCS path to use for output.')
parser.add_argument('--train',
type=str,
help='GCS path of the training csv file.')
parser.add_argument('--eval',
type=str,
help='GCS path of the eval csv file.')
parser.add_argument('--analysis',
type=str,
help='GCS path of the analysis results.')
parser.add_argument('--target', type=str, help='Target column name.')
args = parser.parse_args()
# Remove existing [output]/train and [output]/eval if they exist.
# It should not be done in the run time code because run time code should be portable
# to on-prem while we need gsutil here.
_utils.delete_directory_from_gcs(os.path.join(args.output, 'train'))
_utils.delete_directory_from_gcs(os.path.join(args.output, 'eval'))
code_path = os.path.dirname(os.path.realpath(__file__))
dirname = os.path.basename(__file__).split('.')[0]
runfile_source = os.path.join(code_path, dirname, 'run.py')
dest_files = _utils.copy_resources_to_gcs([runfile_source], args.output)
try:
api = _utils.get_client()
print('Submitting job...')
spark_args = [
'--output', args.output, '--analysis', args.analysis, '--target',
args.target
]
if args.train:
spark_args.extend(['--train', args.train])
if args.eval:
spark_args.extend(['--eval', args.eval])
job_id = _utils.submit_pyspark_job(api, args.project, args.region,
args.cluster, dest_files[0],
spark_args)
print('Job request submitted. Waiting for completion...')
_utils.wait_for_job(api, args.project, args.region, job_id)
with open('/output_train.txt', 'w') as f:
f.write(os.path.join(args.output, 'train', 'part-*'))
with open('/output_eval.txt', 'w') as f:
f.write(os.path.join(args.output, 'eval', 'part-*'))
print('Job completed.')
finally:
_utils.remove_resources_from_gcs(dest_files)