def main(argv=None):
parser = argparse.ArgumentParser(description='ML DataProc Setup')
parser.add_argument('--project', type=str, help='Google Cloud project ID to use.')
parser.add_argument('--region', type=str, help='Which zone for GCE VMs.')
parser.add_argument('--name', type=str, help='The name of the cluster to create.')
parser.add_argument('--staging', type=str, help='GCS path to use for staging.')
parser.add_argument('--output-dir-uri-output-path',
type=str,
default='/output.txt',
help='Local output path for the file containing the output dir URI.')
args = parser.parse_args()
code_path = os.path.dirname(os.path.realpath(__file__))
init_file_source = os.path.join(code_path, 'initialization_actions.sh')
dest_files = _utils.copy_resources_to_gcs([init_file_source], args.staging)
try:
api = _utils.get_client()
print('Creating cluster...')
create_response = _utils.create_cluster(api, args.project, args.region, args.name, dest_files[0])
print('Cluster creation request submitted. Waiting for completion...')
_utils.wait_for_operation(api, create_response['name'])
Path(args.output_dir_uri_output_path).parent.mkdir(parents=True, exist_ok=True)
Path(args.output_dir_uri_output_path).write_text(args.output)
print('Cluster created.')
finally:
_utils.remove_resources_from_gcs(dest_files)
def main(argv=None):
parser = argparse.ArgumentParser(description='ML DataProc Setup')
parser.add_argument('--project',
type=str,
help='Google Cloud project ID to use.')
parser.add_argument('--region', type=str, help='Which zone for GCE VMs.')
parser.add_argument('--name',
type=str,
help='The name of the cluster to create.')
parser.add_argument('--staging',
type=str,
help='GCS path to use for staging.')
args = parser.parse_args()
code_path = os.path.dirname(os.path.realpath(__file__))
dirname = os.path.basename(__file__).split('.')[0]
init_file_source = os.path.join(code_path, dirname,
'initialization_actions.sh')
dest_files = _utils.copy_resources_to_gcs([init_file_source], args.staging)
try:
api = _utils.get_client()
print('Creating cluster...')
create_response = _utils.create_cluster(api, args.project, args.region,
args.name, dest_files[0])
print('Cluster creation request submitted. Waiting for completion...')
_utils.wait_for_operation(api, create_response['name'])
with open('/output.txt', 'w') as f:
f.write(args.name)
print('Cluster created.')
finally:
_utils.remove_resources_from_gcs(dest_files)
def main(argv=None):
parser = argparse.ArgumentParser(description='ML Analyzer')
parser.add_argument('--project', type=str, help='Google Cloud project ID to use.')
parser.add_argument('--region', type=str, help='Which zone to run the analyzer.')
parser.add_argument('--cluster', type=str, help='The name of the cluster to run job.')
parser.add_argument('--output', type=str, help='GCS path to use for output.')
parser.add_argument('--train', type=str, help='GCS path of the training csv file.')
parser.add_argument('--schema', type=str, help='GCS path of the json schema file.')
parser.add_argument('--output-dir-uri-output-path',
type=str,
default='/output.txt',
help='Local output path for the file containing the output dir URI.')
args = parser.parse_args()
code_path = os.path.dirname(os.path.realpath(__file__))
runfile_source = os.path.join(code_path, 'analyze_run.py')
dest_files = _utils.copy_resources_to_gcs([runfile_source], args.output)
try:
api = _utils.get_client()
print('Submitting job...')
spark_args = ['--output', args.output, '--train', args.train, '--schema', args.schema]
job_id = _utils.submit_pyspark_job(
api, args.project, args.region, args.cluster, dest_files[0], spark_args)
print('Job request submitted. Waiting for completion...')
_utils.wait_for_job(api, args.project, args.region, job_id)
Path(args.output_dir_uri_output_path).parent.mkdir(parents=True, exist_ok=True)
Path(args.output_dir_uri_output_path).write_text(args.output)
print('Job completed.')
finally:
_utils.remove_resources_from_gcs(dest_files)
def main(argv=None):
parser = argparse.ArgumentParser(description='ML Analyzer')
parser.add_argument('--project',
type=str,
help='Google Cloud project ID to use.')
parser.add_argument('--region',
type=str,
help='Which zone to run the analyzer.')
parser.add_argument('--cluster',
type=str,
help='The name of the cluster to run job.')
parser.add_argument('--output',
type=str,
help='GCS path to use for output.')
parser.add_argument('--train',
type=str,
help='GCS path of the training csv file.')
parser.add_argument('--schema',
type=str,
help='GCS path of the json schema file.')
args = parser.parse_args()
code_path = os.path.dirname(os.path.realpath(__file__))
dirname = os.path.basename(__file__).split('.')[0]
runfile_source = os.path.join(code_path, dirname, 'run.py')
dest_files = _utils.copy_resources_to_gcs([runfile_source], args.output)
try:
api = _utils.get_client()
print('Submitting job...')
spark_args = [
'--output', args.output, '--train', args.train, '--schema',
args.schema
]
job_id = _utils.submit_pyspark_job(api, args.project, args.region,
args.cluster, dest_files[0],
spark_args)
print('Job request submitted. Waiting for completion...')
_utils.wait_for_job(api, args.project, args.region, job_id)
with open('/output.txt', 'w') as f:
f.write(args.output)
print('Job completed.')
finally:
_utils.remove_resources_from_gcs(dest_files)
def main(argv=None):
parser = argparse.ArgumentParser(description='ML Transfomer')
parser.add_argument('--project',
type=str,
help='Google Cloud project ID to use.')
parser.add_argument('--region',
type=str,
help='Which zone to run the analyzer.')
parser.add_argument('--cluster',
type=str,
help='The name of the cluster to run job.')
parser.add_argument('--output',
type=str,
help='GCS path to use for output.')
parser.add_argument('--train',
type=str,
help='GCS path of the training csv file.')
parser.add_argument('--eval',
type=str,
help='GCS path of the eval csv file.')
parser.add_argument('--analysis',
type=str,
help='GCS path of the analysis results.')
parser.add_argument('--target', type=str, help='Target column name.')
args = parser.parse_args()
# Remove existing [output]/train and [output]/eval if they exist.
# It should not be done in the run time code because run time code should be portable
# to on-prem while we need gsutil here.
_utils.delete_directory_from_gcs(os.path.join(args.output, 'train'))
_utils.delete_directory_from_gcs(os.path.join(args.output, 'eval'))
code_path = os.path.dirname(os.path.realpath(__file__))
dirname = os.path.basename(__file__).split('.')[0]
runfile_source = os.path.join(code_path, dirname, 'run.py')
dest_files = _utils.copy_resources_to_gcs([runfile_source], args.output)
try:
api = _utils.get_client()
print('Submitting job...')
spark_args = [
'--output', args.output, '--analysis', args.analysis, '--target',
args.target
]
if args.train:
spark_args.extend(['--train', args.train])
if args.eval:
spark_args.extend(['--eval', args.eval])
job_id = _utils.submit_pyspark_job(api, args.project, args.region,
args.cluster, dest_files[0],
spark_args)
print('Job request submitted. Waiting for completion...')
_utils.wait_for_job(api, args.project, args.region, job_id)
with open('/output_train.txt', 'w') as f:
f.write(os.path.join(args.output, 'train', 'part-*'))
with open('/output_eval.txt', 'w') as f:
f.write(os.path.join(args.output, 'eval', 'part-*'))
print('Job completed.')
finally:
_utils.remove_resources_from_gcs(dest_files)