def train(job_id, bucket_name, region, config_files, dataset, scale_tier,
master_type, worker_type, worker_count, parameter_server_type,
parameter_server_count):
account = ServiceAccount()
account.validate_region(region)
if bucket_name is None:
bucket_name = 'luminoth-{}'.formata(account.client_id)
click.echo(
'Bucket name not specified. Using "{}".'.format(bucket_name))
# Creates bucket for logs and models if it doesn't exist
bucket = account.get_bucket(bucket_name)
if not job_id:
job_id = 'train_{}'.format(datetime.now().strftime("%Y%m%d_%H%M%S"))
# Define path in bucket to store job's config, logs, etc.
base_path = 'lumi_{}'.format(job_id)
package_path = build_package(bucket, base_path)
job_dir = 'gs://{}/{}'.format(bucket_name, base_path)
override_params = [
'train.job_dir={}'.format(job_dir),
]
if dataset:
# Check if absolute or relative dataset path
if not dataset.startswith('gs://'):
dataset = 'gs://{}'.format(dataset)
override_params.append('dataset.dir={}'.format(dataset))
config = get_config(config_files, override_params=override_params)
# Update final config file to job bucket
config_path = '{}/{}'.format(base_path, DEFAULT_CONFIG_FILENAME)
upload_data(bucket, config_path, dump_config(config))
args = ['--config', '{}/{}'.format(job_dir, DEFAULT_CONFIG_FILENAME)]
cloudml = account.cloud_service('ml')
training_inputs = {
'scaleTier': scale_tier,
'packageUris': ['gs://{}/{}'.format(bucket_name, package_path)],
'pythonModule': 'luminoth.train',
'args': args,
'region': region,
'jobDir': job_dir,
'runtimeVersion': RUNTIME_VERSION
}
if scale_tier == 'CUSTOM':
training_inputs['masterType'] = master_type
if worker_count > 0:
training_inputs['workerCount'] = worker_count
training_inputs['workerType'] = worker_type
if parameter_server_count > 0:
training_inputs['parameterServerCount'] = parameter_server_count
training_inputs['parameterServerType'] = parameter_server_type
job_spec = {'jobId': job_id, 'trainingInput': training_inputs}
jobrequest = cloudml.projects().jobs().create(body=job_spec,
parent='projects/{}'.format(
account.project_id))
try:
click.echo('Submitting training job.')
res = jobrequest.execute()
click.echo('Job submitted successfully.')
click.echo('state = {}, createTime = {}'.format(
res.get('state'), res.get('createTime')))
click.echo('\nJob id: {}'.format(job_id))
save_run(config, environment='gcloud', extra_config=job_spec)
except Exception as err:
click.echo('There was an error creating the training job. '
'Check the details: \n{}'.format(err._get_reason()))
def train(job_id, service_account_json, bucket_name, region, config_files,
dataset, scale_tier, master_type, worker_type, worker_count,
parameter_server_type, parameter_server_count):
project_id = get_project_id(service_account_json)
if project_id is None:
raise ValueError(
'Missing "project_id" in service_account_json "{}"'.format(
service_account_json))
if bucket_name is None:
client_id = get_client_id(service_account_json)
bucket_name = 'luminoth-{}'.format(client_id)
click.echo(
'Bucket name not specified. Using "{}".'.format(bucket_name))
credentials = get_credentials(service_account_json)
validate_region(region, project_id, credentials)
# Creates bucket for logs and models if it doesn't exist
bucket = get_bucket(service_account_json, bucket_name)
if not job_id:
job_id = 'train_{}'.format(datetime.now().strftime("%Y%m%d_%H%M%S"))
# Define path in bucket to store job's config, logs, etc.
base_path = 'lumi_{}'.format(job_id)
package_path = build_package(bucket, base_path)
job_dir = 'gs://{}/{}/'.format(bucket_name, base_path)
override_params = [
'train.job_dir={}'.format(job_dir),
]
if dataset:
# Check if absolute or relative dataset path
if not dataset.startswith('gs://'):
dataset = 'gs://{}'.format(dataset)
override_params.append('dataset.dir={}'.format(dataset))
config = get_config(config_files, override_params=override_params)
# We should validate config before submitting job
# Update final config file to job bucket
config_path = os.path.join(base_path, DEFAULT_CONFIG_FILENAME)
upload_data(bucket, config_path, dump_config(config))
args = ['--config', os.path.join(job_dir, DEFAULT_CONFIG_FILENAME)]
cloudml = cloud_service(credentials, 'ml')
training_inputs = {
'scaleTier': scale_tier,
'packageUris': [
'gs://{}/{}'.format(bucket_name, package_path)
],
'pythonModule': 'luminoth.train',
'args': args,
'region': region,
'jobDir': job_dir,
'runtimeVersion': RUNTIME_VERSION,
}
if scale_tier == 'CUSTOM':
training_inputs['masterType'] = master_type
if worker_count > 0:
training_inputs['workerCount'] = worker_count
training_inputs['workerType'] = worker_type
if parameter_server_count > 0:
training_inputs['parameterServerCount'] = parameter_server_count
training_inputs['parameterServerType'] = parameter_server_type
job_spec = {
'jobId': job_id,
'trainingInput': training_inputs
}
jobrequest = cloudml.projects().jobs().create(
body=job_spec, parent='projects/{}'.format(project_id))
try:
click.echo('Submitting training job.')
res = jobrequest.execute()
click.echo('Job {} submitted successfully.'.format(job_id))
click.echo('state = {}, createTime = {}'.format(
res.get('state'), res.get('createTime')))
save_run(config, environment='gcloud', extra_config=job_spec)
except Exception as err:
click.echo(
'There was an error creating the training job. '
'Check the details: \n{}'.format(err._get_reason())
)
def train(
job_id,
resume_job_id,
bucket_name,
region,
config_files,
dataset,
scale_tier,
master_type,
worker_type,
worker_count,
parameter_server_type,
parameter_server_count,
):
account = ServiceAccount()
account.validate_region(region)
if bucket_name is None:
bucket_name = "luminoth-{}".format(account.client_id)
click.echo(
'Bucket name not specified. Using "{}".'.format(bucket_name))
# Creates bucket for logs and models if it doesn't exist
bucket = account.get_bucket(bucket_name)
if not job_id:
job_id = "train_{}".format(datetime.now().strftime("%Y%m%d_%H%M%S"))
# Path in bucket to store job's config, logs, etc.
# If we are resuming a previous job, then we will use the same path
# that job used, so Luminoth will load the checkpoint from there.
base_path = "lumi_{}".format(resume_job_id if resume_job_id else job_id)
package_path = build_package(bucket, base_path)
job_dir = "gs://{}/{}".format(bucket_name, base_path)
override_params = [
"train.job_dir={}".format(job_dir),
]
if dataset:
# Check if absolute or relative dataset path
if not dataset.startswith("gs://"):
dataset = "gs://{}".format(dataset)
override_params.append("dataset.dir={}".format(dataset))
# Even if we are resuming job, we will use a new config. Thus, we will
# overwrite the config in the old job's dir if it existed.
config = get_config(config_files, override_params=override_params)
# Update final config file to job bucket
config_path = "{}/{}".format(base_path, DEFAULT_CONFIG_FILENAME)
upload_data(bucket, config_path, dump_config(config))
args = ["--config", "{}/{}".format(job_dir, DEFAULT_CONFIG_FILENAME)]
cloudml = account.cloud_service("ml")
training_inputs = {
"scaleTier": scale_tier,
"packageUris": ["gs://{}/{}".format(bucket_name, package_path)],
"pythonModule": "luminoth.train",
"args": args,
"region": region,
"jobDir": job_dir,
"runtimeVersion": RUNTIME_VERSION,
"pythonVersion": PYTHON_VERSION,
}
if scale_tier == "CUSTOM":
training_inputs["masterType"] = master_type
if worker_count > 0:
training_inputs["workerCount"] = worker_count
training_inputs["workerType"] = worker_type
if parameter_server_count > 0:
training_inputs["parameterServerCount"] = parameter_server_count
training_inputs["parameterServerType"] = parameter_server_type
job_spec = {"jobId": job_id, "trainingInput": training_inputs}
jobrequest = (cloudml.projects().jobs().create(body=job_spec,
parent="projects/{}".format(
account.project_id)))
try:
click.echo("Submitting training job.")
res = jobrequest.execute()
click.echo("Job submitted successfully.")
click.echo("state = {}, createTime = {}".format(
res.get("state"), res.get("createTime")))
if resume_job_id:
click.echo(
"\nNote: this job is resuming job {}.\n".format(resume_job_id))
click.echo("Job id: {}".format(job_id))
click.echo("Job directory: {}".format(job_dir))
save_run(config, environment="gcloud", extra_config=job_spec)
except Exception as err:
click.echo("There was an error creating the training job. "
"Check the details: \n{}".format(err._get_reason()))
def train(job_id, service_account_json, bucket_name, region, config_files,
dataset, scale_tier, master_type, worker_type, worker_count,
parameter_server_type, parameter_server_count):
project_id = get_project_id(service_account_json)
if project_id is None:
raise ValueError(
'Missing "project_id" in service_account_json "{}"'.format(
service_account_json))
if bucket_name is None:
client_id = get_client_id(service_account_json)
bucket_name = 'luminoth-{}'.format(client_id)
click.echo(
'Bucket name not specified. Using "{}".'.format(bucket_name))
credentials = get_credentials(service_account_json)
validate_region(region, project_id, credentials)
# Creates bucket for logs and models if it doesn't exist
bucket = get_bucket(service_account_json, bucket_name)
if not job_id:
job_id = 'train_{}'.format(datetime.now().strftime("%Y%m%d_%H%M%S"))
# Define path in bucket to store job's config, logs, etc.
base_path = 'lumi_{}'.format(job_id)
package_path = build_package(bucket, base_path)
# Check if absolute or relative dataset path
if not dataset.startswith('gs://'):
dataset = 'gs://{}'.format(dataset)
args = []
args.extend([
'-o',
'dataset.dir={}'.format(dataset),
])
override_params = [
'dataset.dir={}'.format(dataset),
]
custom_config = load_config(config_files)
model_class = get_model(custom_config.model.type)
config = get_model_config(
model_class.base_config,
custom_config,
override_params,
)
# We should validate config before submitting job
# Update final config file to job bucket
config_path = os.path.join(base_path, DEFAULT_CONFIG_FILENAME)
upload_data(bucket, config_path, dump_config(config))
args = ['--config', 'gs://{}/{}'.format(bucket_name, config_path)]
cloudml = cloud_service(credentials, 'ml')
training_inputs = {
'scaleTier': scale_tier,
'packageUris': ['gs://{}/{}'.format(bucket_name, package_path)],
'pythonModule': 'luminoth.train',
'args': args,
'region': region,
'jobDir': 'gs://{}/{}/'.format(bucket_name, base_path),
'runtimeVersion': RUNTIME_VERSION
}
if scale_tier == 'CUSTOM':
training_inputs['masterType'] = master_type
training_inputs['workerType'] = worker_type
training_inputs['workerCount'] = worker_count
if parameter_server_count > 0:
training_inputs['parameterServerCount'] = parameter_server_count
training_inputs['parameterServerType'] = parameter_server_type
job_spec = {'jobId': job_id, 'trainingInput': training_inputs}
jobrequest = cloudml.projects().jobs().create(
body=job_spec, parent='projects/{}'.format(project_id))
try:
click.echo('Submitting training job.')
res = jobrequest.execute()
click.echo('Job {} submitted successfully.'.format(job_id))
click.echo('state = {}, createTime = {}'.format(
res.get('state'), res.get('createTime')))
save_run(config, environment='gcloud', extra_config=job_spec)
except Exception as err:
click.echo('There was an error creating the training job. '
'Check the details: \n{}'.format(err._get_reason()))