def create_job(self, job_config, service_config):
"""
Create new instance of job based on type,
"""
job_type = job_config.get(self.job_type_key)
if not job_type and self.can_skip:
job_class = NoOpJob
elif not job_type:
raise MashJobException('No job type provided, cannot create job.')
else:
try:
job_class = self.job_types[job_type]
except KeyError:
raise MashJobException(
'Job type {0} is not supported in {1} service.'.format(
job_type, self.service_name))
try:
job = job_class(job_config, service_config)
except Exception as error:
raise MashJobException(
'Invalid job configuration: {0}'.format(error))
return job
def request_credentials(self, accounts, cloud=None):
"""
Request credentials from credential service.
Only send request if credentials not already populated.
"""
if self.credentials:
return
data = {
'cloud': cloud or self.cloud,
'cloud_accounts': accounts,
'requesting_user': self.requesting_user
}
try:
response = handle_request(
self.config.get_credentials_url(),
'credentials/',
'get',
job_data=data
)
self.credentials = response.json()
except Exception:
raise MashJobException(
'Credentials request failed for accounts: {accounts}'.format(
accounts=', '.join(accounts)
)
)
def __init__(self, job_config, config):
self.job_config = job_config
# Properties
self._cloud_image_name = None
self._credentials = None
self._log_callback = None
self._job_file = job_config.get('job_file')
self.config = config
self.status_msg = {'status': UNKOWN, 'errors': []}
try:
self.id = job_config['id']
self.last_service = job_config['last_service']
self.requesting_user = job_config['requesting_user']
self.cloud = job_config['cloud']
self.utctime = job_config['utctime']
except KeyError as error:
raise MashJobException(
'Jobs require a(n) {0} key in the job doc.'.format(
error
)
)
self.post_init()
def validate_azure_job(job_doc):
"""
Validate job.
And update target_account_info for given job doc.
"""
validate_job(job_doc)
cloud_account = get_azure_account(job_doc['cloud_account'],
job_doc['requesting_user'])
attrs = ('region', 'source_container', 'source_resource_group',
'source_storage_account')
publish_args = ('label', 'offer_id', 'publisher_id', 'sku')
for attr in attrs:
if attr not in job_doc:
job_doc[attr] = cloud_account[attr]
services = get_services_by_last_service(job_doc['last_service'])
if 'publish' in services:
for arg in publish_args:
if arg not in job_doc:
raise MashJobException(
'Azure publishing jobs require a(n) '
' {arg} argument in the job document.'.format(arg=arg))
return job_doc
def validate_oci_job(job_doc):
"""
Update target_account_info for given job doc.
"""
job_doc = validate_job(job_doc)
user_id = job_doc['requesting_user']
cloud_account = get_oci_account(job_doc['cloud_account'], user_id)
attrs = [
'region',
'bucket',
'availability_domain',
'compartment_id',
'oci_user_id',
'tenancy',
]
create_args = ['operating_system', 'operating_system_version']
for attr in attrs:
if attr not in job_doc:
job_doc[attr] = cloud_account[attr]
services = get_services_by_last_service(job_doc['last_service'])
if 'create' in services:
for arg in create_args:
if arg not in job_doc:
raise MashJobException(
'OCI jobs that create an image require an '
' {arg} argument in the job document.'.format(arg=arg))
return job_doc
def validate_deprecate_args(data):
"""
Validate required args for image deprecate jobs.
"""
if 'old_cloud_image_name' not in data:
raise MashJobException('Jobs that perform image deprecate require '
'old_cloud_image_name in the job doc.')
def validate_job(data):
"""
Validate job doc.
"""
data = normalize_dictionary(data)
if data.get('use_build_time') and \
'{date}' not in data['cloud_image_name']:
raise MashJobException(
'When use_build_time flag is True the {date} '
'format string is required in cloud_image_name.')
validate_last_service(data)
services_run = get_services_by_last_service(data['last_service'])
if 'create' in services_run:
validate_create_args(data)
if 'deprecate' in services_run:
validate_deprecate_args(data)
if data.get('notify') and not data.get('notification_email'):
user_id = data['requesting_user']
user = get_user_by_id(user_id)
data['notification_email'] = user['email']
return data
def validate_last_service(data):
"""
Validate last service is a valid service name.
"""
if data['last_service'] not in current_app.config['SERVICE_NAMES']:
raise MashJobException('The service name {name} is invalid. '
'Valid service names are: {services}.'.format(
name=data['last_service'],
services=', '.join(
current_app.config['SERVICE_NAMES'])))
def create_job(data):
"""
Create a new job for user.
"""
if data.get('dry_run'):
return None
job_id = get_new_job_id()
data['job_id'] = job_id
user_id = data['requesting_user']
kwargs = {
'job_id': job_id,
'last_service': data['last_service'],
'utctime': data['utctime'],
'image': data['image'],
'download_url': data['download_url'],
'user_id': user_id,
'state': RUNNING,
'current_service': current_app.config['SERVICE_NAMES'][0]
}
if data['utctime'] != 'now':
kwargs['start_time'] = parser.parse(data['utctime'])
if data.get('cloud_architecture'):
kwargs['cloud_architecture'] = data['cloud_architecture']
if data.get('profile'):
kwargs['profile'] = data['profile']
response = handle_request(current_app.config['DATABASE_API_URL'],
'jobs/',
'post',
job_data=kwargs)
try:
publish('jobcreator', 'job_document', json.dumps(data, sort_keys=True))
except Exception:
try:
handle_request(current_app.config['DATABASE_API_URL'],
'jobs/',
'delete',
job_data={
'job_id': job_id,
'user_id': user_id
})
except Exception:
pass # Attempt to cleanup job in database
raise MashJobException('Failed to initialize job.')
return response.json()
def validate_create_args(data):
"""
Validate required args for image creation jobs.
"""
required_args = ['cloud_image_name', 'image_description']
for required_arg in required_args:
if required_arg not in data:
raise MashJobException(
'Jobs that perform image creation require {arg_name} '
'in the job doc.'.format(arg_name=required_arg))
def validate_gce_job(job_doc):
"""
Validate job.
And update target_account_info for given job doc.
"""
job_doc = validate_job(job_doc)
cloud_account = get_gce_account(job_doc['cloud_account'],
job_doc['requesting_user'])
attrs = ['region', 'bucket', 'testing_account']
for attr in attrs:
if attr not in job_doc:
job_doc[attr] = cloud_account.get(attr)
services = get_services_by_last_service(job_doc['last_service'])
if 'create' in services:
if cloud_account['is_publishing_account'] and not job_doc.get(
'family'):
raise MashJobException(
'Jobs using a GCE publishing account require a family.')
if 'test' in services:
if cloud_account['is_publishing_account'] and not job_doc.get(
'testing_account'):
raise MashJobException(
'Jobs using a GCE publishing account require'
' the use of a test account.')
if cloud_account['is_publishing_account'] and not job_doc.get(
'image_project'):
raise MashJobException(
'Jobs using a GCE publishing account require an image_project.'
)
return job_doc
def add_target_ec2_account(
account,
accounts,
cloud_accounts,
helper_images,
use_root_swap=None,
skip_replication=False
):
"""
Update job with account information.
- Append any additional regions
- Update ami for root swap if use_root_swap set
"""
job_doc_data = cloud_accounts.get(account['name'], {})
region_name = job_doc_data.get('region') or account.get('region')
subnet = job_doc_data.get('subnet') or account.get('subnet')
if skip_replication:
regions = [region_name]
if account.get('additional_regions'): # In case an additional region is used
for region in account['additional_regions']:
helper_images[region['name']] = region['helper_image']
else:
regions = get_ec2_regions_by_partition(account['partition'])
if account.get('additional_regions'):
for region in account['additional_regions']:
helper_images[region['name']] = region['helper_image']
regions.append(region['name'])
if use_root_swap:
try:
helper_image = job_doc_data['root_swap_ami']
except KeyError:
raise MashJobException(
'root_swap_ami is required for account {0},'
' when using root swap.'.format(account['name'])
)
else:
helper_image = helper_images[region_name]
accounts[region_name] = {
'account': account['name'],
'partition': account['partition'],
'target_regions': list(set(regions)), # Remove any duplicates
'helper_image': helper_image,
'subnet': subnet
}
def delete_job(job_id, user_id):
"""Delete job for user."""
response = handle_request(current_app.config['DATABASE_API_URL'],
'jobs/',
'delete',
job_data={
'job_id': job_id,
'user_id': user_id
})
try:
rows_deleted = response.json()['rows_deleted']
except KeyError:
raise MashJobException('Delete job failed')
return rows_deleted