def create_job(self, job_config, service_config): """ Create new instance of job based on type, """ job_type = job_config.get(self.job_type_key) if not job_type and self.can_skip: job_class = NoOpJob elif not job_type: raise MashJobException('No job type provided, cannot create job.') else: try: job_class = self.job_types[job_type] except KeyError: raise MashJobException( 'Job type {0} is not supported in {1} service.'.format( job_type, self.service_name)) try: job = job_class(job_config, service_config) except Exception as error: raise MashJobException( 'Invalid job configuration: {0}'.format(error)) return job
def request_credentials(self, accounts, cloud=None): """ Request credentials from credential service. Only send request if credentials not already populated. """ if self.credentials: return data = { 'cloud': cloud or self.cloud, 'cloud_accounts': accounts, 'requesting_user': self.requesting_user } try: response = handle_request( self.config.get_credentials_url(), 'credentials/', 'get', job_data=data ) self.credentials = response.json() except Exception: raise MashJobException( 'Credentials request failed for accounts: {accounts}'.format( accounts=', '.join(accounts) ) )
def __init__(self, job_config, config): self.job_config = job_config # Properties self._cloud_image_name = None self._credentials = None self._log_callback = None self._job_file = job_config.get('job_file') self.config = config self.status_msg = {'status': UNKOWN, 'errors': []} try: self.id = job_config['id'] self.last_service = job_config['last_service'] self.requesting_user = job_config['requesting_user'] self.cloud = job_config['cloud'] self.utctime = job_config['utctime'] except KeyError as error: raise MashJobException( 'Jobs require a(n) {0} key in the job doc.'.format( error ) ) self.post_init()
def validate_azure_job(job_doc): """ Validate job. And update target_account_info for given job doc. """ validate_job(job_doc) cloud_account = get_azure_account(job_doc['cloud_account'], job_doc['requesting_user']) attrs = ('region', 'source_container', 'source_resource_group', 'source_storage_account') publish_args = ('label', 'offer_id', 'publisher_id', 'sku') for attr in attrs: if attr not in job_doc: job_doc[attr] = cloud_account[attr] services = get_services_by_last_service(job_doc['last_service']) if 'publish' in services: for arg in publish_args: if arg not in job_doc: raise MashJobException( 'Azure publishing jobs require a(n) ' ' {arg} argument in the job document.'.format(arg=arg)) return job_doc
def validate_oci_job(job_doc): """ Update target_account_info for given job doc. """ job_doc = validate_job(job_doc) user_id = job_doc['requesting_user'] cloud_account = get_oci_account(job_doc['cloud_account'], user_id) attrs = [ 'region', 'bucket', 'availability_domain', 'compartment_id', 'oci_user_id', 'tenancy', ] create_args = ['operating_system', 'operating_system_version'] for attr in attrs: if attr not in job_doc: job_doc[attr] = cloud_account[attr] services = get_services_by_last_service(job_doc['last_service']) if 'create' in services: for arg in create_args: if arg not in job_doc: raise MashJobException( 'OCI jobs that create an image require an ' ' {arg} argument in the job document.'.format(arg=arg)) return job_doc
def validate_deprecate_args(data): """ Validate required args for image deprecate jobs. """ if 'old_cloud_image_name' not in data: raise MashJobException('Jobs that perform image deprecate require ' 'old_cloud_image_name in the job doc.')
def validate_job(data): """ Validate job doc. """ data = normalize_dictionary(data) if data.get('use_build_time') and \ '{date}' not in data['cloud_image_name']: raise MashJobException( 'When use_build_time flag is True the {date} ' 'format string is required in cloud_image_name.') validate_last_service(data) services_run = get_services_by_last_service(data['last_service']) if 'create' in services_run: validate_create_args(data) if 'deprecate' in services_run: validate_deprecate_args(data) if data.get('notify') and not data.get('notification_email'): user_id = data['requesting_user'] user = get_user_by_id(user_id) data['notification_email'] = user['email'] return data
def validate_last_service(data): """ Validate last service is a valid service name. """ if data['last_service'] not in current_app.config['SERVICE_NAMES']: raise MashJobException('The service name {name} is invalid. ' 'Valid service names are: {services}.'.format( name=data['last_service'], services=', '.join( current_app.config['SERVICE_NAMES'])))
def create_job(data): """ Create a new job for user. """ if data.get('dry_run'): return None job_id = get_new_job_id() data['job_id'] = job_id user_id = data['requesting_user'] kwargs = { 'job_id': job_id, 'last_service': data['last_service'], 'utctime': data['utctime'], 'image': data['image'], 'download_url': data['download_url'], 'user_id': user_id, 'state': RUNNING, 'current_service': current_app.config['SERVICE_NAMES'][0] } if data['utctime'] != 'now': kwargs['start_time'] = parser.parse(data['utctime']) if data.get('cloud_architecture'): kwargs['cloud_architecture'] = data['cloud_architecture'] if data.get('profile'): kwargs['profile'] = data['profile'] response = handle_request(current_app.config['DATABASE_API_URL'], 'jobs/', 'post', job_data=kwargs) try: publish('jobcreator', 'job_document', json.dumps(data, sort_keys=True)) except Exception: try: handle_request(current_app.config['DATABASE_API_URL'], 'jobs/', 'delete', job_data={ 'job_id': job_id, 'user_id': user_id }) except Exception: pass # Attempt to cleanup job in database raise MashJobException('Failed to initialize job.') return response.json()
def validate_create_args(data): """ Validate required args for image creation jobs. """ required_args = ['cloud_image_name', 'image_description'] for required_arg in required_args: if required_arg not in data: raise MashJobException( 'Jobs that perform image creation require {arg_name} ' 'in the job doc.'.format(arg_name=required_arg))
def validate_gce_job(job_doc): """ Validate job. And update target_account_info for given job doc. """ job_doc = validate_job(job_doc) cloud_account = get_gce_account(job_doc['cloud_account'], job_doc['requesting_user']) attrs = ['region', 'bucket', 'testing_account'] for attr in attrs: if attr not in job_doc: job_doc[attr] = cloud_account.get(attr) services = get_services_by_last_service(job_doc['last_service']) if 'create' in services: if cloud_account['is_publishing_account'] and not job_doc.get( 'family'): raise MashJobException( 'Jobs using a GCE publishing account require a family.') if 'test' in services: if cloud_account['is_publishing_account'] and not job_doc.get( 'testing_account'): raise MashJobException( 'Jobs using a GCE publishing account require' ' the use of a test account.') if cloud_account['is_publishing_account'] and not job_doc.get( 'image_project'): raise MashJobException( 'Jobs using a GCE publishing account require an image_project.' ) return job_doc
def add_target_ec2_account( account, accounts, cloud_accounts, helper_images, use_root_swap=None, skip_replication=False ): """ Update job with account information. - Append any additional regions - Update ami for root swap if use_root_swap set """ job_doc_data = cloud_accounts.get(account['name'], {}) region_name = job_doc_data.get('region') or account.get('region') subnet = job_doc_data.get('subnet') or account.get('subnet') if skip_replication: regions = [region_name] if account.get('additional_regions'): # In case an additional region is used for region in account['additional_regions']: helper_images[region['name']] = region['helper_image'] else: regions = get_ec2_regions_by_partition(account['partition']) if account.get('additional_regions'): for region in account['additional_regions']: helper_images[region['name']] = region['helper_image'] regions.append(region['name']) if use_root_swap: try: helper_image = job_doc_data['root_swap_ami'] except KeyError: raise MashJobException( 'root_swap_ami is required for account {0},' ' when using root swap.'.format(account['name']) ) else: helper_image = helper_images[region_name] accounts[region_name] = { 'account': account['name'], 'partition': account['partition'], 'target_regions': list(set(regions)), # Remove any duplicates 'helper_image': helper_image, 'subnet': subnet }
def delete_job(job_id, user_id): """Delete job for user.""" response = handle_request(current_app.config['DATABASE_API_URL'], 'jobs/', 'delete', job_data={ 'job_id': job_id, 'user_id': user_id }) try: rows_deleted = response.json()['rows_deleted'] except KeyError: raise MashJobException('Delete job failed') return rows_deleted