def __submit_job(self, job_index, job): logging.info("Preparing for Job #{0}...".format(job_index)) with open(job['model_file_path']) as xml_file: model_xml_doc = xml_file.read() params = job["params"] params['document'] = model_xml_doc params['bucketname'] = self.output_store_info['bucket_name'] task_id = str(uuid.uuid4()) result = helper.execute_cloud_task( params=params, agent_type=self.AGENT_TYPE, ec2_access_key=self.aws_credentials["AWS_ACCESS_KEY_ID"], ec2_secret_key=self.aws_credentials["AWS_SECRET_ACCESS_KEY"], task_id=task_id, instance_type=self.INSTANCE_TYPE, cost_replay=False, database=self.database) if result["success"]: logging.info("Job #{0} successfully submitted to backend.".format( job_index)) else: logging.info( "Failed to submit Job #{0} to backend.".format(job_index)) logging.debug("result = {0}".format(pprint.pformat(result))) return result
def __submit_job(self, job_index, job): logging.info("Preparing for Job #{0}...".format(job_index)) with open(job['model_file_path']) as xml_file: model_xml_doc = xml_file.read() params = job["params"] params['document'] = model_xml_doc params['bucketname'] = self.output_store_info['bucket_name'] task_id = str(uuid.uuid4()) result = helper.execute_cloud_task(params=params, agent_type=self.AGENT_TYPE, ec2_access_key=self.aws_credentials["AWS_ACCESS_KEY_ID"], ec2_secret_key=self.aws_credentials["AWS_SECRET_ACCESS_KEY"], task_id=task_id, instance_type=self.INSTANCE_TYPE, cost_replay=False, database=self.database) if result["success"]: logging.info("Job #{0} successfully submitted to backend.".format(job_index)) else: logging.info("Failed to submit Job #{0} to backend.".format(job_index)) logging.debug("result = {0}".format(pprint.pformat(result))) return result
def submit_cloud_task(self, params, agent_type=None, cost_replay=False, instance_type=None): logging.debug('submit_cloud_task() params =\n{}\n\n'.format(pprint.pformat(params))) if agent_type is None: if self.active_agent_type is not None: agent_type = self.active_agent_type else: self.isOneOrMoreComputeNodesRunning() if self.active_agent_type is not None: agent_type = self.active_agent_type else: raise Exception("No Cloud resources found") if agent_type not in JobConfig.SUPPORTED_AGENT_TYPES: raise Exception('Unsupported agent type {0}'.format(agent_type)) credentials = self.get_credentials() if agent_type == AgentTypes.EC2: params['resource'] = self.EC2_CLOUD_RESOURCE params['bucketname'] = self.user_data.S3_bucket_name if 'EC2_ACCESS_KEY' not in credentials or credentials['EC2_ACCESS_KEY'] == '': raise Exception('EC2 Access Key is not valid!') if 'EC2_SECRET_KEY' not in credentials or credentials['EC2_SECRET_KEY'] == '': raise Exception('EC2 Secret Key is not valid!') ec2_access_key = credentials['EC2_ACCESS_KEY'] ec2_secret_key = credentials['EC2_SECRET_KEY'] logging.debug('ec2_access_key = {0}, ec2_secret_key = {1}'.format(ec2_access_key, ec2_secret_key)) database = DynamoDB(ec2_access_key, ec2_secret_key) storage_agent = S3StorageAgent(bucket_name=self.user_data.S3_bucket_name, ec2_secret_key=ec2_secret_key, ec2_access_key=ec2_access_key) elif agent_type == AgentTypes.FLEX: params['resource'] = self.FLEX_CLOUD_RESOURCE params['bucketname'] = '' # if flex_credentials == None or 'flex_queue_head' not in flex_credentials \ # or 'flex_db_password' not in flex_credentials: # raise Exception('Please pass valid Flex credentials!') database = FlexDB(ip=credentials['queue_head_ip'], password=credentials['flex_db_password']) flex_queue_head_machine = self.user_data.get_flex_queue_head_machine() storage_agent = FlexStorageAgent(queue_head_ip=flex_queue_head_machine['ip'], queue_head_username=flex_queue_head_machine['username'], queue_head_keyfile= os.path.join('/home', flex_queue_head_machine['username'], FlexConfig.QUEUE_HEAD_KEY_DIR, os.path.basename(flex_queue_head_machine['keyfile'])) ) #queue_head_keyfile=flex_queue_head_machine['keyfile']) ec2_access_key = None ec2_secret_key = None # if there is no taskid explicit, create one the first run if 'rerun_uuid' in params and params['rerun_uuid'] is not None: task_id = params['rerun_uuid'] elif cost_replay: task_id = params['cost_analysis_uuid'] else: task_id = str(uuid.uuid4()) logging.debug('submit_cloud_task: task_id = {}'.format(task_id)) result = helper.execute_cloud_task(params=params, agent_type=agent_type, ec2_access_key=ec2_access_key, ec2_secret_key=ec2_secret_key, task_id=task_id, instance_type=instance_type, cost_replay=cost_replay, database=database, storage_agent=storage_agent) return result