def __submit_job(self, job_index, job):
logging.info("Preparing for Job #{0}...".format(job_index))
with open(job['model_file_path']) as xml_file:
model_xml_doc = xml_file.read()
params = job["params"]
params['document'] = model_xml_doc
params['bucketname'] = self.output_store_info['bucket_name']
task_id = str(uuid.uuid4())
result = helper.execute_cloud_task(
params=params,
agent_type=self.AGENT_TYPE,
ec2_access_key=self.aws_credentials["AWS_ACCESS_KEY_ID"],
ec2_secret_key=self.aws_credentials["AWS_SECRET_ACCESS_KEY"],
task_id=task_id,
instance_type=self.INSTANCE_TYPE,
cost_replay=False,
database=self.database)
if result["success"]:
logging.info("Job #{0} successfully submitted to backend.".format(
job_index))
else:
logging.info(
"Failed to submit Job #{0} to backend.".format(job_index))
logging.debug("result = {0}".format(pprint.pformat(result)))
return result
def __submit_job(self, job_index, job):
logging.info("Preparing for Job #{0}...".format(job_index))
with open(job['model_file_path']) as xml_file:
model_xml_doc = xml_file.read()
params = job["params"]
params['document'] = model_xml_doc
params['bucketname'] = self.output_store_info['bucket_name']
task_id = str(uuid.uuid4())
result = helper.execute_cloud_task(params=params,
agent_type=self.AGENT_TYPE,
ec2_access_key=self.aws_credentials["AWS_ACCESS_KEY_ID"],
ec2_secret_key=self.aws_credentials["AWS_SECRET_ACCESS_KEY"],
task_id=task_id,
instance_type=self.INSTANCE_TYPE,
cost_replay=False,
database=self.database)
if result["success"]:
logging.info("Job #{0} successfully submitted to backend.".format(job_index))
else:
logging.info("Failed to submit Job #{0} to backend.".format(job_index))
logging.debug("result = {0}".format(pprint.pformat(result)))
return result
def submit_cloud_task(self, params, agent_type=None, cost_replay=False, instance_type=None):
logging.debug('submit_cloud_task() params =\n{}\n\n'.format(pprint.pformat(params)))
if agent_type is None:
if self.active_agent_type is not None:
agent_type = self.active_agent_type
else:
self.isOneOrMoreComputeNodesRunning()
if self.active_agent_type is not None:
agent_type = self.active_agent_type
else:
raise Exception("No Cloud resources found")
if agent_type not in JobConfig.SUPPORTED_AGENT_TYPES:
raise Exception('Unsupported agent type {0}'.format(agent_type))
credentials = self.get_credentials()
if agent_type == AgentTypes.EC2:
params['resource'] = self.EC2_CLOUD_RESOURCE
params['bucketname'] = self.user_data.S3_bucket_name
if 'EC2_ACCESS_KEY' not in credentials or credentials['EC2_ACCESS_KEY'] == '':
raise Exception('EC2 Access Key is not valid!')
if 'EC2_SECRET_KEY' not in credentials or credentials['EC2_SECRET_KEY'] == '':
raise Exception('EC2 Secret Key is not valid!')
ec2_access_key = credentials['EC2_ACCESS_KEY']
ec2_secret_key = credentials['EC2_SECRET_KEY']
logging.debug('ec2_access_key = {0}, ec2_secret_key = {1}'.format(ec2_access_key, ec2_secret_key))
database = DynamoDB(ec2_access_key, ec2_secret_key)
storage_agent = S3StorageAgent(bucket_name=self.user_data.S3_bucket_name,
ec2_secret_key=ec2_secret_key,
ec2_access_key=ec2_access_key)
elif agent_type == AgentTypes.FLEX:
params['resource'] = self.FLEX_CLOUD_RESOURCE
params['bucketname'] = ''
# if flex_credentials == None or 'flex_queue_head' not in flex_credentials \
# or 'flex_db_password' not in flex_credentials:
# raise Exception('Please pass valid Flex credentials!')
database = FlexDB(ip=credentials['queue_head_ip'],
password=credentials['flex_db_password'])
flex_queue_head_machine = self.user_data.get_flex_queue_head_machine()
storage_agent = FlexStorageAgent(queue_head_ip=flex_queue_head_machine['ip'],
queue_head_username=flex_queue_head_machine['username'],
queue_head_keyfile= os.path.join('/home', flex_queue_head_machine['username'], FlexConfig.QUEUE_HEAD_KEY_DIR, os.path.basename(flex_queue_head_machine['keyfile']))
)
#queue_head_keyfile=flex_queue_head_machine['keyfile'])
ec2_access_key = None
ec2_secret_key = None
# if there is no taskid explicit, create one the first run
if 'rerun_uuid' in params and params['rerun_uuid'] is not None:
task_id = params['rerun_uuid']
elif cost_replay:
task_id = params['cost_analysis_uuid']
else:
task_id = str(uuid.uuid4())
logging.debug('submit_cloud_task: task_id = {}'.format(task_id))
result = helper.execute_cloud_task(params=params, agent_type=agent_type,
ec2_access_key=ec2_access_key,
ec2_secret_key=ec2_secret_key,
task_id=task_id, instance_type=instance_type,
cost_replay=cost_replay,
database=database,
storage_agent=storage_agent)
return result
