def _run_job(self, api):
cluster_type = clusters.CLUSTER_TYPE_SINGLE_JOB if self.run_on_single_use_cluster \
else clusters.CLUSTER_TYPE_PERSISTENT
cluster_id = None
if self.cluster_size == 0:
# Use local cluster
cluster_id = clusters.LOCAL_CLUSTER_ID
elif not self.run_on_single_use_cluster:
# search for a suitable cluster
usable_clusters = self._get_usable_clusters(api, min_size=self.cluster_size)
if usable_clusters:
# grab the largest usable cluster
largest_cluster = sorted(usable_clusters, key=lambda c: int(c['size']), reverse=True)[0]
logger.info('Using largest running usable cluster with cluster_id [%s], size [%s]' % \
(largest_cluster['cluster_id'], largest_cluster['size']))
cluster_id = largest_cluster['cluster_id']
if cluster_id:
job_id = jobs.post_job_existing_cluster(api, self.project(), self.script(), cluster_id,
git_ref=self._git_ref(), parameters=self.parameters(),
notify_on_job_finish=self.notify_on_job_finish, is_control_script=self.is_control_script(),
pig_version=self.pig_version, pipeline_job_id=self._get_pipeline_job_id())
else:
job_id = jobs.post_job_new_cluster(api, self.project(), self.script(), self.cluster_size,
cluster_type=cluster_type, git_ref=self._git_ref(), parameters=self.parameters(),
notify_on_job_finish=self.notify_on_job_finish, is_control_script=self.is_control_script(),
pig_version=self.pig_version, use_spot_instances=self.use_spot_instances,
pipeline_job_id=self._get_pipeline_job_id())
logger.info('Submitted new job to mortar with job_id [%s]' % job_id)
return job_id
def eval_node(G,node): G.node[node]['running'] = 1 #time.sleep(G.node[node]['script']) #print (node,G.node[node]['script']) job_id = jobs.post_job_new_cluster(api, project_name, G.node[node]['script'], cluster_size) final_job_status = jobs.block_until_job_complete(api, job_id) print (final_job_status) update_linked_inputs(G,node) G.node[node]['running'] = 0 G.node[node]['executed'] = 1 G.node[node]['available'] = 0 return
def _run_job(self, api):
cluster_type = clusters.CLUSTER_TYPE_SINGLE_JOB if self.run_on_single_use_cluster \
else clusters.CLUSTER_TYPE_PERSISTENT
cluster_id = None
if self.cluster_size == 0:
# Use local cluster
cluster_id = clusters.LOCAL_CLUSTER_ID
elif not self.run_on_single_use_cluster:
# search for a suitable cluster
idle_clusters = self._get_idle_clusters(api,
min_size=self.cluster_size)
if idle_clusters:
# grab the idle largest cluster that's big enough to use
largest_cluster = sorted(idle_clusters,
key=lambda c: int(c['size']),
reverse=True)[0]
logger.info('Using largest running idle cluster with cluster_id [%s], size [%s]' % \
(largest_cluster['cluster_id'], largest_cluster['size']))
cluster_id = largest_cluster['cluster_id']
if cluster_id:
job_id = jobs.post_job_existing_cluster(
api,
self.project(),
self.script(),
cluster_id,
git_ref=self._git_ref(),
parameters=self.parameters(),
notify_on_job_finish=self.notify_on_job_finish,
is_control_script=self.is_control_script(),
pig_version=self.pig_version,
pipeline_job_id=self._get_pipeline_job_id())
else:
job_id = jobs.post_job_new_cluster(
api,
self.project(),
self.script(),
self.cluster_size,
cluster_type=cluster_type,
git_ref=self._git_ref(),
parameters=self.parameters(),
notify_on_job_finish=self.notify_on_job_finish,
is_control_script=self.is_control_script(),
pig_version=self.pig_version,
use_spot_instances=self.use_spot_instances,
pipeline_job_id=self._get_pipeline_job_id())
logger.info('Submitted new job to mortar with job_id [%s]' % job_id)
return job_id
def test_post_job_existing_cluster(self):
cluster_id = 'xyz123fdsa'
expected_job_id = 'abcdefghijklmnop'
self.api_mock.post.return_value = {'job_id': expected_job_id}
actual_job_id = jobs.post_job_new_cluster(self.api_mock, self.project_name, self.script_name, cluster_id)
self.assertEquals(expected_job_id, actual_job_id)
