def submit(self, state, partition="Cluster"):
"""
Writing submit file and submitting a job for PBS-like batch managers
Args:
state: what resource requirements a given glidein has
"""
submit_filename = 'submit.pbs'
if 'filename' in self.config["SubmitFile"]:
submit_filename = self.config["SubmitFile"]["filename"]
cluster_config = self.config[partition]
group_jobs = ("group_jobs" in cluster_config and
cluster_config["group_jobs"] and
"count" in state)
self.write_submit_file(submit_filename, state, group_jobs, cluster_config)
num_submits = 1 if group_jobs else state["count"] if "count" in state else 1
for i in xrange(num_submits):
if self.config['StartdLogging']['send_startd_logs'] is True:
startd_logfile_name = '{}_{}.tar.gz'.format(self.config['Glidein']['site'], uuid.uuid4())
presigned_put_url = get_presigned_put_url(startd_logfile_name, self.config, self.secrets)
presigned_get_url = get_presigned_get_url(startd_logfile_name, self.config, self.secrets)
self.write_submit_file(submit_filename,
state,
group_jobs,
cluster_config,
presigned_put_url,
presigned_get_url)
cmd = self.config[partition]["submit_command"] + " " + submit_filename
if not ('Mode' in self.config and 'dryrun' in self.config['Mode'] and
self.config['Mode']['dryrun']):
subprocess.check_call(cmd, shell=True)
def submit(self, state, partition="Cluster"):
"""
Writing submit file and submitting a job for PBS-like batch managers
Args:
state: what resource requirements a given glidein has
"""
submit_filename = 'submit.pbs'
if 'filename' in self.config["SubmitFile"]:
submit_filename = self.config["SubmitFile"]["filename"]
cluster_config = self.config[partition]
group_jobs = ("group_jobs" in cluster_config
and cluster_config["group_jobs"] and "count" in state)
self.write_submit_file(submit_filename, state, group_jobs,
cluster_config)
num_submits = 1 if group_jobs else state[
"count"] if "count" in state else 1
for i in range(num_submits):
if self.config['StartdLogging']['send_startd_logs'] is True:
startd_logfile_name = '{}_{}.tar.gz'.format(
self.config['Glidein']['site'], uuid.uuid4())
presigned_put_url = get_presigned_put_url(
startd_logfile_name, self.config, self.secrets)
presigned_get_url = get_presigned_get_url(
startd_logfile_name, self.config, self.secrets)
self.write_submit_file(submit_filename, state, group_jobs,
cluster_config, presigned_put_url,
presigned_get_url)
cmd = self.config[partition][
"submit_command"] + " " + submit_filename
if not ('Mode' in self.config and 'dryrun' in self.config['Mode']
and self.config['Mode']['dryrun']):
subprocess.check_call(cmd, shell=True)
def submit(self, state, partition="Cluster"):
"""
Writing submit file and submitting a HTCondor job
Args:
state: what resource requirements a given glidein has
"""
submit_filename = 'submit.condor'
if 'filename' in self.config["SubmitFile"]:
submit_filename = self.config["SubmitFile"]["filename"]
env_filename = 'env_wrapper.sh'
if 'env_wrapper_name' in self.config['SubmitFile']:
env_filename = self.config["SubmitFile"]["env_wrapper_name"]
cluster_config = self.config[partition]
group_jobs = ("group_jobs" in cluster_config and
cluster_config["group_jobs"] and
"count" in state)
self.make_env_wrapper(env_filename, cluster_config)
num_submits = 1 if group_jobs else state["count"] if "count" in state else 1
for i in range(num_submits):
if self.config['StartdLogging']['send_startd_logs'] is True:
startd_logfile_name = '{}_{}.tar.gz'.format(self.config['Glidein']['site'],
uuid.uuid4())
presigned_put_url = get_presigned_put_url(startd_logfile_name, self.config,
self.secrets)
presigned_get_url = get_presigned_get_url(startd_logfile_name, self.config,
self.secrets)
self.make_submit_file(submit_filename,
env_filename,
state,
group_jobs,
cluster_config,
presigned_put_url,
presigned_get_url)
else:
self.make_submit_file(submit_filename,
env_filename,
state,
group_jobs,
cluster_config)
cmd = cluster_config["submit_command"] + " " + submit_filename
print(cmd)
subprocess.check_call(cmd, shell=True)
def submit(self, state, partition="Cluster"):
"""
Writing submit file and submitting a HTCondor job
Args:
state: what resource requirements a given glidein has
"""
submit_filename = 'submit.condor'
if 'filename' in self.config["SubmitFile"]:
submit_filename = self.config["SubmitFile"]["filename"]
env_filename = 'env_wrapper.sh'
if 'env_wrapper_name' in self.config['SubmitFile']:
env_filename = self.config["SubmitFile"]["env_wrapper_name"]
cluster_config = self.config[partition]
group_jobs = ("group_jobs" in cluster_config
and cluster_config["group_jobs"] and "count" in state)
self.make_env_wrapper(env_filename, cluster_config)
num_submits = 1 if group_jobs else state[
"count"] if "count" in state else 1
for i in range(num_submits):
if self.config.get('StartdLogging', {}).get(
'send_startd_logs', False) is True:
startd_logfile_name = '{}_{}.tar.gz'.format(
self.config['Glidein']['site'], uuid.uuid4())
presigned_put_url = get_presigned_put_url(
startd_logfile_name, self.config, self.secrets)
presigned_get_url = get_presigned_get_url(
startd_logfile_name, self.config, self.secrets)
self.make_submit_file(submit_filename, env_filename, state,
group_jobs, cluster_config,
presigned_put_url, presigned_get_url)
else:
self.make_submit_file(submit_filename, env_filename, state,
group_jobs, cluster_config)
cmd = cluster_config["submit_command"] + " " + submit_filename
subprocess.check_call(cmd, shell=True)
