def submit(self, state, partition="Cluster"): """ Writing submit file and submitting a job for PBS-like batch managers Args: state: what resource requirements a given glidein has """ submit_filename = 'submit.pbs' if 'filename' in self.config["SubmitFile"]: submit_filename = self.config["SubmitFile"]["filename"] cluster_config = self.config[partition] group_jobs = ("group_jobs" in cluster_config and cluster_config["group_jobs"] and "count" in state) self.write_submit_file(submit_filename, state, group_jobs, cluster_config) num_submits = 1 if group_jobs else state["count"] if "count" in state else 1 for i in xrange(num_submits): if self.config['StartdLogging']['send_startd_logs'] is True: startd_logfile_name = '{}_{}.tar.gz'.format(self.config['Glidein']['site'], uuid.uuid4()) presigned_put_url = get_presigned_put_url(startd_logfile_name, self.config, self.secrets) presigned_get_url = get_presigned_get_url(startd_logfile_name, self.config, self.secrets) self.write_submit_file(submit_filename, state, group_jobs, cluster_config, presigned_put_url, presigned_get_url) cmd = self.config[partition]["submit_command"] + " " + submit_filename if not ('Mode' in self.config and 'dryrun' in self.config['Mode'] and self.config['Mode']['dryrun']): subprocess.check_call(cmd, shell=True)
def submit(self, state, partition="Cluster"): """ Writing submit file and submitting a job for PBS-like batch managers Args: state: what resource requirements a given glidein has """ submit_filename = 'submit.pbs' if 'filename' in self.config["SubmitFile"]: submit_filename = self.config["SubmitFile"]["filename"] cluster_config = self.config[partition] group_jobs = ("group_jobs" in cluster_config and cluster_config["group_jobs"] and "count" in state) self.write_submit_file(submit_filename, state, group_jobs, cluster_config) num_submits = 1 if group_jobs else state[ "count"] if "count" in state else 1 for i in range(num_submits): if self.config['StartdLogging']['send_startd_logs'] is True: startd_logfile_name = '{}_{}.tar.gz'.format( self.config['Glidein']['site'], uuid.uuid4()) presigned_put_url = get_presigned_put_url( startd_logfile_name, self.config, self.secrets) presigned_get_url = get_presigned_get_url( startd_logfile_name, self.config, self.secrets) self.write_submit_file(submit_filename, state, group_jobs, cluster_config, presigned_put_url, presigned_get_url) cmd = self.config[partition][ "submit_command"] + " " + submit_filename if not ('Mode' in self.config and 'dryrun' in self.config['Mode'] and self.config['Mode']['dryrun']): subprocess.check_call(cmd, shell=True)
def submit(self, state, partition="Cluster"): """ Writing submit file and submitting a HTCondor job Args: state: what resource requirements a given glidein has """ submit_filename = 'submit.condor' if 'filename' in self.config["SubmitFile"]: submit_filename = self.config["SubmitFile"]["filename"] env_filename = 'env_wrapper.sh' if 'env_wrapper_name' in self.config['SubmitFile']: env_filename = self.config["SubmitFile"]["env_wrapper_name"] cluster_config = self.config[partition] group_jobs = ("group_jobs" in cluster_config and cluster_config["group_jobs"] and "count" in state) self.make_env_wrapper(env_filename, cluster_config) num_submits = 1 if group_jobs else state["count"] if "count" in state else 1 for i in range(num_submits): if self.config['StartdLogging']['send_startd_logs'] is True: startd_logfile_name = '{}_{}.tar.gz'.format(self.config['Glidein']['site'], uuid.uuid4()) presigned_put_url = get_presigned_put_url(startd_logfile_name, self.config, self.secrets) presigned_get_url = get_presigned_get_url(startd_logfile_name, self.config, self.secrets) self.make_submit_file(submit_filename, env_filename, state, group_jobs, cluster_config, presigned_put_url, presigned_get_url) else: self.make_submit_file(submit_filename, env_filename, state, group_jobs, cluster_config) cmd = cluster_config["submit_command"] + " " + submit_filename print(cmd) subprocess.check_call(cmd, shell=True)
def submit(self, state, partition="Cluster"): """ Writing submit file and submitting a HTCondor job Args: state: what resource requirements a given glidein has """ submit_filename = 'submit.condor' if 'filename' in self.config["SubmitFile"]: submit_filename = self.config["SubmitFile"]["filename"] env_filename = 'env_wrapper.sh' if 'env_wrapper_name' in self.config['SubmitFile']: env_filename = self.config["SubmitFile"]["env_wrapper_name"] cluster_config = self.config[partition] group_jobs = ("group_jobs" in cluster_config and cluster_config["group_jobs"] and "count" in state) self.make_env_wrapper(env_filename, cluster_config) num_submits = 1 if group_jobs else state[ "count"] if "count" in state else 1 for i in range(num_submits): if self.config.get('StartdLogging', {}).get( 'send_startd_logs', False) is True: startd_logfile_name = '{}_{}.tar.gz'.format( self.config['Glidein']['site'], uuid.uuid4()) presigned_put_url = get_presigned_put_url( startd_logfile_name, self.config, self.secrets) presigned_get_url = get_presigned_get_url( startd_logfile_name, self.config, self.secrets) self.make_submit_file(submit_filename, env_filename, state, group_jobs, cluster_config, presigned_put_url, presigned_get_url) else: self.make_submit_file(submit_filename, env_filename, state, group_jobs, cluster_config) cmd = cluster_config["submit_command"] + " " + submit_filename subprocess.check_call(cmd, shell=True)