def init(self, datadir, query_servers): if self.locator: self.query_servers = ServerList(self.locator, "CUSTOM", "QUERY_SERVER") else: self.query_servers = FixedServerList(query_servers, "9091", "/services/query/", "QUERY_SERVER") query_wsdl = "file:" + os.path.join(self.get_datadir(datadir,"query"), "aviary-query.wsdl") self.query_client_pool = ClientPool(query_wsdl, None)
def init(self, datadir, job_servers): if self.locator: self.job_servers = ServerList(self.locator, "SCHEDULER", "JOB") else: self.job_servers = FixedServerList(job_servers, "9090", "/services/job/", "JOB") job_wsdl = "file:" + os.path.join(self.get_datadir(datadir,"job"), "aviary-job.wsdl") self.job_client_pool = ClientPool(job_wsdl, None)
class _AviaryQueryMethods(object): # Do this here rather than __init__ so we don't have to worry about # matching parameter lists in multiple inheritance cases with super def init(self, datadir, query_servers): if self.locator: self.query_servers = ServerList(self.locator, "CUSTOM", "QUERY_SERVER") else: self.query_servers = FixedServerList(query_servers, "9091", "/services/query/", "QUERY_SERVER") query_wsdl = "file:" + os.path.join(self.get_datadir(datadir,"query"), "aviary-query.wsdl") self.query_client_pool = ClientPool(query_wsdl, None) def fetch_job_data(self, job_server, job_id, ftype, file, start, end, scheduler_name, submission, *args, **kwargs): ''' kwargs will be searched for 'default' and 'timeout' arguments. These optional arguments were moved to kwargs for compatibility with another implementation of the same routine. ''' default = "default" in kwargs and kwargs["default"] or None timeout = "timeout" in kwargs and kwargs["timeout"] or 5 # Aviary doesn't use the file name as does QMF, instead it # specifies the file type and lets condor figure out the path. def my_process_results(result): # Fix up the exception message if necessary result = self._pretty_result(result, job_server.Machine) if isinstance(result, Exception): status = result data = None else: status = _AviaryCommon._get_status(result.status) if status == "OK" and hasattr(result, "content"): # Match the format expected by Cumin. This is # the format used by the QMF call... data = {'Data': result.content} else: data = None return (status, data) client = self.query_client_pool.get_object() self._setup_client(client, self.query_servers, # server lookup object job_server.Machine, # host we want "getJobData") # Make a job data parameter (see query wsdl) jobData = client.factory.create('ns0:JobData') jobData.id.job = job_id jobData.id.pool = job_server.Pool jobData.id.scheduler = scheduler_name jobData.id.submission.name = submission.Name jobData.id.submission.owner = submission.Owner # Translate cumin file type to Aviary file type if ftype == "e": jobData.type = "ERR" elif ftype == "o": jobData.type = "OUT" elif ftype == "u": jobData.type = "LOG" else: # We can't translate the type. # Let Aviary throw an error on this instead of us. jobData.type = ftype from_end = start < 0 max_bytes = abs(end - start) res = self._call_sync(my_process_results, self.call_client_retry, client, "getJobData", jobData, max_bytes, from_end) self.query_client_pool.return_object(client) return res; def get_job_ad(self, job_server, job_id, scheduler_name, submission, *args, **kwargs): ''' kwargs will be searched for 'default' and 'timeout' arguments. These optional arguments were moved to kwargs for compatibility with another implementation of the same routine. ''' default = "default" in kwargs and kwargs["default"] or None timeout = "timeout" in kwargs and kwargs["timeout"] or 5 def make_tuple(attr): # Attempt to cast the value into the specified type if attr.type in self.aviary_to_type: try: v = self.aviary_to_type[attr.type](attr.value) except: v = attr.value else: v = attr.value return (attr.name, v) def make_dict(attrs): return dict([make_tuple(attr) for attr in attrs]) def my_process_results(result): # Fix up the exception message if necessary result = self._pretty_result(result, job_server.Machine) if isinstance(result, Exception): status = result data = None else: status = _AviaryCommon._get_status(result[0].status) if status == "OK": # Match the format expected by Cumin. This is # the format used by the QMF call. We have a list # of attributes in attrs that we need to make into # a dictionary ads = make_dict(result[0].details.attrs) data = {'JobAd': ads} else: data = None return (status, data) client = self.query_client_pool.get_object() self._setup_client(client, self.query_servers, # server lookup object job_server.Machine, # host we want "getJobDetails") # Make a job id parameter (see job wsdl) jobId = client.factory.create('ns0:JobID') jobId.job = job_id jobId.pool = job_server.Pool jobId.scheduler = scheduler_name jobId.submission.name = submission.Name jobId.submission.owner = submission.Owner res = self._call_sync(my_process_results, self.call_client_retry, client, "getJobDetails", jobId) self.query_client_pool.return_object(client) return res; def get_job_summaries(self, submission, callback, machine_name): assert callback def to_int_seconds(dt): # Change a datetime.datetime into int seconds since epoch # Note, this works nicely if the datetime happens to include microseconds # since the call to timetuple will drop them. Stuff coming back from # condor should not have microseconds anyway. return int(time.mktime(dt.timetuple())) def get_string(job, attr): # Cast suds text types into str so we have standard Py types # Handles optional strings as well if hasattr(job, attr): return str(getattr(job, attr)) return "" def adapt(jobs): # Make an aviary job summary look like the canonical form # that cumin is expecting (actually the QMF form because of history). result = list() for job in jobs: cluster, proc = job.id.job.split(".") j = dict() j["ClusterId"] = int(cluster) j["Cmd"] = str(job.cmd) j["EnteredCurrentStatus"] = to_int_seconds(job.last_update) # Note, GlobalJobId here will not match the same value from # QMF because the qdate portion of the name is missing j["GlobalJobId"] = job.id.scheduler + \ "#" + job.id.job j["JobStatus"] = str(job.job_status) j["ProcId"] = int(proc) j["QDate"] = to_int_seconds(job.queued) # These may be null... j["Args"] = get_string(job, "args1") j["ReleaseReason"] = get_string(job, "released") j["HoldReason"] = get_string(job, "held") result.append(j) return result def my_callback(result): query_client.set_enable_attributes(False) self.query_client_pool.return_object(query_client) result = self._pretty_result(result, machine_name) if isinstance(result, Exception): callback(result, None) else: status = _AviaryCommon._get_status(result[0].status) if status == "OK" and hasattr(result[0], "jobs"): data = {"Jobs": adapt(result[0].jobs)} else: data = {"Jobs": None} callback(status, data) query_client = self.query_client_pool.get_object() self._setup_client(query_client, self.query_servers, # server lookup object machine_name, # host we want "getSubmissionSummary") # What we really want here is the job summaries from the # submission summary response. To get those, we have to # set an extra attribute on the client... query_client.set_enable_attributes(True) query_client.set_attributes({"includeJobSummaries": "true"}) # Make a submission id. (see query wsdl) subId = query_client.factory.create('ns0:SubmissionID') subId.name = submission.Name subId.owner = submission.Owner t = CallThread(self.call_client_retry, my_callback, query_client, "getSubmissionSummary", subId) t.start()
class _AviaryJobMethods(object): # Do this here rather than __init__ so we don't have to worry about # matching parameter lists in multiple inheritance cases with super def init(self, datadir, job_servers): if self.locator: self.job_servers = ServerList(self.locator, "SCHEDULER", "JOB") else: self.job_servers = FixedServerList(job_servers, "9090", "/services/job/", "JOB") job_wsdl = "file:" + os.path.join(self.get_datadir(datadir,"job"), "aviary-job.wsdl") self.job_client_pool = ClientPool(job_wsdl, None) def set_job_attribute(self, scheduler, job_id, name, value, callback, submission): assert callback def my_callback(result): self.job_client_pool.return_object(job_client) result = self._pretty_result(result, scheduler.Machine) # massage results for use by standard callback cb_args = self._cb_args_dataless(result) callback(*cb_args) job_client = self.job_client_pool.get_object() self._setup_client(job_client, self.job_servers, # server lookup object scheduler.Machine, # host we want "setJobAttribute") # Make a job id parameter (see job wsdl) jobId = job_client.factory.create('ns0:JobID') jobId.job = job_id jobId.pool = scheduler.Pool jobId.scheduler = scheduler.Name jobId.submission.name = submission.Name jobId.submission.owner = submission.Owner # Make attribute parameter from name and value aviary_attr = job_client.factory.create('ns0:Attribute') aviary_attr.name = name aviary_attr.type = "STRING" aviary_attr.value = value t = CallThread(self.call_client_retry, my_callback, job_client, "setJobAttribute", jobId, aviary_attr) t.start() def submit_job(self, scheduler, ad, callback): assert callback def my_callback(result): # Turn this back off before we put it back in the pool # so allow_overrides isn't set for someone else... job_client.set_enable_attributes(False) self.job_client_pool.return_object(job_client) result = self._pretty_result(result, scheduler.Machine) if isinstance(result, Exception): callback(result, None) else: # the aviary response has the job id available, # we'll pass it anyway even though Cumin does not care # at the present time status = _AviaryCommon._get_status(result.status) if status == "OK" and hasattr(result, "id"): id = result.id else: id = None callback(status, id) job_client = self.job_client_pool.get_object() self._setup_client(job_client, self.job_servers, # server lookup object scheduler.Machine, # host we want "submitJob") # Set basic attributes in the order defined by aviary-job.wsdl. args = list() basic_attrs = ("Cmd", "Args", "Owner", "Iwd", "Submission") for attr in basic_attrs: try: args.append(ad[attr]) except: # Someone may be unhappy if this is a required param! # Let the downstream code generate an error pass # Add empty list for Aviary's basic requirement value... args.append([]) # and let's let Requirements remain an unrestricted expression so that # we can just pass through the value from Cumin without interfering. # To do that, we need to specify Requirements through the # "extras" fields and set allowOverrides to True. # (otherwise, Requirements will be limited to particular # resource constraint types defined by aviary) job_client.set_enable_attributes(True) job_client.set_attributes({"allowOverrides": True}) extras = list() for k, v in ad.iteritems(): # We don't need to send descriptors down to aviary # and basic_attrs have already been filled in if k == "!!descriptors" or k in basic_attrs: continue extra = job_client.factory.create('ns0:Attribute') extra.name = k # But we do need to look in descriptors to find expressions... if k in ad["!!descriptors"]: extra.type = "EXPRESSION" else: try: extra.type = self.type_to_aviary[type(v)] except KeyError: extra.type = "UNDEFINED" extra.value = v extras.append(extra) # Important, extras itself must be added as an embedded list or # suds will consider only a single item args.append(extras) t = CallThread(self.call_client_retry, my_callback, job_client, "submitJob", *args) t.start() def control_job(self, cmd, scheduler, job_id, reason, submission, *args, **kwargs): ''' This method is asynchronous iff 'callback' is supplied. Values for cmd are case sensitive (although the first letter may actually be either case) and may be one of the following: 'holdJob', 'releaseJob', 'suspendJob', 'continueJob', 'removeJob' kwargs will be searched for 'callback', 'default' and 'timeout' arguments. ''' # Aviary and QMF command names for job control differ in one respect, # which is that QMF uses an initial capital and Aviary does not. cmd = cmd[0].lower() + cmd[1:] return self._control_job(scheduler, job_id, reason, submission, cmd, *args, **kwargs) def _control_job(self, scheduler, job_id, reason, submission, meth_name, *args, **kwargs): callback = "callback" in kwargs and kwargs["callback"] or None default = "default" in kwargs and kwargs["default"] or None timeout = "timeout" in kwargs and kwargs["timeout"] or 5 client = self.job_client_pool.get_object() self._setup_client(client, self.job_servers, # server lookup object scheduler.Machine, # host we want meth_name) meth = getattr(client.service, meth_name) # Make a job id parameter (see job wsdl) jobId = client.factory.create('ns0:JobID') jobId.job = job_id jobId.pool = scheduler.Pool jobId.scheduler = scheduler.Name jobId.submission.name = submission.Name jobId.submission.owner = submission.Owner if callback: def my_callback(result): self.job_client_pool.return_object(client) # Fix up the exception message if necessary result = self._pretty_result(result, scheduler.Machine) cb_args = self._cb_args_dataless(result) callback(*cb_args) t = CallThread(self.call_client_retry, my_callback, client, meth_name, jobId, reason) t.start() else: def my_process_results(result): # Fix up the exception message if necessary result = self._pretty_result(result, scheduler.Machine) return self._cb_args_dataless(result) res = self._call_sync(my_process_results, self.call_client_retry, client, meth_name, jobId, reason) self.job_client_pool.return_object(client) return res;