def query(self, jobId, queryStructs=list()): ''' Query the object store for objects with the filters specified by attributes @param jobId: The id of the job to which the query is addressed @param queryStructs: A list of queryStructures. ''' objReq = ObjectStore_pb2.ObjectRequest() objReq.job = jobId objReq.type = enumwrap.getValue(objReq, "RequestType", "QUERY") for qS in queryStructs: attributes = qS.getAttributes() negate = qS.getNegate() pbObj = ow.fromObject(attributes) for attr in pbObj.attrs: queryObj = objReq.query.add() val = getattr(attributes, attr.name) queryObj.attr_name = attr.name queryObj.negate = negate if val is None: queryObj.type = enumwrap.getValue( queryObj, "QueryType", "BY_ATTR_NAME") else: queryObj.type = enumwrap.getValue( queryObj, "QueryType", "BY_ATTR_VALUE") queryObj.attr_value.CopyFrom(attr) objResp = self.sendRequest(objReq) return objResp.objects
def objectsPut(self, jobId, taskId, objects, raw=False): ''' Update objects in the object store. @param jobId The id of the job to which the objects belong @param taskId The id of the task to which the objects belong @param objects The list of objects (internal format) which were added @param raw Whether this is supposed to be a raw put (used in imports) @return: List of object ids. ''' logging.debug("Performing ObjectRequest PUT request") if len(objects) == 0: return None objReq = ObjectStore_pb2.ObjectRequest() objReq.job = jobId objReq.task_id = taskId if raw: objReq.type = enumwrap.getValue(objReq, "RequestType", "PUT_RAW") else: objReq.type = enumwrap.getValue(objReq, "RequestType", "PUT") logging.debug("Objects being added:") logging.debug(objects) for obj in ow.fromObjects(objects): objData = objReq.data.add() objData.CopyFrom(obj) logging.debug(objReq) objResp = self.sendRequest(objReq) return objResp.objects
def fromObject(intObject): ''' Process the internal format of the object to the external one. In the current case this changes a Python object into a Protocol Buffers message. @param intObject: An object in internal format. @return: An object in external format. ''' pbObject = Object_pb2.ObjectData() objId = intObject.getObjectId() if objId is not None: pbObject.id = objId for attr_name, value_type in intObject.getTypeStore().iteritems(): attr = pbObject.attrs.add() attr.name = attr_name attr.type = enumwrap.getValue(attr, "Type", value_type) value_name = types.get(value_type) if value_name is not None: if value_type == "BYTES": ref = getattr(intObject, attr_name) (refKey, refStore) = ref.getBoth() attr.data_bytes.key = refKey if refStore is not None: attr.data_bytes.store = refStore else: setattr(attr, value_name, getattr(intObject, attr_name)) return pbObject
def taskError(self, reason='DEFUNCT', desc=""): ''' Send the TaskError message for the current task. @param reason: The reason which is to be reported. Needs to be a name from the enum list. @param desc: String description of the error. ''' te = Process_pb2.TaskError() te.task_id = self.currentTask.task_id te.job = self.currentTask.job te.reason = enumwrap.getValue(te, "ReasonType", reason) te.description = desc logging.warning(reason + " " + desc) self.fwBus.sendCommand("fw", "TaskError", te, False) return True
def objectsUpdate(self, jobId, objects, overwrite=False): ''' Update objects in the object store. @param jobId The id of the job to which the objects belong @param objects The list of objects (internal format) which were modified. @param overwrite Whether to overwrite previously set attributes [default=False] ''' logging.debug("Performing ObjectRequest UPDATE request") if len(objects) == 0: return None objReq = ObjectStore_pb2.ObjectRequest() objReq.job = jobId objReq.type = enumwrap.getValue(objReq, "RequestType", "UPDATE") objReq.overwrite = overwrite logging.debug("Objects being updated:") logging.debug(objects) for obj in ow.fromObjects(objects): objData = objReq.data.add() objData.CopyFrom(obj) logging.debug(objReq) self.sendRequest(objReq)
def objectsGet(self, jobId, objects): ''' Retrieve the objects by their ids. @param jobId The id of the job to which the objects belong @param objects A list of objects Ids to fetch. List as in []. @return: list of retrieved objects in the internal format. ''' logging.debug("Performing ObjectRequest GET request") if len(objects) == 0: logging.debug("No objects passed to objectsGet") return None objReq = ObjectStore_pb2.ObjectRequest() objReq.job = jobId objReq.type = enumwrap.getValue(objReq, "RequestType", "GET") for obj in objects: objReq.objects.append(obj) logging.info( "requesting objects " + str(objects) + " from " + str(jobId)) objResp = self.sendRequest(objReq) self.missing = objResp.missing currentObjects = ow.toObjects(objResp.data) return currentObjects
def sendRequest(self, objReq): ''' Sends a prepared ObjectRequest to the object store. @param objReq: A previously prepared ObjectRequest. @return: ObjectResponse ''' waiting = True tries = 1 objResp = None while waiting and self.keepRunning: try: (mtype, reponse) = self.bus.sendCommand( "os", "ObjectRequest", objReq, sync=1, timeout=self.timeout) if mtype != "ObjectResponse": raise BadMessageException("ObjectResponse", mtype) # TODO: should implement appropriate mechanisms for various # responses. objResp = ObjectStore_pb2.ObjectResponse() objResp.ParseFromString(reponse) if enumwrap.getValue(objResp, "ResponseType", objResp.type) == "FAILURE": logging.error("Failed ObjectRequest: " + str(objResp)) break # logging.debug(objResp) waiting = False except BusTimeoutException: if tries >= self.maxTries: raise ObjectStoreException( "Object store not responding. Tried %d times." % tries) else: tries = tries + 1 logging.info( "ObjectRequest reply not received yet. Resending request") pass if objResp is None: raise ShutdownException("Termination of service while requesting objects.") return objResp