def query(self, jobId, queryStructs=list()):
'''
Query the object store for objects with the filters specified by attributes
@param jobId: The id of the job to which the query is addressed
@param queryStructs: A list of queryStructures.
'''
objReq = ObjectStore_pb2.ObjectRequest()
objReq.job = jobId
objReq.type = enumwrap.getValue(objReq, "RequestType", "QUERY")
for qS in queryStructs:
attributes = qS.getAttributes()
negate = qS.getNegate()
pbObj = ow.fromObject(attributes)
for attr in pbObj.attrs:
queryObj = objReq.query.add()
val = getattr(attributes, attr.name)
queryObj.attr_name = attr.name
queryObj.negate = negate
if val is None:
queryObj.type = enumwrap.getValue(
queryObj, "QueryType", "BY_ATTR_NAME")
else:
queryObj.type = enumwrap.getValue(
queryObj, "QueryType", "BY_ATTR_VALUE")
queryObj.attr_value.CopyFrom(attr)
objResp = self.sendRequest(objReq)
return objResp.objects
def objectsPut(self, jobId, taskId, objects, raw=False):
'''
Update objects in the object store.
@param jobId The id of the job to which the objects belong
@param taskId The id of the task to which the objects belong
@param objects The list of objects (internal format) which were added
@param raw Whether this is supposed to be a raw put (used in imports)
@return: List of object ids.
'''
logging.debug("Performing ObjectRequest PUT request")
if len(objects) == 0:
return None
objReq = ObjectStore_pb2.ObjectRequest()
objReq.job = jobId
objReq.task_id = taskId
if raw:
objReq.type = enumwrap.getValue(objReq, "RequestType", "PUT_RAW")
else:
objReq.type = enumwrap.getValue(objReq, "RequestType", "PUT")
logging.debug("Objects being added:")
logging.debug(objects)
for obj in ow.fromObjects(objects):
objData = objReq.data.add()
objData.CopyFrom(obj)
logging.debug(objReq)
objResp = self.sendRequest(objReq)
return objResp.objects
def fromObject(intObject):
'''
Process the internal format of the object to the external one.
In the current case this changes a Python object into a Protocol Buffers message.
@param intObject: An object in internal format.
@return: An object in external format.
'''
pbObject = Object_pb2.ObjectData()
objId = intObject.getObjectId()
if objId is not None:
pbObject.id = objId
for attr_name, value_type in intObject.getTypeStore().iteritems():
attr = pbObject.attrs.add()
attr.name = attr_name
attr.type = enumwrap.getValue(attr, "Type", value_type)
value_name = types.get(value_type)
if value_name is not None:
if value_type == "BYTES":
ref = getattr(intObject, attr_name)
(refKey, refStore) = ref.getBoth()
attr.data_bytes.key = refKey
if refStore is not None:
attr.data_bytes.store = refStore
else:
setattr(attr, value_name, getattr(intObject, attr_name))
return pbObject
def taskError(self, reason='DEFUNCT', desc=""):
'''
Send the TaskError message for the current task.
@param reason: The reason which is to be reported. Needs to be a name from the enum list.
@param desc: String description of the error.
'''
te = Process_pb2.TaskError()
te.task_id = self.currentTask.task_id
te.job = self.currentTask.job
te.reason = enumwrap.getValue(te, "ReasonType", reason)
te.description = desc
logging.warning(reason + " " + desc)
self.fwBus.sendCommand("fw", "TaskError", te, False)
return True
def objectsUpdate(self, jobId, objects, overwrite=False):
'''
Update objects in the object store.
@param jobId The id of the job to which the objects belong
@param objects The list of objects (internal format) which were modified.
@param overwrite Whether to overwrite previously set attributes [default=False]
'''
logging.debug("Performing ObjectRequest UPDATE request")
if len(objects) == 0:
return None
objReq = ObjectStore_pb2.ObjectRequest()
objReq.job = jobId
objReq.type = enumwrap.getValue(objReq, "RequestType", "UPDATE")
objReq.overwrite = overwrite
logging.debug("Objects being updated:")
logging.debug(objects)
for obj in ow.fromObjects(objects):
objData = objReq.data.add()
objData.CopyFrom(obj)
logging.debug(objReq)
self.sendRequest(objReq)
def objectsGet(self, jobId, objects):
'''
Retrieve the objects by their ids.
@param jobId The id of the job to which the objects belong
@param objects A list of objects Ids to fetch. List as in [].
@return: list of retrieved objects in the internal format.
'''
logging.debug("Performing ObjectRequest GET request")
if len(objects) == 0:
logging.debug("No objects passed to objectsGet")
return None
objReq = ObjectStore_pb2.ObjectRequest()
objReq.job = jobId
objReq.type = enumwrap.getValue(objReq, "RequestType", "GET")
for obj in objects:
objReq.objects.append(obj)
logging.info(
"requesting objects " + str(objects) + " from " + str(jobId))
objResp = self.sendRequest(objReq)
self.missing = objResp.missing
currentObjects = ow.toObjects(objResp.data)
return currentObjects
def sendRequest(self, objReq):
'''
Sends a prepared ObjectRequest to the object store.
@param objReq: A previously prepared ObjectRequest.
@return: ObjectResponse
'''
waiting = True
tries = 1
objResp = None
while waiting and self.keepRunning:
try:
(mtype, reponse) = self.bus.sendCommand(
"os", "ObjectRequest", objReq, sync=1, timeout=self.timeout)
if mtype != "ObjectResponse":
raise BadMessageException("ObjectResponse", mtype)
# TODO: should implement appropriate mechanisms for various
# responses.
objResp = ObjectStore_pb2.ObjectResponse()
objResp.ParseFromString(reponse)
if enumwrap.getValue(objResp, "ResponseType", objResp.type) == "FAILURE":
logging.error("Failed ObjectRequest: " + str(objResp))
break
# logging.debug(objResp)
waiting = False
except BusTimeoutException:
if tries >= self.maxTries:
raise ObjectStoreException(
"Object store not responding. Tried %d times." % tries)
else:
tries = tries + 1
logging.info(
"ObjectRequest reply not received yet. Resending request")
pass
if objResp is None:
raise ShutdownException("Termination of service while requesting objects.")
return objResp