def getObjects(self, request, context):
try:
object_ids = set()
for oid in request.objectIDS:
object_ids.add(Utils.get_id(oid))
already_obtained_objects = set()
for oid in request.alreadyObtainedObjects:
already_obtained_objects.add(Utils.get_id(oid))
result = self.execution_environment.get_objects(Utils.get_id(request.sessionID),
object_ids, already_obtained_objects,
request.recursive,
Utils.get_id(request.destBackendID),
request.updateReplicaLocs)
obj_list = []
for entry in result:
obj_list.append(Utils.get_obj_with_data_param_or_return(entry))
return dataservice_messages_pb2.GetObjectsResponse(objects=obj_list)
except Exception as ex:
traceback.print_exc()
return dataservice_messages_pb2.GetObjectsResponse(
excInfo=self.get_exception_info(ex)
)
def ds_get_objects(self, session_id, object_ids, recursive, moving):
object_ids_list = []
for oid in object_ids:
object_ids_list.append(Utils.get_msg_options['object'](oid))
request = dataservice_messages_pb2.GetObjectsRequest(
sessionID=Utils.get_msg_options['session'](session_id),
objectIDS=object_ids_list,
recursive=recursive,
moving=moving)
try:
response = self.ds_stub.getObjects(request)
except RuntimeError as e:
raise e
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
serialized_objs = list()
for obj_with_data in response.objects:
serialized_objs.append(
Utils.get_obj_with_data_param_or_return(obj_with_data))
return serialized_objs
def ds_get_federated_objects(self, dc_instance_id, object_ids):
object_ids_list = []
for oid in object_ids:
object_ids_list.append(Utils.get_msg_options['object'](oid))
request = dataservice_messages_pb2.GetFederatedObjectsRequest(
extDataClayID=Utils.get_msg_options['dataclay_instance'](
dc_instance_id),
objectIDS=object_ids_list)
try:
response = self.ds_stub.getFederatedObjects(request)
except RuntimeError as e:
raise e
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
serialized_objs = list()
for obj_with_data in response.objectsList:
serialized_objs.append(
Utils.get_obj_with_data_param_or_return(obj_with_data))
return serialized_objs
def ds_store_objects(self, session_id, objects, moving, ids_with_alias):
obj_list = []
id_with_alias_list = []
for obj in objects:
obj_list.append(Utils.get_obj_with_data_param_or_return(obj))
if ids_with_alias is not None:
for id_with_alias in ids_with_alias:
if id_with_alias is not None:
id_with_alias_list.append(
Utils.get_msg_options['object'](id_with_alias))
request = dataservice_messages_pb2.StoreObjectsRequest(
sessionID=Utils.get_msg_options['session'](session_id),
objects=obj_list,
moving=moving,
idsWithAlias=id_with_alias_list)
try:
response = self.ds_stub.storeObjects(request)
except RuntimeError as e:
traceback.print_exc(file=sys.stdout)
raise e
if response.isException:
raise DataClayException(response.exceptionMessage)
def ds_get_objects(self, session_id, object_ids, already_obtained_obs,
recursive, dest_backend_id, update_replica_locs):
object_ids_list = []
for oid in object_ids:
object_ids_list.append(Utils.get_msg_id(oid))
already_obtained_objects = []
for oid in already_obtained_obs:
already_obtained_objects.append(Utils.get_msg_id(oid))
request = dataservice_messages_pb2.GetObjectsRequest(
sessionID=Utils.get_msg_id(session_id),
objectIDS=object_ids_list,
alreadyObtainedObjects=already_obtained_objects,
recursive=recursive,
destBackendID=Utils.get_msg_id(dest_backend_id),
updateReplicaLocs=update_replica_locs)
try:
response = self.ds_stub.getObjects(request,
metadata=self.metadata_call)
except RuntimeError as e:
raise e
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
serialized_objs = list()
for obj_with_data in response.objects:
serialized_objs.append(
Utils.get_obj_with_data_param_or_return(obj_with_data))
return serialized_objs
def notifyFederation(self, request, context):
try:
logger.verbose("Notify Federation started")
objects_to_persist = []
for vol_param in request.objects:
param = Utils.get_obj_with_data_param_or_return(vol_param)
objects_to_persist.append(param)
session_id = Utils.get_id(request.sessionID)
self.execution_environment.notify_federation(session_id, objects_to_persist)
logger.verbose("Notify Federation finished, sending response")
return common_messages_pb2.ExceptionInfo()
except Exception as ex:
return self.get_exception_info(ex)
def upsertObjects(self, request, context):
try:
session_id = Utils.get_id(request.sessionID)
objects = []
for entry in request.bytesUpdate:
objects.append(Utils.get_obj_with_data_param_or_return(entry))
self.execution_environment.upsert_objects(session_id, objects)
return common_messages_pb2.ExceptionInfo()
except Exception as ex:
return self.get_exception_info(ex)
def makePersistent(self, request, context):
try:
objects_to_persist = []
for vol_param in request.objects:
param = Utils.get_obj_with_data_param_or_return(vol_param)
objects_to_persist.append(param)
session_id = Utils.get_id(request.sessionID)
self.execution_environment.make_persistent(session_id, objects_to_persist)
logger.verbose("MakePersistent finished, sending response")
return common_messages_pb2.ExceptionInfo()
except Exception as ex:
traceback.print_exc()
return self.get_exception_info(ex)
def ds_upsert_objects(self, session_id, object_bytes):
obj_byt_list = []
for entry in object_bytes:
obj_byt_list.append(Utils.get_obj_with_data_param_or_return(entry))
request = dataservice_messages_pb2.UpsertObjectsRequest(
sessionID=Utils.get_msg_options['session'](session_id),
bytesUpdate=obj_byt_list)
try:
response = self.ds_stub.upsertObjects(request)
except RuntimeError as e:
raise e
if response.isException:
raise DataClayException(response.exceptionMessage)
def storeObjects(self, request, context):
try:
objects_list = []
for vol_param in request.objects:
param = Utils.get_obj_with_data_param_or_return(vol_param)
objects_list.append(param)
ids_with_alias_set = set()
session_id = Utils.get_id(request.sessionID)
if request.idsWithAlias is not None and len(request.idsWithAlias) > 0:
for ids_with_alias in request.idsWithAlias:
ids_with_alias_set.add(Utils.get_id(ids_with_alias))
self.execution_environment.store_objects(session_id, objects_list, request.moving, ids_with_alias_set)
return common_messages_pb2.ExceptionInfo()
except Exception as ex:
return self.get_exception_info(ex)
def getFederatedObjects(self, request, context):
try:
object_ids = set()
for oid in request.objectIDS:
object_ids.add(Utils.get_id(oid))
result = self.execution_environment.get_federated_objects(
Utils.get_id(request.extDataClayID), object_ids)
obj_list = []
for entry in result:
obj_list.append(Utils.get_obj_with_data_param_or_return(entry))
return dataservice_messages_pb2.GetFederatedObjectsResponse(
objects=obj_list)
except DataClayException as ex:
return dataservice_messages_pb2.GetFederatedObjectsResponse(
excInfo=self.get_exception_info(ex))
def notify_federation(self, session_id, params):
obj_list = []
for entry in params:
obj_list.append(Utils.get_obj_with_data_param_or_return(entry))
request = dataservice_messages_pb2.NotifyFederationRequest(
sessionID=Utils.get_msg_id(session_id),
objects=obj_list,
)
try:
response = self.ds_stub.notifyFederation(
request, metadata=self.metadata_call)
except RuntimeError as e:
logger.error('Failed to federate', exc_info=True)
raise e
if response.isException:
raise DataClayException(response.exceptionMessage)
def make_persistent(self, session_id, params):
logger.debug("Client performing MakePersistent")
obj_list = []
for entry in params:
obj_list.append(Utils.get_obj_with_data_param_or_return(entry))
request = dataservice_messages_pb2.MakePersistentRequest(
sessionID=Utils.get_msg_id(session_id),
objects=obj_list,
)
try:
response = self.ds_stub.makePersistent(request,
metadata=self.metadata_call)
except RuntimeError as e:
logger.error('Failed to make persistent', exc_info=True)
raise e
if response.isException:
raise DataClayException(response.exceptionMessage)
