def ds_move_objects(self, session_id, object_id, dest_st_location,
recursive):
request = dataservice_messages_pb2.MoveObjectsRequest(
sessionID=Utils.get_msg_id(session_id),
objectID=Utils.get_msg_id(object_id),
destLocID=Utils.get_msg_id(dest_st_location),
recursive=recursive)
try:
response = self.ds_stub.moveObjects(request,
metadata=self.metadata_call)
except RuntimeError as e:
raise e
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
result = set()
for oid in response.movedObjects:
result.add(Utils.get_id(oid))
return result
def ds_get_objects(self, session_id, object_ids, already_obtained_obs,
recursive, dest_backend_id, update_replica_locs):
object_ids_list = []
for oid in object_ids:
object_ids_list.append(Utils.get_msg_id(oid))
already_obtained_objects = []
for oid in already_obtained_obs:
already_obtained_objects.append(Utils.get_msg_id(oid))
request = dataservice_messages_pb2.GetObjectsRequest(
sessionID=Utils.get_msg_id(session_id),
objectIDS=object_ids_list,
alreadyObtainedObjects=already_obtained_objects,
recursive=recursive,
destBackendID=Utils.get_msg_id(dest_backend_id),
updateReplicaLocs=update_replica_locs)
try:
response = self.ds_stub.getObjects(request,
metadata=self.metadata_call)
except RuntimeError as e:
raise e
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
serialized_objs = list()
for obj_with_data in response.objects:
serialized_objs.append(
Utils.get_obj_with_data_param_or_return(obj_with_data))
return serialized_objs
def removeObjects(self, request, context):
try:
object_ids = set()
for oid in request.getObjectsIDSList():
object_ids.add(Utils.get_id(oid))
result = self.client.ds_remove_objects(Utils.get_id(request.sessionID),
object_ids,
request.recursive,
request.moving,
Utils.get_id(request.newHint))
rem_obj = dict()
for k, v in result.items():
rem_obj[str(k)] = str(v)
return dataservice_messages_pb2.RemoveObjectsResponse(
removedObjects=rem_obj
)
except Exception as ex:
return dataservice_messages_pb2.RemoveObjectsResponse(
excInfo=self.get_exception_info(ex)
)
def ds_execute_implementation(self, object_id, implementation_id,
session_id, params):
logger.debug("Client performing ExecuteImplementation")
request = dataservice_messages_pb2.ExecuteImplementationRequest(
sessionID=Utils.get_msg_options['session'](session_id),
implementationID=Utils.get_msg_options['implem'](
implementation_id),
params=Utils.get_param_or_return(params),
objectID=Utils.get_msg_options['object'](object_id))
try:
response = self.ds_stub.executeImplementation(request)
except RuntimeError as e:
logger.error('Failed to execute implementation', exc_info=True)
raise e
if response.excInfo.isException:
try:
exception = pickle.loads(response.excInfo.serializedException)
except:
raise DataClayException(response.excInfo.exceptionMessage)
else:
raise exception
if response.ret is not None:
return Utils.get_param_or_return(response.ret)
else:
return None
def ds_store_objects(self, session_id, objects, moving, ids_with_alias):
obj_list = []
id_with_alias_list = []
for obj in objects:
obj_list.append(Utils.get_obj_with_data_param_or_return(obj))
if ids_with_alias is not None:
for id_with_alias in ids_with_alias:
if id_with_alias is not None:
id_with_alias_list.append(Utils.get_msg_id(id_with_alias))
request = dataservice_messages_pb2.StoreObjectsRequest(
sessionID=Utils.get_msg_id(session_id),
objects=obj_list,
moving=moving,
idsWithAlias=id_with_alias_list)
try:
response = self.ds_stub.storeObjects(request,
metadata=self.metadata_call)
except RuntimeError as e:
traceback.print_exc(file=sys.stdout)
raise e
if response.isException:
raise DataClayException(response.exceptionMessage)
def set_dataset_id_from_garbage_collector(self, object_id, dataset_id):
request = logicmodule_messages_pb2.SetDataSetIDFromGarbageCollectorRequest(
objectID=Utils.get_msg_id(object_id),
datasetID=Utils.get_msg_id(dataset_id))
# ToDo: In Java at this point override the onNext/onError/onCompleted methods of responseObserver
try:
logger.trace(
"Asynchronous call to register object from Garbage Collector for object %s",
object_id)
# ToDo: check async
six.advance_iterator(async_req_send)
resp_future = self.lm_stub.setDataSetIDFromGarbageCollector.future.future(
request=request, metadata=self.metadata_call)
resp_future.result()
if resp_future.done():
six.advance_iterator(async_req_rec)
except RuntimeError as e:
raise e
if resp_future.isException:
raise DataClayException(resp_future.exceptionMessage)
def getObjects(self, request, context):
try:
object_ids = set()
for oid in request.objectIDS:
object_ids.add(Utils.get_id(oid))
already_obtained_objects = set()
for oid in request.alreadyObtainedObjects:
already_obtained_objects.add(Utils.get_id(oid))
result = self.execution_environment.get_objects(Utils.get_id(request.sessionID),
object_ids, already_obtained_objects,
request.recursive,
Utils.get_id(request.destBackendID),
request.updateReplicaLocs)
obj_list = []
for entry in result:
obj_list.append(Utils.get_obj_with_data_param_or_return(entry))
return dataservice_messages_pb2.GetObjectsResponse(objects=obj_list)
except Exception as ex:
traceback.print_exc()
return dataservice_messages_pb2.GetObjectsResponse(
excInfo=self.get_exception_info(ex)
)
def ds_new_persistent_instance(self, session_id, class_id,
implementation_id, i_face_bitmaps, params):
logger.debug(
"Ready to call to a DS to build a new persistent instance for class {%s}",
class_id)
temp_iface_b = dict()
temp_param = None
if i_face_bitmaps is not None:
for k, v in i_face_bitmaps.items():
temp_iface_b[k] = v
if params is not None:
temp_param = Utils.get_param_or_return(params)
request = dataservice_messages_pb2.NewPersistentInstanceRequest(
sessionID=Utils.get_msg_options['session'](session_id),
classID=Utils.get_msg_options['meta_class'](class_id),
implementationID=Utils.get_msg_options['implem'](
implementation_id),
ifaceBitMaps=temp_iface_b,
params=temp_param)
try:
response = self.ds_stub.newPersistentInstance(request)
except RuntimeError as e:
raise e
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
return Utils.get_id(response.objectID)
def ds_new_version(self, session_id, object_id, metadata_info):
logger.info("This new_version is called somewhere?")
request = dataservice_messages_pb2.NewVersionRequest(
sessionID=Utils.get_msg_options['session'](session_id),
objectID=Utils.get_msg_options['object'](object_id),
metadataInfo=dataclay_yaml_dump(metadata_info))
try:
response = self.ds_stub.newVersion(request)
except RuntimeError as e:
raise e
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
result = dict()
oid = Utils.get_id(response.objectID)
for k, v in response.versionedIDs:
result[Utils.get_id_from_uuid(k)] = Utils.get_id_from_uuid(v)
t = (oid, result)
return t
def ds_remove_objects(self, session_id, object_ids, recursive, moving,
new_hint):
obj_ids_list = []
for oid in object_ids:
obj_ids_list.append(Utils.get_msg_options['object'](oid))
request = dataservice_messages_pb2.RemoveObjectsRequest(
sessionID=Utils.get_msg_options['session'](session_id),
objectIDs=obj_ids_list,
recursive=recursive,
moving=moving,
newHint=Utils.get_msg_options['exec_env'](new_hint))
try:
response = self.ds_stub.removeObjects(request)
except RuntimeError as e:
raise e
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
result = dict()
for k, v in response.removedObjects.items():
result[Utils.get_id_from_uuid(k)] = Utils.get_id_from_uuid(v)
return result
def new_class(self, account_id, credential, language, new_classes):
new_cl = {}
for klass in new_classes:
yaml_str = dataclay_yaml_dump(klass)
new_cl[klass.name] = yaml_str
request = logicmodule_messages_pb2.NewClassRequest(
accountID=Utils.get_msg_id(account_id),
credential=Utils.get_credential(credential),
language=language,
newClasses=new_cl)
lm_function = lambda request: self.lm_stub.newClass.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
result = dict()
for k, v in response.newClasses.items():
result[k] = dataclay_yaml_load(v)
return result
def new_session(self, account_id, credential, contracts, data_sets,
data_set_for_store, new_session_lang):
contracts_list = []
for con_id in contracts:
contracts_list.append(Utils.get_msg_id(con_id))
data_set_list = []
for data_set in data_sets:
data_set_list.append(Utils.get_msg_id(data_set))
request = logicmodule_messages_pb2.NewSessionRequest(
accountID=Utils.get_msg_id(account_id),
credential=Utils.get_credential(credential),
contractIDs=contracts_list,
dataSetIDs=data_set_list,
storeDataSet=Utils.get_msg_id(data_set_for_store),
sessionLang=new_session_lang)
lm_function = lambda request: self.lm_stub.newSession.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
return dataclay_yaml_load(response.sessionInfo)
def get_stubs(self, applicant_account_id, applicant_credential, language,
contracts_ids):
cid_list = []
for cID in contracts_ids:
cid_list.append(Utils.get_msg_id(cID))
request = logicmodule_messages_pb2.GetStubsRequest(
applicantAccountID=Utils.get_msg_id(applicant_account_id),
credentials=Utils.get_credential(applicant_credential),
language=language,
contractIDs=cid_list)
lm_function = lambda request: self.lm_stub.getStubs.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
result = dict()
for k, v in response.stubs.items():
result[k] = v
return result
def detachObjectFromSession(self, request, context):
try:
self.execution_environment.detach_object_from_session(Utils.get_id(request.objectID),
Utils.get_id(request.sessionID))
return common_messages_pb2.ExceptionInfo()
except Exception as ex:
return self.get_exception_info(ex)
def deleteAlias(self, request, context):
try:
self.execution_environment.delete_alias(Utils.get_id(request.sessionID),
Utils.get_id(request.objectID))
return common_messages_pb2.ExceptionInfo()
except Exception as ex:
return self.get_exception_info(ex)
def set_object_read_write(self, session_id, object_id):
request = logicmodule_messages_pb2.SetObjectReadWriteRequest(
sessionID=Utils.get_msg_id(session_id),
objectID=Utils.get_msg_id(object_id))
lm_function = lambda request: self.lm_stub.setObjectReadWrite.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.isException:
raise DataClayException(response.exceptionMessage)
def get_external_dataclay_info(self, ext_dataclay_id):
request = logicmodule_messages_pb2.GetExtDataClayInfoRequest(
extDataClayID=Utils.get_msg_id(ext_dataclay_id))
lm_function = lambda request: self.lm_stub.getExternalDataClayInfo.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
return Utils.get_dataclay_instance(response.extDataClayInfo)
def consolidateVersion(self, request, context):
try:
self.execution_environment.consolidate_version(Utils.get_id(request.sessionID),
Utils.get_id(request.versionObjectID))
return common_messages_pb2.ExceptionInfo()
except Exception as ex:
return self.get_exception_info(ex)
def remove_class(self, account_id, credential, namespace_id, class_name):
request = logicmodule_messages_pb2.RemoveClassRequest(
accountID=Utils.get_msg_id(account_id),
credential=Utils.get_credential(credential),
className=class_name)
lm_function = lambda request: self.lm_stub.removeClass.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.isException:
raise DataClayException(response.exceptionMessage)
def delete_alias(self, session_id, object_id):
request = dataservice_messages_pb2.DeleteAliasRequest(
sessionID=Utils.get_msg_id(session_id),
objectID=Utils.get_msg_id(object_id))
try:
response = self.ds_stub.deleteAlias(request,
metadata=self.metadata_call)
except RuntimeError as e:
raise e
if response.isException:
raise DataClayException(response.exceptionMessage)
def detach_object_from_session(self, object_id, session_id):
request = dataservice_messages_pb2.DetachObjectFromSessionRequest(
objectID=Utils.get_msg_id(object_id),
sessionID=Utils.get_msg_id(session_id))
try:
response = self.ds_stub.detachObjectFromSession(
request, metadata=self.metadata_call)
except RuntimeError as e:
raise e
if response.isException:
raise DataClayException(response.exceptionMessage)
def delete_alias(self, session_id, alias):
request = logicmodule_messages_pb2.DeleteAliasRequest(
sessionID=Utils.get_msg_id(session_id), alias=alias)
lm_function = lambda request: self.lm_stub.deleteAlias.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
return Utils.get_id(response.objectID)
def federate(self, request, context):
try:
logger.verbose("Federation started")
objects_to_persist = Utils.get_param_or_return(request.params)
session_id = Utils.get_id(request.sessionID)
self.execution_environment.federate(session_id, objects_to_persist)
logger.verbose("Federation finished, sending response")
return common_messages_pb2.ExceptionInfo()
except Exception as ex:
return self.get_exception_info(ex)
def unfederate(self, request, context):
try:
logger.verbose("Unfederation started")
self.execution_environment.unfederate(Utils.get_id(request.sessionID),
Utils.get_id(request.objectID),
Utils.get_id(request.externalExecutionEnvironmentID),
request.recursive)
logger.verbose("Unfederation finished, sending response")
return common_messages_pb2.ExceptionInfo()
except Exception as ex:
return self.get_exception_info(ex)
def get_storage_location_info(self, st_loc_id, from_backend=False):
request = logicmodule_messages_pb2.GetStorageLocationInfoRequest(
storageLocationID=Utils.get_msg_id(st_loc_id),
fromBackend=from_backend)
lm_function = lambda request: self.lm_stub.getStorageLocationInfo.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
return Utils.get_storage_location(response.storageLocationInfo)
def get_external_executionenvironment_info(self, backend_id):
request = logicmodule_messages_pb2.GetExternalExecutionEnvironmentInfoRequest(
execEnvID=Utils.get_msg_id(backend_id))
lm_function = lambda request: self.lm_stub.getExternalExecutionEnvironmentInfo.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
return Utils.get_external_execution_environment(
response.executionEnvironmentInfo)
def unfederate_object(self, session_id, object_id, ext_dataclay_id,
recursive):
request = logicmodule_messages_pb2.UnfederateObjectRequest(
sessionID=Utils.get_msg_id(session_id),
objectID=Utils.get_msg_id(object_id),
extDataClayID=Utils.get_msg_id(ext_dataclay_id),
recursive=recursive)
lm_function = lambda request: self.lm_stub.unfederateObject.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.isException:
raise DataClayException(response.exceptionMessage)
def get_object_info(self, session_id, object_id):
request = logicmodule_messages_pb2.GetObjectInfoRequest(
sessionID=Utils.get_msg_id(session_id),
objectID=Utils.get_msg_id(object_id))
lm_function = lambda request: self.lm_stub.getObjectInfo.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
return response.classname, response.namespace
def synchronize(self, request, context):
try:
object_id = Utils.get_id(request.objectID)
implementation_id = Utils.get_id(request.implementationID)
serialized_params = Utils.get_param_or_return(request.params)
session_id = Utils.get_id(request.sessionID)
calling_backend_id = Utils.get_id(request.callingBackendID)
self.execution_environment.synchronize(session_id, object_id, implementation_id,
serialized_params, calling_backend_id)
return common_messages_pb2.ExceptionInfo()
except DataClayException as ex:
return self.get_exception_info(ex)
def register_event_listener_implementation(self, account_id, credential,
new_event_listener):
request = logicmodule_messages_pb2.RegisterECARequest(
applicantAccountID=Utils.get_msg_id(account_id),
credentials=Utils.get_credential(credential),
eca=dataclay_yaml_dump(new_event_listener))
lm_function = lambda request: self.lm_stub.registerECA.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
