def ds_remove_objects(self, session_id, object_ids, recursive, moving,
new_hint):
obj_ids_list = []
for oid in object_ids:
obj_ids_list.append(Utils.get_msg_id(oid))
request = dataservice_messages_pb2.RemoveObjectsRequest(
sessionID=Utils.get_msg_id(session_id),
objectIDs=obj_ids_list,
recursive=recursive,
moving=moving,
newHint=Utils.get_msg_id(new_hint))
try:
response = self.ds_stub.removeObjects(request,
metadata=self.metadata_call)
except RuntimeError as e:
raise e
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
result = dict()
for k, v in response.removedObjects.items():
result[Utils.get_idd(k)] = Utils.get_id(v)
return result
def register_objects(self, reg_infos, backend_id, lang):
reg_infos_msg = list()
for reg_info in reg_infos:
msg_reg_info = CommonMessages.RegistrationInfo(
objectID=Utils.get_msg_id(reg_info.object_id),
classID=Utils.get_msg_id(reg_info.class_id),
sessionID=Utils.get_msg_id(reg_info.store_session_id),
dataSetID=Utils.get_msg_id(reg_info.dataset_id),
alias=reg_info.alias)
reg_infos_msg.append(msg_reg_info)
request = logicmodule_messages_pb2.RegisterObjectsRequest(
regInfos=reg_infos_msg,
backendID=Utils.get_msg_id(backend_id),
lang=lang)
lm_function = lambda request: self.lm_stub.registerObjects.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
result = list()
for oid in response.objectIDs:
result.append(Utils.get_id(oid))
return result
def ds_get_objects(self, session_id, object_ids, already_obtained_obs,
recursive, dest_backend_id, update_replica_locs):
object_ids_list = []
for oid in object_ids:
object_ids_list.append(Utils.get_msg_id(oid))
already_obtained_objects = []
for oid in already_obtained_obs:
already_obtained_objects.append(Utils.get_msg_id(oid))
request = dataservice_messages_pb2.GetObjectsRequest(
sessionID=Utils.get_msg_id(session_id),
objectIDS=object_ids_list,
alreadyObtainedObjects=already_obtained_objects,
recursive=recursive,
destBackendID=Utils.get_msg_id(dest_backend_id),
updateReplicaLocs=update_replica_locs)
try:
response = self.ds_stub.getObjects(request,
metadata=self.metadata_call)
except RuntimeError as e:
raise e
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
serialized_objs = list()
for obj_with_data in response.objects:
serialized_objs.append(
Utils.get_obj_with_data_param_or_return(obj_with_data))
return serialized_objs
def ds_move_objects(self, session_id, object_id, dest_st_location,
recursive):
request = dataservice_messages_pb2.MoveObjectsRequest(
sessionID=Utils.get_msg_id(session_id),
objectID=Utils.get_msg_id(object_id),
destLocID=Utils.get_msg_id(dest_st_location),
recursive=recursive)
try:
response = self.ds_stub.moveObjects(request,
metadata=self.metadata_call)
except RuntimeError as e:
raise e
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
result = set()
for oid in response.movedObjects:
result.add(Utils.get_id(oid))
return result
def set_dataset_id_from_garbage_collector(self, object_id, dataset_id):
request = logicmodule_messages_pb2.SetDataSetIDFromGarbageCollectorRequest(
objectID=Utils.get_msg_id(object_id),
datasetID=Utils.get_msg_id(dataset_id))
# ToDo: In Java at this point override the onNext/onError/onCompleted methods of responseObserver
try:
logger.trace(
"Asynchronous call to register object from Garbage Collector for object %s",
object_id)
# ToDo: check async
six.advance_iterator(async_req_send)
resp_future = self.lm_stub.setDataSetIDFromGarbageCollector.future.future(
request=request, metadata=self.metadata_call)
resp_future.result()
if resp_future.done():
six.advance_iterator(async_req_rec)
except RuntimeError as e:
raise e
if resp_future.isException:
raise DataClayException(resp_future.exceptionMessage)
def ds_execute_implementation(self, object_id, implementation_id,
session_id, params):
logger.debug("Client performing ExecuteImplementation")
request = dataservice_messages_pb2.ExecuteImplementationRequest(
sessionID=Utils.get_msg_id(session_id),
implementationID=Utils.get_msg_id(implementation_id),
params=Utils.get_param_or_return(params),
objectID=Utils.get_msg_id(object_id))
try:
response = self.ds_stub.executeImplementation(
request, metadata=self.metadata_call)
except RuntimeError as e:
logger.error('Failed to execute implementation', exc_info=True)
raise e
if response.excInfo.isException:
try:
exception = pickle.loads(response.excInfo.serializedException)
except:
raise DataClayException(response.excInfo.exceptionMessage)
else:
raise exception
if response.ret is not None:
return Utils.get_param_or_return(response.ret)
else:
return None
def get_stubs(self, applicant_account_id, applicant_credential, language,
contracts_ids):
cid_list = []
for cID in contracts_ids:
cid_list.append(Utils.get_msg_id(cID))
request = logicmodule_messages_pb2.GetStubsRequest(
applicantAccountID=Utils.get_msg_id(applicant_account_id),
credentials=Utils.get_credential(applicant_credential),
language=language,
contractIDs=cid_list)
lm_function = lambda request: self.lm_stub.getStubs.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
result = dict()
for k, v in response.stubs.items():
result[k] = v
return result
def ds_store_objects(self, session_id, objects, moving, ids_with_alias):
obj_list = []
id_with_alias_list = []
for obj in objects:
obj_list.append(Utils.get_obj_with_data_param_or_return(obj))
if ids_with_alias is not None:
for id_with_alias in ids_with_alias:
if id_with_alias is not None:
id_with_alias_list.append(Utils.get_msg_id(id_with_alias))
request = dataservice_messages_pb2.StoreObjectsRequest(
sessionID=Utils.get_msg_id(session_id),
objects=obj_list,
moving=moving,
idsWithAlias=id_with_alias_list)
try:
response = self.ds_stub.storeObjects(request,
metadata=self.metadata_call)
except RuntimeError as e:
traceback.print_exc(file=sys.stdout)
raise e
if response.isException:
raise DataClayException(response.exceptionMessage)
def new_session(self, account_id, credential, contracts, data_sets,
data_set_for_store, new_session_lang):
contracts_list = []
for con_id in contracts:
contracts_list.append(Utils.get_msg_id(con_id))
data_set_list = []
for data_set in data_sets:
data_set_list.append(Utils.get_msg_id(data_set))
request = logicmodule_messages_pb2.NewSessionRequest(
accountID=Utils.get_msg_id(account_id),
credential=Utils.get_credential(credential),
contractIDs=contracts_list,
dataSetIDs=data_set_list,
storeDataSet=Utils.get_msg_id(data_set_for_store),
sessionLang=new_session_lang)
lm_function = lambda request: self.lm_stub.newSession.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
return dataclay_yaml_load(response.sessionInfo)
def ds_new_persistent_instance(self, session_id, class_id,
implementation_id, i_face_bitmaps, params):
logger.debug(
"Ready to call to a DS to build a new persistent instance for class {%s}",
class_id)
temp_iface_b = dict()
temp_param = None
if i_face_bitmaps is not None:
for k, v in i_face_bitmaps.items():
temp_iface_b[k] = v
if params is not None:
temp_param = Utils.get_param_or_return(params)
request = dataservice_messages_pb2.NewPersistentInstanceRequest(
sessionID=Utils.get_msg_id(session_id),
classID=Utils.get_msg_id(class_id),
implementationID=Utils.get_msg_id(implementation_id),
ifaceBitMaps=temp_iface_b,
params=temp_param)
try:
response = self.ds_stub.newPersistentInstance(
request, metadata=self.metadata_call)
except RuntimeError as e:
raise e
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
return Utils.get_id(response.objectID)
def set_object_read_write(self, session_id, object_id):
request = logicmodule_messages_pb2.SetObjectReadWriteRequest(
sessionID=Utils.get_msg_id(session_id),
objectID=Utils.get_msg_id(object_id))
lm_function = lambda request: self.lm_stub.setObjectReadWrite.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.isException:
raise DataClayException(response.exceptionMessage)
def detach_object_from_session(self, object_id, session_id):
request = dataservice_messages_pb2.DetachObjectFromSessionRequest(
objectID=Utils.get_msg_id(object_id),
sessionID=Utils.get_msg_id(session_id))
try:
response = self.ds_stub.detachObjectFromSession(
request, metadata=self.metadata_call)
except RuntimeError as e:
raise e
if response.isException:
raise DataClayException(response.exceptionMessage)
def delete_alias(self, session_id, object_id):
request = dataservice_messages_pb2.DeleteAliasRequest(
sessionID=Utils.get_msg_id(session_id),
objectID=Utils.get_msg_id(object_id))
try:
response = self.ds_stub.deleteAlias(request,
metadata=self.metadata_call)
except RuntimeError as e:
raise e
if response.isException:
raise DataClayException(response.exceptionMessage)
def unfederate_object(self, session_id, object_id, ext_dataclay_id,
recursive):
request = logicmodule_messages_pb2.UnfederateObjectRequest(
sessionID=Utils.get_msg_id(session_id),
objectID=Utils.get_msg_id(object_id),
extDataClayID=Utils.get_msg_id(ext_dataclay_id),
recursive=recursive)
lm_function = lambda request: self.lm_stub.unfederateObject.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.isException:
raise DataClayException(response.exceptionMessage)
def get_object_info(self, session_id, object_id):
request = logicmodule_messages_pb2.GetObjectInfoRequest(
sessionID=Utils.get_msg_id(session_id),
objectID=Utils.get_msg_id(object_id))
lm_function = lambda request: self.lm_stub.getObjectInfo.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
return response.classname, response.namespace
def get_metadata_by_oid(self, session_id, object_id):
request = logicmodule_messages_pb2.GetMetadataByOIDRequest(
sessionID=Utils.get_msg_id(session_id),
objectID=Utils.get_msg_id(object_id))
lm_function = lambda request: self.lm_stub.getMetadataByOID.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
logger.debug("Obtained metadata info %s" % str(response))
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
return Utils.get_metadata_info(response.mdInfo)
def consolidate_version(self, session_id, version_object_id):
request = dataservice_messages_pb2.ConsolidateVersionRequest(
sessionID=Utils.get_msg_id(session_id),
versionObjectID=Utils.get_msg_id(version_object_id),
)
try:
response = self.ds_stub.consolidateVersion(
request, metadata=self.metadata_call)
except RuntimeError as e:
raise e
if response.isException:
raise DataClayException(response.exceptionMessage)
def ds_update_object(self, session_id, into_object_id, from_object):
request = dataservice_messages_pb2.UpdateObjectRequest(
sessionID=Utils.get_msg_id(session_id),
intoObjectID=Utils.get_msg_id(into_object_id),
fromObject=Utils.get_param_or_return(from_object))
try:
response = self.ds_stub.updateObject(request,
metadata=self.metadata_call)
except RuntimeError as e:
raise e
if response.isException:
raise DataClayException(response.exceptionMessage)
def new_version(self, session_id, object_id, dest_backend_id):
request = dataservice_messages_pb2.NewVersionRequest(
sessionID=Utils.get_msg_id(session_id),
objectID=Utils.get_msg_id(object_id),
destBackendID=Utils.get_msg_id(dest_backend_id))
try:
response = self.ds_stub.newVersion(request,
metadata=self.metadata_call)
except RuntimeError as e:
raise e
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
return Utils.get_id(response.objectID)
def synchronize(self, session_id, object_id, implementation_id, params,
calling_backend_id):
request = dataservice_messages_pb2.SynchronizeRequest(
sessionID=Utils.get_msg_id(session_id),
objectID=Utils.get_msg_id(object_id),
implementationID=Utils.get_msg_id(implementation_id),
params=Utils.get_param_or_return(params),
callingBackendID=Utils.get_msg_id(calling_backend_id))
try:
response = self.ds_stub.synchronize(request,
metadata=self.metadata_call)
except RuntimeError as e:
raise e
if response.isException:
raise DataClayException(response.exceptionMessage)
def delete_to_db(self, execution_environment_id, object_id):
request = dataservice_messages_pb2.DeleteToDBRequest(
executionEnvironmentID=Utils.get_msg_id(execution_environment_id),
objectID=Utils.get_msg_id(object_id),
)
try:
response = self.ds_stub.deleteToDB(request,
metadata=self.metadata_call)
except RuntimeError as e:
raise e
if response.isException:
raise DataClayException(response.exceptionMessage)
def new_class(self, account_id, credential, language, new_classes):
new_cl = {}
for klass in new_classes:
yaml_str = dataclay_yaml_dump(klass)
new_cl[klass.name] = yaml_str
request = logicmodule_messages_pb2.NewClassRequest(
accountID=Utils.get_msg_id(account_id),
credential=Utils.get_credential(credential),
language=language,
newClasses=new_cl)
lm_function = lambda request: self.lm_stub.newClass.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
result = dict()
for k, v in response.newClasses.items():
result[k] = dataclay_yaml_load(v)
return result
def notify_execution_environment_shutdown(self, exec_env_id):
request = logicmodule_messages_pb2.NotifyExecutionEnvironmentShutdownRequest(
executionEnvironmentID=Utils.get_msg_id(exec_env_id))
lm_function = lambda request: self.lm_stub.notifyExecutionEnvironmentShutdown.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.isException:
raise DataClayException(response.exceptionMessage)
def get_from_db(self, execution_environment_id, object_id):
request = dataservice_messages_pb2.GetFromDBRequest(
executionEnvironmentID=Utils.get_msg_id(execution_environment_id),
objectID=Utils.get_msg_id(object_id),
)
try:
response = self.ds_stub.getFromDB(request,
metadata=self.metadata_call)
except RuntimeError as e:
raise e
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
return response.objBytes
def unfederate(self, session_id, object_id, external_execution_env_id,
recursive):
request = dataservice_messages_pb2.UnfederateRequest(
sessionID=Utils.get_msg_id(session_id),
objectID=Utils.get_msg_id(object_id),
externalExecutionEnvironmentID=Utils.get_msg_id(
external_execution_env_id),
recursive=recursive)
try:
response = self.ds_stub.unfederate(request,
metadata=self.metadata_call)
except RuntimeError as e:
logger.error('Failed to unfederate', exc_info=True)
raise e
if response.isException:
raise DataClayException(response.exceptionMessage)
def close_session(self, session_id):
request = logicmodule_messages_pb2.CloseSessionRequest(
sessionID=Utils.get_msg_id(session_id))
lm_function = lambda request: self.lm_stub.closeSession.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.isException:
raise DataClayException(response.exceptionMessage)
def add_alias(self, object_id, alias):
request = logicmodule_messages_pb2.AddAliasRequest(
objectIDToHaveAlias=Utils.get_msg_id(object_id), alias=alias)
lm_function = lambda request: self.lm_stub.addAlias.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.isException:
raise DataClayException(response.exceptionMessage)
def import_models_from_external_dataclay(self, namespace, ext_dataclay_id):
request = logicmodule_messages_pb2.ImportModelsFromExternalDataClayRequest(
namespaceName=namespace,
dataClayID=Utils.get_msg_id(ext_dataclay_id))
lm_function = lambda request: self.lm_stub.importModelsFromExternalDataClay.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.isException:
raise DataClayException(response.exceptionMessage)
def register_objects_from_gc(self, reg_info, backend_id):
reg_info_set = CommonMessages.RegistrationInfo(
objectID=Utils.get_msg_id(reg_info.object_id),
classID=Utils.get_msg_id(reg_info.class_id),
sessionID=Utils.get_msg_id(reg_info.store_session_id),
dataSetID=Utils.get_msg_id(reg_info.dataset_id),
alias=reg_info.alias)
request = logicmodule_messages_pb2.RegisterObjectForGCRequest(
regInfo=reg_info_set, backendID=Utils.get_msg_id(backend_id))
lm_function = lambda request: self.lm_stub.registerObjectFromGC.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.isException:
raise DataClayException(response.exceptionMessage)
def object_exists_in_dataclay(self, object_id):
request = logicmodule_messages_pb2.ObjectExistsInDataClayRequest(
objectID=Utils.get_msg_id(object_id))
lm_function = lambda request: self.lm_stub.objectExistsInDataClay.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
return response.exists
