def get_all_locations(self, object_id):
self.logger.debug("Getting all locations of object %s", object_id)
locations = set()
obj = self.get_from_heap(object_id)
if obj is not None:
replica_locs = obj.get_replica_locations()
if replica_locs is not None:
locations.update(replica_locs)
if obj.get_origin_location() is not None:
locations.add(obj.get_origin_location())
try:
metadata = self.get_metadata(object_id)
for loc in metadata.locations:
locations.add(loc)
# add hint location
if obj is not None:
hint = obj.get_hint()
if hint is not None:
locations.add(hint)
return locations
except:
self.logger.debug("Object %s has no metadata", object_id)
if obj is not None:
hint = obj.get_hint()
if hint is not None:
self.logger.debug("Returning list with hint from heap object")
locations = dict()
locations[hint] = self.get_execution_environment_info(hint)
return locations
else:
raise DataClayException("The object %s is not initialized well, hint missing or not exist" % object_id)
else:
raise DataClayException("The object %s is not initialized" % object_id)
def ds_new_persistent_instance(self, session_id, class_id,
implementation_id, i_face_bitmaps, params):
logger.debug(
"Ready to call to a DS to build a new persistent instance for class {%s}",
class_id)
temp_iface_b = dict()
temp_param = None
if i_face_bitmaps is not None:
for k, v in i_face_bitmaps.items():
temp_iface_b[k] = v
if params is not None:
temp_param = Utils.get_param_or_return(params)
request = dataservice_messages_pb2.NewPersistentInstanceRequest(
sessionID=Utils.get_msg_options['session'](session_id),
classID=Utils.get_msg_options['meta_class'](class_id),
implementationID=Utils.get_msg_options['implem'](
implementation_id),
ifaceBitMaps=temp_iface_b,
params=temp_param)
try:
response = self.ds_stub.newPersistentInstance(request)
except RuntimeError as e:
raise e
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
return Utils.get_id(response.objectID)
def ds_get_objects(self, session_id, object_ids, already_obtained_obs,
recursive, dest_backend_id, update_replica_locs):
object_ids_list = []
for oid in object_ids:
object_ids_list.append(Utils.get_msg_id(oid))
already_obtained_objects = []
for oid in already_obtained_obs:
already_obtained_objects.append(Utils.get_msg_id(oid))
request = dataservice_messages_pb2.GetObjectsRequest(
sessionID=Utils.get_msg_id(session_id),
objectIDS=object_ids_list,
alreadyObtainedObjects=already_obtained_objects,
recursive=recursive,
destBackendID=Utils.get_msg_id(dest_backend_id),
updateReplicaLocs=update_replica_locs)
try:
response = self.ds_stub.getObjects(request,
metadata=self.metadata_call)
except RuntimeError as e:
raise e
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
serialized_objs = list()
for obj_with_data in response.objects:
serialized_objs.append(
Utils.get_obj_with_data_param_or_return(obj_with_data))
return serialized_objs
def set_dataset_id_from_garbage_collector(self, object_id, dataset_id):
request = logicmodule_messages_pb2.SetDataSetIDFromGarbageCollectorRequest(
objectID=Utils.get_msg_id(object_id),
datasetID=Utils.get_msg_id(dataset_id))
# ToDo: In Java at this point override the onNext/onError/onCompleted methods of responseObserver
try:
logger.trace(
"Asynchronous call to register object from Garbage Collector for object %s",
object_id)
# ToDo: check async
six.advance_iterator(async_req_send)
resp_future = self.lm_stub.setDataSetIDFromGarbageCollector.future.future(
request=request, metadata=self.metadata_call)
resp_future.result()
if resp_future.done():
six.advance_iterator(async_req_rec)
except RuntimeError as e:
raise e
if resp_future.isException:
raise DataClayException(resp_future.exceptionMessage)
def register_objects(self, reg_infos, backend_id, lang):
reg_infos_msg = list()
for reg_info in reg_infos:
msg_reg_info = CommonMessages.RegistrationInfo(
objectID=Utils.get_msg_id(reg_info.object_id),
classID=Utils.get_msg_id(reg_info.class_id),
sessionID=Utils.get_msg_id(reg_info.store_session_id),
dataSetID=Utils.get_msg_id(reg_info.dataset_id),
alias=reg_info.alias)
reg_infos_msg.append(msg_reg_info)
request = logicmodule_messages_pb2.RegisterObjectsRequest(
regInfos=reg_infos_msg,
backendID=Utils.get_msg_id(backend_id),
lang=lang)
lm_function = lambda request: self.lm_stub.registerObjects.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
result = list()
for oid in response.objectIDs:
result.append(Utils.get_id(oid))
return result
def new_class(self, account_id, credential, language, new_classes):
new_cl = {}
for klass in new_classes:
yaml_str = dataclay_yaml_dump(klass)
new_cl[klass.name] = yaml_str
request = logicmodule_messages_pb2.NewClassRequest(
accountID=Utils.get_msg_id(account_id),
credential=Utils.get_credential(credential),
language=language,
newClasses=new_cl)
lm_function = lambda request: self.lm_stub.newClass.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
result = dict()
for k, v in response.newClasses.items():
result[k] = dataclay_yaml_load(v)
return result
def new_session(self, account_id, credential, contracts, data_sets,
data_set_for_store, new_session_lang):
contracts_list = []
for con_id in contracts:
contracts_list.append(Utils.get_msg_id(con_id))
data_set_list = []
for data_set in data_sets:
data_set_list.append(Utils.get_msg_id(data_set))
request = logicmodule_messages_pb2.NewSessionRequest(
accountID=Utils.get_msg_id(account_id),
credential=Utils.get_credential(credential),
contractIDs=contracts_list,
dataSetIDs=data_set_list,
storeDataSet=Utils.get_msg_id(data_set_for_store),
sessionLang=new_session_lang)
lm_function = lambda request: self.lm_stub.newSession.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
return dataclay_yaml_load(response.sessionInfo)
def execute_implementation(self, session_id, operation_id,
remote_implementation, object_id, params):
request = logicmodule_messages_pb2.ExecuteImplementationRequest(
sessionID=Utils.get_msg_id(session_id),
operationID=Utils.get_msg_id(operation_id),
implementationID=Utils.get_msg_id(remote_implementation[0]),
contractID=Utils.get_msg_id(remote_implementation[1]),
interfaceID=Utils.get_msg_id(remote_implementation[2]),
params=Utils.get_param_or_return(params),
objectID=Utils.get_msg_id(object_id))
lm_function = lambda request: self.lm_stub.executeImplementation.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.excInfo.isException:
try:
exception = pickle.loads(response.excInfo.serializedException)
except:
raise DataClayException(response.excInfo.exceptionMessage)
else:
raise exception
if response.ret is not None:
return Utils.get_param_or_return(response.ret)
else:
return None
def get_stubs(self, applicant_account_id, applicant_credential, language,
contracts_ids):
cid_list = []
for cID in contracts_ids:
cid_list.append(Utils.get_msg_id(cID))
request = logicmodule_messages_pb2.GetStubsRequest(
applicantAccountID=Utils.get_msg_id(applicant_account_id),
credentials=Utils.get_credential(applicant_credential),
language=language,
contractIDs=cid_list)
lm_function = lambda request: self.lm_stub.getStubs.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
result = dict()
for k, v in response.stubs.items():
result[k] = v
return result
def ds_new_version(self, session_id, object_id, metadata_info):
logger.info("This new_version is called somewhere?")
request = dataservice_messages_pb2.NewVersionRequest(
sessionID=Utils.get_msg_options['session'](session_id),
objectID=Utils.get_msg_options['object'](object_id),
metadataInfo=dataclay_yaml_dump(metadata_info))
try:
response = self.ds_stub.newVersion(request)
except RuntimeError as e:
raise e
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
result = dict()
oid = Utils.get_id(response.objectID)
for k, v in response.versionedIDs:
result[Utils.get_id_from_uuid(k)] = Utils.get_id_from_uuid(v)
t = (oid, result)
return t
def ds_get_objects(self, session_id, object_ids, recursive, moving):
object_ids_list = []
for oid in object_ids:
object_ids_list.append(Utils.get_msg_options['object'](oid))
request = dataservice_messages_pb2.GetObjectsRequest(
sessionID=Utils.get_msg_options['session'](session_id),
objectIDS=object_ids_list,
recursive=recursive,
moving=moving)
try:
response = self.ds_stub.getObjects(request)
except RuntimeError as e:
raise e
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
serialized_objs = list()
for obj_with_data in response.objects:
serialized_objs.append(
Utils.get_obj_with_data_param_or_return(obj_with_data))
return serialized_objs
def ds_get_federated_objects(self, dc_instance_id, object_ids):
object_ids_list = []
for oid in object_ids:
object_ids_list.append(Utils.get_msg_options['object'](oid))
request = dataservice_messages_pb2.GetFederatedObjectsRequest(
extDataClayID=Utils.get_msg_options['dataclay_instance'](
dc_instance_id),
objectIDS=object_ids_list)
try:
response = self.ds_stub.getFederatedObjects(request)
except RuntimeError as e:
raise e
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
serialized_objs = list()
for obj_with_data in response.objectsList:
serialized_objs.append(
Utils.get_obj_with_data_param_or_return(obj_with_data))
return serialized_objs
def add_class(self, klass):
"""Add a class to this factory, from the class' Python object.
Note that the caller provides de class, which should be an instance of
ExecutionGateway.
:param klass: The class object.
"""
if not issubclass(klass, DataClayObject):
raise DataClayException("Can only use DataClayObject classes")
logger.verbose("Adding class %s to the MetaClassFactory", klass)
class_container = klass._prepare_metaclass(self._namespace,
self._responsible_account)
# Save to the list, and bookmark the MetaClass
# (for valid recursive behaviour, e.g. cycles)
complete_name = class_container.name
logger.debug("[add_class] Using `%s` as `name` field of Type",
complete_name)
if complete_name not in self.types:
self.types[complete_name] = UserType(
signature="L{};".format(complete_name).replace(".", "/"),
includes=[],
namespace=self._namespace,
typeName=complete_name)
self.classes.append(class_container)
parent = klass.__bases__[0]
if parent is not DataClayObject:
self.add_class(parent)
logger.debug("Class %s finished", class_container.name)
def ds_store_objects(self, session_id, objects, moving, ids_with_alias):
obj_list = []
id_with_alias_list = []
for obj in objects:
obj_list.append(Utils.get_obj_with_data_param_or_return(obj))
if ids_with_alias is not None:
for id_with_alias in ids_with_alias:
if id_with_alias is not None:
id_with_alias_list.append(
Utils.get_msg_options['object'](id_with_alias))
request = dataservice_messages_pb2.StoreObjectsRequest(
sessionID=Utils.get_msg_options['session'](session_id),
objects=obj_list,
moving=moving,
idsWithAlias=id_with_alias_list)
try:
response = self.ds_stub.storeObjects(request)
except RuntimeError as e:
traceback.print_exc(file=sys.stdout)
raise e
if response.isException:
raise DataClayException(response.exceptionMessage)
def ds_move_objects(self, session_id, object_id, dest_st_location,
recursive):
request = dataservice_messages_pb2.MoveObjectsRequest(
sessionID=Utils.get_msg_id(session_id),
objectID=Utils.get_msg_id(object_id),
destLocID=Utils.get_msg_id(dest_st_location),
recursive=recursive)
try:
response = self.ds_stub.moveObjects(request,
metadata=self.metadata_call)
except RuntimeError as e:
raise e
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
result = set()
for oid in response.movedObjects:
result.add(Utils.get_id(oid))
return result
def deactivate_tracing(self):
lm_function = lambda request: self.lm_stub.deactivateTracing.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(CommonMessages.EmptyMessage(),
lm_function)
if response.isException:
raise DataClayException(response.exceptionMessage)
def ds_execute_implementation(self, object_id, implementation_id,
session_id, params):
logger.debug("Client performing ExecuteImplementation")
request = dataservice_messages_pb2.ExecuteImplementationRequest(
sessionID=Utils.get_msg_options['session'](session_id),
implementationID=Utils.get_msg_options['implem'](
implementation_id),
params=Utils.get_param_or_return(params),
objectID=Utils.get_msg_options['object'](object_id))
try:
response = self.ds_stub.executeImplementation(request)
except RuntimeError as e:
logger.error('Failed to execute implementation', exc_info=True)
raise e
if response.excInfo.isException:
try:
exception = pickle.loads(response.excInfo.serializedException)
except:
raise DataClayException(response.excInfo.exceptionMessage)
else:
raise exception
if response.ret is not None:
return Utils.get_param_or_return(response.ret)
else:
return None
def ds_remove_objects(self, session_id, object_ids, recursive, moving,
new_hint):
obj_ids_list = []
for oid in object_ids:
obj_ids_list.append(Utils.get_msg_options['object'](oid))
request = dataservice_messages_pb2.RemoveObjectsRequest(
sessionID=Utils.get_msg_options['session'](session_id),
objectIDs=obj_ids_list,
recursive=recursive,
moving=moving,
newHint=Utils.get_msg_options['exec_env'](new_hint))
try:
response = self.ds_stub.removeObjects(request)
except RuntimeError as e:
raise e
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
result = dict()
for k, v in response.removedObjects.items():
result[Utils.get_id_from_uuid(k)] = Utils.get_id_from_uuid(v)
return result
def check_alive(self):
lm_function = lambda request: self.lm_stub.checkAlive.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(CommonMessages.EmptyMessage(),
lm_function)
if response.isException:
raise DataClayException(response.exceptionMessage)
def get_location(self, object_id):
self.logger.debug("Looking for location of object %s", str(object_id))
try:
obj = self.get_from_heap(object_id)
if obj is not None:
hint = obj.get_hint()
if hint is not None:
self.logger.debug("Returning hint from heap object")
return hint
else:
raise DataClayException("The object %s is not initialized well, hint missing or not exist" % object_id)
else:
raise DataClayException("The object %s is not initialized" % object_id)
except DataClayException as e:
# If the object is not initialized well trying to obtain location from metadata
metadata = self.get_metadata(object_id)
return six.advance_iterator(iter(metadata.locations))
def activate_tracing(self, task_id):
request = logicmodule_messages_pb2.ActivateTracingRequest(
taskid=task_id)
lm_function = lambda request: self.lm_stub.activateTracing.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.isException:
raise DataClayException(response.exceptionMessage)
def notify_execution_environment_shutdown(self, exec_env_id):
request = logicmodule_messages_pb2.NotifyExecutionEnvironmentShutdownRequest(
executionEnvironmentID=Utils.get_msg_id(exec_env_id))
lm_function = lambda request: self.lm_stub.notifyExecutionEnvironmentShutdown.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.isException:
raise DataClayException(response.exceptionMessage)
def get_num_objects(self):
lm_function = lambda request: self.lm_stub.getNumObjects.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(CommonMessages.EmptyMessage(),
lm_function)
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
return response.numObjs
def is_prefetching_enabled(self):
lm_function = lambda request: self.lm_stub.isPrefetchingEnabled.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(CommonMessages.EmptyMessage(),
lm_function)
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
return response.enabled
def close_session(self, session_id):
request = logicmodule_messages_pb2.CloseSessionRequest(
sessionID=Utils.get_msg_id(session_id))
lm_function = lambda request: self.lm_stub.closeSession.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.isException:
raise DataClayException(response.exceptionMessage)
def add_alias(self, object_id, alias):
request = logicmodule_messages_pb2.AddAliasRequest(
objectIDToHaveAlias=Utils.get_msg_id(object_id), alias=alias)
lm_function = lambda request: self.lm_stub.addAlias.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.isException:
raise DataClayException(response.exceptionMessage)
def advise_event(self, new_event):
request = logicmodule_messages_pb2.AdviseEventRequest(
eventYaml=dataclay_yaml_dump(new_event))
lm_function = lambda request: self.lm_stub.adviseEvent.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.isException:
raise DataClayException(response.exceptionMessage)
def import_models_from_external_dataclay(self, namespace, ext_dataclay_id):
request = logicmodule_messages_pb2.ImportModelsFromExternalDataClayRequest(
namespaceName=namespace,
dataClayID=Utils.get_msg_id(ext_dataclay_id))
lm_function = lambda request: self.lm_stub.importModelsFromExternalDataClay.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.isException:
raise DataClayException(response.exceptionMessage)
def deactivate_tracing(self):
try:
response = self.ds_stub.deactivateTracing(
CommonMessages.EmptyMessage())
except RuntimeError as e:
raise e
if response.isException:
raise DataClayException(response.exceptionMessage)
def object_exists_in_dataclay(self, object_id):
request = logicmodule_messages_pb2.ObjectExistsInDataClayRequest(
objectID=Utils.get_msg_id(object_id))
lm_function = lambda request: self.lm_stub.objectExistsInDataClay.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
return response.exists
