def ds_move_objects(self, session_id, object_id, dest_st_location, recursive): request = dataservice_messages_pb2.MoveObjectsRequest( sessionID=Utils.get_msg_id(session_id), objectID=Utils.get_msg_id(object_id), destLocID=Utils.get_msg_id(dest_st_location), recursive=recursive) try: response = self.ds_stub.moveObjects(request, metadata=self.metadata_call) except RuntimeError as e: raise e if response.excInfo.isException: raise DataClayException(response.excInfo.exceptionMessage) result = set() for oid in response.movedObjects: result.add(Utils.get_id(oid)) return result
def ds_get_objects(self, session_id, object_ids, already_obtained_obs, recursive, dest_backend_id, update_replica_locs): object_ids_list = [] for oid in object_ids: object_ids_list.append(Utils.get_msg_id(oid)) already_obtained_objects = [] for oid in already_obtained_obs: already_obtained_objects.append(Utils.get_msg_id(oid)) request = dataservice_messages_pb2.GetObjectsRequest( sessionID=Utils.get_msg_id(session_id), objectIDS=object_ids_list, alreadyObtainedObjects=already_obtained_objects, recursive=recursive, destBackendID=Utils.get_msg_id(dest_backend_id), updateReplicaLocs=update_replica_locs) try: response = self.ds_stub.getObjects(request, metadata=self.metadata_call) except RuntimeError as e: raise e if response.excInfo.isException: raise DataClayException(response.excInfo.exceptionMessage) serialized_objs = list() for obj_with_data in response.objects: serialized_objs.append( Utils.get_obj_with_data_param_or_return(obj_with_data)) return serialized_objs
def removeObjects(self, request, context): try: object_ids = set() for oid in request.getObjectsIDSList(): object_ids.add(Utils.get_id(oid)) result = self.client.ds_remove_objects(Utils.get_id(request.sessionID), object_ids, request.recursive, request.moving, Utils.get_id(request.newHint)) rem_obj = dict() for k, v in result.items(): rem_obj[str(k)] = str(v) return dataservice_messages_pb2.RemoveObjectsResponse( removedObjects=rem_obj ) except Exception as ex: return dataservice_messages_pb2.RemoveObjectsResponse( excInfo=self.get_exception_info(ex) )
def ds_execute_implementation(self, object_id, implementation_id, session_id, params): logger.debug("Client performing ExecuteImplementation") request = dataservice_messages_pb2.ExecuteImplementationRequest( sessionID=Utils.get_msg_options['session'](session_id), implementationID=Utils.get_msg_options['implem']( implementation_id), params=Utils.get_param_or_return(params), objectID=Utils.get_msg_options['object'](object_id)) try: response = self.ds_stub.executeImplementation(request) except RuntimeError as e: logger.error('Failed to execute implementation', exc_info=True) raise e if response.excInfo.isException: try: exception = pickle.loads(response.excInfo.serializedException) except: raise DataClayException(response.excInfo.exceptionMessage) else: raise exception if response.ret is not None: return Utils.get_param_or_return(response.ret) else: return None
def ds_store_objects(self, session_id, objects, moving, ids_with_alias): obj_list = [] id_with_alias_list = [] for obj in objects: obj_list.append(Utils.get_obj_with_data_param_or_return(obj)) if ids_with_alias is not None: for id_with_alias in ids_with_alias: if id_with_alias is not None: id_with_alias_list.append(Utils.get_msg_id(id_with_alias)) request = dataservice_messages_pb2.StoreObjectsRequest( sessionID=Utils.get_msg_id(session_id), objects=obj_list, moving=moving, idsWithAlias=id_with_alias_list) try: response = self.ds_stub.storeObjects(request, metadata=self.metadata_call) except RuntimeError as e: traceback.print_exc(file=sys.stdout) raise e if response.isException: raise DataClayException(response.exceptionMessage)
def set_dataset_id_from_garbage_collector(self, object_id, dataset_id): request = logicmodule_messages_pb2.SetDataSetIDFromGarbageCollectorRequest( objectID=Utils.get_msg_id(object_id), datasetID=Utils.get_msg_id(dataset_id)) # ToDo: In Java at this point override the onNext/onError/onCompleted methods of responseObserver try: logger.trace( "Asynchronous call to register object from Garbage Collector for object %s", object_id) # ToDo: check async six.advance_iterator(async_req_send) resp_future = self.lm_stub.setDataSetIDFromGarbageCollector.future.future( request=request, metadata=self.metadata_call) resp_future.result() if resp_future.done(): six.advance_iterator(async_req_rec) except RuntimeError as e: raise e if resp_future.isException: raise DataClayException(resp_future.exceptionMessage)
def getObjects(self, request, context): try: object_ids = set() for oid in request.objectIDS: object_ids.add(Utils.get_id(oid)) already_obtained_objects = set() for oid in request.alreadyObtainedObjects: already_obtained_objects.add(Utils.get_id(oid)) result = self.execution_environment.get_objects(Utils.get_id(request.sessionID), object_ids, already_obtained_objects, request.recursive, Utils.get_id(request.destBackendID), request.updateReplicaLocs) obj_list = [] for entry in result: obj_list.append(Utils.get_obj_with_data_param_or_return(entry)) return dataservice_messages_pb2.GetObjectsResponse(objects=obj_list) except Exception as ex: traceback.print_exc() return dataservice_messages_pb2.GetObjectsResponse( excInfo=self.get_exception_info(ex) )
def ds_new_persistent_instance(self, session_id, class_id, implementation_id, i_face_bitmaps, params): logger.debug( "Ready to call to a DS to build a new persistent instance for class {%s}", class_id) temp_iface_b = dict() temp_param = None if i_face_bitmaps is not None: for k, v in i_face_bitmaps.items(): temp_iface_b[k] = v if params is not None: temp_param = Utils.get_param_or_return(params) request = dataservice_messages_pb2.NewPersistentInstanceRequest( sessionID=Utils.get_msg_options['session'](session_id), classID=Utils.get_msg_options['meta_class'](class_id), implementationID=Utils.get_msg_options['implem']( implementation_id), ifaceBitMaps=temp_iface_b, params=temp_param) try: response = self.ds_stub.newPersistentInstance(request) except RuntimeError as e: raise e if response.excInfo.isException: raise DataClayException(response.excInfo.exceptionMessage) return Utils.get_id(response.objectID)
def ds_new_version(self, session_id, object_id, metadata_info): logger.info("This new_version is called somewhere?") request = dataservice_messages_pb2.NewVersionRequest( sessionID=Utils.get_msg_options['session'](session_id), objectID=Utils.get_msg_options['object'](object_id), metadataInfo=dataclay_yaml_dump(metadata_info)) try: response = self.ds_stub.newVersion(request) except RuntimeError as e: raise e if response.excInfo.isException: raise DataClayException(response.excInfo.exceptionMessage) result = dict() oid = Utils.get_id(response.objectID) for k, v in response.versionedIDs: result[Utils.get_id_from_uuid(k)] = Utils.get_id_from_uuid(v) t = (oid, result) return t
def ds_remove_objects(self, session_id, object_ids, recursive, moving, new_hint): obj_ids_list = [] for oid in object_ids: obj_ids_list.append(Utils.get_msg_options['object'](oid)) request = dataservice_messages_pb2.RemoveObjectsRequest( sessionID=Utils.get_msg_options['session'](session_id), objectIDs=obj_ids_list, recursive=recursive, moving=moving, newHint=Utils.get_msg_options['exec_env'](new_hint)) try: response = self.ds_stub.removeObjects(request) except RuntimeError as e: raise e if response.excInfo.isException: raise DataClayException(response.excInfo.exceptionMessage) result = dict() for k, v in response.removedObjects.items(): result[Utils.get_id_from_uuid(k)] = Utils.get_id_from_uuid(v) return result
def new_class(self, account_id, credential, language, new_classes): new_cl = {} for klass in new_classes: yaml_str = dataclay_yaml_dump(klass) new_cl[klass.name] = yaml_str request = logicmodule_messages_pb2.NewClassRequest( accountID=Utils.get_msg_id(account_id), credential=Utils.get_credential(credential), language=language, newClasses=new_cl) lm_function = lambda request: self.lm_stub.newClass.future( request=request, metadata=self.metadata_call) response = self._call_logicmodule(request, lm_function) if response.excInfo.isException: raise DataClayException(response.excInfo.exceptionMessage) result = dict() for k, v in response.newClasses.items(): result[k] = dataclay_yaml_load(v) return result
def new_session(self, account_id, credential, contracts, data_sets, data_set_for_store, new_session_lang): contracts_list = [] for con_id in contracts: contracts_list.append(Utils.get_msg_id(con_id)) data_set_list = [] for data_set in data_sets: data_set_list.append(Utils.get_msg_id(data_set)) request = logicmodule_messages_pb2.NewSessionRequest( accountID=Utils.get_msg_id(account_id), credential=Utils.get_credential(credential), contractIDs=contracts_list, dataSetIDs=data_set_list, storeDataSet=Utils.get_msg_id(data_set_for_store), sessionLang=new_session_lang) lm_function = lambda request: self.lm_stub.newSession.future( request=request, metadata=self.metadata_call) response = self._call_logicmodule(request, lm_function) if response.excInfo.isException: raise DataClayException(response.excInfo.exceptionMessage) return dataclay_yaml_load(response.sessionInfo)
def get_stubs(self, applicant_account_id, applicant_credential, language, contracts_ids): cid_list = [] for cID in contracts_ids: cid_list.append(Utils.get_msg_id(cID)) request = logicmodule_messages_pb2.GetStubsRequest( applicantAccountID=Utils.get_msg_id(applicant_account_id), credentials=Utils.get_credential(applicant_credential), language=language, contractIDs=cid_list) lm_function = lambda request: self.lm_stub.getStubs.future( request=request, metadata=self.metadata_call) response = self._call_logicmodule(request, lm_function) if response.excInfo.isException: raise DataClayException(response.excInfo.exceptionMessage) result = dict() for k, v in response.stubs.items(): result[k] = v return result
def detachObjectFromSession(self, request, context): try: self.execution_environment.detach_object_from_session(Utils.get_id(request.objectID), Utils.get_id(request.sessionID)) return common_messages_pb2.ExceptionInfo() except Exception as ex: return self.get_exception_info(ex)
def deleteAlias(self, request, context): try: self.execution_environment.delete_alias(Utils.get_id(request.sessionID), Utils.get_id(request.objectID)) return common_messages_pb2.ExceptionInfo() except Exception as ex: return self.get_exception_info(ex)
def set_object_read_write(self, session_id, object_id): request = logicmodule_messages_pb2.SetObjectReadWriteRequest( sessionID=Utils.get_msg_id(session_id), objectID=Utils.get_msg_id(object_id)) lm_function = lambda request: self.lm_stub.setObjectReadWrite.future( request=request, metadata=self.metadata_call) response = self._call_logicmodule(request, lm_function) if response.isException: raise DataClayException(response.exceptionMessage)
def get_external_dataclay_info(self, ext_dataclay_id): request = logicmodule_messages_pb2.GetExtDataClayInfoRequest( extDataClayID=Utils.get_msg_id(ext_dataclay_id)) lm_function = lambda request: self.lm_stub.getExternalDataClayInfo.future( request=request, metadata=self.metadata_call) response = self._call_logicmodule(request, lm_function) if response.excInfo.isException: raise DataClayException(response.excInfo.exceptionMessage) return Utils.get_dataclay_instance(response.extDataClayInfo)
def consolidateVersion(self, request, context): try: self.execution_environment.consolidate_version(Utils.get_id(request.sessionID), Utils.get_id(request.versionObjectID)) return common_messages_pb2.ExceptionInfo() except Exception as ex: return self.get_exception_info(ex)
def remove_class(self, account_id, credential, namespace_id, class_name): request = logicmodule_messages_pb2.RemoveClassRequest( accountID=Utils.get_msg_id(account_id), credential=Utils.get_credential(credential), className=class_name) lm_function = lambda request: self.lm_stub.removeClass.future( request=request, metadata=self.metadata_call) response = self._call_logicmodule(request, lm_function) if response.isException: raise DataClayException(response.exceptionMessage)
def delete_alias(self, session_id, object_id): request = dataservice_messages_pb2.DeleteAliasRequest( sessionID=Utils.get_msg_id(session_id), objectID=Utils.get_msg_id(object_id)) try: response = self.ds_stub.deleteAlias(request, metadata=self.metadata_call) except RuntimeError as e: raise e if response.isException: raise DataClayException(response.exceptionMessage)
def detach_object_from_session(self, object_id, session_id): request = dataservice_messages_pb2.DetachObjectFromSessionRequest( objectID=Utils.get_msg_id(object_id), sessionID=Utils.get_msg_id(session_id)) try: response = self.ds_stub.detachObjectFromSession( request, metadata=self.metadata_call) except RuntimeError as e: raise e if response.isException: raise DataClayException(response.exceptionMessage)
def delete_alias(self, session_id, alias): request = logicmodule_messages_pb2.DeleteAliasRequest( sessionID=Utils.get_msg_id(session_id), alias=alias) lm_function = lambda request: self.lm_stub.deleteAlias.future( request=request, metadata=self.metadata_call) response = self._call_logicmodule(request, lm_function) if response.excInfo.isException: raise DataClayException(response.excInfo.exceptionMessage) return Utils.get_id(response.objectID)
def federate(self, request, context): try: logger.verbose("Federation started") objects_to_persist = Utils.get_param_or_return(request.params) session_id = Utils.get_id(request.sessionID) self.execution_environment.federate(session_id, objects_to_persist) logger.verbose("Federation finished, sending response") return common_messages_pb2.ExceptionInfo() except Exception as ex: return self.get_exception_info(ex)
def unfederate(self, request, context): try: logger.verbose("Unfederation started") self.execution_environment.unfederate(Utils.get_id(request.sessionID), Utils.get_id(request.objectID), Utils.get_id(request.externalExecutionEnvironmentID), request.recursive) logger.verbose("Unfederation finished, sending response") return common_messages_pb2.ExceptionInfo() except Exception as ex: return self.get_exception_info(ex)
def get_storage_location_info(self, st_loc_id, from_backend=False): request = logicmodule_messages_pb2.GetStorageLocationInfoRequest( storageLocationID=Utils.get_msg_id(st_loc_id), fromBackend=from_backend) lm_function = lambda request: self.lm_stub.getStorageLocationInfo.future( request=request, metadata=self.metadata_call) response = self._call_logicmodule(request, lm_function) if response.excInfo.isException: raise DataClayException(response.excInfo.exceptionMessage) return Utils.get_storage_location(response.storageLocationInfo)
def get_external_executionenvironment_info(self, backend_id): request = logicmodule_messages_pb2.GetExternalExecutionEnvironmentInfoRequest( execEnvID=Utils.get_msg_id(backend_id)) lm_function = lambda request: self.lm_stub.getExternalExecutionEnvironmentInfo.future( request=request, metadata=self.metadata_call) response = self._call_logicmodule(request, lm_function) if response.excInfo.isException: raise DataClayException(response.excInfo.exceptionMessage) return Utils.get_external_execution_environment( response.executionEnvironmentInfo)
def unfederate_object(self, session_id, object_id, ext_dataclay_id, recursive): request = logicmodule_messages_pb2.UnfederateObjectRequest( sessionID=Utils.get_msg_id(session_id), objectID=Utils.get_msg_id(object_id), extDataClayID=Utils.get_msg_id(ext_dataclay_id), recursive=recursive) lm_function = lambda request: self.lm_stub.unfederateObject.future( request=request, metadata=self.metadata_call) response = self._call_logicmodule(request, lm_function) if response.isException: raise DataClayException(response.exceptionMessage)
def get_object_info(self, session_id, object_id): request = logicmodule_messages_pb2.GetObjectInfoRequest( sessionID=Utils.get_msg_id(session_id), objectID=Utils.get_msg_id(object_id)) lm_function = lambda request: self.lm_stub.getObjectInfo.future( request=request, metadata=self.metadata_call) response = self._call_logicmodule(request, lm_function) if response.excInfo.isException: raise DataClayException(response.excInfo.exceptionMessage) return response.classname, response.namespace
def synchronize(self, request, context): try: object_id = Utils.get_id(request.objectID) implementation_id = Utils.get_id(request.implementationID) serialized_params = Utils.get_param_or_return(request.params) session_id = Utils.get_id(request.sessionID) calling_backend_id = Utils.get_id(request.callingBackendID) self.execution_environment.synchronize(session_id, object_id, implementation_id, serialized_params, calling_backend_id) return common_messages_pb2.ExceptionInfo() except DataClayException as ex: return self.get_exception_info(ex)
def register_event_listener_implementation(self, account_id, credential, new_event_listener): request = logicmodule_messages_pb2.RegisterECARequest( applicantAccountID=Utils.get_msg_id(account_id), credentials=Utils.get_credential(credential), eca=dataclay_yaml_dump(new_event_listener)) lm_function = lambda request: self.lm_stub.registerECA.future( request=request, metadata=self.metadata_call) response = self._call_logicmodule(request, lm_function) if response.excInfo.isException: raise DataClayException(response.excInfo.exceptionMessage)