def babel_stubs_load(stream):
# Note the explicit list, just in case the caller wants to close the provided file/buffer
map_babel_stubs = dataclay_yaml_load(stream, Loader=Loader)
result = list()
for k, v in map_babel_stubs.items():
result.append(dataclay_yaml_load(v))
return result
def new_session(self, account_id, credential, contracts, data_sets,
data_set_for_store, new_session_lang):
contracts_list = []
for con_id in contracts:
contracts_list.append(Utils.get_msg_id(con_id))
data_set_list = []
for data_set in data_sets:
data_set_list.append(Utils.get_msg_id(data_set))
request = logicmodule_messages_pb2.NewSessionRequest(
accountID=Utils.get_msg_id(account_id),
credential=Utils.get_credential(credential),
contractIDs=contracts_list,
dataSetIDs=data_set_list,
storeDataSet=Utils.get_msg_id(data_set_for_store),
sessionLang=new_session_lang)
lm_function = lambda request: self.lm_stub.newSession.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
return dataclay_yaml_load(response.sessionInfo)
def new_class(self, account_id, credential, language, new_classes):
new_cl = {}
for klass in new_classes:
yaml_str = dataclay_yaml_dump(klass)
new_cl[klass.name] = yaml_str
request = logicmodule_messages_pb2.NewClassRequest(
accountID=Utils.get_msg_id(account_id),
credential=Utils.get_credential(credential),
language=language,
newClasses=new_cl)
lm_function = lambda request: self.lm_stub.newClass.future(
request=request, metadata=self.metadata_call)
response = self._call_logicmodule(request, lm_function)
if response.excInfo.isException:
raise DataClayException(response.excInfo.exceptionMessage)
result = dict()
for k, v in response.newClasses.items():
result[k] = dataclay_yaml_load(v)
return result
def consolidateVersion(self, request, context):
try:
self.execution_environment.consolidate_version(
Utils.get_id(request.sessionID),
dataclay_yaml_load(request.versionInfo))
return common_messages_pb2.ExceptionInfo()
except Exception as ex:
return self.get_exception_info(ex)
def load_metaclass(namespace, class_name):
"""Load binary data into a MetaClass container.
:param namespace: The name of the namespace for the class.
:param class_name: Name of the class.
:return: A MetaClass container, deserialized from previously deployed binary data.
"""
try:
return cached_metaclasses[class_name]
except KeyError:
namespace_path = os.path.join(settings.deploy_path, namespace)
with open(os.path.join(namespace_path, class_name + ".mcs"),
'rb') as f:
mc = dataclay_yaml_load(f.read())
cached_metaclasses[class_name] = mc
return mc
def newMetaData(self, request, context):
try:
md_infos = {}
for k, v in request.mdInfos.items():
md_infos[Utils.get_id_from_uuid(k)] = dataclay_yaml_load(v)
self.client.ds_new_metadata(md_infos)
return common_messages_pb2.ExceptionInfo()
except Exception as ex:
return self.get_exception_info(ex)
def newVersion(self, request, context):
try:
result = self.execution_environment.new_version(
Utils.get_id(request.sessionID),
Utils.get_id(request.objectID),
dataclay_yaml_load(request.metadataInfo))
vers_ids = dict()
for k, v in result[1].items():
vers_ids[Utils.prepare_bytes(str(k))] = Utils.prepare_bytes(
str(v))
return dataservice_messages_pb2.NewVersionResponse(
objectID=Utils.get_msg_options['object'](result[0]),
versionedIDs=vers_ids)
except Exception as ex:
return dataservice_messages_pb2.NewVersionResponse(
excInfo=self.get_exception_info(ex))
def migrateObjectsToBackends(self, request, context):
try:
backends = dict()
for k, v in request.destStorageLocs.items():
backends[Utils.get_id_from_uuid(k)] = dataclay_yaml_load(v)
result = self.client.ds_migrate_objects_to_backends(backends)
migr_obj_res = dict()
for k, v in result[0].items():
migrated_obj_list = list()
for oid in v:
migrated_obj_list.append(
Utils.get_msg_options['object'](oid))
migrated_obj_builder = dataservice_messages_pb2.MigratedObjects(
objs=migrated_obj_list)
migr_obj_res[str(k)] = migrated_obj_builder
non_migrated_objs_list = list()
for oid in result[1]:
non_migrated_objs_list.append(
Utils.get_msg_options['object'](oid))
non_migrated_objs_builder = dataservice_messages_pb2.MigratedObjects(
objs=non_migrated_objs_list)
return dataservice_messages_pb2.MigrateObjectsResponse(
migratedObjs=migr_obj_res,
nonMigratedObjs=non_migrated_objs_builder)
except Exception as ex:
return dataservice_messages_pb2.MigrateObjectsResponse(
excInfo=self.get_exception_info(ex))
def register_model(username, password, namespace, python_path):
client = _establish_client()
credential = (None, password)
user_id = client.get_account_id(username)
mfc = MetaClassFactory(namespace=namespace, responsible_account=username)
full_python_path = os.path.abspath(python_path)
# Load the model_package from the path
if full_python_path not in sys.path:
added_to_path = True
sys.path.insert(0, full_python_path)
else:
added_to_path = False
n_base_skip = len(full_python_path.split(os.sep))
for dirpath, dirnames, filenames in os.walk(full_python_path):
base_import = ".".join(dirpath.split(os.sep)[n_base_skip:])
for f in filenames:
if not f.endswith(".py"):
continue
if f == "__init__.py":
if not base_import:
print(
"Ignoring `__init__.py` at the root of the model folder (do not put classes there!)",
file=sys.stderr)
continue
import_str = base_import
else:
if not base_import:
import_str = os.path.splitext(f)[0]
else:
import_str = "%s.%s" % (base_import,
os.path.splitext(f)[0])
# Try to import
mfc.import_and_add(import_str)
result = client.new_class(user_id, credential, LANG_PYTHON, mfc.classes)
print("Was gonna register: %s\nEventually registered: %s" %
(mfc.classes, result.keys()),
file=sys.stderr)
if len(result.keys()) == 0:
print("No classes registered, exiting", file=sys.stderr)
return
interfaces = list()
interfaces_in_contract = list()
for class_name, class_info in result.items():
ref_class_name = class_name.replace('.', '')
interfaces.append(
Template("""
{{ class_name }}interface: &{{ ref_class_name }}iface !!es.bsc.dataclay.util.management.interfacemgr.Interface
providerAccountName: {{ username }}
namespace: {{ namespace }}
classNamespace: {{ namespace }}
className: {{ class_name }}
propertiesInIface: !!set {% if class_info.properties|length == 0 %} { } {% endif %}
{% for property in class_info.properties %}
? {{ property.name }}
{% endfor %}
operationsSignatureInIface: !!set {% if class_info.operations|length == 0 %} { } {% endif %}
{% for operation in class_info.operations %}
? {{ operation.nameAndDescriptor }}
{% endfor %}
""").render(class_name=class_name,
ref_class_name=ref_class_name,
username=username,
namespace=namespace,
class_info=class_info))
interfaces_in_contract.append(
Template("""
- !!es.bsc.dataclay.util.management.contractmgr.InterfaceInContract
iface: *{{ ref_class_name }}iface
implementationsSpecPerOperation: !!set {% if class_info.operations|length == 0 %} { } {% endif %}
{% for operation in class_info.operations %}
? !!es.bsc.dataclay.util.management.contractmgr.OpImplementations
operationSignature: {{ operation.nameAndDescriptor }}
numLocalImpl: 0
numRemoteImpl: 0
{% endfor %}
""").render(class_name=class_name,
ref_class_name=ref_class_name,
class_info=class_info))
contract = Template("""
{{ namespace }}contract: !!es.bsc.dataclay.util.management.contractmgr.Contract
beginDate: 1980-01-01T00:00:01
endDate: 2055-12-31T23:59:58
namespace: {{ namespace }}
providerAccountID: {{ user_id }}
applicantsAccountsIDs:
? {{ user_id }}
interfacesInContractSpecs:
{{ interfaces_in_contract }}
publicAvailable: True
""").render(user_id=user_id,
namespace=namespace,
interfaces_in_contract="\n".join(interfaces_in_contract))
yaml_request = "\n".join(interfaces) + contract
yaml_response = client.perform_set_of_operations(user_id, credential,
yaml_request)
response = dataclay_yaml_load(yaml_response)
print(" ===> The ContractID for the registered classes is:",
file=sys.stderr)
print(response["contracts"]["%scontract" % namespace])
# Remove from sys.path the model
if added_to_path:
sys.path.remove(full_python_path)
return str(response["contracts"]["%scontract" %
namespace]) # For mock testing
def _execute_from_command_line(argv=None):
client = getRuntime().ready_clients["@LM"]
if len(argv) < 2:
print("You should provide a command to the tool.", file=sys.stderr)
print(USAGE_TEXT, file=sys.stderr)
return
# ToDo: A smarter strategy should be used here. At the moment, all commands
# ToDo: are hardcoded and behaviour is programmed somewhat ad hoc.
if argv[1] == "createuser":
# Create a NORMAL_ROLE user.
admin_id = client.get_account_id(argv[2])
admin_credential = (None, argv[3])
username = argv[4]
password = argv[5]
yaml_request_template = Template("""
---
- !!es.bsc.dataclay.util.management.accountmgr.Account
username: {{ username }}
credential: !!es.bsc.dataclay.util.management.accountmgr.PasswordCredential
password: {{ password }}
role: NORMAL_ROLE
""")
yaml_request = yaml_request_template.render(
username=username,
password=password,
)
client.perform_set_of_new_accounts(admin_id, admin_credential,
yaml_request)
elif argv[1] == "registertodataclaypubliccontract":
username = argv[2]
credential = (None, argv[3])
user_id = client.get_account_id(username)
#########################################################################
# First, we prepare the namespace:
try:
namespace_id = client.get_namespace_id(user_id, credential,
"dc_classes")
except Exception:
yaml_request = """
---
{namespace}: !!es.bsc.dataclay.util.management.namespacemgr.Namespace
providerAccountName: {consumer_name}
name: {namespace_name}
language: LANG_PYTHON
""".format(consumer_name=username,
namespace="dc_classes",
namespace_name="dc_classes")
yaml_response = client.perform_set_of_operations(
user_id, credential, yaml_request)
response = dataclay_yaml_load(yaml_response)
namespace_id = response["namespaces"]["dc_classes"]
#########################################################################
# Then we prepare the classes
from dataclay import contrib
modules = contrib.MODULES_TO_REGISTER
mfc = MetaClassFactory(namespace="dc_classes",
responsible_account=username)
for m_str in modules:
m = import_module("dataclay.contrib.%s" % m_str)
for c_str in getattr(m, "CLASSES_TO_REGISTER"):
mfc.add_class(getattr(m, c_str))
client = getRuntime().ready_clients["@LM"]
result = client.new_class(user_id, LANG_PYTHON, mfc.classes)
if not result:
raise RuntimeError(
"No classes successfully registered --cannot continue")
class_interface_template = Template("""
{{ brief_name }}interface: &{{ brief_name }}iface !!es.bsc.dataclay.util.management.interfacemgr.Interface
providerAccountName: {{ username }}
namespace: dc_classes
classNamespace: dc_classes
className: {{ class_name }}
propertiesInIface: !!set {% if class_info.properties|length == 0 %} { } {% endif %}
{% for property in class_info.properties %}
? {{ property.name }}
{% endfor %}
operationsSignatureInIface: !!set {% if class_info.operations|length == 0 %} { } {% endif %}
{% for operation in class_info.operations %}
? {{ operation.nameAndDescriptor }}
{% endfor %}
""")
class_interface_in_contract_template = Template("""
- !!es.bsc.dataclay.util.management.contractmgr.InterfaceInContract
iface: *{{ brief_name }}iface
implementationsSpecPerOperation: !!set {% if class_info.operations|length == 0 %} { } {% endif %}
{% for operation in class_info.operations %}
? !!es.bsc.dataclay.util.management.contractmgr.OpImplementations
operationSignature: {{ operation.nameAndDescriptor }}
numLocalImpl: 0
numRemoteImpl: 0
{% endfor %}
""")
classes_render = list()
incontract_render = list()
for class_name, class_info in result.items():
brief_name = class_name.rsplit(".", 1)[-1].lower()
classes_render.append(
class_interface_template.render(username=username,
brief_name=brief_name,
class_name=class_name,
class_info=class_info))
incontract_render.append(
class_interface_in_contract_template.render(
brief_name=brief_name, class_info=class_info))
yaml_request_template = Template("""
---
{% for class_iface in class_interfaces %}
{{ class_iface }}
{% endfor %}
contribcontract: !!es.bsc.dataclay.util.management.contractmgr.Contract
beginDate: 2012-09-10T20:00:03
endDate: 2020-09-10T20:00:04
namespace: dc_classes
providerAccountID: {{ user_id }}
applicantsAccountsIDs:
? {{ user_id }}
interfacesInContractSpecs:
{% for contract in contracts %}
{{ contract }}
{% endfor %}
publicAvailable: True
""")
yaml_request = yaml_request_template.render(
user_id=user_id,
class_interfaces=classes_render,
contracts=incontract_render)
yaml_response = client.perform_set_of_operations(
user_id, credential, yaml_request)
response = dataclay_yaml_load(yaml_response)
#########################################################################
# Now (hopefully) the contract for the public classes has been obtained
print(" ===> The ContractID for the registered classes is:",
file=sys.stderr)
print(response["contracts"]["contribcontract"], file=sys.stderr)
elif argv[1] == "__new__registermodel":
namespace = argv[2]
python_path = argv[3]
username = argv[4]
password = argv[5]
# Ugly stuff related to the namespace first...
credential = (None, password)
user_id = client.get_account_id(username)
yaml_request_template = Template("""
---
{{ namespace }}: !!es.bsc.dataclay.util.management.namespacemgr.Namespace
providerAccountName: {{ username }}
name: {{ namespace }}
language: LANG_PYTHON
""")
yaml_request = yaml_request_template.render(namespace=namespace,
username=username)
try:
client.perform_set_of_operations(user_id, credential, yaml_request)
except grpc.RpcError:
# We assume that the namespace already exists
pass
# Then use the new register_model shiny stuff
from .functions import register_model
register_model(namespace=namespace,
python_path=python_path,
username=username,
password=password)
elif argv[1] == "registermodule":
namespace = argv[2]
module_name = argv[3]
username = argv[4]
credential = (None, argv[5])
user_id = client.get_account_id(username)
yaml_request_template = Template("""
---
{{ namespace }}: !!es.bsc.dataclay.util.management.namespacemgr.Namespace
providerAccountName: {{ username }}
name: {{ namespace }}
language: LANG_PYTHON
""")
yaml_request = yaml_request_template.render(namespace=namespace,
username=username)
try:
client.perform_set_of_operations(user_id, credential, yaml_request)
except grpc.RpcError:
# We assume that the namespace already exists
pass
mfc = MetaClassFactory(namespace=namespace,
responsible_account=username)
# Scrap the classes in the module
registered_classes = list()
module = import_module(module_name)
for thing_name in dir(module):
thing = getattr(module, thing_name)
if not isinstance(thing, ExecutionGateway):
continue
# Thing seems to be a DataClayObject class
if thing.__module__ != module_name:
print(
"The module for %s is %s, ignoring because it does not equals %s"
% (thing, thing.__module__, module_name),
file=sys.stderr)
continue
# Ok, that's a valid class
mfc.add_class(thing)
registered_classes.append(thing.__name__)
registrator_id = client.get_account_id(username)
result = client.new_class(registrator_id, credential, LANG_PYTHON,
mfc.classes)
print("Was gonna register: %s\nEventually registered: %s" %
(registered_classes, result.keys()),
file=sys.stderr)
if len(result.keys()) == 0:
print("No classes registered, exiting", file=sys.stderr)
return
interfaces = list()
interfaces_in_contract = list()
for class_name, class_info in result.items():
ref_class_name = class_name.replace('.', '')
interfaces.append(
Template("""
{{ class_name }}interface: &{{ ref_class_name }}iface !!es.bsc.dataclay.util.management.interfacemgr.Interface
providerAccountName: {{ username }}
namespace: {{ namespace }}
classNamespace: {{ namespace }}
className: "{{ reg_class_name }}"
propertiesInIface: !!set {% if class_info.properties|length == 0 %} { } {% endif %}
{% for property in class_info.properties %}
? {{ property.name }}
{% endfor %}
operationsSignatureInIface: !!set {% if class_info.operations|length == 0 %} { } {% endif %}
{% for operation in class_info.operations %}
? {{ operation.nameAndDescriptor }}
{% endfor %}
""").render(class_name=ref_class_name,
reg_class_name=class_name,
username=username,
namespace=namespace,
class_info=class_info))
interfaces_in_contract.append(
Template("""
- !!es.bsc.dataclay.util.management.contractmgr.InterfaceInContract
iface: *{{ class_name }}iface
implementationsSpecPerOperation: !!set {% if class_info.operations|length == 0 %} { } {% endif %}
{% for operation in class_info.operations %}
? !!es.bsc.dataclay.util.management.contractmgr.OpImplementations
operationSignature: {{ operation.nameAndDescriptor }}
numLocalImpl: 0
numRemoteImpl: 0
{% endfor %}
""").render(class_name=ref_class_name, class_info=class_info))
contract = Template("""
{{ namespace }}contract: !!es.bsc.dataclay.util.management.contractmgr.Contract
beginDate: 1980-01-01T00:00:01
endDate: 2055-12-31T23:59:58
namespace: {{ namespace }}
providerAccountID: {{ 00000000-00000000-00000000-00000000-00000000 }}
applicantsNames:
? {{ username }}
interfacesInContractSpecs:
{{ interfaces_in_contract }}
publicAvailable: True
""").render(namespace=namespace,
username=username,
interfaces_in_contract="\n".join(interfaces_in_contract))
yaml_request = "\n".join(interfaces) + contract
print(" ===> The yaml for performing is %s" % (yaml_request),
file=sys.stderr)
yaml_response = client.perform_set_of_operations(
user_id, credential, yaml_request)
response = dataclay_yaml_load(yaml_response)
print(" ===> The ContractID for the registered classes is: ",
file=sys.stderr)
print(response["contracts"]["%scontract" % namespace])
elif argv[1] == "registerdataset":
dataset = argv[2]
username = argv[3]
credential = (None, argv[4])
user_id = client.get_account_id(username)
yaml_request_template = Template("""
{{ dataset }}: !!es.bsc.dataclay.util.management.datasetmgr.DataSet
dataClayID: {{ 11111111-00000000-00000000-00000000-00000000 }}
providerAccountID: {{ 00000000-00000000-00000000-00000000-00000000 }}
name: {{ dataset }}
{{ dataset }}datacontract: !!es.bsc.dataclay.util.management.datacontractmgr.DataContract
beginDate: 1980-01-01T00:00:01
endDate: 2055-12-31T23:59:58
providerAccountID: {{ 00000000-00000000-00000000-00000000-00000000 }}
providerDataSetID: {{ 11111111-00000000-00000000-00000000-00000000 }}
applicantsNames:
? {{ username }}
publicAvailable: True
""")
yaml_request = yaml_request_template.render(dataset=dataset,
username=username)
try:
client.perform_set_of_operations(user_id, credential, yaml_request)
except grpc.RpcError as e:
print("Tried to do that operation and received: %s" % e,
file=sys.stderr)
elif argv[1] == "getstubs":
# TODO: If this part is still used, check that contract_ids should be a list.
contract_ids = map(UUID, argv[2].split(','))
path = argv[3]
username = argv[4]
credential = (None, argv[5])
user_id = client.get_account_id(username)
prepare_storage(path)
babel_data = client.get_babel_stubs(user_id, credential, contract_ids)
with open(os.path.join(path, "babelstubs.yml"), 'wb') as f:
f.write(babel_data)
all_stubs = client.get_stubs(user_id, credential, LANG_PYTHON,
contract_ids)
for key, value in all_stubs.items():
with open(os.path.join(path, key), 'wb') as f:
f.write(value)
deploy_stubs(path)
else:
print("Unknown command.", file=sys.stderr)
print(USAGE_TEXT, file=sys.stderr)
return