def session_detach(self):
"""
Detach object from session, i.e. remove reference from current session provided to current object,
'dear garbage-collector, the current session is not using this object anymore'
"""
getRuntime().detach_object_from_session(self.get_object_id(),
self.get_hint())
def _post_stealing(self, stolen_object):
"""Once an object has been stolen, perform the movement."""
# FIXME: We should not assume that this is server-side
from dataclay.commonruntime.Runtime import getRuntime
from dataclay.commonruntime.Settings import settings
getRuntime().move_object(stolen_object, settings.storage_id)
return stolen_object
def __get__(self, obj, type_=None):
"""Getter for the dataClay property
If the object is loaded, perform the getter to the local instance (this
is the scenario for local instances and Execution Environment fully
loaded instances).
If the object is not loaded, perform a remote execution (this is the
scenario for client remote instances and also Execution Environment
non-loaded instances, which may either "not-yet-loaded" or remote)
"""
is_exec_env = getRuntime().is_exec_env()
logger.debug(
"Calling getter for property %s in %s", self.p_name,
"an execution environment" if is_exec_env else "the client")
if (is_exec_env and obj.is_loaded()) or (not is_exec_env
and not obj.is_persistent()):
try:
obj.set_dirty(
True
) # set dirty = true for language types like lists, dicts, that are get and modified. TODO: improve this.
return object.__getattribute__(
obj, "%s%s" % (DCLAY_PROPERTY_PREFIX, self.p_name))
except AttributeError:
logger.warning(
"Received AttributeError while accessing property %s on object %r",
self.p_name, obj)
logger.debug("Internal dictionary of the object: %s",
obj.__dict__)
raise
else:
return getRuntime().execute_implementation_aux(
DCLAY_GETTER_PREFIX + self.p_name, obj, (), obj.get_hint())
def make_persistent(self, alias=None, backend_id=None, recursive=True):
if alias == "":
raise AttributeError('Alias cannot be empty')
getRuntime().make_persistent(self,
alias=alias,
backend_id=backend_id,
recursive=recursive)
def __set__(self, obj, value):
"""Setter for the dataClay property
See the __get__ method for the basic behavioural explanation.
"""
logger.debug("Calling replicated setter for property %s", self.p_name)
is_client = not getRuntime().is_exec_env()
if is_client and not obj.is_persistent():
object.__setattr__(obj,
"%s%s" % (DCLAY_PROPERTY_PREFIX, self.p_name),
value)
elif not is_client and not obj.is_loaded():
getRuntime().execute_implementation_aux(
DCLAY_SETTER_PREFIX + self.p_name, obj, (value, ),
obj.get_hint())
else:
if self.inMaster:
logger.debug(
"Calling update in master [%s] for property %s with value %s",
obj.get_master_location, self.p_name, value)
getRuntime().execute_implementation_aux(
'__setUpdate__', obj,
(obj, self.p_name, value, self.beforeUpdate,
self.afterUpdate), obj.get_master_location())
else:
logger.debug(
"Calling update locally for property %s with value %s",
self.p_name, value)
obj.__setUpdate__(obj, self.p_name, value, self.beforeUpdate,
self.afterUpdate)
obj.set_dirty(True)
def dc_put(self, alias, backend_id=None, recursive=True):
if alias is None or alias is "":
raise AttributeError('Alias cannot be null or empty')
getRuntime().make_persistent(self,
alias=alias,
backend_id=backend_id,
recursive=recursive)
def split(self, split_class=None):
# TODO: this could be improved, library could cache stuff, or a joint call could be added
from dataclay.commonruntime.Runtime import getRuntime
location_chunks = sorted((
# Overly complex way to get a single ExecutionEnvironmentID
# works in both Python2 & Python3
next(
iter(((getRuntime().get_all_locations(ch.get_object_id())
).keys()))),
ch) for ch in self.get_chunks())
result = list()
# FIXME: once support for imports has been implemented and tested, clean this up
from itertools import groupby
from operator import itemgetter
from dataclay.contrib import splitting
if split_class is None:
split_class = splitting.GenericSplit
elif isinstance(split_class, basestring):
split_class = getattr(splitting, split_class)
else:
raise NotImplementedError(
"I could not understand %s (of type %s)" %
(split_class, type(split_class)))
unused_exec_envs = set(
getRuntime().get_execution_environments_info().keys())
# FIXME: **********************************************************************************
# FIXME: not using real split_class due to registration issues, but should be done that way
# FIXME: **********************************************************************************
for loc, chunks in groupby(location_chunks, key=itemgetter(0)):
# Previous code using split intelligently:
# split_object = split_class(map(itemgetter(1), chunks), loc)
# split_object.make_persistent(backend_id=loc)
# unused_exec_envs.remove(loc)
# result.append(split_object)
# Hacked around code:
result.append(map(itemgetter(1), chunks))
# Remove this
return result
# FIXME This is useful for WorkStealingSplit, but not for the others...
for ee in unused_exec_envs:
split_object = split_class(list(), ee)
split_object.make_persistent(backend_id=ee)
result.append(split_object)
for split_object in result:
# Note that this result is implicitly copied given that split is persistent
# (maybe do the copy explicit? even when it is useless?)
split_object.split_brothers = result
return result
def initialize_object(self, new_object_id=None):
"""
@postcondition: Initialize the object
@param new_object_id: Object Id of the object
"""
if new_object_id is not None: # TODO: remove this if once ExecutionGateway is not initializing object id twice
self.set_object_id(new_object_id)
getRuntime().add_to_heap(self)
def dc_update(self, from_object):
"""
@postcondition: Updates all fields of this object with the values of the specified object
@param from_object: instance from which values must be retrieved to set fields of current object
"""
if from_object is None:
return
else:
getRuntime().update_object(self, from_object)
def federate(self, ext_dataclay_id, recursive=True):
"""
@postcondition: Federates this object with an external dataClay instance
@param ext_dataclay_id: id of the external dataClay instance
@param recursive: Indicates if all sub-objects must be federated as well.
"""
getRuntime().federate_object(self.get_object_id(), ext_dataclay_id,
recursive,
self.get_class_extradata().class_id,
self.get_hint())
def _update_object_id(self, new_object_id):
"""
@postcondition: Set a new object id for the object and
calls an update of its heap references
@param new_object_id: object id
"""
if self.is_persistent():
raise DataClayException(
"Cannot change the id of a persistent object")
old_object_id = self.get_object_id()
self.__dclay_instance_extradata.object_id = new_object_id
getRuntime().update_object_id(old_object_id, new_object_id)
def unfederate(self, ext_dataclay_id=None, recursive=True):
"""
@postcondition: Unfederate this object with an external dataClay instance
@param ext_dataclay_id: id of the external dataClay instance (none means to unfederate with all dcs)
@param recursive: Indicates if all sub-objects must be unfederated as well.
"""
if ext_dataclay_id is not None:
getRuntime().unfederate_object(self.get_object_id(),
ext_dataclay_id, recursive)
else:
getRuntime().unfederate_object_with_all_dcs(
self.get_object_id(), recursive)
def initialize_object_as_volatile(self):
"""
@postcondition: Initialize object with state 'volatile' with proper flags. Usually, volatile state is created by a stub, app, exec
class,.. See same function in DataClayExecutionObject for a different initialization. This design is intended to be
clear with object state.
"""
# TODO: improve this using an specialization (dgasull)
if getRuntime().is_exec_env():
# *** Execution Environment flags
self.set_persistent(True) # All objects in the EE are persistent
self.set_loaded(True)
self.set_pending_to_register(True)
self.set_hint(getRuntime().get_hint())
self.set_owner_session_id(getRuntime().get_session_id())
def _dclayMethod(f, self, *args, **kwargs):
"""Helper function for DataClayObject method decoration"""
logger.verbose("Calling function %s", f.__name__)
is_exec_env = getRuntime().is_exec_env()
try:
if (is_exec_env and self.is_loaded()) \
or (not is_exec_env and not self.is_persistent())\
or f._dclay_local:
return f(self, *args, **kwargs)
else:
return getRuntime().execute_implementation_aux(
f.__name__, self, args, self.get_hint())
except Exception:
traceback.print_exc()
raise
def getByID(object_strid):
"""Get a Persistent Object from its OID.
:param object_strid: The string identifying object (contains both ObjectID and hint)
:return: The (Persistent) DataClayObject
"""
try:
object_id, hint, class_id = object_strid.split(":")
ret = getRuntime().get_object_by_id(uuid.UUID(object_id),
class_id=uuid.UUID(class_id),
hint=uuid.UUID(hint))
except ValueError: # this can fail for both [not enough semicolons]|[invalid uuid]
# Fallback behaviour: no extra fields, the whole string is the ObjectID UUID
object_id = object_strid
ret = getRuntime().get_object_by_id(uuid.UUID(object_id))
return ret
def test(self):
"""Test. note that all test method names must begin with 'test.'"""
"""WARNING: IT IS HIGHLY RECOMMENDED TO HAVE ONE TEST ONLY TO ISOLATE FUNCTIONAL TESTS FROM EACH OTHER. i.e.
Start a new Python Interpreter and JVM for each test. In the end, it means only one test in this class. """
from dataclay.api import init
logger.debug('**Starting init**')
init()
"""
Imports. Imports must be located here in order to simulate "import" order in a real scenario.
VERY IMPORTANT: Imports must be located AFTER init
"""
from model.classes import Person
from dataclay.commonruntime.Runtime import getRuntime
self.session_initialized = True
"""
Test. From now on, the Functional Test itself.
"""
p = Person('foo', 100)
execution_environments = list(getRuntime().get_execution_environments_info().keys())
self.assertTrue(len(execution_environments) > 1)
p.make_persistent(backend_id=execution_environments[0])
p.new_replica(backend_id=execution_environments[1])
self.assertEqual(p.run_remote(execution_environments[0], 'getMyMasterLocation', None), execution_environments[0])
self.assertEqual(p.run_remote(execution_environments[1], 'getMyMasterLocation', None), execution_environments[0])
logger.debug("Test OK!")
def get_federation_targets(self):
""" Retrieve dataClay instances ids where the object is federated
:return: dataClay instances ids where this object is federated
:rtype: set of UUID
"""
return getRuntime().get_dataclays_object_is_federated_with(
self.get_object_id())
def get_federation_source(self):
""" Retrieve dataClay instance id where the object comes from or NULL
:return: dataClay instance ids where this object is federated
:rtype: UUID
"""
return getRuntime().get_external_source_of_dataclay_object(
self.get_object_id())
def _dclayEmptyMethod(f, self, *args, **kwargs):
"""Similar to dclayMethod, but without actual Python implementation."""
logger.verbose("Calling (languageless) function %s", f.__name__)
# Let it fail elsewhere, if the user hacks around into an invalid state
# (like a loaded&local non-persistent instance with an dclayEmptyMethod,
# something that should not happen normally)
return getRuntime().execute_implementation_aux(f.__name__, self, args, self.get_hint())
def get_external_dataclay_info(self, dataclay_id):
""" Get external dataClay information
:param dataclay_id: external dataClay ID
:return: DataClayInstance information
:type dataclay_id: UUID
:rtype: DataClayInstance
"""
return getRuntime().get_external_dataclay_info(dataclay_id)
def __set__(self, obj, value):
"""Setter for the dataClay property
See the __get__ method for the basic behavioural explanation.
"""
logger.debug("Calling setter for property %s", self.p_name)
is_exec_env = getRuntime().is_exec_env()
if (is_exec_env and obj.is_loaded()) or (not is_exec_env
and not obj.is_persistent()):
object.__setattr__(obj,
"%s%s" % (DCLAY_PROPERTY_PREFIX, self.p_name),
value)
if is_exec_env:
obj.set_dirty(True)
else:
getRuntime().execute_implementation_aux(
DCLAY_SETTER_PREFIX + self.p_name, obj, (value, ),
obj.get_hint())
def serialize_reference_counting(self, dc_obj, io_file):
""" TODO: IMPORTANT: this should be removed in new serialization by using paddings to directly access reference counters inside
metadata. """
"""
@postcondition: Serialize reference counting (garbage collector information)
@param dc_obj: dc object with ref counting
@param io_file: Buffer in which to serialize
@param reference_counting: Reference counting to serialize
"""
self.external_references = 0
if dc_obj.get_alias() is not None and dc_obj.get_alias() != "":
logger.trace("Found alias reference")
self.external_references = self.external_references + 1
cur_dataclay_id = getRuntime().get_dataclay_id()
if dc_obj.get_replica_locations() is not None and len(
dc_obj.get_replica_locations()) != 0:
for replica_loc in dc_obj.get_replica_locations():
replica_dataclay_id = getRuntime(
).get_execution_environment_info(
replica_loc).dataclay_instance_id
if replica_dataclay_id != cur_dataclay_id:
logger.trace("Found federation reference")
self.external_references = self.external_references + 1
break
logger.trace(
f"Serializing reference counting external references = {self.external_references}"
)
IntegerWrapper().write(io_file, self.external_references)
IntegerWrapper().write(io_file, len(self.reference_counting))
for location, ref_counting_in_loc in self.reference_counting.items():
if location is None:
BooleanWrapper().write(io_file, True)
else:
BooleanWrapper().write(io_file, False)
StringWrapper().write(io_file, str(location))
IntegerWrapper().write(io_file, len(ref_counting_in_loc))
for oid, counter in ref_counting_in_loc.items():
StringWrapper().write(io_file, str(oid))
IntegerWrapper().write(io_file, counter)
def load_metaclass_info(metaclass_id):
"""Load the namespace and class name for a certain MetaClassID.
:param metaclass_id: The dataClay UUID of the MetaClass.
:return: A tuple (class_name, namespace).
"""
try:
return cached_metaclass_info[metaclass_id]
except KeyError:
class_name, namespace = getRuntime(
).ready_clients["@LM"].get_classname_and_namespace_for_ds(metaclass_id)
cached_metaclass_info[metaclass_id] = (class_name, namespace)
return class_name, namespace
def test(self):
"""Test. note that all test method names must begin with 'test.'"""
"""WARNING: IT IS HIGHLY RECOMMENDED TO HAVE ONE TEST ONLY TO ISOLATE FUNCTIONAL TESTS FROM EACH OTHER. i.e.
Start a new Python Interpreter and JVM for each test. In the end, it means only one test in this class. """
from dataclay.api import init
logger.debug('**Starting init**')
init()
"""
Imports. Imports must be located here in order to simulate "import" order in a real scenario.
VERY IMPORTANT: Imports must be located AFTER init
"""
from model.classes import Person
from dataclay.DataClayObjProperties import DCLAY_GETTER_PREFIX
from dataclay.commonruntime.Runtime import getRuntime
self.session_initialized = True
"""
Test. From now on, the Functional Test itself.
"""
p = Person('foo', 100)
execution_environments = list(
getRuntime().get_execution_environments_info().keys())
self.assertTrue(len(execution_environments) > 1)
p.make_persistent(backend_id=execution_environments[0])
p.new_replica(backend_id=execution_environments[1])
# Name is a replicated attribute so the after method should be called after the setter
p.name = 'aaa'
# Assert that the attribute 'name' was properly changed in both dataservices
self.assertEqual(
p.run_remote(execution_environments[0],
DCLAY_GETTER_PREFIX + 'name', None), 'aaa')
self.assertEqual(
p.run_remote(execution_environments[1],
DCLAY_GETTER_PREFIX + 'name', None), 'aaa')
# Assert that the attribute 'years' was changed only in one dataservice
p.years = 1000
years0 = p.run_remote(execution_environments[0],
DCLAY_GETTER_PREFIX + 'years', None)
years1 = p.run_remote(execution_environments[1],
DCLAY_GETTER_PREFIX + 'years', None)
self.assertEqual(abs(years0 - years1), 900)
logger.debug("Test OK!")
def initialize_object_as_persistent(self):
"""
@postcondition: object is initialized as a persistent object.
Flags for "persistent" state might be different in EE and client.
"""
# TODO: improve this using an specialization (dgasull)
if getRuntime().is_exec_env():
# *** Execution Environment flags
self.set_persistent(True)
# by default, loaded = true for volatiles created inside executions
# this function (initialize as persistent) is used for objects being
# deserialized and therefore they might be unloaded
# same happens for pending to register flag.
self.set_loaded(False)
self.set_pending_to_register(False)
else:
# *** Client flags
self.set_persistent(True)
def track_local_available_classes():
"""Track the available classes into the commonruntime.local_available_classes.
Note that no deployment is done in this function: the deployment should be
done beforehand through the deploy_stubs function.
This function returns all the contracts that have been found.
"""
babel_data = load_babel_data()
contracts = set()
for class_data in babel_data:
contracts.update(class_data.contracts)
namespace = class_data.namespace
full_name = class_data.className
getRuntime().local_available_classes[class_data.classID] = \
"%s.%s" % (namespace, full_name)
logger.verbose("Using the following contracts: %s", contracts)
return contracts
def ass_client(self):
self.client = getRuntime().ready_clients["@STORAGE"]
def deserialize(self, io_file, iface_bitmaps, metadata,
cur_deserialized_python_objs):
"""Reciprocal to serialize."""
logger.verbose("Deserializing object %s", str(self.get_object_id()))
# Put slow debugging info inside here:
#
# NOTE: new implementation of ExecutionGateway assert is not needed and wrong
# if logger.isEnabledFor(DEBUG):
# klass = self.__class__
# logger.debug("Deserializing instance %r from class %s",
# self, klass.__name__)
# logger.debug("The previous class is from module %s, in file %s",
# klass.__module__, inspect.getfile(klass))
# logger.debug("The class extradata is:\n%s", klass._dclay_class_extradata)
# assert klass._dclay_class_extradata == self._dclay_class_extradata
#
# LOADED FLAG = TRUE only once deserialization is finished to avoid concurrent problems!
# # This may be due to race conditions. It may need to do some extra locking
# if self.__dclay_instance_extradata.loaded_flag:
# logger.debug("Loaded Flag is True")
# else:
# self.__dclay_instance_extradata.loaded_flag = True
""" reference counting """
""" discard padding """
IntegerWrapper().read(io_file)
""" deserialize master_location """
des_master_loc_str = StringWrapper().read(io_file)
if des_master_loc_str == "x":
self.__dclay_instance_extradata.master_location = None
else:
self.__dclay_instance_extradata.master_location = UUID(
des_master_loc_str)
if hasattr(self, "__setstate__"):
# The object has a user-defined deserialization method.
# Use pickle, and use that method instead
if six.PY2:
import cPickle as pickle
elif six.PY3:
import _pickle as pickle
state = pickle.loads(StringWrapper(mode="binary").read(io_file))
self.__setstate__(state)
else:
# Regular dataClay provided deserialization
# Start by getting the properties
properties = sorted(self.get_class_extradata().properties.values(),
key=attrgetter('position'))
logger.trace("Tell io_file before loop: %s", io_file.tell())
logger.verbose("Deserializing list of properties: %s", properties)
for p in properties:
logger.trace("Tell io_file in loop: %s", io_file.tell())
not_null = BooleanWrapper().read(io_file)
value = None
if not_null:
logger.debug("Not null property %s", p.name)
if isinstance(p.type, UserType):
try:
logger.debug("Property %s is an association",
p.name)
value = DeserializationLibUtilsSingleton.deserialize_association(
io_file, iface_bitmaps, metadata,
cur_deserialized_python_objs, getRuntime())
except KeyError as e:
logger.error('Failed to deserialize association',
exc_info=True)
else:
try:
upck = Unpickler(io_file)
upck.persistent_load = PersistentLoadPicklerHelper(
metadata, cur_deserialized_python_objs,
getRuntime())
value = upck.load()
except:
traceback.print_exc()
logger.debug("Setting value %s for property %s", value, p.name)
object.__setattr__(self,
"%s%s" % (DCLAY_PROPERTY_PREFIX, p.name),
value)
""" reference counting bytes here """
""" TODO: discard bytes? """
def set_in_dataclay_instance(self, dc_info, field_name, params):
from dataclay.DataClayObjProperties import DCLAY_SETTER_PREFIX
getRuntime().synchronize_federated(self, params,
DCLAY_SETTER_PREFIX + field_name,
dc_info)
def set_in_backend(self, backend_id, field_name, value):
from dataclay.DataClayObjProperties import DCLAY_SETTER_PREFIX
return getRuntime().run_remote(self.get_object_id(), backend_id,
DCLAY_SETTER_PREFIX + field_name, value)