def _deserialize_attribute(mainitem,
subitems,
sep,
original_class=None,
original_pk=None,
lesserrors=False):
"""Deserialize a single attribute.
:param mainitem: the main item (either the attribute itself for base
types (None, string, ...) or the main item for lists and dicts.
Must contain the 'key' key and also the following keys:
datatype, tval, fval, ival, bval, dval.
NOTE that a type check is not performed! tval is expected to be a string,
dval a date, etc.
:param subitems: must be a dictionary of dictionaries. In the top-level dictionary,
the key must be the key of the attribute, stripped of all prefixes
(i.e., if the mainitem has key 'a.b' and we pass subitems
'a.b.0', 'a.b.1', 'a.b.1.c', their keys must be '0', '1', '1.c').
It must be None if the value is not iterable (int, str,
float, ...).
It is an empty dictionary if there are no subitems.
:param sep: a string, the separator between subfields (to separate the
name of a dictionary from the keys it contains, for instance)
:param original_class: if these elements come from a specific subclass
of DbMultipleValueAttributeBaseClass, pass here the class (note: the class,
not the instance!). This is used only in case the wrong number of elements
is found in the raw data, to print a more meaningful message (if the class
has a dbnode associated to it)
:param original_pk: if the elements come from a specific subclass
of DbMultipleValueAttributeBaseClass that has a dbnode associated to it,
pass here the PK integer. This is used only in case the wrong number
of elements is found in the raw data, to print a more meaningful message
:param lesserrors: If set to True, in some cases where the content of the
DB is not consistent but data is still recoverable,
it will just log the message rather than raising
an exception (e.g. if the number of elements of a dictionary is different
from the number declared in the ival field).
:return: the deserialized value
:raise aiida.backends.djsite.db.migrations.DeserializationException: if an error occurs"""
from aiida.common import json
from aiida.common.timezone import (is_naive, make_aware,
get_current_timezone)
if mainitem['datatype'] in ['none', 'bool', 'int', 'float', 'txt']:
if subitems:
raise DeserializationException("'{}' is of a base type, "
'but has subitems!'.format(
mainitem.key))
return _deserialize_basic_type(mainitem)
if mainitem['datatype'] == 'date':
if subitems:
raise DeserializationException("'{}' is of a base type, "
'but has subitems!'.format(
mainitem.key))
if is_naive(mainitem['dval']):
return make_aware(mainitem['dval'], get_current_timezone())
return mainitem['dval']
if mainitem['datatype'] == 'list':
return deserialize_list(mainitem, subitems, sep, original_class,
original_pk, lesserrors)
if mainitem['datatype'] == 'dict':
return deserialize_dict(mainitem, subitems, sep, original_class,
original_pk, lesserrors)
if mainitem['datatype'] == 'json':
try:
return json.loads(mainitem['tval'])
except ValueError:
raise DeserializationException(
'Error in the content of the json field')
else:
raise DeserializationException(
"The type field '{}' is not recognized".format(
mainitem['datatype']))
def create_value(cls,
key,
value,
subspecifier_value=None,
other_attribs={}):
"""
Create a new list of attributes, without storing them, associated
with the current key/value pair (and to the given subspecifier,
e.g. the DbNode for DbAttributes and DbExtras).
:note: No hits are done on the DB, in particular no check is done
on the existence of the given nodes.
:param key: a string with the key to create (can contain the
separator cls._sep if this is a sub-attribute: indeed, this
function calls itself recursively)
:param value: the value to store (a basic data type or a list or a dict)
:param subspecifier_value: must be None if this class has no
subspecifier set (e.g., the DbSetting class).
Must be the value of the subspecifier (e.g., the dbnode) for classes
that define it (e.g. DbAttribute and DbExtra)
:param other_attribs: a dictionary of other parameters, to store
only on the level-zero attribute (e.g. for description in DbSetting).
:return: always a list of class instances; it is the user
responsibility to store such entries (typically with a Django
bulk_create() call).
"""
import datetime
from aiida.common import json
from aiida.common.timezone import is_naive, make_aware, get_current_timezone
if cls._subspecifier_field_name is None:
if subspecifier_value is not None:
raise ValueError('You cannot specify a subspecifier value for '
'class {} because it has no subspecifiers'
''.format(cls.__name__))
if issubclass(cls, DbAttributeFunctionality):
new_entry = db_attribute_base_model(key=key, **other_attribs)
else:
new_entry = db_extra_base_model(key=key, **other_attribs)
else:
if subspecifier_value is None:
raise ValueError(
'You also have to specify a subspecifier value '
'for class {} (the {})'.format(
cls.__name__, cls._subspecifier_field_name))
further_params = other_attribs.copy()
further_params.update(
{cls._subspecifier_field_name: subspecifier_value})
# new_entry = cls(key=key, **further_params)
if issubclass(cls, DbAttributeFunctionality):
new_entry = db_attribute_base_model(key=key, **further_params)
else:
new_entry = db_extra_base_model(key=key, **further_params)
list_to_return = [new_entry]
if value is None:
new_entry.datatype = 'none'
new_entry.bval = None
new_entry.tval = ''
new_entry.ival = None
new_entry.fval = None
new_entry.dval = None
elif isinstance(value, bool):
new_entry.datatype = 'bool'
new_entry.bval = value
new_entry.tval = ''
new_entry.ival = None
new_entry.fval = None
new_entry.dval = None
elif isinstance(value, six.integer_types):
new_entry.datatype = 'int'
new_entry.ival = value
new_entry.tval = ''
new_entry.bval = None
new_entry.fval = None
new_entry.dval = None
elif isinstance(value, float):
new_entry.datatype = 'float'
new_entry.fval = value
new_entry.tval = ''
new_entry.ival = None
new_entry.bval = None
new_entry.dval = None
elif isinstance(value, six.string_types):
new_entry.datatype = 'txt'
new_entry.tval = value
new_entry.bval = None
new_entry.ival = None
new_entry.fval = None
new_entry.dval = None
elif isinstance(value, datetime.datetime):
# current timezone is taken from the settings file of django
if is_naive(value):
value_to_set = make_aware(value, get_current_timezone())
else:
value_to_set = value
new_entry.datatype = 'date'
# TODO: time-aware and time-naive datetime objects, see
# https://docs.djangoproject.com/en/dev/topics/i18n/timezones/#naive-and-aware-datetime-objects
new_entry.dval = value_to_set
new_entry.tval = ''
new_entry.bval = None
new_entry.ival = None
new_entry.fval = None
elif isinstance(value, (list, tuple)):
new_entry.datatype = 'list'
new_entry.dval = None
new_entry.tval = ''
new_entry.bval = None
new_entry.ival = len(value)
new_entry.fval = None
for i, subv in enumerate(value):
# I do not need get_or_create here, because
# above I deleted all children (and I
# expect no concurrency)
# NOTE: I do not pass other_attribs
list_to_return.extend(
cls.create_value(key=('{}{}{:d}'.format(key, cls._sep, i)),
value=subv,
subspecifier_value=subspecifier_value))
elif isinstance(value, dict):
new_entry.datatype = 'dict'
new_entry.dval = None
new_entry.tval = ''
new_entry.bval = None
new_entry.ival = len(value)
new_entry.fval = None
for subk, subv in value.items():
cls.validate_key(subk)
# I do not need get_or_create here, because
# above I deleted all children (and I
# expect no concurrency)
# NOTE: I do not pass other_attribs
list_to_return.extend(
cls.create_value(key='{}{}{}'.format(key, cls._sep, subk),
value=subv,
subspecifier_value=subspecifier_value))
else:
try:
jsondata = json.dumps(value)
except TypeError:
raise ValueError(
'Unable to store the value: it must be either a basic datatype, or json-serializable: {}'
.format(value))
new_entry.datatype = 'json'
new_entry.tval = jsondata
new_entry.bval = None
new_entry.ival = None
new_entry.fval = None
return list_to_return
def _deserialize_attribute(mainitem,
subitems,
sep,
original_class=None,
original_pk=None,
lesserrors=False):
"""
Deserialize a single attribute.
:param mainitem: the main item (either the attribute itself for base
types (None, string, ...) or the main item for lists and dicts.
Must contain the 'key' key and also the following keys:
datatype, tval, fval, ival, bval, dval.
NOTE that a type check is not performed! tval is expected to be a string,
dval a date, etc.
:param subitems: must be a dictionary of dictionaries. In the top-level dictionary,
the key must be the key of the attribute, stripped of all prefixes
(i.e., if the mainitem has key 'a.b' and we pass subitems
'a.b.0', 'a.b.1', 'a.b.1.c', their keys must be '0', '1', '1.c').
It must be None if the value is not iterable (int, str,
float, ...).
It is an empty dictionary if there are no subitems.
:param sep: a string, the separator between subfields (to separate the
name of a dictionary from the keys it contains, for instance)
:param original_class: if these elements come from a specific subclass
of DbMultipleValueAttributeBaseClass, pass here the class (note: the class,
not the instance!). This is used only in case the wrong number of elements
is found in the raw data, to print a more meaningful message (if the class
has a dbnode associated to it)
:param original_pk: if the elements come from a specific subclass
of DbMultipleValueAttributeBaseClass that has a dbnode associated to it,
pass here the PK integer. This is used only in case the wrong number
of elements is found in the raw data, to print a more meaningful message
:param lesserrors: If set to True, in some cases where the content of the
DB is not consistent but data is still recoverable,
it will just log the message rather than raising
an exception (e.g. if the number of elements of a dictionary is different
from the number declared in the ival field).
:return: the deserialized value
:raise aiida.backends.djsite.db.migrations.DeserializationException: if an error occurs
"""
from aiida.common import json
from aiida.common.timezone import (is_naive, make_aware,
get_current_timezone)
from aiida.common import AIIDA_LOGGER
if mainitem['datatype'] == 'none':
if subitems:
raise DeserializationException("'{}' is of a base type, "
'but has subitems!'.format(
mainitem.key))
return None
elif mainitem['datatype'] == 'bool':
if subitems:
raise DeserializationException("'{}' is of a base type, "
'but has subitems!'.format(
mainitem.key))
return mainitem['bval']
elif mainitem['datatype'] == 'int':
if subitems:
raise DeserializationException("'{}' is of a base type, "
'but has subitems!'.format(
mainitem.key))
return mainitem['ival']
elif mainitem['datatype'] == 'float':
if subitems:
raise DeserializationException("'{}' is of a base type, "
'but has subitems!'.format(
mainitem.key))
return mainitem['fval']
elif mainitem['datatype'] == 'txt':
if subitems:
raise DeserializationException("'{}' is of a base type, "
'but has subitems!'.format(
mainitem.key))
return mainitem['tval']
elif mainitem['datatype'] == 'date':
if subitems:
raise DeserializationException("'{}' is of a base type, "
'but has subitems!'.format(
mainitem.key))
if is_naive(mainitem['dval']):
return make_aware(mainitem['dval'], get_current_timezone())
else:
return mainitem['dval']
elif mainitem['datatype'] == 'list':
# subitems contains all subitems, here I store only those of
# deepness 1, i.e. if I have subitems '0', '1' and '1.c' I
# store only '0' and '1'
firstlevelsubdict = {k: v for k, v in subitems.items() if sep not in k}
# For checking, I verify the expected values
expected_set = set(['{:d}'.format(i) for i in range(mainitem['ival'])])
received_set = set(firstlevelsubdict.keys())
# If there are more entries than expected, but all expected
# ones are there, I just issue an error but I do not stop.
if not expected_set.issubset(received_set):
if (original_class is not None
and original_class._subspecifier_field_name is not None):
subspecifier_string = '{}={} and '.format(
original_class._subspecifier_field_name, original_pk)
else:
subspecifier_string = ''
if original_class is None:
sourcestr = 'the data passed'
else:
sourcestr = original_class.__name__
raise DeserializationException(
'Wrong list elements stored in {} for '
"{}key='{}' ({} vs {})".format(sourcestr, subspecifier_string,
mainitem['key'], expected_set,
received_set))
if expected_set != received_set:
if (original_class is not None
and original_class._subspecifier_field_name is not None):
subspecifier_string = '{}={} and '.format(
original_class._subspecifier_field_name, original_pk)
else:
subspecifier_string = ''
if original_class is None:
sourcestr = 'the data passed'
else:
sourcestr = original_class.__name__
msg = ('Wrong list elements stored in {} for '
"{}key='{}' ({} vs {})".format(sourcestr,
subspecifier_string,
mainitem['key'],
expected_set, received_set))
if lesserrors:
AIIDA_LOGGER.error(msg)
else:
raise DeserializationException(msg)
# I get the values in memory as a dictionary
tempdict = {}
for firstsubk, firstsubv in firstlevelsubdict.items():
# I call recursively the same function to get subitems
newsubitems = {
k[len(firstsubk) + len(sep):]: v
for k, v in subitems.items() if k.startswith(firstsubk + sep)
}
tempdict[firstsubk] = _deserialize_attribute(
mainitem=firstsubv,
subitems=newsubitems,
sep=sep,
original_class=original_class,
original_pk=original_pk)
# And then I put them in a list
retlist = [tempdict['{:d}'.format(i)] for i in range(mainitem['ival'])]
return retlist
elif mainitem['datatype'] == 'dict':
# subitems contains all subitems, here I store only those of
# deepness 1, i.e. if I have subitems '0', '1' and '1.c' I
# store only '0' and '1'
firstlevelsubdict = {k: v for k, v in subitems.items() if sep not in k}
if len(firstlevelsubdict) != mainitem['ival']:
if (original_class is not None
and original_class._subspecifier_field_name is not None):
subspecifier_string = '{}={} and '.format(
original_class._subspecifier_field_name, original_pk)
else:
subspecifier_string = ''
if original_class is None:
sourcestr = 'the data passed'
else:
sourcestr = original_class.__name__
msg = ('Wrong dict length stored in {} for '
"{}key='{}' ({} vs {})".format(sourcestr,
subspecifier_string,
mainitem['key'],
len(firstlevelsubdict),
mainitem['ival']))
if lesserrors:
AIIDA_LOGGER.error(msg)
else:
raise DeserializationException(msg)
# I get the values in memory as a dictionary
tempdict = {}
for firstsubk, firstsubv in firstlevelsubdict.items():
# I call recursively the same function to get subitems
newsubitems = {
k[len(firstsubk) + len(sep):]: v
for k, v in subitems.items() if k.startswith(firstsubk + sep)
}
tempdict[firstsubk] = _deserialize_attribute(
mainitem=firstsubv,
subitems=newsubitems,
sep=sep,
original_class=original_class,
original_pk=original_pk)
return tempdict
elif mainitem['datatype'] == 'json':
try:
return json.loads(mainitem['tval'])
except ValueError:
raise DeserializationException(
'Error in the content of the json field')
else:
raise DeserializationException(
"The type field '{}' is not recognized".format(
mainitem['datatype']))