def _make_new_dataset(dataset_dict):
#If the user is not allowed to use the existing dataset, a new
#one must be created. This means a unique hash must be created
#To create a unique hash, add a unique piece of metadata.
new_dataset = copy.deepcopy(dataset_dict)
new_dataset['metadata']['created_at'] = datetime.datetime.now()
new_hash = generate_data_hash(new_dataset)
new_dataset['data_hash'] = new_hash
return new_dataset
def _make_new_dataset(dataset_dict):
#If the user is not allowed to use the existing dataset, a new
#one must be created. This means a unique hash must be created
#To create a unique hash, add a unique piece of metadata.
new_dataset = copy.deepcopy(dataset_dict)
new_dataset['metadata']['created_at'] = datetime.datetime.now()
new_hash = generate_data_hash(new_dataset)
new_dataset['data_hash'] = new_hash
return new_dataset
def _process_incoming_data(data, user_id=None, source=None):
datasets = {}
for d in data:
val = d.parse_value()
if val is None:
log.info(
"Cannot parse data (dataset_id=%s). "
"Value not available.", d)
continue
data_dict = {
'data_type': d.type,
'data_name': d.name,
'data_units': d.unit,
'created_by': user_id,
'frequency': None,
'start_time': None,
}
# Assign dimension if necessary
if d.unit is not None and d.dimension in (None, 'dimensionless'):
data_dict['data_dimen'] = hydra_units.get_unit_dimension(d.unit)
else:
data_dict['data_dimen'] = d.dimension
db_val = _get_db_val(d.type, val)
data_dict['value'] = db_val
if d.metadata is not None:
if isinstance(d.metadata, str) or isinstance(d.metadata, unicode):
metadata_dict = json.loads(d.metadata)
else:
metadata_dict = d.metadata
else:
metadata_dict = {}
metadata_keys = [k.lower() for k in metadata_dict]
if user_id is not None and 'user_id' not in metadata_keys:
metadata_dict[u'user_id'] = unicode(user_id)
if source is not None and 'source' not in metadata_keys:
metadata_dict[u'source'] = unicode(source)
data_dict['metadata'] = metadata_dict
d.data_hash = generate_data_hash(data_dict)
data_dict['data_hash'] = d.data_hash
datasets[d.data_hash] = data_dict
return datasets
def _process_incoming_data(data, user_id=None, source=None):
datasets = {}
for d in data:
val = d.parse_value()
if val is None:
log.info("Cannot parse data (dataset_id=%s). "
"Value not available.",d)
continue
data_dict = {
'data_type':d.type,
'data_name':d.name,
'data_units': d.unit,
'created_by' : user_id,
'frequency' : None,
'start_time': None,
}
# Assign dimension if necessary
if d.unit is not None and d.dimension in (None, 'dimensionless'):
data_dict['data_dimen'] = hydra_units.get_unit_dimension(d.unit)
else:
data_dict['data_dimen'] = d.dimension
db_val = _get_db_val(d.type, val)
data_dict['value'] = db_val
if d.metadata is not None:
if isinstance(d.metadata, str) or isinstance(d.metadata, unicode):
metadata_dict = json.loads(d.metadata)
else:
metadata_dict=d.metadata
else:
metadata_dict={}
metadata_keys = [k.lower() for k in metadata_dict]
if user_id is not None and 'user_id' not in metadata_keys:
metadata_dict[u'user_id'] = unicode(user_id)
if source is not None and 'source' not in metadata_keys:
metadata_dict[u'source'] = unicode(source)
data_dict['metadata'] = metadata_dict
d.data_hash = generate_data_hash(data_dict)
data_dict['data_hash'] = d.data_hash
datasets[d.data_hash] = data_dict
return datasets
def get_hash(self, val, metadata):
if metadata is None:
metadata = self.get_metadata_as_dict()
if val is None:
value = self.parse_value()
else:
value = val
dataset_dict = {'data_name' : self.name,
'data_units': self.unit,
'data_dimen': self.dimension,
'data_type' : self.type.lower(),
'value' : value,
'metadata' : metadata,}
data_hash = generate_data_hash(dataset_dict)
return data_hash