def get_resource_attributes(ref_key, ref_id, type_id=None, **kwargs):
"""
Get all the resource attributes for a given resource.
If type_id is specified, only
return the resource attributes within the type.
"""
user_id = kwargs.get('user_id')
resource_attr_qry = DBSession.query(ResourceAttr).filter(
ResourceAttr.ref_key == ref_key,
or_(
ResourceAttr.network_id==ref_id,
ResourceAttr.node_id==ref_id,
ResourceAttr.link_id==ref_id,
ResourceAttr.group_id==ref_id
))
if type_id is not None:
attr_ids = []
rs = DBSession.query(TypeAttr).filter(TypeAttr.type_id==type_id).all()
for r in rs:
attr_ids.append(r.attr_id)
resource_attr_qry = resource_attr_qry.filter(ResourceAttr.attr_id.in_(attr_ids))
resource_attrs = resource_attr_qry.all()
return resource_attrs
def add_resource_attribute(resource_type, resource_id, attr_id, is_var,**kwargs):
"""
Add a resource attribute attribute to a resource.
attr_is_var indicates whether the attribute is a variable or not --
this is used in simulation to indicate that this value is expected
to be filled in by the simulator.
"""
attr = DBSession.query(Attr).filter(Attr.attr_id==attr_id).first()
if attr is None:
raise ResourceNotFoundError("Attribute with ID %s does not exist."%attr_id)
resource_i = _get_resource(resource_type, resource_id)
for ra in resource_i.attributes:
if ra.attr_id == attr_id:
raise HydraError("Duplicate attribute. %s %s already has attribute %s"
%(resource_type, resource_i.get_name(), attr.attr_name))
attr_is_var = 'Y' if is_var else 'N'
new_ra = resource_i.add_attribute(attr_id, attr_is_var)
DBSession.flush()
return new_ra
def add_attribute(attr,**kwargs):
"""
Add a generic attribute, which can then be used in creating
a resource attribute, and put into a type.
.. code-block:: python
(Attr){
id = 1020
name = "Test Attr"
dimen = "very big"
}
"""
log.debug("Adding attribute: %s", attr.name)
if attr.dimen is None or attr.dimen.lower() == 'dimensionless':
log.info("Setting 'dimesionless' on attribute %s", attr.name)
attr.dimen = 'dimensionless'
try:
attr_i = DBSession.query(Attr).filter(Attr.attr_name == attr.name,
Attr.attr_dimen == attr.dimen).one()
log.info("Attr already exists")
except NoResultFound:
attr_i = Attr(attr_name = attr.name, attr_dimen = attr.dimen)
attr_i.attr_description = attr.description
DBSession.add(attr_i)
DBSession.flush()
log.info("New attr added")
return attr_i
def update_attribute(attr,**kwargs):
"""
Add a generic attribute, which can then be used in creating
a resource attribute, and put into a type.
.. code-block:: python
(Attr){
id = 1020
name = "Test Attr"
dimen = "very big"
}
"""
if attr.dimen is None or attr.dimen.lower() == 'dimensionless':
log.info("Setting 'dimesionless' on attribute %s", attr.name)
attr.dimen = 'dimensionless'
log.debug("Adding attribute: %s", attr.name)
attr_i = _get_attr(Attr.attr_id)
attr_i.attr_name = attr.name
attr_i.attr_dimen = attr.dimension
attr_i.attr_description = attr.description
#Make sure an update hasn't caused an inconsistency.
check_attr_dimension(attr_i.attr_id)
DBSession.flush()
return attr_i
def bulk_update_resourcedata(scenario_ids, resource_scenarios,**kwargs):
"""
Update the data associated with a list of scenarios.
"""
user_id = kwargs.get('user_id')
res = None
res = {}
net_ids = DBSession.query(Scenario.network_id).filter(Scenario.scenario_id.in_(scenario_ids)).all()
if len(set(net_ids)) != 1:
raise HydraError("Scenario IDS are not in the same network")
for scenario_id in scenario_ids:
_check_can_edit_scenario(scenario_id, kwargs['user_id'])
scen_i = _get_scenario(scenario_id, False, False)
res[scenario_id] = []
for rs in resource_scenarios:
if rs.value is not None:
updated_rs = _update_resourcescenario(scen_i, rs, user_id=user_id, source=kwargs.get('app_name'))
res[scenario_id].append(updated_rs)
else:
_delete_resourcescenario(scenario_id, rs)
DBSession.flush()
return res
def add_project(project, **kwargs):
"""
Add a new project
returns a project complexmodel
"""
user_id = kwargs.get('user_id')
# check_perm(user_id, 'add_project')
proj_i = Project()
proj_i.project_name = project.name
proj_i.project_description = project.description
proj_i.created_by = user_id
if project.layout is not None:
proj_i.layout = project.get_layout()
attr_map = add_attributes(proj_i, project.attributes)
proj_data = _add_project_attribute_data(proj_i, attr_map,
project.attribute_data)
proj_i.attribute_data = proj_data
proj_i.set_owner(user_id)
DBSession.add(proj_i)
DBSession.flush()
return proj_i
def delete_resourcegroupitem(item_id,**kwargs):
group_item_i = _get_item(item_id)
scenario._check_can_edit_scenario(group_item_i.scenario_id, kwargs['user_id'])
DBSession.delete(group_item_i)
DBSession.flush()
return 'OK'
def get_attribute_data(attr_ids, node_ids, **kwargs):
"""
For a given attribute or set of attributes, return all the resources and
resource scenarios in the network
"""
node_attrs = DBSession.query(ResourceAttr).\
options(joinedload_all('attr')).\
filter(ResourceAttr.node_id.in_(node_ids),
ResourceAttr.attr_id.in_(attr_ids)).all()
ra_ids = []
for ra in node_attrs:
ra_ids.append(ra.resource_attr_id)
resource_scenarios = DBSession.query(ResourceScenario).filter(ResourceScenario.resource_attr_id.in_(ra_ids)).options(joinedload('resourceattr')).options(joinedload_all('dataset.metadata')).order_by(ResourceScenario.scenario_id).all()
for rs in resource_scenarios:
if rs.dataset.hidden == 'Y':
try:
rs.dataset.check_read_permission(kwargs.get('user_id'))
except:
rs.dataset.value = None
rs.dataset.frequency = None
rs.dataset.start_time = None
DBSession.expunge(rs)
return node_attrs, resource_scenarios
def get_resource_attributes(ref_key, ref_id, type_id=None, **kwargs):
"""
Get all the resource attributes for a given resource.
If type_id is specified, only
return the resource attributes within the type.
"""
user_id = kwargs.get('user_id')
resource_attr_qry = DBSession.query(ResourceAttr).filter(
ResourceAttr.ref_key == ref_key,
or_(ResourceAttr.network_id == ref_id, ResourceAttr.node_id == ref_id,
ResourceAttr.link_id == ref_id, ResourceAttr.group_id == ref_id))
if type_id is not None:
attr_ids = []
rs = DBSession.query(TypeAttr).filter(
TypeAttr.type_id == type_id).all()
for r in rs:
attr_ids.append(r.attr_id)
resource_attr_qry = resource_attr_qry.filter(
ResourceAttr.attr_id.in_(attr_ids))
resource_attrs = resource_attr_qry.all()
return resource_attrs
def _get_existing_data(hashes):
str_hashes = [str(h) for h in hashes]
hash_dict = {}
datasets = []
if len(str_hashes) > qry_in_threshold:
idx = 0
extent =qry_in_threshold
while idx < len(str_hashes):
log.info("Querying %s datasets", len(str_hashes[idx:extent]))
rs = DBSession.query(Dataset).filter(Dataset.data_hash.in_(str_hashes[idx:extent])).all()
datasets.extend(rs)
idx = idx + qry_in_threshold
if idx + qry_in_threshold > len(str_hashes):
extent = len(str_hashes)
else:
extent = extent + qry_in_threshold
else:
datasets = DBSession.query(Dataset).filter(Dataset.data_hash.in_(str_hashes))
for r in datasets:
hash_dict[r.data_hash] = r
log.info("Retrieved %s datasets", len(hash_dict))
return hash_dict
def update_resourcedata(scenario_id, resource_scenarios,**kwargs):
"""
Update the data associated with a scenario.
Data missing from the resource scenario will not be removed
from the scenario. Use the remove_resourcedata for this task.
If the resource scenario does not exist, it will be created.
If the value of the resource scenario is specified as being None, the
resource scenario will be deleted.
If the value of the resource scenario does not exist, it will be created.
If the both the resource scenario and value already exist, the resource scenario
will be updated with the ID of the dataset.
If the dataset being set is being changed, already exists,
and is only used by a single resource scenario,
then the dataset itself is updated, rather than a new one being created.
"""
user_id = kwargs.get('user_id')
res = None
_check_can_edit_scenario(scenario_id, kwargs['user_id'])
scen_i = _get_scenario(scenario_id, False, False)
res = []
for rs in resource_scenarios:
if rs.value is not None:
updated_rs = _update_resourcescenario(scen_i, rs, user_id=user_id, source=kwargs.get('app_name'))
res.append(updated_rs)
else:
_delete_resourcescenario(scenario_id, rs)
DBSession.flush()
return res
def set_network_permission(network_id, usernames, read, write, share,
**kwargs):
"""
Set permissions on a network to a list of users, identifed by
their usernames. The read flag ('Y' or 'N') sets read access, the write
flag sets write access. If the read flag is 'N', then there is
automatically no write access or share access.
"""
user_id = kwargs.get('user_id')
net_i = _get_network(network_id)
#Check if the user is allowed to share this network.
net_i.check_share_permission(user_id)
#You cannot edit something you cannot see.
if read == 'N':
write = 'N'
share = 'N'
for username in usernames:
user_i = _get_user(username)
#The creator of a network must always have read and write access
#to their project
if net_i.created_by == user_i.user_id:
raise HydraError("Cannot set permissions on network %s"
" for user %s as tis user is the creator." %
(network_id, username))
net_i.set_owner(user_i.user_id, read=read, write=write, share=share)
DBSession.flush()
def _get_datasets(dataset_ids):
"""
Get all the datasets in a list of dataset IDS. This must be done in chunks of 999,
as sqlite can only handle 'in' with < 1000 elements.
"""
dataset_dict = {}
datasets = []
if len(dataset_ids) > qry_in_threshold:
idx = 0
extent =qry_in_threshold
while idx < len(dataset_ids):
log.info("Querying %s datasets", len(dataset_ids[idx:extent]))
rs = DBSession.query(Dataset).filter(Dataset.dataset_id.in_(dataset_ids[idx:extent])).all()
datasets.extend(rs)
idx = idx + qry_in_threshold
if idx + qry_in_threshold > len(dataset_ids):
extent = len(dataset_ids)
else:
extent = extent + qry_in_threshold
else:
datasets = DBSession.query(Dataset).filter(Dataset.dataset_id.in_(dataset_ids))
for r in datasets:
dataset_dict[r.dataset_id] = r
log.info("Retrieved %s datasets", len(dataset_dict))
return dataset_dict
def _get_existing_data(hashes):
str_hashes = [str(h) for h in hashes]
hash_dict = {}
datasets = []
if len(str_hashes) > qry_in_threshold:
idx = 0
extent = qry_in_threshold
while idx < len(str_hashes):
log.info("Querying %s datasets", len(str_hashes[idx:extent]))
rs = DBSession.query(Dataset).filter(
Dataset.data_hash.in_(str_hashes[idx:extent])).all()
datasets.extend(rs)
idx = idx + qry_in_threshold
if idx + qry_in_threshold > len(str_hashes):
extent = len(str_hashes)
else:
extent = extent + qry_in_threshold
else:
datasets = DBSession.query(Dataset).filter(
Dataset.data_hash.in_(str_hashes))
for r in datasets:
hash_dict[r.data_hash] = r
log.info("Retrieved %s datasets", len(hash_dict))
return hash_dict
def _get_datasets(dataset_ids):
"""
Get all the datasets in a list of dataset IDS. This must be done in chunks of 999,
as sqlite can only handle 'in' with < 1000 elements.
"""
dataset_dict = {}
datasets = []
if len(dataset_ids) > qry_in_threshold:
idx = 0
extent = qry_in_threshold
while idx < len(dataset_ids):
log.info("Querying %s datasets", len(dataset_ids[idx:extent]))
rs = DBSession.query(Dataset).filter(
Dataset.dataset_id.in_(dataset_ids[idx:extent])).all()
datasets.extend(rs)
idx = idx + qry_in_threshold
if idx + qry_in_threshold > len(dataset_ids):
extent = len(dataset_ids)
else:
extent = extent + qry_in_threshold
else:
datasets = DBSession.query(Dataset).filter(
Dataset.dataset_id.in_(dataset_ids))
for r in datasets:
dataset_dict[r.dataset_id] = r
log.info("Retrieved %s datasets", len(dataset_dict))
return dataset_dict
def set_rs_dataset(resource_attr_id, scenario_id, dataset_id, **kwargs):
rs = DBSession.query(ResourceScenario).filter(
ResourceScenario.resource_attr_id == resource_attr_id,
ResourceScenario.scenario_id == scenario_id).first()
if rs is None:
raise ResourceNotFoundError(
"Resource scenario for resource attr %s not found in scenario %s" %
(resource_attr_id, scenario_id))
dataset = DBSession.query(Dataset).filter(
Dataset.dataset_id == dataset_id).first()
if dataset is None:
raise ResourceNotFoundError("Dataset %s not found" % (dataset_id, ))
rs.dataset_id = dataset_id
DBSession.flush()
rs = DBSession.query(ResourceScenario).filter(
ResourceScenario.resource_attr_id == resource_attr_id,
ResourceScenario.scenario_id == scenario_id).first()
return rs
def _get_metadata(dataset_ids):
"""
Get all the metadata for a given list of datasets
"""
metadata = []
if len(dataset_ids) == 0:
return []
if len(dataset_ids) > qry_in_threshold:
idx = 0
extent = qry_in_threshold
while idx < len(dataset_ids):
log.info("Querying %s metadatas", len(dataset_ids[idx:extent]))
rs = DBSession.query(Metadata).filter(
Metadata.dataset_id.in_(dataset_ids[idx:extent])).all()
metadata.extend(rs)
idx = idx + qry_in_threshold
if idx + qry_in_threshold > len(dataset_ids):
extent = len(dataset_ids)
else:
extent = extent + qry_in_threshold
else:
metadata_qry = DBSession.query(Metadata).filter(
Metadata.dataset_id.in_(dataset_ids))
for m in metadata_qry:
metadata.append(m)
return metadata
def _get_group(group_id):
try:
DBSession.query(ResourceGroup).filter(
ResourceGroup.group_id == group_id).one()
except NoResultFound:
raise ResourceNotFoundError("ResourceGroup %s not found" %
(group_id, ))
def get_scenario_data(scenario_id, **kwargs):
"""
Get all the datasets from the group with the specified name
@returns a list of dictionaries
"""
user_id = kwargs.get('user_id')
scenario_data = DBSession.query(Dataset).filter(
Dataset.dataset_id == ResourceScenario.dataset_id,
ResourceScenario.scenario_id == scenario_id).options(
joinedload_all('metadata')).distinct().all()
for sd in scenario_data:
if sd.hidden == 'Y':
try:
sd.check_read_permission(user_id)
except:
sd.value = None
sd.frequency = None
sd.start_time = None
sd.metadata = []
DBSession.expunge_all()
log.info("Retrieved %s datasets", len(scenario_data))
return scenario_data
def get_attribute_data(attr_ids, node_ids, **kwargs):
"""
For a given attribute or set of attributes, return all the resources and
resource scenarios in the network
"""
node_attrs = DBSession.query(ResourceAttr).\
options(joinedload_all('attr')).\
filter(ResourceAttr.node_id.in_(node_ids),
ResourceAttr.attr_id.in_(attr_ids)).all()
ra_ids = []
for ra in node_attrs:
ra_ids.append(ra.resource_attr_id)
resource_scenarios = DBSession.query(ResourceScenario).filter(
ResourceScenario.resource_attr_id.in_(ra_ids)).options(
joinedload('resourceattr')).options(
joinedload_all('dataset.metadata')).order_by(
ResourceScenario.scenario_id).all()
for rs in resource_scenarios:
if rs.dataset.hidden == 'Y':
try:
rs.dataset.check_read_permission(kwargs.get('user_id'))
except:
rs.dataset.value = None
rs.dataset.frequency = None
rs.dataset.start_time = None
DBSession.expunge(rs)
return node_attrs, resource_scenarios
def get_resourcescenarios(resource_attr_ids, scenario_ids, **kwargs):
"""
Retrieve all the datasets in a scenario for a given attribute.
Also return the resource attributes so there is a reference to the node/link
"""
#Make sure the resource_attr_ids are valid
check_ra_qry = DBSession.query(ResourceAttr).filter(
ResourceAttr.resource_attr_id.in_(resource_attr_ids)).all()
if len(check_ra_qry) != len(resource_attr_ids):
raise HydraError(
"Unrecognised resource attribues %s were found in list" %
(resource_attr_ids, ))
#Make sure the scenario ids are valid
scen_qry = DBSession.query(Scenario).filter(
Scenario.scenario_id.in_(scenario_ids)).all()
if len(scen_qry) != len(scenario_ids):
raise HydraError(
"Unrecognised resource attribues %s were found in list" %
(scenario_ids, ))
rs_result = DBSession.query(ResourceScenario).filter(
ResourceScenario.scenario_id.in_(scenario_ids),
ResourceScenario.resource_attr_id.in_(resource_attr_ids)).all()
return rs_result
def bulk_update_resourcedata(scenario_ids, resource_scenarios, **kwargs):
"""
Update the data associated with a list of scenarios.
"""
user_id = kwargs.get('user_id')
res = None
res = {}
net_ids = DBSession.query(Scenario.network_id).filter(
Scenario.scenario_id.in_(scenario_ids)).all()
if len(set(net_ids)) != 1:
raise HydraError("Scenario IDS are not in the same network")
for scenario_id in scenario_ids:
_check_can_edit_scenario(scenario_id, kwargs['user_id'])
scen_i = _get_scenario(scenario_id, False, False)
res[scenario_id] = []
for rs in resource_scenarios:
if rs.value is not None:
updated_rs = _update_resourcescenario(
scen_i, rs, user_id=user_id, source=kwargs.get('app_name'))
res[scenario_id].append(updated_rs)
else:
_delete_resourcescenario(scenario_id, rs)
DBSession.flush()
return res
def hide_dataset(dataset_id, exceptions, read, write, share, **kwargs):
"""
Hide a particular piece of data so it can only be seen by its owner.
Only an owner can hide (and unhide) data.
Data with no owner cannot be hidden.
The exceptions paramater lists the usernames of those with permission to view the data
read, write and share indicate whether these users can read, edit and share this data.
"""
user_id = kwargs.get('user_id')
dataset_i = _get_dataset(dataset_id)
#check that I can hide the dataset
if dataset_i.created_by != int(user_id):
raise HydraError('Permission denied. '
'User %s is not the owner of dataset %s' %
(user_id, dataset_i.data_name))
dataset_i.hidden = 'Y'
if exceptions is not None:
for username in exceptions:
user_i = _get_user(username)
dataset_i.set_owner(user_i.user_id,
read=read,
write=write,
share=share)
DBSession.flush()
def _add_resourcegroupitem(group_item, scenario_id):
"""
Add a single resource group item (no DB flush, as it's an internal function)
"""
if group_item.id and group_item.id > 0:
try:
group_item_i = DBSession.query(ResourceGroupItem).filter(
ResourceGroupItem.item_id == group_item.id).one()
except NoResultFound:
raise ResourceNotFoundError("ResourceGroupItem %s not found" %
(group_item.id))
else:
group_item_i = ResourceGroupItem()
group_item_i.group_id = group_item.group_id
if scenario_id is not None:
group_item_i.scenario_id = scenario_id
ref_key = group_item.ref_key
group_item_i.ref_key = ref_key
if ref_key == 'NODE':
group_item_i.node_id = group_item.ref_id
elif ref_key == 'LINK':
group_item_i.link_id = group_item.ref_id
elif ref_key == 'GROUP':
group_item_i.subgroup_id = group_item.ref_id
DBSession.add(group_item_i)
return group_item_i
def set_network_permission(network_id, usernames, read, write, share,**kwargs):
"""
Set permissions on a network to a list of users, identifed by
their usernames. The read flag ('Y' or 'N') sets read access, the write
flag sets write access. If the read flag is 'N', then there is
automatically no write access or share access.
"""
user_id = kwargs.get('user_id')
net_i = _get_network(network_id)
#Check if the user is allowed to share this network.
net_i.check_share_permission(user_id)
#You cannot edit something you cannot see.
if read == 'N':
write = 'N'
share = 'N'
for username in usernames:
user_i = _get_user(username)
#The creator of a network must always have read and write access
#to their project
if net_i.created_by == user_i.user_id:
raise HydraError("Cannot set permissions on network %s"
" for user %s as tis user is the creator." %
(network_id, username))
net_i.set_owner(user_i.user_id, read=read, write=write, share=share)
DBSession.flush()
def add_attribute(attr, **kwargs):
"""
Add a generic attribute, which can then be used in creating
a resource attribute, and put into a type.
.. code-block:: python
(Attr){
id = 1020
name = "Test Attr"
dimen = "very big"
}
"""
log.debug("Adding attribute: %s", attr.name)
if attr.dimen is None or attr.dimen.lower() == 'dimensionless':
log.info("Setting 'dimesionless' on attribute %s", attr.name)
attr.dimen = 'dimensionless'
try:
attr_i = DBSession.query(Attr).filter(
Attr.attr_name == attr.name, Attr.attr_dimen == attr.dimen).one()
log.info("Attr already exists")
except NoResultFound:
attr_i = Attr(attr_name=attr.name, attr_dimen=attr.dimen)
attr_i.attr_description = attr.description
DBSession.add(attr_i)
DBSession.flush()
log.info("New attr added")
return attr_i
def _add_resourcegroupitem(group_item, scenario_id):
"""
Add a single resource group item (no DB flush, as it's an internal function)
"""
if group_item.id and group_item.id > 0:
try:
group_item_i = DBSession.query(ResourceGroupItem).filter(ResourceGroupItem.item_id == group_item.id).one()
except NoResultFound:
raise ResourceNotFoundError("ResourceGroupItem %s not found" % (group_item.id))
else:
group_item_i = ResourceGroupItem()
group_item_i.group_id = group_item.group_id
if scenario_id is not None:
group_item_i.scenario_id = scenario_id
ref_key = group_item.ref_key
group_item_i.ref_key = ref_key
if ref_key == 'NODE':
group_item_i.node_id =group_item.ref_id
elif ref_key == 'LINK':
group_item_i.link_id =group_item.ref_id
elif ref_key == 'GROUP':
group_item_i.subgroup_id =group_item.ref_id
DBSession.add(group_item_i)
return group_item_i
def update_attribute(attr, **kwargs):
"""
Add a generic attribute, which can then be used in creating
a resource attribute, and put into a type.
.. code-block:: python
(Attr){
id = 1020
name = "Test Attr"
dimen = "very big"
}
"""
if attr.dimen is None or attr.dimen.lower() == 'dimensionless':
log.info("Setting 'dimesionless' on attribute %s", attr.name)
attr.dimen = 'dimensionless'
log.debug("Adding attribute: %s", attr.name)
attr_i = _get_attr(Attr.attr_id)
attr_i.attr_name = attr.name
attr_i.attr_dimen = attr.dimension
attr_i.attr_description = attr.description
#Make sure an update hasn't caused an inconsistency.
check_attr_dimension(attr_i.attr_id)
DBSession.flush()
return attr_i
def delete_rule(rule_id, **kwargs):
rule_i = _get_rule(rule_id)
rule_i.status = 'X'
DBSession.flush()
def add_resource_attribute(resource_type, resource_id, attr_id, is_var,
**kwargs):
"""
Add a resource attribute attribute to a resource.
attr_is_var indicates whether the attribute is a variable or not --
this is used in simulation to indicate that this value is expected
to be filled in by the simulator.
"""
attr = DBSession.query(Attr).filter(Attr.attr_id == attr_id).first()
if attr is None:
raise ResourceNotFoundError("Attribute with ID %s does not exist." %
attr_id)
resource_i = _get_resource(resource_type, resource_id)
for ra in resource_i.attributes:
if ra.attr_id == attr_id:
raise HydraError(
"Duplicate attribute. %s %s already has attribute %s" %
(resource_type, resource_i.get_name(), attr.attr_name))
attr_is_var = 'Y' if is_var else 'N'
new_ra = resource_i.add_attribute(attr_id, attr_is_var)
DBSession.flush()
return new_ra
def _get_metadata(dataset_ids):
"""
Get all the metadata for a given list of datasets
"""
metadata = []
if len(dataset_ids) == 0:
return []
if len(dataset_ids) > qry_in_threshold:
idx = 0
extent = qry_in_threshold
while idx < len(dataset_ids):
log.info("Querying %s metadatas", len(dataset_ids[idx:extent]))
rs = DBSession.query(Metadata).filter(Metadata.dataset_id.in_(dataset_ids[idx:extent])).all()
metadata.extend(rs)
idx = idx + qry_in_threshold
if idx + qry_in_threshold > len(dataset_ids):
extent = len(dataset_ids)
else:
extent = extent +qry_in_threshold
else:
metadata_qry = DBSession.query(Metadata).filter(Metadata.dataset_id.in_(dataset_ids))
for m in metadata_qry:
metadata.append(m)
return metadata
def update_rule(rule, **kwargs):
rule_i = _get_rule(rule.id)
if rule.ref_key != rule_i.ref_key:
raise HydraError("Cannot convert a %s rule to a %s rule. Please create a new rule instead."%(rule_i.ref_key, rule.ref_key))
if rule.ref_key == 'NETWORK':
rule_i.network_id = rule.ref_id
elif rule.ref_key == 'NODE':
rule_i.node_id = rule.ref_id
elif rule.ref_key == 'LINK':
rule_i.link_id = rule.ref_id
elif rule.ref_key == 'GROUP':
rule_i.group_id = rule.group_id
else:
raise HydraError("Ref Key %s not recognised.")
rule_i.scenario_id = rule.scenario_id
rule_i.rule_name = rule.name
rule_i.rule_description = rule.description
rule_i.rule_text = rule.text
DBSession.flush()
return rule_i
def assign_value(rs, data_type, val,
units, name, dimension, metadata={}, data_hash=None, user_id=None, source=None):
"""
Insert or update a piece of data in a scenario.
If the dataset is being shared by other resource scenarios, a new dataset is inserted.
If the dataset is ONLY being used by the resource scenario in question, the dataset
is updated to avoid unnecessary duplication.
"""
log.debug("Assigning value %s to rs %s in scenario %s",
name, rs.resource_attr_id, rs.scenario_id)
if rs.scenario.locked == 'Y':
raise PermissionError("Cannot assign value. Scenario %s is locked"
%(rs.scenario_id))
#Check if this RS is the only RS in the DB connected to this dataset.
#If no results is found, the RS isn't in the DB yet, so the condition is false.
update_dataset = False # Default behaviour is to create a new dataset.
if rs.dataset is not None:
#Has this dataset changed?
if rs.dataset.data_hash == data_hash:
log.debug("Dataset has not changed. Returning.")
return
connected_rs = DBSession.query(ResourceScenario).filter(ResourceScenario.dataset_id==rs.dataset.dataset_id).all()
#If there's no RS found, then the incoming rs is new, so the dataset can be altered
#without fear of affecting something else.
if len(connected_rs) == 0:
#If it's 1, the RS exists in the DB, but it's the only one using this dataset or
#The RS isn't in the DB yet and the datset is being used by 1 other RS.
update_dataset = True
if len(connected_rs) == 1 :
if connected_rs[0].scenario_id == rs.scenario_id and connected_rs[0].resource_attr_id==rs.resource_attr_id:
update_dataset = True
else:
update_dataset=False
if update_dataset is True:
log.info("Updating dataset '%s'", name)
dataset = data.update_dataset(rs.dataset.dataset_id, name, data_type, val, units, dimension, metadata, **dict(user_id=user_id))
rs.dataset = dataset
rs.dataset_id = dataset.dataset_id
else:
log.info("Creating new dataset %s in scenario %s", name, rs.scenario_id)
dataset = data.add_dataset(data_type,
val,
units,
dimension,
metadata=metadata,
name=name,
**dict(user_id=user_id))
rs.dataset = dataset
rs.source = source
DBSession.flush()
def delete_resourcegroupitem(item_id, **kwargs):
group_item_i = _get_item(item_id)
scenario._check_can_edit_scenario(group_item_i.scenario_id,
kwargs['user_id'])
DBSession.delete(group_item_i)
DBSession.flush()
return 'OK'
def _delete_resourcescenario(scenario_id, resource_scenario):
ra_id = resource_scenario.resource_attr_id
try:
sd_i = DBSession.query(ResourceScenario).filter(ResourceScenario.scenario_id==scenario_id, ResourceScenario.resource_attr_id==ra_id).one()
except NoResultFound:
raise HydraError("ResourceAttr %s does not exist in scenario %s."%(ra_id, scenario_id))
DBSession.delete(sd_i)
def update_dataset(dataset_id, name, data_type, val, units, dimension, metadata={}, **kwargs):
"""
Update an existing dataset
"""
if dataset_id is None:
raise HydraError("Dataset must have an ID to be updated.")
user_id = kwargs.get('user_id')
dataset = DBSession.query(Dataset).filter(Dataset.dataset_id==dataset_id).one()
#This dataset been seen before, so it may be attached
#to other scenarios, which may be locked. If they are locked, we must
#not change their data, so new data must be created for the unlocked scenarios
locked_scenarios = []
unlocked_scenarios = []
for dataset_rs in dataset.resourcescenarios:
if dataset_rs.scenario.locked == 'Y':
locked_scenarios.append(dataset_rs)
else:
unlocked_scenarios.append(dataset_rs)
#Are any of these scenarios locked?
if len(locked_scenarios) > 0:
#If so, create a new dataset and assign to all unlocked datasets.
dataset = add_dataset(data_type,
val,
units,
dimension,
metadata=metadata,
name=name,
user_id=kwargs['user_id'])
for unlocked_rs in unlocked_scenarios:
unlocked_rs.dataset = dataset
else:
dataset.set_val(data_type, val)
dataset.set_metadata(metadata)
dataset.data_type = data_type
dataset.data_units = units
dataset.data_name = name
dataset.data_dimen = dimension
dataset.created_by = kwargs['user_id']
dataset.data_hash = dataset.set_hash()
#Is there a dataset in the DB already which is identical to the updated dataset?
existing_dataset = DBSession.query(Dataset).filter(Dataset.data_hash==dataset.data_hash, Dataset.dataset_id != dataset.dataset_id).first()
if existing_dataset is not None and existing_dataset.check_user(user_id):
log.warn("An identical dataset %s has been found to dataset %s."
" Deleting dataset and returning dataset %s",
existing_dataset.dataset_id, dataset.dataset_id, existing_dataset.dataset_id)
DBSession.delete(dataset)
dataset = existing_dataset
return dataset
def update_dataset(dataset_id, name, data_type, val, units, dimension, metadata={}, **kwargs):
"""
Update an existing dataset
"""
if dataset_id is None:
raise HydraError("Dataset must have an ID to be updated.")
user_id = kwargs.get('user_id')
dataset = DBSession.query(Dataset).filter(Dataset.dataset_id==dataset_id).one()
#This dataset been seen before, so it may be attached
#to other scenarios, which may be locked. If they are locked, we must
#not change their data, so new data must be created for the unlocked scenarios
locked_scenarios = []
unlocked_scenarios = []
for dataset_rs in dataset.resourcescenarios:
if dataset_rs.scenario.locked == 'Y':
locked_scenarios.append(dataset_rs)
else:
unlocked_scenarios.append(dataset_rs)
#Are any of these scenarios locked?
if len(locked_scenarios) > 0:
#If so, create a new dataset and assign to all unlocked datasets.
dataset = add_dataset(data_type,
val,
units,
dimension,
metadata=metadata,
name=name,
user_id=kwargs['user_id'])
for unlocked_rs in unlocked_scenarios:
unlocked_rs.dataset = dataset
else:
dataset.set_val(data_type, val)
dataset.set_metadata(metadata)
dataset.data_type = data_type
dataset.data_units = units
dataset.data_name = name
dataset.data_dimen = dimension
dataset.created_by = kwargs['user_id']
dataset.data_hash = dataset.set_hash()
#Is there a dataset in the DB already which is identical to the updated dataset?
existing_dataset = DBSession.query(Dataset).filter(Dataset.data_hash==dataset.data_hash, Dataset.dataset_id != dataset.dataset_id).first()
if existing_dataset is not None and existing_dataset.check_user(user_id):
log.warn("An identical dataset %s has been found to dataset %s."
" Deleting dataset and returning dataset %s",
existing_dataset.dataset_id, dataset.dataset_id, existing_dataset.dataset_id)
DBSession.delete(dataset)
dataset = existing_dataset
return dataset
def bulk_insert_data(data, **kwargs):
datasets = _bulk_insert_data(data, user_id=kwargs.get('user_id'), source=kwargs.get('app_name'))
#This line exists to make the DBSession 'dirty',
#thereby telling it to flush the bulk insert.
datasets[0].data_name = datasets[0].data_name
DBSession.flush()
return datasets
def purge_note(note_id, **kwargs):
"""
Remove a note from the DB permenantly
"""
note_i = _get_note(note_id)
DBSession.delete(note_i)
DBSession.flush()
def bulk_insert_data(data, **kwargs):
datasets = _bulk_insert_data(data, user_id=kwargs.get('user_id'), source=kwargs.get('app_name'))
#This line exists to make the DBSession 'dirty',
#thereby telling it to flush the bulk insert.
datasets[0].data_name = datasets[0].data_name
DBSession.flush()
return datasets
def delete_dataset_collection(collection_id,**kwargs):
try:
collection = DBSession.query(DatasetCollection).filter(DatasetCollection.collection_id==collection_id).one()
except NoResultFound:
raise ResourceNotFoundError("No dataset collection found with id %s"%collection_id)
DBSession.delete(collection)
DBSession.flush()
def add_perm(perm,**kwargs):
"""
"""
#check_perm(kwargs.get('user_id'), 'add_perm')
perm_i = Perm(perm_name=perm.name, perm_code=perm.code)
DBSession.add(perm_i)
DBSession.flush()
return perm_i
def delete_resourcegroup(group_id, **kwargs):
"""
Add a new group to a scenario.
"""
group_i = _get_group(group_id)
#This should cascaded to delete all the group items.
DBSession.delete(group_i)
return 'OK'
def delete_dataset_collection(collection_id,**kwargs):
try:
collection = DBSession.query(DatasetCollection).filter(DatasetCollection.collection_id==collection_id).one()
except NoResultFound:
raise ResourceNotFoundError("No dataset collection found with id %s"%collection_id)
DBSession.delete(collection)
DBSession.flush()
def add_perm(perm,**kwargs):
"""
"""
check_perm(kwargs.get('user_id'), 'add_perm')
perm_i = Perm(perm_name=perm.name, perm_code=perm.code)
DBSession.add(perm_i)
DBSession.flush()
return perm_i
def add_role(role,**kwargs):
"""
"""
check_perm(kwargs.get('user_id'), 'add_role')
role_i = Role(role_name=role.name, role_code=role.code)
DBSession.add(role_i)
DBSession.flush()
return role_i
def add_role(role,**kwargs):
"""
"""
#check_perm(kwargs.get('user_id'), 'add_role')
role_i = Role(role_name=role.name, role_code=role.code)
DBSession.add(role_i)
DBSession.flush()
return role_i
def delete_resourcegroup(group_id,**kwargs):
"""
Add a new group to a scenario.
"""
group_i = _get_group(group_id)
#This should cascaded to delete all the group items.
DBSession.delete(group_i)
return 'OK'
def purge_scenario(scenario_id, **kwargs):
"""
Set the status of a scenario.
"""
_check_can_edit_scenario(scenario_id, kwargs['user_id'])
scenario_i = _get_scenario(scenario_id, False, False)
DBSession.delete(scenario_i)
DBSession.flush()
return 'OK'
def set_role_perm(role_id, perm_id,**kwargs):
check_perm(kwargs.get('user_id'), 'edit_perm')
_get_perm(perm_id)
_get_role(role_id)
roleperm_i = RolePerm(role_id=role_id, perm_id=perm_id)
DBSession.add(roleperm_i)
DBSession.flush()
return roleperm_i.role
def _delete_resourcescenario(scenario_id, resource_scenario):
ra_id = resource_scenario.resource_attr_id
try:
sd_i = DBSession.query(ResourceScenario).filter(
ResourceScenario.scenario_id == scenario_id,
ResourceScenario.resource_attr_id == ra_id).one()
except NoResultFound:
raise HydraError("ResourceAttr %s does not exist in scenario %s." %
(ra_id, scenario_id))
DBSession.delete(sd_i)
def delete_project(project_id,**kwargs):
"""
Set the status of a project to 'X'
"""
user_id = kwargs.get('user_id')
#check_perm(user_id, 'delete_project')
project = _get_project(project_id)
project.check_write_permission(user_id)
DBSession.delete(project)
DBSession.flush()
def add_dataset_collection(collection,**kwargs):
coln_i = DatasetCollection(collection_name=collection.name)
for dataset_id in collection.dataset_ids:
datasetitem = DatasetCollectionItem(dataset_id=dataset_id)
coln_i.items.append(datasetitem)
DBSession.add(coln_i)
DBSession.flush()
return coln_i
def delete_project(project_id, **kwargs):
"""
Set the status of a project to 'X'
"""
user_id = kwargs.get('user_id')
# check_perm(user_id, 'delete_project')
project = _get_project(project_id)
project.check_write_permission(user_id)
DBSession.delete(project)
DBSession.flush()
def purge_scenario(scenario_id, **kwargs):
"""
Set the status of a scenario.
"""
_check_can_edit_scenario(scenario_id, kwargs['user_id'])
scenario_i = _get_scenario(scenario_id, False, False)
DBSession.delete(scenario_i)
DBSession.flush()
return 'OK'
def delete_role(role_id,**kwargs):
"""
"""
check_perm(kwargs.get('user_id'), 'edit_role')
try:
role_i = DBSession.query(Role).filter(Role.role_id==role_id).one()
DBSession.delete(role_i)
except InvalidRequestError:
raise ResourceNotFoundError("Role (role_id=%s) does not exist"%(role_id))
return 'OK'
def delete_resourcegroupitems(scenario_id, item_ids, **kwargs):
"""
Delete specified items in a group, in a scenario.
"""
user_id = int(kwargs.get('user_id'))
scenario = _get_scenario(scenario_id, include_data=False, include_items=False)
_check_network_ownership(scenario.network_id, user_id)
for item_id in item_ids:
rgi = DBSession.query(ResourceGroupItem).\
filter(ResourceGroupItem.item_id==item_id).one()
DBSession.delete(rgi)
def delete_perm(perm_id,**kwargs):
"""
"""
check_perm(kwargs.get('user_id'), 'edit_perm')
try:
perm_i = DBSession.query(Perm).filter(Perm.perm_id==perm_id).one()
DBSession.delete(perm_i)
except InvalidRequestError:
raise ResourceNotFoundError("Permission (id=%s) does not exist"%(perm_id))
return 'OK'
