def _validate_type_hid(self, field, value):
"""
General validation for inserting devices (the name of the function is forced by Cerberus, not a good one).
- Tries to create hid and validates it.
- If hid cannot be created:
- If it has a parent, ensures that the device is unique.
- If it has not a parent, validates that the device has an user provided _id.
"""
from ereuse_devicehub.resources.device.component.domain import ComponentDomain
from ereuse_devicehub.resources.device.exceptions import DeviceNotFound
try:
self.document['hid'] = Naming.url_word(self.document['manufacturer']) + \
'-' + Naming.url_word(self.document['serialNumber']) + \
'-' + Naming.url_word(self.document['model'])
except KeyError:
del self.document['hid']
self.document['isUidSecured'] = False
if '_id' not in self.document: # We do not validate here the unique constraint of _id
if 'parent' in self.document:
with suppress(KeyError, DeviceNotFound):
component = ComponentDomain.get_similar_component(self.document, self.document['parent'])
self._error('model', json_util.dumps({'NotUnique': component}))
else:
# If device has no parent and no hid, user needs to: or provide _id or forcing creating it
if 'forceCreation' not in self.document or not self.document['forceCreation']:
self._error('_id', json_util.dumps({'NeedsId': self.document}))
# else: user forces us to create the device, it will be assigned an _id
# else: user provided _id. We accept this, however is unsecured.
else:
self._validate_regex(HID_REGEX, field, self.document['hid'])
self._validate_unique(True, field, self.document['hid'])
def try_prefix(original_type, prefix, supposed_resource_name):
"""
:param original_type: The resource type
:param prefix:
:param supposed_resource_name: What should the resource name be?
"""
type_name = Naming.new_type(original_type, prefix)
equal = original_type if prefix is None else '{}{}{}'.format(
prefix, Naming.TYPE_PREFIX, original_type)
assert_that(type_name).is_equal_to(equal)
resource_name = Naming.resource(type_name)
equal = supposed_resource_name if prefix is None else '{}{}{}'.format(
prefix, Naming.RESOURCE_PREFIX, supposed_resource_name)
assert_that(resource_name).is_equal_to(equal)
def try_prefix(original_type, prefix, supposed_resource_name):
"""
:param original_type: The resource type
:param prefix:
:param supposed_resource_name: What should the resource name be?
"""
type_name = Naming.new_type(original_type, prefix)
equal = original_type if prefix is None else "{}{}{}".format(prefix, Naming.TYPE_PREFIX, original_type)
assert_that(type_name).is_equal_to(equal)
resource_name = Naming.resource(type_name)
equal = (
supposed_resource_name
if prefix is None
else "{}{}{}".format(prefix, Naming.RESOURCE_PREFIX, supposed_resource_name)
)
assert_that(resource_name).is_equal_to(equal)
def delete_events_in_device(resource_name: str, device: dict):
"""
Deletes the references of the given device in all the events, and deletes the full event if it references only to
the device.
"""
if resource_name in Device.resource_types:
_id = device['_id']
qin = {'$in': [_id]}
query = {'$or': [{'device': _id}, {'devices': qin}, {'components': qin}], '@type': {'$ne': 'devices:Register'}}
sort = {'_created': pymongo.ASCENDING} # Order is important to find the first Snapshot (see below)
first_snapshot_found = False
for event in DeviceEventDomain.get({'$query': query, '$orderby': sort}):
if not first_snapshot_found and event['@type'] == 'devices:Snapshot':
# We cannot delete the Snapshot that created the device, because there is a change to create
# an infinite loop: Snapshot that created device -> Register -> DEL /device -> Snapshot that created...
first_snapshot_found = True
else:
event_resource = Naming.resource(event['@type'])
if event.get('device', None) == _id: # Am I the 'device' of the event?
execute_delete(event_resource, event['_id'])
elif [_id] == event.get('devices', []): # Is there no more 'devices' in the event, apart from me?
execute_delete(event_resource, event['_id'])
# All events that do not use 'components' for materialization (aka Add/Remove) should be erased
# if there are no more components
elif event['@type'] in ('devices:Add', 'devices:Remove') and event['components'] == [_id]:
execute_delete(event_resource, event['_id'])
else: # Keep the event; just delete my reference
DeviceEventDomain.update_raw(event['_id'], {'$pull': {'devices': qin, 'components': qin}})
def create_migrate(migrates: list):
"""
Manages the creation of a Migrate event, like doing so for the devices.
Heavily inspired by the hook 'on_insert_snapshot', uses the same idea on a group of devices.
"""
all_events = [] # We will delete all events if exception
try:
for migrate in migrates:
if 'from' in migrate:
setattr(g, MIGRATE_RETURNED_SAME_AS, dict())
devices_id = []
migrate['components'] = []
for device in migrate['devices']:
if device['@type'] in Component.types:
raise SchemaError('devices', 'You cannot directly migrate components.')
creator = MigrateCreator(device, device.pop('components'))
events = creator.execute()
all_events += events
migrate['events'] = [new_events['_id'] for new_events in events]
devices_id.append(device['_id'])
migrate['components'] += [component['_id'] for component in creator.components]
migrate['unsecured'] = creator.unsecured
getattr(g, MIGRATE_RETURNED_SAME_AS).update(creator.returned_same_as)
from ereuse_devicehub.resources.hooks import set_date
set_date(None, migrates)
migrate['devices'] = devices_id
except Exception as e:
for event in all_events:
# Could result in 404 (ex: delete an 'Add' after deleting 'Register' of the same device)
execute_delete(Naming.resource(event['@type']), event['_id'])
raise e
def _execute_register(device: dict, created: str, log: list):
"""
Tries to POST the device and updates the `device` dict with the resource from the database; if the device could
not be uploaded the `device` param will contain the database version of the device, not the inputting one. This is
because the majority of the information of a device is immutable (in concrete the fields used to compute
the ETAG).
:param device: Inputting device. It is replaced (keeping the reference) with the db version.
:param created: Set the _created value to be the same for the device as for the register
:param log: A log where to append the resulting device if execute_register has been successful
:raise InnerRequestError: any internal error in the POST that is not about the device already existing.
"""
device['hid'] = 'dummy'
new = True
try:
if created:
device['created'] = created
db_device = execute_post_internal(Naming.resource(device['@type']), device)
except InnerRequestError as e:
new = False
try:
db_device = _get_existing_device(e)
# We add a benchmark todo move to another place?
device['_id'] = db_device['_id']
ComponentDomain.benchmark(device)
except DeviceNotFound:
raise e
else:
log.append(db_device)
device.clear()
device.update(db_device)
device['new'] = new # Note that the device is 'cleared' before
return db_device
def test_resource(self):
"""
Creates a resource endpoint extending ResourceSettings and RDFS,
checking that both processes are done correctly,
performing requests to the endpoint.
"""
# The endpoint is defined in setUp and created as part of the regular workflow in DeviceHub.
# Let's validate it.
resource_name = Naming.resource(self.dummy_device_name)
self.assertIn(resource_name, self.domain)
settings = self.domain[resource_name]
assert_that(self.dummy_device_class).is_subset_of(settings['schema'])
assert_that(self.dummy_device_settings).is_subset_of(settings)
# Let's check we can perform some actions
# Let's create a dummy device
dummy_device = {'serialNumber': '33', 'model': 'model1', 'manufacturer': '234', self.dummy_field: 'dummyField',
'@type': self.dummy_device_name}
register = {'@type': 'Register', 'device': dummy_device}
self.post_and_check('{}/{}'.format(self.DEVICE_EVENT, 'register'), register)
# And now without the necessary new field
# wrong_dummy_device = dummy_device
# del wrong_dummy_device[self.dummy_field]
# _, status_code = self.post(self.DEVICES, wrong_dummy_device)
# self.assert422(status_code)
# Let's get a collection of dummy devices
_, status_code = self.get(self.DEVICES)
self.assert200(status_code)
# Let's get the first dummy device
dummy_device = self.get_first(self.DEVICES)
dummy_device['@type'] = self.dummy_device_name
# Let's try posting other devices, so we know our new resource doesn't affect others
self.get_fixtures_computers()
def exec_hard_drive_events(self, event_log, events):
for i, event in events:
event['device'] = self.components[i]['_id']
self.set_created_conditionally(event)
event_log.append(
execute_post_internal(Naming.resource(event['@type']), event))
event.update(event_log[-1])
def delete_events_in_device(resource_name: str, device: dict):
"""
Deletes the references of the given device in all the events, and deletes the full event if it references only to
the device.
"""
if resource_name in Device.resource_types:
_id = device["_id"]
qin = {"$in": [_id]}
query = {"$or": [{"device": _id}, {"devices": qin}, {"components": qin}], "@type": {"$ne": "devices:Register"}}
sort = {"_created": pymongo.ASCENDING} # Order is important to find the first Snapshot (see below)
first_snapshot_found = False
for event in DeviceEventDomain.get({"$query": query, "$orderby": sort}):
if not first_snapshot_found and event["@type"] == "devices:Snapshot":
# We cannot delete the Snapshot that created the device, because there is a change to create
# an infinite loop: Snapshot that created device -> Register -> DEL /device -> Snapshot that created...
first_snapshot_found = True
else:
event_resource = Naming.resource(event["@type"])
if event.get("device", None) == _id: # Am I the 'device' of the event?
execute_delete(event_resource, event["_id"])
elif [_id] == event.get("devices", []): # Is there no more 'devices' in the event, apart from me?
execute_delete(event_resource, event["_id"])
# All events that do not use 'components' for materialization (aka Add/Remove) should be erased
# if there are no more components
elif event["@type"] in ("devices:Add", "devices:Remove") and event["components"] == [_id]:
execute_delete(event_resource, event["_id"])
else: # Keep the event; just delete my reference
DeviceEventDomain.update_raw(event["_id"], {"$pull": {"devices": qin, "components": qin}})
def parse_event(self, event):
with self.app.app_context():
event = parse(event, Naming.resource(event['@type']))
if 'components' in event:
for device in event['components'] + [event['device']]:
device.update(self.parse_device(device))
return event
def create_migrate(migrates: list):
"""
Manages the creation of a Migrate event, like doing so for the devices.
Heavily inspired by the hook 'on_insert_snapshot', uses the same idea on a group of devices.
"""
all_events = [] # We will delete all events if exception
try:
for migrate in migrates:
if 'from' in migrate:
setattr(g, MIGRATE_RETURNED_SAME_AS, dict())
devices_id = []
migrate['components'] = []
for device in migrate['devices']:
if device['@type'] in Component.types:
raise SchemaError('devices', 'You cannot directly migrate components.')
creator = MigrateCreator(device, device.pop('components'))
events = creator.execute()
all_events += events
migrate['events'] = [new_events['_id'] for new_events in events]
devices_id.append(device['_id'])
migrate['components'] += [component['_id'] for component in creator.components]
migrate['unsecured'] = creator.unsecured
getattr(g, MIGRATE_RETURNED_SAME_AS).update(creator.returned_same_as)
from ereuse_devicehub.resources.hooks import set_date
set_date(None, migrates)
migrate['devices'] = devices_id
except Exception as e:
for event in all_events:
# Could result in 404 (ex: delete an 'Add' after deleting 'Register' of the same device)
execute_delete(Naming.resource(event['@type']), event['_id'])
raise e
def clean_device(device: dict):
"""Removes values that are not supposed to be sent, like materialized or readonly ones."""
schema = current_app.config['DOMAIN'][Naming.resource(device['@type'])]['schema']
_id = device['_id']
for field in copy.copy(device):
if '_' in field or not {'materialized', 'readonly'}.isdisjoint(set(schema[field].keys())):
del device[field]
device['url'] = DeviceDomain.url_agent_for(AccountDomain.get_requested_database(), _id)
def redirect_to_first_snapshot(resource, request, lookup):
"""
DELETE /device should be an internal method, but it is open to redirect to the door that is going to effictevely
delete the device; this is the first Snapshot that made the Register that made the device.
"""
if resource in Device.resource_types:
snapshot_id = str(DeviceEventDomain.get_first_snapshot(lookup['_id'])['_id'])
execute_delete(Naming.resource('devices:Snapshot'), snapshot_id)
raise RequestAnother('', 204)
def redirect_to_first_snapshot(resource, request, lookup):
"""
DELETE /device should be an internal method, but it is open to redirect to the door that is going to effictevely
delete the device; this is the first Snapshot that made the Register that made the device.
"""
if resource in Device.resource_types:
snapshot_id = str(
DeviceEventDomain.get_first_snapshot(lookup['_id'])['_id'])
execute_delete(Naming.resource('devices:Snapshot'), snapshot_id)
raise RequestAnother('', 204)
def execute(self, database):
DeviceDomain.update_many_raw({}, {'$set': {'events': []}})
for event in DeviceEventDomain.get({
'$query': {},
'$orderby': {
'_created': pymongo.ASCENDING
}
}):
MaterializeEvents.materialize_events(
Naming.resource(event['@type']), [event])
def transfer_property(receives: list):
for receive in receives:
if receive['automaticallyAllocate']:
allocate_type = DeviceEventDomain.new_type('Allocate')
a = execute_post_internal(Naming.resource(allocate_type), {
'@type': allocate_type,
'to': receive['receiver'],
'devices': receive['devices']
})
receive['_created'] = receive['_updated'] = a['_created'] + datetime.timedelta(milliseconds=1)
def post_devices(registers: list):
"""
Main function for Register. For the given devices, POST the new ones. This method rollbacks the database when
raising exceptions, like when no device has been POSTed.
If the function is called by post_internal(), as the method keeps the reference of the passed in devices, the
caller will see how their devices are replaced by the db versions, plus a 'new' property acting as a flag
to indicate if the device is new or not.
If a device exists, the input device is replaced by the version of the database, loosing any change the
input device was introducing (except benchmarks, which are updated). See `_execute_register` for more info.
:raise InnerRequestError: for any error provoked by a failure in the POST of a device (except if the device already
existed). It carries the original error sent by the POST.
:raise NoDevicesToProcess: Raised to avoid creating empty registers, that actually did not POST any device
"""
log = []
try:
for register in registers:
caller_device = register[
'device'] # Keep the reference from where register['device'] points to
_execute_register(caller_device, register.get('created'), log)
register['device'] = caller_device['_id']
if 'components' in register:
caller_components = register['components']
register['components'] = []
for component in caller_components:
component['parent'] = caller_device['_id']
_execute_register(component, register.get('created'), log)
if component[
'new']: # todo put new in g., don't use device
register['components'].append(component['_id'])
if caller_device['new']:
set_components(register)
elif not register['components']:
text = 'Device {} and components {} already exist.'.format(
register['device'], register['components'])
raise NoDevicesToProcess(text)
else:
add_components([
register
]) # The device is not new but we have new computers
# Note that we only need to copy a place from the parent if this already existed
if 'place' in caller_device:
inherit_place(caller_device['place'],
register['device'],
register['components'])
except Exception as e:
for device in reversed(log): # Rollback
deleteitem_internal(Naming.resource(device['@type']), device)
raise e
else:
from ereuse_devicehub.resources.hooks import set_date
set_date(None,
registers) # Let's get the time AFTER creating the devices
def re_materialize_events():
DeviceDomain.update_many_raw({}, {'$set': {'events': []}})
for event in DeviceEventDomain.get({
'$query': {},
'$orderby': {
'_created': pymongo.ASCENDING
}
}):
MaterializeEvents.materialize_events(
Naming.resource(event['@type']), [event])
print('Events re-materialized.')
def clean_device(device: dict):
"""Removes values that are not supposed to be sent, like materialized or readonly ones."""
schema = current_app.config['DOMAIN'][Naming.resource(
device['@type'])]['schema']
_id = device['_id']
for field in copy.copy(device):
if '_' in field or not {'materialized', 'readonly'}.isdisjoint(
set(schema[field].keys())):
del device[field]
device['url'] = DeviceDomain.url_agent_for(
AccountDomain.get_requested_database(), _id)
def generate_url(self, original_resource, translated_resource):
device_identifier = self.translator.hid_or_url(original_resource['device'])
if not re.compile(HID_REGEX).match(device_identifier): # It is not a HID, so it is an URL
device_identifier = quote_plus(device_identifier.replace('/', '!')) # Adapt it to GRD needs
url = self.domain + '/api/devices/'
event_type = translated_resource['@type']
if event_type == DeviceEventDomain.new_type('Register'):
url += 'register'
else:
url += '{}/{}'.format(device_identifier, Naming.resource(event_type))
return url
def parse_device(self, device):
"""
Parses the device using the standard way of parsing input data, using the settings of DeviceHub.
This is needed when comparing a fixture with the device it represents, when this comparison happens
inside with ... app_context()
:param device:
:return:
"""
with self.app.app_context():
return parse(copy.deepcopy(device), Naming.resource(device['@type']))
def _execute_register(device: dict, created: str, log: list):
"""
Tries to POST the device and updates the `device` dict with the resource from the database; if the device could
not be uploaded the `device` param will contain the database version of the device, not the inputting one. This is
because the majority of the information of a device is immutable (in concrete the fields used to compute
the ETAG).
:param device: Inputting device. It is replaced (keeping the reference) with the db version.
:param created: Set the _created value to be the same for the device as for the register
:param log: A log where to append the resulting device if execute_register has been successful
:raise InnerRequestError: any internal error in the POST that is not about the device already existing.
"""
new = True
try:
if created:
device['created'] = created
db_device = execute_post_internal(Naming.resource(device['@type']),
device)
except InnerRequestError as e:
new = False
try:
db_device = _get_existing_device(e)
# We add a benchmark todo move to another place?
device['_id'] = db_device['_id']
ComponentDomain.benchmark(device)
external_synthetic_id_fields = pick(
device, *DeviceDomain.external_synthetic_ids)
# If the db_device was a placeholder
# We want to override it with the new device
if db_device.get('placeholder', False):
# Eve do not generate defaults from sub-resources
# And we really need the placeholder default set, specially when
# discovering a device
device['placeholder'] = False
# We create hid when we validate (wrong thing) so we need to manually set it here as we won't
# validate in this db operation
device['hid'] = DeviceDomain.hid(device['manufacturer'],
device['serialNumber'],
device['model'])
DeviceDomain.update_one_raw(db_device['_id'], {'$set': device})
elif not is_empty(external_synthetic_id_fields):
# External Synthetic identifiers are not intrinsically inherent
# of devices, and thus can be added later in other Snapshots
# Note that the device / post and _get_existing_device() have already validated those ids
DeviceDomain.update_one_raw(
db_device['_id'], {'$set': external_synthetic_id_fields})
except DeviceNotFound:
raise e
else:
log.append(db_device)
device.clear()
device.update(db_device)
device['new'] = new # Note that the device is 'cleared' before
return db_device
def transfer_property(receives: list):
for receive in receives:
if receive['automaticallyAllocate']:
allocate_type = DeviceEventDomain.new_type('Allocate')
a = execute_post_internal(
Naming.resource(allocate_type), {
'@type': allocate_type,
'to': receive['receiver'],
'devices': receive['devices']
})
receive['_created'] = receive['_updated'] = a[
'_created'] + datetime.timedelta(milliseconds=1)
def get_field(self, field_name: str, schema: dict, **options) -> list:
"""
Returns a list of a) the passed-in field, and b) inner fields represented by passed-in-fieldname.inner-feldname:
- reference: the typeName of a data_relation
- type: the type of the field
- attr: dictionary of attributes. Nonexisting values are set as None, all keys exist.
- name: the name of the field
"""
result_field = {'type': schema['type'], 'name': field_name}
if self.special_cases(result_field, schema, **options):
return [result_field] + result_field.pop('_inner_fields', [])
if 'data_relation' in schema:
result_field['reference'] = Naming.type(schema['data_relation']['resource'])
elif schema['type'] == 'list' and 'schema' in schema:
subschema = schema['schema']
if 'data_relation' in subschema:
result_field['reference'] = Naming.type(subschema['data_relation']['resource'])
elif subschema['type'] == 'dict':
self.get_dict(result_field, subschema['schema'], **options)
elif schema['type'] == 'dict' and 'schema' in schema:
self.get_dict(result_field, schema['schema'], **options)
result_field['attr'] = {
'Unique': schema.get('unique'),
'Default': schema.get('default'),
'Allowed': schema.get('allowed'),
'Required': schema.get('required'),
'Description': schema.get('description'),
'Write only': schema.get('writeonly'),
'Read only': schema.get('readonly'),
'Modifiable': schema.get('modifiable'),
'Sink': schema.get('sink', 0),
'Unit Code': schema.get('unitCode'),
'Doc': schema.get('doc'),
'Roles with writing permission': schema.get(ALLOWED_WRITE_ROLES),
'OR': schema.get('or'),
'Excludes': schema.get('excludes')
}
if 'unitCode' in schema:
result_field['attr']['Unit Code'] = UnitCodes.humanize(schema['unitCode']) + ' ({})'.format(schema['unitCode'])
return [result_field] + result_field.pop('_inner_fields', [])
def add_remove(self, input_snapshot):
component = choice(input_snapshot['components'])
while True:
ignore_fields = self.app.config['DOMAIN'][Naming.resource(component['@type'])]['etag_ignore_fields']
key = choice(list(component.keys()))
if key not in ignore_fields:
break
if type(component[key]) is int or type(component[key]) is float:
component[key] += 10
elif type(component[key]) is str:
import uuid
component[key] = uuid.uuid4().hex[:6].upper()
self.post_snapshot_get_full_events(input_snapshot, 3)
def materialize_public_in_components_update(resource: str, device: dict, original: dict):
"""
The same as :func materialize_public_in_components: but thought to be used with PATCH methods.
:param resource:
:param device:
:param original:
"""
# PATCH doesn't need to include components if we are not changing them
# We have already saved the device to the DB so we can securely modify the device dictionary
if original['@type'] in Device.types:
if 'components' not in device:
device['components'] = original['components']
materialize_public_in_components(Naming.resource(original['@type']), [device])
def register(self, event_log: list):
with suppress(NoDevicesToProcess):
register = {
'@type': DeviceEventDomain.new_type('Register'),
'device': self.device,
'components': self.components
}
self.set_created_conditionally(register)
event_log.append(execute_post_internal(Naming.resource(register['@type']), register))
for device in [self.device] + self.components:
if 'hid' not in device and 'pid' not in device:
self._append_unsecured(device, 'model')
elif 'pid' in device:
self._append_unsecured(device, 'pid')
def materialize_public_in_components_update(resource: str, device: dict,
original: dict):
"""
The same as :func materialize_public_in_components: but thought to be used with PATCH methods.
:param resource:
:param device:
:param original:
"""
# PATCH doesn't need to include components if we are not changing them
# We have already saved the device to the DB so we can securely modify the device dictionary
if original['@type'] in Device.types:
if 'components' not in device:
device['components'] = original['components']
materialize_public_in_components(Naming.resource(original['@type']),
[device])
def generate_url(self, original_resource, translated_resource):
device_identifier = self.translator.hid_or_url(
original_resource['device'])
if not re.compile(HID_REGEX).match(
device_identifier): # It is not a HID, so it is an URL
device_identifier = quote_plus(device_identifier.replace(
'/', '!')) # Adapt it to GRD needs
url = self.domain + '/api/devices/'
event_type = translated_resource['@type']
if event_type == DeviceEventDomain.new_type('Register'):
url += 'register'
else:
url += '{}/{}'.format(device_identifier,
Naming.resource(event_type))
return url
def register(self, event_log: list):
with suppress(NoDevicesToProcess):
register = {
'@type': DeviceEventDomain.new_type('Register'),
'device': self.device,
'components': self.components
}
self.set_created_conditionally(register)
event_log.append(
execute_post_internal(Naming.resource(register['@type']),
register))
for device in [self.device] + self.components:
if 'hid' not in device and 'pid' not in device:
self._append_unsecured(device, 'model')
elif 'pid' in device:
self._append_unsecured(device, 'pid')
def actual_fields(cls):
"""
Gets only the actual fields of the class (ignoring parents'), alongside with the URL.
"""
if not hasattr(cls, '_schema'):
raise TypeError('Resource does not have any schema')
fields = super(ResourceSettings, cls).actual_fields()
super_resources = super(ResourceSettings, cls).superclasses(2)
# Creating the url for the resource
names = []
for resource in reversed(super_resources):
if getattr(resource, '_schema', False):
# We get the URL: first by getting settings['url'] or using the name of the class
names.append(resource._schema._settings.get('url', Naming.resource(resource._schema.__name__)))
fields['url'] = '/'.join(names)
return fields
def process(self) -> list:
"""
Executes all events stored.
First execute the inserts so the stored devices can get the _id and then executes the rest of events.
:return: A list of the executed events
"""
log = [] # log done events
for event_name, common_reference_dict in self.events.items():
for reference, unique in common_reference_dict.items():
device = self.references[reference]
log.append(execute_post_internal(Naming.resource(event_name), {
'@type': event_name,
'device': device['_id'],
'components': [str(x['_id']) for x in unique]
}))
return log
def delete_events(_, snapshot: dict):
"""
Deletes the events that were created with the snapshot.
"""
if snapshot.get('@type') == 'devices:Snapshot':
for event_id in snapshot['events']:
try:
# If the first event is Register, erasing the device will erase the rest of events
event = DeviceEventDomain.get_one(event_id)
except EventNotFound:
pass
else:
try:
execute_delete(Naming.resource(event['@type']), event['_id'])
except InnerRequestError as e:
if e.status_code != 404:
raise e
def delete_events(_, snapshot: dict):
"""
Deletes the events that were created with the snapshot.
"""
if snapshot.get('@type') == 'devices:Snapshot':
for event_id in snapshot['events']:
try:
# If the first event is Register, erasing the device will erase the rest of events
event = DeviceEventDomain.get_one(event_id)
except EventNotFound:
pass
else:
try:
execute_delete(Naming.resource(event['@type']),
event['_id'])
except InnerRequestError as e:
if e.status_code != 404:
raise e
def actual_fields(cls):
"""
Gets only the actual fields of the class (ignoring parents'), alongside with the URL.
"""
if not hasattr(cls, '_schema'):
raise TypeError('Resource does not have any schema')
fields = super(ResourceSettings, cls).actual_fields()
super_resources = super(ResourceSettings, cls).superclasses(2)
# Creating the url for the resource
names = []
for resource in reversed(super_resources):
if getattr(resource, '_schema', False):
# We get the URL: first by getting settings['url'] or using the name of the class
names.append(
resource._schema._settings.get(
'url', Naming.resource(resource._schema.__name__)))
fields['url'] = '/'.join(names)
return fields
def process(self) -> list:
"""
Executes all events stored.
First execute the inserts so the stored devices can get the _id and then executes the rest of events.
:return: A list of the executed events
"""
log = [] # log done events
for event_name, common_reference_dict in list(self.events.items()):
for reference, unique in list(common_reference_dict.items()):
device = self.references[reference]
log.append(
execute_post_internal(
Naming.resource(event_name), {
'@type': event_name,
'device': device['_id'],
'components': [str(x['_id']) for x in unique]
}))
return log
def post_devices(registers: list):
"""
Main function for Register. For the given devices, POST the new ones. This method rollbacks the database when
raising exceptions, like when no device has been POSTed.
If the function is called by post_internal(), as the method keeps the reference of the passed in devices, the
caller will see how their devices are replaced by the db versions, plus a 'new' property acting as a flag
to indicate if the device is new or not.
If a device exists, the input device is replaced by the version of the database, loosing any change the
input device was introducing (except benchmarks, which are updated). See `_execute_register` for more info.
:raise InnerRequestError: for any error provoked by a failure in the POST of a device (except if the device already
existed). It carries the original error sent by the POST.
:raise NoDevicesToProcess: Raised to avoid creating empty registers, that actually did not POST any device
"""
log = []
try:
for register in registers:
caller_device = register['device'] # Keep the reference from where register['device'] points to
_execute_register(caller_device, register.get('created'), log)
register['device'] = caller_device['_id']
if 'components' in register:
caller_components = register['components']
register['components'] = []
for component in caller_components:
component['parent'] = caller_device['_id']
_execute_register(component, register.get('created'), log)
if component['new']: # todo put new in g., don't use device
register['components'].append(component['_id'])
if caller_device['new']:
set_components(register)
elif not register['components']:
raise NoDevicesToProcess()
except Exception as e:
for device in reversed(log): # Rollback
deleteitem_internal(Naming.resource(device['@type']), device)
raise e
else:
from ereuse_devicehub.resources.hooks import set_date
set_date(None, registers) # Let's get the time AFTER creating the devices
def test_resource(self):
"""
Creates a resource endpoint extending ResourceSettings and RDFS,
checking that both processes are done correctly,
performing requests to the endpoint.
"""
# The endpoint is defined in setUp and created as part of the regular workflow in DeviceHub.
# Let's validate it.
resource_name = Naming.resource(self.dummy_device_name)
self.assertIn(resource_name, self.domain)
settings = self.domain[resource_name]
assert_that(self.dummy_device_class).is_subset_of(settings['schema'])
assert_that(self.dummy_device_settings).is_subset_of(settings)
# Let's check we can perform some actions
# Let's create a dummy device
dummy_device = {
'serialNumber': '33',
'model': 'model1',
'manufacturer': '234',
self.dummy_field: 'dummyField',
'@type': self.dummy_device_name
}
register = {'@type': 'Register', 'device': dummy_device}
self.post_and_check('{}/{}'.format(self.DEVICE_EVENT, 'register'),
register)
# And now without the necessary new field
# wrong_dummy_device = dummy_device
# del wrong_dummy_device[self.dummy_field]
# _, status_code = self.post(self.DEVICES, wrong_dummy_device)
# self.assert422(status_code)
# Let's get a collection of dummy devices
_, status_code = self.get(self.DEVICES)
self.assert200(status_code)
# Let's get the first dummy device
dummy_device = self.get_first(self.DEVICES)
dummy_device['@type'] = self.dummy_device_name
# Let's try posting other devices, so we know our new resource doesn't affect others
self.get_fixtures_computers()
def generate_etag(resource: str, items: list):
if resource in Device.resource_types:
for item in items:
item['_etag'] = document_etag(
item, app.config['DOMAIN'][Naming.resource(
item['@type'])]['etag_ignore_fields'])
def hid(manufacturer: str, serial_number: str, model: str) -> str:
"""Generates the HID"""
return Naming.url_word(manufacturer) + \
'-' + Naming.url_word(serial_number) + \
'-' + Naming.url_word(model)
def generate_etag(resource: str, items: list):
if resource in Device.resource_types:
for item in items:
item['_etag'] = document_etag(item,
app.config['DOMAIN'][Naming.resource(item['@type'])]['etag_ignore_fields'])
def resource_name(cls):
return Naming.resource(cls.type_name)
def resource_types(cls):
return {Naming.resource(type_name) for type_name in cls.types}
def type_name(cls):
return Naming.new_type(cls.__name__, cls._settings['prefix'])
def resource_name(cls):
return Naming.resource(cls.type_name)
def test_migrate(self):
"""Tests a basic migrate of multiple devices, from one database (db1) to another database (db2)."""
assert_that(self.domain).contains(Naming.resource('devices:Migrate'))
fixture_migrate_to = self.get_fixture('migrate', 'migrate_to')
fixture_migrate_to['devices'] = self.devices_id
fixture_migrate_to['to']['database'] = self.db2
migrate_db1_to_db2 = self.post_and_check('{}/{}'.format(self.DEVICE_EVENT, self.MIGRATE), fixture_migrate_to)
migrate_db1_to_db2, _ = self.get(Migrate.resource_name, '', migrate_db1_to_db2['_id'])
del fixture_migrate_to['to'] # to contains a new field
assert_that(fixture_migrate_to).is_subset_of(migrate_db1_to_db2)
assert_that(migrate_db1_to_db2['to']['url']).contains(migrate_db1_to_db2['to']['baseUrl'])
# Le'ts see if the migrate in db2 is correct
fixture_migrate_from = self.get_fixture('migrate', 'migrate_from')
migrate_from_db1, status = self._get(migrate_db1_to_db2['to']['url'], self.token_b)
self.assert200(status)
assert_that(fixture_migrate_from).is_subset_of(migrate_from_db1)
assert_that(migrate_from_db1['from']).contains(migrate_db1_to_db2['_links']['self']['href'])
# Let's check that the devices in db2 are the same than in db1
assert_that(migrate_from_db1).contains('devices')
assert_that(migrate_from_db1['devices']).is_not_empty()
devices_db1 = [self.get(self.DEVICES, '', device_id)[0] for device_id in self.devices_id]
devices_db2 = [self.get(self.DEVICES, '', device_id, True, self.db2)[0] for device_id in
migrate_from_db1['devices']]
self.assertSimilarDevices(devices_db1, devices_db2, True)
# We should not be able to perform an event with one of those devices in db1
patched_place = {
'_id': self.place['_id'],
'@type': 'Place',
'devices': self.devices_id
}
_, status = self.patch('{}/{}'.format(self.PLACES, self.place['_id']), patched_place)
# todo it should be 422 however eve's patch renders 400 if exception does not extend werkzeug's HTTPException
self.assert400(status)
del patched_place['devices'] # By removing the devices it should work (empty place)
_, status = self.patch('{}/{}'.format(self.PLACES, self.place['_id']), patched_place)
self.assert200(status)
allocate = self.get_fixture('allocate', 'allocate') # Let's post another event
allocate['to'] = self.get_first('accounts')['_id']
allocate['devices'] = self.devices_id
_, status = self.post(self.DEVICE_EVENT + '/allocate', allocate)
self.assert422(status)
# Let's get the devices in the actual state to use later
devices_in_db1 = [self.get(self.DEVICES, '', device_id)[0] for device_id in self.devices_id]
number_devices_in_db1 = len(self.get(self.DEVICES)[0])
# Let's perform the same but in db2
# Note that we will need to create the place in db2
place = self.get_fixture(self.PLACES, 'place')
place['devices'] = self.devices_id
place_in_db2, status = self._post('{}/{}'.format(self.db2, self.PLACES), place, self.token_b)
self.assert201(status)
# Let's perform a Migrate back to db1, moving the devices to db1
fixture_migrate_to = self.get_fixture('migrate', 'migrate_to')
fixture_migrate_to['to']['database'] = self.db1
fixture_migrate_to['devices'] = self.devices_id
fixture_migrate_to['label'] = 'Migrate back to db1'
url = '{}/{}/{}'.format(self.db2, self.DEVICE_EVENT, self.MIGRATE)
migrate_db2_to_db1, status = self._post(url, fixture_migrate_to, self.token_b)
self.assert201(status)
migrate_db2_to_db1, _ = self._get('{}/{}/{}'.format(self.db2, self.EVENTS, migrate_db2_to_db1['_id']),
self.token_b)
del fixture_migrate_to['to'] # to contains a new field
assert_that(fixture_migrate_to).is_subset_of(migrate_db2_to_db1)
assert_that(migrate_db2_to_db1['to']['url']).contains(migrate_db2_to_db1['to']['baseUrl'])
# Let's check that the devices in db1 only have one more event: the migrate (no Add, Register...)
for device in devices_in_db1:
device_db1_after_migrate, _ = self.get(self.DEVICES, '?embedded={"events":1}', device['_id'])
assert_that(device_db1_after_migrate['events']).is_length(len(device['events']) + 1)
# The last event is the resulting migrate (the one with 'from') executed in db1 by migrate_db2_to_db1
migrate_from_db2, _ = self._get(migrate_db2_to_db1['to']['url'], self.token)
assert_that(device_db1_after_migrate['events'][0]['_id']).is_equal_to(migrate_from_db2['_id'])
# The one before is the migrate with 'to' called in db1 migrate_db1_to_db2
assert_that(device_db1_after_migrate['events'][1]['_id']).is_equal_to(migrate_db1_to_db2['_id'])
# And the first event is always a register
assert_that(device_db1_after_migrate['events'][-1]['@type']).is_equal_to(Register.type_name)
# Let's check that there are exactly the same number of devices now than before, and in db2
# Note that the system can create a new device if there is no HID, but by issuing the URL
# with the device, the system should be able to identify a device with the `URL in sameAs`
number_devices_in_db1_after = len(self.get(self.DEVICES)[0])
assert_that(number_devices_in_db1).is_equal_to(number_devices_in_db1_after)
number_devices_in_db2 = len(self.get(self.DEVICES, '', None, True, self.db2)[0])
assert_that(number_devices_in_db1).is_equal_to(number_devices_in_db2)
# Finally, let's perform an event with the devices in db1
_, status = self.patch('{}/{}'.format(self.PLACES, self.place['_id']), patched_place)
self.assert200(status)
# And to db2: devices are not here anymore so:
# a) they have been automatically removed from their place in db2
place_in_db2, status = self._get('{}/{}/{}'.format(self.db2, self.PLACES, place_in_db2['_id']), self.token_b)
self.assert200(status)
# assert_that(place_in_db2).does_not_contain('devices')
assert_that(place_in_db2['devices']).does_not_contain(*[device['_id'] for device in devices_db2])
# b) you cannot POST/PUT/PATCH/DELETE anything with a reference to those devices
new_patch_for_place_in_db2 = {
'@type': 'Place',
'devices': self.devices_id
}
url = '{}/{}/{}'.format(self.db2, self.PLACES, place_in_db2['_id'])
_, status = self._patch(url, new_patch_for_place_in_db2, self.token_b)
self.assert400(status)
def test_migrate(self):
"""Tests a basic migrate of multiple devices, from one database (db1) to another database (db2)."""
assert_that(self.domain).contains(Naming.resource('devices:Migrate'))
fixture_migrate_to = self.get_fixture('migrate', 'migrate_to')
fixture_migrate_to['devices'] = self.devices_id
fixture_migrate_to['to']['database'] = self.db2
migrate_db1_to_db2 = self.post_and_check(
'{}/{}'.format(self.DEVICE_EVENT, self.MIGRATE),
fixture_migrate_to)
migrate_db1_to_db2, _ = self.get(Migrate.resource_name, '',
migrate_db1_to_db2['_id'])
del fixture_migrate_to['to'] # to contains a new field
assert_that(fixture_migrate_to).is_subset_of(migrate_db1_to_db2)
assert_that(migrate_db1_to_db2['to']['url']).contains(
migrate_db1_to_db2['to']['baseUrl'])
# Le'ts see if the migrate in db2 is correct
fixture_migrate_from = self.get_fixture('migrate', 'migrate_from')
migrate_from_db1, status = self._get(migrate_db1_to_db2['to']['url'],
self.token_b)
self.assert200(status)
assert_that(fixture_migrate_from).is_subset_of(migrate_from_db1)
assert_that(migrate_from_db1['from']).contains(
migrate_db1_to_db2['_links']['self']['href'])
# Let's check that the devices in db2 are the same than in db1
assert_that(migrate_from_db1).contains('devices')
assert_that(migrate_from_db1['devices']).is_not_empty()
devices_db1 = [
self.get(self.DEVICES, '', device_id)[0]
for device_id in self.devices_id
]
devices_db2 = [
self.get(self.DEVICES, '', device_id, True, self.db2)[0]
for device_id in migrate_from_db1['devices']
]
self.assertSimilarDevices(devices_db1, devices_db2, True)
# We should not be able to perform an event with one of those devices in db1
patched_place = {
'_id': self.place['_id'],
'@type': 'Place',
'devices': self.devices_id
}
_, status = self.patch('{}/{}'.format(self.PLACES, self.place['_id']),
patched_place)
# todo it should be 422 however eve's patch renders 400 if exception does not extend werkzeug's HTTPException
self.assert400(status)
del patched_place[
'devices'] # By removing the devices it should work (empty place)
_, status = self.patch('{}/{}'.format(self.PLACES, self.place['_id']),
patched_place)
self.assert200(status)
allocate = self.get_fixture('allocate',
'allocate') # Let's post another event
allocate['to'] = self.get_first('accounts')['_id']
allocate['devices'] = self.devices_id
_, status = self.post(self.DEVICE_EVENT + '/allocate', allocate)
self.assert422(status)
# Let's get the devices in the actual state to use later
devices_in_db1 = [
self.get(self.DEVICES, '', device_id)[0]
for device_id in self.devices_id
]
number_devices_in_db1 = len(self.get(self.DEVICES)[0])
# Let's perform the same but in db2
# Note that we will need to create the place in db2
place = self.get_fixture(self.PLACES, 'place')
place['devices'] = self.devices_id
place_in_db2, status = self._post(
'{}/{}'.format(self.db2, self.PLACES), place, self.token_b)
self.assert201(status)
# Let's perform a Migrate back to db1, moving the devices to db1
fixture_migrate_to = self.get_fixture('migrate', 'migrate_to')
fixture_migrate_to['to']['database'] = self.db1
fixture_migrate_to['devices'] = self.devices_id
fixture_migrate_to['label'] = 'Migrate back to db1'
url = '{}/{}/{}'.format(self.db2, self.DEVICE_EVENT, self.MIGRATE)
migrate_db2_to_db1, status = self._post(url, fixture_migrate_to,
self.token_b)
self.assert201(status)
migrate_db2_to_db1, _ = self._get(
'{}/{}/{}'.format(self.db2, self.EVENTS,
migrate_db2_to_db1['_id']), self.token_b)
del fixture_migrate_to['to'] # to contains a new field
assert_that(fixture_migrate_to).is_subset_of(migrate_db2_to_db1)
assert_that(migrate_db2_to_db1['to']['url']).contains(
migrate_db2_to_db1['to']['baseUrl'])
# Let's check that the devices in db1 only have one more event: the migrate (no Add, Register...)
for device in devices_in_db1:
device_db1_after_migrate, _ = self.get(self.DEVICES,
'?embedded={"events":1}',
device['_id'])
assert_that(device_db1_after_migrate['events']).is_length(
len(device['events']) + 1)
# The last event is the resulting migrate (the one with 'from') executed in db1 by migrate_db2_to_db1
migrate_from_db2, _ = self._get(migrate_db2_to_db1['to']['url'],
self.token)
assert_that(
device_db1_after_migrate['events'][0]['_id']).is_equal_to(
migrate_from_db2['_id'])
# The one before is the migrate with 'to' called in db1 migrate_db1_to_db2
assert_that(
device_db1_after_migrate['events'][1]['_id']).is_equal_to(
migrate_db1_to_db2['_id'])
# And the first event is always a register
assert_that(
device_db1_after_migrate['events'][-1]['@type']).is_equal_to(
Register.type_name)
# Let's check that there are exactly the same number of devices now than before, and in db2
# Note that the system can create a new device if there is no HID, but by issuing the URL
# with the device, the system should be able to identify a device with the `URL in sameAs`
number_devices_in_db1_after = len(self.get(self.DEVICES)[0])
assert_that(number_devices_in_db1).is_equal_to(
number_devices_in_db1_after)
number_devices_in_db2 = len(
self.get(self.DEVICES, '', None, True, self.db2)[0])
assert_that(number_devices_in_db1).is_equal_to(number_devices_in_db2)
# Finally, let's perform an event with the devices in db1
_, status = self.patch('{}/{}'.format(self.PLACES, self.place['_id']),
patched_place)
self.assert200(status)
# And to db2: devices are not here anymore so:
# a) they have been automatically removed from their place in db2
place_in_db2, status = self._get(
'{}/{}/{}'.format(self.db2, self.PLACES, place_in_db2['_id']),
self.token_b)
self.assert200(status)
# assert_that(place_in_db2).does_not_contain('devices')
assert_that(place_in_db2['devices']).does_not_contain(
*[device['_id'] for device in devices_db2])
# b) you cannot POST/PUT/PATCH/DELETE anything with a reference to those devices
new_patch_for_place_in_db2 = {
'@type': 'Place',
'devices': self.devices_id
}
url = '{}/{}/{}'.format(self.db2, self.PLACES, place_in_db2['_id'])
_, status = self._patch(url, new_patch_for_place_in_db2, self.token_b)
self.assert400(status)
def type_name(cls):
return Naming.new_type(cls.__name__, cls._settings['prefix'])
def generate_etag(device: dict) -> str:
domain = current_app.config["DOMAIN"]
return document_etag(device, domain[Naming.resource(device["@type"])]["etag_ignore_fields"])
def delete_device(_, register):
if register.get('@type') == Register.type_name:
for device_id in [register['device']] + register.get('components', []):
execute_delete(
Naming.resource(DeviceDomain.get_one(device_id)['@type']),
device_id)
def re_materialize_events():
DeviceDomain.update_many_raw({}, {'$set': {'events': []}})
for event in DeviceEventDomain.get({'$query': {}, '$orderby': {'_created': pymongo.ASCENDING}}):
MaterializeEvents.materialize_events(Naming.resource(event['@type']), [event])
print('Events re-materialized.')
def exec_hard_drive_events(self, event_log, events):
for i, event in events:
event['device'] = self.components[i]['_id']
self.set_created_conditionally(event)
event_log.append(execute_post_internal(Naming.resource(event['@type']), event))
event.update(event_log[-1])
def delete_device(_, register):
if register.get('@type') == Register.type_name:
for device_id in [register['device']] + register.get('components', []):
execute_delete(Naming.resource(DeviceDomain.get_one(device_id)['@type']), device_id)
def generate_etag(device: dict) -> str:
domain = current_app.config['DOMAIN']
return document_etag(
device,
domain[Naming.resource(device['@type'])]['etag_ignore_fields'])
def new_type(type_name):
from ereuse_devicehub.resources.event.device.settings import DeviceEvent
return Naming.new_type(type_name, DeviceEvent._settings['prefix'])
def resource_types(cls):
return {Naming.resource(type_name) for type_name in cls.types}
VALIDATION_ERROR_AS_LIST = True # We always use list to show errors
ITEM_CACHE = 120 # We differentiate from Resource cache (Cache setting from Eve) from Item cache
# Role settings
from ereuse_devicehub.resources.account.role import Role
ALLOWED_WRITE_ROLES = {Role.AMATEUR}
ALLOWED_ITEM_WRITE_ROLES = {Role.AMATEUR}
ALLOWED_READ_ROLES = {Role.BASIC}
ALLOWED_ITEM_READ_ROLES = {Role.BASIC}
# GRD Settings, do not change them
_events_in_grd = ('Deallocate', 'Migrate', 'Allocate', 'Receive', 'Remove',
'Add', 'Register', 'Locate', 'UsageProof', 'Recycle')
EVENTS_IN_GRD = [
Naming.resource(DeviceEventDomain.new_type(event))
for event in _events_in_grd
]
# Generation of the API (DOMAIN)
from ereuse_devicehub.resources.device.settings import DeviceSettings
from ereuse_devicehub.resources.account.settings import AccountSettings
from ereuse_devicehub.resources.event.settings import EventSettings
from ereuse_devicehub.resources.place.settings import PlaceSettings
DOMAIN = {
'devices': DeviceSettings,
'events': EventSettings,
'accounts': AccountSettings,
'places': PlaceSettings
}
