def materialize_owners(devices_id):
"""Re-computes 'owners' for the given devices and components."""
# First let's erase all owners
DeviceDomain.update_many_raw({'_id': {
'$in': devices_id
}}, {'$set': {
'owners': []
}})
# Then let's execute again the materialize hooks for Allocate/Deallocate
query = {
'$or': [{
'@type': 'devices:Allocate'
}, {
'@type': 'devices:Deallocate'
}],
'devices': {
'$in': devices_id
}
}
order_by = {'_created': pymongo.ASCENDING}
for event in DeviceEventDomain.get({
'$query': query,
'$orderby': order_by
}):
if event['@type'] == 'devices:Allocate':
materialize_actual_owners_add([event])
else:
modified = materialize_actual_owners_remove([event])
if modified == 0: # This Remove does nothing and should be erased
DeviceEventDomain.delete({'_id': event['_id']})
def execute(self, database):
SNAPSHOT_SOFTWARE = {
'DDI': 'Workbench',
'Scan': 'AndroidApp',
'DeviceHubClient': 'Web'
}
for snapshot in DeviceEventDomain.get({'@type': "devices:Snapshot"}):
with suppress(KeyError):
snapshot['snapshotSoftware'] = SNAPSHOT_SOFTWARE[snapshot.get(
'snapshotSoftware', 'DDI')]
DeviceEventDomain.update_one_raw(
snapshot['_id'],
{'$set': {
'snapshotSoftware': snapshot['snapshotSoftware']
}})
for device in DeviceDomain.get({'events._id': snapshot['_id']}):
materialized_snapshot = find(
device['events'],
lambda event: event['_id'] == snapshot['_id'])
materialized_snapshot['snapshotSoftware'] = snapshot[
'snapshotSoftware']
DeviceDomain.update_one_raw(
device['_id'], {'$set': {
'events': device['events']
}})
def execute(self, database):
DeviceDomain.update_many_raw({}, {'$set': {'events': []}})
for event in DeviceEventDomain.get({
'$query': {},
'$orderby': {
'_created': pymongo.ASCENDING
}
}):
MaterializeEvents.materialize_events(
Naming.resource(event['@type']), [event])
def materialize_events(cls, resource: str, events: list):
if resource in DeviceEvent.resource_types:
for event in events:
trimmed_event = {field_name: event[field_name] for field_name in cls.fields if field_name in event}
query = {'$push': {'events': {'$each': [trimmed_event], '$position': 0}}}
devices = [event['device']] if 'device' in event else event['devices']
if 'parent' in event: # Let's materialize the events (test, erasure...) of the component to the parent
devices.append(event['parent'])
DeviceDomain.update_raw(devices, query)
DeviceDomain.update_raw(event.get('components', []), query)
def _execute_register(device: dict, created: str, log: list):
"""
Tries to POST the device and updates the `device` dict with the resource from the database; if the device could
not be uploaded the `device` param will contain the database version of the device, not the inputting one. This is
because the majority of the information of a device is immutable (in concrete the fields used to compute
the ETAG).
:param device: Inputting device. It is replaced (keeping the reference) with the db version.
:param created: Set the _created value to be the same for the device as for the register
:param log: A log where to append the resulting device if execute_register has been successful
:raise InnerRequestError: any internal error in the POST that is not about the device already existing.
"""
new = True
try:
if created:
device['created'] = created
db_device = execute_post_internal(Naming.resource(device['@type']),
device)
except InnerRequestError as e:
new = False
try:
db_device = _get_existing_device(e)
# We add a benchmark todo move to another place?
device['_id'] = db_device['_id']
ComponentDomain.benchmark(device)
external_synthetic_id_fields = pick(
device, *DeviceDomain.external_synthetic_ids)
# If the db_device was a placeholder
# We want to override it with the new device
if db_device.get('placeholder', False):
# Eve do not generate defaults from sub-resources
# And we really need the placeholder default set, specially when
# discovering a device
device['placeholder'] = False
# We create hid when we validate (wrong thing) so we need to manually set it here as we won't
# validate in this db operation
device['hid'] = DeviceDomain.hid(device['manufacturer'],
device['serialNumber'],
device['model'])
DeviceDomain.update_one_raw(db_device['_id'], {'$set': device})
elif not is_empty(external_synthetic_id_fields):
# External Synthetic identifiers are not intrinsically inherent
# of devices, and thus can be added later in other Snapshots
# Note that the device / post and _get_existing_device() have already validated those ids
DeviceDomain.update_one_raw(
db_device['_id'], {'$set': external_synthetic_id_fields})
except DeviceNotFound:
raise e
else:
log.append(db_device)
device.clear()
device.update(db_device)
device['new'] = new # Note that the device is 'cleared' before
return db_device
def materialize_public_in_components(resource: str, devices: list):
"""
Materializes the 'public' field of the passed in devices in their components.
:param resource:
:param devices:
:return:
"""
if resource in Device.resource_types:
for device in devices:
if device.get('components'): # If empty do not try to execute
DeviceDomain.update_raw(device['components'], {'$set': {'public': device.get('public', False)}})
def re_materialize_events():
DeviceDomain.update_many_raw({}, {'$set': {'events': []}})
for event in DeviceEventDomain.get({
'$query': {},
'$orderby': {
'_created': pymongo.ASCENDING
}
}):
MaterializeEvents.materialize_events(
Naming.resource(event['@type']), [event])
print('Events re-materialized.')
def dematerialize_event(cls, _, event: dict):
if event.get('@type') in DeviceEvent.types:
device = [event['device']] if 'device' in event else []
parent = [event['parent']] if 'parent' in device else []
for device_id in event.get('devices', []) + event.get(
'components', []) + device + parent:
DeviceDomain.update_raw(
device_id, {'$pull': {
'events': {
'_id': event['_id']
}
}})
def materialize_component_info():
devices = DeviceDomain.get({'@type': Computer.type_name})
for device in devices:
if 'components' in device:
# Let's reset the ram and hard-drive counter
for _type in ('totalRamSize', 'totalHardDriveSize'):
DeviceDomain.update_one_raw(device['_id'],
{'$set': {
_type: 0
}})
update_materialized_computer(device['_id'],
device['components'],
add=True)
def materialize_owners(devices_id):
"""Re-computes 'owners' for the given devices and components."""
# First let's erase all owners
DeviceDomain.update_many_raw({'_id': {'$in': devices_id}}, {'$set': {'owners': []}})
# Then let's execute again the materialize hooks for Allocate/Deallocate
query = {'$or': [{'@type': 'devices:Allocate'}, {'@type': 'devices:Deallocate'}], 'devices': {'$in': devices_id}}
order_by = {'_created': pymongo.ASCENDING}
for event in DeviceEventDomain.get({'$query': query, '$orderby': order_by}):
if event['@type'] == 'devices:Allocate':
materialize_actual_owners_add([event])
else:
modified = materialize_actual_owners_remove([event])
if modified == 0: # This Remove does nothing and should be erased
DeviceEventDomain.delete({'_id': event['_id']})
def materialize_public_in_components(resource: str, devices: list):
"""
Materializes the 'public' field of the passed in devices in their components.
:param resource:
:param devices:
:return:
"""
if resource in Device.resource_types:
for device in devices:
if device.get('components'): # If empty do not try to execute
DeviceDomain.update_raw(
device['components'],
{'$set': {
'public': device.get('public', False)
}})
def process_same_as(self, device: dict) -> (bool, list):
"""
Tries to match the input device with an existing one by detecting a similar url in sameAs and url fields, and
then processes them.
Given a set of sameAs and url in the param device, it tries to find at least one of those url ones in another
device in the database. If found, both devices are thought to be the same and both 'sameAs' fields are merged
(note that Register does update the sameAs).
:param device: The device to check if whose sameAs and URL match any of the already existing ones.
:return: A boolean indicating if a device was found in the database with at least one matching url and a list
of sameAs for the device
"""
urls = [device['url']] + device.get('sameAs', [])
self.urls.append(device.pop('url'))
matching_same_as = False
try:
already_existing_device = DeviceDomain.get_one(
{'sameAs': {
'$in': urls
}})
except DeviceNotFound:
same_as = device['sameAs'] = urls
else:
device['_id'] = already_existing_device['_id']
urls_set = set(urls)
# we do not want the url of the existing device to be part of the sameAs
urls_set.discard(already_existing_device.get('url'))
same_as = device['sameAs'] = list(
set(already_existing_device['sameAs']) | urls_set)
matching_same_as = True
return matching_same_as, same_as
def generate_returned_same_as(self):
"""
Populates self.returned_same_as with a dict where the URL of devices in the caller agent as keys and a copy of
'sameAs' in this agent.
This value will be returned to the caller agent (see
:py:func:`ereuse_devicehub.resources.event.device.migrate.hooks.return_same_as`) which can use it to update
its 'sameAs' values with new references.
"""
for device, url in zip([self.device] + self.components, self.urls):
device = DeviceDomain.get_one(device['_id'])
same_as = set(device['sameAs'])
same_as.remove(url)
same_as.add(
DeviceDomain.url_agent_for(
AccountDomain.get_requested_database(), device['_id']))
self.returned_same_as[url] = list(same_as)
def clean_device(device: dict):
"""Removes values that are not supposed to be sent, like materialized or readonly ones."""
schema = current_app.config['DOMAIN'][Naming.resource(device['@type'])]['schema']
_id = device['_id']
for field in copy.copy(device):
if '_' in field or not {'materialized', 'readonly'}.isdisjoint(set(schema[field].keys())):
del device[field]
device['url'] = DeviceDomain.url_agent_for(AccountDomain.get_requested_database(), _id)
def clean_device(device: dict):
"""Removes values that are not supposed to be sent, like materialized or readonly ones."""
schema = current_app.config['DOMAIN'][Naming.resource(
device['@type'])]['schema']
_id = device['_id']
for field in copy.copy(device):
if '_' in field or not {'materialized', 'readonly'}.isdisjoint(
set(schema[field].keys())):
del device[field]
device['url'] = DeviceDomain.url_agent_for(
AccountDomain.get_requested_database(), _id)
def device_set_place(device_id: str, place_id: str):
device = DeviceDomain.get_one(device_id)
if 'place' in device:
current_app.data.driver.db['places'].update_one(
{'_id': device['place']}, {'$pull': {
'devices': device_id
}})
current_app.data.driver.db['devices'].update_one(
{'_id': device_id}, {'$set': {
'place': place_id
}})
def assertSimilarDevice(self, input_device: dict or str, created_device: dict or str):
"""
Checks that the createdDevice is the same as the input one, removing computed values as hid... It uses etag.
:param input_device Input device needs all the float values to have, by default, ".0", or it won't work
"""
# todo make sure .0 doesn't crush in real program
parsed_device = self.parse_device(input_device)
with self.app.app_context():
from ereuse_devicehub.resources.device.domain import DeviceDomain
self.assertTrue(DeviceDomain.seem_equal(self.full(self.DEVICES, parsed_device),
self.full(self.DEVICES, created_device)))
def materialize_events(cls, resource: str, events: list):
if resource in DeviceEvent.resource_types:
for event in events:
trimmed_event = {
field_name: event[field_name]
for field_name in cls.FIELDS if field_name in event
}
query = {
'$push': {
'events': {
'$each': [trimmed_event],
'$position': 0
}
}
}
devices = [event['device']
] if 'device' in event else event['devices']
if 'parent' in event: # Let's materialize the events (test, erasure...) of the component to the parent
devices.append(event['parent'])
DeviceDomain.update_raw(devices, query)
DeviceDomain.update_raw(event.get('components', []), query)
def execute(self, database):
for device in DeviceDomain.get({'hid': 'dummy'}):
try:
hid = DeviceDomain.hid(device['manufacturer'], device['serialNumber'], device['model'])
DeviceDomain.update_one_raw(device['_id'], {'$set': {'hid': hid}})
except KeyError:
DeviceDomain.update_one_raw(device['_id'], {'$unset': {'hid': ''}, '$set': {'isUidSecured': False}})
def get_add_remove(self, device: dict, new_parent: dict):
"""
Get the changes (events) that will need to be triggered for the given device.
Changes will we saved in the same device in the reserved key '_change'.
The function doesn't execute any event per se or validate that the user can do it.
:param device: The device must have an hid
:param new_parent:
"""
if not device['new']:
try:
old_parent = ComponentDomain.get_parent(device['_id'])
except DeviceNotFound: # The component exists but had no parent device, until now
self.events.append_add(device, new_parent)
else:
if not DeviceDomain.seem_equal(old_parent, new_parent):
self.events.append_remove(device, old_parent)
self.events.append_add(device, new_parent)
def get_add_remove(self, device: dict, new_parent: dict):
"""
Get the changes (events) that will need to be triggered for the given device.
Changes will we saved in the same device in the reserved key '_change'.
The function doesn't execute any event per se or validate that the user can do it.
:param device: The device must have an hid
:param new_parent:
"""
if not device['new']:
try:
old_parent = ComponentDomain.get_parent(device['_id'])
except DeviceNotFound: # The component exists but had no parent device, until now
self.events.append_add(device, new_parent)
else:
if not DeviceDomain.seem_equal(old_parent, new_parent):
self.events.append_remove(device, old_parent)
self.events.append_add(device, new_parent)
def avoid_repeating_allocations(allocates: list):
"""
Checks that we are not allocating to an account that is already an owner of the device
This method must execute after
:param allocates:
:return:
"""
for allocate in allocates:
devices_with_repeating_owners = DeviceDomain.get({
'$or': [
{'_id': {'$in': [allocate['devices']]}},
{'owners': {'$in': [allocate['to']]}}
]
})
ids = [device['_id'] for device in devices_with_repeating_owners]
allocate['devices'] = list(set(allocate['devices']) - set(ids))
if len(allocate['devices']) == 0:
raise AlreadyAllocated()
def _validate_type_hid(self, field, _):
"""
General validation for inserting devices (the name of the function is forced by Cerberus, not a good one).
- Tries to create hid and validates it.
- If hid cannot be created:
- If it has a parent, ensures that the device is unique.
- If it has not a parent, validates that the device has an user provided _id.
"""
from ereuse_devicehub.resources.device.component.domain import ComponentDomain
from ereuse_devicehub.resources.device.exceptions import DeviceNotFound
try:
from ereuse_devicehub.resources.device.domain import DeviceDomain
# todo this should not be done in the validation. Prove of this is that it needs to be done in
# register/hooks again for placeholders
self.document['hid'] = DeviceDomain.hid(
self.document['manufacturer'], self.document['serialNumber'],
self.document['model'])
except KeyError:
self.document['isUidSecured'] = False
if '_id' not in self.document: # We do not validate here the unique constraint of _id
if 'parent' in self.document:
with suppress(KeyError, DeviceNotFound):
component = ComponentDomain.get_similar_component(
self.document, self.document['parent'])
self._error('model',
json_util.dumps({'NotUnique': component}))
else:
# If device has no parent and no hid, user needs to: or provide _id or forcing creating it
if 'forceCreation' not in self.document or not self.document[
'forceCreation']:
self._error(
'_id', json_util.dumps({'NeedsId': self.document}))
# else: user forces us to create the device, it will be assigned an _id
# else: user provided _id. We accept this, however is unsecured.
else:
from ereuse_devicehub.resources.device.settings import HID_REGEX
self._validate_regex(HID_REGEX, field, self.document['hid'])
self._validate_unique(True, field, self.document['hid'])
def avoid_repeating_allocations(allocates: list):
"""
Checks that we are not allocating to an account that is already an owner of the device
This method must execute after
:param allocates:
:return:
"""
for allocate in allocates:
devices_with_repeating_owners = DeviceDomain.get({
'$or': [{
'_id': {
'$in': [allocate['devices']]
}
}, {
'owners': {
'$in': [allocate['to']]
}
}]
})
ids = [device['_id'] for device in devices_with_repeating_owners]
allocate['devices'] = list(set(allocate['devices']) - set(ids))
if len(allocate['devices']) == 0:
raise AlreadyAllocated()
def delete_device(_, register):
if register.get('@type') == Register.type_name:
for device_id in [register['device']] + register.get('components', []):
execute_delete(
Naming.resource(DeviceDomain.get_one(device_id)['@type']),
device_id)
def re_materialize_owners():
for device in DeviceDomain.get({'@type': 'Computer'}):
materialize_owners([device['_id']])
print('Owners re-materialized in devices.')
def materialize_actual_owners_remove(events: list):
for event in events:
properties = {'$pull': {'owners': event['from']}}
DeviceDomain.update_raw(event.get('components', []), properties)
return DeviceDomain.update_raw(event['devices'], properties)
def dematerialize_event(cls, resource: str, event: dict):
if event.get('@type') in DeviceEvent.types:
device = [event['device']] if 'device' in event else []
parent = [event['parent']] if 'parent' in device else []
for device_id in event.get('devices', []) + event.get('components', []) + device + parent:
DeviceDomain.update_raw(device_id, {'$pull': {'events': {'_id': event['_id']}}})
def device_set_place(device_id: str, place_id: str):
device = DeviceDomain.get_one(device_id)
if 'place' in device:
current_app.data.driver.db['places'].update_one({'_id': device['place']}, {'$pull': {'devices': device_id}})
current_app.data.driver.db['devices'].update_one({'_id': device_id}, {'$set': {'place': place_id}})
def _update_same_as(returned_same_as: dict):
for url, same_as in returned_same_as.items():
_id = url.split('/')[-1]
device = DeviceDomain.get_one({'_id': _id})
same_as = set(device.get('sameAs', [])) | set(same_as)
DeviceDomain.update_one_raw(_id, {'$set': {'sameAs': list(same_as)}})
def delete_device(_, register):
if register.get('@type') == Register.type_name:
for device_id in [register['device']] + register.get('components', []):
execute_delete(Naming.resource(DeviceDomain.get_one(device_id)['@type']), device_id)
def re_materialize_events():
DeviceDomain.update_many_raw({}, {'$set': {'events': []}})
for event in DeviceEventDomain.get({'$query': {}, '$orderby': {'_created': pymongo.ASCENDING}}):
MaterializeEvents.materialize_events(Naming.resource(event['@type']), [event])
print('Events re-materialized.')
def re_materialize_owners():
for device in DeviceDomain.get({'@type': 'Computer'}):
materialize_owners([device['_id']])
print('Owners re-materialized in devices.')
def _update_same_as(device: dict, same_as: list):
DeviceDomain.update_one_raw(device['_id'],
{'$set': {
'sameAs': same_as
}})
def materialize_actual_owners_remove(events: list):
for event in events:
properties = {'$pull': {'owners': event['from']}}
DeviceDomain.update_raw(event.get('components', []), properties)
return DeviceDomain.update_raw(event['devices'], properties)
def materialize_actual_owners_add(allocates: list):
for allocate in allocates:
properties = {'$addToSet': {'owners': allocate['to']}}
DeviceDomain.update_raw(allocate['devices'], properties)
DeviceDomain.update_raw(allocate.get('components', []), properties)
def _update_same_as(returned_same_as: dict):
for url, same_as in list(returned_same_as.items()):
_id = url.split('/')[-1]
device = DeviceDomain.get_one({'_id': _id})
same_as = set(device.get('sameAs', [])) | set(same_as)
DeviceDomain.update_one_raw(_id, {'$set': {'sameAs': list(same_as)}})
def materialize_actual_owners_add(allocates: list):
for allocate in allocates:
properties = {'$addToSet': {'owners': allocate['to']}}
DeviceDomain.update_raw(allocate['devices'], properties)
DeviceDomain.update_raw(allocate.get('components', []), properties)
