def setUp(self):
self.maxDiff = None
client = MongoClient()
client.drop_database('metaphor2_test_db')
self.db = client.metaphor2_test_db
self.schema = Schema(self.db)
self.updater = Updater(self.schema)
self.employee_spec = self.schema.add_spec('employee')
self.schema.add_field(self.employee_spec, 'name', 'str')
self.schema.add_field(self.employee_spec, 'age', 'int')
self.section_spec = self.schema.add_spec('section')
self.schema.add_field(self.section_spec, 'name', 'str')
self.schema.add_field(self.section_spec, 'members', 'linkcollection',
'employee')
self.division_spec = self.schema.add_spec('division')
self.schema.add_field(self.division_spec, 'name', 'str')
self.schema.add_field(self.division_spec, 'employees', 'collection',
'employee')
self.schema.add_field(self.division_spec, 'sections', 'collection',
'section')
self.schema.add_field(self.schema.root, 'divisions', 'collection',
'division')
self.aggregator = ReverseAggregator(self.schema)
def setUp(self):
self.maxDiff = None
client = MongoClient()
client.drop_database('metaphor2_test_db')
self.db = client.metaphor2_test_db
self.schema = Schema(self.db)
self.schema.create_initial_schema()
self.updater = Updater(self.schema)
self.employee_spec = self.schema.create_spec('employee')
self.schema.create_field('employee', 'name', 'str')
self.schema.create_field('employee', 'age', 'int')
self.division_spec = self.schema.create_spec('division')
self.schema.create_field('division', 'name', 'str')
self.schema.create_field('division', 'employees', 'collection',
'employee')
self.schema.create_field('root', 'divisions', 'collection', 'division')
def setUp(self):
self.maxDiff = None
client = MongoClient()
client.drop_database('metaphor2_test_db')
self.db = client.metaphor2_test_db
self.schema = Schema(self.db)
self.updater = Updater(self.schema)
self.employee_spec = self.schema.add_spec('employee')
self.schema.add_field(self.employee_spec, 'name', 'str')
self.schema.add_field(self.employee_spec, 'age', 'int')
self.schema.add_field(self.schema.root, 'current_employees', 'collection', 'employee')
self.schema.add_field(self.schema.root, 'former_employees', 'collection', 'employee')
self.calcs_spec = self.schema.add_spec('calcs')
def setUp(self):
self.maxDiff = None
client = MongoClient()
client.drop_database('metaphor2_test_db')
self.db = client.metaphor2_test_db
self.schema = Schema(self.db)
self.updater = Updater(self.schema)
self.employee_spec = self.schema.add_spec('employee')
self.schema.add_field(self.employee_spec, 'name', 'str')
self.schema.add_field(self.schema.root, 'employees', 'collection',
'employee')
self.api = Api(self.schema)
self.db.delete_calc.create_index([
('update_id', pymongo.ASCENDING),
('resource_id', pymongo.ASCENDING),
],
unique=True)
class UpdaterTest(unittest.TestCase):
def setUp(self):
self.maxDiff = None
client = MongoClient()
client.drop_database('metaphor2_test_db')
self.db = client.metaphor2_test_db
self.schema = Schema(self.db)
self.updater = Updater(self.schema)
self.employee_spec = self.schema.add_spec('employee')
self.schema.add_field(self.employee_spec, 'name', 'str')
self.schema.add_field(self.employee_spec, 'age', 'int')
self.division_spec = self.schema.add_spec('division')
self.schema.add_field(self.division_spec, 'name', 'str')
self.schema.add_field(self.division_spec, 'employees', 'collection', 'employee')
self.schema.add_field(self.schema.root, 'divisions', 'collection', 'division')
def test_update_simple_field(self):
self.schema.add_calc(self.division_spec, 'older_employees', 'self.employees[age>30]')
division_id_1 = self.schema.insert_resource(
'division', {'name': 'sales'}, 'divisions')
employee_id_1 = self.schema.insert_resource(
'employee', {'name': 'bob', 'age': 31}, 'employees', 'division', division_id_1)
self.updater.update_calc('division', 'older_employees', division_id_1)
division_data = self.db.resource_division.find_one()
self.assertEquals({
'_id': self.schema.decodeid(division_id_1),
'_grants': [],
'name': 'sales',
'_canonical_url': '/divisions/%s' % division_id_1,
'_parent_canonical_url': '/',
'_parent_field_name': 'divisions',
'_parent_id': None,
'_parent_type': 'root',
'older_employees': [ObjectId(employee_id_1[2:])],
}, division_data)
self.updater.update_fields('employee', employee_id_1, {"age": 20})
division_data = self.db.resource_division.find_one()
self.assertEquals({
'_id': self.schema.decodeid(division_id_1),
'_grants': [],
'name': 'sales',
'_canonical_url': '/divisions/%s' % division_id_1,
'_parent_canonical_url': '/',
'_parent_field_name': 'divisions',
'_parent_id': None,
'_parent_type': 'root',
'older_employees': [],
}, division_data)
def test_update_containing_collection(self):
self.schema.add_calc(self.division_spec, 'older_employees', 'self.employees[age>30]')
division_id_1 = self.schema.insert_resource(
'division', {'name': 'sales'}, 'divisions')
self.updater.update_calc('division', 'older_employees', division_id_1)
division_data = self.db.resource_division.find_one()
self.assertEquals({
'_id': self.schema.decodeid(division_id_1),
'_grants': [],
'_canonical_url': '/divisions/%s' % division_id_1,
'name': 'sales',
'_parent_canonical_url': '/',
'_parent_field_name': 'divisions',
'_parent_id': None,
'_parent_type': 'root',
'older_employees': [],
}, division_data)
employee_id_1 = self.updater.create_resource('employee', 'division', 'employees', division_id_1, {
'name': 'Bob',
'age': 41,
})
division_data = self.db.resource_division.find_one()
self.assertEquals({
'_id': self.schema.decodeid(division_id_1),
'_grants': [],
'_canonical_url': '/divisions/%s' % division_id_1,
'name': 'sales',
'_parent_canonical_url': '/',
'_parent_field_name': 'divisions',
'_parent_id': None,
'_parent_type': 'root',
'older_employees': [ObjectId(employee_id_1[2:])],
}, division_data)
employee_data = self.db.resource_employee.find_one()
self.assertEquals({
'_id': self.schema.decodeid(employee_id_1),
'_grants': [],
'_canonical_url': '/divisions/%s/employees/%s' % (division_id_1, employee_id_1),
'name': 'Bob',
'age': 41,
'_parent_canonical_url': '/divisions/%s' % division_id_1,
'_parent_field_name': 'employees',
'_parent_id': ObjectId(division_id_1[2:]),
'_parent_type': 'division',
}, employee_data)
def test_update_link_collection(self):
self.schema.add_field(self.division_spec, 'managers', 'linkcollection', 'employee')
self.schema.add_calc(self.division_spec, 'older_managers', 'self.managers[age>30]')
self.schema.add_calc(self.division_spec, 'older_non_retired_managers', 'self.older_managers[age<65]')
log.debug("start")
division_id_1 = self.schema.insert_resource(
'division', {'name': 'sales'}, 'divisions')
log.debug("inserted")
employee_id_1 = self.updater.create_resource('employee', 'division', 'employees', division_id_1, {
'name': 'Bob',
'age': 41
})
log.debug("created 1")
employee_id_2 = self.updater.create_resource('employee', 'division', 'employees', division_id_1, {
'name': 'Ned',
'age': 70
})
log.debug("created 2")
employee_id_3 = self.updater.create_resource('employee', 'division', 'employees', division_id_1, {
'name': 'Fred',
'age': 25
})
log.debug("created 3")
self.updater.create_linkcollection_entry('division', division_id_1, 'managers', employee_id_1)
log.debug("created entry 1")
self.updater.create_linkcollection_entry('division', division_id_1, 'managers', employee_id_2)
log.debug("created entry 2")
self.updater.create_linkcollection_entry('division', division_id_1, 'managers', employee_id_3)
log.debug("created entry 3")
division_data = self.db.resource_division.find_one()
self.assertEquals("sales", division_data['name'])
self.assertEquals(3, len(division_data['managers']))
self.assertTrue({"_id" : self.schema.decodeid(employee_id_1)} in division_data['managers'])
self.assertTrue({"_id" : self.schema.decodeid(employee_id_2)} in division_data['managers'])
self.assertTrue({"_id" : self.schema.decodeid(employee_id_3)} in division_data['managers'])
self.assertEquals(sorted([
self.schema.decodeid(employee_id_1),
self.schema.decodeid(employee_id_2),
]), sorted(division_data['older_managers']))
self.assertEquals([
self.schema.decodeid(employee_id_1)],
division_data['older_non_retired_managers'])
self.assertEquals({
"_id" : self.schema.decodeid(division_id_1),
'_grants': [],
'_canonical_url': '/divisions/%s' % division_id_1,
"_parent_field_name" : "divisions",
"_parent_id" : None,
"_parent_type" : "root",
"_parent_canonical_url" : '/',
"name" : "sales",
"managers" : [
{
"_id" : self.schema.decodeid(employee_id_1)
},
{
"_id" : self.schema.decodeid(employee_id_2)
},
{
"_id" : self.schema.decodeid(employee_id_3)
}
],
"older_managers" : [
self.schema.decodeid(employee_id_1),
self.schema.decodeid(employee_id_2),
],
"older_non_retired_managers" : [
self.schema.decodeid(employee_id_1),
]
}, division_data)
def test_reverse_aggregation_loopback(self):
self.schema.add_field(self.division_spec, 'managers', 'linkcollection', 'employee')
self.schema.add_calc(self.employee_spec, 'all_my_subordinates', 'self.link_division_managers.employees')
division_id_1 = self.schema.insert_resource(
'division', {'name': 'sales'}, 'divisions')
employee_id_1 = self.updater.create_resource('employee', 'division', 'employees', division_id_1, {
'name': 'bob', 'age': 21})
employee_id_2 = self.updater.create_resource('employee', 'division', 'employees', division_id_1, {
'name': 'ned', 'age': 31})
employee_id_3 = self.updater.create_resource('employee', 'division', 'employees', division_id_1, {
'name': 'fred', 'age': 41})
employee_id_4 = self.updater.create_resource('employee', 'division', 'employees', division_id_1, {
'name': 'mike', 'age': 51})
# add manager
calc_spec = self.schema.calc_trees[('employee', 'all_my_subordinates')]
self.assertEquals({'division.managers', 'division.employees'}, calc_spec.get_resource_dependencies())
self.updater.create_linkcollection_entry('division', division_id_1, 'managers', employee_id_1)
employee_data = self.db.resource_employee.find_one()
self.assertEquals({
"_id" : self.schema.decodeid(employee_id_1),
'_grants': [],
'_canonical_url': '/divisions/%s/employees/%s' % (division_id_1, employee_id_1),
"_parent_field_name" : "employees",
"_parent_id" : self.schema.decodeid(division_id_1),
"_parent_type" : "division",
"_parent_canonical_url" : "/divisions/%s" % division_id_1,
"name" : "bob",
"age": 21,
"all_my_subordinates" : [
self.schema.decodeid(employee_id_1),
self.schema.decodeid(employee_id_2),
self.schema.decodeid(employee_id_3),
self.schema.decodeid(employee_id_4),
]}, employee_data)
def test_resource_deps_for_field(self):
self.schema.add_calc(self.employee_spec, 'all_ages', 'divisions.employees.age + 10')
# add manager
calc_spec = self.schema.calc_trees[('employee', 'all_ages')]
self.assertEquals({'division.employees', 'root.divisions', 'employee.age'}, calc_spec.get_resource_dependencies())
def __init__(self, schema):
self.schema = schema
self.updater = Updater(schema)
class AdminApi(object):
def __init__(self, schema):
self.schema = schema
self.updater = Updater(schema)
def format_schema(self, include_admindata=False):
return SchemaSerializer(include_admindata).serialize(self.schema)
def create_spec(self, spec_name):
self.schema.create_spec(spec_name)
def create_field(self,
spec_name,
field_name,
field_type,
field_target=None,
calc_str=None):
if spec_name != 'root' and spec_name not in self.schema.specs:
raise HTTPError(None, 404, 'Not Found', None, None)
try:
self.schema.create_field(spec_name, field_name, field_type,
field_target, calc_str)
except MalformedFieldException as me:
raise HTTPError(None, 400, str(me), None, None)
self._update_for_calc_field(spec_name, field_name, field_type)
def _update_for_calc_field(self, spec_name, field_name, field_type):
if field_type == 'calc':
for resource in self.schema.db['resource_%s' % spec_name].find(
{}, {'_id': 1}):
self.updater.update_calc(spec_name, field_name,
self.schema.encodeid(resource['_id']))
def update_field(self,
spec_name,
field_name,
field_type,
field_target=None,
calc_str=None):
try:
self.schema.update_field(spec_name, field_name, field_type,
field_target, calc_str)
except MalformedFieldException as me:
raise HTTPError(None, 400, str(me), None, None)
self._update_for_calc_field(spec_name, field_name, field_type)
def delete_field(self, spec_name, field_name):
if spec_name != 'root' and spec_name not in self.schema.specs:
raise HTTPError(None, 404, 'Not Found', None, None)
try:
self.schema.delete_field(spec_name, field_name)
except DependencyException as de:
raise HTTPError(None, 400, str(de), None, None)
self.updater.remove_spec_field(spec_name, field_name)
def list_integrations(self):
return self.schema.list_integrations()
def create_integration(self, integration_data):
self.schema.create_integration(integration_data)
def update_integration(self, integration_data):
self.schema.update_integration(integration_data)
def delete_integration(self, integration_id):
self.schema.delete_integration(integration_id)
def export_schema(self):
schema = self.schema.db['metaphor_schema'].find_one()
schema.pop('_id')
return schema
def import_schema(self, schema_data):
self.schema.save_imported_schema_data(schema_data)
class Api(object):
def __init__(self, schema):
self.schema = schema
self.updater = Updater(schema)
@staticmethod
def _has_grants(url_path, canonical_url, grants):
url_path = re.sub(r'\[.*?]', '', url_path.strip('/'))
canonical_url = canonical_url.strip('/')
def match_grant(url, grant_url, recurse=False):
match_re = grant_url.replace('/*', '\/ID[0-9a-f]*')
if recurse and match_re != '/': # allow for root (admin) grant
match_re = match_re + r'(/.*)?$'
if not recurse:
match_re = match_re + r'$'
return re.match(match_re, url)
if url_path.split('/')[0] == 'ego':
return any(
match_grant('/' + url_path, grant_url['url'])
for grant_url in grants)
else:
return any(
match_grant('/' + canonical_url, grant_url['url'], True)
for grant_url in grants)
def _check_grants(self, path, canonical_path, grants):
if not Api._has_grants(path, canonical_path, grants):
raise HTTPError('', 403, "Not Allowed", None, None)
def _check_expand_ego_grants(self, path, expand_dict, grants):
if expand_dict:
for field_name, nested in expand_dict.items():
nested_url = os.path.join(path, field_name)
if Api._has_grants(nested_url, '',
grants) or self._check_expand_ego_grants(
nested_url, expand_dict[field_name],
grants):
return True
return False
def patch(self, path, data, user=None):
path = path.strip().strip('/')
try:
tree = parse_canonical_url(path, self.schema.root)
except SyntaxError as te:
raise HTTPError('', 404, "Not Found", None, None)
aggregate_query, spec, is_aggregate = tree.aggregation(None)
if is_aggregate:
raise HTTPError('', 400, 'PATCH not supported on collections',
None, None)
if path.split('/')[0] == 'ego':
aggregate_query = [{
"$match": {
"username": user.username
}
}] + aggregate_query
cursor = tree.root_collection().aggregate(aggregate_query)
resource = next(cursor)
if user:
# TODO: also need to check read access to target if link
# checking for /ego paths first, then all other paths
self._check_grants(path, resource['_canonical_url'],
user.update_grants)
if spec.name == 'user' and data.get('password'):
data['password'] = generate_password_hash(data['password'])
return self.updater.update_fields(
spec.name, self.schema.encodeid(resource['_id']), data)
def put(self, path, data, user=None):
path = path.strip().strip('/')
from_path = data['_from'].strip('/') if data.get('_from') else None
at_index = data.get('_at')
if '/' in path:
parent_path, field_name = path.rsplit('/', 1)
try:
tree = parse_canonical_url(parent_path, self.schema.root)
except SyntaxError as te:
raise HTTPError('', 404, "Not Found", None, None)
aggregate_query, spec, is_aggregate = tree.aggregation(None)
field_spec = spec.fields[field_name]
if path.split('/')[0] == 'ego':
aggregate_query = [{
"$match": {
"username": user.username
}
}] + aggregate_query
# if we're using a simplified parser we can probably just pull the id off the path
cursor = tree.root_collection().aggregate(aggregate_query)
parent_resource = next(cursor)
# check permissions
if user:
self._check_grants(
path,
os.path.join(parent_resource['_canonical_url'],
field_name), user.put_grants)
self._check_grants(from_path, from_path, user.read_grants)
self._check_grants(from_path, from_path, user.delete_grants)
# do put update
try:
parse_url(from_path, self.schema.root)
except SyntaxError as te:
raise HTTPError('', 400, from_path, None, None)
if field_spec.field_type == 'collection':
return self.updater.move_resource(parent_resource['_id'],
spec.name, field_name, path,
from_path)
else:
raise HTTPError('', 400, from_path, None, None)
else:
if path not in self.schema.root.fields:
raise HTTPError('', 404, "Not Found", None, None)
root_field_spec = self.schema.root.fields[path]
root_spec = self.schema.specs[root_field_spec.target_spec_name]
# check permissions
if user:
self._check_grants(path, path, user.put_grants)
self._check_grants(from_path, from_path, user.read_grants)
self._check_grants(from_path, from_path, user.delete_grants)
if root_field_spec.target_spec_name == 'user':
data['password'] = generate_password_hash(data['password'])
# do put update
try:
if from_path:
parse_url(from_path, self.schema.root)
except SyntaxError as te:
raise HTTPError('', 400, from_path, None, None)
return self.updater.move_resource(None, 'root', path, path,
from_path)
def post(self, path, data, user=None):
path = path.strip().strip('/')
if '/' in path:
parent_path, field_name = path.rsplit('/', 1)
try:
tree = parse_canonical_url(parent_path, self.schema.root)
except SyntaxError as te:
raise HTTPError('', 404, "Not Found", None, None)
aggregate_query, spec, is_aggregate = tree.aggregation(None)
field_spec = spec.fields[field_name]
if path.split('/')[0] == 'ego':
aggregate_query = [{
"$match": {
"username": user.username
}
}] + aggregate_query
# if we're using a simplified parser we can probably just pull the id off the path
cursor = tree.root_collection().aggregate(aggregate_query)
parent_resource = next(cursor)
# check permissions
if user:
# TODO: also need to check read access to target if link
# checking for /ego paths first, then all other paths
self._check_grants(
path,
os.path.join(parent_resource['_canonical_url'],
field_name), user.create_grants)
parent_id = self.schema.encodeid(parent_resource['_id'])
if field_spec.field_type == 'linkcollection':
return self.updater.create_linkcollection_entry(
spec.name, parent_id, field_name, data['id'])
elif field_spec.field_type == 'orderedcollection':
return self.updater.create_orderedcollection_entry(
field_spec.target_spec_name, spec.name, field_name,
parent_id, data, self.schema.read_root_grants(path))
else:
return self.updater.create_resource(
field_spec.target_spec_name, spec.name, field_name,
parent_id, data, self.schema.read_root_grants(path))
else:
if path not in self.schema.root.fields:
raise HTTPError('', 404, "Not Found", None, None)
root_field_spec = self.schema.root.fields[path]
root_spec = self.schema.specs[root_field_spec.target_spec_name]
# check permissions
if user:
self._check_grants(path, path, user.create_grants)
if root_field_spec.target_spec_name == 'user':
data['password'] = generate_password_hash(data['password'])
# add to root spec no need to check existance
return self.updater.create_resource(
root_field_spec.target_spec_name, 'root', path, None, data,
self.schema.read_root_grants(path))
def delete(self, path, user=None):
path = path.strip().strip('/')
if '/' in path:
parent_field_path = '/'.join(path.split('/')[:-1])
resource_id = path.split('/')[-1]
try:
tree = parse_canonical_url(path, self.schema.root)
except ValueError as ve:
raise HTTPError('', 404, "Not Found", None, None)
parent_field_tree = parse_canonical_url(parent_field_path,
self.schema.root)
parent_path = '/'.join(parent_field_path.split('/')[:-1])
field_name = parent_field_path.split('/')[-1]
if type(parent_field_tree) == LinkCollectionResourceRef:
parent_tree = parse_canonical_url(parent_path,
self.schema.root)
aggregate_query, spec, is_aggregate = parent_tree.aggregation(
None)
if path.split('/')[0] == 'ego':
aggregate_query = [{
"$match": {
"username": user.username
}
}] + aggregate_query
# if we're using a simplified parser we can probably just pull the id off the path
cursor = tree.root_collection().aggregate(aggregate_query)
parent_resource = next(cursor)
if user:
# checking for /ego paths first, then all other paths
self._check_grants(parent_field_path,
parent_resource['_canonical_url'],
user.delete_grants)
return self.updater.delete_linkcollection_entry(
spec.name, parent_resource['_id'], field_name, resource_id)
elif type(parent_field_tree) == OrderedCollectionResourceRef:
parent_tree = parse_canonical_url(parent_path,
self.schema.root)
aggregate_query, spec, is_aggregate = parent_tree.aggregation(
None)
if path.split('/')[0] == 'ego':
aggregate_query = [{
"$match": {
"username": user.username
}
}] + aggregate_query
# if we're using a simplified parser we can probably just pull the id off the path
cursor = tree.root_collection().aggregate(aggregate_query)
parent_resource = next(cursor)
if user:
self._check_grants(parent_field_path,
parent_resource['_canonical_url'],
user.delete_grants)
return self.updater.delete_orderedcollection_entry(
spec.name, parent_resource['_id'], field_name, resource_id)
else:
aggregate_query, spec, is_aggregate = parent_field_tree.aggregation(
None)
if path.split('/')[0] == 'ego':
aggregate_query = [{
"$match": {
"username": user.username
}
}] + aggregate_query
cursor = tree.root_collection().aggregate(aggregate_query)
parent_resource = next(cursor)
if user:
self._check_grants(parent_field_path,
parent_resource['_canonical_url'],
user.delete_grants)
parent_spec_name = parent_field_tree.parent_spec.name if parent_field_tree.parent_spec else None
return self.updater.delete_resource(spec.name, resource_id,
parent_spec_name,
field_name)
else:
raise HTTPError('', 400, "Cannot delete root resource", None, None)
def _get_root(self):
root_resource = {
'auth': '/auth',
'ego': '/ego',
'users': '/users',
'groups': '/groups',
'employees': '/employees',
'divisions': '/division',
}
return root_resource
def get(self, path, args=None, user=None):
args = args or {}
expand = args.get('expand')
page = int(args.get('page', 0))
page_size = int(args.get('page_size', 10))
expand_dict = create_expand_dict(expand)
path = path.strip().strip('/')
if not path:
return self._get_root()
try:
tree = parse_url(path, self.schema.root)
except SyntaxError as te:
return None
aggregate_query, spec, is_aggregate = tree.aggregation(None)
if path.split('/')[0] == 'ego':
aggregate_query = [{
"$match": {
"username": user.username
}
}] + aggregate_query
if is_aggregate:
page_agg = self.create_pagination_aggregations(page, page_size)
if expand:
if user and self._check_expand_ego_grants(
path, expand_dict, user.read_grants):
expand_agg = self.create_field_expansion_aggregations(
spec, expand_dict)
else:
expand_agg = self.create_field_expansion_aggregations(
spec, expand_dict, user)
page_agg['$facet']["results"].extend(expand_agg)
aggregate_query.append(page_agg)
# run mongo query from from root_resource collection
cursor = tree.root_collection().aggregate(aggregate_query)
page_results = next(cursor)
results = list(page_results['results'])
count = page_results['count'][0]['total'] if page_results[
'count'] else 0
if user and count:
# TODO: also need to check read access to target if link
# checking for /ego paths first, then all other paths
self._check_grants(path, results[0]['_canonical_url'],
user.read_grants)
return {
"results": [
self.encode_resource(spec, row, expand_dict)
for row in results
],
"count":
count,
"next":
self._next_link(path, args, count, page, page_size),
"previous":
self._previous_link(path, args, count, page, page_size),
'_meta': {
'spec': {
'name': spec.name,
},
'is_collection':
True,
'can_create':
type(tree)
in (CollectionResourceRef, OrderedCollectionResourceRef,
RootResourceRef),
'can_link':
type(tree) in (LinkCollectionResourceRef, ),
}
}
else:
if expand:
if user and self._check_expand_ego_grants(
path, expand_dict, user.read_grants):
expand_agg = self.create_field_expansion_aggregations(
spec, expand_dict)
else:
expand_agg = self.create_field_expansion_aggregations(
spec, expand_dict, user)
aggregate_query.extend(expand_agg)
# run mongo query from from root_resource collection
cursor = tree.root_collection().aggregate(aggregate_query)
result = next(cursor, None)
if user and result:
# TODO: also need to check read access to target if link
# checking for /ego paths first, then all other paths
self._check_grants(path, result['_canonical_url'],
user.read_grants)
if result:
return self.encode_resource(spec, result, expand_dict)
else:
return None
def _next_link(self, path, args, count, page, page_size):
if count >= (page + 1) * page_size:
query = urlparse(request.url)
new_args = dict(args)
new_args['page'] = page + 1
new_args['page_size'] = page_size
new_q = urlencode(new_args)
query = query._replace(query=new_q)
return query.geturl()
else:
return None
def _previous_link(self, path, args, count, page, page_size):
if page:
query = urlparse(request.url)
new_args = dict(args)
new_args['page'] = page - 1
new_args['page_size'] = page_size
new_q = urlencode(new_args)
query = query._replace(query=new_q)
return query.geturl()
else:
return None
def get_spec_for(self, path, user=None):
path = path.strip().strip('/')
tree = parse_url(path, self.schema.root)
aggregate_query, spec, is_aggregate = tree.aggregation(None, user)
return (
spec,
is_aggregate,
type(tree) in (CollectionResourceRef, RootResourceRef),
type(tree) in (LinkCollectionResourceRef, ),
)
def create_pagination_aggregations(self, page, page_size):
return {
"$facet": {
"count": [{
"$count": "total"
}],
"results": [{
"$skip": page * page_size
}, {
"$limit": page_size
}],
}
}
def create_field_expansion_aggregations(self,
spec,
expand_dict,
user=None):
def lookup_agg(from_field, local_field, foreign_field, as_field,
expand_further):
agg = {
"$lookup": {
"from":
from_field,
"as":
as_field,
"let": {
"v_id": "$%s" % local_field,
},
"pipeline": [{
"$match": {
"$expr": {
"$eq": ["$$v_id",
"$%s" % foreign_field]
}
}
}]
}
}
for inner_field_name in expand_further:
inner_spec = spec.schema.specs[
spec.fields[inner_field_name].target_spec_name]
agg['$lookup']['pipeline'].extend(
self.create_field_expansion_aggregations(
inner_spec, expand_further[inner_field_name], user))
return agg
def lookup_collection_agg(from_field, local_field, foreign_field,
as_field, expand_further):
agg = {
"$lookup": {
"from":
from_field,
"as":
as_field,
"let": {
"v_id": "$%s" % local_field,
},
"pipeline": [{
"$match": {
"$expr": {
"$in": ["$%s" % foreign_field, "$$v_id"]
}
}
}]
}
}
for inner_field_name in expand_further:
inner_spec = spec.schema.specs[
spec.fields[inner_field_name].target_spec_name]
agg['$lookup']['pipeline'].extend(
self.create_field_expansion_aggregations(
inner_spec, expand_further[inner_field_name], user))
return agg
aggregate_query = []
for field_name in expand_dict:
if field_name not in spec.fields:
raise HTTPError(
'', 400, '%s not a field of %s' % (field_name, spec.name),
None, None)
field = spec.fields[field_name]
if field.field_type == 'link':
# add check for ' if in "expand" parameter'
aggregate_query.append(
lookup_agg("resource_%s" % field.target_spec_name,
field_name, "_id", "_expanded_%s" % field_name,
expand_dict))
aggregate_query.append(
{"$unwind": "$_expanded_%s" % field_name})
aggregate_query.append(
{"$set": {
field_name: "$_expanded_%s" % field_name
}})
elif field.field_type == 'reverse_link':
aggregate_query.append(
lookup_agg("resource_%s" % field.target_spec_name, "_id",
field.reverse_link_field,
"_expanded_%s" % field_name, expand_dict))
aggregate_query.append(
{"$unwind": "$_expanded_%s" % field_name})
aggregate_query.append(
{"$set": {
field_name: "$_expanded_%s" % field_name
}})
elif field.field_type in ('linkcollection', 'orderedcollection'):
aggregate_query.append(
lookup_collection_agg(
"resource_%s" % field.target_spec_name,
"%s._id" % field.name, "_id",
"_expanded_%s" % field_name, expand_dict))
aggregate_query.append(
{"$set": {
field_name: "$_expanded_%s" % field_name
}})
elif field.field_type == 'collection':
aggregate_query.append(
lookup_agg("resource_%s" % field.target_spec_name, "_id",
"_parent_id", "_expanded_%s" % field_name,
expand_dict))
aggregate_query.append(
{"$set": {
field_name: "$_expanded_%s" % field_name
}})
elif field.field_type == 'parent_collection':
aggregate_query.append(
lookup_agg("resource_%s" % spec.name, "_parent_id", "_id",
"_expanded_%s" % field_name, expand_dict))
aggregate_query.append(
{"$set": {
field_name: "$_expanded_%s" % field_name
}})
else:
raise HTTPError(
'', 400, 'Unable to expand field %s of type %s' %
(field_name, field.field_type), None, None)
if user:
aggregate_query.append(
{"$match": {
"_grants": {
"$in": user.grants
}
}})
return aggregate_query
def encode_resource(self, spec, resource_data, expand_dict):
self_url = os.path.join(resource_data['_parent_canonical_url'],
resource_data['_parent_field_name'],
self.schema.encodeid(resource_data['_id']))
encoded = {
'id': self.schema.encodeid(resource_data['_id']),
'self': self_url,
'_meta': {
'spec': {
'name': spec.name,
},
'is_collection': False,
}
}
for field_name, field in spec.fields.items():
field_value = resource_data.get(field_name)
if field.field_type == 'link':
if field_value:
if field_name in expand_dict:
encoded[field_name] = self.encode_resource(
self.schema.specs[field.target_spec_name],
resource_data[field_name], expand_dict[field_name])
else:
encoded[field_name] = resource_data['_canonical_url_%s'
% field_name]
else:
encoded[field_name] = None
elif field.field_type == 'parent_collection' and resource_data.get(
'_parent_id'):
encoded[field_name] = resource_data['_parent_canonical_url']
elif field.field_type in ('reverse_link', ):
if field_name in expand_dict:
encoded[field_name] = self.encode_resource(
self.schema.specs[field.target_spec_name],
resource_data[field_name], expand_dict[field_name])
else:
# TODO: A canonical link would be better
encoded[field_name] = os.path.join(self_url, field_name)
elif field.field_type in ('linkcollection', 'collection',
'reverse_link_collection',
'orderedcollection'):
if field_name in expand_dict:
encoded[field_name] = [
self.encode_resource(
self.schema.specs[field.target_spec_name], citem,
expand_dict[field_name])
for citem in resource_data[field_name]
]
else:
encoded[field_name] = os.path.join(self_url, field_name)
elif field.field_type == 'calc':
tree = parse(field.calc_str, spec)
res_type = tree.infer_type()
calc_result = resource_data.get(field_name)
if res_type.is_primitive():
if tree.is_collection() and calc_result is not None:
encoded[field_name] = [
res[res_type.name] for res in calc_result
]
else:
encoded[field_name] = calc_result
elif tree.is_collection():
encoded[field_name] = os.path.join(self_url, field_name)
else:
encoded[field_name] = resource_data[
'_canonical_url_%s' %
field.name] if calc_result else None
elif spec.name == 'user' and field_name == 'password':
encoded[field_name] = '<password>'
else:
encoded[field_name] = field_value
return encoded
def search_resource(self, spec_name, query_str, page=0, page_size=10):
spec = self.schema.specs[spec_name]
if query_str:
query = parse_filter(query_str, spec)
query = query.condition_aggregation(spec, None)
else:
query = {}
pagination = self.create_pagination_aggregations(page, page_size)
aggregation = [
{
"$match": query
},
pagination,
]
cursor = self.schema.db['resource_%s' %
spec_name].aggregate(aggregation)
page_results = next(cursor)
results = list(page_results['results'])
count = page_results['count'][0]['total'] if page_results[
'count'] else 0
return {
"results":
[self.encode_resource(spec, row, {}) for row in results],
"count": count,
"next": self._next_link(None, {}, count, page, page_size),
"previous": self._previous_link(None, {}, count, page, page_size),
'_meta': {
'spec': {
'name': spec.name,
},
'is_collection': True,
}
}
class UpdaterTest(unittest.TestCase):
def setUp(self):
self.maxDiff = None
client = MongoClient()
client.drop_database('metaphor2_test_db')
self.db = client.metaphor2_test_db
self.schema = Schema(self.db)
self.updater = Updater(self.schema)
self.employee_spec = self.schema.add_spec('employee')
self.schema.add_field(self.employee_spec, 'name', 'str')
self.schema.add_field(self.employee_spec, 'age', 'int')
self.division_spec = self.schema.add_spec('division')
self.schema.add_field(self.division_spec, 'name', 'str')
self.schema.add_field(self.division_spec, 'employees',
'linkcollection', 'employee')
self.schema.add_calc(self.division_spec, 'older_employees',
'self.employees[age>30]')
self.schema.add_field(self.schema.root, 'divisions', 'collection',
'division')
self.schema.add_field(self.schema.root, 'employees', 'collection',
'employee')
def test_update_only_linked_resources(self):
employee_id_1 = self.schema.insert_resource('employee', {
'name': 'ned',
'age': 10
}, 'employees')
employee_id_2 = self.schema.insert_resource('employee', {
'name': 'bob',
'age': 10
}, 'employees')
division_id_1 = self.schema.insert_resource('division',
{'name': 'sales'},
'divisions')
division_id_2 = self.schema.insert_resource('division',
{'name': 'marketting'},
'divisions')
self.schema.create_linkcollection_entry('division', division_id_1,
'employees', employee_id_1)
self.schema.create_linkcollection_entry('division', division_id_2,
'employees', employee_id_1)
self.schema.create_linkcollection_entry('division', division_id_2,
'employees', employee_id_2)
self.assertEqual([
self.schema.decodeid(division_id_1),
self.schema.decodeid(division_id_2)
],
self.updater.get_affected_ids_for_resource(
'division', 'older_employees', self.employee_spec,
employee_id_1))
self.assertEqual([self.schema.decodeid(division_id_2)],
self.updater.get_affected_ids_for_resource(
'division', 'older_employees', self.employee_spec,
employee_id_2))
class UpdaterTest(unittest.TestCase):
def setUp(self):
self.maxDiff = None
client = MongoClient()
client.drop_database('metaphor2_test_db')
self.db = client.metaphor2_test_db
self.schema = Schema(self.db)
self.updater = Updater(self.schema)
self.schema.create_initial_schema()
self.company_spec = self.schema.add_spec('company')
self.employee_spec = self.schema.add_spec('employee')
self.division_spec = self.schema.add_spec('division')
self.schema.add_field(self.company_spec, 'divisions', 'collection', 'division')
self.schema.add_field(self.employee_spec, 'name', 'str')
self.schema.add_field(self.division_spec, 'employees', 'collection', 'employee')
self.schema.add_field(self.schema.root, 'companies', 'collection', 'company')
def test_delete_group(self):
self.admin_group_id = self.updater.create_resource('group', 'root', 'groups', None, {'name': 'admin'}, self.schema.read_root_grants('groups'))
self.grant_id = self.updater.create_resource('grant', 'group', 'grants', self.admin_group_id, {'type': 'read', 'url': '/companies'})
self.user_id = self.updater.create_resource('user', 'root', 'users', None, {'username': '******', 'password': '******', 'admin': True}, self.schema.read_root_grants('users'))
self.updater.create_linkcollection_entry('user', self.user_id, 'groups', self.admin_group_id)
user_db = self.db['resource_user'].find_one()
self.assertEqual(1, len(user_db['read_grants']))
self.updater.delete_linkcollection_entry('user', self.schema.decodeid(self.user_id), 'groups', self.admin_group_id)
user_db = self.db['resource_user'].find_one()
self.assertEqual(0, len(user_db['read_grants']))
def test_root_grants(self):
group_id = self.updater.create_resource('group', 'root', 'groups', None, {'name': 'readall'}, self.schema.read_root_grants('groups'))
grant_id = self.updater.create_resource('grant', 'group', 'grants', group_id, {'type': 'read', 'url': '/'})
company_id = self.updater.create_resource('company', 'root', 'companies', None, {}, self.schema.read_root_grants('companies'))
company_data = self.db['resource_company'].find_one({})
self.assertEqual([self.schema.decodeid(grant_id)], company_data['_grants'])
def test_nested_grants(self):
group_id = self.updater.create_resource('group', 'root', 'groups', None, {'name': 'readall'}, self.schema.read_root_grants('groups'))
grant_id = self.updater.create_resource('grant', 'group', 'grants', group_id, {'type': 'read', 'url': '/'})
company_id = self.updater.create_resource('company', 'root', 'companies', None, {}, self.schema.read_root_grants('companies'))
company_path = "companies/%s" % company_id
division_id_1 = self.updater.create_resource('division', 'company', 'divisions', company_id, {}, self.schema.read_root_grants(company_path))
division_id_2 = self.updater.create_resource('division', 'company', 'divisions', company_id, {}, self.schema.read_root_grants(company_path))
division_1_path = "companies/%s/divisions/%s" % (company_id, division_id_1)
employee_id_1 = self.updater.create_resource('employee', 'division', 'employees', division_id_1, {}, self.schema.read_root_grants(division_1_path))
employee_id_2 = self.updater.create_resource('employee', 'division', 'employees', division_id_1, {}, self.schema.read_root_grants(division_1_path))
division_2_path = "companies/%s/divisions/%s" % (company_id, division_id_2)
employee_id_3 = self.updater.create_resource('employee', 'division', 'employees', division_id_2, {}, self.schema.read_root_grants(division_2_path))
employee_id_4 = self.updater.create_resource('employee', 'division', 'employees', division_id_2, {}, self.schema.read_root_grants(division_2_path))
# check grants
company_data = self.db['resource_company'].find_one({})
self.assertEqual([self.schema.decodeid(grant_id)], company_data['_grants'])
division_1_data = self.db['resource_division'].find_one({"_id": self.schema.decodeid(division_id_1)})
self.assertEqual([self.schema.decodeid(grant_id)], division_1_data['_grants'])
division_2_data = self.db['resource_division'].find_one({"_id": self.schema.decodeid(division_id_2)})
self.assertEqual([self.schema.decodeid(grant_id)], division_2_data['_grants'])
employee_1_data = self.db['resource_employee'].find_one({"_id": self.schema.decodeid(employee_id_1)})
self.assertEqual([self.schema.decodeid(grant_id)], employee_1_data['_grants'])
employee_2_data = self.db['resource_employee'].find_one({"_id": self.schema.decodeid(employee_id_2)})
self.assertEqual([self.schema.decodeid(grant_id)], employee_2_data['_grants'])
employee_3_data = self.db['resource_employee'].find_one({"_id": self.schema.decodeid(employee_id_3)})
self.assertEqual([self.schema.decodeid(grant_id)], employee_3_data['_grants'])
employee_4_data = self.db['resource_employee'].find_one({"_id": self.schema.decodeid(employee_id_4)})
self.assertEqual([self.schema.decodeid(grant_id)], employee_4_data['_grants'])
def test_deleting_grant_removes_grant_id(self):
group_id = self.updater.create_resource('group', 'root', 'groups', None, {'name': 'readall'}, self.schema.read_root_grants('groups'))
grant_id = self.updater.create_resource('grant', 'group', 'grants', group_id, {'type': 'read', 'url': '/'})
company_id = self.updater.create_resource('company', 'root', 'companies', None, {}, self.schema.read_root_grants('companies'))
self.updater.delete_resource('grant', grant_id, 'group', 'grants')
company_data = self.db['resource_company'].find_one({})
self.assertEqual([], company_data['_grants'])
class UpdaterTest(unittest.TestCase):
def setUp(self):
self.maxDiff = None
client = MongoClient()
client.drop_database('metaphor2_test_db')
self.db = client.metaphor2_test_db
self.schema = Schema(self.db)
self.schema.create_initial_schema()
self.updater = Updater(self.schema)
self.employee_spec = self.schema.create_spec('employee')
self.schema.create_field('employee', 'name', 'str')
self.schema.create_field('employee', 'age', 'int')
self.division_spec = self.schema.create_spec('division')
self.schema.create_field('division', 'name', 'str')
self.schema.create_field('division', 'employees', 'collection',
'employee')
self.schema.create_field('root', 'divisions', 'collection', 'division')
def test_updater(self):
self.schema.add_calc(self.division_spec, 'older_employees',
'self.employees[age>30]')
division_id_1 = self.schema.insert_resource('division',
{'name': 'sales'},
'divisions')
employee_id_1 = self.schema.insert_resource('employee', {
'name': 'bob',
'age': 31
}, 'employees', 'division', division_id_1)
self.updater.update_calc('division', 'older_employees', division_id_1)
division_data = self.db.resource_division.find_one()
self.assertEquals(
{
'_id': self.schema.decodeid(division_id_1),
'_grants': [],
'_canonical_url': '/divisions/%s' % division_id_1,
'name': 'sales',
'_parent_canonical_url': '/',
'_parent_field_name': 'divisions',
'_parent_id': None,
'_parent_type': 'root',
'older_employees': [ObjectId(employee_id_1[2:])],
}, division_data)
employee_id_2 = self.schema.insert_resource('employee', {
'name': 'Ned',
'age': 41
}, 'employees', 'division', division_id_1)
# check again
self.updater.update_calc('division', 'older_employees', division_id_1)
division_data = self.db.resource_division.find_one()
self.assertEquals(
{
'_id':
self.schema.decodeid(division_id_1),
'_grants': [],
'_canonical_url':
'/divisions/%s' % division_id_1,
'name':
'sales',
'_parent_canonical_url':
'/',
'_parent_field_name':
'divisions',
'_parent_id':
None,
'_parent_type':
'root',
'older_employees':
[ObjectId(employee_id_1[2:]),
ObjectId(employee_id_2[2:])],
}, division_data)
def test_reverse_aggregation(self):
self.schema.add_calc(self.division_spec, 'older_employees',
'self.employees[age>30]')
self.schema.add_calc(self.division_spec, 'average_age',
'average(self.employees.age)')
division_id_1 = self.schema.insert_resource('division',
{'name': 'sales'},
'divisions')
employee_id_1 = self.schema.insert_resource('employee', {
'name': 'bob',
'age': 31
}, 'employees', 'division', division_id_1)
division_id_2 = self.schema.insert_resource('division',
{'name': 'sales'},
'divisions')
employee_id_2 = self.schema.insert_resource('employee', {
'name': 'bob',
'age': 31
}, 'employees', 'division', division_id_2)
average_agg = self.updater.build_reverse_aggregations_to_calc(
'division', 'average_age', self.employee_spec, employee_id_1)
self.assertEquals([[
{
"$match": {
"_id": self.schema.decodeid(employee_id_1)
}
},
{
"$lookup": {
"from": "resource_division",
"localField": "_parent_id",
"foreignField": "_id",
"as": "_field_employees",
}
},
{
'$group': {
'_id': '$_field_employees'
}
},
{
"$unwind": "$_id"
},
{
"$replaceRoot": {
"newRoot": "$_id"
}
},
]], average_agg)
affected_ids = self.updater.get_affected_ids_for_resource(
'division', 'average_age', self.employee_spec, employee_id_1)
self.assertEquals([self.schema.decodeid(division_id_1)],
list(affected_ids))
# check another collection
employee_id_3 = self.schema.insert_resource('employee', {
'name': 'bob',
'age': 31
}, 'employees', 'division', division_id_2)
affected_ids = self.updater.get_affected_ids_for_resource(
'division', 'average_age', self.employee_spec, employee_id_3)
self.assertEquals([self.schema.decodeid(division_id_2)],
list(affected_ids))
# different calc
older_agg = self.updater.build_reverse_aggregations_to_calc(
'division', 'older_employees', self.employee_spec, employee_id_1)
self.assertEquals([[
{
"$match": {
"_id": self.schema.decodeid(employee_id_1)
}
},
{
"$lookup": {
"from": "resource_division",
"localField": "_parent_id",
"foreignField": "_id",
"as": "_field_employees",
}
},
{
'$group': {
'_id': '$_field_employees'
}
},
{
"$unwind": "$_id"
},
{
"$replaceRoot": {
"newRoot": "$_id"
}
},
]], average_agg)
def test_reverse_aggregation_link(self):
self.schema.add_field(self.division_spec, 'manager', 'link',
'employee')
self.schema.add_calc(self.division_spec, 'manager_age',
'self.manager.age')
division_id_1 = self.schema.insert_resource('division',
{'name': 'sales'},
'divisions')
employee_id_1 = self.schema.insert_resource('employee', {
'name': 'bob',
'age': 31
}, 'employees', 'division', division_id_1)
self.schema.update_resource_fields('division', division_id_1,
{'manager': employee_id_1})
agg = self.updater.build_reverse_aggregations_to_calc(
'division', 'manager_age', self.employee_spec, employee_id_1)
self.assertEquals([[
{
"$match": {
"_id": self.schema.decodeid(employee_id_1)
}
},
{
"$lookup": {
"from": "resource_division",
"localField": "_id",
"foreignField": "manager",
"as": "_field_manager",
}
},
{
'$group': {
'_id': '$_field_manager'
}
},
{
"$unwind": "$_id"
},
{
"$replaceRoot": {
"newRoot": "$_id"
}
},
]], agg)
# check affected ids
affected_ids = self.updater.get_affected_ids_for_resource(
'division', 'manager_age', self.employee_spec, employee_id_1)
self.assertEquals([self.schema.decodeid(division_id_1)],
list(affected_ids))
# check having two links
division_id_2 = self.schema.insert_resource('division',
{'name': 'sales'},
'divisions')
self.schema.update_resource_fields('division', division_id_2,
{'manager': employee_id_1})
affected_ids = self.updater.get_affected_ids_for_resource(
'division', 'manager_age', self.employee_spec, employee_id_1)
self.assertEquals([
self.schema.decodeid(division_id_1),
self.schema.decodeid(division_id_2)
], list(affected_ids))
def test_reverse_aggregation_link_collection(self):
self.schema.add_field(self.division_spec, 'managers', 'linkcollection',
'employee')
self.schema.add_calc(self.division_spec, 'average_manager_age',
'average(self.managers.age)')
division_id_1 = self.schema.insert_resource('division',
{'name': 'sales'},
'divisions')
employee_id_1 = self.schema.insert_resource('employee', {
'name': 'bob',
'age': 31
}, 'employees', 'division', division_id_1)
self.schema.create_linkcollection_entry('division', division_id_1,
'managers', employee_id_1)
agg = self.updater.build_reverse_aggregations_to_calc(
'division', 'average_manager_age', self.employee_spec,
employee_id_1)
self.assertEquals([[
{
"$match": {
"_id": self.schema.decodeid(employee_id_1)
}
},
{
"$lookup": {
"from": "resource_division",
"foreignField": "managers._id",
"localField": "_id",
"as": "_field_managers",
}
},
{
'$group': {
'_id': '$_field_managers'
}
},
{
"$unwind": "$_id"
},
{
"$replaceRoot": {
"newRoot": "$_id"
}
},
]], agg)
# check affected ids
affected_ids = self.updater.get_affected_ids_for_resource(
'division', 'average_manager_age', self.employee_spec,
employee_id_1)
self.assertEquals([self.schema.decodeid(division_id_1)],
list(affected_ids))
division_id_2 = self.schema.insert_resource('division',
{'name': 'marketting'},
'divisions')
self.schema.create_linkcollection_entry('division', division_id_2,
'managers', employee_id_1)
affected_ids = self.updater.get_affected_ids_for_resource(
'division', 'average_manager_age', self.employee_spec,
employee_id_1)
self.assertEquals([
self.schema.decodeid(division_id_1),
self.schema.decodeid(division_id_2)
], list(affected_ids))
def test_reverse_aggregation_calc_through_calc(self):
self.schema.add_calc(self.division_spec, 'older_employees',
'self.employees[age>30]')
self.schema.add_calc(self.division_spec, 'older_employees_called_ned',
'self.older_employees[name="ned"]')
division_id_1 = self.schema.insert_resource('division',
{'name': 'sales'},
'divisions')
division_id_2 = self.schema.insert_resource('division',
{'name': 'marketting'},
'divisions')
employee_id_1 = self.schema.insert_resource('employee', {
'name': 'bob',
'age': 21
}, 'employees', 'division', division_id_1)
employee_id_2 = self.schema.insert_resource('employee', {
'name': 'ned',
'age': 31
}, 'employees', 'division', division_id_1)
employee_id_3 = self.schema.insert_resource('employee', {
'name': 'fred',
'age': 41
}, 'employees', 'division', division_id_1)
employee_id_4 = self.schema.insert_resource('employee', {
'name': 'sam',
'age': 25
}, 'employees', 'division', division_id_2)
employee_id_5 = self.schema.insert_resource('employee', {
'name': 'ned',
'age': 35
}, 'employees', 'division', division_id_2)
self.updater.update_calc('division', 'older_employees', division_id_1)
self.updater.update_calc('division', 'older_employees_called_ned',
division_id_1)
self.updater.update_calc('division', 'older_employees', division_id_2)
self.updater.update_calc('division', 'older_employees_called_ned',
division_id_2)
agg = self.updater.build_reverse_aggregations_to_calc(
'division', 'older_employees_called_ned', self.employee_spec,
employee_id_2)
self.assertEquals([[
{
"$match": {
"_id": self.schema.decodeid(employee_id_2)
}
},
{
"$lookup": {
"from": "resource_division",
"foreignField": "older_employees",
"localField": "_id",
"as": "_field_older_employees",
}
},
{
'$group': {
'_id': '$_field_older_employees'
}
},
{
"$unwind": "$_id"
},
{
"$replaceRoot": {
"newRoot": "$_id"
}
},
]], agg)
# check affected ids
affected_ids = self.updater.get_affected_ids_for_resource(
'division', 'older_employees_called_ned', self.employee_spec,
employee_id_2)
self.assertEquals([self.schema.decodeid(division_id_1)],
list(affected_ids))
def test_reverse_aggregation_parent_link(self):
self.schema.add_calc(self.employee_spec, 'division_name',
'self.parent_division_employees.name')
division_id_1 = self.schema.insert_resource('division',
{'name': 'sales'},
'divisions')
employee_id_1 = self.schema.insert_resource('employee', {
'name': 'bob',
'age': 31
}, 'employees', 'division', division_id_1)
self.updater.update_calc('employee', 'division_name', employee_id_1)
agg = self.updater.build_reverse_aggregations_to_calc(
'employee', 'division_name', self.division_spec, division_id_1)
self.assertEquals([[
{
"$match": {
"_id": self.schema.decodeid(division_id_1)
}
},
{
"$lookup": {
"from": "resource_employee",
"foreignField": "_parent_id",
"localField": "_id",
"as": "_field_parent_division_employees",
}
},
{
'$group': {
'_id': '$_field_parent_division_employees'
}
},
{
"$unwind": "$_id"
},
{
"$replaceRoot": {
"newRoot": "$_id"
}
},
]], agg)
# check affected ids
affected_ids = self.updater.get_affected_ids_for_resource(
'employee', 'division_name', self.division_spec, division_id_1)
self.assertEquals([self.schema.decodeid(employee_id_1)],
list(affected_ids))
def test_reverse_aggregation_reverse_link(self):
self.schema.add_field(self.division_spec, 'manager', 'link',
'employee')
self.schema.add_calc(self.employee_spec, 'divisions_i_manage',
'self.link_division_manager')
division_id_1 = self.schema.insert_resource('division',
{'name': 'sales'},
'divisions')
division_id_2 = self.schema.insert_resource('division',
{'name': 'marketting'},
'divisions')
employee_id_1 = self.schema.insert_resource('employee', {
'name': 'bob',
'age': 31
}, 'employees', 'division', division_id_1)
self.schema.update_resource_fields('division', division_id_1,
{'manager': employee_id_1})
self.schema.update_resource_fields('division', division_id_2,
{'manager': employee_id_1})
self.updater.update_calc('employee', 'divisions_i_manage',
employee_id_1)
agg = self.updater.build_reverse_aggregations_to_calc(
'employee', 'divisions_i_manage', self.division_spec,
division_id_1)
self.assertEquals([[
{
"$match": {
"_id": self.schema.decodeid(division_id_1)
}
},
{
"$lookup": {
"from": "resource_employee",
"foreignField": "_id",
"localField": "manager",
"as": "_field_link_division_manager",
}
},
{
'$group': {
'_id': '$_field_link_division_manager'
}
},
{
"$unwind": "$_id"
},
{
"$replaceRoot": {
"newRoot": "$_id"
}
},
]], agg)
division_id_2 = self.schema.insert_resource('division',
{'name': 'marketting'},
'divisions')
self.schema.create_linkcollection_entry('division', division_id_2,
'managers', employee_id_1)
def test_reverse_aggregation_loopback(self):
self.schema.add_field(self.division_spec, 'managers', 'linkcollection',
'employee')
self.schema.add_calc(self.employee_spec, 'all_my_subordinates',
'self.link_division_managers.employees')
division_id_1 = self.schema.insert_resource('division',
{'name': 'sales'},
'divisions')
employee_id_1 = self.schema.insert_resource('employee', {
'name': 'bob',
'age': 21
}, 'employees', 'division', division_id_1)
employee_id_2 = self.schema.insert_resource('employee', {
'name': 'ned',
'age': 31
}, 'employees', 'division', division_id_1)
employee_id_3 = self.schema.insert_resource('employee', {
'name': 'fred',
'age': 41
}, 'employees', 'division', division_id_1)
employee_id_4 = self.schema.insert_resource('employee', {
'name': 'mike',
'age': 51
}, 'employees', 'division', division_id_1)
# add manager
self.updater.create_linkcollection_entry('division', division_id_1,
'managers', employee_id_1)
# bobs addition alters bobs calc
self.assertEquals([self.schema.decodeid(employee_id_1)],
list(
self.updater.get_affected_ids_for_resource(
'employee', 'all_my_subordinates',
self.employee_spec, employee_id_1)))
# a little unsure of this
agg = self.updater.build_reverse_aggregations_to_calc(
'employee', 'all_my_subordinates', self.division_spec,
division_id_1)
self.assertEquals([[
{
"$match": {
"_id": self.schema.decodeid(division_id_1)
}
},
{
'$lookup': {
'as': '_field_link_division_managers',
'foreignField': '_id',
'from': 'resource_employee',
'localField': 'managers._id'
}
},
{
'$group': {
'_id': '$_field_link_division_managers'
}
},
{
'$unwind': '$_id'
},
{
"$replaceRoot": {
"newRoot": "$_id"
}
},
]], agg)
def test_reverse_aggregation_simple_collection(self):
self.schema.add_calc(self.division_spec, 'all_employees',
'self.employees')
division_id_1 = self.schema.insert_resource('division',
{'name': 'sales'},
'divisions')
employee_id_1 = self.updater.create_resource('employee', 'division',
'employees',
division_id_1, {
'name': 'bob',
'age': 21
})
employee_id_2 = self.updater.create_resource('employee', 'division',
'employees',
division_id_1, {
'name': 'ned',
'age': 31
})
self.assertEquals([self.schema.decodeid(division_id_1)],
list(
self.updater.get_affected_ids_for_resource(
'division', 'all_employees',
self.employee_spec, employee_id_1)))
def test_reverse_aggregation_switch(self):
self.schema.add_calc(
self.division_spec, 'all_employees',
'self.name -> ("sales": (self.employees[age > 25]), "marketting": self.employees)'
)
division_id_1 = self.schema.insert_resource('division',
{'name': 'sales'},
'divisions')
employee_id_1 = self.updater.create_resource('employee', 'division',
'employees',
division_id_1, {
'name': 'bob',
'age': 21
})
employee_id_2 = self.updater.create_resource('employee', 'division',
'employees',
division_id_1, {
'name': 'ned',
'age': 31
})
self.assertEquals({self.schema.decodeid(division_id_1)},
set(
self.updater.get_affected_ids_for_resource(
'division', 'all_employees',
self.employee_spec, employee_id_1)))
def test_reverse_aggregation_ternary(self):
self.schema.add_calc(
self.division_spec, 'all_employees',
'self.name = "sales" -> (self.employees[age > 25]) : self.employees'
)
division_id_1 = self.schema.insert_resource('division',
{'name': 'sales'},
'divisions')
employee_id_1 = self.updater.create_resource('employee', 'division',
'employees',
division_id_1, {
'name': 'bob',
'age': 21
})
employee_id_2 = self.updater.create_resource('employee', 'division',
'employees',
division_id_1, {
'name': 'ned',
'age': 31
})
self.assertEquals({self.schema.decodeid(division_id_1)},
set(
self.updater.get_affected_ids_for_resource(
'division', 'all_employees',
self.employee_spec, employee_id_1)))
def test_delete_resource_deletes_children(self):
division_id_1 = self.schema.insert_resource('division',
{'name': 'sales'},
'divisions')
employee_id_1 = self.updater.create_resource('employee', 'division',
'employees',
division_id_1, {
'name': 'bob',
'age': 21
})
employee_id_2 = self.updater.create_resource('employee', 'division',
'employees',
division_id_1, {
'name': 'ned',
'age': 31
})
self.assertEqual(1, self.db['resource_division'].count())
self.assertEqual(2, self.db['resource_employee'].count())
self.updater.delete_resource('division', division_id_1, None,
'divisions')
self.assertEqual(0, self.db['resource_division'].count())
self.assertEqual(0, self.db['resource_employee'].count())
def test_delete_resource_deletes_links_to_resource(self):
self.schema.add_field(self.division_spec, 'employees',
'linkcollection', 'employee')
self.schema.add_field(self.division_spec, 'manager', 'link',
'employee')
division_id_1 = self.schema.insert_resource('division',
{'name': 'sales'},
'divisions')
employee_id_1 = self.schema.insert_resource('employee',
{'name': 'Fred'},
'employees')
self.schema.create_linkcollection_entry('division', division_id_1,
'employees', employee_id_1)
self.schema.update_resource_fields('division', division_id_1,
{'manager': employee_id_1})
self.updater.delete_resource('employee', employee_id_1, 'root',
'employees')
self.assertEqual(
{
'_canonical_url': '/divisions/%s' % division_id_1,
'_canonical_url_manager': '/employees/%s' % employee_id_1,
'_grants': [],
'_id': self.schema.decodeid(division_id_1),
'_parent_canonical_url': '/',
'_parent_field_name': 'divisions',
'_parent_id': None,
'_parent_type': 'root',
'employees': [],
'manager': None,
'name': 'sales'
}, self.db['resource_division'].find_one())
def test_updates_calc_linked_to_calc(self):
self.schema.create_field('root', 'parttimers', 'collection',
'employee')
self.schema.create_field('employee', 'income', 'int')
self.schema.create_field('employee', 'vat', 'int')
self.schema.create_field('employee',
'income_after_vat',
'calc',
calc_str='self.income - self.vat')
self.schema.create_field('division', 'parttimers', 'linkcollection',
'employee')
self.schema.create_field(
'division',
'employee_total',
'calc',
calc_str="sum(self.employees.income_after_vat)")
self.schema.create_field(
'division',
'parttime_total',
'calc',
calc_str="sum(self.parttimers.income_after_vat)")
division_id_1 = self.updater.create_resource('division', 'root',
'divisions', None,
{'name': 'sales'})
employee_id_1 = self.updater.create_resource('employee', 'division',
'employees',
division_id_1, {
'name': 'Fred',
'income': 10000,
'vat': 2000
})
employee_id_2 = self.updater.create_resource('employee', 'division',
'employees',
division_id_1, {
'name': 'Ned',
'income': 20000,
'vat': 4000
})
employee_id_3 = self.updater.create_resource('employee', 'root',
'parttimers', None, {
'name': 'Bob',
'income': 40000,
'vat': 8000
})
self.updater.create_linkcollection_entry('division', division_id_1,
'parttimers', employee_id_3)
self.assertEqual(
24000, self.db['resource_division'].find_one()['employee_total'])
self.assertEqual(
32000, self.db['resource_division'].find_one()['parttime_total'])
# assert calc change propagates
self.updater.update_fields('employee', employee_id_3, {'vat': 9000})
self.assertEqual(
24000, self.db['resource_division'].find_one()['employee_total'])
self.assertEqual(
31000, self.db['resource_division'].find_one()['parttime_total'])
def test_update_adjacent_calc_after_update(self):
self.schema.create_field(
'employee',
'division_name',
'calc',
calc_str='self.parent_division_employees.name')
self.schema.create_field('employee',
'both_names',
'calc',
calc_str='self.name + self.division_name')
division_id_1 = self.updater.create_resource('division', 'root',
'divisions', None,
{'name': 'sales'})
employee_id_1 = self.updater.create_resource('employee', 'division',
'employees',
division_id_1,
{'name': 'Fred'})
self.assertEqual('Fredsales',
self.db['resource_employee'].find_one()['both_names'])
self.updater.update_fields('division', division_id_1,
{'name': 'marketting'})
self.assertEqual('Fredmarketting',
self.db['resource_employee'].find_one()['both_names'])