def make_fixtures(self):
self.app.post('/api/1/networks',
headers=AUTHZ_HEADER,
data=NETWORK_FIXTURE)
network = Network.by_slug(NETWORK_FIXTURE['slug'])
Schema.create(network, Schema.RELATION,
h.TEST_RELATION_SCHEMA)
Schema.create(network, Schema.ENTITY,
h.TEST_ENTITY_SCHEMA)
res = self.app.post('/api/1/net/entities',
headers=AUTHZ_HEADER,
data=ENTITY1_FIXTURE,
follow_redirects=True)
body = json.loads(res.data)
self.source_id = body['id']
res = self.app.post('/api/1/net/entities',
headers=AUTHZ_HEADER,
data=ENTITY2_FIXTURE,
follow_redirects=True)
body = json.loads(res.data)
self.target_id = body['id']
RELATION_FIXTURE['source'] = self.source_id
RELATION_FIXTURE['target'] = self.target_id
res = self.app.post('/api/1/net/relations',
headers=AUTHZ_HEADER,
data=RELATION_FIXTURE,
follow_redirects=True)
print res.data
body = json.loads(res.data)
self.id = body['id']
def make_fixtures(self):
self.app.post('/api/1/networks',
headers=AUTHZ_HEADER,
data=NETWORK_FIXTURE)
network = Network.by_slug(NETWORK_FIXTURE['slug'])
Schema.create(network, Schema.RELATION, h.TEST_RELATION_SCHEMA)
Schema.create(network, Schema.ENTITY, h.TEST_ENTITY_SCHEMA)
res = self.app.post('/api/1/net/entities',
headers=AUTHZ_HEADER,
data=ENTITY1_FIXTURE,
follow_redirects=True)
body = json.loads(res.data)
self.source_id = body['id']
res = self.app.post('/api/1/net/entities',
headers=AUTHZ_HEADER,
data=ENTITY2_FIXTURE,
follow_redirects=True)
body = json.loads(res.data)
self.target_id = body['id']
RELATION_FIXTURE['source'] = self.source_id
RELATION_FIXTURE['target'] = self.target_id
res = self.app.post('/api/1/net/relations',
headers=AUTHZ_HEADER,
data=RELATION_FIXTURE,
follow_redirects=True)
print res.data
body = json.loads(res.data)
self.id = body['id']
def setUp(self):
self.app = make_test_app()
self.app.post('/api/1/networks',
headers=AUTHZ_HEADER,
data=NETWORK_FIXTURE)
network = Network.by_slug(NETWORK_FIXTURE['slug'])
Schema.create(network, Schema.RELATION, h.TEST_RELATION_SCHEMA)
Schema.create(network, Schema.ENTITY, h.TEST_ENTITY_SCHEMA)
db.session.commit()
def setUp(self):
self.client = h.make_test_app()
self.network = Network.create({'title': 'Net', 'slug': 'net'})
self.schema = Schema.create(self.network, Schema.ENTITY,
h.TEST_ENTITY_SCHEMA)
self.rschema = Schema.create(self.network, Schema.RELATION,
h.TEST_RELATION_SCHEMA)
db.session.commit()
self.context = ValidationContext(network=self.network)
def setUp(self):
self.client = h.make_test_app()
self.network = Network.create({'title': 'Net', 'slug': 'net'})
self.schema = Schema.create(self.network, Schema.ENTITY,
h.TEST_ENTITY_SCHEMA)
self.rschema = Schema.create(self.network, Schema.RELATION,
h.TEST_RELATION_SCHEMA)
db.session.commit()
self.context = ValidationContext(network=self.network)
def decode(self, node, cstruct):
if isinstance(cstruct, Schema):
return cstruct
if isinstance(cstruct, basestring):
return Schema.by_name(self.project, cstruct)
if isinstance(cstruct, dict):
if cstruct.get('name'):
return Schema.by_name(self.project, cstruct.get('name'))
return None
def decode(self, node, cstruct):
if isinstance(cstruct, Schema):
return cstruct
if isinstance(cstruct, basestring):
return Schema.by_name(self.project, cstruct)
if isinstance(cstruct, dict):
if cstruct.get('name'):
return Schema.by_name(self.project,
cstruct.get('name'))
return None
def save(data, schema=None):
""" Create a schema. """
schema_val = SchemaValidator(validator=check_attributes)
data = schema_val.deserialize(data)
if schema is None:
schema = Schema()
schema.name = data.get('name')
schema.project = data.get('project')
schema.label = data.get('label')
schema.label_in = data.get('label_in') or schema.label
schema.label_out = data.get('label_out') or schema.label
schema.obj = data.get('obj')
schema.hidden = data.get('hidden')
schema.project.updated_at = datetime.utcnow()
db.session.add(schema)
names = []
for attribute in data.get('attributes', []):
attribute['schema'] = schema
attr = attributes.save(attribute)
schema.attributes.append(attr)
names.append(attr.name)
for attr in schema.attributes:
if attr.name not in names:
db.session.delete(attr)
return schema
def setUp(self):
self.app = make_test_app()
self.app.post('/api/1/networks',
headers=AUTHZ_HEADER,
data=NETWORK_FIXTURE)
network = Network.by_slug(NETWORK_FIXTURE['slug'])
Schema.create(network, Schema.RELATION,
h.TEST_RELATION_SCHEMA)
Schema.create(network, Schema.ENTITY,
h.TEST_ENTITY_SCHEMA)
db.session.commit()
def save(data, schema=None):
""" Create a schema. """
schema_val = SchemaValidator(validator=check_attributes)
data = schema_val.deserialize(data)
if schema is None:
schema = Schema()
schema.name = data.get('name')
schema.project = data.get('project')
schema.label = data.get('label')
schema.label_in = data.get('label_in') or schema.label
schema.label_out = data.get('label_out') or schema.label
schema.obj = data.get('obj')
schema.hidden = data.get('hidden')
schema.project.updated_at = datetime.utcnow()
db.session.add(schema)
names = []
for attribute in data.get('attributes', []):
attribute['schema'] = schema
attr = attributes.save(attribute)
schema.attributes.append(attr)
names.append(attr.name)
for attr in schema.attributes:
if attr.name not in names:
attributes.delete(attr)
return schema
def make_fixtures(self):
self.app.post('/api/1/networks',
headers=AUTHZ_HEADER,
data=NETWORK_FIXTURE)
network = Network.by_slug(NETWORK_FIXTURE['slug'])
Schema.create(network, Schema.RELATION, h.TEST_RELATION_SCHEMA)
Schema.create(network, Schema.ENTITY, h.TEST_ENTITY_SCHEMA)
db.session.commit()
res = self.app.post('/api/1/net/entities',
headers=AUTHZ_HEADER,
data=ENTITY_FIXTURE,
follow_redirects=True)
body = json.loads(res.data)
self.id = body['id']
def apply_alias(project, author, canonical_name, alias_name):
""" Given two names, find out if there are existing entities for one or
both of them. If so, merge them into a single entity - or, if only the
entity associated with the alias exists - re-name the entity. """
canonical_name = canonical_name.strip()
# Don't import meaningless aliases.
if canonical_name == alias_name or not len(canonical_name):
return log.info("No alias: %s", canonical_name)
canonical = Entity.by_name(project, canonical_name)
alias = Entity.by_name(project, alias_name)
schema = Schema.by_name(project, 'base')
# Don't artificially increase entity counts.
if canonical is None and alias is None:
return log.info("Neither alias nor canonical exist: %s", canonical_name)
# Rename an alias to its new, canonical name.
if canonical is None:
properties_logic.set(alias, author, 'name', schema, canonical_name,
active=True, source_url=None)
_entity_changed.delay(alias.id)
return log.info("Renamed: %s", alias_name)
# Already done, thanks.
if canonical == alias:
return log.info("Already aliased: %s", canonical_name)
# Merge two existing entities, declare one as "same_as"
if canonical is not None and alias is not None:
_merge_entities(alias, canonical)
_entity_changed.delay(canonical.id)
return log.info("Mapped: %s -> %s", alias.id, canonical.id)
def delete(slug, name):
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_manage(project))
schema = object_or_404(Schema.by_name(project, name))
schemata.delete(schema)
db.session.commit()
raise Gone()
def view(slug, name):
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_read(project))
if not project.private:
validate_cache(last_modified=project.updated_at)
schema = object_or_404(Schema.by_name(project, name))
return jsonify(schema)
def delete(slug, name):
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_manage(project))
schema = object_or_404(Schema.by_name(project, name))
schemata.delete(schema)
db.session.commit()
raise Gone()
def view(slug, name):
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_read(project))
if not project.private:
validate_cache(last_modified=project.updated_at)
schema = object_or_404(Schema.by_name(project, name))
return jsonify(schema)
def make_fixtures(self):
self.app.post('/api/1/networks',
headers=AUTHZ_HEADER,
data=NETWORK_FIXTURE)
network = Network.by_slug(NETWORK_FIXTURE['slug'])
Schema.create(network, Schema.RELATION,
h.TEST_RELATION_SCHEMA)
Schema.create(network, Schema.ENTITY,
h.TEST_ENTITY_SCHEMA)
db.session.commit()
res = self.app.post('/api/1/net/entities',
headers=AUTHZ_HEADER,
data=ENTITY_FIXTURE,
follow_redirects=True)
body = json.loads(res.data)
self.id = body['id']
def update(slug, name):
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_manage(project))
schema = object_or_404(Schema.by_name(project, name))
data = request_data({'project': project})
schema = schemata.save(data, schema=schema)
db.session.commit()
return jsonify(schema)
def index(slug):
project = object_or_404(Project.by_slug(slug))
validate_cache(last_modified=project.updated_at)
query = Schema.all()
query = query.filter_by(project=project)
pager = Pager(query)
conv = lambda es: [schemata.to_rest_index(e) for e in es]
return jsonify(pager.to_dict(conv))
def index(slug):
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_read(project))
validate_cache(last_modified=project.updated_at)
query = Schema.all()
query = query.filter_by(project=project)
pager = Pager(query, slug=slug)
return jsonify(pager, index=not arg_bool("full"))
def update(slug, name):
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_manage(project))
schema = object_or_404(Schema.by_name(project, name))
data = request_data({'project': project})
project = schemata.save(data, schema=schema)
db.session.commit()
return jsonify(schemata.to_rest(schema))
def facet_schema_list(obj, facets):
results = []
project = Project.by_slug('openinterests')
for facet in facets:
schema = Schema.by_name(project, facet.get('term'))
if schema is not None and not schema.hidden:
results.append((schema, facet.get('count')))
return results
def index(slug):
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_read(project))
validate_cache(last_modified=project.updated_at)
query = Schema.all()
query = query.filter_by(project=project)
pager = Pager(query, slug=slug)
return jsonify(pager, index=not arg_bool('full'))
def index(slug):
project = object_or_404(Project.by_slug(slug))
validate_cache(last_modified=project.updated_at)
query = Schema.all()
query = query.filter_by(project=project)
pager = Pager(query)
conv = lambda es: [schemata.to_rest_index(e) for e in es]
return jsonify(pager.to_dict(conv))
def setUp(self):
self.client = h.make_test_app()
self.network = Network.create({'title': 'Net', 'slug': 'net'})
self.schema = Schema.create(self.network, Schema.RELATION,
h.TEST_RELATION_SCHEMA)
db.session.commit()
self.eschema = Schema.create(self.network, Schema.ENTITY,
h.TEST_ENTITY_SCHEMA)
self.context = ValidationContext(network=self.network)
entity = deepcopy(TEST_ENTITY)
entity['network'] = self.network.slug
entity = validate_entity(entity, self.eschema, self.context)
entity['title'] = 'Alice'
a = self.network.Entity.create(self.eschema, entity)
entity['title'] = 'Bob'
b = self.network.Entity.create(self.eschema, entity)
TEST_RELATION['source'] = a.id
TEST_RELATION['target'] = b.id
db.session.commit()
def delete(slug, name):
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_manage(project))
schema = object_or_404(Schema.by_name(project, name))
deleted = schemata.delete(schema)
db.session.commit()
if deleted:
raise Gone()
else:
return jsonify(schema)
def setUp(self):
self.client = h.make_test_app()
self.network = Network.create({'title': 'Net', 'slug': 'net'})
self.schema = Schema.create(self.network, Schema.RELATION,
h.TEST_RELATION_SCHEMA)
db.session.commit()
self.eschema = Schema.create(self.network, Schema.ENTITY,
h.TEST_ENTITY_SCHEMA)
self.context = ValidationContext(network=self.network)
entity = deepcopy(TEST_ENTITY)
entity['network'] = self.network.slug
entity = validate_entity(entity, self.eschema, self.context)
entity['title'] = 'Alice'
a = self.network.Entity.create(self.eschema, entity)
entity['title'] = 'Bob'
b = self.network.Entity.create(self.eschema, entity)
TEST_RELATION['source'] = a.id
TEST_RELATION['target'] = b.id
db.session.commit()
def delete(slug, name):
project = object_or_404(Project.by_slug(slug))
authz.require(authz.project_manage(project))
schema = object_or_404(Schema.by_name(project, name))
deleted = schemata.delete(schema)
db.session.commit()
if deleted:
raise Gone()
else:
return jsonify(schema)
def export_schema(project, path):
if not os.path.exists(path):
os.makedirs(path)
for schema in Schema.all().filter_by(project=project):
fn = os.path.join(path, schema.name + '.yaml')
with open(fn, 'w') as fh:
dumped = yaml.safe_dump(schema.to_dict(schema),
canonical=False,
default_flow_style=False,
indent=4)
fh.write(dumped)
def export_schema(project, path):
if not os.path.exists(path):
os.makedirs(path)
for schema in Schema.all().filter_by(project=project):
fn = os.path.join(path, schema.name + '.yaml')
with open(fn, 'w') as fh:
dumped = yaml.safe_dump(schema.to_dict(schema),
canonical=False,
default_flow_style=False,
indent=4)
fh.write(dumped)
def create(slug, type):
""" Create a new schema. """
network = _get_network(slug)
require.schema.create(network)
_valid_schema(type)
data = request_content(request)
data = validate_schema(dict(data.items()))
schema = Schema.create(network, type, data)
db.session.commit()
url = url_for('.get', slug=network.slug,
type=schema.entity, name=schema.name)
return jsonify(schema, status=201, headers={'location': url})
def import_schema(project, fh):
data = yaml.load(fh.read())
if isinstance(data, dict):
data = [data]
try:
for cur in data:
schema = Schema.by_name(project, cur.get('name'))
cur['project'] = project
save(cur, schema=schema)
db.session.commit()
except Invalid, inv:
pprint(inv.asdict())
def import_schema(project, fh):
data = yaml.load(fh.read())
if isinstance(data, dict):
data = [data]
try:
for cur in data:
schema = Schema.by_name(project, cur.get('name'))
cur['project'] = project
save(cur, schema=schema)
db.session.commit()
except Invalid, inv:
pprint(inv.asdict())
def create(slug, type):
""" Create a new schema. """
network = _get_network(slug)
require.schema.create(network)
_valid_schema(type)
data = request_content(request)
data = validate_schema(dict(data.items()))
schema = Schema.create(network, type, data)
db.session.commit()
url = url_for('.get',
slug=network.slug,
type=schema.entity,
name=schema.name)
return jsonify(schema, status=201, headers={'location': url})
def apply_alias(project, author, canonical_name, alias_name):
""" Given two names, find out if there are existing entities for one or
both of them. If so, merge them into a single entity - or, if only the
entity associated with the alias exists - re-name the entity. """
canonical_name = canonical_name.strip()
# Don't import meaningless aliases.
if canonical_name == alias_name or not len(canonical_name):
return log.info("No alias: %s", canonical_name)
canonical = Entity.by_name(project, canonical_name)
alias = Entity.by_name(project, alias_name)
schema = Schema.by_name(project, 'base')
# Don't artificially increase entity counts.
if canonical is None and alias is None:
return log.info("Neither alias nor canonical exist: %s",
canonical_name)
# Rename an alias to its new, canonical name.
if canonical is None:
properties_logic.set(alias,
author,
'name',
schema,
canonical_name,
active=True,
source_url=None)
_entity_changed.delay(alias.id)
return log.info("Renamed: %s", alias_name)
# Already done, thanks.
if canonical == alias:
return log.info("Already aliased: %s", canonical_name)
# Merge two existing entities, declare one as "same_as"
if canonical is not None and alias is not None:
_merge_entities(alias, canonical)
_entity_changed.delay(canonical.id)
return log.info("Mapped: %s -> %s", alias.id, canonical.id)
def view(slug, name):
project = object_or_404(Project.by_slug(slug))
validate_cache(last_modified=project.updated_at)
schema = object_or_404(Schema.by_name(project, name))
return jsonify(schemata.to_rest(schema))
def test_apply_schema(self):
base = mapping('entity')
schema = Schema.create(self.network, Schema.ENTITY,
h.TEST_ENTITY_SCHEMA)
base = apply_schema(base, schema)
assert len(base.children) == len(h.TEST_ENTITY_SCHEMA['attributes'])
def view(slug, name):
project = object_or_404(Project.by_slug(slug))
validate_cache(last_modified=project.updated_at)
schema = object_or_404(Schema.by_name(project, name))
return jsonify(schemata.to_rest(schema))
def save(data, schema=None):
""" Create a schema. """
data = validate(data)
operation = 'create' if schema is None else 'update'
if schema is None:
schema = Schema()
schema.name = data.get('name')
schema.project = data.get('project')
schema.label = data.get('label')
schema.obj = data.get('obj')
schema.hidden = data.get('hidden')
schema.meta = data.get('meta')
schema.parent = data.get('parent')
if schema.name in DEFAULTS.values():
schema.parent = None
elif schema.parent is None or schema.is_circular():
schema.parent = Schema.by_name(schema.project,
DEFAULTS.get(schema.obj))
schema.project.updated_at = datetime.utcnow()
db.session.add(schema)
inherited = [a.name for a in schema.inherited_attributes]
names = []
for attribute in data.get('attributes', []):
if attribute.get('name') in inherited:
continue
attribute['schema'] = schema
attr = attributes.save(attribute)
schema.local_attributes.append(attr)
names.append(attr.name)
for attr in schema.local_attributes:
if attr.name not in names:
attributes.delete(attr)
db.session.flush()
_schema_changed(schema.project.slug, schema.name, operation)
return schema
def apply_alias(project, author, canonical_name, alias_name, source_url=None):
""" Given two names, find out if there are existing entities for one or
both of them. If so, merge them into a single entity - or, if only the
entity associated with the alias exists - re-name the entity. """
# Don't import meaningless aliases.
if not len(canonical_name) or not len(alias_name):
return log.info("Not an alias: %s", canonical_name)
canonical = None
# de-duplicate existing entities with the same name.
known_names = set()
for existing in Entity.by_name_many(project, canonical_name):
for prop in existing.properties:
if prop.name != 'name':
continue
known_names.add(prop.value)
# make sure the canonical name is actually active
# TODO: is this desirable?
if prop.value == canonical_name:
prop.active = True
else:
prop.active = False
if canonical is not None and canonical.id != existing.id:
canonical = merge(existing, canonical)
else:
canonical = existing
schema = Schema.by_name(project, 'base')
attribute = schema.get_attribute('name')
# Find aliases, i.e. entities with the alias name which are not
# the canonical entity.
q = Entity.by_name_many(project, alias_name)
if canonical is not None:
q = q.filter(Entity.id != canonical.id)
aliases = q.all()
# If there are no existing aliases with that name, add the alias
# name to the canonical entity.
if not len(aliases) and canonical is not None:
if alias_name not in known_names:
data = {
'value': alias_name,
'schema': schema,
'attribute': attribute,
'active': False,
'name': 'name',
'source_url': source_url
}
properties_logic.save(canonical, data)
_entity_changed.delay(canonical.id, 'update')
log.info("Alias: %s -> %s", alias_name, canonical_name)
for alias in aliases:
if canonical is None:
# Rename an alias to its new, canonical name.
data = {
'value': canonical_name,
'schema': schema,
'attribute': attribute,
'active': True,
'name': 'name',
'source_url': source_url
}
properties_logic.save(alias, data)
_entity_changed.delay(alias.id, 'update')
log.info("Renamed: %s -> %s", alias_name, canonical_name)
else:
# Merge two existing entities, declare one as "same_as"
merge(alias, canonical)
log.info("Mapped: %s -> %s", alias.id, canonical.id)
db.session.commit()
def save(data, schema=None):
""" Create a schema. """
data = validate(data)
operation = 'create' if schema is None else 'update'
if schema is None:
schema = Schema()
schema.name = data.get('name')
schema.project = data.get('project')
schema.label = data.get('label')
schema.obj = data.get('obj')
schema.hidden = data.get('hidden')
schema.meta = data.get('meta')
schema.parent = data.get('parent')
if schema.name in DEFAULTS.values():
schema.parent = None
elif schema.parent is None or schema.is_circular():
schema.parent = Schema.by_name(schema.project,
DEFAULTS.get(schema.obj))
schema.project.updated_at = datetime.utcnow()
db.session.add(schema)
inherited = [a.name for a in schema.inherited_attributes]
names = []
for attribute in data.get('attributes', []):
if attribute.get('name') in inherited:
continue
attribute['schema'] = schema
attr = attributes.save(attribute)
schema.local_attributes.append(attr)
names.append(attr.name)
for attr in schema.local_attributes:
if attr.name not in names:
attributes.delete(attr)
db.session.flush()
_schema_changed(schema.project.slug, schema.name, operation)
return schema
def apply_alias(project, author, canonical_name, alias_name, source_url=None):
""" Given two names, find out if there are existing entities for one or
both of them. If so, merge them into a single entity - or, if only the
entity associated with the alias exists - re-name the entity. """
# Don't import meaningless aliases.
if not len(canonical_name) or not len(alias_name):
return log.info("Not an alias: %s", canonical_name)
canonical = None
# de-duplicate existing entities with the same name.
known_names = set()
for existing in Entity.by_name_many(project, canonical_name):
for prop in existing.properties:
if prop.name != 'name':
continue
known_names.add(prop.value)
# make sure the canonical name is actually active
# TODO: is this desirable?
if prop.value == canonical_name:
prop.active = True
else:
prop.active = False
if canonical is not None and canonical.id != existing.id:
canonical = merge(existing, canonical)
else:
canonical = existing
schema = Schema.by_name(project, ENTITY_DEFAULT_SCHEMA)
attribute = schema.get_attribute('name')
# Find aliases, i.e. entities with the alias name which are not
# the canonical entity.
q = Entity.by_name_many(project, alias_name)
if canonical is not None:
q = q.filter(Entity.id != canonical.id)
aliases = q.all()
# If there are no existing aliases with that name, add the alias
# name to the canonical entity.
if not len(aliases) and canonical is not None:
if alias_name not in known_names:
data = {
'value': alias_name,
'schema': schema,
'attribute': attribute,
'active': False,
'name': 'name',
'source_url': source_url
}
properties_logic.save(canonical, data)
_entity_changed.delay(canonical.id, 'update')
log.info("Alias: %s -> %s", alias_name, canonical_name)
for alias in aliases:
if canonical is None:
# Rename an alias to its new, canonical name.
data = {
'value': canonical_name,
'schema': schema,
'attribute': attribute,
'active': True,
'name': 'name',
'source_url': source_url
}
properties_logic.save(alias, data)
_entity_changed.delay(alias.id, 'update')
log.info("Renamed: %s -> %s", alias_name, canonical_name)
else:
# Merge two existing entities, declare one as "same_as"
merge(alias, canonical)
log.info("Mapped: %s -> %s", alias.id, canonical.id)
db.session.commit()
def test_apply_schema(self):
base = mapping('entity')
schema = Schema.create(self.network, Schema.ENTITY, h.TEST_ENTITY_SCHEMA)
base = apply_schema(base, schema)
assert len(base.children)==len(h.TEST_ENTITY_SCHEMA['attributes'])