def home():
"""
Render the index page
:return: index.html
"""
log.info("Accessed route: index")
return render_template('index.html')
def sync_synic(data=None):
log.info('Synchronizing Synic API')
try:
emit('synic.sync', controller.sync_synic())
except Exception as e:
emit('synic.sync.fail', None)
emit('danger', {'title': 'Error', 'message': str(e)})
def remove_entity(data):
log.info('Removing entity', extra=data)
entity_id = data.get('id')
try:
controller.remove_entity(entity_id)
emit('info', {'title': 'Remove Entity', 'message': 'Entity removed successfully'})
except Exception as e:
emit('danger', {'title': 'Error', 'message': str(e)})
def add_entity(data):
log.info('Adding entity', extra=data)
try:
entity_id = controller.add_entity(data)
emit('entity.sync', controller.sync_entity(entity_id))
emit('entity.add.success', None)
except Exception as e:
emit('danger', {'title': 'Error', 'message': str(e)})
emit('entity.add.failure', None)
def export_entities(null):
log.info('Exporting entities')
try:
controller.export_entities()
emit('entity.export.all.success', None)
except Exception as e:
emit('danger', {'title': 'Error', 'message': str(e)})
def clean_empty_entity_attrs():
with session_scope() as session:
entity_attrs = Query.EntityAttribute.all(session)
for entity_attr in entity_attrs:
fields = Query.EntityAttributeField.filter(
session, entity_attribute_id=entity_attr.id)
if len(fields) == 0:
log.info('Removing empty Entity Attribute: {} ({})'.format(
entity_attr.id, entity_attr.entity_id))
session.delete(entity_attr)
def update_entity(data):
log.info('Updating entity', extra=data)
entity_id = data.get('id')
try:
controller.update_entity(entity_id, data)
emit('entity.sync', controller.sync_entity(entity_id))
except Exception as e:
emit('danger', {'title': 'Error', 'message': str(e)})
def remove_attribute(data):
log.info('Removing attribute', extra=data)
entity_id = data.get('entityId')
attribute_id = data.get('attributeId')
try:
controller.remove_attribute(attribute_id)
emit('entity.sync', controller.sync_entity(entity_id))
except Exception as e:
emit('danger', {'title': 'Error', 'message': str(e)})
def sync_synic(data):
log.info('Sending entities to Synthesys for ingestion')
try:
kg_name = data.get('kg')
if kg_name is None:
raise Exception('Knowledge graph cannot be null')
emit('synic.ingest.success', controller.ingest_entities(kg_name))
except Exception as e:
emit('danger', {'title': 'Error', 'message': str(e)})
def add_attribute(data):
log.info('Adding attribute', extra=data)
entity_id = data.get('entityId')
attribute_id = data.get('attributeId')
fields = data.get('fields')
try:
controller.add_attribute(entity_id, attribute_id, fields)
emit('entity.sync', controller.sync_entity(entity_id))
except Exception as e:
emit('danger', {'title': 'Error', 'message': str(e)})
def update_attribute(data):
log.info('Updating attribute', extra=data)
entity_id = data.get('entityId')
field_id = data.get('fieldId')
value = data.get('value')
if value is None or len(value) == 0:
emit('danger', {'title':'Invalid Request', 'message': 'The field cannot be empty. Please try again'})
else:
try:
controller.update_attribute(field_id, value)
emit('entity.sync', controller.sync_entity(entity_id))
except Exception as e:
emit('danger', {'title': 'Error', 'message': str(e)})
def upload():
def allowed_file(filename):
return '.' in filename and \
filename.rsplit('.', 1)[1].lower() == 'xml'
if 'file' not in request.files:
log.error('No file in request')
file = request.files['file']
if file.filename == '':
log.error('No filename in the request')
return Response(json.dumps({'status': 'Fail'}), status=400, mimetype='application/json')
if not allowed_file(file.filename):
log.error('File type not allowed')
return Response(json.dumps({'status': 'Fail'}), status=400, mimetype='application/json')
log.info('Saving file')
filename = secure_filename(file.filename)
file.save(os.path.join(FileConfig.DATA_DIR, filename))
log.info('File saved successfully')
return Response(json.dumps({'status': 'Success'}), status=200, mimetype='application/json')
def import_entities(entities_json):
log.info('Importing entities')
with session_scope() as session:
for e in entities_json:
entity_id = e['id']
entity = Query.Entity.get(session, entity_id)
if entity is None:
if Query.EntityType.get(session, e['type']) is None:
session.add(
EntityType(id=e['type'],
name=e['type'].replace('_', ' ').title()))
session.commit()
entity = Entity(id=e['id'],
name=e['name'],
type=e['type'],
canonical=e['canonical'])
log.info('Adding {} entity {} (id: {})'.format(
entity.type, entity.name, entity.id))
session.add(entity)
else:
entity.name = e['name']
entity.type = e['type']
entity.canonical = e['canonical']
log.info('Updating {} entity {} (id: {})'.format(
entity.type, entity.name, entity.id))
for a in e['attributes']:
attr_id = a.pop('id', None)
attribute = Query.Attribute.get(session, attr_id)
if attribute is None:
if a.get('sid') is not None:
attr_type = 'entity'
else:
attr_type = 'string'
session.add(
Attribute(id=attr_id,
name=attr_id.title(),
required=False,
arity=ArityTypes.FEW.id,
ko_name=None,
description='attribute added via XML import',
type=attr_type))
session.commit()
for field_id in a.keys():
session.add(LinkedAttributeField(attr_id, field_id))
existing_attrs = Query.EntityAttribute.filter(
session, entity_id, attr_id)
duplicate = False
for existing_attr in existing_attrs:
if duplicate:
break
matches = 0
num_fields = len(a)
for field_id, value in a.items():
field = Query.AttributeField.get(session, field_id)
if field is not None:
linked_field = Query.LinkedAttributeField.filter(
session, attr_id, field_id)
existing_attr_field = Query.EntityAttributeField.filter(
session, existing_attr.id, linked_field.id)
if existing_attr_field is not None and existing_attr_field.value == value:
matches += 1
if matches == num_fields:
log.info('Duplicate field identified, skipping.')
duplicate = True
if not duplicate:
entity_attr = EntityAttribute(entity_id, attr_id)
session.add(entity_attr)
for field_id, value in a.items():
field = Query.AttributeField.get(session, field_id)
if field is None:
log.error(
'Undefined field {} for attribute {} for entity {}, skipping.'
.format(field_id, attr_id, entity_id))
else:
linked_field = Query.LinkedAttributeField.filter(
session, attr_id, field_id)
if linked_field is None:
log.error(
'Field {} not allowed for attribute {} for entity {}, skipping.'
.format(field_id, attr_id, entity_id))
else:
# TODO: Arity enforcement
ea_field = EntityAttributeField(
entity_attr.id, linked_field.id, value)
session.add(ea_field)
def init_db():
"""Initializes SQLAlchemy database tables
Models must be imported before schema creation
Schema is not currently configured to migrate / update automatically
Database volume must be re-initialized manually upon schema updates
If database is not volume-mounted, just delete the container
"""
def initialize_settings():
"""Initializes application settings"""
log.debug('Initializing application settings')
with session_scope() as session:
for setting in ApplicationDefaults.list():
try:
existing_setting = Query.Setting.get(session, setting.id)
if existing_setting is None:
log.debug('Initializing application default: {} -> {}'.format(setting.id, setting.safe_value()))
session.add(setting)
else:
log.debug(
'Application default exists: {} -> {}'.format(setting.id, setting.safe_value()))
except Exception as e:
log.error('Failed to initialize setting: {} due to: {}'.format(setting.id, str(e)))
log.debug('Application setting initialization complete')
def initialize_entity_schema():
"""Initializes entity schema"""
log.debug('Initializing entity schema defaults')
with session_scope() as session:
for defaults in AttributeTypes.list(), EntityTypes.list(), ArityTypes.list(), AttributeFields.list():
for default in defaults:
def_type = type(default)
try:
def_existing = session.query(def_type).get(default.id)
if def_existing is None:
log.debug('Initializing default {}: {}'.format(def_type.__name__, default.name))
session.add(default)
# else:
# log.debug('Entry exists for {}: {}'.format(def_type.__name__, default.name))
except Exception as e:
log.error(
'Failed to add default {}: {} due to: {}'.format(def_type.__name__, default.name, str(e)))
session.rollback()
log.debug('Entity initialization complete')
def initialize_attributes():
"""Initializes default attributes"""
log.debug('Initializing attribute defaults')
with session_scope() as session:
attr_templates = load_yml(FileConfig.ATTR_SCHEMA_FILE)
for attr_id, attr_template in attr_templates.items():
try:
attribute = Attribute(id=attr_id,
name=attr_template['name'],
required=attr_template.get('required', False),
arity=attr_template['arity'],
ko_name=attr_template['ko_name'],
description=attr_template['description'],
type=attr_template['type'])
attribute_existing = Query.Attribute.get(session, attr_id)
if attribute_existing is None:
log.debug('Initializing attribute: {}'.format(attribute.name))
session.add(attribute)
# else:
# log.debug('Attribute exists, skipping: {}'.format(attribute.name))
for field_id in attr_template['fields']:
attr_field = Query.AttributeField.get(session, field_id)
if attr_field is not None:
linked_attr_field = LinkedAttributeField(attribute.id, attr_field.id)
linked_attr_field_existing = Query.LinkedAttributeField.get(session, linked_attr_field.id)
if linked_attr_field_existing is None:
log.debug('Creating linked attribute field {} for attribute {}'.format(attr_field.name,
attribute.name))
session.add(linked_attr_field)
else:
log.error('Attribute field {} is not defined in field definitions. Please add field to '
'enumeration located in definitions.py. Skipping for now.'.format(field_id))
except Exception as e:
log.error('Failed to initialize attribute: {} due to: {}'.format(attr_id, str(e)))
log.debug('Attribute initialization complete')
initialized = False
while not initialized:
try:
add_engine_pidguard(engine)
set_tx_isolation(engine)
celery.Task = DatabaseTask
for model in MODELS:
if not engine.dialect.has_table(engine, model.__tablename__):
model.__table__.create(engine)
#initialize_settings()
initialize_entity_schema()
initialize_attributes()
initialized = True
log.info('Database initialized')
except Exception as e:
log.exception(e)
log.info('Waiting for database to initialize')
sleep(5)
