def home(): """ Render the index page :return: index.html """ log.info("Accessed route: index") return render_template('index.html')
def sync_synic(data=None): log.info('Synchronizing Synic API') try: emit('synic.sync', controller.sync_synic()) except Exception as e: emit('synic.sync.fail', None) emit('danger', {'title': 'Error', 'message': str(e)})
def remove_entity(data): log.info('Removing entity', extra=data) entity_id = data.get('id') try: controller.remove_entity(entity_id) emit('info', {'title': 'Remove Entity', 'message': 'Entity removed successfully'}) except Exception as e: emit('danger', {'title': 'Error', 'message': str(e)})
def add_entity(data): log.info('Adding entity', extra=data) try: entity_id = controller.add_entity(data) emit('entity.sync', controller.sync_entity(entity_id)) emit('entity.add.success', None) except Exception as e: emit('danger', {'title': 'Error', 'message': str(e)}) emit('entity.add.failure', None)
def export_entities(null): log.info('Exporting entities') try: controller.export_entities() emit('entity.export.all.success', None) except Exception as e: emit('danger', {'title': 'Error', 'message': str(e)})
def clean_empty_entity_attrs(): with session_scope() as session: entity_attrs = Query.EntityAttribute.all(session) for entity_attr in entity_attrs: fields = Query.EntityAttributeField.filter( session, entity_attribute_id=entity_attr.id) if len(fields) == 0: log.info('Removing empty Entity Attribute: {} ({})'.format( entity_attr.id, entity_attr.entity_id)) session.delete(entity_attr)
def update_entity(data): log.info('Updating entity', extra=data) entity_id = data.get('id') try: controller.update_entity(entity_id, data) emit('entity.sync', controller.sync_entity(entity_id)) except Exception as e: emit('danger', {'title': 'Error', 'message': str(e)})
def remove_attribute(data): log.info('Removing attribute', extra=data) entity_id = data.get('entityId') attribute_id = data.get('attributeId') try: controller.remove_attribute(attribute_id) emit('entity.sync', controller.sync_entity(entity_id)) except Exception as e: emit('danger', {'title': 'Error', 'message': str(e)})
def sync_synic(data): log.info('Sending entities to Synthesys for ingestion') try: kg_name = data.get('kg') if kg_name is None: raise Exception('Knowledge graph cannot be null') emit('synic.ingest.success', controller.ingest_entities(kg_name)) except Exception as e: emit('danger', {'title': 'Error', 'message': str(e)})
def add_attribute(data): log.info('Adding attribute', extra=data) entity_id = data.get('entityId') attribute_id = data.get('attributeId') fields = data.get('fields') try: controller.add_attribute(entity_id, attribute_id, fields) emit('entity.sync', controller.sync_entity(entity_id)) except Exception as e: emit('danger', {'title': 'Error', 'message': str(e)})
def update_attribute(data): log.info('Updating attribute', extra=data) entity_id = data.get('entityId') field_id = data.get('fieldId') value = data.get('value') if value is None or len(value) == 0: emit('danger', {'title':'Invalid Request', 'message': 'The field cannot be empty. Please try again'}) else: try: controller.update_attribute(field_id, value) emit('entity.sync', controller.sync_entity(entity_id)) except Exception as e: emit('danger', {'title': 'Error', 'message': str(e)})
def upload(): def allowed_file(filename): return '.' in filename and \ filename.rsplit('.', 1)[1].lower() == 'xml' if 'file' not in request.files: log.error('No file in request') file = request.files['file'] if file.filename == '': log.error('No filename in the request') return Response(json.dumps({'status': 'Fail'}), status=400, mimetype='application/json') if not allowed_file(file.filename): log.error('File type not allowed') return Response(json.dumps({'status': 'Fail'}), status=400, mimetype='application/json') log.info('Saving file') filename = secure_filename(file.filename) file.save(os.path.join(FileConfig.DATA_DIR, filename)) log.info('File saved successfully') return Response(json.dumps({'status': 'Success'}), status=200, mimetype='application/json')
def import_entities(entities_json): log.info('Importing entities') with session_scope() as session: for e in entities_json: entity_id = e['id'] entity = Query.Entity.get(session, entity_id) if entity is None: if Query.EntityType.get(session, e['type']) is None: session.add( EntityType(id=e['type'], name=e['type'].replace('_', ' ').title())) session.commit() entity = Entity(id=e['id'], name=e['name'], type=e['type'], canonical=e['canonical']) log.info('Adding {} entity {} (id: {})'.format( entity.type, entity.name, entity.id)) session.add(entity) else: entity.name = e['name'] entity.type = e['type'] entity.canonical = e['canonical'] log.info('Updating {} entity {} (id: {})'.format( entity.type, entity.name, entity.id)) for a in e['attributes']: attr_id = a.pop('id', None) attribute = Query.Attribute.get(session, attr_id) if attribute is None: if a.get('sid') is not None: attr_type = 'entity' else: attr_type = 'string' session.add( Attribute(id=attr_id, name=attr_id.title(), required=False, arity=ArityTypes.FEW.id, ko_name=None, description='attribute added via XML import', type=attr_type)) session.commit() for field_id in a.keys(): session.add(LinkedAttributeField(attr_id, field_id)) existing_attrs = Query.EntityAttribute.filter( session, entity_id, attr_id) duplicate = False for existing_attr in existing_attrs: if duplicate: break matches = 0 num_fields = len(a) for field_id, value in a.items(): field = Query.AttributeField.get(session, field_id) if field is not None: linked_field = Query.LinkedAttributeField.filter( session, attr_id, field_id) existing_attr_field = Query.EntityAttributeField.filter( session, existing_attr.id, linked_field.id) if existing_attr_field is not None and existing_attr_field.value == value: matches += 1 if matches == num_fields: log.info('Duplicate field identified, skipping.') duplicate = True if not duplicate: entity_attr = EntityAttribute(entity_id, attr_id) session.add(entity_attr) for field_id, value in a.items(): field = Query.AttributeField.get(session, field_id) if field is None: log.error( 'Undefined field {} for attribute {} for entity {}, skipping.' .format(field_id, attr_id, entity_id)) else: linked_field = Query.LinkedAttributeField.filter( session, attr_id, field_id) if linked_field is None: log.error( 'Field {} not allowed for attribute {} for entity {}, skipping.' .format(field_id, attr_id, entity_id)) else: # TODO: Arity enforcement ea_field = EntityAttributeField( entity_attr.id, linked_field.id, value) session.add(ea_field)
def init_db(): """Initializes SQLAlchemy database tables Models must be imported before schema creation Schema is not currently configured to migrate / update automatically Database volume must be re-initialized manually upon schema updates If database is not volume-mounted, just delete the container """ def initialize_settings(): """Initializes application settings""" log.debug('Initializing application settings') with session_scope() as session: for setting in ApplicationDefaults.list(): try: existing_setting = Query.Setting.get(session, setting.id) if existing_setting is None: log.debug('Initializing application default: {} -> {}'.format(setting.id, setting.safe_value())) session.add(setting) else: log.debug( 'Application default exists: {} -> {}'.format(setting.id, setting.safe_value())) except Exception as e: log.error('Failed to initialize setting: {} due to: {}'.format(setting.id, str(e))) log.debug('Application setting initialization complete') def initialize_entity_schema(): """Initializes entity schema""" log.debug('Initializing entity schema defaults') with session_scope() as session: for defaults in AttributeTypes.list(), EntityTypes.list(), ArityTypes.list(), AttributeFields.list(): for default in defaults: def_type = type(default) try: def_existing = session.query(def_type).get(default.id) if def_existing is None: log.debug('Initializing default {}: {}'.format(def_type.__name__, default.name)) session.add(default) # else: # log.debug('Entry exists for {}: {}'.format(def_type.__name__, default.name)) except Exception as e: log.error( 'Failed to add default {}: {} due to: {}'.format(def_type.__name__, default.name, str(e))) session.rollback() log.debug('Entity initialization complete') def initialize_attributes(): """Initializes default attributes""" log.debug('Initializing attribute defaults') with session_scope() as session: attr_templates = load_yml(FileConfig.ATTR_SCHEMA_FILE) for attr_id, attr_template in attr_templates.items(): try: attribute = Attribute(id=attr_id, name=attr_template['name'], required=attr_template.get('required', False), arity=attr_template['arity'], ko_name=attr_template['ko_name'], description=attr_template['description'], type=attr_template['type']) attribute_existing = Query.Attribute.get(session, attr_id) if attribute_existing is None: log.debug('Initializing attribute: {}'.format(attribute.name)) session.add(attribute) # else: # log.debug('Attribute exists, skipping: {}'.format(attribute.name)) for field_id in attr_template['fields']: attr_field = Query.AttributeField.get(session, field_id) if attr_field is not None: linked_attr_field = LinkedAttributeField(attribute.id, attr_field.id) linked_attr_field_existing = Query.LinkedAttributeField.get(session, linked_attr_field.id) if linked_attr_field_existing is None: log.debug('Creating linked attribute field {} for attribute {}'.format(attr_field.name, attribute.name)) session.add(linked_attr_field) else: log.error('Attribute field {} is not defined in field definitions. Please add field to ' 'enumeration located in definitions.py. Skipping for now.'.format(field_id)) except Exception as e: log.error('Failed to initialize attribute: {} due to: {}'.format(attr_id, str(e))) log.debug('Attribute initialization complete') initialized = False while not initialized: try: add_engine_pidguard(engine) set_tx_isolation(engine) celery.Task = DatabaseTask for model in MODELS: if not engine.dialect.has_table(engine, model.__tablename__): model.__table__.create(engine) #initialize_settings() initialize_entity_schema() initialize_attributes() initialized = True log.info('Database initialized') except Exception as e: log.exception(e) log.info('Waiting for database to initialize') sleep(5)