def flush(collection_id, mapping_id): """Flush all entities loaded by mapping with id `mapping_id`. --- post: summary: Flush entities loaded by a mapping parameters: - description: The collection id. in: path name: collection_id required: true schema: minimum: 1 type: integer example: 2 - description: The mapping id. in: path name: mapping_id required: true schema: minimum: 1 type: integer example: 2 responses: '202': description: No Content tags: - Collection """ collection = get_db_collection(collection_id, request.authz.WRITE) mapping = obj_or_404(Mapping.by_id(mapping_id)) queue_task(collection, OP_FLUSH_MAPPING, job_id=get_session_id(), payload={'mapping_id': mapping.id}) return ('', 202)
def load_mapping(collection, mapping_id, sync=False): """Flush and reload all entities generated by a mapping.""" mapping = Mapping.by_id(mapping_id) if mapping is None: return log.error("Could not find mapping: %s", mapping_id) origin = mapping_origin(mapping.id) aggregator = get_aggregator(collection) aggregator.delete(origin=origin) delete_entities(collection.id, origin=origin, sync=True) if mapping.disabled: return log.info("Mapping is disabled: %s", mapping_id) publish( Events.LOAD_MAPPING, params={ "collection": collection, "table": mapping.table_id }, channels=[collection, mapping.role], actor_id=mapping.role_id, ) try: map_to_aggregator(collection, mapping, aggregator) aggregate_model(collection, aggregator) index_aggregator(collection, aggregator, sync=sync) mapping.set_status(status=Status.SUCCESS) db.session.commit() except Exception as exc: mapping.set_status(status=Status.FAILED, error=str(exc)) db.session.commit() aggregator.delete(origin=origin) finally: aggregator.close()
def delete(collection_id, mapping_id): """Delete a mapping. --- delete: summary: Delete a mapping parameters: - description: The collection id. in: path name: collection_id required: true schema: minimum: 1 type: integer example: 2 - description: The mapping id. in: path name: mapping_id required: true schema: minimum: 1 type: integer example: 2 responses: '204': description: No Content tags: - Collection - Mapping """ get_db_collection(collection_id, request.authz.WRITE) mapping = obj_or_404(Mapping.by_id(mapping_id)) mapping.delete() db.session.commit() return ('', 204)
def view(collection_id, mapping_id): """Return the mapping with id `mapping_id`. --- get: summary: Fetch a mapping parameters: - description: The collection id. in: path name: collection_id required: true schema: minimum: 1 type: integer example: 2 - description: The mapping id. in: path name: mapping_id required: true schema: minimum: 1 type: integer example: 2 responses: '200': content: application/json: schema: $ref: '#/components/schemas/Mapping' description: OK tags: - Collection """ get_db_collection(collection_id, request.authz.WRITE) mapping = obj_or_404(Mapping.by_id(mapping_id)) return MappingSerializer.jsonify(mapping)
def load_mapping(stage, collection, mapping_id): """Flush and reload all entities generated by a mapping.""" mapping = Mapping.by_id(mapping_id) if mapping is None: return log.error("Could not find mapping: %s", mapping_id) flush_mapping(stage, collection, mapping_id) publish(Events.LOAD_MAPPING, params={'collection': collection, 'table': mapping.table_id}, channels=[collection, mapping.role], actor_id=mapping.role_id) mapper = make_mapper(collection, mapping) aggregator = get_aggregator(collection) try: writer = aggregator.bulk() entities_count = 0 entity_ids = set() for idx, record in enumerate(mapper.source.records, 1): for entity in mapper.map(record).values(): if entity.schema.is_a('Thing'): entity.add('proof', mapping.table_id) entity = collection.ns.apply(entity) entity_ids.add(entity.id) entities_count += 1 fragment = '%s-%s' % (mapping.id, idx) writer.put(entity, fragment=fragment) if idx > 0 and idx % 500 == 0: payload = { 'entity_ids': entity_ids, 'mapping_id': mapping.id } queue_task(collection, OP_INDEX, job_id=stage.job.id, payload=payload) entity_ids = set() stage.report_finished(500) log.info("[%s] Loaded %s records, %s entities...", collection.foreign_id, idx, entities_count) writer.flush() payload = { 'entity_ids': entity_ids, 'mapping_id': mapping.id } queue_task(collection, OP_INDEX, job_id=stage.job.id, payload=payload) mapping.set_status(status=Mapping.SUCCESS) log.info("[%s] Mapping done (%s entities)", mapping.id, entities_count) except Exception as exc: mapping.set_status(status=Mapping.FAILED, error=str(exc)) finally: aggregator.close()
def trigger(collection_id, mapping_id): """Load entities by running the mapping with id `mapping_id`. Flushes previously loaded entities before loading new entities. --- post: summary: Load entities from a mapping parameters: - description: The collection id. in: path name: collection_id required: true schema: minimum: 1 type: integer example: 2 - description: The mapping id. in: path name: mapping_id required: true schema: minimum: 1 type: integer example: 2 responses: '202': description: No Content tags: - Collection - Mapping """ collection = get_db_collection(collection_id, request.authz.WRITE) mapping = obj_or_404(Mapping.by_id(mapping_id)) mapping.disabled = False mapping.set_status(Status.PENDING) db.session.commit() job_id = get_session_id() queue_task(collection, OP_LOAD_MAPPING, job_id=job_id, mapping_id=mapping.id) mapping = obj_or_404(Mapping.by_id(mapping_id)) return MappingSerializer.jsonify(mapping, status=202)
def update(collection_id, mapping_id): """Update the mapping with id `mapping_id`. --- post: summary: Update a mapping parameters: - description: The collection id. in: path name: collection_id required: true schema: minimum: 1 type: integer example: 2 - description: The mapping id. in: path name: mapping_id required: true schema: minimum: 1 type: integer example: 2 requestBody: content: application/json: schema: $ref: '#/components/schemas/MappingCreate' responses: '200': content: application/json: schema: $ref: '#/components/schemas/Mapping' description: OK tags: - Collection - Mapping """ get_db_collection(collection_id, request.authz.WRITE) mapping = obj_or_404(Mapping.by_id(mapping_id)) data = parse_request("MappingCreate") mapping.update( query=load_query(), table_id=get_table_id(data), entityset_id=get_entityset_id(data), ) db.session.commit() return MappingSerializer.jsonify(mapping)
def update(collection_id, mapping_id): """Update the mapping with id `mapping_id`. --- post: summary: Update a mapping parameters: - description: The collection id. in: path name: collection_id required: true schema: minimum: 1 type: integer example: 2 - description: The mapping id. in: path name: mapping_id required: true schema: minimum: 1 type: integer example: 2 requestBody: content: application/json: schema: $ref: '#/components/schemas/MappingCreate' responses: '200': content: application/json: schema: $ref: '#/components/schemas/Mapping' description: OK tags: - Collection """ get_db_collection(collection_id, request.authz.WRITE) mapping = obj_or_404(Mapping.by_id(mapping_id)) data = parse_request('MappingCreate') entity_id = data.get('table_id') query = load_query() entity = get_index_entity(entity_id, request.authz.READ) mapping.update(query=query, table_id=entity.get('id')) return MappingSerializer.jsonify(mapping)
def flush(collection_id, mapping_id): """Flush all entities loaded by mapping with id `mapping_id`. --- post: summary: Flush entities loaded by a mapping parameters: - description: The collection id. in: path name: collection_id required: true schema: minimum: 1 type: integer example: 2 - description: The mapping id. in: path name: mapping_id required: true schema: minimum: 1 type: integer example: 2 responses: '202': description: No Content tags: - Collection - Mapping """ collection = get_db_collection(collection_id, request.authz.WRITE) mapping = obj_or_404(Mapping.by_id(mapping_id)) mapping.disabled = True mapping.last_run_status = None mapping.last_run_err_msg = None db.session.add(mapping) db.session.commit() queue_task( collection, OP_FLUSH_MAPPING, job_id=get_session_id(), mapping_id=mapping_id, ) return ("", 202)
def delete(collection_id, mapping_id): """Delete a mapping. --- delete: summary: Delete a mapping parameters: - description: The collection id. in: path name: collection_id required: true schema: minimum: 1 type: integer example: 2 - description: The mapping id. in: path name: mapping_id required: true schema: minimum: 1 type: integer example: 2 responses: '204': description: No Content tags: - Collection - Mapping """ collection = get_db_collection(collection_id, request.authz.WRITE) mapping = obj_or_404(Mapping.by_id(mapping_id)) mapping.delete() db.session.commit() queue_task( collection, OP_FLUSH_MAPPING, job_id=get_session_id(), payload={"mapping_id": mapping_id}, ) return ("", 204)
def trigger(collection_id, mapping_id): """Load entities by running the mapping with id `mapping_id`. Flushes previously loaded entities before loading new entities. --- post: summary: Load entities from a mapping parameters: - description: The collection id. in: path name: collection_id required: true schema: minimum: 1 type: integer example: 2 - description: The mapping id. in: path name: mapping_id required: true schema: minimum: 1 type: integer example: 2 responses: '202': description: No Content tags: - Collection - Mapping """ collection = get_db_collection(collection_id, request.authz.WRITE) mapping = obj_or_404(Mapping.by_id(mapping_id)) job_id = get_session_id() payload = {'mapping_id': mapping.id} queue_task(collection, OP_LOAD_MAPPING, job_id=job_id, payload=payload) collection.touch() db.session.commit() return ('', 202)