def flush(collection_id, mapping_id):
"""Flush all entities loaded by mapping with id `mapping_id`.
---
post:
summary: Flush entities loaded by a mapping
parameters:
- description: The collection id.
in: path
name: collection_id
required: true
schema:
minimum: 1
type: integer
example: 2
- description: The mapping id.
in: path
name: mapping_id
required: true
schema:
minimum: 1
type: integer
example: 2
responses:
'202':
description: No Content
tags:
- Collection
"""
collection = get_db_collection(collection_id, request.authz.WRITE)
mapping = obj_or_404(Mapping.by_id(mapping_id))
queue_task(collection,
OP_FLUSH_MAPPING,
job_id=get_session_id(),
payload={'mapping_id': mapping.id})
return ('', 202)
def load_mapping(collection, mapping_id, sync=False):
"""Flush and reload all entities generated by a mapping."""
mapping = Mapping.by_id(mapping_id)
if mapping is None:
return log.error("Could not find mapping: %s", mapping_id)
origin = mapping_origin(mapping.id)
aggregator = get_aggregator(collection)
aggregator.delete(origin=origin)
delete_entities(collection.id, origin=origin, sync=True)
if mapping.disabled:
return log.info("Mapping is disabled: %s", mapping_id)
publish(
Events.LOAD_MAPPING,
params={
"collection": collection,
"table": mapping.table_id
},
channels=[collection, mapping.role],
actor_id=mapping.role_id,
)
try:
map_to_aggregator(collection, mapping, aggregator)
aggregate_model(collection, aggregator)
index_aggregator(collection, aggregator, sync=sync)
mapping.set_status(status=Status.SUCCESS)
db.session.commit()
except Exception as exc:
mapping.set_status(status=Status.FAILED, error=str(exc))
db.session.commit()
aggregator.delete(origin=origin)
finally:
aggregator.close()
def delete(collection_id, mapping_id):
"""Delete a mapping.
---
delete:
summary: Delete a mapping
parameters:
- description: The collection id.
in: path
name: collection_id
required: true
schema:
minimum: 1
type: integer
example: 2
- description: The mapping id.
in: path
name: mapping_id
required: true
schema:
minimum: 1
type: integer
example: 2
responses:
'204':
description: No Content
tags:
- Collection
- Mapping
"""
get_db_collection(collection_id, request.authz.WRITE)
mapping = obj_or_404(Mapping.by_id(mapping_id))
mapping.delete()
db.session.commit()
return ('', 204)
def view(collection_id, mapping_id):
"""Return the mapping with id `mapping_id`.
---
get:
summary: Fetch a mapping
parameters:
- description: The collection id.
in: path
name: collection_id
required: true
schema:
minimum: 1
type: integer
example: 2
- description: The mapping id.
in: path
name: mapping_id
required: true
schema:
minimum: 1
type: integer
example: 2
responses:
'200':
content:
application/json:
schema:
$ref: '#/components/schemas/Mapping'
description: OK
tags:
- Collection
"""
get_db_collection(collection_id, request.authz.WRITE)
mapping = obj_or_404(Mapping.by_id(mapping_id))
return MappingSerializer.jsonify(mapping)
def load_mapping(stage, collection, mapping_id):
"""Flush and reload all entities generated by a mapping."""
mapping = Mapping.by_id(mapping_id)
if mapping is None:
return log.error("Could not find mapping: %s", mapping_id)
flush_mapping(stage, collection, mapping_id)
publish(Events.LOAD_MAPPING,
params={'collection': collection, 'table': mapping.table_id},
channels=[collection, mapping.role],
actor_id=mapping.role_id)
mapper = make_mapper(collection, mapping)
aggregator = get_aggregator(collection)
try:
writer = aggregator.bulk()
entities_count = 0
entity_ids = set()
for idx, record in enumerate(mapper.source.records, 1):
for entity in mapper.map(record).values():
if entity.schema.is_a('Thing'):
entity.add('proof', mapping.table_id)
entity = collection.ns.apply(entity)
entity_ids.add(entity.id)
entities_count += 1
fragment = '%s-%s' % (mapping.id, idx)
writer.put(entity, fragment=fragment)
if idx > 0 and idx % 500 == 0:
payload = {
'entity_ids': entity_ids,
'mapping_id': mapping.id
}
queue_task(collection, OP_INDEX,
job_id=stage.job.id,
payload=payload)
entity_ids = set()
stage.report_finished(500)
log.info("[%s] Loaded %s records, %s entities...",
collection.foreign_id,
idx, entities_count)
writer.flush()
payload = {
'entity_ids': entity_ids,
'mapping_id': mapping.id
}
queue_task(collection, OP_INDEX,
job_id=stage.job.id,
payload=payload)
mapping.set_status(status=Mapping.SUCCESS)
log.info("[%s] Mapping done (%s entities)",
mapping.id, entities_count)
except Exception as exc:
mapping.set_status(status=Mapping.FAILED, error=str(exc))
finally:
aggregator.close()
def trigger(collection_id, mapping_id):
"""Load entities by running the mapping with id `mapping_id`. Flushes
previously loaded entities before loading new entities.
---
post:
summary: Load entities from a mapping
parameters:
- description: The collection id.
in: path
name: collection_id
required: true
schema:
minimum: 1
type: integer
example: 2
- description: The mapping id.
in: path
name: mapping_id
required: true
schema:
minimum: 1
type: integer
example: 2
responses:
'202':
description: No Content
tags:
- Collection
- Mapping
"""
collection = get_db_collection(collection_id, request.authz.WRITE)
mapping = obj_or_404(Mapping.by_id(mapping_id))
mapping.disabled = False
mapping.set_status(Status.PENDING)
db.session.commit()
job_id = get_session_id()
queue_task(collection,
OP_LOAD_MAPPING,
job_id=job_id,
mapping_id=mapping.id)
mapping = obj_or_404(Mapping.by_id(mapping_id))
return MappingSerializer.jsonify(mapping, status=202)
def update(collection_id, mapping_id):
"""Update the mapping with id `mapping_id`.
---
post:
summary: Update a mapping
parameters:
- description: The collection id.
in: path
name: collection_id
required: true
schema:
minimum: 1
type: integer
example: 2
- description: The mapping id.
in: path
name: mapping_id
required: true
schema:
minimum: 1
type: integer
example: 2
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/MappingCreate'
responses:
'200':
content:
application/json:
schema:
$ref: '#/components/schemas/Mapping'
description: OK
tags:
- Collection
- Mapping
"""
get_db_collection(collection_id, request.authz.WRITE)
mapping = obj_or_404(Mapping.by_id(mapping_id))
data = parse_request("MappingCreate")
mapping.update(
query=load_query(),
table_id=get_table_id(data),
entityset_id=get_entityset_id(data),
)
db.session.commit()
return MappingSerializer.jsonify(mapping)
def update(collection_id, mapping_id):
"""Update the mapping with id `mapping_id`.
---
post:
summary: Update a mapping
parameters:
- description: The collection id.
in: path
name: collection_id
required: true
schema:
minimum: 1
type: integer
example: 2
- description: The mapping id.
in: path
name: mapping_id
required: true
schema:
minimum: 1
type: integer
example: 2
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/MappingCreate'
responses:
'200':
content:
application/json:
schema:
$ref: '#/components/schemas/Mapping'
description: OK
tags:
- Collection
"""
get_db_collection(collection_id, request.authz.WRITE)
mapping = obj_or_404(Mapping.by_id(mapping_id))
data = parse_request('MappingCreate')
entity_id = data.get('table_id')
query = load_query()
entity = get_index_entity(entity_id, request.authz.READ)
mapping.update(query=query, table_id=entity.get('id'))
return MappingSerializer.jsonify(mapping)
def flush(collection_id, mapping_id):
"""Flush all entities loaded by mapping with id `mapping_id`.
---
post:
summary: Flush entities loaded by a mapping
parameters:
- description: The collection id.
in: path
name: collection_id
required: true
schema:
minimum: 1
type: integer
example: 2
- description: The mapping id.
in: path
name: mapping_id
required: true
schema:
minimum: 1
type: integer
example: 2
responses:
'202':
description: No Content
tags:
- Collection
- Mapping
"""
collection = get_db_collection(collection_id, request.authz.WRITE)
mapping = obj_or_404(Mapping.by_id(mapping_id))
mapping.disabled = True
mapping.last_run_status = None
mapping.last_run_err_msg = None
db.session.add(mapping)
db.session.commit()
queue_task(
collection,
OP_FLUSH_MAPPING,
job_id=get_session_id(),
mapping_id=mapping_id,
)
return ("", 202)
def delete(collection_id, mapping_id):
"""Delete a mapping.
---
delete:
summary: Delete a mapping
parameters:
- description: The collection id.
in: path
name: collection_id
required: true
schema:
minimum: 1
type: integer
example: 2
- description: The mapping id.
in: path
name: mapping_id
required: true
schema:
minimum: 1
type: integer
example: 2
responses:
'204':
description: No Content
tags:
- Collection
- Mapping
"""
collection = get_db_collection(collection_id, request.authz.WRITE)
mapping = obj_or_404(Mapping.by_id(mapping_id))
mapping.delete()
db.session.commit()
queue_task(
collection,
OP_FLUSH_MAPPING,
job_id=get_session_id(),
payload={"mapping_id": mapping_id},
)
return ("", 204)
def trigger(collection_id, mapping_id):
"""Load entities by running the mapping with id `mapping_id`. Flushes
previously loaded entities before loading new entities.
---
post:
summary: Load entities from a mapping
parameters:
- description: The collection id.
in: path
name: collection_id
required: true
schema:
minimum: 1
type: integer
example: 2
- description: The mapping id.
in: path
name: mapping_id
required: true
schema:
minimum: 1
type: integer
example: 2
responses:
'202':
description: No Content
tags:
- Collection
- Mapping
"""
collection = get_db_collection(collection_id, request.authz.WRITE)
mapping = obj_or_404(Mapping.by_id(mapping_id))
job_id = get_session_id()
payload = {'mapping_id': mapping.id}
queue_task(collection, OP_LOAD_MAPPING, job_id=job_id, payload=payload)
collection.touch()
db.session.commit()
return ('', 202)
