def update(id):
collection = get_db_collection(id, request.authz.WRITE)
data = parse_request(CollectionSchema)
collection.update(data)
db.session.commit()
update_collection(collection)
return serialize_data(collection, CollectionSchema)
def update(id):
collection = get_collection(id, request.authz.WRITE)
data = parse_request(schema=CollectionSchema)
collection.update(data)
db.session.commit()
update_collection(collection)
return view(id)
def bulk_load(config):
"""Bulk load entities from a CSV file or SQL database.
This is done by mapping the rows in the source data to entities and links
which can be understood by the entity index.
"""
for foreign_id, data in config.items():
collection = Collection.by_foreign_id(foreign_id)
if collection is None:
collection = Collection.create({
'foreign_id': foreign_id,
'managed': True,
'label': data.get('label') or foreign_id,
'summary': data.get('summary'),
'category': data.get('category'),
})
for role_fk in dict_list(data, 'roles', 'role'):
role = Role.by_foreign_id(role_fk)
if role is not None:
Permission.grant(collection, role, True, False)
else:
log.warning("Could not find role: %s", role_fk)
db.session.commit()
update_collection(collection)
for query in dict_list(data, 'queries', 'query'):
load_query(collection, query)
def create_collection(self, creator=None, **kwargs):
authz = Authz.from_role(creator)
collection = Collection.create(kwargs, authz)
db.session.add(collection)
db.session.commit()
update_collection(collection, sync=True)
return collection
def crawldir(path, language=None, country=None, foreign_id=None):
"""Crawl the given directory."""
path = decode_path(path)
if path is None or not os.path.exists(path):
log.error("Invalid path: %r", path)
return
path = os.path.abspath(os.path.normpath(path))
path_name = os.path.basename(path)
if foreign_id is None:
foreign_id = 'directory:%s' % slugify(path)
collection = Collection.by_foreign_id(foreign_id)
if collection is None:
collection = Collection.create({
'foreign_id': foreign_id,
'label': path_name,
'managed': True
})
if language is not None:
collection.languages = [language]
if country is not None:
collection.countries = [country]
db.session.commit()
update_collection(collection)
log.info('Crawling %r to %r...', path, collection.foreign_id)
document = Document.by_keys(collection=collection,
foreign_id=path)
document.file_name = path_name
ingest_document(document, path)
def delete(id):
entity = get_db_entity(id, request.authz.WRITE)
delete_entity(entity)
db.session.commit()
update_collection(entity.collection)
refresh_index(entities_index())
return ('', 204)
def publish(foreign_id):
"""Make a collection visible to all users."""
collection = get_collection(foreign_id)
role = Role.by_foreign_id(Role.SYSTEM_GUEST)
editor = Role.load_cli_user()
update_permission(role, collection, True, False, editor_id=editor.id)
update_collection(collection)
def create():
require(request.authz.logged_in)
data = parse_request(CollectionSchema)
collection = Collection.create(data, request.authz.role)
db.session.commit()
update_collection(collection)
return view(collection.id)
def update(document_id):
document = get_db_document(document_id, request.authz.WRITE)
data = parse_request(DocumentUpdateSchema)
document.update(data)
db.session.commit()
update_document(document)
update_collection(document.collection)
return view(document_id)
def create():
data = parse_request(schema=EntityCreateSchema)
collection = get_db_collection(data['collection_id'], request.authz.WRITE)
entity = Entity.create(data, collection)
db.session.commit()
update_entity(entity)
update_collection(collection)
return view(entity.id)
def flush_mapping(stage, collection, mapping_id, sync=False):
"""Delete entities loaded by a mapping"""
log.debug("Flushing entities for mapping: %s", mapping_id)
delete_entities(collection.id, mapping_id=mapping_id, sync=True)
drop_aggregator(collection)
collection.touch()
db.session.commit()
update_collection(collection)
def create():
data = parse_request(EntityCreateSchema)
collection = get_db_collection(data['collection_id'], request.authz.WRITE)
entity = Entity.create(data, collection)
db.session.commit()
data = update_entity(entity)
update_collection(collection)
return jsonify(data, schema=CombinedSchema)
def flush_mapping(collection, mapping_id, sync=True):
"""Delete entities loaded by a mapping"""
log.debug("Flushing entities for mapping: %s", mapping_id)
origin = mapping_origin(mapping_id)
aggregator = get_aggregator(collection)
aggregator.delete(origin=origin)
delete_entities(collection.id, origin=origin, sync=sync)
update_collection(collection, sync=sync)
def publish(foreign_id):
"""Make a collection visible to all users."""
collection = Collection.by_foreign_id(foreign_id)
if collection is None:
raise ValueError("No such collection: %r" % foreign_id)
role = Role.by_foreign_id(Role.SYSTEM_GUEST)
update_permission(role, collection, True, False)
update_collection(collection, roles=True)
def create():
data = parse_request(EntityCreateSchema)
collection = get_db_collection(data['collection_id'], request.authz.WRITE)
entity = Entity.create(data, collection)
db.session.commit()
data = update_entity(entity)
update_collection(collection)
refresh_index(entities_index())
return serialize_data(data, CombinedSchema)
def publish(foreign_id):
"""Make a collection visible to all users."""
collection = Collection.by_foreign_id(foreign_id)
if collection is None:
raise ValueError("No such collection: %r" % foreign_id)
role = Role.by_foreign_id(Role.SYSTEM_GUEST)
editor = Role.load_cli_user()
update_permission(role, collection, True, False, editor_id=editor.id)
update_collection(collection)
def update(collection_id):
"""
---
post:
summary: Update permissions for a collection
description: >
Update permissions for the collection with id `collection_id`
parameters:
- in: path
name: collection_id
required: true
schema:
type: integer
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/PermissionUpdateList'
responses:
'200':
description: OK
content:
application/json:
schema:
type: object
allOf:
- $ref: '#/components/schemas/QueryResponse'
properties:
results:
type: array
items:
$ref: '#/components/schemas/Permission'
tags:
- Permission
- Collection
"""
collection = get_db_collection(collection_id, request.authz.WRITE)
for permission in parse_request("PermissionUpdateList"):
role_obj = ensure_dict(permission.get("role"))
role_id = permission.get("role_id", role_obj.get("id"))
role = Role.by_id(role_id)
if not check_visible(role, request.authz):
continue
if role.is_public:
permission["write"] = False
if collection.casefile and role.is_public:
permission["read"] = False
update_permission(
role,
collection,
permission["read"],
permission["write"],
editor_id=request.authz.id,
)
update_collection(collection)
return index(collection_id)
def crawldir(path, language=None, foreign_id=None):
"""Crawl the given directory."""
path = Path(path)
if foreign_id is None:
foreign_id = 'directory:%s' % slugify(path)
collection = ensure_collection(foreign_id, path.name)
log.info('Crawling %s to %s (%s)...', path, foreign_id, collection.id)
crawl_directory(collection, path)
log.info('Complete. Make sure a worker is running :)')
update_collection(collection)
def update(id):
_, entity = get_entity(id, request.authz.WRITE)
data = parse_request(schema=EntitySchema)
if arg_bool('merge'):
data['data'] = merge_data(data.get('data') or {}, entity.data or {})
entity.update(data)
db.session.commit()
update_entity(entity)
update_collection(entity.collection)
return view(entity.id)
def test_index(self):
update_collection(self.col)
self.flush_index()
res = self.client.get('/api/2/collections')
assert res.status_code == 200, res
assert res.json['total'] == 0, res.json
_, headers = self.login(is_admin=True)
res = self.client.get('/api/2/collections', headers=headers)
assert res.status_code == 200, res
assert res.json['total'] == 1, res.json
def test_admin_all_access(self):
self.wl = Collection()
self.wl.label = "Test Collection"
self.wl.foreign_id = "test"
self.wl.creator = self.create_user("watcher")
db.session.add(self.wl)
db.session.commit()
update_collection(self.wl)
_, headers = self.login(foreign_id="admin", is_admin=True)
res = self.client.get("/api/2/collections/%s" % self.wl.id, headers=headers)
assert res.status_code == 200, res
def flush_mapping(stage, collection, mapping_id, sync=True):
"""Delete entities loaded by a mapping"""
log.debug("Flushing entities for mapping: %s", mapping_id)
origin = mapping_origin(mapping_id)
aggregator = get_aggregator(collection)
aggregator.delete(origin=origin)
aggregator.close()
delete_entities(collection.id, origin=origin, sync=sync)
collection.touch()
db.session.commit()
update_collection(collection, sync=sync)
def update(id):
entity = get_db_entity(id, request.authz.WRITE)
data = parse_request(schema=EntityUpdateSchema)
if as_bool(request.args.get('merge')):
props = merge_data(data.get('properties'), entity.data)
data['properties'] = props
entity.update(data)
db.session.commit()
update_entity(entity)
update_collection(entity.collection)
return view(entity.id)
def test_admin_all_access(self):
self.wl = Collection()
self.wl.label = "Test Collection"
self.wl.foreign_id = 'test'
self.wl.creator = self.create_user('watcher')
db.session.add(self.wl)
db.session.commit()
update_collection(self.wl)
_, headers = self.login(foreign_id='admin', is_admin=True)
res = self.client.get('/api/2/collections/%s' % self.wl.id,
headers=headers)
assert res.status_code == 200, res
def ingest_complete(collection, role_id=None):
"""Operations supposed to be performed when an ingest process completes."""
from aleph.logic.collections import update_collection, collection_url # noqa
update_collection(collection)
role = Role.by_id(role_id)
if role is not None and not collection.managed:
# notify the user that their import is completed.
notify_role_template(role,
collection.label,
'email/ingest.html',
collection=collection,
url=collection_url(collection.id))
def merge(id, other_id):
entity = get_db_entity(id, request.authz.WRITE)
other = get_db_entity(other_id, request.authz.WRITE)
try:
entity.merge(other)
except ValueError as ve:
raise BadRequest(ve.message)
db.session.commit()
data = update_entity(entity)
update_entity(other)
update_collection(entity.collection)
return serialize_data(data, CombinedSchema)
def merge(id, other_id):
entity = get_db_entity(id, request.authz.WRITE)
other = get_db_entity(other_id, request.authz.WRITE)
try:
entity.merge(other)
except ValueError as ve:
raise BadRequest(ve.message)
db.session.commit()
update_entity(entity)
update_entity(other)
update_collection(entity.collection)
return view(entity.id)
def permissions_update(id):
collection = get_db_collection(id, request.authz.WRITE)
for permission in parse_request(PermissionSchema, many=True):
role_id = permission.get('role', {}).get('id')
role = Role.by_id(role_id).first()
if not check_visible(role, request.authz):
continue
update_permission(role,
collection,
permission['read'],
permission['write'])
update_collection(collection, roles=True)
return permissions_index(id)
def update(id):
collection = get_db_collection(id, request.authz.WRITE)
data = parse_request(CollectionSchema)
collection.update(data)
db.session.commit()
data = update_collection(collection)
return jsonify(data, schema=CollectionSchema)
def load_query(collection, query):
if 'database' in query or 'databases' in query:
query = DatabaseQuery(collection, query)
else:
query = CSVQuery(collection, query)
rows = []
for row_idx, row in enumerate(query.iterrows(), 1):
rows.append(row)
if len(rows) >= PAGE:
load_rows(query, rows)
rows = []
if len(rows):
load_rows(query, rows)
update_collection(collection)
def update(collection_id):
collection = get_db_collection(collection_id, request.authz.WRITE)
data = parse_request(CollectionUpdateSchema)
sync = get_flag('sync')
collection.update(data)
db.session.commit()
data = update_collection(collection, sync=sync)
return CollectionSerializer.jsonify(data)
def update(id):
collection = get_db_collection(id, request.authz.WRITE)
data = parse_request(CollectionSchema)
sync = get_flag('sync')
collection.update(data)
db.session.commit()
data = update_collection(collection, sync=sync)
return serialize_data(data, CollectionSchema)
def update(id):
collection = get_db_collection(id, request.authz.WRITE)
for permission in parse_request(PermissionSchema, many=True):
role_id = permission.get('role_id')
role = Role.by_id(role_id)
if not check_visible(role, request.authz):
continue
if role.is_public:
permission['write'] = False
if collection.casefile and role.is_public:
permission['read'] = False
update_permission(role,
collection,
permission['read'],
permission['write'],
editor_id=request.authz.id)
update_collection(collection)
return index(id)
def create_collection(self, creator=None, **kwargs):
collection = Collection.create(kwargs, role=creator)
db.session.add(collection)
db.session.commit()
update_collection(collection)
return collection
def grant(self, collection, role, read, write):
Permission.grant(collection, role, read, write)
db.session.commit()
update_collection(collection)
