def delete_role(role):
"""Fully delete a role from the database and transfer the
ownership of documents and entities created by it to the
system user."""
# Doesn't update the search index, so they're out of sync.
fallback = Role.load_cli_user()
def _del(cls, col):
pq = db.session.query(cls)
pq = pq.filter(col == role.id)
pq.delete(synchronize_session=False)
def _repo(cls, col):
pq = db.session.query(cls).filter(col == role.id)
pq.update({col: fallback.id}, synchronize_session=False)
_del(Alert, Alert.role_id)
_del(Permission, Permission.role_id)
_del(membership, membership.c.group_id)
_del(membership, membership.c.member_id)
_repo(Collection, Collection.creator_id)
_repo(Document, Document.role_id)
_repo(Entity, Entity.role_id)
_repo(EntitySet, EntitySet.role_id)
_repo(EntitySetItem, EntitySetItem.added_by_id)
_repo(Mapping, Mapping.role_id)
db.session.delete(role)
db.session.commit()
def publish(foreign_id):
"""Make a collection visible to all users."""
collection = get_collection(foreign_id)
role = Role.by_foreign_id(Role.SYSTEM_GUEST)
editor = Role.load_cli_user()
update_permission(role, collection, True, False, editor_id=editor.id)
update_collection(collection)
def sample_entities(secret, properties, schematas, seed, sample_pct, limit,
outfile):
"""Sample random entities"""
random.seed(seed)
authz = Authz.from_role(Role.load_cli_user())
collections = list(Collection.all_by_secret(secret, authz))
random.shuffle(collections)
iter_proxies_kwargs = {
"authz": authz,
"schemata": schematas or None,
"randomize": True,
"random_seed": seed,
}
n_entities = 0
for collection in collections:
for entity in iter_proxies(collection_id=collection.id,
**iter_proxies_kwargs):
if properties and not any(
entity.properties.get(prop) for prop in properties):
continue
if not sample_pct or random.random() < sample_pct:
write_object(outfile, entity)
n_entities += 1
if limit and n_entities >= limit:
return
def crawldir(path, language=None, country=None, foreign_id=None):
"""Crawl the given directory."""
path = decode_path(path)
if path is None or not os.path.exists(path):
log.error("Invalid path: %r", path)
return
path = os.path.abspath(os.path.normpath(path))
path_name = os.path.basename(path)
if foreign_id is None:
foreign_id = 'directory:%s' % slugify(path)
role = Role.load_cli_user()
collection = Collection.by_foreign_id(foreign_id)
if collection is None:
collection = Collection.create(
{
'foreign_id': foreign_id,
'label': path_name,
'casefile': False
},
role=role)
if language is not None:
collection.languages = [language]
if country is not None:
collection.countries = [country]
db.session.commit()
update_collection(collection)
log.info('Crawling %r to %r...', path, collection.foreign_id)
document = Document.by_keys(collection=collection, foreign_id=path)
document.file_name = path_name
ingest_document(document, path, role_id=role.id)
def ensure_collection(foreign_id, label):
authz = Authz.from_role(Role.load_cli_user())
config = {
'foreign_id': foreign_id,
'label': label,
}
create_collection(config, authz)
return Collection.by_foreign_id(foreign_id)
def publish(foreign_id):
"""Make a collection visible to all users."""
collection = Collection.by_foreign_id(foreign_id)
if collection is None:
raise ValueError("No such collection: %r" % foreign_id)
role = Role.by_foreign_id(Role.SYSTEM_GUEST)
editor = Role.load_cli_user()
update_permission(role, collection, True, False, editor_id=editor.id)
update_collection(collection)
def create_collection(data, role=None):
role = role or Role.load_cli_user()
created_at = datetime.utcnow()
collection = Collection.create(data, role=role, created_at=created_at)
if collection.created_at == created_at:
publish(Events.CREATE_COLLECTION,
actor_id=role.id,
params={'collection': collection})
db.session.commit()
index.index_collection(collection)
return collection
def create_collection(data, role=None, sync=False):
role = role or Role.load_cli_user()
created_at = datetime.utcnow()
collection = Collection.create(data, creator=role, created_at=created_at)
publish(Events.CREATE_COLLECTION,
params={'collection': collection},
actor_id=role.id)
db.session.commit()
Authz.flush()
refresh_collection(collection.id)
return index.index_collection(collection, sync=sync)
def crawldir(path, language=None, foreign_id=None):
"""Crawl the given directory."""
path = Path(path)
if foreign_id is None:
foreign_id = 'directory:%s' % slugify(path)
authz = Authz.from_role(Role.load_cli_user())
config = {'foreign_id': foreign_id, 'label': path.name, 'casefile': False}
create_collection(config, authz)
collection = Collection.by_foreign_id(foreign_id)
log.info('Crawling %s to %s (%s)...', path, foreign_id, collection.id)
crawl_directory(collection, path)
log.info('Complete. Make sure a worker is running :)')
def create_collection(data, role=None, sync=False):
role = role or Role.load_cli_user()
created_at = datetime.utcnow()
collection = Collection.create(data, role=role, created_at=created_at)
if collection.created_at == created_at:
publish(Events.CREATE_COLLECTION,
actor_id=role.id,
params={'collection': collection})
db.session.commit()
Authz.flush()
refresh_collection(collection.id)
return index.index_collection(collection, sync=sync)
def metadata():
"""Get operational metadata for the frontend.
---
get:
summary: Retrieve system metadata from the application.
responses:
'200':
description: OK
content:
application/json:
schema:
type: object
tags:
- System
"""
locale = get_locale()
auth = {}
if settings.PASSWORD_LOGIN:
auth['password_login_uri'] = url_for('sessions_api.password_login')
auth['registration_uri'] = url_for('roles_api.create_code')
if settings.OAUTH:
auth['oauth_uri'] = url_for('sessions_api.oauth_init')
locales = settings.UI_LANGUAGES
locales = {l: Locale(l).get_language_name(l) for l in locales}
data = {
'status': 'ok',
'maintenance': request.authz.in_maintenance,
'app': {
'title': settings.APP_TITLE,
'description': settings.APP_DESCRIPTION,
'version': __version__,
'banner': settings.APP_BANNER,
'ui_uri': settings.APP_UI_URL,
'samples': settings.SAMPLE_SEARCHES,
'logo': settings.APP_LOGO,
'favicon': settings.APP_FAVICON,
'locale': str(locale),
'locales': locales
},
'categories': Collection.CATEGORIES,
'model': model,
'token': None,
'auth': auth
}
if settings.SINGLE_USER:
role = Role.load_cli_user()
authz = Authz.from_role(role)
data['token'] = authz.to_token(role=role)
return jsonify(data)
def create_collection(foreign_id, data, role=None):
role = role or Role.load_cli_user()
collection = Collection.by_foreign_id(foreign_id)
if collection is None:
data['foreign_id'] = foreign_id
collection = Collection.create(data, role=role)
else:
languages = ensure_list(data.get('languages'))
if len(languages):
collection.languages = languages
db.session.commit()
index.index_collection(collection)
return collection
def _metadata_locale(locale):
# This is cached in part because latency on this endpoint is
# particularly relevant to the first render being shown to a
# user.
auth = {}
if settings.PASSWORD_LOGIN:
auth['password_login_uri'] = url_for('sessions_api.password_login')
auth['registration_uri'] = url_for('roles_api.create_code')
if settings.OAUTH:
auth['oauth_uri'] = url_for('sessions_api.oauth_init')
locales = settings.UI_LANGUAGES
locales = {l: Locale(l).get_language_name(l) for l in locales}
data = {
'status': 'ok',
'maintenance': request.authz.in_maintenance,
'app': {
'title': settings.APP_TITLE,
'description': settings.APP_DESCRIPTION,
'version': __version__,
'banner': settings.APP_BANNER,
'ui_uri': settings.APP_UI_URL,
'samples': settings.SAMPLE_SEARCHES,
'logo': settings.APP_LOGO,
'favicon': settings.APP_FAVICON,
'locale': locale,
'locales': locales
},
'categories': Collection.CATEGORIES,
'frequencies': Collection.FREQUENCIES,
'pages': load_pages(locale),
'model': model.to_dict(),
'token': None,
'auth': auth
}
if settings.SINGLE_USER:
role = Role.load_cli_user()
authz = Authz.from_role(role)
data['token'] = authz.to_token(role=role)
return jsonify(data)
def load_entities(foreign_id, infile, unsafe=False):
"""Load FtM entities from the specified iJSON file."""
collection = ensure_collection(foreign_id, foreign_id)
def read_entities():
for idx in count(1):
line = infile.readline()
if not line:
return
if idx % 1000 == 0:
log.info("[%s] Loaded %s entities from: %s", collection, idx,
infile.name)
yield json.loads(line)
role = Role.load_cli_user()
bulk_write(collection,
read_entities(),
unsafe=unsafe,
role_id=role.id,
index=False)
reindex_collection(collection)
def metadata():
"""Get operational metadata for the frontend.
---
get:
summary: Retrieve system metadata from the application.
responses:
'200':
description: OK
content:
application/json:
schema:
type: object
tags:
- System
"""
request.rate_limit = None
locale = str(get_locale())
data = _metadata_locale(locale)
if settings.SINGLE_USER:
role = Role.load_cli_user()
authz = Authz.from_role(role)
data["token"] = authz.to_token(role=role)
return jsonify(data)
def create_system_roles():
log.info("Creating system roles...")
Role.load_or_create(Role.SYSTEM_GUEST, Role.SYSTEM, "All visitors")
Role.load_or_create(Role.SYSTEM_USER, Role.SYSTEM, "Logged-in users")
Role.load_cli_user()
db.session.commit()
