def delete_role(role): """Fully delete a role from the database and transfer the ownership of documents and entities created by it to the system user.""" # Doesn't update the search index, so they're out of sync. fallback = Role.load_cli_user() def _del(cls, col): pq = db.session.query(cls) pq = pq.filter(col == role.id) pq.delete(synchronize_session=False) def _repo(cls, col): pq = db.session.query(cls).filter(col == role.id) pq.update({col: fallback.id}, synchronize_session=False) _del(Alert, Alert.role_id) _del(Permission, Permission.role_id) _del(membership, membership.c.group_id) _del(membership, membership.c.member_id) _repo(Collection, Collection.creator_id) _repo(Document, Document.role_id) _repo(Entity, Entity.role_id) _repo(EntitySet, EntitySet.role_id) _repo(EntitySetItem, EntitySetItem.added_by_id) _repo(Mapping, Mapping.role_id) db.session.delete(role) db.session.commit()
def publish(foreign_id): """Make a collection visible to all users.""" collection = get_collection(foreign_id) role = Role.by_foreign_id(Role.SYSTEM_GUEST) editor = Role.load_cli_user() update_permission(role, collection, True, False, editor_id=editor.id) update_collection(collection)
def sample_entities(secret, properties, schematas, seed, sample_pct, limit, outfile): """Sample random entities""" random.seed(seed) authz = Authz.from_role(Role.load_cli_user()) collections = list(Collection.all_by_secret(secret, authz)) random.shuffle(collections) iter_proxies_kwargs = { "authz": authz, "schemata": schematas or None, "randomize": True, "random_seed": seed, } n_entities = 0 for collection in collections: for entity in iter_proxies(collection_id=collection.id, **iter_proxies_kwargs): if properties and not any( entity.properties.get(prop) for prop in properties): continue if not sample_pct or random.random() < sample_pct: write_object(outfile, entity) n_entities += 1 if limit and n_entities >= limit: return
def crawldir(path, language=None, country=None, foreign_id=None): """Crawl the given directory.""" path = decode_path(path) if path is None or not os.path.exists(path): log.error("Invalid path: %r", path) return path = os.path.abspath(os.path.normpath(path)) path_name = os.path.basename(path) if foreign_id is None: foreign_id = 'directory:%s' % slugify(path) role = Role.load_cli_user() collection = Collection.by_foreign_id(foreign_id) if collection is None: collection = Collection.create( { 'foreign_id': foreign_id, 'label': path_name, 'casefile': False }, role=role) if language is not None: collection.languages = [language] if country is not None: collection.countries = [country] db.session.commit() update_collection(collection) log.info('Crawling %r to %r...', path, collection.foreign_id) document = Document.by_keys(collection=collection, foreign_id=path) document.file_name = path_name ingest_document(document, path, role_id=role.id)
def ensure_collection(foreign_id, label): authz = Authz.from_role(Role.load_cli_user()) config = { 'foreign_id': foreign_id, 'label': label, } create_collection(config, authz) return Collection.by_foreign_id(foreign_id)
def publish(foreign_id): """Make a collection visible to all users.""" collection = Collection.by_foreign_id(foreign_id) if collection is None: raise ValueError("No such collection: %r" % foreign_id) role = Role.by_foreign_id(Role.SYSTEM_GUEST) editor = Role.load_cli_user() update_permission(role, collection, True, False, editor_id=editor.id) update_collection(collection)
def create_collection(data, role=None): role = role or Role.load_cli_user() created_at = datetime.utcnow() collection = Collection.create(data, role=role, created_at=created_at) if collection.created_at == created_at: publish(Events.CREATE_COLLECTION, actor_id=role.id, params={'collection': collection}) db.session.commit() index.index_collection(collection) return collection
def create_collection(data, role=None, sync=False): role = role or Role.load_cli_user() created_at = datetime.utcnow() collection = Collection.create(data, creator=role, created_at=created_at) publish(Events.CREATE_COLLECTION, params={'collection': collection}, actor_id=role.id) db.session.commit() Authz.flush() refresh_collection(collection.id) return index.index_collection(collection, sync=sync)
def crawldir(path, language=None, foreign_id=None): """Crawl the given directory.""" path = Path(path) if foreign_id is None: foreign_id = 'directory:%s' % slugify(path) authz = Authz.from_role(Role.load_cli_user()) config = {'foreign_id': foreign_id, 'label': path.name, 'casefile': False} create_collection(config, authz) collection = Collection.by_foreign_id(foreign_id) log.info('Crawling %s to %s (%s)...', path, foreign_id, collection.id) crawl_directory(collection, path) log.info('Complete. Make sure a worker is running :)')
def create_collection(data, role=None, sync=False): role = role or Role.load_cli_user() created_at = datetime.utcnow() collection = Collection.create(data, role=role, created_at=created_at) if collection.created_at == created_at: publish(Events.CREATE_COLLECTION, actor_id=role.id, params={'collection': collection}) db.session.commit() Authz.flush() refresh_collection(collection.id) return index.index_collection(collection, sync=sync)
def metadata(): """Get operational metadata for the frontend. --- get: summary: Retrieve system metadata from the application. responses: '200': description: OK content: application/json: schema: type: object tags: - System """ locale = get_locale() auth = {} if settings.PASSWORD_LOGIN: auth['password_login_uri'] = url_for('sessions_api.password_login') auth['registration_uri'] = url_for('roles_api.create_code') if settings.OAUTH: auth['oauth_uri'] = url_for('sessions_api.oauth_init') locales = settings.UI_LANGUAGES locales = {l: Locale(l).get_language_name(l) for l in locales} data = { 'status': 'ok', 'maintenance': request.authz.in_maintenance, 'app': { 'title': settings.APP_TITLE, 'description': settings.APP_DESCRIPTION, 'version': __version__, 'banner': settings.APP_BANNER, 'ui_uri': settings.APP_UI_URL, 'samples': settings.SAMPLE_SEARCHES, 'logo': settings.APP_LOGO, 'favicon': settings.APP_FAVICON, 'locale': str(locale), 'locales': locales }, 'categories': Collection.CATEGORIES, 'model': model, 'token': None, 'auth': auth } if settings.SINGLE_USER: role = Role.load_cli_user() authz = Authz.from_role(role) data['token'] = authz.to_token(role=role) return jsonify(data)
def create_collection(foreign_id, data, role=None): role = role or Role.load_cli_user() collection = Collection.by_foreign_id(foreign_id) if collection is None: data['foreign_id'] = foreign_id collection = Collection.create(data, role=role) else: languages = ensure_list(data.get('languages')) if len(languages): collection.languages = languages db.session.commit() index.index_collection(collection) return collection
def _metadata_locale(locale): # This is cached in part because latency on this endpoint is # particularly relevant to the first render being shown to a # user. auth = {} if settings.PASSWORD_LOGIN: auth['password_login_uri'] = url_for('sessions_api.password_login') auth['registration_uri'] = url_for('roles_api.create_code') if settings.OAUTH: auth['oauth_uri'] = url_for('sessions_api.oauth_init') locales = settings.UI_LANGUAGES locales = {l: Locale(l).get_language_name(l) for l in locales} data = { 'status': 'ok', 'maintenance': request.authz.in_maintenance, 'app': { 'title': settings.APP_TITLE, 'description': settings.APP_DESCRIPTION, 'version': __version__, 'banner': settings.APP_BANNER, 'ui_uri': settings.APP_UI_URL, 'samples': settings.SAMPLE_SEARCHES, 'logo': settings.APP_LOGO, 'favicon': settings.APP_FAVICON, 'locale': locale, 'locales': locales }, 'categories': Collection.CATEGORIES, 'frequencies': Collection.FREQUENCIES, 'pages': load_pages(locale), 'model': model.to_dict(), 'token': None, 'auth': auth } if settings.SINGLE_USER: role = Role.load_cli_user() authz = Authz.from_role(role) data['token'] = authz.to_token(role=role) return jsonify(data)
def load_entities(foreign_id, infile, unsafe=False): """Load FtM entities from the specified iJSON file.""" collection = ensure_collection(foreign_id, foreign_id) def read_entities(): for idx in count(1): line = infile.readline() if not line: return if idx % 1000 == 0: log.info("[%s] Loaded %s entities from: %s", collection, idx, infile.name) yield json.loads(line) role = Role.load_cli_user() bulk_write(collection, read_entities(), unsafe=unsafe, role_id=role.id, index=False) reindex_collection(collection)
def metadata(): """Get operational metadata for the frontend. --- get: summary: Retrieve system metadata from the application. responses: '200': description: OK content: application/json: schema: type: object tags: - System """ request.rate_limit = None locale = str(get_locale()) data = _metadata_locale(locale) if settings.SINGLE_USER: role = Role.load_cli_user() authz = Authz.from_role(role) data["token"] = authz.to_token(role=role) return jsonify(data)
def create_system_roles(): log.info("Creating system roles...") Role.load_or_create(Role.SYSTEM_GUEST, Role.SYSTEM, "All visitors") Role.load_or_create(Role.SYSTEM_USER, Role.SYSTEM, "Logged-in users") Role.load_cli_user() db.session.commit()