print('Please do not forget to set proper ownership and read+write permissions') print('for web server user (www-data on Ubuntu)') print('When done, please restart KonText Gunicorn server and also worker server (Celery)') print('') if __name__ == '__main__': sys.path.insert(0, os.path.realpath('%s/../..' % os.path.dirname(os.path.realpath(__file__)))) sys.path.insert(0, os.path.realpath('%s/../../../scripts/' % os.path.dirname(os.path.realpath(__file__)))) import autoconf import initializer settings = autoconf.settings logger = autoconf.logger initializer.init_plugin('db') initializer.init_plugin('sessions') initializer.init_plugin('auth') parser = argparse.ArgumentParser( description='Archive old records from Redis to a SQLite3 database') parser.add_argument('action', metavar='ACTION', type=str, help='One of {new_archive, backup}') parser.add_argument('-n', '--num_proc', type=int, default=1000, help='For backup, how many items to process. Default is 1000') parser.add_argument('-d', '--dry-run', action='store_true', default=False, help=('allows running without affecting storage data ' '(not 100%% error prone as it reads/writes to Redis)')) args = parser.parse_args() if args.action == 'backup': ans = run(conf=settings, num_proc=args.num_proc, dry_run=args.dry_run) print(ans)
APP_PATH = os.path.realpath('%s/..' % os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, '%s/../lib' % APP_PATH) import settings import initializer import translation from bgcalc.stderr2f import stderr_redirector settings.load(os.path.join(APP_PATH, 'conf', 'config.xml')) if settings.get('global', 'manatee_path', None): sys.path.insert(0, settings.get('global', 'manatee_path')) import manatee os.environ['MANATEE_REGISTRY'] = settings.get('corpora', 'manatee_registry') initializer.init_plugin('db') initializer.init_plugin('sessions') initializer.init_plugin('auth') initializer.init_plugin('conc_cache') initializer.init_plugin('query_storage') initializer.init_plugin('conc_persistence') initializer.init_plugin('sessions') initializer.init_plugin('user_items') initializer.init_plugin('corparch') initializer.init_plugin('token_connect', optional=True) initializer.init_plugin('live_attributes', optional=True) initializer.init_plugin('dispatch_hook', optional=True) translation.load_translations(settings.get('global', 'translations')) translation.activate('en_US') # background jobs do not need localization
# as published by the Free Software Foundation; version 2 # dated June, 1991. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. import os import sys sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '../../../scripts'))) import autoconf import plugins import initializer initializer.init_plugin('db') tmp_key = 'user_index:new' uindex_key = 'user_index' if __name__ == '__main__': db = plugins.runtime.DB.instance redis_db = getattr(db, 'redis') data = redis_db.hgetall(uindex_key) for k, v in data.items(): print('%s -> %s' % (k, v)) db.hash_set(tmp_key, k, v) db.rename(uindex_key, 'user_index:old') db.rename(tmp_key, uindex_key)
# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. import os import sys sys.path.insert( 0, os.path.realpath( os.path.join(os.path.dirname(__file__), '../../../scripts'))) import autoconf import plugins import initializer initializer.init_plugin('db') tmp_key = 'user_index:new' uindex_key = 'user_index' if __name__ == '__main__': db = plugins.runtime.DB.instance redis_db = getattr(db, 'redis') data = redis_db.hgetall(uindex_key) for k, v in list(data.items()): print(('%s -> %s' % (k, v))) db.hash_set(tmp_key, k, v) db.rename(uindex_key, 'user_index:old') db.rename(tmp_key, uindex_key)
parser.add_argument( '-k', '--ucnk', action='store_const', const=True, help='Customize the script for use with UCNK (CNC) database') args = parser.parse_args() import manatee import initializer import settings conf_path = args.conf if args.conf else os.path.realpath( os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', 'conf', 'config.xml')) settings.load(conf_path, defaultdict(lambda: None)) initializer.init_plugin('db') initializer.init_plugin('integration_db') initializer.init_plugin('user_items') initializer.init_plugin('corparch') db = plugins.runtime.INTEGRATION_DB.instance if args.ucnk: rbackend = Backend(db, user_table='user', corp_table='corpora', group_acc_table='relation', user_acc_table='user_corpus_relation', user_acc_corp_attr='corpus_id', group_acc_corp_attr='corpora', group_acc_group_attr='corplist') wbackend = WriteBackend(db,
# This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. import sys import os import argparse import json sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..')) sys.path.insert( 0, os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', 'scripts')) import autoconf import initializer initializer.init_plugin('integration_db') initializer.init_plugin('auth') import plugins from plugins.abstract.auth.hash import mk_pwd_hash_default def import_user(data): with plugins.runtime.AUTH as auth: cursor = auth.db.cursor() data['pwd_hash'] = mk_pwd_hash_default( data['pwd']) if data['pwd'] else None del data['pwd'] try: cursor.execute( 'INSERT INTO kontext_user (username, firstname, lastname, email, affiliation, pwd_hash) ' 'VALUES (%s, %s, %s, %s, %s, %s)',
CURR_PATH = os.path.realpath(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, '%s/lib' % CURR_PATH) import settings import initializer import plugins import translation from bgcalc.stderr2f import stderr_redirector settings.load('%s/conf/config.xml' % CURR_PATH) if settings.get('global', 'manatee_path', None): sys.path.insert(0, settings.get('global', 'manatee_path')) import manatee os.environ['MANATEE_REGISTRY'] = settings.get('corpora', 'manatee_registry') initializer.init_plugin('db') initializer.init_plugin('sessions') initializer.init_plugin('auth') initializer.init_plugin('conc_cache') initializer.init_plugin('query_storage') initializer.init_plugin('conc_persistence') initializer.init_plugin('sessions') initializer.init_plugin('user_items') initializer.init_plugin('corparch') initializer.init_plugin('live_attributes', optional=True) translation.load_translations(settings.get('global', 'translations')) translation.activate('en_US') # background jobs do not need localization import concworker import task
CURR_PATH = os.path.realpath(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, '%s/lib' % CURR_PATH) import settings import initializer import plugins import translation from stderr2f import stderr_redirector settings.load('%s/conf/config.xml' % CURR_PATH) if settings.get('global', 'manatee_path', None): sys.path.insert(0, settings.get('global', 'manatee_path')) import manatee os.environ['MANATEE_REGISTRY'] = settings.get('corpora', 'manatee_registry') initializer.init_plugin('db') initializer.init_plugin('sessions') initializer.init_plugin('auth') initializer.init_plugin('locking') initializer.init_plugin('conc_cache') initializer.init_plugin('conc_persistence') initializer.init_plugin('sessions') translation.load_translations(settings.get('global', 'translations')) translation.activate('en_US') # background jobs do not need localization from concworker import wcelery import task import freq_calc _, conf = settings.get_full('corpora', 'conc_calc_backend')
def run(conf, num_proc, dry_run): from_db = redis_connection(conf.get('plugins', 'db')['default:host'], conf.get('plugins', 'db')['default:port'], conf.get('plugins', 'db')['default:id']) to_db = SQLite3Ops(conf.get('plugins')['conc_persistence']['ucnk:archive_db_path']) archive_queue_key = conf.get('plugins')['conc_persistence']['ucnk:archive_queue_key'] archiver = Archiver(from_db=from_db, to_db=to_db, archive_queue_key=archive_queue_key) return archiver.run(num_proc, dry_run) if __name__ == '__main__': sys.path.insert(0, os.path.realpath('%s/../..' % os.path.dirname(os.path.realpath(__file__)))) sys.path.insert(0, os.path.realpath('%s/../../../scripts/' % os.path.dirname(os.path.realpath(__file__)))) import autoconf import initializer settings = autoconf.settings logger = autoconf.logger initializer.init_plugin('db') initializer.init_plugin('sessions') initializer.init_plugin('auth') parser = argparse.ArgumentParser(description='Archive old records from Synchronize data from mysql db to redis') parser.add_argument('num_proc', metavar='NUM_PROC', type=int) parser.add_argument('-d', '--dry-run', action='store_true', help='allows running without affecting storage data (not 100% error prone as it reads/writes to Redis)') args = parser.parse_args() ans = run(conf=settings, num_proc=args.num_proc, dry_run=args.dry_run) print(ans)
x for x in getattr(self._db_plugin, 'redis').scan_iter( f'{startswith}*') ] def list_get(self, key, from_idx=0, to_idx=-1): return self._db_plugin.list_get(key, from_idx, to_idx) if __name__ == '__main__': logging.basicConfig() conf_path = os.path.realpath( os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', 'conf', 'config.xml')) settings.load(conf_path, defaultdict(lambda: None)) initializer.init_plugin('integration_db') initializer.init_plugin('db') initializer.init_plugin('query_persistence') with plugins.runtime.DB as db, plugins.runtime.INTEGRATION_DB as integration_db, plugins.runtime.QUERY_PERSISTENCE as qp: full_data = [] custom_db = CustomDB(db) for query_history_user in custom_db.keys('query_history:user:'******':')[-1]) for item in custom_db.list_get(query_history_user): if 'query_id' in item: query_id = item['query_id'] q_supertype = item.get('q_supertype', item.get('qtype', 'conc')) try: corpora = qp.open(query_id)['corpora']
A simple script for manual cache clean-up """ import os import sys sys.path.insert( 0, os.path.realpath(os.path.join(os.path.dirname(__file__), '../..'))) sys.path.insert( 0, os.path.realpath( os.path.join(os.path.dirname(__file__), '../../../scripts'))) import autoconf import plugins import initializer initializer.init_plugin('db') initializer.init_plugin('conc_cache') from plugins.default_conc_cache import cleanup from plugins.default_conc_cache import DefaultCacheMapping if __name__ == '__main__': import argparse def mk_key(corpus_id): return DefaultCacheMapping.KEY_TEMPLATE.format(corpus_id) parser = argparse.ArgumentParser( description='A script to control UCNK concordance cache') parser.add_argument('--dry-run', '-d', action='store_true',
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. """ A simple script for manual cache clean-up """ import os import sys sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '../..'))) sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '../../../scripts'))) import autoconf import plugins import initializer initializer.init_plugin('db') initializer.init_plugin('conc_cache') from plugins.redis_conc_cache import cleanup from plugins.redis_conc_cache import RedisCacheMapping if __name__ == '__main__': import argparse def mk_key(corpus_id): return RedisCacheMapping.KEY_TEMPLATE % corpus_id parser = argparse.ArgumentParser(description='A script to control UCNK metadata cache') parser.add_argument('--dry-run', '-d', action='store_true', help='Just analyze, do not modify anything') parser.add_argument('--exclude', '-x', type=str, default=None,
APP_PATH = os.path.realpath(f'{os.path.dirname(os.path.abspath(__file__))}/..') sys.path.insert(0, os.path.join(APP_PATH, 'lib')) # application libraries import settings from conclib.calc import cancel_conc_task from corplib.corpus import KCorpus from corplib import CorpusManager from bgcalc.errors import CalcTaskNotFoundError from bgcalc import calc_backend_client import plugins import initializer settings.load(os.path.join(APP_PATH, 'conf', 'config.xml')) os.environ['MANATEE_REGISTRY'] = settings.get('corpora', 'manatee_registry') initializer.init_plugin('db') initializer.init_plugin('integration_db') initializer.init_plugin('query_persistence') initializer.init_plugin('conc_cache') initializer.init_plugin('auth') TASK_LIMIT = settings.get_int('calc_backend', 'task_time_limit') JOB_REFRESH_PERIOD = 1.0 # in seconds CONC_CACHE_STATUS_REFRESH_PERIOD = 0.2 # in seconds STATUS_KONTEXT_MAP = dict( queued='PENDING', started='STARTED', deferred='deferred', # TODO Rq specific finished='SUCCESS', failed='FAILURE')