print('Please do not forget to set proper ownership and read+write permissions')
print('for web server user (www-data on Ubuntu)')
print('When done, please restart KonText Gunicorn server and also worker server (Celery)')
print('')
if __name__ == '__main__':
sys.path.insert(0, os.path.realpath('%s/../..' % os.path.dirname(os.path.realpath(__file__))))
sys.path.insert(0, os.path.realpath('%s/../../../scripts/' %
os.path.dirname(os.path.realpath(__file__))))
import autoconf
import initializer
settings = autoconf.settings
logger = autoconf.logger
initializer.init_plugin('db')
initializer.init_plugin('sessions')
initializer.init_plugin('auth')
parser = argparse.ArgumentParser(
description='Archive old records from Redis to a SQLite3 database')
parser.add_argument('action', metavar='ACTION', type=str, help='One of {new_archive, backup}')
parser.add_argument('-n', '--num_proc', type=int, default=1000,
help='For backup, how many items to process. Default is 1000')
parser.add_argument('-d', '--dry-run', action='store_true', default=False,
help=('allows running without affecting storage data '
'(not 100%% error prone as it reads/writes to Redis)'))
args = parser.parse_args()
if args.action == 'backup':
ans = run(conf=settings, num_proc=args.num_proc, dry_run=args.dry_run)
print(ans)
APP_PATH = os.path.realpath('%s/..' %
os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, '%s/../lib' % APP_PATH)
import settings
import initializer
import translation
from bgcalc.stderr2f import stderr_redirector
settings.load(os.path.join(APP_PATH, 'conf', 'config.xml'))
if settings.get('global', 'manatee_path', None):
sys.path.insert(0, settings.get('global', 'manatee_path'))
import manatee
os.environ['MANATEE_REGISTRY'] = settings.get('corpora', 'manatee_registry')
initializer.init_plugin('db')
initializer.init_plugin('sessions')
initializer.init_plugin('auth')
initializer.init_plugin('conc_cache')
initializer.init_plugin('query_storage')
initializer.init_plugin('conc_persistence')
initializer.init_plugin('sessions')
initializer.init_plugin('user_items')
initializer.init_plugin('corparch')
initializer.init_plugin('token_connect', optional=True)
initializer.init_plugin('live_attributes', optional=True)
initializer.init_plugin('dispatch_hook', optional=True)
translation.load_translations(settings.get('global', 'translations'))
translation.activate('en_US') # background jobs do not need localization
# as published by the Free Software Foundation; version 2
# dated June, 1991.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
import os
import sys
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '../../../scripts')))
import autoconf
import plugins
import initializer
initializer.init_plugin('db')
tmp_key = 'user_index:new'
uindex_key = 'user_index'
if __name__ == '__main__':
db = plugins.runtime.DB.instance
redis_db = getattr(db, 'redis')
data = redis_db.hgetall(uindex_key)
for k, v in data.items():
print('%s -> %s' % (k, v))
db.hash_set(tmp_key, k, v)
db.rename(uindex_key, 'user_index:old')
db.rename(tmp_key, uindex_key)
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
import os
import sys
sys.path.insert(
0,
os.path.realpath(
os.path.join(os.path.dirname(__file__), '../../../scripts')))
import autoconf
import plugins
import initializer
initializer.init_plugin('db')
tmp_key = 'user_index:new'
uindex_key = 'user_index'
if __name__ == '__main__':
db = plugins.runtime.DB.instance
redis_db = getattr(db, 'redis')
data = redis_db.hgetall(uindex_key)
for k, v in list(data.items()):
print(('%s -> %s' % (k, v)))
db.hash_set(tmp_key, k, v)
db.rename(uindex_key, 'user_index:old')
db.rename(tmp_key, uindex_key)
parser.add_argument(
'-k',
'--ucnk',
action='store_const',
const=True,
help='Customize the script for use with UCNK (CNC) database')
args = parser.parse_args()
import manatee
import initializer
import settings
conf_path = args.conf if args.conf else os.path.realpath(
os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', 'conf',
'config.xml'))
settings.load(conf_path, defaultdict(lambda: None))
initializer.init_plugin('db')
initializer.init_plugin('integration_db')
initializer.init_plugin('user_items')
initializer.init_plugin('corparch')
db = plugins.runtime.INTEGRATION_DB.instance
if args.ucnk:
rbackend = Backend(db,
user_table='user',
corp_table='corpora',
group_acc_table='relation',
user_acc_table='user_corpus_relation',
user_acc_corp_attr='corpus_id',
group_acc_corp_attr='corpora',
group_acc_group_attr='corplist')
wbackend = WriteBackend(db,
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
import sys
import os
import argparse
import json
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..'))
sys.path.insert(
0,
os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', 'scripts'))
import autoconf
import initializer
initializer.init_plugin('integration_db')
initializer.init_plugin('auth')
import plugins
from plugins.abstract.auth.hash import mk_pwd_hash_default
def import_user(data):
with plugins.runtime.AUTH as auth:
cursor = auth.db.cursor()
data['pwd_hash'] = mk_pwd_hash_default(
data['pwd']) if data['pwd'] else None
del data['pwd']
try:
cursor.execute(
'INSERT INTO kontext_user (username, firstname, lastname, email, affiliation, pwd_hash) '
'VALUES (%s, %s, %s, %s, %s, %s)',
CURR_PATH = os.path.realpath(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, '%s/lib' % CURR_PATH)
import settings
import initializer
import plugins
import translation
from bgcalc.stderr2f import stderr_redirector
settings.load('%s/conf/config.xml' % CURR_PATH)
if settings.get('global', 'manatee_path', None):
sys.path.insert(0, settings.get('global', 'manatee_path'))
import manatee
os.environ['MANATEE_REGISTRY'] = settings.get('corpora', 'manatee_registry')
initializer.init_plugin('db')
initializer.init_plugin('sessions')
initializer.init_plugin('auth')
initializer.init_plugin('conc_cache')
initializer.init_plugin('query_storage')
initializer.init_plugin('conc_persistence')
initializer.init_plugin('sessions')
initializer.init_plugin('user_items')
initializer.init_plugin('corparch')
initializer.init_plugin('live_attributes', optional=True)
translation.load_translations(settings.get('global', 'translations'))
translation.activate('en_US') # background jobs do not need localization
import concworker
import task
CURR_PATH = os.path.realpath(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, '%s/lib' % CURR_PATH)
import settings
import initializer
import plugins
import translation
from stderr2f import stderr_redirector
settings.load('%s/conf/config.xml' % CURR_PATH)
if settings.get('global', 'manatee_path', None):
sys.path.insert(0, settings.get('global', 'manatee_path'))
import manatee
os.environ['MANATEE_REGISTRY'] = settings.get('corpora', 'manatee_registry')
initializer.init_plugin('db')
initializer.init_plugin('sessions')
initializer.init_plugin('auth')
initializer.init_plugin('locking')
initializer.init_plugin('conc_cache')
initializer.init_plugin('conc_persistence')
initializer.init_plugin('sessions')
translation.load_translations(settings.get('global', 'translations'))
translation.activate('en_US') # background jobs do not need localization
from concworker import wcelery
import task
import freq_calc
_, conf = settings.get_full('corpora', 'conc_calc_backend')
def run(conf, num_proc, dry_run):
from_db = redis_connection(conf.get('plugins', 'db')['default:host'],
conf.get('plugins', 'db')['default:port'],
conf.get('plugins', 'db')['default:id'])
to_db = SQLite3Ops(conf.get('plugins')['conc_persistence']['ucnk:archive_db_path'])
archive_queue_key = conf.get('plugins')['conc_persistence']['ucnk:archive_queue_key']
archiver = Archiver(from_db=from_db, to_db=to_db, archive_queue_key=archive_queue_key)
return archiver.run(num_proc, dry_run)
if __name__ == '__main__':
sys.path.insert(0, os.path.realpath('%s/../..' % os.path.dirname(os.path.realpath(__file__))))
sys.path.insert(0, os.path.realpath('%s/../../../scripts/' % os.path.dirname(os.path.realpath(__file__))))
import autoconf
import initializer
settings = autoconf.settings
logger = autoconf.logger
initializer.init_plugin('db')
initializer.init_plugin('sessions')
initializer.init_plugin('auth')
parser = argparse.ArgumentParser(description='Archive old records from Synchronize data from mysql db to redis')
parser.add_argument('num_proc', metavar='NUM_PROC', type=int)
parser.add_argument('-d', '--dry-run', action='store_true',
help='allows running without affecting storage data (not 100% error prone as it reads/writes to Redis)')
args = parser.parse_args()
ans = run(conf=settings, num_proc=args.num_proc, dry_run=args.dry_run)
print(ans)
x for x in getattr(self._db_plugin, 'redis').scan_iter(
f'{startswith}*')
]
def list_get(self, key, from_idx=0, to_idx=-1):
return self._db_plugin.list_get(key, from_idx, to_idx)
if __name__ == '__main__':
logging.basicConfig()
conf_path = os.path.realpath(
os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', 'conf',
'config.xml'))
settings.load(conf_path, defaultdict(lambda: None))
initializer.init_plugin('integration_db')
initializer.init_plugin('db')
initializer.init_plugin('query_persistence')
with plugins.runtime.DB as db, plugins.runtime.INTEGRATION_DB as integration_db, plugins.runtime.QUERY_PERSISTENCE as qp:
full_data = []
custom_db = CustomDB(db)
for query_history_user in custom_db.keys('query_history:user:'******':')[-1])
for item in custom_db.list_get(query_history_user):
if 'query_id' in item:
query_id = item['query_id']
q_supertype = item.get('q_supertype',
item.get('qtype', 'conc'))
try:
corpora = qp.open(query_id)['corpora']
A simple script for manual cache clean-up
"""
import os
import sys
sys.path.insert(
0, os.path.realpath(os.path.join(os.path.dirname(__file__), '../..')))
sys.path.insert(
0,
os.path.realpath(
os.path.join(os.path.dirname(__file__), '../../../scripts')))
import autoconf
import plugins
import initializer
initializer.init_plugin('db')
initializer.init_plugin('conc_cache')
from plugins.default_conc_cache import cleanup
from plugins.default_conc_cache import DefaultCacheMapping
if __name__ == '__main__':
import argparse
def mk_key(corpus_id):
return DefaultCacheMapping.KEY_TEMPLATE.format(corpus_id)
parser = argparse.ArgumentParser(
description='A script to control UCNK concordance cache')
parser.add_argument('--dry-run',
'-d',
action='store_true',
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
"""
A simple script for manual cache clean-up
"""
import os
import sys
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '../..')))
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '../../../scripts')))
import autoconf
import plugins
import initializer
initializer.init_plugin('db')
initializer.init_plugin('conc_cache')
from plugins.redis_conc_cache import cleanup
from plugins.redis_conc_cache import RedisCacheMapping
if __name__ == '__main__':
import argparse
def mk_key(corpus_id):
return RedisCacheMapping.KEY_TEMPLATE % corpus_id
parser = argparse.ArgumentParser(description='A script to control UCNK metadata cache')
parser.add_argument('--dry-run', '-d', action='store_true',
help='Just analyze, do not modify anything')
parser.add_argument('--exclude', '-x', type=str, default=None,
APP_PATH = os.path.realpath(f'{os.path.dirname(os.path.abspath(__file__))}/..')
sys.path.insert(0, os.path.join(APP_PATH, 'lib')) # application libraries
import settings
from conclib.calc import cancel_conc_task
from corplib.corpus import KCorpus
from corplib import CorpusManager
from bgcalc.errors import CalcTaskNotFoundError
from bgcalc import calc_backend_client
import plugins
import initializer
settings.load(os.path.join(APP_PATH, 'conf', 'config.xml'))
os.environ['MANATEE_REGISTRY'] = settings.get('corpora', 'manatee_registry')
initializer.init_plugin('db')
initializer.init_plugin('integration_db')
initializer.init_plugin('query_persistence')
initializer.init_plugin('conc_cache')
initializer.init_plugin('auth')
TASK_LIMIT = settings.get_int('calc_backend', 'task_time_limit')
JOB_REFRESH_PERIOD = 1.0 # in seconds
CONC_CACHE_STATUS_REFRESH_PERIOD = 0.2 # in seconds
STATUS_KONTEXT_MAP = dict(
queued='PENDING',
started='STARTED',
deferred='deferred', # TODO Rq specific
finished='SUCCESS',
failed='FAILURE')
CURR_PATH = os.path.realpath(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, '%s/lib' % CURR_PATH)
import settings
import initializer
import plugins
import translation
from stderr2f import stderr_redirector
settings.load('%s/conf/config.xml' % CURR_PATH)
if settings.get('global', 'manatee_path', None):
sys.path.insert(0, settings.get('global', 'manatee_path'))
import manatee
os.environ['MANATEE_REGISTRY'] = settings.get('corpora', 'manatee_registry')
initializer.init_plugin('db')
initializer.init_plugin('sessions')
initializer.init_plugin('auth')
initializer.init_plugin('locking')
initializer.init_plugin('conc_cache')
initializer.init_plugin('conc_persistence')
initializer.init_plugin('sessions')
translation.load_translations(settings.get('global', 'translations'))
translation.activate('en_US') # background jobs do not need localization
from concworker import wcelery
import task
import freq_calc
_, conf = settings.get_full('corpora', 'conc_calc_backend')
