def __print_error(restore_backup=False):
print(wrap_text_in_a_box("The kit encountered errors in migrating passwords. Please contact The Invenio developers in order to get support providing all the printed messages."))
if restore_backup:
if __restoring_from_backup():
print(wrap_text_in_a_box("Passwords were restored from the backup, but you still need to migrate them in order to use this version of Invenio."))
else:
print(wrap_text_in_a_box("The kit was't unable to restore passwords from the backup"))
sys.exit(1)
def populate(yes_i_know=False):
"""Load CDS general demorecords."""
from invenio.utils.text import wrap_text_in_a_box, wait_for_user
from invenio.config import CFG_PREFIX
from invenio.modules.scheduler.models import SchTASK
wait_for_user(
wrap_text_in_a_box(
"WARNING: You are going to override data in tables!"))
print(">>> Going to load demo records...")
xml_data = pkg_resources.resource_filename(
'cds', os.path.join('demosite', 'data', 'cds-demobibdata.xml'))
job_id = SchTASK.query.count()
for cmd in [
"bin/bibupload -u admin -i %s" % (xml_data, ),
"bin/bibupload %d" % (job_id + 1, ),
"bin/bibindex -u admin",
"bin/bibindex %d" % (job_id + 2, ),
"bin/bibindex -u admin -w global",
"bin/bibindex %d" % (job_id + 3, ),
"bin/bibreformat -u admin -o HB",
"bin/bibreformat %d" % (job_id + 4, ),
"bin/webcoll -u admin",
"bin/webcoll %d" % (job_id + 5, ),
]:
cmd = os.path.join(CFG_PREFIX, cmd)
if os.system(cmd):
print("ERROR: failed execution of", cmd)
sys.exit(1)
print(">>> CDS Demo records loaded successfully.")
def _display_ask_string_box(self, msg, default=""):
"""Utility to display confirmation boxes."""
msg = wrap_text_in_a_box(msg, style='no_border')
rows = msg.split('\n')
height = len(rows) + 3
width = max([len(row) for row in rows]) + 4
self.win = self.curses.newwin(
height,
width,
(self.height - height) / 2 + 1,
(self.width - width) / 2 + 1
)
self.panel = self.curses.panel.new_panel(self.win)
self.panel.top()
self.win.border()
i = 1
for row in rows:
self.win.addstr(i, 2, row, self.current_attr)
i += 1
self.win.refresh()
self.win.move(height - 2, 2)
self.curses.echo()
for c in reversed(default):
self.curses.ungetch(c)
ret = self.win.getstr()
self.curses.noecho()
self.panel = None
return ret
def cmd_upgrade(upgrader=None):
"""Command for applying upgrades."""
from flask import current_app
from invenio.utils.text import wrap_text_in_a_box, wait_for_user
logfilename = os.path.join(current_app.config['CFG_LOGDIR'],
'invenio_upgrader.log')
if not upgrader:
upgrader = InvenioUpgrader()
logger = upgrader.get_logger(logfilename=logfilename)
try:
upgrades = upgrader.get_upgrades()
if not upgrades:
logger.info("All upgrades have been applied.")
return
logger.info("Following upgrade(s) will be applied:")
for u in upgrades:
title = u['__doc__']
if title:
logger.info(" * %s (%s)" % (u['id'], title))
else:
logger.info(" * %s" % u['id'])
logger.info("Running pre-upgrade checks...")
upgrader.pre_upgrade_checks(upgrades)
logger.info("Calculating estimated upgrade time...")
estimate = upgrader.human_estimate(upgrades)
wait_for_user(wrap_text_in_a_box(
"WARNING: You are going to upgrade your installation "
"(estimated time: %s)!" % estimate))
for u in upgrades:
title = u['__doc__']
if title:
logger.info("Applying %s (%s)" % (u['id'], title))
else:
logger.info("Applying %s" % u['id'])
upgrader.apply_upgrade(u)
logger.info("Running post-upgrade checks...")
upgrader.post_upgrade_checks(upgrades)
if upgrader.has_warnings():
logger.warning("Upgrade completed with %s warnings - please check "
"log-file for further information:\nless %s"
% (upgrader.get_warnings_count(), logfilename))
else:
logger.info("Upgrade completed successfully.")
except RuntimeError as e:
for msg in e.args:
logger.error(unicode(msg))
logger.info("Please check log file for further information:\n"
"less %s" % logfilename)
sys.exit(1)
def _display_YN_box(self, msg):
"""Utility to display confirmation boxes."""
msg += ' (Y/N)'
msg = wrap_text_in_a_box(msg, style='no_border')
rows = msg.split('\n')
height = len(rows) + 2
width = max([len(row) for row in rows]) + 4
self.win = self.curses.newwin(
height,
width,
(self.height - height) / 2 + 1,
(self.width - width) / 2 + 1
)
self.panel = self.curses.panel.new_panel(self.win)
self.panel.top()
self.win.border()
i = 1
for row in rows:
self.win.addstr(i, 2, row, self.current_attr)
i += 1
self.win.refresh()
try:
while 1:
c = self.win.getch()
if c in (ord('y'), ord('Y')):
return True
elif c in (ord('n'), ord('N')):
return False
finally:
self.panel = None
def cmd_upgrade(upgrader=None):
"""Command for applying upgrades."""
from invenio.config import CFG_LOGDIR
from invenio.utils.text import wrap_text_in_a_box, wait_for_user
logfilename = os.path.join(CFG_LOGDIR, 'invenio_upgrader.log')
if not upgrader:
upgrader = InvenioUpgrader()
logger = upgrader.get_logger(logfilename=logfilename)
try:
upgrades = upgrader.get_upgrades()
if not upgrades:
logger.info("All upgrades have been applied.")
return
logger.info("Following upgrade(s) will be applied:")
for u in upgrades:
title = u['__doc__']
if title:
logger.info(" * %s (%s)" % (u['id'], title))
else:
logger.info(" * %s" % u['id'])
logger.info("Running pre-upgrade checks...")
upgrader.pre_upgrade_checks(upgrades)
logger.info("Calculating estimated upgrade time...")
estimate = upgrader.human_estimate(upgrades)
wait_for_user(
wrap_text_in_a_box(
"WARNING: You are going to upgrade your installation "
"(estimated time: %s)!" % estimate))
for u in upgrades:
title = u['__doc__']
if title:
logger.info("Applying %s (%s)" % (u['id'], title))
else:
logger.info("Applying %s" % u['id'])
upgrader.apply_upgrade(u)
logger.info("Running post-upgrade checks...")
upgrader.post_upgrade_checks(upgrades)
if upgrader.has_warnings():
logger.warning("Upgrade completed with %s warnings - please check "
"log-file for further information:\nless %s" %
(upgrader.get_warnings_count(), logfilename))
else:
logger.info("Upgrade completed successfully.")
except RuntimeError as e:
for msg in e.args:
logger.error(unicode(msg))
logger.info("Please check log file for further information:\n"
"less %s" % logfilename)
sys.exit(1)
def _display_YN_box(self, msg):
"""Utility to display confirmation boxes."""
msg += ' (Y/N)'
msg = wrap_text_in_a_box(msg, style='no_border')
rows = msg.split('\n')
height = len(rows) + 2
width = max([len(row) for row in rows]) + 4
self.win = self.curses.newwin(height, width,
(self.height - height) / 2 + 1,
(self.width - width) / 2 + 1)
self.panel = self.curses.panel.new_panel(self.win)
self.panel.top()
self.win.border()
i = 1
for row in rows:
self.win.addstr(i, 2, row, self.current_attr)
i += 1
self.win.refresh()
try:
while 1:
c = self.win.getch()
if c in (ord('y'), ord('Y')):
return True
elif c in (ord('n'), ord('N')):
return False
finally:
self.panel = None
def _display_ask_string_box(self, msg, default=""):
"""Utility to display confirmation boxes."""
msg = wrap_text_in_a_box(msg, style='no_border')
rows = msg.split('\n')
height = len(rows) + 3
width = max([len(row) for row in rows]) + 4
self.win = self.curses.newwin(height, width,
(self.height - height) / 2 + 1,
(self.width - width) / 2 + 1)
self.panel = self.curses.panel.new_panel(self.win)
self.panel.top()
self.win.border()
i = 1
for row in rows:
self.win.addstr(i, 2, row, self.current_attr)
i += 1
self.win.refresh()
self.win.move(height - 2, 2)
self.curses.echo()
for c in reversed(default):
self.curses.ungetch(c)
ret = self.win.getstr()
self.curses.noecho()
self.panel = None
return ret
def drop(yes_i_know=False, quiet=False):
"""Drop database tables."""
print(">>> Going to drop tables and related data on filesystem ...")
from invenio.utils.date import get_time_estimator
from invenio.utils.text import wrap_text_in_a_box, wait_for_user
from invenio.ext.sqlalchemy.utils import test_sqla_connection, \
test_sqla_utf8_chain
from invenio.ext.sqlalchemy import db, models
from invenio.modules.jsonalchemy.wrappers import StorageEngine
# Step 0: confirm deletion
wait_for_user(
wrap_text_in_a_box(
"WARNING: You are going to destroy your database tables and related "
"data on filesystem!"))
# Step 1: test database connection
test_sqla_connection()
test_sqla_utf8_chain()
list(models)
# Step 2: destroy associated data
tables = list(reversed(db.metadata.sorted_tables))
def _dropper(items, prefix, dropper):
N = len(items)
prefix = prefix.format(N)
e = get_time_estimator(N)
dropped = 0
if quiet:
print(prefix)
for i, table in enumerate(items):
try:
if not quiet:
print_progress(1.0 * (i + 1) / N,
prefix=prefix,
suffix=str(
datetime.timedelta(seconds=e()[0])))
dropper(table)
dropped += 1
except Exception:
print('\r>>> problem with dropping {0}'.format(table))
current_app.logger.exception(table)
if dropped == N:
print(">>> Everything has been dropped successfully.")
else:
print("ERROR: not all items were properly dropped.")
print(">>> Dropped", dropped, 'out of', N)
_dropper(StorageEngine.__storage_engine_registry__,
'>>> Dropping {0} storage engines ...',
lambda api: api.storage_engine.drop())
_dropper(tables, '>>> Dropping {0} tables ...',
lambda table: table.drop(bind=db.engine, checkfirst=True))
def drop(yes_i_know=False, quiet=False):
"""Drop database tables."""
print(">>> Going to drop tables and related data on filesystem ...")
from invenio.utils.date import get_time_estimator
from invenio.utils.text import wrap_text_in_a_box, wait_for_user
from invenio.ext.sqlalchemy.utils import test_sqla_connection, \
test_sqla_utf8_chain
from invenio.ext.sqlalchemy import db, models
from invenio.modules.jsonalchemy.wrappers import StorageEngine
# Step 0: confirm deletion
wait_for_user(wrap_text_in_a_box(
"WARNING: You are going to destroy your database tables and related "
"data on filesystem!"))
# Step 1: test database connection
test_sqla_connection()
test_sqla_utf8_chain()
list(models)
# Step 2: destroy associated data
tables = list(reversed(db.metadata.sorted_tables))
def _dropper(items, prefix, dropper):
N = len(items)
prefix = prefix.format(N)
e = get_time_estimator(N)
dropped = 0
if quiet:
print(prefix)
for i, table in enumerate(items):
try:
if not quiet:
print_progress(
1.0 * (i + 1) / N, prefix=prefix,
suffix=str(datetime.timedelta(seconds=e()[0])))
dropper(table)
dropped += 1
except Exception:
print('\r>>> problem with dropping {0}'.format(table))
current_app.logger.exception(table)
if dropped == N:
print(">>> Everything has been dropped successfully.")
else:
print("ERROR: not all items were properly dropped.")
print(">>> Dropped", dropped, 'out of', N)
_dropper(StorageEngine.__storage_engine_registry__,
'>>> Dropping {0} storage engines ...',
lambda api: api.storage_engine.drop())
_dropper(tables, '>>> Dropping {0} tables ...',
lambda table: table.drop(bind=db.engine, checkfirst=True))
def test_sqla_connection():
"""Test connection with the database."""
print("Test connection... \t", end="")
try:
from sqlalchemy import inspect
from invenio.ext.sqlalchemy import db
inspector = inspect(db.engine)
inspector.get_table_names()
except OperationalError as err:
from invenio.utils.text import wrap_text_in_a_box
from invenio.config import (
CFG_DATABASE_HOST,
CFG_DATABASE_PORT,
CFG_DATABASE_NAME,
CFG_DATABASE_USER,
CFG_DATABASE_PASS,
)
print(" [ERROR]")
print(
wrap_text_in_a_box(
"""\
DATABASE CONNECTIVITY ERROR:
%(errmsg)s.\n
Perhaps you need to set up database and connection rights?
If yes, then please execute:
console> inveniomanage database init --yes-i-know
The values printed above were detected from your
configuration. If they are not right, then please edit your
invenio-local.conf file and rerun 'inveniocfg --update-all' first.
If the problem is of different nature, then please inspect
the above error message and fix the problem before continuing."""
% {
"errmsg": err.args[0],
"dbname": CFG_DATABASE_NAME,
"dbhost": CFG_DATABASE_HOST,
"dbport": CFG_DATABASE_PORT,
"dbuser": CFG_DATABASE_USER,
"dbpass": CFG_DATABASE_PASS,
"webhost": (
CFG_DATABASE_HOST == "localhost" and "localhost" or os.popen("hostname -f", "r").read().strip()
),
}
)
)
sys.exit(1)
print(" [OK]")
def test_sqla_connection():
"""Test connection with the database."""
print("Test connection... \t", end="")
try:
from sqlalchemy import inspect
from invenio.ext.sqlalchemy import db
inspector = inspect(db.engine)
inspector.get_table_names()
except OperationalError as err:
from invenio.utils.text import wrap_text_in_a_box
from invenio.config import CFG_DATABASE_HOST, \
CFG_DATABASE_PORT, CFG_DATABASE_NAME, CFG_DATABASE_USER, \
CFG_DATABASE_PASS
print(" [ERROR]")
print(
wrap_text_in_a_box(
"""\
DATABASE CONNECTIVITY ERROR:
%(errmsg)s.\n
Perhaps you need to set up database and connection rights?
If yes, then please execute:
console> inveniomanage database init --yes-i-know
The values printed above were detected from your
configuration. If they are not right, then please edit your
invenio-local.conf file and rerun 'inveniocfg --update-all' first.
If the problem is of different nature, then please inspect
the above error message and fix the problem before continuing.""" % {
'errmsg':
err.args[0],
'dbname':
CFG_DATABASE_NAME,
'dbhost':
CFG_DATABASE_HOST,
'dbport':
CFG_DATABASE_PORT,
'dbuser':
CFG_DATABASE_USER,
'dbpass':
CFG_DATABASE_PASS,
'webhost':
(CFG_DATABASE_HOST == 'localhost' and 'localhost'
or os.popen('hostname -f', 'r').read().strip()),
}))
sys.exit(1)
print(" [OK]")
def get_callable_documentation(the_callable):
"""
Returns a string with the callable signature and its docstring.
:param the_callable: the callable to be analyzed.
:type the_callable: function/callable.
:return: the signature.
"""
return wrap_text_in_a_box(
title=get_callable_signature_as_string(the_callable),
body=(getattr(the_callable, "__doc__") or "No documentation").replace("\n", "\n\n"),
style="ascii_double",
)
def get_callable_documentation(the_callable):
"""
Returns a string with the callable signature and its docstring.
:param the_callable: the callable to be analyzed.
:type the_callable: function/callable.
:return: the signature.
"""
return wrap_text_in_a_box(
title=get_callable_signature_as_string(the_callable),
body=(getattr(the_callable, '__doc__') or 'No documentation').replace(
'\n', '\n\n'),
style='ascii_double')
def init(user='******', password='', yes_i_know=False):
"""Initialize database and user."""
from invenio.ext.sqlalchemy import db
from invenio.utils.text import wrap_text_in_a_box, wait_for_user
# Step 0: confirm deletion
wait_for_user(
wrap_text_in_a_box(
"WARNING: You are going to destroy your database tables! Run first"
" `inveniomanage database drop`."))
# Step 1: drop database and recreate it
if db.engine.name == 'mysql':
# FIXME improve escaping
args = dict((k, str(v).replace('$', '\$'))
for (k, v) in iteritems(current_app.config)
if k.startswith('CFG_DATABASE'))
args = dict(zip(args, map(quote, args.values())))
prefix = ('{cmd} -u {user} --password={password} '
'-h {CFG_DATABASE_HOST} -P {CFG_DATABASE_PORT} ')
cmd_prefix = prefix.format(cmd='mysql',
user=user,
password=password,
**args)
cmd_admin_prefix = prefix.format(cmd='mysqladmin',
user=user,
password=password,
**args)
cmds = [
cmd_prefix + '-e "DROP DATABASE IF EXISTS {CFG_DATABASE_NAME}"',
(cmd_prefix + '-e "CREATE DATABASE IF NOT EXISTS '
'{CFG_DATABASE_NAME} DEFAULT CHARACTER SET utf8 '
'COLLATE utf8_general_ci"'),
# Create user and grant access to database.
(cmd_prefix + '-e "GRANT ALL PRIVILEGES ON '
'{CFG_DATABASE_NAME}.* TO {CFG_DATABASE_USER}@localhost '
'IDENTIFIED BY {CFG_DATABASE_PASS}"'),
(cmd_prefix + '-e "GRANT ALL PRIVILEGES ON '
'{CFG_DATABASE_NAME}.* TO {CFG_DATABASE_USER}@\'%\' '
'IDENTIFIED BY {CFG_DATABASE_PASS}"'),
cmd_admin_prefix + 'flush-privileges'
]
for cmd in cmds:
cmd = cmd.format(**args)
print(cmd)
if os.system(cmd):
print("ERROR: failed execution of", cmd, file=sys.stderr)
sys.exit(1)
print('>>> Database has been installed.')
def init(user='******', password='', yes_i_know=False):
"""Initialize database and user."""
from invenio.ext.sqlalchemy.utils import initialize_database_user
from invenio.utils.text import wrap_text_in_a_box, wait_for_user
from sqlalchemy_utils.functions import database_exists, create_database, \
drop_database
from sqlalchemy.engine.url import URL
from sqlalchemy import create_engine
# Step 0: confirm deletion
wait_for_user(wrap_text_in_a_box(
"WARNING: You are going to destroy your database tables! Run first"
" `inveniomanage database drop`."
))
# Step 1: create URI to connect admin user
cfg = current_app.config
SQLALCHEMY_DATABASE_URI = URL(
cfg.get('CFG_DATABASE_TYPE', 'mysql'),
username=user,
password=password,
host=cfg.get('CFG_DATABASE_HOST'),
database=cfg.get('CFG_DATABASE_NAME'),
port=cfg.get('CFG_DATABASE_PORT'),
)
# Step 2: drop the database if already exists
if database_exists(SQLALCHEMY_DATABASE_URI):
drop_database(SQLALCHEMY_DATABASE_URI)
print('>>> Database has been dropped.')
# Step 3: create the database
create_database(SQLALCHEMY_DATABASE_URI, encoding='utf8')
print('>>> Database has been created.')
# Step 4: setup connection with special user
engine = create_engine(SQLALCHEMY_DATABASE_URI)
engine.connect()
# Step 5: grant privileges for the user
initialize_database_user(
engine=engine,
database_name=current_app.config['CFG_DATABASE_NAME'],
database_user=current_app.config['CFG_DATABASE_USER'],
database_pass=current_app.config['CFG_DATABASE_PASS'],
)
print('>>> Database user has been initialized.')
def migrate_passwords():
print(wrap_text_in_a_box(title="Password migration kit (for Invenio prior to v0.92.1)", style='conclusion'))
print("Checking your installation...")
if __check_update_possibility():
print("Tables are OK.")
else:
print("No need to update.")
sys.exit(1)
admin_pwd = __get_admin_pwd()
print("There are %i passwords to migrate (including guest users)" % __count_current_passwords())
print("=========================================================")
print("Creating backup...", end=' ')
sys.stdout.flush()
if __creating_backup():
print("OK")
else:
__print_error()
print("Migrating all Null password to ''...", end=' ')
sys.stdout.flush()
if __migrate_passwords_from_null_to_empty_quotes():
print("OK")
else:
__print_error(True)
print("Changing the column type from string to blob...", end=' ')
sys.stdout.flush()
if __migrate_column_from_string_to_blob():
print("OK")
else:
__print_error(True)
print("Encrypting passwords...", end=' ')
sys.stdout.flush()
if __encrypt_password():
print("OK")
else:
__print_error(True)
print("Checking that admin could enter...", end=' ')
sys.stdout.flush()
if __check_admin(admin_pwd):
print("OK")
else:
__print_error(True)
print("Removing backup...", end=' ')
sys.stdout.flush()
if __removing_backup():
print("OK")
else:
__print_error()
print("=========================================================")
print("Migration to encrypted local passwords has been successful.")
def display_task_options(self):
"""Nicely display information about current process."""
msg = ' id: %i\n\n' % self.currentrow[0]
pid = get_task_pid(self.currentrow[0])
if pid is not None:
msg += ' pid: %s\n\n' % pid
msg += ' priority: %s\n\n' % self.currentrow[8]
msg += ' proc: %s\n\n' % self.currentrow[1]
msg += ' user: %s\n\n' % self.currentrow[2]
msg += ' runtime: %s\n\n' % self.currentrow[3].strftime("%Y-%m-%d %H:%M:%S")
msg += ' sleeptime: %s\n\n' % self.currentrow[4]
msg += ' status: %s\n\n' % self.currentrow[5]
msg += ' progress: %s\n\n' % self.currentrow[6]
arguments = marshal.loads(self.currentrow[7])
if type(arguments) is dict:
# FIXME: REMOVE AFTER MAJOR RELEASE 1.0
msg += ' options : %s\n\n' % arguments
else:
msg += 'executable : %s\n\n' % arguments[0]
args_str = ' '.join(arguments[1:])
if len(args_str) > 500:
args_str = args_str[:500] + '...'
msg += ' arguments : %s\n\n' % args_str
msg += '\n\nPress q to quit this panel...'
msg = wrap_text_in_a_box(msg, style='no_border', break_long=True)
rows = msg.split('\n')
height = len(rows) + 2
width = max([len(row) for row in rows]) + 4
try:
self.win = self.curses.newwin(
height,
width,
(self.height - height) / 2 + 1,
(self.width - width) / 2 + 1
)
except self.curses.error:
return
self.panel = self.curses.panel.new_panel(self.win)
self.panel.top()
self.win.border()
i = 1
for row in rows:
self.win.addstr(i, 2, row, self.current_attr)
i += 1
self.win.refresh()
while self.win.getkey() != 'q':
pass
self.panel = None
def display_task_options(self):
"""Nicely display information about current process."""
msg = ' id: %i\n\n' % self.currentrow[0]
pid = get_task_pid(self.currentrow[0])
if pid is not None:
msg += ' pid: %s\n\n' % pid
msg += ' priority: %s\n\n' % self.currentrow[8]
msg += ' proc: %s\n\n' % self.currentrow[1]
msg += ' user: %s\n\n' % self.currentrow[2]
msg += ' runtime: %s\n\n' % self.currentrow[3].strftime("%Y-%m-%d %H:%M:%S")
msg += ' sleeptime: %s\n\n' % self.currentrow[4]
msg += ' status: %s\n\n' % self.currentrow[5]
msg += ' progress: %s\n\n' % self.currentrow[6]
arguments = marshal.loads(self.currentrow[7])
if type(arguments) is dict:
# FIXME: REMOVE AFTER MAJOR RELEASE 1.0
msg += ' options : %s\n\n' % arguments
else:
msg += 'executable : %s\n\n' % arguments[0]
args_str = ' '.join(arguments[1:])
if len(args_str) > 500:
args_str = args_str[:500] + '...'
msg += ' arguments : %s\n\n' % args_str
msg += '\n\nPress q to quit this panel...'
msg = wrap_text_in_a_box(msg, style='no_border', break_long=True)
rows = msg.split('\n')
height = len(rows) + 2
width = max([len(row) for row in rows]) + 4
try:
self.win = self.curses.newwin(
height,
width,
(self.height - height) / 2 + 1,
(self.width - width) / 2 + 1
)
except self.curses.error:
return
self.panel = self.curses.panel.new_panel(self.win)
self.panel.top()
self.win.border()
i = 1
for row in rows:
self.win.addstr(i, 2, row, self.current_attr)
i += 1
self.win.refresh()
while self.win.getkey() != 'q':
pass
self.panel = None
def main():
"""Core loop."""
logfilename = '%s/fulltext_files_migration_kit-%s.log' % (CFG_LOGDIR, datetime.today().strftime('%Y%m%d%H%M%S'))
try:
logfile = open(logfilename, 'w')
except IOError as e:
print(wrap_text_in_a_box('NOTE: it\'s impossible to create the log:\n\n %s\n\nbecause of:\n\n %s\n\nPlease run this migration kit as the same user who runs Invenio (e.g. Apache)' % (logfilename, e), style='conclusion', break_long=False))
sys.exit(1)
recids = retrieve_fulltext_recids()
print(wrap_text_in_a_box ("""This script migrate the filesystem structure used to store fulltext files to the new stricter structure.
This script must not be run during normal Invenio operations.
It is safe to run this script. No file will be deleted.
Anyway it is recommended to run a backup of the filesystem structure just in case.
A backup of the database tables involved will be automatically performed.""", style='important'))
print("%s records will be migrated/fixed." % len(recids))
print("Please type yes if you want to go further:", end=' ')
if not check_yes():
print("INTERRUPTED")
sys.exit(1)
print("Backing up database tables")
try:
if not backup_tables():
print(wrap_text_in_a_box("""It appears that is not the first time that you run this script.
Backup tables have been already created by a previous run.
In order for the script to go further they need to be removed.""", style='important'))
print("Please, type yes if you agree to remove them and go further:", end=' ')
if not check_yes():
print(wrap_text_in_a_box("INTERRUPTED", style='conclusion'))
sys.exit(1)
print("Backing up database tables (after dropping previous backup)", end=' ')
backup_tables(drop=True)
print("-> OK")
else:
print("-> OK")
except Exception as e:
print(wrap_text_in_a_box("Unexpected error while backing up tables. Please, do your checks: %s" % e, style='conclusion'))
sys.exit(1)
print("Created a complete log file into %s" % logfilename)
for recid in recids:
if not fix_recid(recid, logfile):
logfile.close()
print(wrap_text_in_a_box(title="INTERRUPTED BECAUSE OF ERROR!", body="""Please see the log file %s for what was the status of record %s prior to the error. Contact %s in case of problems, attaching the log.""" % (logfilename, recid, CFG_SITE_SUPPORT_EMAIL),
style='conclusion'))
sys.exit(1)
print(wrap_text_in_a_box("DONE", style='conclusion'))
def populate(packages=[],
default_data=True,
files=None,
job_id=0,
extra_info=None,
yes_i_know=False):
"""Load demo records. Useful for testing purposes."""
from invenio.utils.text import wrap_text_in_a_box, wait_for_user
# Load cli interfaces for tools that we are going to need
from invenio_records.manage import main as rmanager
# Step 0: confirm deletion
wait_for_user(
wrap_text_in_a_box(
"WARNING: You are going to override data in tables!"))
if not default_data:
print('>>> Default data has been skiped (--no-data).')
return
if not packages:
packages = ['invenio_demosite.base']
from werkzeug.utils import import_string
map(import_string, packages)
print(">>> Going to load demo records...")
if files is None:
files = [
pkg_resources.resource_filename(
'invenio',
os.path.join('testsuite', 'data', 'demo_record_marc_data.xml'))
]
# upload demo site files:
bibupload_flags = '-t marcxml'
for f in files:
job_id += 1
for cmd in ((rmanager,
"records create %s %s" % (bibupload_flags, f)), ):
if run_py_func(*cmd, passthrough=True).exit_code:
print("ERROR: failed execution of", *cmd)
sys.exit(1)
print(">>> Demo records loaded successfully.")
def upload_to_site(marcxml, yes_i_know):
"""
makes the appropriate calls to bibupload to get the MARCXML record onto
the site.
@param: marcxml (string): the absolute location of the MARCXML that was
generated by this programme
@param: yes_i_know (boolean): if true, no confirmation. if false, prompt.
@output: a new record on the invenio site
@return: None
"""
if not yes_i_know:
wait_for_user(wrap_text_in_a_box('You are going to upload new ' +
'plots to the server.'))
task_low_level_submission('bibupload', 'admin', '-a', marcxml)
def init(user='******', password='', yes_i_know=False):
"""Initialize database and user."""
from invenio.ext.sqlalchemy import db
from invenio.utils.text import wrap_text_in_a_box, wait_for_user
# Step 0: confirm deletion
wait_for_user(wrap_text_in_a_box(
"WARNING: You are going to destroy your database tables! Run first"
" `inveniomanage database drop`."
))
# Step 1: drop database and recreate it
if db.engine.name == 'mysql':
# FIXME improve escaping
args = dict((k, str(v).replace('$', '\$'))
for (k, v) in iteritems(current_app.config)
if k.startswith('CFG_DATABASE'))
args = dict(zip(args, map(quote, args.values())))
prefix = ('{cmd} -u {user} --password={password} '
'-h {CFG_DATABASE_HOST} -P {CFG_DATABASE_PORT} ')
cmd_prefix = prefix.format(cmd='mysql', user=user, password=password,
**args)
cmd_admin_prefix = prefix.format(cmd='mysqladmin', user=user,
password=password,
**args)
cmds = [
cmd_prefix + '-e "DROP DATABASE IF EXISTS {CFG_DATABASE_NAME}"',
(cmd_prefix + '-e "CREATE DATABASE IF NOT EXISTS '
'{CFG_DATABASE_NAME} DEFAULT CHARACTER SET utf8 '
'COLLATE utf8_general_ci"'),
# Create user and grant access to database.
(cmd_prefix + '-e "GRANT ALL PRIVILEGES ON '
'{CFG_DATABASE_NAME}.* TO {CFG_DATABASE_USER}@localhost '
'IDENTIFIED BY {CFG_DATABASE_PASS}"'),
(cmd_prefix + '-e "GRANT ALL PRIVILEGES ON '
'{CFG_DATABASE_NAME}.* TO {CFG_DATABASE_USER}@\'%\' '
'IDENTIFIED BY {CFG_DATABASE_PASS}"'),
cmd_admin_prefix + 'flush-privileges'
]
for cmd in cmds:
cmd = cmd.format(**args)
print(cmd)
if os.system(cmd):
print("ERROR: failed execution of", cmd, file=sys.stderr)
sys.exit(1)
print('>>> Database has been installed.')
def test_sqla_connection():
"""Test connection with the database."""
print("Test connection... \t", end="")
try:
from sqlalchemy import inspect
from invenio.ext.sqlalchemy import db
inspector = inspect(db.engine)
inspector.get_table_names()
except OperationalError as err:
from invenio.utils.text import wrap_text_in_a_box
from invenio.config import CFG_DATABASE_HOST, \
CFG_DATABASE_PORT, CFG_DATABASE_NAME, CFG_DATABASE_USER, \
CFG_DATABASE_PASS
print(" [ERROR]")
print(wrap_text_in_a_box("""\
DATABASE CONNECTIVITY ERROR:
%(errmsg)s.\n
Perhaps you need to set up database and connection rights?
If yes, then please execute:
console> inveniomanage database init --yes-i-know
The values printed above were detected from your
configuration. If they are not right, then please edit your
instance configuration file (invenio.cfg).
If the problem is of different nature, then please inspect
the above error message and fix the problem before continuing.""" % {
'errmsg': err.args[0],
'dbname': CFG_DATABASE_NAME,
'dbhost': CFG_DATABASE_HOST,
'dbport': CFG_DATABASE_PORT,
'dbuser': CFG_DATABASE_USER,
'dbpass': CFG_DATABASE_PASS,
'webhost': (CFG_DATABASE_HOST == 'localhost' and 'localhost' or
os.popen('hostname -f', 'r').read().strip()),
}))
sys.exit(1)
print(" [OK]")
def populate(packages=[], default_data=True, files=None,
job_id=0, extra_info=None, yes_i_know=False):
"""Load demo records. Useful for testing purposes."""
from invenio.utils.text import wrap_text_in_a_box, wait_for_user
# Load cli interfaces for tools that we are going to need
from invenio_records.manage import main as rmanager
# Step 0: confirm deletion
wait_for_user(wrap_text_in_a_box(
"WARNING: You are going to override data in tables!"
))
if not default_data:
print('>>> Default data has been skiped (--no-data).')
return
if not packages:
packages = ['invenio_demosite.base']
from werkzeug.utils import import_string
map(import_string, packages)
print(">>> Going to load demo records...")
if files is None:
files = [pkg_resources.resource_filename(
'invenio',
os.path.join('testsuite', 'data', 'demo_record_marc_data.xml'))]
# upload demo site files:
bibupload_flags = '-t marcxml'
for f in files:
job_id += 1
for cmd in (
(rmanager, "records create %s %s" % (bibupload_flags, f)),
):
if run_py_func(*cmd, passthrough=True).exit_code:
print("ERROR: failed execution of", *cmd)
sys.exit(1)
print(">>> Demo records loaded successfully.")
def _make_upload(name, description="Going to load demo records"):
"""Upload the demodata from the given file.
:param str name: the demodata file name
:param str description: the user message
"""
from invenio.utils.text import wrap_text_in_a_box, wait_for_user
from invenio.config import CFG_PREFIX
from invenio.modules.scheduler.models import SchTASK
wait_for_user(wrap_text_in_a_box(
"WARNING: You are going to override data in tables!"
))
print(">>> {0}".format(description))
xml_data = pkg_resources.resource_filename(
'cds',
os.path.join('demosite', 'data', name))
job_id = SchTASK.query.count()
for cmd in ["bin/bibupload -u admin -i %s" % (xml_data, ),
"bin/bibupload %d" % (job_id + 1, ),
"bin/bibindex -u admin",
"bin/bibindex %d" % (job_id + 2,),
"bin/bibindex -u admin -w global",
"bin/bibindex %d" % (job_id + 3,),
"bin/bibreformat -u admin -o HB",
"bin/bibreformat %d" % (job_id + 4,),
"bin/webcoll -u admin",
"bin/webcoll %d" % (job_id + 5,),
]:
cmd = os.path.join(CFG_PREFIX, cmd)
if os.system(cmd):
print("ERROR: failed execution of", cmd)
sys.exit(1)
print(">>> CDS Demo records loaded successfully.")
def _make_upload(name, description="Going to load demo records"):
"""Upload the demodata from the given file.
:param str name: the demodata file name
:param str description: the user message
"""
from invenio.utils.text import wrap_text_in_a_box, wait_for_user
from invenio.config import CFG_PREFIX
from invenio.modules.scheduler.models import SchTASK
wait_for_user(
wrap_text_in_a_box(
"WARNING: You are going to override data in tables!"))
print(">>> {0}".format(description))
xml_data = pkg_resources.resource_filename(
'cds', os.path.join('demosite', 'data', name))
job_id = SchTASK.query.count()
for cmd in [
"bin/bibupload -u admin -i %s" % (xml_data, ),
"bin/bibupload %d" % (job_id + 1, ),
"bin/bibindex -u admin",
"bin/bibindex %d" % (job_id + 2, ),
"bin/bibindex -u admin -w global",
"bin/bibindex %d" % (job_id + 3, ),
"bin/bibreformat -u admin -o HB",
"bin/bibreformat %d" % (job_id + 4, ),
"bin/webcoll -u admin",
"bin/webcoll %d" % (job_id + 5, ),
]:
cmd = os.path.join(CFG_PREFIX, cmd)
if os.system(cmd):
print("ERROR: failed execution of", cmd)
sys.exit(1)
print(">>> CDS Demo records loaded successfully.")
def create_config(force=False, no_ssl=False):
"""
Create Apache configuration files for this site, keeping previous
files in a backup copy.
"""
import os
import pwd
import pkg_resources
import sys
import shutil
from flask import current_app
from jinja2 import TemplateNotFound
from invenio.ext.template import render_template_to_string
from invenio.utils.text import wrap_text_in_a_box
CFG_PREFIX = current_app.config.get('CFG_PREFIX', '')
def get_context():
conf = current_app.config
## Apache vhost conf file is distro specific, so analyze needs:
# Gentoo (and generic defaults):
listen_directive_needed = True
ssl_pem_directive_needed = False
ssl_pem_path = CFG_PREFIX + '/etc/apache/ssl/apache.pem'
ssl_crt_path = CFG_PREFIX + '/etc/apache/ssl/server.crt'
ssl_key_path = CFG_PREFIX + '/etc/apache/ssl/server.key'
vhost_ip_address_needed = False
wsgi_socket_directive_needed = False
# Debian:
if os.path.exists(os.path.sep + 'etc' + os.path.sep + 'debian_version'):
listen_directive_needed = False
ssl_pem_directive_needed = True
ssl_pem_path = '/etc/apache2/ssl/apache.pem'
ssl_crt_path = '/etc/apache2/ssl/server.crt'
ssl_key_path = '/etc/apache2/ssl/server.key'
# RHEL/SLC:
if os.path.exists(os.path.sep + 'etc' + os.path.sep + 'redhat-release'):
listen_directive_needed = False
ssl_crt_path = '/etc/pki/tls/certs/localhost.crt'
ssl_key_path = '/etc/pki/tls/private/localhost.key'
vhost_ip_address_needed = True
wsgi_socket_directive_needed = True
# maybe we are using non-standard ports?
vhost_site_url = conf.get('CFG_SITE_URL').replace("http://", "")
if vhost_site_url.startswith("https://"):
## The installation is configured to require HTTPS for any connection
vhost_site_url = vhost_site_url.replace("https://", "")
vhost_site_url_port = '80'
vhost_site_secure_url = conf.get('CFG_SITE_SECURE_URL').replace("https://", "").replace("http://", "")
vhost_site_secure_url_port = '443'
if ':' in vhost_site_url:
vhost_site_url, vhost_site_url_port = vhost_site_url.split(':', 1)
if ':' in vhost_site_secure_url:
vhost_site_secure_url, vhost_site_secure_url_port = vhost_site_secure_url.split(':', 1)
if vhost_site_url_port != '80' or vhost_site_secure_url_port != '443':
listen_directive_needed = True
static_root = current_app.config['COLLECT_STATIC_ROOT']
if not os.path.exists(static_root):
os.mkdir(static_root)
def prepare_alias(filename):
if os.path.isdir(os.path.join(static_root, filename)):
return '/%s/' % (filename, )
return '/%s' % (filename, )
aliases = map(prepare_alias, os.listdir(static_root))
apc1 = {'vhost_site_url_port': vhost_site_url_port,
'servername': vhost_site_url,
'serveralias': vhost_site_url.split('.')[0],
'vhost_ip_address': vhost_ip_address_needed and
_detect_ip_address() or '*',
'wsgi_socket_directive_needed': wsgi_socket_directive_needed,
'listen_directive_needed': listen_directive_needed,
'aliases': aliases,
}
apc2 = {'vhost_site_url_port': vhost_site_secure_url_port,
'servername': vhost_site_secure_url,
'serveralias': vhost_site_secure_url.split('.')[0],
'vhost_ip_address': vhost_ip_address_needed and
_detect_ip_address() or '*',
'wsgi_socket_directive_needed': wsgi_socket_directive_needed,
'ssl_pem_directive': ssl_pem_directive_needed and
'SSLCertificateFile %s' % ssl_pem_path or
'#SSLCertificateFile %s' % ssl_pem_path,
'ssl_crt_directive': ssl_pem_directive_needed and
'#SSLCertificateFile %s' % ssl_crt_path or
'SSLCertificateFile %s' % ssl_crt_path,
'ssl_key_directive': ssl_pem_directive_needed and
'#SSLCertificateKeyFile %s' % ssl_key_path or
'SSLCertificateKeyFile %s' % ssl_key_path,
'listen_directive_needed': listen_directive_needed,
'aliases': aliases,
}
return [apc1, apc2]
current_app.config.update(
SYS_PREFIX=sys.prefix,
CFG_RUNNING_AS_USER=pwd.getpwuid(os.getuid())[0],
CFG_WSGIDIR=os.path.abspath(
pkg_resources.resource_filename('invenio', '')))
apache_conf_dir = current_app.instance_path + os.sep + 'apache'
print(">>> Going to create Apache conf files...")
conf_files = ['invenio-apache-vhost.conf', 'invenio-apache-vhost-ssl.conf']
conf_files = conf_files[:1 if no_ssl else 2]
if not os.path.exists(apache_conf_dir):
os.mkdir(apache_conf_dir)
for local_file, context in zip(conf_files,
get_context()[:1 if no_ssl else 2]):
print(">>> Writing %s ..." % local_file)
try:
apache_vhost_file = apache_conf_dir + os.sep + local_file
if os.path.exists(apache_vhost_file):
shutil.copy(apache_vhost_file,
apache_vhost_file + '.OLD')
with open(apache_vhost_file, 'w') as f:
out = render_template_to_string(local_file + '.tpl', os=os,
**context)
print(out, file=f)
except TemplateNotFound:
print("Could not find template %s".format(local_file),
file=sys.stderr)
print(wrap_text_in_a_box("""\
Apache virtual host configuration file(s) for your Invenio site
was(were) created. Please check created file(s) and activate virtual
host(s). For example, you can put the following include statements in
your httpd.conf:\n
%s
Please see the INSTALL file for more details.
""" % '\n\n'.join(tuple(map(
lambda x: "Include " + apache_conf_dir.encode('utf-8') + os.sep + x,
list(conf_files[:1 if no_ssl else 2]))))))
print(">>> Apache conf files created.")
def main():
"""Core loop."""
logfilename = '%s/fulltext_files_migration_kit-%s.log' % (
CFG_LOGDIR, datetime.today().strftime('%Y%m%d%H%M%S'))
try:
logfile = open(logfilename, 'w')
except IOError as e:
print(
wrap_text_in_a_box(
'NOTE: it\'s impossible to create the log:\n\n %s\n\nbecause of:\n\n %s\n\nPlease run this migration kit as the same user who runs Invenio (e.g. Apache)'
% (logfilename, e),
style='conclusion',
break_long=False))
sys.exit(1)
recids = retrieve_fulltext_recids()
print(
wrap_text_in_a_box(
"""This script migrate the filesystem structure used to store fulltext files to the new stricter structure.
This script must not be run during normal Invenio operations.
It is safe to run this script. No file will be deleted.
Anyway it is recommended to run a backup of the filesystem structure just in case.
A backup of the database tables involved will be automatically performed.""",
style='important'))
print("%s records will be migrated/fixed." % len(recids))
print("Please type yes if you want to go further:", end=' ')
if not check_yes():
print("INTERRUPTED")
sys.exit(1)
print("Backing up database tables")
try:
if not backup_tables():
print(
wrap_text_in_a_box(
"""It appears that is not the first time that you run this script.
Backup tables have been already created by a previous run.
In order for the script to go further they need to be removed.""",
style='important'))
print(
"Please, type yes if you agree to remove them and go further:",
end=' ')
if not check_yes():
print(wrap_text_in_a_box("INTERRUPTED", style='conclusion'))
sys.exit(1)
print(
"Backing up database tables (after dropping previous backup)",
end=' ')
backup_tables(drop=True)
print("-> OK")
else:
print("-> OK")
except Exception as e:
print(
wrap_text_in_a_box(
"Unexpected error while backing up tables. Please, do your checks: %s"
% e,
style='conclusion'))
sys.exit(1)
print("Created a complete log file into %s" % logfilename)
for recid in recids:
if not fix_recid(recid, logfile):
logfile.close()
print(
wrap_text_in_a_box(
title="INTERRUPTED BECAUSE OF ERROR!",
body=
"""Please see the log file %s for what was the status of record %s prior to the error. Contact %s in case of problems, attaching the log."""
% (logfilename, recid, CFG_SITE_SUPPORT_EMAIL),
style='conclusion'))
sys.exit(1)
print(wrap_text_in_a_box("DONE", style='conclusion'))
def main():
"""Core loop."""
check_running_process_user()
logfilename = '%s/fulltext_files_migration_kit-%s.log' % (CFG_LOGDIR, datetime.today().strftime('%Y%m%d%H%M%S'))
try:
logfile = open(logfilename, 'w')
except IOError as e:
print(wrap_text_in_a_box('NOTE: it\'s impossible to create the log:\n\n %s\n\nbecause of:\n\n %s\n\nPlease run this migration kit as the same user who runs Invenio (e.g. Apache)' % (logfilename, e), style='conclusion', break_long=False))
sys.exit(1)
bibdoc_bibdoc = retrieve_bibdoc_bibdoc()
print(wrap_text_in_a_box ("""This script migrate the filesystem structure used to store icons files to the new stricter structure.
This script must not be run during normal Invenio operations.
It is safe to run this script. No file will be deleted.
Anyway it is recommended to run a backup of the filesystem structure just in case.
A backup of the database tables involved will be automatically performed.""", style='important'))
if not bibdoc_bibdoc:
print(wrap_text_in_a_box("No need for migration", style='conclusion'))
return
print("%s icons will be migrated/fixed." % len(bibdoc_bibdoc))
wait_for_user()
print("Backing up database tables")
try:
if not backup_tables():
print(wrap_text_in_a_box("""It appears that is not the first time that you run this script.
Backup tables have been already created by a previous run.
In order for the script to go further they need to be removed.""", style='important'))
wait_for_user()
print("Backing up database tables (after dropping previous backup)", end=' ')
backup_tables(drop=True)
print("-> OK")
else:
print("-> OK")
except Exception as e:
print(wrap_text_in_a_box("Unexpected error while backing up tables. Please, do your checks: %s" % e, style='conclusion'))
sys.exit(1)
to_fix_marc = intbitset()
print("Created a complete log file into %s" % logfilename)
try:
try:
for id_bibdoc1, id_bibdoc2 in bibdoc_bibdoc:
try:
record_does_exist = True
recids = get_recid_from_docid(id_bibdoc1)
if not recids:
print("Skipping %s" % id_bibdoc1)
continue
for recid in recids:
if record_exists(recid[0]) > 0:
to_fix_marc.add(recid[0])
else:
record_does_exist = False
if not fix_bibdoc_bibdoc(id_bibdoc1, id_bibdoc2, logfile):
if record_does_exist:
raise StandardError("Error when correcting document ID %s" % id_bibdoc1)
except Exception as err:
print("ERROR: %s" % err, file=logfile)
print(wrap_text_in_a_box("DONE", style='conclusion'))
except:
logfile.close()
register_exception()
print(wrap_text_in_a_box(
title = "INTERRUPTED BECAUSE OF ERROR!",
body = """Please see the log file %s for what was the status prior to the error. Contact %s in case of problems, attaching the log.""" % (logfilename, CFG_SITE_SUPPORT_EMAIL),
style = 'conclusion'))
sys.exit(1)
finally:
print("Scheduling FIX-MARC to synchronize MARCXML for updated records.")
cli_fix_marc(options={}, explicit_recid_set=to_fix_marc)
def populate(packages=[],
default_data=True,
files=None,
job_id=0,
extra_info=None,
yes_i_know=False):
"""Load demo records. Useful for testing purposes."""
from invenio.utils.text import wrap_text_in_a_box, wait_for_user
# Load cli interfaces for tools that we are going to need
from invenio.legacy.bibupload.engine import main as bibupload
from invenio.legacy.bibindex.engine import main as bibindex
from invenio.legacy.bibformat.bibreformat import main as bibreformat
from invenio.legacy.oairepository.updater import main as oairepositoryupdater
from invenio.legacy.bibsort.daemon import main as bibsort
from invenio.legacy.bibdocfile.cli import main as bibdocfile
from invenio.legacy.bibrank.cli import main as bibrank
## Step 0: confirm deletion
wait_for_user(
wrap_text_in_a_box(
"WARNING: You are going to override data in tables!"))
if not default_data:
print('>>> Default data has been skiped (--no-data).')
return
if not packages:
packages = ['invenio_demosite.base']
from werkzeug.utils import import_string
map(import_string, packages)
from invenio.ext.sqlalchemy import db
print(">>> Going to load demo records...")
db.session.execute("TRUNCATE schTASK")
db.session.commit()
if files is None:
files = [
pkg_resources.resource_filename(
'invenio',
os.path.join('testsuite', 'data', 'demo_record_marc_data.xml'))
]
# upload demo site files:
bibupload_flags = '-i'
if extra_info is not None and 'force-recids' in extra_info:
bibupload_flags = '-i -r --force'
for f in files:
job_id += 1
for cmd in ((bibupload,
"bibupload -u admin %s %s" % (bibupload_flags, f)),
(bibupload, "bibupload %d" % (job_id))):
if run_py_func(*cmd, passthrough=True).exit_code:
print("ERROR: failed execution of", *cmd)
sys.exit(1)
i = count(job_id + 1).next
for cmd in (
(bibdocfile, "bibdocfile --textify --with-ocr --recid 97"),
(bibdocfile, "bibdocfile --textify --all"),
(bibindex, "bibindex -u admin"),
(bibindex, "bibindex %d" % i()),
(bibindex, "bibindex -u admin -w global"),
(bibindex, "bibindex %d" % i()),
(bibreformat, "bibreformat -u admin -o HB"),
(bibreformat, "bibreformat %d" % i()),
(bibrank, "bibrank -u admin"),
(bibrank, "bibrank %d" % i()),
(bibsort, "bibsort -u admin -R"),
(bibsort, "bibsort %d" % i()),
(oairepositoryupdater, "oairepositoryupdater -u admin"),
(oairepositoryupdater, "oairepositoryupdater %d" % i()),
(bibupload, "bibupload %d" % i()),
):
if run_py_func(*cmd, passthrough=True).exit_code:
print("ERROR: failed execution of", *cmd)
sys.exit(1)
print(">>> Demo records loaded successfully.")
def main():
"""Core loop."""
check_running_process_user()
logfilename = '%s/fulltext_files_migration_kit-%s.log' % (
CFG_LOGDIR, datetime.today().strftime('%Y%m%d%H%M%S'))
try:
logfile = open(logfilename, 'w')
except IOError as e:
print(
wrap_text_in_a_box(
'NOTE: it\'s impossible to create the log:\n\n %s\n\nbecause of:\n\n %s\n\nPlease run this migration kit as the same user who runs Invenio (e.g. Apache)'
% (logfilename, e),
style='conclusion',
break_long=False))
sys.exit(1)
bibdoc_bibdoc = retrieve_bibdoc_bibdoc()
print(
wrap_text_in_a_box(
"""This script migrate the filesystem structure used to store icons files to the new stricter structure.
This script must not be run during normal Invenio operations.
It is safe to run this script. No file will be deleted.
Anyway it is recommended to run a backup of the filesystem structure just in case.
A backup of the database tables involved will be automatically performed.""",
style='important'))
if not bibdoc_bibdoc:
print(wrap_text_in_a_box("No need for migration", style='conclusion'))
return
print("%s icons will be migrated/fixed." % len(bibdoc_bibdoc))
wait_for_user()
print("Backing up database tables")
try:
if not backup_tables():
print(
wrap_text_in_a_box(
"""It appears that is not the first time that you run this script.
Backup tables have been already created by a previous run.
In order for the script to go further they need to be removed.""",
style='important'))
wait_for_user()
print(
"Backing up database tables (after dropping previous backup)",
end=' ')
backup_tables(drop=True)
print("-> OK")
else:
print("-> OK")
except Exception as e:
print(
wrap_text_in_a_box(
"Unexpected error while backing up tables. Please, do your checks: %s"
% e,
style='conclusion'))
sys.exit(1)
to_fix_marc = intbitset()
print("Created a complete log file into %s" % logfilename)
try:
try:
for id_bibdoc1, id_bibdoc2 in bibdoc_bibdoc:
try:
record_does_exist = True
recids = get_recid_from_docid(id_bibdoc1)
if not recids:
print("Skipping %s" % id_bibdoc1)
continue
for recid in recids:
if record_exists(recid[0]) > 0:
to_fix_marc.add(recid[0])
else:
record_does_exist = False
if not fix_bibdoc_bibdoc(id_bibdoc1, id_bibdoc2, logfile):
if record_does_exist:
raise StandardError(
"Error when correcting document ID %s" %
id_bibdoc1)
except Exception as err:
print("ERROR: %s" % err, file=logfile)
print(wrap_text_in_a_box("DONE", style='conclusion'))
except:
logfile.close()
register_exception()
print(
wrap_text_in_a_box(
title="INTERRUPTED BECAUSE OF ERROR!",
body=
"""Please see the log file %s for what was the status prior to the error. Contact %s in case of problems, attaching the log."""
% (logfilename, CFG_SITE_SUPPORT_EMAIL),
style='conclusion'))
sys.exit(1)
finally:
print(
"Scheduling FIX-MARC to synchronize MARCXML for updated records.")
cli_fix_marc(options={}, explicit_recid_set=to_fix_marc)
def populate(packages=[], default_data=True, files=None, job_id=0, extra_info=None, yes_i_know=False):
"""Load demo records. Useful for testing purposes."""
from invenio.utils.text import wrap_text_in_a_box, wait_for_user
## Step 0: confirm deletion
wait_for_user(wrap_text_in_a_box("WARNING: You are going to override data in tables!"))
if not default_data:
print(">>> Default data has been skiped (--no-data).")
return
if not packages:
packages = ["invenio_demosite.base"]
from werkzeug.utils import import_string
from invenio.config import CFG_PREFIX
map(import_string, packages)
from invenio.ext.sqlalchemy import db
print(">>> Going to load demo records...")
db.session.execute("TRUNCATE schTASK")
db.session.commit()
if files is None:
files = [
pkg_resources.resource_filename("invenio", os.path.join("testsuite", "data", "demo_record_marc_data.xml"))
]
# upload demo site files:
bibupload_flags = "-i"
if extra_info is not None and "force-recids" in extra_info:
bibupload_flags = "-i -r --force"
for f in files:
job_id += 1
for cmd in [
"%s/bin/bibupload -u admin %s %s" % (CFG_PREFIX, bibupload_flags, f),
"%s/bin/bibupload %d" % (CFG_PREFIX, job_id),
]:
if os.system(cmd):
print("ERROR: failed execution of", cmd)
sys.exit(1)
i = count(1).next
for cmd in [
"bin/bibdocfile --textify --with-ocr --recid 97",
"bin/bibdocfile --textify --all",
"bin/bibindex -u admin",
"bin/bibindex %d" % (job_id + i(),),
"bin/bibindex -u admin -w global",
"bin/bibindex %d" % (job_id + i(),),
"bin/bibreformat -u admin -o HB",
"bin/bibreformat %d" % (job_id + i(),),
"bin/bibrank -u admin",
"bin/bibrank %d" % (job_id + i(),),
"bin/bibsort -u admin -R",
"bin/bibsort %d" % (job_id + i(),),
"bin/oairepositoryupdater -u admin",
"bin/oairepositoryupdater %d" % (job_id + i(),),
"bin/bibupload %d" % (job_id + i(),),
]:
cmd = os.path.join(CFG_PREFIX, cmd)
if os.system(cmd):
print("ERROR: failed execution of", cmd)
sys.exit(1)
print(">>> Demo records loaded successfully.")
def create_config(force=False, no_ssl=False):
"""
Create Apache configuration files for this site, keeping previous
files in a backup copy.
"""
import os
import pwd
import pkg_resources
import sys
import shutil
from flask import current_app
from jinja2 import TemplateNotFound
from invenio.ext.template import render_template_to_string
from invenio.utils.text import wrap_text_in_a_box
CFG_PREFIX = current_app.config.get('CFG_PREFIX', '')
def get_context():
conf = current_app.config
## Apache vhost conf file is distro specific, so analyze needs:
# Gentoo (and generic defaults):
listen_directive_needed = True
ssl_pem_directive_needed = False
ssl_pem_path = CFG_PREFIX + '/etc/apache/ssl/apache.pem'
ssl_crt_path = CFG_PREFIX + '/etc/apache/ssl/server.crt'
ssl_key_path = CFG_PREFIX + '/etc/apache/ssl/server.key'
vhost_ip_address_needed = False
wsgi_socket_directive_needed = False
# Debian:
if os.path.exists(os.path.sep + 'etc' + os.path.sep +
'debian_version'):
listen_directive_needed = False
ssl_pem_directive_needed = True
ssl_pem_path = '/etc/apache2/ssl/apache.pem'
ssl_crt_path = '/etc/apache2/ssl/server.crt'
ssl_key_path = '/etc/apache2/ssl/server.key'
# RHEL/SLC:
if os.path.exists(os.path.sep + 'etc' + os.path.sep +
'redhat-release'):
listen_directive_needed = False
ssl_crt_path = '/etc/pki/tls/certs/localhost.crt'
ssl_key_path = '/etc/pki/tls/private/localhost.key'
vhost_ip_address_needed = True
wsgi_socket_directive_needed = True
# maybe we are using non-standard ports?
vhost_site_url = conf.get('CFG_SITE_URL').replace("http://", "")
if vhost_site_url.startswith("https://"):
## The installation is configured to require HTTPS for any connection
vhost_site_url = vhost_site_url.replace("https://", "")
vhost_site_url_port = '80'
vhost_site_secure_url = conf.get('CFG_SITE_SECURE_URL').replace(
"https://", "").replace("http://", "")
vhost_site_secure_url_port = '443'
if ':' in vhost_site_url:
vhost_site_url, vhost_site_url_port = vhost_site_url.split(':', 1)
if ':' in vhost_site_secure_url:
vhost_site_secure_url, vhost_site_secure_url_port = vhost_site_secure_url.split(
':', 1)
if vhost_site_url_port != '80' or vhost_site_secure_url_port != '443':
listen_directive_needed = True
static_root = current_app.config['COLLECT_STATIC_ROOT']
if not os.path.exists(static_root):
os.mkdir(static_root)
def prepare_alias(filename):
if os.path.isdir(os.path.join(static_root, filename)):
return '/%s/' % (filename, )
return '/%s' % (filename, )
aliases = map(prepare_alias, os.listdir(static_root))
apc1 = {
'vhost_site_url_port':
vhost_site_url_port,
'servername':
vhost_site_url,
'serveralias':
vhost_site_url.split('.')[0],
'vhost_ip_address':
vhost_ip_address_needed and _detect_ip_address() or '*',
'wsgi_socket_directive_needed':
wsgi_socket_directive_needed,
'listen_directive_needed':
listen_directive_needed,
'aliases':
aliases,
}
apc2 = {
'vhost_site_url_port':
vhost_site_secure_url_port,
'servername':
vhost_site_secure_url,
'serveralias':
vhost_site_secure_url.split('.')[0],
'vhost_ip_address':
vhost_ip_address_needed and _detect_ip_address() or '*',
'wsgi_socket_directive_needed':
wsgi_socket_directive_needed,
'ssl_pem_directive':
ssl_pem_directive_needed and 'SSLCertificateFile %s' % ssl_pem_path
or '#SSLCertificateFile %s' % ssl_pem_path,
'ssl_crt_directive':
ssl_pem_directive_needed
and '#SSLCertificateFile %s' % ssl_crt_path
or 'SSLCertificateFile %s' % ssl_crt_path,
'ssl_key_directive':
ssl_pem_directive_needed
and '#SSLCertificateKeyFile %s' % ssl_key_path
or 'SSLCertificateKeyFile %s' % ssl_key_path,
'listen_directive_needed':
listen_directive_needed,
'aliases':
aliases,
}
return [apc1, apc2]
current_app.config.update(SYS_PREFIX=sys.prefix,
CFG_RUNNING_AS_USER=pwd.getpwuid(os.getuid())[0],
CFG_WSGIDIR=os.path.abspath(
pkg_resources.resource_filename(
'invenio', '')))
apache_conf_dir = current_app.instance_path + os.sep + 'apache'
print(">>> Going to create Apache conf files...")
conf_files = ['invenio-apache-vhost.conf', 'invenio-apache-vhost-ssl.conf']
conf_files = conf_files[:1 if no_ssl else 2]
if not os.path.exists(apache_conf_dir):
os.mkdir(apache_conf_dir)
for local_file, context in zip(conf_files,
get_context()[:1 if no_ssl else 2]):
print(">>> Writing %s ..." % local_file)
try:
apache_vhost_file = apache_conf_dir + os.sep + local_file
if os.path.exists(apache_vhost_file):
shutil.copy(apache_vhost_file, apache_vhost_file + '.OLD')
with open(apache_vhost_file, 'w') as f:
out = render_template_to_string(local_file + '.tpl',
os=os,
**context)
print(out, file=f)
except TemplateNotFound:
print("Could not find template %s".format(local_file),
file=sys.stderr)
print(
wrap_text_in_a_box("""\
Apache virtual host configuration file(s) for your Invenio site
was(were) created. Please check created file(s) and activate virtual
host(s). For example, you can put the following include statements in
your httpd.conf:\n
%s
Please see the INSTALL file for more details.
""" % '\n\n'.join(
tuple(
map(
lambda x: "Include " + apache_conf_dir.encode('utf-8') + os
.sep + x, list(conf_files[:1 if no_ssl else 2]))))))
print(">>> Apache conf files created.")
def drop(yes_i_know=False):
"""Drops database tables"""
print(">>> Going to drop tables and related data on filesystem ...")
from sqlalchemy import event
from invenio.utils.date import get_time_estimator
from invenio.utils.text import wrap_text_in_a_box, wait_for_user
from invenio.legacy.webstat.api import destroy_customevents
from invenio.legacy.inveniocfg import test_db_connection
from invenio.ext.sqlalchemy import db, models
from invenio.legacy.bibdocfile.api import _make_base_dir
## Step 0: confirm deletion
wait_for_user(wrap_text_in_a_box(
"WARNING: You are going to destroy your database tables and related "
"data on filesystem!"))
## Step 1: test database connection
test_db_connection()
list(models)
## Step 2: disable foreign key checks
if db.engine.name == 'mysql':
db.engine.execute('SET FOREIGN_KEY_CHECKS=0;')
## Step 3: destroy associated data
try:
msg = destroy_customevents()
if msg:
print(msg)
except:
print("ERROR: Could not destroy customevents.")
## FIXME: move to bibedit_model
def bibdoc_before_drop(target, connection_dummy, **kw_dummy):
print
print(">>> Going to remove records data...")
for (docid,) in db.session.query(target.c.id).all():
directory = _make_base_dir(docid)
if os.path.isdir(directory):
print(' >>> Removing files for docid =', docid)
shutil.rmtree(directory)
db.session.commit()
print(">>> Data has been removed.")
from invenio.modules.editor.models import Bibdoc
event.listen(Bibdoc.__table__, "before_drop", bibdoc_before_drop)
tables = list(reversed(db.metadata.sorted_tables))
N = len(tables)
prefix = '>>> Dropping %d tables ...' % N
e = get_time_estimator(N)
dropped = 0
for i, table in enumerate(tables):
try:
print_progress(1.0 * i / N, prefix=prefix,
suffix=str(datetime.timedelta(seconds=e()[0])))
table.drop(bind=db.engine)
dropped += 1
except:
print('\r', '>>> problem with dropping table', table)
print
if dropped == N:
print(">>> Tables dropped successfully.")
else:
print("ERROR: not all tables were properly dropped.")
print(">>> Dropped", dropped, 'out of', N)
def populate(packages=[], default_data=True, files=None,
job_id=0, extra_info=None, yes_i_know=False):
"""Load demo records. Useful for testing purposes."""
from invenio.utils.text import wrap_text_in_a_box, wait_for_user
# Load cli interfaces for tools that we are going to need
from invenio.legacy.bibupload.engine import main as bibupload
from invenio.legacy.bibindex.engine import main as bibindex
from invenio.legacy.oairepository.updater import main as oairepositoryupdater
from invenio.legacy.bibsort.daemon import main as bibsort
from invenio.legacy.bibdocfile.cli import main as bibdocfile
from invenio.legacy.bibrank.cli import main as bibrank
## Step 0: confirm deletion
wait_for_user(wrap_text_in_a_box(
"WARNING: You are going to override data in tables!"
))
if not default_data:
print('>>> Default data has been skiped (--no-data).')
return
if not packages:
packages = ['invenio_demosite.base']
from werkzeug.utils import import_string
map(import_string, packages)
from invenio.ext.sqlalchemy import db
print(">>> Going to load demo records...")
db.session.execute("TRUNCATE schTASK")
db.session.commit()
if files is None:
files = [pkg_resources.resource_filename(
'invenio',
os.path.join('testsuite', 'data', 'demo_record_marc_data.xml'))]
# upload demo site files:
bibupload_flags = '-i'
if extra_info is not None and 'force-recids' in extra_info:
bibupload_flags = '-i -r --force'
for f in files:
job_id += 1
for cmd in (
(bibupload, "bibupload -u admin %s %s" % (bibupload_flags, f)),
(bibupload, "bibupload %d" % (job_id))
):
if run_py_func(*cmd, passthrough=True).exit_code:
print("ERROR: failed execution of", *cmd)
sys.exit(1)
i = count(job_id + 1).next
for cmd in (
#(bibdocfile, "bibdocfile --textify --with-ocr --recid 97"),
#(bibdocfile, "bibdocfile --textify --all"),
(bibindex, "bibindex -u admin"),
(bibindex, "bibindex %d" % i()),
(bibindex, "bibindex -u admin -w global"),
(bibindex, "bibindex %d" % i()),
(bibrank, "bibrank -u admin"),
(bibrank, "bibrank %d" % i()),
(bibsort, "bibsort -u admin -R"),
(bibsort, "bibsort %d" % i()),
(oairepositoryupdater, "oairepositoryupdater -u admin"),
(oairepositoryupdater, "oairepositoryupdater %d" % i()),
(bibupload, "bibupload %d" % i()),
):
if run_py_func(*cmd, passthrough=True).exit_code:
print("ERROR: failed execution of", *cmd)
sys.exit(1)
print(">>> Demo records loaded successfully.")
def populate(packages=[], default_data=True, files=None,
job_id=0, extra_info=None, yes_i_know=False):
"""Load demo records. Useful for testing purposes."""
from invenio.utils.text import wrap_text_in_a_box, wait_for_user
## Step 0: confirm deletion
wait_for_user(wrap_text_in_a_box(
"WARNING: You are going to override data in tables!"
))
if not default_data:
print('>>> Default data has been skiped (--no-data).')
return
if not packages:
packages = ['invenio_demosite.base']
from werkzeug.utils import import_string
from invenio.config import CFG_PREFIX
map(import_string, packages)
from invenio.ext.sqlalchemy import db
print(">>> Going to load demo records...")
db.session.execute("TRUNCATE schTASK")
db.session.commit()
if files is None:
files = [pkg_resources.resource_filename(
'invenio',
os.path.join('testsuite', 'data', 'demo_record_marc_data.xml'))]
# upload demo site files:
bibupload_flags = '-i'
if extra_info is not None and 'force-recids' in extra_info:
bibupload_flags = '-i -r --force'
for f in files:
job_id += 1
for cmd in ["%s/bin/bibupload -u admin %s %s" % (CFG_PREFIX, bibupload_flags, f),
"%s/bin/bibupload %d" % (CFG_PREFIX, job_id)]:
if os.system(cmd):
print("ERROR: failed execution of", cmd)
sys.exit(1)
for cmd in ["bin/bibdocfile --textify --with-ocr --recid 97",
"bin/bibdocfile --textify --all",
"bin/bibindex -u admin",
"bin/bibindex %d" % (job_id + 1,),
"bin/bibindex -u admin -w global",
"bin/bibindex %d" % (job_id + 2,),
"bin/bibreformat -u admin -o HB",
"bin/bibreformat %d" % (job_id + 3,),
"bin/webcoll -u admin",
"bin/webcoll %d" % (job_id + 4,),
"bin/bibrank -u admin",
"bin/bibrank %d" % (job_id + 5,),
"bin/bibsort -u admin -R",
"bin/bibsort %d" % (job_id + 6,),
"bin/oairepositoryupdater -u admin",
"bin/oairepositoryupdater %d" % (job_id + 7,),
"bin/bibupload %d" % (job_id + 8,)]:
cmd = os.path.join(CFG_PREFIX, cmd)
if os.system(cmd):
print("ERROR: failed execution of", cmd)
sys.exit(1)
print(">>> Demo records loaded successfully.")
