def cmd_upgrade(dummy_conf, upgrader=None):
""" Command for applying upgrades """
from invenio.config import CFG_LOGDIR
from invenio.textutils import wrap_text_in_a_box, wait_for_user
logfilename = os.path.join(CFG_LOGDIR, 'invenio_upgrader.log')
if not upgrader:
upgrader = InvenioUpgrader()
logger = upgrader.get_logger(logfilename=logfilename)
try:
upgrades = upgrader.get_upgrades()
if not upgrades:
logger.info("All upgrades have been applied.")
return
logger.info("Following upgrade(s) will be applied:")
for u in upgrades:
title = u['__doc__']
if title:
logger.info(" * %s (%s)" % (u['id'], title))
else:
logger.info(" * %s" % u['id'])
logger.info("Running pre-upgrade checks...")
upgrader.pre_upgrade_checks(upgrades)
logger.info("Calculating estimated upgrade time...")
estimate = upgrader.human_estimate(upgrades)
wait_for_user(
wrap_text_in_a_box(
"""WARNING: You are going to upgrade your installation (estimated time: %s)!"""
% estimate))
for u in upgrades:
title = u['__doc__']
if title:
logger.info("Applying %s (%s)" % (u['id'], title))
else:
logger.info("Applying %s" % u['id'])
upgrader.apply_upgrade(u)
logger.info("Running post-upgrade checks...")
upgrader.post_upgrade_checks(upgrades)
if upgrader.has_warnings():
logger.warning("Upgrade completed with %s warnings - please check "
"log-file for further information:\nless %s" %
(upgrader.get_warnings_count(), logfilename))
else:
logger.info("Upgrade completed successfully.")
except RuntimeError, e:
for msg in e.args:
logger.error(unicode(msg))
logger.info("Please check log file for further information:\n"
"less %s" % logfilename)
sys.exit(1)
def pre_upgrade():
"""
Run pre-upgrade check conditions to ensure the upgrade can be applied
without problems.
"""
logger = logging.getLogger("invenio_upgrader")
global DB_VERSION # Needed because we assign to it
DB_VERSION = _invenio_schema_version_guesser()
if DB_VERSION == "unknown":
raise RuntimeError(
"Your Invenio database schema version could not be" " determined. Please upgrade to Invenio v1.0.0 first."
)
if DB_VERSION == "pre-0.99.0":
raise RuntimeError(
"Upgrading from Invenio versions prior to 0.99 is" " not supported. Please upgrade to 0.99.0 first."
)
if DB_VERSION == "0.99.0":
raise RuntimeError(
"It seems like you are running Invenio version "
"0.99.0. Please run the upgrade in the following special way:\n"
"make install\ninveniocfg --update-all\n"
"make update-v0.99.0-tables\nmake update-v0.99.6-tables\n"
"inveniocfg --upgrade\n\nNote: Most warnings printed during "
"inveniocfg --upgrade can safely be ignored when upgrading from"
" Invenio 0.99.0."
)
if DB_VERSION in ["0.99.x", "0.99.x-1.0.0"]:
raise RuntimeError(
"It seems like you are running Invenio version "
"v0.99.1-v0.99.x. Please run the upgrade in the following special"
" way:\nmake install\ninveniocfg --update-all\n"
"make update-v0.99.6-tables\n"
"inveniocfg --upgrade\n\nNote: Most warnings printed during "
"inveniocfg --upgrade can safely be ignored when upgrading from"
" Invenio v0.99.1-0.99.x."
)
if DB_VERSION == "master":
warnings.warn("Invenio database schema is on a development version" " between 1.0.x and 1.1.0")
# Run import here, since we are on 1.0-1.1 we know the import will work
from invenio.textutils import wait_for_user
try:
wait_for_user(
"\nUPGRADING TO 1.1.0 FROM A DEVELOPMENT VERSION"
" WILL LEAD TO MANY WARNINGS! Please thoroughly"
" test the upgrade on non-production systems first,"
" and pay close attention to warnings.\n"
)
except SystemExit:
raise RuntimeError("Upgrade aborted by user.")
else:
logger.info("Invenio version v%s detected." % DB_VERSION)
def cmd_upgrade(dummy_conf, upgrader=None):
""" Command for applying upgrades """
from invenio.config import CFG_LOGDIR
from invenio.textutils import wrap_text_in_a_box, wait_for_user
logfilename = os.path.join(CFG_LOGDIR, 'invenio_upgrader.log')
if not upgrader:
upgrader = InvenioUpgrader()
logger = upgrader.get_logger(logfilename=logfilename)
try:
upgrades = upgrader.get_upgrades()
if not upgrades:
logger.info("All upgrades have been applied.")
return
logger.info("Following upgrade(s) will be applied:")
for u in upgrades:
title = u['__doc__']
if title:
logger.info(" * %s (%s)" % (u['id'], title))
else:
logger.info(" * %s" % u['id'])
logger.info("Running pre-upgrade checks...")
upgrader.pre_upgrade_checks(upgrades)
logger.info("Calculating estimated upgrade time...")
estimate = upgrader.human_estimate(upgrades)
wait_for_user(wrap_text_in_a_box(
"""WARNING: You are going to upgrade your installation (estimated time: %s)!""" % estimate))
for u in upgrades:
title = u['__doc__']
if title:
logger.info("Applying %s (%s)" % (u['id'], title))
else:
logger.info("Applying %s" % u['id'])
upgrader.apply_upgrade(u)
logger.info("Running post-upgrade checks...")
upgrader.post_upgrade_checks(upgrades)
if upgrader.has_warnings():
logger.warning("Upgrade completed with %s warnings - please check "
"log-file for further information:\nless %s"
% (upgrader.get_warnings_count(), logfilename))
else:
logger.info("Upgrade completed successfully.")
except RuntimeError, e:
for msg in e.args:
logger.error(unicode(msg))
logger.info("Please check log file for further information:\n"
"less %s" % logfilename)
sys.exit(1)
def pre_upgrade():
"""
Run pre-upgrade check conditions to ensure the upgrade can be applied
without problems.
"""
logger = logging.getLogger('invenio_upgrader')
global DB_VERSION # Needed because we assign to it
DB_VERSION = _invenio_schema_version_guesser()
if DB_VERSION == 'unknown':
raise RuntimeError(
"Your Invenio database schema version could not be"
" determined. Please upgrade to Invenio v1.0.0 first.")
if DB_VERSION == 'pre-0.99.0':
raise RuntimeError("Upgrading from Invenio versions prior to 0.99 is"
" not supported. Please upgrade to 0.99.0 first.")
if DB_VERSION == '0.99.0':
raise RuntimeError(
"It seems like you are running Invenio version "
"0.99.0. Please run the upgrade in the following special way:\n"
"make install\ninveniocfg --update-all\n"
"make update-v0.99.0-tables\nmake update-v0.99.6-tables\n"
"inveniocfg --upgrade\n\nNote: Most warnings printed during "
"inveniocfg --upgrade can safely be ignored when upgrading from"
" Invenio 0.99.0.")
if DB_VERSION in ['0.99.x', '0.99.x-1.0.0']:
raise RuntimeError(
"It seems like you are running Invenio version "
"v0.99.1-v0.99.x. Please run the upgrade in the following special"
" way:\nmake install\ninveniocfg --update-all\n"
"make update-v0.99.6-tables\n"
"inveniocfg --upgrade\n\nNote: Most warnings printed during "
"inveniocfg --upgrade can safely be ignored when upgrading from"
" Invenio v0.99.1-0.99.x.")
if DB_VERSION == 'master':
warnings.warn("Invenio database schema is on a development version"
" between 1.0.x and 1.1.0")
# Run import here, since we are on 1.0-1.1 we know the import will work
from invenio.textutils import wait_for_user
try:
wait_for_user("\nUPGRADING TO 1.1.0 FROM A DEVELOPMENT VERSION"
" WILL LEAD TO MANY WARNINGS! Please thoroughly"
" test the upgrade on non-production systems first,"
" and pay close attention to warnings.\n")
except SystemExit:
raise RuntimeError("Upgrade aborted by user.")
else:
logger.info("Invenio version v%s detected." % DB_VERSION)
def upload_to_site(marcxml, yes_i_know, upload_mode="append"):
"""
makes the appropriate calls to bibupload to get the MARCXML record onto
the site. Uploads in "correct" mode.
@param: marcxml (string): the absolute location of the MARCXML that was
generated by this programme
@param: yes_i_know (boolean): if true, no confirmation. if false, prompt.
@output: a new record on the invenio site
@return: None
"""
if not yes_i_know:
wait_for_user(wrap_text_in_a_box("You are going to upload new " + "plots to the server."))
task_low_level_submission("bibupload", "admin", upload_mode and "--" + upload_mode or "", marcxml)
def upload_to_site(marcxml, yes_i_know):
"""
makes the appropriate calls to bibupload to get the MARCXML record onto
the site.
@param: marcxml (string): the absolute location of the MARCXML that was
generated by this programme
@param: yes_i_know (boolean): if true, no confirmation. if false, prompt.
@output: a new record on the invenio site
@return: None
"""
if not yes_i_know:
wait_for_user(wrap_text_in_a_box('You are going to upload new ' + \
'plots to the server.'))
task_low_level_submission('bibupload', 'admin', '-a', marcxml)
def pre_upgrade():
"""Check for potentially invalid revisions"""
res = run_sql("""SELECT DISTINCT(changeset_id) FROM bibHOLDINGPEN
WHERE LENGTH(changeset_xml) = %s""", [2**16-1])
if res:
warnings.warn("""You have %s holding pen entries with potentially corrupt data!
You can find the rows affected with the sql command:
SELECT DISTINCT(changeset_id) FROM bibHOLDINGPEN
WHERE LENGTH(changeset_xml) = 65535""" % len(res))
from invenio.textutils import wait_for_user
try:
wait_for_user("\nThis upgrade will delete all the corrupted entries. A backup table bibHOLDINGPEN_backup will be created.\n")
except SystemExit:
raise RuntimeError("Upgrade aborted by user.")
for r in res:
run_sql("""DELETE FROM bibHOLDINGPEN WHERE changeset_id=%s""" % r[0])
except IOError, e:
print wrap_text_in_a_box('NOTE: it\'s impossible to create the log:\n\n %s\n\nbecause of:\n\n %s\n\nPlease run this migration kit as the same user who runs Invenio (e.g. Apache)' % (logfilename, e), style='conclusion', break_long=False)
sys.exit(1)
bibdoc_bibdoc = retrieve_bibdoc_bibdoc()
print wrap_text_in_a_box ("""This script migrate the filesystem structure used to store icons files to the new stricter structure.
This script must not be run during normal Invenio operations.
It is safe to run this script. No file will be deleted.
Anyway it is recommended to run a backup of the filesystem structure just in case.
A backup of the database tables involved will be automatically performed.""", style='important')
if not bibdoc_bibdoc:
print wrap_text_in_a_box("No need for migration", style='conclusion')
return
print "%s icons will be migrated/fixed." % len(bibdoc_bibdoc)
wait_for_user()
print "Backing up database tables"
try:
if not backup_tables():
print wrap_text_in_a_box("""It appears that is not the first time that you run this script.
Backup tables have been already created by a previous run.
In order for the script to go further they need to be removed.""", style='important')
wait_for_user()
print "Backing up database tables (after dropping previous backup)",
backup_tables(drop=True)
print "-> OK"
else:
print "-> OK"
except Exception, e:
print wrap_text_in_a_box("Unexpected error while backing up tables. Please, do your checks: %s" % e, style='conclusion')
sys.exit(1)
bibdoc_bibdoc = retrieve_bibdoc_bibdoc()
print wrap_text_in_a_box(
"""This script migrate the filesystem structure used to store icons files to the new stricter structure.
This script must not be run during normal Invenio operations.
It is safe to run this script. No file will be deleted.
Anyway it is recommended to run a backup of the filesystem structure just in case.
A backup of the database tables involved will be automatically performed.""",
style='important')
if not bibdoc_bibdoc:
print wrap_text_in_a_box("No need for migration", style='conclusion')
return
print "%s icons will be migrated/fixed." % len(bibdoc_bibdoc)
wait_for_user()
print "Backing up database tables"
try:
if not backup_tables():
print wrap_text_in_a_box(
"""It appears that is not the first time that you run this script.
Backup tables have been already created by a previous run.
In order for the script to go further they need to be removed.""",
style='important')
wait_for_user()
print "Backing up database tables (after dropping previous backup)",
backup_tables(drop=True)
print "-> OK"
else:
print "-> OK"
TestBatchEngineFunctions,
TestDaemonFunctions)
if __name__ == "__main__":
wait_for_user("""
#######################################################
# This is the test suite for the BibEncode module #
# #
# You need to have installed ffmpeg with H.264, WebM #
# and Theora support! Please see the manual! #
# #
# Please be aware that not every aspect can be tested #
# due to the nature of video encoding and wrapping #
# external libraries like ffmpeg. The results should #
# only be seen as an indicator and do not necessarily #
# mean that there is something wrong. #
# #
# You should evaluate the output manually in the tmp #
# folder of your Invenio installation #
# #
# The test suite will download and create several #
# gigabytes of video material to perform the test! #
# The whole test might take up half an hour #
# #
# Do you wich to continue? Then enter "Yes, I know!". #
# Else press 'ctrl + c' to leave this tool. #
#######################################################
""")
run_test_suite(TEST_SUITE)
parser.error("The specified label %s does not exist" % label)
redirection_data = get_redirection_data(label)
plugin = options.plugin or redirection_data['plugin']
parameters = options.parameters or redirection_data['parameters']
if not plugin in CFG_GOTO_PLUGINS:
parser.error("%s is not a valid plugin" % plugin)
update_redirection(label, plugin, parameters=None)
print "The redirection %s was successfully updated for the plugin %s with parameters %s" % (
label, plugin, parameters)
elif options.get_redirection:
label = options.get_redirection
if not is_redirection_label_already_taken(label):
parser.error("The specified label %s does not exist" % label)
print get_redirection_data(label)
elif options.drop_redirection:
label = options.drop_redirection
if is_redirection_label_already_taken(label):
wait_for_user(
"Are you sure you want to drop the redirection: %s\n%s" %
(label, get_redirection_data(label)))
drop_redirection(label)
print "The redirection %s was successfully dropped" % label
else:
print "The specified label %s is not registered with any redirection" % label
else:
parser.print_help()
if __name__ == "__main__":
main()
print "The redirection %s was successfully registered for the plugin %s with parameters %s" % (label, plugin, parameters)
elif options.update:
label = options.update
if not is_redirection_label_already_taken(label):
parser.error("The specified label %s does not exist" % label)
redirection_data = get_redirection_data(label)
plugin = options.plugin or redirection_data['plugin']
parameters = options.parameters or redirection_data['parameters']
if not plugin in CFG_GOTO_PLUGINS:
parser.error("%s is not a valid plugin" % plugin)
update_redirection(label, plugin, parameters=None)
print "The redirection %s was successfully updated for the plugin %s with parameters %s" % (label, plugin, parameters)
elif options.get_redirection:
label = options.get_redirection
if not is_redirection_label_already_taken(label):
parser.error("The specified label %s does not exist" % label)
print get_redirection_data(label)
elif options.drop_redirection:
label = options.drop_redirection
if is_redirection_label_already_taken(label):
wait_for_user("Are you sure you want to drop the redirection: %s\n%s" % (label, get_redirection_data(label)))
drop_redirection(label)
print "The redirection %s was successfully dropped" % label
else:
print "The specified label %s is not registered with any redirection" % label
else:
parser.print_help()
if __name__ == "__main__":
main()
## TestMetadataFunctions,
TestBatchEngineFunctions,
TestDaemonFunctions)
if __name__ == "__main__":
wait_for_user("""
#######################################################
# This is the test suite for the BibEncode module #
# #
# You need to have installed ffmpeg with H.264, WebM #
# and Theora support! Please see the manual! #
# #
# Please be aware that not every aspect can be tested #
# due to the nature of video encoding and wrapping #
# external libraries like ffmpeg. The results should #
# only be seen as an indicator and do not necessarily #
# mean that there is something wrong. #
# #
# You should evaluate the output manually in the tmp #
# folder of your Invenio installation #
# #
# The test suite will download and create several #
# gigabytes of video material to perform the test! #
# The whole test might take up half an hour #
# #
# Do you wich to continue? Then enter "Yes, I know!". #
# Else press 'ctrl + c' to leave this tool. #
#######################################################
""")
run_test_suite(TEST_SUITE)
def drop(yes_i_know=False):
"""Drops database tables"""
print ">>> Going to drop tables and related data on filesystem ..."
from sqlalchemy import event
from invenio.dateutils import get_time_estimator
from invenio.textutils import wrap_text_in_a_box, wait_for_user
from invenio.webstat import destroy_customevents
from invenio.inveniocfg import test_db_connection
from invenio.sqlalchemyutils import db
from invenio.bibdocfile import _make_base_dir
## Step 0: confirm deletion
wait_for_user(
wrap_text_in_a_box(
"""WARNING: You are going to destroy your database tables and related data on filesystem!"""
))
## Step 1: test database connection
test_db_connection()
## Step 2: disable foreign key checks
if db.engine.name == 'mysql':
db.engine.execute('SET FOREIGN_KEY_CHECKS=0;')
## Step 3: destroy associated data
try:
msg = destroy_customevents()
if msg:
print msg
except:
print "ERROR: Could not destroy customevents."
## FIXME: move to bibedit_model
def bibdoc_before_drop(target, connection_dummy, **kw_dummy):
print
print ">>> Going to remove records data..."
for (docid, ) in db.session.query(target.c.id).all():
directory = _make_base_dir(docid)
if os.path.isdir(directory):
print ' >>> Removing files for docid =', docid
shutil.rmtree(directory)
db.session.commit()
print ">>> Data has been removed."
from invenio.bibedit_model import Bibdoc
event.listen(Bibdoc.__table__, "before_drop", bibdoc_before_drop)
tables = list(reversed(db.metadata.sorted_tables))
N = len(tables)
prefix = '>>> Dropping %d tables ...' % N
e = get_time_estimator(N)
dropped = 0
for i, table in enumerate(tables):
try:
print_progress(1.0 * i / N,
prefix=prefix,
suffix=str(datetime.timedelta(seconds=e()[0])))
table.drop(bind=db.engine)
dropped += 1
except:
print '\r', '>>> problem with dropping table', table
print
if dropped == N:
print ">>> Tables dropped successfully."
else:
print "ERROR: not all tables were properly dropped."
print ">>> Dropped", dropped, 'out of', N
