def get_nice_bibsched_related_message(curdir, ln=CFG_SITE_LANG):
"""
@return: a message suitable to display to the user, explaining the current
status of the system.
@rtype: string
"""
bibupload_id = ParamFromFile(os.path.join(curdir, 'bibupload_id'))
if not bibupload_id:
## No BibUpload scheduled? Then we don't care about bibsched
return ""
## Let's get an estimate about how many processes are waiting in the queue.
## Our bibupload might be somewhere in it, but it's not really so important
## WRT informing the user.
_ = gettext_set_language(ln)
res = run_sql(
"SELECT id,proc,runtime,status,priority FROM schTASK WHERE (status='WAITING' AND runtime<=NOW()) OR status='SLEEPING'"
)
pre = _(
"Note that your submission has been inserted into the bibliographic task queue and is waiting for execution.\n"
)
if server_pid():
## BibSched is up and running
msg = _(
"The task queue is currently running in automatic mode, and there are currently %(x_num)s tasks waiting to be executed. Your record should be available within a few minutes and searchable within an hour or thereabouts.\n",
x_num=(len(res)))
else:
msg = _(
"Because of a human intervention or a temporary problem, the task queue is currently set to the manual mode. Your submission is well registered but may take longer than usual before it is fully integrated and searchable.\n"
)
return pre + msg
def change_auto_mode(self, new_mode, duration=None):
if not server_pid():
program = os.path.join(CFG_BINDIR, "bibsched")
COMMAND = "%s -q start" % program
os.system(COMMAND)
# Enable automatic mode
if new_mode:
run_sql("""UPDATE "schSTATUS" SET value = '' WHERE name = 'resume_after'""")
run_sql("""UPDATE "schSTATUS" SET value = '1' WHERE name = 'auto_mode'""")
log('queue changed to automatic mode')
# Enable manual mode
else:
run_sql("""UPDATE "schSTATUS" SET value = '0' WHERE name = 'auto_mode'""")
if duration:
resume_after = datetime.now() + timedelta(seconds=duration)
resume_after = resume_after.strftime("%Y-%m-%d %H:%M:%S")
else:
resume_after = ""
run_sql("""REPLACE INTO "schSTATUS" (name, value) VALUES ('resume_after', %s)""", [resume_after])
if duration:
log('queue changed to manual mode for %ss' % duration)
else:
log('queue changed to manual mode')
self.auto_mode = not self.auto_mode
# We need to refresh the color of the header and footer
self.repaint()
def change_auto_mode(self, new_mode, duration=None):
if not server_pid():
program = os.path.join(CFG_BINDIR, "bibsched")
COMMAND = "%s -q start" % program
os.system(COMMAND)
# Enable automatic mode
if new_mode:
run_sql(
'UPDATE schSTATUS SET value = "" WHERE name = "resume_after"')
run_sql(
'UPDATE schSTATUS SET value = "1" WHERE name = "auto_mode"')
log('queue changed to automatic mode')
# Enable manual mode
else:
run_sql(
'UPDATE schSTATUS SET value = "0" WHERE name = "auto_mode"')
if duration:
resume_after = datetime.now() + timedelta(seconds=duration)
resume_after = resume_after.strftime("%Y-%m-%d %H:%M:%S")
else:
resume_after = ""
run_sql(
'REPLACE INTO schSTATUS (name, value) VALUES ("resume_after", %s)',
[resume_after])
if duration:
log('queue changed to manual mode for %ss' % duration)
else:
log('queue changed to manual mode')
self.auto_mode = not self.auto_mode
# We need to refresh the color of the header and footer
self.repaint()
def reset_rec_cache(output_format, get_record, split_by=1000):
"""It either stores or does not store the output_format.
If CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE is changed, this function
will adapt the database to either store or not store the output_format."""
import sys
try:
from six.moves import cPickle as pickle
except:
import pickle
from itertools import islice
from intbitset import intbitset
from invenio.legacy.bibsched.cli import server_pid, pidfile
from invenio.ext.sqlalchemy import db
from invenio.modules.records.models import Record as Bibrec
from invenio.modules.formatter.models import Bibfmt
pid = server_pid(ping_the_process=False)
if pid:
print("ERROR: bibsched seems to run with pid {0}, according to {1}.".
format(pid, pidfile),
file=sys.stderr)
print(" Please stop bibsched before running this procedure.",
file=sys.stderr)
sys.exit(1)
if current_app.config.get('CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE'):
print(
">>> Searching records which need %s cache resetting; this may take a while..."
% (output_format, ))
all_recids = intbitset(db.session.query(Bibrec.id).all())
#TODO: prevent doing all records?
recids = all_recids
print(">>> Generating %s cache..." % (output_format, ))
tot = len(recids)
count = 0
it = iter(recids)
while True:
rec_group = tuple(islice(it, split_by))
if not len(rec_group):
break
Bibfmt.query.filter(
db.and_(Bibfmt.id_bibrec.in_(rec_group),
Bibfmt.format == output_format)).delete(
synchronize_session=False)
db.session.commit()
#TODO: Update the cache or wait for the first access
map(get_record, rec_group)
count += len(rec_group)
print(" ... done records %s/%s" % (count, tot))
if len(rec_group) < split_by or count >= tot:
break
print(">>> %s cache generated successfully." % (output_format, ))
else:
print(">>> Cleaning %s cache..." % (output_format, ))
Bibfmt.query.filter(Bibfmt.format == output_format).delete(
synchronize_session=False)
db.session.commit()
def fetch_auto_mode(self):
# If the daemon is not running at all, we are in manual mode
if not server_pid():
status = 0
else:
# Otherwise check the daemon status
r = run_sql('SELECT value FROM schSTATUS WHERE name = "auto_mode"')
try:
status = int(r[0][0])
except (ValueError, IndexError):
status = 0
return status
def fetch_auto_mode(self):
# If the daemon is not running at all, we are in manual mode
if not server_pid():
status = 0
else:
# Otherwise check the daemon status
r = run_sql("""SELECT value FROM "schSTATUS" WHERE name = 'auto_mode'""")
try:
status = int(r[0][0])
except (ValueError, IndexError):
status = 0
return status
def reset_rec_cache(output_format, get_record, split_by=1000):
"""It either stores or does not store the output_format.
If CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE is changed, this function
will adapt the database to either store or not store the output_format."""
import sys
try:
from six.moves import cPickle as pickle
except:
import pickle
from itertools import islice
from intbitset import intbitset
from invenio.legacy.bibsched.cli import server_pid, pidfile
from invenio.ext.sqlalchemy import db
from invenio_records.models import Record as Bibrec
from invenio.modules.formatter.models import Bibfmt
pid = server_pid(ping_the_process=False)
if pid:
print("ERROR: bibsched seems to run with pid {0}, according to {1}.".format(pid, pidfile), file=sys.stderr)
print(" Please stop bibsched before running this procedure.", file=sys.stderr)
sys.exit(1)
if current_app.config.get("CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE"):
print(">>> Searching records which need %s cache resetting; this may take a while..." % (output_format,))
all_recids = intbitset(db.session.query(Bibrec.id).all())
# TODO: prevent doing all records?
recids = all_recids
print(">>> Generating %s cache..." % (output_format,))
tot = len(recids)
count = 0
it = iter(recids)
while True:
rec_group = tuple(islice(it, split_by))
if not len(rec_group):
break
Bibfmt.query.filter(db.and_(Bibfmt.id_bibrec.in_(rec_group), Bibfmt.format == output_format)).delete(
synchronize_session=False
)
db.session.commit()
# TODO: Update the cache or wait for the first access
map(get_record, rec_group)
count += len(rec_group)
print(" ... done records %s/%s" % (count, tot))
if len(rec_group) < split_by or count >= tot:
break
print(">>> %s cache generated successfully." % (output_format,))
else:
print(">>> Cleaning %s cache..." % (output_format,))
Bibfmt.query.filter(Bibfmt.format == output_format).delete(synchronize_session=False)
db.session.commit()
def get_nice_bibsched_related_message(curdir, ln=CFG_SITE_LANG):
"""
@return: a message suitable to display to the user, explaining the current
status of the system.
@rtype: string
"""
bibupload_id = ParamFromFile(os.path.join(curdir, 'bibupload_id'))
if not bibupload_id:
## No BibUpload scheduled? Then we don't care about bibsched
return ""
## Let's get an estimate about how many processes are waiting in the queue.
## Our bibupload might be somewhere in it, but it's not really so important
## WRT informing the user.
_ = gettext_set_language(ln)
res = run_sql("SELECT id,proc,runtime,status,priority FROM schTASK WHERE (status='WAITING' AND runtime<=NOW()) OR status='SLEEPING'")
pre = _("Note that your submission has been inserted into the bibliographic task queue and is waiting for execution.\n")
if server_pid():
## BibSched is up and running
msg = _("The task queue is currently running in automatic mode, and there are currently %(x_num)s tasks waiting to be executed. Your record should be available within a few minutes and searchable within an hour or thereabouts.\n", x_num=(len(res)))
else:
msg = _("Because of a human intervention or a temporary problem, the task queue is currently set to the manual mode. Your submission is well registered but may take longer than usual before it is fully integrated and searchable.\n")
return pre + msg
