def get_nice_bibsched_related_message(curdir, ln=CFG_SITE_LANG): """ @return: a message suitable to display to the user, explaining the current status of the system. @rtype: string """ bibupload_id = ParamFromFile(os.path.join(curdir, 'bibupload_id')) if not bibupload_id: ## No BibUpload scheduled? Then we don't care about bibsched return "" ## Let's get an estimate about how many processes are waiting in the queue. ## Our bibupload might be somewhere in it, but it's not really so important ## WRT informing the user. _ = gettext_set_language(ln) res = run_sql( "SELECT id,proc,runtime,status,priority FROM schTASK WHERE (status='WAITING' AND runtime<=NOW()) OR status='SLEEPING'" ) pre = _( "Note that your submission has been inserted into the bibliographic task queue and is waiting for execution.\n" ) if server_pid(): ## BibSched is up and running msg = _( "The task queue is currently running in automatic mode, and there are currently %(x_num)s tasks waiting to be executed. Your record should be available within a few minutes and searchable within an hour or thereabouts.\n", x_num=(len(res))) else: msg = _( "Because of a human intervention or a temporary problem, the task queue is currently set to the manual mode. Your submission is well registered but may take longer than usual before it is fully integrated and searchable.\n" ) return pre + msg
def change_auto_mode(self, new_mode, duration=None): if not server_pid(): program = os.path.join(CFG_BINDIR, "bibsched") COMMAND = "%s -q start" % program os.system(COMMAND) # Enable automatic mode if new_mode: run_sql("""UPDATE "schSTATUS" SET value = '' WHERE name = 'resume_after'""") run_sql("""UPDATE "schSTATUS" SET value = '1' WHERE name = 'auto_mode'""") log('queue changed to automatic mode') # Enable manual mode else: run_sql("""UPDATE "schSTATUS" SET value = '0' WHERE name = 'auto_mode'""") if duration: resume_after = datetime.now() + timedelta(seconds=duration) resume_after = resume_after.strftime("%Y-%m-%d %H:%M:%S") else: resume_after = "" run_sql("""REPLACE INTO "schSTATUS" (name, value) VALUES ('resume_after', %s)""", [resume_after]) if duration: log('queue changed to manual mode for %ss' % duration) else: log('queue changed to manual mode') self.auto_mode = not self.auto_mode # We need to refresh the color of the header and footer self.repaint()
def change_auto_mode(self, new_mode, duration=None): if not server_pid(): program = os.path.join(CFG_BINDIR, "bibsched") COMMAND = "%s -q start" % program os.system(COMMAND) # Enable automatic mode if new_mode: run_sql( 'UPDATE schSTATUS SET value = "" WHERE name = "resume_after"') run_sql( 'UPDATE schSTATUS SET value = "1" WHERE name = "auto_mode"') log('queue changed to automatic mode') # Enable manual mode else: run_sql( 'UPDATE schSTATUS SET value = "0" WHERE name = "auto_mode"') if duration: resume_after = datetime.now() + timedelta(seconds=duration) resume_after = resume_after.strftime("%Y-%m-%d %H:%M:%S") else: resume_after = "" run_sql( 'REPLACE INTO schSTATUS (name, value) VALUES ("resume_after", %s)', [resume_after]) if duration: log('queue changed to manual mode for %ss' % duration) else: log('queue changed to manual mode') self.auto_mode = not self.auto_mode # We need to refresh the color of the header and footer self.repaint()
def reset_rec_cache(output_format, get_record, split_by=1000): """It either stores or does not store the output_format. If CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE is changed, this function will adapt the database to either store or not store the output_format.""" import sys try: from six.moves import cPickle as pickle except: import pickle from itertools import islice from intbitset import intbitset from invenio.legacy.bibsched.cli import server_pid, pidfile from invenio.ext.sqlalchemy import db from invenio.modules.records.models import Record as Bibrec from invenio.modules.formatter.models import Bibfmt pid = server_pid(ping_the_process=False) if pid: print("ERROR: bibsched seems to run with pid {0}, according to {1}.". format(pid, pidfile), file=sys.stderr) print(" Please stop bibsched before running this procedure.", file=sys.stderr) sys.exit(1) if current_app.config.get('CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE'): print( ">>> Searching records which need %s cache resetting; this may take a while..." % (output_format, )) all_recids = intbitset(db.session.query(Bibrec.id).all()) #TODO: prevent doing all records? recids = all_recids print(">>> Generating %s cache..." % (output_format, )) tot = len(recids) count = 0 it = iter(recids) while True: rec_group = tuple(islice(it, split_by)) if not len(rec_group): break Bibfmt.query.filter( db.and_(Bibfmt.id_bibrec.in_(rec_group), Bibfmt.format == output_format)).delete( synchronize_session=False) db.session.commit() #TODO: Update the cache or wait for the first access map(get_record, rec_group) count += len(rec_group) print(" ... done records %s/%s" % (count, tot)) if len(rec_group) < split_by or count >= tot: break print(">>> %s cache generated successfully." % (output_format, )) else: print(">>> Cleaning %s cache..." % (output_format, )) Bibfmt.query.filter(Bibfmt.format == output_format).delete( synchronize_session=False) db.session.commit()
def fetch_auto_mode(self): # If the daemon is not running at all, we are in manual mode if not server_pid(): status = 0 else: # Otherwise check the daemon status r = run_sql('SELECT value FROM schSTATUS WHERE name = "auto_mode"') try: status = int(r[0][0]) except (ValueError, IndexError): status = 0 return status
def fetch_auto_mode(self): # If the daemon is not running at all, we are in manual mode if not server_pid(): status = 0 else: # Otherwise check the daemon status r = run_sql("""SELECT value FROM "schSTATUS" WHERE name = 'auto_mode'""") try: status = int(r[0][0]) except (ValueError, IndexError): status = 0 return status
def reset_rec_cache(output_format, get_record, split_by=1000): """It either stores or does not store the output_format. If CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE is changed, this function will adapt the database to either store or not store the output_format.""" import sys try: from six.moves import cPickle as pickle except: import pickle from itertools import islice from intbitset import intbitset from invenio.legacy.bibsched.cli import server_pid, pidfile from invenio.ext.sqlalchemy import db from invenio_records.models import Record as Bibrec from invenio.modules.formatter.models import Bibfmt pid = server_pid(ping_the_process=False) if pid: print("ERROR: bibsched seems to run with pid {0}, according to {1}.".format(pid, pidfile), file=sys.stderr) print(" Please stop bibsched before running this procedure.", file=sys.stderr) sys.exit(1) if current_app.config.get("CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE"): print(">>> Searching records which need %s cache resetting; this may take a while..." % (output_format,)) all_recids = intbitset(db.session.query(Bibrec.id).all()) # TODO: prevent doing all records? recids = all_recids print(">>> Generating %s cache..." % (output_format,)) tot = len(recids) count = 0 it = iter(recids) while True: rec_group = tuple(islice(it, split_by)) if not len(rec_group): break Bibfmt.query.filter(db.and_(Bibfmt.id_bibrec.in_(rec_group), Bibfmt.format == output_format)).delete( synchronize_session=False ) db.session.commit() # TODO: Update the cache or wait for the first access map(get_record, rec_group) count += len(rec_group) print(" ... done records %s/%s" % (count, tot)) if len(rec_group) < split_by or count >= tot: break print(">>> %s cache generated successfully." % (output_format,)) else: print(">>> Cleaning %s cache..." % (output_format,)) Bibfmt.query.filter(Bibfmt.format == output_format).delete(synchronize_session=False) db.session.commit()
def get_nice_bibsched_related_message(curdir, ln=CFG_SITE_LANG): """ @return: a message suitable to display to the user, explaining the current status of the system. @rtype: string """ bibupload_id = ParamFromFile(os.path.join(curdir, 'bibupload_id')) if not bibupload_id: ## No BibUpload scheduled? Then we don't care about bibsched return "" ## Let's get an estimate about how many processes are waiting in the queue. ## Our bibupload might be somewhere in it, but it's not really so important ## WRT informing the user. _ = gettext_set_language(ln) res = run_sql("SELECT id,proc,runtime,status,priority FROM schTASK WHERE (status='WAITING' AND runtime<=NOW()) OR status='SLEEPING'") pre = _("Note that your submission has been inserted into the bibliographic task queue and is waiting for execution.\n") if server_pid(): ## BibSched is up and running msg = _("The task queue is currently running in automatic mode, and there are currently %(x_num)s tasks waiting to be executed. Your record should be available within a few minutes and searchable within an hour or thereabouts.\n", x_num=(len(res))) else: msg = _("Because of a human intervention or a temporary problem, the task queue is currently set to the manual mode. Your submission is well registered but may take longer than usual before it is fully integrated and searchable.\n") return pre + msg