def _dump_database(dirname, filename):
"""
Dump Invenio database into SQL file called FILENAME living in
DIRNAME.
"""
write_message("... writing %s" % dirname + os.sep + filename)
cmd = CFG_PATH_MYSQL + 'dump'
if not os.path.exists(cmd):
msg = "ERROR: cannot find %s." % cmd
write_message(msg, stream=sys.stderr)
raise StandardError(msg)
cmd += " --skip-opt --add-drop-table --add-locks --create-options " \
" --quick --extended-insert --set-charset --disable-keys " \
" --host=%s --user=%s --password=%s %s | %s -c " % \
(escape_shell_arg(CFG_DATABASE_HOST),
escape_shell_arg(CFG_DATABASE_USER),
escape_shell_arg(CFG_DATABASE_PASS),
escape_shell_arg(CFG_DATABASE_NAME),
CFG_PATH_GZIP)
dummy1, dummy2, dummy3 = run_shell_command(cmd, None, dirname + os.sep + filename)
if dummy1:
msg = "ERROR: mysqldump exit code is %s." % repr(dummy1)
write_message(msg, stream=sys.stderr)
raise StandardError(msg)
if dummy2:
msg = "ERROR: mysqldump stdout is %s." % repr(dummy1)
write_message(msg, stream=sys.stderr)
raise StandardError(msg)
if dummy3:
msg = "ERROR: mysqldump stderr is %s." % repr(dummy1)
write_message(msg, stream=sys.stderr)
raise StandardError(msg)
def ticket_set_attribute(self, uid, ticketid, attribute, new_value):
"""change the ticket's attribute. Returns 1 on success, 0 on failure"""
#check that the attribute is accepted..
if attribute not in BibCatalogSystem.TICKET_ATTRIBUTES:
return 0
#we cannot change read-only values.. including text that is an attachment. pity
if attribute in [
'creator', 'date', 'ticketid', 'url_close', 'url_display',
'recordid', 'text'
]:
return 0
#check attribute
setme = ""
if attribute == 'priority':
try:
dummy = int(new_value)
except ValueError:
return 0
setme = "set Priority=" + str(new_value)
if attribute == 'subject':
subject = escape_shell_arg(new_value)
setme = "set Subject='" + subject + "'"
if attribute == 'owner':
#convert from invenio to RT
ownerprefs = invenio.legacy.webuser.get_user_preferences(new_value)
if "bibcatalog_username" not in ownerprefs:
return 0
else:
owner = escape_shell_arg(ownerprefs["bibcatalog_username"])
setme = " set owner='" + owner + "'"
if attribute == 'status':
setme = " set status='" + escape_shell_arg(new_value) + "'"
if attribute == 'queue':
setme = " set queue='" + escape_shell_arg(new_value) + "'"
#make sure ticketid is numeric
try:
dummy = int(ticketid)
except ValueError:
return 0
command = CFG_BIBCATALOG_SYSTEM_RT_CLI + " edit ticket/" + ticketid + setme
command_out = self._run_rt_command(command, uid)
if command_out is None:
return 0
mylines = command_out.split("\n")
for line in mylines:
if line.count('updated') > 0:
return 1
return 0
def ticket_set_attribute(self, uid, ticketid, attribute, new_value):
"""change the ticket's attribute. Returns 1 on success, 0 on failure"""
# check that the attribute is accepted..
if attribute not in BibCatalogSystem.TICKET_ATTRIBUTES:
return 0
# we cannot change read-only values.. including text that is an attachment. pity
if attribute in ["creator", "date", "ticketid", "url_close", "url_display", "recordid", "text"]:
return 0
# check attribute
setme = ""
if attribute == "priority":
try:
dummy = int(new_value)
except ValueError:
return 0
setme = "set Priority=" + str(new_value)
if attribute == "subject":
subject = escape_shell_arg(new_value)
setme = "set Subject='" + subject + "'"
if attribute == "owner":
# convert from invenio to RT
ownerprefs = invenio.legacy.webuser.get_user_preferences(new_value)
if "bibcatalog_username" not in ownerprefs:
return 0
else:
owner = escape_shell_arg(ownerprefs["bibcatalog_username"])
setme = " set owner='" + owner + "'"
if attribute == "status":
setme = " set status='" + escape_shell_arg(new_value) + "'"
if attribute == "queue":
setme = " set queue='" + escape_shell_arg(new_value) + "'"
# make sure ticketid is numeric
try:
dummy = int(ticketid)
except ValueError:
return 0
command = CFG_BIBCATALOG_SYSTEM_RT_CLI + " edit ticket/" + ticketid + setme
command_out = self._run_rt_command(command, uid)
if command_out is None:
return 0
mylines = command_out.split("\n")
for line in mylines:
if line.count("updated") > 0:
return 1
return 0
def _run_rt_command(self, command, uid=None):
"""
This function will run a RT CLI command as given user. If no user is specified
the default RT user will be used, if configured.
Should any of the configuration parameters be missing this function will return
None. Otherwise it will return the standard output from the CLI command.
@param command: RT CLI command to execute
@type command: string
@param uid: the Invenio user id to submit on behalf of. Optional.
@type uid: int
@return: standard output from the command given. None, if any errors.
@rtype: string
"""
if not CFG_BIBCATALOG_SYSTEM_RT_URL:
return None
if uid:
username, passwd = get_bibcat_from_prefs(uid)
else:
username = CFG_BIBCATALOG_SYSTEM_RT_DEFAULT_USER
passwd = CFG_BIBCATALOG_SYSTEM_RT_DEFAULT_PWD
httppart, siteandpath = CFG_BIBCATALOG_SYSTEM_RT_URL.split("//")
bibcatalog_rt_server = httppart + "//" + username + ":" + passwd + "@" + siteandpath
# set as env var
os.environ["RTUSER"] = username
os.environ["RTSERVER"] = bibcatalog_rt_server
passwd = escape_shell_arg(passwd)
error_code, myout, error_output = run_shell_command("echo " + passwd + " | " + command)
if error_code > 0:
raise ValueError('Problem running "%s": %d - %s' % (command, error_code, error_output))
return myout
def _task_submit(argv, authorization_action, authorization_msg):
"""Submits task to the BibSched task queue. This is what people will
be invoking via command line."""
## check as whom we want to submit?
check_running_process_user()
## sanity check: remove eventual "task" option:
## authenticate user:
_TASK_PARAMS['user'] = authenticate(_TASK_PARAMS["user"], authorization_action, authorization_msg)
## submit task:
if _TASK_PARAMS['task_specific_name']:
task_name = '%s:%s' % (_TASK_PARAMS['task_name'], _TASK_PARAMS['task_specific_name'])
else:
task_name = _TASK_PARAMS['task_name']
write_message("storing task options %s\n" % argv, verbose=9)
verbose_argv = 'Will execute: %s' % ' '.join([escape_shell_arg(str(arg)) for arg in argv])
_TASK_PARAMS['task_id'] = run_sql("""INSERT INTO schTASK (proc,user,
runtime,sleeptime,status,progress,arguments,priority,sequenceid)
VALUES (%s,%s,%s,%s,'WAITING',%s,%s,%s,%s)""",
(task_name, _TASK_PARAMS['user'], _TASK_PARAMS["runtime"],
_TASK_PARAMS["sleeptime"], verbose_argv, marshal.dumps(argv), _TASK_PARAMS['priority'], _TASK_PARAMS['sequence-id']))
## update task number:
write_message("Task #%d submitted." % _TASK_PARAMS['task_id'])
return _TASK_PARAMS['task_id']
def ticket_comment(self, uid, ticketid, comment):
"""comment on a given ticket. Returns 1 on success, 0 on failure"""
command = "%s comment -m %s %s" % (CFG_BIBCATALOG_SYSTEM_RT_CLI, escape_shell_arg(comment), ticketid)
command_out = self._run_rt_command(command, uid)
if command_out is None:
return None
return 1
def check_system(self, uid=None):
"""return an error string if there are problems"""
if uid:
rtuid, rtpw = get_bibcat_from_prefs(uid)
else:
# Assume default RT user
rtuid = CFG_BIBCATALOG_SYSTEM_RT_DEFAULT_USER
rtpw = CFG_BIBCATALOG_SYSTEM_RT_DEFAULT_PWD
if not rtuid and not rtpw:
return "No valid RT user login specified"
if not CFG_BIBCATALOG_SYSTEM == "RT":
return "CFG_BIBCATALOG_SYSTEM is not RT though this is an RT module"
if not CFG_BIBCATALOG_SYSTEM_RT_CLI:
return "CFG_BIBCATALOG_SYSTEM_RT_CLI not defined or empty"
if not os.path.exists(CFG_BIBCATALOG_SYSTEM_RT_CLI):
return "CFG_BIBCATALOG_SYSTEM_RT_CLI " + CFG_BIBCATALOG_SYSTEM_RT_CLI + " file does not exists"
# Check that you can execute the binary.. this is a safe call unless someone can fake CFG_BIBCATALOG_SYSTEM_RT_CLI (unlikely)
dummy, myout, myerr = run_shell_command(CFG_BIBCATALOG_SYSTEM_RT_CLI + " help")
helpfound = False
if myerr.count("help") > 0:
helpfound = True
if not helpfound:
return (
"Execution of CFG_BIBCATALOG_SYSTEM_RT_CLI "
+ CFG_BIBCATALOG_SYSTEM_RT_CLI
+ " help did not produce output 'help'"
)
if not CFG_BIBCATALOG_SYSTEM_RT_URL:
return "CFG_BIBCATALOG_SYSTEM_RT_URL not defined or empty"
# Construct URL, split RT_URL at //
if not CFG_BIBCATALOG_SYSTEM_RT_URL.startswith("http://") and not CFG_BIBCATALOG_SYSTEM_RT_URL.startswith(
"https://"
):
return "CFG_BIBCATALOG__SYSTEM_RT_URL does not start with 'http://' or 'https://'"
httppart, siteandpath = CFG_BIBCATALOG_SYSTEM_RT_URL.split("//")
# Assemble by http://user:password@RT_URL
bibcatalog_rt_server = httppart + "//" + rtuid + ":" + rtpw + "@" + siteandpath
# set as env var
os.environ["RTUSER"] = rtuid
os.environ["RTSERVER"] = bibcatalog_rt_server
# try to talk to RT server
# this is a safe call since rtpw is the only variable in it, and it is escaped
rtpw = escape_shell_arg(rtpw)
dummy, myout, myerr = run_shell_command(
"echo " + rtpw + " | " + CFG_BIBCATALOG_SYSTEM_RT_CLI + " ls \"Subject like 'F00'\""
)
if len(myerr) > 0:
return "could not connect to " + bibcatalog_rt_server + " " + myerr
# finally, check that there is some sane output like tickets or 'No matching result'
saneoutput = (myout.count("matching") > 0) or (myout.count("1") > 0)
if not saneoutput:
return CFG_BIBCATALOG_SYSTEM_RT_CLI + " returned " + myout + " instead of 'matching' or '1'"
return ""
def ticket_comment(self, uid, ticketid, comment):
"""comment on a given ticket. Returns 1 on success, 0 on failure"""
command = '%s comment -m %s %s' % (CFG_BIBCATALOG_SYSTEM_RT_CLI,
escape_shell_arg(comment), ticketid)
command_out = self._run_rt_command(command, uid)
if command_out is None:
return None
return 1
def check_system(self, uid=None):
"""return an error string if there are problems"""
if uid:
rtuid, rtpw = get_bibcat_from_prefs(uid)
else:
# Assume default RT user
rtuid = CFG_BIBCATALOG_SYSTEM_RT_DEFAULT_USER
rtpw = CFG_BIBCATALOG_SYSTEM_RT_DEFAULT_PWD
if not rtuid and not rtpw:
return "No valid RT user login specified"
if not CFG_BIBCATALOG_SYSTEM == 'RT':
return "CFG_BIBCATALOG_SYSTEM is not RT though this is an RT module"
if not CFG_BIBCATALOG_SYSTEM_RT_CLI:
return "CFG_BIBCATALOG_SYSTEM_RT_CLI not defined or empty"
if not os.path.exists(CFG_BIBCATALOG_SYSTEM_RT_CLI):
return "CFG_BIBCATALOG_SYSTEM_RT_CLI " + CFG_BIBCATALOG_SYSTEM_RT_CLI + " file does not exists"
# Check that you can execute the binary.. this is a safe call unless someone can fake CFG_BIBCATALOG_SYSTEM_RT_CLI (unlikely)
dummy, myout, myerr = run_shell_command(CFG_BIBCATALOG_SYSTEM_RT_CLI +
" help")
helpfound = False
if myerr.count("help") > 0:
helpfound = True
if not helpfound:
return "Execution of CFG_BIBCATALOG_SYSTEM_RT_CLI " + CFG_BIBCATALOG_SYSTEM_RT_CLI + " help did not produce output 'help'"
if not CFG_BIBCATALOG_SYSTEM_RT_URL:
return "CFG_BIBCATALOG_SYSTEM_RT_URL not defined or empty"
# Construct URL, split RT_URL at //
if not CFG_BIBCATALOG_SYSTEM_RT_URL.startswith('http://') and \
not CFG_BIBCATALOG_SYSTEM_RT_URL.startswith('https://'):
return "CFG_BIBCATALOG__SYSTEM_RT_URL does not start with 'http://' or 'https://'"
httppart, siteandpath = CFG_BIBCATALOG_SYSTEM_RT_URL.split("//")
# Assemble by http://user:password@RT_URL
bibcatalog_rt_server = httppart + "//" + rtuid + ":" + rtpw + "@" + siteandpath
#set as env var
os.environ["RTUSER"] = rtuid
os.environ["RTSERVER"] = bibcatalog_rt_server
#try to talk to RT server
#this is a safe call since rtpw is the only variable in it, and it is escaped
rtpw = escape_shell_arg(rtpw)
dummy, myout, myerr = run_shell_command("echo " + rtpw + " | " +
CFG_BIBCATALOG_SYSTEM_RT_CLI +
" ls \"Subject like 'F00'\"")
if len(myerr) > 0:
return "could not connect to " + bibcatalog_rt_server + " " + myerr
#finally, check that there is some sane output like tickets or 'No matching result'
saneoutput = (myout.count('matching') > 0) or (myout.count('1') > 0)
if not saneoutput:
return CFG_BIBCATALOG_SYSTEM_RT_CLI + " returned " + myout + " instead of 'matching' or '1'"
return ""
def _ticket_submit(self,
uid=None,
subject="",
recordid=-1,
text="",
queue="",
priority="",
owner="",
requestor=""):
"""creates a ticket. return ticket_id string on success, otherwise None"""
queueset = ""
textset = ""
priorityset = ""
ownerset = ""
subjectset = ""
requestorset = ""
if subject:
cleaned_subject = " ".join(str(subject).splitlines())
subjectset = " subject=" + escape_shell_arg(cleaned_subject)
recidset = " CF-RecordID=" + escape_shell_arg(str(recordid))
if priority:
priorityset = " priority=" + escape_shell_arg(str(priority))
if queue:
queueset = " queue=" + escape_shell_arg(queue)
if requestor:
requestorset = " requestor=" + escape_shell_arg(requestor)
if owner:
#get the owner name from prefs
ownerprefs = invenio.legacy.webuser.get_user_preferences(owner)
if "bibcatalog_username" in ownerprefs:
owner = ownerprefs["bibcatalog_username"]
ownerset = " owner=" + escape_shell_arg(owner)
if text:
if '\n' in text:
# contains newlines (\n) return with error
raise Exception(
"Newlines are not allowed in text parameter. Use ticket_comment() instead."
)
else:
textset = " text=" + escape_shell_arg(text)
# make a command.. note that all set 'set' parts have been escaped
command = CFG_BIBCATALOG_SYSTEM_RT_CLI + " create -t ticket set " + subjectset + recidset + \
queueset + textset + priorityset + ownerset + requestorset
command_out = self._run_rt_command(command, uid)
if command_out is None:
return None
ticket_id = None
ticket_created_re = re.compile(r'Ticket (\d+) created')
for line in command_out.split("\n"):
matches = ticket_created_re.search(line)
if matches:
ticket_id = matches.groups()[0]
return ticket_id
def _ticket_submit(self, uid=None, subject="", recordid=-1, text="", queue="", priority="", owner="", requestor=""):
"""creates a ticket. return ticket_id string on success, otherwise None"""
queueset = ""
textset = ""
priorityset = ""
ownerset = ""
subjectset = ""
requestorset = ""
if subject:
cleaned_subject = " ".join(str(subject).splitlines())
subjectset = " subject=" + escape_shell_arg(cleaned_subject)
recidset = " CF-RecordID=" + escape_shell_arg(str(recordid))
if priority:
priorityset = " priority=" + escape_shell_arg(str(priority))
if queue:
queueset = " queue=" + escape_shell_arg(queue)
if requestor:
requestorset = " requestor=" + escape_shell_arg(requestor)
if owner:
# get the owner name from prefs
ownerprefs = invenio.legacy.webuser.get_user_preferences(owner)
if "bibcatalog_username" in ownerprefs:
owner = ownerprefs["bibcatalog_username"]
ownerset = " owner=" + escape_shell_arg(owner)
if text:
if "\n" in text:
# contains newlines (\n) return with error
raise Exception("Newlines are not allowed in text parameter. Use ticket_comment() instead.")
else:
textset = " text=" + escape_shell_arg(text)
# make a command.. note that all set 'set' parts have been escaped
command = (
CFG_BIBCATALOG_SYSTEM_RT_CLI
+ " create -t ticket set "
+ subjectset
+ recidset
+ queueset
+ textset
+ priorityset
+ ownerset
+ requestorset
)
command_out = self._run_rt_command(command, uid)
if command_out is None:
return None
ticket_id = None
ticket_created_re = re.compile(r"Ticket (\d+) created")
for line in command_out.split("\n"):
matches = ticket_created_re.search(line)
if matches:
ticket_id = matches.groups()[0]
return ticket_id
def ticket_submit(self, uid=None, subject="", recordid=-1, text="", queue="",
priority="", owner="", requestor=""):
"""creates a ticket. return ticket num on success, otherwise None"""
queueset = ""
textset = ""
priorityset = ""
ownerset = ""
subjectset = ""
requestorset = ""
if subject:
subjectset = " subject=" + escape_shell_arg(subject)
recidset = " CF-RecordID=" + escape_shell_arg(str(recordid))
if priority:
priorityset = " priority=" + escape_shell_arg(str(priority))
if queue:
queueset = " queue=" + escape_shell_arg(queue)
if requestor:
requestorset = " requestor=" + escape_shell_arg(requestor)
if owner:
#get the owner name from prefs
ownerprefs = invenio.legacy.webuser.get_user_preferences(owner)
if "bibcatalog_username" in ownerprefs:
owner = ownerprefs["bibcatalog_username"]
ownerset = " owner=" + escape_shell_arg(owner)
if text:
if '\n' in text:
# contains newlines (\n) return with error
return "Newlines are not allowed in text parameter. Use ticket_comment() instead."
else:
textset = " text=" + escape_shell_arg(text)
# make a command.. note that all set 'set' parts have been escaped
command = CFG_BIBCATALOG_SYSTEM_RT_CLI + " create -t ticket set " + subjectset + recidset + \
queueset + textset + priorityset + ownerset + requestorset
command_out = self._run_rt_command(command, uid)
if command_out == None:
return None
inum = -1
for line in command_out.split("\n"):
if line.count(' ') > 0:
stuff = line.split(' ')
try:
inum = int(stuff[2])
except:
pass
if inum > 0:
return inum
return None
def _run_rt_command(self, command, uid=None):
"""
This function will run a RT CLI command as given user. If no user is specified
the default RT user will be used, if configured.
Should any of the configuration parameters be missing this function will return
None. Otherwise it will return the standard output from the CLI command.
@param command: RT CLI command to execute
@type command: string
@param uid: the Invenio user id to submit on behalf of. Optional.
@type uid: int
@return: standard output from the command given. None, if any errors.
@rtype: string
"""
if not CFG_BIBCATALOG_SYSTEM_RT_URL:
return None
if uid:
username, passwd = get_bibcat_from_prefs(uid)
else:
username = CFG_BIBCATALOG_SYSTEM_RT_DEFAULT_USER
passwd = CFG_BIBCATALOG_SYSTEM_RT_DEFAULT_PWD
httppart, siteandpath = CFG_BIBCATALOG_SYSTEM_RT_URL.split("//")
bibcatalog_rt_server = httppart + "//" + username + ":" + passwd + "@" + siteandpath
#set as env var
os.environ["RTUSER"] = username
os.environ["RTSERVER"] = bibcatalog_rt_server
passwd = escape_shell_arg(passwd)
error_code, myout, error_output = run_shell_command("echo " + passwd +
" | " + command)
if error_code > 0:
raise ValueError('Problem running "%s": %d - %s' %
(command, error_code, error_output))
return myout
def ticket_submit(self, uid=None, subject="", recordid=-1, text="", queue="", priority="", owner="", requestor=""):
"""creates a ticket. return true on success, otherwise false"""
if not EMAIL_SUBMIT_CONFIGURED:
register_exception(stream='warning',
subject='bibcatalog email not configured',
prefix="please configure bibcatalog email sending in CFG_BIBCATALOG_SYSTEM and CFG_BIBCATALOG_SYSTEM_EMAIL_ADDRESS")
ticket_id = self._get_ticket_id()
priorityset = ""
queueset = ""
requestorset = ""
ownerset = ""
recidset = " cf-recordID:" + escape_shell_arg(str(recordid)) + '\n'
textset = ""
subjectset = ""
if subject:
subjectset = 'ticket #' + ticket_id + ' - ' + escape_shell_arg(subject)
if priority:
priorityset = " priority:" + escape_shell_arg(str(priority)) + '\n'
if queue:
queueset = " queue:" + escape_shell_arg(queue) + '\n'
if requestor:
requestorset = " requestor:" + escape_shell_arg(requestor) + '\n'
if owner:
ownerprefs = invenio.legacy.webuser.get_user_preferences(owner)
if "bibcatalog_username" in ownerprefs:
owner = ownerprefs["bibcatalog_username"]
ownerset = " owner:" + escape_shell_arg(owner) + '\n'
textset = textset + ownerset + requestorset + recidset + queueset + priorityset + '\n'
textset = textset + escape_shell_arg(text) + '\n'
ok = send_email(fromaddr=FROM_ADDRESS, toaddr=TO_ADDRESS, subject=subjectset, header='Hello,\n\n', content=textset)
if ok:
return ticket_id
return None
def test_escape_double_quoted(self):
"""shellutils - escaping strings containing double-quotes"""
self.assertEqual("""'"hello world"'""",
escape_shell_arg('"hello world"'))
def test_escape_quoted(self):
"""shellutils - escaping strings containing single quotes"""
self.assertEqual("'hello'\\''world'",
escape_shell_arg("hello'world"))
def test_escape_number_sign(self):
"""shellutils - escaping strings containing the number sign"""
self.assertEqual(r"'Python comments start with #.'",
escape_shell_arg(r'Python comments start with #.'))
def task_low_level_submission(name, user, *argv):
"""Let special lowlevel enqueuing of a task on the bibsche queue.
@param name: is the name of the bibtask. It must be a valid executable under
C{CFG_BINDIR}.
@type name: string
@param user: is a string that will appear as the "user" submitting the task.
Since task are submitted via API it make sense to set the
user to the name of the module/function that called
task_low_level_submission.
@type user: string
@param argv: are all the additional CLI parameters that would have been
passed on the CLI (one parameter per variable).
e.g.:
>>> task_low_level_submission('bibupload', 'admin', '-a', '/tmp/z.xml')
@type: strings
@return: the task identifier when the task is correctly enqueued.
@rtype: int
@note: use absolute paths in argv
"""
def get_priority(argv):
"""Try to get the priority by analysing the arguments."""
priority = 0
argv = list(argv)
while True:
try:
opts, args = getopt.gnu_getopt(argv, 'P:', ['priority='])
except getopt.GetoptError as err:
## We remove one by one all the non recognized parameters
if len(err.opt) > 1:
argv = [arg for arg in argv if arg != '--%s' % err.opt and not arg.startswith('--%s=' % err.opt)]
else:
argv = [arg for arg in argv if not arg.startswith('-%s' % err.opt)]
else:
break
for opt in opts:
if opt[0] in ('-P', '--priority'):
try:
priority = int(opt[1])
except ValueError:
pass
return priority
def get_special_name(argv):
"""Try to get the special name by analysing the arguments."""
special_name = ''
argv = list(argv)
while True:
try:
opts, args = getopt.gnu_getopt(argv, 'N:', ['name='])
except getopt.GetoptError as err:
## We remove one by one all the non recognized parameters
if len(err.opt) > 1:
argv = [arg for arg in argv if arg != '--%s' % err.opt and not arg.startswith('--%s=' % err.opt)]
else:
argv = [arg for arg in argv if not arg.startswith('-%s' % err.opt)]
else:
break
for opt in opts:
if opt[0] in ('-N', '--name'):
special_name = opt[1]
return special_name
def get_runtime(argv):
"""Try to get the runtime by analysing the arguments."""
runtime = time.strftime("%Y-%m-%d %H:%M:%S")
argv = list(argv)
while True:
try:
opts, args = getopt.gnu_getopt(argv, 't:', ['runtime='])
except getopt.GetoptError as err:
## We remove one by one all the non recognized parameters
if len(err.opt) > 1:
argv = [arg for arg in argv if arg != '--%s' % err.opt and not arg.startswith('--%s=' % err.opt)]
else:
argv = [arg for arg in argv if not arg.startswith('-%s' % err.opt)]
else:
break
for opt in opts:
if opt[0] in ('-t', '--runtime'):
try:
runtime = get_datetime(opt[1])
except ValueError:
pass
return runtime
def get_sleeptime(argv):
"""Try to get the runtime by analysing the arguments."""
sleeptime = ""
argv = list(argv)
while True:
try:
opts, args = getopt.gnu_getopt(argv, 's:', ['sleeptime='])
except getopt.GetoptError as err:
## We remove one by one all the non recognized parameters
if len(err.opt) > 1:
argv = [arg for arg in argv if arg != '--%s' % err.opt and not arg.startswith('--%s=' % err.opt)]
else:
argv = [arg for arg in argv if not arg.startswith('-%s' % err.opt)]
else:
break
for opt in opts:
if opt[0] in ('-s', '--sleeptime'):
try:
sleeptime = opt[1]
except ValueError:
pass
return sleeptime
def get_sequenceid(argv):
"""Try to get the sequenceid by analysing the arguments."""
sequenceid = None
argv = list(argv)
while True:
try:
opts, args = getopt.gnu_getopt(argv, 'I:', ['sequence-id='])
except getopt.GetoptError as err:
## We remove one by one all the non recognized parameters
if len(err.opt) > 1:
argv = [arg for arg in argv if arg != '--%s' % err.opt and not arg.startswith('--%s=' % err.opt)]
else:
argv = [arg for arg in argv if not arg.startswith('-%s' % err.opt)]
else:
break
for opt in opts:
if opt[0] in ('-I', '--sequence-id'):
try:
sequenceid = opt[1]
except ValueError:
pass
return sequenceid
task_id = None
try:
if not name in CFG_BIBTASK_VALID_TASKS:
raise StandardError('%s is not a valid task name' % name)
new_argv = []
for arg in argv:
if isinstance(arg, unicode):
arg = arg.encode('utf8')
new_argv.append(arg)
argv = new_argv
priority = get_priority(argv)
special_name = get_special_name(argv)
runtime = get_runtime(argv)
sleeptime = get_sleeptime(argv)
sequenceid = get_sequenceid(argv)
argv = tuple([os.path.join(CFG_BINDIR, name)] + list(argv))
if special_name:
name = '%s:%s' % (name, special_name)
verbose_argv = 'Will execute: %s' % ' '.join([escape_shell_arg(str(arg)) for arg in argv])
## submit task:
task_id = run_sql("""INSERT INTO schTASK (proc,user,
runtime,sleeptime,status,progress,arguments,priority,sequenceid)
VALUES (%s,%s,%s,%s,'WAITING',%s,%s,%s,%s)""",
(name, user, runtime, sleeptime, verbose_argv[:254], marshal.dumps(argv), priority, sequenceid))
except Exception:
register_exception(alert_admin=True)
if task_id:
run_sql("""DELETE FROM schTASK WHERE id=%s""", (task_id, ))
raise
return task_id
def test_escape_simple(self):
"""shellutils - escaping simple strings"""
self.assertEqual("'hello'",
escape_shell_arg("hello"))
def test_escape_less_than(self):
"""shellutils - escaping strings containing the less-than sign"""
self.assertEqual(r"'5 < 10'",
escape_shell_arg(r'5 < 10'))
def test_escape_greater_than(self):
"""shellutils - escaping strings containing the greater-than sign"""
self.assertEqual(r"'10 > 5'",
escape_shell_arg(r'10 > 5'))
def test_escape_unix_style_path(self):
"""shellutils - escaping strings containing unix-style file paths"""
self.assertEqual(r"'/tmp/z_temp.txt'",
escape_shell_arg(r'/tmp/z_temp.txt'))
def test_escape_complex_quoted(self):
"""shellutils - escaping strings containing complex quoting"""
self.assertEqual(r"""'"Who is this `Eve'\'', Bob?", asked Alice.'""",
escape_shell_arg(r""""Who is this `Eve', Bob?", asked Alice."""))
def ticket_search(
self,
uid,
recordid=-1,
subject="",
text="",
creator="",
owner="",
date_from="",
date_until="",
status="",
priority="",
queue="",
):
"""returns a list of ticket ID's related to this record or by
matching the subject, creator or owner of the ticket."""
search_atoms = [] # the search expression will be made by and'ing these
if recordid > -1:
# search by recid
search_atoms.append("CF.{RecordID} = " + escape_shell_arg(str(recordid)))
if subject:
# search by subject
search_atoms.append("Subject like " + escape_shell_arg(str(subject)))
if text:
search_atoms.append("Content like " + escape_shell_arg(str(text)))
if str(creator):
# search for this person's bibcatalog_username in preferences
creatorprefs = invenio.legacy.webuser.get_user_preferences(creator)
creator = "Nobody can Have This Kind of Name"
if "bibcatalog_username" in creatorprefs:
creator = creatorprefs["bibcatalog_username"]
search_atoms.append("Creator = " + escape_shell_arg(str(creator)))
if str(owner):
ownerprefs = invenio.legacy.webuser.get_user_preferences(owner)
owner = "Nobody can Have This Kind of Name"
if "bibcatalog_username" in ownerprefs:
owner = ownerprefs["bibcatalog_username"]
search_atoms.append("Owner = " + escape_shell_arg(str(owner)))
if date_from:
search_atoms.append("Created >= " + escape_shell_arg(str(date_from)))
if date_until:
search_atoms.append("Created <= " + escape_shell_arg(str(date_until)))
if str(status) and isinstance(status, type("this is a string")):
search_atoms.append("Status = " + escape_shell_arg(str(status)))
if str(priority):
# Try to convert to int
intpri = -1
try:
intpri = int(priority)
except ValueError:
pass
if intpri > -1:
search_atoms.append("Priority = " + str(intpri))
if queue:
search_atoms.append("Queue = " + escape_shell_arg(queue))
searchexp = " and ".join(search_atoms)
tickets = []
if len(searchexp) == 0:
return tickets
command = CFG_BIBCATALOG_SYSTEM_RT_CLI + ' ls -l "' + searchexp + '"'
command_out = self._run_rt_command(command, uid)
if command_out is None:
return tickets
statuses = []
for line in command_out.split("\n"):
# if there are matching lines they will look like NUM:subj.. so pick num
if line.count("id: ticket/") > 0:
dummy, tnum = line.split("/") # get the ticket id
try:
dummy = int(tnum)
except ValueError:
pass
else:
tickets.append(tnum)
if line.count("Status: ") > 0:
dummy, tstatus = line.split("Status: ")
statuses.append(tstatus)
if isinstance(status, list):
# take only those tickets whose status matches with one of the status list
alltickets = tickets
tickets = []
for i in range(len(alltickets)):
tstatus = statuses[i]
tnum = alltickets[i]
if status.count(tstatus) > 0: # match
tickets.append(tnum)
return tickets
def test_escape_windows_style_path(self):
"""shellutils - escaping strings containing windows-style file paths"""
self.assertEqual(r"'C:\Users\Test User\My Documents" \
"\funny file name (for testing).pdf'",
escape_shell_arg(r'C:\Users\Test User\My Documents' \
'\funny file name (for testing).pdf'))
def build_icon(path_workingdir,
source_filename,
source_filetype,
icon_name,
icon_filetype,
multipage_icon,
multipage_icon_delay,
icon_scale):
"""Whereas create_icon acts as the API for icon creation and therefore
deals with argument washing, temporary working directory creation,
etc, the build_icon function takes care of the actual creation of the
icon file itself by calling various shell tools.
To accomplish this, it relies upon the following parameters:
@param path_workingdir: (string) - the path to the working directory
in which all files related to the icon creation are stored.
@param source_filename: (string) - the filename of the original image
file.
@param source_filetype: (string) - the file type of the original image
file.
@param icon_name: (string) - the name that is to be given to the icon.
@param icon_filetype: (string) - the file type of the icon that is
to be created.
@param multipage_icon: (boolean) - a flag indicating whether or not
an icon with multiple pages (i.e. an animated gif icon) should be
created.
@param multipage_icon_delay: (integer) - the delay to be used between
frame changing for an icon with multiple pages (i.e. an animated gif.)
@param icon_scale: (integer) - the scaling information for the created
icon.
@return: (string) - the name of the created icon file (which will have
been created in the working directory "path_workingdir".)
@Exceptions raised: (InvenioWebSubmitIconCreatorError) - raised when
the icon creation process fails.
"""
##
## If the source file is a PS, convert it into a PDF:
if source_filetype == "ps":
## Convert the subject file from PostScript to PDF:
if source_filename[-3:].lower() == ".ps":
## The name of the file to be stamped has a PostScript extension.
## Strip it and give the name of the PDF file to be created a
## PDF extension:
created_pdfname = "%s.pdf" % source_filename[:-3]
elif len(source_filename.split(".")) > 1:
## The file name has an extension - strip it and add a PDF
## extension:
raw_name = source_filename[:source_filename.rfind(".")]
if raw_name != "":
created_pdfname = "%s.pdf" % raw_name
else:
## It would appear that the file had no extension and that its
## name started with a period. Just use the original name with
## a .pdf suffix:
created_pdfname = "%s.pdf" % source_filename
else:
## No extension - use the original name with a .pdf suffix:
created_pdfname = "%s.pdf" % source_filename
## Build the distilling command:
cmd_distill = """%(distiller)s %(ps-file-path)s """ \
"""%(pdf-file-path)s 2>/dev/null""" % \
{ 'distiller' : CFG_PATH_PS2PDF,
'ps-file-path' : escape_shell_arg("%s/%s" % \
(path_workingdir, \
source_filename)),
'pdf-file-path' : escape_shell_arg("%s/%s" % \
(path_workingdir, \
created_pdfname)),
}
## Distill the PS into a PDF:
errcode_distill = os.system(cmd_distill)
## Test to see whether the PS was distilled into a PDF without error:
if errcode_distill or \
not os.access("%s/%s" % (path_workingdir, created_pdfname), os.F_OK):
## The PDF file was not correctly created in the working directory.
## Unable to continue.
msg = "Error: Unable to correctly convert PostScript file [%s] to" \
" PDF. Cannot create icon." % source_filename
raise InvenioWebSubmitIconCreatorError(msg)
## Now assign the name of the created PDF file to subject_file:
source_filename = created_pdfname
##
## Treat the name of the icon:
if icon_name in (None, ""):
## Since no name has been provided for the icon, give it the same name
## as the source file, but with the prefix "icon-":
icon_name = "icon-%s" % source_filename
## Now if the icon name has an extension, strip it and add that of the
## icon file type:
if len(icon_name.split(".")) > 1:
## The icon file name has an extension - strip it and add the icon
## file type extension:
raw_name = icon_name[:icon_name.rfind(".")]
if raw_name != "":
icon_name = "%s.%s" % (raw_name, icon_filetype)
else:
## It would appear that the file had no extension and that its
## name started with a period. Just use the original name with
## the icon file type's suffix:
icon_name = "%s.%s" % (icon_name, icon_filetype)
else:
## The icon name had no extension. Use the original name with the
## icon file type's suffix:
icon_name = "%s.%s" % (icon_name, icon_filetype)
##
## If the source file type is PS or PDF, it may be necessary to separate
## the first page from the rest of the document and keep it for use as
## the icon. Do this if necessary:
if source_filetype in ("ps", "pdf") and \
(icon_filetype != "gif" or not multipage_icon):
## Either (a) the icon type isn't GIF (in which case it cannot
## be animated and must therefore be created _only_ from the
## document's first page; or (b) the icon type is GIF, but the
## icon is to be created from the first page of the document only.
## The first page of the PDF document must be separated and is to
## be used for icon creation:
source_file_first_page = "p1-%s" % source_filename
## Perform the separation:
cmd_get_first_page = \
"%(pdftk)s A=%(source-file-path)s " \
"cat A1 output %(first-page-path)s " \
"2>/dev/null" \
% { 'pdftk' : CFG_PATH_PDFTK,
'source-file-path' : escape_shell_arg("%s/%s" % \
(path_workingdir, source_filename)),
'first-page-path' : escape_shell_arg("%s/%s" % \
(path_workingdir, \
source_file_first_page)),
}
errcode_get_first_page = os.system(cmd_get_first_page)
## Check that the separation was successful:
if errcode_get_first_page or \
not os.access("%s/%s" % (path_workingdir, \
source_file_first_page), os.F_OK):
## Separation was unsuccessful.
msg = "Error: Unable to create an icon for file [%s/%s] - it " \
"wasn't possible to separate the first page from the " \
"rest of the document (error code [%s].)" \
% (path_workingdir, source_filename, errcode_get_first_page)
raise InvenioWebSubmitIconCreatorError(msg)
else:
## Successfully extracted the first page. Treat it as the source
## file for icon creation from now on:
source_filename = source_file_first_page
##
## Create the icon:
## If a delay is necessary for an animated gif icon, create the
## delay string:
delay_info = ""
if source_filetype in ("ps", "pdf") and \
icon_filetype == "gif" and multipage_icon:
## Include delay information:
delay_info = "-delay %s" % escape_shell_arg(str(multipage_icon_delay))
## Command for icon creation:
cmd_create_icon = "%(convert)s -colorspace rgb -scale %(scale)s %(delay)s " \
"%(source-file-path)s %(icon-file-path)s 2>/dev/null" \
% { 'convert' : CFG_PATH_CONVERT,
'scale' : \
escape_shell_arg(icon_scale),
'delay' : delay_info,
'source-file-path' : \
escape_shell_arg("%s/%s" \
% (path_workingdir, \
source_filename)),
'icon-file-path' : \
escape_shell_arg("%s/%s" \
% (path_workingdir, \
icon_name)),
}
errcode_create_icon = os.system(cmd_create_icon)
## Check that the icon creation was successful:
if errcode_create_icon or \
not os.access("%s/%s" % (path_workingdir, icon_name), os.F_OK):
## Icon creation was unsuccessful.
msg = "Error: Unable to create an icon for file [%s/%s] (error " \
"code [%s].)" \
% (path_workingdir, source_filename, errcode_create_icon)
raise InvenioWebSubmitIconCreatorError(msg)
##
## The icon was successfully created. Return its name:
return icon_name
def test_escape_less_than(self):
"""shellutils - escaping strings containing the less-than sign"""
self.assertEqual(r"'5 < 10'",
escape_shell_arg(r'5 < 10'))
def test_escape_complex_quoted(self):
"""shellutils - escaping strings containing complex quoting"""
self.assertEqual(r"""'"Who is this `Eve'\'', Bob?", asked Alice.'""",
escape_shell_arg(r""""Who is this `Eve', Bob?", asked Alice."""))
def test_escape_ampersand(self):
"""shellutils - escaping strings containing ampersand"""
self.assertEqual(r"'Today the weather is hot & sunny'",
escape_shell_arg(r'Today the weather is hot & sunny'))
def test_escape_windows_style_path(self):
"""shellutils - escaping strings containing windows-style file paths"""
self.assertEqual(r"'C:\Users\Test User\My Documents" \
"\funny file name (for testing).pdf'",
escape_shell_arg(r'C:\Users\Test User\My Documents' \
'\funny file name (for testing).pdf'))
def test_escape_ampersand(self):
"""shellutils - escaping strings containing ampersand"""
self.assertEqual(r"'Today the weather is hot & sunny'",
escape_shell_arg(r'Today the weather is hot & sunny'))
def test_escape_number_sign(self):
"""shellutils - escaping strings containing the number sign"""
self.assertEqual(r"'Python comments start with #.'",
escape_shell_arg(r'Python comments start with #.'))
def test_escape_double_quoted(self):
"""shellutils - escaping strings containing double-quotes"""
self.assertEqual("""'"hello world"'""",
escape_shell_arg('"hello world"'))
def test_escape_greater_than(self):
"""shellutils - escaping strings containing the greater-than sign"""
self.assertEqual(r"'10 > 5'",
escape_shell_arg(r'10 > 5'))
def test_escape_quoted(self):
"""shellutils - escaping strings containing single quotes"""
self.assertEqual("'hello'\\''world'",
escape_shell_arg("hello'world"))
def ticket_search(self,
uid,
recordid=-1,
subject="",
text="",
creator="",
owner="",
date_from="",
date_until="",
status="",
priority="",
queue=""):
"""returns a list of ticket ID's related to this record or by
matching the subject, creator or owner of the ticket."""
search_atoms = [
] # the search expression will be made by and'ing these
if (recordid > -1):
#search by recid
search_atoms.append("CF.{RecordID} = " +
escape_shell_arg(str(recordid)))
if subject:
#search by subject
search_atoms.append("Subject like " +
escape_shell_arg(str(subject)))
if text:
search_atoms.append("Content like " + escape_shell_arg(str(text)))
if str(creator):
#search for this person's bibcatalog_username in preferences
creatorprefs = invenio.legacy.webuser.get_user_preferences(creator)
creator = "Nobody can Have This Kind of Name"
if "bibcatalog_username" in creatorprefs:
creator = creatorprefs["bibcatalog_username"]
search_atoms.append("Creator = " + escape_shell_arg(str(creator)))
if str(owner):
ownerprefs = invenio.legacy.webuser.get_user_preferences(owner)
owner = "Nobody can Have This Kind of Name"
if "bibcatalog_username" in ownerprefs:
owner = ownerprefs["bibcatalog_username"]
search_atoms.append("Owner = " + escape_shell_arg(str(owner)))
if date_from:
search_atoms.append("Created >= " +
escape_shell_arg(str(date_from)))
if date_until:
search_atoms.append("Created <= " +
escape_shell_arg(str(date_until)))
if str(status) and isinstance(status, type("this is a string")):
search_atoms.append("Status = " + escape_shell_arg(str(status)))
if str(priority):
# Try to convert to int
intpri = -1
try:
intpri = int(priority)
except ValueError:
pass
if intpri > -1:
search_atoms.append("Priority = " + str(intpri))
if queue:
search_atoms.append("Queue = " + escape_shell_arg(queue))
searchexp = " and ".join(search_atoms)
tickets = []
if len(searchexp) == 0:
return tickets
command = CFG_BIBCATALOG_SYSTEM_RT_CLI + " ls -l \"" + searchexp + "\""
command_out = self._run_rt_command(command, uid)
if command_out is None:
return tickets
statuses = []
for line in command_out.split("\n"):
#if there are matching lines they will look like NUM:subj.. so pick num
if line.count('id: ticket/') > 0:
dummy, tnum = line.split('/') # get the ticket id
try:
dummy = int(tnum)
except ValueError:
pass
else:
tickets.append(tnum)
if line.count('Status: ') > 0:
dummy, tstatus = line.split('Status: ')
statuses.append(tstatus)
if isinstance(status, list):
#take only those tickets whose status matches with one of the status list
alltickets = tickets
tickets = []
for i in range(len(alltickets)):
tstatus = statuses[i]
tnum = alltickets[i]
if status.count(tstatus) > 0: # match
tickets.append(tnum)
return tickets
def test_escape_backtick(self):
"""shellutils - escaping strings containing backticks"""
self.assertEqual(r"'hello `world`'",
escape_shell_arg(r'hello `world`'))
def _task_run(task_run_fnc):
"""Runs the task by fetching arguments from the BibSched task queue.
This is what BibSched will be invoking via daemon call.
The task prints Fibonacci numbers for up to NUM on the stdout, and some
messages on stderr.
@param task_run_fnc: will be called as the main core function. Must return
False in case of errors.
Return True in case of success and False in case of failure."""
from invenio.legacy.bibsched.bibtasklet import _TASKLETS
## We prepare the pid file inside /prefix/var/run/taskname_id.pid
check_running_process_user()
try:
CFG_BIBTASK_RUN_DIR = os.path.join(current_app.instance_path, 'run')
if not os.path.exists(CFG_BIBTASK_RUN_DIR):
os.mkdir(CFG_BIBTASK_RUN_DIR)
pidfile_name = os.path.join(CFG_BIBTASK_RUN_DIR,
'bibsched_task_%d.pid' % _TASK_PARAMS['task_id'])
pidfile = open(pidfile_name, 'w')
pidfile.write(str(os.getpid()))
pidfile.close()
except OSError:
register_exception(alert_admin=True)
task_update_status("ERROR")
return False
## check task status:
task_status = task_read_status()
if task_status not in ("WAITING", "SCHEDULED"):
write_message("Error: The task #%d is %s. I expected WAITING or SCHEDULED." %
(_TASK_PARAMS['task_id'], task_status), sys.stderr)
return False
time_now = datetime.datetime.now()
if _TASK_PARAMS['runtime_limit'] is not None and os.environ.get('BIBSCHED_MODE', 'manual') != 'manual':
if not _TASK_PARAMS['runtime_limit'][0][0] <= time_now <= _TASK_PARAMS['runtime_limit'][0][1]:
if time_now <= _TASK_PARAMS['runtime_limit'][0][0]:
new_runtime = _TASK_PARAMS['runtime_limit'][0][0].strftime("%Y-%m-%d %H:%M:%S")
else:
new_runtime = _TASK_PARAMS['runtime_limit'][1][0].strftime("%Y-%m-%d %H:%M:%S")
progress = run_sql("SELECT progress FROM schTASK WHERE id=%s", (_TASK_PARAMS['task_id'], ))
if progress:
progress = progress[0][0]
else:
progress = ''
g = re.match(r'Postponed (\d+) time\(s\)', progress)
if g:
postponed_times = int(g.group(1))
else:
postponed_times = 0
if _TASK_PARAMS['sequence-id']:
## Also postponing other dependent tasks.
run_sql("UPDATE schTASK SET runtime=%s, progress=%s WHERE sequenceid=%s AND status='WAITING'", (new_runtime, 'Postponed as task %s' % _TASK_PARAMS['task_id'], _TASK_PARAMS['sequence-id'])) # kwalitee: disable=sql
run_sql("UPDATE schTASK SET runtime=%s, status='WAITING', progress=%s, host='' WHERE id=%s", (new_runtime, 'Postponed %d time(s)' % (postponed_times + 1), _TASK_PARAMS['task_id'])) # kwalitee: disable=sql
write_message("Task #%d postponed because outside of runtime limit" % _TASK_PARAMS['task_id'])
return True
# Make sure the host field is updated
# It will not be updated properly when we run
# a task from the cli (without using the bibsched monitor)
host = bibsched_get_host(_TASK_PARAMS['task_id'])
if host and host != gethostname():
write_message("Error: The task #%d is bound to %s." %
(_TASK_PARAMS['task_id'], host), sys.stderr)
return False
else:
bibsched_set_host(_TASK_PARAMS['task_id'], gethostname())
## initialize signal handler:
signal.signal(signal.SIGUSR2, signal.SIG_IGN)
signal.signal(signal.SIGTSTP, _task_sig_sleep)
signal.signal(signal.SIGTERM, _task_sig_stop)
signal.signal(signal.SIGQUIT, _task_sig_stop)
signal.signal(signal.SIGABRT, _task_sig_suicide)
signal.signal(signal.SIGINT, _task_sig_stop)
## we can run the task now:
write_message("Task #%d started." % _TASK_PARAMS['task_id'])
task_update_status("RUNNING")
## run the task:
_TASK_PARAMS['task_starting_time'] = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
sleeptime = _TASK_PARAMS['sleeptime']
try:
try:
if callable(task_run_fnc) and task_run_fnc():
task_update_status("DONE")
else:
task_update_status("DONE WITH ERRORS")
except SystemExit:
pass
except:
write_message(traceback.format_exc()[:-1])
register_exception(alert_admin=True)
if task_get_task_param('stop_queue_on_error'):
task_update_status("ERROR")
else:
task_update_status("CERROR")
finally:
task_status = task_read_status()
if sleeptime:
argv = _task_get_options(_TASK_PARAMS['task_id'], _TASK_PARAMS['task_name'])
verbose_argv = 'Will execute: %s' % ' '.join([escape_shell_arg(str(arg)) for arg in argv])
# Here we check if the task can shift away of has to be run at
# a fixed time
if task_get_task_param('fixed_time') or _TASK_PARAMS['task_name'] in CFG_BIBTASK_FIXEDTIMETASKS:
old_runtime = run_sql("SELECT runtime FROM schTASK WHERE id=%s", (_TASK_PARAMS['task_id'], ))[0][0]
else:
old_runtime = None
new_runtime = get_datetime(sleeptime, now=old_runtime)
## The task is a daemon. We resubmit it
if task_status == 'DONE':
## It has finished in a good way. We recycle the database row
run_sql("UPDATE schTASK SET runtime=%s, status='WAITING', progress=%s, host='' WHERE id=%s", (new_runtime, verbose_argv, _TASK_PARAMS['task_id']))
write_message("Task #%d finished and resubmitted." % _TASK_PARAMS['task_id'])
elif task_status == 'STOPPED':
run_sql("UPDATE schTASK SET status='WAITING', progress=%s, host='' WHERE id=%s", (verbose_argv, _TASK_PARAMS['task_id'], ))
write_message("Task #%d stopped and resubmitted." % _TASK_PARAMS['task_id'])
else:
## We keep the bad result and we resubmit with another id.
#res = run_sql('SELECT proc,user,sleeptime,arguments,priority FROM schTASK WHERE id=%s', (_TASK_PARAMS['task_id'], ))
#proc, user, sleeptime, arguments, priority = res[0]
#run_sql("""INSERT INTO schTASK (proc,user,
#runtime,sleeptime,status,arguments,priority)
#VALUES (%s,%s,%s,%s,'WAITING',%s, %s)""",
#(proc, user, new_runtime, sleeptime, arguments, priority))
write_message("Task #%d finished but not resubmitted. [%s]" % (_TASK_PARAMS['task_id'], task_status))
else:
## we are done:
write_message("Task #%d finished. [%s]" % (_TASK_PARAMS['task_id'], task_status))
## Removing the pid
os.remove(pidfile_name)
#Lets call the post-process tasklets
if task_get_task_param("post-process"):
split = re.compile(r"(bst_.*)\[(.*)\]")
for tasklet in task_get_task_param("post-process"):
if not split.match(tasklet): # wrong syntax
_usage(1, "There is an error in the post processing option "
"for this task.")
aux_tasklet = split.match(tasklet)
_TASKLETS[aux_tasklet.group(1)](**eval("dict(%s)" % (aux_tasklet.group(2))))
return True
def test_escape_simple(self):
"""shellutils - escaping simple strings"""
self.assertEqual("'hello'",
escape_shell_arg("hello"))
def test_escape_backtick(self):
"""shellutils - escaping strings containing backticks"""
self.assertEqual(r"'hello `world`'",
escape_shell_arg(r'hello `world`'))
def test_escape_unix_style_path(self):
"""shellutils - escaping strings containing unix-style file paths"""
self.assertEqual(r"'/tmp/z_temp.txt'",
escape_shell_arg(r'/tmp/z_temp.txt'))
def dump_database(dump_path, host=CFG_DATABASE_HOST, port=CFG_DATABASE_PORT, \
user=CFG_DATABASE_USER, passw=CFG_DATABASE_PASS, \
name=CFG_DATABASE_NAME, params=None, compress=False, \
ignore_tables=None):
"""
Dump Invenio database into SQL file located at DUMP_PATH.
Will perform the command to mysqldump with the given host configuration
and user credentials.
Optional mysqldump parameters can also be passed. Otherwise, a default
set of parameters will be used.
@param dump_path: path on the filesystem to save the dump to.
@type dump_path: string
@param host: hostname of mysql database node to connect to.
@type host: string
@param port: port of mysql database node to connect to
@type port: string
@param user: username to connect with
@type user: string
@param passw: password to connect to with
@type passw: string
@param name: name of mysql database node to dump
@type name: string
@param params: command line parameters to pass to mysqldump. Optional.
@type params: string
@param compress: should the dump be compressed through gzip?
@type compress: bool
@param ignore_tables: list of tables to ignore in the dump
@type ignore: list of string
"""
write_message("... writing %s" % (dump_path,))
partial_dump_path = dump_path + ".part"
# Is mysqldump installed or in the right path?
cmd_prefix = CFG_PATH_MYSQL + 'dump'
if not os.path.exists(cmd_prefix):
raise StandardError("%s is not installed." % (cmd_prefix))
if not params:
# No parameters set, lets use the default ones.
params = " --skip-opt --add-drop-table --add-locks --create-options" \
" --quick --extended-insert --set-charset --disable-keys" \
" --lock-tables=false --max_allowed_packet=2G "
if ignore_tables:
params += " ".join([escape_shell_arg("--ignore-table=%s.%s" % (CFG_DATABASE_NAME, table)) for table in ignore_tables])
dump_cmd = "%s %s " \
" --host=%s --port=%s --user=%s --password=%s %s" % \
(cmd_prefix, \
params, \
escape_shell_arg(host), \
escape_shell_arg(str(port)), \
escape_shell_arg(user), \
escape_shell_arg(passw), \
escape_shell_arg(name))
if compress:
dump_cmd = "%s | %s -cf; exit ${PIPESTATUS[0]}" % \
(dump_cmd, \
CFG_PATH_GZIP)
dump_cmd = "bash -c %s" % (escape_shell_arg(dump_cmd),)
write_message(dump_cmd, verbose=2)
exit_code, stdout, stderr = run_shell_command(dump_cmd, None, partial_dump_path)
if exit_code:
raise StandardError("ERROR: mysqldump exit code is %s. stderr: %s stdout: %s" % \
(repr(exit_code), \
repr(stderr), \
repr(stdout)))
else:
os.rename(partial_dump_path, dump_path)
write_message("... completed writing %s" % (dump_path,))
def build_icon(path_workingdir,
source_filename,
source_filetype,
icon_name,
icon_filetype,
multipage_icon,
multipage_icon_delay,
icon_scale):
"""Whereas create_icon acts as the API for icon creation and therefore
deals with argument washing, temporary working directory creation,
etc, the build_icon function takes care of the actual creation of the
icon file itself by calling various shell tools.
To accomplish this, it relies upon the following parameters:
@param path_workingdir: (string) - the path to the working directory
in which all files related to the icon creation are stored.
@param source_filename: (string) - the filename of the original image
file.
@param source_filetype: (string) - the file type of the original image
file.
@param icon_name: (string) - the name that is to be given to the icon.
@param icon_filetype: (string) - the file type of the icon that is
to be created.
@param multipage_icon: (boolean) - a flag indicating whether or not
an icon with multiple pages (i.e. an animated gif icon) should be
created.
@param multipage_icon_delay: (integer) - the delay to be used between
frame changing for an icon with multiple pages (i.e. an animated gif.)
@param icon_scale: (integer) - the scaling information for the created
icon.
@return: (string) - the name of the created icon file (which will have
been created in the working directory "path_workingdir".)
@Exceptions raised: (InvenioWebSubmitIconCreatorError) - raised when
the icon creation process fails.
"""
##
## If the source file is a PS, convert it into a PDF:
if source_filetype == "ps":
## Convert the subject file from PostScript to PDF:
if source_filename[-3:].lower() == ".ps":
## The name of the file to be stamped has a PostScript extension.
## Strip it and give the name of the PDF file to be created a
## PDF extension:
created_pdfname = "%s.pdf" % source_filename[:-3]
elif len(source_filename.split(".")) > 1:
## The file name has an extension - strip it and add a PDF
## extension:
raw_name = source_filename[:source_filename.rfind(".")]
if raw_name != "":
created_pdfname = "%s.pdf" % raw_name
else:
## It would appear that the file had no extension and that its
## name started with a period. Just use the original name with
## a .pdf suffix:
created_pdfname = "%s.pdf" % source_filename
else:
## No extension - use the original name with a .pdf suffix:
created_pdfname = "%s.pdf" % source_filename
## Build the distilling command:
cmd_distill = """%(distiller)s %(ps-file-path)s """ \
"""%(pdf-file-path)s 2>/dev/null""" % \
{ 'distiller' : CFG_PATH_PS2PDF,
'ps-file-path' : escape_shell_arg("%s/%s" % \
(path_workingdir, \
source_filename)),
'pdf-file-path' : escape_shell_arg("%s/%s" % \
(path_workingdir, \
created_pdfname)),
}
## Distill the PS into a PDF:
errcode_distill = os.system(cmd_distill)
## Test to see whether the PS was distilled into a PDF without error:
if errcode_distill or \
not os.access("%s/%s" % (path_workingdir, created_pdfname), os.F_OK):
## The PDF file was not correctly created in the working directory.
## Unable to continue.
msg = "Error: Unable to correctly convert PostScript file [%s] to" \
" PDF. Cannot create icon." % source_filename
raise InvenioWebSubmitIconCreatorError(msg)
## Now assign the name of the created PDF file to subject_file:
source_filename = created_pdfname
##
## Treat the name of the icon:
if icon_name in (None, ""):
## Since no name has been provided for the icon, give it the same name
## as the source file, but with the prefix "icon-":
icon_name = "icon-%s" % source_filename
## Now if the icon name has an extension, strip it and add that of the
## icon file type:
if len(icon_name.split(".")) > 1:
## The icon file name has an extension - strip it and add the icon
## file type extension:
raw_name = icon_name[:icon_name.rfind(".")]
if raw_name != "":
icon_name = "%s.%s" % (raw_name, icon_filetype)
else:
## It would appear that the file had no extension and that its
## name started with a period. Just use the original name with
## the icon file type's suffix:
icon_name = "%s.%s" % (icon_name, icon_filetype)
else:
## The icon name had no extension. Use the original name with the
## icon file type's suffix:
icon_name = "%s.%s" % (icon_name, icon_filetype)
##
## If the source file type is PS or PDF, it may be necessary to separate
## the first page from the rest of the document and keep it for use as
## the icon. Do this if necessary:
if source_filetype in ("ps", "pdf") and \
(icon_filetype != "gif" or not multipage_icon):
## Either (a) the icon type isn't GIF (in which case it cannot
## be animated and must therefore be created _only_ from the
## document's first page; or (b) the icon type is GIF, but the
## icon is to be created from the first page of the document only.
## The first page of the PDF document must be separated and is to
## be used for icon creation:
source_file_first_page = "p1-%s" % source_filename
## Perform the separation:
cmd_get_first_page = \
"%(pdftk)s A=%(source-file-path)s " \
"cat A1 output %(first-page-path)s " \
"2>/dev/null" \
% { 'pdftk' : CFG_PATH_PDFTK,
'source-file-path' : escape_shell_arg("%s/%s" % \
(path_workingdir, source_filename)),
'first-page-path' : escape_shell_arg("%s/%s" % \
(path_workingdir, \
source_file_first_page)),
}
errcode_get_first_page = os.system(cmd_get_first_page)
## Check that the separation was successful:
if errcode_get_first_page or \
not os.access("%s/%s" % (path_workingdir, \
source_file_first_page), os.F_OK):
## Separation was unsuccessful.
msg = "Error: Unable to create an icon for file [%s/%s] - it " \
"wasn't possible to separate the first page from the " \
"rest of the document (error code [%s].)" \
% (path_workingdir, source_filename, errcode_get_first_page)
raise InvenioWebSubmitIconCreatorError(msg)
else:
## Successfully extracted the first page. Treat it as the source
## file for icon creation from now on:
source_filename = source_file_first_page
##
## Create the icon:
## If a delay is necessary for an animated gif icon, create the
## delay string:
delay_info = ""
if source_filetype in ("ps", "pdf") and \
icon_filetype == "gif" and multipage_icon:
## Include delay information:
delay_info = "-delay %s" % escape_shell_arg(str(multipage_icon_delay))
## Command for icon creation:
cmd_create_icon = "%(convert)s -colorspace rgb -auto-orient -scale %(scale)s %(delay)s " \
"%(source-file-path)s %(icon-file-path)s 2>/dev/null" \
% { 'convert' : CFG_PATH_CONVERT,
'scale' : \
escape_shell_arg(icon_scale),
'delay' : delay_info,
'source-file-path' : \
escape_shell_arg("%s/%s" \
% (path_workingdir, \
source_filename)),
'icon-file-path' : \
escape_shell_arg("%s/%s" \
% (path_workingdir, \
icon_name)),
}
errcode_create_icon = os.system(cmd_create_icon)
## Check that the icon creation was successful:
if errcode_create_icon or \
not os.access("%s/%s" % (path_workingdir, icon_name), os.F_OK):
## Icon creation was unsuccessful.
msg = "Error: Unable to create an icon for file [%s/%s] (error " \
"code [%s].)" \
% (path_workingdir, source_filename, errcode_create_icon)
raise InvenioWebSubmitIconCreatorError(msg)
##
## The icon was successfully created. Return its name:
return icon_name