def _createlog(self): """ Create the logfile and the link to the logfile (if requested). """ if self.toemail and self.fromemail and self.smtphost: # Use the email logger as the first logger, so that when sending the email (in :meth:`EmailLogger.close`) fails, it will still be logged to the log file/stdout/stderr self._loggers.append(EmailLogger(self)) if self.log2stderr: self._loggers.append(StreamLogger(self, sys.stderr, self._formatlogline)) if self.log2stdout: self._loggers.append(StreamLogger(self, sys.stdout, self._formatlogline)) if self.log2file: # Create the log file logfilename = ul4c.Template(self.logfilename, "logfilename").renders(job=self) logfilename = url.File(logfilename).abs() self.logfileurl = str(url.Ssh(misc.sysinfo.user_name, misc.sysinfo.host_fqdn or misc.sysinfo.host_name, logfilename.local())) skipurls = [logfilename] logfile = logfilename.open(mode="w", encoding=self.encoding, errors=self.errors) if self.loglinkname is not None: # Create the log link loglinkname = ul4c.Template(self.loglinkname, "loglinkname").renders(job=self) loglinkname = url.File(loglinkname).abs() skipurls.append(loglinkname) logfilename = logfilename.relative(loglinkname) try: logfilename.symlink(loglinkname) except OSError as exc: if exc.errno == errno.EEXIST: loglinkname.remove() logfilename.symlink(loglinkname) else: raise self._loggers.append(URLResourceLogger(self, logfile, skipurls, self._formatlogline))
def from_args(self, args):
"""
Sets the attributes of :obj:`self` from the object :obj:`args` (which
must be an instance of :class:`argparse.Namespace`).
Returns the main template.
"""
templates = {}
maintemplate = None
for templatename in args.templates:
if templatename == "-":
templatesource = sys.stdin.read()
templatename = "stdin"
else:
with open(templatename, "r", encoding=args.encoding) as f:
templatesource = f.read()
templatename = os.path.basename(templatename)
if os.path.extsep in templatename:
templatename = templatename.rpartition(os.extsep)[0]
templatename = fixname(templatename)
if args.stacktrace == "short":
try:
template = ul4c.Template(templatesource,
name=templatename,
whitespace=args.whitespace)
except Exception as exc:
print_exception_chain(exc)
raise SystemExit(1)
else:
template = ul4c.Template(templatesource,
name=templatename,
whitespace=args.whitespace)
# The first template is the main template
if maintemplate is None:
maintemplate = template
templates[template.name] = template
self.templates = templates
self.vars = dict(args.vars) if args.vars is not None else {}
def option(name):
if getattr(args, name):
if getattr(self, name) is None:
delattr(self, name)
else:
if getattr(self, name) is not None:
setattr(self, name, None)
option("oracle")
option("mysql")
option("sqlite")
option("redis")
option("system")
option("load")
option("save")
option("compile")
return maintemplate
def test_oracle_execute_procedure_out(globals):
if globals:
template = ul4c.Template("""
<?code db = globals.oracle(globals.vars.connectstring)?>
<?code db.execute('''
create or replace procedure ul4test(p_intarg out integer, p_numberarg out number, p_strarg out varchar2, p_clobarg out clob, p_datearg out timestamp)
as
begin
p_intarg := 42;
p_numberarg := 42.5;
p_strarg := 'foo';
dbms_lob.createtemporary(p_clobarg, true);
for i in 1..100000 loop
dbms_lob.writeappend(p_clobarg, 3, 'foo');
end loop;
p_datearg := to_date('05.10.2014 16:17:18', 'DD.MM.YYYY HH24:MI:SS');
end;
''')?>
<?code vint = db.int()?>
<?code vnumber = db.number()?>
<?code vstr = db.str()?>
<?code vclob = db.clob()?>
<?code vdate = db.date()?>
<?code db.execute('call ul4test(', vint, ', ', vnumber, ', ', vstr, ', ', vclob, ', ', vdate, ')')?>
<?print vint.value?>|<?print vnumber.value?>|<?print vstr.value?>|<?print vclob.value?>|<?print vdate.value?>
<?code db.execute('drop procedure ul4test')?>
""",
whitespace="strip")
assert template.renders(
globals=globals) == "42|42.5|foo|{}|2014-10-05 16:17:18".format(
100000 * "foo")
def test_oracle_query(globals):
if globals:
template = ul4c.Template("""
<?code db = globals.oracle(globals.vars.connectstring)?>
<?code db.execute('''
create table ul4test(
ul4_int integer,
ul4_char varchar2(1000),
ul4_clob clob)
''')?>
<?code db.execute('insert into ul4test values(1, ', 'first', ', ', 10000*'first', ')')?>
<?code db.execute('insert into ul4test values(2, ', 'second', ', ', 10000*'second', ')')?>
<?code db.execute('insert into ul4test values(3, ', 'third', ', ', 10000*'third', ')')?>
<?code vin = db.int(2)?>
<?for row in db.query('select * from ul4test where ul4_int <= ', vin, ' order by ul4_int')?>
<?print row.ul4_int?>|
<?print row.ul4_char?>|
<?print row.ul4_clob?>|
<?end for?>
<?code db.execute('drop table ul4test')?>
""",
whitespace="strip")
assert template.renders(
globals=globals) == "1|first|{}|2|second|{}|".format(
10000 * "first", 10000 * "second")
def compile(self,
source,
name=None,
whitespace="keep",
signature=None,
startdelim="<?",
enddelim="?>"):
"""
Compile the UL4 source ``source`` into a :class:`~ll.ul4c.Template` object
and return it. All other parameters are passed to the
:class:`~ll.ul4c.Template` constructor too.
"""
return ul4c.Template(source,
name=name,
whitespace=whitespace,
signature=signature,
startdelim=startdelim,
enddelim=enddelim)
def test_oracle_execute_function(globals):
if globals:
template = ul4c.Template("""
<?code db = globals.oracle(globals.vars.connectstring)?>
<?code db.execute('''
create or replace function ul4test(p_arg integer)
return integer
as
begin
return 2*p_arg;
end;
''')?>
<?code vin = db.int(42)?>
<?code vout = db.int()?>
<?code db.execute('begin ', vout, ' := ul4test(', vin, '); end;')?>
<?print vout.value?>
<?code db.execute('drop function ul4test')?>
""",
whitespace="strip")
assert template.renders(globals=globals) == "84"
def test_template(t):
template = ul4c.Template("<?for i in range(10)?>(<?print i?>)<?end for?>")
assert template.renders() == t(template).renders()
def _handleexecution(self):
"""
Handle executing the job including handling of duplicate or hanging jobs.
"""
if self.jobname is None:
self.jobname = self.__class__.__qualname__
self._tasks = []
self._loggers = []
self._exceptioncount = 0
self._originalproctitle = setproctitle.getproctitle() if self.setproctitle and setproctitle else None
# Obtain a lock on the script file to make sure we're the only one running
with open(misc.sysinfo.script_name, "rb") as f:
if fcntl is not None:
try:
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError as exc:
if exc.errno != errno.EWOULDBLOCK: # some other error
raise
# The previous invocation of the job is still running
return # Return without calling :meth:`execute`
# we were able to obtain the lock, so we are the only one running
self.starttime = datetime.datetime.now()
self.endtime = None
self._getscriptsource() # Get source code
self._getcrontab() # Get crontab
self.log = Tag(self._log) # Create tagged logger for files
self._formatlogline = ul4c.Template(self.formatlogline, "formatlogline", whitespace="strip") # Log line formatting template
self._formatemailsubject = ul4c.Template(self.formatemailsubject, "formatemailsubject", whitespace="strip") # Email subject formatting template
self._formatemailbodytext = ul4c.Template(self.formatemailbodytext, "formatemailbodytext", whitespace="strip") # Email body formatting template (plain text)
self._formatemailbodyhtml = ul4c.Template(self.formatemailbodyhtml, "formatemailbodyhtml", whitespace="strip") # Email body formatting template (HTML)
self._createlog() # Create loggers
maxtime = self.getmaxtime()
self.log.sisyphus.init(f"{misc.sysinfo.script_name} (max time {maxtime}; pid {misc.sysinfo.pid})")
# Check for support of various thing we'd like to use
if fcntl is None:
self.log.sisyphus.init.warning("Can't lock script file (module fcntl not available)")
if self.fork and not hasattr(os, "fork"):
self.log.sisyphus.init.warning("Can't fork (function os.fork not available)")
self.fork = False
if not hasattr(signal, "SIGALRM"):
self.log.sisyphus.init.warning("Can't use signals (signal.SIGALRM not available)")
self.fork = False
if self.setproctitle and setproctitle is None:
self.log.sisyphus.init.warning("Can't set process title (module setproctitle not available)")
if self.fork: # Forking mode?
# Fork the process; the child will do the work; the parent will monitor the maximum runtime
self.killpid = pid = os.fork()
if pid: # We are the parent process
msg = f"logging to {self.logfileurl}" if self.logfileurl else "no logging"
self.setproctitle("parent", f"{msg} (max time {maxtime})")
# set a signal to kill the child process after the maximum runtime
if hasattr(signal, "SIGALRM"):
signal.signal(signal.SIGALRM, self._alarm_fork)
signal.alarm(self.getmaxtime_seconds())
try:
os.waitpid(pid, 0) # Wait for the child process to terminate
except BaseException as exc:
pass
return # Exit normally
# Here we are in the child process
self.setproctitle("child")
self.log.sisyphus.init(f"forked worker child (child pid {os.getpid()})")
else: # We didn't fork
# set a signal to kill ourselves after the maximum runtime
if hasattr(signal, "SIGALRM"):
signal.signal(signal.SIGALRM, self._alarm_nofork)
signal.alarm(self.getmaxtime_seconds())
self.setproctitle("child", "Setting up")
self.notifystart()
result = None
try:
with url.Context():
self.setproctitle("child", "Working")
result = self.execute()
except Exception as exc:
self.endtime = datetime.datetime.now()
self.setproctitle("child", "Handling exception")
result = f"failed with {misc.format_exception(exc)}"
# log the error to the logfile, because the job probably didn't have a chance to do it
self.log.sisyphus.email(exc)
self.log.sisyphus.result.fail(result)
self.failed()
# Make sure that exceptions at the top level get propagated if they are not reported by email
if not (self.toemail and self.fromemail and self.smtphost):
raise
else:
self.endtime = datetime.datetime.now()
self.setproctitle("child", "Finishing")
# log the result
if self._exceptioncount:
self.log.sisyphus.result.errors(result)
else:
self.log.sisyphus.result.ok(result)
finally:
self.setproctitle("child", "Cleaning up logs")
for logger in self._loggers:
logger.close()
self.notifyfinish(result)
if fcntl is not None:
fcntl.flock(f, fcntl.LOCK_UN | fcntl.LOCK_NB)
if self.fork:
os._exit(0)