def __init__(self, dir):
"""Constructor for file-based storage backend.
Takes a path to the base directory in which the files are to be
stored."""
self.dir = dir
self.breakdigits = 3
self.outext = 'txt'
self.errext = 'err'
self.tabext = 'txt'
if not os.path.isdir(self.dir):
raise CrabError('file store error: invalid base directory')
if not os.access(self.dir, os.W_OK):
raise CrabError('file store error: unwritable base directory')
self.outputdir = os.path.join(dir, 'output')
self.tabdir = os.path.join(dir, 'crontab')
for directory in [self.outputdir, self.tabdir]:
if not os.path.exists(directory):
try:
os.mkdir(directory)
except OSError as err:
if err.errno != errno.EEXIST:
raise CrabError('file store error: '
'could not make directory ' +
directory + ': ' + str(err))
def _read_json(self, url):
"""Performs an HTTP GET on the given URL and interprets the
response as JSON."""
try:
try:
conn = self._get_conn()
conn.request('GET', url)
res = conn.getresponse()
if res.status != 200:
raise CrabError('server error: ' + self._read_error(res))
return json.loads(latin_1_decode(res.read(), 'replace')[0])
# except HTTPException as err:
except HTTPException:
err = sys.exc_info()[1]
raise CrabError('HTTP error: ' + str(err))
# except socket.error as err:
except socket.error:
err = sys.exc_info()[1]
raise CrabError('socket error: ' + str(err))
# except ValueError as err:
except ValueError:
err = sys.exc_info()[1]
raise CrabError('did not understand response: ' + str(err))
finally:
conn.close()
def __init__(self, specifier, timezone):
"""Construct a CrabSchedule object from a cron time specifier
and the associated timezone name.
The timezone string, if provided, is converted into an object
using the pytz module."""
try:
item = CronTab.__init__(self, specifier)
except ValueError as err:
raise CrabError('Failed to parse cron time specifier ' +
specifier + ' reason: ' + str(err))
self.timezone = None
if timezone is not None:
try:
# pytz returns the same object if called twice
# with the same timezone, so we don't need to cache
# the timezone objects by zone name.
self.timezone = pytz.timezone(timezone)
except pytz.UnknownTimeZoneError:
logger.warning(
'Warning: unknown time zone {}'.format(timezone))
def write_notification(self, notifyid, configid, host, user, method,
address, time, timezone, skip_ok, skip_warning,
skip_error, include_output):
"""Adds or updates a notification record in the database."""
if configid is not None and ((host is not None) or (user is not None)):
raise CrabError('writing notification: job config and match '
'parameters both specified')
with self.lock as c:
if notifyid is None:
c.execute(
'INSERT INTO jobnotify (configid, host, user, '
'method, address, time, timezone, skip_ok, '
'skip_warning, skip_error, include_output) '
'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [
configid, host, user, method, address, time, timezone,
skip_ok, skip_warning, skip_error, include_output
])
else:
c.execute(
'UPDATE jobnotify SET configid=?, host=?, '
'user=?, method=?, address=?, time=?, '
'timezone=?, skip_ok=?, skip_warning=?, '
'skip_error=?, include_output=? '
'WHERE id=?', [
configid, host, user, method, address, time, timezone,
skip_ok, skip_warning, skip_error, include_output,
notifyid
])
def log_start(self, host, user, crabid, command):
"""Inserts a job start record into the database.
Returns a dictionary including a boolean value indicating
whether the job inhibit setting is active or not."""
data = {'inhibit': False}
with self.lock:
try:
id_ = self._check_job(host, user, crabid, command)
with closing(self.conn.cursor()) as c:
c.execute('INSERT INTO jobstart (jobid, command) '
'VALUES (?, ?)',
[id_, command])
# Read the job configuration in order to determine whether
# this job is currently inhibited.
config = self._get_job_config(id_)
if config is not None and config['inhibit']:
data['inhibit'] = True
except DatabaseError as err:
raise CrabError('database error: ' + str(err))
return data
def _query_to_dict_list(self, sql, param=[]):
"""Execute an SQL query and return the result as a list of
Python dict objects.
The dict keys are retrieved from the SQL result using the
description method of the DB cursor object."""
c = self.conn.cursor()
output = []
try:
c.execute(sql, param)
while True:
row = c.fetchone()
if row is None:
break
dict = {}
for (i, coldescription) in enumerate(c.description):
dict[coldescription[0]] = row[i]
output.append(dict)
except DatabaseError as err:
raise CrabError('database error: ' + str(err))
finally:
c.close()
return output
def crontab(self, host, user, raw=False):
"""CherryPy handler for the crontab action.
Allows the client to PUT a new crontab, or use a GET
request to see a crontab-style representation of the
job information held in the the storage backend."""
if cherrypy.request.method == 'GET':
try:
if raw:
crontab = self.store.get_raw_crontab(host, user)
else:
crontab = self.store.get_crontab(host, user)
return json.dumps({'crontab': crontab})
except CrabError as err:
cherrypy.log.error('CrabError: read error: ' + str(err))
raise HTTPError(message='read error: ' + str(err))
elif cherrypy.request.method == 'PUT':
try:
data = self._read_json()
crontab = data.get('crontab')
if crontab is None:
raise CrabError('no crontab received')
warning = self.store.save_crontab(
host, user, crontab, timezone=data.get('timezone'))
return json.dumps({'warning': warning})
except CrabError as err:
cherrypy.log.error('CrabError: write error: ' + str(err))
raise HTTPError(message='write error: ' + str(err))
def _update_job(self, id_,
crabid=None, command=None, time=None, timezone=None):
"""Marks a job as not deleted, and updates its information.
Only fields not given as None are updated."""
fields = ['installed=CURRENT_TIMESTAMP', 'deleted=NULL']
params = []
if crabid is not None:
fields.append('crabid=?')
params.append(crabid)
if command is not None:
fields.append('command=?')
params.append(command)
if time is not None:
fields.append('time=?')
params.append(time)
if timezone is not None:
fields.append('timezone=?')
params.append(timezone)
params.append(id_)
with closing(self.conn.cursor()) as c:
try:
c.execute('UPDATE job SET ' + ', '.join(fields) + ' '
'WHERE id=?', params)
except DatabaseError as err:
raise CrabError('database error: ' + str(err))
def write_raw_crontab(self, host, user, crontab):
if self.outputstore is not None and hasattr(self.outputstore,
'write_raw_crontab'):
return self.outputstore.write_raw_crontab(host, user, crontab)
with self.lock:
entry = self._query_to_dict('SELECT id FROM rawcrontab ' +
'WHERE host = ? AND user = ?',
[host, user])
c = self.conn.cursor()
try:
if entry is None:
c.execute('INSERT INTO rawcrontab (host, user, crontab) '
'VALUES (?, ?, ?)',
[host, user, '\n'.join(crontab)])
else:
c.execute('UPDATE rawcrontab SET crontab = ? WHERE id = ?',
['\n'.join(crontab), entry['id']])
except DatabaseError as err:
raise CrabError('database error: ' + str(err))
finally:
c.close()
def write_job_output(self, finishid, host, user, id_, crabid,
stdout, stderr):
"""Writes the job output to the database.
This method does not require the host, user, or job ID
number, but will pass them to the outputstore's corresponding
method if it is defined rather than performing this action
with the database."""
if self.outputstore is not None:
return self.outputstore.write_job_output(finishid, host, user,
id_, crabid, stdout, stderr)
with self.lock:
c = self.conn.cursor()
try:
c.execute('INSERT INTO joboutput (finishid, stdout, stderr) ' +
'VALUES (?, ?, ?)',
[finishid, stdout, stderr])
except DatabaseError as err:
raise CrabError('database error: ' + str(err))
finally:
c.close()
def get_job_output(self, finishid, host, user, id_, crabid):
"""Fetches the standard output and standard error for the
given finish ID.
The result is returned as a two element list. The parameters
host, user and id number are passed on to the outputstore's
get_job_output method if an outputstore was provided to the
contructor, allowing the outputstore to organise its
information hierarchically if desired. Otherwise this method
does not make use of those parameters. Returns a pair of empty
strings if no output is found."""
if self.outputstore is not None:
return self.outputstore.get_job_output(finishid, host, user,
id_, crabid)
with self.lock:
c = self.conn.cursor()
try:
c.execute('SELECT stdout, stderr FROM joboutput ' +
'WHERE finishid=?', [finishid])
row = c.fetchone()
if row is None:
return ('', '')
return row
except DatabaseError as err:
raise CrabError('database error: ' + str(err))
finally:
c.close()
def _write_json(self, url, obj, read=False):
"""Converts the given object to JSON and sends it with an
HTTP PUT to the given URL.
Optionally attempts to read JSON from the response."""
try:
try:
conn = self._get_conn()
conn.request('PUT', url, json.dumps(obj))
res = conn.getresponse()
if res.status != 200:
raise CrabError('server error: ' + self._read_error(res))
if read:
response = latin_1_decode(res.read(), 'replace')[0]
# Check we got a response before attempting to decode
# it as JSON. (Some messages did not have responses
# for previous server versions.)
if response:
return json.loads(response)
else:
return {}
#except HTTPException as err:
#except HTTPException, err:
except HTTPException:
err = sys.exc_info()[1]
raise CrabError('HTTP error: ' + str(err))
#except socket.error as err:
#except socket.error, err:
except socket.error:
err = sys.exc_info()[1]
raise CrabError('socket error: ' + str(err))
#except ValueError as err:
#except ValueError, err:
except ValueError:
err = sys.exc_info()[1]
raise CrabError('did not understand response: ' + str(err))
finally:
conn.close()
def write_job_config(self, id_, graceperiod=None, timeout=None,
success_pattern=None, warning_pattern=None, fail_pattern=None,
note=None, inhibit=False):
"""Writes configuration data for a job by ID number.
Returns the configuration ID number."""
with self.lock:
row = self._query_to_dict('SELECT id as configid '
'FROM jobconfig '
'WHERE jobid = ?', [id_])
c = self.conn.cursor()
try:
if row is None:
c.execute('INSERT INTO jobconfig (jobid, graceperiod, '
'timeout, success_pattern, warning_pattern, '
'fail_pattern, note, inhibit) '
'VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
[id_, graceperiod, timeout,
success_pattern, warning_pattern, fail_pattern,
note, inhibit])
return c.lastrowid
else:
configid = row['configid']
if configid is None:
raise CrabError('job config: got null id')
c.execute('UPDATE jobconfig SET graceperiod=?, timeout=?, '
'success_pattern=?, warning_pattern=?, '
'fail_pattern=?, note=?, inhibit=? '
'WHERE id=?',
[graceperiod, timeout,
success_pattern, warning_pattern, fail_pattern,
note, inhibit, configid])
return configid
except DatabaseError as err:
raise CrabError('database error: ' + str(err))
finally:
c.close()
def write_job_output(self, finishid, host, user, id_, crabid, stdout,
stderr):
"""Write the cron job output to a file.
The only parameter required to uniquely identify the event
associated with this output is the "finishid", but the
host, user and job identifiers are also provided to allow
hierarchical storage.
Only writes a stdout file (extension set by self.outext, by default
txt), and a stderr file (extension self.errext, default err)
if they are not empty."""
path = self._make_output_path(finishid, host, user, id_, crabid)
(dir, file) = os.path.split(path)
if not os.path.exists(dir):
try:
os.makedirs(dir)
except OSError as err:
if err.errno != errno.EEXIST:
raise CrabError(
'file store error: could not make directory: ' +
str(err))
outfile = path + '.' + self.outext
errfile = path + '.' + self.errext
if os.path.exists(outfile) or os.path.exists(errfile):
raise CrabError('file store error: file already exists: ' + path)
try:
if stdout:
with open(outfile, 'w') as file:
file.write(stdout)
if stderr:
with open(errfile, 'w') as file:
file.write(stderr)
except IOError as err:
raise CrabError('file store error: could not write files: ' +
str(err))
def delete_notification(self, notifyid):
"""Removes a notification from the database."""
with self.lock:
with closing(self.conn.cursor()) as c:
try:
c.execute('DELETE FROM jobnotify WHERE id=?', [notifyid])
except DatabaseError as err:
raise CrabError('database error: ' + str(err))
def finish(self, host, user, crabid=None):
"""CherryPy handler allowing clients to report jobs finishing."""
try:
data = self._read_json()
command = data.get('command')
status = data.get('status')
if command is None or status is None:
raise CrabError('insufficient information to log finish')
if status not in CrabStatus.VALUES:
raise CrabError('invalid finish status')
self.store.log_finish(host, user, crabid, command, status,
data.get('stdout'), data.get('stderr'))
except CrabError as err:
cherrypy.log.error('CrabError: log error: ' + str(err))
raise HTTPError(message='log error: ' + str(err))
def __exit__(self, type_, value, tb):
new_exception = None
# Use try-finally block to ensure we release the lock whatever
# happens (for extra safety -- the try-except blocks should
# allow us to keep going anyway).
try:
# Close and delete cursor.
try:
self.cursor.close()
del self.cursor
except Exception as err:
new_exception = CrabError('database error (closing cursor): ' +
str(err))
# Commit the transaction, or roll back if an exception occurred
# during the transaction.
try:
if type_ is None:
self.conn.commit()
else:
self.conn.rollback()
except Exception as err:
new_exception = CrabError(
'database error (ending transaction): ' + str(err))
finally:
self.lock.release()
# If an exception happened during the transaction, raise if it was
# database error, otherwise leave it alone (do nothing). If there
# wasn't an exception, but we have a new one, raise it.
if type_ is not None:
if issubclass(type_, self.error_class):
raise CrabError('database error: ' + str(value))
elif new_exception is not None:
raise new_exception
def relink_job_config(self, configid, id_):
with self.lock:
c = self.conn.cursor()
try:
c.execute('UPDATE jobconfig SET jobid = ? '
'WHERE id = ?', [id_, configid])
except DatabseError as err:
raise CrabError('database error: ' + str(err))
finally:
c.close()
def _get_jobs(self,
c,
host,
user,
include_deleted=False,
crabid=None,
command=None,
without_crabid=False):
"""Private/protected version of get_jobs which does not
acquire the lock, and takes more search parameters."""
params = []
conditions = []
if not include_deleted:
conditions.append('deleted IS NULL')
if host is not None:
conditions.append('host=?')
params.append(host)
if user is not None:
conditions.append('user=?')
params.append(user)
if crabid is not None:
conditions.append('crabid=?')
params.append(crabid)
if command is not None:
conditions.append('command=?')
params.append(command)
if without_crabid:
if crabid is not None:
raise CrabError(
'_get_jobs called with crabid and without_crabid')
conditions.append('crabid IS NULL')
if conditions:
where_clause = 'WHERE ' + ' AND '.join(conditions)
else:
where_clause = ''
return self._query_to_dict_list(
c, 'SELECT id, host, user, crabid, command, time, timezone, '
'installed AS "installed [timestamp]", '
'deleted AS "deleted [timestamp]" '
'FROM job ' + where_clause + ' '
'ORDER BY host ASC, user ASC, crabid ASC, installed ASC', params)
def write_raw_crontab(self, host, user, crontab):
"""Writes the given crontab to a file."""
pathname = self._make_crontab_path(host, user)
(dir, file) = os.path.split(pathname)
if not os.path.exists(dir):
try:
os.makedirs(dir)
except OSError as err:
if err.errno != errno.EEXIST:
raise CrabError(
'file store error: could not make directory: ' +
str(err))
try:
with open(pathname, 'w') as file:
file.write('\n'.join(crontab))
except IOError as err:
raise CrabError('file store error: could not write crontab: ' +
str(err))
def _delete_job(self, id_):
"""Marks a job as deleted in the database."""
c = self.conn.cursor()
try:
c.execute('UPDATE job SET deleted=CURRENT_TIMESTAMP ' +
'WHERE id=?',
[id_])
except DatabaseError as err:
raise CrabError('database error: ' + str(err))
finally:
c.close()
def get_raw_crontab(self, host, user):
"""Reads the given user's crontab from a file."""
pathname = self._make_crontab_path(host, user)
if not os.path.exists(pathname):
return None
try:
with open(pathname) as file:
crontab = file.read()
except IOError as err:
raise CrabError('file store error: could not read crontab: ' +
str(err))
return crontab.split('\n')
def start(self, host, user, crabid=None):
"""CherryPy handler allowing clients to report jobs starting."""
try:
data = self._read_json()
command = data.get('command')
if command is None:
raise CrabError('cron command not specified')
data = self.store.log_start(host, user, crabid, command)
return json.dumps({'inhibit': data['inhibit']})
except CrabError as err:
cherrypy.log.error('CrabError: log error: ' + str(err))
raise HTTPError(message='log error: ' + str(err))
def _insert_job(self, host, user, crabid, time, command, timezone):
"""Inserts a job record into the database."""
c = self.conn.cursor()
try:
c.execute('INSERT INTO job (host, user, crabid, ' +
'time, command, timezone)' +
'VALUES (?, ?, ?, ?, ?, ?)',
[host, user, crabid, time, command, timezone])
return c.lastrowid
except DatabaseError as err:
raise CrabError('database error: ' + str(err))
finally:
c.close()
def get_job_output(self, finishid, host, user, id_, crabid):
"""Find the file containing the cron job output and read it.
As for write_job_output, only the "finishid" is logically required,
but this method makes use of the host, user and job identifiers
to read from a directory hierarchy.
Requires there to be an stdout file but allows the
stderr file to be absent."""
path = self._make_output_path(finishid, host, user, id_, crabid)
outfile = path + '.' + self.outext
errfile = path + '.' + self.errext
if not (os.path.exists(outfile) or os.path.exists(errfile)):
if crabid is not None:
# Try again with no crabid. This is to handle the case where
# a job is imported with no name, but is subsequently named.
path = self._make_output_path(finishid, host, user, id_, None)
outfile = path + '.' + self.outext
errfile = path + '.' + self.errext
else:
# Return now just to avoid testing the same files again.
return ('', '')
try:
if os.path.exists(outfile):
with open(outfile) as file:
stdout = file.read()
else:
stdout = ''
if os.path.exists(errfile):
with open(errfile) as file:
stderr = file.read()
else:
stderr = ''
except IOError as err:
raise CrabError('file store error: could not read files: ' +
str(err))
return (stdout, stderr)
def log_finish(self, host, user, crabid, command, status,
stdout=None, stderr=None):
"""Inserts a job finish record into the database.
The output will be passed to the write_job_output method,
unless both stdout and stderr are empty."""
with self.lock:
c = self.conn.cursor()
try:
id_ = self._check_job(host, user, crabid, command)
# Fetch the configuration so that we can check the status.
config = self._get_job_config(id_)
if config is not None:
status = check_status_patterns(
status, config,
'\n'.join((x for x in (stdout, stderr)
if x is not None)))
c.execute('INSERT INTO jobfinish (jobid, command, status) ' +
'VALUES (?, ?, ?)',
[id_, command, status])
finishid = c.lastrowid
except DatabaseError as err:
raise CrabError('database error: ' + str(err))
finally:
c.close()
if stdout or stderr:
# If a crabid was not specified, check whether the job
# actually has one. This is to avoid sending misleading
# parameters to write_job_output, which can cause the
# file-based output store to default to a numeric directory name.
if crabid is None:
info = self.get_job_info(id_)
crabid = info['crabid']
self.write_job_output(finishid, host, user, id_, crabid,
stdout, stderr)
def log_alarm(self, id_, status):
"""Inserts an alarm regarding a job into the database.
This is for alarms generated interally by crab, for example
from the monitor thread. Such alarms are currently stored
in an separate table and do not have any associated output
records."""
with self.lock:
c = self.conn.cursor()
try:
c.execute('INSERT INTO jobalarm (jobid, status) VALUES (?, ?)',
[id_, status])
except DatabaseError as err:
raise CrabError('database error: ' + str(err))
finally:
c.close()
def __enter__(self):
self.lock.acquire(True)
# Open a cursor, but be sure to release the lock again if this
# fails.
try:
if self.ping:
self.conn.ping(reconnect=True, attempts=2, delay=5)
self.cursor = self.conn.cursor(**self.cursor_args)
except self.error_class as err:
self.lock.release()
raise CrabError('database error (opening cursor): ' + str(err))
except:
self.lock.release()
raise
return self.cursor
def disable_inhibit(self, id_):
"""Disable the inhibit setting for a job.
This is a convenience routine to simply disable the inhibit
job configuration parameter without having to read and write
the rest of the configuration.
"""
with self.lock:
c = self.conn.cursor()
try:
c.execute('UPDATE jobconfig SET inhibit=0 WHERE jobid=?',
[id_])
except DatabaseError as err:
raise CrabError('database error: ' + str(err))
finally:
c.close()