def get_html_report(cfg_file=None, cfg=None):
"""
Format a report in HTML
"""
rval = ""
if cfg is not None:
# use it
pass
elif cfg_file is not None:
cfg = CrawlConfig.add_config(filename=cfg_file)
else:
cfg = CrawlConfig.add_config()
db = CrawlDBI.DBI(dbtype="crawler")
last_rpt_time = rpt_lib.get_last_rpt_time(db)
rval += ('<head><meta http-equiv="refresh" content="60">\n')
rval += ("<title>HPSSIC Dashboard</title></head>")
rval += ("<body><center><h1>HPSS Integrity Crawler Dashboard</h1>" +
"<br><h4>Version %s</h4>" % version.__version__ +
"</center>\n")
rval += ("Report generated at %s\n" % time.strftime("%Y.%m%d %H:%M:%S"))
rval += ("<br>Based on data from %s\n" %
time.strftime("%Y.%m%d %H:%M:%S", time.localtime(last_rpt_time)))
rval += get_html_cv_report(db, last_rpt_time)
rval += get_html_mpra_report(db, last_rpt_time)
rval += get_html_tcc_report(db, last_rpt_time)
rval += "</body>"
db.close()
return rval
def cvv_show_next(argv):
"""show_next - Report the Checkables in the order they will be checked
usage: cvtool show_next
"""
p = optparse.OptionParser()
p.add_option('-c', '--config',
action='store', default='', dest='config',
help='alternate configuration')
p.add_option('-d', '--debug',
action='store_true', default=False, dest='debug',
help='run the debugger')
p.add_option('-i', '--id',
action='store', default='', dest='id',
help='id of entry to be checked')
p.add_option('-l', '--limit',
action='store', default=-1, dest='limit', type=int,
help='max records to get')
p.add_option('-p', '--path',
action='store', default='', dest='path',
help='name of path to be checked')
p.add_option('-v', '--verbose',
action='store_true', default=False, dest='verbose',
help='more information')
try:
(o, a) = p.parse_args(argv)
except SystemExit:
return
if o.debug:
pdb.set_trace()
if o.config:
cfg = CrawlConfig.add_config(close=True, filename=o.config)
else:
cfg = CrawlConfig.add_config()
if o.limit < 0:
limit = int(cfg.get_d('cv', 'operations', '10'))
else:
limit = o.limit
clist = Checkable.Checkable.get_list(limit)
for c in clist:
if c.last_check == 0:
print("%18d %s %s" % (c.last_check,
c.type,
c.path))
else:
print("%s %s %s" % (U.ymdhms(c.last_check),
c.type,
c.path))
def running_pid(proc_required=True, context=None):
"""
Return a list of pids if the crawler is running (per ps(1)) or [] otherwise
"""
cfg = CrawlConfig.add_config()
rval = []
if proc_required:
result = pidcmd()
for line in result.split("\n"):
if 'crawl start' in line:
pid = int(line.split()[0])
pfpath = "%s/%d" % (CrawlConfig.pid_dir(), pid)
if os.path.exists(pfpath):
(ctx, xpath) = util.contents(pfpath).strip().split()
rval.append((pid, ctx, xpath))
elif not os.path.exists(pfpath + '.DEFUNCT'):
# crawler is running but the pid file has been lost
ctx = context or cfg.get('crawler', 'context')
xpath = cfg.get_d('crawler', 'exitpath', '%s.exit' % ctx)
make_pidfile(pid, ctx, xpath)
rval.append((pid, ctx, xpath))
# if pfpath + '.DEFUNCT' exists, the crawler is shutting down
# so we don't want to recreate the pid file.
else:
pid_l = glob.glob("%s/*" % CrawlConfig.pid_dir())
for pid_n in pid_l:
pid = int(os.path.basename(pid_n))
(ctx, xpath) = util.contents(pid_n).strip().split()
rval.append((pid, ctx, xpath))
return rval
def running_pid(proc_required=True, context=None):
"""
Return a list of pids if the crawler is running (per ps(1)) or [] otherwise
"""
cfg = CrawlConfig.add_config()
rval = []
if proc_required:
result = pidcmd()
for line in result.split("\n"):
if 'crawl start' in line:
pid = int(line.split()[0])
pfpath = "%s/%d" % (CrawlConfig.pid_dir(), pid)
if os.path.exists(pfpath):
(ctx, xpath) = util.contents(pfpath).strip().split()
rval.append((pid, ctx, xpath))
elif not os.path.exists(pfpath + '.DEFUNCT'):
# crawler is running but the pid file has been lost
ctx = context or cfg.get('crawler', 'context')
xpath = cfg.get_d('crawler', 'exitpath', '%s.exit' % ctx)
make_pidfile(pid, ctx, xpath)
rval.append((pid, ctx, xpath))
# if pfpath + '.DEFUNCT' exists, the crawler is shutting down
# so we don't want to recreate the pid file.
else:
pid_l = glob.glob("%s/*" % CrawlConfig.pid_dir())
for pid_n in pid_l:
pid = int(os.path.basename(pid_n))
(ctx, xpath) = util.contents(pid_n).strip().split()
rval.append((pid, ctx, xpath))
return rval
def clean_defunct_pidfiles(context):
"""
Remove .DEFUNCT pid files for *context*
"""
cfg = CrawlConfig.add_config()
pdir = CrawlConfig.pid_dir()
for path in glob.glob(os.path.join(pdir, '*.DEFUNCT')):
c = util.contents(path)
if context in c:
os.unlink(path)
def clean_defunct_pidfiles(context):
"""
Remove .DEFUNCT pid files for *context*
"""
cfg = CrawlConfig.add_config()
pdir = CrawlConfig.pid_dir()
for path in glob.glob(os.path.join(pdir, '*.DEFUNCT')):
c = util.contents(path)
if context in c:
os.unlink(path)
def load_recheck_list(cls, how_many):
"""
Look to see whether any of the already checksummed items in the
database have a last check time over the threshold for rechecking. If
so, we'll shove some of them to the front of the list based on the
configuration.
"""
cfg = CrawlConfig.add_config()
r_fraction = float(cfg.get_d('cv', 'recheck_fraction', '0.0'))
r_age = cfg.get_time('cv', 'recheck_age', 365*24*3600)
threshold = int(time.time() - r_age)
CrawlConfig.log("threshold = %s (%d)", U.ymdhms(threshold), threshold)
if r_fraction == 0.0:
return []
limit = round(r_fraction * how_many)
db = CrawlDBI.DBI(dbtype='crawler')
kw = {'table': 'checkables',
'fields': ['rowid',
'path',
'type',
'cos',
'cart',
'ttypes',
'checksum',
'last_check',
'fails',
'reported'],
'where': 'checksum <> 0 and last_check < %d' % threshold,
'orderby': 'last_check',
'limit': limit}
rows = db.select(**kw)
db.close()
rval = []
for row in rows:
tmp = list(row)
new = Checkable(rowid=tmp.pop(0),
path=tmp.pop(0),
type=tmp.pop(0),
cos=tmp.pop(0),
cart=tmp.pop(0),
ttypes=tmp.pop(0),
checksum=tmp.pop(0),
last_check=tmp.pop(0),
fails=tmp.pop(0),
reported=tmp.pop(0),
in_db=True,
dirty=False)
rval.append(new)
return rval
def load_recheck_list(cls, how_many):
"""
Look to see whether any of the already checksummed items in the
database have a last check time over the threshold for rechecking. If
so, we'll shove some of them to the front of the list based on the
configuration.
"""
cfg = CrawlConfig.add_config()
r_fraction = float(cfg.get_d('cv', 'recheck_fraction', '0.0'))
r_age = cfg.get_time('cv', 'recheck_age', 365 * 24 * 3600)
threshold = int(time.time() - r_age)
CrawlConfig.log("threshold = %s (%d)", U.ymdhms(threshold), threshold)
if r_fraction == 0.0:
return []
limit = round(r_fraction * how_many)
db = CrawlDBI.DBI(dbtype='crawler')
kw = {
'table':
'checkables',
'fields': [
'rowid', 'path', 'type', 'cos', 'cart', 'ttypes', 'checksum',
'last_check', 'fails', 'reported'
],
'where':
'checksum <> 0 and last_check < %d' % threshold,
'orderby':
'last_check',
'limit':
limit
}
rows = db.select(**kw)
db.close()
rval = []
for row in rows:
tmp = list(row)
new = Checkable(rowid=tmp.pop(0),
path=tmp.pop(0),
type=tmp.pop(0),
cos=tmp.pop(0),
cart=tmp.pop(0),
ttypes=tmp.pop(0),
checksum=tmp.pop(0),
last_check=tmp.pop(0),
fails=tmp.pop(0),
reported=tmp.pop(0),
in_db=True,
dirty=False)
rval.append(new)
return rval
def history_load(loadlist, filename):
"""
Each plugin's sublib has a load_history() routine that knows how to load
its data to the history file.
Unfortunately, we do have to know here something special about plugin 'cv'
to warn the user when a filename was specified without 'cv' in the load
list or vice versa and when to pass filename to the plugin's load_history()
method.
"""
cfg = CrawlConfig.add_config()
pluglist = U.csv_list(cfg.get_d('crawler', 'plugins', U.default_plugins()))
ll = U.csv_list(loadlist)
if 'all' in ll or ll == []:
ll = copy.deepcopy(pluglist)
if filename is None and 'cv' in ll:
print(MSG.history_cv_not_loaded)
ll.remove('cv')
elif filename is not None and 'cv' not in ll:
print(MSG.history_filename_ignored)
unk_plugs = [x for x in ll if x not in pluglist]
if 0 < len(unk_plugs):
print(MSG.unrecognized_plugin_S % ', '.join(unk_plugs))
map(ll.remove, unk_plugs)
if ll == []:
return
dbschem.make_table('history')
for plug in [x for x in ll if x in pluglist]:
print("loading %s..." % plug)
if plug == 'cv' and filename is not None:
args = [filename]
else:
args = []
p = CrawlPlugin.CrawlPlugin(name=plug, cfg=cfg)
p.load_history(*args)
def history_load(loadlist, filename):
"""
Each plugin's sublib has a load_history() routine that knows how to load
its data to the history file.
Unfortunately, we do have to know here something special about plugin 'cv'
to warn the user when a filename was specified without 'cv' in the load
list or vice versa and when to pass filename to the plugin's load_history()
method.
"""
cfg = CrawlConfig.add_config()
pluglist = U.csv_list(cfg.get_d('crawler', 'plugins', U.default_plugins()))
ll = U.csv_list(loadlist)
if 'all' in ll or ll == []:
ll = copy.deepcopy(pluglist)
if filename is None and 'cv' in ll:
print(MSG.history_cv_not_loaded)
ll.remove('cv')
elif filename is not None and 'cv' not in ll:
print(MSG.history_filename_ignored)
unk_plugs = [x for x in ll if x not in pluglist]
if 0 < len(unk_plugs):
print(MSG.unrecognized_plugin_S % ', '.join(unk_plugs))
map(ll.remove, unk_plugs)
if ll == []:
return
dbschem.make_table('history')
for plug in [x for x in ll if x in pluglist]:
print("loading %s..." % plug)
if plug == 'cv' and filename is not None:
args = [filename]
else:
args = []
p = CrawlPlugin.CrawlPlugin(name=plug, cfg=cfg)
p.load_history(*args)
def __init__(self, *args, **kwargs):
"""
Initialize a Checkable object -- set the path, type, checksum, cos, and
last_check to default values, then update them based on the arguments.
"""
# where the item is in HPSS
self.path = '---'
# file ('f') or directory ('d')
self.type = '-'
# which COS the file is in (empty for directories)
self.cos = ''
# which tape cartridge(s) the file is stored on
self.cart = None
# the type of tape cartridge(s)
self.ttypes = None
# 1 if we have a checksum stored, else 0
self.checksum = 0
# how many times we've tried and failed to retrieve the file content
self.fails = 0
# whether we've reported that retrievals are failing for this file
self.reported = 0
# when was the last check of this file (epoch time)
self.last_check = 0
# this item's row id in the database
self.rowid = None
# how likely are we to add an item to the sample?
self.probability = 0.1
# whether this object is in the database
self.in_db = False
# whether this object has been changed
self.dirty = False
# non keyword arguments
self.args = args
for k in kwargs:
if k not in ['rowid',
'path',
'type',
'checksum',
'cos',
'cart',
'ttypes',
'dim',
'fails',
'reported',
'last_check',
'probability',
'in_db',
'dirty']:
raise StandardError("Attribute %s is invalid for Checkable" %
k)
setattr(self, k, kwargs[k])
for attr in ['checksum', 'fails', 'reported']:
if getattr(self, attr) is None:
setattr(self, attr, 0)
# Set up dimensions based on configuration. If no dimensions option is
# set in the configuration, we just leave the dimensions dict emtpy.
# Since this class is only used by the cv_plugin, it makes no sense for
# this code to be running if there is no cv section in the
# configuration, so we'll let that exception get thrown up the stack.
cfg = CrawlConfig.add_config()
self.dim = {}
try:
dim_l = util.csv_list(cfg.get('cv', 'dimensions'))
for dname in dim_l:
self.dim[dname] = Dimension.get_dim(dname)
except CrawlConfig.NoOptionError:
pass
super(Checkable, self).__init__()
def get_list(cls, how_many=-1, prob=0.1, rootlist=[]):
"""
Return the current list of Checkables from the database.
"""
if how_many < 0:
cfg = CrawlConfig.add_config()
how_many = int(cfg.get_d('cv', 'operations', '30'))
rval = Checkable.load_priority_list()
if how_many <= len(rval):
return rval
rval.extend(Checkable.load_recheck_list(how_many))
if how_many <= len(rval):
return rval
db = CrawlDBI.DBI(dbtype='crawler')
kw = {'table': 'checkables',
'fields': ['rowid',
'path',
'type',
'cos',
'cart',
'ttypes',
'checksum',
'last_check',
'fails',
'reported'],
'orderby': 'last_check'}
if 0 < how_many:
kw['limit'] = how_many
rows = db.select(**kw)
# check whether any roots from rootlist are missing and if so, add them
# to the table
reselect = False
pathlist = [x[1] for x in rows]
for root in rootlist:
if root not in pathlist:
nr = Checkable(path=root, type='d')
nr.load()
nr.persist()
reselect = True
if reselect:
rows = db.select(**kw)
for row in rows:
tmp = list(row)
new = Checkable(rowid=tmp.pop(0),
path=tmp.pop(0),
type=tmp.pop(0),
cos=tmp.pop(0),
cart=tmp.pop(0),
ttypes=tmp.pop(0),
checksum=tmp.pop(0),
last_check=tmp.pop(0),
fails=tmp.pop(0),
reported=tmp.pop(0),
probability=prob,
in_db=True,
dirty=False)
if new not in rval:
rval.append(new)
if how_many <= len(rval):
break
db.close()
CrawlConfig.log("returning %d items" % len(rval))
return rval
def __init__(self, *args, **kwargs):
"""
Initialize a Checkable object -- set the path, type, checksum, cos, and
last_check to default values, then update them based on the arguments.
"""
# where the item is in HPSS
self.path = '---'
# file ('f') or directory ('d')
self.type = '-'
# which COS the file is in (empty for directories)
self.cos = ''
# which tape cartridge(s) the file is stored on
self.cart = None
# the type of tape cartridge(s)
self.ttypes = None
# 1 if we have a checksum stored, else 0
self.checksum = 0
# how many times we've tried and failed to retrieve the file content
self.fails = 0
# whether we've reported that retrievals are failing for this file
self.reported = 0
# when was the last check of this file (epoch time)
self.last_check = 0
# this item's row id in the database
self.rowid = None
# how likely are we to add an item to the sample?
self.probability = 0.1
# whether this object is in the database
self.in_db = False
# whether this object has been changed
self.dirty = False
# non keyword arguments
self.args = args
for k in kwargs:
if k not in [
'rowid', 'path', 'type', 'checksum', 'cos', 'cart',
'ttypes', 'dim', 'fails', 'reported', 'last_check',
'probability', 'in_db', 'dirty'
]:
raise StandardError("Attribute %s is invalid for Checkable" %
k)
setattr(self, k, kwargs[k])
for attr in ['checksum', 'fails', 'reported']:
if getattr(self, attr) is None:
setattr(self, attr, 0)
# Set up dimensions based on configuration. If no dimensions option is
# set in the configuration, we just leave the dimensions dict emtpy.
# Since this class is only used by the cv_plugin, it makes no sense for
# this code to be running if there is no cv section in the
# configuration, so we'll let that exception get thrown up the stack.
cfg = CrawlConfig.add_config()
self.dim = {}
try:
dim_l = util.csv_list(cfg.get('cv', 'dimensions'))
for dname in dim_l:
self.dim[dname] = Dimension.get_dim(dname)
except CrawlConfig.NoOptionError:
pass
super(Checkable, self).__init__()
def get_list(cls, how_many=-1, prob=0.1, rootlist=[]):
"""
Return the current list of Checkables from the database.
"""
if how_many < 0:
cfg = CrawlConfig.add_config()
how_many = int(cfg.get_d('cv', 'operations', '30'))
rval = Checkable.load_priority_list()
if how_many <= len(rval):
return rval
rval.extend(Checkable.load_recheck_list(how_many))
if how_many <= len(rval):
return rval
db = CrawlDBI.DBI(dbtype='crawler')
kw = {
'table':
'checkables',
'fields': [
'rowid', 'path', 'type', 'cos', 'cart', 'ttypes', 'checksum',
'last_check', 'fails', 'reported'
],
'orderby':
'last_check'
}
if 0 < how_many:
kw['limit'] = how_many
rows = db.select(**kw)
# check whether any roots from rootlist are missing and if so, add them
# to the table
reselect = False
pathlist = [x[1] for x in rows]
for root in rootlist:
if root not in pathlist:
nr = Checkable(path=root, type='d')
nr.load()
nr.persist()
reselect = True
if reselect:
rows = db.select(**kw)
for row in rows:
tmp = list(row)
new = Checkable(rowid=tmp.pop(0),
path=tmp.pop(0),
type=tmp.pop(0),
cos=tmp.pop(0),
cart=tmp.pop(0),
ttypes=tmp.pop(0),
checksum=tmp.pop(0),
last_check=tmp.pop(0),
fails=tmp.pop(0),
reported=tmp.pop(0),
probability=prob,
in_db=True,
dirty=False)
if new not in rval:
rval.append(new)
if how_many <= len(rval):
break
db.close()
CrawlConfig.log("returning %d items" % len(rval))
return rval
def dispatch(self):
"""
Figure out where we're supposed to send this alert and send it.
Possible destinations are the log file, one or more e-mail addresses,
and/or a shell program.
It's also possible for a 'use' option to show up in the alerts section.
In this case, we're being redirected to another section, also 'use' can
also point to the current alerts section. There's no reason to ever do
this, but it could happen so we want to handle it in a reasonable way.
That's why we sort the config options in the while statement below --
to make 'use' get handled last, so any other options in the section
will get handled. Once we process 'use', anything not yet processed in
the current section is ignored.
"""
if self.cfg is not None:
cfg = self.cfg
else:
cfg = CrawlConfig.add_config()
if self.caller != '':
section = cfg.get(self.caller, 'alerts')
else:
section = 'alerts'
done = False
while not done:
for opt in sorted(cfg.options(section)):
if opt == 'log':
# write to log
fmt = cfg.get(section, 'log')
CrawlConfig.log(fmt, self.msg)
done = True
elif opt == 'shell':
# run the program
cmd = cfg.get(section, 'shell')
if '%s' in cmd:
cmdline = cmd % (self.msg)
else:
cmdline = cmd
os.system(cmdline)
CrawlConfig.log("ran: '%s'" % (cmdline))
done = True
elif opt == 'email':
CrawlMail.send(cfg=cfg,
to="%s.email" % section,
subj="HPSS Integrity Crawler ALERT",
msg=self.msg)
done = True
elif opt == 'use':
# delegate to another section
done = True
new_section = cfg.get(section, 'use')
# if it's the same section, ignore the 'use', but we don't
# want to break the rule that all options after a 'use' are
# ignored. So we set done to True to terminate the while
# loop and break unconditionally at the end of this clause
# to get out of the for loop
if new_section != section:
section = new_section
done = False
break
def crl_history(argv):
"""history - access to the plugin history
usage: crawl history [--load|--show|--reset]
--load {all,cv,mpra,tcc,rpt}
Load the history table from listed plugin tables, log file
--show
Read the history table and report its contents.
--reset
Drop the history table.
--read-log FILENAME
If --load is specified and includes 'cv', read FILENAME and load cv
history from it.
To load just cv data, --load cv --read-log FILENAME
To load just mpra data, --load mpra
To load all plugins, --load all (or "") --read-log FILENAME
If --load contains 'cv' but --read-log is not specified, an error message
will be issued.
If --load contains 'all' or is empty and --read-log is not specified, a
warning will be issued to notify the user that cv data is not being loaded.
If --load does not contain 'cv' or 'all' and is not empty and --read-log is
specified, a warning will be issued that the log file is not being read and
cv data is not being loaded.
"""
p = optparse.OptionParser()
p.add_option('-d', '--debug',
action='store_true', default=False, dest='debug',
help='run the debugger')
p.add_option('-n', '--dry-run',
action='store_true', default=False, dest='dryrun',
help='just report')
p.add_option('-l', '--load',
action='store', default=None, dest='loadlist',
help='plugins to load')
p.add_option('-r', '--read-log',
action='store', default=None, dest='filename',
help='log file for cv history')
p.add_option('-R', '--reset',
action='store_true', default=False, dest='reset',
help='drop the history table')
p.add_option('-s', '--show',
action='store', default='unset', dest='show',
help='Report the contents of the history table')
(o, a) = p.parse_args(argv)
if o.debug:
pdb.set_trace()
# This is saying, if any two of our primary command line options are set,
# we have a problem since they are all mutually exclusive.
if o.show == 'unset':
o.show = None
if any([all([o.loadlist is not None, o.reset]),
all([o.loadlist is not None, o.show]),
all([o.reset, o.show])]):
raise SystemExit(MSG.history_options)
if o.dryrun:
cfg = CrawlConfig.add_config()
table = cfg.get('dbi-crawler', 'tbl_prefix') + '_history'
dbname = cfg.get('dbi-crawler', 'dbname')
hostname = cfg.get('dbi-crawler', 'hostname')
if o.show:
# This option is non-destructive, so we ignore --dry-run for it.
history_show(o.show)
elif o.reset:
if o.dryrun:
print(MSG.history_reset_dryrun_SSS % (table, dbname, hostname))
else:
print(dbschem.drop_table(table='history'))
elif o.loadlist is not None:
if o.dryrun:
print(MSG.history_load_dryrun_SSSS %
(table, dbname, hostname, o.filename))
else:
history_load(o.loadlist, o.filename)
def crl_history(argv):
"""history - access to the plugin history
usage: crawl history [--load|--show|--reset]
--load {all,cv,mpra,tcc,rpt}
Load the history table from listed plugin tables, log file
--show
Read the history table and report its contents.
--reset
Drop the history table.
--read-log FILENAME
If --load is specified and includes 'cv', read FILENAME and load cv
history from it.
To load just cv data, --load cv --read-log FILENAME
To load just mpra data, --load mpra
To load all plugins, --load all (or "") --read-log FILENAME
If --load contains 'cv' but --read-log is not specified, an error message
will be issued.
If --load contains 'all' or is empty and --read-log is not specified, a
warning will be issued to notify the user that cv data is not being loaded.
If --load does not contain 'cv' or 'all' and is not empty and --read-log is
specified, a warning will be issued that the log file is not being read and
cv data is not being loaded.
"""
p = optparse.OptionParser()
p.add_option('-d',
'--debug',
action='store_true',
default=False,
dest='debug',
help='run the debugger')
p.add_option('-n',
'--dry-run',
action='store_true',
default=False,
dest='dryrun',
help='just report')
p.add_option('-l',
'--load',
action='store',
default=None,
dest='loadlist',
help='plugins to load')
p.add_option('-r',
'--read-log',
action='store',
default=None,
dest='filename',
help='log file for cv history')
p.add_option('-R',
'--reset',
action='store_true',
default=False,
dest='reset',
help='drop the history table')
p.add_option('-s',
'--show',
action='store',
default='unset',
dest='show',
help='Report the contents of the history table')
(o, a) = p.parse_args(argv)
if o.debug:
pdb.set_trace()
# This is saying, if any two of our primary command line options are set,
# we have a problem since they are all mutually exclusive.
if o.show == 'unset':
o.show = None
if any([
all([o.loadlist is not None, o.reset]),
all([o.loadlist is not None, o.show]),
all([o.reset, o.show])
]):
raise SystemExit(MSG.history_options)
if o.dryrun:
cfg = CrawlConfig.add_config()
table = cfg.get('dbi-crawler', 'tbl_prefix') + '_history'
dbname = cfg.get('dbi-crawler', 'dbname')
hostname = cfg.get('dbi-crawler', 'hostname')
if o.show:
# This option is non-destructive, so we ignore --dry-run for it.
history_show(o.show)
elif o.reset:
if o.dryrun:
print(MSG.history_reset_dryrun_SSS % (table, dbname, hostname))
else:
print(dbschem.drop_table(table='history'))
elif o.loadlist is not None:
if o.dryrun:
print(MSG.history_load_dryrun_SSSS %
(table, dbname, hostname, o.filename))
else:
history_load(o.loadlist, o.filename)
def cvv_show_next(argv):
"""show_next - Report the Checkables in the order they will be checked
usage: cvtool show_next
"""
p = optparse.OptionParser()
p.add_option('-c',
'--config',
action='store',
default='',
dest='config',
help='alternate configuration')
p.add_option('-d',
'--debug',
action='store_true',
default=False,
dest='debug',
help='run the debugger')
p.add_option('-i',
'--id',
action='store',
default='',
dest='id',
help='id of entry to be checked')
p.add_option('-l',
'--limit',
action='store',
default=-1,
dest='limit',
type=int,
help='max records to get')
p.add_option('-p',
'--path',
action='store',
default='',
dest='path',
help='name of path to be checked')
p.add_option('-v',
'--verbose',
action='store_true',
default=False,
dest='verbose',
help='more information')
try:
(o, a) = p.parse_args(argv)
except SystemExit:
return
if o.debug:
pdb.set_trace()
if o.config:
cfg = CrawlConfig.add_config(close=True, filename=o.config)
else:
cfg = CrawlConfig.add_config()
if o.limit < 0:
limit = int(cfg.get_d('cv', 'operations', '10'))
else:
limit = o.limit
clist = Checkable.Checkable.get_list(limit)
for c in clist:
if c.last_check == 0:
print("%18d %s %s" % (c.last_check, c.type, c.path))
else:
print("%s %s %s" % (U.ymdhms(c.last_check), c.type, c.path))
def dispatch(self):
"""
Figure out where we're supposed to send this alert and send it.
Possible destinations are the log file, one or more e-mail addresses,
and/or a shell program.
It's also possible for a 'use' option to show up in the alerts section.
In this case, we're being redirected to another section, also 'use' can
also point to the current alerts section. There's no reason to ever do
this, but it could happen so we want to handle it in a reasonable way.
That's why we sort the config options in the while statement below --
to make 'use' get handled last, so any other options in the section
will get handled. Once we process 'use', anything not yet processed in
the current section is ignored.
"""
if self.cfg is not None:
cfg = self.cfg
else:
cfg = CrawlConfig.add_config()
if self.caller != '':
section = cfg.get(self.caller, 'alerts')
else:
section = 'alerts'
done = False
while not done:
for opt in sorted(cfg.options(section)):
if opt == 'log':
# write to log
fmt = cfg.get(section, 'log')
CrawlConfig.log(fmt, self.msg)
done = True
elif opt == 'shell':
# run the program
cmd = cfg.get(section, 'shell')
if '%s' in cmd:
cmdline = cmd % (self.msg)
else:
cmdline = cmd
os.system(cmdline)
CrawlConfig.log("ran: '%s'" % (cmdline))
done = True
elif opt == 'email':
CrawlMail.send(cfg=cfg,
to="%s.email" % section,
subj="HPSS Integrity Crawler ALERT",
msg=self.msg)
done = True
elif opt == 'use':
# delegate to another section
done = True
new_section = cfg.get(section, 'use')
# if it's the same section, ignore the 'use', but we don't
# want to break the rule that all options after a 'use' are
# ignored. So we set done to True to terminate the while
# loop and break unconditionally at the end of this clause
# to get out of the for loop
if new_section != section:
section = new_section
done = False
break