def record_history(name, when, errors): """ Record a plugin name and runtime in the history table """ db = CrawlDBI.DBI(dbtype='crawler') if not db.table_exists(table='history'): dbschem.make_table('history') db.insert(table='history', fields=['plugin', 'runtime', 'errors'], data=[(name, when, errors)]) db.close()
def mpra_record_recent(type, start, end, hits): """ Record the most recent record reported so we don't report records repeatedly. However, if recent is not later than the time already stored, we don't want to update it. """ dbschem.make_table('mpra') db = CrawlDBI.DBI(dbtype="crawler") db.insert(table='mpra', fields=['type', 'scan_time', 'start_time', 'end_time', 'hits'], data=[(type, int(time.time()), int(start), int(end), hits)]) db.close()
def cvv_ttype_table(argv): """ttype_table - create (or drop) table tape_types usage: cv ttype_table [-d] {-D|-r /opt/hpss} Without the -D/--drop option, create the table tape_types in the mysql database. Populate it with information from an HPSS build tree (default is /opt/hpss). With -D or --drop, drop the table. """ p = optparse.OptionParser() p.add_option('-D', '--drop', action='store_true', default=False, dest='drop', help='drop the table') p.add_option('-d', '--debug', action='store_true', default=False, dest='debug', help='run the debugger') p.add_option('-r', '--root', action='store', default='', dest='hpssroot', help='where to look for data') try: (o, a) = p.parse_args(argv) except SystemExit: return if o.debug: pdb.set_trace() # lookup and report tape type for each pathname specified if o.drop: result = dbschem.drop_table(table="tape_types") print result else: dbschem.make_table("tape_types") hpssroot = o.hpssroot if hpssroot == '': hpssroot = os.getenv("HPSS_ROOT") if hpssroot is None: hpssroot = "/opt/hpss" tape_types_populate(hpssroot)
def record_checked_ids(cfg, low, high, correct, error): """ Save checked NSOBJECT ids in the HPSSIC database. If we check a range and get no hits (i.e., no NSOBJECT ids exist in the range), we'll store (<time>, <low-id>, <high-id>, 0, 0) If we get a hit with the right copy count, we store it by itself as (<time>, <hit-id>, <hit-id>, 1, 0) If we get a hit with the wrong copy count, we store it by itself as (<time>, <hit-id>, <hit-id>, 0, 1) """ tabname = cfg.get(sectname(), 'table_name') result = dbschem.make_table(tabname) ts = int(time.time()) CrawlConfig.log("recording checked ids %d to %d at %d" % (low, high, ts)) db = CrawlDBI.DBI(dbtype="crawler") db.insert(table=tabname, fields=['check_time', 'low_nsobj_id', 'high_nsobj_id', 'correct', 'error'], data=[(ts, low, high, correct, error)]) db.close()
def record_checked_ids(cfg, low, high, correct, error): """ Save checked NSOBJECT ids in the HPSSIC database. If we check a range and get no hits (i.e., no NSOBJECT ids exist in the range), we'll store (<time>, <low-id>, <high-id>, 0, 0) If we get a hit with the right copy count, we store it by itself as (<time>, <hit-id>, <hit-id>, 1, 0) If we get a hit with the wrong copy count, we store it by itself as (<time>, <hit-id>, <hit-id>, 0, 1) """ tabname = cfg.get(sectname(), 'table_name') result = dbschem.make_table(tabname) ts = int(time.time()) CrawlConfig.log("recording checked ids %d to %d at %d" % (low, high, ts)) db = CrawlDBI.DBI(dbtype="crawler") db.insert(table=tabname, fields=[ 'check_time', 'low_nsobj_id', 'high_nsobj_id', 'correct', 'error' ], data=[(ts, low, high, correct, error)]) db.close()
def cvv_ttype_table(argv): """ttype_table - create (or drop) table tape_types usage: cv ttype_table [-d] {-D|-r /opt/hpss} Without the -D/--drop option, create the table tape_types in the mysql database. Populate it with information from an HPSS build tree (default is /opt/hpss). With -D or --drop, drop the table. """ p = optparse.OptionParser() p.add_option('-D', '--drop', action='store_true', default=False, dest='drop', help='drop the table') p.add_option('-d', '--debug', action='store_true', default=False, dest='debug', help='run the debugger') p.add_option('-r', '--root', action='store', default='', dest='hpssroot', help='where to look for data') try: (o, a) = p.parse_args(argv) except SystemExit: return if o.debug: pdb.set_trace() # lookup and report tape type for each pathname specified if o.drop: result = dbschem.drop_table(table="tape_types") print result else: dbschem.make_table("tape_types") hpssroot = o.hpssroot if hpssroot == '': hpssroot = os.getenv("HPSS_ROOT") if hpssroot is None: hpssroot = "/opt/hpss" tape_types_populate(hpssroot)
def history_load(loadlist, filename): """ Each plugin's sublib has a load_history() routine that knows how to load its data to the history file. Unfortunately, we do have to know here something special about plugin 'cv' to warn the user when a filename was specified without 'cv' in the load list or vice versa and when to pass filename to the plugin's load_history() method. """ cfg = CrawlConfig.add_config() pluglist = U.csv_list(cfg.get_d('crawler', 'plugins', U.default_plugins())) ll = U.csv_list(loadlist) if 'all' in ll or ll == []: ll = copy.deepcopy(pluglist) if filename is None and 'cv' in ll: print(MSG.history_cv_not_loaded) ll.remove('cv') elif filename is not None and 'cv' not in ll: print(MSG.history_filename_ignored) unk_plugs = [x for x in ll if x not in pluglist] if 0 < len(unk_plugs): print(MSG.unrecognized_plugin_S % ', '.join(unk_plugs)) map(ll.remove, unk_plugs) if ll == []: return dbschem.make_table('history') for plug in [x for x in ll if x in pluglist]: print("loading %s..." % plug) if plug == 'cv' and filename is not None: args = [filename] else: args = [] p = CrawlPlugin.CrawlPlugin(name=plug, cfg=cfg) p.load_history(*args)
def history_load(loadlist, filename): """ Each plugin's sublib has a load_history() routine that knows how to load its data to the history file. Unfortunately, we do have to know here something special about plugin 'cv' to warn the user when a filename was specified without 'cv' in the load list or vice versa and when to pass filename to the plugin's load_history() method. """ cfg = CrawlConfig.add_config() pluglist = U.csv_list(cfg.get_d('crawler', 'plugins', U.default_plugins())) ll = U.csv_list(loadlist) if 'all' in ll or ll == []: ll = copy.deepcopy(pluglist) if filename is None and 'cv' in ll: print(MSG.history_cv_not_loaded) ll.remove('cv') elif filename is not None and 'cv' not in ll: print(MSG.history_filename_ignored) unk_plugs = [x for x in ll if x not in pluglist] if 0 < len(unk_plugs): print(MSG.unrecognized_plugin_S % ', '.join(unk_plugs)) map(ll.remove, unk_plugs) if ll == []: return dbschem.make_table('history') for plug in [x for x in ll if x in pluglist]: print("loading %s..." % plug) if plug == 'cv' and filename is not None: args = [filename] else: args = [] p = CrawlPlugin.CrawlPlugin(name=plug, cfg=cfg) p.load_history(*args)
def ex_nihilo(cls, dataroot='/'): """ Start from scratch. Create the database if necessary. Create the table(s) if necessary. Bootstrap the queue by adding the root director(ies). Field path is the location of the file or directory in the HPSS archive. Field type is 'f' for files or 'd' for directories. Field cos is the class of service for the file. For directories, cos is empty. Field cart starts with a null value. When populated from hsi, it may be set to the name of a tape cartridge or to ''. Empty files take up no space on any cartridge, so for them the field is empty. Field checksum is 0 if we have not computed or discoverd a checksum for the file. Once we know a checksum has been stored for the file, we set this to 1. Field last_check is the epoch time at which the file was last checked. Field fails is the number of times hashcreate and/or hashverify has failed on the file. Field reported is 0 or 1 indicating whether we've reported """ dbschem.make_table("checkables") if type(dataroot) == str: dataroot = [dataroot] if type(dataroot) == list: for root in dataroot: r = Checkable(path=root, type='d', in_db=False, dirty=True) r.load() r.persist()
def ex_nihilo(cls, dataroot='/'): """ Start from scratch. Create the database if necessary. Create the table(s) if necessary. Bootstrap the queue by adding the root director(ies). Field path is the location of the file or directory in the HPSS archive. Field type is 'f' for files or 'd' for directories. Field cos is the class of service for the file. For directories, cos is empty. Field cart starts with a null value. When populated from hsi, it may be set to the name of a tape cartridge or to ''. Empty files take up no space on any cartridge, so for them the field is empty. Field checksum is 0 if we have not computed or discoverd a checksum for the file. Once we know a checksum has been stored for the file, we set this to 1. Field last_check is the epoch time at which the file was last checked. Field fails is the number of times hashcreate and/or hashverify has failed on the file. Field reported is 0 or 1 indicating whether we've reported """ dbschem.make_table("checkables") if type(dataroot) == str: dataroot = [dataroot] if type(dataroot) == list: for root in dataroot: r = Checkable(path=root, type='d', in_db=False, dirty=True) r.load() r.persist()
def update_stats(cmf): """ Record the values in tuple cmf in table cvstats in the database. If the table does not exist, create it. """ result = dbschem.make_table(stats_table) db = CrawlDBI.DBI(dbtype="crawler") if result == "Created": db.insert(table=stats_table, fields=["rowid", "matches", "failures"], data=[(1, 0, 0)]) db.update(table=stats_table, fields=["matches", "failures"], data=[cmf], where="rowid = 1") db.close()
def update_stats(cmf): """ Record the values in tuple cmf in table cvstats in the database. If the table does not exist, create it. """ result = dbschem.make_table(stats_table) db = CrawlDBI.DBI(dbtype="crawler") if result == "Created": db.insert(table=stats_table, fields=["rowid", "matches", "failures"], data=[(1, 0, 0)]) db.update(table=stats_table, fields=["matches", "failures"], data=[cmf], where="rowid = 1") db.close()
def get_last_rpt_time(db): """ Retrieve the last report time from the report table. If the table does not exist before make_table ('Created' in result), the table is empty so we just return 0 to indicate no last report time. """ result = dbschem.make_table("report") if "Created" in result: rval = 0 else: rows = db.select(table='report', fields=['max(report_time)']) (rval) = rows[0][0] if rval is None: rval = 0 CrawlConfig.log("time of last report: %d" % rval) return rval
def get_last_rpt_time(db): """ Retrieve the last report time from the report table. If the table does not exist before make_table ('Created' in result), the table is empty so we just return 0 to indicate no last report time. """ result = dbschem.make_table("report") if "Created" in result: rval = 0 else: rows = db.select(table='report', fields=['max(report_time)']) (rval) = rows[0][0] if rval is None: rval = 0 CrawlConfig.log("time of last report: %d" % rval) return rval
def lscos_populate(): """ If table lscos already exists, we're done. Otherwise, retrieve the lscos info from hsi, create the table, and fill the table in. We store the min_size and max_size for each COS as text strings containing digits because the largest sizes are already within three orders of magnitude of a mysql bigint and growing. """ db = CrawlDBI.DBI(dbtype="crawler") tabname = 'lscos' st = dbschem.make_table(tabname) szrgx = "(\d+([KMGT]B)?)" rgx = ("\s*(\d+)\s*(([-_a-zA-Z0-9]+\s)+)\s+[UGAN]*\s+(\d+)" + "\s+(ALL)?\s+%s\s+-\s+%s" % (szrgx, szrgx)) if "Created" == st: H = hpss.HSI() raw = H.lscos() H.quit() z = [x.strip() for x in raw.split('\r')] rules = [q for q in z if '----------' in q] first = z.index(rules[0]) + 1 second = z[first:].index(rules[0]) + first lines = z[first:second] data = [] for line in lines: m = U.rgxin(rgx, line) (cos, desc, copies, lo_i, hi_i) = (m[0], m[1].strip(), m[3], U.scale(m[5], kb=1024), U.scale(m[7], kb=1024)) data.append((cos, desc, copies, lo_i, hi_i)) db.insert(table=tabname, fields=['cos', 'name', 'copies', 'min_size', 'max_size'], data=data) rval = MSG.table_created_S % tabname else: rval = MSG.table_already_S % tabname db.close() return rval