def _restructure(): """major restructure of reporting data""" # run any migrations from the previous schema try: from Bcfg2.Server.Reports.updatefix import update_database update_database() except: logger.error("Failed to run legacy schema updates", exc_info=1) return False # try to avoid dangling transactions if not migrate_stage1(): return try: entries = {} for ent in legacy_models.Entries.objects.all(): entries[ent.id] = ent except: logger.error("Failed to populate entries dict", exc_info=1) return False try: fperms = _migrate_perms() except: logger.error("Failed create FilePerms objects", exc_info=1) return False failures = [] int_count = legacy_models.Interaction.objects.count() if int_count == 0: logger.error("Found no legacy interactions") return False int_ctr = 0 start_time = 0 for inter in BatchFetch(legacy_models.Interaction.objects.select_related("metadata", "client").all()): if int_ctr % 1000 == 0: if int_ctr > 0: logger.info("Migrated %s of %s interactions in %ss" % (int_ctr, int_count, time.time() - start_time)) else: logger.info("Migrating interactions") start_time = time.time() try: _migrate_transaction(inter, entries, fperms) except: logger.error("Failed to migrate interaction %s for %s" % (inter.id, inter.client.name), exc_info=1) failures.append(inter.id) int_ctr += 1 if not failures: logger.info("Successfully restructured reason data") return True logger.info("Updating recent interactions") for newint in new_models.Interaction.objects.recent(): try: newint.save() except: logger.error("Failed to set current interaction %s for %s" % (newint.id, newint.client.name), exc_info=1)
def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.ThreadedStatistics.__init__(self, core, datastore) Bcfg2.Server.Plugin.PullSource.__init__(self) self.cpath = "%s/Metadata/clients.xml" % datastore self.core = core logger.debug("Searching for new models to add to the statistics database") try: update_database() except Exception, inst: logger.debug(str(inst)) logger.debug(str(type(inst)))
def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.ThreadedStatistics.__init__(self, core, datastore) Bcfg2.Server.Plugin.PullSource.__init__(self) self.cpath = "%s/Metadata/clients.xml" % datastore self.core = core logger.debug( "Searching for new models to add to the statistics database") try: update_database() except Exception: inst = sys.exc_info()[1] logger.debug(str(inst)) logger.debug(str(type(inst)))
def __call__(self, args): Bcfg2.Server.Admin.Mode.__call__(self, args) if len(args) == 0 or args[0] == '-h': print(self.__usage__) raise SystemExit(0) verb = 0 if '-v' in args or '--verbose' in args: self.log.setLevel(logging.DEBUG) verb = 1 if '-q' in args or '--quiet' in args: self.log.setLevel(logging.WARNING) # FIXME - dry run if args[0] in self.django_commands: self.django_command_proxy(args[0]) elif args[0] == 'scrub': self.scrub() elif args[0] == 'init': update_database() elif args[0] == 'update': update_database() elif args[0] == 'load_stats': quick = '-O3' in args stats_file=None clients_file=None i=1 while i < len(args): if args[i] == '-s' or args[i] == '--stats': stats_file = args[i+1] if stats_file[0] == '-': self.errExit("Invalid statistics file: %s" % stats_file) elif args[i] == '-c' or args[i] == '--clients-file': clients_file = args[i+1] if clients_file[0] == '-': self.errExit("Invalid clients file: %s" % clients_file) i = i + 1 self.load_stats(stats_file, clients_file, verb, quick) else: print "Unknown command: %s" % args[0]
def _restructure(): """major restructure of reporting data""" # run any migrations from the previous schema try: from Bcfg2.Server.Reports.updatefix import update_database update_database() except: logger.error("Failed to run legacy schema updates", exc_info=1) return False # try to avoid dangling transactions if not migrate_stage1(): return try: entries = {} for ent in legacy_models.Entries.objects.all(): entries[ent.id] = ent except: logger.error("Failed to populate entries dict", exc_info=1) return False try: fperms = _migrate_perms() except: logger.error("Failed create FilePerms objects", exc_info=1) return False failures = [] int_count = legacy_models.Interaction.objects.count() if int_count == 0: logger.error("Found no legacy interactions") return False int_ctr = 0 start_time = 0 for inter in BatchFetch(legacy_models.Interaction.objects.\ select_related('metadata', 'client').all()): if int_ctr % 1000 == 0: if int_ctr > 0: logger.info("Migrated %s of %s interactions in %ss" % \ (int_ctr, int_count, time.time() - start_time)) else: logger.info("Migrating interactions") start_time = time.time() try: _migrate_transaction(inter, entries, fperms) except: logger.error("Failed to migrate interaction %s for %s" % (inter.id, inter.client.name), exc_info=1) failures.append(inter.id) int_ctr += 1 if not failures: logger.info("Successfully restructured reason data") return True logger.info("Updating recent interactions") for newint in new_models.Interaction.objects.recent(): try: newint.save() except: logger.error("Failed to set current interaction %s for %s" % (newint.id, newint.client.name), exc_info=1)
print "Could not read bcfg2.conf; exiting" raise SystemExit, 1 try: statsdata = XML(open(statpath).read()) except (IOError, XMLSyntaxError): print("StatReports: Failed to parse %s"%(statpath)) raise SystemExit, 1 if not clientpath: try: clientspath = "%s/Metadata/clients.xml" % \ cf.get('server', 'repository') except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): print "Could not read bcfg2.conf; exiting" raise SystemExit, 1 try: clientsdata = XML(open(clientspath).read()) except (IOError, XMLSyntaxError): print("StatReports: Failed to parse %s"%(clientspath)) raise SystemExit, 1 q = '-O3' in sys.argv try: location = cf.get('components', 'bcfg2') except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): print "Could not read bcfg2 from bcfg2.conf; exiting" raise SystemExit, 1 # Be sure the database is ready for new schema update_database() load_stats(clientsdata, statsdata, verb, quick=q, location=location)
def __call__(self, args): Bcfg2.Server.Admin.Mode.__call__(self, args) if len(args) == 0 or args[0] == '-h': print(self.__usage__) raise SystemExit(0) verb = 0 if '-v' in args or '--verbose' in args: self.log.setLevel(logging.DEBUG) verb = 1 if '-q' in args or '--quiet' in args: self.log.setLevel(logging.WARNING) # FIXME - dry run if args[0] in self.django_commands: self.django_command_proxy(args[0]) elif args[0] == 'scrub': self.scrub() elif args[0] == 'init': update_database() elif args[0] == 'update': update_database() elif args[0] == 'load_stats': quick = '-O3' in args stats_file = None clients_file = None i = 1 while i < len(args): if args[i] == '-s' or args[i] == '--stats': stats_file = args[i + 1] if stats_file[0] == '-': self.errExit("Invalid statistics file: %s" % stats_file) elif args[i] == '-c' or args[i] == '--clients-file': clients_file = args[i + 1] if clients_file[0] == '-': self.errExit("Invalid clients file: %s" % clients_file) i = i + 1 self.load_stats(stats_file, clients_file, verb, quick) elif args[0] == 'purge': expired = False client = None maxdate = None state = None i = 1 while i < len(args): if args[i] == '-c' or args[i] == '--client': if client: self.errExit("Only one client per run") client = args[i + 1] print(client) i = i + 1 elif args[i] == '--days': if maxdate: self.errExit("Max date specified multiple times") try: maxdate = datetime.datetime.now() - datetime.timedelta(days=int(args[i + 1])) except: self.log.error("Invalid number of days: %s" % args[i + 1]) raise SystemExit(-1) i = i + 1 elif args[i] == '--expired': expired = True i = i + 1 if expired: if state: self.log.error("--state is not valid with --expired") raise SystemExit(-1) self.purge_expired(maxdate) else: self.purge(client, maxdate, state) else: print("Unknown command: %s" % args[0])
raise SystemExit(1) try: encoding = cf.get('components', 'encoding') except: encoding = 'UTF-8' if not clientpath: try: clientspath = "%s/Metadata/clients.xml" % \ cf.get('server', 'repository') except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): print("Could not read bcfg2.conf; exiting") raise SystemExit(1) try: clientsdata = XML(open(clientspath).read()) except (IOError, XMLSyntaxError): print("StatReports: Failed to parse %s" % (clientspath)) raise SystemExit(1) q = '-O3' in sys.argv # Be sure the database is ready for new schema update_database() load_stats(clientsdata, statsdata, encoding, verb, logger, quick=q, location=platform.node())
def __call__(self, args): Bcfg2.Server.Admin.Mode.__call__(self, args) if len(args) == 0 or args[0] == '-h': print(self.__usage__) raise SystemExit(0) verb = 0 if '-v' in args or '--verbose' in args: self.log.setLevel(logging.DEBUG) verb = 1 if '-q' in args or '--quiet' in args: self.log.setLevel(logging.WARNING) # FIXME - dry run if args[0] in self.django_commands: self.django_command_proxy(args[0]) elif args[0] == 'scrub': self.scrub() elif args[0] == 'init': update_database() elif args[0] == 'update': update_database() elif args[0] == 'load_stats': quick = '-O3' in args stats_file = None clients_file = None i = 1 while i < len(args): if args[i] == '-s' or args[i] == '--stats': stats_file = args[i + 1] if stats_file[0] == '-': self.errExit("Invalid statistics file: %s" % stats_file) elif args[i] == '-c' or args[i] == '--clients-file': clients_file = args[i + 1] if clients_file[0] == '-': self.errExit("Invalid clients file: %s" % clients_file) i = i + 1 self.load_stats(stats_file, clients_file, verb, quick) elif args[0] == 'purge': expired = False client = None maxdate = None state = None i = 1 while i < len(args): if args[i] == '-c' or args[i] == '--client': if client: self.errExit("Only one client per run") client = args[i + 1] print(client) i = i + 1 elif args[i] == '--days': if maxdate: self.errExit("Max date specified multiple times") try: maxdate = datetime.datetime.now() - datetime.timedelta( days=int(args[i + 1])) except: self.log.error("Invalid number of days: %s" % args[i + 1]) raise SystemExit(-1) i = i + 1 elif args[i] == '--expired': expired = True i = i + 1 if expired: if state: self.log.error("--state is not valid with --expired") raise SystemExit(-1) self.purge_expired(maxdate) else: self.purge(client, maxdate, state) else: print("Unknown command: %s" % args[0])