def _restructure():
"""major restructure of reporting data"""
# run any migrations from the previous schema
try:
from Bcfg2.Server.Reports.updatefix import update_database
update_database()
except:
logger.error("Failed to run legacy schema updates", exc_info=1)
return False
# try to avoid dangling transactions
if not migrate_stage1():
return
try:
entries = {}
for ent in legacy_models.Entries.objects.all():
entries[ent.id] = ent
except:
logger.error("Failed to populate entries dict", exc_info=1)
return False
try:
fperms = _migrate_perms()
except:
logger.error("Failed create FilePerms objects", exc_info=1)
return False
failures = []
int_count = legacy_models.Interaction.objects.count()
if int_count == 0:
logger.error("Found no legacy interactions")
return False
int_ctr = 0
start_time = 0
for inter in BatchFetch(legacy_models.Interaction.objects.select_related("metadata", "client").all()):
if int_ctr % 1000 == 0:
if int_ctr > 0:
logger.info("Migrated %s of %s interactions in %ss" % (int_ctr, int_count, time.time() - start_time))
else:
logger.info("Migrating interactions")
start_time = time.time()
try:
_migrate_transaction(inter, entries, fperms)
except:
logger.error("Failed to migrate interaction %s for %s" % (inter.id, inter.client.name), exc_info=1)
failures.append(inter.id)
int_ctr += 1
if not failures:
logger.info("Successfully restructured reason data")
return True
logger.info("Updating recent interactions")
for newint in new_models.Interaction.objects.recent():
try:
newint.save()
except:
logger.error("Failed to set current interaction %s for %s" % (newint.id, newint.client.name), exc_info=1)
def __init__(self, core, datastore):
Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
Bcfg2.Server.Plugin.ThreadedStatistics.__init__(self, core, datastore)
Bcfg2.Server.Plugin.PullSource.__init__(self)
self.cpath = "%s/Metadata/clients.xml" % datastore
self.core = core
logger.debug("Searching for new models to add to the statistics database")
try:
update_database()
except Exception, inst:
logger.debug(str(inst))
logger.debug(str(type(inst)))
def __init__(self, core, datastore):
Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
Bcfg2.Server.Plugin.ThreadedStatistics.__init__(self, core, datastore)
Bcfg2.Server.Plugin.PullSource.__init__(self)
self.cpath = "%s/Metadata/clients.xml" % datastore
self.core = core
logger.debug(
"Searching for new models to add to the statistics database")
try:
update_database()
except Exception:
inst = sys.exc_info()[1]
logger.debug(str(inst))
logger.debug(str(type(inst)))
def __call__(self, args):
Bcfg2.Server.Admin.Mode.__call__(self, args)
if len(args) == 0 or args[0] == '-h':
print(self.__usage__)
raise SystemExit(0)
verb = 0
if '-v' in args or '--verbose' in args:
self.log.setLevel(logging.DEBUG)
verb = 1
if '-q' in args or '--quiet' in args:
self.log.setLevel(logging.WARNING)
# FIXME - dry run
if args[0] in self.django_commands:
self.django_command_proxy(args[0])
elif args[0] == 'scrub':
self.scrub()
elif args[0] == 'init':
update_database()
elif args[0] == 'update':
update_database()
elif args[0] == 'load_stats':
quick = '-O3' in args
stats_file=None
clients_file=None
i=1
while i < len(args):
if args[i] == '-s' or args[i] == '--stats':
stats_file = args[i+1]
if stats_file[0] == '-':
self.errExit("Invalid statistics file: %s" % stats_file)
elif args[i] == '-c' or args[i] == '--clients-file':
clients_file = args[i+1]
if clients_file[0] == '-':
self.errExit("Invalid clients file: %s" % clients_file)
i = i + 1
self.load_stats(stats_file, clients_file, verb, quick)
else:
print "Unknown command: %s" % args[0]
def _restructure():
"""major restructure of reporting data"""
# run any migrations from the previous schema
try:
from Bcfg2.Server.Reports.updatefix import update_database
update_database()
except:
logger.error("Failed to run legacy schema updates", exc_info=1)
return False
# try to avoid dangling transactions
if not migrate_stage1():
return
try:
entries = {}
for ent in legacy_models.Entries.objects.all():
entries[ent.id] = ent
except:
logger.error("Failed to populate entries dict", exc_info=1)
return False
try:
fperms = _migrate_perms()
except:
logger.error("Failed create FilePerms objects", exc_info=1)
return False
failures = []
int_count = legacy_models.Interaction.objects.count()
if int_count == 0:
logger.error("Found no legacy interactions")
return False
int_ctr = 0
start_time = 0
for inter in BatchFetch(legacy_models.Interaction.objects.\
select_related('metadata', 'client').all()):
if int_ctr % 1000 == 0:
if int_ctr > 0:
logger.info("Migrated %s of %s interactions in %ss" % \
(int_ctr, int_count, time.time() - start_time))
else:
logger.info("Migrating interactions")
start_time = time.time()
try:
_migrate_transaction(inter, entries, fperms)
except:
logger.error("Failed to migrate interaction %s for %s" %
(inter.id, inter.client.name),
exc_info=1)
failures.append(inter.id)
int_ctr += 1
if not failures:
logger.info("Successfully restructured reason data")
return True
logger.info("Updating recent interactions")
for newint in new_models.Interaction.objects.recent():
try:
newint.save()
except:
logger.error("Failed to set current interaction %s for %s" %
(newint.id, newint.client.name),
exc_info=1)
print "Could not read bcfg2.conf; exiting"
raise SystemExit, 1
try:
statsdata = XML(open(statpath).read())
except (IOError, XMLSyntaxError):
print("StatReports: Failed to parse %s"%(statpath))
raise SystemExit, 1
if not clientpath:
try:
clientspath = "%s/Metadata/clients.xml" % \
cf.get('server', 'repository')
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
print "Could not read bcfg2.conf; exiting"
raise SystemExit, 1
try:
clientsdata = XML(open(clientspath).read())
except (IOError, XMLSyntaxError):
print("StatReports: Failed to parse %s"%(clientspath))
raise SystemExit, 1
q = '-O3' in sys.argv
try:
location = cf.get('components', 'bcfg2')
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
print "Could not read bcfg2 from bcfg2.conf; exiting"
raise SystemExit, 1
# Be sure the database is ready for new schema
update_database()
load_stats(clientsdata, statsdata, verb, quick=q, location=location)
def __call__(self, args):
Bcfg2.Server.Admin.Mode.__call__(self, args)
if len(args) == 0 or args[0] == '-h':
print(self.__usage__)
raise SystemExit(0)
verb = 0
if '-v' in args or '--verbose' in args:
self.log.setLevel(logging.DEBUG)
verb = 1
if '-q' in args or '--quiet' in args:
self.log.setLevel(logging.WARNING)
# FIXME - dry run
if args[0] in self.django_commands:
self.django_command_proxy(args[0])
elif args[0] == 'scrub':
self.scrub()
elif args[0] == 'init':
update_database()
elif args[0] == 'update':
update_database()
elif args[0] == 'load_stats':
quick = '-O3' in args
stats_file = None
clients_file = None
i = 1
while i < len(args):
if args[i] == '-s' or args[i] == '--stats':
stats_file = args[i + 1]
if stats_file[0] == '-':
self.errExit("Invalid statistics file: %s" % stats_file)
elif args[i] == '-c' or args[i] == '--clients-file':
clients_file = args[i + 1]
if clients_file[0] == '-':
self.errExit("Invalid clients file: %s" % clients_file)
i = i + 1
self.load_stats(stats_file, clients_file, verb, quick)
elif args[0] == 'purge':
expired = False
client = None
maxdate = None
state = None
i = 1
while i < len(args):
if args[i] == '-c' or args[i] == '--client':
if client:
self.errExit("Only one client per run")
client = args[i + 1]
print(client)
i = i + 1
elif args[i] == '--days':
if maxdate:
self.errExit("Max date specified multiple times")
try:
maxdate = datetime.datetime.now() - datetime.timedelta(days=int(args[i + 1]))
except:
self.log.error("Invalid number of days: %s" % args[i + 1])
raise SystemExit(-1)
i = i + 1
elif args[i] == '--expired':
expired = True
i = i + 1
if expired:
if state:
self.log.error("--state is not valid with --expired")
raise SystemExit(-1)
self.purge_expired(maxdate)
else:
self.purge(client, maxdate, state)
else:
print("Unknown command: %s" % args[0])
raise SystemExit(1)
try:
encoding = cf.get('components', 'encoding')
except:
encoding = 'UTF-8'
if not clientpath:
try:
clientspath = "%s/Metadata/clients.xml" % \
cf.get('server', 'repository')
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
print("Could not read bcfg2.conf; exiting")
raise SystemExit(1)
try:
clientsdata = XML(open(clientspath).read())
except (IOError, XMLSyntaxError):
print("StatReports: Failed to parse %s" % (clientspath))
raise SystemExit(1)
q = '-O3' in sys.argv
# Be sure the database is ready for new schema
update_database()
load_stats(clientsdata,
statsdata,
encoding,
verb,
logger,
quick=q,
location=platform.node())
def __call__(self, args):
Bcfg2.Server.Admin.Mode.__call__(self, args)
if len(args) == 0 or args[0] == '-h':
print(self.__usage__)
raise SystemExit(0)
verb = 0
if '-v' in args or '--verbose' in args:
self.log.setLevel(logging.DEBUG)
verb = 1
if '-q' in args or '--quiet' in args:
self.log.setLevel(logging.WARNING)
# FIXME - dry run
if args[0] in self.django_commands:
self.django_command_proxy(args[0])
elif args[0] == 'scrub':
self.scrub()
elif args[0] == 'init':
update_database()
elif args[0] == 'update':
update_database()
elif args[0] == 'load_stats':
quick = '-O3' in args
stats_file = None
clients_file = None
i = 1
while i < len(args):
if args[i] == '-s' or args[i] == '--stats':
stats_file = args[i + 1]
if stats_file[0] == '-':
self.errExit("Invalid statistics file: %s" %
stats_file)
elif args[i] == '-c' or args[i] == '--clients-file':
clients_file = args[i + 1]
if clients_file[0] == '-':
self.errExit("Invalid clients file: %s" % clients_file)
i = i + 1
self.load_stats(stats_file, clients_file, verb, quick)
elif args[0] == 'purge':
expired = False
client = None
maxdate = None
state = None
i = 1
while i < len(args):
if args[i] == '-c' or args[i] == '--client':
if client:
self.errExit("Only one client per run")
client = args[i + 1]
print(client)
i = i + 1
elif args[i] == '--days':
if maxdate:
self.errExit("Max date specified multiple times")
try:
maxdate = datetime.datetime.now() - datetime.timedelta(
days=int(args[i + 1]))
except:
self.log.error("Invalid number of days: %s" %
args[i + 1])
raise SystemExit(-1)
i = i + 1
elif args[i] == '--expired':
expired = True
i = i + 1
if expired:
if state:
self.log.error("--state is not valid with --expired")
raise SystemExit(-1)
self.purge_expired(maxdate)
else:
self.purge(client, maxdate, state)
else:
print("Unknown command: %s" % args[0])
