def purge_expired(self, maxdate=None): '''Purge expired clients from the database''' if maxdate: if not isinstance(maxdate, datetime.datetime): raise TypeError("maxdate is not a DateTime object") self.log.debug("Filtering by maxdate: %s" % maxdate) clients = Client.objects.filter(expiration__lt=maxdate) else: clients = Client.objects.filter(expiration__isnull=False) for client in clients: self.log.debug("Purging client %s" % client) Interaction.objects.filter(client=client).delete() client.delete() self.log.debug("Pruning orphan Performance objects") Performance.prune_orphans()
def _import_interaction(self, interaction): """Real import function""" hostname = interaction['hostname'] stats = etree.fromstring(interaction['stats']) metadata = interaction['metadata'] server = metadata['server'] client = cache.get(hostname) if not client: client, created = Client.objects.get_or_create(name=hostname) if created: self.logger.debug("Client %s added to the db" % hostname) cache.set(hostname, client) timestamp = datetime(*strptime(stats.get('time'))[0:6]) if len(Interaction.objects.filter(client=client, timestamp=timestamp)) > 0: self.logger.warn("Interaction for %s at %s already exists" % (hostname, timestamp)) return if 'profile' in metadata: profile, created = \ Group.objects.get_or_create(name=metadata['profile']) else: profile = None flags = {'dry_run': False, 'only_important': False} for flag in stats.findall('./Flags/Flag'): value = flag.get('value', default='false').lower() == 'true' name = flag.get('name') if name in flags: flags[name] = value inter = Interaction(client=client, timestamp=timestamp, state=stats.get('state', default="unknown"), repo_rev_code=stats.get('revision', default="unknown"), good_count=stats.get('good', default="0"), total_count=stats.get('total', default="0"), server=server, profile=profile, **flags) inter.save() self.logger.debug("Interaction for %s at %s with INSERTED in to db" % (client.id, timestamp)) # FIXME - this should be more efficient for group_name in metadata['groups']: group = cache.get("GROUP_" + group_name) if not group: group, created = Group.objects.get_or_create(name=group_name) if created: self.logger.debug("Added group %s" % group) cache.set("GROUP_" + group_name, group) inter.groups.add(group) for bundle_name in metadata.get('bundles', []): bundle = cache.get("BUNDLE_" + bundle_name) if not bundle: bundle, created = \ Bundle.objects.get_or_create(name=bundle_name) if created: self.logger.debug("Added bundle %s" % bundle) cache.set("BUNDLE_" + bundle_name, bundle) inter.bundles.add(bundle) inter.save() counter_fields = {TYPE_BAD: 0, TYPE_MODIFIED: 0, TYPE_EXTRA: 0} pattern = [('Bad/*', TYPE_BAD), ('Extra/*', TYPE_EXTRA), ('Modified/*', TYPE_MODIFIED)] updates = dict([(etype, []) for etype in Interaction.entry_types]) for (xpath, state) in pattern: for entry in stats.findall(xpath): counter_fields[state] = counter_fields[state] + 1 # handle server failures differently failure = entry.get('failure', '') if failure: act_dict = dict(name=entry.get("name"), entry_type=entry.tag, message=failure) newact = FailureEntry.entry_get_or_create(act_dict) updates['failures'].append(newact) continue updatetype = entry.tag.lower() + "s" update = getattr(self, "_import_%s" % entry.tag, self._import_unknown)(entry, state) if update is not None: updates[updatetype].append(update) inter.bad_count = counter_fields[TYPE_BAD] inter.modified_count = counter_fields[TYPE_MODIFIED] inter.extra_count = counter_fields[TYPE_EXTRA] inter.save() for entry_type in updates.keys(): # batch this for sqlite i = 0 while (i < len(updates[entry_type])): getattr(inter, entry_type).add(*updates[entry_type][i:i + 100]) i += 100 # performance metrics for times in stats.findall('OpStamps'): for metric, value in list(times.items()): Performance(interaction=inter, metric=metric, value=value).save()
def purge(self, client=None, maxdate=None, state=None): '''Purge historical data from the database''' filtered = False # indicates whether or not a client should be deleted if not client and not maxdate and not state: self.errExit("Reports.prune: Refusing to prune all data") ipurge = Interaction.objects if client: try: cobj = Client.objects.get(name=client) ipurge = ipurge.filter(client=cobj) except Client.DoesNotExist: self.log.error("Client %s not in database" % client) raise SystemExit(-1) self.log.debug("Filtering by client: %s" % client) if maxdate: filtered = True if not isinstance(maxdate, datetime.datetime): raise TypeError("maxdate is not a DateTime object") self.log.debug("Filtering by maxdate: %s" % maxdate) ipurge = ipurge.filter(timestamp__lt=maxdate) if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3': grp_limit = 100 else: grp_limit = 1000 if state: filtered = True if state not in ('dirty', 'clean', 'modified'): raise TypeError("state is not one of the following values " + \ "('dirty','clean','modified')") self.log.debug("Filtering by state: %s" % state) ipurge = ipurge.filter(state=state) count = ipurge.count() rnum = 0 try: while rnum < count: grp = list(ipurge[:grp_limit].values("id")) # just in case... if not grp: break Interaction.objects.filter(id__in=[x['id'] for x in grp]).delete() rnum += len(grp) self.log.debug("Deleted %s of %s" % (rnum, count)) except: self.log.error("Failed to remove interactions") (a, b, c) = sys.exc_info() msg = traceback.format_exception(a, b, c, limit=2)[-1][:-1] del a, b, c self.log.error(msg) # bulk operations bypass the Interaction.delete method self.log.debug("Pruning orphan Performance objects") Performance.prune_orphans() self.log.debug("Pruning orphan Reason objects") Reason.prune_orphans() if client and not filtered: '''Delete the client, ping data is automatic''' try: self.log.debug("Purging client %s" % client) cobj.delete() except: self.log.error("Failed to delete client %s" % client) (a, b, c) = sys.exc_info() msg = traceback.format_exception(a, b, c, limit=2)[-1][:-1] del a, b, c self.log.error(msg)