def load_stats(cdata, sdata, vlevel, quick=False, location=''): cursor = connection.cursor() clients = {} cursor.execute("SELECT name, id from reports_client;") [clients.__setitem__(a, b) for a, b in cursor.fetchall()] for node in sdata.findall('Node'): name = node.get('name') if not name in clients: cursor.execute(\ "INSERT INTO reports_client VALUES (NULL, %s, %s, NULL, NULL)", [datetime.now(), name]) clients[name] = cursor.lastrowid if vlevel > 0: print("Client %s added to db" % name) else: if vlevel > 0: print("Client %s already exists in db" % name) pingability = {} [pingability.__setitem__(n.get('name'), n.get('pingable', default='N')) \ for n in cdata.findall('Client')] for node in sdata.findall('Node'): name = node.get('name') c_inst = Client.objects.filter(id=clients[name])[0] try: pingability[name] except KeyError: pingability[name] = 'N' for statistics in node.findall('Statistics'): t = strptime(statistics.get('time')) # Maybe replace with django.core.db typecasts typecast_timestamp()? # import from django.backends util timestamp = datetime(t[0], t[1], t[2], t[3], t[4], t[5]) ilist = Interaction.objects.filter(client=c_inst, timestamp=timestamp) if ilist: current_interaction = ilist[0] if vlevel > 0: print("Interaction for %s at %s with id %s already exists"%(clients[name], datetime(t[0],t[1],t[2],t[3],t[4],t[5]),current_interaction.id)) continue else: newint = Interaction(client=c_inst, timestamp = timestamp, state = statistics.get('state', default="unknown"), repo_rev_code = statistics.get('revision',default="unknown"), client_version = statistics.get('client_version',default="unknown"), goodcount = statistics.get('good',default="0"), totalcount = statistics.get('total',default="0"), server = location) newint.save() current_interaction = newint if vlevel > 0: print("Interaction for %s at %s with id %s INSERTED in to db"%(clients[name], timestamp, current_interaction.id)) pattern = [('Bad/*', TYPE_CHOICES[0]), ('Extra/*', TYPE_CHOICES[2]), ('Modified/*', TYPE_CHOICES[1]),] for (xpath, type) in pattern: for x in statistics.findall(xpath): kargs = build_reason_kwargs(x) if not quick: rls = Reason.objects.filter(**kargs) else: rls = [] if rls: rr = rls[0] if vlevel > 0: print "Reason exists: %s"% (rr.id) else: rr = Reason(**kargs) rr.save() if vlevel > 0: print "Created reason: %s" % rr.id links = Entries.objects.filter(name=x.get('name'), kind=x.tag) if links: entry = links[0] else: entry = Entries(name=x.get('name'), kind=x.tag) entry.save() interaction = Entries_interactions(entry=entry, reason=rr, interaction=current_interaction, type=type[0]) interaction.save() if vlevel > 0: print "%s interaction created with reason id %s and entry %s" % (xpath, rr.id, entry.id) for times in statistics.findall('OpStamps'): for metric, value in times.items(): if not quick: mmatch = Performance.objects.filter(metric=metric, value=value) else: mmatch = [] if mmatch: item_id = mmatch[0].id else: mperf = Performance(metric=metric, value=value) mperf.save() item_id = mperf.id try: cursor.execute("INSERT INTO reports_performance_interaction VALUES (NULL, %s, %s);", [item_id, current_interaction.id]) except: pass if vlevel > 1: print("----------------INTERACTIONS SYNCED----------------") cursor.execute("select reports_interaction.id, x.client_id from (select client_id, MAX(timestamp) as timer from reports_interaction Group BY client_id) x, reports_interaction where reports_interaction.client_id = x.client_id AND reports_interaction.timestamp = x.timer") for row in cursor.fetchall(): cursor.execute("UPDATE reports_client SET current_interaction_id = %s where reports_client.id = %s", [row[0],row[1]]) if vlevel > 1: print("------------LATEST INTERACTION SET----------------") for key in pingability.keys(): if key not in clients: #print "Ping Save Problem with client %s" % name continue cmatch = Client.objects.filter(id=clients[key])[0] pmatch = Ping.objects.filter(client=cmatch).order_by('-endtime') if pmatch: if pmatch[0].status == pingability[key]: pmatch[0].endtime = datetime.now() pmatch[0].save() else: newp = Ping(client=cmatch, status=pingability[key], starttime=datetime.now(), endtime=datetime.now()) newp.save() else: newp = Ping(client=cmatch, status=pingability[key], starttime=datetime.now(), endtime=datetime.now()) newp.save() if vlevel > 1: print "---------------PINGDATA SYNCED---------------------" connection._commit()