コード例 #1
0
ファイル: test_psql.py プロジェクト: holgerBerger/ludalo
  def __init__(self, filename, hostfile=None, dbFile='sqlite_new.db'):

    self.filename = filename  
    self.myDB = PSQLObject(dbFile)
    self.filesize = os.path.getsize(filename)

    if hostfile:
      self.readhostfile(hostfile)
コード例 #2
0
ファイル: test_psql.py プロジェクト: holgerBerger/ludalo
class Logfile:

  def __init__(self, filename, hostfile=None, dbFile='sqlite_new.db'):

    self.filename = filename  
    self.myDB = PSQLObject(dbFile)
    self.filesize = os.path.getsize(filename)

    if hostfile:
      self.readhostfile(hostfile)

  def eta(self,secs):
    if secs<60:
      return "%2.2d sec       "%secs
    else:
      return "%d min %2.2d sec" % (secs/60, secs%60)
    
#------------------------------------------------------------------------------
  def read(self):
    ''' action is HERE'''
    f = open(self.filename,"r")
    counter=0
    acounter=0
    starttime = time.time()

    #1.0;hmds1;time;mdt;reqs;
    #1.0;hoss3;time;ost;rio,rb,wio,wb;
    for line in f:
      sp = line[:-1].split(";")  # ignore the line break at the end 

      if line.startswith("#"): # this is a head line
        server = sp[1]
        timestamp = sp[2] 
        stype = sp[3] # mdt or ost

        # add server and type to the db
        self.myDB.insert_server(server, stype)


        # add nids to the database 
        for nid in sp[5:]:
            self.insert_nid_server(server, nid)

      else:
#--------------------- if not headline -----------------------------------------
        # form a hash digest of the line and check if line is already in database,
        # if so, do not add line again to avoid bloat and wrong results for sums over data
        hexdigest = hashlib.sha224(line).hexdigest()
        if self.myDB.has_hash(hexdigest):
          #print "line collision"
          continue

        server = sp[0]
        timestamp = sp[1]
        source = sp[2]
        value_tupel = sp[3] # values for ost

        self.myDB.insert_timestamp(timestamp) 
        self.myDB.insert_source(source)
        
        # add ost global
        self.insert_ost_global(server, value_tupel, timestamp)

        self.insert_nids(server, timestamp, source, sp[4:])
        
#--------------------- progress bar --------------------------------------------
      counter+=1
      if counter%10 == 0:
        duration = (time.time() - starttime)
        fraction = (float(f.tell())/float(self.filesize))
        endtime = duration * (1.0/ fraction) - duration
        #printString = str("\rinserted %d records / %d%% ETA = %s"
        #                 %(counter,int(fraction*100.0), self.eta(endtime)))
        printString = str("\rinserted %9d records [%s] ETA = %s"
                         %(counter,"|"*int(fraction*20.0)+"\\|/-"[acounter%4]+"-"*(19-int(fraction*20.0)), self.eta(endtime)))
        print printString,
        sys.stdout.flush()
        acounter+=1
#------------------------------------------------------------------------------
    endtime = time.time()
    print "used %s to insert data." % self.eta(endtime-starttime)

#-------------------------------------------------------------------------------

  def readhostfile(self, hostfile):
    try:
      f = open(hostfile, "r")
    except:
      return
    for l in f:
      if not l.startswith('#'):
        sp = l[:-1].split()
        if len(sp)==0: continue
        ip = sp[0]
        name = sp[1]
        self.myDB.hostfilemap[ip]=name
    print "read",len(self.myDB.hostfilemap),"host mappings"
    f.close()
#-------------------------------------------------------------------------------
  def insert_ost_global(self, server, tuple, timestamp):
      self.myDB.insert_ost_global(server, tuple, timestamp)

  def insert_nid_server(self, server, one_nid):
      self.myDB.add_nid_server(server, one_nid)

  def insert_nid(self, server, timeStamp, source, nidvals_Tup, nidID):
      ''' methode to insert only one nid value tuple '''
      self.myDB.add_nid_values(server, timeStamp, source, nidvals_Tup, nidID)

  def insert_nids(self, server, timestamp, source, nidvals):
    stype = self.myDB.servertype[server]
    il_ost = []
    il_mdt = []

    for i in range(len(nidvals)):
      nidid = self.myDB.globalnidmap[self.myDB.per_server_nids[server][i]]
      timeid = self.myDB.timestamps[timestamp]
      sourceid = self.myDB.sources[source]

      if nidvals[i]!="":
        if stype == 'ost':
          temp = [timeid, sourceid, nidid]
          temp.extend(nidvals[i].split(','))
          il_ost.append(temp)

        if stype == 'mdt':
          il_mdt.append((timeid, sourceid, nidid, nidvals[i]))
    
    self.myDB.insert_ost_samples(il_ost)
    self.myDB.insert_mdt_samples(il_mdt)