def importXmlFile(self, bid, filename):
db = self.db
if filename.endswith('.gz'):
f = mygzip(filename)
else:
f = open(filename)
count = 0
error = 0
with f as xml_file:
tree = etree.iterparse(xml_file)
for events, row in tree:
tag_name = row.tag.lower()
if tag_name == 'httpsample':
tag_name = 'sample'
table_name = self._prefix + tag_name
if table_name not in self.table_names:
continue
try:
logging.debug(row.attrib.keys())
cols = 'bid' + ', ' + ', '.join(row.attrib.keys())
values = ('?, ' * (len(row.attrib.keys()) + 1))[:-2]
data = row.attrib.values()
data.insert(0, bid)
db.execute(INSERT_QUERY.format(
table=table_name,
columns=cols,
values=values), data)
count += 1
except Exception, e:
logging.warning(e)
error += 1
finally:
row.clear()
def importXmlFile(self, bid, filename):
db = self.db
if filename.endswith('.gz'):
f = mygzip(filename)
else:
f = open(filename)
count = 0
error = 0
with f as xml_file:
tree = etree.iterparse(xml_file)
for events, row in tree:
tag_name = row.tag.lower()
if tag_name == 'httpsample':
tag_name = 'sample'
table_name = self._prefix + tag_name
if table_name not in self.table_names:
continue
try:
logging.debug(row.attrib.keys())
# wrapping each column in quotes
cols = 'bid' + ', ' + ', '.join('"{0}"'.format(w) for w in row.attrib.keys())
values = ('?, ' * (len(row.attrib.keys()) + 1))[:-2]
data = row.attrib.values()
data.insert(0, bid)
sql_query = INSERT_QUERY.format(table=table_name, columns=cols, values=values)
logging.debug(sql_query)
db.execute(sql_query, data)
count += 1
except Exception, e:
logging.warning(e)
error += 1
finally:
row.clear()
def importOtherFormat(self, bid, filename):
"""Handle CSV file"""
db = self.db
if filename.endswith('.gz'):
f = mygzip(filename)
else:
f = open(filename)
jtlReader = csv.reader(f)
count = 0
error = 0
insert_query = None
for row in jtlReader:
row = [unicode(cell, 'utf-8').encode('ascii', 'ignore') for cell in row]
if insert_query is None:
if len(row) not in CSV_COLS.keys():
raise ValueError('Unsupported format')
values = ('?, ' * (len(row) + 1))[:-2]
insert_query = 'INSERT INTO j_sample (bid, ' + ', '.join(CSV_COLS[len(row)]) + ') VALUES (' + values + ')'
try:
db.execute(insert_query, [bid, ] + row)
count += 1
except Exception, e:
logging.warning(e)
error += 1
print "x",
def doImport(self, bid, filename):
c = self.db.cursor()
options = self.options
host = options.host
t = (bid, host, options.comment)
logging.info("Importing sar file %s into bid: %s" % (filename, bid))
c.execute('INSERT INTO host (bid, host, comment) VALUES (?, ?, ?)', t)
if filename.endswith('.gz'):
f = mygzip(filename)
else:
f = open(filename)
in_cpu = False
in_disk = False
count = 0
while True:
line = f.readline()
if not line:
break
if 'CPU %usr' in line:
in_cpu = True
continue
if 'DEV tps' in line:
in_disk = True
continue
if in_cpu:
if not 'all' in line:
continue
if 'Average' in line:
in_cpu = False
continue
t = [bid, host] + line.split()
t.remove('all')
c.execute(
'INSERT INTO cpu (bid, host, date, usr, nice, sys, iowait, steal, irq, soft, '
'guest, idle) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
t)
count += 1
elif in_disk:
if 'Average' in line:
in_disk = False
break
r = line.split()
t = [bid, host, r[0], r[1], r[2], r[3], r[4], r[9]]
c.execute(
'INSERT INTO disk (bid, host, date, dev, tps, rd_sec_per_s, wr_sec_per_s, util)'
' VALUES (?, ?, ?, ?, ?, ?, ?, ?)', t)
count += 1
c.close()
self.db.commit()
logging.info('%d lines imported.' % count)
def doImport(self, bid, filename):
c = self.db.cursor()
options = self.options
host = options.host
t = (bid, host, options.comment)
logging.info("Importing sar file %s into bid: %s" % (filename, bid))
c.execute('INSERT INTO host (bid, host, comment) VALUES (?, ?, ?)', t)
if filename.endswith('.gz'):
f = mygzip(filename)
else:
f = open(filename)
in_cpu = False
in_disk = False
count = 0
while True:
line = f.readline()
if not line:
break
if 'CPU %usr' in line:
in_cpu = True
continue
if 'DEV tps' in line:
in_disk = True
continue
if in_cpu:
if not 'all' in line:
continue
if 'Average' in line:
in_cpu = False
continue
t = [bid, host] + line.split()
t.remove('all')
c.execute('INSERT INTO cpu (bid, host, date, usr, nice, sys, iowait, steal, irq, soft, '
'guest, idle) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', t)
count += 1
elif in_disk:
if 'Average' in line:
in_disk = False
break
r = line.split()
t = [bid, host, r[0], r[1], r[2], r[3], r[4], r[9]]
c.execute('INSERT INTO disk (bid, host, date, dev, tps, rd_sec_per_s, wr_sec_per_s, util)'
' VALUES (?, ?, ?, ?, ?, ?, ?, ?)', t)
count += 1
c.close()
self.db.commit()
logging.info('%d lines imported.' % count)