def print_job(self, jobName):
# test if job exist
jobID = self.getJobID(jobName)
if not jobID:
print '404', jobName
exit(1)
# test if job running
job_start_end = self.get_job_start_end(jobID)
if not job_start_end:
print 'Not enough samples for job', jobName
exit(1)
jobRunning = self.job_running(jobID)
if jobRunning:
job_end = int(time.time())
else:
job_end = job_start_end[1]
job_start = job_start_end[0]
if self.verbose:
print 'job id ', jobID
self.explainJob(jobID)
print 'job Running? ', jobRunning
print 'job_end ', job_end
print 'job_start ', job_start, '\n'
query = ''' select
jobs.nodelist
from
jobs,
users
where
jobs.id = %s
and users.id = jobs.owner'''
self.c.execute(query, jobID)
non = self.c.fetchall()
non = non[0][0]
non = len(non.split(','))
# get job data
option = (jobID, job_start, job_end)
query = '''
select
timestamps.c_timestamp,
sum(samples_ost.wb),
sum(samples_ost.wio),
sum(samples_ost.rb),
sum(samples_ost.rio)
from
nids
join
nodelist ON nids.id = nodelist.nid
join
jobs ON jobs.id = nodelist.job
and jobs.id = %s
join
samples_ost ON nids.id = samples_ost.nid
join
timestamps ON timestamps.id = samples_ost.timestamp_id,
filesystems
where
timestamps.c_timestamp between %s and %s
group by timestamps.c_timestamp
'''
values_np = self.query_to_npArray(query, option)
if values_np is not None and values_np.any():
query = ''' select
c_timestamp
from
timestamps
where
c_timestamp
between
%s and %s
'''
allTimestamps = self.query_to_npArray(query, (job_start, job_end))
values_np = self.np_fillAndSort(values_np, allTimestamps)
# ignore division by zero
# it is posibel that wbs has a value but wio is zero. in this case
# replaced with 0 (np.nan_to_num)
np.seterr(divide='ignore', invalid='ignore')
# transform data
timestamps = values_np[:, 0]
duration = max(timestamps) - min(timestamps) # in seconds
duration = duration / 60 # in minutes
wbs = values_np[:, 1]
wbs_per_second = wbs / 60
wbs_kb_per_s = wbs_per_second / 1024
wbs_mb_per_s = wbs_kb_per_s / 1024
wio = values_np[:, 2]
wio_volume_in_kb = np.nan_to_num((wbs / wio) / 1024)
rbs = values_np[:, 3]
rbs_per_second = rbs / 60
rbs_kb_per_s = rbs_per_second / 1024
rbs_mb_per_s = rbs_kb_per_s / 1024
rio = values_np[:, 4]
rio_volume_in_kb = np.nan_to_num((rbs / rio) / 1024)
# print job data
if jobRunning:
path = '/var/www/ludalo-web/calc/jobs/' + str(jobName)
else:
path = '/var/www/ludalo-web/calc/jobs/' + str(jobName)
print jobName, get_fingerprint(duration, wbs, rbs, rio, wio, non)
if self.verbose:
plotJob(timestamps,
rbs_mb_per_s, rio_volume_in_kb,
wbs_mb_per_s, wio_volume_in_kb,
path, self.verbose)
print 'Read Informations: \n -----------'
print ' Total bytes:', np.sum(rbs), 'Avr [MB/s]:', round(np.average(rbs_mb_per_s), 2), 'std [MB/s]:', round(np.std(rbs_mb_per_s), 2)
print ' Total IOs:', np.sum(rio), 'Avr [IO/s]:', round(np.average(rio / 60), 2), 'std [IO/s]:', round(np.std(rio / 60), 2)
print ' Avr [KB/IO]', round(np.average(rio_volume_in_kb), 2), 'std [KB/IO]:', round(np.std(rio_volume_in_kb), 2), '\n'
print 'Write Informations: \n -----------'
print ' Total bytes:', np.sum(wbs), 'Avr [MB/s]:', round(np.average(wbs_mb_per_s), 2), 'std [MB/s]:', round(np.std(wbs_mb_per_s), 2)
print ' Total IOs:', round(np.sum(wio), 2), 'Avr [IO/s]:', round(np.average(wio / 60), 2), 'std [IO/s]:', round(np.std(wio / 60), 2)
print ' Avr [KB/IO]', round(np.average(wio_volume_in_kb), 2), 'std [KB/IO]:', round(np.std(wio_volume_in_kb), 2), '\n'
print 'done'
else:
print 'job not in sample range'
def print_Filesystem(self, window, fs='univ_1', png_path='/var/www/ludalo-web/calc/'):
# used in main
# getting all informations out of the database
query = ('''
select
timestamps.c_timestamp, sum(wb), sum(wio), sum(rb), sum(rio)
from
ost_values,
targets,
filesystems,
timestamps
where
ost_values.target = targets.id
and targets.fsid = filesystems.id
and ost_values.timestamp_id = timestamps.id
and filesystems.filesystem = %s
and c_timestamp between
unix_timestamp()-%s and unix_timestamp()
group by timestamps.c_timestamp
order by timestamps.c_timestamp''')
values_np = self.query_to_npArray(query, (fs, int(window)))
if values_np is not None and not values_np.any():
print 'no fs data found'
exit(1)
query = ''' select
c_timestamp
from
timestamps
where
c_timestamp
between
unix_timestamp()-%s and unix_timestamp()
'''
allTimestamps = self.query_to_npArray(query, int(window))
if values_np is not None and not values_np.any():
print 'no fs timestamps found'
exit(1)
values_np = self.np_fillAndSort(values_np, allTimestamps)
# ignore division by zero
# it is posibel that wbs has a value but wio is zero. in this case
# replaced with 0 (np.nan_to_num)
np.seterr(divide='ignore', invalid='ignore')
timestamps = values_np[:, 0]
wbs = values_np[:, 1]
wbs_per_second = wbs / 60
wbs_kb_per_s = wbs_per_second / 1024
wbs_mb_per_s = wbs_kb_per_s / 1024
wio = values_np[:, 2]
wio_volume_in_kb = np.nan_to_num((wbs / wio) / 1024)
rbs = values_np[:, 3]
rbs_per_second = rbs / 60
rbs_kb_per_s = rbs_per_second / 1024
rbs_mb_per_s = rbs_kb_per_s / 1024
rio = values_np[:, 4]
rio_volume_in_kb = np.nan_to_num((rbs / rio) / 1024)
path = png_path + str(fs)
plotJob(timestamps,
rbs_mb_per_s, rio_volume_in_kb,
wbs_mb_per_s, wio_volume_in_kb,
path, self.verbose)
update_query = '''
UPDATE web_fs_cashe
SET
t_time=%s,
topSpeedWB=%s, topSpeedRB=%s,
avrSpeedWB=%s, avrSpeedRB=%s,
ioSumWB=%s, ioSumRB=%s,
totalWB=%s, totalRB=%s,
avrIoSizeW=%s, avrIoSizeR=%s
WHERE fs=%s
'''
self.c.execute(update_query, (np.max(timestamps),
np.max(wbs), np.max(rbs),
np.average(wbs), np.average(rbs),
np.sum(wio), np.sum(rio),
np.sum(wbs), np.sum(rbs),
np.average(wio_volume_in_kb),
np.average(rio_volume_in_kb),
fs))
self.conn.commit()
if self.verbose:
print 'Read Informations: \n -----------'
print 'Total bytes:', np.sum(rbs), 'Avr [MB/s]:', round(np.average(rbs_mb_per_s), 2), 'std [MB/s]:', round(np.std(rbs_mb_per_s), 2)
print 'Total IOs:', np.sum(rio), 'Avr [IO/s]:', round(np.average(rio / 60), 2), 'std [IO/s]:', round(np.std(rio / 60), 2)
print 'Avr [KB/IO]', round(np.average(rio_volume_in_kb), 2), 'std [KB/IO]:', round(np.std(rio_volume_in_kb), 2), '\n'
print 'Write Informations: \n -----------'
print 'Total bytes:', np.sum(wbs), 'Avr [MB/s]:', round(np.average(wbs_mb_per_s), 2), 'std [MB/s]:', round(np.std(wbs_mb_per_s), 2)
print 'Total IOs:', round(np.sum(wio), 2), 'Avr [IO/s]:', round(np.average(wio / 60), 2), 'std [IO/s]:', round(np.std(wio / 60), 2)
print 'Avr [KB/IO]', round(np.average(wio_volume_in_kb), 2), 'std [KB/IO]:', round(np.std(wio_volume_in_kb), 2), '\n'
print 'done'
