def update_pytomo_rrd(self):
'''Insert data from the list of tuples (timestamp, parameter1, ...)
to the rrd.
'''
if not self.has_values:
config_pytomo.LOG.warn('RRD data update aborted')
return 1
# insert into rrd all the values for the extracted parameters to plot
# data[][TIMESTAMP_POSITION] is the timestamp
# data[][:TIMESTAMP_POSITION] represents the parameters to plot
for row in self.data:
# transform timestamp to epoch in local time
# TODO: check problems related to timezone
timestamp = row[TIMESTAMP_POSITION]
parameter_values = row[:TIMESTAMP_POSITION]
# function used in order to take advantage of the * operator and
# retrieve all the elements of an argument
identity = lambda *x: x
try:
rrdtool.update(self.rrd_file,
update_data_types(parameter_values) %
identity(lib_database.time_to_epoch(timestamp),
*format_null_values(*parameter_values)))
except rrdtool.error, mes:
config_pytomo.LOG.debug('Could not update the rrd with error'
' %s' % mes)
continue
#config_pytomo.LOG.debug('Updated rrd data: (%s, %s)' %
# (timestamp, str(format_null_values(*parameter_values))))
for index, parameter in enumerate(parameter_values):
if parameter is None:
self.unknown_values[index] += 1
def update_pytomo_rrd(self):
'''Insert data from the list of tuples (timestamp, parameter1, ...)
to the rrd.
'''
if not self.has_values:
config_pytomo.LOG.warn('RRD data update aborted')
return 1
# insert into rrd all the values for the extracted parameters to plot
# data[][TIMESTAMP_POSITION] is the timestamp
# data[][:TIMESTAMP_POSITION] represents the parameters to plot
for row in self.data:
# transform timestamp to epoch in local time
# TODO: check problems related to timezone
timestamp = row[TIMESTAMP_POSITION]
parameter_values = row[:TIMESTAMP_POSITION]
# function used in order to take advantage of the * operator and
# retrieve all the elements of an argument
identity = lambda *x: x
try:
rrdtool.update(self.rrd_file,
update_data_types(parameter_values) %
identity(lib_database.time_to_epoch(timestamp),
*format_null_values(*parameter_values)))
except rrdtool.error, mes:
config_pytomo.LOG.debug('Could not update the rrd with error'
' %s' % mes)
continue
#config_pytomo.LOG.debug('Updated rrd data: (%s, %s)' %
# (timestamp, str(format_null_values(*parameter_values))))
for index, parameter in enumerate(parameter_values):
if parameter is None:
self.unknown_values[index] += 1
def compute_average_values(data):
'''Function to return a tuple (start_crawl_time, end_crawl_time,
nr_videos, average_ping, average_download_time,
average_download_interruptions)
'''
# data is retrieved sorted from the database
start_crawl_time = data[0][TIMESTAMP_POSITION]\
[:MAX_TIMESTAMP_LENGTH]
end_crawl_time = data[-1][TIMESTAMP_POSITION]\
[:MAX_TIMESTAMP_LENGTH]
# total crawl time (sync time is added at the beginning and end for
# the plots)
total_time = (datetime.fromtimestamp(
lib_database.time_to_epoch(data[-1][TIMESTAMP_POSITION])) -
datetime.fromtimestamp(
lib_database.time_to_epoch(data[0][TIMESTAMP_POSITION])))
# each row in the dataset represents a video
nr_videos = len(data)
# filter values that are not None (can be zero)
not_none = lambda x: x is not None
# for some videos data cannot be retrieved
# average download time represents the average of DownloadTime
average_list = filter(
not_none, map(itemgetter(AVERAGE_PARAM.index('DownloadTime')), data))
average_download_time = average(average_list, len(average_list))
# average download interruptions represents the average of
# DownloadInterruptions
average_list = filter(
not_none,
map(itemgetter(AVERAGE_PARAM.index('DownloadInterruptions')), data))
average_download_interruptions = average(average_list, len(average_list))
# average ping represents the average of PingAvg
average_list = filter(
not_none, map(itemgetter(AVERAGE_PARAM.index('PingAvg')), data))
average_ping = average(average_list, len(average_list))
return (total_time, start_crawl_time, end_crawl_time, nr_videos,
average_download_time, average_download_interruptions,
average_ping)
def compute_average_values(data):
'''Function to return a tuple (start_crawl_time, end_crawl_time,
nr_videos, average_ping, average_download_time,
average_download_interruptions)
'''
# data is retrieved sorted from the database
start_crawl_time = data[0][TIMESTAMP_POSITION]\
[:MAX_TIMESTAMP_LENGTH]
end_crawl_time = data[-1][TIMESTAMP_POSITION]\
[:MAX_TIMESTAMP_LENGTH]
# total crawl time (sync time is added at the beginning and end for
# the plots)
total_time = (datetime.fromtimestamp(lib_database.time_to_epoch(
data[-1][TIMESTAMP_POSITION])) -
datetime.fromtimestamp(lib_database.time_to_epoch(
data[0][TIMESTAMP_POSITION])))
# each row in the dataset represents a video
nr_videos = len(data)
# filter values that are not None (can be zero)
not_none = lambda x: x is not None
# for some videos data cannot be retrieved
# average download time represents the average of DownloadTime
average_list = filter(not_none,
map(itemgetter(AVERAGE_PARAM.index('DownloadTime')), data))
average_download_time = average(average_list, len(average_list))
# average download interruptions represents the average of
# DownloadInterruptions
average_list = filter(not_none, map(itemgetter(
AVERAGE_PARAM.index('DownloadInterruptions')), data))
average_download_interruptions = average(average_list, len(average_list))
# average ping represents the average of PingAvg
average_list = filter(not_none, map(itemgetter(
AVERAGE_PARAM.index('PingAvg')), data))
average_ping = average(average_list, len(average_list))
return (total_time, start_crawl_time,
end_crawl_time, nr_videos, average_download_time,
average_download_interruptions, average_ping)