def process_results(results):
""" process the results and gather data """
conn = get_connection()
new_results = []
for d in results:
mirror = d.get('mirror')
if mirror in IGNORE_MIRRORS:
# skip mirrors we want to ignore.
continue
status = d.get('status')
location = get_location_for_mirror(mirror)
d['location'] = location_name(location)
if status != 'Unavailable':
resp_time = d.get('response_time')
age = get_total_seconds(d.get('time_diff'))
conn.rpush(cache_key('RESPTIME', mirror), resp_time)
conn.rpush(cache_key('AGE', mirror), age)
resp_list = conn.lrange(cache_key('RESPTIME', mirror), -60, -1)
age_list = conn.lrange(cache_key('AGE', mirror), -60, -1)
d['num_packages'] = find_number_of_packages(mirror)
d['resp_list'] = ",".join(resp_list)
d['age_list'] = ",".join(age_list)
new_results.append(d)
return new_results
def process_results(results):
""" process the results and gather data """
conn = get_connection()
new_results = []
for d in results:
mirror = d.get('mirror')
if mirror in IGNORE_MIRRORS:
# skip mirrors we want to ignore.
continue
status = d.get('status')
location = get_location_for_mirror(mirror)
d['location'] = location_name(location)
if status != 'Unavailable':
resp_time = d.get('response_time')
age = get_total_seconds(d.get('time_diff'))
conn.rpush(cache_key('RESPTIME', mirror), resp_time )
conn.rpush(cache_key('AGE', mirror), age)
resp_list = conn.lrange(cache_key('RESPTIME', mirror), -60, -1)
age_list = conn.lrange(cache_key('AGE', mirror), -60, -1)
d['num_packages'] = find_number_of_packages(mirror)
d['resp_list'] = ",".join(resp_list)
d['age_list'] = ",".join(age_list)
new_results.append(d)
return new_results
def main():
a = cache_key('cluster-test', base_cluster, 100000)
res, cnt, loo = a['res'], a['cnt'], a['loo']
b = 0
q = {}
for w, v in res.iteritems():
#v = dict(filter(lambda x: x[1] > 1, v.iteritems()))
# if b >= 10000:
# break
# b += 1
if loo[w] < 10:
continue
v = dict(filter(lambda x: x[1] > 5, res[w].iteritems()))
q[w] = cnt[w]
if v:
print_dict(v,
title='word: %s %d' % (w, cnt[w]),
cmp_key=lambda x: x[1])
print_dict(q, cmp_key=lambda x: x[1])
def get_location_for_mirror(mirror):
""" get the location for the mirror """
conn = get_connection()
loc_key = cache_key('IPLOC', mirror)
value = conn.get(loc_key)
if value:
return pickle.loads(value)
# if we have a mirror name like mirror.domain.suffix/blah it won't work
try:
hostname = urlparse("http://{0}".format(mirror)).netloc
except Exception as exc:
# if error, just default to mirror that works most of the time
print("Error getting location for {0} \n {1}".format(mirror, exc))
hostname = mirror
ip = socket.gethostbyname(hostname)
location = ping_ip2loc(ip)
if location:
conn.setex(loc_key, 86400, pickle.dumps(location)) # 1 day cache
return location
# if we get here, no good, return None
return None
def get_location_for_mirror(mirror):
""" get the location for the mirror """
conn = get_connection()
loc_key = cache_key('IPLOC', mirror)
value = conn.get(loc_key)
if value:
return pickle.loads(value)
# if we have a mirror name like mirror.domain.suffix/blah it won't work
try:
hostname = urlparse("http://{0}".format(mirror)).netloc
except Exception as exc:
# if error, just default to mirror that works most of the time
print("Error getting location for {0} \n {1}".format(mirror, exc))
hostname = mirror
ip = socket.gethostbyname(hostname)
location = ping_ip2loc(ip)
if location:
conn.setex(loc_key, 86400, pickle.dumps(location)) # 1 day cache
return location
# if we get here, no good, return None
return None