def retrieve_fields(self):
ip, reqlink, reqtype, response, virtualm, keytemp = [], [], [], [], [], []
bytes, avg_time, count, uniq_vis, total_vis = 0, 0, 0, 0, len(self.keys)
for key in self.keys:
keytemp.append(str(key))
# creating an ordered dictionary containing log data retrieved from column_family
log = self.cass_conn.multiget(keytemp)
# starting a pool of 5 worker processes
pool = Pool()
pool.ncpus = 5
for item in log.values():
# appending lists with their respective values
ip.append(item['host']), reqlink.append(item['request_link']), reqtype.append(item['request_type']), response.append(str(item['response_code'])), virtualm.append(item['virtual_machine'])
if item['byte_transfer'] != '-':
bytes += item['byte_transfer']
if item['response_time'] != '-':
avg_time += item['response_time']
count += 1
avg_time = avg_time/count
# using the pool of workers to get results
results = pool.map(self.unique_count, [ip, reqtype, reqlink, response, virtualm])
pool.close()
pool.join()
uniq_vis = len(results[0][0])
return self.time, results[0][0], results[0][1], results[1][0], results[1][1], results[2][0], results[2][1], results[3][0], results[3][1], results[4][0], results[4][1], bytes, avg_time, uniq_vis, total_vis
# build from lambda functions
squ = lambda x:x**2
if __name__ is '__main__':
from pathos.helpers import freeze_support
freeze_support()
from pathos.pools import ProcessPool as Pool
from pathos.pools import ThreadPool as TPool
pool = Pool()
tpool = TPool()
# test 'dilled' multiprocessing for inner
print("Evaluate 10 items on 2 proc:")
pool.ncpus = 2
print(pool)
print(pool.map(add_me, range(10)))
print('')
# test 'dilled' multiprocessing for lambda
print("Evaluate 10 items on 4 proc:")
pool.ncpus = 4
print(pool)
print(pool.map(squ, range(10)))
print('')
# test for lambda, but with threads
print("Evaluate 10 items on 4 threads:")
tpool.nthreads = 4
print(tpool)
# build from lambda functions
squ = lambda x:x**2
if __name__ is '__main__':
from pathos.helpers import freeze_support
freeze_support()
from pathos.pools import ProcessPool as Pool
from pathos.pools import ThreadPool as TPool
pool = Pool()
tpool = TPool()
# test 'dilled' multiprocessing for inner
print("Evaluate 10 items on 2 proc:")
pool.ncpus = 2
print(pool)
print(pool.map(add_me, range(10)))
print('')
# test 'dilled' multiprocessing for lambda
print("Evaluate 10 items on 4 proc:")
pool.ncpus = 4
print(pool)
print(pool.map(squ, range(10)))
print('')
# test for lambda, but with threads
print("Evaluate 10 items on 4 threads:")
tpool.nthreads = 4
print(tpool)
