def get_top(self):
if len(self.results) == 0:
rds = store.RedisStorage(self.host, self.port, self.db)
rds.store_top(self.file_name, {})
return
res_json = {}
#res_json['count_per_five_second'] = self.results['count_per_five_second']
res_json['time'] = self.time_dic
res_json['source'] = self.results['source']
for prov in self.results:
if prov == '默认':
continue
if prov == 'count_per_five_second':
continue
if prov == 'source':
continue
res_json[prov] = {}
prov_count = self.results[prov]['count']
self.results[prov]['top'] = []
prov_vid_count = {}
#get top 10 of all city in the prov
res_json[prov]['city'] = {}
#count total prov vid
for city in prov_count:
if city == '未知':
pass
res_json[prov]['city'][city] = {}
res_json[prov]['city'][city]['count'] = len(prov_count[city])
for vid in prov_count[city]:
if prov_vid_count.get(vid) == None:
prov_vid_count[vid] = 0
prov_vid_count[vid] += prov_count[city][vid]
res_json[prov]['count'] = prov_vid_count
prov_vid_count = None
prov_count = None
res_json[prov]['total'] = self.results[prov]['total']
'''
print('{', prov, ':')
print(res_json[prov])
print('\}')
'''
rds = store.RedisStorage(self.host, self.port, self.db)
rds.store_top(self.file_name, res_json)
res_json = None
self.results = None
def test_retry_set_on_connection_error(self):
redis_storage = store.RedisStorage()
redis_storage.db.connected = False
redis_storage.db.set = MagicMock(side_effect=ConnectionError())
storage = store.Storage(redis_storage)
self.assertEqual(storage.cache_set("key", "value"), None)
self.assertEqual(redis_storage.db.set.call_count, store.Storage.MAX_RETRIES)
def split_file(self, file_name):
cmd = "split -l " + str(
config.split_lines
) + " " + "../src-logs/" + file_name + " " + file_name + "_part_"
print(cmd)
try:
subprocess.call(cmd, shell=True)
except subprocess.CalledProcessError as e:
print(e)
return
cmd2 = 'ls ' + file_name + '_part_*'
p = subprocess.Popen(cmd2,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=True)
p.wait()
if p.returncode == 0:
files = []
for line in p.stdout:
line = line.decode('utf-8')
file = line[0:-1]
files.append(file)
print(files)
length = len(files)
rds = store.RedisStorage()
rds.store_part_length(file_name.split('/')[-1], length)
#if file is split remove the old file
os.remove(file_name)
for file_path in files:
#if received finished processed Message, just to delete the split file
self.file_deque.append(file_path.split('/')[-1])
#self.subscribe_finish()
self.reportNewPartFile(file_path, length)
else:
#if file cann't be split, reserve the old file
#once received finish processed Message, delete the tmp zero size file
self.file_deque.append(file_name.split('/')[-1])
#self.subscribe_finish()
rds = store.RedisStorage()
rds.store_part_length(file_name.split('/')[-1], 0)
self.reportNewPartFile(file_name, 0)
def __init__(self):
self.lock_time = 5000
self.dlm = redlock.Redlock(config.redis_servers)
self.resource = "reduce"
self.lock = None
self.queue = 'task_finished'
self.redis_client = store.RedisStorage()
self.redis_final_result_key = "result"
self.total = 31
pass
class MainHTTPHandler(BaseHTTPRequestHandler):
router = {"method": method_handler}
store = store.Storage(store.RedisStorage())
def get_request_id(self, headers):
return headers.get('HTTP_X_REQUEST_ID', uuid.uuid4().hex)
def do_POST(self):
response, code = {}, OK
context = {"request_id": self.get_request_id(self.headers)}
request = None
try:
data_string = self.rfile.read(int(self.headers['Content-Length']))
data_string = data_string.decode("utf-8")
request = json.loads(data_string)
except:
code = BAD_REQUEST
if request:
path = self.path.strip("/")
logging.info("%s: %s %s" %
(self.path, data_string, context["request_id"]))
if path in self.router:
try:
response, code = self.router[path]({
"body": request,
"headers": self.headers
}, context, self.store)
except Exception as e:
logging.exception("Unexpected error: %s" % e)
code = INTERNAL_ERROR
else:
code = NOT_FOUND
self.send_response(code)
self.send_header("Content-Type", "application/json")
self.end_headers()
if code not in ERRORS:
r = {"response": response, "code": code}
else:
r = {
"error": response or ERRORS.get(code, "Unknown Error"),
"code": code
}
context.update(r)
logging.info(context)
self.wfile.write(bytes(json.dumps(r), "utf-8"))
return
def setUp(self):
self.context = {}
self.headers = {}
self.settings = store.Storage(store.RedisStorage())
def test_cache_set(self):
redis_storage = store.RedisStorage()
redis_storage.db.connected = False
storage = store.Storage(redis_storage)
self.assertEqual(storage.cache_set("key", "value"), True)
def test_cache_get(self):
redis_storage = store.RedisStorage()
redis_storage.db.connected = False
storage = store.Storage(redis_storage)
self.assertEqual(storage.cache_get("key"), None)
def __init__(self):
self.lock_time = 5000
self.dlm = redlock.Redlock(config.redis_servers)
self.queue = 'part_task_finished'
self.redis_client = store.RedisStorage()
self.lock = None
def test_get_raises_on_connection_error(self):
redis_storage = store.RedisStorage()
redis_storage.db.connected = False
storage = store.Storage(redis_storage)
with self.assertRaises(ConnectionError):
storage.get("key")
def test_not_raises_on_connection_error(self):
redis_storage = store.RedisStorage()
redis_storage.db.connected = False
storage = store.Storage(redis_storage)
self.assertEqual(storage.cache_get("key"), None)
self.assertEqual(storage.cache_set("key", "value"), None)