def do_GET(self):
if self.path=="/api/data/total_faces":
self._set_response()
self.wfile.write(JsonHandler(filename).get_data_from_file("totalFaces"))
elif self.path=="/api/data/avg_age":
self._set_response()
self.wfile.write(JsonHandler(filename).get_data_from_file("currentAverageAge"))
elif self.path=="/api/data/parity":
self._set_response()
self.wfile.write(JsonHandler(filename).get_data_from_file("parity"))
elif self.path=="/api/data/expressions":
self._set_response()
self.wfile.write(JsonHandler(filename).get_data_from_file("expressions"))
def __init__(self, sizes, dataHandler, save_rate, checkpoints_dir,
activation_function):
self.num_layers = len(sizes)
self.sizes = sizes
self.dataHandler = dataHandler
self.layers = layer.create_layers(self.sizes, activation_function)
self.save_rate = save_rate
self.checkpoints_dir = checkpoints_dir
self.json_handler = JsonHandler()
def do_POST(self): if self.path=="/compute_stats": content_length = int(self.headers['Content-Length']) # <--- Gets the size of data post_data = self.rfile.read(content_length) # <--- Gets the data itself self._set_response() JsonHandler(filename).process_data_from_json(post_data) self.wfile.write(post_data)
def mc_worker(jobs, stats, ctl, store, timeout=5):
logging.info("mc_worker started")
while ctl["run_ok"]:
try:
root, parents, val = jobs.get(block=True, timeout=timeout)
except Queue.Empty:
logging.debug("mc_worker hasn't received jobs for %s seconds" %
timeout)
continue
start = time.time()
for server in val:
try:
ip, port = server.split(":")
except (ValueError, AttributeError), e:
logging.error("unable to collect mc stats from %s : %s" %
(server, e))
continue
mc_server = Server(ip)
# get bucket name from root and parent nodes
bucket = DataHelper.get_bucket(root, parents)
# initialize memcached source
mc_source = MemcachedSource(mc_server, bucket)
# initialize handlers to dump data json doc
j_handler = JsonHandler()
s_hanlder = SerieslyHandler(store)
# collect data from source and emit to handlers
mc_coll = MemcachedCollector([mc_source], [j_handler, s_hanlder])
mc_coll.collect()
mc_coll.emit()
stats.put([mc_source.fast, mc_source.meta], block=True)
stats.put([mc_source.slow, mc_source.meta], block=True)
delta = time.time() - start
logging.debug("collected mc stats from %s, took %s seconds" %
(val, delta))
if delta < timeout:
logging.debug("mc_worker sleep for %s seconds" % (timeout - delta))
time.sleep(timeout - delta)
async def init(loop, args):
if args.tags:
tags = args.tags.split(',')
else:
tags = [Model.default_tag]
model = Model(loop)
if args.model:
await model.set_model(args.model, tags)
batcher = Batcher(model, loop, args.batch_size)
web_app = web.Application(loop=loop, client_max_size=args.request_size)
web_app.on_shutdown.append(on_shutdown)
web_app.router.add_get('/stats', batcher.stats_handler)
json_handler = JsonHandler(model, batcher, args.batch_transpose)
if args.no_cors:
web_app.router.add_get('/', batcher.info_handler)
web_app.router.add_post('/{method}', json_handler.handler)
web_app.router.add_post('/', json_handler.handler)
else:
cors = aiohttp_cors.setup(web_app,
defaults={
"*":
aiohttp_cors.ResourceOptions(
allow_credentials=True,
expose_headers="*",
allow_headers="*")
})
get_resource = cors.add(web_app.router.add_resource('/'))
cors.add(get_resource.add_route("GET", batcher.info_handler))
post_resource = cors.add(web_app.router.add_resource('/{method}'))
cors.add(post_resource.add_route("POST", json_handler.handler))
post_resource = cors.add(web_app.router.add_resource('/'))
cors.add(post_resource.add_route("POST", json_handler.handler))
if args.static_path:
web_app.router.add_static('/web/',
path=args.static_path,
name='static')
grpc_app = Server([GrpcHandler(model, batcher)], loop=loop)
return web_app, grpc_app
parser.add_option('-j', '--jsonFile', dest='json_file')
options, otherjunk = parser.parse_args(argv)
return options
options = parse_commands(sys.argv[1:])
header_list = [
"person_id", "level", "chapterIndex", "contentIndex", "questionIndex",
"derivedIndex", "duration", "point", "clearDateTime",
"incorrectAnswerCount"
]
rest_handler = RestHandler()
json_handler = JsonHandler()
csv_handler = CsvHandler(filepath=options.drag_file, header_list=header_list)
#f = open(options.json_file, 'w')
date_list_json = rest_handler.get_json_of_date_list()
date_list = json_handler.json_to_date_list(date_list_json)
for idx, date in enumerate(date_list):
#for idx, date in enumerate(['1970-01-01','2019-02-07']):
#result_dict_list = []
if date == '1970/01/01':
print('Date 1970/01/01')
continue
print('[{}], ({}/{}) Now Collecting'.format(date, idx + 1, len(date_list)))
for mobile_os in ('iOS', 'Android'):
person_list_json = rest_handler.get_json_of_person_id_by_date(
date, mobile_os)
import click
import os
import copy
import convert
from abc import ABCMeta
from itertools import groupby
from collections import namedtuple
from datetime import datetime
from json_handler import JsonHandler
jh = JsonHandler()
LEAGUES_DATA = jh.load_leagues()
def get_writer(output_format='stdout', output_file=None):
return globals()[output_format.capitalize()](output_file)
class BaseWriter(object):
__metaclass__ = ABCMeta
def __init__(self, output_file):
self.output_filename = output_file
class Stdout(BaseWriter):
def __init__(self, output_file):
super().__init__(output_file)
class QueryMatcher(object):
jsonHandler = JsonHandler()
nlp = None
def __init__(self, nlp):
self.nlp = nlp
def getQuery(self, statement):
properties = self.getProperties(statement)
queries = self.searchPropertyMatch(properties)
maxQueryID, maxReplaces, maxScore = self.getBestQueryProperties(
queries, statement)
return self.buildOriginalQuery(maxQueryID, maxReplaces), maxScore
def getProperties(self, statement):
doc = self.nlp(statement)
properties = {"time": "0", "price": "0"}
for each in doc.ents:
if str(each.label_) == "MONEY" or str(each.label_) == "CARDINAL":
properties.__setitem__("price", "1")
elif str(each.label_) == "TIME":
properties.__setitem__("time", "1")
return properties
def searchPropertyMatch(self, properties):
queries = self.jsonHandler.matchProperties(properties)
return queries
def getBestQueryProperties(self, queries, statement):
#do the partial matching for each query as well as the entity matching
maxScore = 0.0
maxQueryID = ""
maxReplaces = None
for ID in queries:
totScore = self.getMaxHit(statement,
self.jsonHandler.getGeneralizedQuery(ID))
replaces = self.jsonHandler.getReplaces(ID)
numOfReplaces = len(replaces)
for i in range(0, len(replaces)):
replace = replaces[i]
score, key = self.getMaximumScoringKey(replace, statement)
totScore += score / numOfReplaces
replaces[i] = key
if (maxScore >= totScore):
continue
maxScore = totScore
maxQueryID = ID
maxReplaces = replaces
return maxQueryID, maxReplaces, maxScore
def getMaxHit(self, statement, generalizedQueries):
maxScore = 0.0
queryList = generalizedQueries.split(",")
for each in queryList:
score = fuzz.partial_ratio(statement, each)
if (score > maxScore):
maxScore = score
return maxScore
def buildOriginalQuery(self, ID, replaces):
if ID == "":
return
originalQuery = self.jsonHandler.getOriginalQuery(ID)
for i in range(0, len(replaces)):
originalQuery = originalQuery.replace("xxx" + str(i), replaces[i])
return originalQuery
def getMaximumScoringKey(self, replace, statement):
if (replace == "price"):
doc = self.nlp(statement)
for ent in doc.ents:
if str(ent.label_) == "MONEY" or str(ent.label_) == "CARDINAL":
return 0, self.get_first_nbr_from_str(str(ent))
else:
categorySet = replace.split(",")
maxScore = 0.0
maxScoreReplace = ""
for each in categorySet:
keySet = self.jsonHandler.getKeywordList(each)
for eachInner in keySet:
score = fuzz.partial_ratio(eachInner, statement)
if score > maxScore:
maxScore = score
maxScoreReplace = eachInner
return maxScore, maxScoreReplace
def get_first_nbr_from_str(self, input_str):
if not input_str and not isinstance(input_str, str):
return 0
out_number = ''
for ele in input_str:
if (ele == '.' and '.' not in out_number) or ele.isdigit():
out_number += ele
elif out_number:
break
return out_number
