def format_for_insert(cls, req):
'''将请求数据转化为{"tkey":[],"tvalue":"","tstr":""}形式'''
res_dict = DictObj()
req_post = cls().check_data(req)
if isinstance(req_post, dict):
tkey = []
tvalue = ""
nlist = []
for key, value in req_post.items():
key_str = '`%s`' % key
value_str = '"%s",' % value
tkey.append(key_str)
tvalue += value_str
if key != 'id':
equal_str = '`%s`="%s" ' % (key, value)
nlist.append(equal_str)
tstr = ",".join(nlist)
tkey_str = ",".join(tkey)
tvalue = tvalue[:-1]
res_dict.tkey = '(' + tkey_str + ')'
res_dict.tvalue = '(' + tvalue + ')'
res_dict.tstring = tstr
return res_dict
else:
raise TypeError("param is not a dict!")
def start_deploy():
'''func doc'''
logger.info("deploying...")
final_result = DictObj()
# final_result.start_master = execute(start_master)
final_result.start_slave = execute(start_slave)
# final_result.stop_slave = execute(stop_slave)
logger.info(final_result)
def check_data(self, req):
'''func doc'''
data = DictObj(**req.POST.dict())
if data.id:
data.modify_time = curtime()
else:
data.create_time = curtime()
# data.operate_ip = req.META['HTTP_X_REAL_IP']
return data
def __init__(self, db, auth, profile):
self.db = db
self.auth = auth
self.profile = profile
self.handlers = Result(
{
"SummationHandler": SummationHandler(),
"AdditionHandler": AdditionHandler(),
"ConversionHandler": ConversionHandler(),
}
)
self.type = enum(Science=1, Technology=2, Engineering=3, Art=4, Math=5)
self.problems = self.db.get_table("problems")
self.problemsets = self.db.get_table("problemsets")
self.links = self.db.get_table("set_links")
self.urls = self.db.get_table("problem_urls")
self.instances = self.db.get_table("problem_instances")
def format_for_delete(cls, req):
data = DictObj(**req.POST.dict())
ids_list = [int(client_id) for client_id in data.id.split(',')]
return ids_list
def dictfetchall(cls,cursor):
'''将cursor查询到的每行数据作为dict元素包含在list中返回'''
columns = [col[0] for col in cursor.description]
data = [DictObj(columns, row) for row in cursor.fetchall()]
return data
async def run_test(
cls, test_name: str, steps: List[STEP] = None
) -> Union[None, Dict[str, Any]]:
from config import Config
from utils import DictObj
if test_name not in Config.tests.keys():
logger.error('Specified test is not defined in loaded config.')
logger.debug(f'Loaded tests: {list(Config.tests.keys())}')
exit()
test: List[STEP] = Config.tests[test_name]
results = {
'test': Config.tests[test_name],
'status': 'PASSED',
'success_rate': 100,
'stats': {'passed': 0, 'failed': 0, 'skipped': 0, 'total': 0},
'steps': [],
}
step_failed = False
for i in range(len(test)):
results['stats']['total'] += 1
step = copy.deepcopy(test[i])
if steps and i not in steps:
results['stats']['total'] -= 1
results['stats']['skipped'] += 1
continue
if step_failed and not Config.test_force:
results['stats']['skipped'] += 1
continue
if step._step == 'AUTH':
try:
STEP.validate_step(step=step)
step = STEP.CALL(
module='session',
method='auth',
doc={
step._args['var']: step._args['val'],
'hash': step._args['hash'],
},
)
except InvalidTestStepException as e:
logger.error(
f'Can\'t process test step \'AUTH\' with error: {e} Exiting.'
)
exit()
elif step._step == 'SIGNOUT':
step = STEP.CALL(
module='session', method='signout', query=[{'_id': '$__session'}]
)
else:
try:
STEP.validate_step(step=step)
except InvalidTestStepException as e:
logger.error(f'{e} Exiting.')
exit()
if step._step == 'CALL':
call_results = await cls.run_call(results=results, **step._args)
if 'session' in call_results.keys():
logger.debug(
'Updating session after detecting \'session\' in call results.'
)
if str(call_results['session']._id) == 'f00000000000000000000012':
cls.session = DictObj(
{
**Config.compile_anon_session(),
'user': DictObj(Config.compile_anon_user()),
}
)
else:
cls.session = call_results['session']
results['steps'].append(call_results)
elif step._step == 'TEST':
test_results = await cls.run_test(
test_name=step._args['test'], steps=step._args['steps']
)
if test_results['status'] == 'PASSED':
test_results['status'] = True
else:
test_results['status'] = False
results['steps'].append(test_results)
elif step._step == 'SET_REALM':
try:
if step._args['realm'].startswith('$__'):
step._args['realm'] = extract_attr(
scope=results, attr_path=step._args['realm']
)
logger.debug(f'Updating realm to \'{step._args["realm"]}\'.')
cls.env['realm'] = step._args['realm']
results['steps'].append(
{
'step': 'set_realm',
'realm': step._args['realm'],
'status': True,
}
)
except Exception as e:
logger.error(e)
results['steps'].append(
{
'step': 'set_realm',
'realm': step._args['realm'],
'status': False,
}
)
if not results['steps'][-1]['status']:
results['stats']['failed'] += 1
if not Config.test_force:
step_failed = True
else:
results['stats']['passed'] += 1
if len(results['steps']) == 0:
logger.error('No steps tested. Exiting.')
exit()
results['success_rate'] = int(
(results['stats']['passed'] / results['stats']['total']) * 100
)
if results['success_rate'] == 0:
results['status'] = 'FAILED'
elif results['success_rate'] == 100:
results['status'] = 'PASSED'
else:
results['status'] = 'PARTIAL'
if test_name == Config.test:
logger.debug(
'Finished testing %s steps [Passed: %s, Failed: %s, Skipped: %s] with success rate of: %s%%',
results['stats']['total'],
results['stats']['passed'],
results['stats']['failed'],
results['stats']['skipped'],
results['success_rate'],
)
tests_log = os.path.join(
Config._limp_location,
'tests',
f'LIMP-TEST_{test_name}_{datetime.datetime.utcnow().strftime("%d-%b-%Y")}',
)
if os.path.exists(f'{tests_log}.json'):
i = 1
while True:
if os.path.exists(f'{tests_log}.{i}.json'):
i += 1
else:
tests_log = f'{tests_log}.{i}'
break
tests_log += '.json'
with open(tests_log, 'w') as f:
f.write(json.dumps(json.loads(JSONEncoder().encode(results)), indent=4))
logger.debug(f'Full tests log available at: {tests_log}')
else:
return results
class ProblemManager(object):
def __init__(self, db, auth, profile):
self.db = db
self.auth = auth
self.profile = profile
self.handlers = Result(
{
"SummationHandler": SummationHandler(),
"AdditionHandler": AdditionHandler(),
"ConversionHandler": ConversionHandler(),
}
)
self.type = enum(Science=1, Technology=2, Engineering=3, Art=4, Math=5)
self.problems = self.db.get_table("problems")
self.problemsets = self.db.get_table("problemsets")
self.links = self.db.get_table("set_links")
self.urls = self.db.get_table("problem_urls")
self.instances = self.db.get_table("problem_instances")
# ----------
# - Get Handlers
# ----------
def getHandlers(self):
return self.handlers.keys()
# ----------
# - Get Problem
# ----------
def getProblem(self, problem_id):
problems = self.db.select(self.problems, self.problems.c.id == problem_id)
if len(problems):
return Result({"success": False, "ecode": 0, "message": "Invalid Problem ID"})
urls = self.db.select(self.urls, self.urls.problem_id == problem_id)
return Result(
{
"success": True,
"problem_id": problems[0].id,
"type": problems[0].type,
"name": problems[0].name,
"slug": problems[0].slug,
"desc": problems[0].desc,
"creator": self.profile.getUser(user_id=problems[0].creator),
"background": problems[0].background,
"handler": problems[0].handler,
"urls": [url.url for url in urls],
}
)
# ----------
# - Get All Problems
# ----------
def getAllProblems(self):
return [
Result(
{
"problem_id": problem.id,
"type": problem.type,
"name": problem.name,
"slug": problem.slug,
"desc": problem.desc,
"creator": self.profile.getUser(user_id=problem.creator),
"background": problem.background,
"handler": problem.handler,
}
)
for problem in self.db.select(self.problems)
]
# ----------
# - Get Problem Instance
# ----------
def getProblemInstance(self, problem_id):
instances = self.db.select(self.instances, self.instances.c.id == problem_id)
if not len(instances):
return Result({"success": False, "ecode": 0, "message": "Instance does not exist"})
return Result(
{
"success": True,
"problem_id": instances[0].problem_id,
"user_id": instances[0].user_id,
"completed": instances[0].completed,
"data": instances[0].data,
"start": instances[0].start,
"end": instances[0].end,
}
)
# ----------
# - Get Problem URLs
# ----------
def getProblemURLs(self, problem_id):
urls = self.db.select(self.urls, self.urls.c.problem_id == problem_id)
return {"success": True, "urls": [url.url for url in urls]}
# ----------
# - Get All Problem URLs
# ----------
def getAllProblemURLs(self):
urls = self.db.select(self.urls)
return [Result({"url_id": url.id, "problem_id": url.problem_id, "url": url.url}) for url in urls]
# ----------
# - Get Problem Set
# ----------
def getProblemSet(self, set_id):
sets = self.db.select(self.problemsets, self.problemsets.c.id == set_id)
if not len(sets):
return Result({"success": False, "ecode": 0, "message": "Invalid Set ID"})
links = self.db.select(self.links, self.links.c.set_id == sets[0].id)
problems = [self.db.select(self.problems, self.problems.c.id == link.problem_id)[0] for link in links]
return Result(
{
"success": True,
"set_id": sets[0].id,
"name": sets[0].name,
"slug": sets[0].slug,
"desc": sets[0].desc,
"creator": self.profile.getUser(user_id=sets[0].creator),
"problems": [
Result(
{
"problem_id": problem.id,
"type": problem.type,
"name": problem.name,
"slug": problem.slug,
"creator": self.profile.getUser(user_id=problem.creator),
}
)
for problem in problems
],
}
)
# ----------
# - Get All Problem Sets
# ----------
def getAllProblemSets(self):
sets = self.db.select(self.problemsets)
return [Result({"success": True, "set_id": set.id, "name": set.name}) for set in sets]
# ----------
# - Get Set Link
# ----------
def getSetLink(self, set_id, problem_id):
links = self.db.select(self.links, and_(self.links.c.set_id == set_id, self.links.c.problem_id == problem_id))
if not len(links):
return Result({"success": False, "ecode": 0, "message": "Set Link Does Not Exist"})
return Result({"success": True, "set_id": links[0].set_id, "problem_id": links[0].problem_id})
# ----------
# - Get Set Links
# ----------
def getSetLinks(self, set_id):
links = self.db.select(self.links, self.links.c.set_id == set_id)
return Result(
{
"success": True,
"links": [{"link_id": link.id, "set_id": link.set_id, "problem_id": link.problem_id} for link in links],
}
)
# ----------
# - Get All Set Links
# ----------
def getAllSetLinks(self):
links = self.db.select(self.links)
return [Result({"link_id": link.id, "set_id": link.set_id, "problem_id": link.problem_id}) for link in links]
# ----------
# - Create Problem
# ----------
def createProblem(self, ptype, name, slug, desc, background, handler, creator, urls):
problems = self.db.select(self.problems, or_(self.problems.c.name == name, self.problems.c.slug == slug))
if len(problems):
return Result({"success": False, "ecode": 0, "message": "Problem with that name already exits"})
self.db.insert(
self.problems,
{
"type": ptype,
"name": name,
"slug": slug,
"desc": desc,
"background": background,
"handler": handler,
"creator": creator,
},
)
problems = self.db.select(self.problems, self.problems.c.name == name)
for url in urls:
self.createProblemURL(problems[0].id, url)
return Result({"success": True})
# ----------
# - Create Problem Instance
# ----------
def createProblemInstance(self, problem_id):
instance = self.getProblemInstance(problem_id)
if instance.success:
return Result({"success": False, "ecode": 0, "message": "Problem Instance already exists"})
user = self.auth.getSession()
self.db.insert(self.instances, {"problem_id": problem_id, "user_id": user.user_id})
return Result({"success": True})
# ----------
# - Create Problem URL
# ----------
def createProblemURL(self, problem_id, url):
self.db.insert(self.urls, {"problem_id": problem_id, "url": url})
return Result({"success": True})
# ----------
# - Create Problem Set
# ----------
def createProblemSet(self, name, slug, desc):
sets = self.db.select(self.problemsets, or_(self.problemsets.c.name == name, self.problemsets.c.slug == slug))
if len(sets):
return Result({"success": False, "ecode": 0, "message": "Problem set with that name already exits"})
self.db.insert(self.problemsets, {"name": name, "slug": slug, "desc": desc})
return Result({"success": True})
# ----------
# - Create Set Link
# ----------
def createSetLink(self, set_id, problem_id):
links = self.db.select(self.links, and_(self.links.c.set_id == set_id, self.links.c.problem_id == problem_id))
if len(links):
return Result({"success": False, "ecode": 0, "message": "Problem set link already exists"})
self.db.insert(self.links, {"set_id": set_id, "problem_id": problem_id})
return Result({"success": True})
# ----------
# - Delete Problem
# ----------
def deleteProblem(self, problem_id):
self.db.delete(self.problems, self.problems.c.id == problem_id)
self.db.delete(self.links, self.links.c.problem_id == problem_id)
self.db.delete(self.urls, self.urls.c.problem_id == problem_id)
return Result({"success": True})
# ----------
# - Delete Problem URL
# ----------
def deleteProblemURL(self, url_id):
self.db.delete(self.urls, self.urls.c.id == url_id)
return Result({"success": True})
# ----------
# - Delete All Problem URLs
# ----------
def deleteAllProblemURLs(self, problem_id):
self.db.delete(self.urls, self.urls.c.problem_id == problem_id)
return Result({"success": True})
# ----------
# - Delete Problem Instance
# ----------
def deleteProblemInstance(self, instance_id):
self.db.delete(self.instances, self.instances.c.id == instance_id)
return Result({"success": True})
# ----------
# - Delete Problem Set
# ----------
def deleteProblemSet(self, set_id):
self.db.delete(self.problemsets, self.problemsets.c.id == set_id)
self.db.delete(self.links, self.links.c.set_id == set_id)
return Result({"success": True})
# ----------
# - Delete Set Link
# ----------
def deleteSetLink(self, link_id=None, set_id=None, problem_id=None):
if link_id:
self.db.delete(self.links, self.links.c.id == link_id)
else:
self.db.delete(self.links, and_(self.links.c.set_id == set_id, self.links.c.problem_id == problem_id))
# ----------
# - Delete All Set Links
# ----------
def deleteAllSetLinks(self, set_id=None, problem_id=None):
if not set_id:
self.db.delete(self.links, self.links.c.problem_id == problem_id)
elif not problem_id:
self.db.delete(self.links, self.links.c.set_id == set_id)
return Result({"success": True})
async def signout(self, skip_events=[], env={}, query=[], doc={}):
if str(query['_id'][0]) == 'f00000000000000000000012':
return self.status(
status=400,
msg='Singout is not allowed for \'__ANON\' user.',
args={'code': 'ANON_SIGNOUT'},
)
results = await self.read(skip_events=[Event.PERM],
env=env,
query=[{
'_id': query['_id'][0]
}])
if not results.args.count:
return self.status(status=403,
msg='Session is invalid.',
args={'code': 'INVALID_SESSION'})
results = await self.delete(skip_events=[Event.PERM],
env=env,
query=[{
'_id': env['session']._id
}])
# [DOC] Create CONN_AUTH Analytic doc
if Config.analytics_events['session_conn_deauth']:
analytic_doc = {
'event': 'CONN_DEAUTH',
'subevent': env['client_app'],
'args': {
'user': env['session'].user._id,
'session': env['session']._id,
'REMOTE_ADDR': env['REMOTE_ADDR'],
'HTTP_USER_AGENT': env['HTTP_USER_AGENT'],
},
}
analytic_results = await Config.modules['analytic'].create(
skip_events=[Event.PERM], env=env, doc=analytic_doc)
if analytic_results.status != 200:
logger.error(
f'Failed to create \'Analytic\' doc: {analytic_doc}. Results: {analytic_results}'
)
# [DOC] Create USER_AUTH Analytic doc
if Config.analytics_events['session_user_deauth']:
analytic_doc = {
'event': 'USER_DEAUTH',
'subevent': env['session'].user._id,
'args': {
'session': env['session']._id,
'REMOTE_ADDR': env['REMOTE_ADDR'],
'HTTP_USER_AGENT': env['HTTP_USER_AGENT'],
'client_app': env['client_app'],
},
}
analytic_results = await Config.modules['analytic'].create(
skip_events=[Event.PERM], env=env, doc=analytic_doc)
if analytic_results.status != 200:
logger.error(
f'Failed to create \'Analytic\' doc: {analytic_doc}. Results: {analytic_results}'
)
return self.status(
status=200,
msg='You are successfully signed-out.',
args={'session': DictObj({'_id': 'f00000000000000000000012'})},
)
from gevent.queue import Queue, Empty, Full
import six
from six.moves import xrange
import psutil
from data_runner import events
from data_runner import rpc, Message
from utils import DictObj
from runner_settings import LBS_IP_STR, LBS_PORT, OP_INTERVAL, CLASS_DURATION, DATA_RUNNER_MASTER
from runner_settings import get_logger
# 全局runner
global_runner = None
#用于汇总数据的全局变量
global_stats = DictObj()
push_uid_list = None
#current process
cur_process = psutil.Process()
STATE_INIT, STATE_READY, STATE_RUNNING, STATE_DONE, STATE_STOPPED = [
"connected", "ready", "running", "done", "stopped"
]
SLAVE_REPORT_INTERVAL = 2.0
QUEUE_MONITOR_INTERVAL = 2.0
#master数据节点的ip与port
MASTER_HOST = DATA_RUNNER_MASTER()
MASTER_BIND_PORT = 6666
