def format_for_insert(cls, req): '''将请求数据转化为{"tkey":[],"tvalue":"","tstr":""}形式''' res_dict = DictObj() req_post = cls().check_data(req) if isinstance(req_post, dict): tkey = [] tvalue = "" nlist = [] for key, value in req_post.items(): key_str = '`%s`' % key value_str = '"%s",' % value tkey.append(key_str) tvalue += value_str if key != 'id': equal_str = '`%s`="%s" ' % (key, value) nlist.append(equal_str) tstr = ",".join(nlist) tkey_str = ",".join(tkey) tvalue = tvalue[:-1] res_dict.tkey = '(' + tkey_str + ')' res_dict.tvalue = '(' + tvalue + ')' res_dict.tstring = tstr return res_dict else: raise TypeError("param is not a dict!")
def start_deploy(): '''func doc''' logger.info("deploying...") final_result = DictObj() # final_result.start_master = execute(start_master) final_result.start_slave = execute(start_slave) # final_result.stop_slave = execute(stop_slave) logger.info(final_result)
def check_data(self, req): '''func doc''' data = DictObj(**req.POST.dict()) if data.id: data.modify_time = curtime() else: data.create_time = curtime() # data.operate_ip = req.META['HTTP_X_REAL_IP'] return data
def __init__(self, db, auth, profile): self.db = db self.auth = auth self.profile = profile self.handlers = Result( { "SummationHandler": SummationHandler(), "AdditionHandler": AdditionHandler(), "ConversionHandler": ConversionHandler(), } ) self.type = enum(Science=1, Technology=2, Engineering=3, Art=4, Math=5) self.problems = self.db.get_table("problems") self.problemsets = self.db.get_table("problemsets") self.links = self.db.get_table("set_links") self.urls = self.db.get_table("problem_urls") self.instances = self.db.get_table("problem_instances")
def format_for_delete(cls, req): data = DictObj(**req.POST.dict()) ids_list = [int(client_id) for client_id in data.id.split(',')] return ids_list
def dictfetchall(cls,cursor): '''将cursor查询到的每行数据作为dict元素包含在list中返回''' columns = [col[0] for col in cursor.description] data = [DictObj(columns, row) for row in cursor.fetchall()] return data
async def run_test( cls, test_name: str, steps: List[STEP] = None ) -> Union[None, Dict[str, Any]]: from config import Config from utils import DictObj if test_name not in Config.tests.keys(): logger.error('Specified test is not defined in loaded config.') logger.debug(f'Loaded tests: {list(Config.tests.keys())}') exit() test: List[STEP] = Config.tests[test_name] results = { 'test': Config.tests[test_name], 'status': 'PASSED', 'success_rate': 100, 'stats': {'passed': 0, 'failed': 0, 'skipped': 0, 'total': 0}, 'steps': [], } step_failed = False for i in range(len(test)): results['stats']['total'] += 1 step = copy.deepcopy(test[i]) if steps and i not in steps: results['stats']['total'] -= 1 results['stats']['skipped'] += 1 continue if step_failed and not Config.test_force: results['stats']['skipped'] += 1 continue if step._step == 'AUTH': try: STEP.validate_step(step=step) step = STEP.CALL( module='session', method='auth', doc={ step._args['var']: step._args['val'], 'hash': step._args['hash'], }, ) except InvalidTestStepException as e: logger.error( f'Can\'t process test step \'AUTH\' with error: {e} Exiting.' ) exit() elif step._step == 'SIGNOUT': step = STEP.CALL( module='session', method='signout', query=[{'_id': '$__session'}] ) else: try: STEP.validate_step(step=step) except InvalidTestStepException as e: logger.error(f'{e} Exiting.') exit() if step._step == 'CALL': call_results = await cls.run_call(results=results, **step._args) if 'session' in call_results.keys(): logger.debug( 'Updating session after detecting \'session\' in call results.' ) if str(call_results['session']._id) == 'f00000000000000000000012': cls.session = DictObj( { **Config.compile_anon_session(), 'user': DictObj(Config.compile_anon_user()), } ) else: cls.session = call_results['session'] results['steps'].append(call_results) elif step._step == 'TEST': test_results = await cls.run_test( test_name=step._args['test'], steps=step._args['steps'] ) if test_results['status'] == 'PASSED': test_results['status'] = True else: test_results['status'] = False results['steps'].append(test_results) elif step._step == 'SET_REALM': try: if step._args['realm'].startswith('$__'): step._args['realm'] = extract_attr( scope=results, attr_path=step._args['realm'] ) logger.debug(f'Updating realm to \'{step._args["realm"]}\'.') cls.env['realm'] = step._args['realm'] results['steps'].append( { 'step': 'set_realm', 'realm': step._args['realm'], 'status': True, } ) except Exception as e: logger.error(e) results['steps'].append( { 'step': 'set_realm', 'realm': step._args['realm'], 'status': False, } ) if not results['steps'][-1]['status']: results['stats']['failed'] += 1 if not Config.test_force: step_failed = True else: results['stats']['passed'] += 1 if len(results['steps']) == 0: logger.error('No steps tested. Exiting.') exit() results['success_rate'] = int( (results['stats']['passed'] / results['stats']['total']) * 100 ) if results['success_rate'] == 0: results['status'] = 'FAILED' elif results['success_rate'] == 100: results['status'] = 'PASSED' else: results['status'] = 'PARTIAL' if test_name == Config.test: logger.debug( 'Finished testing %s steps [Passed: %s, Failed: %s, Skipped: %s] with success rate of: %s%%', results['stats']['total'], results['stats']['passed'], results['stats']['failed'], results['stats']['skipped'], results['success_rate'], ) tests_log = os.path.join( Config._limp_location, 'tests', f'LIMP-TEST_{test_name}_{datetime.datetime.utcnow().strftime("%d-%b-%Y")}', ) if os.path.exists(f'{tests_log}.json'): i = 1 while True: if os.path.exists(f'{tests_log}.{i}.json'): i += 1 else: tests_log = f'{tests_log}.{i}' break tests_log += '.json' with open(tests_log, 'w') as f: f.write(json.dumps(json.loads(JSONEncoder().encode(results)), indent=4)) logger.debug(f'Full tests log available at: {tests_log}') else: return results
class ProblemManager(object): def __init__(self, db, auth, profile): self.db = db self.auth = auth self.profile = profile self.handlers = Result( { "SummationHandler": SummationHandler(), "AdditionHandler": AdditionHandler(), "ConversionHandler": ConversionHandler(), } ) self.type = enum(Science=1, Technology=2, Engineering=3, Art=4, Math=5) self.problems = self.db.get_table("problems") self.problemsets = self.db.get_table("problemsets") self.links = self.db.get_table("set_links") self.urls = self.db.get_table("problem_urls") self.instances = self.db.get_table("problem_instances") # ---------- # - Get Handlers # ---------- def getHandlers(self): return self.handlers.keys() # ---------- # - Get Problem # ---------- def getProblem(self, problem_id): problems = self.db.select(self.problems, self.problems.c.id == problem_id) if len(problems): return Result({"success": False, "ecode": 0, "message": "Invalid Problem ID"}) urls = self.db.select(self.urls, self.urls.problem_id == problem_id) return Result( { "success": True, "problem_id": problems[0].id, "type": problems[0].type, "name": problems[0].name, "slug": problems[0].slug, "desc": problems[0].desc, "creator": self.profile.getUser(user_id=problems[0].creator), "background": problems[0].background, "handler": problems[0].handler, "urls": [url.url for url in urls], } ) # ---------- # - Get All Problems # ---------- def getAllProblems(self): return [ Result( { "problem_id": problem.id, "type": problem.type, "name": problem.name, "slug": problem.slug, "desc": problem.desc, "creator": self.profile.getUser(user_id=problem.creator), "background": problem.background, "handler": problem.handler, } ) for problem in self.db.select(self.problems) ] # ---------- # - Get Problem Instance # ---------- def getProblemInstance(self, problem_id): instances = self.db.select(self.instances, self.instances.c.id == problem_id) if not len(instances): return Result({"success": False, "ecode": 0, "message": "Instance does not exist"}) return Result( { "success": True, "problem_id": instances[0].problem_id, "user_id": instances[0].user_id, "completed": instances[0].completed, "data": instances[0].data, "start": instances[0].start, "end": instances[0].end, } ) # ---------- # - Get Problem URLs # ---------- def getProblemURLs(self, problem_id): urls = self.db.select(self.urls, self.urls.c.problem_id == problem_id) return {"success": True, "urls": [url.url for url in urls]} # ---------- # - Get All Problem URLs # ---------- def getAllProblemURLs(self): urls = self.db.select(self.urls) return [Result({"url_id": url.id, "problem_id": url.problem_id, "url": url.url}) for url in urls] # ---------- # - Get Problem Set # ---------- def getProblemSet(self, set_id): sets = self.db.select(self.problemsets, self.problemsets.c.id == set_id) if not len(sets): return Result({"success": False, "ecode": 0, "message": "Invalid Set ID"}) links = self.db.select(self.links, self.links.c.set_id == sets[0].id) problems = [self.db.select(self.problems, self.problems.c.id == link.problem_id)[0] for link in links] return Result( { "success": True, "set_id": sets[0].id, "name": sets[0].name, "slug": sets[0].slug, "desc": sets[0].desc, "creator": self.profile.getUser(user_id=sets[0].creator), "problems": [ Result( { "problem_id": problem.id, "type": problem.type, "name": problem.name, "slug": problem.slug, "creator": self.profile.getUser(user_id=problem.creator), } ) for problem in problems ], } ) # ---------- # - Get All Problem Sets # ---------- def getAllProblemSets(self): sets = self.db.select(self.problemsets) return [Result({"success": True, "set_id": set.id, "name": set.name}) for set in sets] # ---------- # - Get Set Link # ---------- def getSetLink(self, set_id, problem_id): links = self.db.select(self.links, and_(self.links.c.set_id == set_id, self.links.c.problem_id == problem_id)) if not len(links): return Result({"success": False, "ecode": 0, "message": "Set Link Does Not Exist"}) return Result({"success": True, "set_id": links[0].set_id, "problem_id": links[0].problem_id}) # ---------- # - Get Set Links # ---------- def getSetLinks(self, set_id): links = self.db.select(self.links, self.links.c.set_id == set_id) return Result( { "success": True, "links": [{"link_id": link.id, "set_id": link.set_id, "problem_id": link.problem_id} for link in links], } ) # ---------- # - Get All Set Links # ---------- def getAllSetLinks(self): links = self.db.select(self.links) return [Result({"link_id": link.id, "set_id": link.set_id, "problem_id": link.problem_id}) for link in links] # ---------- # - Create Problem # ---------- def createProblem(self, ptype, name, slug, desc, background, handler, creator, urls): problems = self.db.select(self.problems, or_(self.problems.c.name == name, self.problems.c.slug == slug)) if len(problems): return Result({"success": False, "ecode": 0, "message": "Problem with that name already exits"}) self.db.insert( self.problems, { "type": ptype, "name": name, "slug": slug, "desc": desc, "background": background, "handler": handler, "creator": creator, }, ) problems = self.db.select(self.problems, self.problems.c.name == name) for url in urls: self.createProblemURL(problems[0].id, url) return Result({"success": True}) # ---------- # - Create Problem Instance # ---------- def createProblemInstance(self, problem_id): instance = self.getProblemInstance(problem_id) if instance.success: return Result({"success": False, "ecode": 0, "message": "Problem Instance already exists"}) user = self.auth.getSession() self.db.insert(self.instances, {"problem_id": problem_id, "user_id": user.user_id}) return Result({"success": True}) # ---------- # - Create Problem URL # ---------- def createProblemURL(self, problem_id, url): self.db.insert(self.urls, {"problem_id": problem_id, "url": url}) return Result({"success": True}) # ---------- # - Create Problem Set # ---------- def createProblemSet(self, name, slug, desc): sets = self.db.select(self.problemsets, or_(self.problemsets.c.name == name, self.problemsets.c.slug == slug)) if len(sets): return Result({"success": False, "ecode": 0, "message": "Problem set with that name already exits"}) self.db.insert(self.problemsets, {"name": name, "slug": slug, "desc": desc}) return Result({"success": True}) # ---------- # - Create Set Link # ---------- def createSetLink(self, set_id, problem_id): links = self.db.select(self.links, and_(self.links.c.set_id == set_id, self.links.c.problem_id == problem_id)) if len(links): return Result({"success": False, "ecode": 0, "message": "Problem set link already exists"}) self.db.insert(self.links, {"set_id": set_id, "problem_id": problem_id}) return Result({"success": True}) # ---------- # - Delete Problem # ---------- def deleteProblem(self, problem_id): self.db.delete(self.problems, self.problems.c.id == problem_id) self.db.delete(self.links, self.links.c.problem_id == problem_id) self.db.delete(self.urls, self.urls.c.problem_id == problem_id) return Result({"success": True}) # ---------- # - Delete Problem URL # ---------- def deleteProblemURL(self, url_id): self.db.delete(self.urls, self.urls.c.id == url_id) return Result({"success": True}) # ---------- # - Delete All Problem URLs # ---------- def deleteAllProblemURLs(self, problem_id): self.db.delete(self.urls, self.urls.c.problem_id == problem_id) return Result({"success": True}) # ---------- # - Delete Problem Instance # ---------- def deleteProblemInstance(self, instance_id): self.db.delete(self.instances, self.instances.c.id == instance_id) return Result({"success": True}) # ---------- # - Delete Problem Set # ---------- def deleteProblemSet(self, set_id): self.db.delete(self.problemsets, self.problemsets.c.id == set_id) self.db.delete(self.links, self.links.c.set_id == set_id) return Result({"success": True}) # ---------- # - Delete Set Link # ---------- def deleteSetLink(self, link_id=None, set_id=None, problem_id=None): if link_id: self.db.delete(self.links, self.links.c.id == link_id) else: self.db.delete(self.links, and_(self.links.c.set_id == set_id, self.links.c.problem_id == problem_id)) # ---------- # - Delete All Set Links # ---------- def deleteAllSetLinks(self, set_id=None, problem_id=None): if not set_id: self.db.delete(self.links, self.links.c.problem_id == problem_id) elif not problem_id: self.db.delete(self.links, self.links.c.set_id == set_id) return Result({"success": True})
async def signout(self, skip_events=[], env={}, query=[], doc={}): if str(query['_id'][0]) == 'f00000000000000000000012': return self.status( status=400, msg='Singout is not allowed for \'__ANON\' user.', args={'code': 'ANON_SIGNOUT'}, ) results = await self.read(skip_events=[Event.PERM], env=env, query=[{ '_id': query['_id'][0] }]) if not results.args.count: return self.status(status=403, msg='Session is invalid.', args={'code': 'INVALID_SESSION'}) results = await self.delete(skip_events=[Event.PERM], env=env, query=[{ '_id': env['session']._id }]) # [DOC] Create CONN_AUTH Analytic doc if Config.analytics_events['session_conn_deauth']: analytic_doc = { 'event': 'CONN_DEAUTH', 'subevent': env['client_app'], 'args': { 'user': env['session'].user._id, 'session': env['session']._id, 'REMOTE_ADDR': env['REMOTE_ADDR'], 'HTTP_USER_AGENT': env['HTTP_USER_AGENT'], }, } analytic_results = await Config.modules['analytic'].create( skip_events=[Event.PERM], env=env, doc=analytic_doc) if analytic_results.status != 200: logger.error( f'Failed to create \'Analytic\' doc: {analytic_doc}. Results: {analytic_results}' ) # [DOC] Create USER_AUTH Analytic doc if Config.analytics_events['session_user_deauth']: analytic_doc = { 'event': 'USER_DEAUTH', 'subevent': env['session'].user._id, 'args': { 'session': env['session']._id, 'REMOTE_ADDR': env['REMOTE_ADDR'], 'HTTP_USER_AGENT': env['HTTP_USER_AGENT'], 'client_app': env['client_app'], }, } analytic_results = await Config.modules['analytic'].create( skip_events=[Event.PERM], env=env, doc=analytic_doc) if analytic_results.status != 200: logger.error( f'Failed to create \'Analytic\' doc: {analytic_doc}. Results: {analytic_results}' ) return self.status( status=200, msg='You are successfully signed-out.', args={'session': DictObj({'_id': 'f00000000000000000000012'})}, )
from gevent.queue import Queue, Empty, Full import six from six.moves import xrange import psutil from data_runner import events from data_runner import rpc, Message from utils import DictObj from runner_settings import LBS_IP_STR, LBS_PORT, OP_INTERVAL, CLASS_DURATION, DATA_RUNNER_MASTER from runner_settings import get_logger # 全局runner global_runner = None #用于汇总数据的全局变量 global_stats = DictObj() push_uid_list = None #current process cur_process = psutil.Process() STATE_INIT, STATE_READY, STATE_RUNNING, STATE_DONE, STATE_STOPPED = [ "connected", "ready", "running", "done", "stopped" ] SLAVE_REPORT_INTERVAL = 2.0 QUEUE_MONITOR_INTERVAL = 2.0 #master数据节点的ip与port MASTER_HOST = DATA_RUNNER_MASTER() MASTER_BIND_PORT = 6666