def open(self, path): ''' open store or create it if not exist :return: ''' try: with Lock(self.lock): #init store path self.path = path if not path_exists(self.path): #create database self._create() else: # load database self._load() self._rebuild_tindex() return self Logger.info("open store %s...success. %d tables.", self.path, len(self.tables)) except Exception as e: Logger.error("open store %s...failed. error: %s", self.path, str(e)) raise e
def create(self, dbc, table): ''' create table :return self ''' try: #initialize table parameters self.dbc = dbc self.table = table self.name = table.name #check if table has exists if self._exists_table(): #exists table old_table = self.desc() if old_table != self.table: if is_subset(old_table.nfields(), self.table.nfields()): #upgrade table self._upgrade_table() else: #replace table self._replace_table else: #table is the same as in database pass else: #create new table self._create_table() Logger.info("create table %s...success.", self.name) return self except Exception as e: Logger.error("create table %s...failed. error: %s", self.name, str(e)) raise e
def select(self): ''' select all data from table :return: ''' try: nfields = self.table.nfields() sql, models = "select %s from %s;" % (",".join(quotes( nfields, '`')), self.name), [] cursor = self.dbc.cursor() cursor.execute(sql) results = cursor.fetchall() for result in results: model = {} for idx in range(0, len(result)): nfield = nfields[idx] vfield = result[idx] if isinstance(vfield, str): vfield = unescapes(vfield) model[nfield] = vfield models.append(model) Logger.info("select from table %s...success", self.name) return models except Exception as e: Logger.error("select from table %s...failed. error %s", self.name, str(e)) raise e
def open(self, host, user, pwd, dbn, port=3306): ''' open database or create it if not exist :return: ''' #init store path try: self.host, self.port, self.user, self.pwd = host, port, user, pwd self.dbn = dbn self.dbc = pymysql.connect(host=host, user=user, passwd=pwd, port=port) if not self._exists(): #create database self._create() self._use() else: # load database self._use() self._load() self._rebuild_tindex() return self Logger.info("open store mysql://%s:%s@%s:%d/%s...success. %d tables.", user, pwd, host, port, self.dbn, len(self.tables)) except Exception as e: Logger.error("open store mysql://%s:%s@%s:%d/%s...failed. error: %s", user, pwd, host, port, self.dbn, str(e)) raise e
def open_script_config(productModel): dict_script = { "default": "default", #没有适配的手机 "Xiaomi": "default", "samsung": "default", "360": "default", "Lenovo": "default", "vivo": "default", "HUAWEI": "default", "OPPO": "default", "gionee": "default", "xlj": "default", "yunso": "default", "oysin": "default" } try: if productModel in dict_script: return dict_script[productModel] else: return dict_script['default'] except: log = Logger() log.error("model_config.py, open_model_config : get model error") return dict_script['default']
def drop(self): ''' drop table :return: ''' try: remove_dir(self.path) except Exception as e: Logger.error("drop table %s...failed. error %s", self.name, str(e)) raise e
def drop(self): ''' drop table :return: ''' try: sql = "drop table if exists %s;" % self.name self.dbc.cursor().execute(sql) Logger.info("drop table %s...success", self.name) except Exception as e: Logger.error("drop table %s...failed. error %s", self.name, str(e)) raise e
def truncate(self): ''' truncate table :return: ''' try: with Lock(self.lock): remove_files(self.data_file) self._create_data_file() except Exception as e: Logger.error("truncate table %s...failed. error %s", self.name, str(e)) raise e
def truncate(self): ''' truncate table :return: ''' try: sql = "truncate table %s;" % table.name self.dbc.cursor().execute(sql) Logger.info("truncate table %s...success", self.name) except Exception as e: Logger.error("truncate table %s...failed. error %s", self.name, str(e)) raise e
def create(self, dbpath, table): ''' create table :return self ''' try: #initialize table parameters self.table = table self.name = table.name self.path = join_paths(dbpath, table.name) self.table_file = join_paths(self.path, "table") self.data_file = join_paths(self.path, "data") #create table directory if it is not exists make_dirs(self.path) #create or replace table file if is_file(self.table_file): #replace old table file if needed old_table = self.desc() if self.table != old_table: #replace table file self._replace_table_file() else: #new table is same as exists table pass else: #create new table file self._create_table_file() #create or upgrade or replace data file if is_file(self.data_file): #replace old data file if needed with open(self.data_file) as fdata: nfields = strips(fdata.readline().split(",")) if self.table.nfields() != nfields: if is_subset(nfields, self.table.nfields()): self._upgrade_data_file() else: self._replace_data_file() else: #create new data file self._create_data_file() Logger.info("create table %s...success.", self.name) return self except Exception as e: Logger.error("create table %s...failed. error: %s", self.name, str(e)) raise e
def test_device_connect(self): try: outInfo = subprocess.Popen('adb get-serialno', shell=True, stdout=subprocess.PIPE) out, err = outInfo.communicate() if out: return True # 连接成功 else: log = Logger() log.error( "device_handle.py, test_device_connect : no devices found") exit(1) except Exception: #print e #此处需要打印log exit(1)
def desc(self): ''' descrite table from store :return: Table ''' try: sql = "show create table %s;" % self.name cursor = self.dbc.cursor() cursor.execute(sql) create_sql = cursor.fetchall()[0][1] table = Table().fromsql(create_sql) Logger.info("describe table %s...success", self.name) return table except Exception as e: Logger.error("describe table %s...failed. error %s", self.name, str(e)) raise e
def get_uidump(self): ''' 获取当前Activity控制树 ''' temp_parent_path = os.getcwd() xml_path_parent = temp_parent_path + '\\temp' if (os.path.exists(xml_path_parent) == False): os.makedirs(xml_path_parent) xml_path = xml_path_parent + '\\uidump.xml' #time_start = time.clock() try: run_command = "adb pull /data/local/tmp/uidump.xml {path}".format( path=xml_path) os.popen("adb shell uiautomator dump /data/local/tmp/uidump.xml") os.popen(run_command) except: log = Logger() log.error("install_app_by_sax.py, get_uidump : xml can not pull") exit(1)
def open_model_config(deviceModel): dict_deviceModel={ "default" : "default", "OPPO_R9_Plusm_A": "OPPO", "vivo_Y67A" : "vivo", "MI_5": "Xiaomi", "Redmi_Note_4": "Xiaomi", "2014501": "Xiaomi", "Redmi_Note_3": "Xiaomi", "MI_2" : "Xiaomi", #安装后无法打开 "SM-G9300": "samsung", "1503-A01": "360", "Lenovo_K30-TM": "Lenovo", "Lenovo_A828t": "Lenovo", "F103": "gionee", "HLJ6": "xlj", "LA-S6": "xlj", "D9": "yunso", "OYSIN_X8": "oysin" } try: if deviceModel in dict_deviceModel: return dict_deviceModel[deviceModel] else: return dict_deviceModel['default'] except: log = Logger() log.error("model_config.py, open_model_config : get model error") return dict_deviceModel['default']
def insert(self, models): ''' insert data to table :param models: :return: ''' try: #get fields except default auto increment field nfields = [] for field in self.table.fields: if not isinstance(field.default, AutoIncValue): nfields.append(field.name) #prepare the values to be inserted values = [] for model in models: value = [] for nfield in nfields: vfield = model.get(nfield) if isinstance(vfield, str): value.append(quotes(escapes(vfield))) else: value.append(objtostr(vfield)) values.append("(%s)" % ",".join(value)) sql = "insert into %s(%s) values %s;" % (self.name, ",".join( quotes(nfields, '`')), ",".join(values)) self.dbc.cursor().execute(sql) self.dbc.commit() Logger.info("insert into table %s...success", self.name) except Exception as e: Logger.error("insert into table %s...failed. error %s", self.name, str(e)) raise e
class Worker(object): def __init__(self, queue_name, route, persister, wnum, port, logfile=sys.stdout): self.queue_name = queue_name self.route = route self.persister = persister self.stop = False self.pid = os.getpid() self.worker_id = '-'.join(['worker', str(wnum), queue_name, route, str(self.pid)]) self.log = Logger(self.worker_id,logfile=logfile, loglevel=logging.DEBUG) self.log.info("starting") self.host = socket.gethostbyname(socket.gethostname()) self.port = port self.register() def begin_execution(self, *args): while not self.stop: try: job = self.persister.get_job_from_queue(self.queue_name, self.worker_id, self.route) if job: self.do_job(job) sleep(0.1) except TriggeredInterrupt: # we were cut off by an interrupt trigger self.log.warn("received interrupt request; stopping current job") self.log.warn("no result will be committed and this job will not be restarted") self.stop_worker() def register(self): self.persister.add_worker(self.worker_id, self.host, self.port) def unregister(self): self.persister.delete_worker(self.worker_id) def do_job(self, job): # depickle body = pickle.loads(job['body']) directory = body['dir'] # FIXME a horrible hack where we add ourselves to the pythonpath sys.path.append(directory) mod = __import__(body['mod']) self.log.debug("successfully imported module "+str(mod)) if job['fn_type'] == 'method': parent = dill.loads(body['parent']) fn = getattr(parent, body['fn']) else: fn = getattr(mod, body['fn']) args = body['args'] kwargs = body['kwargs'] # call it self.persister.set_working(self.worker_id) try: ret = fn(*args, **kwargs) self.persister.save_result(job['job_id'], ret, 'complete') self.log.info(ret) except Exception as e: self.persister.save_result(job['job_id'], None, 'error') self.log.error(str(e)) exc_t, exc_v, exc_tr = sys.exc_info() self.log.error(str( '\n'.join(traceback.format_exception(exc_t, exc_v, exc_tr)))) self.log.debug("python path is %s" % str(sys.path)) finally: a = sys.path.pop() self.persister.unset_working(self.worker_id) def stop_worker(self): self.log.info("shutting down") self.unregister() self.stop = True
from config import Config import requests from COPASI import * from sim_spec_manager import SimulationSpecManager from util.log import Logger logger = Logger(push_to_crbmapi=True) # create a datamodel try: dataModel = CRootContainer.addDatamodel() except: dataModel = CRootContainer.getUndefinedFunction() sim_spec_manager = SimulationSpecManager() if not sim_spec_manager.parse_status: logger.error("Error encountered while parsing omex") sys.exit() def main(): # the only argument to the main routine should be the name of an SBML file # if len(args) != 1: # logger.error("Usage: copasi_sim SBMLFILE\n") # return 1 filename = sim_spec_manager.sbml_path try: # load the model if not dataModel.importSBML(filename): logger.info("Couldn't load {0}:".format(filename)) logger.info(CCopasiMessage.getAllMessageText())
class AbTemplate(BaseCase): step = "prepare to start" # Running times # If timeout, # This is used to determine whether to run setUp again. flag = 1 # Tester account login info login_account = "*****@*****.**" login_password = "******" # This should pass any validate code login_captcha = "eu3hfs7envhs8#" # case value username_value = "林嘉敏" # Define login page and index page login_page = login.LoginPage() index_page = index.IndexPage() def get_browser_img(self): if not os.path.exists(UrlUtil.image_dir): os.makedirs(UrlUtil.image_dir) self.driver.get_screenshot_as_file(UrlUtil.image_dir + "\\" + self.case_id + ".png") # utils load def util_load(self, file): file_name = os.path.basename(file) file_name = file_name.replace(".pyc", "_case").replace(".py", "_case") name_info = file_name.split("_", 1) self.case_id = name_info[0] self.case_name = name_info[1] self.logger = Logger(self.case_id) self.case_info = CaseInfo(self.case_id, self.case_name, "Functional testing") self.case_info.set_log_path(self.logger.log_file_rel_report) def timeout_method(self): self.flag += 1 self.case_info.step = self.step self.logger.error("time out exception") self.case_info.error_info = "time out exception" self.get_browser_img() raise TimeoutException def exception_method(self, err): self.case_info.step = self.step self.logger.error(err) self.case_info.error_info = err self.get_browser_img() def finally_method(self): self.case_info.end_time = DateUtil.get_date_to_second() time.sleep(2) self.driver.quit() # def setUp(self): self.driver = CaseUtil.get_driver() self.step = "convert to login page" self.convert_to_url(UrlUtil.get_login_page_url()) self.step = "fill login_name" self.fill(self.login_page.account_box, self.login_account) self.step = "fill password" self.fill(self.login_page.password_box, self.login_password) self.step = "fill verification_code" self.fill(self.login_page.captcha_box, self.login_captcha) self.step = "click login button" self.enter(self.login_page.submit_button) self.find_element(self.index_page.username_info) def tearDown(self): ReportUtil.write_to_report(self.case_info)
class Worker: def __init__(self, code: CodeBlock): self.code = BlockStmt(code) self.log = Logger("Algo") self.strict_typing = False self.callback_stop = lambda: () self.callback_input = None self.callback_print = None self.map = { DisplayStmt: self.exec_display, InputStmt: self.exec_input, AssignStmt: self.exec_assign, IfStmt: self.exec_if, ForStmt: self.exec_for, WhileStmt: self.exec_while, BreakStmt: self.exec_break, ContinueStmt: self.exec_continue, FuncStmt: self.exec_function, ReturnStmt: self.exec_return, CallStmt: self.exec_call, ElseStmt: self.exec_else, BaseStmt: lambda _: (), CommentStmt: lambda _: (), StopStmt: self.exec_stop, SleepStmt: self.exec_sleep } def reset_eval(self): """Resets the worker's evaluator object.""" self.evaluator = Evaluator() self.evaluator.log = self.log self.evaluator.strict_typing = self.strict_typing def stmt_input(self, prompt: str = None, text: bool = False) -> Any: """Executes an input statement.""" if self.callback_input is not None: res = self.callback_input(prompt) else: res = input(prompt) if text: return res p = Parser(str(res)) return self.evaluator.eval_node(p.parse()) def stmt_print(self, *args, end="\n"): """Executes a print statement.""" if self.callback_print is not None: self.callback_print(*args, end=end) return print(*args, end=end) def iterate_for(self, stmt: ForStmt) -> bool: """Updates a for loop.""" current = self.evaluator.get_variable(stmt.variable) begin = self.evaluator.eval_node(stmt.begin) end = self.evaluator.eval_node(stmt.end) step = None if stmt.step is None else self.evaluator.eval_node( stmt.step) if step is None: if end < begin: step = -1 else: step = 1 current = self.evaluator.binary_operation(current, step, "+") self.evaluator.set_variable(stmt.variable, current) return self.check_for_condition(stmt, current, step, begin, end) def check_for_condition(self, stmt: ForStmt, current=None, step=None, begin=None, end=None) -> bool: """Checks for the condition of a for loop.""" begin = self.evaluator.eval_node( stmt.begin) if begin is None else begin end = self.evaluator.eval_node(stmt.end) if end is None else end step = (None if stmt.step is None else self.evaluator.eval_node( stmt.step)) if step is None else step if step is None: if end < begin: step = -1 else: step = 1 condition_1 = bool( self.evaluator.binary_operation(current, begin, "><"[step < 0] + "=")) condition_2 = bool( self.evaluator.binary_operation(current, end, "<>"[step < 0] + "=")) return condition_1 and condition_2 def find_parent( self, types: Union[type, typing.Iterable[type]] ) -> Optional[Tuple[int, StackFrame]]: """Finds the nearest frame of the specified type.""" if not isinstance(types, Iterable): types = [types] for idx, frame in enumerate(reversed(self.stack)): if type(frame[0]) in types: return idx, frame return None def next_stmt(self) -> Optional[BaseStmt]: """Returns the next statement to be executed.""" while True: stmt, index = self.stack[-1] index += 1 if index >= len(stmt.children): if len(self.stack) == 1: self.finish(True) return None if isinstance(stmt, ForStmt): if self.iterate_for(stmt): self.stack[-1] = (stmt, -1) continue elif isinstance(stmt, WhileStmt): if bool(self.evaluator.eval_node(stmt.condition)): self.stack[-1] = (stmt, -1) continue elif isinstance(stmt, FuncStmt): self.calls.append(None) self.exit_block() return None self.exit_block() continue break self.stack[-1] = (stmt, index) return stmt.children[index] def peek_following(self) -> BaseStmt: """Returns the immediately following statement. Does not handle loops or functions.""" stmt, index = self.stack[-1] if index + 1 < len(stmt.children): return stmt.children[index + 1] if len(self.stack) == 1: self.finish() return None return None def enter_block(self, stmt: BlockStmt, value=None): """Pushes a new frame to the stack.""" self.stack.append((stmt, -1)) self.evaluator.enter_frame(value) def exit_block(self): """Pops the last frame from the stack.""" self.evaluator.exit_frame() return self.stack.pop() def exec_display(self, stmt: DisplayStmt): """Executes a display statement.""" self.stmt_print(str(self.evaluator.eval_node(stmt.content)), end="\n" if stmt.newline else "") def exec_input(self, stmt: InputStmt): """Executes an input statement.""" prompt = (translate("Algo", "Variable {var} = ").format( var=stmt.variable.code()) ) if stmt.prompt is None else self.evaluator.eval_node( stmt.prompt) self.assign(stmt.variable, self.stmt_input(prompt, stmt.text)) def assign(self, target: AstNode, value): """Assigns the specified value to the target (either variable or array access).""" if isinstance(target, IdentifierNode): self.evaluator.set_variable(target.value, value) elif isinstance(target, ArrayAccessNode): array = self.evaluator.eval_node(target.array) if not type(array) == list: self.log.error( translate("Algo", "Array access target must be of array type")) self.finish() return index = self.evaluator.eval_node(target.index) while index >= len(array): array.append(0) if index < len(array): array[index] = value else: self.log.error( translate( "Algo", "Index '{idx}' too big for array").format(idx=index)) return None else: self.log.error( translate( "Algo", "Assignment target must be either variable or array item")) self.finish() return def exec_assign(self, stmt: AssignStmt): """Executes an assignment statement.""" value = None if stmt.value is None else self.evaluator.eval_node( stmt.value) self.assign(stmt.variable, value) def exec_if(self, stmt: IfStmt): """Executes an if block.""" self.enter_block(stmt) condition = bool(self.evaluator.eval_node(stmt.condition)) self.if_status = (len(self.stack) - 1, condition) if not condition: self.exit_block() def exec_while(self, stmt: WhileStmt): """Executes a while loop.""" self.enter_block(stmt) predicate = bool(self.evaluator.eval_node(stmt.condition)) if not predicate: self.exit_block() def exec_for(self, stmt: ForStmt): """Executes a for loop.""" self.enter_block(stmt) self.evaluator.set_variable(stmt.variable, self.evaluator.eval_node(stmt.begin), local=True) if not self.check_for_condition( stmt, self.evaluator.get_variable(stmt.variable), None if stmt.step is None else self.evaluator.eval_node(stmt.step)): self.exit_block() def exec_break(self, stmt: BreakStmt): """Executes a break statement.""" if not self.find_parent(Loops): self.log.error( translate("Algo", "BREAK can only be used inside a loop")) self.finish() return while True: if isinstance(self.exit_block()[0], Loops): break def exec_continue(self, stmt: ContinueStmt): """Executes a continue statement.""" if not self.find_parent(Loops): self.log.error( translate("Algo", "CONTINUE can only be used inside a loop")) self.finish() return while not isinstance(self.stack[-1][0], Loops): self.exit_block() stmt, index = self.stack[-1] index = len(stmt.children) self.stack[-1] = stmt, index def exec_function(self, stmt: FuncStmt): """Executes a function definition block.""" frames = [x.copy() for x in self.evaluator.frames[1:]] def wrapper(*args): for frame in frames: self.evaluator.enter_frame(frame) result = self.call_function(stmt, *list(args)) for frame in frames: self.evaluator.exit_frame() return result self.evaluator.set_variable(stmt.name, wrapper) def exec_return(self, stmt: ReturnStmt): """Executes a return statement.""" if not self.find_parent(FuncStmt): self.log.error( translate("Algo", "RETURN can only be used inside a function")) self.finish() return self.calls.append( self.evaluator.eval_node(stmt.value) if stmt.value else None) while True: if isinstance(self.exit_block()[0], FuncStmt): break def call_function(self, stmt: FuncStmt, *args) -> Optional[Any]: """Calls the specified function.""" self.enter_block( stmt, {stmt.parameters[idx]: arg for idx, arg in enumerate(args)}) length = len(self.stack) while len(self.stack) >= length and not self.finished: self.step() if len(self.stack) != length - 1: self.log.error( translate( "Algo", "Stack corruption after calling function %s" % stmt.name)) return None return self.calls.pop() def exec_call(self, stmt: CallStmt): """Executes a function call statement.""" self.evaluator.eval_node(stmt.to_node()) def exec_else(self, stmt: ElseStmt): """Executes an else block.""" if self.if_status is None: self.log.error( translate("Algo", "ELSE can only be used after an IF block")) self.finish() return if not self.if_status[1]: self.enter_block(stmt) self.if_status = None def exec_stop(self, stmt: StopStmt): """Executes a breakpoint statement.""" self.stopped = True self.callback_stop(stmt) def exec_sleep(self, stmt: SleepStmt): """Executes a sleep statement""" time.sleep(self.evaluator.eval_node(stmt.duration)) def step(self): """Executes a step.""" stmt = self.next_stmt() self.last = stmt if stmt is None: return self.exec_stmt(stmt) if self.break_on_error and self.log.messages: self.finish() def exec_stmt(self, stmt): """Executes the specified statement.""" self.last = stmt if self.if_status is not None and type(stmt) != ElseStmt and len( self.stack) <= self.if_status[0]: self.if_status = None if type(stmt) not in self.map: self.log.error( translate( "Algo", "Unknown statement type: {type}").format(type=type(stmt))) self.finish() return self.map[type(stmt)](stmt) def finish(self, normal=False): """Ends the execution.""" self.finished = True self.error = not normal self.evaluator.exit_frame() def init(self): """Initialises the worker.""" self.reset_eval() self.stack = [(self.code, -1)] self.calls = [] self.if_status = None self.finished = False self.error = False self.evaluator.enter_frame() self.stopped = False self.break_on_error = False self.last = None def run(self): """Runs the program continuously.""" self.init() while not self.finished: self.step()
logger = Logger().get_logger() if __name__ == '__main__': input_triples = "result/triples.txt" client = MongoClient() client = MongoClient('127.0.0.1', 27017) db = client.relation_extraction # 连接数据库,没有则自动创建 triples = db.distant_supervised # 使用集合,没有则自动创建 triples_sents = IOHelper.read_lines(input_triples) triples_sents = list(set(triples_sents)) if triples_sents == None: logger.error('read failed!') sys.exit(0) begin = datetime.datetime.now() try: count = 1 for sent in triples_sents: if sent.strip() == '': continue doc = {} doc['e1'], doc['rel'], doc['e2'] = sent.strip().split('\t') if doc['rel'] == '中文名': # 中文名字涉及的实体是一样的,所以过滤 continue triples.insert(doc) logger.info('insert {} triples'.format(count)) count += 1 except Exception as e:
class Parser: """Main parser class. Transforms a string into an AST tree.""" expression: str = None tokens: List[Token] = None index: int = None log: Logger = None def __init__(self, expr: str): """Initializes the Parser instance. expr -- the expression to be parsed""" self.expression = expr self.tokens = [] self.index = 0 self.log = Logger("Parser") def fix_mul_tok(self): """Fixes implicit multiplication (e.g. 2pi) at token level.""" result = [] previous = (None, None) for tok in self.tokens: if previous[0] == TokenType.NUMBER and tok[ 0] == TokenType.IDENTIFIER: result.append((TokenType.OPERATOR, "*")) result.append(tok) previous = tok self.tokens = result def next_token(self) -> Token: """Reads the next token and advances the position.""" self.index += 1 return self.tokens[self.index - 1] def peek_token(self) -> Optional[Token]: """Reads the next token without affecting position.""" if not self.can_read(): return None return self.tokens[self.index] def match_token(self, token_type: TokenType, value=None) -> bool: """Checks if the next token matches the specified token type and (optional) value.""" return self.can_read() \ and self.peek_token()[0] == token_type \ and ((not value) or self.peek_token()[1] in (value if type(value) == list else [value])) def accept_token(self, token_type: TokenType, value=None) -> bool: """If the next token matches, advance and return True, otherwise return False without advancing.""" if self.match_token(token_type, value): self.index += 1 return True return False def accept_operator(self, operator: str) -> bool: """Wrapper for accept(OPERATOR, operator).""" return self.accept_token(TokenType.OPERATOR, operator) def expect_token(self, token_type: TokenType, value=None): """Asserts the next token is of the specified type and (optional) value. Explodes otherwise.""" if not self.match_token(token_type, value): self.log.error( translate("Parser", "Expected token ({type}) '{val}'").format( type=TokenType.get_name(token_type), val=value)) return None return self.next_token() def can_read(self) -> bool: """Checks if there is still anything to read.""" return self.index < len(self.tokens) def tokenize(self): """Converts the expression string into a linear list of tokens.""" regex = re.compile( r"(\+|-|/|%|\^|\*\*|\*|" r"==|!=|<=|>=|<|>|" r"\(|\)|\[|\]|{|\}|" r"\bET\b|\bAND\b|\bOU\b|\bOR\b|\bXOR\b|\bNON\b|\bNOT\b|" r"\bVRAI\b|\bFAUX\b|\bTRUE\b|\bFALSE\b|" r"&|\||,| |\"(?:[^\"]*)\")", re.IGNORECASE) tokenized = [ x.strip() for x in regex.split(self.expression) if x.strip() ] # fix exponents new_tokens = [] idx = 0 while idx < len(tokenized): current = tokenized[idx] if idx < len(tokenized) - 2 and current[0].isdigit( ) and current[-1].upper() == "E" and tokenized[idx + 1] in ["+", "-"]: # if there is an E followed by a number, this is an exponent notation current += tokenized[idx + 1] current += tokenized[idx + 2] idx += 2 new_tokens.append(current) idx += 1 for token in new_tokens: if token.upper() in Operators.ops: self.tokens.append((TokenType.OPERATOR, token.upper())) elif token.upper() in TokenType.TrueVal: self.tokens.append((TokenType.BOOLEAN, True)) elif token.upper() in TokenType.FalseVal: self.tokens.append((TokenType.BOOLEAN, False)) elif token == ",": self.tokens.append((TokenType.COMMA, ",")) elif token in ["(", ")"]: self.tokens.append((TokenType.PAREN, token)) elif token in ["[", "]"]: self.tokens.append((TokenType.BRACK, token)) elif token in ["{", "}"]: self.tokens.append((TokenType.BRACE, token)) else: if token[0] == token[-1] == '"': self.tokens.append((TokenType.STRING, token[1:-1])) else: try: if re.search('^[0-9]+$', token): num = int(token) else: num = float(token) self.tokens.append((TokenType.NUMBER, num)) except: if re.search('^[a-zA-Z_0-9]+$', token): match = re.search('^([0-9]+)([a-zA-Z_0-9]+$)', token) if match: factor, variable = match.groups() self.tokens.append( (TokenType.NUMBER, int(factor))) self.tokens.append((TokenType.OPERATOR, "*")) self.tokens.append( (TokenType.IDENTIFIER, variable)) else: self.tokens.append( (TokenType.IDENTIFIER, token)) else: self.tokens.append((TokenType.INVALID, token)) self.fix_mul_tok() def match_operator(self, expected: List[str]) -> str: """Checks if any of the specified operators are to be found.""" for x in expected: if self.accept_operator(x): return x def parse(self) -> Optional[nodes.AstNode]: """Main parsing routine.""" self.tokenize() result = self.parse_expression() if self.can_read(): self.log.error( translate( "Parser", "Unexpected token ({type}) '{val}' after end of expression" ).format(type=TokenType.get_name(self.peek_token()[0]), val=self.peek_token()[1])) return None return result def parse_expression(self) -> nodes.AstNode: """Parses an expression.""" return self.parse_or() def parse_or(self) -> nodes.AstNode: """Parses an OR operation.""" expr = self.parse_xor() while self.match_token(TokenType.OPERATOR): op = self.match_operator(["OR", "OU", "|"]) if op: expr = nodes.BinOpNode(expr, self.parse_xor(), "|") continue break return expr def parse_xor(self) -> nodes.AstNode: """Parses a XOR operation.""" expr = self.parse_and() while self.match_token(TokenType.OPERATOR): op = self.match_operator(["XOR"]) if op: expr = nodes.BinOpNode(expr, self.parse_and(), "XOR") continue break return expr def parse_and(self) -> nodes.AstNode: """Parses an AND operation.""" expr = self.parse_equality() while self.match_token(TokenType.OPERATOR): op = self.match_operator(["AND", "ET", "&"]) if op: expr = nodes.BinOpNode(expr, self.parse_equality(), "&") continue break return expr def parse_equality(self) -> nodes.AstNode: """Parses a comparison/equality.""" expr = self.parse_additive() while self.match_token(TokenType.OPERATOR): op = self.match_operator(Operators.comp) if op: expr = nodes.BinOpNode(expr, self.parse_additive(), op) continue break return expr def parse_additive(self) -> nodes.AstNode: """Parses an addition or subtraction.""" expr = self.parse_multiplicative() while self.match_token(TokenType.OPERATOR): op = self.match_operator(["+", "-"]) if op: expr = nodes.BinOpNode(expr, self.parse_multiplicative(), op) continue break return expr def parse_multiplicative(self) -> nodes.AstNode: """Parses a product, division, or modulus.""" expr = self.parse_exponent() while self.match_token(TokenType.OPERATOR): op = self.match_operator(["*", "/", "%"]) if op: expr = nodes.BinOpNode(expr, self.parse_exponent(), op) continue break return expr def parse_exponent(self) -> nodes.AstNode: """Parses an exponentiation.""" expr = self.parse_unary() while self.match_token(TokenType.OPERATOR): op = self.match_operator(["^", "**"]) if op: expr = nodes.BinOpNode(expr, self.parse_unary(), op) continue break return expr def parse_unary(self) -> nodes.AstNode: """Parses an unary operation.""" op = self.match_operator(["+", "-", "NON", "NOT", "*"]) if op: return nodes.UnaryOpNode(self.parse_unary(), op) return self.parse_call_pre() def parse_call_pre(self) -> nodes.AstNode: """Parses a function call (1).""" return self.parse_call(self.parse_term()) def parse_arg_list(self, array=False) -> List[nodes.AstNode]: """Parses an argument list.""" result = [] if array: tok_type = TokenType.BRACK sym_open = "[" sym_end = "]" else: tok_type = TokenType.PAREN sym_open = "(" sym_end = ")" self.expect_token(tok_type, sym_open) while not self.match_token(tok_type, sym_end): result.append(self.parse_expression()) if not self.accept_token(TokenType.COMMA): break self.expect_token(tok_type, sym_end) return result def parse_param_list(self) -> List[str]: """Parses a lambda function parameter list.""" result = [] self.expect_token(TokenType.BRACE, "{") while not self.match_token(TokenType.BRACE, "}"): result.append(self.expect_token(TokenType.IDENTIFIER)[1]) if not self.accept_token(TokenType.COMMA): break self.expect_token(TokenType.BRACE, "}") return result def parse_indexer(self) -> nodes.AstNode: """Parses an indexer expression.""" self.expect_token(TokenType.BRACK, "[") expr = self.parse_expression() self.expect_token(TokenType.BRACK, "]") return expr def parse_call(self, left: nodes.AstNode) -> nodes.AstNode: """Parses a function call (2).""" if self.match_token(TokenType.PAREN, "("): return self.parse_call(nodes.CallNode(left, self.parse_arg_list())) elif self.match_token(TokenType.BRACK, "["): return self.parse_call( nodes.ArrayAccessNode(left, self.parse_indexer())) else: return left def parse_term(self) -> Optional[nodes.AstNode]: """Parses an atomic term.""" if self.match_token(TokenType.NUMBER): return nodes.NumberNode(self.next_token()[1]) elif self.match_token(TokenType.BOOLEAN): return nodes.NumberNode(bool(self.next_token()[1])) elif self.match_token(TokenType.STRING): return nodes.StringNode(self.next_token()[1]) elif self.match_token(TokenType.IDENTIFIER): return nodes.IdentifierNode(self.next_token()[1]) elif self.accept_token(TokenType.PAREN, "("): stmt = self.parse_expression() self.expect_token(TokenType.PAREN, ")") return stmt elif self.match_token(TokenType.BRACK, "["): stmt = nodes.ListNode(self.parse_arg_list(True)) return stmt elif self.match_token(TokenType.BRACE, "{"): args = self.parse_param_list() self.expect_token(TokenType.PAREN, "(") expr = self.parse_expression() self.expect_token(TokenType.PAREN, ")") return nodes.LambdaNode(args, expr) else: if not self.can_read(): self.log.error(translate("Parser", "Unexpected EOL")) else: self.log.error( translate( "Parser", "Unexpected token ({type}) '{val}'").format( type=TokenType.get_name(self.peek_token()[0]), val=self.peek_token()[1])) return None def beautify(self): """Beautifies the expression (adds spaces between operators).""" result = "" prev2: Token = None prev1: Token = None for typ, val in self.tokens: # remove space between operator and term only if operator is unary if (result # only if string contains contents and result[-1] == " " # only if there is a space to remove and (typ in TokenType.Term or val in TokenType.Opening ) # only if this is a term or block and ((prev1[1] in TokenType.UnaryVal) # only for unary op and ((not prev2) or (prev2[1] in TokenType.Opening + TokenType.UnaryVal ) # no space between opening and unary op or (prev2[0] == TokenType.COMMA) # no space before comma ))): result = result[:-1] # add space before operator only if after term or closing if (typ == TokenType.OPERATOR # only if operator and prev1 # and something before and (prev1[1] not in TokenType.Opening) # no space after opening and (prev1[0] != TokenType.OPERATOR or (prev2 and prev2[1] not in TokenType.Opening))): result += " " # remove space for i if typ == TokenType.IDENTIFIER and val == "i": if prev1 and prev1 == (TokenType.OPERATOR, "*"): result = result[:-3] # token if typ in [TokenType.NUMBER, TokenType.BOOLEAN]: result += proper_str(val) elif typ == TokenType.STRING: result += '"' + str(val) + '"' else: result += str(val) # comma is always followed by space if typ == TokenType.COMMA: result += " " if (typ == TokenType.OPERATOR and prev1 and prev1[0] != TokenType.OPERATOR and prev1[1] not in TokenType.Opening): result += " " prev2 = prev1 prev1 = (typ, val) # remove double whitespaces return re.sub("\s\s+", " ", result)
class DeviceHandle: def __init__(self): self.log = Logger() # 检测手机是否连接成功 def getID(self): adbOutInfo = [] outInfo = subprocess.Popen('adb devices', shell=True, stdout=subprocess.PIPE) out, err = outInfo.communicate() for line in out.splitlines(): adbOutInfo.append(line) ID = adbOutInfo[1].split('\t')[0] return ID # 获取手机型号 def getModel(self): try: adbOutModel = [] outInfo = subprocess.Popen('adb -d shell getprop ro.product.model', shell=True, stdout=subprocess.PIPE) out, err = outInfo.communicate() for line in out.splitlines(): adbOutModel.append(line) model = adbOutModel[0] model = '_'.join(model.split()) # 将空格全部变成下划线_ return model except: self.log.error("device_handle.py, getModel : devices disconnect") exit(1) #获取api_level def getAPILeve(self): outInfo = subprocess.Popen('adb shell getprop ro.build.version.sdk', shell=True, stdout=subprocess.PIPE) apiLevel, err = outInfo.communicate() return int(apiLevel) #获取手机商标 oppo vivo def getBrand(self): try: outInfo = subprocess.Popen('adb shell getprop ro.product.brand', shell=True, stdout=subprocess.PIPE) brand, err = outInfo.communicate() return brand except: self.log.error("device_handle.py, getBrand : devices disconnect") exit(1) # 获取巡行的进程 def getProcess(self): adbOutProcess = [] outInfo = subprocess.Popen('adb shell ps com.Nrush.forensictool', shell=True, stdout=subprocess.PIPE) out, err = outInfo.communicate() for line in out.splitlines(): adbOutProcess.append(line) try: if adbOutProcess[3]: return True except: return False #获取安装apk列表 def get_apk_install(self): try: outInfo = subprocess.Popen( 'adb shell pm list packages com.Nrush.forensictool', shell=True, stdout=subprocess.PIPE) out, err = outInfo.communicate() return out except: self.log.error( "device_handle.py, get_apk_install : devices disconnect") exit(1) #启动 def activity_app(self): try: subprocess.Popen( 'adb shell am start -n com.Nrush.forensictool/.MainActivity', shell=True, stdout=subprocess.PIPE) except: self.log.error( "device_handle.py, activity_app : devices disconnect") exit(1) #获取手机系统版本 def getRelease(self): adbOutRelease = [] outInfo = subprocess.Popen( 'adb shell getprop ro.build.version.release', shell=True, stdout=subprocess.PIPE) out, err = outInfo.communicate() for line in out.splitlines(): adbOutRelease.append(line) release = adbOutRelease[0] return release # 检查是否连接成功 def test_device_connect(self): try: outInfo = subprocess.Popen('adb get-serialno', shell=True, stdout=subprocess.PIPE) out, err = outInfo.communicate() if out: return True # 连接成功 else: log = Logger() log.error( "device_handle.py, test_device_connect : no devices found") exit(1) except Exception: #print e #此处需要打印log exit(1)
class Evaluator: frames = None arguments = None log = None beautified = None strict_typing = False node_tree = None def __init__(self, strict=False): self.frames = [{}] for name, item in mlib.__dict__.items(): if isinstance(item, types.ModuleType): for member_name, member in item.__dict__.items(): if callable(member): # if function doc_func = mlib.find_function(member_name) if doc_func: member.doc_spec = doc_func self.frames[0][member_name] = member elif member_name.startswith("c_"): # if constant self.frames[0][member_name[2:]] = member for alias, func in mlib.docs.ext_aliases: self.frames[0][alias] = self.frames[0][func] self.log = Logger("Eval") self.strict_typing = strict def enter_frame(self, value: Dict[str, Any] = None): """Pushes the a new frame / the specified frame to the stack. value -- Frame (optional)""" self.frames.append(value or {}) def exit_frame(self): """Pops the last frame from the stack.""" return self.frames.pop() def set_variable(self, variable: str, value: Any, local=False): """Sets the value of the specified variable. variable -- Variable name value -- Variable value local -- If true, forces the creation of a new variable in the current frame. Otherwise, sets the value in the nearest frame containing the variable.""" if not local: for frame in reversed(self.frames): if variable in frame: frame[variable] = value return self.frames[-1][variable] = value def get_variable(self, variable: str) -> Optional[Any]: """Returns the value of the specified variable. variable -- Variable name""" for frame in reversed(self.frames): if variable in frame: return frame[variable] self.log.error( translate("Evaluator", "Cannot find variable or function {name}").format( name=variable)) return None def evaluate(self, expr: str) -> Optional[Any]: """Evaluates the specified expression. If an error occurs during the parsing, the result will be None. If an error occurs during the evaluation, the result will be undefined.""" parser = Parser(expr) parser.log = self.log node = None try: node = parser.parse() except: if DEBUG: raise self.log.error( translate("Evaluator", "Parser: ") + str(sys.exc_info()[1])) self.beautified = parser.beautify() if not node: return None return self.evaluate_parsed(node) def evaluate_parsed(self, node: nodes.AstNode): """Evaluates the specified root node. node -- Node to be evaluated""" self.node_tree = node self.beautified = self.node_tree.code() result = None try: result = self.eval_node(self.node_tree) except: if DEBUG: raise self.log.error(str(sys.exc_info()[1])) return result def eval_node(self, node: nodes.AstNode): """Evaluates the specified node. Basically just a wrapper for eval_node_real that handles all the IEEE754 floating point rounding fuckery.""" value = self.eval_node_real(node) if value is not None and is_num(value) and not isinstance(value, bool): if isinstance(value, complex): # if real, convert to float directly if is_real(value): value = value.real # only convert to int if it fits well if is_int(value) and 1 <= abs(value) <= 1e15: value = int(round(value)) # if zero, use zero directly if is_zero(value): value = 0 else: if not (type(value) == int and value != int(float(value))): value = close_round(value, 12) return value def call_lambda(self, node: nodes.LambdaNode, *args): """Calls the specified lambda function. node -- Function node args -- Argument list""" args = list(args) if len(args) != len(node.args): self.log.error( translate( "Evaluator", "Argument count mismatch (expected {exp}, got {act})"). format(exp=len(node.args), act=len(args))) return None frame = {node.args[idx]: arg for idx, arg in enumerate(args)} self.enter_frame(frame) result = self.eval_node(node.expr) if callable(result): if not hasattr(result, "frames"): result.frames = [] result.frames.append(frame.copy()) # pop arguments after use self.exit_frame() return result def eval_node_real(self, node: nodes.AstNode): """Evaluates the specified node. node -- Node to be evaluated""" if isinstance(node, nodes.ListNode): return [self.eval_node(x) for x in node.value] if isinstance(node, (nodes.NumberNode, nodes.StringNode)): return node.value if isinstance(node, nodes.IdentifierNode): return self.get_variable(node.value) if isinstance(node, nodes.UnaryOpNode): return self.eval_unary(node) if isinstance(node, nodes.BinOpNode): return self.eval_binary(node) if isinstance(node, nodes.CallNode): return self.eval_call(node) if isinstance(node, nodes.ArrayAccessNode): return self.eval_array_access(node) if isinstance(node, nodes.LambdaNode): return self.eval_lambda(node) # if the object is not a node, it must be a remnant of an already-parsed value # return it directly if not isinstance(node, nodes.AstNode): return node # if it's an unknown descendant of AstNode # this should never happen, but we put a message just in case self.log.error( translate("Evaluator", "Unknown node type: {type}").format(type=type(node))) return None def eval_lambda(self, node: nodes.LambdaNode): """Evaluates the specified lambda function node.""" return lambda *args: self.call_lambda(node, *list(args)) def eval_array_access(self, node: nodes.ArrayAccessNode): """Evaluates the specified array access node.""" array = self.eval_node(node.array) index = int(self.eval_node(node.index)) if type(array) != list: self.log.error( translate("Evaluator", "Array access target must be of array type")) return None if index < len(array): return array[index] else: self.log.error( translate("Evaluator", "Index '{idx}' too big for array").format(idx=index)) return None def eval_call(self, node: nodes.CallNode): """Evaluates the specified function call node.""" function = self.eval_node(node.func) if function is None: self.log.error(translate("Evaluator", "Callee is None")) return None if (len(node.args) == 1 and isinstance(node.args[0], nodes.UnaryOpNode) and node.args[0].operator == "*"): # expand list of arguments arg_list = self.eval_node(node.args[0].value) if type(arg_list) != list: self.log.error( translate("Evaluator", "Only lists can be expanded")) return None args = arg_list else: args = [self.eval_node(x) for x in node.args] if hasattr(function, "doc_spec") and not hasattr(function, "listfunc"): arg_spec = function.doc_spec[1][1] num_opt = sum(1 for arg in arg_spec if len(arg) >= 4) if not (len(arg_spec) - num_opt <= len(args) <= len(arg_spec)): self.log.error( translate( "Evaluator", "Argument count mismatch (expected {exp}, got {act})"). format(exp=len(arg_spec) - num_opt, act=len(args))) return None for idx, (arg, spec) in enumerate(zip(args, arg_spec)): if not check_type(arg, spec[1]): self.log.error( translate( "Evaluator", "Type mismatch for argument #{idx} '{arg}' (expected {exp})" ).format(idx=idx + 1, arg=spec[0], exp=spec[1])) return None if hasattr(function, "frames"): for f in function.frames: self.enter_frame(f) result = function.__call__(*args) if hasattr(function, "frames"): for _ in function.frames: self.exit_frame() return result def eval_unary(self, node: nodes.UnaryOpNode): """Evaluates the specified unary operation node.""" value = self.eval_node(node.value) value_type = ValueType.get_type(value) if node.operator == "+": return value if node.operator == "-" and (is_num(value) and (not self.strict_typing or not is_bool(value))): return -value if node.operator == "-" and value_type == ValueType.LIST: return value[::-1] if node.operator == "NOT" and (is_bool(value) or (not self.strict_typing and is_num(value))): return not value self.log.error( translate( "Evaluator", "Invalid unary operator '{op}'").format(op=node.operator)) return None def eval_binary(self, node: nodes.BinOpNode): """Evaluates the specified binary operation node. Wrapper for binary_operation.""" return self.binary_operation(self.eval_node(node.left), self.eval_node(node.right), node.operator) def binary_operation(self, left, right, operator): """Evaluates the specified binary operation.""" left_type = ValueType.get_type(left) right_type = ValueType.get_type(right) if left is None or right is None: self.log.error(translate("Evaluator", "Trying to use None")) return None if operator == "+" and ValueType.STRING in [left_type, right_type]: # if one of the operands is a string return str(left) + str(right) if operator in [ "*" ] and right_type == ValueType.LIST and left_type != ValueType.LIST: # if one operand is list and not the other, then put the list at left # so we don't have to handle both cases afterwards (left, left_type, right, right_type) = (right, right_type, left, left_type) result = None if self.strict_typing: if left_type != right_type: self.log.error( translate( "Evaluator", "Type mismatch: operands have different types ({left} and {right})" ).format(left=ValueType.get_name(left_type), right=ValueType.get_name(right_type))) return None if left_type == ValueType.BOOLEAN: allowed = Operators.boolean elif left_type == ValueType.NUMBER: allowed = Operators.math + Operators.comp elif left_type == ValueType.STRING: allowed = Operators.eq + ["+"] elif left_type == ValueType.LIST: allowed = Operators.eq + ["+", "-", "&", "|"] else: error_pos = [] if left_type is None: error_pos.append(translate("Evaluator", "left")) if right_type is None: error_pos.append(translate("Evaluator", "right")) self.log.error( translate( "Evaluator", "Invalid value type for {val} and operator '{op}'"). format(val=translate("Evaluator", " and ").join(error_pos), op=operator)) return None if operator not in allowed: self.log.error( translate( "Evaluator", "Operator '{op}' not allowed for value type {type}"). format(op=operator, type=ValueType.get_name(left_type))) return None # arithmetic if operator == "+": result = left + right elif operator == "-": if left_type == right_type == ValueType.LIST: result = [x for x in left if x not in right] else: result = left - right elif operator == "*": if left_type == ValueType.LIST: if not is_int(right): self.log.error( translate( "Evaluator", "Trying to multiply List by non-integer ({val})"). format(val=right)) return None else: result = left * int(right) else: result = left * right elif operator == "/": if right == 0: self.log.error( translate("Evaluator", "Trying to divide by zero")) return None result = left / right elif operator == "%": result = math.fmod(left, right) elif operator in ["^", "**"]: result = left**right # comparison elif operator == "<=": result = left <= right or is_close(left, right) elif operator == "<": result = left < right elif operator == ">": result = left > right elif operator == ">=": result = left >= right or is_close(left, right) # equality elif operator == "==": result = is_close(left, right) elif operator == "!=": result = not is_close(left, right) # logic / bitwise elif operator == "&": if left_type == right_type == ValueType.LIST: result = [x for x in left if x in right] else: result = int(left) & int(right) elif operator == "|": if left_type == right_type == ValueType.LIST: result = list(set(left + right)) else: result = int(left) | int(right) elif operator == "XOR": if left_type == right_type == ValueType.LIST: result = list( set(x for x in left + right if x not in left or x not in right)) else: result = int(left) ^ int(right) if result is None: self.log.error( translate( "Evaluator", "Invalid binary operator '{op}' for '{left}' and '{right}'" ).format(op=operator, left=left, right=right)) else: if is_bool(left) and is_bool(right): # if both operands are bool, then cast the whole thing to bool so it looks like we're professional result = bool(result) return result