def _rewrite(self, javascript_code): first_token = Frewriter.tokenizer(javascript_code) token = first_token while token: if token.type == TokenType.IDENTIFIER and \ token.string == 'RES': start_paren = token.next quote = start_paren.next # print 'type: [%s] string: [%s]' % (quote.type, quote.string) if quote.type != TokenType.SINGLE_QUOTE_STRING_START and \ quote.type != TokenType.DOUBLE_QUOTE_STRING_START: logging.error('Invalid call RES() function @%s.', token.line_number) token = quote continue resource = quote.next if resource.type != TokenType.STRING_TEXT: logging.error('RES()\' parameter must be const STRING.') token = resource continue print 'type: [%s] string: [%s]' % (resource.type, resource.string) new_resource = self._found_resource(None, resource.string) if new_resource: resource.string = new_resource token = resource.next token = token.next return Frewriter.dump_source(first_token)
def find_widget_config(code): first_token = Frewriter.tokenizer(code) token = Frewriter.find_token(first_token, 'WIDGET_CONFIG') start_block = Frewriter.find_token(token, "{", TokenType.START_BLOCK) if start_block is None: return None end_block = Frewriter.find_end_token(start_block.next) config = Frewriter.dump_source(start_block, end_block) return config.strip().rstrip(';')
def testFindEndToken(self): token = Frewriter.find_token(self.first_token, "a.this_is_a_module.config") self.assertEquals(token.type, TokenType.SIMPLE_LVALUE) self.assertEquals(token.values['identifier'], 'a.this_is_a_module.config') start_block = Frewriter.find_token(token, "{", TokenType.START_BLOCK) self.assertTrue(start_block is not None) end_block = Frewriter.find_end_token(start_block.next) self.assertTrue(end_block is not None)
def testFindToken(self): token = Frewriter.find_token(self.first_token, "a.this_is_a_module.config") self.assertEquals(token.type, TokenType.SIMPLE_LVALUE) self.assertEquals(token.values['identifier'], 'a.this_is_a_module.config') token = Frewriter.find_token(self.first_token, "a.this_is_a_module.Fields") self.assertEquals(token.type, TokenType.SIMPLE_LVALUE) self.assertEquals(token.values['identifier'], 'a.this_is_a_module.Fields') token = Frewriter.find_token(self.first_token, "a.this_is_a_module.data") self.assertEquals(token.type, TokenType.SIMPLE_LVALUE) self.assertEquals(token.values['identifier'], 'a.this_is_a_module.data') self.assertEquals(token.string, 'a.this_is_a_module.data') token = Frewriter.find_token(self.first_token, "/*", TokenType.START_BLOCK_COMMENT) self.assertTrue(token is not None) self.assertEquals(token.type, TokenType.START_BLOCK_COMMENT) self.assertEquals(token.string, "/*") token = Frewriter.find_token(self.first_token, "a.this_is_a_module.NotFound") self.assertTrue(token is None)
def find_ad_config(code): first_token = Frewriter.tokenizer(code) token = Frewriter.find_token(first_token, 'AD_CONFIG') token = Frewriter.find_token(token, 'AD_CONFIG') start_block = token.next.next.next.next # whitespace operator whitespace START_BLOCK # start_block = Frewriter.find_token(token, "{", TokenType.START_BLOCK) before_start_block = start_block.previous if start_block is None or start_block.type != TokenType.START_BLOCK: return (None, None) end_block = Frewriter.find_end_token(start_block.next) config = Frewriter.dump_source(start_block, end_block) remove_tokens_between(start_block, end_block) # 插入var AD_CONFIG = %AD_CONFIG%; # 保证后续如果想把处理之后的AD_CONFIG重新放回到code的时候,有一个 # 可以确定的位置。 tokenutil.InsertTokenAfter(Token(';', TokenType.SEMICOLON, token.line, token.line_number), before_start_block) tokenutil.InsertTokenAfter(Token('%AD_CONFIG%', TokenType.IDENTIFIER, token.line, token.line_number), before_start_block) code = Frewriter.dump_source(first_token) config = config.strip().rstrip(';') return (code, config)
def register_action(module_path, action_path, app_cfg): # 修改module.js,添加path和action关系 abs_path = os.path.join(module_path, 'module.js') module_config_identifier = "%s.config" % app_cfg["app.module"] first_token = Frewriter.tokenizer(open(abs_path).read()) token = Frewriter.find_token(first_token, module_config_identifier) start_block = Frewriter.find_token(token, "{", TokenType.START_BLOCK) end_block = Frewriter.find_end_token(start_block.next) map_code = "\n// Autogenerated at %s\n%s['action'].push({'location':'%s','action':'%s'});" % ( datetime.now().strftime('%Y/%m/%d %H:%M:%S'), module_config_identifier, action_path, app_cfg['app.name']) Frewriter.append_code(end_block.previous, Frewriter.tokenizer(map_code)) open(abs_path, 'wb').write(Frewriter.dump_source(first_token)) logging.info('M %s' % abs_path)
def testInsertActionConfig(self): token = Frewriter.find_token(self.first_token, "a.this_is_a_module.config") self.assertEquals(token.type, TokenType.SIMPLE_LVALUE) self.assertEquals(token.values['identifier'], 'a.this_is_a_module.config') start_block = Frewriter.find_token(token, "{", TokenType.START_BLOCK) self.assertTrue(start_block is not None) end_block = Frewriter.find_end_token(start_block.next) self.assertTrue(end_block is not None) # a.this_is_a_module.config['action'].push({'location':'','action':''}); code = "a.this_is_a_module.config['action'].push({'location':'/jn/demo/xxx','action':'jn.demo.Xxx'});" Frewriter.append_code(end_block, Frewriter.tokenizer(code)) self.assertTrue(self._dump_source(self.first_token).find(code) > 0)
def create_material_config(path_prefix, action_name, app_cfg): """ 生成widget相关的代码 """ _ = lambda k : os.path.normpath(os.path.join(path_prefix, k)) abs_path = _(action_name + '.js') first_token = Frewriter.tokenizer(open(abs_path).read()) """ token = Frewriter.find_token(first_token, 'material.setWidgets', TokenType.IDENTIFIER) start_paren = Frewriter.find_token(token, '(', TokenType.START_PAREN) end_paren = Frewriter.find_end_token(start_paren.next, TokenType.START_PAREN, TokenType.END_PAREN) """ widgets = [] def ignore_whitespace(token): while token and token.type == TokenType.WHITESPACE: token = token.next return token token = first_token while token: if token.type == TokenType.IDENTIFIER: next_token = ignore_whitespace(token.next) if next_token and \ next_token.type == TokenType.START_PAREN and \ token.string.startswith('ad.widget'): # FIXME(leeight) 检测方式不合理 widgets.append(token.string) token = token.next if len(widgets) <= 0: logging.error('Can\'t find any widgets in %s.', abs_path) return # 建立namespace和filename的对应关系 action_name_regex = re.compile('(?P<package>([a-z\d_]+\.)+)(?P<action_name>[A-Z]\w+)', re.M|re.S) pattern = re.compile(r'goog.addDependency\("([^"]+)",\s*\[([^\)]*)\],\s*\[([^\)]*)\]\)') deps = {} for line in open('src/deps.js').read().splitlines(): match = pattern.findall(line) if match: filename = match[0][0] provides = match[0][1] for item in provides.replace('\'', '').split(','): if item.strip(): deps[item.strip()] = filename config = "{\n'id' : 'ec-ma-8964',\n'main_url' : 'http://www.baidu.com'" for widget in set(widgets): logging.debug("%s => %s" , widget, deps[widget]) abs_path = os.path.join('src', deps[widget].replace('.js', '.config.js')) if os.path.exists(abs_path): key = os.path.basename(abs_path).replace(".config.js", "") # XXX 不是所有人写的JSON都是那么规范,因此使用json.loads的话,可能 # 导致很多问题 config += (",\n'%s' : " % key) xxx = find_widget_config(open(abs_path, 'rb').read()) if xxx is None: logging.error(abs_path) else: config += xxx config += "\n}" app_cfg['material.config'] = config generate_code('material.config.js', _(action_name + '.config.js'), app_cfg)
def testAppendCode(self): token_stream1 = Frewriter.tokenizer("var hello = 'world';") token_stream2 = Frewriter.tokenizer("var happy = 'new year';\n\n\n\n\n\n//THIS IS COMMENT") Frewriter.append_code(token_stream1.next.next.next.next.next.next.next.next.next, token_stream2) self.assertEquals("var hello = 'world';var happy = 'new year';\n\n\n\n\n//THIS IS COMMENT\n", self._dump_source(token_stream1))
def setUp(self): base, ext = os.path.splitext(__file__) code = open(base + '.js').read() self.first_token = Frewriter.tokenizer(code)