def _rewrite(self, javascript_code):
first_token = Frewriter.tokenizer(javascript_code)
token = first_token
while token:
if token.type == TokenType.IDENTIFIER and \
token.string == 'RES':
start_paren = token.next
quote = start_paren.next
# print 'type: [%s] string: [%s]' % (quote.type, quote.string)
if quote.type != TokenType.SINGLE_QUOTE_STRING_START and \
quote.type != TokenType.DOUBLE_QUOTE_STRING_START:
logging.error('Invalid call RES() function @%s.', token.line_number)
token = quote
continue
resource = quote.next
if resource.type != TokenType.STRING_TEXT:
logging.error('RES()\' parameter must be const STRING.')
token = resource
continue
print 'type: [%s] string: [%s]' % (resource.type, resource.string)
new_resource = self._found_resource(None, resource.string)
if new_resource:
resource.string = new_resource
token = resource.next
token = token.next
return Frewriter.dump_source(first_token)
def find_widget_config(code):
first_token = Frewriter.tokenizer(code)
token = Frewriter.find_token(first_token, 'WIDGET_CONFIG')
start_block = Frewriter.find_token(token, "{", TokenType.START_BLOCK)
if start_block is None:
return None
end_block = Frewriter.find_end_token(start_block.next)
config = Frewriter.dump_source(start_block, end_block)
return config.strip().rstrip(';')
def testFindEndToken(self):
token = Frewriter.find_token(self.first_token, "a.this_is_a_module.config")
self.assertEquals(token.type, TokenType.SIMPLE_LVALUE)
self.assertEquals(token.values['identifier'], 'a.this_is_a_module.config')
start_block = Frewriter.find_token(token, "{", TokenType.START_BLOCK)
self.assertTrue(start_block is not None)
end_block = Frewriter.find_end_token(start_block.next)
self.assertTrue(end_block is not None)
def testFindToken(self):
token = Frewriter.find_token(self.first_token, "a.this_is_a_module.config")
self.assertEquals(token.type, TokenType.SIMPLE_LVALUE)
self.assertEquals(token.values['identifier'], 'a.this_is_a_module.config')
token = Frewriter.find_token(self.first_token, "a.this_is_a_module.Fields")
self.assertEquals(token.type, TokenType.SIMPLE_LVALUE)
self.assertEquals(token.values['identifier'], 'a.this_is_a_module.Fields')
token = Frewriter.find_token(self.first_token, "a.this_is_a_module.data")
self.assertEquals(token.type, TokenType.SIMPLE_LVALUE)
self.assertEquals(token.values['identifier'], 'a.this_is_a_module.data')
self.assertEquals(token.string, 'a.this_is_a_module.data')
token = Frewriter.find_token(self.first_token, "/*", TokenType.START_BLOCK_COMMENT)
self.assertTrue(token is not None)
self.assertEquals(token.type, TokenType.START_BLOCK_COMMENT)
self.assertEquals(token.string, "/*")
token = Frewriter.find_token(self.first_token, "a.this_is_a_module.NotFound")
self.assertTrue(token is None)
def find_ad_config(code):
first_token = Frewriter.tokenizer(code)
token = Frewriter.find_token(first_token, 'AD_CONFIG')
token = Frewriter.find_token(token, 'AD_CONFIG')
start_block = token.next.next.next.next # whitespace operator whitespace START_BLOCK
# start_block = Frewriter.find_token(token, "{", TokenType.START_BLOCK)
before_start_block = start_block.previous
if start_block is None or start_block.type != TokenType.START_BLOCK:
return (None, None)
end_block = Frewriter.find_end_token(start_block.next)
config = Frewriter.dump_source(start_block, end_block)
remove_tokens_between(start_block, end_block)
# 插入var AD_CONFIG = %AD_CONFIG%;
# 保证后续如果想把处理之后的AD_CONFIG重新放回到code的时候，有一个
# 可以确定的位置。
tokenutil.InsertTokenAfter(Token(';', TokenType.SEMICOLON,
token.line, token.line_number), before_start_block)
tokenutil.InsertTokenAfter(Token('%AD_CONFIG%', TokenType.IDENTIFIER,
token.line, token.line_number), before_start_block)
code = Frewriter.dump_source(first_token)
config = config.strip().rstrip(';')
return (code, config)
def register_action(module_path, action_path, app_cfg):
# 修改module.js，添加path和action关系
abs_path = os.path.join(module_path, 'module.js')
module_config_identifier = "%s.config" % app_cfg["app.module"]
first_token = Frewriter.tokenizer(open(abs_path).read())
token = Frewriter.find_token(first_token, module_config_identifier)
start_block = Frewriter.find_token(token, "{", TokenType.START_BLOCK)
end_block = Frewriter.find_end_token(start_block.next)
map_code = "\n// Autogenerated at %s\n%s['action'].push({'location':'%s','action':'%s'});" % (
datetime.now().strftime('%Y/%m/%d %H:%M:%S'), module_config_identifier,
action_path, app_cfg['app.name'])
Frewriter.append_code(end_block.previous, Frewriter.tokenizer(map_code))
open(abs_path, 'wb').write(Frewriter.dump_source(first_token))
logging.info('M %s' % abs_path)
def testInsertActionConfig(self):
token = Frewriter.find_token(self.first_token, "a.this_is_a_module.config")
self.assertEquals(token.type, TokenType.SIMPLE_LVALUE)
self.assertEquals(token.values['identifier'], 'a.this_is_a_module.config')
start_block = Frewriter.find_token(token, "{", TokenType.START_BLOCK)
self.assertTrue(start_block is not None)
end_block = Frewriter.find_end_token(start_block.next)
self.assertTrue(end_block is not None)
# a.this_is_a_module.config['action'].push({'location':'','action':''});
code = "a.this_is_a_module.config['action'].push({'location':'/jn/demo/xxx','action':'jn.demo.Xxx'});"
Frewriter.append_code(end_block, Frewriter.tokenizer(code))
self.assertTrue(self._dump_source(self.first_token).find(code) > 0)
def create_material_config(path_prefix, action_name, app_cfg):
""" 生成widget相关的代码 """
_ = lambda k : os.path.normpath(os.path.join(path_prefix, k))
abs_path = _(action_name + '.js')
first_token = Frewriter.tokenizer(open(abs_path).read())
"""
token = Frewriter.find_token(first_token, 'material.setWidgets', TokenType.IDENTIFIER)
start_paren = Frewriter.find_token(token, '(', TokenType.START_PAREN)
end_paren = Frewriter.find_end_token(start_paren.next, TokenType.START_PAREN, TokenType.END_PAREN)
"""
widgets = []
def ignore_whitespace(token):
while token and token.type == TokenType.WHITESPACE:
token = token.next
return token
token = first_token
while token:
if token.type == TokenType.IDENTIFIER:
next_token = ignore_whitespace(token.next)
if next_token and \
next_token.type == TokenType.START_PAREN and \
token.string.startswith('ad.widget'): # FIXME(leeight) 检测方式不合理
widgets.append(token.string)
token = token.next
if len(widgets) <= 0:
logging.error('Can\'t find any widgets in %s.', abs_path)
return
# 建立namespace和filename的对应关系
action_name_regex = re.compile('(?P<package>([a-z\d_]+\.)+)(?P<action_name>[A-Z]\w+)', re.M|re.S)
pattern = re.compile(r'goog.addDependency\("([^"]+)",\s*\[([^\)]*)\],\s*\[([^\)]*)\]\)')
deps = {}
for line in open('src/deps.js').read().splitlines():
match = pattern.findall(line)
if match:
filename = match[0][0]
provides = match[0][1]
for item in provides.replace('\'', '').split(','):
if item.strip():
deps[item.strip()] = filename
config = "{\n'id' : 'ec-ma-8964',\n'main_url' : 'http://www.baidu.com'"
for widget in set(widgets):
logging.debug("%s => %s" , widget, deps[widget])
abs_path = os.path.join('src', deps[widget].replace('.js', '.config.js'))
if os.path.exists(abs_path):
key = os.path.basename(abs_path).replace(".config.js", "")
# XXX 不是所有人写的JSON都是那么规范，因此使用json.loads的话，可能
# 导致很多问题
config += (",\n'%s' : " % key)
xxx = find_widget_config(open(abs_path, 'rb').read())
if xxx is None:
logging.error(abs_path)
else:
config += xxx
config += "\n}"
app_cfg['material.config'] = config
generate_code('material.config.js', _(action_name + '.config.js'), app_cfg)
def testAppendCode(self):
token_stream1 = Frewriter.tokenizer("var hello = 'world';")
token_stream2 = Frewriter.tokenizer("var happy = 'new year';\n\n\n\n\n\n//THIS IS COMMENT")
Frewriter.append_code(token_stream1.next.next.next.next.next.next.next.next.next, token_stream2)
self.assertEquals("var hello = 'world';var happy = 'new year';\n\n\n\n\n//THIS IS COMMENT\n",
self._dump_source(token_stream1))
def setUp(self): base, ext = os.path.splitext(__file__) code = open(base + '.js').read() self.first_token = Frewriter.tokenizer(code)
