def __to_python_format(js):
ctx = PyV8.JSContext()
ctx.enter()
ctx.eval(js)
expect_all = dict((d[0].strftime('%Y/%m/%d'), d[1]) for d in PyV8.convert(ctx.locals["expectAll"]["data"]))
result_all = dict((d[0].strftime('%Y/%m/%d'), d[1]) for d in PyV8.convert(ctx.locals["resultAll"]["data"]))
return {"expectAll": expect_all, "resultAll": result_all}
def fetch(self, url, postdata=None, headers={}):
try:
if postdata:
postdata = urllib.urlencode(PyV8.convert(postdata))
r = urllib2.Request(url=url, data=postdata, headers=PyV8.convert(headers))
r.add_header('user-agent', 'DrEvalFetch/%s (%s)' % (doctoreval.__version__, doctoreval.__url__))
f = urllib2.urlopen(r)
return JSObject({"content": f.read(), "code": f.getcode(), "headers": JSObject(f.info().dict)})
except (urllib2.HTTPError, urllib2.URLError), e:
self._context.throw(str(e))
def log(self, *args):
args2 = []
for arg in args:
arg = PyV8.convert(arg)
args2.append(arg)
print(" ".join([str(x) for x in args2]))
def eval_coffee_footprint(coffee):
meta = eval_coffee_meta(coffee)
if 'format' not in meta:
raise Exception("Missing mandatory #format meta field")
else:
format = meta['format']
if format not in supported_formats:
raise Exception("Unsupported file format. Supported formats: %s" % (supported_formats))
# only compile the compiler once
global js_make_js_ctx
global js_make_js_from_coffee
global js_ctx_cleanup_count
js_ctx_cleanup_count = js_ctx_cleanup_count + 1
# HACK: occationally cleanup the context to avoid compiler slowdown
# will need a better approach in the future
if js_ctx_cleanup_count == 10:
js_make_js_ctx = None
js_make_js_from_coffee = None
js_ctx_cleanup_count = 0
if js_make_js_ctx == None:
prepare_coffee_compiler()
try:
js_make_js_ctx.enter()
ground = pkg_resources.resource_string(grind.__name__, "ground-%s.coffee" % (format))
ground_js = js_make_js_from_coffee(ground)
js = js_make_js_from_coffee(coffee + "\nreturn footprint()\n")
with PyV8.JSContext() as ctxt:
js_res = ctxt.eval("(function() {\n" + ground_js + js + "\n}).call(this);\n")
pl = PyV8.convert(js_res)
pl.append(meta)
return pl
finally:
js_make_js_ctx.leave()
def log(self, *args):
args2 = []
for arg in args:
arg = PyV8.convert(arg)
args2.append(arg)
print(" ".join([str(x) for x in args2]))
def setUp(self):
self.assertIsNotNone(self.jsparser_source)
self.ctxt = PyV8.JSContext(Global())
self.ctxt.enter()
self.ctxt.eval(self.jsparser)
testcode = "var s='%s'; var t='%s';parseStructure(s, t);"
self.jsparser = lambda x, y: convert_to_unicode(PyV8.convert(
self.ctxt.eval(testcode % (x.replace('\n', "' + \n '"), y))))
self.parser = {"pyparser": pyparser, "jsparser": self.jsparser}
def wrapper(*args):
if args:
if isinstance(args[-1], JSObject):
kws = v8.convert(args[-1])
args = args[:-1]
else:
kws = {}
return f(*args, **kws)
else:
return f()
def parse_price(resp, rule):
print '++++++++++++++++++++++++++++++++++++++++++++++++++++++++'
print resp.text
jscode = resp.text #[{'p': '769.00', 'm': '859.00', 'id': 'J_954086'}]
obj = Global()
info = None
with PyV8.JSContext(obj) as ctx:
c = ctx.eval(jscode)
info = PyV8.convert(c)
print info
print '++++++++++++++++++++++++++++++++++++++++++++++++++++++++'
return info
def parse_item_info(nodes, name, values, item):
# print '================================================================================'
if len(nodes) < 1:
return
node = nodes[0]
jscode = node.text_content()
# print 'js content' ,jscode
info = None
obj = Global()
with PyV8.JSContext(obj) as ctx:
ctx.eval(jscode)
info = PyV8.convert(ctx.locals.pageConfig)
prod = info['product']
values.update(prod)
def jd_save2db(data, opt):
skus = data['skuid']
url = 'http://p.3.cn/prices/mgets?skuIds=J_%s&type=1' % skus #(',J_'.join(skus))
o = {'referer': opt.url}
# print opt
resp = spider.fetch(url, o)
jscode = resp.text #[{'p': '769.00', 'm': '859.00', 'id': 'J_954086'}]
obj = Global()
info = None
with PyV8.JSContext(obj) as ctx:
c = ctx.eval(jscode)
info = PyV8.convert(c)
# print info
print '++++++++++++++++++++++++++++++++++++++++++++++++++++++++'
data['price'] = info[0]['p']
data['old_price'] = info[0]['m']
data['title'] = data['name'].decode('utf8')
save2db(data, opt)
def jd_parse_item_info(nodes, name, values, item):
# print '================================================================================'
if len(nodes) < 1:
return
node = nodes[0]
jscode = node.text_content()
# print 'js content' ,jscode
info = None
obj = Global()
with pyv8_jslocker:
with PyV8.JSContext(obj) as ctx:
ctx.eval(jscode)
info = PyV8.convert(ctx.locals.pageConfig)
prod = info['product']
values.update(prod)
if not values.get('title'):
pass
def script(self, obj={}):
obj = PyV8.convert(obj)
self._context.eval("function _runscript(%s) { %s }" % (', '.join(obj.keys()), self._script))
return self._context.eval("_runscript(%s);" % ', '.join([simplejson.dumps(v) for v in obj.values()]))
