def __init__(self, result=None, error=None): response = {} if result: response.update(result=result) if error: response.update(error=error) self.response = json.dumps(response) self.request = None
def add_sample(source): """ Method for adding samples to test samples file (use from console) """ count = 0 while os.path.exists("%s_%d.json" % (SAMPLES_FILE_PREFIX, count)): count += 1 open("%s_%d.html" % (SAMPLES_FILE_PREFIX, count), "wb").write(unicode_to_str(source)) parsed = list(parse_html(source)) open("%s_%d.json" % (SAMPLES_FILE_PREFIX, count), "wb")\ .write(json.dumps(parsed, default=_encode_element, indent=8))
def jsonrpc_client_call(url, method, *args, **kwargs): """Execute a JSON-RPC call on the given url""" _urllib = kwargs.pop('_urllib', urllib) req = {'jsonrpc': '2.0', 'method': method, 'params': args or kwargs, 'id': 1} res = json.loads(_urllib.urlopen(url, json.dumps(req)).read()) if 'result' in res: return res['result'] elif 'error' in res: er = res['error'] raise JsonRpcError(er['code'], er['message'], er['data']) else: msg = "JSON-RPC response must contain 'result' or 'error': %s" % res raise ValueError(msg)
def test_encode_decode(self): sr = self.spref sp1 = self.spider1 sp2 = self.spider2 ref1 = sr.get_reference_from_spider(sp1) ref2 = sr.get_reference_from_spider(sp2) dt = datetime.datetime(2010, 1, 2, 10, 11, 12) dts = "2010-01-02 10:11:12" d = datetime.date(2010, 1, 2) ds = "2010-01-02" t = datetime.time(10, 11, 12) ts = "10:11:12" dec = Decimal("1000.12") decs = "1000.12" examples_encode_decode = [ ('lala', 'lala'), (sp1, ref1), (['lala', sp1], ['lala', ref1]), ({'lala': sp1}, {'lala': ref1}), ({sp1: sp2}, {ref1: ref2}), ({sp1: {sp2: ['lala', sp1]}}, {ref1: {ref2: ['lala', ref1]}}) ] for spiders, refs in examples_encode_decode: self.assertEqual(self.encoder.encode(spiders), json.dumps(refs)) self.assertEqual(self.decoder.decode(json.dumps(refs)), spiders) examples_encode_only = [ ({sp1: dt}, {ref1: dts}), ({sp1: d}, {ref1: ds}), ({sp1: t}, {ref1: ts}), ({sp1: dec}, {ref1: decs}), ] for spiders, refs in examples_encode_only: self.assertEqual(self.encoder.encode(spiders), json.dumps(refs)) assert 'Deferred' in self.encoder.encode(defer.Deferred())
def __init__(self, *args, **kwargs): req_params = None self.rpc_method = "" if "body" not in kwargs and "rpc_params" in kwargs: self.rpc_method = kwargs.pop("rpc_method") req_params = {"jsonrpc": "2.0", "method": self.rpc_method, "params": kwargs.pop("rpc_params"), "id": 1} # spec defines that requests must use POST method kwargs.setdefault("method", "POST") # rpc query multiples times over the same url kwargs.setdefault("dont_filter", True) super(JsonRpcRequest, self).__init__(*args, **kwargs) if req_params: self.body = json.dumps(req_params)
def add_sample(source): """ Method for adding samples to test samples file (use from console) """ samples = [] if os.path.exists(SAMPLES_FILE): for line in GzipFile(os.path.join(path, SAMPLES_FILE), "r").readlines(): samples.append(json.loads(line)) new_sample = {"source": source} new_sample["parsed"] = list(parse_html(source)) samples.append(new_sample) samples_file = GzipFile(os.path.join(path, SAMPLES_FILE), "wb") for sample in samples: samples_file.write(json.dumps(sample, default=_encode_element) + "\n") samples_file.close()
def __init__(self, *args, **kwargs): req_params = None self.rpc_method = "" if 'body' not in kwargs and 'rpc_params' in kwargs: self.rpc_method = kwargs.pop('rpc_method') req_params = {'jsonrpc': '2.0', 'method': self.rpc_method, 'params': kwargs.pop("rpc_params"), 'id': 1} # spec defines that requests must use POST method kwargs.setdefault('method', 'POST') # rpc query multiples times over the same url kwargs.setdefault('dont_filter', True) super(JsonRpcRequest, self).__init__(*args, **kwargs) if req_params: self.body = json.dumps(req_params)
def jsonrpc_client_call(url, method, *args, **kwargs): """Execute a JSON-RPC call on the given url""" _urllib = kwargs.pop('_urllib', urllib) if args and kwargs: raise ValueError( "Pass *args or **kwargs but not both to jsonrpc_client_call") req = { 'jsonrpc': '2.0', 'method': method, 'params': args or kwargs, 'id': 1 } res = json.loads(_urllib.urlopen(url, json.dumps(req)).read()) if 'result' in res: return res['result'] elif 'error' in res: er = res['error'] raise JsonRpcError(er['code'], er['message'], er['data']) else: msg = "JSON-RPC response must contain 'result' or 'error': %s" % res raise ValueError(msg)
def add(self, name, **spider_args): d = spider_args.copy() d['name'] = name msg = Message(body=json.dumps(d)) return threads.deferToThread(self._queue_method, 'write', msg)
def encode(self, obj): return json.dumps(obj)
def test_encode_decode(self): sr = self.spref sp1 = self.spider1 sp2 = self.spider2 ref1 = sr.get_reference_from_spider(sp1) ref2 = sr.get_reference_from_spider(sp2) dt = datetime.datetime(2010, 1, 2, 10, 11, 12) dts = "2010-01-02 10:11:12" d = datetime.date(2010, 1, 2) ds = "2010-01-02" t = datetime.time(10, 11, 12) ts = "10:11:12" dec = Decimal("1000.12") decs = "1000.12" examples_encode_decode = [('lala', 'lala'), (sp1, ref1), (['lala', sp1], ['lala', ref1]), ({ 'lala': sp1 }, { 'lala': ref1 }), ({ sp1: sp2 }, { ref1: ref2 }), ({ sp1: { sp2: ['lala', sp1] } }, { ref1: { ref2: ['lala', ref1] } })] for spiders, refs in examples_encode_decode: self.assertEqual(self.encoder.encode(spiders), json.dumps(refs)) self.assertEqual(self.decoder.decode(json.dumps(refs)), spiders) examples_encode_only = [ ({ sp1: dt }, { ref1: dts }), ({ sp1: d }, { ref1: ds }), ({ sp1: t }, { ref1: ts }), ({ sp1: dec }, { ref1: decs }), ] for spiders, refs in examples_encode_only: self.assertEqual(self.encoder.encode(spiders), json.dumps(refs)) assert 'Deferred' in self.encoder.encode(defer.Deferred())
def add(self, name, **spider_args): d = spider_args.copy() d["name"] = name msg = Message(body=json.dumps(d)) return threads.deferToThread(self._queue_method, "write", msg)
#!/usr/bin/env python import sys from boto.sqs.connection import SQSConnection from boto.sqs.message import Message from scrapy.utils.py26 import json from scrapy.conf import settings qname = settings['SQS_QUEUE'] if len(sys.argv) <= 1: print "usage: %s <command> [args]" % sys.argv[0] print print "available commands:" print " put <spider_name> - append spider to queue" print print "SQS queue: %s" % qname print sys.exit() cmd, args = sys.argv[1], sys.argv[2:] if cmd == 'put': conn = SQSConnection(settings['AWS_ACCESS_KEY_ID'], \ settings['AWS_SECRET_ACCESS_KEY']) q = conn.create_queue(qname) msg = Message(body=json.dumps({'name': args[0]})) q.write(msg)