def __init__(self, result=None, error=None):
response = {}
if result:
response.update(result=result)
if error:
response.update(error=error)
self.response = json.dumps(response)
self.request = None
def __init__(self, result=None, error=None):
response = {}
if result:
response.update(result=result)
if error:
response.update(error=error)
self.response = json.dumps(response)
self.request = None
def add_sample(source):
"""
Method for adding samples to test samples file
(use from console)
"""
count = 0
while os.path.exists("%s_%d.json" % (SAMPLES_FILE_PREFIX, count)):
count += 1
open("%s_%d.html" % (SAMPLES_FILE_PREFIX, count), "wb").write(unicode_to_str(source))
parsed = list(parse_html(source))
open("%s_%d.json" % (SAMPLES_FILE_PREFIX, count), "wb")\
.write(json.dumps(parsed, default=_encode_element, indent=8))
def jsonrpc_client_call(url, method, *args, **kwargs):
"""Execute a JSON-RPC call on the given url"""
_urllib = kwargs.pop('_urllib', urllib)
req = {'jsonrpc': '2.0', 'method': method, 'params': args or kwargs, 'id': 1}
res = json.loads(_urllib.urlopen(url, json.dumps(req)).read())
if 'result' in res:
return res['result']
elif 'error' in res:
er = res['error']
raise JsonRpcError(er['code'], er['message'], er['data'])
else:
msg = "JSON-RPC response must contain 'result' or 'error': %s" % res
raise ValueError(msg)
def test_encode_decode(self):
sr = self.spref
sp1 = self.spider1
sp2 = self.spider2
ref1 = sr.get_reference_from_spider(sp1)
ref2 = sr.get_reference_from_spider(sp2)
dt = datetime.datetime(2010, 1, 2, 10, 11, 12)
dts = "2010-01-02 10:11:12"
d = datetime.date(2010, 1, 2)
ds = "2010-01-02"
t = datetime.time(10, 11, 12)
ts = "10:11:12"
dec = Decimal("1000.12")
decs = "1000.12"
examples_encode_decode = [
('lala', 'lala'),
(sp1, ref1),
(['lala', sp1], ['lala', ref1]),
({'lala': sp1}, {'lala': ref1}),
({sp1: sp2}, {ref1: ref2}),
({sp1: {sp2: ['lala', sp1]}}, {ref1: {ref2: ['lala', ref1]}})
]
for spiders, refs in examples_encode_decode:
self.assertEqual(self.encoder.encode(spiders), json.dumps(refs))
self.assertEqual(self.decoder.decode(json.dumps(refs)), spiders)
examples_encode_only = [
({sp1: dt}, {ref1: dts}),
({sp1: d}, {ref1: ds}),
({sp1: t}, {ref1: ts}),
({sp1: dec}, {ref1: decs}),
]
for spiders, refs in examples_encode_only:
self.assertEqual(self.encoder.encode(spiders), json.dumps(refs))
assert 'Deferred' in self.encoder.encode(defer.Deferred())
def __init__(self, *args, **kwargs):
req_params = None
self.rpc_method = ""
if "body" not in kwargs and "rpc_params" in kwargs:
self.rpc_method = kwargs.pop("rpc_method")
req_params = {"jsonrpc": "2.0", "method": self.rpc_method, "params": kwargs.pop("rpc_params"), "id": 1}
# spec defines that requests must use POST method
kwargs.setdefault("method", "POST")
# rpc query multiples times over the same url
kwargs.setdefault("dont_filter", True)
super(JsonRpcRequest, self).__init__(*args, **kwargs)
if req_params:
self.body = json.dumps(req_params)
def add_sample(source):
"""
Method for adding samples to test samples file
(use from console)
"""
samples = []
if os.path.exists(SAMPLES_FILE):
for line in GzipFile(os.path.join(path, SAMPLES_FILE), "r").readlines():
samples.append(json.loads(line))
new_sample = {"source": source}
new_sample["parsed"] = list(parse_html(source))
samples.append(new_sample)
samples_file = GzipFile(os.path.join(path, SAMPLES_FILE), "wb")
for sample in samples:
samples_file.write(json.dumps(sample, default=_encode_element) + "\n")
samples_file.close()
def __init__(self, *args, **kwargs):
req_params = None
self.rpc_method = ""
if 'body' not in kwargs and 'rpc_params' in kwargs:
self.rpc_method = kwargs.pop('rpc_method')
req_params = {'jsonrpc': '2.0',
'method': self.rpc_method,
'params': kwargs.pop("rpc_params"),
'id': 1}
# spec defines that requests must use POST method
kwargs.setdefault('method', 'POST')
# rpc query multiples times over the same url
kwargs.setdefault('dont_filter', True)
super(JsonRpcRequest, self).__init__(*args, **kwargs)
if req_params:
self.body = json.dumps(req_params)
def jsonrpc_client_call(url, method, *args, **kwargs):
"""Execute a JSON-RPC call on the given url"""
_urllib = kwargs.pop('_urllib', urllib)
if args and kwargs:
raise ValueError(
"Pass *args or **kwargs but not both to jsonrpc_client_call")
req = {
'jsonrpc': '2.0',
'method': method,
'params': args or kwargs,
'id': 1
}
res = json.loads(_urllib.urlopen(url, json.dumps(req)).read())
if 'result' in res:
return res['result']
elif 'error' in res:
er = res['error']
raise JsonRpcError(er['code'], er['message'], er['data'])
else:
msg = "JSON-RPC response must contain 'result' or 'error': %s" % res
raise ValueError(msg)
def add(self, name, **spider_args):
d = spider_args.copy()
d['name'] = name
msg = Message(body=json.dumps(d))
return threads.deferToThread(self._queue_method, 'write', msg)
def encode(self, obj):
return json.dumps(obj)
def test_encode_decode(self):
sr = self.spref
sp1 = self.spider1
sp2 = self.spider2
ref1 = sr.get_reference_from_spider(sp1)
ref2 = sr.get_reference_from_spider(sp2)
dt = datetime.datetime(2010, 1, 2, 10, 11, 12)
dts = "2010-01-02 10:11:12"
d = datetime.date(2010, 1, 2)
ds = "2010-01-02"
t = datetime.time(10, 11, 12)
ts = "10:11:12"
dec = Decimal("1000.12")
decs = "1000.12"
examples_encode_decode = [('lala', 'lala'), (sp1, ref1),
(['lala', sp1], ['lala', ref1]),
({
'lala': sp1
}, {
'lala': ref1
}), ({
sp1: sp2
}, {
ref1: ref2
}),
({
sp1: {
sp2: ['lala', sp1]
}
}, {
ref1: {
ref2: ['lala', ref1]
}
})]
for spiders, refs in examples_encode_decode:
self.assertEqual(self.encoder.encode(spiders), json.dumps(refs))
self.assertEqual(self.decoder.decode(json.dumps(refs)), spiders)
examples_encode_only = [
({
sp1: dt
}, {
ref1: dts
}),
({
sp1: d
}, {
ref1: ds
}),
({
sp1: t
}, {
ref1: ts
}),
({
sp1: dec
}, {
ref1: decs
}),
]
for spiders, refs in examples_encode_only:
self.assertEqual(self.encoder.encode(spiders), json.dumps(refs))
assert 'Deferred' in self.encoder.encode(defer.Deferred())
def encode(self, obj):
return json.dumps(obj)
def add(self, name, **spider_args):
d = spider_args.copy()
d["name"] = name
msg = Message(body=json.dumps(d))
return threads.deferToThread(self._queue_method, "write", msg)
#!/usr/bin/env python
import sys
from boto.sqs.connection import SQSConnection
from boto.sqs.message import Message
from scrapy.utils.py26 import json
from scrapy.conf import settings
qname = settings['SQS_QUEUE']
if len(sys.argv) <= 1:
print "usage: %s <command> [args]" % sys.argv[0]
print
print "available commands:"
print " put <spider_name> - append spider to queue"
print
print "SQS queue: %s" % qname
print
sys.exit()
cmd, args = sys.argv[1], sys.argv[2:]
if cmd == 'put':
conn = SQSConnection(settings['AWS_ACCESS_KEY_ID'], \
settings['AWS_SECRET_ACCESS_KEY'])
q = conn.create_queue(qname)
msg = Message(body=json.dumps({'name': args[0]}))
q.write(msg)
