def create_connection(addr, port):
con = SuperSocket()
Client(con)
con.add_map(CONNECT, on_connect, addr, port)
con.add_map(CONNECT_ERR, on_connect_err, addr, port)
con.add_map(CONNECT, lambda con: die())
con.add_map(CONNECT_ERR, lambda con, err: die())
con.connect_ex((addr, port))
def scan(addr, min, max):
task = Task()
for ind in range(min, max):
task.add(create_connection(addr, ind), CONNECT, CONNECT_ERR)
task.start()
task.add_map(DONE, lambda task: die())
def setup(con, msg):
SockReader(con)
SockWriter(con)
con.dump(msg.encode('utf8'))
con.settimeout(1)
con.add_map(DUMPED, lambda con: die('Msg dumped!'))
def terminate(self):
lose(self.ssock)
lose(self.server)
die()
def handle_done(self, client, a, b):
self.assertEqual(a, b'abc' * 100)
self.assertEqual(b, b'efg' * 100)
lose(client)
lose(self.server)
die()
def handle_done(self, client, data):
print('Received bytes from:', self.client)
lose(client)
lose(self.server)
die()
def handle_done(self, ssock, data):
print('Received bytes from:', ssock)
lose(ssock)
lose(self.server)
die()
from untwisted.core import die
from untwisted import core
from untwisted.task import Task
from untwisted.job import Job, DONE
import time
def sum(x, y):
time.sleep(3)
return x + y
def show(job, result):
print(result)
task = Task()
# Tell the task it can start trigging events.
task.start()
for ind in range(100):
job = Job(sum, ind, 1000)
job.add_map(DONE, show)
task.add(job, DONE)
task.add_map(DONE, lambda task: die())
core.gear.mainloop()
def handle_done(self, task):
self.drive(self.EMPTY)
die()
def handle_connect(self, client):
print('Connected!', client)
lose(client)
lose(self.server)
die()
def handle_connect_err(ssock, err):
print('Connect err.', err)
die()
def handle_close(ssock, err):
print('Closed.')
die()
def on_close(expect):
print('Closing..')
die()
def setup(con, msg):
Stdout(con)
Stdin(con)
con.dump(msg)
xmap(con, DUMPED, lambda con: die('Msg dumped!'))
def update(self):
die()
def handle_done(self, job, retval):
self.retval = retval
die()
class Miner(list):
task = Task()
task.add_map(DONE, lambda task: die())
task.start()
def __init__(self,
url,
headers=default_headers,
args={},
method='get',
payload=None,
auth=None,
attempts=5):
"""
Resource
Param: url
Headers to be send.
Param: headers
Url query.
Param: args
The HTTP method.
Param: method
The payload data in case of method is 'post'.
Param: payload
Authentication user/pass.
Param: auth
The number of times a given url should be tried
in case of corrupted response.
Param: attempts.
"""
self.url = url
self.auth = auth
self.args = args
self.encoding = 'utf-8'
self.response = None
self.headers = headers
self.payload = payload
self.method = method
self.attempts = attempts
self.urlparser = urlparse(url)
super(list, self).__init__()
self.next(self.url)
def setup(self, response):
# Reset the fd so it can be reread later.
data = response.fd.read()
response.fd.seek(0)
type = response.headers.get('content-type',
'text/html; charset=%s' % self.encoding)
# Sets the encoding for later usage
# in self.geturl for example.
params = cgi.parse_header(type)
self.encoding = params[1]['charset']
self.response = response
data = data.decode(self.encoding, 'ignore')
self.build_dom(data)
def build_dom(self, data):
pass
def handle_success(self, request, response):
self.setup(response)
def fetcher(self):
request = Get(self.url,
headers=self.headers,
auth=self.auth,
attempts=self.attempts)
self.task.add(request, ResponseHandle.ERROR, ResponseHandle.DONE)
request.add_map('200', self.handle_success)
return request
def poster(self):
request = Post(self.url,
headers=self.headers,
payload=self.payload,
auth=self.auth,
attempts=self.attempts)
self.task.add(request, ResponseHandle.ERROR, ResponseHandle.DONE)
request.add_map('200', self.handle_success)
return request
def geturl(self, reference):
"""
"""
urlparser = urlparse(reference)
if not urlparser.scheme:
return urljoin(
'%s://%s' % (self.urlparser.scheme, self.urlparser.hostname),
reference)
return reference
def next(self, reference):
self.url = self.geturl(reference)
self.urlparser = urlparse(self.url)
if self.method == 'get':
return self.fetcher()
return self.poster()
def run(self, dom):
"""
Implement your rules here.
"""
pass
def setup(con, msg):
Stdout(con)
Stdin(con)
con.dump(msg.encode('utf8'))
xmap(con, DUMPED, lambda con: die('Msg dumped!'))
def handle_accept(self, server, ssock):
self.ssock = ssock
self.ssock.add_map(CLOSE, lambda ssock, err: die())
class Miner(list):
task = Task()
task.add_map(DONE, lambda task: die())
task.start()
def __init__(self, url, pool=None,
headers=HEADERS, method='get', payload={}, auth=()):
self.pool = pool
self.url = url
self.urlparser = urlparse(url)
self.headers = headers
self.method = method
self.payload = payload
self.auth = auth
self.encoding = 'utf-8'
self.response = None
super(list, self).__init__()
self.expand()
def expand(self):
"""
No exception being raised.
"""
try:
self.create_connection()
except Exception as excpt:
print(excpt)
def setup(self, response):
data = response.fd.read()
# Reset the fd so it can be reread later.
response.fd.seek(0)
type = response.headers.get('content-type',
'text/html; charset=%s' % self.encoding)
params = cgi.parse_header(type)
# Sets the encoding for later usage
# in self.geturl for example.
self.encoding = params[1]['charset']
self.response = response
data = data.decode(self.encoding, 'ignore')
self.build_dom(data)
def build_dom(self, data):
pass
def create_connection(self):
if self.method == 'get':
return Fetcher(self)
return Poster(self)
def geturl(self, reference):
"""
"""
# It is necessary to encode back the url
# because websnake get method inserts the host header
# with the wrong encoding and some web servers wouldnt
# accept it as valid header.
reference = reference
urlparser = urlparse(reference)
url = urljoin('%s://%s' % (self.urlparser.scheme,
self.urlparser.hostname), reference) \
if not urlparser.scheme else reference
return url
def next(self, reference):
self.url = self.geturl(reference)
self.urlparser = urlparse(self.url)
self.expand()
def run(self, dom):
"""
Implement your rules here.
"""
pass
def handle_accept(self, server, ssock):
print('Accepted:', ssock)
lose(ssock)
lose(self.server)
die()
