class ECWriter(object): """ Writes an EC chunk """ def __init__(self, chunk, conn): self._chunk = chunk self._conn = conn self.failed = False self.bytes_transferred = 0 self.checksum = hashlib.md5() @property def chunk(self): return self._chunk @property def conn(self): return self._conn @classmethod def connect(cls, chunk, sysmeta, reqid=None): raw_url = chunk["url"] parsed = urlparse(raw_url) chunk_path = parsed.path.split('/')[-1] h = {} h["transfer-encoding"] = "chunked" h[chunk_headers["content_id"]] = sysmeta['id'] h[chunk_headers["content_path"]] = sysmeta['content_path'] h[chunk_headers["content_chunkmethod"]] = sysmeta['chunk_method'] h[chunk_headers["container_id"]] = sysmeta['container_id'] h[chunk_headers["chunk_pos"]] = chunk["pos"] h[chunk_headers["chunk_id"]] = chunk_path h[chunk_headers["content_policy"]] = sysmeta['policy'] h[chunk_headers["content_version"]] = sysmeta['version'] if reqid: h['X-oio-req-id'] = reqid # in the trailer # metachunk_size & metachunk_hash h["Trailer"] = (chunk_headers["metachunk_size"], chunk_headers["metachunk_hash"]) with ConnectionTimeout(io.CONNECTION_TIMEOUT): conn = io.http_connect( parsed.netloc, 'PUT', parsed.path, h) conn.chunk = chunk return cls(chunk, conn) def start(self, pool): # we use eventlet Queue to pass data to the send coroutine self.queue = Queue(io.PUT_QUEUE_DEPTH) # spawn the send coroutine pool.spawn(self._send) def _send(self): # this is the send coroutine loop while True: # fetch input data from the queue d = self.queue.get() # use HTTP transfer encoding chunked # to write data to RAWX if not self.failed: # format the chunk to_send = "%x\r\n%s\r\n" % (len(d), d) try: with ChunkWriteTimeout(io.CHUNK_TIMEOUT): self.conn.send(to_send) self.bytes_transferred += len(d) except (Exception, ChunkWriteTimeout) as e: self.failed = True msg = str(e) logger.warn("Failed to write to %s (%s)", self.chunk, msg) self.chunk['error'] = msg self.queue.task_done() def wait(self): # wait until all data in the queue # has been processed by the send coroutine if self.queue.unfinished_tasks: self.queue.join() def send(self, data): # do not send empty data because # this will end the chunked body if not data: return # put the data to send into the queue # it will be processed by the send coroutine self.queue.put(data) def finish(self, metachunk_size, metachunk_hash): parts = [ '0\r\n', '%s: %s\r\n' % (chunk_headers['metachunk_size'], metachunk_size), '%s: %s\r\n' % (chunk_headers['metachunk_hash'], metachunk_hash), '\r\n' ] to_send = "".join(parts) self.conn.send(to_send) def getresponse(self): # read the HTTP response from the connection with Timeout(io.CHUNK_TIMEOUT): self.resp = self.conn.getresponse() return self.resp
class Crawler(object): """ A crawler will traverse all the pages of a site and process the content in a defined way. :param init_urls: the very first urls to start with. :param q: the queue that stores all urls to be crawled :param urls: a set stores all urls already crawled """ def __init__(self, init_urls, max_workers=200): self.init_urls = init_urls self.max_workers = max_workers self.q = Queue() self.urls = set() self.s = requests.Session() self.root_hosts = set() for url in init_urls: self.q.put(url) self.urls.add(url) self.root_hosts.add(get_netloc(url)) def url_allowed(self, url): """Check if given url will be crawled. Current, only if the url belongs to the same host as init_urls. """ return get_netloc(url) in self.root_hosts def save(self, response): """Save data at the given url.""" raise NotImplementedError( "Please implement your own save logic in subclass.") def parse(self, response): self.save(response) new_links = set() for url in self.find_links(response): if url not in self.urls and self.url_allowed(url): new_links.add(url) self.urls.add(url) self.q.put(url) if len(new_links) != 0: print("Find %d new urls to crawl" % len(new_links)) def fetch(self, url): """Fetch content of the url from network.""" response = self.s.get(url) print("Getting content from %s, length: %d" % (url, len(response.content))) return response def work(self, i): """Define the work process. Retrieve a url from queue, fetch the content from it, process it and get new urls to crawl. Continue the process until all pages are crawled. :param i: indicate the worker number """ while True: url = self.q.get() print("Worker %d: Getting url %s from queue." % (i, url)) response = self.fetch(url) self.parse(response) self.q.task_done() def run(self): """Start the crawling process. This is the main entrance for our crawler. It will start several workers, crawling in parallel. """ pool = eventlet.GreenPool() start = time.time() for i in range(self.max_workers): pool.spawn(self.work, i) self.q.join() end = time.time() print("Finished crawling, takes %s seconds." % str(end - start)) print("Have fun hacking!")
class EcChunkWriter(object): """ Writes an EC chunk """ def __init__(self, chunk, conn, write_timeout=None, **_kwargs): self._chunk = chunk self._conn = conn self.failed = False self.bytes_transferred = 0 self.checksum = hashlib.md5() self.write_timeout = write_timeout or io.CHUNK_TIMEOUT # we use eventlet Queue to pass data to the send coroutine self.queue = Queue(io.PUT_QUEUE_DEPTH) @property def chunk(self): return self._chunk @property def conn(self): return self._conn @classmethod def connect(cls, chunk, sysmeta, reqid=None, connection_timeout=None, write_timeout=None, **_kwargs): raw_url = chunk["url"] parsed = urlparse(raw_url) chunk_path = parsed.path.split('/')[-1] hdrs = headers_from_object_metadata(sysmeta) if reqid: hdrs['X-oio-req-id'] = reqid hdrs[chunk_headers["chunk_pos"]] = chunk["pos"] hdrs[chunk_headers["chunk_id"]] = chunk_path # in the trailer # metachunk_size & metachunk_hash hdrs["Trailer"] = ', '.join( (chunk_headers["metachunk_size"], chunk_headers["metachunk_hash"], chunk_headers["chunk_hash"])) with green.ConnectionTimeout(connection_timeout or io.CONNECTION_TIMEOUT): conn = io.http_connect(parsed.netloc, 'PUT', parsed.path, hdrs) conn.chunk = chunk return cls(chunk, conn, write_timeout=write_timeout) def start(self, pool): """Spawn the send coroutine""" pool.spawn(self._send) def _send(self): """Send coroutine loop""" while True: # fetch input data from the queue data = self.queue.get() # use HTTP transfer encoding chunked # to write data to RAWX if not self.failed: # format the chunk to_send = "%x\r\n%s\r\n" % (len(data), data) try: with green.ChunkWriteTimeout(self.write_timeout): self.conn.send(to_send) self.bytes_transferred += len(data) except (Exception, green.ChunkWriteTimeout) as exc: self.failed = True msg = str(exc) logger.warn("Failed to write to %s (%s)", self.chunk, msg) self.chunk['error'] = 'write: %s' % msg self.queue.task_done() def wait(self): """ Wait until all data in the queue has been processed by the send coroutine """ if self.queue.unfinished_tasks: self.queue.join() def send(self, data): # do not send empty data because # this will end the chunked body if not data: return # put the data to send into the queue # it will be processed by the send coroutine self.queue.put(data) def finish(self, metachunk_size, metachunk_hash): """Send metachunk_size and metachunk_hash as trailers""" parts = [ '0\r\n', '%s: %s\r\n' % (chunk_headers['metachunk_size'], metachunk_size), '%s: %s\r\n' % (chunk_headers['metachunk_hash'], metachunk_hash), '%s: %s\r\n' % (chunk_headers['chunk_hash'], self.checksum.hexdigest()), '\r\n' ] to_send = "".join(parts) self.conn.send(to_send) def getresponse(self): """Read the HTTP response from the connection""" # As the server may buffer data before writing it to non-volatile # storage, we don't know if we have to wait while sending data or # while reading response, thus we apply the same timeout to both. with Timeout(self.write_timeout): return self.conn.getresponse()
class ECWriter(object): """ Writes an EC chunk """ def __init__(self, chunk, conn): self._chunk = chunk self._conn = conn self.failed = False self.bytes_transferred = 0 self.checksum = hashlib.md5() @property def chunk(self): return self._chunk @property def conn(self): return self._conn @classmethod def connect(cls, chunk, sysmeta): raw_url = chunk["url"] parsed = urlparse(raw_url) chunk_path = parsed.path.split('/')[-1] h = {} h["transfer-encoding"] = "chunked" h[chunk_headers["content_id"]] = sysmeta['id'] h[chunk_headers["content_path"]] = sysmeta['content_path'] h[chunk_headers["content_chunkmethod"]] = sysmeta['chunk_method'] h[chunk_headers["container_id"]] = sysmeta['container_id'] h[chunk_headers["chunk_pos"]] = chunk["pos"] h[chunk_headers["chunk_id"]] = chunk_path h[chunk_headers["content_policy"]] = sysmeta['policy'] h[chunk_headers["content_version"]] = sysmeta['version'] # in the trailer # metachunk_size & metachunk_hash h["Trailer"] = (chunk_headers["metachunk_size"], chunk_headers["metachunk_hash"]) with ConnectionTimeout(io.CONNECTION_TIMEOUT): conn = io.http_connect(parsed.netloc, 'PUT', parsed.path, h) conn.chunk = chunk return cls(chunk, conn) def start(self, pool): # we use eventlet Queue to pass data to the send coroutine self.queue = Queue(io.PUT_QUEUE_DEPTH) # spawn the send coroutine pool.spawn(self._send) def _send(self): # this is the send coroutine loop while True: # fetch input data from the queue d = self.queue.get() # use HTTP transfer encoding chunked # to write data to RAWX if not self.failed: # format the chunk to_send = "%x\r\n%s\r\n" % (len(d), d) try: with ChunkWriteTimeout(io.CHUNK_TIMEOUT): self.conn.send(to_send) self.bytes_transferred += len(d) except (Exception, ChunkWriteTimeout) as e: self.failed = True msg = str(e) logger.warn("Failed to write to %s (%s)", self.chunk, msg) self.chunk['error'] = msg self.queue.task_done() def wait(self): # wait until all data in the queue # has been processed by the send coroutine if self.queue.unfinished_tasks: self.queue.join() def send(self, data): # do not send empty data because # this will end the chunked body if not data: return # put the data to send into the queue # it will be processed by the send coroutine self.queue.put(data) def finish(self, metachunk_size, metachunk_hash): parts = [ '0\r\n', '%s: %s\r\n' % (chunk_headers['metachunk_size'], metachunk_size), '%s: %s\r\n' % (chunk_headers['metachunk_hash'], metachunk_hash), '\r\n' ] to_send = "".join(parts) self.conn.send(to_send) def getresponse(self): # read the HTTP response from the connection with Timeout(io.CHUNK_TIMEOUT): self.resp = self.conn.getresponse() return self.resp