def get_images(self, response, request, info): path = self.file_path(request, response=response, info=info) try: orig_image = Image.open(BytesIO(response.body)) except IOError: tempimage = tempfile.gettempdir() + '\\' + path.split( '.')[0].split('/')[-1] + '.png' urlretrieve(request.url, tempimage) image = cv2.imread(tempimage) cv2.imwrite(tempimage, image) orig_image = Image.open(tempimage) width, height = orig_image.size if width < self.min_width or height < self.min_height: raise FileException( "Image too small (%dx%d < %dx%d)" % (width, height, self.min_width, self.min_height)) image, buf = self.convert_image(orig_image) yield path, image, buf for thumb_id, size in six.iteritems(self.thumbs): thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) thumb_image, thumb_buf = self.convert_image(image, size) yield thumb_path, thumb_image, thumb_buf
def item_completed(self, results, item, info): ok, x = results[0] if not ok: raise FileException("Item not contains") item['files'] = [x['path']] tags = item['tags'] item['tags'] = {} self._save_tags(x['path'], tags) self._constraint_image(x['path']) return item
def file_downloaded(self, response, request, info): expected_csum = request.meta.get('sha256') if expected_csum: response_csum = sha256sum(BytesIO(response.body)) logger.debug('Request %s SHA256: expected %s, actual %s', request, expected_csum, response_csum) if expected_csum != response_csum: logger.warning( 'File (checksum-mismatch): Error downloading %s ' 'from %s referred in <%s>: expected SHA256 digest %s, ' 'got %s', self.MEDIA_NAME, request, referer_str(request), expected_csum, response_csum, extra={'spider': info.spider}) raise FileException('checksum-mismatch') return super(OpenWrtDownloaderPipeline, self).file_downloaded( response, request, info)
def media_downloaded(self, response, request, info): referer = referer_str(request) if response.status != 200: logger.warning( 'File (code: %(status)s): Error downloading file from ' '%(request)s referred in <%(referer)s>', { 'status': response.status, 'request': request, 'referer': referer }, extra={'spider': info.spider}) raise FileException('download-error') if not response.body: logger.warning( 'File (empty-content): Empty file from %(request)s referred ' 'in <%(referer)s>: no-content', { 'request': request, 'referer': referer }, extra={'spider': info.spider}) raise FileException('empty-content') status = 'cached' if 'cached' in response.flags else 'downloaded' logger.debug( 'File (%(status)s): Downloaded file from %(request)s referred in ' '<%(referer)s>', { 'status': status, 'request': request, 'referer': referer }, extra={'spider': info.spider}) self.inc_stats(info.spider, status) try: path = self.file_path(request, response=response, info=info) # HERE ARE CUSTOM CHANGES checksum, most_common = self.file_downloaded( response, request, info) except FileException as exc: logger.warning( 'File (error): Error processing file from %(request)s ' 'referred in <%(referer)s>: %(errormsg)s', { 'request': request, 'referer': referer, 'errormsg': str(exc) }, extra={'spider': info.spider}, exc_info=True) raise except Exception as exc: logger.error( 'File (unknown-error): Error processing file from %(request)s ' 'referred in <%(referer)s>', { 'request': request, 'referer': referer }, exc_info=True, extra={'spider': info.spider}) raise FileException(str(exc)) # HERE ARE ALSO CUSTOM CHANGES return { 'url': request.url, 'path': path, 'checksum': checksum, 'status': status, 'most_common': most_common }
def test_should_remove_req_res_references_before_caching_the_results(self): """Regression test case to prevent a memory leak in the Media Pipeline. The memory leak is triggered when an exception is raised when a Response scheduled by the Media Pipeline is being returned. For example, when a FileException('download-error') is raised because the Response status code is not 200 OK. It happens because we are keeping a reference to the Response object inside the FileException context. This is caused by the way Twisted return values from inline callbacks. It raises a custom exception encapsulating the original return value. The solution is to remove the exception context when this context is a _DefGen_Return instance, the BaseException used by Twisted to pass the returned value from those inline callbacks. Maybe there's a better and more reliable way to test the case described here, but it would be more complicated and involve running - or at least mocking - some async steps from the Media Pipeline. The current test case is simple and detects the problem very fast. On the other hand, it would not detect another kind of leak happening due to old object references being kept inside the Media Pipeline cache. This problem does not occur in Python 2.7 since we don't have Exception Chaining (https://www.python.org/dev/peps/pep-3134/). """ # Create sample pair of Request and Response objects request = Request('http://url') response = Response('http://url', body=b'', request=request) # Simulate the Media Pipeline behavior to produce a Twisted Failure try: # Simulate a Twisted inline callback returning a Response # The returnValue method raises an exception encapsulating the value returnValue(response) except BaseException as exc: def_gen_return_exc = exc try: # Simulate the media_downloaded callback raising a FileException # This usually happens when the status code is not 200 OK raise FileException('download-error') except Exception as exc: file_exc = exc # Simulate Twisted capturing the FileException # It encapsulates the exception inside a Twisted Failure failure = Failure(file_exc) # The Failure should encapsulate a FileException ... self.assertEqual(failure.value, file_exc) # ... and it should have the returnValue exception set as its context self.assertEqual(failure.value.__context__, def_gen_return_exc) # Let's calculate the request fingerprint and fake some runtime data... fp = request_fingerprint(request) info = self.pipe.spiderinfo info.downloading.add(fp) info.waiting[fp] = [] # When calling the method that caches the Request's result ... self.pipe._cache_result_and_execute_waiters(failure, fp, info) # ... it should store the Twisted Failure ... self.assertEqual(info.downloaded[fp], failure) # ... encapsulating the original FileException ... self.assertEqual(info.downloaded[fp].value, file_exc) # ... but it should not store the returnValue exception on its context context = getattr(info.downloaded[fp].value, '__context__', None) self.assertIsNone(context)
def media_downloaded(self, response, request, info): """ Handler for success downloads. """ referer = request.headers.get('Referer') if response.status != 200: logger.msg( format= '%(medianame)s (code: %(status)s): Error downloading %(medianame)s from %(request)s referred in <%(referer)s>', level=logger.WARNING, spider=info.spider, medianame=self.MEDIA_NAME, status=response.status, request=request, referer=referer) raise FileException(request.url, '%s: download-error' % (request.url, )) if not response.body: log.msg( format= '%(medianame)s (empty-content): Empty %(medianame)s from %(request)s referred in <%(referer)s>: no-content', level=log.WARNING, spider=info.spider, medianame=self.MEDIA_NAME, request=request, referer=referer) raise FileException(request.url, '%s: empty-content' % (request.url, )) status = 'cached' if 'cached' in response.flags else 'downloaded' logger.msg( format= '%(medianame)s (%(status)s): Downloaded %(medianame)s from %(request)s referred in <%(referer)s>', level=logger.DEBUG, spider=info.spider, medianame=self.MEDIA_NAME, status=status, request=request, referer=referer) if self.is_valid_content_type(response): raise FileException(request.url, '%s: invalid-content_type' % (request.url, )) filename = self.get_file_name(request, response) if not filename: raise FileException(request.url, '%s: noaccess-filename' % (request.url, )) self.inc_stats(info.spider, status) try: key = self.file_key( info.spider.name, request.url) # return the SHA1 hash of the file url book_file_id, checksum = self.store.persist_file(key, response.body, info, filename, url=request.url) except FileException as exc: whyfmt = '%(medianame)s (error): Error processing %(medianame)s from %(request)s referred in <%(referer)s>: %(errormsg)s' logger.msg(format=whyfmt, level=log.WARNING, spider=info.spider, medianame=self.MEDIA_NAME, request=request, referer=referer, errormsg=str(exc)) raise # print {'url': request.url, 'file_id': book_file_id, 'checksum': checksum} return { 'url': request.url, 'file_id': book_file_id, 'checksum': checksum }
def media_downloaded(self, response, request, info): """ 从content-dispositon中取文件名 :param response: :param request: :param info: :return: """ referer = referer_str(request) if response.status != 200: logger.warning( 'File (code: %(status)s): Error downloading file from ' '%(request)s referred in <%(referer)s>', {'status': response.status, 'request': request, 'referer': referer}, extra={'spider': info.spider} ) raise FileException('download-error') if not response.body: logger.warning( 'File (empty-content): Empty file from %(request)s referred ' 'in <%(referer)s>: no-content', {'request': request, 'referer': referer}, extra={'spider': info.spider} ) raise FileException('empty-content') status = 'cached' if 'cached' in response.flags else 'downloaded' logger.debug( 'File (%(status)s): Downloaded file from %(request)s referred in ' '<%(referer)s>', {'status': status, 'request': request, 'referer': referer}, extra={'spider': info.spider} ) self.inc_stats(info.spider, status) try: containFileName = response.headers.get('Content-Disposition') or response.headers.get('content-disposition') if containFileName is not None: pattern_marks = re.compile(r'filename="(.*)"') pattern_no_marks = re.compile(r'filename=(.*)') try: file_name = pattern_marks.search(containFileName.decode('utf-8')) or pattern_no_marks.search(containFileName.decode('utf-8')) except: file_name = pattern_marks.search(str(containFileName).split("'")[1]) or pattern_no_marks.search(str(containFileName).split("'")[1]) if file_name is not None: file_name = urlparse.unquote(file_name.group(1).strip()) else: file_name = urlparse.unquote(os.path.basename(urlparse.unquote(response.request.url))) else: file_name = urlparse.unquote(os.path.basename(urlparse.unquote(response.request.url))) media_ext = os.path.splitext(file_name)[1] if "." not in media_ext or "\\" in media_ext or "/" in media_ext or ":" in media_ext or "*" in media_ext or "?" in media_ext or '"' in media_ext or "<" in media_ext or ">" in media_ext or "|" in media_ext or ";" in media_ext: content_type = response.headers.get('Content-Type') or response.headers.get('content-type') file_type = "." + content_type.decode('utf-8').split("/")[-1] file_name = str(int(time.time())) + file_type media_ext = file_type if response.meta.get("data"): url = request.url + urlparse.urlencode(response.meta["data"]) else: url = request.url media_guid = hashlib.sha1(to_bytes(url)).hexdigest() path = 'full/%s%s' % (media_guid, media_ext) checksum = self.file_downloaded(response, request, info, path) except FileException as exc: logger.warning( 'File (error): Error processing file from %(request)s ' 'referred in <%(referer)s>: %(errormsg)s', {'request': request, 'referer': referer, 'errormsg': str(exc)}, extra={'spider': info.spider}, exc_info=True ) raise except Exception as exc: logger.error( 'File (unknown-error): Error processing file from %(request)s ' 'referred in <%(referer)s>', {'request': request, 'referer': referer}, exc_info=True, extra={'spider': info.spider} ) raise FileException(str(exc)) return {'url': urlparse.unquote(url), 'path': path, 'checksum': checksum, "name": file_name}
def media_downloaded(self, response, request, info): referer = referer_str(request) if response.status != 200: logger.warning( 'File (code: %(status)s): Error downloading file from ' '%(request)s referred in <%(referer)s>', { 'status': response.status, 'request': request, 'referer': referer }, extra={'spider': info.spider}) raise FileException('download-error') buf = BytesIO(response.body) # Tag files that have errors in them has_error = True if 'A PHP Error was encountered'\ in buf.read() else False if not response.body: logger.warning( 'File (empty-content): Empty file from %(request)s referred ' 'in <%(referer)s>: no-content', { 'request': request, 'referer': referer }, extra={'spider': info.spider}) raise FileException('empty-content') status = 'cached' if 'cached' in response.flags else 'downloaded' logger.debug( 'File (%(status)s): Downloaded file from %(request)s referred in ' '<%(referer)s>', { 'status': status, 'request': request, 'referer': referer }, extra={'spider': info.spider}) self.inc_stats(info.spider, status) try: path = self.file_path(request, response=response, info=info) checksum = self.file_downloaded(response, request, info) except FileException as exc: logger.warning( 'File (error): Error processing file from %(request)s ' 'referred in <%(referer)s>: %(errormsg)s', { 'request': request, 'referer': referer, 'errormsg': str(exc) }, extra={'spider': info.spider}, exc_info=True) raise except Exception as exc: logger.error( 'File (unknown-error): Error processing file from %(request)s ' 'referred in <%(referer)s>', { 'request': request, 'referer': referer }, exc_info=True, extra={'spider': info.spider}) raise FileException(str(exc)) return { 'url': request.url, 'path': path, 'checksum': checksum, 'has_error': has_error }
def media_downloaded(self, response, request, info): referer = referer_str(request) if response.status != 200: logger.warning( 'File (code: %(status)s): Error downloading file from ' '%(request)s referred in <%(referer)s>', { 'status': response.status, 'request': request, 'referer': referer }, extra={'spider': info.spider}) raise FileException('download-error') if not response.body: logger.warning( 'File (empty-content): Empty file from %(request)s referred ' 'in <%(referer)s>: no-content', { 'request': request, 'referer': referer }, extra={'spider': info.spider}) raise FileException('empty-content') status = 'cached' if 'cached' in response.flags else 'downloaded' logger.debug( 'File (%(status)s): Downloaded file from %(request)s referred in ' '<%(referer)s>', { 'status': status, 'request': request, 'referer': referer }, extra={'spider': info.spider}) self.inc_stats(info.spider, status) try: # path = self.file_path(request, response=response, info=info) width, height, url_sha2, phash, checksum = self.file_downloaded( response, request, info) except FileException as exc: logger.warning( 'File (error): Error processing file from %(request)s ' 'referred in <%(referer)s>: %(errormsg)s', { 'request': request, 'referer': referer, 'errormsg': str(exc) }, extra={'spider': info.spider}, exc_info=True) raise except Exception as exc: logger.error( 'File (unknown-error): Error processing file from %(request)s ' 'referred in <%(referer)s>', { 'request': request, 'referer': referer }, exc_info=True, extra={'spider': info.spider}) raise FileException(str(exc)) resultDict = { 'url': request.url, 'url_sha2': url_sha2, 'checksum': checksum, 'width': width, 'heigth': height, 'phash': phash } print json.dumps(resultDict) raise DropItem("Printed to console")