def get_images(self, response, request, info):
path = self.file_path(request, response=response, info=info)
try:
orig_image = Image.open(BytesIO(response.body))
except IOError:
tempimage = tempfile.gettempdir() + '\\' + path.split(
'.')[0].split('/')[-1] + '.png'
urlretrieve(request.url, tempimage)
image = cv2.imread(tempimage)
cv2.imwrite(tempimage, image)
orig_image = Image.open(tempimage)
width, height = orig_image.size
if width < self.min_width or height < self.min_height:
raise FileException(
"Image too small (%dx%d < %dx%d)" %
(width, height, self.min_width, self.min_height))
image, buf = self.convert_image(orig_image)
yield path, image, buf
for thumb_id, size in six.iteritems(self.thumbs):
thumb_path = self.thumb_path(request,
thumb_id,
response=response,
info=info)
thumb_image, thumb_buf = self.convert_image(image, size)
yield thumb_path, thumb_image, thumb_buf
def item_completed(self, results, item, info):
ok, x = results[0]
if not ok:
raise FileException("Item not contains")
item['files'] = [x['path']]
tags = item['tags']
item['tags'] = {}
self._save_tags(x['path'], tags)
self._constraint_image(x['path'])
return item
def file_downloaded(self, response, request, info):
expected_csum = request.meta.get('sha256')
if expected_csum:
response_csum = sha256sum(BytesIO(response.body))
logger.debug('Request %s SHA256: expected %s, actual %s',
request, expected_csum, response_csum)
if expected_csum != response_csum:
logger.warning(
'File (checksum-mismatch): Error downloading %s '
'from %s referred in <%s>: expected SHA256 digest %s, '
'got %s',
self.MEDIA_NAME, request, referer_str(request),
expected_csum, response_csum,
extra={'spider': info.spider})
raise FileException('checksum-mismatch')
return super(OpenWrtDownloaderPipeline, self).file_downloaded(
response, request, info)
def media_downloaded(self, response, request, info):
referer = referer_str(request)
if response.status != 200:
logger.warning(
'File (code: %(status)s): Error downloading file from '
'%(request)s referred in <%(referer)s>', {
'status': response.status,
'request': request,
'referer': referer
},
extra={'spider': info.spider})
raise FileException('download-error')
if not response.body:
logger.warning(
'File (empty-content): Empty file from %(request)s referred '
'in <%(referer)s>: no-content', {
'request': request,
'referer': referer
},
extra={'spider': info.spider})
raise FileException('empty-content')
status = 'cached' if 'cached' in response.flags else 'downloaded'
logger.debug(
'File (%(status)s): Downloaded file from %(request)s referred in '
'<%(referer)s>', {
'status': status,
'request': request,
'referer': referer
},
extra={'spider': info.spider})
self.inc_stats(info.spider, status)
try:
path = self.file_path(request, response=response, info=info)
# HERE ARE CUSTOM CHANGES
checksum, most_common = self.file_downloaded(
response, request, info)
except FileException as exc:
logger.warning(
'File (error): Error processing file from %(request)s '
'referred in <%(referer)s>: %(errormsg)s', {
'request': request,
'referer': referer,
'errormsg': str(exc)
},
extra={'spider': info.spider},
exc_info=True)
raise
except Exception as exc:
logger.error(
'File (unknown-error): Error processing file from %(request)s '
'referred in <%(referer)s>', {
'request': request,
'referer': referer
},
exc_info=True,
extra={'spider': info.spider})
raise FileException(str(exc))
# HERE ARE ALSO CUSTOM CHANGES
return {
'url': request.url,
'path': path,
'checksum': checksum,
'status': status,
'most_common': most_common
}
def test_should_remove_req_res_references_before_caching_the_results(self):
"""Regression test case to prevent a memory leak in the Media Pipeline.
The memory leak is triggered when an exception is raised when a Response
scheduled by the Media Pipeline is being returned. For example, when a
FileException('download-error') is raised because the Response status
code is not 200 OK.
It happens because we are keeping a reference to the Response object
inside the FileException context. This is caused by the way Twisted
return values from inline callbacks. It raises a custom exception
encapsulating the original return value.
The solution is to remove the exception context when this context is a
_DefGen_Return instance, the BaseException used by Twisted to pass the
returned value from those inline callbacks.
Maybe there's a better and more reliable way to test the case described
here, but it would be more complicated and involve running - or at least
mocking - some async steps from the Media Pipeline. The current test
case is simple and detects the problem very fast. On the other hand, it
would not detect another kind of leak happening due to old object
references being kept inside the Media Pipeline cache.
This problem does not occur in Python 2.7 since we don't have Exception
Chaining (https://www.python.org/dev/peps/pep-3134/).
"""
# Create sample pair of Request and Response objects
request = Request('http://url')
response = Response('http://url', body=b'', request=request)
# Simulate the Media Pipeline behavior to produce a Twisted Failure
try:
# Simulate a Twisted inline callback returning a Response
# The returnValue method raises an exception encapsulating the value
returnValue(response)
except BaseException as exc:
def_gen_return_exc = exc
try:
# Simulate the media_downloaded callback raising a FileException
# This usually happens when the status code is not 200 OK
raise FileException('download-error')
except Exception as exc:
file_exc = exc
# Simulate Twisted capturing the FileException
# It encapsulates the exception inside a Twisted Failure
failure = Failure(file_exc)
# The Failure should encapsulate a FileException ...
self.assertEqual(failure.value, file_exc)
# ... and it should have the returnValue exception set as its context
self.assertEqual(failure.value.__context__, def_gen_return_exc)
# Let's calculate the request fingerprint and fake some runtime data...
fp = request_fingerprint(request)
info = self.pipe.spiderinfo
info.downloading.add(fp)
info.waiting[fp] = []
# When calling the method that caches the Request's result ...
self.pipe._cache_result_and_execute_waiters(failure, fp, info)
# ... it should store the Twisted Failure ...
self.assertEqual(info.downloaded[fp], failure)
# ... encapsulating the original FileException ...
self.assertEqual(info.downloaded[fp].value, file_exc)
# ... but it should not store the returnValue exception on its context
context = getattr(info.downloaded[fp].value, '__context__', None)
self.assertIsNone(context)
def media_downloaded(self, response, request, info):
"""
Handler for success downloads.
"""
referer = request.headers.get('Referer')
if response.status != 200:
logger.msg(
format=
'%(medianame)s (code: %(status)s): Error downloading %(medianame)s from %(request)s referred in <%(referer)s>',
level=logger.WARNING,
spider=info.spider,
medianame=self.MEDIA_NAME,
status=response.status,
request=request,
referer=referer)
raise FileException(request.url,
'%s: download-error' % (request.url, ))
if not response.body:
log.msg(
format=
'%(medianame)s (empty-content): Empty %(medianame)s from %(request)s referred in <%(referer)s>: no-content',
level=log.WARNING,
spider=info.spider,
medianame=self.MEDIA_NAME,
request=request,
referer=referer)
raise FileException(request.url,
'%s: empty-content' % (request.url, ))
status = 'cached' if 'cached' in response.flags else 'downloaded'
logger.msg(
format=
'%(medianame)s (%(status)s): Downloaded %(medianame)s from %(request)s referred in <%(referer)s>',
level=logger.DEBUG,
spider=info.spider,
medianame=self.MEDIA_NAME,
status=status,
request=request,
referer=referer)
if self.is_valid_content_type(response):
raise FileException(request.url,
'%s: invalid-content_type' % (request.url, ))
filename = self.get_file_name(request, response)
if not filename:
raise FileException(request.url,
'%s: noaccess-filename' % (request.url, ))
self.inc_stats(info.spider, status)
try:
key = self.file_key(
info.spider.name,
request.url) # return the SHA1 hash of the file url
book_file_id, checksum = self.store.persist_file(key,
response.body,
info,
filename,
url=request.url)
except FileException as exc:
whyfmt = '%(medianame)s (error): Error processing %(medianame)s from %(request)s referred in <%(referer)s>: %(errormsg)s'
logger.msg(format=whyfmt,
level=log.WARNING,
spider=info.spider,
medianame=self.MEDIA_NAME,
request=request,
referer=referer,
errormsg=str(exc))
raise
# print {'url': request.url, 'file_id': book_file_id, 'checksum': checksum}
return {
'url': request.url,
'file_id': book_file_id,
'checksum': checksum
}
def media_downloaded(self, response, request, info):
"""
从content-dispositon中取文件名
:param response:
:param request:
:param info:
:return:
"""
referer = referer_str(request)
if response.status != 200:
logger.warning(
'File (code: %(status)s): Error downloading file from '
'%(request)s referred in <%(referer)s>',
{'status': response.status,
'request': request, 'referer': referer},
extra={'spider': info.spider}
)
raise FileException('download-error')
if not response.body:
logger.warning(
'File (empty-content): Empty file from %(request)s referred '
'in <%(referer)s>: no-content',
{'request': request, 'referer': referer},
extra={'spider': info.spider}
)
raise FileException('empty-content')
status = 'cached' if 'cached' in response.flags else 'downloaded'
logger.debug(
'File (%(status)s): Downloaded file from %(request)s referred in '
'<%(referer)s>',
{'status': status, 'request': request, 'referer': referer},
extra={'spider': info.spider}
)
self.inc_stats(info.spider, status)
try:
containFileName = response.headers.get('Content-Disposition') or response.headers.get('content-disposition')
if containFileName is not None:
pattern_marks = re.compile(r'filename="(.*)"')
pattern_no_marks = re.compile(r'filename=(.*)')
try:
file_name = pattern_marks.search(containFileName.decode('utf-8')) or pattern_no_marks.search(containFileName.decode('utf-8'))
except:
file_name = pattern_marks.search(str(containFileName).split("'")[1]) or pattern_no_marks.search(str(containFileName).split("'")[1])
if file_name is not None:
file_name = urlparse.unquote(file_name.group(1).strip())
else:
file_name = urlparse.unquote(os.path.basename(urlparse.unquote(response.request.url)))
else:
file_name = urlparse.unquote(os.path.basename(urlparse.unquote(response.request.url)))
media_ext = os.path.splitext(file_name)[1]
if "." not in media_ext or "\\" in media_ext or "/" in media_ext or ":" in media_ext or "*" in media_ext or "?" in media_ext or '"' in media_ext or "<" in media_ext or ">" in media_ext or "|" in media_ext or ";" in media_ext:
content_type = response.headers.get('Content-Type') or response.headers.get('content-type')
file_type = "." + content_type.decode('utf-8').split("/")[-1]
file_name = str(int(time.time())) + file_type
media_ext = file_type
if response.meta.get("data"):
url = request.url + urlparse.urlencode(response.meta["data"])
else:
url = request.url
media_guid = hashlib.sha1(to_bytes(url)).hexdigest()
path = 'full/%s%s' % (media_guid, media_ext)
checksum = self.file_downloaded(response, request, info, path)
except FileException as exc:
logger.warning(
'File (error): Error processing file from %(request)s '
'referred in <%(referer)s>: %(errormsg)s',
{'request': request, 'referer': referer, 'errormsg': str(exc)},
extra={'spider': info.spider}, exc_info=True
)
raise
except Exception as exc:
logger.error(
'File (unknown-error): Error processing file from %(request)s '
'referred in <%(referer)s>',
{'request': request, 'referer': referer},
exc_info=True, extra={'spider': info.spider}
)
raise FileException(str(exc))
return {'url': urlparse.unquote(url), 'path': path, 'checksum': checksum, "name": file_name}
def media_downloaded(self, response, request, info):
referer = referer_str(request)
if response.status != 200:
logger.warning(
'File (code: %(status)s): Error downloading file from '
'%(request)s referred in <%(referer)s>', {
'status': response.status,
'request': request,
'referer': referer
},
extra={'spider': info.spider})
raise FileException('download-error')
buf = BytesIO(response.body)
# Tag files that have errors in them
has_error = True if 'A PHP Error was encountered'\
in buf.read() else False
if not response.body:
logger.warning(
'File (empty-content): Empty file from %(request)s referred '
'in <%(referer)s>: no-content', {
'request': request,
'referer': referer
},
extra={'spider': info.spider})
raise FileException('empty-content')
status = 'cached' if 'cached' in response.flags else 'downloaded'
logger.debug(
'File (%(status)s): Downloaded file from %(request)s referred in '
'<%(referer)s>', {
'status': status,
'request': request,
'referer': referer
},
extra={'spider': info.spider})
self.inc_stats(info.spider, status)
try:
path = self.file_path(request, response=response, info=info)
checksum = self.file_downloaded(response, request, info)
except FileException as exc:
logger.warning(
'File (error): Error processing file from %(request)s '
'referred in <%(referer)s>: %(errormsg)s', {
'request': request,
'referer': referer,
'errormsg': str(exc)
},
extra={'spider': info.spider},
exc_info=True)
raise
except Exception as exc:
logger.error(
'File (unknown-error): Error processing file from %(request)s '
'referred in <%(referer)s>', {
'request': request,
'referer': referer
},
exc_info=True,
extra={'spider': info.spider})
raise FileException(str(exc))
return {
'url': request.url,
'path': path,
'checksum': checksum,
'has_error': has_error
}
def media_downloaded(self, response, request, info):
referer = referer_str(request)
if response.status != 200:
logger.warning(
'File (code: %(status)s): Error downloading file from '
'%(request)s referred in <%(referer)s>', {
'status': response.status,
'request': request,
'referer': referer
},
extra={'spider': info.spider})
raise FileException('download-error')
if not response.body:
logger.warning(
'File (empty-content): Empty file from %(request)s referred '
'in <%(referer)s>: no-content', {
'request': request,
'referer': referer
},
extra={'spider': info.spider})
raise FileException('empty-content')
status = 'cached' if 'cached' in response.flags else 'downloaded'
logger.debug(
'File (%(status)s): Downloaded file from %(request)s referred in '
'<%(referer)s>', {
'status': status,
'request': request,
'referer': referer
},
extra={'spider': info.spider})
self.inc_stats(info.spider, status)
try:
# path = self.file_path(request, response=response, info=info)
width, height, url_sha2, phash, checksum = self.file_downloaded(
response, request, info)
except FileException as exc:
logger.warning(
'File (error): Error processing file from %(request)s '
'referred in <%(referer)s>: %(errormsg)s', {
'request': request,
'referer': referer,
'errormsg': str(exc)
},
extra={'spider': info.spider},
exc_info=True)
raise
except Exception as exc:
logger.error(
'File (unknown-error): Error processing file from %(request)s '
'referred in <%(referer)s>', {
'request': request,
'referer': referer
},
exc_info=True,
extra={'spider': info.spider})
raise FileException(str(exc))
resultDict = {
'url': request.url,
'url_sha2': url_sha2,
'checksum': checksum,
'width': width,
'heigth': height,
'phash': phash
}
print json.dumps(resultDict)
raise DropItem("Printed to console")
