def action(self, record):
if record.warc_type != 'response':
return
if not isinstance(record.content_block, model.BlockWithPayload):
return
if not isinstance(record.content_block.fields, model.HTTPHeaders):
return
if not record.content_block.fields.status_code == http.client.OK:
return
url = record.header.fields['WARC-Target-URI']
binary_block = record.content_block.binary_block
file_obj = binary_block.get_file()
data = file_obj.read(binary_block.length)
response = util.parse_http_response(data)
path_list = util.split_url_to_filename(url)
path_list = util.truncate_filename_parts(path_list)
path = os.path.join(self.out_dir, *path_list)
dir_path = os.path.dirname(path)
if os.path.isdir(path):
path = util.append_index_filename(path)
_logger.debug('Extracting %s to %s', record.record_id, path)
util.rename_filename_dirs(path)
os.makedirs(dir_path, exist_ok=True)
try:
with open(path, 'wb') as f:
shutil.copyfileobj(response, f)
except http.client.IncompleteRead as error:
_logger.warning('Malformed HTTP response: %s', error)
with open(path, 'wb') as f:
f.write(error.partial)
last_modified_str = response.getheader('Last-Modified')
if last_modified_str:
try:
last_modified = util.parse_http_date(last_modified_str)
except ValueError:
pass
else:
timestamp = time.mktime(last_modified.utctimetuple())
os.utime(path, (time.time(), timestamp))
_logger.debug('Apply mtime %d to %s', timestamp, path)
_logger.info('Extracted %s to %s', record.record_id, path)
def action(self, record):
if record.warc_type != 'response':
return
if not isinstance(record.content_block, model.BlockWithPayload):
return
if not isinstance(record.content_block.fields, model.HTTPHeaders):
return
if not record.content_block.fields.status_code == http.client.OK:
return
url = record.header.fields['WARC-Target-URI']
binary_block = record.content_block.binary_block
file_obj = binary_block.get_file()
data = file_obj.read(binary_block.length)
response = util.parse_http_response(data)
path_list = util.split_url_to_filename(url)
path_list = util.truncate_filename_parts(path_list)
path = os.path.join(self.out_dir, *path_list)
dir_path = os.path.dirname(path)
if os.path.isdir(path):
path = util.append_index_filename(path)
_logger.debug('Extracting %s to %s', record.record_id, path)
util.rename_filename_dirs(path)
os.makedirs(dir_path, exist_ok=True)
try:
with open(path, 'wb') as f:
shutil.copyfileobj(response, f)
except http.client.IncompleteRead as error:
_logger.warning('Malformed HTTP response: %s', error)
with open(path, 'wb') as f:
f.write(error.partial)
last_modified_str = response.getheader('Last-Modified')
if last_modified_str:
try:
last_modified = util.parse_http_date(last_modified_str)
except ValueError:
pass
else:
timestamp = time.mktime(last_modified.utctimetuple())
os.utime(path, (time.time(), timestamp))
_logger.debug('Apply mtime %d to %s', timestamp, path)
_logger.info('Extracted %s to %s', record.record_id, path)
def test_truncate_filename_parts(self):
self.assertEqual(['a', 'b', 'c' * 160 + '_4658bd', 'd'],
util.truncate_filename_parts(
['a', 'b', 'c' * 400, 'd']))
def test_truncate_filename_parts(self):
self.assertEqual(
["a", "b", "c" * 160 + "_4658bd", "d"], util.truncate_filename_parts(["a", "b", "c" * 400, "d"])
)
