def test_parse_http_date(self):
self.assertEqual(
datetime.datetime(1995,
11,
20,
19,
12,
8,
tzinfo=datetime.timezone(
datetime.timedelta(-1, 68400))),
util.parse_http_date('Mon, 20 Nov 1995 19:12:08 -0500'))
def action(self, record):
if record.warc_type != 'response':
return
if not isinstance(record.content_block, model.BlockWithPayload):
return
if not isinstance(record.content_block.fields, model.HTTPHeaders):
return
if not record.content_block.fields.status_code == http.client.OK:
return
url = record.header.fields['WARC-Target-URI']
binary_block = record.content_block.binary_block
file_obj = binary_block.get_file()
data = file_obj.read(binary_block.length)
response = util.parse_http_response(data)
path_list = util.split_url_to_filename(url)
path_list = util.truncate_filename_parts(path_list)
path = os.path.join(self.out_dir, *path_list)
dir_path = os.path.dirname(path)
if os.path.isdir(path):
path = util.append_index_filename(path)
_logger.debug('Extracting %s to %s', record.record_id, path)
util.rename_filename_dirs(path)
os.makedirs(dir_path, exist_ok=True)
try:
with open(path, 'wb') as f:
shutil.copyfileobj(response, f)
except http.client.IncompleteRead as error:
_logger.warning('Malformed HTTP response: %s', error)
with open(path, 'wb') as f:
f.write(error.partial)
last_modified_str = response.getheader('Last-Modified')
if last_modified_str:
try:
last_modified = util.parse_http_date(last_modified_str)
except ValueError:
pass
else:
timestamp = time.mktime(last_modified.utctimetuple())
os.utime(path, (time.time(), timestamp))
_logger.debug('Apply mtime %d to %s', timestamp, path)
_logger.info('Extracted %s to %s', record.record_id, path)
def action(self, record):
if record.warc_type != 'response':
return
if not isinstance(record.content_block, model.BlockWithPayload):
return
if not isinstance(record.content_block.fields, model.HTTPHeaders):
return
if not record.content_block.fields.status_code == http.client.OK:
return
url = record.header.fields['WARC-Target-URI']
binary_block = record.content_block.binary_block
file_obj = binary_block.get_file()
data = file_obj.read(binary_block.length)
response = util.parse_http_response(data)
path_list = util.split_url_to_filename(url)
path_list = util.truncate_filename_parts(path_list)
path = os.path.join(self.out_dir, *path_list)
dir_path = os.path.dirname(path)
if os.path.isdir(path):
path = util.append_index_filename(path)
_logger.debug('Extracting %s to %s', record.record_id, path)
util.rename_filename_dirs(path)
os.makedirs(dir_path, exist_ok=True)
try:
with open(path, 'wb') as f:
shutil.copyfileobj(response, f)
except http.client.IncompleteRead as error:
_logger.warning('Malformed HTTP response: %s', error)
with open(path, 'wb') as f:
f.write(error.partial)
last_modified_str = response.getheader('Last-Modified')
if last_modified_str:
try:
last_modified = util.parse_http_date(last_modified_str)
except ValueError:
pass
else:
timestamp = time.mktime(last_modified.utctimetuple())
os.utime(path, (time.time(), timestamp))
_logger.debug('Apply mtime %d to %s', timestamp, path)
_logger.info('Extracted %s to %s', record.record_id, path)
def test_parse_http_date(self):
self.assertEqual(datetime.datetime(1995, 11, 20, 19, 12, 8,
tzinfo=datetime.timezone(datetime.timedelta(-1, 68400))),
util.parse_http_date('Mon, 20 Nov 1995 19:12:08 -0500'))