def test_append_index_filename(self):
self.assertEqual('_index_da39a3', util.append_index_filename(''))
self.assertEqual('index.php_index_bb6499',
util.append_index_filename('index.php'))
self.assertEqual(
os.path.join('hello', 'index.php_index_bb6499'),
util.append_index_filename(os.path.join('hello', 'index.php')))
def test_append_index_filename(self):
self.assertEqual('_index_da39a3', util.append_index_filename(''))
self.assertEqual('index.php_index_bb6499',
util.append_index_filename('index.php'))
self.assertEqual(os.path.join('hello', 'index.php_index_bb6499'),
util.append_index_filename(os.path.join('hello', 'index.php'))
)
def test_append_index_filename(self):
self.assertEqual("_index_da39a3", util.append_index_filename(""))
self.assertEqual("index.php_index_bb6499", util.append_index_filename("index.php"))
self.assertEqual(
os.path.join("hello", "index.php_index_bb6499"),
util.append_index_filename(os.path.join("hello", "index.php")),
)
def action(self, record):
if record.warc_type != 'response':
return
if not isinstance(record.content_block, model.BlockWithPayload):
return
if not isinstance(record.content_block.fields, model.HTTPHeaders):
return
if not record.content_block.fields.status_code == http.client.OK:
return
url = record.header.fields['WARC-Target-URI']
binary_block = record.content_block.binary_block
file_obj = binary_block.get_file()
data = file_obj.read(binary_block.length)
response = util.parse_http_response(data)
path_list = util.split_url_to_filename(url)
path_list = util.truncate_filename_parts(path_list)
path = os.path.join(self.out_dir, *path_list)
dir_path = os.path.dirname(path)
if os.path.isdir(path):
path = util.append_index_filename(path)
_logger.debug('Extracting %s to %s', record.record_id, path)
util.rename_filename_dirs(path)
os.makedirs(dir_path, exist_ok=True)
try:
with open(path, 'wb') as f:
shutil.copyfileobj(response, f)
except http.client.IncompleteRead as error:
_logger.warning('Malformed HTTP response: %s', error)
with open(path, 'wb') as f:
f.write(error.partial)
last_modified_str = response.getheader('Last-Modified')
if last_modified_str:
try:
last_modified = util.parse_http_date(last_modified_str)
except ValueError:
pass
else:
timestamp = time.mktime(last_modified.utctimetuple())
os.utime(path, (time.time(), timestamp))
_logger.debug('Apply mtime %d to %s', timestamp, path)
_logger.info('Extracted %s to %s', record.record_id, path)
def action(self, record):
if record.warc_type != 'response':
return
if not isinstance(record.content_block, model.BlockWithPayload):
return
if not isinstance(record.content_block.fields, model.HTTPHeaders):
return
if not record.content_block.fields.status_code == http.client.OK:
return
url = record.header.fields['WARC-Target-URI']
binary_block = record.content_block.binary_block
file_obj = binary_block.get_file()
data = file_obj.read(binary_block.length)
response = util.parse_http_response(data)
path_list = util.split_url_to_filename(url)
path_list = util.truncate_filename_parts(path_list)
path = os.path.join(self.out_dir, *path_list)
dir_path = os.path.dirname(path)
if os.path.isdir(path):
path = util.append_index_filename(path)
_logger.debug('Extracting %s to %s', record.record_id, path)
util.rename_filename_dirs(path)
os.makedirs(dir_path, exist_ok=True)
try:
with open(path, 'wb') as f:
shutil.copyfileobj(response, f)
except http.client.IncompleteRead as error:
_logger.warning('Malformed HTTP response: %s', error)
with open(path, 'wb') as f:
f.write(error.partial)
last_modified_str = response.getheader('Last-Modified')
if last_modified_str:
try:
last_modified = util.parse_http_date(last_modified_str)
except ValueError:
pass
else:
timestamp = time.mktime(last_modified.utctimetuple())
os.utime(path, (time.time(), timestamp))
_logger.debug('Apply mtime %d to %s', timestamp, path)
_logger.info('Extracted %s to %s', record.record_id, path)