示例#1
0
 def test_parse_http_date(self):
     self.assertEqual(
         datetime.datetime(1995,
                           11,
                           20,
                           19,
                           12,
                           8,
                           tzinfo=datetime.timezone(
                               datetime.timedelta(-1, 68400))),
         util.parse_http_date('Mon, 20 Nov 1995 19:12:08 -0500'))
示例#2
0
    def action(self, record):
        if record.warc_type != 'response':
            return
        if not isinstance(record.content_block, model.BlockWithPayload):
            return
        if not isinstance(record.content_block.fields, model.HTTPHeaders):
            return
        if not record.content_block.fields.status_code == http.client.OK:
            return

        url = record.header.fields['WARC-Target-URI']
        binary_block = record.content_block.binary_block
        file_obj = binary_block.get_file()
        data = file_obj.read(binary_block.length)
        response = util.parse_http_response(data)
        path_list = util.split_url_to_filename(url)
        path_list = util.truncate_filename_parts(path_list)
        path = os.path.join(self.out_dir, *path_list)
        dir_path = os.path.dirname(path)

        if os.path.isdir(path):
            path = util.append_index_filename(path)

        _logger.debug('Extracting %s to %s', record.record_id, path)
        util.rename_filename_dirs(path)
        os.makedirs(dir_path, exist_ok=True)

        try:
            with open(path, 'wb') as f:
                shutil.copyfileobj(response, f)
        except http.client.IncompleteRead as error:
            _logger.warning('Malformed HTTP response: %s', error)

            with open(path, 'wb') as f:
                f.write(error.partial)

        last_modified_str = response.getheader('Last-Modified')

        if last_modified_str:
            try:
                last_modified = util.parse_http_date(last_modified_str)
            except ValueError:
                pass
            else:
                timestamp = time.mktime(last_modified.utctimetuple())
                os.utime(path, (time.time(), timestamp))
                _logger.debug('Apply mtime %d to %s', timestamp, path)

        _logger.info('Extracted %s to %s', record.record_id, path)
示例#3
0
文件: tool.py 项目: chfoo/warcat
    def action(self, record):
        if record.warc_type != 'response':
            return
        if not isinstance(record.content_block, model.BlockWithPayload):
            return
        if not isinstance(record.content_block.fields, model.HTTPHeaders):
            return
        if not record.content_block.fields.status_code == http.client.OK:
            return

        url = record.header.fields['WARC-Target-URI']
        binary_block = record.content_block.binary_block
        file_obj = binary_block.get_file()
        data = file_obj.read(binary_block.length)
        response = util.parse_http_response(data)
        path_list = util.split_url_to_filename(url)
        path_list = util.truncate_filename_parts(path_list)
        path = os.path.join(self.out_dir, *path_list)
        dir_path = os.path.dirname(path)

        if os.path.isdir(path):
            path = util.append_index_filename(path)

        _logger.debug('Extracting %s to %s', record.record_id, path)
        util.rename_filename_dirs(path)
        os.makedirs(dir_path, exist_ok=True)

        try:
            with open(path, 'wb') as f:
                shutil.copyfileobj(response, f)
        except http.client.IncompleteRead as error:
            _logger.warning('Malformed HTTP response: %s', error)

            with open(path, 'wb') as f:
                f.write(error.partial)

        last_modified_str = response.getheader('Last-Modified')

        if last_modified_str:
            try:
                last_modified = util.parse_http_date(last_modified_str)
            except ValueError:
                pass
            else:
                timestamp = time.mktime(last_modified.utctimetuple())
                os.utime(path, (time.time(), timestamp))
                _logger.debug('Apply mtime %d to %s', timestamp, path)

        _logger.info('Extracted %s to %s', record.record_id, path)
示例#4
0
 def test_parse_http_date(self):
     self.assertEqual(datetime.datetime(1995, 11, 20, 19, 12, 8,
         tzinfo=datetime.timezone(datetime.timedelta(-1, 68400))),
         util.parse_http_date('Mon, 20 Nov 1995 19:12:08 -0500'))