def _extract_b2_metadata(self, response, obj_key): '''Extract metadata from HTTP response object''' headers = CaseInsensitiveDict() for k, v in response.headers.items(): # we convert to lower case in order to do case-insensitive comparison if k.lower().startswith(info_header_prefix.lower() + 'meta-'): headers[k] = self._b2_url_decode(v) format_ = headers.get('%smeta-format' % info_header_prefix, 'raw') if format_ != 'raw2': # Current metadata format raise CorruptedObjectError('invalid metadata format: %s' % format_) parts = [] for i in count(): part = headers.get('%smeta-%03d' % (info_header_prefix, i), None) if part is None: break parts.append(part) buffer = urllib.parse.unquote(''.join(parts)) meta = literal_eval('{ %s }' % buffer) # Decode bytes values for (k, v) in meta.items(): if not isinstance(v, bytes): continue try: meta[k] = base64.b64decode(v) except binascii.Error: # This should trigger a MD5 mismatch below meta[k] = None # Check MD5. There is a case to be made for treating a mismatch as a # `CorruptedObjectError` rather than a `BadDigestError`, because the MD5 # sum is not calculated on-the-fly by the server but stored with the # object, and therefore does not actually verify what the server has # sent over the wire. However, it seems more likely for the data to get # accidentally corrupted in transit than to get accidentally corrupted # on the server (which hopefully checksums its storage devices). md5 = base64.b64encode(checksum_basic_mapping(meta)).decode('ascii') if md5 != headers.get('%smeta-md5' % info_header_prefix, None): log.warning('MD5 mismatch in metadata for %s', obj_key) # When trying to read file system revision 23 or earlier, we will # get a MD5 error because the checksum was calculated # differently. In order to get a better error message, we special # case the s3ql_passphrase and s3ql_metadata object (which are only # retrieved once at program start). if obj_key in ('s3ql_passphrase', 's3ql_metadata'): raise CorruptedObjectError('Meta MD5 for %s does not match' % obj_key) raise BadDigestError(400, 'bad_digest', 'Meta MD5 for %s does not match' % obj_key) return meta