def test_get_object_if_different(self): """Test getting object content only when the local file is different http://en.wikipedia.org/wiki/HTTP_ETag Make a conditional request for an object using the If-None-Match header, it should get downloaded only if the local file is different, otherwise the response code should be 304 Not Modified """ object_name, data = self.create_object(self.container_name) # local copy is identical, no download object_md5 = md5(data, usedforsecurity=False).hexdigest() headers = {'If-None-Match': object_md5} url = "%s/%s" % (self.container_name, object_name) resp, _ = self.object_client.get(url, headers=headers) self.assertEqual(resp['status'], '304') # When the file is not downloaded from Swift server, response does # not contain 'X-Timestamp' header. This is the special case, therefore # the existence of response headers is checked without custom matcher. self.assertIn('date', resp) # Check only the format of common headers with custom matcher self.assertThat(resp, custom_matchers.AreAllWellFormatted()) # local copy is different, download local_data = "something different" other_md5 = md5(local_data.encode(), usedforsecurity=False).hexdigest() headers = {'If-None-Match': other_md5} resp, _ = self.object_client.get(url, headers=headers) self.assertHeaders(resp, 'Object', 'GET')
def _create_manifest(self): # Create a manifest file for SLO uploading object_name = data_utils.rand_name(name='TestObject') object_name_base_1 = object_name + '_01' object_name_base_2 = object_name + '_02' data_size = MIN_SEGMENT_SIZE self.content = data_utils.random_bytes(data_size) self._create_object(self.container_name, object_name_base_1, self.content) self._create_object(self.container_name, object_name_base_2, self.content) path_object_1 = '/%s/%s' % (self.container_name, object_name_base_1) path_object_2 = '/%s/%s' % (self.container_name, object_name_base_2) data_manifest = [{ 'path': path_object_1, 'etag': md5(self.content, usedforsecurity=False).hexdigest(), 'size_bytes': data_size }, { 'path': path_object_2, 'etag': md5(self.content, usedforsecurity=False).hexdigest(), 'size_bytes': data_size }] return json.dumps(data_manifest)
def test_delete_volume(self): self.mock_object(self._driver, '_get_vrts_lun_list') self.mock_object(self._driver, '_access_api') va_lun_name = self._driver._get_va_lun_name(self.volume.id) length = len(self.volume.id) index = int(length / 2) name1 = self.volume.id[:index] name2 = self.volume.id[index:] crc1 = md5(name1.encode('utf-8'), usedforsecurity=False).hexdigest()[:5] crc2 = md5(name2.encode('utf-8'), usedforsecurity=False).hexdigest()[:5] volume_name_to_ret = 'cinder' + '-' + crc1 + '-' + crc2 lun = {} lun['lun_name'] = va_lun_name lun['target_name'] = 'iqn.2017-02.com.veritas:faketarget' lun_list = {'output': {'output': {'luns': [lun]}}} self._driver._get_vrts_lun_list.return_value = lun_list self._driver._access_api.return_value = True self._driver.delete_volume(self.volume) self.assertEqual(volume_name_to_ret, va_lun_name) self.assertEqual(1, self._driver._access_api.call_count)
def _get_va_lun_name(self, name): length = len(name) index = int(length / 2) name1 = name[:index] name2 = name[index:] crc1 = md5(name1.encode('utf-8'), usedforsecurity=False).hexdigest()[:5] crc2 = md5(name2.encode('utf-8'), usedforsecurity=False).hexdigest()[:5] return 'cinder' + '-' + crc1 + '-' + crc2
def test_md5_with_data(self): digest = secretutils.md5(self._test_data).digest() self.assertEqual(digest, self._md5_digest) digest = secretutils.md5(self._test_data, usedforsecurity=True).digest() self.assertEqual(digest, self._md5_digest) digest = secretutils.md5(self._test_data, usedforsecurity=False).digest() self.assertEqual(digest, self._md5_digest)
def test_md5_without_data(self): md5 = secretutils.md5() md5.update(self._test_data) digest = md5.digest() self.assertEqual(digest, self._md5_digest) md5 = secretutils.md5(usedforsecurity=True) md5.update(self._test_data) digest = md5.digest() self.assertEqual(digest, self._md5_digest) md5 = secretutils.md5(usedforsecurity=False) md5.update(self._test_data) digest = md5.digest() self.assertEqual(digest, self._md5_digest)
def test_add(self): """Test that we can add an image via the filesystem backend.""" filesystem.ChunkedFile.CHUNKSIZE = units.Ki expected_image_id = str(uuid.uuid4()) expected_file_size = 5 * units.Ki # 5K expected_file_contents = b"*" * expected_file_size expected_checksum = md5(expected_file_contents, usedforsecurity=False).hexdigest() expected_location = "file://%s/%s" % (self.test_dir, expected_image_id) image_file = six.BytesIO(expected_file_contents) loc, size, checksum, metadata = self.store.add(expected_image_id, image_file, expected_file_size) self.assertEqual(expected_location, loc) self.assertEqual(expected_file_size, size) self.assertEqual(expected_checksum, checksum) self.assertEqual(u"file1", metadata['store']) uri = "file:///%s/%s" % (self.test_dir, expected_image_id) loc = location.get_location_from_uri_and_backend(uri, 'file1', conf=self.conf) (new_image_file, new_image_size) = self.store.get(loc) new_image_contents = b"" new_image_file_size = 0 for chunk in new_image_file: new_image_file_size += len(chunk) new_image_contents += chunk self.assertEqual(expected_file_contents, new_image_contents) self.assertEqual(expected_file_size, new_image_file_size)
def test_add_size_zero(self, mock_api_session, fake_size, fake_select_datastore): """ Test that when specifying size zero for the image to add, the actual size of the image is returned. """ fake_select_datastore.return_value = self.store.datastores[0][0] expected_image_id = str(uuid.uuid4()) expected_size = FIVE_KB expected_contents = b"*" * expected_size hash_code = secretutils.md5(expected_contents, usedforsecurity=False) expected_checksum = hash_code.hexdigest() fake_size.__get__ = mock.Mock(return_value=expected_size) with mock.patch('hashlib.md5') as md5: md5.return_value = hash_code expected_location = format_location( VMWARE_DS['vmware_server_host'], VMWARE_DS['vmware_store_image_dir'], expected_image_id, VMWARE_DS['vmware_datastores']) image = six.BytesIO(expected_contents) with mock.patch('requests.Session.request') as HttpConn: HttpConn.return_value = utils.fake_response() location, size, checksum, metadata = self.store.add( expected_image_id, image, 0) self.assertEqual("vmware1", metadata["store"]) self.assertEqual(utils.sort_url_by_qs_keys(expected_location), utils.sort_url_by_qs_keys(location)) self.assertEqual(expected_size, size) self.assertEqual(expected_checksum, checksum)
def update_lock(self): prop = self.settings.get('system.guid') guid = prop.get('value') path = '%s:%s' % (guid, self.path) if isinstance(path, six.text_type): path = path.encode('utf-8') self.lock = md5(path, usedforsecurity=False).hexdigest()
def test_add(self, fake_api_session, fake_size, fake_select_datastore, fake_cookie): """Test that we can add an image via the VMware backend.""" fake_select_datastore.return_value = self.store.datastores[0][0] expected_image_id = str(uuid.uuid4()) expected_size = FIVE_KB expected_contents = b"*" * expected_size hash_code = secretutils.md5(expected_contents, usedforsecurity=False) expected_checksum = hash_code.hexdigest() fake_size.__get__ = mock.Mock(return_value=expected_size) expected_cookie = 'vmware_soap_session=fake-uuid' fake_cookie.return_value = expected_cookie expected_headers = { 'Content-Length': six.text_type(expected_size), 'Cookie': expected_cookie } with mock.patch('hashlib.md5') as md5: md5.return_value = hash_code expected_location = format_location( VMWARE_DS['vmware_server_host'], VMWARE_DS['vmware_store_image_dir'], expected_image_id, VMWARE_DS['vmware_datastores']) image = six.BytesIO(expected_contents) with mock.patch('requests.Session.request') as HttpConn: HttpConn.return_value = utils.fake_response() location, size, checksum, metadata = self.store.add( expected_image_id, image, expected_size) _, kwargs = HttpConn.call_args self.assertEqual(expected_headers, kwargs['headers']) self.assertEqual("vmware1", metadata["store"]) self.assertEqual(utils.sort_url_by_qs_keys(expected_location), utils.sort_url_by_qs_keys(location)) self.assertEqual(expected_size, size) self.assertEqual(expected_checksum, checksum)
def _derive_key_and_iv(self, password, salt, key_length, iv_length): d = d_i = b'' while len(d) < key_length + iv_length: md5_str = d_i + password + salt d_i = md5(md5_str, usedforsecurity=True).digest() d += d_i return d[:key_length], d[key_length:key_length + iv_length]
def close(self): media = http.MediaIoBaseUpload(io.BytesIO(self.data), 'application/octet-stream', chunksize=self.chunk_size, resumable=self.resumable) resp = self.conn.objects().insert( bucket=self.bucket, name=self.object_name, body={}, media_body=media).execute(num_retries=self.num_retries) etag = resp['md5Hash'] md5 = secretutils.md5(self.data, usedforsecurity=False).digest() md5 = md5.encode('utf-8') etag = bytes(etag, 'utf-8') md5 = base64.b64encode(md5) if etag != md5: err = _('MD5 of object: %(object_name)s before: ' '%(md5)s and after: %(etag)s is not same.') % { 'object_name': self.object_name, 'md5': md5, 'etag': etag, } raise exception.InvalidBackup(reason=err) else: LOG.debug('MD5 before: %(md5)s and after: %(etag)s ' 'writing object: %(object_name)s in GCS.', {'etag': etag, 'md5': md5, 'object_name': self.object_name, }) return md5
def _get_mount_point_for_share(self, nfs_share): """Returns path to mount point NFS share. :param nfs_share: example 172.18.194.100:/var/nfs """ nfs_share = nfs_share.encode('utf-8') return os.path.join(self.configuration.nexenta_mount_point_base, md5(nfs_share, usedforsecurity=False).hexdigest())
def _get_partition(self, data): if self._hash_function == 'md5': hashed_key = self._hash2int(md5(data, usedforsecurity=False)) else: hashed_key = self._hash2int( hashlib.new(self._hash_function, data)) position = bisect.bisect(self._partitions, hashed_key) return position if position < len(self._partitions) else 0
def _convert_host_name(name): if name is None: return "" if len(name) > 32: name = md5(name.encode('utf-8'), usedforsecurity=False).hexdigest() else: name = name.replace('.', '-').lower() return name
def get_hash_str(base_str): """Returns string that represents MD5 hash of base_str (in hex format). If base_str is a Unicode string, encode it to UTF-8. """ if isinstance(base_str, str): base_str = base_str.encode('utf-8') return md5(base_str, usedforsecurity=False).hexdigest()
def _validate_etag(self, req, volume_id): if not req.if_match: return True context = req.environ['cinder.context'] metadata = self._get_metadata(context, volume_id) data = jsonutils.dumps({"metadata": metadata}) data = data.encode('utf-8') checksum = md5(data, usedforsecurity=False).hexdigest() return checksum in req.if_match.etags
def __init__(self, name, parsed_url, options): super(PostgresLock, self).__init__(name) self.acquired = False self._conn = None self._parsed_url = parsed_url self._options = options h = md5(usedforsecurity=False) h.update(name) self.key = h.digest()[0:2]
def get_fingerprint(self): """Patch paramiko This method needs to be patched to allow paramiko to work under FIPS. Until the patch to do this merges, patch paramiko here. TODO(alee) Remove this when paramiko is patched. See https://github.com/paramiko/paramiko/pull/1928 """ return md5(self.asbytes(), usedforsecurity=False).digest()
def add(self, image_id, image_file, image_size, hashing_algo, context=None, verifier=None): """This is a 0.26.0+ add, returns a 5-tuple""" if hashing_algo == 'md5': hasher = md5(usedforsecurity=False) else: hasher = hashlib.new(str(hashing_algo)) # assume 'image_file' will be bytes for these tests hasher.update(image_file) backend_url = "backend://%s" % image_id bytes_written = len(image_file) checksum = md5(image_file, usedforsecurity=False).hexdigest() multihash = hasher.hexdigest() metadata_dict = {"verifier_obj": verifier.name if verifier else None, "context_obj": context.name if context else None} return (backend_url, bytes_written, checksum, multihash, metadata_dict)
def test_update_lock(self, get_settings): guid = uuid.uuid4().hex settings = {'value': guid} get_settings.return_value = settings self.assertIsNone(self.proxy.update_lock()) path = '%s:%s' % (guid, self.proxy.path) if isinstance(path, six.text_type): path = path.encode('utf-8') expected = md5(path, usedforsecurity=False).hexdigest() self.assertEqual(expected, self.proxy.lock)
def test_get_object_with_if_none_match(self): """Test getting object with if_none_match""" object_name = data_utils.rand_name(name='TestObject') data = data_utils.random_bytes() create_md5 = md5(data, usedforsecurity=False).hexdigest() create_metadata = {'Etag': create_md5} self.object_client.create_object(self.container_name, object_name, data, metadata=create_metadata) list_data = data_utils.random_bytes() list_md5 = md5(list_data, usedforsecurity=False).hexdigest() list_metadata = {'If-None-Match': list_md5} resp, body = self.object_client.get_object(self.container_name, object_name, metadata=list_metadata) self.assertHeaders(resp, 'Object', 'GET') self.assertEqual(body, data)
def test_configure_add_same_dir_multiple_times_same_priority(self): """Tests handling of same dir in config multiple times. Tests BadStoreConfiguration exception is raised if same directory is specified multiple times in filesystem_store_datadirs with the same priority. """ store_map = [ self.useFixture(fixtures.TempDir()).path, self.useFixture(fixtures.TempDir()).path ] self.conf.set_override('filesystem_store_datadir', override=None, group='file1') self.conf.set_override('filesystem_store_datadirs', [ store_map[0] + ":100", store_map[1] + ":200", store_map[0] + ":100" ], group='file1') try: self.store.configure() except exceptions.BadStoreConfiguration: self.fail("configure() raised BadStoreConfiguration unexpectedly!") # Test that we can add an image via the filesystem backend filesystem.ChunkedFile.CHUNKSIZE = 1024 expected_image_id = str(uuid.uuid4()) expected_file_size = 5 * units.Ki # 5K expected_file_contents = b"*" * expected_file_size expected_checksum = md5(expected_file_contents, usedforsecurity=False).hexdigest() expected_location = "file://%s/%s" % (store_map[1], expected_image_id) image_file = six.BytesIO(expected_file_contents) loc, size, checksum, metadata = self.store.add(expected_image_id, image_file, expected_file_size) self.assertEqual(u"file1", metadata["store"]) self.assertEqual(expected_location, loc) self.assertEqual(expected_file_size, size) self.assertEqual(expected_checksum, checksum) loc = location.get_location_from_uri_and_backend(expected_location, "file1", conf=self.conf) (new_image_file, new_image_size) = self.store.get(loc) new_image_contents = b"" new_image_file_size = 0 for chunk in new_image_file: new_image_file_size += len(chunk) new_image_contents += chunk self.assertEqual(expected_file_contents, new_image_contents) self.assertEqual(expected_file_size, new_image_file_size)
def ftok(name, project): # Similar to ftok & http://semanchuk.com/philip/sysv_ipc/#ftok_weakness # but hopefully without as many weaknesses... h = md5(usedforsecurity=False) if not isinstance(project, bytes): project = project.encode('ascii') h.update(project) if not isinstance(name, bytes): name = name.encode('ascii') h.update(name) return (int(h.hexdigest(), 16) % _KEY_RANGE) + sysv_ipc.KEY_MIN
def index(self, req, volume_id): req_version = req.api_version_request metadata = super(Controller, self).index(req, volume_id) if req_version.matches(mv.ETAGS): data = jsonutils.dumps(metadata) data = data.encode('utf-8') resp = webob.Response() resp.headers['Etag'] = md5(data, usedforsecurity=False).hexdigest() resp.body = data return resp return metadata
def _get_hash_list(self, accounts_list): hash_list = [] hash_fields = ( preprov_creds.PreProvisionedCredentialProvider.HASH_CRED_FIELDS) for account in accounts_list: hash = md5(usedforsecurity=False) account_for_hash = dict( (k, v) for (k, v) in account.items() if k in hash_fields) hash.update(str(account_for_hash).encode('utf-8')) temp_hash = hash.hexdigest() hash_list.append(temp_hash) return hash_list
def setUp(self): super(TestBackCompatWrapper, self).setUp() self.fake_store = _FakeStore() self.fake_context = _FakeContext() self.fake_verifier = _FakeVerifier() self.img_id = '1234' self.img_file = b'0123456789' self.img_size = 10 self.img_checksum = md5(self.img_file, usedforsecurity=False).hexdigest() self.hashing_algo = 'sha256' self.img_sha256 = hashlib.sha256(self.img_file).hexdigest()
def test_reader_partial(self): content = b'XXX' image = six.BytesIO(content) expected_checksum = secretutils.md5(b'X', usedforsecurity=False).hexdigest() expected_multihash = hashlib.sha256(b'X').hexdigest() reader = vm_store._Reader(image, self.hash_algo) ret = reader.read(1) self.assertEqual(b'X', ret) self.assertEqual(expected_checksum, reader.checksum.hexdigest()) self.assertEqual(expected_multihash, reader.os_hash_value.hexdigest()) self.assertEqual(1, reader.size)
def get_hasher(hash_algo, usedforsecurity=True): """ Returns the required hasher, given the hashing algorithm. This is primarily to ensure that the hash algorithm is correctly chosen when executed on a FIPS enabled system :param hash_algo: hash algorithm requested :param usedforsecurity: whether the hashes are used in a security context """ if str(hash_algo) == 'md5': return md5(usedforsecurity=usedforsecurity) else: return hashlib.new(str(hash_algo))
def test_add_different_backend(self, mock_client): self.store = s3.Store(self.conf, backend="s3_region2") self.store.configure() self.register_store_backend_schemes(self.store, 's3', 's3_region2') expected_image_id = str(uuid.uuid4()) expected_s3_size = FIVE_KB expected_s3_contents = b"*" * expected_s3_size expected_checksum = md5(expected_s3_contents, usedforsecurity=False).hexdigest() expected_multihash = hashlib.sha256(expected_s3_contents).hexdigest() expected_location = format_s3_location(S3_CONF['s3_store_access_key'], S3_CONF['s3_store_secret_key'], 'http://s3-region2.com', S3_CONF['s3_store_bucket'], expected_image_id) image_s3 = six.BytesIO(expected_s3_contents) fake_s3_client = botocore.session.get_session().create_client('s3') with stub.Stubber(fake_s3_client) as stubber: stubber.add_response( method='head_bucket', service_response={}, expected_params={'Bucket': S3_CONF['s3_store_bucket']}) stubber.add_client_error(method='head_object', service_error_code='404', service_message='', expected_params={ 'Bucket': S3_CONF['s3_store_bucket'], 'Key': expected_image_id }) stubber.add_response(method='put_object', service_response={}, expected_params={ 'Bucket': S3_CONF['s3_store_bucket'], 'Key': expected_image_id, 'Body': botocore.stub.ANY }) mock_client.return_value = fake_s3_client loc, size, checksum, multihash, metadata = \ self.store.add(expected_image_id, image_s3, expected_s3_size, self.hash_algo) self.assertEqual("s3_region2", metadata["store"]) self.assertEqual(expected_location, loc) self.assertEqual(expected_s3_size, size) self.assertEqual(expected_checksum, checksum) self.assertEqual(expected_multihash, multihash)