def test_validate_sync_to(self): for goodurl in ('http://1.1.1.1/v1/a/c/o', 'http://1.1.1.1:8080/a/c/o', 'http://2.2.2.2/a/c/o', 'https://1.1.1.1/v1/a/c/o'): self.assertEquals( utils.validate_sync_to(goodurl, ['1.1.1.1', '2.2.2.2']), None) for badurl in ('http://1.1.1.1', 'httpq://1.1.1.1/v1/a/c/o', 'http://1.1.1.1/v1/a/c/o?query', 'http://1.1.1.1/v1/a/c/o#frag', 'http://1.1.1.1/v1/a/c/o?query#frag', 'http://1.1.1.1/v1/a/c/o?query=param', 'http://1.1.1.1/v1/a/c/o?query=param#frag', 'http://1.1.1.2/v1/a/c/o'): self.assertNotEquals( utils.validate_sync_to(badurl, ['1.1.1.1', '2.2.2.2']), None)
def test_validate_sync_to(self): for goodurl in ('http://1.1.1.1/v1/a/c/o', 'http://1.1.1.1:8080/a/c/o', 'http://2.2.2.2/a/c/o', 'https://1.1.1.1/v1/a/c/o'): self.assertEquals(utils.validate_sync_to(goodurl, ['1.1.1.1', '2.2.2.2']), None) for badurl in ('http://1.1.1.1', 'httpq://1.1.1.1/v1/a/c/o', 'http://1.1.1.1/v1/a/c/o?query', 'http://1.1.1.1/v1/a/c/o#frag', 'http://1.1.1.1/v1/a/c/o?query#frag', 'http://1.1.1.1/v1/a/c/o?query=param', 'http://1.1.1.1/v1/a/c/o?query=param#frag', 'http://1.1.1.2/v1/a/c/o'): self.assertNotEquals(utils.validate_sync_to(badurl, ['1.1.1.1', '2.2.2.2']), None)
def container_sync(self, path): """ Checks the given path for a container database, determines if syncing is turned on for that database and, if so, sends any updates to the other container. :param path: the path to a container db """ try: if not path.endswith('.db'): return broker = ContainerBroker(path) info = broker.get_info() x, nodes = self.container_ring.get_nodes(info['account'], info['container']) for ordinal, node in enumerate(nodes): if node['ip'] in self._myips and node['port'] == self._myport: break else: return if not broker.is_deleted(): sync_to = None sync_key = None sync_point1 = info['x_container_sync_point1'] sync_point2 = info['x_container_sync_point2'] for key, (value, timestamp) in broker.metadata.iteritems(): if key.lower() == 'x-container-sync-to': sync_to = value elif key.lower() == 'x-container-sync-key': sync_key = value if not sync_to or not sync_key: self.container_skips += 1 return sync_to = sync_to.rstrip('/') err = validate_sync_to(sync_to, self.allowed_sync_hosts) if err: self.logger.info( _('ERROR %(db_file)s: %(validate_sync_to_err)s'), { 'db_file': broker.db_file, 'validate_sync_to_err': err }) self.container_failures += 1 return stop_at = time() + self.container_time while time() < stop_at and sync_point2 < sync_point1: rows = broker.get_items_since(sync_point2, 1) if not rows: break row = rows[0] if row['ROWID'] >= sync_point1: break key = hash_path(info['account'], info['container'], row['name'], raw_digest=True) # This node will only intially sync out one third of the # objects (if 3 replicas, 1/4 if 4, etc.). This section # will attempt to sync previously skipped rows in case the # other nodes didn't succeed. if unpack_from('>I', key)[0] % \ self.container_ring.replica_count != ordinal: if not self.container_sync_row(row, sync_to, sync_key, broker, info): return sync_point2 = row['ROWID'] broker.set_x_container_sync_points(None, sync_point2) while time() < stop_at: rows = broker.get_items_since(sync_point1, 1) if not rows: break row = rows[0] key = hash_path(info['account'], info['container'], row['name'], raw_digest=True) # This node will only intially sync out one third of the # objects (if 3 replicas, 1/4 if 4, etc.). It'll come back # around to the section above and attempt to sync # previously skipped rows in case the other nodes didn't # succeed. if unpack_from('>I', key)[0] % \ self.container_ring.replica_count == ordinal: if not self.container_sync_row(row, sync_to, sync_key, broker, info): return sync_point1 = row['ROWID'] broker.set_x_container_sync_points(sync_point1, None) self.container_syncs += 1 except (Exception, Timeout), err: self.container_failures += 1 self.logger.exception(_('ERROR Syncing %s'), (broker.db_file))
def container_sync(self, path): """ Checks the given path for a container database, determines if syncing is turned on for that database and, if so, sends any updates to the other container. :param path: the path to a container db """ try: if not path.endswith('.db'): return broker = ContainerBroker(path) info = broker.get_info() x, nodes = self.container_ring.get_nodes(info['account'], info['container']) for ordinal, node in enumerate(nodes): if node['ip'] in self._myips and node['port'] == self._myport: break else: return if not broker.is_deleted(): sync_to = None sync_key = None sync_point1 = info['x_container_sync_point1'] sync_point2 = info['x_container_sync_point2'] for key, (value, timestamp) in broker.metadata.iteritems(): if key.lower() == 'x-container-sync-to': sync_to = value elif key.lower() == 'x-container-sync-key': sync_key = value if not sync_to or not sync_key: self.container_skips += 1 return sync_to = sync_to.rstrip('/') err = validate_sync_to(sync_to, self.allowed_sync_hosts) if err: self.logger.info( _('ERROR %(db_file)s: %(validate_sync_to_err)s'), {'db_file': broker.db_file, 'validate_sync_to_err': err}) self.container_failures += 1 return stop_at = time() + self.container_time while time() < stop_at and sync_point2 < sync_point1: rows = broker.get_items_since(sync_point2, 1) if not rows: break row = rows[0] if row['ROWID'] >= sync_point1: break key = hash_path(info['account'], info['container'], row['name'], raw_digest=True) # This node will only intially sync out one third of the # objects (if 3 replicas, 1/4 if 4, etc.). This section # will attempt to sync previously skipped rows in case the # other nodes didn't succeed. if unpack_from('>I', key)[0] % \ self.container_ring.replica_count != ordinal: if not self.container_sync_row(row, sync_to, sync_key, broker, info): return sync_point2 = row['ROWID'] broker.set_x_container_sync_points(None, sync_point2) while time() < stop_at: rows = broker.get_items_since(sync_point1, 1) if not rows: break row = rows[0] key = hash_path(info['account'], info['container'], row['name'], raw_digest=True) # This node will only intially sync out one third of the # objects (if 3 replicas, 1/4 if 4, etc.). It'll come back # around to the section above and attempt to sync # previously skipped rows in case the other nodes didn't # succeed. if unpack_from('>I', key)[0] % \ self.container_ring.replica_count == ordinal: if not self.container_sync_row(row, sync_to, sync_key, broker, info): return sync_point1 = row['ROWID'] broker.set_x_container_sync_points(sync_point1, None) self.container_syncs += 1 except (Exception, Timeout), err: self.container_failures += 1 self.logger.exception(_('ERROR Syncing %s'), (broker.db_file))
return HTTPNotFound() def PUT(self, req): """Handle HTTP PUT request.""" try: drive, part, account, container, obj = split_path( unquote(req.path), 4, 5, True) except ValueError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) if 'x-timestamp' not in req.headers or \ not check_float(req.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=req, content_type='text/plain') if 'x-container-sync-to' in req.headers: err = validate_sync_to(req.headers['x-container-sync-to'], self.allowed_sync_hosts) if err: return HTTPBadRequest(err) if self.mount_check and not check_mount(self.root, drive): return Response(status='507 %s is not mounted' % drive) timestamp = normalize_timestamp(req.headers['x-timestamp']) broker = self._get_container_broker(drive, part, account, container) if obj: # put container object if account.startswith(self.auto_create_account_prefix) and \ not os.path.exists(broker.db_file): broker.initialize(timestamp) if not os.path.exists(broker.db_file): return HTTPNotFound() broker.put_object(obj, timestamp, int(req.headers['x-size']), req.headers['x-content-type'], req.headers['x-etag']) return HTTPCreated(request=req)
def PUT(self, req): """Handle HTTP PUT request.""" try: drive, part, account, container, obj = split_path( unquote(req.path), 4, 5, True) except ValueError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) if 'x-timestamp' not in req.headers or \ not check_float(req.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=req, content_type='text/plain') if 'x-container-sync-to' in req.headers: err = validate_sync_to(req.headers['x-container-sync-to'], self.allowed_sync_hosts) if err: return HTTPBadRequest(err) if self.mount_check and not check_mount(self.root, drive): return Response(status='507 %s is not mounted' % drive) timestamp = normalize_timestamp(req.headers['x-timestamp']) broker = self._get_container_broker(drive, part, account, container) if obj: # put container object if account.startswith(self.auto_create_account_prefix) and \ not os.path.exists(broker.db_file): broker.initialize(timestamp) if not os.path.exists(broker.db_file): return HTTPNotFound() broker.put_object(obj, timestamp, int(req.headers['x-size']), req.headers['x-content-type'], req.headers['x-etag'])