def test_mkdir_returns_a_dir():
requester = Mock()
requester.put.side_effect = get_response
client = Client(requester=requester)
directory = client.mkdir('/key_dir')
assert isinstance(directory, Directory)
assert '/key_dir' == directory.key
def test_set_without_value_key_sends_none():
requester = Mock()
requester.put.side_effect = get_response
client = Client(requester=requester)
key = '/update_key'
client.set(key)
requester.put.assert_called_with(key, data={'value': None, 'dir': False})
def test_update_key_returns_a_node_with_prev_node():
requester = Mock()
requester.put.side_effect = get_response
client = Client(requester=requester)
node = client.set('/update_key', value=1)
assert 'key_val' == node.value
assert 'old_key_val' == node.prev_node.value
def test_mkdir_raises_error_if_already_exists(response):
j = {
u'cause': u'/somedir',
u'errorCode': 105,
u'index': 30,
u'message': u'Key already exists'
}
response.json = Mock(return_value=j)
response.status_code = 412
session = Mock()
session.put.return_value = response
client = Client()
client.requester.session = session
with pytest.raises(errors.KeyAlreadyExists):
client.mkdir('/somedir')
def __init__(self, dst_path):
self.downloader = AuditsDownloader(dst_path)
self.is_downloading = False
self.etcd_client = Client(address=ETCD_SERVICE_HOST, port='2379')
with self.blocking_channel() as channel:
# Make sure queue is there
channel.queue_declare(queue=QUEUE_NAME)
logging.info('queue {} declared'.format(QUEUE_NAME))
class Collector(object):
def __init__(self, dst_path):
self.downloader = AuditsDownloader(dst_path)
self.is_downloading = False
self.etcd_client = Client(address=ETCD_SERVICE_HOST, port='2379')
with self.blocking_channel() as channel:
# Make sure queue is there
channel.queue_declare(queue=QUEUE_NAME)
logging.info('queue {} declared'.format(QUEUE_NAME))
@contextmanager
def blocking_channel(self):
connection = pika.BlockingConnection(
pika.ConnectionParameters(RABBITMQ_SERVICE_HOST))
channel = connection.channel()
yield channel
channel.close()
connection.close()
def add_to_queue(self, filepath):
with self.blocking_channel() as channel:
channel.basic_publish(exchange='',
routing_key=QUEUE_NAME,
body=filepath)
logging.info('Placed in queue file {}'.format(filepath))
def set_url(self):
try:
url = self.etcd_client.get('/data/collector/url').value
if url.lower() not in ['none', 'null']:
self.downloader.url = url
except KeyError:
logging.info('No hdfs address set yet')
@contextmanager
def downloading(self):
self.is_downloading = True
yield
self.is_downloading = False
@property
def delay(self):
try:
return int(self.etcd_client.get('/data/collector/delay').value)
except KeyError:
return 60
def run(self):
self.set_url()
with self.downloading():
self.downloader.download(callback=self.add_to_queue)
def run_periodically(self):
self.run()
scheduler = sched.scheduler(time.time, time.sleep)
while True:
if not self.is_downloading: # Add to queue
scheduler.enter(self.delay, 1, self.run, [])
scheduler.run()
def test_get_keys(key, recursive, expected):
requester = Mock()
requester.get = Mock(side_effect=get_response)
client = Client(requester=requester)
assert expected == client.get_keys(key=key, recursive=recursive)
def test_get_raises_an_error_if_directory():
requester = Mock()
requester.get = Mock(side_effect=get_response)
client = Client(requester=requester)
with pytest.raises(errors.NotAFile):
client.get('/k')