def test_jobqueue():
signal.signal(signal.SIGALRM, lambda: pytest.fail())
signal.alarm(5)
with pytest.raises(ValueError):
JobQueue(0)
source = list(range(1024))
queue = Queue()
jq = JobQueue(8)
for n in source:
jq.put(store, (n, queue))
while not jq.queue.empty():
time.sleep(.1)
assert (set(source) == set(queue.queue))
if config.get('debug'):
jobs.logger.addHandler(logging.StreamHandler())
jobs.logger.setLevel(logging.DEBUG)
postgresql = config.get('postgresql', {})
threads = postgresql.get('threads', 1)
pg_uri = postgresql.get('uri')
kafka = config.get('kafka', {})
topics = kafka.pop('topics', '')
if isinstance(topics, str):
topics = (topics, )
kafka['value_deserializer'] = json_deserialize
if 'auto_offset_reset' not in kafka:
kafka['auto_offset_reset'] = 'earliest'
print('Setting up PostgreSQL...')
pool = ThreadedConnectionPool(1, threads, pg_uri)
with db.get_conn(pool) as c:
db.run_ddl(c)
print('Connecting to Kafka...')
consumer = KafkaConsumer(*topics, **kafka)
jq = JobQueue(threads)
print('\nWebChecker-pg service is running\n')
for message in consumer:
jq.put(jobs.commit_message, (pool, message))