def test(test_cases):
try:
logfile = edex_tools.find_latest_log()
except OSError as e:
log.error('Error fetching latest log file - %s', e)
return
purge_edex(logfile)
last_instrument = None
num_files = 0
total_timeout = 0
for i, test_case in enumerate(test_cases):
logger.remove_handler(last_instrument)
logger.add_handler(test_case.instrument, dir=output_dir)
last_instrument = test_case.instrument
total_timeout += test_case.timeout
log.debug('Processing test case: %s', test_case)
sensor = 'MDA-%.1f-%08d' % (time.time(), i)
for source in test_case.source_data:
num_files += load_files(test_case.resource, test_case.endpoint,
source, sensor)
if not edex_tools.watch_log_for('Ingest: EDEX: Ingest',
logfile=logfile,
timeout=total_timeout,
expected_count=num_files):
log.error('Timed out waiting for ingest complete message')
time.sleep(1)
mio_analysis(hostname='localhost', output_dir=output_dir)
def wait_for_ingest_complete():
"""
Wait for ingestion to complete.
:return: True when the EDEX log file indicates completion, False if message does not appear within expected
timeout.
"""
return edex_tools.watch_log_for('Ingest: EDEX: Ingest')
def wait_for_ingest_complete():
"""
Wait for ingestion to complete.
:return: True when the EDEX log file indicates completion, False if message does not appear within expected
timeout.
"""
return edex_tools.watch_log_for('Ingest: EDEX: Ingest')
def test(test_cases):
try:
logfile = edex_tools.find_latest_log()
except OSError as e:
log.error('Error fetching latest log file - %s', e)
return
purge_edex(logfile)
last_instrument = None
num_files = 0
total_timeout = 0
for i, test_case in enumerate(test_cases):
logger.remove_handler(last_instrument)
logger.add_handler(test_case.instrument, dir=output_dir)
last_instrument = test_case.instrument
total_timeout += test_case.timeout
log.debug('Processing test case: %s', test_case)
sensor = 'MDA-%.1f-%08d' % (time.time(), i)
for source in test_case.source_data:
num_files += load_files(test_case.resource, test_case.endpoint, source, sensor)
if not edex_tools.watch_log_for('Ingest: EDEX: Ingest', logfile=logfile,
timeout=total_timeout, expected_count=num_files):
log.error('Timed out waiting for ingest complete message')
time.sleep(1)
mio_analysis(hostname='localhost', output_dir=output_dir)
def test(my_test_cases):
try:
logfile = edex_tools.find_latest_log()
except OSError as e:
log.error('Error fetching latest log file - %s', e)
return {}
total_timeout = 0
count = 0
sc = {}
pool = ThreadPool(MAX_THREADS)
pool.map(execute_test, my_test_cases)
for tc in my_test_cases:
total_timeout += tc.timeout
count += tc.count
# wait for all ingestion to complete
if not edex_tools.watch_log_for('EDEX - Ingest complete for file', logfile=logfile,
expected_count=count, timeout=total_timeout):
log.error('Timed out waiting for ingest complete message')
log.info('All files ingested, testing results')
for tc in pool.map(evaluate_test_case, test_cases):
sc.update(tc)
return sc
def purge_edex(logfile=None):
edex_tools.purge_edex()
return edex_tools.watch_log_for(
'Purge Operation: PURGE_ALL_DATA completed', logfile=logfile)
def purge_edex(logfile=None):
edex_tools.purge_edex()
return edex_tools.watch_log_for('Purge Operation: PURGE_ALL_DATA completed', logfile=logfile)
def test(my_test_cases):
try:
logfile = edex_tools.find_latest_log()
except OSError as e:
log.error('Error fetching latest log file - %s', e)
return {}
purge_edex(logfile)
total_timeout = 0
test_queue = Queue.Queue()
expected_queue = Queue.Queue()
results_queue = Queue.Queue()
threads = []
sc = {}
# execute all tests and load expected results
for count, case in enumerate(my_test_cases):
total_timeout += case.timeout
for index, _ in enumerate(case.pairs):
test_queue.put((case, index, count))
for _ in xrange(NUM_THREADS):
t = Thread(target=execute_test, args=[test_queue, expected_queue])
t.start()
threads.append(t)
for t in threads:
t.join()
# wait for all ingestion to complete
if not edex_tools.watch_log_for('Ingest: EDEX: Ingest', logfile=logfile,
expected_count=expected_queue.qsize(), timeout=total_timeout):
log.error('Timed out waiting for ingest complete message')
# pull expected results from the queue, break them down into individual streams then put them back in
temp_list = []
while True:
try:
instrument, sensor, test_file, yaml_file, expected = expected_queue.get_nowait()
for stream in expected:
temp_list.append((instrument, sensor, test_file, yaml_file, stream, expected[stream]))
except Queue.Empty:
break
for each in temp_list:
expected_queue.put_nowait(each)
# retrieve results and compare with expected
for _ in xrange(NUM_THREADS):
t = Thread(target=execute_validate, args=[expected_queue, results_queue])
t.start()
threads.append(t)
for t in threads:
t.join()
while True:
try:
instrument, test_file, yaml_file, stream, results = results_queue.get_nowait()
sc.setdefault(instrument, {}).setdefault(test_file, {}).setdefault(yaml_file, {})[stream] = results
except Queue.Empty:
break
return sc
for count, case in enumerate(my_test_cases):
total_timeout += case.timeout
for index, _ in enumerate(case.pairs):
test_queue.put((case, index, count))
for _ in xrange(NUM_THREADS):
t = Thread(target=execute_test, args=[test_queue, expected_queue])
t.start()
threads.append(t)
for t in threads:
t.join()
# wait for all ingestion to complete
if not edex_tools.watch_log_for('Ingest: EDEX: Ingest',
logfile=logfile,
expected_count=expected_queue.qsize(),
timeout=total_timeout):
log.error('Timed out waiting for ingest complete message')
# pull expected results from the queue, break them down into individual streams then put them back in
temp_list = []
while True:
try:
instrument, sensor, test_file, yaml_file, expected = expected_queue.get_nowait(
)
for stream in expected:
temp_list.append((instrument, sensor, test_file, yaml_file,
stream, expected[stream]))
except Queue.Empty:
break