def test(test_cases): try: logfile = edex_tools.find_latest_log() except OSError as e: log.error('Error fetching latest log file - %s', e) return purge_edex(logfile) last_instrument = None num_files = 0 total_timeout = 0 for i, test_case in enumerate(test_cases): logger.remove_handler(last_instrument) logger.add_handler(test_case.instrument, dir=output_dir) last_instrument = test_case.instrument total_timeout += test_case.timeout log.debug('Processing test case: %s', test_case) sensor = 'MDA-%.1f-%08d' % (time.time(), i) for source in test_case.source_data: num_files += load_files(test_case.resource, test_case.endpoint, source, sensor) if not edex_tools.watch_log_for('Ingest: EDEX: Ingest', logfile=logfile, timeout=total_timeout, expected_count=num_files): log.error('Timed out waiting for ingest complete message') time.sleep(1) mio_analysis(hostname='localhost', output_dir=output_dir)
def wait_for_ingest_complete(): """ Wait for ingestion to complete. :return: True when the EDEX log file indicates completion, False if message does not appear within expected timeout. """ return edex_tools.watch_log_for('Ingest: EDEX: Ingest')
def wait_for_ingest_complete(): """ Wait for ingestion to complete. :return: True when the EDEX log file indicates completion, False if message does not appear within expected timeout. """ return edex_tools.watch_log_for('Ingest: EDEX: Ingest')
def test(test_cases): try: logfile = edex_tools.find_latest_log() except OSError as e: log.error('Error fetching latest log file - %s', e) return purge_edex(logfile) last_instrument = None num_files = 0 total_timeout = 0 for i, test_case in enumerate(test_cases): logger.remove_handler(last_instrument) logger.add_handler(test_case.instrument, dir=output_dir) last_instrument = test_case.instrument total_timeout += test_case.timeout log.debug('Processing test case: %s', test_case) sensor = 'MDA-%.1f-%08d' % (time.time(), i) for source in test_case.source_data: num_files += load_files(test_case.resource, test_case.endpoint, source, sensor) if not edex_tools.watch_log_for('Ingest: EDEX: Ingest', logfile=logfile, timeout=total_timeout, expected_count=num_files): log.error('Timed out waiting for ingest complete message') time.sleep(1) mio_analysis(hostname='localhost', output_dir=output_dir)
def test(my_test_cases): try: logfile = edex_tools.find_latest_log() except OSError as e: log.error('Error fetching latest log file - %s', e) return {} total_timeout = 0 count = 0 sc = {} pool = ThreadPool(MAX_THREADS) pool.map(execute_test, my_test_cases) for tc in my_test_cases: total_timeout += tc.timeout count += tc.count # wait for all ingestion to complete if not edex_tools.watch_log_for('EDEX - Ingest complete for file', logfile=logfile, expected_count=count, timeout=total_timeout): log.error('Timed out waiting for ingest complete message') log.info('All files ingested, testing results') for tc in pool.map(evaluate_test_case, test_cases): sc.update(tc) return sc
def purge_edex(logfile=None): edex_tools.purge_edex() return edex_tools.watch_log_for( 'Purge Operation: PURGE_ALL_DATA completed', logfile=logfile)
def purge_edex(logfile=None): edex_tools.purge_edex() return edex_tools.watch_log_for('Purge Operation: PURGE_ALL_DATA completed', logfile=logfile)
def test(my_test_cases): try: logfile = edex_tools.find_latest_log() except OSError as e: log.error('Error fetching latest log file - %s', e) return {} purge_edex(logfile) total_timeout = 0 test_queue = Queue.Queue() expected_queue = Queue.Queue() results_queue = Queue.Queue() threads = [] sc = {} # execute all tests and load expected results for count, case in enumerate(my_test_cases): total_timeout += case.timeout for index, _ in enumerate(case.pairs): test_queue.put((case, index, count)) for _ in xrange(NUM_THREADS): t = Thread(target=execute_test, args=[test_queue, expected_queue]) t.start() threads.append(t) for t in threads: t.join() # wait for all ingestion to complete if not edex_tools.watch_log_for('Ingest: EDEX: Ingest', logfile=logfile, expected_count=expected_queue.qsize(), timeout=total_timeout): log.error('Timed out waiting for ingest complete message') # pull expected results from the queue, break them down into individual streams then put them back in temp_list = [] while True: try: instrument, sensor, test_file, yaml_file, expected = expected_queue.get_nowait() for stream in expected: temp_list.append((instrument, sensor, test_file, yaml_file, stream, expected[stream])) except Queue.Empty: break for each in temp_list: expected_queue.put_nowait(each) # retrieve results and compare with expected for _ in xrange(NUM_THREADS): t = Thread(target=execute_validate, args=[expected_queue, results_queue]) t.start() threads.append(t) for t in threads: t.join() while True: try: instrument, test_file, yaml_file, stream, results = results_queue.get_nowait() sc.setdefault(instrument, {}).setdefault(test_file, {}).setdefault(yaml_file, {})[stream] = results except Queue.Empty: break return sc
for count, case in enumerate(my_test_cases): total_timeout += case.timeout for index, _ in enumerate(case.pairs): test_queue.put((case, index, count)) for _ in xrange(NUM_THREADS): t = Thread(target=execute_test, args=[test_queue, expected_queue]) t.start() threads.append(t) for t in threads: t.join() # wait for all ingestion to complete if not edex_tools.watch_log_for('Ingest: EDEX: Ingest', logfile=logfile, expected_count=expected_queue.qsize(), timeout=total_timeout): log.error('Timed out waiting for ingest complete message') # pull expected results from the queue, break them down into individual streams then put them back in temp_list = [] while True: try: instrument, sensor, test_file, yaml_file, expected = expected_queue.get_nowait( ) for stream in expected: temp_list.append((instrument, sensor, test_file, yaml_file, stream, expected[stream])) except Queue.Empty: break