def test_all_plugins():
#skip = ['MST_tracing','fastmarching_spanningtree','LCM_boost','Advantra','nctuTW','NeuroStalker','smartTrace','tips_GD']
skip = [] #['nctuTW']
START = 0
STOP = 30
filename = VAA3D_TEST_INPUT_FILE_5
prepare_test_files_local([filename])
i = START
errored = []
while i < STOP and i < len(JOB_TYPE_PLUGINS['Neuron Tracing']):
plugin = JOB_TYPE_PLUGINS['Neuron Tracing'][i]
if plugin not in skip:
try:
print "PLUGIN: " + plugin + ", FILENAME: " + filename
input_file_path = os.path.abspath(filename)
runtimes = PLUGINS[plugin]['runtime']
max_runtime = timeout.get_timeout_from_file(
input_file_path, runtimes['bytes_per_sec'],
runtimes['max'], runtimes['min'])
print input_file_path
run_plugin_local(plugin, PLUGINS[plugin], filename,
input_file_path, max_runtime)
except Exception, e:
print "####### Plugin " + plugin + " failed ########\n" + str(
e)
errored.append([plugin, filename])
i += 1
def test_all_plugins(): #skip = ['MST_tracing','fastmarching_spanningtree','LCM_boost','Advantra','nctuTW','NeuroStalker','smartTrace','tips_GD'] skip = [] #['nctuTW'] START = 0 STOP = 30 filename = VAA3D_TEST_INPUT_FILE_5 prepare_test_files_local([filename]) i = START errored = [] while i < STOP and i < len(JOB_TYPE_PLUGINS['Neuron Tracing']): plugin = JOB_TYPE_PLUGINS['Neuron Tracing'][i] if plugin not in skip: try: print "PLUGIN: " + plugin + ", FILENAME: " + filename input_file_path = os.path.abspath(filename) runtimes = PLUGINS[plugin]['runtime'] max_runtime = timeout.get_timeout_from_file(input_file_path, runtimes['bytes_per_sec'], runtimes['max'], runtimes['min']) print input_file_path run_plugin_local(plugin, PLUGINS[plugin], filename, input_file_path, max_runtime) except Exception, e: print "####### Plugin " + plugin + " failed ########\n" + str(e) errored.append([plugin,filename]) i+=1
def process_zip_file(job_item, zip_file_path, max_runtime):
"""
Unzip compressed file
Create new job_item record(s)
Upload new uncompressed file(s) to s3
"""
output_dir = os.path.dirname(zip_file_path)
zip_archive = zipfile.ZipFile(zip_file_path, "r")
filenames = zip_archive.namelist()
if len(filenames) > 1:
items_log.info("found more than 1 file inside .zip")
output_dir = os.path.join(output_dir, zip_file_path[:zip_file_path.find(zipper.ZIP_FILE_EXT)])
zipper.expand_zip_archive(zip_archive, output_dir)
zip_archive.close()
create_job_items_from_directory(job_item, output_dir)
shutil.rmtree(output_dir)
status = "COMPLETE"
else:
items_log.info("found only 1 file inside .zip")
filename = filenames[0]
file_path = os.path.join(output_dir, filename)
zipper.extract_file_from_archive(zip_archive, filename, file_path)
zip_archive.close()
job_item['input_filename'] = filename
job_item['output_filename'] = filename + OUTPUT_FILE_SUFFIXES[job_item['plugin']]
items_log.info("New output filename: " + job_item['output_filename'])
runtimes = PLUGINS[job_item['plugin']]['runtime']
max_runtime = timeout.get_timeout_from_file(file_path, runtimes['bytes_per_sec'],
runtimes['max'], runtimes['min'])
status = run_job_item(job_item, max_runtime)
os.remove(zip_file_path)
return status
def test_single_plugin(): filenames = [VAA3D_TEST_INPUT_FILE_5, VAA3D_TEST_INPUT_FILE_1] prepare_test_files_local(filenames) plugin_name = 'Vaa3D_Neuron2' #'MST_tracing' for f in filenames: input_file_path = os.path.abspath(f) runtimes = PLUGINS[plugin_name]['runtime'] max_runtime = timeout.get_timeout_from_file(input_file_path, runtimes['bytes_per_sec'], runtimes['max'], runtimes['min']) run_plugin_local(plugin_name, PLUGINS[plugin_name], f, input_file_path, max_runtime) cleanup_all([input_file_path])
def process_job_item(job_item, max_runtime=None):
job_item['status_id'] = get_job_item_status_id("IN_PROGRESS")
save_job_item(job_item)
local_file_path = os.path.abspath(job_item['input_filename'])
bucket_name = s3.get_bucket_name_from_filename(job_item['input_filename'],
[S3_INPUT_BUCKET, S3_WORKING_INPUT_BUCKET])
s3.download_file(job_item['input_filename'], local_file_path, bucket_name)
if not max_runtime:
runtimes = PLUGINS[job_item['plugin']]['runtime']
max_runtime = timeout.get_timeout_from_file(local_file_path, runtimes['bytes_per_sec'],
runtimes['max'], runtimes['min'])
status = run_job_item(job_item, max_runtime)
return status
def test_single_plugin():
filenames = [VAA3D_TEST_INPUT_FILE_5, VAA3D_TEST_INPUT_FILE_1]
prepare_test_files_local(filenames)
plugin_name = 'Vaa3D_Neuron2' #'MST_tracing'
for f in filenames:
input_file_path = os.path.abspath(f)
runtimes = PLUGINS[plugin_name]['runtime']
max_runtime = timeout.get_timeout_from_file(input_file_path,
runtimes['bytes_per_sec'],
runtimes['max'],
runtimes['min'])
run_plugin_local(plugin_name, PLUGINS[plugin_name], f, input_file_path,
max_runtime)
cleanup_all([input_file_path])
def process_job_item(job_item, max_runtime=None):
job_item['status_id'] = get_job_item_status_id("IN_PROGRESS")
save_job_item(job_item)
local_file_path = os.path.abspath(job_item['input_filename'])
bucket_name = s3.get_bucket_name_from_filename(
job_item['input_filename'], [S3_INPUT_BUCKET, S3_WORKING_INPUT_BUCKET])
s3.download_file(job_item['input_filename'], local_file_path, bucket_name)
if not max_runtime:
runtimes = PLUGINS[job_item['plugin']]['runtime']
max_runtime = timeout.get_timeout_from_file(local_file_path,
runtimes['bytes_per_sec'],
runtimes['max'],
runtimes['min'])
status = run_job_item(job_item, max_runtime)
return status
def process_zip_file(job_item, zip_file_path, max_runtime):
"""
Unzip compressed file
Create new job_item record(s)
Upload new uncompressed file(s) to s3
"""
output_dir = os.path.dirname(zip_file_path)
zip_archive = zipfile.ZipFile(zip_file_path, "r")
filenames = zip_archive.namelist()
if len(filenames) > 1:
items_log.info("found more than 1 file inside .zip")
output_dir = os.path.join(
output_dir,
zip_file_path[:zip_file_path.find(zipper.ZIP_FILE_EXT)])
zipper.expand_zip_archive(zip_archive, output_dir)
zip_archive.close()
create_job_items_from_directory(job_item, output_dir)
shutil.rmtree(output_dir)
status = "COMPLETE"
else:
items_log.info("found only 1 file inside .zip")
filename = filenames[0]
file_path = os.path.join(output_dir, filename)
zipper.extract_file_from_archive(zip_archive, filename, file_path)
zip_archive.close()
job_item['input_filename'] = filename
job_item['output_filename'] = filename + OUTPUT_FILE_SUFFIXES[
job_item['plugin']]
items_log.info("New output filename: " + job_item['output_filename'])
runtimes = PLUGINS[job_item['plugin']]['runtime']
max_runtime = timeout.get_timeout_from_file(file_path,
runtimes['bytes_per_sec'],
runtimes['max'],
runtimes['min'])
status = run_job_item(job_item, max_runtime)
os.remove(zip_file_path)
return status
