def run():
simulator_no_flask.get_files(dir,
0,
None,
None,
"yes",
hio_config=hio_config_hw_image_proc,
stream_id_tag='dummy_set')
def file_walk_post():
file_path = request.form["file_path"]
period = request.form["period"]
binning = int(request.form["binning"])
color_channel = request.form.getlist("color_channel")
connect_kafka = request.form["kafka"] # Review: better to parse to a boolean here,
# rather than checking against a string in the other places
# Review2: better to spec which streaming fw to use?
period = float(period)
simulator_no_flask.get_files(file_path, period, binning, color_channel, connect_kafka)
return "You have now streamed all your files"
def run():
# get_files(file_path, period, binning, color_channel, send_to_target):
stream_id = simulator_no_flask.get_files(dir, 0, None, None, "yes",
hio_config=hio_config_hw_image_proc,
stream_id_tag='from_al')
return stream_id
# This pipeline was for ben to test simulator integration
hio_config_ben_test_haste_example = {
'master_host': ip,
'master_port': 8080,
'container_name': 'benblamey/haste-example:latest',
'container_os': 'ubuntu'
}
# This pipeline was for ben to test simulator integration
hio_config_ben_test = {
'master_host': '130.239.81.84',
'master_port': 8080,
'container_name': 'hakanwie/test:batch_hist2',
'container_os': 'ubuntu'
}
# This is the production pipeline
hio_config_hokan = {
'master_host': '130.239.81.126',
'master_port': 8080,
'container_name': 'hakanwie/test:batch_hist2',
'container_os': 'ubuntu'
}
simulator_no_flask.get_files(dir,
0.01,
1, ["1"],
"yes",
hio_config=hio_config_hokan,
stream_id_tag='test_set')
'master_host': ip,
'master_port': 8080,
'container_name': 'benblamey/hio-example:latest',
'container_os': 'ubuntu'
}
# This pipeline was for ben to test simulator integration
hio_config_ben_test_haste_example = {
'master_host': ip,
'master_port': 8080,
'container_name': 'benblamey/haste-example:latest',
'container_os': 'ubuntu'
}
# This is the production pipeline
hio_config_hokan = {
'master_host': '130.239.81.126',
'master_port': 8080,
'container_name': 'hakanwie/test:batch_hist2',
'container_os': 'ubuntu'
}
# get_files(file_path, period, binning, color_channel, send_to_target):
simulator_no_flask.get_files(dir,
0.01,
None,
None,
"yes",
hio_config=hio_config_ben_test_haste_example,
stream_id_tag='dummy_set')
'master_port': 8080,
'container_name': 'benblamey/haste-image-proc:latest',
'container_os': 'ubuntu'}
hio_config_hw_image_proc_profiling = {'master_host': ip,
'master_port': 8080,
'container_name': 'benblamey/haste-image-proc:latest-profiling',
'container_os': 'ubuntu'}
benchmarking.enable()
benchmark_full_pipeline_start = benchmarking.start_benchmark()
# get_files(file_path, period, binning, color_channel, send_to_target):
stream_id = simulator_no_flask.get_files(dir, 0, None, None, "yes",
hio_config=hio_config_hw_image_proc,
stream_id_tag='from_al')
# Poll the database to check for completion
mongo_client = MongoClient('mongodb://192.168.1.7:27017')
#mongo_client = MongoClient('mongodb://metadata-db-prod:27017')
mongo_db = mongo_client.streams
# The name of the MongoDB collection is the stream ID:
collection = mongo_db['strm_' + stream_id]
wait_for_record_count(collection, target_count=500)
benchmarking.end_benchmark('benchmark_full_pipeline', 'full', benchmark_full_pipeline_start)
# ** This script should be maintained to work on the production pipeline in the SNIC cloud **
ip = '192.168.1.24'
dir = '/mnt/ImageData/testDatasets/0'
hio_config_ben_test_hio_example = {'master_host': ip,
'master_port': 8080,
'container_name': 'benblamey/hio-example:latest',
'container_os': 'ubuntu'}
hio_config_ben_test_haste_example = {'master_host': ip,
'master_port': 8080,
'container_name': 'benblamey/haste-example:latest',
'container_os': 'ubuntu'}
hio_config_hokan = {'master_host': '130.239.81.126',
'master_port': 8080,
'container_name': 'hakanwie/test:batch_hist2',
'container_os': 'ubuntu'}
hio_config_hw_image_proc = {'master_host': ip,
'master_port': 8080,
'container_name': 'benblamey/haste-image-proc:latest',
'container_os': 'ubuntu'}
# get_files(file_path, period, binning, color_channel, send_to_target):
simulator_no_flask.get_files(dir, 0.01, None, None, "yes",
hio_config=hio_config_hw_image_proc,
stream_id_tag='test_dataset_0')