def __init__(self):
self.pipeline = Pipeline()
self.history_pipeline = Pipeline()
for plsc in pipelinestages.getPipelineFeedStageClasses():
self.pipeline.appendStage(plsc())
for plsc in pipelinestages.getPipelineHistoryFeedStageClasses():
self.history_pipeline.appendStage(plsc())
def test_resource_group_get_all_mentioned(self):
p = Pipeline()
t = p.new_task()
t.declare_resource_group(foo={'bed': '{root}.bed', 'bim': '{root}.bim'})
t.command(f"cat {t.foo.bed}")
assert(t.foo.bed in t._mentioned)
assert(t.foo.bim not in t._mentioned)
def test_add_extension_input_resource_file(self):
input_file1 = '/tmp/data/example1.txt.bgz.foo'
p = Pipeline()
in1 = p.read_input(input_file1, extension='.txt.bgz.foo')
with self.assertRaises(Exception):
in1.add_extension('.baz')
assert in1._value.endswith('.txt.bgz.foo')
def test_resource_group_get_all_mentioned_dependent_tasks(self):
p = Pipeline()
t = p.new_task()
t.declare_resource_group(foo={'bed': '{root}.bed', 'bim': '{root}.bim'})
t.command(f"cat")
t2 = p.new_task()
t2.command(f"cat {t.foo}")
def test_iteration(self):
p1 = Pipe(name='p1')
p1.pipe_cache = ['ok']
p2 = Pipe(name='p2')
p3 = Pipe(name='p3')
pline = Pipeline(pipes=[p1,p2,p3])
self.assertEqual(pline.next(),'ok')
def main_pipeline(filedir):
#point it somewhere with a bunch of text files
pipe = Pipeline(getFileNames(filedir), frequencyCount, pruneCommon, writer)
pipe.run()
while(not pipe.isDone()):
pass
out.close()
def on_display(self):
"""
Rendering callback.
"""
self._camera.render()
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
self._scale += 0.1
pipeline = Pipeline(rotation=[0, self._scale, 0],
translation=[0, 0, 6],
projection=self._projection)
pipeline.set_camera(self._camera)
self._effect.set_wvp(pipeline.get_wvp())
self._effect.set_directional_light(
self._dir_light_color, self._dir_light_ambient_intensity)
position, tex_coord = 0, 1
glEnableVertexAttribArray(position)
glEnableVertexAttribArray(tex_coord)
glBindBuffer(GL_ARRAY_BUFFER, self._vbo)
glVertexAttribPointer(position, 3, GL_FLOAT, GL_FALSE, 20, ctypes.c_void_p(0))
glVertexAttribPointer(tex_coord, 2, GL_FLOAT, GL_FALSE, 20, ctypes.c_void_p(12))
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, self._ibo)
self._texture.bind(GL_TEXTURE0)
glDrawElements(GL_TRIANGLES, 18, GL_UNSIGNED_INT, ctypes.c_void_p(0))
glDisableVertexAttribArray(position)
glDisableVertexAttribArray(tex_coord)
glutSwapBuffers()
def test_when_consumer_yield_none(self):
pipeline = Pipeline().add(
PipeBuilder().alias("yield_none").consumer(lambda m: m if m == 0 else None).buffer_size(100).number_of_consumer(2)
)
expected = [0]
actual = [x for x in pipeline.stream(range(100))]
self.assertEquals(expected, actual)
def bas(target, fluxcal, phasecal, fringefinder, bpcal, catlist, FLAG_INT,
FLAG_BATCH, FLAG_DEBUG, FLAG_SILENT, FLAG_LOG, converts,
StageSelection, confpath, conffile, inpath, inext, outpath, outprefix,
logpath, logfile, comment, timeformat, prompt):
# Get all those parameters from above ^
settings = locals()
# Inform the CASA logger of what's happening.
casalog.origin("bas")
casalog.post("Output from this task is not appended to the CASA log.",
priority="INFO")
# Make sure we can see all the files we need.
obit_path = "/home/rowell/src/obit/python"
parseltongue_path = "/usr/local/share/parseltongue/python"
sys.path.append(obit_path)
sys.path.append(parseltongue_path)
# Do what we came here to do.
p = Pipeline()
p.set(settings)
p.start()
# Return the settings.
s = p.settings
return s
def main(input_file_path, output_file_path):
pdf_file = open_pdf_file(open(input_file_path, 'rwb'))
image_processor = Pipeline(pipeline_provider,
[layout_handler, ayat_handler, soura_handler],
pipeline_consumer, pipeline_validator)
image_processor.follow(retrive_page_as_image(pdf_file))
print 'success'
def test_pipeline():
p = Pipeline()
assert(len(p) == 0)
p = Pipeline([ajob])
assert(len(p) == 1)
assert_raises(ValueError, Pipeline, [notjob])
p.append(ajob)
assert(len(p) == 2)
def test_config_dict(pipeline_config):
pipeline = Pipeline([Filename], [PipelineResult], **pipeline_config)
config_dict = pipeline.get_config()
print(config_dict)
assert('Localizer' in config_dict)
assert('Decoder' in config_dict)
assert(config_dict['Localizer']['model_path'] == 'REQUIRED')
assert(config_dict['Decoder']['model_path'] == 'REQUIRED')
def test_resource_group_get_all_inputs(self):
p = Pipeline()
input = p.read_input_group(fasta="foo",
idx="bar")
t = p.new_task()
t.command(f"cat {input.fasta}")
assert(input.fasta in t._inputs)
assert(input.idx in t._inputs)
def test_resource_group_mentioned(self):
p = Pipeline()
t = p.new_task()
t.declare_resource_group(foo={'bed': '{root}.bed'})
t.command(f'echo "hello" > {t.foo}')
t2 = p.new_task()
t2.command(f'echo "hello" >> {t.foo.bed}')
p.run()
def test_one_pipe(self):
pipeline = Pipeline().add(
PipeBuilder().alias("multiplier").consumer(lambda m: m * m).buffer_size(100).number_of_consumer(10)
)
expected = [0, 1, 4, 9, 16, 25, 36, 49, 64, 81]
actual = [x for x in pipeline.stream(range(10))]
actual.sort()
self.assertEquals(expected, actual)
def __init__(self, data_folder='Data', models_folder='Models', working_folder='Working_Folder'):
"""
Constructor.
:param data_folder: the folder containing the data
:param models_folder: the folder containing the models
:param working_folder: the folder to store the results
"""
Pipeline.__init__(self, data_folder, models_folder, working_folder)
def test_multiple_pipes(self):
pipeline = Pipeline().add(
PipeBuilder("aggregator").aggregation_size(2).buffer_size(10)
).add(
PipeBuilder("summation").consumer(lambda aggr: sum(aggr)).number_of_consumer(1).buffer_size(10)
).add(
PipeBuilder("nth_triangular").consumer(lambda n: (n * n + n) / 2).number_of_consumer(1).buffer_size(10)
)
expect = [1, 15, 45, 91]
self.assertEquals(expect, [x for x in pipeline.stream(range(8))])
def test_two_actions(self):
arguments = {}
action1 = Mock()
action1.execute = Mock(return_value='somathin')
action2 = Mock()
action2.execute = Mock(return_value='other thing')
pipeline = Pipeline([ action1, action2 ])
result = pipeline.execute(arguments)
action1.execute.assert_called_once_with(arguments)
action2.execute.assert_called_once_with('somathin')
self.assertEqual('other thing', result)
def test_threads(self):
pipe_threads = Pipeline(range(1, 3), threads=10).apply(sleep)
threads_time = timeit("pipe_threads.run()",
number=1,
globals={"pipe_threads": pipe_threads})
self.assertLess(threads_time, 3)
pipe_no_threads = Pipeline(range(1, 3), threads=1).apply(sleep)
no_threads_time = timeit("pipe_no_threads.run()",
number=1,
globals={"pipe_no_threads": pipe_no_threads})
self.assertGreater(no_threads_time, 3)
def test_unnamed_action_in_pipeline():
"""Test that an unnamed action will get the module.task_name name."""
actions = [
TaskAction(
'stuff_increment_source',
amount='1'
),
]
executor = Pipeline(actions)
result = executor.schedule(1).get()
assert 'stuff_increment_source' in result.results
def get_default_pipeline():
cypress = CypressStage()
spirit = SpiritStage(ispca=False,thresh=0.01,ebounds=(0.96,1.1),startm=3)
kalman = KalmanStage()
pipeline = Pipeline()
draw = DrawStage('../etc/tmp/abilene', False)
pipeline.append_stage(cypress)
pipeline.append_stage(spirit)
#pipeline.append_stage(kalman)
#pipeline.append_stage(draw)
return pipeline
def searchImagery(permissions, token, extent):
validateAccessRights([os.getenv('IMG_AVAILABILITY')], permissions)
url = utils.SK_IMAGE_API + '/search'
pipeline = Pipeline(url, token, prepare_searchReq(extent))
pipeline.start()
logger.info("Created Pipeline. Waiting for results...")
response = pipeline.join()
if not response or 'results' not in response or len(
response['results']) == 0:
raise SpaceKnowError('Any imagery found in the response', 500)
return response['results']
def evaluatesCosts(scenes, extent, permissions, token):
validateAccessRights([os.getenv('KRAKEN_DRY_RUN')], permissions)
data = createEvaluationRequest(scenes, extent)
url = utils.SK_KRAKEN_API + '/dry-run'
pipeline = Pipeline(url, token, data)
pipeline.start()
logger.info("Created Pipeline. Waiting for results...")
analysis = pipeline.join()
if not analysis or 'allocatedCredits' not in analysis:
raise SpaceKnowError('Cost analysis is not available', 503)
return analysis
def pipeline_wrapper(dic_exp_conf, dic_agent_conf, dic_traffic_env_conf,
dic_path):
ppl = Pipeline(dic_exp_conf=dic_exp_conf,
dic_agent_conf=dic_agent_conf,
dic_traffic_env_conf=dic_traffic_env_conf,
dic_path=dic_path)
global multi_process
ppl.run(multi_process=multi_process)
print("pipeline_wrapper end")
return
def worker_thread(p, organizations, auto_create_repositories, deployment_map,
parameter_store):
LOGGER.debug("Worker Thread started for %s", p.get('name'))
pipeline = Pipeline(p)
if auto_create_repositories == 'enabled':
code_account_id = p.get('default_providers',
{}).get('source',
{}).get('properties',
{}).get('account_id', {})
has_custom_repo = p.get('default_providers',
{}).get('source',
{}).get('properties',
{}).get('repository', {})
if auto_create_repositories and code_account_id and str(
code_account_id).isdigit() and not has_custom_repo:
repo = Repo(code_account_id, p.get('name'), p.get('description'))
repo.create_update()
regions = []
for target in p.get('targets', []):
target_structure = TargetStructure(target)
for step in target_structure.target:
regions = step.get('regions',
p.get('regions', DEPLOYMENT_ACCOUNT_REGION))
paths_tags = []
for path in step.get('path', []):
paths_tags.append(path)
if step.get('tags') is not None:
paths_tags.append(step.get('tags', {}))
for path_or_tag in paths_tags:
pipeline.stage_regions.append(regions)
pipeline_target = Target(path_or_tag, target_structure,
organizations, step, regions)
pipeline_target.fetch_accounts_for_target()
pipeline.template_dictionary["targets"].append(
target.target_structure.generate_waves())
if DEPLOYMENT_ACCOUNT_REGION not in regions:
pipeline.stage_regions.append(DEPLOYMENT_ACCOUNT_REGION)
pipeline.generate_input()
ssm_params = fetch_required_ssm_params(pipeline.input["regions"]
or [DEPLOYMENT_ACCOUNT_REGION])
deployment_map.update_deployment_parameters(pipeline)
store_regional_parameter_config(pipeline, parameter_store)
with open(f'cdk_inputs/{pipeline.input["name"]}.json',
mode='w',
encoding='utf-8') as outfile:
data = {}
data['input'] = pipeline.input
data['input']['default_scm_branch'] = ssm_params.get(
'default_scm_branch')
data['ssm_params'] = ssm_params
json.dump(data, outfile)
def pipeline_wrapper(dic_exp_conf, dic_agent_conf, dic_traffic_env_conf, dic_path):
ppl = Pipeline(dic_exp_conf=dic_exp_conf, # experiment config
dic_agent_conf=dic_agent_conf, # RL agent config
dic_traffic_env_conf=dic_traffic_env_conf, # the simolation configuration
dic_path=dic_path # where should I save the logs?
)
global multi_process
ppl.run(multi_process=multi_process)
pass#print("pipeline_wrapper end")
return
def test_as_rows(passing_stage, failing_stage, second_passing_stage):
stages = [passing_stage, failing_stage, second_passing_stage]
pipeline = Pipeline(stages, trigger="commits")
runs = pipeline.simulation(now, [commit1], timedelta(minutes=60))
rows = as_rows(stages, runs)
assert ",".join(
map(str, rows[0])
) == "Start Time,Changes Included,Build,Build,AcceptanceTest,End Time,Deploy Time"
assert ",".join(
map(str, rows[1])
) == "2018-04-03 08:00:00,['#0001'],ok,fail,skip (previous failure),2018-04-03 08:20:00,"
def __init__(self,
name,
fields,
universe_file=None,
field_rename_map={},
build_meta=False,
fuzzy_thresh=0.75,
field_weights=None,
exact=[],
udelim='\t'):
self.name = name
self.fields = fields
self.universe_file = universe_file
self.field_rename_map = field_rename_map
#if target has different names for same fields, specify and rename target fields when matching
self.LSH = {}
self.threshold = fuzzy_thresh
self.field_weights = field_weights
self.exact = exact
self.__pipeline__ = Pipeline(name)
self.__preprocessor__ = PreprocessPiper(self.__pipeline__)
if (universe_file and os.path.exists(universe_file)):
self.__pipeline__.connect()
if (not self.__pipeline__.__client__[Pipeline.__database__][
'meta_{}'.format(name)].find_one(
{'sha1': sha1_file(universe_file)})):
try:
self.__preprocessor__.upload_universe_file(
universe_file, build_meta=build_meta, delim=udelim)
self.__pipeline__.__client__[Pipeline.__database__][
'meta_{}'.format(name)].insert_one({
'sha1':
sha1_file(universe_file),
'source':
universe_file
})
except pymongo.errors.BulkWriteError:
##File might have changed, still uploads new records, does not upload duplicates
##Throws error when any duplicates, any new records still get uploaded
pass
self.filters = self.__getfilters__()
for exit_status in self.__buildfilters__():
if (exit_status[1] != 0):
raise RuntimeError(
'When building {obj}, exit code returned non-zero ({code})'
.format(obj=exit_status[0], code=exit_status[1]))
def input_fn():
with tf.device('/cpu:0'), tf.name_scope('input_pipeline'):
pipeline = Pipeline(filenames,
batch_size=batch_size,
image_size=image_size,
time_step=time_step,
repeat=is_training,
shuffle=is_training,
augmentation=is_training)
features, labels = pipeline.get_batch()
return features, labels
def on_message(channel, method_frame, header_frame, body):
global counter
global conn
channel.basic_ack(delivery_tag=method_frame.delivery_tag)
lines = body.decode()
pln = Pipeline(lines, conn, nlp)
pln.run()
counter = counter + 1
if counter % 50 == 0:
logging.info("consumer {}: {} messages consumed ".format(
queue_name, counter))
def detect(input_path, output_directory, pipeline_file):
# if input_path is just a single file, we don't need all the multicore
# setup.
if os.path.isfile(input_path):
pipeline = Pipeline(pipeline_file, os.path.dirname(input_path), output_directory)
pipeline.execute(input_path)
elif os.path.isdir(input_path):
multiexecutor = MultiPipelineExecutor()
multiexecutor.execute(pipeline_file, input_path, output_directory)
else:
print("Input is not an image file or directory:", input_path)
def test_single_action_in_pipeline():
"""Test a single action scheduled by the executor.
"""
actions = [
TaskAction('stuff_increment_source',
name= 'increment',
amount='1'
)
]
executor = Pipeline(actions)
result = executor.schedule(1).get()
assert result.results['increment'] == 2
def main():
"""
Parses arguments; initialises logger; initialises camera driver if
necessary; loads single image from disk if necessary; and runs desired parts
of pipeline, or loads output from previous execution for printout.
"""
options, args = argparse.run()
loginit.run(options.verbosity)
logger = logging.getLogger('main')
logger.info(' '.join(sys.argv[1:]))
if options.simulate == 0:
options.simulate = None
l = DC1394Library()
elif options.simulate > 0:
options.simulate -= 1
elif options.simtime is None:
options.simtime = 36000
global pipeline
pipeline = Pipeline(options)
if options.disk:
logger.info('using poses from disk')
pipe = Pipeline()
pipe.options = options
printer = Printer(pipe=pipe)
printer.final()
logger.info('done. exiting')
sys.exit(0)
if args:
try:
image = cv2.imread('images/'+args[0], cv2.CV_LOAD_IMAGE_GRAYSCALE)
pipeline.set_image(image)
logger.info('opening image file %s from disk' % args[0])
except IOError:
logger.error('image file not found: %s' % args[0])
exit(1)
elif options.simulate is not None:
logger.info('running in simulation mode')
else:
try:
fwcam = handle_common_options(options, l)
pipeline.set_fwcam(fwcam)
logger.info('init. pydc1394 camera object')
logger.info('camera: %s' % fwcam.model)
logger.info('mode: %s' % fwcam.mode)
logger.info('framerate: %d' % fwcam.framerate.val)
except:
logger.error('unable to open camera capture')
exit(1)
pipeline.run()
def configure(self):
self.logger.info('Loading configuration: %r' % self.path)
# push implicit top-level context
self.pipeline = Pipeline()
with self.pipeline:
try:
execfile(self.path)
except Exception as ex:
trace = sys.exc_info()[2]
raise ConfigException(ex), None, trace
# pipeline has no final output queue
self.pipeline.setup(None)
def __init__(self, name):
self.crawl = Crawl()
self.analysis = Analysis()
self.pipe = Pipeline()
self.options = webdriver.ChromeOptions()
# 指定下载位置
prefs = {
'profile.default_content_settings.popups': 0,
'download.default_directory': os.path.abspath('DATA')
}
self.options.add_experimental_option('prefs', prefs)
self.driver = webdriver.Chrome(chrome_options=self.options)
self.name = str(name.encode('gbk'))[2:-1].replace('\\x', '%').upper()
def test_context_and_kwargs_application_in_pipeline():
"""Test that both build context and user-supplied kwargs are applied
to a series of tasks in a chain.
"""
actions = [
TaskAction('increment', num='0'),
TaskAction('increment', name='increment_again', num='{{ increment }}'),
TaskAction('increment', name='increment_once_more', num='{{ increment_again }}')
]
source = None
executor = Pipeline(source, actions)
result = executor.schedule().get()
assert result.results['increment_once_more'] == 3
def __init__(
self,
data_folder="Data",
models_folder="Models",
working_folder="Working_Folder",
sim_data_folder="param_estim_data",
sim_plots_folder="param_estim_plots",
):
__doc__ = Pipeline.__init__.__doc__
Pipeline.__init__(self, data_folder, models_folder, working_folder, sim_data_folder, sim_plots_folder)
# The folder containing the updated Copasi models
self.__updated_models_folder = "updated_models"
def test_resource_group_get_all_outputs(self):
p = Pipeline()
t1 = p.new_task()
t1.declare_resource_group(foo={'bed': '{root}.bed', 'bim': '{root}.bim'})
t1.command(f"cat {t1.foo.bed}")
t2 = p.new_task()
t2.command(f"cat {t1.foo.bed}")
for r in [t1.foo.bed, t1.foo.bim]:
assert(r in t1._outputs)
assert(r in t2._inputs)
assert(r in t1._mentioned)
assert(r not in t2._mentioned)
def main(argv):
Common.init("/mnt/config/config.yml", docker_container=argv[1])
Common.message(_('Loading pipeline...'))
pipeline = Pipeline(path="/mnt/pipeline/pipeline.yml", host_path=argv[0])
if len(argv) > 2 and argv[2] == "test":
pipeline.run_tests()
elif len(argv) > 2 and argv[2]:
Common.message(_('Unknown argument')+": "+argv[2])
else:
Web(pipeline, host='0.0.0.0')
def add_pics(rep, paths, process, recipe=None):
"""
Add pictures to repository.
Arguments:
rep -- Add pictures to this repository.
paths -- Paths of the pictures to be added (check if path exists).
process -- Boolean flag if added pictures should be processed.
recipe -- Recipe to use for picture processing.
"""
for path in paths:
if not os.path.exists(path):
log.warning("File not found: '%s'. Skipping it." % path)
pics = [Picture(path) for path in paths if os.path.exists(path)]
rep.index.add(pics)
if process:
log.info("Processing pictures.")
if not recipe: # set up pipeline
process_recipe = \
Recipe.fromString(rep.config['recipes.default'])
pl = Pipeline('Pipeline1', process_recipe,
path=rep.connector.url.path)
for pic in pics:
pl.put(pic)
pl.start() # start processing threads
pl.join() # wait until threads exit
log.info("Saving index to file.")
with rep.connector.connected():
rep.save_index_to_disk()
return rep
def test_star_apply(self):
pipe = Pipeline([10, 20, 30])
self.assertIsInstance(Pipeline([]).star_apply(func=func), Pipeline)
with self.assertRaises(ValueError):
Pipeline([]).star_apply(object())
with self.assertRaises(TypeError):
Pipeline([]).star_apply(not_a_param=object())
with self.assertRaises(TypeError):
out, err = pipe.star_apply(add_two_params).run()
def error_func():
raise err[0]
error_func()
out, err = pipe.apply(lambda x: [x, x]).star_apply(
add_two_params).run()
self.assertEqual(len(out), 3)
self.assertEqual(sum(out), 120)
self.assertEqual(len(err), 0)
out, err = pipe.apply(lambda x: {
"a": x,
"b": x
}).star_apply(add_two_params).run()
self.assertEqual(len(out), 3)
self.assertEqual(sum(out), 120)
self.assertEqual(len(err), 0)
def testGetAllDataDependencies(self):
pipeline = Pipeline('project-input.0.txt')
pipeline.get_all_data_dependencies()
self.assertEqual(pipeline.data_dep[0], (3,1))
self.assertEqual(pipeline.data_dep[1], (3,2))
self.assertEqual(pipeline.data_dep[2], (4,3))
self.assertEqual(pipeline.data_dep[3], (6,2))
self.assertEqual(pipeline.data_dep[4], (6,3))
self.assertEqual(pipeline.data_dep[5], (6,5))
self.assertEqual(pipeline.data_dep[6], (7,3))
self.assertEqual(pipeline.data_dep[7], (7,5))
self.assertEqual(pipeline.data_dep[8], (8,1))
self.assertEqual(pipeline.data_dep[9], (8,2))
self.assertEqual(pipeline.data_dep[10], (8,6))
def test_named_actions_in_pipeline():
"""Test that named actions store their name in build context.
"""
dct = {
"name": "mytask",
"task": "named_action",
}
actions = [TaskAction(dct['task'], name=dct['name'])]
executor = Pipeline(actions)
ret = executor.schedule('42').get()
assert 'mytask' in ret.results.keys()
assert bool(ret.results['mytask'])
def main():
# Read input file path from command line argument
if len(sys.argv) != 2:
print 'Please use the application as follows: python main.py <filepath>.wav'
sys.exit(2)
fpath = sys.argv[1]
# If the file path ends with .wav only the given file will be processed
# Else we assume that the input is a path to all the .wav files to be batch processed
audio_files = []
if fpath.endswith('.wav'):
audio_files.append(fpath)
else:
audio_files.extend(glob.glob(fpath + '/*.wav'))
# TODO: Compare human/model recognition
# TODO: Maybe save (for instance) spectral images of sound_generation that failed
# Setup logging
logging.basicConfig(filename='pitch_perception.log', level=logging.INFO)
n_channels = int(Config.get_config_option('n_channels'))
transducer = BrianTransducer(n_channels)
available_pitch_extractors = {'naive': NaivePitchExtractor, 'spectral': SpectralPitchExtractor(n_channels),
'temporal': TemporalPitchExtractor, 'xcorr': XcorrPitchExtractor(n_channels)}
pitch_extractor = available_pitch_extractors[Config.get_config_option('pitch_extraction')]
# Init pipeline
pipeline = Pipeline(transducer, pitch_extractor, test_mode=False)
# Collect results
results = []
for af in audio_files:
# Run processing
pitch = pipeline.process(af)
log_string = 'File: %s\tPitch: %i' % (af, pitch)
logging.info(log_string)
# Output final pitch
print log_string
results.append((af, pitch))
# Export results
CsvExporter.export('results.csv', pitch_extractor.__class__.__name__, results)
def process_image(self):
'''
Process the image by thresholding and other techniques to make the cells easy to extract.
Return value
----------
returns a black and white version of the image
'''
# build the prepocessing pipleine
pipeline = Pipeline([
Helpers.convert_to_grayscale, lambda image: Helpers.blur(image, 5),
Helpers.thresholdify, Helpers.ellipse_morph
])
return pipeline.process_pipeline(self.image)
def handle_data():
# Get Lyrics
lyrics = request.form['lyrics']
if not lyrics:
return "No Lyrics Found!"
# Convert to list if it is not.
if type(lyrics) == type(""):
lyrics = [lyrics]
# Instantiate a pipeline object
pipeline = Pipeline(lyrics)
# Get the results.
return pipeline.vectorize()
def downloadMap(mapType, scene, extent, token):
url = buildURL(utils.SK_KRAKEN_API, 'release', mapType, 'geojson')
data = json.dumps({'sceneId': scene, 'extent': extent})
try:
pipeline = Pipeline(url, token, data)
pipeline.start()
spaceKnowLogger.info('Making Request for scene %s' % scene)
jsonMap = pipeline.join()
if not jsonMap or 'mapId' not in jsonMap or 'maxZoom' not in jsonMap or \
'tiles' not in jsonMap:
raise SpaceKnowError('Receive invalid map for scene %s' % scene,
500)
return jsonMap
except SpaceKnowError as e:
spaceKnowLogger.error('Error %d: %s' % (e.status_code, e.error))
def paintGL(self):
self.step += 0.1
self.camera.render()
projection = ProjParams(self.width, self.height, 1.0, 100.0, 60.0)
self.pipeline = Pipeline(rotation=[0, 30*self.step, 0],
translation=[0, 0, 3],
projection=projection)
self.pipeline.set_camera(self.camera)
glClear(GL_COLOR_BUFFER_BIT)
glEnableVertexAttribArray(0)
world_location = glGetUniformLocation(self.program, "gWorld")
glUniformMatrix4fv(world_location, 1, GL_TRUE, self.pipeline.get_wvp())
glDrawElements(GL_TRIANGLES, self.index.shape[0],
GL_UNSIGNED_INT, ctypes.c_void_p(0))
glDisableVertexAttribArray(0)
def test_pipeline_unknown(self):
os.environ["IN_KIND"] = "MEM"
os.environ["OUT_KIND"] = "MEM"
pipeline = Pipeline(args=["unknown"])
del os.environ["IN_KIND"]
del os.environ["OUT_KIND"]
assert pipeline is not None
def test_localizer(pipeline_config):
pipeline = Pipeline([Filename], [Regions, LocalizerPositions], **pipeline_config)
expected_stages = [ImageReader,
LocalizerPreprocessor,
Localizer]
_assert_types(pipeline.pipeline, expected_stages)
fname = os.path.dirname(__file__) + '/data/Cam_2_20150821161530_884267.jpeg'
outputs = pipeline([fname])
assert len(outputs) == 2
assert Regions in outputs
assert LocalizerPositions in outputs
regions = outputs[Regions]
assert(len(regions) > 0)
positions = outputs[LocalizerPositions]
assert(len(regions) == len(positions))
for pos in positions:
assert(pos[0] >= 0 and pos[0] < 3000)
assert(pos[1] >= 0 and pos[1] < 4000)
def get_output_tasks(my):
process = my.get_value("process")
parent = my.get_parent()
# get the pipeline
pipeline_code = parent.get_value("pipeline_code", no_exception=True)
if not pipeline_code:
return []
pipeline = Pipeline.get_by_code(pipeline_code)
if not pipeline:
return []
processes = pipeline.get_output_processes(process)
if not processes:
return []
tasks = []
process_names = [x.get_name() for x in processes]
search = Search("sthpw/task")
search.add_filters("process", process_names)
search.add_parent_filter(parent)
tasks = search.get_sobjects()
return tasks
def test_resource_group_get_all_outputs(self):
p = Pipeline()
t1 = p.new_task()
t1.declare_resource_group(foo={
'bed': '{root}.bed',
'bim': '{root}.bim'
})
t1.command(f"cat {t1.foo.bed}")
t2 = p.new_task()
t2.command(f"cat {t1.foo.bed}")
for r in [t1.foo.bed, t1.foo.bim]:
assert (r in t1._outputs)
assert (r in t2._inputs)
assert (r in t1._mentioned)
assert (r not in t2._mentioned)
def test_parse_normal(self):
self.assertEqual(
Pipeline.parse('Hello!'),
Message(tokens=[
PlainText(name='Hello!')
])
)
def test_generator_processor(tmpdir, bees_image, pipeline_config):
def image_generator():
ts = time.time()
data_source = DataSource.new_message(filename='bees.jpeg')
for i in range(2):
img = imread(bees_image)
yield data_source, img, ts + i
repo = Repository(str(tmpdir))
pipeline = Pipeline([Image, Timestamp], [PipelineResult], **pipeline_config)
gen_processor = GeneratorProcessor(
pipeline, lambda: BBBinaryRepoSink(repo, camId=2))
gen_processor(image_generator())
gen_processor(image_generator())
fnames = list(repo.iter_fnames())
assert len(fnames) == 2
last_ts = 0
for fname in repo.iter_fnames():
print("{}: {}".format(fname, os.path.getsize(fname)))
with open(fname, 'rb') as f:
fc = FrameContainer.read(f)
assert fc.dataSources[0].filename == 'bees.jpeg'
assert last_ts < fc.fromTimestamp
last_ts = fc.fromTimestamp
def test_imagereader(bees_image, pipeline_config):
pipeline = Pipeline([Filename], [Image, Timestamp, CameraIndex], **pipeline_config)
expected_stages = [ImageReader]
_assert_types(pipeline.pipeline, expected_stages)
outputs = pipeline([bees_image])
assert(len(outputs) == 3)
assert Image in outputs
assert Timestamp in outputs
assert CameraIndex in outputs
im = outputs[Image]
ts = outputs[Timestamp]
idx = outputs[CameraIndex]
tz = pytz.timezone('Europe/Berlin')
dt = datetime.datetime.fromtimestamp(ts, tz=pytz.utc)
dt = dt.astimezone(tz)
assert(im.shape == (3000, 4000))
assert(dt.year == 2015)
assert(dt.month == 8)
assert(dt.day == 21)
assert(dt.hour == 16)
assert(dt.minute == 15)
assert(dt.second == 30)
assert(dt.microsecond == 884267)
assert(idx == 2)
def sort_shot_tasks( tasks):
''' sort task by pipeline. It can be for assets or shots'''
# first sort by pipeline
# get the pipeline of the first task
sobject = tasks[0].get_parent()
if not sobject:
return tasks
pipeline = Pipeline.get_by_sobject(sobject)
if not pipeline:
return tasks
# assign a number value to each process in the pipeline
processes = pipeline.get_process_names()
process_dict = {}
count = 0
for process in processes:
process_dict[process] = count
count += 1
def process_compare(x,y):
x_process = x.get_value("process")
y_process = y.get_value("process")
x_value = process_dict.get(x_process)
y_value = process_dict.get(y_process)
return cmp(x_value, y_value)
tasks.sort(process_compare)
return tasks
def test_decoder(pipeline_config):
pipeline = Pipeline([Filename], [LocalizerPositions, IDs, Radii], **pipeline_config)
expected_stages = [ImageReader,
LocalizerPreprocessor,
Localizer,
Decoder]
_assert_types(pipeline.pipeline, expected_stages)
fname = os.path.dirname(__file__) + '/data/Cam_2_20150821161530_884267.jpeg'
outputs = pipeline([fname])
assert len(outputs) == 3
assert IDs in outputs
assert LocalizerPositions in outputs
assert Radii in outputs
positions = outputs[LocalizerPositions]
ids = outputs[IDs]
radii = outputs[Radii]
assert(len(ids) == len(positions))
assert(len(ids) == len(radii))
for pos, id, radius in zip(positions, ids, radii):
pos = np.round(pos).astype(np.int)
id = ''.join([str(int(b)) for b in (np.round(id))])
print('Detection at ({}, {}) \t ID: {} \t Radius: {}'.format(pos[0], pos[1], id, radius))
def test_update_deployment_parameters(cls):
cls.parameter_store = Mock()
cls.parameter_store.put_parameter.return_value = None
pipeline = Pipeline({
"name": "pipeline",
"params": [{"key": "value"}],
"targets": [],
"pipeline_type": "some_type"
})
pipeline.template_dictionary = {
"targets": [[{"name": "some_pipeline", "path": "/fake/path"}]]
}
cls.update_deployment_parameters(pipeline)
assert cls.account_ou_names['some_pipeline'] == '/fake/path'
def test_parse_escaped(self):
self.assertEqual(
Pipeline.parse('\\\\\\#\\#Hello\\\\world!\\#\\#'), # means '\\\#\#Hello\\world!\#\#'
Message(tokens=[
PlainText(name='\\##Hello\\world!##') # means '\##Hello\world!##'
])
)
def test_tagSimilarityEncoder(pipeline_config):
pipeline = Pipeline([Filename], [Descriptors], **pipeline_config)
fname = os.path.dirname(__file__) + '/data/Cam_2_20150821161530_884267.jpeg'
outputs = pipeline([fname])
assert Descriptors in outputs
assert len(outputs[Descriptors]) > 20
