def test_streams4(self):
gen = Generate("gen", size=10, out_streams="*")
double = Double("double", in_streams=["even", "odd"])
p = Pipeline(gen | double)
with self.assertRaises(Exception) as context:
p.run()
self.assertTrue('Node %s emits %i items' %
(gen, 1) in str(context.exception))
def test_gen_reverse_stream(self):
gen = Generate("gen", size=10, reverse=True, out_streams="num")
p = Pipeline(NodeGraph(gen))
p.run()
output = get_output()
expected_out = reversed([str(x) for x in range(10)])
self.assertCountEqual(output, expected_out)
def test_streams_odd(self):
gen = EvenOddGenerate("gen", size=20, out_streams=["even", "odd"])
printer = Printer("printer", in_streams="odd")
p = Pipeline(gen | printer)
p.run()
output = get_output()
expected_out = [str([x + 1 for x in range(20)[::2]])]
self.assertCountEqual(output, expected_out)
def test_streams(self):
gen = EvenOddGenerate("gen", size=20, out_streams=["even", "odd"])
printer = Printer("printer", batch_size=Node.BATCH_SIZE_ALL)
p = Pipeline(gen | printer)
p.run()
output = get_output()
expected_out = [str([[x, x + 1] for x in range(20)[::2]])]
self.assertCountEqual(output, expected_out)
def test_printer_batch_stream(self):
gen = Generate("gen", size=10, out_streams="num")
printer = Printer("printer", batch_size=Node.BATCH_SIZE_ALL)
p = Pipeline(gen | printer)
p.run()
output = get_output()
expected_out = [str([x for x in range(10)])]
self.assertCountEqual(output, expected_out)
def test_double(self):
gen = Generate("gen", size=10)
double = Double("double")
p = Pipeline(gen | double)
p.run()
output = get_output()
expected_out = [str(x * 2) for x in range(10)]
self.assertCountEqual(output, expected_out)
def test_printer(self):
gen = Generate("gen", size=10)
printer = Printer("printer", batch_size=1)
p = Pipeline(gen | printer)
p.run()
output = get_output()
expected_out = [str(x) for x in range(10)]
self.assertCountEqual(output, expected_out)
def test_split_multiple_input_streams(self):
gen = EvenOddGenerate("gen", size=20, out_streams=["even", "odd"])
printer = Printer("p1", in_streams=["even", "odd"], batch_size=1)
p = Pipeline(gen | printer, quiet=False)
p.run()
output = get_output()
expected_out = [str(x) for x in range(20)]
self.assertCountEqual(output, expected_out)
def test_streams3(self):
gen = Generate("gen", size=10, out_streams="*")
double = Double("double", out_streams="num")
p = Pipeline(gen | double)
p.run()
output = get_output()
expected_out = [str(x * 2) for x in range(10)]
self.maxDiff = None
self.assertCountEqual(output, expected_out)
def test_double_square_stream(self):
gen = Generate("gen", size=10, out_streams="num")
double = Double("double", out_streams="num", in_streams="num")
square = Square("square", out_streams="num", in_streams="num")
p = Pipeline(gen | double | square)
p.run()
output = get_output()
expected_out = [str((x * 2)**2) for x in range(10)]
self.assertCountEqual(output, expected_out)
def test_double_and_square(self):
gen = Generate("gen", size=10)
double = Double("double")
square = Square("square")
p = Pipeline(gen | [double, square])
p.run()
output = get_output()
expected_out = [str(x * 2)
for x in range(10)] + [str(x**2) for x in range(10)]
self.assertCountEqual(output, expected_out)
def test_streams_complex(self):
gen = EvenOddGenerate("gen", size=20, out_streams=["even", "odd"])
double = Double("double", out_streams="num", in_streams="even")
square = Square("square", out_streams="num", in_streams="odd")
printer1 = Printer("p1",
in_streams="num",
batch_size=Node.BATCH_SIZE_ALL)
printer2 = Printer("p2",
in_streams="num",
batch_size=Node.BATCH_SIZE_ALL)
p = Pipeline(gen | [double | printer1, square | printer2], quiet=False)
p.run()
output = get_output()
expected_out = [str([x + x for x in range(20)[::2]])
] + [str([(x + 1)**2 for x in range(20)[::2]])]
self.assertCountEqual(output, expected_out)
print(data)
class TqdmUpdate(tqdm):
def update(self, done, total_size=None):
if total_size is not None:
self.total = total_size
self.n = done
super().refresh()
if __name__ == '__main__':
gen = Generate("gen", size=100)
double = Double("double")
square = Square("square")
printer1 = Printer("printer1", batch_size=1)
printer2 = Printer("printer2", batch_size=1)
sleeper = Sleep("sleep")
sleeper1 = Sleep("sleep1")
# p = Pipeline(gen | [sleeper, sleeper1], quiet=False, n_threads=50)
# p.run()
# p = Pipeline(gen | double | printer, n_threads=1)
# p.run()
p = Pipeline(gen | [double, sleeper], n_threads=5, quiet=True)
with TqdmUpdate(desc="Progress") as pbar:
# pbar=None
p.run(update_callback=pbar.update)
if __name__ == "__main__":
CurrentPath = os.getcwd()
FramesDirectory = "Frames"
FramesDirectoryPath = os.path.join(CurrentPath, FramesDirectory)
EncodingsFolder = "Encodings"
EncodingsFolderPath = os.path.join(CurrentPath, EncodingsFolder)
if os.path.exists(EncodingsFolderPath):
shutil.rmtree(EncodingsFolderPath, ignore_errors=True)
time.sleep(0.5)
os.makedirs(EncodingsFolderPath)
pipeline = Pipeline(
FramesProvider("Files source", sourcePath=FramesDirectoryPath)
| FaceEncoder("Encode faces") | DatastoreManager(
"Store encoding", encodingsOutputPath=EncodingsFolderPath),
n_threads=4,
quiet=True)
pbar = TqdmUpdate()
pipeline.run(update_callback=pbar.update)
print()
print('[INFO] Encodings extracted')
picklesListCollator = PicklesListCollator(
picklesInputDirectory=EncodingsFolderPath)
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=FutureWarning)
picklesListCollator.GeneratePickle("encodings.pickle")
as well as pass it on directly using pipelineData
@authors: Thom Kirwan-Evans, Jeremy Revell
"""
from pyPiper import Pipeline
from FNC.COVIDPipeline import *
import yaml
# specify the config file to load
with open("./configs/config_pipeline.yml", 'r') as f:
config = yaml.safe_load(f)
# Create the pipelineData object
# This will be passed from node to node
pipelineData = {'config': config, 'data': {}}
# Run the pipeline
pipeline = Pipeline(
Initialise("Initialise", pipelineData=pipelineData) # adds dates to config
| GatherData("Gather") # adds 'data' and 'covar_data'
| Infer("Infer") # generates posterior and saves it
| Summarise("Summary") # simulates and adds 'summaryData'
| MakeGPKG("GPKG") # creates a geopackage output
| InspectData("Inspector") # creates plots for data checks
# | PlotResults("Plotter") # creates plots of results
| PipelineEnd('End') # stops the pipeline
)
pipeline.run()
print('Finished')
def main() -> None:
ip_address, port = sys.argv[1:3]
mon_client = air_client.MonitorClient("monitoring_uds")
audio_input = audio_tools.AudioInput(sample_rate=SAMPLE_RATE)
audio_input.start()
samples = audio_input.seconds_to_samples(WINDOW_SIZE_SEC)
fft_node = nodes.FastFourierTransform(
"fft",
samples=samples,
sample_delta=audio_input.sample_delta,
monitor_client=mon_client)
pipeline = Pipeline(
nodes.AudioGenerator("mic",
audio_input=audio_input,
samples=samples,
monitor_client=mon_client)
| nodes.Hamming("hamming", samples=samples, monitor_client=mon_client)
| fft_node
| nodes.AWeighting("a-weighting",
frequencies=fft_node.fourier_frequencies,
monitor_client=mon_client)
# | nodes.OctaveSubsampler(
# "sampled",
# start_octave=FIRST_OCTAVE,
# samples_per_octave=BEAMS / NUM_OCTAVES,
# num_octaves=NUM_OCTAVES,
# frequencies=fft_node.fourier_frequencies,
# monitor_client=mon_client,
# )
| nodes.ExponentialSubsampler("sampled",
start_frequency=65,
stop_frequency=1046,
samples=18,
frequencies=fft_node.fourier_frequencies,
monitor_client=mon_client)
# | nodes.FoldingNode("folded", samples_per_octave=BEAMS, monitor_client=mon_client)
# | nodes.SumMatrixVertical("sum", monitor_client=mon_client)
# | nodes.MaxMatrixVertical("max", monitor_client=mon_client)
| nodes.Normalizer(
"normalized",
min_threshold=VOLUME_MIN_THRESHOLD,
falloff=VOLUME_FALLOFF,
monitor_client=mon_client,
)
| nodes.Square("square", monitor_client=mon_client)
# | nodes.Logarithm("log", i_0=0.03, monitor_client=mon_client)
# | nodes.Fade("fade", falloff=FADE_FALLOFF, monitor_client=mon_client)
# | nodes.Shift("clip", minimum=0.14)
| nodes.Mirror("mirrored", reverse=False, monitor_client=mon_client)
| nodes.Roll("rolled", shift=16, monitor_client=mon_client)
| nodes.Star(
"ring",
ip_address=ip_address,
port=port,
led_per_beam=LED_PER_BEAM,
beams=BEAMS,
octaves=NUM_OCTAVES,
))
pipeline.run()