def main(): stage1 = Stage(Accumulator) stage2 = OrderedStage(echo, 50) stage1.link(stage2) pipe = Pipeline(stage1) size = 1000 prices = np.linspace(0, np.pi * 10, size) prices = np.sin(prices) + 1 for price in prices: pipe.put(price) pipe.put(None)
def sub2Pdf(path, backup, remove_dir, debug): stage1 = Stage(GetImages, 2) stage2 = Stage(MakePdf, 7) if backup: stage3 = Stage(MakeBackup, 7) stage1.link(stage3) if not os.path.exists(path + '\\BACKUP'): os.makedirs(path + '\\BACKUP') stage1.link(stage2) pipe = Pipeline(stage1) lt_dir = get_subdiretories(path) for folder in lt_dir: pipe.put(folder) pipe.put(None) if debug: print("Converting to pdf :\n%s\n" % ('Backup activated' if backup else 'Backup deactivated')) # This allows to wait until all task are done for res in pipe.results(): try: if debug: print(' Done :=> ' + res) except Exception: pass if remove_dir: delete_dirs(lt_dir) print('\nFinished treating : %s' % get_leaf_from_path(path))
def main(): stage = Stage(Echo, do_stop_task=True) pipe = Pipeline(stage) for number in range(10): pipe.put(number) pipe.put(None)
def main(): stage1 = Stage(Adder, 1, number=5) pipe = Pipeline(stage1) for number in range(10): pipe.put(number) pipe.put(None) for result in pipe.results(): print(result)
def main(): s1 = Stage(Echo, do_stop_task=True) s2 = FilterStage( (s1, ), max_tasks=999, do_stop_task=True, ) pipe = Pipeline(s2) for number in range(10): pipe.put(number) pipe.put(None)
def main(): stage = Stage(Yes, 4, disable_result=True) pipe = Pipeline(stage) for number in range(10): pipe.put(number) pipe.put(None) count = 0 for _ in pipe.results(): count += 1 print(count)
def prepare_stages(tokens): stages = dict() m =0 for t in tokens: stages[m]=Stage(eval('wf_library.{g}'.format(g=t))) m = m+1 for i in range(0,len(stages)-1): stages[i].link(stages[i+1]) return stages
from mpipe import Stage, OrderedWorker, FilterStage, Pipeline class Echo(OrderedWorker): def doTask(self, value): print(value) s1 = Stage(Echo, do_stop_task=True) s2 = FilterStage( (s1,), max_tasks=999, do_stop_task=True, ) pipe = Pipeline(s2) for number in range(10): pipe.put(number) pipe.put(None)
from mpipe import UnorderedWorker, Stage, Pipeline class Yes(UnorderedWorker): def doTask(self, value): return value stage = Stage(Yes, 4, disable_result=True) pipe = Pipeline(stage) for number in range(10): pipe.put(number) pipe.put(None) count = 0 for result in pipe.results(): count += 1 print(count)
from mpipe import OrderedWorker, Stage, OrderedStage, Pipeline last10 = deque() junk = 'http://ws.cdyne.com/delayedstockquote/delayedstockquote.asmx/GetQuote?StockSymbol=fac&LicenseKey=0' j = 'http://www.google.com/ig/api?stock=AAPL' class Accumulator(OrderedWorker): def doTask(self, price): if last10: if price < min(last10): self.putResult(price) last10.append(price) if len(last10) > 10: last10.popleft() def echo(value): print('value = {0}'.format(value)) stage1 = Stage(Accumulator) stage2 = OrderedStage(echo, 50) stage1.link(stage2) pipe = Pipeline(stage1) SIZE = 1000 prices = np.linspace(0, np.pi*10, SIZE) prices = np.sin(prices) + 1 for price in prices: pipe.put(price) pipe.put(None)
from mpipe import OrderedWorker, Stage, Pipeline class Adder(OrderedWorker): def __init__(self, number): self.number = number def doTask(self, value): return value + self.number stage1 = Stage(Adder, 1, number=5) pipe = Pipeline(stage1) for number in range(10): pipe.put(number) pipe.put(None) for result in pipe.results(): print(result)
def main(): config = load_config() clean_data = True if "True" in config["CLEAN_DATA"] else False # create stages stage_setup = Stage(SetupDatabase, 1) stage_load_ex = Stage(LoadExchange, 1) save_exchange = Stage(SaveGetExchange) get_symbols = Stage(GetSymbols) save_symbols = Stage(SaveSymbol, 1) save_curency_pair = Stage(SaveCurrencyPair) trade_dates = Stage(GetTradeRanges, 1) add_price = Stage(AddPriceData, 1) # link stages stage_setup.link(stage_load_ex) stage_load_ex.link(save_exchange) save_exchange.link(get_symbols) #get_symbols.link(save_symbols) get_symbols.link(save_curency_pair) save_curency_pair.link(trade_dates) trade_dates.link(add_price) # setup pipeline pipe = Pipeline(stage_setup) pipe.put(clean_data) pipe.put(None) insert_count = [] for result in pipe.results(): print 'pipe result %s' % (result) insert_count.append(result) print "inserted %s records" % (sum(insert_count)) close1 = CloseSymbolRepoSession() close1.doTask("") close2 = CloseCurrencyPairRepoSession() close2.doTask("") close3 = CloseDateRepoSession() close3.doTask("") close4 = ClosePriceRepoSession() close4.doTask("")
from mpipe import OrderedWorker, Stage, Pipeline class Incrementor(OrderedWorker): def doTask(self, value): return value + 1 class Doubler(OrderedWorker): def doTask(self, value): return value * 2 class Printer(OrderedWorker): def doTask(self, value): print(value) stage1 = Stage(Incrementor) stage2 = Stage(Doubler) stage3 = Stage(Printer) stage4 = Stage(Printer) stage1.link(stage2) stage1.link(stage3) stage2.link(stage4) pipe = Pipeline(stage1) for number in range(10): pipe.put(number) pipe.put(None)
class Incrementor(OrderedWorker): def doTask(self, value): return value + 1 class Doubler(OrderedWorker): def doTask(self, value): return value * 2 class Printer(OrderedWorker): def doTask(self, value): print(value) stage1 = Stage(Incrementor) stage2 = Stage(Doubler) stage3 = Stage(Printer) stage4 = Stage(Printer) stage1.link(stage2) stage1.link(stage3) stage2.link(stage4) pipe = Pipeline(stage1) for number in range(10): pipe.put(number) pipe.put(None)