def main():
stage1 = Stage(Accumulator)
stage2 = OrderedStage(echo, 50)
stage1.link(stage2)
pipe = Pipeline(stage1)
size = 1000
prices = np.linspace(0, np.pi * 10, size)
prices = np.sin(prices) + 1
for price in prices:
pipe.put(price)
pipe.put(None)
def sub2Pdf(path, backup, remove_dir, debug):
stage1 = Stage(GetImages, 2)
stage2 = Stage(MakePdf, 7)
if backup:
stage3 = Stage(MakeBackup, 7)
stage1.link(stage3)
if not os.path.exists(path + '\\BACKUP'):
os.makedirs(path + '\\BACKUP')
stage1.link(stage2)
pipe = Pipeline(stage1)
lt_dir = get_subdiretories(path)
for folder in lt_dir:
pipe.put(folder)
pipe.put(None)
if debug:
print("Converting to pdf :\n%s\n" %
('Backup activated' if backup else 'Backup deactivated'))
# This allows to wait until all task are done
for res in pipe.results():
try:
if debug:
print(' Done :=> ' + res)
except Exception:
pass
if remove_dir:
delete_dirs(lt_dir)
print('\nFinished treating : %s' % get_leaf_from_path(path))
def main():
stage = Stage(Echo, do_stop_task=True)
pipe = Pipeline(stage)
for number in range(10):
pipe.put(number)
pipe.put(None)
def main():
stage1 = Stage(Adder, 1, number=5)
pipe = Pipeline(stage1)
for number in range(10):
pipe.put(number)
pipe.put(None)
for result in pipe.results():
print(result)
def main():
s1 = Stage(Echo, do_stop_task=True)
s2 = FilterStage(
(s1, ),
max_tasks=999,
do_stop_task=True,
)
pipe = Pipeline(s2)
for number in range(10):
pipe.put(number)
pipe.put(None)
def main():
stage = Stage(Yes, 4, disable_result=True)
pipe = Pipeline(stage)
for number in range(10):
pipe.put(number)
pipe.put(None)
count = 0
for _ in pipe.results():
count += 1
print(count)
def prepare_stages(tokens):
stages = dict()
m =0
for t in tokens:
stages[m]=Stage(eval('wf_library.{g}'.format(g=t)))
m = m+1
for i in range(0,len(stages)-1):
stages[i].link(stages[i+1])
return stages
from mpipe import Stage, OrderedWorker, FilterStage, Pipeline
class Echo(OrderedWorker):
def doTask(self, value):
print(value)
s1 = Stage(Echo, do_stop_task=True)
s2 = FilterStage(
(s1,),
max_tasks=999,
do_stop_task=True,
)
pipe = Pipeline(s2)
for number in range(10):
pipe.put(number)
pipe.put(None)
from mpipe import UnorderedWorker, Stage, Pipeline
class Yes(UnorderedWorker):
def doTask(self, value):
return value
stage = Stage(Yes, 4, disable_result=True)
pipe = Pipeline(stage)
for number in range(10):
pipe.put(number)
pipe.put(None)
count = 0
for result in pipe.results():
count += 1
print(count)
from mpipe import OrderedWorker, Stage, OrderedStage, Pipeline
last10 = deque()
junk = 'http://ws.cdyne.com/delayedstockquote/delayedstockquote.asmx/GetQuote?StockSymbol=fac&LicenseKey=0'
j = 'http://www.google.com/ig/api?stock=AAPL'
class Accumulator(OrderedWorker):
def doTask(self, price):
if last10:
if price < min(last10):
self.putResult(price)
last10.append(price)
if len(last10) > 10:
last10.popleft()
def echo(value):
print('value = {0}'.format(value))
stage1 = Stage(Accumulator)
stage2 = OrderedStage(echo, 50)
stage1.link(stage2)
pipe = Pipeline(stage1)
SIZE = 1000
prices = np.linspace(0, np.pi*10, SIZE)
prices = np.sin(prices) + 1
for price in prices:
pipe.put(price)
pipe.put(None)
from mpipe import OrderedWorker, Stage, Pipeline
class Adder(OrderedWorker):
def __init__(self, number):
self.number = number
def doTask(self, value):
return value + self.number
stage1 = Stage(Adder, 1, number=5)
pipe = Pipeline(stage1)
for number in range(10):
pipe.put(number)
pipe.put(None)
for result in pipe.results():
print(result)
def main():
config = load_config()
clean_data = True if "True" in config["CLEAN_DATA"] else False
# create stages
stage_setup = Stage(SetupDatabase, 1)
stage_load_ex = Stage(LoadExchange, 1)
save_exchange = Stage(SaveGetExchange)
get_symbols = Stage(GetSymbols)
save_symbols = Stage(SaveSymbol, 1)
save_curency_pair = Stage(SaveCurrencyPair)
trade_dates = Stage(GetTradeRanges, 1)
add_price = Stage(AddPriceData, 1)
# link stages
stage_setup.link(stage_load_ex)
stage_load_ex.link(save_exchange)
save_exchange.link(get_symbols)
#get_symbols.link(save_symbols)
get_symbols.link(save_curency_pair)
save_curency_pair.link(trade_dates)
trade_dates.link(add_price)
# setup pipeline
pipe = Pipeline(stage_setup)
pipe.put(clean_data)
pipe.put(None)
insert_count = []
for result in pipe.results():
print 'pipe result %s' % (result)
insert_count.append(result)
print "inserted %s records" % (sum(insert_count))
close1 = CloseSymbolRepoSession()
close1.doTask("")
close2 = CloseCurrencyPairRepoSession()
close2.doTask("")
close3 = CloseDateRepoSession()
close3.doTask("")
close4 = ClosePriceRepoSession()
close4.doTask("")
from mpipe import OrderedWorker, Stage, Pipeline
class Incrementor(OrderedWorker):
def doTask(self, value):
return value + 1
class Doubler(OrderedWorker):
def doTask(self, value):
return value * 2
class Printer(OrderedWorker):
def doTask(self, value):
print(value)
stage1 = Stage(Incrementor)
stage2 = Stage(Doubler)
stage3 = Stage(Printer)
stage4 = Stage(Printer)
stage1.link(stage2)
stage1.link(stage3)
stage2.link(stage4)
pipe = Pipeline(stage1)
for number in range(10):
pipe.put(number)
pipe.put(None)
class Incrementor(OrderedWorker):
def doTask(self, value):
return value + 1
class Doubler(OrderedWorker):
def doTask(self, value):
return value * 2
class Printer(OrderedWorker):
def doTask(self, value):
print(value)
stage1 = Stage(Incrementor)
stage2 = Stage(Doubler)
stage3 = Stage(Printer)
stage4 = Stage(Printer)
stage1.link(stage2)
stage1.link(stage3)
stage2.link(stage4)
pipe = Pipeline(stage1)
for number in range(10):
pipe.put(number)
pipe.put(None)