Example #1
0
def main():
    stage1 = Stage(Accumulator)
    stage2 = OrderedStage(echo, 50)
    stage1.link(stage2)
    pipe = Pipeline(stage1)

    size = 1000
    prices = np.linspace(0, np.pi * 10, size)
    prices = np.sin(prices) + 1
    for price in prices:
        pipe.put(price)

    pipe.put(None)
Example #2
0
def sub2Pdf(path, backup, remove_dir, debug):
    stage1 = Stage(GetImages, 2)
    stage2 = Stage(MakePdf, 7)

    if backup:
        stage3 = Stage(MakeBackup, 7)
        stage1.link(stage3)
        if not os.path.exists(path + '\\BACKUP'):
            os.makedirs(path + '\\BACKUP')

    stage1.link(stage2)

    pipe = Pipeline(stage1)
    lt_dir = get_subdiretories(path)
    for folder in lt_dir:
        pipe.put(folder)

    pipe.put(None)

    if debug:
        print("Converting to pdf :\n%s\n" %
              ('Backup activated' if backup else 'Backup deactivated'))

    # This allows to wait until all task are done
    for res in pipe.results():
        try:
            if debug:
                print('    Done :=> ' + res)
        except Exception:
            pass

    if remove_dir:
        delete_dirs(lt_dir)

    print('\nFinished treating : %s' % get_leaf_from_path(path))
Example #3
0
def main():
    stage = Stage(Echo, do_stop_task=True)
    pipe = Pipeline(stage)

    for number in range(10):
        pipe.put(number)
    pipe.put(None)
Example #4
0
def main():
    stage1 = Stage(Adder, 1, number=5)
    pipe = Pipeline(stage1)

    for number in range(10):
        pipe.put(number)

    pipe.put(None)

    for result in pipe.results():
        print(result)
Example #5
0
def main():
    s1 = Stage(Echo, do_stop_task=True)
    s2 = FilterStage(
        (s1, ),
        max_tasks=999,
        do_stop_task=True,
    )
    pipe = Pipeline(s2)

    for number in range(10):
        pipe.put(number)
    pipe.put(None)
Example #6
0
def main():
    stage = Stage(Yes, 4, disable_result=True)
    pipe = Pipeline(stage)

    for number in range(10):
        pipe.put(number)
    pipe.put(None)

    count = 0
    for _ in pipe.results():
        count += 1

    print(count)
def prepare_stages(tokens):
    stages = dict()
    m =0

    for t in tokens:

       stages[m]=Stage(eval('wf_library.{g}'.format(g=t)))
       m = m+1



    for i in range(0,len(stages)-1):
       stages[i].link(stages[i+1])


    return stages
Example #8
0
from mpipe import Stage, OrderedWorker, FilterStage, Pipeline

class Echo(OrderedWorker):
    def doTask(self, value):
        print(value)

s1 = Stage(Echo, do_stop_task=True)
s2 = FilterStage(
    (s1,), 
    max_tasks=999,
    do_stop_task=True,
    )
pipe = Pipeline(s2)
for number in range(10):
    pipe.put(number)

pipe.put(None)
Example #9
0
from mpipe import UnorderedWorker, Stage, Pipeline


class Yes(UnorderedWorker):
    def doTask(self, value):
        return value


stage = Stage(Yes, 4, disable_result=True)
pipe = Pipeline(stage)

for number in range(10):
    pipe.put(number)

pipe.put(None)

count = 0
for result in pipe.results():
    count += 1

print(count)
Example #10
0
from mpipe import OrderedWorker, Stage, OrderedStage, Pipeline

last10 = deque()
junk = 'http://ws.cdyne.com/delayedstockquote/delayedstockquote.asmx/GetQuote?StockSymbol=fac&LicenseKey=0'
j = 'http://www.google.com/ig/api?stock=AAPL'

class Accumulator(OrderedWorker):
    def doTask(self, price):
        if last10:
            if price < min(last10):
                self.putResult(price)
        last10.append(price)
        if len(last10) > 10:
            last10.popleft()

def echo(value):
    print('value = {0}'.format(value))

stage1 = Stage(Accumulator)
stage2 = OrderedStage(echo, 50)
stage1.link(stage2)
pipe = Pipeline(stage1)

SIZE = 1000
prices = np.linspace(0, np.pi*10, SIZE)
prices = np.sin(prices) + 1
for price in prices:
    pipe.put(price)

pipe.put(None)
Example #11
0
from mpipe import OrderedWorker, Stage, Pipeline


class Adder(OrderedWorker):
    def __init__(self, number):
        self.number = number

    def doTask(self, value):
        return value + self.number


stage1 = Stage(Adder, 1, number=5)
pipe = Pipeline(stage1)

for number in range(10):
    pipe.put(number)

pipe.put(None)

for result in pipe.results():
    print(result)
Example #12
0
def main():
    config = load_config()
    clean_data = True if "True" in config["CLEAN_DATA"] else False

    # create stages
    stage_setup = Stage(SetupDatabase, 1)
    stage_load_ex = Stage(LoadExchange, 1)
    save_exchange = Stage(SaveGetExchange)
    get_symbols = Stage(GetSymbols)
    save_symbols = Stage(SaveSymbol, 1)
    save_curency_pair = Stage(SaveCurrencyPair)
    trade_dates = Stage(GetTradeRanges, 1)
    add_price = Stage(AddPriceData, 1)
    # link stages
    stage_setup.link(stage_load_ex)
    stage_load_ex.link(save_exchange)
    save_exchange.link(get_symbols)
    #get_symbols.link(save_symbols)
    get_symbols.link(save_curency_pair)
    save_curency_pair.link(trade_dates)
    trade_dates.link(add_price)
    # setup pipeline
    pipe = Pipeline(stage_setup)
    pipe.put(clean_data)
    pipe.put(None)
    insert_count = []
    for result in pipe.results():
        print 'pipe result %s' % (result)
        insert_count.append(result)
    print "inserted %s records" % (sum(insert_count))
    close1 = CloseSymbolRepoSession()
    close1.doTask("")
    close2 = CloseCurrencyPairRepoSession()
    close2.doTask("")
    close3 = CloseDateRepoSession()
    close3.doTask("")
    close4 = ClosePriceRepoSession()
    close4.doTask("")
Example #13
0
from mpipe import OrderedWorker, Stage, Pipeline

class Incrementor(OrderedWorker):
    def doTask(self, value):
        return value + 1

class Doubler(OrderedWorker):
    def doTask(self, value):
        return value * 2

class Printer(OrderedWorker):
    def doTask(self, value):
        print(value)

stage1 = Stage(Incrementor)
stage2 = Stage(Doubler)
stage3 = Stage(Printer)
stage4 = Stage(Printer)

stage1.link(stage2)
stage1.link(stage3)
stage2.link(stage4)

pipe = Pipeline(stage1)

for number in range(10):
    pipe.put(number)

pipe.put(None)
Example #14
0
File: fork.py Project: todo/mpipe
class Incrementor(OrderedWorker):
    def doTask(self, value):
        return value + 1


class Doubler(OrderedWorker):
    def doTask(self, value):
        return value * 2


class Printer(OrderedWorker):
    def doTask(self, value):
        print(value)


stage1 = Stage(Incrementor)
stage2 = Stage(Doubler)
stage3 = Stage(Printer)
stage4 = Stage(Printer)

stage1.link(stage2)
stage1.link(stage3)
stage2.link(stage4)

pipe = Pipeline(stage1)

for number in range(10):
    pipe.put(number)

pipe.put(None)