示例#1
0
文件: drano.py 项目: volks73/mpipe
def main():
    pipe = Pipeline(UnorderedStage(increment))
    pipe2 = Pipeline(UnorderedStage(Pull(pipe)))
    pipe2.put(True)

    # for task in range(sys.maxint):
    for task in range(10000):
        pipe.put(task)
    pipe.put(None)

    pipe2.put(None)
示例#2
0
文件: link_uo.py 项目: volks73/mpipe
def main():
    stage1 = UnorderedStage(increment, 3)
    stage2 = OrderedStage(double, 3)
    stage1.link(stage2)
    pipe = Pipeline(stage1)

    for number in range(10):
        pipe.put(number)
    pipe.put(None)

    for result in pipe.results():
        print(result)
示例#3
0
文件: clog.py 项目: volks73/mpipe
def main():
    stage = UnorderedStage(increment)
    pipe = Pipeline(stage)

    # for task in range(sys.maxint if sys.version_info.major <= 2 else sys.maxsize):
    for task in range(10000):
        pipe.put(task)

    pipe.put(None)

    for result in pipe.results():
        print(result)
示例#4
0
    def __init__(self, function, jobs, SchedulerClass):
        self._results = dict()
        self._targets = set()
        self._update_lock = Lock()

        self._runner = Pipeline(UnorderedStage(function, jobs))

        def fetch():
            for target, result in self._runner.results():
                if target in self._targets:
                    self._results[target.host] = result
                else:
                    print(f'dropping obsolete result for {target}')

        self._fetcher = Thread(target=fetch)
        self._fetcher.start()
        self.scheduler = SchedulerClass(self._runner)
示例#5
0
文件: drano.py 项目: yijxiang/mpipe
import sys
from mpipe import UnorderedStage, Pipeline


def increment(value):
    return value + 1


stage = UnorderedStage(increment)
pipe = Pipeline(stage)


def pull(value):
    for result in pipe.results():
        print(result)


pipe2 = Pipeline(UnorderedStage(pull))
pipe2.put(True)

for task in xrange(sys.maxint):
    pipe.put(task)

pipe.put(None)
pipe2.put(None)
示例#6
0
import sys
from mpipe import UnorderedStage, Pipeline


def increment(value):
    return value + 1


stage = UnorderedStage(increment)
pipe = Pipeline(stage)

for task in xrange(sys.maxint):
    pipe.put(task)

pipe.put(None)

for result in pipe.results():
    print(result)
示例#7
0
文件: link_ou.py 项目: yijxiang/mpipe
from mpipe import OrderedStage, UnorderedStage, Pipeline


def increment(value):
    return value + 1


def double(value):
    return value * 2


stage1 = OrderedStage(increment, 3)
stage2 = UnorderedStage(double, 3)
stage1.link(stage2)
pipe = Pipeline(stage1)

for number in range(10):
    pipe.put(number)
pipe.put(None)

for result in pipe.results():
    print(result)
示例#8
0
from mpipe import OrderedStage, UnorderedStage, Pipeline


def increment(value):
    return value + 1


def double(value):
    return value * 2


stage1 = UnorderedStage(increment, 3)
stage2 = OrderedStage(double, 3)
stage1.link(stage2)
pipe = Pipeline(stage1)

for number in range(10):
    pipe.put(number)
pipe.put(None)

for result in pipe.results():
    print(result)
            stderr=PIPE,
            cwd=directory)
        stdout, stderr = process.communicate()

        console("rm -r " + directory)

        if len(stderr) > 0 or process.returncode != 0:
            print("MODELLER ERROR:", stderr)
        else:
            print("new model:", target, template)
            return ("ok", time.time() - start)
    return ("bad", 0)


processes_number = 48
pipe = Pipeline(UnorderedStage(run_modeller, processes_number))

waiting_candidates = 0
processed_candidates = 0
sum_time = 0


def get_pipe():
    global pipe
    global waiting_candidates
    global sum_time
    global processed_candidates
    res = pipe.get()
    waiting_candidates -= 1
    if res[0] == "ok":
        processed_candidates += 1
        errorfile.write(bundle[3] + '\n')
        errorfile.flush()
        return

    jso = bundle[2]
    data = json.dumps(jso)
    outfile.write(data + '\n')
    outfile.flush()


if __name__ == '__main__':

    infile = infile[1:]
    count = 0

    stage1 = UnorderedStage(processpage, 6)
    stage2 = UnorderedStage(writetofile, 1)
    stage1.link(stage2)
    pipe = Pipeline(stage1)

    for line in infile:
        line = line.strip()
        count += 1
        bundle = []
        bundle.append(str(count))
        bundle.append(
            'https://ntrl.ntis.gov/NTRL/dashboard/searchResults/titleDetail/' +
            line + '.xhtml')

        pipe.put(bundle)
示例#11
0
from mpipe import UnorderedStage, Pipeline


def increment(value):
    return value + 1


def double(value):
    return value * 2


stage1 = UnorderedStage(increment, 2)
stage2 = UnorderedStage(double, 2)
stage1.link(stage2)
pipe = Pipeline(stage1)

for number in range(10):
    pipe.put(number)

pipe.put(None)

for result in pipe.results():
    print(result)
示例#12
0
            for jsonstring in extratedfile:
                linecount += 1
                result = json.load(createtable.readablestring(jsonstring))
                uniquerowkey = result['id']
                query = queryfy.create_query_from_json(createtable.readablestring(jsonstring))
                table.put(uniquerowkey, query)

        except Exception:
            error = traceback.format_exc()
            bundle.error=error

    logfile.write('LOGS:'+bundle.logs+'\n'+'ERRORS:'+bundle.error+'\n'+'TIME TAKEN: '+str(datetime.datetime.now()-bundle.starttime))
    logfile.close()


if __name__ == '__main__':

    stage1 = UnorderedStage(fetch, 4)
    stage2 = UnorderedStage(extractandclean, 4)
    stage3 = UnorderedStage(updatetable, 4)
    stage1.link(stage2)
    stage2.link(stage3)
    pipe = Pipeline(stage1)


    for number in range(0,8):
        pipe.put(number)

    pipe.put(None)