def main():
pipe = Pipeline(UnorderedStage(increment))
pipe2 = Pipeline(UnorderedStage(Pull(pipe)))
pipe2.put(True)
# for task in range(sys.maxint):
for task in range(10000):
pipe.put(task)
pipe.put(None)
pipe2.put(None)
def main():
stage1 = UnorderedStage(increment, 3)
stage2 = OrderedStage(double, 3)
stage1.link(stage2)
pipe = Pipeline(stage1)
for number in range(10):
pipe.put(number)
pipe.put(None)
for result in pipe.results():
print(result)
def main():
stage = UnorderedStage(increment)
pipe = Pipeline(stage)
# for task in range(sys.maxint if sys.version_info.major <= 2 else sys.maxsize):
for task in range(10000):
pipe.put(task)
pipe.put(None)
for result in pipe.results():
print(result)
def __init__(self, function, jobs, SchedulerClass):
self._results = dict()
self._targets = set()
self._update_lock = Lock()
self._runner = Pipeline(UnorderedStage(function, jobs))
def fetch():
for target, result in self._runner.results():
if target in self._targets:
self._results[target.host] = result
else:
print(f'dropping obsolete result for {target}')
self._fetcher = Thread(target=fetch)
self._fetcher.start()
self.scheduler = SchedulerClass(self._runner)
import sys
from mpipe import UnorderedStage, Pipeline
def increment(value):
return value + 1
stage = UnorderedStage(increment)
pipe = Pipeline(stage)
def pull(value):
for result in pipe.results():
print(result)
pipe2 = Pipeline(UnorderedStage(pull))
pipe2.put(True)
for task in xrange(sys.maxint):
pipe.put(task)
pipe.put(None)
pipe2.put(None)
import sys
from mpipe import UnorderedStage, Pipeline
def increment(value):
return value + 1
stage = UnorderedStage(increment)
pipe = Pipeline(stage)
for task in xrange(sys.maxint):
pipe.put(task)
pipe.put(None)
for result in pipe.results():
print(result)
from mpipe import OrderedStage, UnorderedStage, Pipeline
def increment(value):
return value + 1
def double(value):
return value * 2
stage1 = OrderedStage(increment, 3)
stage2 = UnorderedStage(double, 3)
stage1.link(stage2)
pipe = Pipeline(stage1)
for number in range(10):
pipe.put(number)
pipe.put(None)
for result in pipe.results():
print(result)
from mpipe import OrderedStage, UnorderedStage, Pipeline
def increment(value):
return value + 1
def double(value):
return value * 2
stage1 = UnorderedStage(increment, 3)
stage2 = OrderedStage(double, 3)
stage1.link(stage2)
pipe = Pipeline(stage1)
for number in range(10):
pipe.put(number)
pipe.put(None)
for result in pipe.results():
print(result)
stderr=PIPE,
cwd=directory)
stdout, stderr = process.communicate()
console("rm -r " + directory)
if len(stderr) > 0 or process.returncode != 0:
print("MODELLER ERROR:", stderr)
else:
print("new model:", target, template)
return ("ok", time.time() - start)
return ("bad", 0)
processes_number = 48
pipe = Pipeline(UnorderedStage(run_modeller, processes_number))
waiting_candidates = 0
processed_candidates = 0
sum_time = 0
def get_pipe():
global pipe
global waiting_candidates
global sum_time
global processed_candidates
res = pipe.get()
waiting_candidates -= 1
if res[0] == "ok":
processed_candidates += 1
errorfile.write(bundle[3] + '\n')
errorfile.flush()
return
jso = bundle[2]
data = json.dumps(jso)
outfile.write(data + '\n')
outfile.flush()
if __name__ == '__main__':
infile = infile[1:]
count = 0
stage1 = UnorderedStage(processpage, 6)
stage2 = UnorderedStage(writetofile, 1)
stage1.link(stage2)
pipe = Pipeline(stage1)
for line in infile:
line = line.strip()
count += 1
bundle = []
bundle.append(str(count))
bundle.append(
'https://ntrl.ntis.gov/NTRL/dashboard/searchResults/titleDetail/' +
line + '.xhtml')
pipe.put(bundle)
from mpipe import UnorderedStage, Pipeline
def increment(value):
return value + 1
def double(value):
return value * 2
stage1 = UnorderedStage(increment, 2)
stage2 = UnorderedStage(double, 2)
stage1.link(stage2)
pipe = Pipeline(stage1)
for number in range(10):
pipe.put(number)
pipe.put(None)
for result in pipe.results():
print(result)
for jsonstring in extratedfile:
linecount += 1
result = json.load(createtable.readablestring(jsonstring))
uniquerowkey = result['id']
query = queryfy.create_query_from_json(createtable.readablestring(jsonstring))
table.put(uniquerowkey, query)
except Exception:
error = traceback.format_exc()
bundle.error=error
logfile.write('LOGS:'+bundle.logs+'\n'+'ERRORS:'+bundle.error+'\n'+'TIME TAKEN: '+str(datetime.datetime.now()-bundle.starttime))
logfile.close()
if __name__ == '__main__':
stage1 = UnorderedStage(fetch, 4)
stage2 = UnorderedStage(extractandclean, 4)
stage3 = UnorderedStage(updatetable, 4)
stage1.link(stage2)
stage2.link(stage3)
pipe = Pipeline(stage1)
for number in range(0,8):
pipe.put(number)
pipe.put(None)