def sub2Pdf(path, backup, remove_dir, debug): stage1 = Stage(GetImages, 2) stage2 = Stage(MakePdf, 7) if backup: stage3 = Stage(MakeBackup, 7) stage1.link(stage3) if not os.path.exists(path + '\\BACKUP'): os.makedirs(path + '\\BACKUP') stage1.link(stage2) pipe = Pipeline(stage1) lt_dir = get_subdiretories(path) for folder in lt_dir: pipe.put(folder) pipe.put(None) if debug: print("Converting to pdf :\n%s\n" % ('Backup activated' if backup else 'Backup deactivated')) # This allows to wait until all task are done for res in pipe.results(): try: if debug: print(' Done :=> ' + res) except Exception: pass if remove_dir: delete_dirs(lt_dir) print('\nFinished treating : %s' % get_leaf_from_path(path))
def main(): pipe = Pipeline(OrderedStage(yes, disable_result=True)) for number in range(10): pipe.put(number) pipe.put(None) for result in pipe.results(): print(result)
def evaluate_workflow(stages,inp): # if __name__ == '__main__': # freeze_support() pipe = Pipeline(stages[0]) pipe.put(inp) pipe.put(None) return pipe.results()
def main(): stage1 = Stage(Adder, 1, number=5) pipe = Pipeline(stage1) for number in range(10): pipe.put(number) pipe.put(None) for result in pipe.results(): print(result)
def main(): stage1 = UnorderedStage(increment, 3) stage2 = OrderedStage(double, 3) stage1.link(stage2) pipe = Pipeline(stage1) for number in range(10): pipe.put(number) pipe.put(None) for result in pipe.results(): print(result)
def main(): stage = UnorderedStage(increment) pipe = Pipeline(stage) # for task in range(sys.maxint if sys.version_info.major <= 2 else sys.maxsize): for task in range(10000): pipe.put(task) pipe.put(None) for result in pipe.results(): print(result)
def main(): stage = Stage(Yes, 4, disable_result=True) pipe = Pipeline(stage) for number in range(10): pipe.put(number) pipe.put(None) count = 0 for _ in pipe.results(): count += 1 print(count)
class ExecutionCache(): def __init__(self, function, jobs, SchedulerClass): self._results = dict() self._targets = set() self._update_lock = Lock() self._runner = Pipeline(UnorderedStage(function, jobs)) def fetch(): for target, result in self._runner.results(): if target in self._targets: self._results[target.host] = result else: print(f'dropping obsolete result for {target}') self._fetcher = Thread(target=fetch) self._fetcher.start() self.scheduler = SchedulerClass(self._runner) def teardown(self): self.scheduler.cancel_all() self._runner.put(None) self._fetcher.join() def update_targets(self, targets): with self._update_lock: new_targets = set(targets) old_targets = self._targets new_hosts = {target.host for target in targets} old_hosts = {target.host for target in self._targets} removed_hosts = old_hosts - new_hosts self._targets = new_targets for target in old_targets - new_targets: self.scheduler.remove(target) if target.host in removed_hosts: try: del self._results[target.host] except KeyError: pass for target in new_targets - old_targets: self.scheduler.add(target) def __getitem__(self, key): return self._results[key]
def extract_feat_from_FCL(): input_shape = (224, 224, 3) model = VGG16(weights='imagenet', input_shape=(input_shape[0], input_shape[1], input_shape[2]), pooling='max', include_top=True) t1 = time.time() layer_name = "fc2" intermediate_layer_model = Model( inputs=model.input, outputs=model.get_layer(layer_name).output) list = get_imlist("img_cut") stage1 = OrderedStage(send_batch) stage2 = OrderedStage(create_bag_of_window) stage1.link(stage2) pipe = Pipeline(stage1) batch_size = 1 total_batch = math.ceil(len(list) / batch_size) for p in range(total_batch): v = list[p * batch_size:(p + 1) * batch_size] print("Batch number %s" % p) pipe.put(v) pipe.put(None) for result in pipe.results(): t0 = time.time() print("Predicting...") feature_tensor = intermediate_layer_model.predict( np.vstack((r for r in result))) t1 = time.time() print("time to predict : %ss" % (t1 - t0)) # gc.collect() print(feature_tensor.shape) del result # feature_tensor = intermediate_layer_model.predict_generator(generator=generator_, steps=1, max_queue_size=1) t2 = time.time() print("Total time to predict: " + str(t2 - t1)) print(feature_tensor.shape) return feature_tensor
def main(): config = load_config() clean_data = True if "True" in config["CLEAN_DATA"] else False # create stages stage_setup = Stage(SetupDatabase, 1) stage_load_ex = Stage(LoadExchange, 1) save_exchange = Stage(SaveGetExchange) get_symbols = Stage(GetSymbols) save_symbols = Stage(SaveSymbol, 1) save_curency_pair = Stage(SaveCurrencyPair) trade_dates = Stage(GetTradeRanges, 1) add_price = Stage(AddPriceData, 1) # link stages stage_setup.link(stage_load_ex) stage_load_ex.link(save_exchange) save_exchange.link(get_symbols) #get_symbols.link(save_symbols) get_symbols.link(save_curency_pair) save_curency_pair.link(trade_dates) trade_dates.link(add_price) # setup pipeline pipe = Pipeline(stage_setup) pipe.put(clean_data) pipe.put(None) insert_count = [] for result in pipe.results(): print 'pipe result %s' % (result) insert_count.append(result) print "inserted %s records" % (sum(insert_count)) close1 = CloseSymbolRepoSession() close1.doTask("") close2 = CloseCurrencyPairRepoSession() close2.doTask("") close3 = CloseDateRepoSession() close3.doTask("") close4 = ClosePriceRepoSession() close4.doTask("")
from mpipe import OrderedStage, Pipeline def increment(value): return value + 1 def double(value): return value * 2 stage1 = OrderedStage(increment) stage2 = OrderedStage(double) stage1.link(stage2) pipe = Pipeline(stage1) for number in range(10): pipe.put(number) pipe.put(None) for result in pipe.results(): print(result)
from mpipe import OrderedStage as OStage, Pipeline def magnify(value): return value*10 p1 = Pipeline( OStage(magnify).link( OStage(magnify).link( OStage(magnify).link( OStage(magnify) ) ) ) ) for val in list(range(10)) + [None]: p1.put(val) for result in p1.results(): print(result)
from mpipe import OrderedStage as OStage, Pipeline def magnify(value): return value * 10 p1 = Pipeline( OStage(magnify).link( OStage(magnify).link(OStage(magnify).link(OStage(magnify))))) for val in list(range(10)) + [None]: p1.put(val) for result in p1.results(): print(result)