def doPageRank(self, ranks): self.totalResult = None self.single_ResultList = [] self.partitionFile(self.current_WorkingOn_FileName) self.setPartionsToWorker(self.partitions) # print 'self.partitions: %s'% self.partitions output = StringIO.StringIO() pickler = cloudpickle.CloudPickler(output) pickler.dump(ranks) rankSent = output.getvalue() for worker in self.workers: # print 'call next worker start work' try: c = zerorpc.Client(timeout=9999999) c.connect('tcp://' + worker['ip'] + ':' + worker['port']) print colored('[Master:]', 'white'), colored( 'call worker: %s start working', 'red') % worker['port'] c.doPageRank(rankSent, async=True) except TimeoutExpired: continue except LostRemote: continue while not self.totalResult: gevent.sleep(0.5) return self.totalResult
def setJob_async(self, pickle_object): # unpickle input = StringIO.StringIO(pickle_object) unpickler = pickle.Unpickler(input) intermediateRDD = unpickler.load() input_filename = intermediateRDD.get_ancester().get_input_filename() workerlist = self.create_RDD_workerlist() print workerlist intermediateRDD.set_params_recv(workerlist) output = StringIO.StringIO() pickler = cloudpickle.CloudPickler(output) pickler.dump(intermediateRDD) pickle_object = output.getvalue() # save pickle_object for failure handling self.current_intermediateRDD = intermediateRDD for w in self.workers: self.assign_rdd_to_worker(w, pickle_object) print "in setjob_async" pass
def dumps(o, enable_trace=True, use_zlib=False): # type: (...) -> bytes """For internal use only; no backwards-compatibility guarantees.""" with _pickle_lock: with io.BytesIO() as file: pickler = cloudpickle.CloudPickler(file) try: pickler.dispatch_table[type(flags.FLAGS)] = _pickle_absl_flags except NameError: pass pickler.dump(o) s = file.getvalue() # Compress as compactly as possible (compresslevel=9) to decrease peak memory # usage (of multiple in-memory copies) and to avoid hitting protocol buffer # limits. # WARNING: Be cautious about compressor change since it can lead to pipeline # representation change, and can break streaming job update compatibility on # runners such as Dataflow. if use_zlib: c = zlib.compress(s, 9) else: c = bz2.compress(s, compresslevel=9) del s # Free up some possibly large and no-longer-needed memory. return base64.b64encode(c)
def test_save_unsupported(self): sio = StringIO() pickler = cloudpickle.CloudPickler(sio, 2) with pytest.raises(pickle.PicklingError) as excinfo: pickler.save_unsupported("test") assert "Cannot pickle objects of type" in str(excinfo.value)
def test_cloudpickle_to_file(EN): f = tempfile.NamedTemporaryFile(delete=False) p = cloudpickle.CloudPickler(f) p.dump(EN) f.close() loaded_en = cloudpickle.load(open(f.name)) os.unlink(f.name) doc = loaded_en(unicode('test parse')) assert len(doc) == 2
def __execute(self, stage, conn): ''' Send the stage task to worker and execute this stage in the worker ''' output = StringIO.StringIO() pickler = cloudpickle.CloudPickler(output) pickler.dump(stage) objstr = output.getvalue() conn.run(objstr)
def log_query(filename, keyword): logfile = TextFile(filename) fil = Filter(logfile, lambda x: (keyword in x)) fil.rdd_collect() log_output = StringIO.StringIO() pickler = cloudpickle.CloudPickler(log_output) pickler.dump(fil) return log_output.getvalue()
def dump_plugins(self): """Save the plugins into a pickle object.""" message("Dumping plugins", force=True) f_handler = open(CmdManager.dumped_plugin_path, "wb") pick = cloudpickle.CloudPickler(f_handler) pick.dump((self.cmd_obj_list, self.parser)) f_handler.close()
def word_count(filename): textfile = TextFile(filename) flat = FlatMap(textfile, lambda x: x.split()) map = Map(flat, lambda x: (x, 1)) red = ReduceByKey(map, lambda x, y: x + y) red.rdd_collect( ) #set the collect to be true then we will collect all the data to driver wc_output = StringIO.StringIO() pickler = cloudpickle.CloudPickler(wc_output) pickler.dump(red) return wc_output.getvalue()
def sendResultToDriver(self, result, pName): if type(result) is int: objstr = result else: output = StringIO.StringIO() pickler = cloudpickle.CloudPickler(output) pickler.dump(result) objstr = output.getvalue() # print '6@~~~' c = zerorpc.Client() c.connect('tcp://0.0.0.0:4040') c.getSingleResult_FromWorker(objstr, self.port, pName, async=True)
def collect(self): #1. send serialized RDD to each worker #2. worker do the calculation #3. collect all results output = StringIO.StringIO() pickler = cloudpickle.CloudPickler(output) pickler.dump(self)#rdd pickle_object = output.getvalue() # if self.if_context is params.NO_CONTEXT: # master_addr = self.master_address # elif self.if_context is params.USE_CONTEXT: # master_addr = Context().getMasterAddress() # else: # return "Unknown Type:" + str(self.if_context) # master_addr = self.master_address c = zerorpc.Client(timeout=params.GENERAL_TIMEOUT) c.connect("tcp://"+master_addr) c.set_job(pickle_object) worker_ips = c.result_is_ready() #print "####worker_ips for collect: " + str(worker_ips) final_results = [] if worker_ips is None: print "#############################################" print "Insufficient worker(s) to finish partitions " print "#############################################" return None else: for w in worker_ips: c = zerorpc.Client(timeout=params.GENERAL_TIMEOUT) c.connect("tcp://"+w) result = c.getResults() if isinstance(result, int): if isinstance(final_results, list): final_results = 0 final_results += int(result) else: final_results += result return final_results
def setUp(self): self.file_obj = StringIO() self.cloudpickler = cloudpickle.CloudPickler(self.file_obj, 2)
def ping_worker(self, w, type): while True: if self.workerState[w] != "LOSS": try: if type == WORKER_NORMAL: self.workers[w].ping() else: self.workers_standby[w].ping() except Exception: if type == WORKER_NORMAL: self.workerState[w] = "LOSS" print "lost connection" print "old workers:" + str(self.workers) # remove lost worker self.workers.pop(w, None) # select new worker from standby list selected_worker = self.select_worker_from_standby() # pop from standby list c = self.workers_standby.pop(selected_worker, None) if c is not None: # add select_worker to worker list self.workers[selected_worker] = c self.workerState[selected_worker] = "READY" print "new workers: " + str(self.workers) # update workerlist_for_RDD print "old workerlist:" + str( self.workerlist_for_RDD) if self.workerlist_for_RDD is None or len( self.workerlist_for_RDD.keys()) is 0: continue # 1.pop loss worker with index value index = self.workerlist_for_RDD.pop(w[0] + ":" + w[1]) # 2. assign select_worker with this index self.workerlist_for_RDD[selected_worker[0] + ":" + selected_worker[1]] = index print "new workerlist:" + str( self.workerlist_for_RDD) self.current_intermediateRDD.set_params_recv( self.workerlist_for_RDD) output = StringIO.StringIO() pickler = cloudpickle.CloudPickler(output) pickler.dump(self.current_intermediateRDD) pickle_object = output.getvalue() print "####" + str( self.current_intermediateRDD.workerlist) self.set_job_for_single_worker( selected_worker, pickle_object) gevent.spawn( self.ping_worker, (selected_worker[0], selected_worker[1]), WORKER_NORMAL) self.update_RDD_workerlists() else: """ since there is no enough standby workers. Master may need to cancel the job, and let driver knows """ print "########################## should shut down" self.broadcast_threatening() pass print "new workers:" + str(self.workers) print "standby: " + str(self.workers_standby) else: pass """ the loss of standby worker can be caused by either worker is down, or being removed from the list. """ break else: break gevent.sleep(1)
def controller(self): gevent.sleep(10) # print '1~~' while True: # print '2~~' if self.method != 'repl': # print '3~~' printList = [] for worker in self.workers: # print '4~~' try: # print '5~~' c = zerorpc.Client(timeout=1) c.connect('tcp://' + worker['ip'] + ':' + worker['port']) c.ping() printList.append(worker['port']) except TimeoutExpired: # print '6~~~' # if not worker['current_task']: # # print '7~~~' # # continue # print '1test~~~~' self.workers.remove(worker) if worker['current_task']: # print '2test~~~~' temp = [] for p in worker['current_task']: if not p['status']: temp.append(p) # print 'temp %s' % temp # partiton_num = len(temp) # worker_num = len(self.workers) if len(temp) % len(self.workers) == 0: pop_num = len(temp) / len(self.workers) else: pop_num = len(temp) / len(self.workers) + 1 # print 'pop_num: %s' % pop_num for w in self.workers: if worker['status'] == StatusEnum.Status_Down: continue else: # print '3test~~~' i = 0 partition_List_toSent_inRecovery = [] partitonName_toPrint = [] if len(temp) == 0: break else: while i < pop_num: temp_p = temp.pop() partition_List_toSent_inRecovery.append( temp_p) partitonName_toPrint.append( temp_p['partition_name']) i += 1 if len(temp) == 0: break try: c = zerorpc.Client() c.connect('tcp://' + w['ip'] + ':' + w['port']) # print w['current_task'] w['status'] = StatusEnum.Status_Working output = StringIO.StringIO() pickler = cloudpickle.CloudPickler( output) pickler.dump( partition_List_toSent_inRecovery ) sent = output.getvalue() c.getPartitionList(sent) print colored( '[Master:]', 'white' ), colored( 'Sent partitons:%s to worker: %s', 'red') % (partitonName_toPrint, w['port']) worker['current_task'].extend( partition_List_toSent_inRecovery ) except LostRemote: continue except TimeoutExpired: continue print colored( '[Master:]', 'white'), colored('(%s:%s) down', 'red') % ( worker['ip'], worker['port']) except LostRemote: print 'LostRemote event ignored' continue # print printList gevent.sleep(1)
def setPartionsToWorker(self, partitions): partitions_usedTosetTask = copy.deepcopy(partitions) # print '1' worker_num = len(self.workers) # print worker_num partition_num = len(partitions_usedTosetTask) # print partition_num if partition_num % worker_num == 0: pop_num = partition_num / worker_num else: pop_num = partition_num / worker_num + 1 for worker in self.workers: # print '-------' # print partitions_usedTosetTask # print '-------' if len(partitions_usedTosetTask) == 0: break # print '2' i = 0 while i < pop_num: p = partitions_usedTosetTask.pop() worker['current_task'].append(p) i += 1 # print i if len(partitions_usedTosetTask) == 0: break # print 'test2' # print worker['current_task'] # # sent partitions to workers worker['status'] = StatusEnum.Status_Working partition_List_toSent = [] p_name_toPrint = [] for ct in worker['current_task']: # print ct['status'] if not ct['status']: partition_List_toSent.append(ct) p_name_toPrint.append(ct['partition_name']) try: c = zerorpc.Client() c.connect('tcp://' + worker['ip'] + ':' + worker['port']) output = StringIO.StringIO() pickler = cloudpickle.CloudPickler(output) pickler.dump(partition_List_toSent) sent = output.getvalue() print colored( '[Master:]', 'white'), colored('Sent partitons:%s to worker: %s', 'red') % (p_name_toPrint, worker['port']) c.getPartitionList(sent, async=True) output = StringIO.StringIO() pickler = cloudpickle.CloudPickler(output) pickler.dump(self.rddLineageList) objstr = output.getvalue() print colored('[Master:]', 'white'), colored( 'sent rdd lineage to worker: %s', 'red') % worker['port'] c.getRddLineage(objstr, async=True) except TimeoutExpired: print colored( '[Exception:]exception happened when sent partition and rdd lineage to workers!', 'red') continue except LostRemote: continue
import StringIO import zerorpc import cloudpickle from src.rdd.rdd import * r = TextFile('myfile') m = Map(r, lambda s: s.split()) f = Filter(m, lambda a: int(a[1]) > 2) output = StringIO.StringIO() pickler = cloudpickle.CloudPickler(output) pickler.dump(f) objstr = output.getvalue() c = zerorpc.Client() c.connect("tcp://127.0.0.1:4242") print c.hello(objstr)