def run(self): print "Test: starting emulation" # Single shuffle (one-to-many) num_transfers = 1 size = 1024*1024*10 transfers = self.genShuffleTransfers(nodes[0], nodes, num_transfers,size) # Parallel shuffle (many-to-many) # num_transfers = 1 # size = 1024 # transfers = [] # for mapper in nodes: # transfers += self.genShuffleTransfers(mapper, nodes, num_transfers,size) # start master if host in master: m = Master(host, nodes) m.start() # start slaves if host in nodes: s = Slave(host, master, nodes, transfers) s.start() if host in nodes: s.join() if host in master: m.join() outfile = open("./output/done.json", 'w') json.dump(m.result, outfile, indent=4, sort_keys=True) outfile.close() return False
def run(self): while True: # poll for devices r = requests.get(self.URI) json_data = r.text #f = open("devices.json", "r") #json_data = f.read() #f.close() deviceList_json = json.loads(json_data)["_embedded"]["devices"] devListNew = [] for deviceEntry in deviceList_json: entry = (deviceEntry["id"], deviceEntry["samplingRate"], deviceEntry["longitude"], deviceEntry["latitude"]) devListNew.append(entry) for device in devListNew: if device not in self.devListAct: self.devListAct.append( device ) newSlave = Slave(self.URI, device[0], device[1]) self.deviceMap[device[0]] = newSlave neuron.updateBias(len(self.devListAct)) newSlave.start() # create new slave for device in self.devListAct: if device not in devListNew: removedDevice = self.deviceMap[device[0]] removedDevice.stop() self.devListAct.remove(device) neuron.updateBias(len(self.devListAct)) # shut down slave #print(self.devListAct) time.sleep(1)
def run(self): print "Test: starting emulation" # Single shuffle (one-to-many) num_transfers = 1 size = 1024 * 1024 * 10 transfers = self.genShuffleTransfers(nodes[0], nodes, num_transfers, size) # Parallel shuffle (many-to-many) # num_transfers = 1 # size = 1024 # transfers = [] # for mapper in nodes: # transfers += self.genShuffleTransfers(mapper, nodes, num_transfers,size) # start master if host in master: m = Master(host, nodes) m.start() # start slaves if host in nodes: s = Slave(host, master, nodes, transfers) s.start() if host in nodes: s.join() if host in master: m.join() outfile = open("./output/done.json", 'w') json.dump(m.result, outfile, indent=4, sort_keys=True) outfile.close() return False
from slave import Slave from config import CONFIG_SLAVES import os import logging logging.basicConfig(level=logging.INFO) if not CONFIG_SLAVES or len(CONFIG_SLAVES) == 0: raise Exception("No configured accounts in `config.py` file") export_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'exports') processes = [] for config in CONFIG_SLAVES: slave = Slave(ig_login=config['ig_login'], ig_password=config['ig_password'], comments=config['comments'], hashtags=config['hashtags'], export_dir=export_dir) processes.append(slave) slave.start()