def run(self): print "Test: starting emulation" # Single shuffle (one-to-many) num_transfers = 1 size = 1024*1024*10 transfers = self.genShuffleTransfers(nodes[0], nodes, num_transfers,size) # Parallel shuffle (many-to-many) # num_transfers = 1 # size = 1024 # transfers = [] # for mapper in nodes: # transfers += self.genShuffleTransfers(mapper, nodes, num_transfers,size) # start master if host in master: m = Master(host, nodes) m.start() # start slaves if host in nodes: s = Slave(host, master, nodes, transfers) s.start() if host in nodes: s.join() if host in master: m.join() outfile = open("./output/done.json", 'w') json.dump(m.result, outfile, indent=4, sort_keys=True) outfile.close() return False
def fetch_estate(): database = torndb.Connection(**dbutil.get_mysql_config()) urls = [] for result in database.query('select distinct communityId from house'): estate_id = result.communityId urls.append('http://www.iwjw.com/estate/%s/' % estate_id) master = Master(rest_period=5, result_model='html', result_dir='../iwjw/estate') fetcher = Fetcher(processor=ps.Processor_hn()) master.add_fetchers(fetcher) master.start(urls)
def fetch_house_from_db(): print 'sales' existed = set([f.replace('.html', '') for f in os.listdir('../iwjw/sale')]) master = Master(rest_period=5, result_model='html', result_dir='../iwjw/sale') fetcher = Fetcher(processor=ps.Processor_hn()) master.add_fetchers(fetcher) database = torndb.Connection(**dbutil.get_mysql_config()) sale_list = database.query('select houseId from house where type=1;') sale_list = [result.houseId for result in sale_list if not result.houseId in existed] sale_list = ['http://www.iwjw.com/sale/%s/' % hid for hid in sale_list] master.start(sale_list) database.close()
def __init__(self): self.parse_options() if self.mode == 'client': client = Client() client.standby() elif self.mode == 'master': master = Master() master.start() elif self.mode == 'server': server = Server() server.standby() else: BenchResultsParser()
def fetch_house(): # print 'sales' # master = Master(rest_period=5, result_model='html', result_dir='../iwjw/sale') # fetcher = Fetcher(processor=ps.Processor_hn()) # master.add_fetchers(fetcher) # sales = list(get_houses('../iwjw/sale_list', 'sale')) # master.start(sales) print 'rent' master = Master(rest_period=5, result_model='html', result_dir='../iwjw/rent') fetcher = Fetcher(processor=ps.Processor_hn()) master.add_fetchers(fetcher) rents = list(get_houses('../iwjw/rent_list', 'chuzu')) master.start(rents)
def run(): args = argparser() path = utils.create_log_dir(sys.argv) utils.start(args.http_port) env = Env(args) agents = [Agent(args) for _ in range(args.n_agent)] master = Master(args) for agent in agents: master.add_agent(agent) master.add_env(env) success_list = [] time_list = [] for idx in range(args.n_episode): print('=' * 80) print("Episode {}".format(idx + 1)) # 서버의 stack, timer 초기화 print("서버를 초기화하는중...") master.reset(path) # 에피소드 시작 master.start() # 에이전트 학습 master.train() print('=' * 80) success_list.append(master.infos["is_success"]) time_list.append(master.infos["end_time"] - master.infos["start_time"]) if (idx + 1) % args.print_interval == 0: print("=" * 80) print("EPISODE {}: Avg. Success Rate / Time: {:.2} / {:.2}".format( idx + 1, np.mean(success_list), np.mean(time_list))) success_list.clear() time_list.clear() print("=" * 80) if (idx + 1) % args.checkpoint_interval == 0: utils.save_checkpoints(path, agents, idx + 1) if args.visual: visualize(path, args) print("끝") utils.close()
def fetch_list(): print 'sale_list' master = Master(rest_period=5, result_model='html', result_dir='../iwjw/sale_list') fetcher = Fetcher(processor=ps.ProcessorIwjw()) master.add_fetchers(fetcher) urls = [line.split('#')[0].strip() for line in codecs.open('../district.id')] urls = map(lambda x: 'http://www.iwjw.com/sale/shanghai/%sp1/' % x, urls) master.start(urls) print 'rent_list' master = Master(rest_period=5, result_model='html', result_dir='../iwjw/rent_list') fetcher = Fetcher(processor=ps.ProcessorIwjw()) master.add_fetchers(fetcher) urls = [line.split('#')[0].strip() for line in codecs.open('../district.id')] urls = map(lambda x: 'http://www.iwjw.com/chuzu/shanghai/%sp1/' % x, urls) master.start(urls)
def run(self): print "Test: starting emulation" # Single shuffle (one-to-many) num_transfers = 1 size = 1024 * 1024 * 10 transfers = self.genShuffleTransfers(nodes[0], nodes, num_transfers, size) # Parallel shuffle (many-to-many) # num_transfers = 1 # size = 1024 # transfers = [] # for mapper in nodes: # transfers += self.genShuffleTransfers(mapper, nodes, num_transfers,size) # start master if host in master: m = Master(host, nodes) m.start() # start slaves if host in nodes: s = Slave(host, master, nodes, transfers) s.start() if host in nodes: s.join() if host in master: m.join() outfile = open("./output/done.json", 'w') json.dump(m.result, outfile, indent=4, sort_keys=True) outfile.close() return False
import sys from master import Master NODES = ['10.11.220.20', '10.11.220.21', '10.11.220.22', '10.11.220.23'] PASSWORD = sys.argv[1] USE_NODES = int(sys.argv[2]) if __name__ == '__main__': print(f'Running for {len(NODES[:USE_NODES])} nodes: {PASSWORD}') master = Master(PASSWORD, nodes=NODES[:USE_NODES], partioning=1) master.start()
config["paths"]["log_path"] = log_path config["paths"]["job_db_path"] = os.path.join(log_path, "jobs.db") config["paths"]["pid_path"] = os.path.join(log_path, "pid") # check configurations check_result = check_config(config) if type(check_result) is str: print(check_result) exit(1) else: (job_manager_class, slave_class, uploader_class) = check_result pid_path = config["paths"]["pid_path"] with open(pid_path, "w") as pid: pid.write(str(os.getpid())) if task == 0: # submit procedure job_manager = job_manager_class(config) job_manager.start() print("New submitted: %d" % job_manager.new_submitted) print("Submit failed: %d" % job_manager.submit_error) print("Ignored: %d" % job_manager.ignore) elif task == 1: # upload procedure master = Master(config, slave_class, uploader_class) master.start() if os.path.isfile(pid_path): os.remove(pid_path)
def monaco(): ''' The management daemon for Monaco ''' # Set up top-level console logger consolehandler = logging.StreamHandler() consolehandler.setLevel(logging.ERROR) consolehandler.setFormatter( logging.Formatter('%(name)s - %(levelname)s - %(message)s')) logger = logging.getLogger('monaco') logger.setLevel(logging.DEBUG) logger.addHandler(consolehandler) # Load config config.initLoggers() # Install reload handler def USR1_handler(signum, frame): ''' SIGUSR1 handler ''' logger.info('Received SIGUSR1, reloading Monaco...') # this kills the current process tree and re-executes the # original call arguments in the same PID (good for upstart) os.execl(sys.executable, *([sys.executable] + sys.argv)) signal.signal(signal.SIGUSR1, USR1_handler) # Setup scheduling sched = Scheduler(daemon=True) atexit.register(lambda: sched.shutdown(wait=False)) sched.start() # Config and start slave thread to maintain app state slave = Slave() slave.update_subs() #slave.start() starting in apsched inherits daemon=True @sched.interval_schedule(seconds=1) def maintain_slave(): ''' This periodically ensures that the slave is subscribed to all the channels it should be. Again, this should be redundant ''' if not hasattr(maintain_slave, 'started'): maintain_slave.started = True slave.start() logger.info('Slave thread started') if not slave.is_alive(): logger.info('Slave thread died') slave.start() # lifeRaft - Use Raft algorithm to maintain management leader liferaft = LifeRaft() # Start master/monitor thread master = Master(liferaft) master.start() # maintain mgmt redis role from LifeRaft r = redis.StrictRedis(port=config.config['mgmt_port']) mastercli = None while True: time.sleep(1) try: master_tup = liferaft.value if not master_tup: logger.warn("Couldn't learn master value!") else: logger.debug('MGMT DB maintaining: %s', repr(master_tup)) if master_tup[0] == config.config['IP']: if r.info()['role'] == 'master': # We're the master, and properly configured continue # Else set to master, rewrite config to persist logger.debug('Promoting self to master') r.slaveof() r.config_rewrite() else: if r.info()['role'] == 'slave' and r.info( )['master_host'] == master_tup[0]: # We're a slave and properly configured to the master continue for idx in xrange(3): if idx != 0: # Decongest on retry time.sleep(0.5) try: # Try to get a connection to the mgmt master and verify it thinks it's master if not mastercli: mastercli = redis.StrictRedis( host=master_tup[0], port=config.config['mgmt_port']) if mastercli.info()['role'] != 'master': # If it doesn't think it's master, delete client and continue to retry del mastercli continue break except Exception: continue else: # We didn't break logger.debug( 'Assigned master (%s) has not yet assumed role', master_tup[0]) continue # Set slave to proper master logger.debug('Assigning self as slaveof %s:6379', master_tup[0]) r.slaveof(host=master_tup[0], port=config.config['mgmt_port']) except Exception: try: r = redis.StrictRedis(port=config.config['mgmt_port']) except Exception: logger.exception("Can't connect to mgmt db!")
def run(self): print "ssss" t = Master(self.cmd) #t.daemon=True t.start() time.sleep(0.5)
from master import Master import logging import time import logging.handlers logger = logging.getLogger("scrapper") logger.setLevel(logging.DEBUG) fh = logging.handlers.RotatingFileHandler("logs/scrapper.log", maxBytes=100000, backupCount=5) formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') formatter.converter = time.gmtime fh.setFormatter(formatter) logger.addHandler(fh) # create console handler with a higher log level ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) logger.addHandler(ch) manager = Master() if __name__ == "__main__": manager.start()
def monaco(): ''' The management daemon for Monaco ''' # Set up top-level console logger consolehandler = logging.StreamHandler() consolehandler.setLevel(logging.ERROR) consolehandler.setFormatter(logging.Formatter('%(name)s - %(levelname)s - %(message)s')) logger = logging.getLogger('monaco') logger.setLevel(logging.DEBUG) logger.addHandler(consolehandler) # Load config config.initLoggers() # Install reload handler def USR1_handler(signum, frame): ''' SIGUSR1 handler ''' logger.info('Received SIGUSR1, reloading Monaco...') # this kills the current process tree and re-executes the # original call arguments in the same PID (good for upstart) os.execl(sys.executable, *([sys.executable]+sys.argv)) signal.signal(signal.SIGUSR1, USR1_handler) # Setup scheduling sched = Scheduler(daemon=True) atexit.register(lambda: sched.shutdown(wait=False)) sched.start() # Config and start slave thread to maintain app state slave = Slave() slave.update_subs() #slave.start() starting in apsched inherits daemon=True @sched.interval_schedule(seconds=1) def maintain_slave(): ''' This periodically ensures that the slave is subscribed to all the channels it should be. Again, this should be redundant ''' if not hasattr(maintain_slave, 'started'): maintain_slave.started = True slave.start() logger.info('Slave thread started') if not slave.is_alive(): logger.info('Slave thread died') slave.start() # lifeRaft - Use Raft algorithm to maintain management leader liferaft = LifeRaft() # Start master/monitor thread master = Master(liferaft) master.start() # maintain mgmt redis role from LifeRaft r = redis.StrictRedis(port=config.config['mgmt_port']) mastercli = None while True: time.sleep(1) try: master_tup = liferaft.value if not master_tup: logger.warn("Couldn't learn master value!") else: logger.debug('MGMT DB maintaining: %s', repr(master_tup)) if master_tup[0] == config.config['IP']: if r.info()['role'] == 'master': # We're the master, and properly configured continue # Else set to master, rewrite config to persist logger.debug('Promoting self to master') r.slaveof() r.config_rewrite() else: if r.info()['role'] == 'slave' and r.info()['master_host'] == master_tup[0]: # We're a slave and properly configured to the master continue for idx in xrange(3): if idx != 0: # Decongest on retry time.sleep(0.5) try: # Try to get a connection to the mgmt master and verify it thinks it's master if not mastercli: mastercli = redis.StrictRedis(host=master_tup[0], port=config.config['mgmt_port']) if mastercli.info()['role'] != 'master': # If it doesn't think it's master, delete client and continue to retry del mastercli continue break except Exception: continue else: # We didn't break logger.debug('Assigned master (%s) has not yet assumed role', master_tup[0]) continue # Set slave to proper master logger.debug('Assigning self as slaveof %s:6379', master_tup[0]) r.slaveof(host=master_tup[0], port=config.config['mgmt_port']) except Exception: try: r = redis.StrictRedis(port=config.config['mgmt_port']) except Exception: logger.exception("Can't connect to mgmt db!")