class DataCurator(object): """ This program runs periodically and sends data to the central server. """ def __init__(self): self.log = Logger(__name__).setup() def run(self): self.log.info("Running data curator") schedule.every().day.at("20:55").do(self.curate) while True: schedule.run_pending() time.sleep(1) def curate(self): self.log.info("Curation job running...")
def _run(self): """ Main run loop. """ NO_WORK_DELAY = 5 # FIXME move this to a settings file self.running = True while self.running: has_work = True while has_work: distro = self.get_next_distribution() if distro: distro.start() else: has_work = False Logger.log("No work to do. Sleeping for {}s.".format( NO_WORK_DELAY )) gevent.sleep(NO_WORK_DELAY)
class MuonDetector(gps.GPSListener, adc.EventListener, object): """ Main entry point into the muon detector program. """ def __init__(self): self.log = Logger(__name__).setup() # Read our node ID self.node_id = eeprom.get_id() # Load the GPS module self.gps_module = gps.GPS(self) # Load the ADC module self.adc_module = adc.ADC(0, self) self.filesystem = client.FileSystem("root://localhost//tmp") self.create_event_file() def run(self): """ Main function that will load all the necessary modules and start taking data. """ self.log.info("Running muon detector") self.gps_module.start() self.adc_module.start() self.gps_module.join() self.adc_module.join() def on_event(self, data): """ Do something when we get an event from the ADC """ event = Event(data, self.gps_module.current_timestamp, self.gps_module.current_lat, self.gps_module.current_lon) self.log.debug("Got response from ADC: %s" % event) # Dump the event to a file print(str(event), file=open('/tmp/event.txt', 'a')) def on_gpgga(self, gpgga): """ Do something when we get a GPS pulse """ self.log.debug("on_gpgga()") def create_event_file(self): """ Creates an event file in the xrootd filesystem (if it doesn't exist) """ status, statinfo = self.filesystem.stat("/tmp/event.txt") if statinfo is None: with client.File() as f: f.open("root://localhost//tmp/event.txt", OpenFlags.NEW)
def __init__(self): self.log = Logger(__name__).setup() # Read our node ID self.node_id = eeprom.get_id() # Load the GPS module self.gps_module = gps.GPS(self) # Load the ADC module self.adc_module = adc.ADC(0, self) self.filesystem = client.FileSystem("root://localhost//tmp") self.create_event_file()
def __init__(self): self.log = Logger(__name__).setup() # Read our node ID self.node_id = eeprom.get_id() # Load the GPS module self.gps_module = gps.GPS(self) # Load the ADC module self.adc_module = adc.ADC(self) # Load the temperature and pressure sensors self.temp_module = temp.TemperatureSensor() self.pressure_module = pressure.PressureSensor() self.filesystem = client.FileSystem("root://localhost//tmp") self.create_event_file() # Register signal handler signal.signal(signal.SIGINT, self.signal_handler)
class DriverFactory: """ # singleton 单例模式,保证全局只有有一个driver """ _instance = None _dirver = None Chrome_driver_path = os.path.join(os.path.dirname(__file__), "drivers" + os.sep + "chromedriver.exe") logger = Logger() def __new__(cls, *args, **kwargs): if not cls._instance: cls._instance = super().__new__(cls) return cls._instance @classmethod def get_driver(cls, browers="chrome"): if cls._dirver is None: cls._dirver = cls.create_driver(browers) cls.logger.info("创建" + browers + "浏览器") return cls._dirver @classmethod def create_driver(cls, browers): driver = None if browers.lower() == "chrome": driver = webdriver.Chrome(executable_path=cls.Chrome_driver_path) elif browers.lower() == "firfox": driver = webdriver.Chrome() elif driver.lower() == "ie": driver = webdriver.Ie() elif driver.lower() == "Safari": driver = webdriver.Safari() else: #后期此处改为logger cls.logger.warn("您传入的参数browers异常") # print("您传入的参数browers异常") return driver
def clear_trash(self): """Finds trash mobs closest to the current fleet location and battles them until the boss spawns """ while self.kills_needed > 0: blacklist = [] tries = 0 if self.resume_previous_sortie: self.resume_previous_sortie = False Utils.find_and_touch('combat_attack') Utils.script_sleep(2) else: self.avoided_ambush = True while not Utils.exists('combat_battle_start'): if Utils.find_and_touch('combat_evade'): if Utils.wait_for_exist('combat_battle_start', 3): self.avoided_ambush = False else: Logger.log_msg('Successfully avoided ambush.') elif Utils.find_and_touch('combat_items_received'): pass else: enemy_coord = self.get_closest_enemy() if tries > 2: blacklist.append(enemy_coord) enemy_coord = self.get_closest_enemy(blacklist) Logger.log_msg( 'Navigating to enemy fleet at {}'.format(enemy_coord)) Utils.touch(enemy_coord) tries += 1 Utils.script_sleep(5) if self.conduct_prebattle_check(): if self.conduct_battle(): self.need_to_refocus = True else: self.resume_previous_sortie = True while not (Utils.exists('home_menu_build')): Utils.touch_randomly(self.region['nav_back']) # Add logic for retirement here? return False if self.avoided_ambush: self.kills_needed -= 1 Logger.log_msg('Kills left for boss to spawn: {}'.format( self.kills_needed)) Utils.script_sleep(1) return True
def read(self): backup_config = deepcopy(self.__dict__) config = configparser.ConfigParser() config.read(self.config_file) self.network['service'] = config.get('Network', 'Service') if config.getboolean('Combat', 'Enabled'): self._read_combat(config) else: self.combat = {'enabled': False} self.commissions['enabled'] = config.getboolean('Modules', 'Commissions') self.enhancement['enabled'] = config.getboolean('Modules', 'Enhancement') self.missions['enabled'] = config.getboolean('Modules', 'Missions') self.retirement['enabled'] = config.getboolean('Modules', 'Retirement') if config.getboolean('Events', 'Enabled'): self._read_event(config) else: self.events = {'enabled': False} self.validate() if (self.ok and not self.initialized): Logger.log_msg("Starting ALAuto!") self.initialized = True self.changed = True elif (not self.ok and not self.initialized): Logger.log_error("Invalid config. Please check your config file.") sys.exit(1) elif (not self.ok and self.initialized): Logger.log_warning("Config change detected, but with problems. Rolling back config.") self._rollback_config(backup_config) elif (self.ok and self.initialized): if backup_config != self.__dict__: Logger.log_warning("Config change detected. Hot-reloading.") self.changed = True
def startup(spider): try: Logger.info("开始[%d]线程顺序抓取..." % spider.max_thread) task_list = [] with ThreadPoolExecutor(spider.max_thread) as executor: for channel in spider.channels: # 异常抓取重试任务 task = executor.submit(retry_failed, spider, channel) task_list.append(task) # 正常抓取 task = executor.submit(get_page, spider, channel) task_list.append(task) for task in as_completed(task_list): Logger.info("线程[%s]执行完成" % str(task)) except Exception as e: Logger.error("线程池启动抓取异常:{} \n {}".format(str(e), traceback.format_exc()))
def build_agents(self): num_agents = self.env.get_num_agents() self.agents = [] Logger.print('') Logger.print('Num Agents: {:d}'.format(num_agents)) agent_files = self.arg_parser.parse_strings('agent_files') assert(len(agent_files) == num_agents or len(agent_files) == 0) model_files = self.arg_parser.parse_strings('model_files') assert(len(model_files) == num_agents or len(model_files) == 0) output_path = self.arg_parser.parse_string('output_path') int_output_path = self.arg_parser.parse_string('int_output_path') for i in range(num_agents): curr_file = agent_files[i] curr_agent = self._build_agent(i, curr_file) if curr_agent is not None: curr_agent.output_dir = output_path curr_agent.int_output_dir = int_output_path Logger.print(str(curr_agent)) if (len(model_files) > 0): curr_model_file = model_files[i] if curr_model_file != 'none': curr_agent.load_model(curr_model_file) self.agents.append(curr_agent) Logger.print('') self.set_enable_training(self.enable_training) return
def enhance_ship(self): """ Method that selects the first (leftmost of the first row) favorite ship and proceeds to enhance her. """ #selects ship Utils.touch_randomly(self.region['first_favorite_ship']) Utils.script_sleep(1) while True: Utils.update_screen() if Utils.find("enhancement/menu_enhance"): Logger.log_debug("Filling with ships.") #taps the "fill" button Utils.touch_randomly(self.region['fill_button']) Utils.update_screen() if Utils.find("enhancement/alert_no_items", 0.85): Logger.log_warning("Not enough ships to enhance.") break if Utils.find("enhancement/menu_level", 0.8): self.handle_retirement() Logger.log_msg("Successfully finished enhancing.") break if Utils.find("enhancement/menu_details"): Logger.log_debug("Opening enhance menu.") if not Utils.find("enhancement/menu_retrofit", 0.9): Utils.touch_randomly( self.region['enhance_tab_normal_ship']) else: Utils.touch_randomly(self.region['enhance_tab_retro_ship']) continue Utils.touch_randomly(self.region['button_go_back']) Utils.script_sleep(1) return
def checkUpdate(self): version = '' latest_version = '' _file = open('version.txt', 'r') if self.config.updates['channel'] == 'Release': version = _file.readline() try: with request.urlopen( "https://api.github.com/repos/egoistically/alauto/releases/latest" ) as f: _json = json.loads(f.read().decode('utf-8')) latest_version = _json["tag_name"] except error.HTTPError as e: Logger.log_error("Couldn't check for updates, {}.".format(e)) else: version = _file.readlines()[1] try: with request.urlopen( "https://raw.githubusercontent.com/Egoistically/ALAuto/master/version.txt" ) as f: _f = f.read().decode('utf-8') latest_version = _f.splitlines()[1] except error.HTTPError as e: Logger.log_error("Couldn't check for updates, {}.".format(e)) _file.close() if version != latest_version: Logger.log_debug("Current version: " + version) Logger.log_debug("Latest version: " + latest_version) return True
def lambda_handler(event, context): dt = datetime.utcnow() logstream = LOG_STREAM.format("OpsAutomatorMain", dt.year, dt.month, dt.day) with Logger(logstream=logstream, context=context, buffersize=20) as logger: logger.info("Ops Automator, version %version%") for handler_name in handlers.all_handlers(): if handlers.get_class_for_handler(handler_name).is_handling_request(event): handler = handlers.create_handler(handler_name, event, context) logger.info("Handler is {}", handler_name) try: result = handler.handle_request() logger.info(MSG_REQUEST_HANLED, handler_name, (datetime.utcnow() - dt).total_seconds()) return safe_dict(result) except Exception as e: logger.error(MSG_ERR_HANDLING_REQUEST, safe_json(event, indent=2), handler_name, e, traceback.format_exc()) return logger.error(MSG_NO_REQUEST_HANDLER, safe_json(event, indent=2))
def collect_dorm_balloons(self): """" This method finds and collects all the dorm tokens and affinity points visible to the script. The various swipes may not work if there is a shipgirl at the starting point of the swipe. For this reason the wrapper to this methoed iterates its cycle for three times, refreshing the dorm. """ Utils.script_sleep(1) # tap dorm eye in order to hide UI Utils.touch_randomly(self.region["dorm_eye_button"]) Logger.log_debug("Collecting all visible dorm tokens/affinity points.") for i in range(0, 4): Utils.wait_update_screen(1) # since a rather low similarity is used, the variable j ensures a finite loop j = 0 while Utils.find_and_touch("headquarters/dorm_token", 0.75) and j < 5: Logger.log_msg("Collected dorm token.") Utils.wait_update_screen() j += 1 j = 0 while Utils.find_and_touch("headquarters/affinity_point", 0.75) and j < 5: Logger.log_msg("Collected affinity points.") Utils.wait_update_screen() j += 1 if i == 0: # swipe right and refresh Utils.swipe(960, 540, 560, 540, 300) continue if i == 1: # swipe left (also countering the previous swipe) and refresh Utils.swipe(960, 540, 1760, 540, 300) continue if i == 2: # undo previous swipe Utils.swipe(960, 540, 560, 540, 300) # swipe up and refresh Utils.swipe(960, 540, 960, 790, 300) continue if i == 3: # swipe bottom (also countering the previous swipe) and refresh Utils.swipe(960, 540, 960, 40, 300) continue # restore UI Utils.touch_randomly(self.region["dorm_eye_button"])
def urgent_handler(self): Utils.touch_randomly(self.region["urgent_tab"]) while True: Utils.update_screen() if Utils.find_and_touch("commission/commission_status"): Logger.log_msg("Found status indicator on urgent commission.") if not self.start_commission(): Logger.log_msg("No more commissions to start.") return False else: Utils.touch_randomly(self.region["daily_tab"]) Logger.log_msg("No urgent commissions left.") break Utils.script_sleep(1) return True
def unable_handler(self, coords, boss=False): """ Method called when the path to the target (boss fleet or mystery node) is obstructed by mobs: it procedes to switch targets to the mobs which are blocking the path. Args: coords (list): coordinate_x, coordinate_y. These coordinates describe the target's location. """ if boss: Logger.log_debug("Unable to reach boss function started.") else: Logger.log_debug( "Unable to reach selected target function started.") self.blacklist.clear() closest_to_unreachable_target = self.get_closest_target(self.blacklist, coords, boss=boss) Utils.touch(closest_to_unreachable_target) Utils.update_screen() if Utils.find("combat/alert_unable_reach"): Logger.log_warning("Unable to reach next to selected target.") self.blacklist.append(closest_to_unreachable_target[0:2]) while True: closest_enemy = self.get_closest_target(self.blacklist) Utils.touch(closest_enemy) Utils.update_screen() if Utils.find("combat/alert_unable_reach"): self.blacklist.append(closest_enemy[0:2]) else: break self.movement_handler(closest_enemy) if not self.battle_handler(): return False return True else: self.movement_handler(closest_to_unreachable_target) if not self.battle_handler(): return False return True
def lambda_handler(event, context): try: dt = datetime.utcnow() log_stream = LOG_STREAM.format(dt.year, dt.month, dt.day) result = {} with Logger(logstream=log_stream, buffersize=20, context=context, debug=util.as_bool(os.getenv(configuration.ENV_TRACE, False))) as logger: # logger.info("InstanceScheduler, Running locally!!") # logger.info("InstanceScheduler, version {}".format(VERSION)) logger.debug("Event is {}", util.safe_json(event, indent=3)) for handler_type in [SchedulerRequestHandler, SchedulerSetupHandler, ScheduleResourceHandler, AdminCliRequestHandler, CloudWatchEventHandler]: # logger.info("handler_type: {}", handler_type) if handler_type.is_handling_request(event): start = time() handler = handler_type(event, context) logger.info("Handler is {}".format(handler_type.__name__)) try: result = handler.handle_request() except Exception as e: logger.error("Error handling request {} by handler {}: ({})\n{}", json.dumps(event), handler_type.__name__, e, traceback.format_exc()) execution_time = round(float((time() - start)), 3) logger.info("Handling took {} seconds", execution_time) return result logger.debug("Request was not handled, no handler was able to handle this type of request {}", json.dumps(event)) finally: configuration.unload_scheduler_configuration()
class AuthServer(RNServer): logger = Logger('AuthServer') def __init__(self, ip='127.0.0.1', port=1001, backlog=10000, password=b'3.25 ND1', database=None): super().__init__((ip, port), backlog, password) self.database = database self.add_handler(RNEvent.UserPacket, self.handle_packet) self.auth_handlers = {} self.register_handler(PacketHeaders.HANDSHAKE.value, Handshake) self.register_handler(PacketHeaders.CLIENT_LOGIN_REQ.value, ClientLoginRequest) def handle_packet(self, packet, address): header = packet[0:8] if header in self.auth_handlers: self.auth_handlers[header].database = self.database res = self.auth_handlers[header].construct_packet(self, packet) if res is not None: self.send(res, address) else: self.logger.warn( 'unable to construct packet for header-{}'.format(header)) else: self.logger.warn( 'no registered handlers found for header-{}'.format(header)) def register_handler(self, header, func): self.auth_handlers[header] = func
def get_datas(self): ''' 新建读取测试数据方法 :return: ''' # 定义空列表 datas = [] try: #打开指定文件 xl = xlrd.open_workbook(self.file_path) #根据sheetName获取指定的sheet sheet = xl.sheet_by_index(self.sheet_id) items = sheet.row_values(0) #遍历excel的行数 for nrow in range(1, sheet.nrows): #定义字典 data = dict() values = sheet.row_values(nrow) for ncol in range(0, len(items)): data[items[ncol]] = values[ncol] datas.append(data) except Exception as e: Logger().error("读取文件出错:{}".format(e)) return datas
def init_logger(): log_level = getattr(logging, config['LOGGING']['LEVEL'], logging.INFO) Logger.setLevel(log_level) stream_handler = logging.StreamHandler() stream_handler.setFormatter( logging.Formatter( '[%(levelname)s -> %(name)s] at %(asctime)s in %(filename)s: %(lineno)s - %(message)s' )) Logger.addHandler(stream_handler) logging.getLogger( 'sqlalchemy.engine.base.Engine').handlers = Logger.handlers app.logger.handlers = Logger.handlers app.logger.setLevel(log_level) Logger.info('Initializing logger...')
def print_stats(self): """Prints a summary of all the stats to console. """ delta = datetime.now() - self.start_time hours = delta.total_seconds() / 3600 if self.config.commissions['enabled']: Logger.log_success("Commissions sent: {} / received: {}".format( self._pretty_perhour(self.commissions_started, hours), self._pretty_perhour(self.commissions_received, hours))) if self.config.combat['enabled']: Logger.log_success("Combat done: {} / attempted: {}".format( self._pretty_perhour(self.combat_done, hours), self._pretty_perhour(self.combat_attempted, hours))) Logger.log_success( "ALAuto has been running for {} (started on {})".format( self._pretty_timedelta(delta), self.start_time.strftime('%Y-%m-%d %H:%M:%S')))
def retry_failed(spider, channel): Logger.info("开始重试任务...") try: retry_list = mysql.query_toretry_task(channel.pk_channel) if len(retry_list) > 0: _chrome = chrome.Chrome() for retry_info in retry_list: get_article(_chrome, retry_info["title"], retry_info["src_url"], retry_info["pub_time"], spider, channel) # 如果任务未失败(retry_info未增加),删除任务 mysql.delete_toretry_task(channel.pk_channel, retry_info["src_url"], retry_info["total_times"]) # 停止重试超过5次的任务 mysql.stop_toretry_task(channel.pk_channel) _chrome.quit() Logger.info("重试任务完成") except Exception as e: Logger.error("重试任务异常:{} \n {}".format(str(e), traceback.format_exc()))
def run(dir, inpath): if not os.path.exists(dir): os.makedirs(dir) logfile = Logger(dir + '/qnrsearch.log') logfile.open() try: run_sieves( [ (dnareader, {'item_limit': 0}), (hmmsearch, {'model_path': sys.path[0]+'/hmm_models/model.hmm', 'hmmsearch_out': dir+'/hmmsearch_out', 'write_only_domain': True}), (sga, {'error_rate': 0.05, 'min_assembly_overlap': 20, 'min_merge_overlap': 20, 'resolve_small': 5 }) ], ['', dir+'/fragments.db', dir+'/fragments_passed.db', dir+'/clusters.db'], #dbs [inpath, dir+'/dnareader.pfa', dir+'/fragments_passed.nfa', dir+'/clusters.nfa'], #files logfile, level, 0,2 ) finally: logfile.close()
def __create_buddies(self): Logger.get_logger().info("Parsing buddies data") data = {name: [] for name in self.names} for _idx, record in enumerate(tqdm(self._raw_data, disable=not self._verbose)): if record["buddies"] is None: Logger.get_logger().debug("Buddies found to be None") continue buddies = [name.lower() for name in record["buddies"]] ts = record["timestamp"] for name in buddies: data[name].append(pendulum.parse(ts)) Logger.get_logger().info("Generating buddies") for name in tqdm(data, disable=not self._verbose): data[name] = Buddy(data[name]) return data
def combat_handler(self): Logger.log_msg("Starting combat.") Utils.touch_randomly(self.region['menu_combat_start']) Utils.script_sleep(4) while True: Utils.wait_update_screen(1) if Utils.find("event/button_no"): Utils.touch_randomly(self.region['combat_button_no']) Utils.script_sleep(1) continue if Utils.find("combat/combat_pause", 0.7): Logger.log_debug("In battle.") Utils.script_sleep(5) continue if Utils.find("combat/menu_touch2continue"): Utils.touch_randomly(self.region['tap_to_continue']) continue if Utils.find("menu/item_found"): Utils.touch_randomly(self.region['tap_to_continue']) Utils.script_sleep(1) continue if Utils.find("combat/button_retry"): Logger.log_msg("Combat ended.") Utils.touch_randomly(self.region['combat_end_confirm']) self.stats.increment_combat_done() Utils.script_sleep(1) return if Utils.find("combat/commander"): # prevents fleet with submarines from getting stuck at combat end screen Utils.touch_randomly( self.region["combat_dismiss_surface_fleet_summary"]) Utils.script_sleep(1) continue if Utils.find("combat/menu_combat_finished"): Utils.touch_randomly(self.region['dismiss_combat_finished']) Utils.script_sleep(1) continue
def init_screencap_mode(cls,mode): consts = UtilConsts.ScreenCapMode cls.screencap_mode = mode if cls.screencap_mode == consts.ASCREENCAP: # Prepare for ascreencap, push the required libraries Adb.exec_out('rm /data/local/tmp/ascreencap') cpuArc = Adb.exec_out('getprop ro.product.cpu.abi').decode('utf-8').strip() sdkVer = int(Adb.exec_out('getprop ro.build.version.sdk').decode('utf-8').strip()) ascreencaplib = 'ascreencap_{}'.format(cpuArc) if sdkVer in range(21, 26) and os.path.isfile(ascreencaplib): Adb.cmd('push {} /data/local/tmp/ascreencap'.format(ascreencaplib)) else: Logger.log_warning( 'No suitable version of aScreenCap lib is available locally, using ascreencap_local...') if os.path.isfile('ascreencap_local'): Adb.cmd('push ascreencap_local /data/local/tmp/ascreencap') else: Logger.log_error( 'File "ascreencap_local" not found. Please download the appropriate version of aScreenCap for your device from github.com/ClnViewer/Android-fast-screen-capture and save it as "ascreencap_local"') Logger.log_warning('Since aScreenCap is not ready, falling back to normal adb screencap') Utils.useAScreenCap = False Adb.shell('chmod 0777 /data/local/tmp/ascreencap')
def battle_handler(self, boss=False): Logger.log_msg("Starting combat.") while not Utils.find("combat/menu_loading", 0.8): Utils.update_screen() if Utils.find("combat/alert_morale_low") or Utils.find( "menu/button_sort"): self.retreat_handler() return False else: Utils.touch_randomly(self.region["menu_combat_start"]) Utils.script_sleep(1) Utils.script_sleep(4) while True: Utils.update_screen() if Utils.find("combat/alert_lock"): Logger.log_msg("Locking received ship.") Utils.touch_randomly(Region(1086, 739, 200, 55)) continue if Utils.find("combat/combat_pause", 0.7): Logger.log_debug("In battle.") Utils.script_sleep(5) continue if Utils.find("combat/menu_touch2continue"): Utils.touch_randomly(Region(661, 840, 598, 203)) continue if Utils.find("menu/item_found"): '''if boss: DM.find_droped() ''' Utils.touch_randomly(Region(661, 840, 598, 203)) Utils.script_sleep(1) continue if Utils.find("menu/drop_ssr"): Logger.log_msg("Received SSR ship as drop.") Utils.touch_randomly(Region(1228, 103, 692, 735)) continue if Utils.find("menu/drop_elite"): Logger.log_msg("Received ELITE ship as drop.") Utils.touch_randomly(Region(1228, 103, 692, 735)) continue if Utils.find("combat/button_confirm"): Logger.log_msg("Combat ended.") Utils.touch_randomly(self.region["combat_end_confirm"]) Utils.script_sleep(1) if boss: return True Utils.update_screen() if Utils.find("commission/button_confirm"): Logger.log_msg("Found commission info message.") Utils.touch_randomly(self.region["combat_com_confirm"]) continue if Utils.find("combat/button_retreat"): Utils.script_sleep(3) Utils.touch_randomly(self.region["hide_strat_menu"]) return if Utils.find("combat/commander"): Utils.touch_randomly(self.region["combat_end_confirm"]) continue
import sys import core.bonds __all__ = ["Calculation", "CalculationCache", "CalculationSettings", "calculated", "count_frames", "calculated_frames", "calculate_cavities", "getresults", "delete_center_cavity_information", "timestamp", "calculate"] logger = Logger("core.calculation") logger.setstream("default", sys.stdout, Logger.WARNING) class CalculationSettings(object): """ Structure to store the parameters for one or more calculation. **Attributes:** `datasets` : Dictionary, which contains filenames (Strings) as keys. Each value is a list of Integers, which contains the frames. The value ``[-1]`` means 'all frames'. `resolution` : resolution parameter for the discretization `domains` :
def main(args): #initial setup if args.checkpoint == '': args.checkpoint = "checkpoints1/ic19val_%s_bs_%d_ep_%d" % ( args.arch, args.batch_size, args.n_epoch) if args.pretrain: if 'synth' in args.pretrain: args.checkpoint += "_pretrain_synth" else: args.checkpoint += "_pretrain_ic17" print(('checkpoint path: %s' % args.checkpoint)) print(('init lr: %.8f' % args.lr)) print(('schedule: ', args.schedule)) sys.stdout.flush() if not os.path.isdir(args.checkpoint): os.makedirs(args.checkpoint) kernel_num = 7 min_scale = 0.4 start_epoch = 0 validation_split = 0.1 random_seed = 42 prev_val_loss = -1 val_loss_list = [] loggertf = tfLogger('./log/' + args.arch) #end #setup data loaders data_loader = IC19Loader(is_transform=True, img_size=args.img_size, kernel_num=kernel_num, min_scale=min_scale) dataset_size = len(data_loader) indices = list(range(dataset_size)) split = int(np.floor(validation_split * dataset_size)) np.random.seed(random_seed) np.random.shuffle(indices) train_incidies, val_indices = indices[split:], indices[:split] train_sampler = SubsetRandomSampler(train_incidies) validate_sampler = SubsetRandomSampler(val_indices) train_loader = torch.utils.data.DataLoader(data_loader, batch_size=args.batch_size, num_workers=3, drop_last=True, pin_memory=True, sampler=train_sampler) validate_loader = torch.utils.data.DataLoader(data_loader, batch_size=args.batch_size, num_workers=3, drop_last=True, pin_memory=True, sampler=validate_sampler) #end #Setup architecture and optimizer if args.arch == "resnet50": model = models.resnet50(pretrained=True, num_classes=kernel_num) elif args.arch == "resnet101": model = models.resnet101(pretrained=True, num_classes=kernel_num) elif args.arch == "resnet152": model = models.resnet152(pretrained=True, num_classes=kernel_num) elif args.arch == "resPAnet50": model = models.resPAnet50(pretrained=True, num_classes=kernel_num) elif args.arch == "resPAnet101": model = models.resPAnet101(pretrained=True, num_classes=kernel_num) elif args.arch == "resPAnet152": model = models.resPAnet152(pretrained=True, num_classes=kernel_num) model = torch.nn.DataParallel(model).cuda() if hasattr(model.module, 'optimizer'): optimizer = model.module.optimizer else: optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, momentum=0.99, weight_decay=5e-4) #end #options to resume/use pretrained model/train from scratch title = 'icdar2019MLT' if args.pretrain: print('Using pretrained model.') assert os.path.isfile( args.pretrain), 'Error: no checkpoint directory found!' checkpoint = torch.load(args.pretrain) model.load_state_dict(checkpoint['state_dict']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Train Acc.', 'Train IOU.', 'Validate Loss', 'Validate Acc', 'Validate IOU' ]) elif args.resume: print('Resuming from checkpoint.') assert os.path.isfile( args.resume), 'Error: no checkpoint directory found!' checkpoint = torch.load(args.resume) start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: print('Training from scratch.') logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names([ 'Learning Rate', 'Train Loss', 'Train Acc.', 'Train IOU.', 'Validate Loss', 'Validate Acc', 'Validate IOU' ]) #end #start training model for epoch in range(start_epoch, args.n_epoch): adjust_learning_rate(args, optimizer, epoch) print(('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.n_epoch, optimizer.param_groups[0]['lr']))) train_loss, train_te_acc, train_ke_acc, train_te_iou, train_ke_iou = train( train_loader, model, dice_loss, optimizer, epoch, loggertf) val_loss, val_te_acc, val_ke_acc, val_te_iou, val_ke_iou = validate( validate_loader, model, dice_loss) #logging on tensorboard loggertf.scalar_summary('Training/Accuracy', train_te_acc, epoch + 1) loggertf.scalar_summary('Training/Loss', train_loss, epoch + 1) loggertf.scalar_summary('Training/IoU', train_te_iou, epoch + 1) loggertf.scalar_summary('Validation/Accuracy', val_te_acc, epoch + 1) loggertf.scalar_summary('Validation/Loss', val_loss, epoch + 1) loggertf.scalar_summary('Validation/IoU', val_te_iou, epoch + 1) #end #Boring Book Keeping print(("End of Epoch %d", epoch + 1)) print(( "Train Loss: {loss:.4f} | Train Acc: {acc: .4f} | Train IOU: {iou_t: .4f}" .format(loss=train_loss, acc=train_te_acc, iou_t=train_te_iou))) print(( "Validation Loss: {loss:.4f} | Validation Acc: {acc: .4f} | Validation IOU: {iou_t: .4f}" .format(loss=val_loss, acc=val_te_acc, iou_t=val_te_iou))) #end #Saving improving and Best Models val_loss_list.append(val_loss) if (val_loss < prev_val_loss or prev_val_loss == -1): checkpointname = "{loss:.3f}".format( loss=val_loss) + "_epoch" + str(epoch + 1) + "_checkpoint.pth.tar" save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'lr': args.lr, 'optimizer': optimizer.state_dict(), }, checkpoint=args.checkpoint, filename=checkpointname) if (val_loss < min(val_loss_list)): checkpointname = "best_checkpoint.pth.tar" save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'lr': args.lr, 'optimizer': optimizer.state_dict(), }, checkpoint=args.checkpoint, filename=checkpointname) #end prev_val_loss = val_loss logger.append([ optimizer.param_groups[0]['lr'], train_loss, train_te_acc, train_te_iou, val_loss, val_te_acc, val_te_iou ]) #end traing model logger.close()
import base64 from django.db.models import F, Q from xos.config import Config from observer.syncstep import SyncStep from core.models import Service from hpc.models import ServiceProvider, ContentProvider, CDNPrefix, OriginServer from util.logger import Logger, logging # hpclibrary will be in steps/.. parentdir = os.path.join(os.path.dirname(__file__), "..") sys.path.insert(0, parentdir) from hpclib import HpcLibrary logger = Logger(level=logging.INFO) class SyncOriginServer(SyncStep, HpcLibrary): provides = [OriginServer] observes = OriginServer requested_interval = 0 def __init__(self, **args): SyncStep.__init__(self, **args) HpcLibrary.__init__(self) def filter_hpc_service(self, objs): hpcService = self.get_hpc_service() return [
def __init__(self, schedule_task_queue): Thread.__init__(self) self.__logger = Logger.get(InfoTransfer.__name__) self.schedule_task_queue = schedule_task_queue
def run_update_check(self): if self.modules['updates']: if self.modules['updates'].checkUpdate(): Logger.log_warning("A new release is available, please check the github.")
help='Use the specified configuration file instead ' + 'of the default config.ini') parser.add_argument('-d', '--debug', help='Enables debugging logs.', action='store_true') parser.add_argument('-l', '--legacy', help='Enables sed usage.', action='store_true') args = parser.parse_args() # check args, and if none provided, load default config if args: if args.config: config = Config(args.config) else: config = Config('config.ini') if args.debug: Logger.log_info("Enabled debugging.") Logger.enable_debugging(Logger) if args.legacy: Logger.log_info("Enabled sed usage.") Adb.enable_legacy(Adb) script = ALAuto(config) script.run_update_check() Adb.service = config.network['service'] Adb.device = '-d' if (Adb.service == 'PHONE') else '-e' adb = Adb() if adb.init(): Logger.log_msg('Successfully connected to the service.') output = Adb.exec_out('wm size').decode('utf-8').strip()
def init_data(): """ 初始化数据 """ print('Initialization of the data will take a long time please wait') TradeCalendarUpdater().start() Logger.info('trade_calendar data has been initialized') StockBasicUpdater().start() Logger.info('stock_basic data has been initialized') StockCompanyUpdater().start() Logger.info('stock_company data has been initialized') StockMonthlyUpdater().start() Logger.info('stock_monthly data has been initialized') StockWeeklyUpdater().start() Logger.info('stock_weekly data has been initialized') StockDailyUpdater().start() Logger.info('stock_daily data has been initialized') DailyBasicUpdater().start() Logger.info('daily_basic data has been initialized') ConceptUpdater().start() Logger.info('concept data has been initialized') ConceptDetailUpdater().start() Logger.info('concept_detail data has been initialized') Logger.info('All initialization of data has been completed')
def __init__(self): self.log = Logger(__name__).setup()
def movement_handler(self, target_info): """ Method that handles the fleet movement until it reach its target (mystery node or enemy node). If the coordinates are wrong, they will be blacklisted and another set of coordinates to work on is obtained. If the target is a mystery node and what is found is ammo, then the method will fall in the blacklist case and search for another enemy: this is inefficient and should be improved, but it works. Args: target_info (list): coordinate_x, coordinate_y, type. Describes the selected target. Returns: (int): 1 if a fight is needed, otherwise 0. """ Logger.log_msg("Moving towards objective.") count = 0 location = [target_info[0], target_info[1]] Utils.script_sleep(1) while True: Utils.update_screen() event = self.check_movement_threads() if event["combat/button_evade"]: Logger.log_msg("Ambush was found, trying to evade.") Utils.touch_randomly(self.region["combat_ambush_evade"]) Utils.script_sleep(0.5) continue if event["combat/alert_failed_evade"]: Logger.log_warning("Failed to evade ambush.") Utils.touch_randomly(self.region["menu_combat_start"]) self.battle_handler() continue if event["menu/item_found"]: Logger.log_msg("Item found on node.") Utils.touch_randomly(Region(661, 840, 598, 203)) if Utils.find("combat/menu_emergency"): Utils.script_sleep(1) Utils.touch_randomly(self.region["hide_strat_menu"]) if target_info[2] == "mystery_node": Logger.log_msg("Target reached.") return 0 continue if event["menu/alert_info"]: Logger.log_debug("Found alert.") Utils.find_and_touch("menu/alert_close") continue if event["combat/menu_formation"] or event["combat/menu_loading"]: return 1 else: if count != 0 and count % 3 == 0: Utils.touch(location) if count > 21: Logger.log_msg( "Blacklisting location and searching for another enemy." ) self.blacklist.append(location) self.l.clear() location = self.get_closest_target(self.blacklist) count = 0 count += 1
def run_SAE_experiment(pretrain_lr=0.1,pretraining_epochs=3300, finetune_lr=0.1,training_epochs=4e5, L1_reg=0.0,L2_reg=1e-4, dataset='KSC.pkl', split_proportions = [6,2,2], hidden_layers_sizes=[20], corruption_levels=[0.], batch_size=20, log_file='log', restart = False, use_rate_schedule=True, load_pretrained_weights=False): """ Reproduce the paper... """ assert not(restart and load_pretrained_weights) assert not(load_pretrained_weights and len(hidden_layers_sizes)!=5) assert len(hidden_layers_sizes)==len(corruption_levels), \ "Error: hidden_layers_sizes and corruption_levels need to be of equal length" pretrain_rate_decay = (type(pretrain_lr)==tuple) train_rate_decay = (type(finetune_lr)==tuple) assert pretrain_rate_decay or type(pretrain_lr)==float assert train_rate_decay or type(finetune_lr)==float assert not (use_rate_schedule and train_rate_decay), ('Error:', 'Can not use adaptive rate schedule and linear rate schedule together' ) #cast number of epochsto int pretraining_epochs = int(pretraining_epochs) training_epochs = int(training_epochs) #check for linear rate schedules if pretrain_rate_decay: linear_pretrain_rates = True pretrain_rates = numpy.linspace(pretrain_lr[0],pretrain_lr[1],pretraining_epochs) else: pretrain_rates = [pretrain_lr]*pretraining_epochs if train_rate_decay: linear_train_rates = True train_rates = numpy.linspace(finetune_lr[0],finetune_lr[1],training_epochs) else: train_rates = [finetune_lr]*training_epochs #create a log object logger = Logger(log_file) #log run params if restart: logger.log("Restarting run using old best_model") logger.log("Running SAE Experiment...") logger.add_newline() logger.log("Runtime params:") logger.log("pretrain_lr=%s" % str(pretrain_lr)) logger.log("pretraining_epochs=%d" % pretraining_epochs) logger.log("finetune_lr=%s" % str(finetune_lr)) logger.log("training_epochs=%d" % training_epochs) logger.log("L1_reg=%f" % L1_reg) logger.log("L2_reg=%f" % L2_reg) logger.log("dataset=%s" % dataset) logger.log("split_proportions=%s" % str(split_proportions)) logger.log("hidden_layers_sizes=%s" % str(hidden_layers_sizes)) logger.log("corruption_levels=%s" % str(corruption_levels)) logger.log("batch_size=%d" % batch_size) logger.log("use_rate_schedule=%s" % use_rate_schedule) logger.log("load_pretrained_weights=%s" % load_pretrained_weights) logger.add_newline() datasets = load_data(dataset,split_proportions,logger) train_set_x, train_set_y = datasets[0] valid_set_x, valid_set_y = datasets[1] test_set_x, test_set_y = datasets[2] # compute number of minibatches for training, validation and testing n_train_batches = train_set_x.get_value(borrow=True).shape[0] n_train_batches /= batch_size # numpy random generator numpy_rng = numpy.random.RandomState(89677) logger.log( '... building the model') # construct the stacked denoising autoencoder class #since labels were cast to int32 need to do this to get the shared variable shared_train_set_y = train_set_y.owner.inputs[0] if not restart: sda = SdA( numpy_rng=numpy_rng, n_ins=train_set_x.get_value(borrow=True).shape[1], hidden_layers_sizes=hidden_layers_sizes, n_outs=numpy.unique(shared_train_set_y.get_value(borrow=True)).size, L1_reg=L1_reg, L2_reg=L2_reg ) elif restart: logger.log("loading model from best_model.pkl") sda = cPickle.load(open('best_model.pkl','r')) elif load_pretrained_weights: logger.log("loading model from pretrained_model.pkl") sda = cPickle.load(open('pretrained_model.pkl','r')) #create dictionary to store training stat accumulation arrays for easy pickling train_stat_dict = {} ######################### # PRETRAINING THE MODEL # ######################### pretrainig_costs = [ [] for i in xrange(sda.n_layers) ] # average pretrainig cost at each epoch train_stat_dict['pretrainig_costs'] = pretrainig_costs if not (restart or load_pretrained_weights or SKIP_PRETRAINING): logger.log( '... getting the pretraining functions') pretraining_fns = sda.pretraining_functions(train_set_x=train_set_x, batch_size=batch_size) logger.log( '... pre-training the model') start_time = timeit.default_timer() ## Pre-train layer-wise for i in xrange(sda.n_layers): # go through pretraining epochs for epoch in xrange(pretraining_epochs): # go through the training set c = [] for batch_index in xrange(n_train_batches): c.append(pretraining_fns[i](index=batch_index, corruption=corruption_levels[i], lr=pretrain_rates[epoch])) logger.log('Pre-training layer %i, epoch %d, cost ' % (i, epoch) ) logger.log( str(numpy.mean(c)) ) pretrainig_costs[i].append(numpy.mean(c)) end_time = timeit.default_timer() #save the pretrained model with open('pretrained_model.pkl', 'w') as f: cPickle.dump(sda, f) logger.log( 'The pretraining code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % ((end_time - start_time) / 60.) ) else: logger.log("skipping pretraining") ######################## # FINETUNING THE MODEL # ######################## # get the training, validation and testing function for the model logger.log( '... getting the finetuning functions') ( train_fn, validate_model_NLL, validate_model_zero_one, test_model ) = sda.build_finetune_functions( datasets=datasets, batch_size=batch_size ) logger.log( '... finetunning the model') # early-stopping parameters patience = 100 * n_train_batches # look as this many examples regardless patience_increase = 2. # wait this much longer when a new best is # found improvement_threshold = 0.995 # a relative improvement of this much is # considered significant validation_frequency = min(n_train_batches, patience / 2) # go through this many # minibatche before checking the network # on the validation set; in this case we # check every epoch best_validation_loss = numpy.inf test_score = 0. start_time = timeit.default_timer() done_looping = False epoch = 0 iter = 0 # global minibatch iteration minibatch_avg_NLL = [] #array to accumulate NLL cost over over minibatches training_NLL = [] # average training NLL cost at each epoch (really after val_freq iters) validation_NLL = [] # average validation NLL cost at each epoch validation_zero_one = [] # average zero one cost at each epoch (% misclassified) train_stat_dict['training_NLL'] = training_NLL train_stat_dict['validation_NLL'] = validation_NLL train_stat_dict['validation_zero_one'] = validation_zero_one while (epoch < training_epochs) and (not done_looping): epoch = epoch + 1 for minibatch_index in xrange(n_train_batches): iter += 1 minibatch_avg_NLL.append( train_fn(minibatch_index,lr= train_rates[epoch-1] ) ) if iter % validation_frequency == 0: """validation zero one loss """ validation_zero_one_losses = validate_model_zero_one() validation_zero_one.append( numpy.mean(validation_zero_one_losses) ) #validation NLL cost validation_NLL_losses = validate_model_NLL() validation_NLL.append( numpy.mean(validation_NLL_losses) ) #training NLL cost training_NLL.append( numpy.mean(minibatch_avg_NLL) ) minibatch_avg_NLL = [] #reset the NLL accumulator logger.log( 'epoch %i, minibatch %i/%i:' % (epoch, minibatch_index + 1, n_train_batches)) logger.log('\ttraining NLL loss: %f ' % training_NLL[-1]) logger.log('\tvalidation NLL loss: %f ' % validation_NLL[-1]) logger.log('\tvalidation zero one loss: %f %%' % (validation_zero_one[-1] * 100.)) # if we got the best validation score until now if validation_zero_one[-1] < best_validation_loss: #improve patience if loss improvement is good enough if ( validation_zero_one[-1] < best_validation_loss * improvement_threshold ): patience = max(patience, iter * patience_increase) else: print "improvemnt not good enough: %f" % (validation_zero_one[-1]/best_validation_loss) # save best validation score and iteration number best_validation_loss = validation_zero_one[-1] best_iter = iter # test it on the test set test_zero_one_losses = test_model() test_score = numpy.mean(test_zero_one_losses) print '\t\ttest zero one loss of best model %f %%' % (test_score * 100.) #save the best model with open('best_model.pkl', 'w') as f: cPickle.dump(sda, f) if patience <= iter: pass #done_looping = True #break if use_rate_schedule and epoch%100==0: if validation_NLL[epoch-100]-validation_NLL[epoch-1]<1e-4: finetune_lr = max(finetune_lr/2.,1e-6) train_rates = [finetune_lr]*training_epochs logger.log("Reducing learning rate. new rate: %f" % finetune_lr) #save train_stat_dict to a .mat file sio.savemat('train_stats.mat',train_stat_dict) #with open('train_stat_dict.pkl','w') as f: # cPickle.dump(train_stat_dict,f) end_time = timeit.default_timer() logger.log( ( 'Optimization complete with best validation score of %f %%, ' 'on iteration %i, ' 'with test performance %f %%' ) % (best_validation_loss * 100., best_iter, test_score * 100.) ) logger.log ('The training code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % ((end_time - start_time) / 60.) ) logger.close()
class MuonDetector(gps.GPSListener, adc.EventListener, object): """ Main entry point into the muon detector program. """ def __init__(self): self.log = Logger(__name__).setup() # Read our node ID self.node_id = eeprom.get_id() # Load the GPS module self.gps_module = gps.GPS(self) # Load the ADC module self.adc_module = adc.ADC(self) # Load the temperature and pressure sensors self.temp_module = temp.TemperatureSensor() self.pressure_module = pressure.PressureSensor() self.filesystem = client.FileSystem("root://localhost//tmp") self.create_event_file() # Register signal handler signal.signal(signal.SIGINT, self.signal_handler) def run(self): """ Main function that will load all the necessary modules and start taking data. """ self.gps_module.start() self.adc_module.start() self.log.info("Muon detector is running") # Wait until the process is killed signal.pause() def on_event(self, data, timestamp): """ Do something when we get an event from the ADC """ event = Event(self.node_id, data, timestamp, self.gps_module.current_timestamp, self.gps_module.current_lat, self.gps_module.current_lon, self.temp_module.read_temperature(), self.temp_module.read_humidity(), self.pressure_module.read_pressure()) self.temp_module.read_firmware_version() self.log.debug("Got event from ADC: %s" % event) # Dump the event to a file print(json.dumps(event.__dict__), file=open('/tmp/event.txt', 'a')) def on_gpgga(self, gpgga): """ Do something when we get a GPS pulse """ self.log.debug("on_gpgga(): %s %s %s" % (gpgga.timestamp, gpgga.latitude, gpgga.longitude)) def create_event_file(self): """ Creates an event file in the xrootd filesystem (if it doesn't exist) """ self.log.debug("Creating initial event file... (did you remember to " "start xrootd? ;)") status, statinfo = self.filesystem.stat("/tmp/event.txt") if statinfo is None: with client.File() as f: f.open("root://localhost//tmp/event.txt", OpenFlags.NEW) def signal_handler(self, signal, frame): """ Called when the process receives SIGINT """ self.log.info('Received Ctrl-C') self.adc_module.cleanup() sys.exit(0)
from __future__ import absolute_import from PyQt5 import QtCore, QtWidgets import gr from qtgr import GRWidget import csv from util.logger import Logger import sys import numpy as np import os from math import floor logger = Logger("gui.histogram_widget") logger.setstream("default", sys.stdout, Logger.DEBUG) class GrHistogramWidget(GRWidget): def __init__(self, *args, **kwargs): super(GrHistogramWidget, self).__init__(*args, **kwargs) self.xvalues = None self.yvalues = None self.title = None self.datapoints = None def setdata(self, xvalues, yvalues, widths, title): self.xvalues = xvalues self.yvalues = yvalues
def combat_logic_wrapper(self): """Method that fires off the necessary child methods that encapsulates the entire action of sortieing combat fleets and resolving combat. Returns: int: 1 if boss was defeated, 2 if morale is too low and 3 if dock is full. """ self.exit = 0 self.l.clear() self.blacklist.clear() while True: Utils.wait_update_screen() if Utils.find("menu/button_sort"): Utils.touch_randomly(Region(1326, 274, 35, 35)) self.exit = 3 if Utils.find("combat/alert_morale_low"): Utils.touch_randomly(Region(1326, 274, 35, 35)) self.exit = 2 break if Utils.find("commission/button_confirm"): Logger.log_msg("Found commission info message.") Utils.touch_randomly(self.region["combat_com_confirm"]) continue if Utils.find("menu/button_battle"): Logger.log_debug("Found menu battle button.") Utils.touch_randomly(self.region["menu_button_battle"]) Utils.wait_update_screen(1) continue if Utils.find("combat/menu_select_fleet"): Logger.log_debug("Found fleet select go button.") Utils.touch_randomly(self.region["fleet_menu_go"]) continue if Utils.find("combat/button_go"): Logger.log_debug("Found map summary go button.") Utils.touch_randomly(self.region["map_summary_go"]) continue if Utils.find("combat/button_retreat"): Logger.log_debug( "Found retreat button, starting clear function.") if not self.clear_map(): self.stats.increment_combat_attempted() break if self.exit == 1: self.stats.increment_combat_done() break if self.exit > 1: self.stats.increment_combat_attempted() break if Utils.find("menu/button_normal_mode"): Logger.log_debug("Disabling hard mode.") Utils.touch_randomly(Region(88, 990, 80, 40)) Utils.wait_update_screen(1) if Utils.find_and_touch('maps/map_{}'.format(self.chapter_map), 0.8): Logger.log_msg("Found specified map.") continue else: self.reach_map() continue Utils.script_sleep(1) Utils.menu_navigate("menu/button_battle") return self.exit
def run_deepNet(dataset = 'KSC.pkl', split_proportions = [6,2,2], hidden_sizes = [20], hidden_nonlinearity = lasagne.nonlinearities.rectify, dropout_probs = [0.5], learning_rate = 0.1, momentum = 0.9, num_epochs = int(5e4), minibatch_size = 64, log_file = 'log'): #create a log object logger = Logger(log_file) #log run params logger.log("Running run_deepNet Experiment...") logger.add_newline() logger.log("Runtime params:") logger.log("dataset=%s" % dataset) logger.log("split_proportions=%s" % str(split_proportions)) logger.log("hidden_sizes=%s" % str(hidden_sizes)) logger.log("hidden_nonlinearity=%s" % str(hidden_nonlinearity)) logger.log("dropout_probs=%s" % str(dropout_probs)) logger.log("learning_rate=%s" % str(learning_rate)) logger.log("momentum=%s" % str(momentum)) logger.log("num_epochs=%d" % num_epochs) logger.log("minibatch_size=%d" % minibatch_size) #Load the data train_set,val_set,test_set = load_data(dataset,split_proportions, logger,shared=False) x_train, y_train = train_set x_val, y_val = val_set x_test, y_test = test_set #normalize data to zero mean unit variance x_mean = np.mean(x_train) x_std = np.std(x_train) x_train = (x_train-x_mean)/x_std x_val = (x_val-x_mean)/x_std x_test = (x_test-x_mean)/x_std #prepare theano variables for inputs and targets input_var = T.matrix('inputs') target_var = T.ivector('targets') #build the model logger.log( '... building the model') input_size = x_train.shape[1] output_size=np.unique(y_train).size #net = build_network(input_var, #input_size, #hidden_sizes, #hidden_nonlinearity, #dropout_probs, #output_size) net = cPickle.load(open('best_model.pkl','r')) layers = lasagne.layers.get_all_layers(net) input_var = layers[0].input_var #create loss expression for training logger.log( '... building expressions and compiling train functions') predicition = lasagne.layers.get_output(net) loss = lasagne.objectives.categorical_crossentropy(predicition,target_var) loss = loss.mean() #create update expressions for training params = lasagne.layers.get_all_params(net,trainable=True) #linearly decay learning rate learning_rate = np.linspace(learning_rate[0],learning_rate[1],num_epochs) lr = theano.shared(np.array(learning_rate[0],dtype=theano.config.floatX)) #linearly grow momentum momentum = np.linspace(momentum[0],momentum[1],num_epochs) mom = theano.shared(np.array(momentum[0],dtype=theano.config.floatX)) updates = lasagne.updates.nesterov_momentum(loss,params, learning_rate=lr, momentum=mom) #create loss expression for validation/testing test_prediction = lasagne.layers.get_output(net,deterministic=True) test_loss = lasagne.objectives.categorical_crossentropy(test_prediction, target_var) test_loss = loss.mean() #create an expression for the classification accuracy test_acc = T.mean(T.eq(T.argmax(test_prediction,axis=1), target_var), dtype=theano.config.floatX) #compile the training function train_fn = theano.function([input_var,target_var], loss, updates=updates) #compile a validation function for the validation loss and accuracy val_fn = theano.function([input_var,target_var], [test_loss,test_acc]) #train the model logger.log( '... training the model') start_time = timeit.default_timer() best_validation_loss = np.inf training_NLL = [] # average training NLL cost at each epoch (really after val_freq iters) validation_NLL = [] # average validation NLL cost at each epoch validation_zero_one = [] # average zero one cost at each epoch (% misclassified) train_stat_dict = {} train_stat_dict['training_NLL'] = training_NLL train_stat_dict['validation_NLL'] = validation_NLL train_stat_dict['validation_zero_one'] = validation_zero_one for epoch in xrange(num_epochs): #do a pass over the training data lr.set_value(learning_rate[epoch]) mom.set_value(momentum[epoch]) train_err=0 train_batches=0 for batch in iterate_minibatches(x_train,y_train, minibatch_size,shuffle=True): inputs, targets = batch train_err += train_fn(inputs, targets) train_batches += 1 #do a pass over the validation data val_err = 0 val_acc = 0 val_batches = 0 for batch in iterate_minibatches(x_val, y_val, minibatch_size,shuffle=False): inputs, targets = batch err, acc = val_fn(inputs, targets) val_err += err val_acc += acc val_batches += 1 #record results training_NLL.append(train_err / train_batches) validation_NLL.append(val_err / val_batches) validation_zero_one.append(1-(val_acc/val_batches)) logger.log( 'epoch %i:' % (epoch)) logger.log('\ttraining NLL loss: %f ' % training_NLL[-1]) logger.log('\tvalidation NLL loss: %f ' % validation_NLL[-1]) logger.log('\tvalidation zero one loss: %f %%' % (validation_zero_one[-1] * 100.)) # if we got the best validation score until now if validation_zero_one[-1] < best_validation_loss: # save best validation score and iteration number best_validation_loss = validation_zero_one[-1] best_epoch = epoch #save the best model with open('best_model.pkl', 'w') as f: cPickle.dump(net,f) # update the best model in a sliding window looking back 50 epochs #window_start = max(len(validation_zero_one)-50,0) #window_end = len(validation_zero_one) #if validation_zero_one[-1] == min(validation_zero_one[window_start:window_end]): ## save best validation score and iteration number #best_window_validation_loss = validation_zero_one[-1] #best_window_epoch = epoch ##save the best model #with open('best_window_model.pkl', 'w') as f: #cPickle.dump(net,f) if (epoch-best_epoch)>1e4: logger.log("Early stopping...") break ######post training####### #save train_stat_dict to a .mat file sio.savemat('train_stats.mat',train_stat_dict) #with open('train_stat_dict.pkl','w') as f: # cPickle.dump(train_stat_dict,f) # After training, we compute and print the test error: #load best model logger.log("loading model from best_model.pkl") net = cPickle.load(open('best_model.pkl','r')) #logger.log("loading model from best_window_model.pkl") #window_net = cPickle.load(open('best_window_model.pkl','r')) test_err, test_acc = predict(net,x_test,y_test) test_score = 1 - test_acc #test_err, test_acc = predict(window_net,x_test,y_test) #window_test_score = 1 - test_acc end_time = timeit.default_timer() logger.log( ( 'Optimization complete with best validation score of %f %%, ' 'on epoch %i, ' 'with test performance %f %%' ) % (best_validation_loss * 100., best_epoch, test_score * 100.) ) logger.log ('The training code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % ((end_time - start_time) / 60.) ) logger.close()