def __init__(self, args): win32serviceutil.ServiceFramework.__init__(self, args) self.hWaitStop = win32event.CreateEvent(None, 0, 0, None) config = get_config(parse_args=False) # Setup the correct options so the agent will use the forwarder opts, args = Values({ 'dd_url': None, 'clean': False, 'use_forwarder': True, 'disabled_dd': False }), [] agentConfig = get_config(parse_args=False, options=opts) self.restart_interval = \ int(agentConfig.get('autorestart_interval', RESTART_INTERVAL)) log.info("Autorestarting the collector ever %s seconds" % self.restart_interval) # Keep a list of running processes so we can start/end as needed. # Processes will start started in order and stopped in reverse order. self.procs = { 'forwarder': DDForwarder(config), 'collector': DDAgent(agentConfig), 'dogstatsd': DogstatsdProcess(config), 'pup': PupProcess(config), }
def create_task(screen_name, consumer_key, consumer_secret, user_key, user_secret): ecs = boto3.client('ecs', region_name=conf.get_config('AWS_REGION_NAME')) ec2 = boto3.client('ec2', region_name=conf.get_config('AWS_REGION_NAME')) try: rw = RandomWords() word = rw.random_words(count=3) password = '******' % (word[0], word[1], word[2]) tn_logger.debug('Calling run_task') task_arn = run_task(ecs, screen_name, consumer_key, consumer_secret, user_key, user_secret, password) tn_logger.debug('Done calling run_task') task_info = get_task_info(ecs, task_arn) ip_address = get_connection_ip(ec2, task_info['instanceId']) try_connecting_neo4j(ip_address, task_info['port']) tn_logger.info('Created instance for tw:%s at %s:%s' % (screen_name,ip_address,task_info['port'])) except Exception as e: tn_logger.exception(e) tn_logger.error('Error creating docker image for: tu:%s' % screen_name) print(traceback.format_exc()) print(e) raise e response_dict = { 'url': 'http://%s:%s' % (ip_address, task_info['port']), 'password': password } return response_dict
def slice_spritesheet(filename, x_size, y_size): spritesheet = Image.open(filename) spritesheet.load() processed_directory = get_config("processed", "directory") x_tiles = int(spritesheet.size[0] / x_size) y_tiles = int(spritesheet.size[1] / y_size) filenames_out = [] tile_number = int(get_config("last_file_index", "image")) # Process row for row in range(y_tiles): # Process tiles for tile in range(x_tiles): left = x_size * tile top = y_size * row right = left + x_size bottom = top + y_size box = (left, top, right, bottom) frame_image = spritesheet.crop(box) tile_number += 1 tile_filename = "".join([str(tile_number), ".png"]) tile_file = path.join(processed_directory, tile_filename) frame_image.save(tile_file) filenames_out.append(tile_filename) set_last_file_index(tile_number) return filenames_out
def modify_req(request, apikey): username = request.user valid, message, userelement, projectelement = validUserPjt(username, apikey) if not valid: return HttpResponse(json.dupms({"success": False, "message": "wrong access"}), "application/json") # 오너가 아니라면 안됨!! if projectelement.owner_uid.id != userelement.id: return HttpResponse(json.dupms({"success": False, "message": "Only the owner"}), "application/json") stagedata = json.loads(get_config("app_stages")) categorydata = json.loads(get_config("app_categories")) platformdata = json.loads(get_config("app_platforms")) projectelement.category = categorydata[request.POST["category"]] projectelement.stage = stagedata[request.POST["stage"]] projectelement.platform = platformdata[request.POST["platform"]] projectelement.name = request.POST["projectname"] projectelement.timezone = request.POST["timezone"] # project modify projectelement.save() return HttpResponse(json.dumps({"success": True, "message": "success modify project"}), "application/json")
def __init__(self, **kwargs): """Initialize the TPAM SSH Client with a paramiko SSH Client""" self.conn = paramiko.SSHClient() self.conn.set_missing_host_key_policy(paramiko.AutoAddPolicy()) # Read client configuration from ~/.tccfg or <cwd>.tccfg cfg_opts = {} config.get_config(cfg_opts) # Override .tccfg settings with anything passed in as keyword args if "tpam_key" in kwargs: kwargs["tpam_key"] = config.expand_user_path(kwargs["tpam_key"]) cfg_opts.update(kwargs) # Validate config and establish SSH connection try: config.validate(cfg_opts) except error.TPAMConfigError: raise else: try: self.conn.connect(cfg_opts["tpam_host"], username=cfg_opts["tpam_user"], key_filename=cfg_opts["tpam_key"] ) except (paramiko.SSHException, socket.error) as se: raise error.TPAMConnectError(se) # Dynamically add API methods to class # TODO: API version? - should ask Dell to include a version API call for method_name in api_2_5_11: self._generate(method_name)
def intercom_companies(): """ Get all companies from Intercom and return the data as a pandas DataFrame. """ Intercom.app_id = get_config('INTERCOM_APP_ID') Intercom.app_api_key = get_config('INTERCOM_API_KEY') company_list = [FlatDict(c.to_dict) for c in Company.all() if 'Verified' in c.custom_attributes and c.custom_attributes['Verified']] companies = [] for c in company_list: dic = {} for k, v in c.iteritems(): kn = k.lower().split(':')[-1].replace(' ', '_') dic[kn] = v companies.append(dic) companies = pd.DataFrame(companies) companies = companies.T.drop( ['0', '1', 'id', 'widget_integrated', 'app_id', 'automatic_confirm_bookings', 'minimum_book_in_advance_hours', 'phone_number', 'monthly_spend']).T companies.last_request_at = companies.last_request_at.apply( lambda x: date.fromtimestamp(x)) companies.created_at = companies.created_at.apply( lambda x: date.fromtimestamp(x)) companies.remote_created_at = companies.remote_created_at.apply( lambda x: date.fromtimestamp(x)) companies.updated_at = companies.updated_at.apply( lambda x: date.fromtimestamp(x)) return companies
def download(url): """ Return the contents of the URL in the argument. The `crawler_io` object will be used to cache the requests and keep track of redirect and error URLs. :param url: URL to download. """ crawler_io = get_crawler_io() if crawler_io.is_error_url(url): return None text = crawler_io.load_str(url) if text: if get_config("CRAWLER_VERBOSE"): print "cached", url else: if get_config("CRAWLER_VERBOSE"): print "downloading", url try: response = requests.get(url) except Exception: crawler_io.add_error_url(url) return None if response.status_code >= 400: crawler_io.add_error_url(url) return None crawler_io.add_redirect(url, response.url) text = response.content crawler_io.save_str(url, text) return text
def package_component(component_name): package_common_lib() binary_dir = os.path.join(config.get_config()['package_dir'], 'temp') output_lib_dirs = os.path.join(config.get_config()['package_dir'], 'libs') package_dir = config.get_config()['package_dir'] component_package_dir = os.path.join(config.get_config()['package_dir'], component_name) if not os.path.exists(component_package_dir): os.makedirs(component_package_dir) if os.path.exists(binary_dir): shutil.rmtree(binary_dir) os.makedirs(binary_dir) build_dir = os.path.join(config.get_config()['build_dir'], component_name) if not os.path.exists(build_dir): print "Build dir does not exists" return shutil.copytree(os.path.join(build_dir, 'MainApp'), os.path.join(binary_dir, 'MainApp')) shutil.copytree(os.path.join(build_dir, 'plugins'), os.path.join(binary_dir, 'plugins')) os.symlink(output_lib_dirs, os.path.join(binary_dir, 'libs')) os.chdir(package_dir) call(['tar', 'cjf', os.path.join(component_package_dir, "{0}_{1}.tar.gz".format(component_name, time.strftime('%y%m%d.%H%M'))), '-h', 'temp']) shutil.rmtree('temp')
def __init__(self): cur_path = os.path.dirname(os.path.abspath(__file__)) self.app = webapp.create_app() self.app.debug = True self.port = get_config().getint("setup", "listen_port") self.pidfile = os.path.join(cur_path, "..", get_config().get("setup", "pidpath")) Daemon.__init__(self, self.pidfile)
def main(): logger = logging.getLogger("pump19") logger.info("Pump19 started.") client_config = config.get_config("irc") client = protocol.Protocol(**client_config) loop = client.loop cmdhdl_config = config.get_config("cmd") # we don't need to remember this instance command.CommandHandler(client, loop=loop, **cmdhdl_config) def shutdown(): logger.info("Shutdown signal received.") client.shutdown() loop.add_signal_handler(signal.SIGTERM, shutdown) logger.info("Running protocol activity.") client.start() loop.run_forever() # before we stop the event loop, make sure all tasks are done pending = asyncio.Task.all_tasks(loop) if pending: loop.run_until_complete(asyncio.wait(pending, timeout=5)) loop.close() logger.info("Protocol activity ceased.") logger.info("Exiting...")
def lisa_config_init(): """ Generate default config from function paramteres. Specific config given by command line argument is implemented in parser_init() function. """ # read confguraton from file, use default values from OrganSegmentation cfg = config.get_default_function_config(OrganSegmentation.__init__) # for parameters without support in OrganSegmentation or to overpower # default OrganSegmentation values use cfgplus cfgplus = { 'datapath': None, 'viewermax': 225, 'viewermin': -125, 'output_datapath': os.path.expanduser("~/lisa_data"), 'input_datapath_start': os.path.expanduser("~/lisa_data") # config_version':[1,1] } cfg.update(cfgplus) # now is in cfg default values cfg = config.get_config("organ_segmentation.config", cfg) user_config_path = os.path.join(cfg['output_datapath'], "organ_segmentation.config") config.check_config_version_and_remove_old_records( user_config_path, version=config_version, records_to_save=['experiment_caption', 'lisa_operator_identifier']) # read user defined config in user data cfg = config.get_config(user_config_path, cfg) return cfg
def __init__(self, args): win32serviceutil.ServiceFramework.__init__(self, args) self.hWaitStop = win32event.CreateEvent(None, 0, 0, None) config = get_config(parse_args=False) # Setup the correct options so the agent will use the forwarder opts, args = Values({ 'autorestart': False, 'dd_url': None, 'use_forwarder': True, 'disabled_dd': False, 'profile': False }), [] agentConfig = get_config(parse_args=False, options=opts) self.hostname = get_hostname(agentConfig) # Watchdog for Windows self._collector_heartbeat, self._collector_send_heartbeat = multiprocessing.Pipe(False) self._collector_failed_heartbeats = 0 self._max_failed_heartbeats = \ MAX_FAILED_HEARTBEATS * agentConfig['check_freq'] / SERVICE_SLEEP_INTERVAL # Watch JMXFetch restarts self._MAX_JMXFETCH_RESTARTS = 3 self._count_jmxfetch_restarts = 0 # Keep a list of running processes so we can start/end as needed. # Processes will start started in order and stopped in reverse order. self.procs = { 'forwarder': ProcessWatchDog("forwarder", DDForwarder(config, self.hostname)), 'collector': ProcessWatchDog("collector", DDAgent(agentConfig, self.hostname, heartbeat=self._collector_send_heartbeat)), 'dogstatsd': ProcessWatchDog("dogstatsd", DogstatsdProcess(config, self.hostname)), 'jmxfetch': ProcessWatchDog("jmxfetch", JMXFetchProcess(config, self.hostname), 3), }
def modify_req(request, apikey): username = request.user valid , message , userelement, projectelement = validUserPjt(username,apikey) if not valid: return HttpResponse(json.dupms({'success' : False , 'message' : 'wrong access'}),'application/json') #오너가 아니라면 안됨!! if(projectelement.owner_uid.id != userelement.id): return HttpResponse(json.dupms({'success' : False , 'message' : 'Only the owner'}),'application/json') stagedata = json.loads(get_config('app_stages')) categorydata = json.loads(get_config('app_categories')) platformdata = json.loads(get_config('app_platforms')) projectelement.category = categorydata[request.POST['category']] projectelement.stage = stagedata[request.POST['stage']] projectelement.platform = platformdata[request.POST['platform']] projectelement.name = request.POST['projectname'] projectelement.timezone = request.POST['timezone'] #project modify projectelement.save(); return HttpResponse(json.dumps({'success' : True , 'message' : 'success modify project'}),'application/json')
def clone_or_get(self,URL,dest): try: git = GitConnect(dest) except IOError: GitConnect.clone(URL,dest) git = GitConnect(dest) git.repoConfig("user.name",get_config()["Git-Username"]) git.repoConfig("user.email",get_config()["Git-Email"]) return git
def switch_version(arguments): component = arguments.component component_package_dir = os.path.join(config.get_config()['package_dir'], component) deploy_dir = config.get_config()['deploy_dir'] if (not os.path.exists(deploy_dir)): os.makedirs(deploy_dir) os.symlink(os.path.join(component_package_dir, arguments.target_version), os.path.join(deploy_dir, component))
def __init__(self, hostname, command): #TODO: don't hard code this here if "gulpin" in hostname or 'dugtrio' in hostname or 'delcatty' in hostname: user = get_config("GULPIN_IMMUSER") else: user = get_config("IMMUSER") command = "ipmitool -I lanplus -H %s -U %s -P '%s' chassis power %s" % \ (hostname, user, get_config("CBMCPASSWD"), command) Command.__init__(self, command)
def registration(request): # step1: login user element가져오기 try: userElement = AuthUser.objects.get(username=request.user) except ObjectDoesNotExist: return HttpResponse('user "%s" not exists' % request.user) categorydata = json.loads(get_config("app_categories")) platformdata = json.loads(get_config("app_platforms")) stagedata = json.loads(get_config("app_stages")) # stagecolordata = json.loads(get_config('app_stages_color')) # avgcolordata = json.loads(get_config('avg_error_score_color')) countcolordata = json.loads(get_config("error_rate_color")) name = request.POST["name"] platformtxt = request.POST["platform"] stagetxt = request.POST["stage"] categorytxt = request.POST["category"] platform = platformdata[platformtxt] stage = stagedata[stagetxt] category = categorydata[categorytxt] color = ErrorRate_for_color(countcolordata, 0) # project name은 중복을 허용한다. # step2: apikey를 발급받는다. apikeysms 8자리 숫자 apikey = newApikey() print "new apikey = %s" % apikey projectElement = Projects( owner_uid=userElement, apikey=apikey, name=name, platform=platform, stage=stage, category=category, timezone="Asia/Seoul", ) projectElement.save() # step3: viwer db에 사용자와 프로젝트를 연결한다. Viewer.objects.create(uid=userElement, pid=projectElement) return HttpResponse( json.dumps( { "success": True, "prjname": name, "apikey": apikey, "color": color, "platform": platformtxt, "stage": stagetxt, } ), "application/json", )
def getSettingDict(projectelement,userelement): isowner = False if(projectelement.owner_uid.id == userelement.id): isowner = True categorydata = json.loads(get_config('app_categories')) platformdata = json.loads(get_config('app_platforms')) stagedata = json.loads(get_config('app_stages')) platformnum = get_dict_value_matchin_number(platformdata,projectelement.platform) categorynum = get_dict_value_matchin_number(categorydata,projectelement.category) stagenum = get_dict_value_matchin_number(stagedata,projectelement.stage) platformtxt = get_dict_value_matchin_key(platformdata,projectelement.platform) count = 0 for zone in pytz.common_timezones: if( zone == projectelement.timezone): break; count += 1 #print len(pytz.common_timezones) #print 'getSettingDict' + str(count) viewerlist = [] ViewerElements = Viewer.objects.select_related().filter(~Q(uid= userelement.id) ,pid = projectelement.pid) for v in ViewerElements: a = v.uid viewerlist.append(a.username) dict = { 'platform_number' : platformnum + 1, 'stage_number' : stagenum + 1, 'category_number' : categorynum + 1, 'timezonelist' : pytz.common_timezones, 'timezone_number' : count + 1, 'project_viewerlist' : viewerlist, 'owner_user' : isowner, 'app_platformlist' : platformdata.items(), 'app_categorylist' : categorydata.items(), 'app_stagelist' : stagedata.items(), 'project_name' : projectelement.name, 'project_platform' : platformtxt, 'map_file_list' : Proguardmap.objects.filter(pid = projectelement), } for mapfile in dict['map_file_list']: mapfile.date = toTimezone(mapfile.uploadtime,projectelement.timezone).__format__('%Y.%m.%d') mapfile.time = toTimezone(mapfile.uploadtime,projectelement.timezone).__format__('%H:%M') #print dict['map_file_list'] return dict
def __init__(self, args): win32serviceutil.ServiceFramework.__init__(self, args) self.hWaitStop = win32event.CreateEvent(None, 0, 0, None) config = get_config(parse_args=False) self.forwarder = DDForwarder(config) self.dogstatsd = DogstatsdThread(config) self.pup = PupThread(config) # Setup the correct options so the agent will use the forwarder opts, args = Values({"dd_url": None, "clean": False, "use_forwarder": True, "disabled_dd": False}), [] agentConfig = get_config(parse_args=False, options=opts) self.agent = DDAgent(agentConfig)
def package_common_lib(): program_lib_dirs = os.path.join(config.get_config()['source_dir'], 'libs') thirdparty_lib_dirs = os.path.join(config.get_config()['source_dir'], 'Thirdparty', 'lib', 'x64') output_lib_dirs = os.path.join(config.get_config()['package_dir'], 'libs') if not os.path.exists(output_lib_dirs): os.makedirs(output_lib_dirs) for filename in glob.glob(os.path.join(program_lib_dirs, "*")): shutil.copy2(filename, output_lib_dirs) for filename in glob.glob(os.path.join(thirdparty_lib_dirs, "*")): shutil.copy2(filename, output_lib_dirs)
def send_confirmation_mail(username, email): mail.send_mail( sender="%s Admin <%s>" % (config.get_config("site_name"), config.get_config("admin_email")), to="%s <%s>" % (username, email), subject="Confirmation mail", body=_( """Hello %(name)s, Thanks for registering on our site. Now, click this link %(link)s to confirm your account. """ ) % {"name": username, "link": generate_confirm_link(username)}, )
def ImageDisk(server_uuid, diskIndex, customerUUID, customerUsername, customerPassword, endpoint, default_user="******", isVerbose=False): # Actually just defines the global variables now (since all config bits are passed on the command line) config.get_config("") config.CUST_UUID = customerUUID config.USER_LOGIN = customerUsername config.USER_PASSWORD = customerPassword config.HOST_NAME = endpoint auth_client = setup() image_state = image_disk(auth_client, server_uuid, default_user, 0) return image_state
def get_crawler_io(): """ Return the instance that handles crawler I/O functionality for caching and keeping track of redirect and error URLs. Is a `IoRethinkdb` instance if CRAWLER_USE_RETHINK is True else `IoFs`. """ global crawler_io if crawler_io is None: if get_config("CRAWLER_USE_RETHINK"): crawler_io = IoRethinkdb(get_config("DB_HOST"), get_config("DB_PORT")) else: crawler_io = IoFs(get_config("CRAWLER_PATH")) return crawler_io
def StopVM(server_uuid, customerUUID, customerUsername, customerPassword, endpoint, isVerbose=False): # Actually just defines the global variables now (since all config bits are passed on the command line) config.get_config("") config.CUST_UUID = customerUUID config.USER_LOGIN = customerUsername config.USER_PASSWORD = customerPassword config.HOST_NAME = endpoint # config.NETWORK_TYPE = networkType auth_client = setup() server_state = stop_server(auth_client, server_uuid) return server_state
def run(infile): """Run an entire Fermi analysis (spectrum) by reading a config file""" config = get_config(infile) folder = config['out'] os.system('mkdir -p ' + folder) runfit,Fit = GenAnalysisObject(config) # create all the fit files and run gtlike runfit.PerformFit(Fit) if config['verbose'] == 'yes' : utils.GetFluxes(Fit,runfit.obs.Emin,runfit.obs.Emax) #print the flux of all the sources Result = runfit.GetAndPrintResults(Fit)#Get and dump the target specific results utils.DumpResult(Result, config) #plot the SED and model map if possible and asked if config['Spectrum']['ResultPlots'] == 'yes' : os.system("mkdir -p "+config['out'] + '/Spectrum/') if float(config['UpperLimit']['TSlimit']) < Fit.Ts(config['target']['name']): runfit.ComputeSED(Fit) outXml = utils._dump_xml(config) if SummedLike == 'yes': # the possiblity of making the model map is checked inside the function runfitback.ModelMap(outXml) runfitfront.ModelMap(outXml) else: runfit.ModelMap(outXml) # Make energy bins by run a *new* analysis Nbin = config['Ebin']['NumEnergyBins'] if int(Nbin) > 0: configfiles = utils.PrepareEbin(Fit, runfit) ind = 0 enricodir = environ.DIRS.get('ENRICO_DIR') fermidir = environ.DIRS.get('FERMI_DIR') for conf in configfiles: pathconf = folder + "/Ebin" + str(Nbin) +"/" + conf Newconfig = get_config(pathconf) cmd = enricodir+"/enrico/RunGTlike.py "+pathconf if Newconfig['Ebin']['Submit'] == 'no' : #run directly os.system(cmd) else : #submit a job to a cluster prefix = Newconfig['out'] + "/Ebin" + str(ind) scriptname = prefix + "_Script.sh" JobLog = prefix + "_Job.log" JobName = (Newconfig['target']['name'] + "_" + Newconfig['analysis']['likelihood'] + "_Ebin_" + str(ind) + "_" + Newconfig['file']['tag']) call(cmd, enricodir, fermidir, scriptname, JobLog, JobName)# submition ind+=1
def init(config_path=None, use_watchdog=False, use_forwarder=False): c = get_config(parse_args=False, cfg_path=config_path, init_logging=True) logger.debug("Configuration dogstatsd") port = c['dogstatsd_port'] interval = int(c['dogstatsd_interval']) normalize = c['dogstatsd_normalize'] api_key = c['api_key'] target = c['dd_url'] if use_forwarder: target = c['dogstatsd_target'] hostname = gethostname(c) # Create the aggregator (which is the point of communication between the # server and reporting threads. normalization_factor = 1.0 if normalize: normalization_factor = 1.0 / interval aggregator = MetricsAggregator(hostname, normalization_factor) # Start the reporting thread. reporter = Reporter(interval, aggregator, target, api_key, use_watchdog) # Start the server. server_host = '' server = Server(aggregator, server_host, port) return reporter, server
def init(config_path=None, use_watchdog=False, use_forwarder=False): c = get_config(parse_args=False, cfg_path=config_path) log.debug("Configuration dogstatsd") port = c['dogstatsd_port'] interval = int(c['dogstatsd_interval']) normalize = c['dogstatsd_normalize'] api_key = c['api_key'] non_local_traffic = c['non_local_traffic'] target = c['dd_url'] if use_forwarder: target = c['dogstatsd_target'] hostname = get_hostname(c) # Create the aggregator (which is the point of communication between the # server and reporting threads. assert 0 < interval aggregator = MetricsAggregator(hostname, interval, recent_point_threshold=c.get('recent_point_threshold', None)) # Start the reporting thread. reporter = Reporter(interval, aggregator, target, api_key, use_watchdog) # Start the server on an IPv4 stack # Default to loopback server_host = '127.0.0.1' # If specified, bind to all addressses if non_local_traffic: server_host = '' server = Server(aggregator, server_host, port) return reporter, server
def get_lc(profile): conf = get_config(profile) driver = get_driver(getattr(Provider, conf.get('driver').upper())) conn = driver(conf.get('access_id'), conf.get('secret_key')) return conn
def __init__(self): import os from config import get_config cfg = get_config() self.base = cfg['base_dir'] self.MAX_RATING = cfg['max_rating'] self.data = [] # self.data_with_links = [] for root, dirs, files in os.walk(self.base): for file in files: if file.endswith(".pdf") or file.endswith(".PDF"): relative_root = root[len(self.base):] file_info = self.__get_file_info(relative_root, file, self.MAX_RATING) self.data.append( { # 'full_path': os.path.join(root, file), 'path': os.path.join(relative_root, file).replace('\\', '/'), 'type': 'pdf', 'rating': file_info['rating'], 'year': file_info['year'], 'author': file_info['author'], 'publisher': file_info['publisher'], 'citations': file_info['citations'], 'title': file_info['title'] } ) # test if not file_info['title']: print(self.data[-1]) self.meta_data = { 'length': len(self.data), 'base': self.base }
def get_person_from_config(): """ get the username value for Ubuntu SSO from the config file """ cfg = get_config() if cfg.has_option("reviews", "username"): return cfg.get("reviews", "username") return None
def trainWithGan(mode, dis_mode,dataset, epochs, loss='l1Loss', gan_loss = 'vanllia', op='momentum', lr=1e-2, batch_size=4, load_gen_model=None, load_dis_model=None, save_dir=None, source=False, start_index = 0, LAMBDA = 1): device = torch.device('cuda:0') # Get model netG = get_model(mode, dataset, source) netG.to(device) netD = get_discriminator(dis_mode, dataset) netD.to(device) #Check if there is trained model if load_gen_model is not None: netG.load_state_dict(torch.load(load_gen_model)) if load_dis_model is not None: netD.load_state_dict(torch.load(load_dis_model)) netG.train() netD.train() # traditional loss function loss_normal = get_loss(loss) loss_normal.to(device) # gan loss loss_gan = get_loss('lsganLoss') loss_gan.to(device) # loss in Gen and Dis optimD = torch.optim.Adam(netD.parameters(), lr=5e-5, betas=(0.5, 0.999)) optimG = get_optim(netG, op, lr) train_count = int(get_config(dataset, 'train_count')) total_it = math.ceil(train_count * epochs / batch_size) epoch = train_count // batch_size if save_dir is not None: if os.path.exists(save_dir) is False: os.mkdir(save_dir) for i in range(total_it): batch_list = list(np.random.randint(1, train_count, size=[batch_size])) images, depths = get_train_data(dataset, batch_list) images = torch.from_numpy(images).cuda().float() depths = torch.from_numpy(depths).cuda() mask = torch.tensor(depths) if dataset is 'Make3D': mask = (depths > 0.0) & (depths < 70.0) elif dataset is 'NyuV2': mask = (depths > 0.0) & (depths < 10.0) #Gen the depth predicticon and this is for fake lable fake_predict = netG(images) fake_predict = torch.unsqueeze(fake_predict, 1) set_required_grad(netD, True) optimD.zero_grad() # backward the netD fake_predict_temp = torch.cat([images, fake_predict.detach()], 1) image_depth_pair = torch.cat([images, torch.unsqueeze(depths, 1)], 1) #fake pred_fake = netD(fake_predict_temp.detach()) loss_D_fake = loss_gan(fake_predict, 0.) #real real_predict = netD(image_depth_pair) loss_D_real = loss_gan(real_predict, 1.) lossD = (loss_D_fake + loss_D_real) * 0.5 lossD.backward(retain_graph=True) optimD.step() # backward netG set_required_grad(netD, False) optimG.zero_grad() fake_predict_temp = torch.cat([images, fake_predict], 1) pred_fake = netD(fake_predict_temp) loss_G_GAN = loss_gan(pred_fake, 1.) loss_l1 = loss_normal(torch.squeeze(fake_predict, 1), depths, mask) loss_G = loss_G_GAN + loss_l1 * LAMBDA loss_G.backward() optimG.step() if i % 100 == 0: print (i, loss_G_GAN.cpu().detach().numpy(), loss_l1.cpu().detach().numpy()) if i % epoch == epoch - 1: torch.save(netG.state_dict(), '{}/gen{}.pkl'.format(save_dir,start_index + i // epoch)) torch.save(netD.state_dict(), '{}/dis{}.pkl'.format(save_dir,start_index + i // epoch)) pass
"scaling_factor": self.scaling_factor, "base_metric": self.base_metric } try: self.resultdb[self.results_collection].insert_one( sizing_result_doc) except Exception as e: self.logger.error( "Unable to store sizing result doc in MongoDB: %s" % str(sizing_result_doc)) raise Exception("Unable to store sizing result doc in MongoDB: " + str(e)) if __name__ == "__main__": config = get_config() sa = SizingAnalyzer(config) end_time = 1517016459493000000 ANALYSIS_WINDOW_SECOND = 300 start_time = end_time - ANALYSIS_WINDOW_SECOND * NANOSECONDS_PER_SECOND status = sa.analyze_node_cpu(start_time, end_time) print("Node cpu finished with status: " + str(status.to_dict())) status = sa.analyze_node_memory(start_time, end_time) print("Node memory finished with status: " + str(status.to_dict())) status = sa.analyze_container_cpu(start_time, end_time) print("Container cpu finished with status: " + str(status.to_dict())) status = sa.analyze_container_memory(start_time, end_time) print("Container memory finished with status: " + str(status.to_dict()))
mse = mean_squared_error(y_true=y_test, y_pred=y_hat) mae = mean_absolute_error(y_test, y_hat) errors[i, 0] = mse errors[i, 1] = mae mean_cv_err = np.round(np.mean(errors, axis=0), 4) std_cv_err = np.round(np.std(errors, axis=0), 4) return mean_cv_err[0], std_cv_err[0], mean_cv_err[1], std_cv_err[ 1], errors if __name__ == "__main__": c = get_config() region_grid = RegionGrid(config=c) region_grid.load_weighted_mtx() region_grid.load_housing_data(c['housing_data_file']) y_house = region_grid.get_target_var("house_price") results = [] # Adjacency Average Model mse, mse_std, mae, mae_std, err_adj = cv_adj_mean( region_grid.regions, region_grid.matrix_idx_map, y_house) results.append(['adjacent avg', mse, mse_std, mae, mae_std]) y_is_valid = np.where(~np.isnan(y_house))[0] y_house = y_house[y_is_valid]
def __init__(self): self.host = config.get_config("database", "host") self.port = int(config.get_config("database", "port")) self.user = config.get_config("database", "user") self.pwd = config.get_config("database", "pwd")
def main(): args = get_config() # cuda if args.cuda and torch.cuda.is_available(): device = torch.device("cuda:0") torch.set_num_threads(1) else: device = torch.device("cpu") torch.set_num_threads(args.n_training_threads) run_dir = Path(args.model_dir) / ("run" + str(args.seed)) / 'eval' if os.path.exists(run_dir): shutil.rmtree(run_dir) os.mkdir(run_dir) log_dir = run_dir / 'logs' os.makedirs(str(log_dir)) logger = SummaryWriter(str(log_dir)) gifs_dir = run_dir / 'gifs' os.makedirs(str(gifs_dir)) num_agents = args.num_agents # actor_critic = torch.load('/home/chenjy/curriculum/results/MPE/simple_spread/check/run10/models/agent_model.pt')['model'].to(device) actor_critic = torch.load( '/home/tsing73/curriculum/results/MPE/simple_spread/homework/run4/models/agent_model.pt' )['model'].to(device) # filename = '/home/tsing73/curriculum/' # for name, para in zip(actor_critic.actor_base.state_dict(),actor_critic.actor_base.parameters()): # pdb.set_trace() # a = para.transpose(0,1).reshape(1,-1) # np.savetxt(filename+ name + '.txt',np.array(a.to('cpu').detach()),delimiter=',\n') # pdb.set_trace() actor_critic.agents_num = 2 actor_critic.boxes_num = 2 num_agents = 2 num_boxes = 2 all_frames = [] cover_rate = 0 random.seed(1) np.random.seed(1) # load files dir_path = '/home/chenjy/curriculum/diversified_left/' + ('archive_' + str(89)) if os.path.exists(dir_path): with open(dir_path, 'r') as fp: archive = fp.readlines() for i in range(len(archive)): archive[i] = np.array(archive[i][1:-2].split(), dtype=float) starts = produce_good_case_grid_pb(500, [-0.6, 0.6, -0.6, 0.6], num_agents, num_boxes) for eval_episode in range(args.eval_episodes): print(eval_episode) eval_env = MPEEnv(args) if args.save_gifs: image = eval_env.render('rgb_array', close=False)[0] all_frames.append(image) # eval_obs, _ = eval_env.reset(num_agents,num_boxes) eval_obs, _ = eval_env.reset(num_agents) # eval_obs = eval_env.new_starts_obs(start,num_agents) # eval_obs = eval_env.new_starts_obs_pb(starts[eval_episode],num_agents,num_boxes) eval_obs = np.array(eval_obs) eval_share_obs = eval_obs.reshape(1, -1) eval_recurrent_hidden_states = np.zeros( (num_agents, args.hidden_size)).astype(np.float32) eval_recurrent_hidden_states_critic = np.zeros( (num_agents, args.hidden_size)).astype(np.float32) eval_masks = np.ones((num_agents, 1)).astype(np.float32) step_cover_rate = np.zeros(shape=(args.episode_length)) for step in range(args.episode_length): calc_start = time.time() eval_actions = [] for agent_id in range(num_agents): if args.share_policy: actor_critic.eval() _, action, _, recurrent_hidden_states, recurrent_hidden_states_critic = actor_critic.act( agent_id, torch.FloatTensor(eval_share_obs), torch.FloatTensor(eval_obs[agent_id].reshape(1, -1)), torch.FloatTensor( eval_recurrent_hidden_states[agent_id]), torch.FloatTensor( eval_recurrent_hidden_states_critic[agent_id]), torch.FloatTensor(eval_masks[agent_id]), None, deterministic=True) else: actor_critic[agent_id].eval() _, action, _, recurrent_hidden_states, recurrent_hidden_states_critic = actor_critic[ agent_id].act( agent_id, torch.FloatTensor(eval_share_obs), torch.FloatTensor(eval_obs[agent_id]), torch.FloatTensor( eval_recurrent_hidden_states[agent_id]), torch.FloatTensor( eval_recurrent_hidden_states_critic[agent_id]), torch.FloatTensor(eval_masks[agent_id]), None, deterministic=True) eval_actions.append(action.detach().cpu().numpy()) eval_recurrent_hidden_states[ agent_id] = recurrent_hidden_states.detach().cpu().numpy() eval_recurrent_hidden_states_critic[ agent_id] = recurrent_hidden_states_critic.detach().cpu( ).numpy() # rearrange action eval_actions_env = [] for agent_id in range(num_agents): one_hot_action = np.zeros(eval_env.action_space[0].n) one_hot_action[eval_actions[agent_id][0]] = 1 eval_actions_env.append(one_hot_action) pdb.set_trace() # Obser reward and next obs eval_obs, eval_rewards, eval_dones, eval_infos, _ = eval_env.step( eval_actions_env) step_cover_rate[step] = eval_infos[0]['cover_rate'] eval_obs = np.array(eval_obs) eval_share_obs = eval_obs.reshape(1, -1) if args.save_gifs: image = eval_env.render('rgb_array', close=False)[0] all_frames.append(image) calc_end = time.time() elapsed = calc_end - calc_start if elapsed < args.ifi: time.sleep(ifi - elapsed) print('cover_rate: ', np.mean(step_cover_rate[-5:])) cover_rate += np.mean(step_cover_rate[-5:]) if args.save_gifs: gif_num = 0 imageio.mimsave(str(gifs_dir / args.scenario_name) + '_%i.gif' % gif_num, all_frames, duration=args.ifi) # if save_gifs: # gif_num = 0 # while os.path.exists('./gifs/' + model_dir + '/%i_%i.gif' % (gif_num, ep_i)): # gif_num += 1 # imageio.mimsave('./gifs/' + model_dir + '/%i_%i.gif' % (gif_num, ep_i), # frames, duration=ifi) print('average_cover_rate: ', cover_rate / args.eval_episodes) eval_env.close()
def get_config(self, name, parse_args=False): """ Small helper function to load fixtures configs """ return get_config(cfg_path=os.path.join(self.CONFIG_FOLDER, name), parse_args=parse_args)
def log(log_list): string = join_and_sanitize(log_list) + '\n' timestamp = time.strftime('%Y-%m-%dT%H:%M:%S ', time.gmtime()) string = timestamp + string with open(config.get_config('log_path'), 'a') as log_file: log_file.write(string)
def test(DATASET="Texas"): CONFIG = get_config(DATASET) _, _, EVALUATE, _ = datasets.fetch(DATASET, **CONFIG) cd = ChangeDetector(**CONFIG) cd.print_all_input_images(EVALUATE)
from sqlalchemy import create_engine, func from orm_schema import Pool, Host, Area from sqlalchemy.engine.url import URL from sqlalchemy.orm import sessionmaker from config import get_config from orm_io import dbIO import pika, json from threading import Thread conf_dict = get_config('conf.json') db = conf_dict['database'] broker = conf_dict['broker'] def choose_Host(cpu, memory, area): postgres_db = { 'drivername': 'postgres', 'username': '******', 'password': '******', 'host': db, 'port': 5432, 'database': 'monitor' } uri = URL(**postgres_db) engine = create_engine(uri) Session = sessionmaker(bind=engine) session = Session() io = dbIO(db) a = io.query(Area, area_name=area)[0] m = memory q = session.query(Host).filter(Host.host_free_memory >= m,
from pycnic.core import WSGI, Handler import json import yaml from model import database_init from auth import requires_auth, change_auth from config import get_config from core.ngrokmanager import NgrokManager from core.error import TunnelInstanceError, TunnelManagerError NM = NgrokManager(get_config('basic', 'docker_url')) class Info(Handler): @requires_auth() def get(self): return { "version": get_config('basic', 'version'), } class Token(Handler): def options(self): set_options(self) return '' def patch(self): old = self.request.data.get('old_token', '') new = self.request.data.get('new_token', '') if not new:
def get(self): return { "version": get_config('basic', 'version'), }
def __init__(self): self.corpus = get_corpus() self.corpus.initialize() self.config = get_config() self.model = None
def api_requests_frequency(): return float(get_config()["requests_frequency"]) # sec
from config import get_config from Learner import face_learner from data.data_pipe import get_val_pair from torchvision import transforms as trans conf = get_config(training=False) learner = face_learner(conf, inference=True) learner.load_state(conf, 'ir_se50.pth', model_only=True) # LFW evaluation lfw, lfw_issame = get_val_pair(conf.emore_folder, 'lfw') accuracy, best_threshold, roc_curve_tensor = learner.evaluate(conf, lfw, lfw_issame, nrof_folds=10, tta=False) print('lfw - accuray:{}, threshold:{}'.format(accuracy, best_threshold)) trans.ToPILImage()(roc_curve_tensor)
from urlparse import urlparse import logging import memcache import config as conf from py2neo import Graph from py2neo import neo4j from py2neo.packages.httpstream import http from py2neo.packages.httpstream import SocketError import create_task application = Flask(__name__) application.debug = True application.secret_key = conf.get_config('SESSION_KEY_SECRET') application.config['SESSION_TYPE'] = 'filesystem' oauth = OAuth() twitter = oauth.remote_app('twitter', base_url='https://api.twitter.com/1/', request_token_url='https://api.twitter.com/oauth/request_token', access_token_url='https://api.twitter.com/oauth/access_token', authorize_url='https://api.twitter.com/oauth/authenticate', consumer_key=conf.get_config('TWITTER_CONSUMER_KEY'), consumer_secret=conf.get_config('TWITTER_CONSUMER_SECRET'), access_token_method='POST' ) logging.getLogger("py2neo.cypher").setLevel(logging.CRITICAL)
def __init__(self): self.api_config = config.get_idex_api_config() self.configs = config.get_config()
def __init__( self, sqlexecute=None, prompt=None, logfile=None, auto_vertical_output=False, warn=None, liteclirc=None, ): self.sqlexecute = sqlexecute self.logfile = logfile # Load config. c = self.config = get_config(liteclirc) self.multi_line = c["main"].as_bool("multi_line") self.key_bindings = c["main"]["key_bindings"] special.set_favorite_queries(self.config) self.formatter = TabularOutputFormatter( format_name=c["main"]["table_format"]) self.formatter.litecli = self self.syntax_style = c["main"]["syntax_style"] self.less_chatty = c["main"].as_bool("less_chatty") self.cli_style = c["colors"] self.output_style = style_factory_output(self.syntax_style, self.cli_style) self.wider_completion_menu = c["main"].as_bool("wider_completion_menu") c_dest_warning = c["main"].as_bool("destructive_warning") self.destructive_warning = c_dest_warning if warn is None else warn self.login_path_as_host = c["main"].as_bool("login_path_as_host") # read from cli argument or user config file self.auto_vertical_output = auto_vertical_output or c["main"].as_bool( "auto_vertical_output") # audit log if self.logfile is None and "audit_log" in c["main"]: try: self.logfile = open(os.path.expanduser(c["main"]["audit_log"]), "a") except (IOError, OSError): self.echo( "Error: Unable to open the audit log file. Your queries will not be logged.", err=True, fg="red", ) self.logfile = False self.completion_refresher = CompletionRefresher() self.logger = logging.getLogger(__name__) self.initialize_logging() prompt_cnf = self.read_my_cnf_files(["prompt"])["prompt"] self.prompt_format = (prompt or prompt_cnf or c["main"]["prompt"] or self.default_prompt) self.prompt_continuation_format = c["main"]["prompt_continuation"] keyword_casing = c["main"].get("keyword_casing", "auto") self.query_history = [] # Initialize completer. self.completer = SQLCompleter( supported_formats=self.formatter.supported_formats, keyword_casing=keyword_casing, ) self._completer_lock = threading.Lock() # Register custom special commands. self.register_special_commands() self.prompt_app = None
from tfutils import biGRU from modules import attention_decoder from modules import highway_network from modules import conv1d_banks from modules import prenet from utils import spectrogram2wav import tensorflow as tf import numpy as np __AUTHOR__ = "kozistr" __VERSION__ = "0.1" cfg, _ = get_config() # configuration # set random seed np.random.seed(cfg.seed) tf.set_random_seed(cfg.seed) class Tacotron: def __init__(self, sess, mode="train", sample_rate=22050, vocab_size=251, embed_size=256, n_mels=80, n_fft=2048,
def main(): try: if os.path.exists('config.py'): from config import get_config config = get_config() else: raise MissingConfigurationError('Configuration is missing. Before running CLI, please copy the file \'config_tpl.pl\' from current dir, rename it to \'config.pl\', and fill') program_name = 'VxWebService Python API Connector' program_version = __version__ vxapi_cli_headers = {'User-agent': 'VxApi CLI Connector'} if config['server'].endswith('/'): config['server'] = config['server'][:-1] map_of_available_actions = OrderedDict([ (ACTION_GET_API_LIMITS, CliApiLimits(ApiApiLimits(config['api_key'], config['api_secret'], config['server']))), (ACTION_GET_ENVIRONMENTS, CliEnvironments(ApiEnvironments(config['api_key'], config['api_secret'], config['server']))), (ACTION_GET_FEED, CliFeed(ApiFeed(config['api_key'], config['api_secret'], config['server']))), (ACTION_GET_RELATIONSHIPS, CliRelationships(ApiRelationships(config['api_key'], config['api_secret'], config['server']))), (ACTION_GET_RESULT, CliResult(ApiResult(config['api_key'], config['api_secret'], config['server']))), (ACTION_GET_PUBLIC_RESULT, CliResultPublic(ApiResultPublic(config['api_key'], config['api_secret'], config['server']))), (ACTION_GET_SAMPLE_DROPPED_FILES, CliSampleDroppedFiles(ApiSampleDroppedFiles(config['api_key'], config['api_secret'], config['server']))), (ACTION_GET_SAMPLE_SCREENSHOTS, CliSampleScreenshots(ApiSampleScreenshots(config['api_key'], config['api_secret'], config['server']))), (ACTION_GET_SCAN, CliScan(ApiScan(config['api_key'], config['api_secret'], config['server']))), (ACTION_GET_STATE, CliState(ApiState(config['api_key'], config['api_secret'], config['server']))), (ACTION_GET_SUMMARY, CliSummary(ApiSummary(config['api_key'], config['api_secret'], config['server']))), (ACTION_GET_SYSTEM_BACKEND, CliSystemBackend(ApiSystemBackend(config['api_key'], config['api_secret'], config['server']))), (ACTION_GET_SYSTEM_IN_PROGRESS, CliSystemInProgress(ApiSystemInProgress(config['api_key'], config['api_secret'], config['server']))), (ACTION_GET_SYSTEM_HEARTBEAT, CliSystemHeartbeat(ApiSystemHeartbeat(config['api_key'], config['api_secret'], config['server']))), (ACTION_GET_SYSTEM_STATE, CliSystemState(ApiSystemState(config['api_key'], config['api_secret'], config['server']))), (ACTION_GET_SYSTEM_STATS, CliSystemStats(ApiSystemStats(config['api_key'], config['api_secret'], config['server']))), (ACTION_GET_SYSTEM_QUEUE_SIZE, CliSystemQueueSize(ApiSystemQueueSize(config['api_key'], config['api_secret'], config['server']))), (ACTION_GET_QUOTA, CliQuota(ApiQuota(config['api_key'], config['api_secret'], config['server']))), (ACTION_GET_URL_HASH, CliUrlHash(ApiUrlHash(config['api_key'], config['api_secret'], config['server']))), (ACTION_REANALYZE_SAMPLE, CliReanalyze(ApiReanalyze(config['api_key'], config['api_secret'], config['server']))), (ACTION_SEARCH, CliSearch(ApiSearch(config['api_key'], config['api_secret'], config['server']))), (ACTION_SUBMIT_DROPPED_FILE, CliDroppedFileSubmit(ApiDroppedFileSubmit(config['api_key'], config['api_secret'], config['server']))), (ACTION_SUBMIT_FILE, CliSubmitFile(ApiSubmitFile(config['api_key'], config['api_secret'], config['server']))), (ACTION_SUBMIT_URL_FILE, CliSubmitUrlFile(ApiSubmitFile(config['api_key'], config['api_secret'], config['server']))), (ACTION_SUBMIT_URL, CliSubmitUrl(ApiSubmitUrl(config['api_key'], config['api_secret'], config['server']))), ]) request_session = requests.Session() api_object_api_key_data = ApiApiKeyData(config['api_key'], config['api_secret'], config['server']) api_object_api_key_data.call(request_session, vxapi_cli_headers) api_key_data_json_response = api_object_api_key_data.get_response_json() if api_object_api_key_data.get_response_status_code() != 200 or api_key_data_json_response['response_code'] != 0: base_error_message = 'Can\'t retrieve data for given API Key \'{}\' in the webservice: \'{}\'. Response status code: \'{}\''.format(config['api_key'], config['server'], api_object_api_key_data.get_response_status_code()) if 'response' in api_key_data_json_response and 'error' in api_key_data_json_response['response']: base_error_message += '. Response message: \'{}\''.format(api_key_data_json_response['response']['error']) raise RetrievingApiKeyDataError(base_error_message) used_api_key_data = api_key_data_json_response['response'] parser = argparse.ArgumentParser(description=program_name, formatter_class=argparse.ArgumentDefaultsHelpFormatter, add_help=False) parser.add_argument('--version', '-ver', action='version', version='{} - version {}'.format(program_name, program_version)) CliArgumentBuilder(parser).add_help_argument() subparsers = parser.add_subparsers(help='Action names for \'{}\' auth level'.format(used_api_key_data['auth_level_name']), dest="chosen_action") for name, cli_object in map_of_available_actions.items(): if cli_object.api_object.endpoint_auth_level <= used_api_key_data['auth_level']: child_parser = subparsers.add_parser(name=name, help=cli_object.help_description, add_help=False) cli_object.add_parser_args(child_parser) args = vars(parser.parse_args()) if args['chosen_action'] is not None: cli_object = map_of_available_actions[args['chosen_action']] cli_object.attach_args(args) if args['verbose'] is True: cli_object.init_verbose_mode() print(Color.control('Running \'{}\' in version \'{}\''.format(program_name, program_version))) if args['chosen_action'] != 'get_api_limits': # API limits checking should be done here, to ensure that user always will be able to run command in help mode. Also there is no need to run it in non verbose mode. api_object_api_limits = ApiApiLimits(config['api_key'], config['api_secret'], config['server']) api_object_api_limits.call(request_session, vxapi_cli_headers) api_limits_response_json = api_object_api_limits.get_response_json() # Ignore when WebService doesn't have that endpoint if api_object_api_limits.get_response_status_code() != 404: if api_object_api_limits.get_response_status_code() != 200 or api_limits_response_json['response_code'] == -1: raise RetrievingApiKeyDataError('Can\'t check API limits before calling requested endpoint in webservice: \'{}\'. Response status code: \'{}\''.format(config['server'], api_object_api_limits.get_response_status_code())) if api_object_api_limits.get_response_status_code() == 200 and api_limits_response_json['response_code'] == 0 and api_limits_response_json['response']['limit_reached'] is True: name_of_reached_limit = api_limits_response_json['response']['name_of_reached_limit'] raise ReachedApiLimitError('Exceeded maximum API requests per {}({}). Please try again later.'.format(name_of_reached_limit, api_limits_response_json['response']['used'][name_of_reached_limit])) if api_object_api_limits.get_response_status_code() == 200 and api_limits_response_json['response_code'] == 0 and api_limits_response_json['response']['used']: api_usage = OrderedDict() api_usage_limits = api_limits_response_json['response']['limits'] is_api_limit_reached = False for period, used_limit in api_limits_response_json['response']['used'].items(): # Given request is made after checking api limits. It means that we have to add 1 to current limits, to simulate that what happen after making requested API call api_usage[period] = used_limit + 1 if is_api_limit_reached is False and api_usage[period] == api_usage_limits[period]: is_api_limit_reached = True print(Color.control('API Limits for used API Key')) print('Webservice API usage limits: {}'.format(api_usage_limits)) print('Current API usage: {}'.format(json.dumps(api_usage))) print('Is limit reached: {}'.format(Color.success('No') if is_api_limit_reached is False else Color.error('Yes'))) print(Color.control('Used API Key')) print('API Key: {}'.format(used_api_key_data['api_key'])) print('Auth Level: {}'.format(used_api_key_data['auth_level_name'])) if used_api_key_data['user'] is not None: print('User: {} ({})'.format(used_api_key_data['user']['name'], used_api_key_data['user']['email'])) print(Color.control('Request was sent at ' + '{:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()))) print('Endpoint URL: {}'.format(cli_object.api_object.get_full_endpoint_url())) print('HTTP Method: {}'.format(cli_object.api_object.request_method_name.upper())) print('Sent GET params: {}'.format(cli_object.api_object.params)) print('Sent POST params: {}'.format(cli_object.api_object.data)) print('Sent files: {}'.format(cli_object.api_object.files)) cli_object.api_object.call(request_session, vxapi_cli_headers) if args['verbose'] is True: print(Color.control('Received response at ' + '{:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()))) print('Response status code: {}'.format(cli_object.get_colored_response_status_code())) print('Message: {}'.format(cli_object.get_colored_prepared_response_msg())) print(Color.control('Showing response')) print(cli_object.get_result_msg()) cli_object.do_post_processing() if args['verbose'] is True: print('\n') else: print(Color.control('No option was selected. To check CLI options, run script in help mode: \'{} -h\''.format(__file__))) except Exception as exc: print(Color.control('During the code execution, error has occurred. Please try again or contact the support.')) print(Color.error('Message: \'{}\'.').format(str(exc)) + '\n') print(traceback.format_exc())
parser.add_argument('--use_senet', help='use_senet', default=False, required=False, type=bool) parser.add_argument('--se_reduction', help='se_reduction', default=16, required=False, type=int) return parser.parse_args() if __name__ == '__main__': conf = get_config() # args = parse_args() # conf.eval.transform = trans.Compose([ # # trans.Resize(conf.model.input_size), # trans.ToTensor(), # trans.Normalize([0.5], [0.5]) # ]) # exp_sequence = ['4@2'] exp_sequence = ['4@3'] # format_sequence = ['rgb', 'depth', 'nir'] format_sequence = ['rgb'] epoch_range = range(8, 13) conf.data_path = Path( '/home/users/jiachen.xue/anti_spoofing/data/CASIA-CeFA/phase1')
def validate_training_SVR(validation_generator, model): ''' This function is used to calculate validation loss during training PointsSVR ''' print("Validating model......") with torch.no_grad(): total_loss = 0 items = 0 for input, gtpt, _, _ in validation_generator: # Image, Point Cloud, model category, model name is given by training generator input = input.cuda() gtpt = gtpt.cuda() predpt = model(input) # Predict a spare point cloud loss, _ = chamfer_distance(predpt, gtpt) total_loss += loss.item() items += 1 return total_loss / items # Return average validation loss if __name__ == '__main__': from config import get_config experiment, opt = get_config() if opt.train == 'AE': train_AE(experiment, opt) elif opt.train == 'SVR': train_SVR(experiment, opt)
def init(config_path=None, use_watchdog=False, use_forwarder=False, args=None): """Configure the server and the reporting thread. """ c = get_config(parse_args=False, cfg_path=config_path) if (not c['use_dogstatsd'] and (args and args[0] in ['start', 'restart'] or not args)): log.info("Dogstatsd is disabled. Exiting") # We're exiting purposefully, so exit with zero (supervisor's expected # code). HACK: Sleep a little bit so supervisor thinks we've started cleanly # and thus can exit cleanly. sleep(4) sys.exit(0) log.debug("Configuring dogstatsd") port = c['dogstatsd_port'] interval = DOGSTATSD_FLUSH_INTERVAL api_key = c['api_key'] aggregator_interval = DOGSTATSD_AGGREGATOR_BUCKET_SIZE non_local_traffic = c['non_local_traffic'] forward_to_host = c.get('statsd_forward_host') forward_to_port = c.get('statsd_forward_port') event_chunk_size = c.get('event_chunk_size') recent_point_threshold = c.get('recent_point_threshold', None) target = c['dd_url'] if use_forwarder: target = c['dogstatsd_target'] hostname = get_hostname(c) # Create the aggregator (which is the point of communication between the # server and reporting threads. assert 0 < interval aggregator = MetricsBucketAggregator( hostname, aggregator_interval, recent_point_threshold=recent_point_threshold, formatter=get_formatter(c), histogram_aggregates=c.get('histogram_aggregates'), histogram_percentiles=c.get('histogram_percentiles'), utf8_decoding=c['utf8_decoding']) # Start the reporting thread. reporter = Reporter(interval, aggregator, target, api_key, use_watchdog, event_chunk_size) # Start the server on an IPv4 stack # Default to loopback server_host = c['bind_host'] # If specified, bind to all addressses if non_local_traffic: server_host = '' server = Server(aggregator, server_host, port, forward_to_host=forward_to_host, forward_to_port=forward_to_port) return reporter, server, c
import config singleDataRoot = 'simulation_data/single_neuron' parser = flagparse.FlagParser() parser.add_flag('--maxThetaFRSweeps') parser.add_flag('--maxThetaFRSweeps_median') parser.add_flag('--maxThetaFRSweeps_std') parser.add_flag('--maxThetaFRHist') parser.add_flag('--seizureProportion') args = parser.parse_args() #ps = ds.getDefaultParamSpaces() env = NoiseEnvironment(user_config=config.get_config()) if args.maxThetaFRSweeps or args.all: env.register_plotter(noisefigs.plotters.MaxMeanThetaFRSweepPlotter) if args.seizureProportion or args.all: env.register_plotter(noisefigs.plotters.PSeizureSweepPlotter) if args.maxThetaFRSweeps_std or args.all: env.register_plotter(noisefigs.plotters.MaxStdThetaFRSweepPlotter) if args.maxThetaFRSweeps_median or args.all: env.register_plotter(noisefigs.plotters.MaxMedianThetaFRSweepPlotter) if args.maxThetaFRHist or args.all: env.register_plotter(noisefigs.plotters.MaxThetaFRHistPlotter)
from datetime import datetime from ranker import features from ranker.estimators import Estimator, LONG_TERM_MODE, SHORT_TERM_MODE from Queue import Queue from threading import Thread from multiprocessing import Pool, Process import uuid from models.wrapper import Dual_Encoder_Wrapper, Human_Imitator_Wrapper, HREDQA_Wrapper, CandidateQuestions_Wrapper, DumbQuestions_Wrapper, DRQA_Wrapper, NQG_Wrapper, Echo_Wrapper, Topic_Wrapper, FactGenerator_Wrapper, AliceBot_Wrapper from models.wrapper import HRED_Wrapper import logging logging.basicConfig( level=logging.INFO, format= '%(asctime)s %(name)s.%(funcName)s +%(lineno)s: %(levelname)-8s [%(process)d] %(message)s', ) conf = config.get_config() # Script to select model conversation # Initialize all the models here and then reply with the best answer # ZMQ version. Here, all models are initialized in a separate thread # Then the main process sends out next response and context as a PUB msg, # which every model_client listens in SUB # Then all models calculate the response and send it to parent using PUSH-PULL # NN model selection algorithm can therefore work in the parent queue # to calculate the score of all incoming msgs # Set a hard time limit to discard the messages which are slow # N.B. `import zmq` **has to be** the first import. nlp = spacy.load('en', parser=False, entity=False) # Utils
from flask import Flask from flask.ext.sqlalchemy import SQLAlchemy from flask.ext.security import Security, SQLAlchemyUserDatastore from flask_mail import Mail from flask_boilerplate_utils import Boilerplate # from flask_boilerplate_utils.menu import Menu from flask.ext.menu import Menu # Initial App Setup app = Flask(__name__) # Configure the app. import config app.config_class = config.get_config() app.config.from_object(app.config_class) # Initialise the boilerplate and do Configuration Magic. Boilerplate(app) # Menu(app) # Setup flask menu Menu(app) # Setup the ORM. import models app.db = SQLAlchemy(app) app.db.register_base(models.Base) # Setup Flask-Security app.user_datastore = SQLAlchemyUserDatastore(app.db, models.User, models.Role) security = Security(app, app.user_datastore) mail = Mail(app)
def main(): # torch.autograd.set_detect_anomaly(True) torch.backends.cudnn.benchmark = True # os.environ['CUDA_LAUNCH_BLOCKING'] = '1' args = get_config()[0] torch.manual_seed(args.train.seed) torch.cuda.manual_seed(args.train.seed) torch.cuda.manual_seed_all(args.train.seed) np.random.seed(args.train.seed) model_dir = os.path.join(args.model_dir, args.exp_name) summary_dir = os.path.join(args.summary_dir, args.exp_name) if not os.path.isdir(model_dir): os.makedirs(model_dir) if not os.path.isdir(summary_dir): os.makedirs(summary_dir) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # torch.manual_seed(args.seed) args.train.num_gpu = torch.cuda.device_count() with open(os.path.join(summary_dir, 'config.yaml'), 'w') as f: yaml.dump(args, f) if args.data.dataset == 'mnist': train_data = MultiMNIST(args, mode='train') test_data = MultiMNIST(args, mode='test') val_data = MultiMNIST(args, mode='val') elif args.data.dataset == 'blender': train_data = Blender(args, mode='train') test_data = Blender(args, mode='test') val_data = Blender(args, mode='val') else: raise NotImplemented train_loader = DataLoader(train_data, batch_size=args.train.batch_size, shuffle=True, drop_last=True, num_workers=6) num_train = len(train_data) test_loader = DataLoader(test_data, batch_size=args.train.batch_size * 4, shuffle=False, drop_last=True, num_workers=6) num_test = len(test_data) val_loader = DataLoader(val_data, batch_size=args.train.batch_size * 4, shuffle=False, drop_last=True, num_workers=6) num_val = len(val_data) model = GNM(args) model.to(device) num_gpu = 1 if device.type == 'cuda' and torch.cuda.device_count() > 1: print("Let's use", torch.cuda.device_count(), "GPUs!") num_gpu = torch.cuda.device_count() model = nn.DataParallel(model) model.train() optimizer = torch.optim.RMSprop(model.parameters(), lr=args.train.lr) global_step = 0 if args.last_ckpt: global_step, args.train.start_epoch = \ load_ckpt(model, optimizer, args.last_ckpt, device) args.train.global_step = global_step args.log.phase_log = False writer = SummaryWriter(summary_dir) end_time = time.time() for epoch in range(int(args.train.start_epoch), args.train.epoch): local_count = 0 last_count = 0 for batch_idx, sample in enumerate(train_loader): imgs = sample.to(device) hyperparam_anneal(args, global_step) global_step += 1 phase_log = global_step % args.log.print_step_freq == 0 or global_step == 1 args.train.global_step = global_step args.log.phase_log = phase_log pa_recon, log_like, kl, _, _, _, log = \ model(imgs) aux_kl_pres, aux_kl_where, aux_kl_depth, aux_kl_what, aux_kl_bg, kl_pres, \ kl_where, kl_depth, kl_what, kl_global_all, kl_bg = kl aux_kl_pres_raw = aux_kl_pres.mean(dim=0) aux_kl_where_raw = aux_kl_where.mean(dim=0) aux_kl_depth_raw = aux_kl_depth.mean(dim=0) aux_kl_what_raw = aux_kl_what.mean(dim=0) aux_kl_bg_raw = aux_kl_bg.mean(dim=0) kl_pres_raw = kl_pres.mean(dim=0) kl_where_raw = kl_where.mean(dim=0) kl_depth_raw = kl_depth.mean(dim=0) kl_what_raw = kl_what.mean(dim=0) kl_bg_raw = kl_bg.mean(dim=0) log_like = log_like.mean(dim=0) aux_kl_pres = aux_kl_pres_raw * args.train.beta_aux_pres aux_kl_where = aux_kl_where_raw * args.train.beta_aux_where aux_kl_depth = aux_kl_depth_raw * args.train.beta_aux_depth aux_kl_what = aux_kl_what_raw * args.train.beta_aux_what aux_kl_bg = aux_kl_bg_raw * args.train.beta_aux_bg kl_pres = kl_pres_raw * args.train.beta_pres kl_where = kl_where_raw * args.train.beta_where kl_depth = kl_depth_raw * args.train.beta_depth kl_what = kl_what_raw * args.train.beta_what kl_bg = kl_bg_raw * args.train.beta_bg kl_global_raw = kl_global_all.sum(dim=-1).mean(dim=0) kl_global = kl_global_raw * args.train.beta_global total_loss = -(log_like - kl_pres - kl_where - kl_depth - kl_what - kl_bg - kl_global - aux_kl_pres - aux_kl_where - aux_kl_depth - aux_kl_what - aux_kl_bg) optimizer.zero_grad() total_loss.backward() clip_grad_norm_(model.parameters(), args.train.cp) optimizer.step() local_count += imgs.data.shape[0] if phase_log: bs = imgs.size(0) time_inter = time.time() - end_time count_inter = local_count - last_count print_schedule(global_step, epoch, local_count, count_inter, num_train, total_loss, time_inter) end_time = time.time() for name, param in model.named_parameters(): writer.add_histogram('param/' + name, param.cpu().detach().numpy(), global_step) if param.grad is not None: writer.add_histogram('grad/' + name, param.grad.cpu().detach(), global_step) if len(param.size()) != 1: writer.add_scalar( 'grad_std/' + name + '.grad', param.grad.cpu().detach().std().item(), global_step) writer.add_scalar( 'grad_mean/' + name + '.grad', param.grad.cpu().detach().mean().item(), global_step) for key, value in log.items(): if value is None: continue if key == 'importance_map_full_res_norm': writer.add_histogram( 'inside_value/' + key, value[value > 0].cpu().detach().numpy(), global_step) else: writer.add_histogram('inside_value/' + key, value.cpu().detach().numpy(), global_step) grid_image = make_grid( imgs.cpu().detach()[:args.log.num_summary_img].view( -1, args.data.inp_channel, args.data.img_h, args.data.img_w), args.log.num_img_per_row, normalize=False, pad_value=1) writer.add_image('train/1-image', grid_image, global_step) grid_image = make_grid(pa_recon[0].cpu().detach() [:args.log.num_summary_img].clamp( 0, 1).view(-1, args.data.inp_channel, args.data.img_h, args.data.img_w), args.log.num_img_per_row, normalize=False, pad_value=1) writer.add_image('train/2-reconstruction_overall', grid_image, global_step) if args.arch.phase_background: grid_image = make_grid(pa_recon[1].cpu().detach() [:args.log.num_summary_img].clamp( 0, 1).view(-1, args.data.inp_channel, args.data.img_h, args.data.img_w), args.log.num_img_per_row, normalize=False, pad_value=1) writer.add_image('train/3-reconstruction-fg', grid_image, global_step) grid_image = make_grid(pa_recon[2].cpu().detach() [:args.log.num_summary_img].clamp( 0, 1).view(-1, 1, args.data.img_h, args.data.img_w), args.log.num_img_per_row, normalize=False, pad_value=1) writer.add_image('train/4-reconstruction-alpha', grid_image, global_step) grid_image = make_grid(pa_recon[3].cpu().detach() [:args.log.num_summary_img].clamp( 0, 1).view(-1, args.data.inp_channel, args.data.img_h, args.data.img_w), args.log.num_img_per_row, normalize=False, pad_value=1) writer.add_image('train/5-reconstruction-bg', grid_image, global_step) bbox = visualize( imgs[:args.log.num_summary_img].cpu(), log['z_pres'].view( bs, args.arch.num_cell**2, -1)[:args.log.num_summary_img].cpu().detach(), log['z_where_scale'].view( bs, args.arch.num_cell**2, -1)[:args.log.num_summary_img].cpu().detach(), log['z_where_shift'].view( bs, args.arch.num_cell**2, -1)[:args.log.num_summary_img].cpu().detach(), only_bbox=True, phase_only_display_pres=False) bbox = bbox.view(args.log.num_summary_img, -1, 3, args.data.img_h, args.data.img_w).sum(1).clamp(0.0, 1.0) bbox_img = imgs[:args.log.num_summary_img].cpu().expand( -1, 3, -1, -1).contiguous() bbox_img[bbox.sum(dim=1, keepdim=True).expand(-1, 3, -1, -1) > 0.5] = \ bbox[bbox.sum(dim=1, keepdim=True).expand(-1, 3, -1, -1) > 0.5] grid_image = make_grid(bbox_img, args.log.num_img_per_row, normalize=False, pad_value=1) writer.add_image('train/6-bbox', grid_image, global_step) bbox_white = visualize( imgs[:args.log.num_summary_img].cpu(), log['z_pres'].view( bs, args.arch.num_cell**2, -1)[:args.log.num_summary_img].cpu().detach(), log['z_where_scale'].view( bs, args.arch.num_cell**2, -1)[:args.log.num_summary_img].cpu().detach(), log['z_where_shift'].view( bs, args.arch.num_cell**2, -1)[:args.log.num_summary_img].cpu().detach(), only_bbox=True, phase_only_display_pres=True) bbox_white = bbox_white.view(args.log.num_summary_img, -1, 3, args.data.img_h, args.data.img_w).sum(1).clamp( 0.0, 1.0) bbox_white_img = imgs[:args.log.num_summary_img].cpu().expand( -1, 3, -1, -1).contiguous() bbox_white_img[bbox_white.sum(dim=1, keepdim=True).expand(-1, 3, -1, -1) > 0.5] = \ bbox_white[bbox_white.sum(dim=1, keepdim=True).expand(-1, 3, -1, -1) > 0.5] grid_image = make_grid(bbox_white_img, args.log.num_img_per_row, normalize=False, pad_value=1) writer.add_image('train/6a-bbox-white', grid_image, global_step) grid_image = make_grid(log['recon_from_q_g'].cpu().detach() [:args.log.num_summary_img].clamp( 0, 1).view(-1, args.data.inp_channel, args.data.img_h, args.data.img_w), args.log.num_img_per_row, normalize=False, pad_value=1) writer.add_image('train/7-reconstruction_from_q_g', grid_image, global_step) if args.arch.phase_background: grid_image = make_grid( log['recon_from_q_g_fg'].cpu().detach() [:args.log.num_summary_img].clamp(0, 1).view( -1, args.data.inp_channel, args.data.img_h, args.data.img_w), args.log.num_img_per_row, normalize=False, pad_value=1) writer.add_image('train/8-recon_from_q_g-fg', grid_image, global_step) grid_image = make_grid( log['recon_from_q_g_alpha'].cpu().detach() [:args.log.num_summary_img].clamp(0, 1).view( -1, 1, args.data.img_h, args.data.img_w), args.log.num_img_per_row, normalize=False, pad_value=1) writer.add_image('train/9-recon_from_q_g-alpha', grid_image, global_step) grid_image = make_grid( log['recon_from_q_g_bg'].cpu().detach() [:args.log.num_summary_img].clamp(0, 1).view( -1, args.data.inp_channel, args.data.img_h, args.data.img_w), args.log.num_img_per_row, normalize=False, pad_value=1) writer.add_image('train/a-background_from_q_g', grid_image, global_step) writer.add_scalar('train/total_loss', total_loss.item(), global_step=global_step) writer.add_scalar('train/log_like', log_like.item(), global_step=global_step) writer.add_scalar('train/What_KL', kl_what.item(), global_step=global_step) writer.add_scalar('train/bg_KL', kl_bg.item(), global_step=global_step) writer.add_scalar('train/Where_KL', kl_where.item(), global_step=global_step) writer.add_scalar('train/Pres_KL', kl_pres.item(), global_step=global_step) writer.add_scalar('train/Depth_KL', kl_depth.item(), global_step=global_step) writer.add_scalar('train/kl_global', kl_global.item(), global_step=global_step) writer.add_scalar('train/What_KL_raw', kl_what_raw.item(), global_step=global_step) writer.add_scalar('train/bg_KL_raw', kl_bg_raw.item(), global_step=global_step) writer.add_scalar('train/Where_KL_raw', kl_where_raw.item(), global_step=global_step) writer.add_scalar('train/Pres_KL_raw', kl_pres_raw.item(), global_step=global_step) writer.add_scalar('train/Depth_KL_raw', kl_depth_raw.item(), global_step=global_step) writer.add_scalar('train/aux_What_KL', aux_kl_what.item(), global_step=global_step) writer.add_scalar('train/aux_bg_KL', aux_kl_bg.item(), global_step=global_step) writer.add_scalar('train/aux_Where_KL', aux_kl_where.item(), global_step=global_step) writer.add_scalar('train/aux_Pres_KL', aux_kl_pres.item(), global_step=global_step) writer.add_scalar('train/aux_Depth_KL', aux_kl_depth.item(), global_step=global_step) writer.add_scalar('train/aux_What_KL_raw', aux_kl_what_raw.item(), global_step=global_step) writer.add_scalar('train/aux_bg_KL_raw', aux_kl_bg_raw.item(), global_step=global_step) writer.add_scalar('train/aux_Where_KL_raw', aux_kl_where_raw.item(), global_step=global_step) writer.add_scalar('train/aux_Pres_KL_raw', aux_kl_pres_raw.item(), global_step=global_step) writer.add_scalar('train/aux_Depth_KL_raw', aux_kl_depth_raw.item(), global_step=global_step) writer.add_scalar('train/kl_global_raw', kl_global_raw.item(), global_step=global_step) writer.add_scalar('train/tau_pres', args.train.tau_pres, global_step=global_step) for i in range(args.arch.draw_step): writer.add_scalar(f'train/kl_global_raw_step_{i}', kl_global_all[:, i].mean().item(), global_step=global_step) writer.add_scalar('train/log_prob_x_given_g', log['log_prob_x_given_g'].mean(0).item(), global_step=global_step) elbo = (log_like.item() - kl_pres_raw.item() - kl_where_raw.item() - kl_depth_raw.item() - kl_what_raw.item() - kl_bg_raw.item() - kl_global_raw.item()) writer.add_scalar('train/elbo', elbo, global_step=global_step) ######################################## generation ######################################## with torch.no_grad(): model.eval() if num_gpu > 1: sample = model.module.sample()[0] else: sample = model.sample()[0] model.train() grid_image = make_grid(sample[0].cpu().detach().clamp(0, 1), args.log.num_img_per_row, normalize=False, pad_value=1) writer.add_image('generation/1-image', grid_image, global_step) if args.arch.phase_background: grid_image = make_grid(sample[1].cpu().detach().clamp( 0, 1), args.log.num_img_per_row, normalize=False, pad_value=1) writer.add_image('generation/2-fg', grid_image, global_step) grid_image = make_grid(sample[2].cpu().detach().clamp( 0, 1), args.log.num_img_per_row, normalize=False, pad_value=1) writer.add_image('generation/3-alpha', grid_image, global_step) grid_image = make_grid(sample[3].cpu().detach().clamp( 0, 1), args.log.num_img_per_row, normalize=False, pad_value=1) writer.add_image('generation/4-bg', grid_image, global_step) ###################################### generation end ###################################### last_count = local_count ###################################### ll computing ###################################### # only for logging, final ll should be computed using 100 particles if epoch % args.log.compute_nll_freq == 0: print(f'val nll at the end of epoch {epoch}') model.eval() args.log.phase_nll = True elbo_list = [] kl_list = [] ll_list = [] with torch.no_grad(): args.log.phase_log = False for batch_idx, sample in enumerate(val_loader): imgs = sample.to(device) ll_sample_list = [] for i in range(args.log.nll_num_sample): _, log_like, kl, log_imp, _, _, _ = \ model(imgs) aux_kl_pres, aux_kl_where, aux_kl_depth, aux_kl_what, \ aux_kl_bg, kl_pres, kl_where, kl_depth, kl_what, \ kl_global_all, kl_bg = kl log_imp_pres, log_imp_depth, log_imp_what, log_imp_where, log_imp_bg, log_imp_g = log_imp ll_sample_list.append( (log_like + log_imp_pres + log_imp_depth + log_imp_what + log_imp_where + log_imp_bg + log_imp_g).cpu()) # Only use one sample for elbo if i == 0: elbo_list.append((log_like - kl_pres - kl_where - kl_depth - kl_what - kl_bg - kl_global_all.sum(dim=1)).cpu()) kl_list.append( (kl_pres + kl_where + kl_depth + kl_what + kl_bg + kl_global_all.sum(dim=1)).cpu()) ll_sample = log_mean_exp(torch.stack(ll_sample_list, dim=1), dim=1) ll_list.append(ll_sample) ll_all = torch.cat(ll_list, dim=0) elbo_all = torch.cat(elbo_list, dim=0) kl_all = torch.cat(kl_list, dim=0) writer.add_scalar('val/ll', ll_all.mean(0).item(), global_step=epoch) writer.add_scalar('val/elbo', elbo_all.mean(0).item(), global_step=epoch) writer.add_scalar('val/kl', kl_all.mean(0).item(), global_step=epoch) args.log.phase_nll = False model.train() if epoch % (args.log.compute_nll_freq * 10) == 0: print(f'test nll at the end of epoch {epoch}') model.eval() args.log.phase_nll = True elbo_list = [] kl_list = [] ll_list = [] with torch.no_grad(): args.log.phase_log = False for batch_idx, sample in enumerate(test_loader): imgs = sample.to(device) ll_sample_list = [] for i in range(args.log.nll_num_sample): _, log_like, kl, log_imp, _, _, _ = \ model(imgs) aux_kl_pres, aux_kl_where, aux_kl_depth, aux_kl_what, \ aux_kl_bg, kl_pres, kl_where, kl_depth, kl_what, \ kl_global_all, kl_bg = kl log_imp_pres, log_imp_depth, log_imp_what, log_imp_where, log_imp_bg, log_imp_g = log_imp ll_sample_list.append( (log_like + log_imp_pres + log_imp_depth + log_imp_what + log_imp_where + log_imp_bg + log_imp_g).cpu()) # Only use one sample for elbo if i == 0: elbo_list.append((log_like - kl_pres - kl_where - kl_depth - kl_what - kl_bg - kl_global_all.sum(dim=1)).cpu()) kl_list.append( (kl_pres + kl_where + kl_depth + kl_what + kl_bg + kl_global_all.sum(dim=1)).cpu()) ll_sample = log_mean_exp(torch.stack(ll_sample_list, dim=1), dim=1) ll_list.append(ll_sample) ll_all = torch.cat(ll_list, dim=0) elbo_all = torch.cat(elbo_list, dim=0) kl_all = torch.cat(kl_list, dim=0) writer.add_scalar('test/ll', ll_all.mean(0).item(), global_step=epoch) writer.add_scalar('test/elbo', elbo_all.mean(0).item(), global_step=epoch) writer.add_scalar('test/kl', kl_all.mean(0).item(), global_step=epoch) args.log.phase_nll = False model.train() if epoch % args.log.save_epoch_freq == 0 and epoch != 0: save_ckpt(model_dir, model, optimizer, global_step, epoch, local_count, args.train.batch_size, num_train) save_ckpt(model_dir, model, optimizer, global_step, epoch, local_count, args.train.batch_size, num_train)
plt.subplot(3, 2, 3) plt.imshow(proposal_img[:,:,::-1].astype(np.uint8)) plt.axis('off') plt.subplot(3, 2, 4) plt.imshow(proposal_label.astype(np.uint8)) plt.axis('off') plt.subplot(3, 2, 5) plt.imshow(clamp_array(proposal_msk*255, 0, 255).astype(np.uint8)) plt.axis('off') fig.savefig(out_path, bbox_inches='tight') plt.close(fig) if cnt > 100: break if __name__ == '__main__': config, unparsed = get_config() np.random.seed(config.seed) random.seed(config.seed) torch.manual_seed(config.seed) if(config.cuda): torch.cuda.manual_seed_all(config.seed) prepare_directories(config) test_cityscapes_syn_dataloader(config)
def exec_test(config_data): add_test_info = AddTestInfo(8, "mon and mon id status") add_test_info.started_info() try: api_mon_ops = APIMonOps(**config_data) api_mon_ops.get_mon() api_mon_ops.get_mon_id() add_test_info.status("ok") except Exception, e: log.error("test error") log.error(e) add_test_info.status("error") add_test_info.completed_info() if __name__ == "__main__": config_data = config.get_config() if not config_data["auth"]: log.error("auth failed") else: exec_test(config_data)
from src.image_watcher import ImagesWatcher from src.config import get_config from src.thread.watcher_thread import WatcherThread from src.thread.heartbeat_thread import HeartbeatThread from src.thread.worker_thread import WorkerThread from src.singleton import SingleInstance # Constants # TODO do this different for the dev vs release # need to allow different configs so I can actually make sure release works APP_DATA = os.getenv('APPDATA') + '\\PathologyWatchUploader' CONFIG_FILE = f'{APP_DATA}\\config.json' LOG_DIR = f'{APP_DATA}\\logs' # Globals CONFIG = get_config(CONFIG_FILE) UPLOAD_QUEUE = queue.Queue() def on_quit(): QApplication.quit() def on_settings(appctxt): def fn(): d = SettingsDialog(appctxt, CONFIG, CONFIG_FILE) d.ui.exec_() return fn
def train_with_perceptual_loss(mode, dataset, epochs, loss='l1Loss', op='momentum', lr=1e-2, batch_size=4, load_model=None, save_dir=None, source=False, start_index = 0, with_grad = False,): device = torch.device('cuda:0') # Get model model = get_model(mode, dataset, source) model.to(device) #Check if there is trained model if load_model is not None: model.load_state_dict(torch.load(load_model)) model.train() loss_fn = get_loss(loss) loss_fn.to(device) optim = get_optim(model, op, lr) train_count = int(get_config(dataset, 'train_count')) total_it = math.ceil(train_count * epochs / batch_size) epoch = train_count // batch_size LAMBDA = 1 cnn = torchvision.models.vgg19(pretrained=True).features.to(device).eval() if save_dir is not None: if os.path.exists(save_dir) is False: os.mkdir(save_dir) for i in range(total_it): batch_list = list(np.random.randint(1, train_count, size=[batch_size])) images, depths = get_train_data(dataset, batch_list) images = torch.from_numpy(images).cuda().float() depths = torch.from_numpy(depths).cuda() optim.zero_grad() predict = model(images) mask = torch.tensor(depths) if dataset is 'Make3D': mask = (depths > 0.0) & (depths < 70.0) elif dataset is 'NyuV2': mask = (depths > 0.0) & (depths < 10.0) # per-pixel loss loss = torch.tensor([0.0]).float().to(device) pixel_loss = loss_fn(predict, depths, mask) if dataset is 'Make3D': depths = depths.clamp_(0., 70.) depths = depths / 70. predict = predict.clamp_(0., 70.) predict = predict / 70. elif dataset is 'NyuV2': depths = depths.clamp_(0., 10.) depths = depths / 10. predict = predict.clamp_(0., 10.) predict = predict / 10. # prepare for perceputal loss perceptual_loss_fn = get_loss('perceptualLoss') content = torch.stack([depths, depths, depths], dim = 1) perceptual_model, content_losses, style_losses = perceptual_loss_fn(cnn, content, content) input_perceptual = torch.stack([predict, predict, predict], dim = 1) perceptual_model(input_perceptual) style_score = 0 content_score = 0 for sl in style_losses: style_score += sl.loss for cl in content_losses: content_score += cl.loss perceptual_loss = content_score + style_score loss = pixel_loss + LAMBDA * perceptual_loss loss.backward() optim.step() if i % 100 == 0: message = 'Epoch [{}/{}]: iter {}: per-pixel loss is {}, features loss is {}'.format( i // epoch, epochs, i, pixel_loss.detach().cpu().item(), perceptual_loss.detach().cpu().item()) print (message) if i % epoch == epoch - 1: torch.save(model.state_dict(), '{}/{}.pkl'.format(save_dir,start_index + i // epoch)) pass