def _init(): session = requests.Session() adapter = requests.adapters.HTTPAdapter( pool_connections=config.get_default('connection_pool'), pool_maxsize=config.get_default('connection_pool'), max_retries=config.get_default('connection_retries')) session.mount('http://', adapter) global _session _session = session
def irc_RPL_WELCOME(bot, prefix, params): """ Begin logging when the bot officially joins the network. """ global logger if logger is None: logger = BufferLogger(config.get_default("log", "dir", "greenbot"))
def run_on_variable(distribution, variable_name, values): measures = [] results = {} for value in values: kwargs = config.get_default() kwargs[variable_name] = value temp = run_exp(distribution, **kwargs) for method in temp: if method not in results: results[method] = {} results[method][str(value)] = temp[method].report() if len(measures) == 0: measures = [x for x in results[method][str(value)]] ofile = open(distribution + '_' + variable_name + '.csv', 'w') for measure in measures: ofile.write(measure + '\n') ofile.write('method') for value in values: ofile.write('\t' + str(value)) ofile.write('\n') for method in config.output_order: if method not in results: continue ofile.write(method) for value in values: ofile.write('\t' + str(results[method][str(value)][measure])) ofile.write('\n') ofile.close()
def _get(url, params, auth): try: r = requests.get( url, params=params, auth=RequestsAuth(auth) if auth is not None else None, timeout=config.get_default('connection_timeout'), headers=_headers) except Exception as e: return None, ResponseInfo(None, e) return __return_wrapper(r)
def _post(url, data, files, auth): if _session is None: _init() try: r = _session.post( url, data=data, files=files, auth=auth, headers=_headers, timeout=config.get_default('connection_timeout')) except Exception as e: return None, ResponseInfo(None, e) return __return_wrapper(r)
def buildProtocol(self, addr): bot = GreenBot() # set session properties that should be set /before/ we connect # the value is chosen from several according to this precedence order: # defaults < configuration file entries < command line arguments bot.factory = self bot.nickname = config.get_default("bot", "nickname", "greenbot") \ if (self.properties['nickname'] is None) \ else self.properties['nickname']; bot.username = config.get_default("bot", "username", "greenbot") \ if (self.properties['username'] is None) \ else self.properties['username']; bot.password = config.get_default("server", "password", "greenbot") \ if (self.properties['password'] is None) \ else self.properties['password']; bot.load_modules() # reset delay timer for reconnecting clients self.resetDelay() return bot
def __init__(self, prefix): self.prefix = prefix self.buffers = {} # if the log directory doesn't exist, create it if not os.path.isdir(prefix): os.mkdir(prefix) cycle_task = LoopingCall(self.cycle_all) duration = 0; try: duration = config.get_default("log", "cycle-duration", 3600*24) except ValueError: duration = 3600*24; cycle_task.start(duration);
def generate_data_on_variable(distribution, variable_name, values): for value in values: kwargs = config.get_default() kwargs[variable_name] = value generate_data(variable_name, distribution, **kwargs)
def main(args): config = cfg.get_default() cfg.set_params(config, args.config_path, args.set) cfg.freeze(config, True) print('- Configuration:') print(config) if config.dataset == 'FluidIceShake': n_groups = 2 n_particles = 348 elif config.dataset == 'RigidFall': n_groups = 3 n_particles = 192 elif config.dataset == 'MassRope': n_groups = 2 n_particles = 95 else: raise ValueError('Unsupported environment') # generate outputs for both train and valid data train_loader = get_dataloader(config, 'train', shuffle=False) valid_loader = get_dataloader(config, 'valid', shuffle=False) # build model model = PointSetNet( config.n_frames, config.pred_hidden, n_particles, n_groups, config.batchnorm, single_out=False, recur_pred=config.recur_pred).to(_DEVICE) # a model checkpoint must be loaded if config.load_path != '': print('- Loading model from {}'.format(config.load_path)) # load model on GPU/CPU if torch.cuda.is_available(): model.load_state_dict(torch.load(config.load_path)) else: model.load_state_dict( torch.load(config.load_path, map_location='cpu')) else: raise ValueError('- Please provide a valid checkpoint') if config.log_eval: # [[train_data_loss], [valid_data_loss]] losses = [] for loader, name in [(train_loader, 'train'), (valid_loader, 'valid')]: # load data with progress bar pbar = tqdm(loader) n_traj = 0 # create directory to save output data save_dir = os.path.join(config.run_dir, 'eval', name) if not os.path.isdir(save_dir): os.makedirs(save_dir) if config.vis_eval: vis_save_dir = os.path.join(save_dir, 'out_vis') if not os.path.isdir(vis_save_dir): os.makedirs(vis_save_dir) if config.log_eval: # [(loss, pos_loss, grp_loss) for all data in current loader] loader_loss = [] for images, positions, groups in pbar: if config.log_eval: model, _, loss, pos_loss, grp_loss = step( config, model, None, images, positions, groups, False) loader_loss.append((loss, pos_loss, grp_loss)) pbar.set_description('Loss {:f}'.format(loss)) pbar.set_description('Generating video outputs') n_traj = generate_outputs(config, model, n_traj, images, save_dir) if config.vis: visualize(config, model, n_traj // config.batch_size, n_particles, images, positions, groups, False) if config.log_eval: losses.append(loader_loss) if config.log_eval: # save all losses into JSON file stats = {} train_losses, valid_losses = losses (stats['train_losses'], stats['train_pos_losses'], stats['train_grp_losses']) = list(zip(*train_losses)) (stats['valid_losses'], stats['valid_pos_losses'], stats['valid_grp_losses']) = list(zip(*valid_losses)) with open(os.path.join(config.run_dir, 'eval_stats.json'), 'w') as fout: json.dump(stats, fout)
class ArgParser(object): """ This class is responsible for parsing the command line arguments (if any) given to gramps, and determining if a GUI or a CLI session must be started. The valid arguments are: Possible: 1/ FAMTREE : Just the family tree (name or database dir) 2/ -O, --open=FAMTREE, Open of a family tree 3/ -i, --import=FILE, Import a family tree of any format understood by an importer, optionally provide -f to indicate format 4/ -e, --export=FILE, export a family tree in required format, optionally provide -f to indicate format 5/ -f, --format=FORMAT : format after a -i or -e option 6/ -a, --action: An action (possible: 'report', 'tool') 7/ -p, --options=OPTIONS_STRING : specify options 8/ -u, --force-unlock: A locked database can be unlocked by giving this argument when opening it If the filename (no flags) is specified, the interactive session is launched using data from filename. In this mode (filename, no flags), the rest of the arguments is ignored. This is a mode suitable by default for GUI launchers, mime type handlers, and the like If no filename or -i option is given, a new interactive session (empty database) is launched, since no data is given anyway. If -O or -i option is given, but no -e or -a options are given, an interactive session is launched with the FILE (specified with -i). If both input (-O or -i) and processing (-e or -a) options are given, interactive session will not be launched. """ def __init__(self, args): """ Pass the command line arguments on creation. """ self.args = args self.open_gui = None self.open = None self.exports = [] self.actions = [] self.imports = [] self.imp_db_path = None self.list = False self.list_more = False self.list_table = False self.help = False self.usage = False self.force_unlock = False self.create = None self.runqml = False self.errors = [] self.parse_args() #------------------------------------------------------------------------- # Argument parser: sorts out given arguments #------------------------------------------------------------------------- def parse_args(self): """ Fill in lists with open, exports, imports, and actions options. Any errors are added to self.errors Possible: 1/ Just the family tree (name or database dir) 2/ -O, --open: Open of a family tree 3/ -i, --import: Import a family tree of any format understood by an importer, optionally provide -f to indicate format 4/ -e, --export: export a family tree in required format, optionally provide -f to indicate format 5/ -f, --format=FORMAT : format after a -i or -e option 6/ -a, --action: An action (possible: 'report', 'tool') 7/ -p, --options=OPTIONS_STRING : specify options 8/ -u, --force-unlock: A locked database can be unlocked by giving this argument when opening it 9/ -s --show : Show config settings 10/ -c --config=config.setting:value : Set config.setting and start Gramps without :value, the actual config.setting is shown """ try: # Convert arguments to unicode, otherwise getopt will not work # if a non latin character is used as an option (by mistake). # getopt will try to treat the first char in an utf-8 sequence. Example: # -Ärik is '-\xc3\x84rik' and getopt will respond : # option -\xc3 not recognized for arg in range(len(self.args) - 1): self.args[arg + 1] = Utils.get_unicode_path_from_env_var( self.args[arg + 1]) options, leftargs = getopt.getopt(self.args[1:], const.SHORTOPTS, const.LONGOPTS) except getopt.GetoptError, msg: # Extract the arguments in the list. # The % operator replaces the list elements with repr() of the list elemements # which is OK for latin characters, but not for non latin characters in list elements cliargs = "[ " for arg in range(len(self.args) - 1): cliargs += self.args[arg + 1] + " " cliargs += "]" # Must first do str() of the msg object. msg = unicode(str(msg)) self.errors += [ (_('Error parsing the arguments'), msg + '\n' + _("Error parsing the arguments: %s \n" "Type gramps --help for an overview of commands, or " "read the manual pages.") % cliargs) ] return if leftargs: # if there were an argument without option, # use it as a file to open and return self.open_gui = leftargs[0] print >> sys.stderr, "Trying to open: %s ..." % leftargs[0] #see if force open is on for opt_ix in range(len(options)): option, value = options[opt_ix] if option in ('-u', '--force-unlock'): self.force_unlock = True break return # Go over all given option and place them into appropriate lists cleandbg = [] need_to_quit = False for opt_ix in range(len(options)): option, value = options[opt_ix] if option in ('-O', '--open'): self.open = value elif option in ('-C', '--create'): self.create = value elif option in ('-i', '--import'): family_tree_format = None if opt_ix < len(options) - 1 \ and options[opt_ix + 1][0] in ( '-f', '--format'): family_tree_format = options[opt_ix + 1][1] self.imports.append((value, family_tree_format)) elif option in ('-e', '--export'): family_tree_format = None if opt_ix < len(options) - 1 \ and options[opt_ix + 1][0] in ( '-f', '--format'): family_tree_format = options[opt_ix + 1][1] self.exports.append((value, family_tree_format)) elif option in ('-a', '--action'): action = value if action not in ('report', 'tool'): print >> sys.stderr, "Unknown action: %s. Ignoring." % action continue options_str = "" if opt_ix < len(options)-1 \ and options[opt_ix+1][0] in ( '-p', '--options' ): options_str = options[opt_ix + 1][1] self.actions.append((action, options_str)) elif option in ('-d', '--debug'): print >> sys.stderr, 'setup debugging', value logger = logging.getLogger(value) logger.setLevel(logging.DEBUG) cleandbg += [opt_ix] elif option in ('-l'): self.list = True elif option in ('-L'): self.list_more = True elif option in ('-t'): self.list_table = True elif option in ('-s', '--show'): print "Gramps config settings from %s:" % \ config.config.filename.encode(sys.getfilesystemencoding()) for section in config.config.data: for setting in config.config.data[section]: print "%s.%s=%s" % ( section, setting, repr(config.config.data[section][setting])) print sys.exit(0) elif option in ('-c', '--config'): setting_name = value set_value = False if setting_name: if ":" in setting_name: setting_name, new_value = setting_name.split(":", 1) set_value = True if config.has_default(setting_name): setting_value = config.get(setting_name) print >> sys.stderr, "Current Gramps config setting: " \ "%s:%s" % (setting_name, repr(setting_value)) if set_value: # does a user want the default config value? if new_value in ("DEFAULT", _("DEFAULT")): new_value = config.get_default(setting_name) else: converter = Utils.get_type_converter( setting_value) new_value = converter(new_value) config.set(setting_name, new_value) # translators: indent "New" to match "Current" print >> sys.stderr, " New Gramps config " \ "setting: %s:%s" % ( setting_name, repr(config.get(setting_name)) ) else: need_to_quit = True else: print >> sys.stderr, "Gramps: no such config setting:" \ " '%s'" % setting_name need_to_quit = True cleandbg += [opt_ix] elif option in ('-h', '-?', '--help'): self.help = True elif option in ('-u', '--force-unlock'): self.force_unlock = True elif option in ('--usage'): self.usage = True elif option in ('--qml'): self.runqml = True #clean options list cleandbg.reverse() for ind in cleandbg: del options[ind] if len(options) > 0 and self.open is None and self.imports == [] \ and not (self.list or self.list_more or self.list_table or self.help or self.runqml): # Extract and convert to unicode the arguments in the list. # The % operator replaces the list elements with repr() of # the list elements, which is OK for latin characters # but not for non-latin characters in list elements cliargs = "[ " for arg in range(len(self.args) - 1): cliargs += Utils.get_unicode_path_from_env_var( self.args[arg + 1]) + " " cliargs += "]" self.errors += [(_('Error parsing the arguments'), _("Error parsing the arguments: %s \n" "To use in the command-line mode, supply at " "least one input file to process.") % cliargs)] if need_to_quit: sys.exit(0)
if opt in ("-h", "--help"): print_help() sys.exit() elif opt == "-x": print("\nDownload flag has been set.") DOWN_FLAG = False elif opt == "-f": FOLD_FLAG = True customFolder = arg elif opt == "--hang": HANG_FLAG = True elif opt in ("-s", "--search-method"): searchMethod = arg print(searchMethod) folderloc = config.get_default("Folder") url = config.get_default("URL") if searchMethod is "": searchMethod = config.get_default("Search_Method")["value"] # Set download and read location to custom folder if -f <folder> option is used if FOLD_FLAG: fileloc = folderloc["value"] + customFolder + "\\" else: date = re.sub( "/", "_", time.strftime("%x")) # Get current date with format: 'MM_DD_YY' fileloc = folderloc["value"] + date + "\\" if not os.path.exists( fileloc): # If file location does not exist, create it
def main(args): config = cfg.get_default() cfg.set_params(config, args.config_path, args.set) cfg.freeze(config, True) print('- Configuration:') print(config) if config.dataset == 'FluidIceShake': n_groups = 2 n_particles = 348 elif config.dataset == 'RigidFall': n_groups = 3 n_particles = 192 elif config.dataset == 'MassRope': n_groups = 2 n_particles = 95 else: raise ValueError('Unsupported environment') train_loader = get_dataloader(config, 'train') valid_loader = get_dataloader(config, 'valid') # build model model = PointSetNet( config.n_frames, config.pred_hidden, n_particles, n_groups, config.batchnorm, single_out=config.single_out, recur_pred=config.recur_pred, use_temp_encoder=config.use_temp_encoder, conv_temp_encoder=config.conv_temp_encoder, temp_embedding_size=config.temp_embedding_size).to(_DEVICE) print('- Model architecture:') print(model) if config.load_path != '': print('- Loading model from {}'.format(config.load_path)) model.load_state_dict(torch.load(config.load_path)) # build optimizer optimizer = torch.optim.Adam(model.parameters(), lr=config.lr) if not config.debug: print('- Training start') stats = { 'epoch': [], 'valid_losses': [], 'train_losses': [], 'train_pos_losses': [], 'train_group_losses': [], 'valid_pos_losses': [], 'valid_group_losses': [], } best_valid_loss = np.Inf for epoch in range(config.n_epochs): # training print('- Training epoch {:d}'.format(epoch)) epoch_train_losses = [] epoch_train_pos_losses = [] epoch_train_grp_losses = [] pbar = tqdm(train_loader) did_vis = False for images, positions, groups in pbar: (model, optimizer, train_loss, train_pos_loss, train_grp_loss) = step( config, model, optimizer, images, positions, groups, True) epoch_train_losses.append(train_loss) epoch_train_pos_losses.append(train_pos_loss) epoch_train_grp_losses.append(train_grp_loss) pbar.set_description('Train loss {:f}'.format(train_loss)) # visualize training results if not did_vis \ and config.vis and (epoch + 1) % config.vis_every == 0: pbar.set_description('Generating video') visualize(config, model, epoch, n_particles, images, positions, groups, True) did_vis = True train_loss = np.average(epoch_train_losses) train_pos_loss = np.average(epoch_train_pos_losses) train_grp_loss = np.average(epoch_train_grp_losses) print(('- Finish training epoch {:d}, training loss {:f},' ' pos loss {:f}, group loss {:f}').format( epoch, train_loss, train_pos_loss, train_grp_loss)) # validation print('- Evaluating epoch {:d}'.format(epoch)) epoch_valid_losses = [] epoch_valid_pos_losses = [] epoch_valid_grp_losses = [] pbar = tqdm(valid_loader) did_vis = False for images, positions, groups in pbar: with torch.no_grad(): (model, _, valid_loss, valid_pos_loss, valid_grp_loss) = step( config, model, optimizer, images, positions, groups, False) epoch_valid_losses.append(valid_loss) epoch_valid_pos_losses.append(valid_pos_loss) epoch_valid_grp_losses.append(valid_grp_loss) pbar.set_description('Valid loss {:f}'.format(valid_loss)) # visualize validation results if not did_vis \ and config.vis and (epoch + 1) % config.vis_every == 0: pbar.set_description('Generating video') visualize(config, model, epoch, n_particles, images, positions, groups, False) did_vis = True valid_loss = np.average(epoch_valid_losses) valid_pos_loss = np.average(epoch_valid_pos_losses) valid_grp_loss = np.average(epoch_valid_grp_losses) print('- Finish eval epoch {:d}, validation loss {:f}'.format( epoch, valid_loss)) if valid_loss < best_valid_loss: print('- Best model') best_valid_loss = valid_loss torch.save(model.state_dict(), os.path.join(config.run_dir, 'checkpoint_best.pth')) torch.save(model.state_dict(), os.path.join(config.run_dir, 'checkpoint_latest.pth')) print() stats['epoch'].append(epoch) stats['train_losses'].append(train_loss) stats['valid_losses'].append(valid_loss) stats['train_pos_losses'].append(train_pos_loss) stats['train_group_losses'].append(train_grp_loss) stats['valid_pos_losses'].append(valid_pos_loss) stats['valid_group_losses'].append(valid_grp_loss) with open(os.path.join(config.run_dir, 'stats.json'), 'w') as fout: json.dump(stats, fout) # Plot loss curves plot_dir = os.path.join(config.run_dir, 'curves') if not os.path.isdir(plot_dir): os.makedirs(plot_dir) utils.plot_curves( x=stats['epoch'], ys=[stats['train_losses'], stats['valid_losses']], save_path=os.path.join(plot_dir, 'loss.png'), curve_labels=['train', 'valid'], x_label='epoch', y_label='total_loss', title='Total loss') utils.plot_curves( x=stats['epoch'], ys=[stats['train_pos_losses'], stats['valid_pos_losses']], save_path=os.path.join(plot_dir, 'loss_pos.png'), curve_labels=['train', 'valid'], x_label='epoch', y_label='pos_loss', title='Position loss') utils.plot_curves( x=stats['epoch'], ys=[stats['train_group_losses'], stats['valid_group_losses']], save_path=os.path.join(plot_dir, 'loss_grp.png'), curve_labels=['train', 'valid'], x_label='epoch', y_label='grp_loss', title='Grouping loss') else: # Debug on a single batch images, positions, groups = next(iter(train_loader)) images = images[:5, :15, ...] positions = positions[:5, :15, ...] groups = groups[:5, ...] for epoch in range(config.n_epochs): (model, optimizer, train_loss, train_pos_loss, train_grp_loss) = step( config, model, optimizer, images, positions, groups, True) print(train_loss, train_pos_loss, train_grp_loss)
def main(): print(config.get_default("connection_pool")) config.set_default(connection_pool=20) print(config.get_default("connection_pool"))
def __init__(self, properties): self.prefix = config.get_default("bot", "prefix", '`') self.properties = properties
def main(argv): global connection ############### Argument Parsing ############### i = 1 while i < len(argv): if arg_matches(argv[i], '--port', '-p'): i += 1 try: connection['port'] = int(argv[i]) except ValueError: print "* Ignoring invalid port number:", argv[i] elif arg_matches(argv[i], '--ssl', '-s'): connection['ssl'] = True elif arg_matches(argv[i], '--username', '-u'): i += 1 connection['username'] = argv[i] elif arg_matches(argv[i], '--nickname', '-n'): i += 1 connection['nickname'] = argv[i] elif arg_matches(argv[i], '--password', '-P'): i += 1 connection['password'] = argv[i] elif arg_matches(argv[i], '--foreground', '-f'): runtime['foreground'] = True elif arg_matches(argv[i], '--config', '-c'): i += 1 runtime['config'] = argv[i] elif arg_matches(argv[i], '--help', '-h'): print_help() return elif arg_matches(argv[i], '--version'): version() return elif (i == len(argv) - 1): connection['addr'] = argv[i] else: print "Unrecognized argument:", argv[i] i += 1 ############### Configuration ############### config.load(runtime['config']) # connection details need to be resolved before we can do anything! if connection['addr'] is None: connection['addr'] = config.get("server", "address") # complain if we still don't have an address, then we die if connection['addr'] is None: print "Error: telepathy failed, unable to connect to the address you didn't provide." return 1 if not connection['port']: try: connection['port'] = int(config.get_default("server", "port", '6667')) except ValueError: print "Error: parameter to --port (-p) must be an integer." return 1 ############### Real Work ############### factory = greenbot.GreenbotFactory(connection) if not runtime['foreground']: # this is a daemon; get forking... print "* Forking into background..." pid = os.fork() if runtime['foreground'] or pid == 0: # child process should run in the background if not runtime['foreground']: os.setsid() # initiate the connection to the server greenbot.start(connection['addr'], connection['port'], factory, connection['ssl'])
directory = args.pop_get("--directory", default="weights") # Distribute vgpus = int(args.pop_get("--vgpu", default=1)) memory_limit = int(args.pop_get("--vram", default=12000)) gpus = args.pop_get("--gpus", default=None) distribute = create_distribute( vgpus=vgpus, memory_limit=memory_limit, gpus=gpus) # Pick up flags first initialize_only = args.pop_check("--initialize") # Default params strategy = args.pop_get("--strategy", "repeat") policy = args.pop_get("--policy", "rnnprop") default = get_default(strategy=strategy, policy=policy) # Build overrides presets = args.pop_get("--presets", "") overrides = [] if presets != "": for p in presets.split(','): overrides += get_preset(p) overrides += args.to_overrides() with distribute.scope(): # Build strategy strategy = l2o.build( default, overrides, directory=directory, strict=True) # Train if not --initialize