def main(argv): """ Entry point for etl module. """ option_parser = optparse.OptionParser(usage=DEFAULT_USAGE_TEXT) option_parser.add_option("-c", "--config", dest="config", default="config.cfg", help="Configuration file") option_parser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False, help="Show verbose output") options, _ = option_parser.parse_args(argv) if not os.path.exists(options.config): sys.stderr.write("ERROR: {} does not exist\n".format(options.config)) option_parser.print_help() return 1 config = read_config(options.config) log_dir = config['general']['log_dir'] if not os.path.exists(log_dir): os.mkdir(log_dir) filename = os.path.join(log_dir, __file__.replace(".py", ".log")) setup_logger(filename, options.verbose) logging.debug("config={}".format(json.dumps(config, indent=2))) retcode = run_etl(config) return retcode
def main(): args = _get_args() mkdirp(conf.get_opt("session_dir"), exit_on_err=True) if args.mode == "list": session_dir = conf.get_opt("session_dir") else: session_dir = os.path.join(conf.get_opt("session_dir"), args.session) if not os.path.exists(session_dir) and args.mode not in ["create", "list"]: fail("Invalid session %s" % args.session) vol_dir = os.path.join(session_dir, args.volume) if not os.path.exists(vol_dir) and args.mode not in ["create", "list"]: fail("Session %s not created with volume %s" % (args.session, args.volume)) mkdirp(os.path.join(conf.get_opt("log_dir"), args.session, args.volume), exit_on_err=True) log_file = os.path.join(conf.get_opt("log_dir"), args.session, args.volume, "cli.log") setup_logger(logger, log_file, args.debug) # globals() will have all the functions already defined. # mode_<args.mode> will be the function name to be called globals()["mode_" + args.mode](session_dir, args)
def main(): global gtmpfilename args = None try: args = _get_args() mkdirp(conf.get_opt("session_dir"), exit_on_err=True) # force the default session name if mode is "query" if args.mode == "query": args.session = "default" if args.mode == "list": session_dir = conf.get_opt("session_dir") else: session_dir = os.path.join(conf.get_opt("session_dir"), args.session) if not os.path.exists(session_dir) and \ args.mode not in ["create", "list", "query"]: fail("Invalid session %s" % args.session) # "default" is a system defined session name if args.mode in ["create", "post", "pre", "delete"] and \ args.session == "default": fail("Invalid session %s" % args.session) vol_dir = os.path.join(session_dir, args.volume) if not os.path.exists(vol_dir) and args.mode not in \ ["create", "list", "query"]: fail("Session %s not created with volume %s" % (args.session, args.volume)) mkdirp(os.path.join(conf.get_opt("log_dir"), args.session, args.volume), exit_on_err=True) log_file = os.path.join(conf.get_opt("log_dir"), args.session, args.volume, "cli.log") setup_logger(logger, log_file, args.debug) # globals() will have all the functions already defined. # mode_<args.mode> will be the function name to be called globals()["mode_" + args.mode](session_dir, args) except KeyboardInterrupt: if args is not None: if args.mode == "pre" or args.mode == "query": # cleanup session if gtmpfilename is not None: # no more interrupts until we clean up signal.signal(signal.SIGINT, signal.SIG_IGN) run_cmd_nodes("cleanup", args, tmpfilename=gtmpfilename) # Interrupted, exit with non zero error code sys.exit(2)
def init_event_server(): utils.setup_logger() # Delete Socket file if Exists try: os.unlink(SERVER_ADDRESS) except OSError: if os.path.exists(SERVER_ADDRESS): print ("Failed to cleanup socket file {0}".format(SERVER_ADDRESS), file=sys.stderr) sys.exit(1) utils.load_all() # Start the Eventing Server, UNIX DOMAIN SOCKET Server GlusterEventsServer() asyncore.loop()
def init_event_server(): utils.setup_logger() utils.load_all() port = utils.get_config("port") if port is None: sys.stderr.write("Unable to get Port details from Config\n") sys.exit(1) # Start the Eventing Server, UDP Server try: server = SocketServer.ThreadingUDPServer( (SERVER_ADDRESS, port), GlusterEventsRequestHandler) except socket.error as e: sys.stderr.write("Failed to start Eventsd: {0}\n".format(e)) sys.exit(1) server.serve_forever()
def mode_cleanup(args): working_dir = os.path.join(conf.get_opt("working_dir"), args.session, args.volume) mkdirp(os.path.join(conf.get_opt("log_dir"), args.session, args.volume), exit_on_err=True) log_file = os.path.join(conf.get_opt("log_dir"), args.session, args.volume, "changelog.log") setup_logger(logger, log_file) try: shutil.rmtree(working_dir, onerror=handle_rm_error) except (OSError, IOError) as e: logger.error("Failed to delete working directory: %s" % e) sys.exit(1)
def __init__(self): print "starting server" self.key = '' # generate public/private key self.peers = {} self.p2pfiles = [] # list of P2PFile self.server_id = os.urandom(8).encode('hex') self.max_peer_sem = threading.Semaphore(MAX_PEERS) # This is to control shard serving requests self._load_files() # load metadata and create dirs self.logger = utils.setup_logger(LOG_FILE) # setup logger self._load_keys() # load publickey self.heartbeat_thrd = threading.Timer(HEARTBEAT_TIMEOUT, self.check_clients) #Thread to monitor alive peers self.heartbeat_thrd.setDaemon(True) self.heartbeat_thrd.start()
def __init__(self, ec2, ec2_client, tag_base_name, **kwargs): """Constructor Args: ec2 (object): Aws Ec2 session ec2_client (object): Aws ec2 session tag_base_name (string): Tag base name **kwargs: Multiple arguments Raises: TypeError: Description """ BaseResources.__init__(self, ec2, ec2_client, tag_base_name) log_level = kwargs.pop("log_level", logging.WARNING) boto_log_level = kwargs.pop("boto_log_level", logging.WARNING) if kwargs: raise TypeError("Unexpected **kwargs: %r" % kwargs) self.logger = setup_logger(__name__, log_level, boto_log_level)
def main(): parser = argparse.ArgumentParser( description='pyrasite - inject code into a running python process', epilog="For updates, visit https://github.com/lmacken/pyrasite" ) parser.add_argument('pid', help="The ID of the process to inject code into") parser.add_argument('filename', help="The second argument must be a filename") parser.add_argument('--gdb-prefix', dest='gdb_prefix', help='GDB prefix (if specified during installation)', default="") parser.add_argument('--verbose', dest='verbose', help='Verbose mode', default=False, action='store_const', const=True) if len(sys.argv) == 1: parser.print_help() sys.exit(1) args = parser.parse_args() log = setup_logger() try: pid = int(args.pid) except ValueError: log.error("Error: The first argument must be a pid") sys.exit(2) filename = args.filename if filename: if not os.path.exists(filename): log.error("Error: Invalid path or file doesn't exist") sys.exit(3) else: log.error("Error: The second argument must be a filename") sys.exit(4) injector = CodeInjector(pid, verbose=args.verbose, gdb_prefix=args.gdb_prefix) injector.inject(filename)
def __init__(self): print "Client running at: %s" % str(LOCAL_ADDRESS) self.server_key = '' # server's public key self.current_peers = {} # current active peers self.peer_times = {} self.p2pfiles = [] # list of p2p files self.shards = [] # list of shards self.peer_id = None self.max_peer_sem = threading.Semaphore(MAX_PEERS) # This is to control shard serving requests self._load_files() self.logger = utils.setup_logger(LOG_FILE) self._reg_with_server() self.ping_thrd = threading.Timer(1.0, self._ping_server_thread) self.ping_thrd.setDaemon(True) self.ping_thrd.start() self.peer_thrd = threading.Timer(1.0, self._peer_contact_thread) self.peer_thrd.setDaemon(True) self.peer_thrd.start() self.timeout_thrd = threading.Timer(1.0, self._peer_timeout_thread) self.timeout_thrd.setDaemon(True) self.timeout_thrd.start()
default=".") parser.add_argument("-N", "--only-namespace-changes", help="List only namespace changes", action="store_true") return parser.parse_args() if __name__ == "__main__": args = _get_args() mkdirp(os.path.join(conf.get_opt("log_dir"), args.session, args.volume), exit_on_err=True) log_file = os.path.join(conf.get_opt("log_dir"), args.session, args.volume, "changelog.log") setup_logger(logger, log_file, args.debug) session_dir = os.path.join(conf.get_opt("session_dir"), args.session) status_file = os.path.join(session_dir, args.volume, "%s.status" % urllib.quote_plus(args.brick)) status_file_pre = status_file + ".pre" mkdirp(os.path.join(session_dir, args.volume), exit_on_err=True, logger=logger) try: with open(status_file) as f: start = int(f.read().strip()) except (ValueError, OSError, IOError): start = args.start
# path to store the doc id and token auth STORAGE_PATH = os.path.join(os.path.expanduser('~'), '.gdoc_shell') FILE_ID_PATH = os.path.join(STORAGE_PATH, 'fid') TOKEN_PATH = os.path.join(STORAGE_PATH, 'token') # drive params # using drive.file since it only creates one file SCOPES = ['https://www.googleapis.com/auth/drive.file'] DOC_MIMETYPE = 'application/vnd.google-apps.document' # msgs output table OUTPUT_INVALID_CMD = 'Command is not in the list of valid commands. Please add it to the config if you want to use it.' OUTPUT_NO_OUTPUT = 'No output' logger, handler = utils.setup_logger(__name__) DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S' def _get_http_client(cred_file): """Uses project credentials (if exists) along with requested OAuth2 scopes. Based from this example: https://github.com/gsuitedevs/python-samples/blob/master/docs/mail-merge/docs_mail_merge.py """ store = file.Storage(TOKEN_PATH) cred = store.get() if not cred or cred.invalid: flow = client.flow_from_clientsecrets(cred_file, SCOPES) cred = tools.run_flow(flow, store)
num_col = [col for col in df.columns if df[col].dtype != 'object' and col not in config.unused] not_num_col = [col for col in df.columns if col not in num_col and col not in config.unused] df = pd.get_dummies(df, columns=not_num_col, dummy_na=True) logger.info('Handling Missing Values') for col in df.columns: if col in num_col: df[col].fillna(df[col].mean(), inplace=True) if config.SCALING: sc = StandardScaler() df[col] = sc.fit_transform(df[col].values.reshape(-1, 1)) logger.info('application shape:{0}'.format(df.shape)) logger.info('Save data to directory {0}'.format(save_path)) train = df[~df['TARGET'].isnull()] test = df[df['TARGET'].isnull()].drop(['TARGET'], axis=1) train.to_pickle(os.path.join(save_path, 'application_train.pickle')) test.to_pickle(os.path.join(save_path, 'application_test.pickle')) logger.info('Finish Preprocessing') if __name__ == '__main__': NOW = datetime.datetime.now().strftime('%Y-%m-%d_%H:%M:%S') logger = setup_logger('./logs/preprocessing_{0}.log'.format(NOW)) train_df = pd.read_csv(os.path.join(config.DATA_PATH, 'application_train.csv'), nrows=None) test_df = pd.read_csv(os.path.join(config.DATA_PATH, 'application_test.csv'), nrows=None) all_df = pd.concat([train_df, test_df]) application_preprocessing(all_df, logger, config.SAVE_PATH)
import time from datetime import datetime from options import Options import utils # using the options from code_live_miccai opt = Options(isTrain=True) opt.parse() opt.save_options() os.environ['CUDA_VISIBLE_DEVICES'] = ','.join( str(x) for x in opt.train['gpus']) # set up logger logger, logger_results = utils.setup_logger(opt) opt.print_options(logger) data_transforms = { 'train': transforms.Compose([ transforms.Resize((299, 299)), transforms.RandomHorizontalFlip(), transforms.RandomVerticalFlip(), transforms.RandomRotation(randint(1, 45)), transforms.RandomAffine(degrees=0, translate=(0.1, 0.1)), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]), 'test':
from tqdm import tqdm import torch import torch.optim as optim import data from model import RNNModel from utils import process_data, build_unigram_noise, setup_parser, setup_logger from generic_model import GenModel from index_gru import IndexGRU from index_linear import IndexLinear parser = setup_parser() args = parser.parse_args() logger = setup_logger('pt-nce-%s' % args.save) logger.info(args) # Set the random seed manually for reproducibility. torch.manual_seed(args.seed) if torch.cuda.is_available(): if not args.cuda: logger.warning('You have a CUDA device, so you should probably run with --cuda') else: torch.cuda.manual_seed(args.seed) ################################################################# # Load data ################################################################# corpus = data.Corpus( path=args.data,
def __init__(self, config, model, criterion, weights_init): config['trainer']['output_dir'] = os.path.join( str(pathlib.Path(os.path.abspath(__name__)).parent), config['trainer']['output_dir']) config['name'] = config['name'] + '_' + model.name self.save_dir = os.path.join(config['trainer']['output_dir'], config['name']) self.checkpoint_dir = os.path.join(self.save_dir, 'checkpoint') if config['trainer']['resume_checkpoint'] == '' and config['trainer'][ 'finetune_checkpoint'] == '': shutil.rmtree(self.save_dir, ignore_errors=True) if not os.path.exists(self.checkpoint_dir): os.makedirs(self.checkpoint_dir) self.global_step = 0 self.start_epoch = 1 self.config = config self.model = model self.criterion = criterion # logger and tensorboard self.tensorboard_enable = self.config['trainer']['tensorboard'] self.epochs = self.config['trainer']['epochs'] self.display_interval = self.config['trainer']['display_interval'] if self.tensorboard_enable: from torch.utils.tensorboard import SummaryWriter self.writer = SummaryWriter(self.save_dir) self.logger = setup_logger(os.path.join(self.save_dir, 'train_log')) self.logger.info(pformat(self.config)) # device torch.manual_seed(self.config['trainer']['seed']) # 为CPU设置随机种子 if len(self.config['trainer']['gpus']) > 0 and torch.cuda.is_available( ): self.with_cuda = True torch.backends.cudnn.benchmark = True self.logger.info('train with gpu {} and pytorch {}'.format( self.config['trainer']['gpus'], torch.__version__)) self.gpus = { i: item for i, item in enumerate(self.config['trainer']['gpus']) } self.device = torch.device("cuda:0") #self.device = torch.device("cuda:1") torch.cuda.manual_seed( self.config['trainer']['seed']) # 为当前GPU设置随机种子 torch.cuda.manual_seed_all( self.config['trainer']['seed']) # 为所有GPU设置随机种子 else: self.with_cuda = False self.logger.info('train with cpu and pytorch {}'.format( torch.__version__)) self.device = torch.device("cpu") self.logger.info('device {}'.format(self.device)) self.metrics = { 'recall': 0, 'precision': 0, 'hmean': 0, 'train_loss': float('inf'), 'best_model': '' } self.optimizer = self._initialize('optimizer', torch.optim, model.parameters()) if self.config['trainer']['resume_checkpoint'] != '': self._laod_checkpoint(self.config['trainer']['resume_checkpoint'], resume=True) elif self.config['trainer']['finetune_checkpoint'] != '': self._laod_checkpoint( self.config['trainer']['finetune_checkpoint'], resume=False) else: if weights_init is not None: model.apply(weights_init) if self.config['lr_scheduler']['type'] != 'PolynomialLR': self.scheduler = self._initialize('lr_scheduler', torch.optim.lr_scheduler, self.optimizer) # 单机多卡 self.model.to(self.device) num_gpus = torch.cuda.device_count() if num_gpus > 1: self.model = nn.DataParallel(self.model) if self.tensorboard_enable: try: # add graph dummy_input = torch.zeros( 1, self.config['data_loader']['args']['dataset'] ['img_channel'], self.config['data_loader']['args'] ['dataset']['input_size'], self.config['data_loader'] ['args']['dataset']['input_size']).to(self.device) self.writer.add_graph(model, dummy_input) except: import traceback # self.logger.error(traceback.format_exc()) self.logger.warn('add graph to tensorboard failed')
0. You just DO WHAT THE F**K YOU WANT TO. """ from utils import setup_logger import ConfigParser import dns.tsigkeyring import dns.query import dns.update import dns.zone import base64 import falcon import uuid import os LOGGER = setup_logger("dnsbin") CONFIG = ConfigParser.ConfigParser() CONFIG.read('/etc/dnsbin.ini') KEYRING = dns.tsigkeyring.from_text({ CONFIG.get('key', 'name') : CONFIG.get('key', 'secret') }) def chunks(l, n): for i in xrange(0, len(l), n): yield l[i:i+n] class Paste(object): def on_post(self, req, resp): update = dns.update.Update(
def __init__(self, config, path, train_idx, val_idx): # base config self.epochs = config.epochs self.train_sample_num = config.train_sample_num self.val_sample_num = config.val_sample_num self.dataset = config.dataset self.train_index = train_idx self.train_test_num = config.train_test_num self.batch_size = config.batch_size self.save_model_path = os.path.join(config.save_model_path, config.dataset, 'LUPVisQ') if not os.path.exists(self.save_model_path): os.makedirs(self.save_model_path) log_name = os.path.join(self.save_model_path, 'train_LUPVisQ.log') if os.path.exists(log_name): os.remove(log_name) self.logger = setup_logger(log_name, 'LUPVisQ') self.logger_info(pformat(config)) # model prepare self.repeat_num = config.repeat_num self.class_num = config.class_num self.channel_num = config.channel_num self.tau = config.tau self.backbone_type = config.backbone_type self.lambda_ = config.lambda_ self.model_LUPVisQ = models.LUPVisQNet(512, 512, 512, self.class_num, self.backbone_type, self.channel_num, self.tau).cuda() # self.model_LUPVisQ.load_state_dict((torch.load('./result/ava_database/LUPVisQ/'))) # optimizer prepare self.lr = config.lr self.momentum = config.momentum self.weight_decay = config.weight_decay self.lr_decay_rate = config.lr_decay_rate paras = [ { 'params': self.model_LUPVisQ.parameters(), 'lr': self.lr }, ] self.optimizer = torch.optim.SGD(paras, momentum=self.momentum, weight_decay=1e-4) # dataset prepare train_loader = data_loader.LUPVisQDataLoader( config.dataset, path, train_idx, config.patch_size, config.train_sample_num, batch_size=config.batch_size, num_workers=config.num_workers, istrain='train') val_loader = data_loader.LUPVisQDataLoader( config.dataset, path, val_idx, config.patch_size, config.val_sample_num, batch_size=config.batch_size, num_workers=config.num_workers, istrain='val') self.train_data = train_loader.get_data() self.val_data = val_loader.get_data() self.logger_info('train with device {} and pytorch {}'.format( 0, torch.__version__))
help="True or False", default=False) args = parser.parse_args() os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu_id) CKPT_BASE_NAME = args.backbone LOGFILE_NAME = CKPT_BASE_NAME + '_logfile' experiment_ckpt_path = args.checkpoint_folder_path + args.dataset + '_' + args.split + '_' + CKPT_BASE_NAME if not os.path.exists(args.checkpoint_folder_path): os.mkdir(args.checkpoint_folder_path) if not os.path.exists(experiment_ckpt_path): os.mkdir(experiment_ckpt_path) os.mkdir(experiment_ckpt_path + '/' + 'backups') formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') logger = setup_logger('logger', formatter, LOGFILE_NAME) dataset_folder_path = os.path.join(args.dataset_folder, args.dataset, args.split) with open(dataset_folder_path + '/train_obj.pkl', 'rb') as f: train_obj = pickle.load(f) with open(dataset_folder_path + '/test_obj.pkl', 'rb') as f: test_obj = pickle.load(f) with open(dataset_folder_path + '/meta.pkl', 'rb') as f: meta_dict = pickle.load(f) with open(dataset_folder_path + '/class_to_idx.pkl', 'rb') as f: class_to_idx = pickle.load(f) with open(dataset_folder_path + '/idx_to_class.pkl', 'rb') as f: idx_to_class = pickle.load(f) if args.dataset in ['mnist', 'svhn', 'cifar10']:
opt.add_option('--dhcp-subnetmask', dest='dhcp_subnetmask', default=DHCP_DEFAULT_SUBNETMASK, action='store', help='DHCP lease subnet mask') opt.add_option('--dhcp-gateway', dest='dhcp_gateway', default=DHCP_DEFAULT_GW, action='store', help='DHCP lease gateway') opt.add_option('--dhcp-dns', dest='dhcp_dns', default=DHCP_DEFAULT_DNS, action='store', help='DHCP lease DNS') opt.add_option('--dhcp-bcast', dest='dhcp_bcast', default=DHCP_DEFAULT_BCAST, action='store', help='DHCP lease broadcast') opt.add_option('--dhcp-fileserver', dest='dhcp_fileserver', default='', action='store', help='DHCP lease fileserver IP (option 66)') opt.add_option('--dhcp-filename', dest='dhcp_filename', default='', action='store', help='DHCP lease filename (option 67)') options, args = opt.parse_args(sys.argv[1:]) main_logger = utils.setup_logger('main_logger', options.logfile, options.debug) sip_logger = utils.setup_logger('sip_logger', options.sip_logfile, options.debug, str_format='%(asctime)s %(message)s') main_logger.info("Starting application") main_logger.debug("SIP: Writing SIP messages in %s log file" % options.sip_logfile) main_logger.debug("SIP: Authentication password: %s" % options.sip_password) main_logger.debug("Logfile: %s" % options.logfile) if not options.terminal: import gui import Tkinter as tk root = tk.Tk() app = gui.MainApplication(root, options, main_logger) root.title(sys.argv[0])
import os import validictory import utils def run_main(main_func, schema, args): if len(args) < 2: print "Usage: " + args[0] + " <json config file>" print "For possible configs: " + args[0] + " help" sys.exit(1) if args[1].lower() in ["--h", "-h", "help", "-help", "--help"]: #pprint.pprint(schema) print json.dumps(schema, indent=2) sys.exit(0) arguments = {} for i in xrange(1, len(args)): try: config = args[i] print >> sys.stderr, "Merging %s into configuration" %(config,) arguments.update(json.load(open(config))) except Exception, e: print e sys.exit(1) validictory.validate(arguments, schema) logging_ini = None if "logging_ini" in arguments: logging_ini = arguments["logging_ini"] utils.setup_logger(logging_ini) main_func(arguments)
def main(): """Parse command line options/arguments and execute.""" try: arg_names = ["help", "version", "quick", "strict", "debug", "stop-tag="] opts, args = getopt.getopt(sys.argv[1:], "hvqsdct:v", arg_names) except getopt.GetoptError: usage(2) detailed = True stop_tag = DEFAULT_STOP_TAG debug = False strict = False color = False for option, arg in opts: if option in ("-h", "--help"): usage(0) if option in ("-v", "--version"): show_version() if option in ("-q", "--quick"): detailed = False if option in ("-t", "--stop-tag"): stop_tag = arg if option in ("-s", "--strict"): strict = True if option in ("-d", "--debug"): debug = True if option in ("-c", "--color"): color = True if not args: usage(2) setup_logger(debug, color) # output info for each file for filename in args: file_start = timeit.default_timer() try: img_file = open(str(filename), 'rb') except IOError: logger.error("'%s' is unreadable", filename) continue logger.info("Opening: %s", filename) tag_start = timeit.default_timer() # get the tags data = process_file(img_file, stop_tag=stop_tag, details=detailed, strict=strict, debug=debug) tag_stop = timeit.default_timer() if not data: logger.warning("No EXIF information found\n") continue if 'JPEGThumbnail' in data: logger.info('File has JPEG thumbnail') del data['JPEGThumbnail'] if 'TIFFThumbnail' in data: logger.info('File has TIFF thumbnail') del data['TIFFThumbnail'] tag_keys = list(data.keys()) tag_keys.sort() for i in tag_keys: try: logger.info('%s (%s): %s', i, FIELD_TYPES[data[i].field_type][2], data[i].printable) except: logger.error("%s : %s", i, str(data[i])) file_stop = timeit.default_timer() logger.debug("Tags processed in %s seconds", tag_stop - tag_start) logger.debug("File processed in %s seconds", file_stop - file_start) print("")
def main(out_dir, num_cores): ''' Function to fetch the latest spot history from AWS and store in a dataframe saved to a local csv file for every availability zone Parameters ---------- out_dir : string base file directory to store the spot history dataframes num_cores: integer number of cores to use ''' # Import packages import boto import datetime import logging import os import pandas as pd from multiprocessing import Process from CPAC.AWS import fetch_creds # Import local packages import utils # Init variables proc_list = [] out_csvs = [] df_list = [] creds_path = '/home2/dclark/secure-creds/aws-keys/dclark_cmi/dclark_cmi_keys.csv' # Set up logger now_date = datetime.datetime.now() log_month = now_date.strftime('%m-%Y') log_path = os.path.join(out_dir, 'spot_history_'+log_month+'.log') sh_log = utils.setup_logger('sh_log', log_path, logging.INFO, to_screen=True) # Get list of regions aws_sak, aws_aki = fetch_creds.return_aws_keys(creds_path) reg_conn = boto.connect_ec2(aws_sak, aws_aki) regions = reg_conn.get_all_regions() reg_conn.close() # Init categories to iterate through instance_types, product_descriptions = init_categories() # Form a list of the combinations of instance types and products instance_products = [(inst_type, prod) for inst_type in instance_types \ for prod in product_descriptions] # Get total lengths reg_len = len(regions) ip_len = len(instance_products) # For each AWS region for reg_idx, region in enumerate(regions): # For each instance_type-product combination for ip_idx, (instance_type, product) in enumerate(instance_products): proc = Process(target=get_df_and_save, args=(None, instance_type, product, region, out_dir)) proc_list.append(proc) # Run in parallel utils.run_in_parallel(proc_list, num_cores) # Gather files to merge into one dataframe sh_log.info('Done fetching and saving histories.\nGathering for merge...') for root, dirs, files in os.walk(out_dir): if files: found_csvs = [os.path.join(root, f) for f in files \ if f.endswith('csv')] out_csvs.extend(found_csvs) # Create data frame list for csv in out_csvs: df_list.append(pd.DataFrame.from_csv(csv)) # Merge dataframes sh_log.info('Merging dataframe list...') big_df = pd.concat(df_list, ignore_index=True) # Save to disk big_csv = os.path.join(out_dir, 'spot_history_%s.csv' % log_month) sh_log.info('Saving data frame to disk as %s' % big_csv) big_df.to_csv(big_csv)
if __name__ == "__main__": # Args: <SESSION> <VOLUME> session = sys.argv[1] volume = sys.argv[2] working_dir = os.path.join(conf.get_opt("working_dir"), session, volume) mkdirp(os.path.join(conf.get_opt("log_dir"), session, volume), exit_on_err=True) log_file = os.path.join(conf.get_opt("log_dir"), session, volume, "changelog.log") setup_logger(logger, log_file) try: def handle_rm_error(func, path, exc_info): if exc_info[1].errno == ENOENT: return raise exc_info[1] shutil.rmtree(working_dir, onerror=handle_rm_error) except (OSError, IOError) as e: logger.error("Failed to delete working directory: %s" % e) sys.exit(1)
options.update({'test.save': os.path.join(sprefix, sfolder, 'test.save')}) options.update({'params.save': os.path.join(sprefix, sfolder, 'params.save')}) if not os.path.exists(os.path.join(sprefix, sfolder)): os.makedirs(os.path.join(sprefix, sfolder)) # current word included train_set, test_set = load_cate_data(options) mlp = init_cate_params(options) run_cate(mlp, train_set, test_set, options) if __name__ == "__main__": log_folder = '../log' main_file = 'main_log' correct_file = 'correct_log' error_file = 'error_log' main_name = 'main' corr_name = 'correct' error_name = 'error' stript_path = os.path.dirname(os.path.abspath(__file__)) log_folder_path = os.path.join(stript_path, log_folder) if not os.path.exists(log_folder_path): os.makedirs(log_folder_path) log_file_names = (main_file, correct_file, error_file) logger_names = (main_name, corr_name, error_name) setup_logger(log_folder_path, log_file_names, logger_names) log_main = logging.getLogger(main_name) log_main.info('\n*****date: %s*****' % datetime.datetime.now()) main()
def __init__(self, root, options, main_logger): self.root = root self.options = options self.frame = tk.Frame(root) self.main_logger = main_logger self.sip_proxy = None self.tftp_server = None self.http_server = None # can enlarge self.frame.columnconfigure(0, weight=1) # first row: setting, cannot vertical enlarge: self.frame.rowconfigure(0, weight=0) # second row: registrar, can vertical enlarge: self.frame.rowconfigure(1, weight=1) # Settings control frame self.settings_frame = tk.LabelFrame(self.frame, text="Settings", padx=5, pady=5) self.settings_frame.grid(row=0, column=0, sticky=tk.N, padx=5, pady=5) # Registrar frame #self.registrar_frame = tk.Frame(self.frame) #self.registrar_frame.rowconfigure(0, weight=1) row = 0 tk.Label(self.settings_frame, text="General settings:", font = "-weight bold").grid(row=row, column=0, sticky=tk.W) row = row + 1 self.gui_debug = tk.BooleanVar() self.gui_debug.set(self.options.debug) tk.Label(self.settings_frame, text="Debug:").grid(row=row, column=0, sticky=tk.W) tk.Checkbutton(self.settings_frame, variable=self.gui_debug, command=self.gui_debug_action).grid(row=row, column=1, sticky=tk.W) #row = row + 1 self.gui_sip_ip_address = tk.StringVar() self.gui_sip_ip_address.set(self.options.ip_address) tk.Label(self.settings_frame, text="IP Address:").grid(row=row, column=2, sticky=tk.W) tk.Entry(self.settings_frame, textvariable=self.gui_sip_ip_address, width=15).grid(row=row, column=3, sticky=tk.W) row = row + 1 tk.Label(self.settings_frame, text="TFTP Server:", font = "-weight bold").grid(row=row, column=0, sticky=tk.W) row = row + 1 self.gui_tftp_port = tk.IntVar() self.gui_tftp_port.set(self.options.tftp_port) tk.Label(self.settings_frame, text="TFTP Port:").grid(row=row, column=0, sticky=tk.W) tk.Entry(self.settings_frame, textvariable=self.gui_tftp_port, width=5).grid(row=row, column=1, sticky=tk.W) #row = row + 1 self.gui_tftp_root = tk.StringVar() self.gui_tftp_root.set(self.options.tftp_root) tk.Label(self.settings_frame, text="TFTP Directory:").grid(row=row, column=2, sticky=tk.W) tk.Entry(self.settings_frame, textvariable=self.gui_tftp_root, width=15).grid(row=row, column=3, sticky=tk.W) row = row + 1 tk.Label(self.settings_frame, text="HTTP Server:", font = "-weight bold").grid(row=row, column=0, sticky=tk.W) row = row + 1 self.gui_http_port = tk.IntVar() self.gui_http_port.set(self.options.http_port) tk.Label(self.settings_frame, text="HTTP Port:").grid(row=row, column=0, sticky=tk.W) tk.Entry(self.settings_frame, textvariable=self.gui_http_port, width=5).grid(row=row, column=1, sticky=tk.W) #row = row + 1 self.gui_http_root = tk.StringVar() self.gui_http_root.set(self.options.http_root) tk.Label(self.settings_frame, text="HTTP Directory:").grid(row=row, column=2, sticky=tk.W) tk.Entry(self.settings_frame, textvariable=self.gui_http_root, width=15).grid(row=row, column=3, sticky=tk.W) row = row + 1 tk.Label(self.settings_frame, text="DHCP Server:", font = "-weight bold").grid(row=row, column=0, sticky=tk.W) row = row + 1 self.gui_dhcp_begin = tk.StringVar() self.gui_dhcp_begin.set(self.options.dhcp_begin) tk.Label(self.settings_frame, text="DHCP Pool start:").grid(row=row, column=0, sticky=tk.W) tk.Entry(self.settings_frame, textvariable=self.gui_dhcp_begin, width=15).grid(row=row, column=1, sticky=tk.W) #row = row + 1 self.gui_dhcp_end = tk.StringVar() self.gui_dhcp_end.set(self.options.dhcp_end) tk.Label(self.settings_frame, text="DHCP Pool end:").grid(row=row, column=2, sticky=tk.W) tk.Entry(self.settings_frame, textvariable=self.gui_dhcp_end, width=15).grid(row=row, column=3, sticky=tk.W) row = row + 1 self.gui_dhcp_subnetmask = tk.StringVar() self.gui_dhcp_subnetmask.set(self.options.dhcp_subnetmask) tk.Label(self.settings_frame, text="DHCP Subnet mask:").grid(row=row, column=0, sticky=tk.W) tk.Entry(self.settings_frame, textvariable=self.gui_dhcp_subnetmask, width=15).grid(row=row, column=1, sticky=tk.W) #row = row + 1 self.gui_dhcp_gateway = tk.StringVar() self.gui_dhcp_gateway.set(self.options.dhcp_gateway) tk.Label(self.settings_frame, text="DHCP Gateway:").grid(row=row, column=2, sticky=tk.W) tk.Entry(self.settings_frame, textvariable=self.gui_dhcp_gateway, width=15).grid(row=row, column=3, sticky=tk.W) row = row + 1 self.gui_dhcp_bcast = tk.StringVar() self.gui_dhcp_bcast.set(self.options.dhcp_bcast) tk.Label(self.settings_frame, text="DHCP Broadcast:").grid(row=row, column=0, sticky=tk.W) tk.Entry(self.settings_frame, textvariable=self.gui_dhcp_bcast, width=15).grid(row=row, column=1, sticky=tk.W) #row = row + 1 self.gui_dhcp_dns = tk.StringVar() self.gui_dhcp_dns.set(self.options.dhcp_dns) tk.Label(self.settings_frame, text="DHCP DNS:").grid(row=row, column=2, sticky=tk.W) tk.Entry(self.settings_frame, textvariable=self.gui_dhcp_dns, width=15).grid(row=row, column=3, sticky=tk.W) row = row + 1 self.gui_dhcp_fileserver = tk.StringVar() self.gui_dhcp_fileserver.set(self.options.dhcp_fileserver) tk.Label(self.settings_frame, text="DHCP Fileserver (opt. 66):").grid(row=row, column=0, sticky=tk.W) tk.Entry(self.settings_frame, textvariable=self.gui_dhcp_fileserver, width=25).grid(row=row, column=1, sticky=tk.W) #row = row + 1 self.gui_dhcp_filename = tk.StringVar() self.gui_dhcp_filename.set(self.options.dhcp_filename) tk.Label(self.settings_frame, text="DHCP Filename (opt. 67):").grid(row=row, column=2, sticky=tk.W) tk.Entry(self.settings_frame, textvariable=self.gui_dhcp_filename, width=25).grid(row=row, column=3, sticky=tk.W) row = row + 1 tk.Label(self.settings_frame, text="SIP Plug&Play:", font = "-weight bold").grid(row=row, column=0, sticky=tk.W) row = row + 1 self.gui_pnp_uri = tk.StringVar() self.gui_pnp_uri.set(self.options.pnp_uri) tk.Label(self.settings_frame, text="PnP URI:").grid(row=row, column=0, sticky=tk.W) tk.Entry(self.settings_frame, textvariable=self.gui_pnp_uri, width=60).grid(row=row, column=1, columnspan=3, sticky=tk.W) row = row + 1 tk.Label(self.settings_frame, text="SIP Proxy:", font = "-weight bold").grid(row=row, column=0, sticky=tk.W) row = row + 1 self.gui_sip_redirect = tk.BooleanVar() self.gui_sip_redirect.set(self.options.sip_redirect) tk.Label(self.settings_frame, text="SIP Redirect server:").grid(row=row, column=0, sticky=tk.W) tk.Checkbutton(self.settings_frame, variable=self.gui_sip_redirect, command=self.gui_sip_redirect_action).grid(row=row, column=1, sticky=tk.W) #row = row + 1 self.gui_sip_port = tk.IntVar() self.gui_sip_port.set(self.options.sip_port) tk.Label(self.settings_frame, text="SIP Port:").grid(row=row, column=2, sticky=tk.W) tk.Entry(self.settings_frame, textvariable=self.gui_sip_port, width=5).grid(row=row, column=3, sticky=tk.W) row = row + 1 self.gui_sip_password = tk.StringVar() self.gui_sip_password.set(self.options.sip_password) tk.Label(self.settings_frame, text="Password:"******"Start SIP Proxy", command=self.start_sip_proxy) self.sip_control_button.grid(row=row, column=0, sticky=tk.N) self.tftp_control_button = tk.Button(self.settings_frame, text="Start TFTP Server", command=self.start_tftp_server) self.tftp_control_button.grid(row=row, column=1, sticky=tk.N) self.http_control_button = tk.Button(self.settings_frame, text="Start HTTP Server", command=self.start_http_server) self.http_control_button.grid(row=row, column=2, sticky=tk.N) self.dhcp_control_button = tk.Button(self.settings_frame, text="Start DHCP Server", command=self.start_dhcp_server) self.dhcp_control_button.grid(row=row, column=3, sticky=tk.N) self.pnp_control_button = tk.Button(self.settings_frame, text="Start PnP Server", command=self.start_pnp_server) self.pnp_control_button.grid(row=row, column=4, sticky=tk.N) #self.registrar_button = tk.Button(self.settings_frame, text="Reload registered", command=self.load_registrar) #self.registrar_button.grid(row=row, column=4, sticky=tk.N) #row = row + 1 #self.registrar_frame.grid(row=1, column=0, sticky=tk.NS) #self.registrar_text = ScrolledText(self.registrar_frame) #self.registrar_text.grid(row=0, column=0, sticky=tk.NS) #self.registrar_text.config(state='disabled') self.sip_queue = Queue.Queue() self.sip_trace_logger = utils.setup_logger('sip_widget_logger', log_file=None, debug=True, str_format='%(asctime)s %(message)s', handler=SipTraceQueueLogger(queue=self.sip_queue)) self.log_queue = Queue.Queue() utils.setup_logger('main_logger', options.logfile, self.options.debug, handler=MessagesQueueLogger(queue=self.log_queue))
parser.add_argument( "opts", help="Modify config options using the command-line", default=None, nargs=argparse.REMAINDER, ) args = parser.parse_args() ood = ["OOD.exclude_back", args.exclude_back, "OOD.ood", args.ood] cfg.merge_from_file(args.cfg) cfg.merge_from_list(ood) cfg.merge_from_list(args.opts) # cfg.freeze() logger = setup_logger(distributed_rank=0) # TODO logger.info("Loaded configuration file {}".format(args.cfg)) logger.info("Running with config:\n{}".format(cfg)) # absolute paths of model weights cfg.MODEL.weights_encoder = os.path.join(cfg.DIR, 'encoder_' + cfg.VAL.checkpoint) cfg.MODEL.weights_decoder = os.path.join(cfg.DIR, 'decoder_' + cfg.VAL.checkpoint) assert os.path.exists(cfg.MODEL.weights_encoder) and \ os.path.exists(cfg.MODEL.weights_decoder), "checkpoint does not exitst!" if not os.path.isdir(os.path.join(cfg.DIR, "result")): os.makedirs(os.path.join(cfg.DIR, "result")) main(cfg, args.gpu)
import argparse import os import sys def setup_args(): options = argparse.ArgumentParser() options.add_argument('--metafile', action="store", default = "splits/train_total.csv") options.add_argument('--featfile', action="store") options.add_argument('--evalfeatfile', action="store") options.add_argument('--save-dir', action="store", dest="save_dir", default="results/ot/") options.add_argument('--eval-save-dir', action="store", dest="eval_save_dir", default=None) options.add_argument('--label1', action="store", default=0, type=float) options.add_argument('--label2', action="store", default=1, type=float) options.add_argument('--reg', action="store", default=1, type=float) options.add_argument('--nbins', action="store", default=2, type=int) options.add_argument('--split-features', action="store_true", default=False) return options.parse_args() if __name__ == "__main__": args = setup_args() os.makedirs(args.save_dir, exist_ok=True) logger = setup_logger(name='ot_log', save_dir=args.save_dir) logger.info(" ".join(sys.argv)) logger.info(args) get_ot_matrix(args, logger) eval_ot_matrix(args, logger)
if test_summary['test_loss'] < best_loss: best_loss = test_summary['test_loss'] current_state['best_loss'] = best_loss save_checkpoint(current_state=current_state, filename=os.path.join(args.save_dir, "models/best.pth")) logger.info("Best loss: %s" % best_loss) if epoch % args.save_freq == 0: save_checkpoint(current_state=current_state, filename=os.path.join( args.save_dir, "models/epoch_%s.pth" % epoch)) save_checkpoint(current_state=current_state, filename=os.path.join(args.save_dir, "models/last.pth")) if torch.cuda.is_available(): net.cuda() if __name__ == "__main__": args = setup_args() os.makedirs(args.save_dir, exist_ok=True) logger = setup_logger(name='training_log', save_dir=args.save_dir) logger.info(" ".join(sys.argv)) logger.info(args) run_training(args, logger)
def main(instructions=None, params=None, do_one_iteration=False): if not instructions: return if not params: params = __import__('params') data_json = "data.json" actions_fname = os.path.abspath(__file__).rsplit("/",1)[0]+"/actions.txt" u.copy_jecs() logger_name = u.setup_logger() logger = logging.getLogger(logger_name) time_stats = [] if os.path.isfile(data_json): with open(data_json, "r") as fhin: data = json.load(fhin) if "time_stats" in data: time_stats = data["time_stats"] all_samples = [] for i in range(5000): if u.proxy_hours_left() < 60 and not params.FORSAKE_HEAVENLY_PROXY: u.proxy_renew() data = { "samples": [], "last_updated": None, "time_stats": time_stats } # read instructions file. if new sample found, add it to list # for existing samples, try to update params (xsec, kfact, etc.) for samp in u.read_samples(instructions): samp["params"] = params if samp not in all_samples: s = Samples.Sample(**samp) all_samples.append(s) else: all_samples[all_samples.index(samp)].update_params(samp) n_done = 0 n_samples = len(all_samples) for isample, s in enumerate(all_samples): try: stat = s.get_status() typ = s.get_type() # grab actions from a text file and act on them, consuming (removing) them if successful for dataset_name, action in u.get_actions(actions_fname=actions_fname,dataset_name=s["dataset"]): if s.handle_action(action): u.consume_actions(dataset_name=s["dataset"],action=action, actions_fname=actions_fname) if not s.pass_tsa_prechecks(): continue if typ == "CMS3": if stat == "new": s.crab_submit() elif stat == "crab": s.crab_parse_status() if s.is_crab_done(): s.make_miniaod_map() s.make_merging_chunks() s.submit_merge_jobs() elif stat == "postprocessing": if s.is_merging_done(): if s.check_output(): s.make_metadata() s.copy_files() else: s.submit_merge_jobs() elif stat == "done": s.do_done_stuff() n_done += 1 elif typ == "BABY": if stat == "new": s.set_baby_inputs() s.submit_baby_jobs() elif stat == "condor" or stat == "postprocessing": if params.open_datasets: s.check_new_merged_for_babies() if not params.open_datasets and s.is_babymaking_done(): s.set_status("done") else: # s.sweep_babies() s.sweep_babies_parallel() s.submit_baby_jobs() elif stat == "done": if params.open_datasets: s.check_new_merged_for_babies() else: s.do_done_stuff() n_done += 1 s.save() data["samples"].append( s.get_slimmed_dict() ) except Exception, err: logger.info( "send an (angry?) email to Nick with the Traceback below!!") logger.info( traceback.format_exc() ) breakdown_crab = u.sum_dicts([samp["crab"]["breakdown"] for samp in data["samples"] if "crab" in samp and "breakdown" in samp["crab"]]) # breakdown_baby = u.sum_dicts([{"baby_"+key:samp["baby"].get(key,0) for key in ["running", "sweepRooted"]} for samp in data["samples"] if samp["type"] == "BABY"]) breakdown_baby = u.sum_dicts([{"running_babies":samp["baby"]["running"], "sweepRooted_babies":samp["baby"]["sweepRooted"]} for samp in data["samples"] if samp["type"] == "BABY"]) tot_breakdown = u.sum_dicts([breakdown_crab, breakdown_baby]) data["last_updated"] = u.get_timestamp() data["time_stats"].append( (u.get_timestamp(), tot_breakdown) ) data["log"] = u.get_last_n_lines(fname=params.log_file, N=100) with open(data_json, "w") as fhout: data["samples"] = sorted(data["samples"], key=lambda x: x.get("status","done")=="done") json.dump(data, fhout, sort_keys = True, indent = 4) u.copy_json(params) if params.exit_when_done and (n_done == n_samples): print ">>> All %i samples are done. Exiting." % n_samples break if not do_one_iteration: sleep_time = 60 if i < 2 else 2*600 logger.debug("sleeping for %i seconds..." % sleep_time) u.smart_sleep(sleep_time, files_to_watch=[actions_fname, instructions]) else: break
def main(): import pydevd_pycharm pydevd_pycharm.settrace('172.26.3.54', port=12345, stdoutToServer=True, stderrToServer=True) parser = argparse.ArgumentParser(description="DFDGAN Showing G pic") parser.add_argument("--config_file", default="./configs/show_pic.yml", help="path to config file", type=str) parser.add_argument("opts", help="Modify config options using the command-line", default=None, nargs=argparse.REMAINDER) args = parser.parse_args() if args.config_file != "": cfg.merge_from_file(args.config_file) cfg.merge_from_list(args.opts) cfg.freeze() output_dir = cfg.OUTPUT_DIR datasets_dir = '' for dataset_name in cfg.DATASETS.NAMES: if datasets_dir != '': datasets_dir += '-' datasets_dir += dataset_name output_dir = os.path.join(output_dir, datasets_dir) time_string = 'show_pic[{}]'.format( time.strftime('%Y-%m-%d-%X', time.localtime(time.time()))) output_dir = os.path.join(output_dir, time_string) if output_dir and not os.path.exists(output_dir): os.makedirs(output_dir) device = cfg.TEST.DEVICE if device == "cuda": os.environ['CUDA_VISIBLE_DEVICES'] = cfg.TEST.DEVICE_ID cudnn.benchmark = True logger = setup_logger("DFDGAN", output_dir, 0) logger.info("Running with config:\n{}".format(cfg)) data_loader, num_classes = make_dataloaders(cfg) E = Encoder(num_classes, cfg.E.LAST_STRIDE, cfg.E.PRETRAIN_PATH, cfg.E.NECK, cfg.TEST.NECK_FEAT, cfg.E.NAME, cfg.E.PRETRAIN_CHOICE).to(device) Ed = Encoder(num_classes, cfg.ED.LAST_STRIDE, cfg.ED.PRETRAIN_PATH, cfg.ED.NECK, cfg.TEST.NECK_FEAT, cfg.ED.NAME, cfg.ED.PRETRAIN_CHOICE).to(device) G = DFDGenerator(cfg.G.PRETRAIN_PATH, cfg.G.PRETRAIN_CHOICE, noise_size=cfg.TRAIN.NOISE_SIZE).to(device) for _, batch in enumerate(data_loader): img_x1, img_x2, img_y1, img_y2, target_pid, target_setid = batch img_x1, img_x2, img_y1, img_y2, target_pid, target_setid = img_x1.to( device), img_x2.to(device), img_y1.to(device), img_y2.to( device), target_pid.to(device), target_setid.to(device) g_img = G(E(img_x1)[0], Ed(img_y1)[0]) img_x1_PIL = transforms.ToPILImage()(img_x1[0].cpu()).convert('RGB') img_x1_PIL.save(os.path.join(output_dir, 'img_x1.jpg')) img_y1_PIL = transforms.ToPILImage()(img_y1[0].cpu()).convert('RGB') img_y1_PIL.save(os.path.join(output_dir, 'img_y1.jpg')) g_img_PIL = transforms.ToPILImage()(g_img[0].cpu()).convert('RGB') g_img_PIL.save(os.path.join(output_dir, 'g_img.jpg')) break
def main(): global best_score, logger, logger_results, slide_weights print('inside train_inception.py ') opt = Options(isTrain=True) opt.parse() opt.save_options() os.environ['CUDA_VISIBLE_DEVICES'] = ','.join( str(x) for x in opt.train['gpus']) # set up logger logger, logger_results = utils.setup_logger(opt) opt.print_options(logger) # ---------- Create model ---------- # model = Inception_v3(opt.model['out_c']) model = model.cuda() # logger.info(model) # ---------- End create model ---------- # # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss() # params are taken from the nas scoring paper lr = 0.00005 logger.info('lr :{}\n'.format(lr)) optimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=0.9) exp_lr_scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.2, patience=2, verbose=False, threshold=0.0001, threshold_mode='rel', cooldown=1, min_lr=1e-7, eps=1e-08) # ---------- Data loading ---------- # data_transforms = { 'train': transforms.Compose([ transforms.Resize((299, 299)), transforms.RandomHorizontalFlip(), transforms.RandomVerticalFlip(), transforms.RandomRotation(randint(1, 45)), transforms.RandomAffine(degrees=0, translate=(0.1, 0.1)), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]), 'test': transforms.Compose([ transforms.Resize((299, 299)), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) } fold_num = opt.exp_num.split('_')[-1] logger.info('Fold number: {:s}'.format(fold_num)) if opt.exp in ['fib']: #base_path = '/dresden/users/aj611/experiments/biomed/he_images/' base_path = '/dresden/users/aj611/experiments/biomed/he_images_check/' else: #base_path = '/dresden/users/aj611/experiments/biomed/he_images_3x/' base_path = '/dresden/users/aj611/experiments/biomed/he_images_3x_check/' #data_dir = opt.train['data_dir'] + opt.exp + '/' + 'fold_{}/'.format(fold_num) data_dir = base_path + opt.exp + '/' + 'fold_{}/'.format(fold_num) print('data_dir :', data_dir) #data_dir = '/dresden/users/aj611/experiments/biomed/he_images_bkp/fibrosis/' batch_size = 8 image_datasets = { x: datasets.ImageFolder(os.path.join(data_dir, x), data_transforms[x]) for x in ['train', 'test'] } #dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size = batch_size, drop_last=True, shuffle = True, num_workers = 8) for x in ['train', 'test']} dataloaders = {} x = 'train' dataloaders[x] = torch.utils.data.DataLoader(image_datasets[x], batch_size=batch_size, drop_last=True, shuffle=True, num_workers=8) x = 'test' dataloaders[x] = torch.utils.data.DataLoader(image_datasets[x], batch_size=1, drop_last=True, shuffle=True, num_workers=8) dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'test']} logger.info('data_dir :{}\n'.format(data_dir)) class_names = image_datasets['train'].classes device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") # ---------- End Data loading ---------- # # ----- optionally load from a checkpoint ----- # # if opt.train['checkpoint']: # model_state_dict, optimizer_state_dict = load_checkpoint(opt.train['checkpoint']) # model.load_state_dict(model_state_dict) # optimizer.load_state_dict(optimizer_state_dict) # ----- End checkpoint loading ----- # # ----- Start training ---- # best_score = 0 for epoch in range(opt.train['epochs']): # train and validate for one epoch test_auc = train_model(model, opt, dataset_sizes, dataloaders, image_datasets, criterion, optimizer, exp_lr_scheduler, epoch) print('epoch {}'.format(epoch)) # remember best accuracy and save checkpoint is_best = test_auc > best_score best_score = max(test_auc, best_score) cp_flag = False save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_score': best_score, 'optimizer': optimizer.state_dict(), }, is_best, opt.train['save_dir'], cp_flag, epoch + 1) # save training results #logger_results.info('{:<6d}| {:<12.4f}{:<12.4f}|| {:<12.4f}{:<12.4f}{:<12.4f}' # .format(epoch, train_loss, train_acc, # test_loss, test_acc, test_auc)) for i in list(logger.handlers): logger.removeHandler(i) i.flush() for i in list(logger.handlers): logger.removeHandler(i) i.flush() i.close() for i in list(logger_results.handlers): logger_results.removeHandler(i) i.flush() i.close()
from __future__ import print_function from __future__ import division import torch import torch.nn as nn from torch.autograd import Variable import torch.nn.functional as F from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence import numpy as np from utils import maybe_cuda, setup_logger, unsort from times_profiler import profiler logger = setup_logger(__name__, 'train.log') profilerLogger = setup_logger("profilerLogger", 'profiler.log', True) def zero_state(module, batch_size): # * 2 is for the two directions return Variable(maybe_cuda(torch.zeros(module.num_layers * 2, batch_size, module.hidden))), \ Variable(maybe_cuda(torch.zeros(module.num_layers * 2, batch_size, module.hidden))) class SentenceEncodingCNN(nn.Module): def __init__(self, D=300, Ci=1, Co=32, Ks=[2, 3]): super(SentenceEncodingCNN, self).__init__() self.convs1 = nn.ModuleList([nn.Conv2d(Ci, Co, (K, D)) for K in Ks]) def forward(self, x): x = x.unsqueeze(1) # (N, Ci, W, D) x = [F.relu(conv(x)).squeeze(3)
from utils import setup_logger from settings import run_folder ### SET all LOGGER_DISABLED to True to disable logging ### WARNING: the mcts log file gets big quite quickly LOGGER_DISABLED = { 'main':False , 'memory':True , 'tourney':False , 'mcts':True , 'model': True} logger_mcts = setup_logger('logger_mcts', run_folder + 'logs/logger_mcts.log') logger_mcts.disabled = LOGGER_DISABLED['mcts'] logger_main = setup_logger('logger_main', run_folder + 'logs/logger_main.log') logger_main.disabled = LOGGER_DISABLED['main'] logger_tourney = setup_logger('logger_tourney', run_folder + 'logs/logger_tourney.log') logger_tourney.disabled = LOGGER_DISABLED['tourney'] logger_memory = setup_logger('logger_memory', run_folder + 'logs/logger_memory.log') logger_memory.disabled = LOGGER_DISABLED['memory'] logger_model = setup_logger('logger_model', run_folder + 'logs/logger_model.log') logger_model.disabled = LOGGER_DISABLED['model']
def main(args): this_dir = osp.join(osp.dirname(__file__), '.') save_dir = osp.join(this_dir, 'checkpoints') if not osp.isdir(save_dir): os.makedirs(save_dir) command = 'python ' + ' '.join(sys.argv) logger = utl.setup_logger(osp.join(this_dir, 'log.txt'), command=command) os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') utl.set_seed(int(args.seed)) model = build_model(args) if osp.isfile(args.checkpoint): checkpoint = torch.load(args.checkpoint, map_location=torch.device('cpu')) model.load_state_dict(checkpoint['model_state_dict']) else: model.apply(utl.weights_init) if args.distributed: model = nn.DataParallel(model) model = model.to(device) criterion = utl.MultiCrossEntropyLoss(ignore_index=21).to(device) optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) if osp.isfile(args.checkpoint): optimizer.load_state_dict(checkpoint['optimizer_state_dict']) for param_group in optimizer.param_groups: param_group['lr'] = args.lr args.start_epoch += checkpoint['epoch'] softmax = nn.Softmax(dim=1).to(device) for epoch in range(args.start_epoch, args.start_epoch + args.epochs): if epoch == 21: args.lr = args.lr * 0.1 for param_group in optimizer.param_groups: param_group['lr'] = args.lr data_loaders = { phase: utl.build_data_loader(args, phase) for phase in args.phases } enc_losses = {phase: 0.0 for phase in args.phases} enc_score_metrics = [] enc_target_metrics = [] enc_mAP = 0.0 dec_losses = {phase: 0.0 for phase in args.phases} dec_score_metrics = [] dec_target_metrics = [] dec_mAP = 0.0 start = time.time() for phase in args.phases: training = phase == 'train' if training: model.train(True) elif not training and args.debug: model.train(False) else: continue with torch.set_grad_enabled(training): for batch_idx, (camera_inputs, motion_inputs, enc_target, dec_target) \ in enumerate(data_loaders[phase], start=1): batch_size = camera_inputs.shape[0] camera_inputs = camera_inputs.to(device) motion_inputs = motion_inputs.to(device) enc_target = enc_target.to(device).view( -1, args.num_classes) dec_target = dec_target.to(device).view( -1, args.num_classes) enc_score, dec_score = model(camera_inputs, motion_inputs) enc_loss = criterion(enc_score, enc_target) dec_loss = criterion(dec_score, dec_target) enc_losses[phase] += enc_loss.item() * batch_size dec_losses[phase] += dec_loss.item() * batch_size if args.verbose: print( 'Epoch: {:2} | iteration: {:3} | enc_loss: {:.5f} dec_loss: {:.5f}' .format(epoch, batch_idx, enc_loss.item(), dec_loss.item())) if training: optimizer.zero_grad() loss = enc_loss + dec_loss loss.backward() optimizer.step() else: # Prepare metrics for encoder enc_score = softmax(enc_score).cpu().numpy() enc_target = enc_target.cpu().numpy() enc_score_metrics.extend(enc_score) enc_target_metrics.extend(enc_target) # Prepare metrics for decoder dec_score = softmax(dec_score).cpu().numpy() dec_target = dec_target.cpu().numpy() dec_score_metrics.extend(dec_score) dec_target_metrics.extend(dec_target) end = time.time() if args.debug: result_file = 'inputs-{}-epoch-{}.json'.format(args.inputs, epoch) # Compute result for encoder enc_mAP = utl.compute_result_multilabel( args.class_index, enc_score_metrics, enc_target_metrics, save_dir, result_file, ignore_class=[0, 21], save=True, ) # Compute result for decoder dec_mAP = utl.compute_result_multilabel( args.class_index, dec_score_metrics, dec_target_metrics, save_dir, result_file, ignore_class=[0, 21], save=False, ) # Output result logger.output(epoch, enc_losses, dec_losses, len(data_loaders['train'].dataset), len(data_loaders['test'].dataset), enc_mAP, dec_mAP, end - start, debug=args.debug) # Save model checkpoint_file = 'inputs-{}-epoch-{}.pth'.format(args.inputs, epoch) torch.save( { 'epoch': epoch, 'model_state_dict': model.module.state_dict() if args.distributed else model.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), }, osp.join(save_dir, checkpoint_file))
import numpy as np import pandas as pd import datetime import os, sys, gc, time, warnings, pickle, psutil, random from tqdm import tqdm from multiprocessing import Pool from utils import setup_logger warnings.filterwarnings('ignore') ########################### logger ################################################################################# NOW = datetime.datetime.now().strftime('%Y-%m-%d_%H:%M:%S') logger = setup_logger(f'./logs/train_{NOW}.log') ########################### Helpers ################################################################################# def seed_everything(seed=0): random.seed(seed) np.random.seed(seed) ## Multiprocess Runs def df_parallelize_run(func, t_split): num_cores = np.min([N_CORES, len(t_split)]) pool = Pool(num_cores) df = pd.concat(pool.map(func, t_split), axis=1) pool.close() pool.join() return df
def main(): global best_score, logger, logger_results, slide_weights opt = Options(isTrain=True) opt.parse() opt.save_options() # tb_writer = SummaryWriter('{:s}/tb_logs'.format(opt.train['save_dir'])) os.environ['CUDA_VISIBLE_DEVICES'] = ','.join(str(x) for x in opt.train['gpus']) # set up logger logger, logger_results = utils.setup_logger(opt) opt.print_options(logger) if opt.train['random_seed'] >= 0: # logger.info("=> Using random seed {:d}".format(opt.random_seed)) torch.manual_seed(opt.train['random_seed']) torch.cuda.manual_seed(opt.train['random_seed']) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False np.random.seed(opt.train['random_seed']) else: torch.backends.cudnn.benchmark = True # ---------- Create model ---------- # #print('in train file', opt.model['use_resnet']) if opt.model['use_resnet'] == 1: #print('opt model use_resnet ', type(opt.model['use_resnet'])) #print('choosing baseline2') model = BaselineNet2(opt.model['out_c'], opt.model['resnet_layers'], opt.model['train_res4']) logger.info('choosing BaselineNet2(resnet based) model') else: model = BaselineNet(opt.model['in_c'], opt.model['out_c']) logger.info('choosing BaselineNet model') if opt.model['pre_train_sup'] == 1: print("=> loading pre-trained model from path", opt.model['sup_model_path']) model.set_fea_extractor(opt.model['sup_model_path']) model = model.cuda() # logger.info(model) # ---------- End create model ---------- # # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.Adam(model.parameters(), opt.train['lr'], weight_decay=opt.train['weight_decay']) # ---------- Data loading ---------- # data_transform = transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), transforms.Normalize(mean=[0.485], std=[0.229]) ]) fold_num = opt.exp_num.split('_')[-1] logger.info('Fold number: {:s}'.format(fold_num)) if opt.model['use_resnet'] == 1: train_set = LiverDataset('{:s}/train{:s}.h5'.format(opt.train['data_dir'], fold_num), data_transform, opt) test_set = LiverDataset('{:s}/test{:s}.h5'.format(opt.train['data_dir'], fold_num), data_transform, opt) else: train_set = LiverDataset('{:s}/train{:s}.h5'.format(opt.train['data_dir'], fold_num), data_transform) test_set = LiverDataset('{:s}/test{:s}.h5'.format(opt.train['data_dir'], fold_num), data_transform) # loading from the pre-trained model #print("=> loading pre-trained model from path", opt.model['sup_model_path']) #model.set_fea_extractor(opt.model['sup_model_path']) # ---------- End Data loading ---------- # # ----- optionally load from a checkpoint ----- # # if opt.train['checkpoint']: # model_state_dict, optimizer_state_dict = load_checkpoint(opt.train['checkpoint']) # model.load_state_dict(model_state_dict) # optimizer.load_state_dict(optimizer_state_dict) # ----- End checkpoint loading ----- # # ----- Start training ---- # best_score = 0 for epoch in range(opt.train['epochs']): # train and validate for one epoch train_loss, train_acc = train(opt, train_set, model, criterion, optimizer, epoch) test_loss, test_acc, test_auc = test(opt, test_set, model, criterion, epoch) # remember best accuracy and save checkpoint is_best = test_auc > best_score best_score = max(test_auc, best_score) cp_flag = False save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_score': best_score, 'optimizer': optimizer.state_dict(), }, is_best, opt.train['save_dir'], cp_flag, epoch+1) # save training results logger_results.info('{:<6d}| {:<12.4f}{:<12.4f}|| {:<12.4f}{:<12.4f}{:<12.4f}' .format(epoch, train_loss, train_acc, test_loss, test_acc, test_auc)) for i in list(logger.handlers): logger.removeHandler(i) i.flush() i.close() for i in list(logger_results.handlers): logger_results.removeHandler(i) i.flush() i.close()
help='A csv containing phenotypic info for the release. Used to create participants.tsv') # For copying inherited files. parser.add_argument('-i', '--inherited', required=False, help='A path to a directory containing task metadata in JSON format, as well as other files to be included at the top level of the BIDS directory (such as the dataset description). Some of this data will be inherited by all scans.') args = parser.parse_args() bids_warehouse = os.path.abspath(args.bids_warehouse) log_dir = os.path.abspath(args.log_dir) # Log errors now = datetime.now() bids_logs=os.path.join(log_dir,'bids_logs') log_path=os.path.join(bids_logs,str(now.year),str(now.month),str(now.day)) if not os.path.exists(log_path): mkdir_p(log_path) bids_nifti_sanity_log=setup_logger('bids_nifti_sanity',os.path.join(log_path,'bids_nifti_sanity.log'), logging.INFO) # Read anonymization key in as dictionary and define output directory. anonmap={} if args.anon_ursi: with open(os.path.abspath(args.anon_ursi),'rU') as map_file: csvreader = csv.reader(map_file,delimiter=',') for row in csvreader: anonmap[row[1]]=row[0] output_dir=os.path.join(bids_warehouse,'Anonymized') else: output_dir=os.path.join(bids_warehouse,'Non-anonymized') mkdir_p(output_dir) if args.inherited: bids_inherit_log=setup_logger('bids_inherit',os.path.join(log_path,'bids_inherit_jsons.log'), logging.INFO)
from api_observer import ApiObserver from bot import YobitBot from constants import BOT_TOKEN from models import Chat from utils import setup_logger if __name__ == '__main__': if not Chat.table_exists(): Chat.create_table() bot = YobitBot(BOT_TOKEN) bot.start() setup_logger() observer = ApiObserver(bot) observer.observe()
def handle(self): data = self.request[0] self.data = data.split("\r\n") self.socket = self.request[1] request_uri = self.data[0] if rx_request_uri.search(request_uri) or rx_code.search(request_uri): self.server.sip_logger.debug("Received from %s:%d (%d bytes):\n\n%s" % (self.client_address[0], self.client_address[1], len(data), data)) self.processRequest() else: if len(data) > 4: self.server.sip_logger.debug("Received from %s:%d (%d bytes):\n\n" % (self.client_address[0], self.client_address[1], len(data))) mess = hexdump(data,' ',16) self.server.sip_logger.debug('PnP Hex data:\n' + '\n'.join(mess)) if __name__ == '__main__': import utils class Options: ip = "127.0.0.1" pnp_uri = "http://test.com" options = Options() HOST = "224.0.1.75" PORT = 5060 main_logger = utils.setup_logger('main_logger', None, True) sip_logger = utils.setup_logger('sip_logger', None, True) pnp_server = SipTracedMcastUDPServer((HOST, PORT), UDPHandler, main_logger, sip_logger, options) pnp_server.serve_forever()
""" import time import sched import pandas import logging import requests from io import StringIO import utils from database import upsert_bpa BPA_SOURCE = 'https://og-production-open-data-bostonma-892364687672.s3.amazonaws.com/resources/f123e65d-dc0e-4c83-9348-ed46fec498c0/tmpke4utrxy.csv?Signature=UAjyK0cDg%2BFiT22SGIeHlr12eQ8%3D&Expires=1575837619&AWSAccessKeyId=AKIAJJIENTAPKHZMIPXQ' MAX_DOWNLOAD_ATTEMPT = 5 DOWNLOAD_PERIOD = 10 # second logger = logging.Logger(__name__) utils.setup_logger(logger, 'data.log') def download_bpa(url=BPA_SOURCE, retries=MAX_DOWNLOAD_ATTEMPT): """Returns BPA text from `BPA_SOURCE` that includes power loads and resources Returns None if network failed """ print('hi') text = None for i in range(retries): try: req = requests.get(url, timeout=0.5) req.raise_for_status() text = req.text except requests.exceptions.HTTPError as e: logger.warning("Retry on HTTP Error: {}".format(e))
def __init__(self, id, base, night, configfile='C:/minerva-control/config/com.ini'): #ipdb.set_trace() #S set the id of self to the memory addresss, unique identifier self.lock = threading.Lock() self.id = id self.base_directory = base self.night = night #S Parses configfile into List based on id #? Not sure if this is how id is working, but makes sense. Need to look #? into *.ini files more. configObj = ConfigObj(configfile) #S Get the corresponding item/object (Typer??) from list of configObj, #S throws if not in configfile, printing to stdout. No log it try: config = configObj[self.id] except: print('ERROR accessing ', self.id, ".", self.id, " was not found in the configuration file", configfile) return #? Not really sure what is going on, need to look into com.ini self.flowcontrol = str(config['Setup']['FLOWCONTROL']) #self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) #S Looks like more fun in com.ini, I think it makes sense though, adds #S allowable serial commands for communication with instrument. self.allowedCmds = config['Setup']['ALLOWEDCMDS'] #S Is this short for 'Termination String'? I think so, for ending a #S command in serila comm. A little confused as \n is CR and LF, may #S get into trouble with picky instruments. if config['Setup']['TERMSTR'] == r"\r": self.termstr = "\r" elif config['Setup']['TERMSTR'] == r"\r\n": self.termstr = "\r\n" elif config['Setup']['TERMSTR'] == r"\n\r": self.termstr = "\n\r" elif config['Setup']['TERMSTR'] == r"\n": self.termstr = "\n" elif config['Setup']['TERMSTR'] == "": self.termstr = "" #S Gives self a Serial class self.ser = serial.Serial() #S Get the port, baud, expected bits, and the stopbit from configfile. self.ser.port = str(config['Setup']['PORT']) self.ser.baudrate = int(config['Setup']['BAUDRATE']) self.ser.databits = int(config['Setup']['DATABITS']) #? Parity commented out? # self.ser.parity = str(config['Setup']['PARITY']) self.ser.stopbits = int(config['Setup']['STOPBITS']) #S Looks like log writing stuff, comes from configfile as well. self.logger_name = config['Setup']['LOGNAME'] #S Nead to read more into logging package, see what's going on here self.logger = utils.setup_logger(self.base_directory, self.night, self.logger_name) #S Ensure that the serial port is definitely closed. self.ser.close()
#!/usr/bin/env python3 import logging from Piloton import Piloton from utils import setup_logger, set_logger_level if __name__ == "__main__": # Set up root logger logger = setup_logger(logging_level=logging.DEBUG) set_logger_level("bleak", logging_level=logging.WARNING) set_logger_level("urllib3", logging_level=logging.WARNING) set_logger_level("asyncio", logging_level=logging.WARNING) # Set up Piloton piloton: Piloton = Piloton() # Run Piloton piloton.app()
def main(argv): """ Entry point for chart generation. """ option_parser = optparse.OptionParser(usage=DEFAULT_USAGE_TEXT) option_parser.add_option("-c", "--config", dest="config", default="config.cfg", help="Configuration file") option_parser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False, help="Show verbose output") option_parser.add_option("-x", "--x_axis", type="choice", choices=[TIME], help="Label for x-axis: time") option_parser.add_option("-X", "--x_output", dest="x_output", help="Unit for x-axis. " "time: daily, month, yearly or range 2000,2003. " "Range for time is separated by comma to allow " "daily range such as -X 2000-01-01,2003-08-20") option_parser.add_option("-y", "--y_axis", type="choice", choices=[SALARY], help="Label for y-axis: salary") option_parser.add_option("-Y", "--y_output", type="choice", choices=[MEDIAN, TOTAL], help="Unit for y-axis. salary: median or total") option_parser.add_option("-d", "--data_type", type="choice", choices=[AGE, COMPANY, GENDER, JOBROLE, MANAGER], help="Data type for plotted lines: " "age, company, gender, jobrole or manager") option_parser.add_option("-D", "--data_output", dest="data_output", help="Unit for plotted lines. " "age: comma-separated list of ages or ranges, " "company: comma-separated list of IDs, " "gender: comma-separated list (male,female, " "undefined or empty for all), " "jobrole: comma-separated list of job roles, " "manager: comma-separated list (true,false)") options, _ = option_parser.parse_args(argv) error = "" if not options.x_axis: error = "--x_axis is required" if not options.x_output: error = "--x_output is required" if not options.y_axis: error = "--y_axis is required" if not options.y_output: error = "--y_output is required" if not options.data_type: error = "--data_type is required" if not options.data_output: error = "--data_output is required" if error: sys.stderr.write("ERROR: {}\n".format(error)) option_parser.print_help() return 1 if not os.path.exists(options.config): sys.stderr.write("ERROR: {} does not exist\n".format(options.config)) option_parser.print_help() return 1 config = read_config(options.config) log_dir = config['general']['log_dir'] if not os.path.exists(log_dir): os.mkdir(log_dir) filename = os.path.join(log_dir, __file__.replace(".py", ".log")) setup_logger(filename, options.verbose) logging.debug("config={}".format(json.dumps(config, indent=2))) retcode = generate_chart(config, options) return retcode
help='maximum length of an episode (default: 100000)') args = parser.parse_args() setup_json = read_config(args.env_config) env_conf = setup_json["Default"] for i in setup_json.keys(): if i in args.env: env_conf = setup_json[i] torch.set_default_tensor_type('torch.FloatTensor') saved_state_path = os.path.join(args.load_model_dir, args.env + '.model') saved_state = torch.load(saved_state_path, map_location=lambda storage, loc: storage) print('Loaded trained model from: {}'.format(saved_state_path)) log = {} setup_logger('{}_mon_log'.format(args.env), r'{0}{1}_mon_log'.format( args.log_dir, args.env)) log['{}_mon_log'.format(args.env)] = logging.getLogger( '{}_mon_log'.format(args.env)) env = atari_env("{}".format(args.env), env_conf) model = A3Clstm(env.observation_space.shape[0], env.action_space) num_tests = 0 reward_total_sum = 0 player = Agent(model, env, args, state=None) player.env = gym.wrappers.Monitor(player.env, "{}_monitor".format(args.env), force=True) player.model.eval() for i_episode in range(args.num_episodes): state = player.env.reset() player.state = torch.from_numpy(state).float() player.eps_len = 0
def __setup_logger(self,name): return utils.setup_logger(name, is_cobblerd=self.is_cobblerd, **self.log_settings)
#!/usr/bin/python # -*- coding: utf-8 -*- # by: pantuts # DISCLAIMER: FOR TESTING AND EDUCATIONAL PURPOSES ONLY # ANYTHING YOU DO WILL NOT AND SHOULD NOT AND CANNOT AFFECT THE AUTHOR, BE RESPONSIBLE! import os import time from bs4 import BeautifulSoup from datetime import datetime from tld import get_fld from utils import setup_logger, request, CSVUtils logger = setup_logger(__name__) csv_utils = CSVUtils() SCRIPT_NAME = os.path.splitext(os.path.basename(__file__))[0].title() NAME = 'CardLife' CSV_FNAME = NAME + '_' + datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d') + '.csv' NOW = datetime.strftime(datetime.now(), '%Y-%m-%d') MAIN_URL = 'https://www.cardlifeapp.com/saas-directory' class Cardlife: def get_categories(self): logger.info('Getting categories.') resp = request(MAIN_URL)
NUMBER_OF_ROWS = 10 ** 2 NUMBER_OF_COLS = 10 ROW_OFFSET = 2 COL_OFFSET = 2 VALUES_BUTTON_MAX_WIDTH = 50 VARIABLE_LIST_COLS_WIDTH = 50 VARIABLE_LIST_COL_N_WIDTH = 20 VARIABLE_LIST_COLS_COUNT = 6 N_ROLE = 0 NAME_ROLE = 1 TYPE_ROLE = 2 WIDTH_ROLE = 3 DECIMAL_ROLE = 4 LABEL_ROLE = 5 if __name__ == '__main__': sys.excepthook = handle_exception setup_logger("FS") app = QApplication(sys.argv) app.setStyle("Fusion") app.blockSignals(True) fill_random() app.blockSignals(False) window = MainWindow() window.show() exit_code = app.exec_() sys.exit(exit_code)
parser.add_argument('--hidden', type=int, default=128, help='Number of hidden units.') parser.add_argument('--nb_heads', type=int, default=8, help='Number of head attentions.') parser.add_argument('--dropout', type=float, default=0.5, help='Dropout rate (1 - keep probability).') parser.add_argument('--alpha', type=float, default=0.2, help='Alpha for the leaky_relu.') parser.add_argument('--patience', type=int, default=10000, help='Patience') args = parser.parse_args() args.cuda = not args.no_cuda and torch.cuda.is_available() random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if args.cuda: torch.cuda.manual_seed(args.seed) logger = setup_logger('baseline', args.output_path, 0) if args.output_path and not os.path.exists(args.output_path): os.makedirs(args.output_path) # Load data # adj, features, labels, idx_train, idx_val, idx_test = load_data() # train_feature_node_path = os.path.join(args.train_path, 'node_features') train_img_path = os.path.join(args.train_path, 'imgs') train_edge_path = os.path.join(args.train_path, 'edge_adjs') train_label_path = os.path.join(args.train_path, 'labels') train_roi_path = os.path.join(args.train_path, 'roi') # train_obj_path = os.path.join(args.train_path, 'mask_objs') # val_feature_node_path = os.path.join(args.val_path, 'node_features')
return celery celery = make_celery(app) # DB db = MongoEngine() db.init_app(app) db_wrapper = Database(db, cfg) if __name__ == '__main__': # TODO - not this __builtin__.flock_app = app # Logging setup_logger(cfg["logging"]["file"], cfg["logging"]["level"]) # Init Rollbar @app.before_first_request def init_rollbar(): rollbar.init( cfg["rollbar"]["token"], cfg["rollbar"]["environment"], root=os.path.dirname(os.path.realpath(__file__)), allow_logging_basic_config=False ) got_request_exception.connect(rollbar.contrib.flask.report_exception, app) class CustomRequest(Request): @property def rollbar_person(self):
def main(sim_dir, proc_time, num_jobs, jobs_per, in_gb, out_gb, out_gb_dl, up_rate, down_rate, bid_ratio, instance_type, av_zone, product, csv_file=None): ''' Function to calculate spot instance run statistics based on job submission parameters; this function will save the statistics and specific spot history in csv dataframes to execution directory Parameters ---------- sim_dir : string base directory where to create the availability zone folders for storing the simulation results proc_time : float the number of minutes a single job of interest takes to run num_jobs : integer total number of jobs to run to complete job submission jobs_per : integer the number of jobs to run per node in_gb : float the total amount of input data for a particular job (in GB) out_gb : float the total amount of output data from a particular job (in GB) out_gb_dl : float the total amount of output data to download from EC2 (in GB) up_rate : float the average upload rate to transfer data to EC2 (in Mb/s) down_rate : float the average download rate to transfer data from EC2 (in Mb/s) bid_ratio : float the ratio to average spot history price to set the bid price to instance_type : string type of instance to run the jobs on and to get spot history for av_zone : string the AWS EC2 availability zone (sub-region) to get spot history from product : string the type of operating system product to get spot history for csv_file : string (optional), default is None the filepath to a csv dataframe to get spot history from; if not specified, the function will just get the most recent 90 days worth of spot price history Returns ------- spot_history : pd.DataFrame object in addition to saving this as './spot_history.csv' the dataframe can also be returned as an object in memory stat_df : pd.DataFrame object in addition to saving this as './<info>_stats.csv' the dataframe can also be returned as an object in memory ''' # Import packages import dateutil import logging import numpy as np import os import pandas as pd import yaml # Import local packages import utils from record_spot_price import return_spot_history # Init variables proc_time *= 60.0 num_nodes = min(np.ceil(float(num_jobs)/jobs_per), 20) # Init simulation market results dataframe sim_df_cols = ['start_time', 'spot_hist_csv', 'proc_time', 'num_datasets', 'jobs_per_node', 'num_jobs_iter', 'bid_ratio', 'bid_price', 'median_history', 'mean_history', 'stdev_history', 'compute_time', 'wait_time', 'per_node_cost', 'num_interrupts', 'first_iter_time'] sim_df = pd.DataFrame(columns=sim_df_cols) # Init full run stats data frame stat_df_cols = ['Total cost', 'Instance cost', 'Storage cost', 'Tranfer cost', 'Total time', 'Run time', 'Wait time', 'Upload time', 'Download time'] stat_df = pd.DataFrame(columns=stat_df_cols) # Set up logger base_dir = os.path.join(sim_dir, av_zone) if not os.path.exists(base_dir): try: os.makedirs(base_dir) except OSError as exc: print 'Found av zone directory %s, continuing...' % av_zone log_path = os.path.join(base_dir, '%s_%d-jobs_%.3f-bid.log' % \ (instance_type, num_jobs, bid_ratio)) stat_log = utils.setup_logger('stat_log', log_path, logging.INFO, to_screen=True) # Check to see if simulation was already run (sim csv file exists) sim_csv = os.path.join(base_dir, '%s_%d-jobs_%.3f-bid_sim.csv' % \ (instance_type, num_jobs, bid_ratio)) if os.path.exists(sim_csv): stat_log.info('Simulation file %s already exists, skipping...' % sim_csv) return # Calculate number of iterations given run configuration # Round up and assume that we're waiting for all jobs to finish # before terminating nodes num_iter = np.ceil(num_jobs/float((jobs_per*num_nodes))) stat_log.info('With %d jobs, %d nodes, and %d jobs running per node...\n' \ 'job iterations: %d' % (num_jobs, num_nodes, jobs_per, num_iter)) # Get spot price history, if we're getting it from a csv dataframe if csv_file: # Parse dataframe to form history spot_history = spothistory_from_dataframe(csv_file, instance_type, product, av_zone) # Get rid of any duplicated timestamps spot_history = spot_history.groupby(spot_history.index).first() # Otherwise, just grab latest 90 days else: sh_list = return_spot_history(None, instance_type, product, av_zone) # Convert history into just timepoints and prices list of tuples timestamps = [dateutil.parser.parse(sh.timestamp) for sh in sh_list] prices = [sh.price for sh in sh_list] # Use pandas timeseries and sort in oldest -> newest spot_history = pd.Series(prices, timestamps) spot_history = spot_history.sort_index() # Write spot history to disk sh_csv = os.path.join(os.getcwd(), 'spot_history.csv') spot_history.to_csv(sh_csv) # Get interpolated times per second (forward fill) interp_seq = pd.date_range(spot_history.index[0], spot_history.index[-1], freq='S') interp_history = spot_history.reindex(interp_seq) interp_history = interp_history.fillna(method='ffill') # Init simulation time series sim_seq = pd.date_range(interp_seq[0], interp_seq[-1], freq='20T') sim_series = interp_history[sim_seq] # Init loop variables sim_idx = 0 sim_length = len(sim_series) beg_time = spot_history.index[0] end_time = spot_history.index[-1] time_needed = num_iter*(proc_time) # Get bid price spot_history_avg = interp_history.mean() bid_price = bid_ratio*spot_history_avg stat_log.info('Spot history average is $%.3f, bid ratio of %.3fx sets ' \ 'bid to $%.3f' % (spot_history_avg, bid_ratio, bid_price)) # Iterate through the interpolated timeseries for start_time, start_price in sim_series.iteritems(): # First see if there's enough time to run jobs time_window = (end_time-start_time).total_seconds() if time_needed > time_window: stat_log.info('Total runtime exceeds time window, ending simulation...') # Simulate running job and get stats from that start time try: run_time, wait_time, pernode_cost, num_interrupts, first_iter_time = \ simulate_market(start_time, spot_history, interp_history, proc_time, num_iter, bid_price) except Exception as exc: stat_log.info('Could not run full simulation because of:\n%s' % exc) continue # Write simulate market output to dataframe sim_df.loc[sim_idx] = [start_time, csv_file, proc_time, num_jobs, jobs_per, num_iter, bid_ratio, bid_price, np.mean(spot_history), np.median(spot_history), np.std(spot_history), run_time, wait_time, pernode_cost, num_interrupts, first_iter_time] # Get complete time and costs from spot market simulation parameters total_cost, instance_cost, stor_cost, xfer_cost, \ total_time, run_time, wait_time, \ xfer_up_time, xfer_down_time = \ calc_ebs_model_costs(run_time, wait_time, pernode_cost, first_iter_time, num_jobs, num_nodes, jobs_per, av_zone, in_gb, out_gb, out_gb_dl, up_rate, down_rate) # Add to output dataframe stat_df.loc[sim_idx] = [total_cost, instance_cost, stor_cost, xfer_cost, total_time/60.0, run_time/60.0, wait_time/60.0, xfer_up_time/60.0, xfer_down_time/60.0] # Print stats stat_log.info('Total cost: $%.3f' % total_cost) stat_log.info('Total time (minutes): %.3f' % (total_time/60.0)) stat_log.info('run time (minutes): %.3f' % (run_time/60.0)) stat_log.info('per-node cost: $%.3f' % pernode_cost) stat_log.info('number of interrupts: %d' % num_interrupts) stat_log.info('wait time (minutes): %.3f' % (wait_time/60.0)) sim_idx += 1 utils.print_loop_status(sim_idx, sim_length) # Add configuration parameters to dataframe sim_df['av_zone'] = av_zone sim_df['in_gb'] = in_gb sim_df['out_gb'] = out_gb sim_df['out_gb_dl'] = out_gb_dl sim_df['up_rate'] = up_rate sim_df['down_rate'] = down_rate # Write simulation dataframe to disk sim_df.to_csv(sim_csv) # Write stats dataframe to disk stat_csv = os.path.join(base_dir, '%s_%d-jobs_%.3f-bid_stats.csv' % \ (instance_type, num_jobs, bid_ratio)) stat_df.to_csv(stat_csv) # Write parameters yaml to disk params_yml = os.path.join(base_dir, '%s_%d-jobs_%.3f-bid_params.yml' % \ (instance_type, num_jobs, bid_ratio)) params = {'proc_time' : proc_time, 'num_jobs' : num_jobs, 'jobs_per' : jobs_per, 'in_gb' : in_gb, 'out_gb' : out_gb, 'out_gb_dl' : out_gb_dl, 'up_rate' : up_rate, 'down_rate' : down_rate, 'bid_ratio' : bid_ratio, 'instance_type' : instance_type, 'av_zone' : av_zone, 'product' : product, 'csv_file' : csv_file} with open(params_yml, 'w') as y_file: y_file.write(yaml.dump(params)) # Give simulation-wide statistics interrupt_avg = sim_df['num_interrupts'].mean() time_avg = stat_df['Total time'].mean() cost_avg = stat_df['Total cost'].mean() # Print simulation statistics stat_log.info('\n' + 72*'-') stat_log.info('Submission of %d job iterations, ' \ 'each takes %.3f mins to run:' % (num_iter, proc_time/60.0)) stat_log.info('Average spot history price for %s in %s\n' \ 'between %s and %s is: $%.3f' % \ (instance_type, av_zone, beg_time, end_time, spot_history_avg)) stat_log.info('Spot ratio of %.3fx the average price set bid to $%.3f' % \ (bid_ratio, bid_price)) stat_log.info('Average total time (mins): %f' % time_avg) stat_log.info('Average total cost: $%.3f' % cost_avg) stat_log.info('Average number of interruptions: %.3f' % interrupt_avg) stat_log.info(72*'-' + '\n') # Return dataframes return spot_history, sim_df, stat_df
import subprocess from pathlib import Path from tempfile import NamedTemporaryFile import h5py import librosa import numpy as np import torch import torch.utils.data as data import tqdm from scipy.io import wavfile import utils from utils import mu_law logger = utils.setup_logger(__name__, 'data.log') def random_of_length(seq, length): limit = seq.size(0) - length if length < 1: # logging.warning("%d %s" % (length, path)) return None start = random.randint(0, limit) end = start + length return seq[start:end] class EncodedFilesDataset(data.Dataset): """
__date__ = "30/07/2012" __copyright__ = "Copyright 2012, Australia Indonesia Facility for " "Disaster Reduction" import os import sys import logging from urllib2 import URLError from zipfile import BadZipfile from ftp_client import FtpClient from sftp_client import SFtpClient from utils import setup_logger, data_dir, is_event_id from shake_event import ShakeEvent # Loading from package __init__ not working in this context so manually doing setup_logger() LOGGER = logging.getLogger("InaSAFE") def process_event(event_id=None, locale="en"): """Launcher that actually runs the event processing. :param event_id: The event id to process. If None the latest event will be downloaded and processed. :type event_id: str :param locale: The locale that will be used. Default to en. :type locale: str """ population_path = os.path.join(data_dir(), "exposure", "IDN_mosaic", "popmap10_all.tif")
optimizer = Adam(shared_model.parameters(), lr=args.lr) criterion = nn.BCELoss() dataset = np.load( os.path.join(Dataset_Dir, '{}_all'.format(Tag_Name[Tag_Dict[args.tag]]), "%s_all.npz" % Tag_Name[Tag_Dict[args.tag]])) targets = dataset["arr_0"] max_accuracy = 0.0 while True: args.epoch += 1 print('=====> Train at epoch %d, Learning rate %0.6f <=====' % (args.epoch, args.lr)) start_time = time.time() log = setup_logger( 0, 'epoch%d' % args.epoch, os.path.join(args.log_dir, 'epoch%d_log.txt' % args.epoch)) log.info('Train time ' + time.strftime( "%Hh %Mm %Ss", time.gmtime(time.time() - start_time)) + ', ' + 'Training started.') order = list(range(targets.shape[0])) random.shuffle(order) losses = 0 correct_cnt = 0 for i in range(targets.shape[0]): idx = order[i] if dataset["arr_%d" % (idx + 1)].shape[0] == 0: continue
def test(args, shared_model, env_conf): log = {} setup_logger('{}_log'.format(args.env), r'{0}{1}_log'.format(args.log_dir, args.env)) log['{}_log'.format(args.env)] = logging.getLogger('{}_log'.format( args.env)) d_args = vars(args) for k in d_args.keys(): log['{}_log'.format(args.env)].info('{0}: {1}'.format(k, d_args[k])) torch.manual_seed(args.seed) env = atari_env(args.env, env_conf) reward_sum = 0 start_time = time.time() num_tests = 0 reward_total_sum = 0 player = Agent(None, env, args, None) player.model = A3Clstm(player.env.observation_space.shape[0], player.env.action_space) player.state = player.env.reset() player.state = torch.from_numpy(player.state).float() player.model.eval() while True: if player.done: player.model.load_state_dict(shared_model.state_dict()) player.action_test() reward_sum += player.reward if player.done: num_tests += 1 player.current_life = 0 reward_total_sum += reward_sum reward_mean = reward_total_sum / num_tests log['{}_log'.format(args.env)].info( "Time {0}, episode reward {1}, episode length {2}, reward mean {3:.4f}" .format( time.strftime("%Hh %Mm %Ss", time.gmtime(time.time() - start_time)), reward_sum, player.eps_len, reward_mean)) sendStatElastic({ "score": reward_sum, 'agent_name': 'pytorch-test', 'game_name': 'a3c-pytorch-SpaceInvaders-v0', 'episode': num_tests, 'frame_count': 0, 'episode_length': player.eps_len }) if reward_sum > args.save_score_level: player.model.load_state_dict(shared_model.state_dict()) state_to_save = player.model.state_dict() torch.save(state_to_save, '{0}{1}.dat'.format(args.save_model_dir, args.env)) reward_sum = 0 player.eps_len = 0 state = player.env.reset() time.sleep(60) player.state = torch.from_numpy(state).float()
def main(): parser = argparse.ArgumentParser(description="ReID Baseline Training") parser.add_argument("--config_file", default="", help="path to config file", type=str) parser.add_argument("opts", help="Modify config options using the command-line", default=None,nargs=argparse.REMAINDER) args = parser.parse_args() if args.config_file != "": cfg.merge_from_file(args.config_file) cfg.merge_from_list(args.opts) cfg.freeze() output_dir = cfg.OUTPUT_DIR if output_dir and not os.path.exists(output_dir): os.makedirs(output_dir) num_gpus = torch.cuda.device_count() logger = setup_logger('reid_baseline', output_dir, 0) logger.info('Using {} GPUS'.format(num_gpus)) logger.info('Running with config:\n{}'.format(cfg)) if cfg.INPUT.SEPNORM.USE: train_dl, val_dl, num_query, num_classes = make_sepnorm_dataloader(cfg, num_gpus) elif cfg.DATASETS.EXEMPLAR.USE: train_dl, val_dl, num_query, num_classes,exemplar_dl = make_dataloader(cfg, num_gpus) else: train_dl, val_dl, num_query, num_classes = make_dataloader(cfg, num_gpus) model = build_model(cfg, num_classes) loss = make_loss(cfg, num_classes) if cfg.SOLVER.CENTER_LOSS.USE == True: trainer = CenterTrainer(cfg, model, train_dl, val_dl, loss, num_query, num_gpus) else: if cfg.SOLVER.MIXUP.USE: trainer = NegMixupTrainer(cfg, model, train_dl, val_dl, loss, num_query, num_gpus) elif cfg.DATASETS.EXEMPLAR.USE: if cfg.DATASETS.EXEMPLAR.MEMORY.USE: trainer = ExemplarMemoryTrainer(cfg, model, train_dl, val_dl,exemplar_dl, loss, num_query, num_gpus) else: trainer = UIRLTrainer(cfg, model, train_dl, val_dl,exemplar_dl, loss, num_query, num_gpus) elif cfg.DATASETS.HIST_LABEL.USE: trainer = HistLabelTrainer(cfg, model, train_dl, val_dl, loss, num_query, num_gpus) else: trainer = BaseTrainer(cfg, model, train_dl, val_dl, loss, num_query, num_gpus) if cfg.INPUT.SEPNORM.USE: logger.info('train transform0: \n{}'.format(train_dl.dataset.transform0)) logger.info('train transform1: \n{}'.format(train_dl.dataset.transform1)) logger.info('valid transform0: \n{}'.format(val_dl.dataset.transform0)) logger.info('valid transform1: \n{}'.format(val_dl.dataset.transform1)) else: logger.info('train transform: \n{}'.format(train_dl.dataset.transform)) logger.info('valid transform: \n{}'.format(val_dl.dataset.transform)) logger.info(type(model)) logger.info(loss) logger.info(trainer) for epoch in range(trainer.epochs): for batch in trainer.train_dl: trainer.step(batch) trainer.handle_new_batch() trainer.handle_new_epoch()
def __init__(self, profile, **kwargs): """Constructor Args: profile (string): AWS profile **kwargs: Multiple arguments Raises: TypeError: Description """ self.log_level = kwargs.pop("log_level", logging.WARNING) self.boto_log_level = kwargs.pop("boto_log_level", logging.WARNING) # Setup logger self.logger = setup_logger(__name__, self.log_level, self.boto_log_level) # Get AWS Session self.session = boto3.Session(profile_name=profile) self.logger.info("AWS Session created") # Get AWS EC2 Resource self.ec2 = self.session.resource("ec2") self.logger.info("AWS EC2 resource created") # Get AWS EC2 Client self.ec2_client = self.ec2.meta.client self.logger.info("AWS EC2 client created") self.eni_mapping = kwargs.pop("eni_mappings", settings.ENI_MAPPING) self.cidr_suffix_ips_number_mapping = kwargs.pop( "cidr_suffix_ips_number_mapping", settings.CIDR_SUFFIX_IPS_NUMBER_MAPPING ) self.tag_base_name = kwargs.pop("tag_base_name", settings.TAG_BASE_NAME) self.hvm_only_instance_types = kwargs.pop( "hvm_only_instance_types", settings.HVM_ONLY_INSTANCE_TYPES ) if kwargs: raise TypeError("Unexpected **kwargs: %r" % kwargs) self.config = { "vpcs": {}, "instances_groups": [] } resources_params = { "ec2": self.ec2, "ec2_client": self.ec2_client, "tag_base_name": self.tag_base_name, "log_level": self.log_level, "boto_log_level": self.boto_log_level } self.vpcs = Vpcs(**resources_params) self.internet_gateways = InternetGateways(**resources_params) self.subnets = Subnets(**resources_params) self.security_groups = SecurityGroups(**resources_params) self.route_tables = RouteTables(**resources_params) self.network_acls = NetworkAcls(**resources_params) self.network_interfaces = NetworkInterfaces(**resources_params) self.instances = Instances(**resources_params)
def test(args, shared_model, env_conf): ptitle('Test Agent') gpu_id = args.gpu_ids[-1] log = {} setup_logger('{}_log'.format(args.env), r'{0}{1}_log'.format(args.log_dir, args.env)) log['{}_log'.format(args.env)] = logging.getLogger( '{}_log'.format(args.env)) d_args = vars(args) for k in d_args.keys(): log['{}_log'.format(args.env)].info('{0}: {1}'.format(k, d_args[k])) torch.manual_seed(args.seed) if gpu_id >= 0: torch.cuda.manual_seed(args.seed) print("test proc:") env = AllowBacktracking(make_local_env(env_conf['game'], env_conf['level'], stack=False, scale_rew=False)) print("test got env:", env.observation_space) reward_sum = 0 start_time = time.time() num_tests = 0 reward_total_sum = 0 player = Agent(None, env, args, None) player.gpu_id = gpu_id player.model = A3Clstm( player.env.observation_space.shape[0], player.env.action_space) player.state = player.env.reset() player.eps_len += 2 player.state = torch.from_numpy(player.state).float() if gpu_id >= 0: with torch.cuda.device(gpu_id): player.model = player.model.cuda() player.state = player.state.cuda() flag = True max_score = 0 while True: if flag: if gpu_id >= 0: with torch.cuda.device(gpu_id): player.model.load_state_dict(shared_model.state_dict()) else: player.model.load_state_dict(shared_model.state_dict()) player.model.eval() flag = False player.action_test() reward_sum += player.reward """ if player.done and player.info['ale.lives'] > 0 and not player.max_length: state = player.env.reset() player.eps_len += 2 player.state = torch.from_numpy(state).float() if gpu_id >= 0: with torch.cuda.device(gpu_id): player.state = player.state.cuda() """ if player.done or player.max_length: flag = True num_tests += 1 reward_total_sum += reward_sum reward_mean = reward_total_sum / num_tests log['{}_log'.format(args.env)].info( "Time {0}, episode reward {1}, episode length {2}, reward mean {3:.4f}". format( time.strftime("%Hh %Mm %Ss", time.gmtime(time.time() - start_time)), reward_sum, player.eps_len, reward_mean)) if args.save_max and reward_sum >= max_score: max_score = reward_sum if gpu_id >= 0: with torch.cuda.device(gpu_id): state_to_save = player.model.state_dict() torch.save(state_to_save, '{0}{1}.dat'.format(args.save_model_dir, args.env)) else: state_to_save = player.model.state_dict() torch.save(state_to_save, '{0}{1}.dat'.format(args.save_model_dir, args.env)) reward_sum = 0 player.eps_len = 0 state = player.env.reset() player.eps_len += 2 time.sleep(10) player.state = torch.from_numpy(state).float() if gpu_id >= 0: with torch.cuda.device(gpu_id): player.state = player.state.cuda()
parser.add_argument("outfile", help="Output File") parser.add_argument("start", help="Start Time", type=float) parser.add_argument("--debug", help="Debug", action="store_true") parser.add_argument("--output-prefix", help="File prefix in output", default=".") return parser.parse_args() if __name__ == "__main__": args = _get_args() session_dir = os.path.join(conf.get_opt("session_dir"), args.session) status_file = os.path.join(session_dir, args.volume, "%s.status" % urllib.quote_plus(args.brick)) status_file_pre = status_file + ".pre" mkdirp(os.path.join(session_dir, args.volume), exit_on_err=True, logger=logger) mkdirp(os.path.join(conf.get_opt("log_dir"), args.session, args.volume), exit_on_err=True) log_file = os.path.join(conf.get_opt("log_dir"), args.session, args.volume, "brickfind.log") setup_logger(logger, log_file, args.debug) time_to_update = int(time.time()) brickfind_crawl(args.brick, args) with open(status_file_pre, "w", buffering=0) as f: f.write(str(time_to_update)) sys.exit(0)
from utils import setup_logger from settings import run_folder ### SET all LOGGER_DISABLED to True to disable logging ### WARNING: the mcts log file gets big quite quickly LOGGER_DISABLED = { 'main':False , 'memory':False , 'tourney':False , 'mcts':False , 'model': False} logger_mcts = setup_logger('logger_mcts', run_folder + 'logs/logger_mcts.log') logger_mcts.disabled = LOGGER_DISABLED['mcts'] logger_main = setup_logger('logger_main', run_folder + 'logs/logger_main.log') logger_main.disabled = LOGGER_DISABLED['main'] logger_tourney = setup_logger('logger_tourney', run_folder + 'logs/logger_tourney.log') logger_tourney.disabled = LOGGER_DISABLED['tourney'] logger_memory = setup_logger('logger_memory', run_folder + 'logs/logger_memory.log') logger_memory.disabled = LOGGER_DISABLED['memory'] logger_model = setup_logger('logger_model', run_folder + 'logs/logger_model.log') logger_model.disabled = LOGGER_DISABLED['model']