def main(): args = make_args() config = configparser.ConfigParser() utils.load_config(config, args.config) for cmd in args.modify: utils.modify_config(config, cmd) with open(os.path.expanduser(os.path.expandvars(args.logging)), 'r') as f: logging.config.dictConfig(yaml.load(f)) cache_dir = utils.get_cache_dir(config) os.makedirs(cache_dir, exist_ok=True) shutil.copyfile( os.path.expanduser(os.path.expandvars(config.get('cache', 'category'))), os.path.join(cache_dir, 'category')) category = utils.get_category(config) category_index = dict([(name, i) for i, name in enumerate(category)]) datasets = config.get('cache', 'datasets').split() for phase in args.phase: path = os.path.join(cache_dir, phase) + '.pkl' logging.info('save cache file: ' + path) data = [] for dataset in datasets: logging.info('load %s dataset' % dataset) module, func = dataset.rsplit('.', 1) module = importlib.import_module(module) func = getattr(module, func) data += func(config, path, category_index) if config.getboolean('cache', 'shuffle'): random.shuffle(data) with open(path, 'wb') as f: pickle.dump(data, f) logging.info('%s data are saved into %s' % (str(args.phase), cache_dir))
def main(): args = make_args() config = configparser.ConfigParser() utils.load_config(config, args.config) for cmd in args.modify: utils.modify_config(config, cmd) with open(os.path.expanduser(os.path.expandvars(args.logging)), 'r') as f: logging.config.dictConfig(yaml.load(f)) cache_dir = utils.get_cache_dir(config) os.makedirs(cache_dir, exist_ok=True) mappers, _ = utils.get_dataset_mappers(config) for phase in args.phase: path = os.path.join(cache_dir, phase) + '.pkl' logging.info('save cache file: ' + path) data = [] for dataset in mappers: logging.info('load %s dataset' % dataset) module, func = dataset.rsplit('.', 1) module = importlib.import_module(module) func = getattr(module, func) data += func(config, path, mappers[dataset]) if config.getboolean('cache', 'shuffle'): random.shuffle(data) with open(path, 'wb') as f: pickle.dump(data, f) logging.info('%s data are saved into %s' % (str(args.phase), cache_dir))
def main(): args = make_args() config = configparser.ConfigParser() utils.load_config(config, args.config) for cmd in args.modify: utils.modify_config(config, cmd) with open(os.path.expanduser(os.path.expandvars(args.logging)), 'r') as f: logging.config.dictConfig(yaml.load(f)) cache_dir = utils.get_cache_dir(config) os.makedirs(cache_dir, exist_ok=True) shutil.copyfile(os.path.expanduser(os.path.expandvars(config.get('cache', 'category'))), os.path.join(cache_dir, 'category')) category = utils.get_category(config) category_index = dict([(name, i) for i, name in enumerate(category)]) datasets = config.get('cache', 'datasets').split() for phase in args.phase: path = os.path.join(cache_dir, phase) + '.pkl' logging.info('save cache file: ' + path) data = [] for dataset in datasets: logging.info('load %s dataset' % dataset) module, func = dataset.rsplit('.', 1) module = importlib.import_module(module) func = getattr(module, func) data += func(config, path, category_index) if config.getboolean('cache', 'shuffle'): random.shuffle(data) with open(path, 'wb') as f: pickle.dump(data, f) logging.info('%s data are saved into %s' % (str(args.phase), cache_dir))
def main(): args = make_args() config = configparser.ConfigParser() utils.load_config(config, args.config) for cmd in args.modify: utils.modify_config(config, cmd) with open(os.path.expanduser(os.path.expandvars(args.logging)), 'r') as f: logging.config.dictConfig(yaml.load(f)) cache_dir = utils.get_cache_dir(config) model_dir = utils.get_model_dir(config) category = utils.get_category( config, cache_dir if os.path.exists(cache_dir) else None) anchors = utils.get_anchors(config) anchors = torch.from_numpy(anchors).contiguous() path, step, epoch = utils.train.load_model(model_dir) state_dict = torch.load(path, map_location=lambda storage, loc: storage) dnn = utils.parse_attr(config.get('model', 'dnn'))(model.ConfigChannels( config, state_dict), anchors, len(category)) dnn.load_state_dict(state_dict) height, width = tuple(map(int, config.get('image', 'size').split())) resize = transform.parse_transform(config, config.get('transform', 'resize_test')) transform_image = transform.get_transform( config, config.get('transform', 'image_test').split()) transform_tensor = transform.get_transform( config, config.get('transform', 'tensor').split()) # load image image_bgr = cv2.imread('image.jpg') image_resized = resize(image_bgr, height, width) image = transform_image(image_resized) tensor = transform_tensor(image).unsqueeze(0) # Checksum for key, var in dnn.state_dict().items(): a = var.cpu().numpy() print('\t'.join( map(str, [ key, a.shape, utils.abs_mean(a), hashlib.md5(a.tostring()).hexdigest() ]))) output = dnn(torch.autograd.Variable(tensor, volatile=True)).data for key, a in [ ('image_bgr', image_bgr), ('image_resized', image_resized), ('tensor', tensor.cpu().numpy()), ('output', output.cpu().numpy()), ]: print('\t'.join( map(str, [ key, a.shape, utils.abs_mean(a), hashlib.md5(a.tostring()).hexdigest() ])))
def main(): config = configparser.ConfigParser() config.read('config.ini') cache_dir = utils.get_cache_dir(config) os.makedirs(cache_dir, exist_ok=True) for phase in ['train', 'val', 'test']: path = os.path.join(cache_dir, phase) + '.pkl' logging.info('save cache file: ' + path) data = cache(config, path) #if config.getboolean('cache', 'shuffle'): # random.shuffle(data) with open(path, 'wb') as f: pickle.dump(data, f) logging.info('data are saved into ' + cache_dir)
model = model_from_json(open(os.path.join(basepath, 'model.json')).read()) model.load_weights(os.path.join(basepath, 'weights.h5')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) return model def analyze(basepath, filename): global LOOKBACK, MAX_MIDPOINT_DELTA # Load the data & model df = load_data(basepath, filename) model = load_model(basepath) # Process data samples, labels = preprocess.process_data(df) # Test model and print results print "Running analysis..." predictions = model.predict_classes(samples, batch_size=32) print_results(labels, predictions) if __name__ == '__main__': if len(sys.argv) != 2: print "usage: %s quotes.csv" % sys.argv[0] sys.exit(0) filename = os.path.basename(sys.argv[1]).replace(".csv", ".labels") analyze(utils.get_cache_dir(), filename)
samples, labels = preprocess.process_data(df) # How many samples are we going to leave out for the test set? nb_test = int(len(labels) * 0.2) split = len(labels) - nb_test # Prepare training and test sets X_train = np.array(samples[:split]) y_train = labels[:split] X_test = np.array(samples[split + 1:]) y_test = labels[split + 1:] print len(X_train), 'train sequences' print len(X_test), 'test sequences' # How many classes? num_classes = np.max(labels) + 1 print num_classes, 'classes' # Train Model train_and_save(X_train, X_test, y_train, y_test, num_classes, basepath) if __name__ == '__main__': if len(sys.argv) != 2: print "usage: %s quotes.csv" % sys.argv[0] sys.exit(0) features_file = os.path.basename(sys.argv[1]).replace(".csv", ".features") labels_file = os.path.basename(sys.argv[1]).replace(".csv", ".labels") learn(utils.get_cache_dir(), features_file, labels_file)
model = model_from_json(open(os.path.join(basepath, 'model.json')).read()) model.load_weights(os.path.join(basepath, 'weights.h5')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) return model def analyze(basepath, filename): global LOOKBACK, MAX_MIDPOINT_DELTA # Load the data & model df = load_data(basepath, filename) model = load_model(basepath) # Process data samples, labels = preprocess.process_data(df) # Test model and print results print "Running analysis..." predictions = model.predict_classes(samples, batch_size=32) print_results(labels, predictions) if __name__ == '__main__': if len(sys.argv) != 2: print "usage: %s quotes.csv" % sys.argv[0] sys.exit(0) filename = os.path.basename(sys.argv[1]).replace(".csv", ".labels") analyze(utils.get_cache_dir(), filename)
computed = pd.DataFrame.from_dict(predictions, orient='columns') utils.print_distribution_graph(computed['label'], 'Distribution of Label Buckets') return computed if __name__ == '__main__': if len(sys.argv) != 2: print "usage: %s quotes.csv" % sys.argv[0] sys.exit(0) try: # Load file from ./cache directory filename = os.path.basename(sys.argv[1]).replace(".csv", ".features") gc.disable() packed = pickle.load( open(os.path.join(utils.get_cache_dir(), filename), 'rb')) df = pd.read_msgpack(packed['data']) gc.enable() if packed['format_version'] != FILE_FORMAT_VERSION: print "Encountered unexpected cache file format version %s" % str( data['file_format_version']) sys.exit(1) except IOError: print 'Cannot open features cache file "%s"' % cache sys.exit(1) # Calculate labels labels = calculate(df) # Pack data and save to the cache directory gc.disable()
# Process features samples, labels = preprocess.process_data(df) # How many samples are we going to leave out for the test set? nb_test = int(len(labels) * 0.2) split = len(labels) - nb_test # Prepare training and test sets X_train = np.array(samples[:split]) y_train = labels[:split] X_test = np.array(samples[split+1:]) y_test = labels[split+1:] print len(X_train), 'train sequences' print len(X_test), 'test sequences' # How many classes? num_classes = np.max(labels)+1 print num_classes, 'classes' # Train Model train_and_save(X_train, X_test, y_train, y_test, num_classes, basepath) if __name__ == '__main__': if len(sys.argv) != 2: print "usage: %s quotes.csv" % sys.argv[0] sys.exit(0) features_file = os.path.basename(sys.argv[1]).replace(".csv", ".features") labels_file = os.path.basename(sys.argv[1]).replace(".csv", ".labels") learn(utils.get_cache_dir(), features_file, labels_file)
try: reader = open(sys.argv[1], 'r') quotebook = researchBook.ResearchBook() for line in reader: if line[0] == 'B': last_tick = quotebook.bid(line.rstrip()) elif line[0] == 'A': last_tick = quotebook.ask(line.rstrip()) else: last_tick = quotebook.trade(line.rstrip()) record(quotebook, last_tick) reader.close() except IOError: print 'Cannot open input file "%s"' % sys.argv[1] sys.exit(1) # Run computations df = compute() # Pack data and save to the cache directory gc.disable() packed = { 'data' : df.to_msgpack(compress='blosc'), 'format_version' : FILE_FORMAT_VERSION } filename = os.path.basename(sys.argv[1]).replace(".csv",".features") with open(os.path.join(utils.get_cache_dir(), filename), "wb") as outfile: pickle.dump(packed, outfile, 2) gc.enable()
predictions['label'].append(bucket_price_delta(pt)) # Print distribution of classes computed = pd.DataFrame.from_dict(predictions, orient='columns') utils.print_distribution_graph(computed['label'], 'Distribution of Label Buckets') return computed if __name__ == '__main__': if len(sys.argv) != 2: print "usage: %s quotes.csv" % sys.argv[0] sys.exit(0) try: # Load file from ./cache directory filename = os.path.basename(sys.argv[1]).replace(".csv",".features") gc.disable() packed = pickle.load(open(os.path.join(utils.get_cache_dir(), filename), 'rb')) df = pd.read_msgpack(packed['data']) gc.enable() if packed['format_version'] != FILE_FORMAT_VERSION: print "Encountered unexpected cache file format version %s" % str(data['file_format_version']) sys.exit(1) except IOError: print 'Cannot open features cache file "%s"' % cache sys.exit(1) # Calculate labels labels = calculate(df) # Pack data and save to the cache directory gc.disable() packed = {