def run(model, properties, targets, filename, start=None, end=None, ratio=[1], shuffle=False, overwrite=False, args={}): if os.path.isfile(filename) and not overwrite: print("Filename %s already exists and the overwrite flag is not set!" % filename) return db.open() if type(properties) != list: properties = [properties] if type(targets) != list: targets = [targets] #generate the dataset dataset = generateDataset(model, properties, targets, start=start, end=end, args=args) if shuffle: #randomize it dataset = randomizeDataset(dataset) print("Randomized dataset and labels.") if len(ratio) == 1: data = dataset else: data = [] split = [] #the lenghts of the dataset pieces mainLen = len(dataset['dataset']) for rat in ratio: split.append( int((rat * mainLen) / np.sum(ratio)) ) #calculate the length by keeping the given ratio print(split, ratio) index = 0 for i, spl in enumerate(split): end = (spl + index) if i != len(split) -1 else None #because of integer division, add anything left on the last iteration newDataset = {} for key in dataset: if type(dataset[key]) != np.ndarray or len(dataset[key]) != mainLen: newDataset[key] = dataset[key] print("Unable to split key %s. Leaving it as is." % key) else: newDataset[key] = dataset[key][index:end] data.append(newDataset) index += spl #save it if save: saveDataset(filename, data) print("saved dataset and labels as %a." % filename) db.close()
def run(action, processor_type, processors, start=None, end=None, list=False, force=False): db.open() processorObjects = propertyObjects if processor_type == "property" else postprocessorObjects if action == 'generate': objects = getObjectsFromNameList(processors, processorObjects) if processor_type == 'property': generateProperties(objects, start=start, end=end, force=force) else: generatePostprocessors(objects) elif action == 'remove': for key in db.list_keys(): meta = db.getMetadata(key) if 'type' in meta: if key in processors or meta['type'] == processor_type: db.remove(key) print("Successfully removed key %s" % key) elif args.action == None or args.list: print("Available properties:", str.join(',', [prop.name for prop in propertyObjects])) db.close()
parser.add_argument( '--course', dest='course', action="store_true", help="Downloads and saves or upgrades historical course data.") parser.add_argument( '--blockchain', dest='blockchain', action="store_true", help="Downloads and saves or upgrades blockchain data.") parser.add_argument('--start', type=int, default=0, help='From which block to start downloading.') parser.add_argument('--end', type=int, default=None, help='Until which block to download.') parser.set_defaults(course=False) parser.set_defaults(blockchain=False) args, _ = parser.parse_known_args() db.open() if args.course: downloadCourse() if args.blockchain: downloadBlockchain(args.start, args.end) db.close()
def __exit__(self, exc_type, exc_value, traceback): db.close()