def run(model, properties, targets, filename, start=None, end=None, ratio=[1], shuffle=False, overwrite=False, args={}):
if os.path.isfile(filename) and not overwrite:
print("Filename %s already exists and the overwrite flag is not set!" % filename)
return
db.open()
if type(properties) != list:
properties = [properties]
if type(targets) != list:
targets = [targets]
#generate the dataset
dataset = generateDataset(model, properties, targets, start=start, end=end, args=args)
if shuffle:
#randomize it
dataset = randomizeDataset(dataset)
print("Randomized dataset and labels.")
if len(ratio) == 1:
data = dataset
else:
data = []
split = [] #the lenghts of the dataset pieces
mainLen = len(dataset['dataset'])
for rat in ratio:
split.append( int((rat * mainLen) / np.sum(ratio)) ) #calculate the length by keeping the given ratio
print(split, ratio)
index = 0
for i, spl in enumerate(split):
end = (spl + index) if i != len(split) -1 else None #because of integer division, add anything left on the last iteration
newDataset = {}
for key in dataset:
if type(dataset[key]) != np.ndarray or len(dataset[key]) != mainLen:
newDataset[key] = dataset[key]
print("Unable to split key %s. Leaving it as is." % key)
else:
newDataset[key] = dataset[key][index:end]
data.append(newDataset)
index += spl
#save it
if save:
saveDataset(filename, data)
print("saved dataset and labels as %a." % filename)
db.close()
def run(action, processor_type, processors, start=None, end=None, list=False, force=False):
db.open()
processorObjects = propertyObjects if processor_type == "property" else postprocessorObjects
if action == 'generate':
objects = getObjectsFromNameList(processors, processorObjects)
if processor_type == 'property':
generateProperties(objects, start=start, end=end, force=force)
else:
generatePostprocessors(objects)
elif action == 'remove':
for key in db.list_keys():
meta = db.getMetadata(key)
if 'type' in meta:
if key in processors or meta['type'] == processor_type:
db.remove(key)
print("Successfully removed key %s" % key)
elif args.action == None or args.list:
print("Available properties:", str.join(',', [prop.name for prop in propertyObjects]))
db.close()
parser.add_argument(
'--course',
dest='course',
action="store_true",
help="Downloads and saves or upgrades historical course data.")
parser.add_argument(
'--blockchain',
dest='blockchain',
action="store_true",
help="Downloads and saves or upgrades blockchain data.")
parser.add_argument('--start',
type=int,
default=0,
help='From which block to start downloading.')
parser.add_argument('--end',
type=int,
default=None,
help='Until which block to download.')
parser.set_defaults(course=False)
parser.set_defaults(blockchain=False)
args, _ = parser.parse_known_args()
db.open()
if args.course: downloadCourse()
if args.blockchain:
downloadBlockchain(args.start, args.end)
db.close()
def __exit__(self, exc_type, exc_value, traceback):
db.close()