def generatePostprocessors(postprocessors): for proc in postprocessors: proc_name = proc.name available_props = [x.name for x in propertyObjects if db.has_key(x.name)] if proc.requires == [] else proc.requires for prop_name in available_props: print(proc_name, prop_name) proc.requires = [prop_name] generateProperties([proc]) new_name = proc_name % prop_name #try: # Problem: renaming the key doesn't change the name of the df columns # db.rename(proc_name, proc_name % prop_name) #except NodeError: print("Renaming %s to %s" % (proc_name, new_name)) data = db.get(proc_name) meta = db.getMetadata(proc_name) data.rename(index=str, columns={prop_name: new_name}) meta['type'] = 'postprocessor' db.save(new_name, data) db.setMetadata(new_name, meta) db.remove(proc_name)
def test_instance_crud(create_clean_db, game): create(create_clean_db, game) gm = Player('*****@*****.**', '') instance = game.create_new_game('our multiplayer game', gm, datetime.utcnow() - timedelta(days=1), datetime.utcnow() + timedelta(days=1)) player = Player('*****@*****.**', '') instance.add_player(player, 'pseudonym', 'one') save(create_clean_db, instance) assert instance.name in [f[1] for f in saves(create_clean_db, player)] assert instance.name not in [f[1] for f in hosts(create_clean_db, player)] assert instance.name in [f[1] for f in hosts(create_clean_db, gm)] db_instance = load(create_clean_db, instance.id.hex) assert db_instance.id == instance.id assert db_instance.player_states[0].first_name == 'pseudonym'
def downloadCourse(): callDataDownloaderCourse(tempFilename) data = loadRawData(tempFilename) #get it os.remove(tempFilename) print("Downloaded data with length " + str(len(data)) + " ticks") #debug processRawCourseData(data) #process a bit to make it suitable for storage df = pd.DataFrame(data) df.set_index('date', inplace=True) print(df.head()) print(df.index[0], type(df.index[0])) db.save('tick', df) # save to db
def saveProperty(name, val, quiet=False): val2 = [] for v in val: v2 = v.copy() #avoid changing the actual values val2.append(v2) df = pd.DataFrame(val2) print(df) df.set_index('date', inplace=True) if not quiet: print("Saving prop " + name + " with values ", df.head(5), df.tail(5)) print("With byte size:", sys.getsizeof(df)) db.save(name, df)
def downloadBlockchain(start=0, targetBlock=None): """Calls the JS script to download a certain block range and saves the result in the DB""" currentBlock = getLatestBlock() + 1 #add 1 for the next block to download if currentBlock < 0: currentBlock = start series = blockSeries if targetBlock is None: targetBlock = 5528000 - series #TODO: Have automatic detection of latest block if series > targetBlock - currentBlock: series = targetBlock - currentBlock print("Starting to download blocks after", currentBlock, " and with target ", targetBlock) attempts = 0 while currentBlock < targetBlock: nextTargetBlock = currentBlock + series - 1 print('Calling js to download ' + str(series) + ' blocks from ' + str(currentBlock)) callDataDownloaderBlockchain(currentBlock, nextTargetBlock, tempFilename) data = loadRawData(tempFilename) #get it if data is None: print("Failed reading", tempFilename, ", redownloading...") attempts += 1 if attempts > attemptsThreshold: raise RuntimeError( "Too many failed data-downloader calls, aborting operation." ) time.sleep( 30 * attempts ) #delay before retrying. Most issues are solved that way. continue attempts = 0 os.remove(tempFilename) data = processRawBlockchainData(data) for key in data: if data[key]: df = pd.DataFrame(data[key]) if key == 'trace': for k in ['gasUsed', 'gas']: fr = df.iloc[0][k] print(fr, type(fr)) df.set_index('date', inplace=True) if key == 'block': #TODO: Ugly fix #the following keys are problematic. Due to a bug in the JS implementation, #some values may be in exponential form, which is more string chars than the max allowance of the DB #so we need to parse them to normal numbers and then convert to string for k in ['totalDifficulty', 'difficulty']: #problematic keys df[k] = df[k].map(lambda x: str(exp_to_int(x))) db.save(key, df) currentBlock += series