def generatePostprocessors(postprocessors):
for proc in postprocessors:
proc_name = proc.name
available_props = [x.name for x in propertyObjects if db.has_key(x.name)] if proc.requires == [] else proc.requires
for prop_name in available_props:
print(proc_name, prop_name)
proc.requires = [prop_name]
generateProperties([proc])
new_name = proc_name % prop_name
#try: # Problem: renaming the key doesn't change the name of the df columns
# db.rename(proc_name, proc_name % prop_name)
#except NodeError:
print("Renaming %s to %s" % (proc_name, new_name))
data = db.get(proc_name)
meta = db.getMetadata(proc_name)
data.rename(index=str, columns={prop_name: new_name})
meta['type'] = 'postprocessor'
db.save(new_name, data)
db.setMetadata(new_name, meta)
db.remove(proc_name)
def test_instance_crud(create_clean_db, game):
create(create_clean_db, game)
gm = Player('*****@*****.**', '')
instance = game.create_new_game('our multiplayer game', gm,
datetime.utcnow() - timedelta(days=1),
datetime.utcnow() + timedelta(days=1))
player = Player('*****@*****.**', '')
instance.add_player(player, 'pseudonym', 'one')
save(create_clean_db, instance)
assert instance.name in [f[1] for f in saves(create_clean_db, player)]
assert instance.name not in [f[1] for f in hosts(create_clean_db, player)]
assert instance.name in [f[1] for f in hosts(create_clean_db, gm)]
db_instance = load(create_clean_db, instance.id.hex)
assert db_instance.id == instance.id
assert db_instance.player_states[0].first_name == 'pseudonym'
def downloadCourse():
callDataDownloaderCourse(tempFilename)
data = loadRawData(tempFilename) #get it
os.remove(tempFilename)
print("Downloaded data with length " + str(len(data)) + " ticks") #debug
processRawCourseData(data) #process a bit to make it suitable for storage
df = pd.DataFrame(data)
df.set_index('date', inplace=True)
print(df.head())
print(df.index[0], type(df.index[0]))
db.save('tick', df) # save to db
def saveProperty(name, val, quiet=False):
val2 = []
for v in val:
v2 = v.copy() #avoid changing the actual values
val2.append(v2)
df = pd.DataFrame(val2)
print(df)
df.set_index('date', inplace=True)
if not quiet:
print("Saving prop " + name + " with values ", df.head(5), df.tail(5))
print("With byte size:", sys.getsizeof(df))
db.save(name, df)
def downloadBlockchain(start=0, targetBlock=None):
"""Calls the JS script to download a certain block range and saves the result in the DB"""
currentBlock = getLatestBlock() + 1 #add 1 for the next block to download
if currentBlock < 0:
currentBlock = start
series = blockSeries
if targetBlock is None:
targetBlock = 5528000 - series #TODO: Have automatic detection of latest block
if series > targetBlock - currentBlock:
series = targetBlock - currentBlock
print("Starting to download blocks after", currentBlock,
" and with target ", targetBlock)
attempts = 0
while currentBlock < targetBlock:
nextTargetBlock = currentBlock + series - 1
print('Calling js to download ' + str(series) + ' blocks from ' +
str(currentBlock))
callDataDownloaderBlockchain(currentBlock, nextTargetBlock,
tempFilename)
data = loadRawData(tempFilename) #get it
if data is None:
print("Failed reading", tempFilename, ", redownloading...")
attempts += 1
if attempts > attemptsThreshold:
raise RuntimeError(
"Too many failed data-downloader calls, aborting operation."
)
time.sleep(
30 * attempts
) #delay before retrying. Most issues are solved that way.
continue
attempts = 0
os.remove(tempFilename)
data = processRawBlockchainData(data)
for key in data:
if data[key]:
df = pd.DataFrame(data[key])
if key == 'trace':
for k in ['gasUsed', 'gas']:
fr = df.iloc[0][k]
print(fr, type(fr))
df.set_index('date', inplace=True)
if key == 'block':
#TODO: Ugly fix
#the following keys are problematic. Due to a bug in the JS implementation,
#some values may be in exponential form, which is more string chars than the max allowance of the DB
#so we need to parse them to normal numbers and then convert to string
for k in ['totalDifficulty',
'difficulty']: #problematic keys
df[k] = df[k].map(lambda x: str(exp_to_int(x)))
db.save(key, df)
currentBlock += series