def test_load(caplog):
caplog.set_level(logging.INFO)
# Save data for other tests
(
pytest.train_data,
pytest.train_labels,
pytest.test_data,
pytest.test_labels,
) = database.load(standardized=True, printSize=True)
assert caplog.record_tuples[3][2] == "---Train samples: 7352"
assert caplog.record_tuples[4][2] == "---Test samples: 2947"
assert caplog.record_tuples[5][2] == "Dataset standardized."
# Save data for other tests
__, __, __, __ = database.load(
standardized=True,
printSize=True,
train_data_path="UCI HAR Dataset/train/X_train.txt",
train_labels_path="UCI HAR Dataset/train/y_train.txt",
test_data_path="UCI HAR Dataset/test/X_test.txt",
test_labels_path="UCI HAR Dataset/test/y_test.txt",
)
assert caplog.record_tuples[3][2] == "---Train samples: 7352"
assert caplog.record_tuples[4][2] == "---Test samples: 2947"
assert caplog.record_tuples[5][2] == "Dataset standardized."
def load_db():
global DB
db.load()
DB['con'] = db.create_engine(db.CONFIG['db_path'], echo=False)
DB['inspector'] = db.Inspector.from_engine(DB['con'])
DB['table_names'] = DB['inspector'].get_table_names()
DB['metadata'] = MetaData(bind=DB['con'])
DB['metadata'].reflect()
def add_to_database(member: discord.Member):
# save the database as members
members = database.load("data.json")
# get the member id (string)
nameid = str(member.id)
# if they already exist then return
if nameid in members:
return
# otherwise add them to the database with default items and save it
else:
members[nameid] = database.load("items.json")
database.save(members, "data.json")
def openfile(self): # TODO: On Windows, access to folders containing special characters fails. May be due to the fact that Windows doesn't use Unicode.
if utils_ui.save_warning(database):
filename = QtWidgets.QFileDialog.getOpenFileName(self, 'Open Database')[0]
if filename:
if filename[-3:] != '.db':
QtWidgets.QMessageBox.warning(self, 'Warning', "Database has wrong extension.", buttons=QtWidgets.QMessageBox.Ok)
else:
database.load(database, filename)
self.update_database_info(os.path.basename(filename))
self.update_widgets()
self.update_log("Database '{}' was loaded successfully".format(os.path.basename(filename)))
def predict(inference_date):
"""
Loads one month of inference data from a GCP bucket and runs inference using
the locally saved model. The predictions are uploaded to GCP.
:param inference_date: datetime object, the first day of the month to perform inference on
"""
inference_date = inference_date.split(' ')[0]
# load inference data from a GCP bucket as a pandas dataframe
inference_df = db.load(f'{data_key}-inference-{inference_date}')
# drop non-feature columns
x_inference = inference_df.drop(['cust_id', 'date'], axis=1)
# insert DataContract validation code here #
############################################
x_inference = x_inference.fillna(0)
# load model from local directory
model = pickle.load(open("fitted_objects/model.pkl", "rb"))
# run inference, and create a dataframe to store the predicted probabilities
probs = model.predict_proba(x_inference)[:, 1]
probs_df = pd.DataFrame(probs, columns=["predicted_churn_probability"])
# save predictions to a GCP bucket
db.save(f'{data_key}-predictions-{inference_date}', probs_df)
def main(args):
# Load data and ground-truth
train_data, train_labels, test_data, test_labels = database.load(
standardized=True,
printSize=True,
train_data_path=args.train_data,
train_labels_path=args.train_labels,
test_data_path=args.test_data,
test_labels_path=args.test_labels,
)
train_labels = train_labels.ravel()
# Training
model = algorithm.train(train_data, train_labels, args)
# Logging scores on train and test sets
logging.info("---Training set accuracy: %f" %
model.score(train_data, train_labels))
logging.info("---Testing set accuracy: %f" %
model.score(test_data, test_labels))
predictions = algorithm.predict(test_data, model)
# Evaluate the predictions
evaluator.evaluate(predictions, test_data, test_labels, args.output_folder,
args.model, model)
def get(artist, song, album=None):
"""Fetch the lyrics as text."""
info = artist, song, album or ''
try:
return database.load(*info)
except LookupError:
return fetcher.fetch(*info)
def get(self):
load()
families = Family.query()
states = {}
for family in families:
if family.state in states:
if not family.city in states[family.state]:
states[family.state].append(family.city)
else:
states[family.state] = [family.city]
states = json.dumps(states)
self.redirect('/search')
def eval(eval_date):
"""
Loads one month of labelled data from a GCP bucket and calculates model performance
on that month.
:param eval_date: datetime object, the first day of the month to perform evaluation on
"""
eval_date = eval_date.split(' ')[0]
# load eval data from a GCP bucket as a pandas dataframe
df = db.load(f'{data_key}-labelled-{eval_date}')
# calculate accuracy, roc_auc, and confusion matrix
no_promo_df = df[df['promo'] == 0]
targets = no_promo_df["churn"]
probs = no_promo_df["predicted_churn_probability"]
preds = [1 if prob > 0.5 else 0 for prob in probs]
accuracy = accuracy_score(targets, preds)
roc_auc = roc_auc_score(targets, probs)
# calculate revenue
n_active_custs = len(df[
df.churn == 0]) # number of active customers in the evaluation month
revenue = revenue_per_cust * n_active_custs
# insert foundations metric tracking here #
###########################################
return accuracy, roc_auc, n_active_custs, revenue
def ingest(record_type, **kwargs):
"""
Run the ingestion flow:
1. acquire data from files or API
2. optionally validate data, filtering invalid records
3. optionally write data to output files
4. optionally load valid records into the database
"""
version = mds.Version(kwargs.pop("version", common.DEFAULT_VERSION))
version.raise_if_unsupported()
datasource = common.get_data(record_type, **kwargs, version=version)
data_key = mds.Schema(record_type).data_key
# validation and filtering
if not kwargs.pop("no_validate", False):
print(f"Validating {record_type} @ {version}")
valid, errors, removed = validation.validate(record_type, datasource, version=version)
seen = sum([len(d["data"][data_key]) for d in datasource])
passed = sum([len(v["data"][data_key]) for v in valid])
failed = sum([len(r["data"][data_key]) for r in removed])
print(f"{seen} records, {passed} passed, {failed} failed")
else:
print("Skipping data validation")
valid = datasource
removed = None
# output to files if needed
output = kwargs.pop("output", None)
if output:
f = mds.DataFile(record_type, output)
f.dump_payloads(valid)
if removed:
f.dump_payloads(removed)
# load to database
loading = not kwargs.pop("no_load", False)
if loading and len(valid) > 0:
database.load(valid, record_type, **kwargs, version=version)
else:
print("Skipping data load")
print(f"{record_type} complete")
def get(self, bank_id):
banking_configuration = bank.load_config(
bank.env()['main_config_file'])
db = database.load(bank.env()['database_folder'])
bank_config = banking_configuration.banks[bank_id]
access_code = database.get_bank_access_code(db, bank_config)
return jsonify(io.encode_object(access_code))
def choose_db():
nonlocal d
nonlocal l0
nonlocal b3
filename = QtWidgets.QFileDialog.getOpenFileName(d, 'Choose Database')[0]
if filename:
if filename.find('.db') != -1:
database.load(module, filename)
l0.setText(os.path.basename(filename))
load_mogs()
else:
QtWidgets.QMessageBox.warning(b3, '', 'Database not in *.db format',
QtWidgets.QMessageBox.Cancel, QtWidgets.QMessageBox.NoButton,
QtWidgets.QMessageBox.NoButton)
l0.setText('')
def get(self, account_id):
banking_configuration = bank.load_config(
bank.env()['main_config_file'])
db = database.load(bank.env()['database_folder'])
account = account_from_config(
banking_configuration.accounts[account_id])
access_code = database.get_account_access_code(db, account)
return jsonify(io.encode_object(access_code))
def login():
if request.method == 'POST':
if (database.checkLogin(request.form['name'],
request.form['password'])):
session['name'] = request.form['name']
session['password'] = request.form['password']
data = database.load(session['name'], session['password'])
if (data != False):
session['items'] = database.load(session['name'],
session['password'])
session.modified = True
return redirect('/')
return redirect('/login')
if 'name' in session.keys():
del session['name']
if 'password' in session.keys():
del session['password']
return render_template('Login.html')
def load_everything(self):
"""Load all the data from the database"""
data = database.load(self.name)
if data:
# Using get will allow backwards compatibility in the future
self.key = data.get('key', '')
self.owner_key = data.get('owner_key', '')
self.layers = data.get('layers', [])
self.timestamp = data.get('timestamp', time_current())
self.has_loaded = True
def get(self, account_id):
banking_configuration = bank.load_config(
bank.env()['main_config_file'])
db = database.load(bank.env()['database_folder'])
account = account_from_config(
banking_configuration.accounts[account_id])
encoded_account = io.encode_account(account)
encoded_account['balance'] = database.get_account_balance(db, account)
return jsonify(encoded_account)
def lookup(code):
val = database.load(code)
if val is None:
return index()
return render_template("index.html",
start_data=repr(
json.dumps({
"code": eval(val[0]),
"skip_tree": bool(val[1]),
"hide_return_frames": bool(val[2])
})))
async def stats(ctx):
# load from the database
members = database.load("data.json")
memberid = str(ctx.author.id)
stats = members[memberid]
statmsg = ""
statmsg = statmsg + f'Current Health: {stats["health"]}\n'
statmsg = statmsg + f'Current Magic: {stats["magic"]}\n'
statmsg = statmsg + f'Kill Count: {stats["kills"]}\n'
statmsg = statmsg + f'Death Count: {stats["deaths"]}\n'
await ctx.send(f'''```{statmsg}```''')
async def craft(ctx, *, item=None):
# load members
members = database.load("data.json")
# define self
nameid = str(ctx.author.id)
# load recipes
recipes = database.load("recipes.json")
# check if recipe is None
if item == None:
msg = '''```Please select an item from below!\n'''
for recipe in recipes:
msg = msg + recipe + "\n"
msg = msg + "```"
await ctx.send(msg)
return
# check if the recipe exists
if members[nameid]["weapon"] == item:
await ctx.send("```You already have that item!```")
return
if item in recipes:
# define the recipe
recipe = recipes[item]
# check if you have every item
for material in recipe:
# if you dont have enough of the item then end loop
if recipe[material] - members[nameid]["items"][material] > 0:
await ctx.send(
"```You don't have enough items! Do some exploring!```")
return
# will only run if the first was successful (because it would have returned)
for material in recipe:
# subtract the item
members[nameid]["items"][material] -= recipe[material]
database.save(members, "data.json")
members[nameid]["weapon"] = item
database.save(members, "data.json")
await ctx.send(f"```Crafted {item}, check your items.```")
else:
await ctx.send("```That item does not exist!```")
def get(self, account_id):
banking_configuration = bank.load_config(
bank.env()['main_config_file'])
db = database.load(bank.env()['database_folder'])
account = account_from_config(
banking_configuration.accounts[account_id])
transactions = database.find_transactions(
db,
account,
sort_direction=database.SortDirection.NEWEST_TRANSACTION_FIRST)
return jsonify(list(map(io.encode_object, transactions)))
def put(self, bank_id):
banking_configuration = bank.load_config(
bank.env()['main_config_file'])
db = database.load(bank.env()['database_folder'])
code = request.get_json()['code']
bank_config = banking_configuration.banks[bank_id]
access_code = datatypes.BankAccessCode(code=code,
date=datetime.utcnow(),
bank_id=bank_config.id)
database.update_bank_access_code(db, bank_config, access_code)
return jsonify(io.encode_object(access_code))
def process(hum):
hum_normalized = normalize(hum)
data = database.load()
l = {}
for alias in data:
original = data[alias]['pitches']
normalized = normalize(original)
dtwdist, dtwpath = dtw(hum_normalized, normalized, euc, 14)
l[alias] = (dtwdist, dtwpath)
for alias in l:
print(alias, l[alias][0])
def analyze(file_in):
"""
Returns (bpm, Key) after analyzing the given audio file
"""
data = load(file_in)
if data is not None:
bpm, key, scale = data
return bpm, Key(key, scale)
bpm = get_bpm(file_in)
key = get_key(file_in)
save(file_in, bpm, key)
return bpm, key
def process(hum):
hum_normalized = normalize(hum)
data = database.load()
l = {}
for alias in data:
original = data[alias]['pitches']
normalized = normalize(original)
dtwdist, dtwpath = dtw(hum_normalized, normalized, euc, 14)
l[alias] = (dtwdist, dtwpath)
for alias in l:
print(alias, l[alias][0])
def execargs():
if sys.argv[0] == '-h' or sys.argv[0] == '--help' or sys.argv[0] == '/?':
funcs.help()
elif sys.argv[0] == '-c':
database.load()
for k in database.database.keys():
print(k)
elif sys.argv[0] == '-w':
if not funcs.areUsure():
print('failed')
database.database = {}
database.dump()
print('\nok\n')
elif sys.argv[0] == '-a' and len(sys.argv) == 3 and os.path.exists(
sys.argv[2]):
database.load()
with open(sys.argv[2], 'r', encoding='utf-8') as f:
database.addCountedWords(process.count(f.read()), sys.argv[1])
database.dump()
print('\nok\n')
elif sys.argv[0] == '-p' and len(sys.argv) == 2 and os.path.exists(
sys.argv[1]):
database.load()
scores = process.getscores(sys.argv[1])
for cat in scores.keys():
print(f'{cat}\t:\t{scores[cat]}')
def main_menu():
"""Run the main menu of the program"""
repeat = False
numberBase = "default"
curDB = "root"
prompt = ("Repeat is %s.\nCurrent Default Database: %s.\n"
"Enter a valid command, or help.")
aliases, dbAliases = init_aliases()
dbConfig, production = config.init_config_file()
database = db.init(dbConfig)
command = ""
while command != "quit":
entry = ui.get_input(
prompt % (("on" if repeat else "off"), curDB), "lower")
command, args = parse_entry(aliases, dbAliases, entry)
if command == "help":
print("\nI'm working on the help file!!")
elif command == "repeat":
repeat = not repeat
print("\nRepeat is now %s." % ("on" if repeat else "off"))
elif command == "run":
tracker.produce(database, production)
elif command in ("switch", "root", "global"):
curDB = change_database(curDB, command)
elif command == "database":
db.basic_actions(database, curDB, *tuple(args))
elif command == "load":
db.load(database, *tuple(args))
elif command == "save":
db.save(database, dbConfig, *tuple(args))
elif command == "wipe":
wipePrompt = "Are you SURE you want to erase the database? Y/N"
if ui.get_binary_choice(wipePrompt):
database = db.init()
elif command != "quit":
print("\n\"%s\" is not a recognized command." % command)
async def help(ctx):
# read the file
helpmsg = database.load("help.json")
# set the embed attributes to the sections from the file
embed = discord.Embed(title=helpmsg["title"],
url=helpmsg["link"],
description=helpmsg["content"],
color=discord.Colour.from_rgb(
helpmsg["color"][0], helpmsg["color"][1],
helpmsg["color"][2]))
embed.set_thumbnail(url=helpmsg["image"])
# send it
await ctx.send(embed=embed)
async def permaloop():
event_log.info("Entering permaloop ...")
await globals.bot.wait_until_ready()
while not globals.bot.is_closed():
checkpoint = time.time()
modification = False
for s in globals.seminars:
modification = await s.make_request("ulohy") # check tasks
for rnd in s.rounds.values():
modification = await s.make_request("vysledky", rnd.id) # check results for each round
if modification:
db.load(cn.FB_SEMINARS, s.name, s.to_dict()) # load to database if changes
management_log.info("Changes loaded to database!")
management_log.info(f"permaloop updated in {int((time.time()-checkpoint)*1000)}ms")
# users update
management_log.info("Creating and uploading {0} users".format(len(globals.server.members)-len(globals.users)))
for mem in globals.server.members:
hp.create_user(str(mem.id))
for userid in globals.users.keys():
db.load(cn.FB_USERS, str(userid), vars(globals.users[userid]))
await asyncio.sleep(cn.MINIMAL_CHECKING_DELAY)
event_log.error("Exited infinite loop")
def put(self, account_id):
banking_configuration = bank.load_config(
bank.env()['main_config_file'])
db = database.load(bank.env()['database_folder'])
code = request.get_json()['code']
account = account_from_config(
banking_configuration.accounts[account_id])
access_code = datatypes.AccountAccessCode(code=code,
date=datetime.utcnow(),
account_id=account.id)
database.update_account_access_code(db, account, access_code)
return jsonify(io.encode_object(access_code))
async def search(ctx, location="None"):
if location == "None":
await ctx.send('''Please provide a location!
Choose from: mine, dessert, ocean, forest''')
else:
members = database.load("data.json")
nameid = str(ctx.author.id)
amount = random.randint(0, 3)
amount2 = random.randint(0, 3)
if location.lower().startswith("m"):
msg = await ctx.send("```Attempting to search the mine...```")
await asyncio.sleep(3)
await msg.edit(
content=
f'''```You found {amount} rocks.\nYou found {amount2} iron.```'''
)
members[nameid]["items"]["rocks"] += amount
members[nameid]["items"]["iron"] += amount2
elif location.lower().startswith("d"):
msg = await ctx.send("```Attempting to search the dessert...```")
await asyncio.sleep(3)
await msg.edit(
content=
f'''```You found {amount} cactus.\nYou found {amount2} cups of sand.```'''
)
members[nameid]["items"]["cactus"] += amount
members[nameid]["items"]["sand"] += amount2
elif location.lower().startswith("o"):
msg = await ctx.send("```Attempting to search the ocean...```")
await asyncio.sleep(3)
await msg.edit(
content=
f'''```You found {amount} fish.\nYou found {amount2} cups of water.```'''
)
members[nameid]["items"]["fish"] += amount
members[nameid]["items"]["water"] += amount2
elif location.lower().startswith("f"):
msg = await ctx.send("```Attempting to search the forest...```")
await asyncio.sleep(3)
await msg.edit(
content=
f'''```You found {amount} apples.\nYou found {amount2} sticks.```'''
)
members[nameid]["items"]["apples"] += amount
members[nameid]["items"]["sticks"] += amount
else:
await ctx.send(
"```Please provide a valid location!```\nChoose from: mine, dessert, ocean, forest"
)
database.save(members, "data.json")
async def updateloop(rnd: Round):
"""Loop responsible for updating seminar countdowns, etc ..."""
event_log.info(f"Entering updateloop for seminar {rnd.sem.name} ...")
await globals.bot.wait_until_ready()
while not globals.bot.is_closed():
# Garbage collector
if rnd.remaining.total_seconds() <= 0:
event_log.info(
f"Exiting updateloop for seminar {rnd.sem.name} ... the round has ended."
)
await rnd.end()
rnd.sem.rounds.pop(rnd.id)
db.load(cn.FB_SEMINARS, rnd.sem.name, rnd.sem.to_dict())
del (rnd) # remove round
return
await asyncio.sleep(cn.UPDATE_DELAY)
try:
await rnd.check_round_message()
except Exception as e:
event_log.warning(
f"{rnd.sem.name} >>> Error occured while updating {e}",
exc_info=True)
def index():
chinese = {
'1': '一',
'2': '二',
'3': '三',
'4': '四',
'5': '五',
'6': '六',
'0': '日'
}
name = request.args.get('name')
zimuzu = request.args.get('zimuzu')
day = request.args.get('day')
if name != "" and zimuzu != "" and name != None and zimuzu != None:
database.dump(name, zimuzu, day)
result = database.load(day)
return render_template('index.html', result=result)
else:
result = []
for days in range(7):
result.append("周" + chinese[str(days)])
result.append(database.load(days))
return render_template('index.html', result=result)
async def items(ctx):
# load from the database
members = database.load("data.json")
memberid = str(ctx.author.id)
items = members[memberid]["items"]
# item display
itemmsg = ""
itemmsg = itemmsg + 'Here is your items:\n'
for item in items:
itemmsg = itemmsg + f'{item}, {members[memberid]["items"][item]}\n'
# money display
itemmsg = itemmsg + f'Here is your money: {members[memberid]["money"]}\n'
# weapon display
itemmsg = itemmsg + f'Here is your weapon: {members[memberid]["weapon"]}'
await ctx.send(f'''```{itemmsg}```''')
def testDB():
# new store
database.store = {}
# add to database
database.addToDB("sec", fid = "test")
database.addToDB("sec", msg = "anotherone")
# get from database
assert len(database.get("sec", database.getDate())) == 2
assert (database.get("sec", database.getDate()))[0]['fid'] == "test"
assert (database.get("sec", database.getDate()))[1]['msg'] == "anotherone"
assert (database.get("oth", database.getDate())) == []
# save database to pickle
database.save("temp.pickle")
# remove from database
database.remove(database.getDate())
assert database.get("sec", database.getDate()) == []
# load database from pickle
database.store = database.load("temp.pickle")
# get from database
assert (database.get("sec", database.getDate()))[0]['fid'] == "test"
assert (database.get("sec", database.getDate()))[1]['msg'] == "anotherone"
os.remove("temp.pickle")
output_list = slack_rtm_output
if output_list and len(output_list) > 0:
for output in output_list:
if output and 'text' in output and "uploaded" in output['text']:
if output and 'file' in output and output['file'] and 'id' in output['file']:
id = "id:"+output['file']['id']
return id, \
output['channel']
if output and 'text' in output and AT_BOT in output['text']:
# return text after the @ mention, whitespace removed
return output['text'].split(AT_BOT)[1].strip().lower(), \
output['channel']
return None, None
if os.path.exists("/volume/slackdb.pickle"):
db.store = db.load("/volume/slackdb.pickle")
else:
pickle.dump({}, open("/volume/slackdb.pickle", 'w+'))
db.store = db.load("/volume/slackdb.pickle")
slack_client = SlackClient(os.environ.get('SLACK_BOT_TOKEN'))
if __name__ == "__main__":
READ_WEBSOCKET_DELAY = 0.5 # 1 second delay between reading from firehose
if slack_client.rtm_connect():
print("StarterBot connected and running!")
while True:
command, channel = parse_slack_output(slack_client.rtm_read())
if command and channel:
handle_command(command, channel)
time.sleep(READ_WEBSOCKET_DELAY)
else:
