def test_load(caplog): caplog.set_level(logging.INFO) # Save data for other tests ( pytest.train_data, pytest.train_labels, pytest.test_data, pytest.test_labels, ) = database.load(standardized=True, printSize=True) assert caplog.record_tuples[3][2] == "---Train samples: 7352" assert caplog.record_tuples[4][2] == "---Test samples: 2947" assert caplog.record_tuples[5][2] == "Dataset standardized." # Save data for other tests __, __, __, __ = database.load( standardized=True, printSize=True, train_data_path="UCI HAR Dataset/train/X_train.txt", train_labels_path="UCI HAR Dataset/train/y_train.txt", test_data_path="UCI HAR Dataset/test/X_test.txt", test_labels_path="UCI HAR Dataset/test/y_test.txt", ) assert caplog.record_tuples[3][2] == "---Train samples: 7352" assert caplog.record_tuples[4][2] == "---Test samples: 2947" assert caplog.record_tuples[5][2] == "Dataset standardized."
def load_db(): global DB db.load() DB['con'] = db.create_engine(db.CONFIG['db_path'], echo=False) DB['inspector'] = db.Inspector.from_engine(DB['con']) DB['table_names'] = DB['inspector'].get_table_names() DB['metadata'] = MetaData(bind=DB['con']) DB['metadata'].reflect()
def add_to_database(member: discord.Member): # save the database as members members = database.load("data.json") # get the member id (string) nameid = str(member.id) # if they already exist then return if nameid in members: return # otherwise add them to the database with default items and save it else: members[nameid] = database.load("items.json") database.save(members, "data.json")
def openfile(self): # TODO: On Windows, access to folders containing special characters fails. May be due to the fact that Windows doesn't use Unicode. if utils_ui.save_warning(database): filename = QtWidgets.QFileDialog.getOpenFileName(self, 'Open Database')[0] if filename: if filename[-3:] != '.db': QtWidgets.QMessageBox.warning(self, 'Warning', "Database has wrong extension.", buttons=QtWidgets.QMessageBox.Ok) else: database.load(database, filename) self.update_database_info(os.path.basename(filename)) self.update_widgets() self.update_log("Database '{}' was loaded successfully".format(os.path.basename(filename)))
def predict(inference_date): """ Loads one month of inference data from a GCP bucket and runs inference using the locally saved model. The predictions are uploaded to GCP. :param inference_date: datetime object, the first day of the month to perform inference on """ inference_date = inference_date.split(' ')[0] # load inference data from a GCP bucket as a pandas dataframe inference_df = db.load(f'{data_key}-inference-{inference_date}') # drop non-feature columns x_inference = inference_df.drop(['cust_id', 'date'], axis=1) # insert DataContract validation code here # ############################################ x_inference = x_inference.fillna(0) # load model from local directory model = pickle.load(open("fitted_objects/model.pkl", "rb")) # run inference, and create a dataframe to store the predicted probabilities probs = model.predict_proba(x_inference)[:, 1] probs_df = pd.DataFrame(probs, columns=["predicted_churn_probability"]) # save predictions to a GCP bucket db.save(f'{data_key}-predictions-{inference_date}', probs_df)
def main(args): # Load data and ground-truth train_data, train_labels, test_data, test_labels = database.load( standardized=True, printSize=True, train_data_path=args.train_data, train_labels_path=args.train_labels, test_data_path=args.test_data, test_labels_path=args.test_labels, ) train_labels = train_labels.ravel() # Training model = algorithm.train(train_data, train_labels, args) # Logging scores on train and test sets logging.info("---Training set accuracy: %f" % model.score(train_data, train_labels)) logging.info("---Testing set accuracy: %f" % model.score(test_data, test_labels)) predictions = algorithm.predict(test_data, model) # Evaluate the predictions evaluator.evaluate(predictions, test_data, test_labels, args.output_folder, args.model, model)
def get(artist, song, album=None): """Fetch the lyrics as text.""" info = artist, song, album or '' try: return database.load(*info) except LookupError: return fetcher.fetch(*info)
def get(self): load() families = Family.query() states = {} for family in families: if family.state in states: if not family.city in states[family.state]: states[family.state].append(family.city) else: states[family.state] = [family.city] states = json.dumps(states) self.redirect('/search')
def eval(eval_date): """ Loads one month of labelled data from a GCP bucket and calculates model performance on that month. :param eval_date: datetime object, the first day of the month to perform evaluation on """ eval_date = eval_date.split(' ')[0] # load eval data from a GCP bucket as a pandas dataframe df = db.load(f'{data_key}-labelled-{eval_date}') # calculate accuracy, roc_auc, and confusion matrix no_promo_df = df[df['promo'] == 0] targets = no_promo_df["churn"] probs = no_promo_df["predicted_churn_probability"] preds = [1 if prob > 0.5 else 0 for prob in probs] accuracy = accuracy_score(targets, preds) roc_auc = roc_auc_score(targets, probs) # calculate revenue n_active_custs = len(df[ df.churn == 0]) # number of active customers in the evaluation month revenue = revenue_per_cust * n_active_custs # insert foundations metric tracking here # ########################################### return accuracy, roc_auc, n_active_custs, revenue
def ingest(record_type, **kwargs): """ Run the ingestion flow: 1. acquire data from files or API 2. optionally validate data, filtering invalid records 3. optionally write data to output files 4. optionally load valid records into the database """ version = mds.Version(kwargs.pop("version", common.DEFAULT_VERSION)) version.raise_if_unsupported() datasource = common.get_data(record_type, **kwargs, version=version) data_key = mds.Schema(record_type).data_key # validation and filtering if not kwargs.pop("no_validate", False): print(f"Validating {record_type} @ {version}") valid, errors, removed = validation.validate(record_type, datasource, version=version) seen = sum([len(d["data"][data_key]) for d in datasource]) passed = sum([len(v["data"][data_key]) for v in valid]) failed = sum([len(r["data"][data_key]) for r in removed]) print(f"{seen} records, {passed} passed, {failed} failed") else: print("Skipping data validation") valid = datasource removed = None # output to files if needed output = kwargs.pop("output", None) if output: f = mds.DataFile(record_type, output) f.dump_payloads(valid) if removed: f.dump_payloads(removed) # load to database loading = not kwargs.pop("no_load", False) if loading and len(valid) > 0: database.load(valid, record_type, **kwargs, version=version) else: print("Skipping data load") print(f"{record_type} complete")
def get(self, bank_id): banking_configuration = bank.load_config( bank.env()['main_config_file']) db = database.load(bank.env()['database_folder']) bank_config = banking_configuration.banks[bank_id] access_code = database.get_bank_access_code(db, bank_config) return jsonify(io.encode_object(access_code))
def choose_db(): nonlocal d nonlocal l0 nonlocal b3 filename = QtWidgets.QFileDialog.getOpenFileName(d, 'Choose Database')[0] if filename: if filename.find('.db') != -1: database.load(module, filename) l0.setText(os.path.basename(filename)) load_mogs() else: QtWidgets.QMessageBox.warning(b3, '', 'Database not in *.db format', QtWidgets.QMessageBox.Cancel, QtWidgets.QMessageBox.NoButton, QtWidgets.QMessageBox.NoButton) l0.setText('')
def get(self, account_id): banking_configuration = bank.load_config( bank.env()['main_config_file']) db = database.load(bank.env()['database_folder']) account = account_from_config( banking_configuration.accounts[account_id]) access_code = database.get_account_access_code(db, account) return jsonify(io.encode_object(access_code))
def login(): if request.method == 'POST': if (database.checkLogin(request.form['name'], request.form['password'])): session['name'] = request.form['name'] session['password'] = request.form['password'] data = database.load(session['name'], session['password']) if (data != False): session['items'] = database.load(session['name'], session['password']) session.modified = True return redirect('/') return redirect('/login') if 'name' in session.keys(): del session['name'] if 'password' in session.keys(): del session['password'] return render_template('Login.html')
def load_everything(self): """Load all the data from the database""" data = database.load(self.name) if data: # Using get will allow backwards compatibility in the future self.key = data.get('key', '') self.owner_key = data.get('owner_key', '') self.layers = data.get('layers', []) self.timestamp = data.get('timestamp', time_current()) self.has_loaded = True
def get(self, account_id): banking_configuration = bank.load_config( bank.env()['main_config_file']) db = database.load(bank.env()['database_folder']) account = account_from_config( banking_configuration.accounts[account_id]) encoded_account = io.encode_account(account) encoded_account['balance'] = database.get_account_balance(db, account) return jsonify(encoded_account)
def lookup(code): val = database.load(code) if val is None: return index() return render_template("index.html", start_data=repr( json.dumps({ "code": eval(val[0]), "skip_tree": bool(val[1]), "hide_return_frames": bool(val[2]) })))
async def stats(ctx): # load from the database members = database.load("data.json") memberid = str(ctx.author.id) stats = members[memberid] statmsg = "" statmsg = statmsg + f'Current Health: {stats["health"]}\n' statmsg = statmsg + f'Current Magic: {stats["magic"]}\n' statmsg = statmsg + f'Kill Count: {stats["kills"]}\n' statmsg = statmsg + f'Death Count: {stats["deaths"]}\n' await ctx.send(f'''```{statmsg}```''')
async def craft(ctx, *, item=None): # load members members = database.load("data.json") # define self nameid = str(ctx.author.id) # load recipes recipes = database.load("recipes.json") # check if recipe is None if item == None: msg = '''```Please select an item from below!\n''' for recipe in recipes: msg = msg + recipe + "\n" msg = msg + "```" await ctx.send(msg) return # check if the recipe exists if members[nameid]["weapon"] == item: await ctx.send("```You already have that item!```") return if item in recipes: # define the recipe recipe = recipes[item] # check if you have every item for material in recipe: # if you dont have enough of the item then end loop if recipe[material] - members[nameid]["items"][material] > 0: await ctx.send( "```You don't have enough items! Do some exploring!```") return # will only run if the first was successful (because it would have returned) for material in recipe: # subtract the item members[nameid]["items"][material] -= recipe[material] database.save(members, "data.json") members[nameid]["weapon"] = item database.save(members, "data.json") await ctx.send(f"```Crafted {item}, check your items.```") else: await ctx.send("```That item does not exist!```")
def get(self, account_id): banking_configuration = bank.load_config( bank.env()['main_config_file']) db = database.load(bank.env()['database_folder']) account = account_from_config( banking_configuration.accounts[account_id]) transactions = database.find_transactions( db, account, sort_direction=database.SortDirection.NEWEST_TRANSACTION_FIRST) return jsonify(list(map(io.encode_object, transactions)))
def put(self, bank_id): banking_configuration = bank.load_config( bank.env()['main_config_file']) db = database.load(bank.env()['database_folder']) code = request.get_json()['code'] bank_config = banking_configuration.banks[bank_id] access_code = datatypes.BankAccessCode(code=code, date=datetime.utcnow(), bank_id=bank_config.id) database.update_bank_access_code(db, bank_config, access_code) return jsonify(io.encode_object(access_code))
def process(hum): hum_normalized = normalize(hum) data = database.load() l = {} for alias in data: original = data[alias]['pitches'] normalized = normalize(original) dtwdist, dtwpath = dtw(hum_normalized, normalized, euc, 14) l[alias] = (dtwdist, dtwpath) for alias in l: print(alias, l[alias][0])
def analyze(file_in): """ Returns (bpm, Key) after analyzing the given audio file """ data = load(file_in) if data is not None: bpm, key, scale = data return bpm, Key(key, scale) bpm = get_bpm(file_in) key = get_key(file_in) save(file_in, bpm, key) return bpm, key
def execargs(): if sys.argv[0] == '-h' or sys.argv[0] == '--help' or sys.argv[0] == '/?': funcs.help() elif sys.argv[0] == '-c': database.load() for k in database.database.keys(): print(k) elif sys.argv[0] == '-w': if not funcs.areUsure(): print('failed') database.database = {} database.dump() print('\nok\n') elif sys.argv[0] == '-a' and len(sys.argv) == 3 and os.path.exists( sys.argv[2]): database.load() with open(sys.argv[2], 'r', encoding='utf-8') as f: database.addCountedWords(process.count(f.read()), sys.argv[1]) database.dump() print('\nok\n') elif sys.argv[0] == '-p' and len(sys.argv) == 2 and os.path.exists( sys.argv[1]): database.load() scores = process.getscores(sys.argv[1]) for cat in scores.keys(): print(f'{cat}\t:\t{scores[cat]}')
def main_menu(): """Run the main menu of the program""" repeat = False numberBase = "default" curDB = "root" prompt = ("Repeat is %s.\nCurrent Default Database: %s.\n" "Enter a valid command, or help.") aliases, dbAliases = init_aliases() dbConfig, production = config.init_config_file() database = db.init(dbConfig) command = "" while command != "quit": entry = ui.get_input( prompt % (("on" if repeat else "off"), curDB), "lower") command, args = parse_entry(aliases, dbAliases, entry) if command == "help": print("\nI'm working on the help file!!") elif command == "repeat": repeat = not repeat print("\nRepeat is now %s." % ("on" if repeat else "off")) elif command == "run": tracker.produce(database, production) elif command in ("switch", "root", "global"): curDB = change_database(curDB, command) elif command == "database": db.basic_actions(database, curDB, *tuple(args)) elif command == "load": db.load(database, *tuple(args)) elif command == "save": db.save(database, dbConfig, *tuple(args)) elif command == "wipe": wipePrompt = "Are you SURE you want to erase the database? Y/N" if ui.get_binary_choice(wipePrompt): database = db.init() elif command != "quit": print("\n\"%s\" is not a recognized command." % command)
async def help(ctx): # read the file helpmsg = database.load("help.json") # set the embed attributes to the sections from the file embed = discord.Embed(title=helpmsg["title"], url=helpmsg["link"], description=helpmsg["content"], color=discord.Colour.from_rgb( helpmsg["color"][0], helpmsg["color"][1], helpmsg["color"][2])) embed.set_thumbnail(url=helpmsg["image"]) # send it await ctx.send(embed=embed)
async def permaloop(): event_log.info("Entering permaloop ...") await globals.bot.wait_until_ready() while not globals.bot.is_closed(): checkpoint = time.time() modification = False for s in globals.seminars: modification = await s.make_request("ulohy") # check tasks for rnd in s.rounds.values(): modification = await s.make_request("vysledky", rnd.id) # check results for each round if modification: db.load(cn.FB_SEMINARS, s.name, s.to_dict()) # load to database if changes management_log.info("Changes loaded to database!") management_log.info(f"permaloop updated in {int((time.time()-checkpoint)*1000)}ms") # users update management_log.info("Creating and uploading {0} users".format(len(globals.server.members)-len(globals.users))) for mem in globals.server.members: hp.create_user(str(mem.id)) for userid in globals.users.keys(): db.load(cn.FB_USERS, str(userid), vars(globals.users[userid])) await asyncio.sleep(cn.MINIMAL_CHECKING_DELAY) event_log.error("Exited infinite loop")
def put(self, account_id): banking_configuration = bank.load_config( bank.env()['main_config_file']) db = database.load(bank.env()['database_folder']) code = request.get_json()['code'] account = account_from_config( banking_configuration.accounts[account_id]) access_code = datatypes.AccountAccessCode(code=code, date=datetime.utcnow(), account_id=account.id) database.update_account_access_code(db, account, access_code) return jsonify(io.encode_object(access_code))
async def search(ctx, location="None"): if location == "None": await ctx.send('''Please provide a location! Choose from: mine, dessert, ocean, forest''') else: members = database.load("data.json") nameid = str(ctx.author.id) amount = random.randint(0, 3) amount2 = random.randint(0, 3) if location.lower().startswith("m"): msg = await ctx.send("```Attempting to search the mine...```") await asyncio.sleep(3) await msg.edit( content= f'''```You found {amount} rocks.\nYou found {amount2} iron.```''' ) members[nameid]["items"]["rocks"] += amount members[nameid]["items"]["iron"] += amount2 elif location.lower().startswith("d"): msg = await ctx.send("```Attempting to search the dessert...```") await asyncio.sleep(3) await msg.edit( content= f'''```You found {amount} cactus.\nYou found {amount2} cups of sand.```''' ) members[nameid]["items"]["cactus"] += amount members[nameid]["items"]["sand"] += amount2 elif location.lower().startswith("o"): msg = await ctx.send("```Attempting to search the ocean...```") await asyncio.sleep(3) await msg.edit( content= f'''```You found {amount} fish.\nYou found {amount2} cups of water.```''' ) members[nameid]["items"]["fish"] += amount members[nameid]["items"]["water"] += amount2 elif location.lower().startswith("f"): msg = await ctx.send("```Attempting to search the forest...```") await asyncio.sleep(3) await msg.edit( content= f'''```You found {amount} apples.\nYou found {amount2} sticks.```''' ) members[nameid]["items"]["apples"] += amount members[nameid]["items"]["sticks"] += amount else: await ctx.send( "```Please provide a valid location!```\nChoose from: mine, dessert, ocean, forest" ) database.save(members, "data.json")
async def updateloop(rnd: Round): """Loop responsible for updating seminar countdowns, etc ...""" event_log.info(f"Entering updateloop for seminar {rnd.sem.name} ...") await globals.bot.wait_until_ready() while not globals.bot.is_closed(): # Garbage collector if rnd.remaining.total_seconds() <= 0: event_log.info( f"Exiting updateloop for seminar {rnd.sem.name} ... the round has ended." ) await rnd.end() rnd.sem.rounds.pop(rnd.id) db.load(cn.FB_SEMINARS, rnd.sem.name, rnd.sem.to_dict()) del (rnd) # remove round return await asyncio.sleep(cn.UPDATE_DELAY) try: await rnd.check_round_message() except Exception as e: event_log.warning( f"{rnd.sem.name} >>> Error occured while updating {e}", exc_info=True)
def index(): chinese = { '1': '一', '2': '二', '3': '三', '4': '四', '5': '五', '6': '六', '0': '日' } name = request.args.get('name') zimuzu = request.args.get('zimuzu') day = request.args.get('day') if name != "" and zimuzu != "" and name != None and zimuzu != None: database.dump(name, zimuzu, day) result = database.load(day) return render_template('index.html', result=result) else: result = [] for days in range(7): result.append("周" + chinese[str(days)]) result.append(database.load(days)) return render_template('index.html', result=result)
async def items(ctx): # load from the database members = database.load("data.json") memberid = str(ctx.author.id) items = members[memberid]["items"] # item display itemmsg = "" itemmsg = itemmsg + 'Here is your items:\n' for item in items: itemmsg = itemmsg + f'{item}, {members[memberid]["items"][item]}\n' # money display itemmsg = itemmsg + f'Here is your money: {members[memberid]["money"]}\n' # weapon display itemmsg = itemmsg + f'Here is your weapon: {members[memberid]["weapon"]}' await ctx.send(f'''```{itemmsg}```''')
def testDB(): # new store database.store = {} # add to database database.addToDB("sec", fid = "test") database.addToDB("sec", msg = "anotherone") # get from database assert len(database.get("sec", database.getDate())) == 2 assert (database.get("sec", database.getDate()))[0]['fid'] == "test" assert (database.get("sec", database.getDate()))[1]['msg'] == "anotherone" assert (database.get("oth", database.getDate())) == [] # save database to pickle database.save("temp.pickle") # remove from database database.remove(database.getDate()) assert database.get("sec", database.getDate()) == [] # load database from pickle database.store = database.load("temp.pickle") # get from database assert (database.get("sec", database.getDate()))[0]['fid'] == "test" assert (database.get("sec", database.getDate()))[1]['msg'] == "anotherone" os.remove("temp.pickle")
output_list = slack_rtm_output if output_list and len(output_list) > 0: for output in output_list: if output and 'text' in output and "uploaded" in output['text']: if output and 'file' in output and output['file'] and 'id' in output['file']: id = "id:"+output['file']['id'] return id, \ output['channel'] if output and 'text' in output and AT_BOT in output['text']: # return text after the @ mention, whitespace removed return output['text'].split(AT_BOT)[1].strip().lower(), \ output['channel'] return None, None if os.path.exists("/volume/slackdb.pickle"): db.store = db.load("/volume/slackdb.pickle") else: pickle.dump({}, open("/volume/slackdb.pickle", 'w+')) db.store = db.load("/volume/slackdb.pickle") slack_client = SlackClient(os.environ.get('SLACK_BOT_TOKEN')) if __name__ == "__main__": READ_WEBSOCKET_DELAY = 0.5 # 1 second delay between reading from firehose if slack_client.rtm_connect(): print("StarterBot connected and running!") while True: command, channel = parse_slack_output(slack_client.rtm_read()) if command and channel: handle_command(command, channel) time.sleep(READ_WEBSOCKET_DELAY) else: