def main(): print("Enter data file name to be read:") f = open(input(), "r") line = f.readline() start = time.time() #------------------------Building the store----------------------------------------- print("building databases...\n") while line: db_create(json.loads(line)) line = f.readline() end = time.time() print("database complete - built in ", end - start) # ---------------------------------------------------------------------------------- menu() f.close()
def backfill_asset(asset, start_date, reset): data_to_write = [] while True: data = fetch_data(asset, start_date) if len(data) == 1: break for candle in data: data_to_write.append([int(candle[0] / 1000), candle[1], candle[2], candle[3], candle[4], candle[5]]) start_date = data[-1][0] data_to_write = conform_data(data_to_write) if reset: database.db_create(asset) database.db_write(asset, data_to_write) else: database.db_write(asset, data_to_write[1:])
def backfill_forex(asset, startdate, reset=False): config = configparser.ConfigParser() config.read('config.ini') access_token = config['OANDA']['AccessToken'] client = oandapyV20.API(access_token=access_token) forex_data = [] while True: logger.debug(f"Backfilling {asset} data from {startdate}") parameters = { 'granularity': 'M1', 'from': startdate.timestamp(), 'count': 5000 } r = instruments.InstrumentsCandles(instrument=asset, params=parameters) data = client.request(r) if len(data['candles']) == 0: break for candle in massage_oanda(data): forex_data.append(candle) startdate = datetime.strptime( data['candles'][-1]['time'].replace("000Z", "+0000"), "%Y-%m-%dT%H:%M:%S.%f%z") + timedelta(minutes=1) if startdate.timestamp() > time.time(): break towrite = np.array(conform_oanda_data(forex_data), dtype=np.float64) if reset: database.db_create(asset) database.db_write(asset, towrite) else: database.db_write(asset, towrite[1:])
def setupUi(self, MainWindow): MainWindow.setObjectName("Database Creator") MainWindow.resize(525, 464) self.centralwidget = QtWidgets.QWidget(MainWindow) self.centralwidget.setObjectName("centralwidget") self.verticalLayout = QtWidgets.QVBoxLayout(self.centralwidget) self.verticalLayout.setObjectName("verticalLayout") self.splitter = QtWidgets.QSplitter(self.centralwidget) self.splitter.setOrientation(QtCore.Qt.Vertical) self.splitter.setObjectName("splitter") self.dblbl = QtWidgets.QLabel(self.splitter) self.dblbl.setObjectName("dblbl") self.createDatabase = QtWidgets.QLineEdit(self.splitter) self.createDatabase.setText("") self.createDatabase.setObjectName("createDatabase") self.createDbBtn = QtWidgets.QPushButton(self.splitter) self.createDbBtn.setObjectName("createDbBtn") self.loadDbBtn = QtWidgets.QPushButton(self.splitter) self.loadDbBtn.setObjectName("loadDbBtn") self.resultMsg = QtWidgets.QLabel(self.splitter) self.resultMsg.setText("") self.resultMsg.setObjectName("resultMsg") self.tableWidget = QtWidgets.QTableWidget(self.splitter) self.tableWidget.setRowCount(14) self.tableWidget.setColumnCount(5) self.tableWidget.setObjectName("tableWidget") self.verticalLayout.addWidget(self.splitter) MainWindow.setCentralWidget(self.centralwidget) self.menubar = QtWidgets.QMenuBar(MainWindow) self.menubar.setGeometry(QtCore.QRect(0, 0, 525, 22)) self.menubar.setObjectName("menubar") MainWindow.setMenuBar(self.menubar) self.statusbar = QtWidgets.QStatusBar(MainWindow) self.statusbar.setObjectName("statusbar") MainWindow.setStatusBar(self.statusbar) self.retranslateUi(MainWindow) QtCore.QMetaObject.connectSlotsByName(MainWindow) self.createDbBtn.clicked.connect( lambda: db_create(self.createDatabase.text())) self.loadDbBtn.clicked.connect(self.db_load)
def commands_messages(message): if message.text == '/start': bot.send_message(message.chat.id, "Добро пожаловать!") elif message.text == '/help': help_message(bot, message.chat.id) elif message.text == '/your_stat': tables = db.db_table_list() data = list() for chat_id in tables: x = db.db_search_stat(chat_id, message.from_user.id) if x: x.append(int(chat_id)) data.append(x) data = sorted(data, key=lambda c: c[0], reverse=True) private_stat(bot, message.chat.id, data) elif message.chat.type in chat_types: if message.text == '/start@carmas_bot': db.db_create(message.chat.id) bot.send_message(message.chat.id, "Добро пожаловать!") elif message.text == '/help@carmas_bot': help_message(bot, message.chat.id) elif message.text == '/stats@carmas_bot': data = db.db_carma_stat(message.chat.id) if match(r'^relation.+', str(data)): return carma_stats(bot, message.chat.id, data) elif message.text == '/your_stat@carmas_bot': crm = db.db_select(message.chat.id, message.from_user.id, ['carma', 'name']) if re.match(r'^relation.+', str(crm)) or not crm: return bot.send_message( message.chat.id, "[{user_name}](tg://user?id={user_id}), " "твоя текущая карма {crm}".format( user_name=crm[0][1], user_id=message.from_user.id, crm=crm[0][0] ), parse_mode='Markdown' ) elif ((message.text == '/block@carmas_bot' or message.text == '/unblock@carmas_bot') and message.reply_to_message): status = bot.get_chat_member(message.chat.id, message.from_user.id) if status.status == 'creator' or status.status == 'administrator': block = (1 if message.text == '/block@carmas_bot' else 0) data = db.db_block( message.chat.id, message.reply_to_message.from_user.id, block ) if match(r'^relation.+', str(data)): return bot.send_message( message.chat.id, 'Пользователь {}.'.format( 'заблокирован' if block else 'разблокирован' )) else: bot.send_message( message.chat.id, 'Только администраторы могут изменять ' 'возможность модификации кармы!' ) elif message.text == '/delete@carmas_bot': status = bot.get_chat_member(message.chat.id, message.from_user.id) if status.status == 'creator' or status.status == 'administrator': bot.send_message( message.chat.id, 'Вы действительно хотите удалить данные по вашему чату?\n' '(Для продолжения ответьте "*да*" или "*нет*" на это ' 'сообщение через "reply"(ответить))', parse_mode='Markdown' ) else: bot.send_message( message.chat.id, 'Эта команда доступна только администраторам!' )
def carma(message): carma_to = message.reply_to_message.from_user.id carma_to_username = (message.reply_to_message.from_user.username if message.reply_to_message.from_user.username else carma_to) carma_to_name = (message.reply_to_message.from_user.first_name if message.reply_to_message.from_user.first_name else carma_to_username) to_lastname = message.reply_to_message.from_user.last_name if to_lastname: carma_to_name += ' ' + to_lastname carma_from = message.from_user.id carma_from_username = (message.from_user.username if message.from_user.username else carma_from) carma_from_name = (message.from_user.first_name if message.from_user.first_name else carma_from_username) from_lastname = message.from_user.last_name if from_lastname: carma_from_name += ' ' + from_lastname if carma_from == carma_to: return x = 0 current_time = int(time()) if message.text.lower() == '+' or match(r'^спасибо\W*$', message.text.lower()): x += 1 change_text = 'увеличил' else: x -= 1 change_text = 'уменьшил' select_from = db.db_select( chat_id=message.chat.id, user_id=carma_from, column=['date', 'block'] ) last_change = 0 if select_from: if match(r'^relation.+', str(select_from)): db.db_create(message.chat.id) db.db_add( chat_id=message.chat.id, user_id=carma_from, user_name=carma_from_username, name=carma_from_name, chat_title=message.chat.title, date=current_time ) elif select_from[0][1]: bot.send_message(message.chat.id, 'Вы не можете модифицировать карму') return else: last_change = select_from[0][0] if (current_time - last_change) < 10: return select_to = db.db_select( chat_id=message.chat.id, user_id=carma_to, column=['carma'] ) if select_to: x += select_to[0][0] db.db_update( chat_id=message.chat.id, user_id=carma_to, user_name=carma_to_username, name=carma_to_name, chat_title=message.chat.title, carma=x ) else: db.db_add( chat_id=message.chat.id, user_id=carma_to, user_name=carma_to_username, name=carma_to_name, chat_title=message.chat.title, carma=x ) if select_from: db.db_update_date( chat_id=message.chat.id, user_id=carma_from, user_name=carma_from_username, name=carma_from_name, chat_title=message.chat.title, date=current_time ) else: db.db_add( chat_id=message.chat.id, user_id=carma_from, user_name=carma_from_username, name=carma_from_name, chat_title=message.chat.title, date=current_time ) carma_params = [carma_to, carma_to_name, carma_from, carma_from_name, x, change_text] carma_change(bot, message.chat.id, carma_params)
def main(): f = open("data_6k.txt", "r") line = f.readline() #------------------------Building the 6k store----------------------------------------- while line: db_create(json.loads(line)) line = f.readline() # ---------------------------------------------------------------------------------- f.close() db_data, kv_data = db_range('created_at', 'text', "Wed Nov 11 23:20:10 +0000 2020", "Wed Nov 12 23:20:10 +0000 2020") f = open("range_result_6k_mysql.txt", "w") for text in db_data: f.write(text[0]) f.close() f = open("range_result_6k_kv.txt", "w") for text in kv_data: f.write(text) f.close() print("Range + create for 6k") print(benchmark_print_results()) benchmark_clear_results() f = open("data_20k.txt", "r") line = f.readline() #------------------------Building the 20k store----------------------------------------- while line: db_create(json.loads(line)) line = f.readline() # ---------------------------------------------------------------------------------- f.close() db_data, kv_data = db_range('created_at', 'text', "Wed Nov 11 23:20:10 +0000 2020", "Wed Nov 12 23:20:10 +0000 2020") f = open("range_result_20k_mysql.txt", "w") for text in db_data: f.write(text[0]) f.close() f = open("range_result_20k_kv.txt", "w") for text in kv_data: f.write(text) f.close() print("Range + create for 20k") print(benchmark_print_results()) benchmark_clear_results() f = open("data_25k.txt", "r") line = f.readline() #------------------------Building the 25k store----------------------------------------- while line: db_create(json.loads(line)) line = f.readline() # ---------------------------------------------------------------------------------- f.close() db_data, kv_data = db_range('created_at', 'text', "Wed Nov 11 23:20:10 +0000 2020", "Wed Nov 12 23:20:10 +0000 2020") f = open("range_result_25k_mysql.txt", "w") for text in db_data: f.write(text[0]) f.close() f = open("range_result_25_kv.txt", "w") for text in kv_data: f.write(text) f.close() print("Range + create for 25k") print(benchmark_print_results()) benchmark_clear_results() f = open("data_100k.txt", "r") line = f.readline() #------------------------Building the 100k store----------------------------------------- while line: db_create(json.loads(line)) line = f.readline() # ---------------------------------------------------------------------------------- f.close() db_data, kv_data = db_range('created_at', 'text', "Wed Nov 11 23:20:10 +0000 2020", "Wed Nov 12 23:20:10 +0000 2020") f = open("range_result_100k_mysql.txt", "w") for text in db_data: f.write(text[0]) f.close() f = open("range_result_100_kv.txt", "w") for text in kv_data: f.write(text) f.close() print("Range + create for 100k") print(benchmark_print_results())