def main():
print("Enter data file name to be read:")
f = open(input(), "r")
line = f.readline()
start = time.time()
#------------------------Building the store-----------------------------------------
print("building databases...\n")
while line:
db_create(json.loads(line))
line = f.readline()
end = time.time()
print("database complete - built in ", end - start)
# ----------------------------------------------------------------------------------
menu()
f.close()
def backfill_asset(asset, start_date, reset):
data_to_write = []
while True:
data = fetch_data(asset, start_date)
if len(data) == 1:
break
for candle in data:
data_to_write.append([int(candle[0] / 1000), candle[1], candle[2], candle[3], candle[4], candle[5]])
start_date = data[-1][0]
data_to_write = conform_data(data_to_write)
if reset:
database.db_create(asset)
database.db_write(asset, data_to_write)
else:
database.db_write(asset, data_to_write[1:])
def backfill_forex(asset, startdate, reset=False):
config = configparser.ConfigParser()
config.read('config.ini')
access_token = config['OANDA']['AccessToken']
client = oandapyV20.API(access_token=access_token)
forex_data = []
while True:
logger.debug(f"Backfilling {asset} data from {startdate}")
parameters = {
'granularity': 'M1',
'from': startdate.timestamp(),
'count': 5000
}
r = instruments.InstrumentsCandles(instrument=asset, params=parameters)
data = client.request(r)
if len(data['candles']) == 0:
break
for candle in massage_oanda(data):
forex_data.append(candle)
startdate = datetime.strptime(
data['candles'][-1]['time'].replace("000Z", "+0000"),
"%Y-%m-%dT%H:%M:%S.%f%z") + timedelta(minutes=1)
if startdate.timestamp() > time.time():
break
towrite = np.array(conform_oanda_data(forex_data), dtype=np.float64)
if reset:
database.db_create(asset)
database.db_write(asset, towrite)
else:
database.db_write(asset, towrite[1:])
def setupUi(self, MainWindow):
MainWindow.setObjectName("Database Creator")
MainWindow.resize(525, 464)
self.centralwidget = QtWidgets.QWidget(MainWindow)
self.centralwidget.setObjectName("centralwidget")
self.verticalLayout = QtWidgets.QVBoxLayout(self.centralwidget)
self.verticalLayout.setObjectName("verticalLayout")
self.splitter = QtWidgets.QSplitter(self.centralwidget)
self.splitter.setOrientation(QtCore.Qt.Vertical)
self.splitter.setObjectName("splitter")
self.dblbl = QtWidgets.QLabel(self.splitter)
self.dblbl.setObjectName("dblbl")
self.createDatabase = QtWidgets.QLineEdit(self.splitter)
self.createDatabase.setText("")
self.createDatabase.setObjectName("createDatabase")
self.createDbBtn = QtWidgets.QPushButton(self.splitter)
self.createDbBtn.setObjectName("createDbBtn")
self.loadDbBtn = QtWidgets.QPushButton(self.splitter)
self.loadDbBtn.setObjectName("loadDbBtn")
self.resultMsg = QtWidgets.QLabel(self.splitter)
self.resultMsg.setText("")
self.resultMsg.setObjectName("resultMsg")
self.tableWidget = QtWidgets.QTableWidget(self.splitter)
self.tableWidget.setRowCount(14)
self.tableWidget.setColumnCount(5)
self.tableWidget.setObjectName("tableWidget")
self.verticalLayout.addWidget(self.splitter)
MainWindow.setCentralWidget(self.centralwidget)
self.menubar = QtWidgets.QMenuBar(MainWindow)
self.menubar.setGeometry(QtCore.QRect(0, 0, 525, 22))
self.menubar.setObjectName("menubar")
MainWindow.setMenuBar(self.menubar)
self.statusbar = QtWidgets.QStatusBar(MainWindow)
self.statusbar.setObjectName("statusbar")
MainWindow.setStatusBar(self.statusbar)
self.retranslateUi(MainWindow)
QtCore.QMetaObject.connectSlotsByName(MainWindow)
self.createDbBtn.clicked.connect(
lambda: db_create(self.createDatabase.text()))
self.loadDbBtn.clicked.connect(self.db_load)
def commands_messages(message):
if message.text == '/start':
bot.send_message(message.chat.id, "Добро пожаловать!")
elif message.text == '/help':
help_message(bot, message.chat.id)
elif message.text == '/your_stat':
tables = db.db_table_list()
data = list()
for chat_id in tables:
x = db.db_search_stat(chat_id, message.from_user.id)
if x:
x.append(int(chat_id))
data.append(x)
data = sorted(data, key=lambda c: c[0], reverse=True)
private_stat(bot, message.chat.id, data)
elif message.chat.type in chat_types:
if message.text == '/start@carmas_bot':
db.db_create(message.chat.id)
bot.send_message(message.chat.id, "Добро пожаловать!")
elif message.text == '/help@carmas_bot':
help_message(bot, message.chat.id)
elif message.text == '/stats@carmas_bot':
data = db.db_carma_stat(message.chat.id)
if match(r'^relation.+', str(data)):
return
carma_stats(bot, message.chat.id, data)
elif message.text == '/your_stat@carmas_bot':
crm = db.db_select(message.chat.id, message.from_user.id,
['carma', 'name'])
if re.match(r'^relation.+', str(crm)) or not crm:
return
bot.send_message(
message.chat.id,
"[{user_name}](tg://user?id={user_id}), "
"твоя текущая карма {crm}".format(
user_name=crm[0][1],
user_id=message.from_user.id,
crm=crm[0][0]
),
parse_mode='Markdown'
)
elif ((message.text == '/block@carmas_bot'
or message.text == '/unblock@carmas_bot')
and message.reply_to_message):
status = bot.get_chat_member(message.chat.id, message.from_user.id)
if status.status == 'creator' or status.status == 'administrator':
block = (1 if message.text == '/block@carmas_bot' else 0)
data = db.db_block(
message.chat.id,
message.reply_to_message.from_user.id,
block
)
if match(r'^relation.+', str(data)):
return
bot.send_message(
message.chat.id,
'Пользователь {}.'.format(
'заблокирован' if block else 'разблокирован'
))
else:
bot.send_message(
message.chat.id,
'Только администраторы могут изменять '
'возможность модификации кармы!'
)
elif message.text == '/delete@carmas_bot':
status = bot.get_chat_member(message.chat.id, message.from_user.id)
if status.status == 'creator' or status.status == 'administrator':
bot.send_message(
message.chat.id,
'Вы действительно хотите удалить данные по вашему чату?\n'
'(Для продолжения ответьте "*да*" или "*нет*" на это '
'сообщение через "reply"(ответить))',
parse_mode='Markdown'
)
else:
bot.send_message(
message.chat.id,
'Эта команда доступна только администраторам!'
)
def carma(message):
carma_to = message.reply_to_message.from_user.id
carma_to_username = (message.reply_to_message.from_user.username
if message.reply_to_message.from_user.username
else carma_to)
carma_to_name = (message.reply_to_message.from_user.first_name
if message.reply_to_message.from_user.first_name
else carma_to_username)
to_lastname = message.reply_to_message.from_user.last_name
if to_lastname:
carma_to_name += ' ' + to_lastname
carma_from = message.from_user.id
carma_from_username = (message.from_user.username
if message.from_user.username
else carma_from)
carma_from_name = (message.from_user.first_name
if message.from_user.first_name
else carma_from_username)
from_lastname = message.from_user.last_name
if from_lastname:
carma_from_name += ' ' + from_lastname
if carma_from == carma_to:
return
x = 0
current_time = int(time())
if message.text.lower() == '+' or match(r'^спасибо\W*$',
message.text.lower()):
x += 1
change_text = 'увеличил'
else:
x -= 1
change_text = 'уменьшил'
select_from = db.db_select(
chat_id=message.chat.id,
user_id=carma_from,
column=['date', 'block']
)
last_change = 0
if select_from:
if match(r'^relation.+', str(select_from)):
db.db_create(message.chat.id)
db.db_add(
chat_id=message.chat.id,
user_id=carma_from,
user_name=carma_from_username,
name=carma_from_name,
chat_title=message.chat.title,
date=current_time
)
elif select_from[0][1]:
bot.send_message(message.chat.id,
'Вы не можете модифицировать карму')
return
else:
last_change = select_from[0][0]
if (current_time - last_change) < 10:
return
select_to = db.db_select(
chat_id=message.chat.id,
user_id=carma_to,
column=['carma']
)
if select_to:
x += select_to[0][0]
db.db_update(
chat_id=message.chat.id,
user_id=carma_to,
user_name=carma_to_username,
name=carma_to_name,
chat_title=message.chat.title,
carma=x
)
else:
db.db_add(
chat_id=message.chat.id,
user_id=carma_to,
user_name=carma_to_username,
name=carma_to_name,
chat_title=message.chat.title,
carma=x
)
if select_from:
db.db_update_date(
chat_id=message.chat.id,
user_id=carma_from,
user_name=carma_from_username,
name=carma_from_name,
chat_title=message.chat.title,
date=current_time
)
else:
db.db_add(
chat_id=message.chat.id,
user_id=carma_from,
user_name=carma_from_username,
name=carma_from_name,
chat_title=message.chat.title,
date=current_time
)
carma_params = [carma_to, carma_to_name,
carma_from, carma_from_name,
x, change_text]
carma_change(bot, message.chat.id, carma_params)
def main():
f = open("data_6k.txt", "r")
line = f.readline()
#------------------------Building the 6k store-----------------------------------------
while line:
db_create(json.loads(line))
line = f.readline()
# ----------------------------------------------------------------------------------
f.close()
db_data, kv_data = db_range('created_at', 'text',
"Wed Nov 11 23:20:10 +0000 2020",
"Wed Nov 12 23:20:10 +0000 2020")
f = open("range_result_6k_mysql.txt", "w")
for text in db_data:
f.write(text[0])
f.close()
f = open("range_result_6k_kv.txt", "w")
for text in kv_data:
f.write(text)
f.close()
print("Range + create for 6k")
print(benchmark_print_results())
benchmark_clear_results()
f = open("data_20k.txt", "r")
line = f.readline()
#------------------------Building the 20k store-----------------------------------------
while line:
db_create(json.loads(line))
line = f.readline()
# ----------------------------------------------------------------------------------
f.close()
db_data, kv_data = db_range('created_at', 'text',
"Wed Nov 11 23:20:10 +0000 2020",
"Wed Nov 12 23:20:10 +0000 2020")
f = open("range_result_20k_mysql.txt", "w")
for text in db_data:
f.write(text[0])
f.close()
f = open("range_result_20k_kv.txt", "w")
for text in kv_data:
f.write(text)
f.close()
print("Range + create for 20k")
print(benchmark_print_results())
benchmark_clear_results()
f = open("data_25k.txt", "r")
line = f.readline()
#------------------------Building the 25k store-----------------------------------------
while line:
db_create(json.loads(line))
line = f.readline()
# ----------------------------------------------------------------------------------
f.close()
db_data, kv_data = db_range('created_at', 'text',
"Wed Nov 11 23:20:10 +0000 2020",
"Wed Nov 12 23:20:10 +0000 2020")
f = open("range_result_25k_mysql.txt", "w")
for text in db_data:
f.write(text[0])
f.close()
f = open("range_result_25_kv.txt", "w")
for text in kv_data:
f.write(text)
f.close()
print("Range + create for 25k")
print(benchmark_print_results())
benchmark_clear_results()
f = open("data_100k.txt", "r")
line = f.readline()
#------------------------Building the 100k store-----------------------------------------
while line:
db_create(json.loads(line))
line = f.readline()
# ----------------------------------------------------------------------------------
f.close()
db_data, kv_data = db_range('created_at', 'text',
"Wed Nov 11 23:20:10 +0000 2020",
"Wed Nov 12 23:20:10 +0000 2020")
f = open("range_result_100k_mysql.txt", "w")
for text in db_data:
f.write(text[0])
f.close()
f = open("range_result_100_kv.txt", "w")
for text in kv_data:
f.write(text)
f.close()
print("Range + create for 100k")
print(benchmark_print_results())