class Crawl:
def __init__(self, log_path):
self.session = requests.Session()
self.log = Log(log_path)
def make_response(self, requests_parameter, sleep):
"""
Crawling page or data using 'requests' module
:param requests_parameter: {
method = 'GET',
url = base_url + path,
params = dictionary or inline string,
headers = dictionary,
}
:param sleep: time.sleep
:return: requests.response
"""
response = self.session.request(**requests_parameter, timeout=10)
if response.headers.get('status'):
if '200' not in response.headers['status']:
self.log.warning('HTTP Status Error!, {}'.format(
str(response.headers)))
time.sleep(sleep)
return response
def __del__(self):
print('session_closed!')
self.session.close()
def __init__(self, port, timeout):
now = datetime.datetime.now()
self.server_log = Log("server_log.txt")
self.running = True
self.timeout = timeout
self.name = "" # Имя сервера
self.clients = [] # Список подключённых ip
self.requests = {} # Карта входящий сообщений адрес: сообщение
self.clients_names = Log.read_and_return_dict(
"clients_names.txt") # Карта ников пользователей адрес: имя
self.public_key, self.private_key = rsa.newkeys(512)
self.keys = [0 for i in range(3)]
self.first_client = socket.socket()
self.second_client = socket.socket()
self.third_client = socket.socket()
self.clients_info = [0 for i in range(3)
] # Индикатор загруженности соккетов
self.client_map = {} # Карта пользователей адрес: индекс
self.max_clients = 3
self.server_socket = socket.socket()
self.server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,
1)
self.server_socket.bind(('localhost', port))
self.server_socket.listen()
self.first_client.settimeout(timeout)
self.second_client.settimeout(timeout)
self.third_client.settimeout(timeout)
self.server_log.save_message("\n\n{}\nServer initialized".format(
str(now)))
def write_file(output_dir, file_to_write, mode, data):
out_file_name = file_to_write
index = file_to_write.rindex(constants.SEPARATOR)
if index is not -1:
out_file_name = file_to_write[index + 1:]
pass
if mode == constants.MODE_DECODE:
out_file_name += '.json'
pass
elif mode == constants.MODE_ENCODE:
out_file_name = data['file_name']
pass
out_file_path = output_dir + constants.SEPARATOR + out_file_name
wtf_dir = out_file_path[:out_file_path.rindex(constants.SEPARATOR)]
try:
os.makedirs(wtf_dir)
pass
except FileExistsError:
pass
if mode == constants.MODE_DECODE:
out_file = open(
out_file_path, 'w+t',
encoding='utf8') # Write file as utf-8 for universal use
pass
elif mode == constants.MODE_ENCODE:
out_file = open(out_file_path,
'w+t',
encoding=config.cfg.input_file_encoding
) # Write file with specified encoding from cfg
pass
else:
Log.log_to_file(True, 'Unknown mode selected?!')
breakpoint()
return None
if mode == constants.MODE_DECODE:
out_file.write(data)
pass
elif mode == constants.MODE_ENCODE:
if data is not None:
out_file.writelines(data['decoded_lines'])
pass
pass
out_file.close()
if mode == constants.MODE_DECODE:
chosen_mode = 'decoded'
pass
else:
chosen_mode = 'encoded'
pass
Log.log_to_file(
True, 'Wrote {0} file \"{1}\"\n'.format(chosen_mode, out_file_name))
return
def kill(self): # Останавливает сервер
self.running = False
sleep(1)
Log.save_dict(self.clients_names, "clients_names.txt")
self.server_log.save_message("Server successfully stopped")
self.server_log.close()
self.first_client.close()
self.server_socket.close()
def download(self):
tp = ThreadPool(100)
result = tp.imap_unordered(self.__worker, self.items)
for item in result:
ok, book_output_dir = item
if not ok:
logger = Log()
logger.write_error("Intro: " + book_output_dir + '\n')
tp.terminate()
def __init__(self):
self.log = Log().setup_logging()
self.progressbar = ProgressBar()
self.database = DatabaseClass(self.log)
self.program_path = os.path.join(utils.get_current_path(),
constants.EXE_PROGRAM_NAME)
self.new_version = None
self.new_version_msg = None
self.client_version = None
class Transaction(metaclass=ABCMeta):
def __init__(self, name):
self.dir = ProjectDir(name)
self.db_connection = None
self.http_session = Crawl(
os.path.join(self.dir.log_dir(), 'session.log'))
self.log = Log(os.path.join(self.dir.log_dir(), 'transaction.log'))
def request(self, sleep=3):
return self.http_session.make_response(self.http_requests_parameter(),
sleep)
def save_raw_file(self, filename, response):
"""
:param filename: filename
:param response: requests.response
"""
encoding = response.encoding
if encoding.lower() != 'utf-8':
self.log.warning('Check Page Encoding!, {}'.format(
str(response.headers)))
with open(os.path.join(self.dir.data_dir(), filename),
mode='w',
encoding=encoding) as f:
f.write(response.content.decode(encoding))
@staticmethod
def remove_file(save_path):
"""
Avoid file collision
"""
if os.path.isfile(save_path):
os.remove(save_path)
@abstractmethod
def http_requests_parameter(self):
"""
requests_parameter: {
'method': 'GET',
'url': base_url + path,
'params': dictionary or inline string,
'headers': dictionary,
}
:return: dictionary
"""
pass
def set_db_connection(self, db_connection):
"""
self.db_connection: Database.connect
"""
self.db_connection = db_connection
def __del__(self):
if self.db_connection is not None:
self.db_connection.close()
def read_config():
global cfg
if os.path.isfile(CONFIG_PATH):
Log.log_to_file(True, 'Reading config...\n')
cfg_file = open(CONFIG_PATH, mode='r+', encoding='utf-8')
cfg = json.load(fp=cfg_file, object_hook=Config.decode)
cfg_file.close()
Log.log_to_file(True, 'Read config file!\n')
pass
else:
Log.log_to_file(True, 'Config does not exist!')
Log.log_to_file(True, 'Creating config file...\n')
cfg = Config(
input_file_encoding=
'shift-jis', # Encoding to read input files - Leave as shift-jis unless you know what you're doing
current_config_type_id='cube_v1',
script_config_types=[
ScriptConfigType(sct_id='cube_v1',
annotation_prefix='@',
annotation_suffix='',
annotation_types=['Talk'],
comment=';',
ignore=[]),
ScriptConfigType(sct_id='babel_v1',
annotation_prefix='[',
annotation_suffix=']',
annotation_types=[],
comment=';',
ignore=['*|']),
ScriptConfigType(sct_id='norn_v1',
annotation_prefix='@',
annotation_suffix='',
annotation_types=['name'],
comment=';',
ignore=['*|'])
])
cfg_file = open(CONFIG_PATH, mode='x+t', encoding='utf-8')
json.dump(cfg,
fp=cfg_file,
ensure_ascii=False,
indent=2,
default=Config.encode)
cfg_file.close()
Log.log_to_file(True, 'Created config file!')
pass
global cfg_script_config_type
for i in cfg.script_config_types:
if i.sct_id == cfg.current_config_type_id:
cfg_script_config_type = i
break
continue
return
def download(self):
"""
Download audio page by page
:return:
"""
for item in self.items:
ok, book_output_dir = self.__worker(item)
if not ok:
logger = Log()
logger.write_error("Listen: " + book_output_dir + '\n')
def test_incorrect_sorted_data():
logs = [
Log(client_id="user1", location="http://shop.ru", referer="http://shop.ru", request_datetime=datetime(2018, 1, 2, 20, 3, 4)),
Log(client_id="user1", location="http://shop.ru", referer="http://shop.ru", request_datetime=datetime(2018, 1, 1, 20, 3, 4))
]
with pytest.raises(ValueError, match=INCORRECT_SORTED_ERROR) as excinfo:
build_statistics(data=logs,
shop_domain="shop.ru",
finish_url="http://shop.ru/checkout",
target_domain="our.com",
another_domains=[])
def test_processor(filename):
input_data = {}
with open(filename) as json_file:
input_data = json.load(json_file)
# build available
available = []
for result_item in input_data['result']:
item = StatItem(result_item['url'])
item.bill_count = result_item['count']
available.append(item)
available = sorted(available, key=lambda x: x.url)
# build input data
logs = []
for result_item in input_data['logs']:
logs.append(Log(client_id=result_item['client_id'],
location=result_item['location'],
referer=result_item['referer'],
request_datetime=datetime.strptime(result_item['datetime'], '%Y-%m-%dT%H:%M:%S.%f')))
result = build_statistics(data=logs,
shop_domain=input_data['shop_domain'],
finish_url=input_data['finish_url'],
target_domain=input_data['target_domain'],
another_domains=input_data['another_domains'])
result = sorted(result, key=lambda x: x.url)
assert result == available
def decode(input_file):
Log.log_to_file(True, 'Decoding input file \"{0}\"...'.format(input_file))
read_input_data = read(input_file)
input_file_name = input_file
index = input_file_name.rindex(constants.SEPARATOR)
if index is not -1:
input_file_name = input_file_name[index + 1:]
pass
read_lines = read_input_data
if read_lines is None:
return None
return json_encode.encode_to_json(input_file_name, read_lines)
def create_widgets(self):
self.lid = Lid(self)
self.com = Com(self)
self.log = Log(self)
self.lpg_meter = Meter('A', self)
self.co_meter = Meter('B', self)
self.lpg_plot = Plot('A', self)
self.co_plot = Plot('B', self)
def connect(): # Исходящее подключение
address = enter_ip_text.get(1.0, END)
address = address[:address.find("\n")]
enter_ip_text.delete(1.0, END)
if server.check_address(address):
if not window_manager.get(address):
load_chat_window(address, nick)
else:
load_error_window("Chat already open")
return
if check_ip(address):
Log.save_with_ignore_same("peers.txt", address)
thread = Thread(target=server.create_connection,
args=(address, server_port))
thread.start()
thread.join(0)
load_chat_window(address, nick)
else:
load_error_window("Can`t connect to: " + address)
return
def __init__(self, qtobj, form):
self.qtobj = qtobj
self.form = form
self.client_version = constants.VERSION
self.log = Log().setup_logging()
self.database = DatabaseClass(self.log)
self.progressbar = ProgressBar()
self.check_program_updates = True
self.check_reshade_updates = True
self.create_screenshots_folder = True
self.show_info_messages = True
self.use_dark_theme = True
self.update_shaders = True
self.need_apply = False
self.selected_game = None
self.game_config_form = None
self.reshade_version = None
self.program_version = None
self.local_reshade_path = None
self.new_version = None
self.remote_reshade_version = None
self.remote_reshade_download_url = None
def load(self):
try:
data = self._load()
except Exception:
raise ReadError(f"Cann't read json file {self.filename}")
schema = LogSchema()
result = []
# Todo: добавить проверку что это список объектов
for item in data:
try:
value = schema.load(item)
log_tem = Log(client_id=value['client_id'],
location=value['location'],
referer=value['referer'],
request_datetime=value['date'])
result.append(log_tem)
except marshmallow.exceptions.ValidationError as e:
if not self.ignore_error:
raise e
return result
def encode(input_file):
Log.log_to_file(True, 'Encoding input file {}...'.format(input_file))
read_json_file = read(input_file)
decoded_json_file = json_decode.decode_from_json(read_json_file)
return decoded_json_file
def insert(file_name,
listbox): # Вытаскиваем из файла значение и суёт их в лист
for line in Log.read_and_return_list(file_name):
listbox.insert(END, line)
def load_chat_window(
address, your_name
): # Загружает окно чата. Нужен адрес пользователя и имя локального клиента
def end():
window_manager.pop(address)
nonlocal running
server.close_connection(address, server.get_ind_by_address(address))
user_log.close()
running = False
time.sleep(2)
window.quit()
window.destroy()
def get_message():
while running:
time.sleep(0.1)
if server.check_request(address):
message = server.get_request(address)
messages.insert(END, "{}: {}".format(user, message))
user_log.save_message("{}: {}".format(user, message))
def send():
text = send_text.get(1.0, END)
text = text[:text.find("\n")]
if text == "":
return
try:
server.send(address, text)
except OSError:
end()
load_error_window("User disconnected")
return
send_text.delete(1.0, END)
messages.insert(END, your_name + ": " + text)
user_log.save_message("{}: {}".format(your_name, text))
user = str(server.get_user_name_by_address(address))
running = True
user_log = Log(user + ".txt")
window_manager.update({address: True})
window = Tk()
window.geometry("400x270+400+400")
window.title("Chat with " + user)
window.resizable(False, False)
send_frame = Frame(window, relief=RAISED, borderwidth=1)
message_frame = Frame(window, relief=RAISED, borderwidth=1)
messages_scrollbar = Scrollbar(message_frame)
messages = Listbox(message_frame, yscrollcommand=messages_scrollbar.set)
send_text = Text(send_frame, height=1)
send_button = Button(send_frame,
text="Send",
command=send,
font=button_font)
quit_button = Button(send_frame,
text="Close",
command=end,
font=button_font)
message_frame.pack(fill=X)
send_frame.pack(fill=X, side=BOTTOM)
messages_scrollbar.pack(fill=Y, side=RIGHT)
messages.pack(padx=5, pady=5, fill=BOTH)
messages_scrollbar.config(command=messages.yview)
send_text.pack(fill=X, padx=5, pady=5)
send_button.pack(fill=BOTH, padx=5)
quit_button.pack(fill=BOTH, padx=5, pady=5)
recieve_thread = Thread(target=get_message)
recieve_thread.start()
recieve_thread.join(0)
insert(user + ".txt", messages)
window.mainloop()
import socket, time
from tkinter import *
from threading import Thread
from src.server import Server
from src.log import Log
config = Log.read_and_return_dict("config.TXT")
server_port = int(config["server_port"])
listen_port = int(config["listen_port"])
check_port = int(config["check_port"])
timeout = float(config["timeout"])
label_font = config["label_font"]
button_font = config["button_font"]
def load_error_window(text): # Загружает окно загрузки с указанным текстом
def end():
root.quit()
root.destroy()
root = Tk()
root.resizable(False, False)
root.title("Error!")
quit_frame = Frame(root, relief=RAISED, borderwidth=1)
error_label = Label(root, text="Error: " + text, font=label_font)
quit_continue = Button(quit_frame, text="Continue", command=end)
quit_frame.pack(fill=BOTH, side=BOTTOM)
# declaring usefull global variables
name = os.path.basename(os.path.abspath(sys.argv[0])) # as in this files name
fileAndPath = inspect.getframeinfo(inspect.currentframe()).filename
path = os.path.dirname(os.path.abspath(fileAndPath))
prePend = "[ " + name + " ] "
description = name + "; " + "Nemesyst, an adversarual neural network based recommender system."
dependancies = ["https://github.com/DreamingRaven/RavenPythonLib"]
args = argz(sys.argv[1:], description=description)
args_json = json.loads(json.dumps(args))
# setting fallback logger here pre-update
log = Log(logLevel=args["loglevel"])
print = log.print
# attempting update/ falling back
try: # TODO: devise a method to make erros in nested try, catch
if (args["toUpdate"] == True):
from RavenPythonLib.updaters.gitUpdate import Gupdater
nucleon = Gupdater(path=path, urls=dependancies)
nucleon.install()
nucleon.update()
print(prePend + "main updater success", 3)
else:
print(
prePend +
"Skipping update/ installation since there is no --toUpdate flag given",
0)
def test_bad_check_location_domain(location_url, domain):
log = Log("user_id", location_url, "https://sth.com/bla-bla",
datetime.now())
assert not log.check_location_domain(domain)
def test_check_bad_finish_url_without_queries(location_url, finish_url):
log = Log("user_id", location_url, "https://sth.com/bla-bla",
datetime.now())
assert not log.is_finish_url(finish_url)
class MainSrc:
def __init__(self, qtobj, form):
self.qtobj = qtobj
self.form = form
self.client_version = constants.VERSION
self.log = Log().setup_logging()
self.database = DatabaseClass(self.log)
self.progressbar = ProgressBar()
self.check_program_updates = True
self.check_reshade_updates = True
self.create_screenshots_folder = True
self.show_info_messages = True
self.use_dark_theme = True
self.update_shaders = True
self.need_apply = False
self.selected_game = None
self.game_config_form = None
self.reshade_version = None
self.program_version = None
self.local_reshade_path = None
self.new_version = None
self.remote_reshade_version = None
self.remote_reshade_download_url = None
def start(self):
self.log.info(f"STARTING {constants.FULL_PROGRAM_NAME}")
self.progressbar.set_values(messages.checking_database, 15)
utils.check_database_connection(self)
utils.check_default_database_tables(self)
utils.check_default_database_configs(self)
self.progressbar.set_values(messages.checking_files, 30)
utils.check_local_files(self)
self.progressbar.set_values(messages.checking_configs, 45)
self.set_variables()
self.register_form_events()
self.progressbar.set_values(messages.downloading_shaders, 60)
if not os.path.isdir(constants.SHADERS_SRC_PATH)\
or (self.update_shaders is not None and self.update_shaders is True):
utils.download_shaders(self)
self.progressbar.set_values(messages.checking_reshade_updates, 80)
utils.check_reshade_updates(self)
utils.check_reshade_dll_files(self)
self.progressbar.set_values(messages.checking_program_updates, 90)
utils.check_program_updates(self)
self.progressbar.close()
self.qtobj.main_tabWidget.setCurrentIndex(0)
self.qtobj.programs_tableWidget.setColumnWidth(2, 130)
self.qtobj.programs_tableWidget.horizontalHeader().setDefaultAlignment(
Qt.AlignLeft)
self.populate_table_widget()
self.enable_widgets(False)
def set_variables(self):
config_sql = ConfigSql(self)
rs_config = config_sql.get_configs()
if rs_config is not None and len(rs_config) > 0:
if not rs_config[0].get("use_dark_theme"):
self.use_dark_theme = False
self.qtobj.yes_dark_theme_radioButton.setChecked(False)
self.qtobj.no_dark_theme_radioButton.setChecked(True)
else:
self.use_dark_theme = True
self.qtobj.yes_dark_theme_radioButton.setChecked(True)
self.qtobj.no_dark_theme_radioButton.setChecked(False)
self.set_style_sheet()
if not rs_config[0].get("check_program_updates"):
self.check_program_updates = False
self.qtobj.yes_check_program_updates_radioButton.setChecked(
False)
self.qtobj.no_check_program_updates_radioButton.setChecked(
True)
else:
self.check_program_updates = True
self.qtobj.yes_check_program_updates_radioButton.setChecked(
True)
self.qtobj.no_check_program_updates_radioButton.setChecked(
False)
if not rs_config[0].get("show_info_messages"):
self.show_info_messages = False
self.qtobj.yes_show_info_messages_radioButton.setChecked(False)
self.qtobj.no_show_info_messages_radioButton.setChecked(True)
else:
self.show_info_messages = True
self.qtobj.yes_show_info_messages_radioButton.setChecked(True)
self.qtobj.no_show_info_messages_radioButton.setChecked(False)
if not rs_config[0].get("check_reshade_updates"):
self.check_reshade_updates = False
self.qtobj.yes_check_reshade_updates_radioButton.setChecked(
False)
self.qtobj.no_check_reshade_updates_radioButton.setChecked(
True)
else:
self.check_reshade_updates = True
self.qtobj.yes_check_reshade_updates_radioButton.setChecked(
True)
self.qtobj.no_check_reshade_updates_radioButton.setChecked(
False)
if not rs_config[0].get("update_shaders"):
self.update_shaders = False
self.qtobj.yes_update_shaders_radioButton.setChecked(False)
self.qtobj.no_update_shaders_radioButton.setChecked(True)
else:
self.update_shaders = True
self.qtobj.yes_update_shaders_radioButton.setChecked(True)
self.qtobj.no_update_shaders_radioButton.setChecked(False)
if not rs_config[0].get("create_screenshots_folder"):
self.create_screenshots_folder = False
self.qtobj.yes_screenshots_folder_radioButton.setChecked(False)
self.qtobj.no_screenshots_folder_radioButton.setChecked(True)
else:
self.create_screenshots_folder = True
self.qtobj.yes_screenshots_folder_radioButton.setChecked(True)
self.qtobj.no_screenshots_folder_radioButton.setChecked(False)
if rs_config[0].get("program_version"):
self.program_version = rs_config[0].get("program_version")
if rs_config[0].get("reshade_version"):
self.reshade_version = rs_config[0].get("reshade_version")
def register_form_events(self):
# TAB 1 - games
self.qtobj.add_button.clicked.connect(lambda: events.add_game(self))
self.qtobj.delete_button.clicked.connect(
lambda: events.delete_game(self))
self.qtobj.edit_path_button.clicked.connect(
lambda: events.edit_game_path(self))
self.qtobj.edit_preset_button.clicked.connect(
lambda: events.open_preset_config_file(self))
self.qtobj.apply_button.clicked.connect(lambda: events.apply_all(self))
self.qtobj.update_button.clicked.connect(
lambda: events.update_clicked())
self.qtobj.programs_tableWidget.clicked.connect(
self._table_widget_clicked)
self.qtobj.programs_tableWidget.itemDoubleClicked.connect(
self._table_widget_double_clicked)
# TAB 2 - configs
self.qtobj.yes_dark_theme_radioButton.clicked.connect(
lambda: events.dark_theme_clicked(self, "YES"))
self.qtobj.no_dark_theme_radioButton.clicked.connect(
lambda: events.dark_theme_clicked(self, "NO"))
self.qtobj.yes_check_program_updates_radioButton.clicked.connect(
lambda: events.check_program_updates_clicked(self, "YES"))
self.qtobj.no_check_program_updates_radioButton.clicked.connect(
lambda: events.check_program_updates_clicked(self, "NO"))
self.qtobj.yes_show_info_messages_radioButton.clicked.connect(
lambda: events.show_info_messages_clicked(self, "YES"))
self.qtobj.no_show_info_messages_radioButton.clicked.connect(
lambda: events.show_info_messages_clicked(self, "NO"))
self.qtobj.yes_check_reshade_updates_radioButton.clicked.connect(
lambda: events.check_reshade_updates_clicked(self, "YES"))
self.qtobj.no_check_reshade_updates_radioButton.clicked.connect(
lambda: events.check_reshade_updates_clicked(self, "NO"))
self.qtobj.yes_update_shaders_radioButton.clicked.connect(
lambda: events.update_shaders_clicked(self, "YES"))
self.qtobj.no_update_shaders_radioButton.clicked.connect(
lambda: events.update_shaders_clicked(self, "NO"))
self.qtobj.yes_screenshots_folder_radioButton.clicked.connect(
lambda: events.create_screenshots_folder_clicked(self, "YES"))
self.qtobj.no_screenshots_folder_radioButton.clicked.connect(
lambda: events.create_screenshots_folder_clicked(self, "NO"))
self.qtobj.edit_default_preset_plugin_button.clicked.connect(
lambda: events.edit_default_preset_plugin_button_clicked(self))
self.qtobj.reset_all_button.clicked.connect(
lambda: events.reset_all_button_clicked(self))
# TAB 3 - about
self.qtobj.donate_button.clicked.connect(
lambda: events.donate_clicked())
def set_style_sheet(self):
if self.use_dark_theme:
self.form.setStyleSheet(open(constants.QSS_PATH, "r").read())
else:
self.form.setStyleSheet("")
def populate_table_widget(self):
self.qtobj.programs_tableWidget.horizontalHeader(
).setStretchLastSection(False)
self.qtobj.programs_tableWidget.setRowCount(0) # cleanning datagrid
games_sql = GamesSql(self)
rs_all_games = games_sql.get_games()
if rs_all_games is not None and len(rs_all_games) > 0:
for i in range(len(rs_all_games)):
self.qtobj.programs_tableWidget.insertRow(i)
self.qtobj.programs_tableWidget.setItem(
i, 0,
QtWidgets.QTableWidgetItem(rs_all_games[i].get("name")))
self.qtobj.programs_tableWidget.setItem(
i, 1,
QtWidgets.QTableWidgetItem(
rs_all_games[i].get("architecture")))
self.qtobj.programs_tableWidget.setItem(
i, 2,
QtWidgets.QTableWidgetItem(rs_all_games[i].get("api")))
self.qtobj.programs_tableWidget.setItem(
i, 3,
QtWidgets.QTableWidgetItem(rs_all_games[i].get("path")))
self.qtobj.programs_tableWidget.resizeColumnsToContents()
highest_column_width = self.qtobj.programs_tableWidget.columnWidth(3)
if highest_column_width < 600:
self.qtobj.programs_tableWidget.horizontalHeader(
).setStretchLastSection(True)
else:
self.qtobj.programs_tableWidget.horizontalHeader(
).setSectionResizeMode(3, QtWidgets.QHeaderView.ResizeToContents)
def enable_form(self, status: bool):
if not status:
self.selected_game = None
self.qtobj.add_button.setEnabled(status)
for i in range(0, self.qtobj.main_tabWidget.count()):
self.qtobj.main_tabWidget.setTabEnabled(i, status)
self.qtobj.main_tabWidget.setCurrentIndex(0)
def enable_widgets(self, status: bool):
if not status:
self.selected_game = None
self._set_state_apply_button()
self.qtobj.delete_button.setEnabled(status)
self.qtobj.edit_path_button.setEnabled(status)
self.qtobj.edit_preset_button.setEnabled(status)
self.qtobj.main_tabWidget.setCurrentIndex(0)
def _set_state_apply_button(self):
len_games = self.qtobj.programs_tableWidget.rowCount()
if len_games == 0:
self.qtobj.apply_button.setEnabled(False)
else:
self.qtobj.apply_button.setEnabled(True)
def _table_widget_clicked(self, item):
events.programs_tableWidget_clicked(self, item)
def _table_widget_double_clicked(self):
if self.selected_game is not None:
qtutils.show_game_config_form(self, self.selected_game.name,
self.selected_game.architecture)
def decode_from_json(json_file):
if json_file is None:
Log.log_to_file(True, 'No json file given?!')
return None
file_name = json_file['file_name']
lines = json_file['lines']
decoded_lines = []
for i in range(len(lines)):
line = lines[i]
line_number = line['line_number']
while len(decoded_lines) + 1 < line_number:
decoded_lines.append(constants.END_LINE)
continue
line_data = line['line']
shift_start = line['shiftStart']
shift_end = line['shiftEnd']
index_start = line['indexStart']
index_end = line['indexEnd']
line_type = line_data['type']
if line_type == 'annotation':
params = line_data['params']
current_line = config.get_script_config(
).annotation_prefix # Apply prefix
current_line += line_data['name'] # Name
# Params
param_args = []
for param in params:
if isinstance(param, dict):
param_name = param['name']
param_value = param['value']
if isinstance(param_value, list):
concat = ''
for pv in param_value:
concat += (pv + ',')
continue
concat = concat[:-1]
param_value = concat
pass
elif isinstance(param_value, str):
# NOOP
pass
else:
Log.log_to_file(True, 'Unexpected param!!')
breakpoint()
pass
param_args.append(param_name + '=' + param_value)
pass
elif isinstance(param, str):
param_args.append(param)
pass
else:
Log.log_to_file(True, 'Unexpected param!!')
breakpoint()
pass
continue
for param_arg in param_args:
current_line += (' ' + param_arg)
continue
# Apply suffix
current_line += config.get_script_config().annotation_suffix
elif line_type == 'text':
current_line = line_data['value']
pass
else:
Log.log_to_file(True, 'Unexpected type found!')
return None
if shift_start > 0:
i = 0
while i < shift_start:
current_line = '\t' + current_line
i += 1
continue
pass
if shift_end > 0:
i = 0
while i < shift_end:
current_line += '\t'
i += 1
continue
pass
current_line += constants.END_LINE
if len(lines) > 1 and line_number == lines[i - 1]['line_number']:
prev_line = decoded_lines[len(decoded_lines) - 1] # Get prev line
prev_line = prev_line[:-1] # Remove EOL
prev_line += current_line # Add 2nd annotation
decoded_lines[len(decoded_lines) -
1] = prev_line # Set 2nd annotation
continue
else:
decoded_lines.append(current_line)
continue
return {'file_name': file_name, 'decoded_lines': decoded_lines}
class Launcher:
def __init__(self):
self.log = Log().setup_logging()
self.progressbar = ProgressBar()
self.database = DatabaseClass(self.log)
self.program_path = os.path.join(utils.get_current_path(),
constants.EXE_PROGRAM_NAME)
self.new_version = None
self.new_version_msg = None
self.client_version = None
def start(self):
self.progressbar.set_values(messages.checking_files, 25)
utils.check_local_files(self)
if not os.path.isfile(self.program_path):
self.program_path = os.path.join(constants.PROGRAM_PATH,
constants.EXE_PROGRAM_NAME)
self.progressbar.set_values(messages.checking_database, 50)
utils.check_database_connection(self)
utils.check_default_database_tables(self)
utils.check_default_database_configs(self)
self.progressbar.set_values(messages.checking_program_updates, 75)
self.check_program_updates()
self.progressbar.close()
self.call_program()
def check_program_updates(self):
config_sql = ConfigSql(self)
rs_config = config_sql.get_configs()
if rs_config[0].get("program_version") is None:
self.client_version = constants.VERSION
else:
self.client_version = rs_config[0].get("program_version")
if rs_config[0].get("check_program_updates"):
new_version_obj = utils.get_new_program_version(self)
if new_version_obj.new_version_available:
self.new_version = new_version_obj.new_version
self.new_version_msg = new_version_obj.new_version_msg
self.download_new_program_version()
def download_new_program_version(self):
program_url = f"{constants.GITHUB_EXE_PROGRAM_URL}{self.new_version}/{constants.EXE_PROGRAM_NAME}"
r = requests.get(program_url)
if r.status_code == 200:
with open(self.program_path, "wb") as outfile:
outfile.write(r.content)
qtutils.show_message_window(
self.log, "info",
f"{messages.program_updated}v{self.new_version}")
else:
qtutils.show_message_window(self.log, "error",
messages.error_dl_new_version)
self.log.error(
f"{messages.error_dl_new_version} {r.status_code} {r}")
def call_program(self):
code = None
try:
process = subprocess.run(self.program_path,
shell=True,
check=True,
universal_newlines=True)
code = process.returncode
except Exception as e:
if code is None and hasattr(e, "returncode"):
self.log.error(
f"cmd:{self.program_path} - code:{e.returncode} - {e}")
msg = f"{messages.error_executing_program}{constants.EXE_PROGRAM_NAME}\n"\
f"{messages.error_check_installation}"
qtutils.show_message_window(self.log, "error", msg)
class Server: # Сервер для чата
def __init__(self, port, timeout):
now = datetime.datetime.now()
self.server_log = Log("server_log.txt")
self.running = True
self.timeout = timeout
self.name = "" # Имя сервера
self.clients = [] # Список подключённых ip
self.requests = {} # Карта входящий сообщений адрес: сообщение
self.clients_names = Log.read_and_return_dict(
"clients_names.txt") # Карта ников пользователей адрес: имя
self.public_key, self.private_key = rsa.newkeys(512)
self.keys = [0 for i in range(3)]
self.first_client = socket.socket()
self.second_client = socket.socket()
self.third_client = socket.socket()
self.clients_info = [0 for i in range(3)
] # Индикатор загруженности соккетов
self.client_map = {} # Карта пользователей адрес: индекс
self.max_clients = 3
self.server_socket = socket.socket()
self.server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,
1)
self.server_socket.bind(('localhost', port))
self.server_socket.listen()
self.first_client.settimeout(timeout)
self.second_client.settimeout(timeout)
self.third_client.settimeout(timeout)
self.server_log.save_message("\n\n{}\nServer initialized".format(
str(now)))
# set - функции
def set_server_name(self, name): # Уставляет имя сервера
self.name = name
self.server_log.save_message("Server name set to {}".format(name))
# add - функции
def add_user(self, address, sock_ind): # Полноценно добавляет клинента
if address == "localhost" or address == "127.0.0.1":
self.clients.append("localhost")
self.clients.append("127.0.0.1")
self.client_map.update({
"localhost": sock_ind,
"127.0.0.1": sock_ind
})
else:
self.clients.append(address)
self.client_map.update({address: sock_ind})
self.server_log.save_message("Added user: {} to sock {}".format(
str(address), str(sock_ind)))
def add_key(self, address, key): # Добавляет RSA publickey пользователю
sock_ind = self.get_ind_by_address(address)
self.keys[sock_ind] = key
def add_user_name(self, address, name): # Добавляет имя пользователя
self.clients_names.update({address: name})
self.server_log.save_message("Added name {} to user {}".format(
name, address))
def add_request(self, address, message): # Добавляет сообщение от клиента
self.requests.update({address: message})
self.server_log.save_message("Added request from {}".format(address))
# del - функции
def del_user(self, address, sock_ind): # Удаляет пользвателя из списка
if address == "localhost" or "127.0.0.1":
try:
self.client_map.pop("localhost")
self.client_map.pop("127.0.0.1")
except KeyError:
pass
try:
self.clients.remove("localhost")
self.clients.remove("127.0.0.1")
except ValueError:
pass
else:
self.client_map.pop(address)
self.clients.remove(address)
self.server_log.save_message("Deleted user {} with socket {}".format(
address, sock_ind))
def del_key(self, address): # Удаляет RSA publickey у пользователя
sock_ind = self.get_ind_by_address(address)
self.keys[sock_ind] = 0
# server - функции
def create_connection(self, address, port): # Создаёт чат-сессию
self.server_log.save_message("Creating connection with {}:{}".format(
address, port))
sock_ind = self.get_free_socket()
self.server_log.save_message("Socket {}".format(sock_ind))
if not sock_ind and sock_ind != 0:
self.server_log.save_message(
"Failed to create connection: {}".format(
"All sockets are in use"))
return OSError
try:
self.server_log.save_message(
"Trying to create connection with {}".format(address))
thread = Thread(target=self.connect,
args=(address, port, sock_ind))
thread.start()
thread.join(0)
connection, address_1 = self.server_socket.accept()
connection.settimeout(0.2)
self.add_user(address, sock_ind)
except OSError:
self.server_log.save_message(
"Failed to create connection: {}".format(
"Socket is busy or offline"))
return OSError
self.raw_send(address, self.public_key.save_pkcs1())
key = connection.recv(162)
key = rsa.PublicKey.load_pkcs1(key)
self.add_key(address, key)
while self.running and self.clients_info[sock_ind]:
try:
data = connection.recv(4096)
except socket.timeout:
continue
except OSError:
self.server_log.save_message(
"Connection with {} lost".format(address))
self.close_connection(address, sock_ind)
return OSError
if data:
data = rsa.decrypt(data, self.private_key)
self.server_log.save_message(
"Received message from {}".format(address))
self.add_request(address, data.decode())
self.close_connection(address, sock_ind)
def send(self, address, message): # Отправляет сообщения с шифрованием
ind = self.get_ind_by_address(address)
try:
if ind == 0:
self.first_client.send(
rsa.encrypt(message.encode(), self.keys[ind]))
elif ind == 1:
self.second_client.send(
rsa.encrypt(message.encode(), self.keys[ind]))
elif ind == 2:
self.third_client.send(
rsa.encrypt(message.encode(), self.keys[ind]))
except OSError:
self.server_log.save_message(
"Error sending message to {}".format(address))
return OSError
self.server_log.save_message(
"Send message to {} with socket {}".format(address, ind))
def raw_send(self, address,
message): # Отправляет сообщение без шифрования
ind = self.get_ind_by_address(address)
try:
if ind == 0:
self.first_client.send(message)
elif ind == 1:
self.second_client.send(message)
elif ind == 2:
self.third_client.send(message)
except OSError:
self.server_log.save_message(
"Error raw sending message to {}".format(address))
return OSError
self.server_log.save_message(
"Raw send message to {} with socket {}".format(address, ind))
def connect(self, address, port,
sock_ind): # Создаёт подключение c соккетом по индексу
if sock_ind == 0:
try:
self.first_client.connect((address, port))
except OSError:
return OSError
elif sock_ind == 1:
try:
self.second_client.connect((address, port))
except OSError:
return OSError
elif sock_ind == 2:
try:
self.third_client.connect((address, port))
except OSError:
return OSError
self.clients_info[sock_ind] = 1
def reload_socket(self,
sock_ind): # Перезагружает соккет с указанным индексом
if sock_ind == 0:
self.first_client.close()
self.first_client = socket.socket()
elif sock_ind == 1:
self.second_client.close()
self.second_client = socket.socket()
elif sock_ind == 2:
self.third_client.close()
self.third_client = socket.socket()
self.clients_info[sock_ind] = 0
self.server_log.save_message(
"Socket {} was successfully reload".format(sock_ind))
def get_free_socket(
self): # Возвращает индекс свободного соккета, иначе ничего
for ind in range(self.max_clients):
if self.clients_info[ind] == 0:
return ind
return None
def get_request(self,
address): # Возвращает сообщение по адресу, иначе ничего
try:
message = self.requests.pop(address)
except KeyError:
message = None
self.server_log.save_message(
"Request from {} was taken".format(address))
return message
def check_request(self, address): # Возвращает Правда, если сообщение есть
message = self.requests.get(address)
return bool(message)
def get_ind_by_address(
self,
address): # Возвращет номер соккета, к которому подключён адрес
ind = self.client_map.get(address)
return ind
def get_user_name_by_address(
self, address): # Возвращает имя пользователя по его адресу
name = self.clients_names.get(address)
return name
def check_address(
self,
address): # Возвращает Правда, если к адрессу уже есть подключение
for i in self.clients:
if i == address:
return True
return False
def close_connection(self, address, sock_ind): # Закрывает соединение
try:
self.reload_socket(sock_ind)
self.del_user(address, sock_ind)
self.server_log.save_message(
"Connection with {} closed".format(address))
except TypeError:
quit()
def kill(self): # Останавливает сервер
self.running = False
sleep(1)
Log.save_dict(self.clients_names, "clients_names.txt")
self.server_log.save_message("Server successfully stopped")
self.server_log.close()
self.first_client.close()
self.server_socket.close()
def test_bad_check_referral_domains(referral_url, domains):
log = Log("user_id", "https://sth.com/bla-bla", referral_url,
datetime.now())
assert not log.check_referral_domains(domains)
def read(input_file: str):
read_lines: list = []
try:
with open(
input_file,
'r+t',
encoding=config.cfg.input_file_encoding,
newline='\r\n'
) as opened_file: # Read input file with specified encoding from cfg
line_number: int = 0
for line in opened_file:
line_number += 1
trimmed_line: str = line
if len(line) >= 2:
end = line[len(line) - 2:]
if end == '\r\n':
trimmed_line = line[0:len(line) - 2]
pass
pass
if line == '\r\n' and trimmed_line == '': # EOL
Log.log_to_file(True, 'Skipped EOL')
continue
elif trimmed_line == '': # Blank
Log.log_to_file(
True, 'Skipped line ({0}):\r\n{1}'.format(
line_number, trimmed_line))
continue
elif len(config.get_script_config().comment) <= len(
trimmed_line
) and trimmed_line[:len(config.get_script_config(
).comment)] == config.get_script_config().comment: # Comment
Log.log_to_file(
True, 'Skipped comment line ({0}):\r\n{1}'.format(
line_number, trimmed_line))
continue
# Check to ignore certain lines
if len(config.get_script_config().ignore) > 0:
should_ignore = False
for i in range(0, len(config.get_script_config().ignore)):
ignore = config.get_script_config().ignore[i]
if len(ignore) <= len(
trimmed_line
): # Add len check in case of len issues
if trimmed_line[:len(ignore)] == ignore:
should_ignore = True
break
pass
continue
if should_ignore:
Log.log_to_file(
True, 'Ignored line ({0}):\r\n{1}'.format(
line_number, trimmed_line))
continue
pass
read_lines.append([line_number, trimmed_line])
Log.log_to_file(
True,
'Appended line ({0}): {1}'.format(line_number,
trimmed_line))
continue
opened_file.close()
pass
except UnicodeDecodeError as e:
Log.log_to_file(
True,
'Failed to read input due to unicode decoding error! \"{0}\"!?'.
format(input_file))
Log.log_to_file(True, 'Reason: {}\n'.format(e.reason))
return None
return read_lines
import os
from src.log import Log
from src.plots import Plot
project_folder = os.path.dirname(__file__).split("src")[0]
l = Log()
#l.create_snapshot_pickles()
p = Plot()
p.event_frequency('event', 'week')
#p.create_graph_visualizations('day')