def update(self, **kwargs):
self.settings.update_params(**kwargs)
if not any(self.settings.rates.values()) or self.settings.update:
print("[INFO]: Trying to get exchange rates from remote server...")
self.exchanger.update_exchange_rates(self.settings.rates)
self.exchanger.save_rates(self.settings.rates)
print(f"[INFO]: Get exchange rates: {self.settings.rates}")
self.collector = DataCollector(self.settings.rates)
self.analyzer = Analyzer(self.settings.save_result)
class ResearcherHH:
"""Main class for searching vacancies and analyze them."""
def __init__(self, config_path: str = SETTINGS_PATH, no_parse: bool = False):
self.settings = Settings(config_path, no_parse=no_parse)
self.exchanger = Exchanger(config_path)
self.collector: Optional[DataCollector] = None
self.analyzer: Optional[Analyzer] = None
self.predictor = Predictor()
def update(self, **kwargs):
self.settings.update_params(**kwargs)
if not any(self.settings.rates.values()) or self.settings.update:
print("[INFO]: Trying to get exchange rates from remote server...")
self.exchanger.update_exchange_rates(self.settings.rates)
self.exchanger.save_rates(self.settings.rates)
print(f"[INFO]: Get exchange rates: {self.settings.rates}")
self.collector = DataCollector(self.settings.rates)
self.analyzer = Analyzer(self.settings.save_result)
def __call__(self):
print("[INFO]: Collect data from JSON. Create list of vacancies...")
vacancies = self.collector.collect_vacancies(
query=self.settings.options, refresh=self.settings.refresh, max_workers=self.settings.max_workers
)
print("[INFO]: Prepare dataframe...")
df = self.analyzer.prepare_df(vacancies)
print("\n[INFO]: Analyze dataframe...")
self.analyzer.analyze_df(df)
print("\n[INFO]: Predict None salaries...")
# total_df = self.predictor.predict(df)
# self.predictor.plot_results(total_df)
print("[INFO]: Done! Exit()")
from src.data_collector import DataCollector #Collect data to be analized
from src.textNotifications import TextNotifier
dc = DataCollector()
dc.printData()
#Texting, will run when some condition I must see is passed
importantEvent = True
if (importantEvent):
texter = TextNotifier()
# texter.sendText()
def __init__(self):
super().__init__()
self.data_collector = DataCollector()
self.data_viewer = DataViewer()
self.predictor = Predictor()
class App(QtWidgets.QMainWindow):
def __init__(self):
super().__init__()
self.data_collector = DataCollector()
self.data_viewer = DataViewer()
self.predictor = Predictor()
def keyPressEvent(self, e):
if e.key() == QtCore.Qt.Key_Escape:
self.close()
def create_log_widget(self):
logTextBox = QTextEditLogger(self)
logTextBox.setFormatter(logging.Formatter('%(asctime)s: %(message)s'))
logging.getLogger().addHandler(logTextBox)
logging.getLogger().setLevel(logging.DEBUG)
return logTextBox.widget
def display_main_menu(self):
self.setWindowTitle('iClicker')
# self.resize(*get_screen_dimensions())
self.resize(800, 600)
# creating layout
main_widget = QtWidgets.QWidget()
main_widget.setLayout(QtWidgets.QVBoxLayout())
self.top_menu_part = QtWidgets.QWidget()
self.top_menu_part.setLayout(QtWidgets.QVBoxLayout())
# self.top_menu_part.layout().addWidget(QtWidgets.QLabel('iClicker'))
self.log_widget = self.create_log_widget()
self.top_menu_part.layout().addWidget(self.log_widget)
self.top_menu_part.resize(100, 200)
self.bottom_menu_part = QtWidgets.QWidget()
main_widget.layout().addWidget(self.top_menu_part)
main_widget.layout().addWidget(self.bottom_menu_part)
self.add_control_buttons()
self.setCentralWidget(main_widget)
self.show()
def add_control_buttons(self):
self.bottom_menu_part.setLayout(QtWidgets.QGridLayout())
collect_data_button = QtWidgets.QPushButton('Colectare date')
collect_data_button.setToolTip('Colectează date')
collect_data_button.clicked.connect(self.collect_data)
process_data_button = QtWidgets.QPushButton('Procesare date')
process_data_button.setToolTip('Procesează datele colectate')
process_data_button.clicked.connect(self.process_collected_data)
train_button = QtWidgets.QPushButton('Antrenare model')
train_button.setToolTip('Antrenează modelul bazat pe datele procesate')
train_button.clicked.connect(self.train_model)
predict_button = QtWidgets.QPushButton('Simulare')
predict_button.setToolTip('Simulează funcționalitățile mouse-ului')
predict_button.clicked.connect(self.predict)
# view_data_button = QtWidgets.QPushButton('View data')
# view_data_button.setToolTip('View collected data')
# view_data_button.clicked.connect(self.view_data)
buttons = [
collect_data_button, process_data_button, train_button,
predict_button
]
for i in range(0, 2):
for j in range(0, 2):
self.bottom_menu_part.layout().addWidget(
buttons[i * 2 + j], i, j)
def predict(self):
if self.predictor.can_predict() == False:
self.error_dialog = QtWidgets.QErrorMessage()
self.error_dialog.showMessage(
'Nu există niciun model antrenat.\nApăsați pe butonul "Antrenare" pentru a antrena un model.'
)
return
self.predictor.start()
def process_collected_data(self):
if can_process_data() == False:
self.error_dialog = QtWidgets.QErrorMessage()
self.error_dialog.showMessage(
'Nu există date pentru a putea fi procesate.\n Apăsați pe butonul "Colectare date" mai întâi pentru a obține date.'
)
return
run_function_on_thread(data_processing_main)
def view_data(self):
print('Getting collected data...')
data = self.data_collector.get_collected_data()
print(f'Displaying random photos from {len(data)} samples')
self.data_viewer.view_data(data)
def train_model(self):
if Trainer.can_train_model() == False:
self.error_dialog = QtWidgets.QErrorMessage()
self.error_dialog.showMessage(
'Nu există date procesate pentru a putea antrena un model.\n Apăsați pe butonul "Procesare date" pentru a procesa datele.'
)
return
run_function_on_thread(Trainer.main)
def collect_data(self):
self.data_collector.collect_data()
from src.data_collector import DataCollector
from datetime import datetime
if __name__ == "__main__":
# Create the DataCollector with default parameters
my_data_collector = DataCollector()
# Start the Data Collection
print(f"[i] Start DataColector{datetime.now()}")
my_data_collector.start()
print(f"[i] Stop DataColector{datetime.now()}")
sys.path.append(
os.path.dirname(
os.path.dirname(
os.path.dirname(os.path.dirname(os.path.realpath(__file__))))))
from src.data_collector import DataCollector
import argparse
import json
BASE_DIR = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
CONFIG_PATH = os.path.join(BASE_DIR, 'config')
CONFIG_FILE = 'newsdatasource.json'
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('-c',
'--config',
dest='config',
help='Absolute path to configuration file.')
args = parser.parse_args()
if not args.config:
print('no config file')
exit()
else:
with open(os.path.join(CONFIG_PATH, CONFIG_FILE), 'r') as file:
__config = json.load(file)
__dataFetchConfig = __config.get('Collection')
obj = DataCollector(__dataFetchConfig)
input('print key to exit')
def __init__(self, DataCollectionconfig):
print('DataFetcher instantiated')
self._dataCollector = DataCollector(DataCollectionconfig)