class JsonTrialReader:
logger = get_logger_()
def __init__(self, *args):
reader = args[READER_CONSTANT]
with open(PATH_TO_FILE + reader, mode=READ_MODE) as f:
try:
self.json = json.load(f)
self.counter = len(self.json)
except JSONDecodeError:
JsonTrialReader.logger.error(EMPTY_JSON)
self.counter = ZERO
def next_trial(self):
if self.counter > ZERO:
json_object = self.json[len(self.json) - self.counter]
self.counter -= 1
class_name = json_object.get(CLASS, None)
if class_name is None:
JsonTrialReader.logger.error(CLASS_NAME_IS_EMPTY)
return None
args = json_object.get(ARGS, None)
if args is None:
JsonTrialReader.logger.error(ARGS_IS_EMPTY)
return None
account = args.get(ACCOUNT, None)
mark1 = args.get(MARK1, INITIAL_VALUE)
mark2 = args.get(MARK2, INITIAL_VALUE)
mark3 = args.get(MARK3, INITIAL_VALUE)
return TrialsFactory.get_trial(class_name, account, mark1, mark2, mark3)
else:
return FINAL_TRIAL
def close(self):
pass
class SqlTrialReader:
logger = get_logger_()
def __init__(self, *args):
reader = args[READER_CONSTANT]
configuration_file_name = args[CONFIG_CONSTANT]
params = reader.split(DOT)
if len(params) < SQL_PARAMS_NUMBER:
raise ValueError(NOT_ENOUGH_PARAMETERS_SQL + reader)
database_name = params[DATABASE_PARAM]
table_name = params[TABLE_PARAM]
if is_table_exist(configuration_file_name, database_name, table_name):
self.SQL = SQL_FOR_SELECTION.format(table_name)
self.counter = COUNTER_START
self.connection = get_connection(configuration_file_name, database_name)
self.cursor = self.connection.cursor(prepared=True)
else:
raise ValueError(TABLE_DOES_NOT_EXIST.format(database_name, table_name))
def next_trial(self):
self.cursor.execute(self.SQL, str(self.counter))
trial = self.cursor.fetchall()
if len(trial) > TRIAL_PARAM:
self.counter += COUNTER_STEP
return TrialsFactory.get_trial(trial[TRIAL_PARAM][CLASS_PARAM],
trial[TRIAL_PARAM][ACCOUNT_PARAM],
trial[TRIAL_PARAM][MARK1_PARAM],
trial[TRIAL_PARAM][MARK2_PARAM],
trial[TRIAL_PARAM][MARK3_PARAM])
else:
return FINAL_TRIAL
def close(self):
self.connection.close()
class CsvTrialReader:
logger = get_logger_()
def __init__(self, *args):
reader = args[READER_CONSTANT]
self.file_reader = open(PATH_TO_FILE + reader, mode=READ_MODE)
def next_trial(self):
trial = self.file_reader.readline().strip()
if trial:
trial = trial.split(CSV_SEPARATOR)
if len(trial) < CSV_PARAMETERS_NUMBER:
CsvTrialReader.logger.error(TOO_FEW_ARGUMENTS_MESSAGE)
return None
class_name, account, mark1, mark2 = trial[:CSV_PARAMETERS_NUMBER]
mark3 = INITIAL_VALUE
if len(trial) > CSV_PARAMETERS_NUMBER:
mark3 = trial[CSV_PARAMETERS_NUMBER]
return TrialsFactory.get_trial(class_name, account, mark1, mark2,
mark3)
else:
return FINAL_TRIAL
def close(self):
self.file_reader.close()
class TrialReadersFactory:
TRIAL_DAO_PATTERNS_DICT = {
CSV: CsvTrialReader,
JSON: JsonTrialReader,
SQL: SqlTrialReader
}
logger = get_logger_()
def __init__(self, configuration_file_name, group,
readers_name_in_properties):
self.trial_dao_list = list()
config = get_config(configuration_file_name)
reader = config.get(group, readers_name_in_properties)
readers = reader.split(CSV_SEPARATOR)
for reader in readers:
try:
file_extension = os.path.splitext(
reader)[EXTENSION_PARAMETER][EXTENSION_PARAMETER:]
reader_implementation = TrialReadersFactory \
.TRIAL_DAO_PATTERNS_DICT.get(file_extension.upper())
self.trial_dao_list.append(
reader_implementation(reader, configuration_file_name))
except TypeError:
TrialReadersFactory.logger.error(EMPTY_READER)
except (FileNotFoundError, ValueError) as e:
TrialReadersFactory.logger.error(e)
if len(self.trial_dao_list) == 0:
raise ValueError(NO_VALID_READERS)
def __enter__(self):
return self
def get_trial_dao(self):
return self.trial_dao_list
def __exit__(self, exc_type, exc_val, exc_tb):
for reader in self.trial_dao_list:
reader.close()
import queue
from concurrent.futures.thread import ThreadPoolExecutor
from configparser import NoOptionError
from readers.TrialReader import TrialReader
from readerwriterfactory import TrialWriterFactory
from readerwriterfactory.TrialReadersFactory import TrialReadersFactory
from utils.ExceptionHandling import get_logger_
from utils.ReaderWriterConstants import CONFIG_FILE_NAME, READER_WRITER_GROUP, \
READER_IN_CONFIG, WRITER_IN_CONFIG, QUEUE_SIZE
from writers.TrialWriter import TrialWriter
logger = get_logger_()
blocking_queue = queue.Queue(QUEUE_SIZE)
try:
with TrialReadersFactory(CONFIG_FILE_NAME,
READER_WRITER_GROUP,
READER_IN_CONFIG) as all_trial_dao, \
TrialWriterFactory.get_consumer(CONFIG_FILE_NAME,
READER_WRITER_GROUP,
WRITER_IN_CONFIG) as trial_consumer, \
ThreadPoolExecutor(max_workers=len(all_trial_dao.get_trial_dao())) as e:
for trial_dao in all_trial_dao.get_trial_dao():
trial_reader = TrialReader(blocking_queue, trial_dao)
e.submit(trial_reader.run)
trial_writer = TrialWriter(blocking_queue, trial_consumer,
len(all_trial_dao.get_trial_dao()))
trial_writer.go()
except (FileNotFoundError, ValueError, NoOptionError) as e:
logger.error(e)