def main(params: InputParams) -> None:
logger = get_logger()
try:
try:
start_time = datetime.utcnow()
csv_rows, db_rows = todb(params)
took_seconds = seconds_between(start_time)
velocity_kBps, velocity_rows_sec = (
path.getsize(params.input_path) /
1000) / took_seconds, csv_rows / took_seconds
success_percentage = db_rows * 100 / csv_rows
logger.info(
"Inserted {} / {} ({:.1f}%) rows in {:.2f}s ({:.1f} kB/s, {:.1f} rows/s)"
.format(db_rows, csv_rows, success_percentage, took_seconds,
velocity_kBps, velocity_rows_sec))
exit(EXIT_CODE_OK)
except Exception as e:
logger.error("Error: {} ()".format(e))
traceback.print_exc()
exit(EXIT_CODE_FAILURE)
except Exception as e:
logger.error(
"Provided arguments were not correct: {} (args: {})".format(
e, params))
exit(EXIT_CODE_USER_ERROR)
def __init__(self, task_queue: mp.Queue, unsuccessful_rows_queue: mp.Queue,
importer: Importer, table_name: str) -> None:
super(ParsingWorker, self).__init__()
self.table_name = table_name
self.importer = importer
self.task_queue = task_queue
self.unsuccessful_rows_queue = unsuccessful_rows_queue
self.logger = get_logger()
def todb(params: InputParams) -> Tuple[int, int]:
logger = get_logger()
columns, pkey, file_config = parse_model_file(params.model_path)
logger.debug("Parsed model columns: {}".format(columns))
executor = ParallelExecutor(params, file_config, columns, pkey,
params.table_name, params.fail_output_path)
csv_rows, db_rows = executor.start(params.input_path)
return csv_rows, db_rows
def __init__(self, params: InputParams, input_file_config: InputFileConfig,
columns: List[ConfColumn], pkey: PrimaryKeyConf,
table_name: str, failed_rows_file: str) -> None:
self.pkey = pkey
self.params = params
self.input_file_config = input_file_config
self.columns = columns
self.table_name = table_name
self.failed_rows_file = failed_rows_file
self.logger = get_logger()
def __init__(self,
db_url: str,
entity_builder: EntityBuilder,
db_engine: Optional[Engine] = None,
ca_file: Optional[str] = None) -> None:
self.db_url = db_url
self.ca_file = ca_file
self.entity_builder = entity_builder
self._db_engine = db_engine
self.logger = get_logger()
self._conn = None # type: Optional[Connection]
def __init__(self, input_file_config: InputFileConfig,
output_file_path: str) -> None:
self.input_file_config = input_file_config
self.output_file_path = output_file_path
self.logger = get_logger()
def __init__(self, columns: List[ConfColumn]) -> None:
self.columns = columns
self.logger = get_logger()
def __init__(self, unsuccessful_rows_queue: mp.Queue,
handler: FailRowHandler) -> None:
super(UnsuccessfulRowsHandlingWorker, self).__init__()
self.unsuccessful_rows_queue = unsuccessful_rows_queue
self.handler = handler
self.logger = get_logger()
def __init__(self, db_client: DbClient) -> None:
self.db_client = db_client
self.logger = get_logger()
def __init__(self, input_file_config: InputFileConfig,
chunk_size_kB: int) -> None:
self.chunk_size_kB = chunk_size_kB
self.input_file_config = input_file_config
self.logger = get_logger()