def sql_import(self):
# TODO: Archive the original sheet in some way, along with the csv conversion
try:
schema_create_if_not_exists(config.config["database"]["mysql_user_database"])
table_name_drop(schema_name=self.schema, table_name=self.import_table_name)
except Exception as ex:
Model.logger.log(logging.ERROR, "Exception: {0} Arguments: {1}".format(type(ex).__name__, ex.args))
raise
else:
try:
# noinspection PyUnresolvedReferences
# from csvkit import convert
in2csv.launch_new_instance()
args = [csvkit_tool("in2csv"), str(self.wb_filename)]
args.extend(["--sheet", self.ws_name_selection]) # Append the worksheet name
if self.header_row != 1:
args.extend(["-K", str(self.header_row - 1)])
args.extend(["|"]) # Convert worksheet into csv, pass to csvcut
args.extend([csvkit_tool("csvcut"), "-l", "|"]) # Add line number to csv output
args.extend([csvkit_tool("csvformat")])
args.extend(["-U", "1"]) # Quote all fields
args.extend(["-Q", self.quote_char]) # Set Quote character
args.extend(["|"]) # Pass to csvsql
args.extend([csvkit_tool("csvsql")])
args.extend(["-q", self.quote_char]) # Set Quote character
args.extend(["-u", "1"]) # Set Quote character
args.extend(["-e", "ansi"]) # Set encoding
args.extend(["--db", get_mysql_conn_string(user_db=self.schema)]) # Set the db connection string
args.extend(["--snifflimit", "0"])
args.extend(["--tables", self.import_table_name]) # Name the table
args.extend(["--insert", "--overwrite"])
Model.logger.log(logging.DEBUG,
"Beginning unified csvkit step: {0}".format(subprocess.list2cmdline(args)))
csvsql_process = subprocess.Popen(args, shell=True, **subprocess_args(False))
out, err = csvsql_process.communicate()
Model.logger.log(logging.DEBUG, "Out: {0}".format(out))
Model.logger.log(logging.DEBUG, "Err: {0}".format(err))
except subprocess.CalledProcessError as ex:
Model.logger.log(logging.ERROR,
"CalledProcessError Exception: {0} Arguments: {1}".format(type(ex).__name__, ex.args))
raise
except Exception as ex:
Model.logger.log(logging.ERROR, "Exception: {0} Arguments: {1}".format(type(ex).__name__, ex.args))
raise
else:
Model.logger.log(logging.DEBUG, "Reflecting table in SQLAlchemy:")
self.table = Table(self.import_table_name,
mysql_base.metadata,
autoload=True,
autoload_with=mysql_engine,
schema=self.schema)
def test_launch_new_instance(self):
with patch.object(
sys, 'argv',
[self.Utility.__name__.lower(), 'examples/dummy.csv']):
launch_new_instance()
def test_launch_new_instance(self):
with patch.object(sys, 'argv', [self.Utility.__name__.lower(), 'examples/dummy.csv']):
launch_new_instance()
def test_launch_new_instance(self):
with patch.object(sys, 'argv', ['in2csv', 'examples/dummy.csv']):
launch_new_instance()
def test_launch_new_instance(self):
with patch.object(sys, 'argv', ['in2csv', 'examples/dummy.csv']):
launch_new_instance()
