def parse_and_execute(self, parsed_args: Dict):
table_name = parsed_args['table-name']
client = create_client_from_parsed_args(parsed_args)
connection = create_sql_connecting_from_parsed_args(parsed_args)
result_df = connection.execute_query_and_save_result(
parsed_args['query'])
inferred_schema = AitoTableSchema.infer_from_pandas_data_frame(
result_df)
converted_tmp_file = tempfile.NamedTemporaryFile(mode='w',
suffix='.ndjson.gz',
delete=False)
DataFrameHandler().df_to_format(result_df, 'ndjson',
converted_tmp_file.name,
{'compression': 'gzip'})
converted_tmp_file.close()
api.create_table(client, table_name, inferred_schema)
with open(converted_tmp_file.name, 'rb') as in_f:
api.upload_binary_file(client=client,
table_name=table_name,
binary_file=in_f)
converted_tmp_file.close()
unlink(converted_tmp_file.name)
return 0
def parse_and_execute(self, parsed_args: Dict):
client = create_client_from_parsed_args(parsed_args)
table_name = parsed_args['table-name']
table_schema = load_json_from_parsed_input_arg(parsed_args['input'],
'table schema')
api.create_table(client=client,
table_name=table_name,
schema=table_schema)
return 0
def create_table(self):
create_table(self.client, self.default_table_name, self.default_table_schema)
def create_table(self):
with (self.input_folder / "invoice_aito_schema.json").open() as f:
table_schema = json.load(f)
create_table(self.client, self.default_table_name, table_schema)
import json
# 1. load the data
df = pandas.read_csv("../datasets/invoice-automation/invoice_data.csv")
print("loaded the invoice data.")
# 2. load the database schema
with open("schema.json") as f:
schema = json.load(f)
print("loaded the invoice schema from schema.json.")
# 3. create the Aito table
aito_instance_url = os.environ.get('AITO_INSTANCE_URL')
aito_api_key = os.environ.get('AITO_API_KEY')
aito_client = AitoClient(instance_url=aito_instance_url, api_key=aito_api_key)
delete_table(client=aito_client, table_name='invoice_data')
create_table(client=aito_client, table_name='invoice_data', schema=schema)
print("created the aito database table.")
# 4. convert the dataframe to have correct types
entries = DataFrameHandler().convert_df_using_aito_table_schema(
df=df,
table_schema=schema
).to_dict(orient="records")
print("converted the dataframe.")
# 5. upload the entries
upload_entries(aito_client, table_name='invoice_data', entries=entries)
print("uploaded the entries.")
def create_default_table_and_check(self):
api.create_table(self.client, self.default_table_name, self.default_table_schema)
self.assertTrue(api.check_table_exists(self.client, self.default_table_name))