def _setup(self,
query,
host='http://localhost',
user='******',
password=None,
port=8123):
if ' format ' in query.lower():
err_msg = 'Please refrain from adding a "FROAMT" statement to the query'
log.error(err_msg)
raise Exception(err_msg)
query = f'{query} FORMAT JSON'
log.info(f'Getting data via the query: "{query}""')
params = {'user': user}
if password is not None:
params['password'] = password
response = requests.post(f'{host}:{port}', data=query, params=params)
try:
data = response.json()['data']
except:
log.error(
f'Got an invalid response from the database: {response.text}')
raise Exception(response.text)
df = pd.DataFrame(data)
col_map = {}
for col in df.columns:
col_map[col] = col
return df, col_map
def fixFileIfPossible(filepath):
"""
Tries to fix a file header if it finds header or encoding issues
:param filepath: the filepath to fix if possible
:return: fixed, error
"""
fixed = False
error = False
rows = []
try:
with open(filepath, newline='') as f:
reader = csv.reader(f)
header = None
max_len = 0
for row in reader:
if header is None:
header = row
for i, col in enumerate(row):
if col in [None, '']:
fixed = True
header[i] = 'col_{i}'.format(i=i+1)
rows += [row]
length = int(len(row))
if length > max_len:
max_len = length
log.info(max_len)
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
error = traceback.format_exception(exc_type, exc_value,
exc_traceback)
return fixed, error
if len(header) < max_len or fixed == True:
rightCell = lambda h, i: 'col_{i}'.format(i=i+1) if i > len(header) else h
row = [rightCell(header_col, i) for i, header_col in enumerate(header)]
rows[0] = row
with open(filepath, 'w', newline='') as f:
writer = csv.writer(f)
writer.writerows(rows)
return fixed, error
f'Got an invalid response from the database: {response.text}')
raise Exception(response.text)
df = pd.DataFrame(data)
col_map = {}
for col in df.columns:
col_map[col] = col
return df, col_map
if __name__ == "__main__":
from mindsdb import Predictor
log.info('Starting ClickhouseDS tests !')
log.info('Inserting data')
requests.post('http://localhost:8123',
data='CREATE DATABASE IF NOT EXISTS test')
requests.post('http://localhost:8123',
data='DROP TABLE IF EXISTS test.mock')
requests.post('http://localhost:8123',
data="""CREATE TABLE test.mock(
col1 String
,col2 Int64
,col3 Array(UInt8)
) ENGINE=Memory""")
requests.post('http://localhost:8123',
data="""INSERT INTO test.mock VALUES ('a',1,[1,2,3])""")
requests.post('http://localhost:8123',
def parse(str):
str, text_var_map = replaceTexts(str)
str = cleanStr(str)
log.info(str)
def test():
log.info(fixFileIfPossible('/Users/jorge/Downloads/tweets (1).csv'))
