def _setup(self, query, host='http://localhost', user='******', password=None, port=8123): if ' format ' in query.lower(): err_msg = 'Please refrain from adding a "FROAMT" statement to the query' log.error(err_msg) raise Exception(err_msg) query = f'{query} FORMAT JSON' log.info(f'Getting data via the query: "{query}""') params = {'user': user} if password is not None: params['password'] = password response = requests.post(f'{host}:{port}', data=query, params=params) try: data = response.json()['data'] except: log.error( f'Got an invalid response from the database: {response.text}') raise Exception(response.text) df = pd.DataFrame(data) col_map = {} for col in df.columns: col_map[col] = col return df, col_map
def fixFileIfPossible(filepath): """ Tries to fix a file header if it finds header or encoding issues :param filepath: the filepath to fix if possible :return: fixed, error """ fixed = False error = False rows = [] try: with open(filepath, newline='') as f: reader = csv.reader(f) header = None max_len = 0 for row in reader: if header is None: header = row for i, col in enumerate(row): if col in [None, '']: fixed = True header[i] = 'col_{i}'.format(i=i+1) rows += [row] length = int(len(row)) if length > max_len: max_len = length log.info(max_len) except: exc_type, exc_value, exc_traceback = sys.exc_info() error = traceback.format_exception(exc_type, exc_value, exc_traceback) return fixed, error if len(header) < max_len or fixed == True: rightCell = lambda h, i: 'col_{i}'.format(i=i+1) if i > len(header) else h row = [rightCell(header_col, i) for i, header_col in enumerate(header)] rows[0] = row with open(filepath, 'w', newline='') as f: writer = csv.writer(f) writer.writerows(rows) return fixed, error
f'Got an invalid response from the database: {response.text}') raise Exception(response.text) df = pd.DataFrame(data) col_map = {} for col in df.columns: col_map[col] = col return df, col_map if __name__ == "__main__": from mindsdb import Predictor log.info('Starting ClickhouseDS tests !') log.info('Inserting data') requests.post('http://localhost:8123', data='CREATE DATABASE IF NOT EXISTS test') requests.post('http://localhost:8123', data='DROP TABLE IF EXISTS test.mock') requests.post('http://localhost:8123', data="""CREATE TABLE test.mock( col1 String ,col2 Int64 ,col3 Array(UInt8) ) ENGINE=Memory""") requests.post('http://localhost:8123', data="""INSERT INTO test.mock VALUES ('a',1,[1,2,3])""") requests.post('http://localhost:8123',
def parse(str): str, text_var_map = replaceTexts(str) str = cleanStr(str) log.info(str)
def test(): log.info(fixFileIfPossible('/Users/jorge/Downloads/tweets (1).csv'))