def summary(self, sqls, params = None):
'''Run summary scripts.'''
params = sql_utils.extend_params(params)
sql_list = sql_utils.get_sql_list(sqls, params)
for sql in sql_list:
self.logger.info('Summary\n%s' % sql)
with self.conn.cursor() as c:
c.execute(sql)
def load(self, sqls, table_name, params = None):
'''
Load data into table, use ${dataFile} variable to indicate file path.
Also supports multiple SQL statement.
'''
params = sql_utils.extend_params(params)
params['dataFile'] = "'%s'" % self.get_data_file(table_name, params)
sql_list = sql_utils.get_sql_list(sqls, params)
for sql in sql_list:
self.logger.info('Load\n%s' % sql)
with self.conn.cursor() as c:
c.execute(sql)
def extract(self, sqls, table_name, params = None):
'''
Run a single query, and store the result into file:
<EXTRACT_FILES_DIR>/<table_name>.tsv
'''
params = sql_utils.extend_params(params)
# check
sql_list = sql_utils.get_sql_list(sqls, params)
self.check_single_select(sql_list)
data_file = self.get_data_file(table_name, params)
self.logger.info('Extract into file %s\n%s' % (data_file, sql_list[0]))
# query
with self.conn.cursor() as c:
c.execute(sql_list[0])
with open(data_file, 'w') as f:
ptrn = re.compile(r'[\r\n\t]+')
for row in c:
line = '\t'.join(ptrn.sub(' ', unicode(col)) for col in row)
f.write(line.encode('utf-8'))
f.write('\n')