def main():
global limit_step
global complex_messages
# configs
config = LoadConfigs('config.ini')
template_paths = GetConfigParameter(config, 'Templates', 'paths').split(',')
filemaxlines = int(GetConfigParameter(config, 'OutputFile', 'filemaxlines'))
delimiter = GetConfigParameter(config, 'OutputFile', 'delimiter')
quotechar = GetConfigParameter(config, 'OutputFile', 'quotechar')
outputdir = GetConfigParameter(config, 'OutputFile', 'outputdir')
analitics_path = outputdir + '/' + GetConfigParameter(config, 'Analitics', 'filename')
metrics = GetConfigParameter(config, 'Analitics', 'metrics').split(',') # add check if it is in query
limit_step = int(GetConfigParameter(config, 'ClickHouse', 'limit_step'))
clickhouse_host = GetConfigParameter(config, 'ClickHouse', 'host')
colums = GetConfigParameter(config, 'ClickHouse', 'colums').split(',')
date_start = sys.argv[1]
date_end = sys.argv[2]
complex_messages = []
total_count = limit_step*2
GetTemplatesFromCSV(template_paths)
client = ClickHouseClient(clickhouse_host, on_progress=OnProgress)
query_format = GenerateQueryFormat(config, colums)
filename = GenerateFileName(outputdir, date_start, date_end)
csvwriter = CSVWriter(filemaxlines, filename, delimiter, quotechar, u'\ufeff', colums + ['FullText', 'MessageType', 'Template'])
analitics = Analitics(analitics_path, metrics)
for single_date in DateRange(date_start, date_end):
date_str = single_date.strftime("%Y-%m-%d")
ColoredPrint('\n[Determiner]', 'Working on date(' + date_str + ')', bcolors.OKGREEN, bcolors.WARNING)
limit_count = 0
msg_count = 0
while limit_count < total_count:
query = query_format.format(date_str, limit_count, limit_step)
result = client.select(query, on_progress=OnProgress, send_progress_in_http_headers=1)
print()
data_len = len(result.data)
total_count = limit_count + data_len + 1
counter = Counter(data_len, 0.2)
for v in result.data:
message = Message(colums, v)
id, c_message = ToComplexMessage(message)
if c_message.isFullFiled():
operator_id = message.get('OperatorGroupId')
originator = message.get('OutgoingOriginator')
c_message.determineTemplate(GetTemplates(operator_id, originator))
c_message.writePartsToCsvFile(csvwriter)
analitics.addData(date_str[:-3], c_message.getParameters(metrics), c_message.getType(), c_message.getCount())
complex_messages.pop(id)
msg_count+=1
counter.step(bcolors.OKGREEN + '[' + date_str + '] ' + bcolors.ENDC + str(msg_count) + ' messages handled')
counter.lastTell(bcolors.OKGREEN + '[' + date_str + '] ' + bcolors.ENDC + str(msg_count) + ' messages handled')
del result
limit_count += limit_step
WriteDownAllRemainingData(csvwriter, analitics, metrics)
csvwriter.close()
SaveAnalitics(analitics, delimiter, quotechar)
from client import ClickHouseClient
from errors import Error as ClickHouseError
def on_progress(total, read, progress):
print(total,read,progress)
try:
client = ClickHouseClient('http://ch00.fin.adfox.ru:8123/', on_progress=on_progress, user='******', password='******')
query = 'SELECT date FROM adfox.dist_elog WHERE date > toDate(0)'
result = client.select(query, send_progress_in_http_headers=1)
print(result.data)
except ClickHouseError as e:
print(e)
except Exception as e:
print(e)