def requires(self):
pre_tasks = []
dateCurrent = self.dateStart
while (dateCurrent < self.dateEnd):
pre_tasks.append(TaskReportNes(dateTarget=dateCurrent))
dateCurrent = util.add_days(dateCurrent, 1)
return pre_tasks
def __init__(self, auth_folder, thread_id, project_id, bq_folder, scripts, date_target, period):
threading.Thread.__init__(self)
self.logger = logging.getLogger(__name__)
self.thread_id = thread_id
self.project_id = project_id
self.scripts = scripts
self.date_target = date_target
self.period = period
self.bq_folder = bq_folder
self.exec_scripts = []
self.auth_folder = auth_folder
self.svc = get_service(auth_folder)
if (self.period == 'W'):
self.date_start = date_target
self.date_end = util.add_days(date_target, 6)
self.date_start_prev = util.add_days(date_target, -7)
else:
self.date_start = util.getDateMonthStart(date_target)
self.date_end = util.getDateMonthEnd(date_target)
self.date_start_prev = util.getDatePrevMonthStart(date_target)
def requires(self):
pre_tasks = {}
print('requires....')
logging.info('requires....')
if self.period == 'W':
date_current = self.date_start
else:
date_current = jbtools.get_date_month_start(self.date_start)
while (date_current <= self.date_end):
date_prev = None
if self.period == 'W':
date_prev = util.add_days(date_current, -7)
else:
date_prev = jbtools.get_date_prev_month_start(date_current)
if self.period == 'W' and date_current > util.add_days(self.date_start, 7):
rpt_task = NesStep41Task(date_target=date_current, project_id='1049530906192', period=self.period)
elif self.period == 'M' and date_current > jbtools.get_date_month_start(self.date_start):
rpt_task = NesStep41Task(date_target=date_current, project_id='1049530906192', period=self.period)
else:
rpt_task = NesStep09Task(date_target=date_current, project_id='1049530906192', period=self.period)
rpt_key = self.get_report_key(date_current, self.period)
pre_tasks[rpt_key] = rpt_task
rpt_key_prev = self.get_report_key(date_prev, self.period)
if rpt_key_prev in pre_tasks:
rpt_task.pre_tasks.append(pre_tasks[rpt_key_prev])
if self.period == 'W':
date_current = util.add_days(date_current, 1)
else:
date_current = jbtools.get_date_next_month_start(date_current)
return pre_tasks
def main(argv):
# example calling: main W|M DATE_START [DATE_END]
if (len(argv) < 3):
print('example calling: main W|M DATE_START [DATE_END]')
logger.warn('example calling: main W|M DATE_START [DATE_END]')
sys.exit(9)
logger.debug('argv: %s', argv)
logger.debug('argv len: %d', len(argv))
conf = config.load_config()
project_id = conf['ProjectId']
period = argv[1]
make_full_trans = True;
date_start = parse(argv[2])
if (len(argv) == 4):
date_end = parse(argv[3])
else:
date_end = date_start
bq_folder = conf['ScriptFolder']
file_path_full_trans = os.path.join(bq_folder, conf['Scripts']['JoinFullTransaction']['Script'])
auth_folder = config.get_auth_folder()
logger.info('project_id: %s', project_id)
logger.info('make_full_trans: %s', make_full_trans)
logger.info('period: %s', period)
logger.info('date_start: %s', date_start)
logger.info('date_end : %s', date_end)
bq_full_trans = read_txt_file(file_path_full_trans)
if (period == 'M'):
date_start = util.getDateMonthStart(date_start)
todo_dates = []
date_current = date_start
while date_end >= date_current:
todo_dates.append(date_current)
if (period == 'M'):
date_current = util.getDateNextMonthStart(date_current)
else:
date_current = util.add_days(date_current, 1)
days_waiting = []
workers_doing = []
workers_done = []
max_concurrent = 15
for d in todo_dates:
logger.info('todo date: %s', d)
days_waiting.append(d)
# threadName = 'worker-nes-{:%Y%m%d}'.format(d)
# workerNes = NesWorker(threadName, project_id, bq_folder=bq_folder, scripts=conf['Scripts']['CalculateNes'], date_target=d, period=period);
# workers_waiting.append(workerNes)
logger.info('count_days_waiting=%s', len(days_waiting))
while(len(days_waiting) > 0):
if (len(workers_doing) < max_concurrent):
day_todo = days_waiting[0]
threadName = 'worker-nes-{:%Y%m%d}'.format(day_todo)
worker = NesWorker(auth_folder, threadName, project_id, bq_folder=bq_folder, scripts=conf['Scripts']['CalculateNes'], date_target=day_todo, period=period);
workers_doing.append(worker)
days_waiting.remove(day_todo)
worker.start()
logger.info('start worker:%s', worker.thread_id)
else:
logger.info('concurrent worker limit reached.')
for w in workers_doing:
if (not w.is_alive()):
workers_done.append(w)
workers_doing.remove(w)
logger.info('worker waiting: %s, doing: %s, done: %s', len(days_waiting), len(workers_doing), len(workers_done))
time.sleep(0.5)
logger.info('all worker started. count_worker_doing=%s', len(workers_doing))
for w in workers_doing:
w.join()
logger.info('worker joined. id=%s', w.thread_id)
logger.info("Exiting Main Thread")
