def stockIntegrity(params, data):
Debug = DebugManager.DebugManager()
Debug.start()
global msia_tz, date_retrieve_limit
result = []
today = DateTime.now(tzinfo=msia_tz)
start_date = DateTime.getDaysAgo(date_retrieve_limit, datefrom=today)
durations = DateTime.getBetween([start_date, today],
element='date',
offset=24)['order']
# offset 24 hour to include today
state_data = fn.getNestedElement(data, 'state')
facility_data_by_state = fn.getNestedElement(data, 'state_facility')
check_data = combinedFacilityList(data=facility_data_by_state)
result = getIntegrity(params={
'durations': durations,
},
data={
'facility': facility_data_by_state,
'state': state_data,
'to_update': result,
'check_data': check_data,
})
updateStateData(result)
result = list(sorted(result, key=lambda k: k['name'], reverse=False))
Debug.end()
Debug.show('Model.Structure.stockIntegrity')
return result
def getMissingDates(data):
dbManager = SharedMemoryManager.getInstance();
db = dbManager.query();
missing_dates = {};
today = DateTime.now(tzinfo=msia_tz); # date only
state_by = 'state_code';
states = list(db['state'].find({},{'_id': 0, state_by: 1}));
current_year = DateTime.getDateCategoryName(date=DateTime.now(tzinfo=msia_tz), element='year');
for rk in data:
row = data[rk];
if rk not in missing_dates:
missing_dates[rk] = [];
dates = groupDates(params={'states': states, 'state_by': state_by}, data=row);
for date in dates['missing']:
end_date_of_month = DateTime.getDaysAgo(days_to_crawl=1, datefrom=DateTime.getNextMonth(DateTime.convertDateTimeFromString(date)));
day_diff = DateTime.getDifferenceBetweenDuration([today, end_date_of_month]);
if day_diff >= 0:
date_str = DateTime.toString(today);
else:
date_str = DateTime.toString(end_date_of_month);
if date_str not in dates['crawled']:
missing_dates[rk].append(date_str);
# Logger.v('day_diff', day_diff);
# Logger.v('date', DateTime.getDaysAgo(days_to_crawl=1, datefrom=DateTime.getNextMonth(DateTime.convertDateTimeFromString(ed))));
missing_dates[rk] = sorted(list(set(missing_dates[rk])), reverse=True);
return missing_dates;
def generateTemplate(params):
result = {};
report_keys = fn.getNestedElement(params, 'keys.report', ['procurement', 'budget']);
first_date = fn.getNestedElement(params, 'first_date');
last_date = fn.getNestedElement(params, 'last_date');
state_by = fn.getNestedElement(params, 'state_by');
states = fn.getNestedElement(params, 'states');
today = DateTime.now(tzinfo=msia_tz); # date only
for rk in report_keys:
if rk not in result:
result[rk] = {};
for date in DateTime.getBetween([first_date, last_date], element='date')['order']:
end_date_of_month = DateTime.getDaysAgo(days_to_crawl=1, datefrom=DateTime.getNextMonth(DateTime.convertDateTimeFromString(date)));
year_month = date[:7];
day_diff = DateTime.getDifferenceBetweenDuration([today, end_date_of_month]);
if day_diff >= 0:
date_str = DateTime.toString(today);
else:
date_str = DateTime.toString(end_date_of_month);
if date_str not in result[rk]:
result[rk][date_str] = {};
result[rk][date_str].update({
'date': date_str,
})
for idx in range(0, len(states)):
state = states[idx][state_by];
result[rk][date_str].update({
state: 0,
});
return result;
def check(params):
global msia_tz, date_retrieve_limit, date_count, collection_name
dbManager = SharedMemoryManager.getInstance()
db = dbManager.query()
today = DateTime.now(tzinfo=msia_tz)
start_date = DateTime.getDaysAgo(date_retrieve_limit, datefrom=today)
durations = DateTime.getBetween([start_date, today],
element='date',
offset=24)['order']
# offset 24 to include today
Logger.v('durations', durations)
data = db[collection_name].aggregate([{
'$match': {
'state_updated_at': {
'$in': durations
},
'facility_updated_at': {
'$in': durations
}
}
}, {
'$project': {
'_id': 0,
'inserted_at': 0,
'updated_at': 0
}
}])
data = list(data)
Logger.v('Total stock issue integrity in', date_retrieve_limit, 'days:',
len(data))
state_data = {}
facility_data_by_state = {}
for idx in range(0, len(data)):
row = data[idx]
state_code = fn.getNestedElement(row, 'state_code')
if state_code not in facility_data_by_state:
facility_data_by_state[state_code] = {}
state_data = addIntegrityData(data={
'row': row,
'to_update': state_data
},
category='state')
facility_data_by_state[state_code] = addIntegrityData(
data={
'row': row,
'to_update': facility_data_by_state[state_code]
},
category='facility')
if date_count > date_retrieve_limit: # limit loop data/ show data in N days
break
date_count = 0
# reset to 0th day
return {
'state': state_data,
'state_facility': facility_data_by_state,
}
def getIntegrity(params, data):
dbManager = SharedMemoryManager.getInstance()
db = dbManager.query()
check_data = fn.getNestedElement(data, 'check_data')
facility = ModelFacility.getActiveFacility()
filter_key = fn.getNestedElement(params, 'filter_key')
durations = fn.getNestedElement(params, 'durations')
result = fn.getNestedElement(data, 'to_update')
state_data = fn.getNestedElement(data, 'state')
facility_data_by_state = fn.getNestedElement(data, 'facility')
data_list = getFacilityByState(params=params, data=check_data)
for key in data_list:
row = fn.getNestedElement(data_list, key)
count = getTotalCount(params={
'filter_key': filter_key,
'key': key
},
data={
'row': row,
'facility': facility
})
obj_ = {
'id': fn.convertToSnakecase(fn.getNestedElement(row, 'id')),
'name': fn.getNestedElement(row, 'name'),
'code': fn.getNestedElement(row, 'code'),
'data': [],
}
for idx in range(len(durations) - 1, -1, -1):
date = durations[idx]
previous_date = DateTime.toString(
DateTime.getDaysAgo(1, datefrom=date))
# Logger.v('date', date, 'previous_date', previous_date);
if filter_key:
date_count = fn.getNestedElement(
facility_data_by_state,
'{0}.{1}.{2}'.format(filter_key, key, date), 0)
if not date_count:
date_count = 0
else:
date_count = 0
# do not include those positive, count missing facility quantity only
# date_count = fn.getNestedElement(state_data, '{0}.{1}'.format(key, date), 0);
if filter_key:
val = date_count - count
else:
val = 0
obj_['data'].append({
previous_date:
val, # negative value is missing, 0 mean complete, positive value is not found from user upload facility
})
if filter_key:
# Logger.v('recursive end')
pass
else:
obj_['facility'] = []
obj_['facility'] = getIntegrity(params={
'filter_key': key,
'durations': durations,
},
data={
'state': state_data,
'facility':
facility_data_by_state,
'to_update': obj_['facility'],
'check_data': check_data,
})
result.append(obj_)
# Logger.v('result', result)
return result
def generateCrawlParam(params):
Debug = DebugManager.DebugManager()
Debug.start()
global pass_month_quantity
dbManager = SharedMemoryManager.getInstance()
db = dbManager.query()
crawl_params = {}
limit_for_test = 10
report_keys = fn.getNestedElement(params, 'keys.report',
['budget', 'procurement'])
interval = fn.getNestedElement(params, 'interval', 1)
filter_facility_code = fn.getNestedElement(params, 'filter.facility_code',
True)
check_empty = fn.getNestedElement(params, 'schedule_params.check_empty',
False)
today = fn.getNestedElement(
params, 'schedule_params.today',
DateTime.toString(DateTime.now(tzinfo=msia_tz)))
# Logger.v('filter_facility_code', filter_facility_code);
if check_empty:
# past_dates = DateTime.getPastDate(count=pass_month_quantity, duration=interval);
past_dates = DateTime.getPastDate(
count=pass_month_quantity,
duration=interval,
end=DateTime.convertDateTimeFromString(today))
# Logger.v('past_dates', past_dates);
# exit();
else:
past_dates = DateTime.getPastDate(count=pass_month_quantity,
duration=interval)
# Logger.v('past_dates', past_dates);
state_codes = retrieveOption(collection_name='state',
show_keys=['state_code'],
hide_keys=['_id'])
state_code = extractListByKey(data=state_codes, key='state_code')
facility_codes = retrieveOption(collection_name='facility',
show_keys=['facility_code'],
hide_keys=['_id'])
facility_code = extractListByKey(data=facility_codes, key='facility_code')
for key in report_keys:
# Logger.v('collection', key, past_dates[0]);
Debug.trace()
if key not in crawl_params:
crawl_params[key] = []
mongo_data = list(db[key].find({}, {}))
if len(mongo_data) == 0:
dates = past_dates[0][:]
else:
dates = past_dates[0][:1]
year = extractYear(data=dates)
# Logger.v('year', year);
# Logger.v('filter_facility_code', filter_facility_code);
if key == 'budget':
if not filter_facility_code:
iteration = 0
total = len(year) * len(state_code)
# fn.printProgressBar(iteration=iteration, total=total);
for y in year:
for sc in state_code:
obj_ = {
'financial_year': y,
'state_code': sc,
'page_type': key,
'upid': '_'.join([sc, y]),
'url': api_links[key].format(sc, y, ''),
'start_date': today,
'end_date': today,
}
if obj_ not in crawl_params[key]:
crawl_params[key].append(obj_)
# Logger.v('len(crawl_param])', len(crawl_params[key]));
iteration += 1
# fn.printProgressBar(iteration=iteration, total=total);
else:
iteration = 0
total = len(year) * len(state_code) * len(
facility_code[:limit_for_test])
# fn.printProgressBar(iteration=iteration, total=total);
for y in year:
for sc in state_code:
for fc in facility_code[:limit_for_test]:
obj_ = {
'financial_year': y,
'state_code': sc,
'page_type': key,
'upid': '_'.join([sc, y, fc]),
'facility_code': fc,
'url': api_links[key].format(sc, y, fc),
'start_date': today,
'end_date': today,
}
if obj_ not in crawl_params[key]:
crawl_params[key].append(obj_)
# Logger.v('len(crawl_param])', len(crawl_params[key]));
iteration += 1
# fn.printProgressBar(iteration=iteration, total=total);
elif key == 'procurement':
if not filter_facility_code:
for past_duration in dates:
start_date = DateTime.toString(
DateTime.getDaysAgo(days_to_crawl=-1,
datefrom=past_duration[0]))
end_date = DateTime.toString(
DateTime.getDaysAgo(days_to_crawl=1,
datefrom=past_duration[1]))
for sc in state_code:
obj_ = {
'state_code':
sc,
'start_date':
start_date,
'end_date':
end_date,
'page_type':
key,
'upid':
'_'.join([sc, start_date, end_date]),
'url':
api_links[key].format(sc,
start_date.replace('-', ''),
end_date.replace('-', ''),
''),
}
if obj_ not in crawl_params[key]:
crawl_params[key].append(obj_)
# Logger.v('len(crawl_param])', len(crawl_params[key]));
else:
for past_duration in dates:
start_date = DateTime.toString(
DateTime.getDaysAgo(days_to_crawl=-1,
datefrom=past_duration[0]))
end_date = DateTime.toString(
DateTime.getDaysAgo(days_to_crawl=1,
datefrom=past_duration[1]))
for sc in state_code:
for fc in facility_code[:limit_for_test]:
obj_ = {
'state_code':
sc,
'start_date':
start_date,
'end_date':
end_date,
'page_type':
key,
'facility_code':
fc,
'upid':
'_'.join([sc, start_date, end_date, fc]),
'url':
api_links[key].format(
sc, start_date.replace('-', ''),
end_date.replace('-', ''), fc)
}
if obj_ not in crawl_params[key]:
crawl_params[key].append(obj_)
# Logger.v('len(crawl_param])', len(crawl_params[key]));
for c in crawl_params:
# Logger.v('crawl_params', c, len(crawl_params[c]));
fn.writeExcelFile(filename='{0}/{1}'.format(test_folder, c),
data=crawl_params[c])
Logger.v('crawl_params', len(crawl_params))
Debug.show('Generate Crawl Params')
return crawl_params