def stockIntegrity(params, data):
Debug = DebugManager.DebugManager()
Debug.start()
global msia_tz, date_retrieve_limit
result = []
today = DateTime.now(tzinfo=msia_tz)
start_date = DateTime.getDaysAgo(date_retrieve_limit, datefrom=today)
durations = DateTime.getBetween([start_date, today],
element='date',
offset=24)['order']
# offset 24 hour to include today
state_data = fn.getNestedElement(data, 'state')
facility_data_by_state = fn.getNestedElement(data, 'state_facility')
check_data = combinedFacilityList(data=facility_data_by_state)
result = getIntegrity(params={
'durations': durations,
},
data={
'facility': facility_data_by_state,
'state': state_data,
'to_update': result,
'check_data': check_data,
})
updateStateData(result)
result = list(sorted(result, key=lambda k: k['name'], reverse=False))
Debug.end()
Debug.show('Model.Structure.stockIntegrity')
return result
def updateDropdownOptions(params):
option_keys = fn.getNestedElement(params, 'keys.option', ['state'])
today = fn.getNestedElement(
params, 'schedule_params.today',
DateTime.toString(DateTime.now(tzinfo=msia_tz)))
data = {}
crawled_data = {}
# crawl from API URL (get options from API)
# for key in keys:
# url = api_links[key];
# # url = generateUrl(api_links[key]);
# response = requests.get(url);
# json_response = json.loads(response.text);
# Logger.v('json_response', json_response);
# crawled_data[key] = json_response;
# Logger.v('Crawled', url);
# Logger.v('Done crawling.');
# save(data);
# read from file
for key in option_keys:
filename = api_files[key]
crawled_data[key] = File.readJson(filename)
# convert key to snakecase, value to lower
for key in crawled_data:
if key not in data:
data[key] = []
for idx in range(0, len(crawled_data[key])):
row = crawled_data[key][idx]
obj_ = {}
for row_key in row:
row_value = row[row_key]
new_key = fn.camelToSnakecase(str=row_key)
if type(row_value) == str:
new_value = row_value.lower()
elif row_value is None:
new_value = 'null'
else:
new_value = row_value
obj_[new_key] = new_value
data[key].append(obj_)
for key in data:
folder_path = '/'.join([crawl_folder, key])
if not os.path.exists(folder_path):
os.makedirs(folder_path)
filename = '{0}/{1}'.format(folder_path, today)
Logger.v('Saving', filename)
fn.writeJSONFile(filename='{0}.json'.format(filename), data=data[key])
for key in option_keys:
directory = '/'.join([crawl_folder, key])
raw = File.readLatestFile(directory=directory)
refresh_collection = refreshIsRequired(data=raw, collection_name=key)
if refresh_collection:
refreshCollection(data=raw, collection_name=key)
Logger.v('refreshed', key)
def check(params):
global msia_tz, date_retrieve_limit, date_count, collection_name
dbManager = SharedMemoryManager.getInstance()
db = dbManager.query()
today = DateTime.now(tzinfo=msia_tz)
start_date = DateTime.getDaysAgo(date_retrieve_limit, datefrom=today)
durations = DateTime.getBetween([start_date, today],
element='date',
offset=24)['order']
# offset 24 to include today
Logger.v('durations', durations)
data = db[collection_name].aggregate([{
'$match': {
'state_updated_at': {
'$in': durations
},
'facility_updated_at': {
'$in': durations
}
}
}, {
'$project': {
'_id': 0,
'inserted_at': 0,
'updated_at': 0
}
}])
data = list(data)
Logger.v('Total stock issue integrity in', date_retrieve_limit, 'days:',
len(data))
state_data = {}
facility_data_by_state = {}
for idx in range(0, len(data)):
row = data[idx]
state_code = fn.getNestedElement(row, 'state_code')
if state_code not in facility_data_by_state:
facility_data_by_state[state_code] = {}
state_data = addIntegrityData(data={
'row': row,
'to_update': state_data
},
category='state')
facility_data_by_state[state_code] = addIntegrityData(
data={
'row': row,
'to_update': facility_data_by_state[state_code]
},
category='facility')
if date_count > date_retrieve_limit: # limit loop data/ show data in N days
break
date_count = 0
# reset to 0th day
return {
'state': state_data,
'state_facility': facility_data_by_state,
}
def recordTime(key):
global global_text
new_key = key.replace('end', 'start')
if new_key not in global_text:
global_text[new_key] = DateTime.now(tzinfo=msia_tz)
if new_key in global_text and not key == new_key:
global_text[key] = DateTime.now(tzinfo=msia_tz)
def getMonthRange(params):
start_month = fn.getNestedElement(params, 'start_month');
number_of_month = fn.getNestedElement(params, 'number_of_month', 1);
month_range = [start_month];
new_month = '{0}-01'.format(start_month);
for month_count in range(0, number_of_month - 1): # included start_month, so total month less 1
new_month = DateTime.getNextMonth(DateTime.convertDateTimeFromString(new_month));
year_month = DateTime.getDateCategoryName(new_month, element='year_month_digit');
month_range.append(year_month);
return month_range;
def checkEmpty(params):
global global_check_data;
dbManager = SharedMemoryManager.getInstance();
db = dbManager.query();
custom_params = copy.deepcopy(params);
report_keys = fn.getNestedElement(params, 'keys.report', ['procurement', 'budget']);
interval = fn.getNestedElement(params, 'interval', 1);
past_dates = DateTime.getPastDate(count=12, duration=interval); # check previous 12 month data
year = Crawl.extractYear(data=past_dates[0]);
first_date = past_dates[0][-1][0];
last_date = past_dates[0][0][1];
# Logger.v('first_date', first_date, 'last_date', last_date);
state_by = 'state_code';
states = list(db['state'].find({},{'_id': 0, state_by: 1}));
result = {};
datetime = DateTime.toString(DateTime.now(tzinfo=msia_tz), date_format='%Y-%m-%d-%H-%M-%S');
custom_params['first_date'] = first_date;
custom_params['last_date'] = last_date;
custom_params['state_by'] = state_by;
custom_params['states'] = states;
temp_result = generateTemplate(params=custom_params);
for rk in report_keys:
if rk not in global_check_data:
global_check_data[rk] = [];
for y in year:
root_path = '{0}/{1}/year_{2}'.format(crawl_folder, rk, y);
openDir(root_path, rk);
for gcd in global_check_data[rk]:
date = gcd.split('_')[0];
state = gcd.split('_')[1];
if DateTime.inrange(date, [first_date, last_date]):
try:
temp_result[rk][date][state] += 1;
except Exception as e:
# Logger.v('Main.checkEmpty:', e);
pass;
for rk in temp_result:
if rk not in result:
result[rk] = [];
for date in temp_result[rk]:
result[rk].append(temp_result[rk][date]);
filename = '{0}/{1}_check_moh_empty'.format(test_folder, rk);
# filename = 'tests/{0}_{1}_check_moh_empty'.format(rk, datetime);
fn.writeExcelFile(filename=filename, data=result[rk]);
global_check_data = {};
return result;
def getMissingDates(data):
dbManager = SharedMemoryManager.getInstance();
db = dbManager.query();
missing_dates = {};
today = DateTime.now(tzinfo=msia_tz); # date only
state_by = 'state_code';
states = list(db['state'].find({},{'_id': 0, state_by: 1}));
current_year = DateTime.getDateCategoryName(date=DateTime.now(tzinfo=msia_tz), element='year');
for rk in data:
row = data[rk];
if rk not in missing_dates:
missing_dates[rk] = [];
dates = groupDates(params={'states': states, 'state_by': state_by}, data=row);
for date in dates['missing']:
end_date_of_month = DateTime.getDaysAgo(days_to_crawl=1, datefrom=DateTime.getNextMonth(DateTime.convertDateTimeFromString(date)));
day_diff = DateTime.getDifferenceBetweenDuration([today, end_date_of_month]);
if day_diff >= 0:
date_str = DateTime.toString(today);
else:
date_str = DateTime.toString(end_date_of_month);
if date_str not in dates['crawled']:
missing_dates[rk].append(date_str);
# Logger.v('day_diff', day_diff);
# Logger.v('date', DateTime.getDaysAgo(days_to_crawl=1, datefrom=DateTime.getNextMonth(DateTime.convertDateTimeFromString(ed))));
missing_dates[rk] = sorted(list(set(missing_dates[rk])), reverse=True);
return missing_dates;
def update(data):
global msia_tz, column_keymap, collection_name
dbManager = SharedMemoryManager.getInstance()
db = dbManager.query()
state_facility_code = '_'.join(
[str(data['state']), str(data['facility_code'])])
if state_facility_code not in list(set(unique_facility)):
state_name = fn.getNestedElement(data, 'state')
state_code = fn.getNestedElement(data, 'state')
facility_name = fn.getNestedElement(data, 'facility_name')
facility_code = fn.getNestedElement(data, 'facility_code')
date = fn.getNestedElement(data, 'date')
date_string = DateTime.toString(date)
values = {
'state_name': state_name,
'state_code': state_code,
'facility_name': facility_name,
'facility_code': facility_code,
'state_updated_at': date_string,
'facility_updated_at': date_string,
'date': date_string,
}
dbManager.addBulkInsert(collection_name, values, batch=True)
unique_facility.append(state_facility_code)
dbManager.executeBulkOperations(collection_name)
def getBackdateList(params):
dbManager = SharedMemoryManager.getInstance()
db = dbManager.query()
dates = []
for idx in range(0, date_retrieve_limit + 1): # 7 days backward + 1 today
if idx == 0:
collection_name = 'stock_latest'
else:
collection_name = 'stock_{0}'.format(idx)
# Logger.v('collection_name', collection_name);
data = list(db[collection_name].find({}, {
'_id': 0,
'date': 1
}).limit(1))
if data:
date = DateTime.toString(data[0]['date'])
# Logger.v('date', date);
dates.append(date)
# Logger.v('data', data);
# Logger.v('dates', sorted(list(set(dates)), reverse=True));
result = {
'date': sorted(list(set(dates)), reverse=True)
}
return result
def save(params, chunk, chunks_info):
global latest_collection_name, history_collection_name
data = File.readChunkData(chunk)
dbManager = SharedMemoryManager.getInstance()
db = dbManager.query()
current_index = fn.getNestedElement(chunks_info, 'current', 0)
total_index = fn.getNestedElement(chunks_info, 'total', len(data))
date = fn.getNestedElement(params, 'date')
datetime = DateTime.convertDateTimeFromString(date)
total_length = len(data)
queue_info = chunks_info['queue']
# Logger.v('Running Index:', chunks_info['queue']['running']);
chunks_info['queue']['current'] += 1
# Logger.v('Saving from... {0}/{1}, current package: {2}'.format(current_index, total_index, total_length) );
fn.printProgressBar(queue_info['current'], queue_info['total'],
'Processing Chunk Insertion')
for idx in range(0, total_length):
# insert stock_latest
row = data[idx]
obj_ = transformToLowercase(data=row, datetime=datetime)
ModelStockIntegrity.update(data=obj_)
dbManager.addBulkInsert(latest_collection_name, obj_, batch=True)
# dbManager.addBulkInsert(history_collection_name, obj_, batch=True); # temporary off (need 7 day data only)
# insert items
# d = data[idx];
ModelItem.saveItem(row)
# fn.printProgressBar(current_index+idx, total_index, 'Processing Item Insertion');
#ensure all data is save properly
# dbManager.executeBulkOperations(history_collection_name); # temporary off (need 7 day data only)
dbManager.executeBulkOperations(latest_collection_name)
return chunks_info
def save(params, chunk, chunks_info):
global collection_name, column_keymap;
upload_date = fn.getNestedElement(params, 'date');
data = File.readChunkData(chunk);
dbManager = SharedMemoryManager.getInstance();
db = dbManager.query();
current_index = fn.getNestedElement(chunks_info, 'current', 0);
total_index = fn.getNestedElement(chunks_info, 'total', len(data));
total_length = len(data);
queue_info = chunks_info['queue']
# Logger.v('Running Index:', chunks_info['queue']['running']);
chunks_info['queue']['current']+=1;
# Logger.v('Saving from... {0}/{1}, current package: {2}'.format(current_index, total_index, total_length) );
fn.printProgressBar(queue_info['current'], queue_info['total'], 'Processing Chunk Insertion');
for idx in range(0, total_length):
row = data[idx];
# Logger.v('row', row);
obj_ = transformToLowercase(row);
date_only = obj_['approved_date'].split(' ')[0];
# Logger.v('date_only', date_only);
obj_.update({
'approved_year_month': DateTime.getDateCategoryName(date=date_only, element='year_month_digit'),
'upload_date': upload_date,
});
dbManager.addBulkInsert(collection_name, obj_, batch=True);
ModelSIIntegrity.update(data=obj_);
retrieveIssueOption(obj_);
#ensure all data is save properly
dbManager.executeBulkOperations(collection_name);
return chunks_info;
def generateTemplate(params):
result = {};
report_keys = fn.getNestedElement(params, 'keys.report', ['procurement', 'budget']);
first_date = fn.getNestedElement(params, 'first_date');
last_date = fn.getNestedElement(params, 'last_date');
state_by = fn.getNestedElement(params, 'state_by');
states = fn.getNestedElement(params, 'states');
today = DateTime.now(tzinfo=msia_tz); # date only
for rk in report_keys:
if rk not in result:
result[rk] = {};
for date in DateTime.getBetween([first_date, last_date], element='date')['order']:
end_date_of_month = DateTime.getDaysAgo(days_to_crawl=1, datefrom=DateTime.getNextMonth(DateTime.convertDateTimeFromString(date)));
year_month = date[:7];
day_diff = DateTime.getDifferenceBetweenDuration([today, end_date_of_month]);
if day_diff >= 0:
date_str = DateTime.toString(today);
else:
date_str = DateTime.toString(end_date_of_month);
if date_str not in result[rk]:
result[rk][date_str] = {};
result[rk][date_str].update({
'date': date_str,
})
for idx in range(0, len(states)):
state = states[idx][state_by];
result[rk][date_str].update({
state: 0,
});
return result;
def getCollectionName(params):
global latest_collection_name
latest_collection_name = 'stock_latest'
# set default;
dbManager = SharedMemoryManager.getInstance()
db = dbManager.query()
data = list(db[latest_collection_name].find({}, {
'_id': 0,
'date': 1
}).limit(1))
if data:
latest_date_string = DateTime.toString(data[0]['date'])
latest_date = DateTime.convertDateTimeFromString(latest_date_string)
date_string = fn.getNestedElement(params, 'date', None)
if date_string:
date = DateTime.convertDateTimeFromString(date_string)
different = latest_date - date
day_diff = math.floor(different.total_seconds() / float(86400))
if day_diff > 0:
latest_collection_name = 'stock_{0}'.format(day_diff)
def extractYear(data):
year = []
for past_duration in data:
# Logger.v('len(past_duration)', len(past_duration))
for idx in range(len(past_duration) - 1, -1, -1):
pd = past_duration[idx]
# Logger.v('pd', pd);
y = DateTime.getDateCategoryName(date=pd, element='year', offset=8)
if y not in year:
year.append(y)
return year
def run(params):
Mail.send(
'{0} Crawl - Store starts'.format(
DateTime.getReadableDate(DateTime.now())),
'Start at: {0}'.format(DateTime.now(tzinfo=msia_tz)))
recordTime(key='create_schedule_start_time')
Debug = DebugManager.DebugManager()
Debug.start()
start_crawl = fn.getNestedElement(params, 'schedule_params.start_crawl',
False)
check_empty = fn.getNestedElement(params, 'schedule_params.check_empty',
False)
Logger.v('Creating schedule:')
updateDropdownOptions(params=params)
crawl_params = generateCrawlParam(params)
Debug.trace('Generate Crawl Params')
createSchedules({'pages': crawl_params})
Debug.trace('Create Schedule')
if start_crawl:
Crawl.start(params)
Debug.trace('crawling')
recordTime(key='create_schedule_end_time')
Debug.show('Run')
def getQuery(params):
duration = fn.getNestedElement(params, 'duration',
['2020-03-30', '2020-03-30'])
item_codes = fn.getNestedElement(params, 'item_codes', [])
item_desc = fn.getNestedElement(params, 'item_desc')
group_by_list = fn.getNestedElement(params, 'group_by', [])
facility_group = fn.getNestedElement(params, 'facility_group', [])
dates = DateTime.getBetween(duration, element='date')['order']
query = {}
if item_desc: # TEST wildcard search
query.update({'item_desc': {
'$regex': item_desc.lower()
}})
if item_codes:
query.update({
'item_code': {
'$in': [c.lower() for c in item_codes]
},
})
if facility_group:
facility_code_list = ModelFacility.getFacilityCodeList(
facility_group=facility_group)
query.update({
'facility_code': {
'$in': facility_code_list
},
})
for gbl in group_by_list:
gbl_id = gbl['id']
gbl_value = gbl['value']
val = gbl_value
if type(val) == str:
val = val.lower()
if gbl_id == 'state':
val = val.replace('_', ' ')
query.update({
query_key[gbl_id]: val,
})
Logger.v('query', query)
return query
def getIntegrity(params, data):
dbManager = SharedMemoryManager.getInstance()
db = dbManager.query()
check_data = fn.getNestedElement(data, 'check_data')
facility = ModelFacility.getActiveFacility()
filter_key = fn.getNestedElement(params, 'filter_key')
durations = fn.getNestedElement(params, 'durations')
result = fn.getNestedElement(data, 'to_update')
state_data = fn.getNestedElement(data, 'state')
facility_data_by_state = fn.getNestedElement(data, 'facility')
data_list = getFacilityByState(params=params, data=check_data)
for key in data_list:
row = fn.getNestedElement(data_list, key)
count = getTotalCount(params={
'filter_key': filter_key,
'key': key
},
data={
'row': row,
'facility': facility
})
obj_ = {
'id': fn.convertToSnakecase(fn.getNestedElement(row, 'id')),
'name': fn.getNestedElement(row, 'name'),
'code': fn.getNestedElement(row, 'code'),
'data': [],
}
for idx in range(len(durations) - 1, -1, -1):
date = durations[idx]
previous_date = DateTime.toString(
DateTime.getDaysAgo(1, datefrom=date))
# Logger.v('date', date, 'previous_date', previous_date);
if filter_key:
date_count = fn.getNestedElement(
facility_data_by_state,
'{0}.{1}.{2}'.format(filter_key, key, date), 0)
if not date_count:
date_count = 0
else:
date_count = 0
# do not include those positive, count missing facility quantity only
# date_count = fn.getNestedElement(state_data, '{0}.{1}'.format(key, date), 0);
if filter_key:
val = date_count - count
else:
val = 0
obj_['data'].append({
previous_date:
val, # negative value is missing, 0 mean complete, positive value is not found from user upload facility
})
if filter_key:
# Logger.v('recursive end')
pass
else:
obj_['facility'] = []
obj_['facility'] = getIntegrity(params={
'filter_key': key,
'durations': durations,
},
data={
'state': state_data,
'facility':
facility_data_by_state,
'to_update': obj_['facility'],
'check_data': check_data,
})
result.append(obj_)
# Logger.v('result', result)
return result
def generateCrawlParam(params):
Debug = DebugManager.DebugManager()
Debug.start()
global pass_month_quantity
dbManager = SharedMemoryManager.getInstance()
db = dbManager.query()
crawl_params = {}
limit_for_test = 10
report_keys = fn.getNestedElement(params, 'keys.report',
['budget', 'procurement'])
interval = fn.getNestedElement(params, 'interval', 1)
filter_facility_code = fn.getNestedElement(params, 'filter.facility_code',
True)
check_empty = fn.getNestedElement(params, 'schedule_params.check_empty',
False)
today = fn.getNestedElement(
params, 'schedule_params.today',
DateTime.toString(DateTime.now(tzinfo=msia_tz)))
# Logger.v('filter_facility_code', filter_facility_code);
if check_empty:
# past_dates = DateTime.getPastDate(count=pass_month_quantity, duration=interval);
past_dates = DateTime.getPastDate(
count=pass_month_quantity,
duration=interval,
end=DateTime.convertDateTimeFromString(today))
# Logger.v('past_dates', past_dates);
# exit();
else:
past_dates = DateTime.getPastDate(count=pass_month_quantity,
duration=interval)
# Logger.v('past_dates', past_dates);
state_codes = retrieveOption(collection_name='state',
show_keys=['state_code'],
hide_keys=['_id'])
state_code = extractListByKey(data=state_codes, key='state_code')
facility_codes = retrieveOption(collection_name='facility',
show_keys=['facility_code'],
hide_keys=['_id'])
facility_code = extractListByKey(data=facility_codes, key='facility_code')
for key in report_keys:
# Logger.v('collection', key, past_dates[0]);
Debug.trace()
if key not in crawl_params:
crawl_params[key] = []
mongo_data = list(db[key].find({}, {}))
if len(mongo_data) == 0:
dates = past_dates[0][:]
else:
dates = past_dates[0][:1]
year = extractYear(data=dates)
# Logger.v('year', year);
# Logger.v('filter_facility_code', filter_facility_code);
if key == 'budget':
if not filter_facility_code:
iteration = 0
total = len(year) * len(state_code)
# fn.printProgressBar(iteration=iteration, total=total);
for y in year:
for sc in state_code:
obj_ = {
'financial_year': y,
'state_code': sc,
'page_type': key,
'upid': '_'.join([sc, y]),
'url': api_links[key].format(sc, y, ''),
'start_date': today,
'end_date': today,
}
if obj_ not in crawl_params[key]:
crawl_params[key].append(obj_)
# Logger.v('len(crawl_param])', len(crawl_params[key]));
iteration += 1
# fn.printProgressBar(iteration=iteration, total=total);
else:
iteration = 0
total = len(year) * len(state_code) * len(
facility_code[:limit_for_test])
# fn.printProgressBar(iteration=iteration, total=total);
for y in year:
for sc in state_code:
for fc in facility_code[:limit_for_test]:
obj_ = {
'financial_year': y,
'state_code': sc,
'page_type': key,
'upid': '_'.join([sc, y, fc]),
'facility_code': fc,
'url': api_links[key].format(sc, y, fc),
'start_date': today,
'end_date': today,
}
if obj_ not in crawl_params[key]:
crawl_params[key].append(obj_)
# Logger.v('len(crawl_param])', len(crawl_params[key]));
iteration += 1
# fn.printProgressBar(iteration=iteration, total=total);
elif key == 'procurement':
if not filter_facility_code:
for past_duration in dates:
start_date = DateTime.toString(
DateTime.getDaysAgo(days_to_crawl=-1,
datefrom=past_duration[0]))
end_date = DateTime.toString(
DateTime.getDaysAgo(days_to_crawl=1,
datefrom=past_duration[1]))
for sc in state_code:
obj_ = {
'state_code':
sc,
'start_date':
start_date,
'end_date':
end_date,
'page_type':
key,
'upid':
'_'.join([sc, start_date, end_date]),
'url':
api_links[key].format(sc,
start_date.replace('-', ''),
end_date.replace('-', ''),
''),
}
if obj_ not in crawl_params[key]:
crawl_params[key].append(obj_)
# Logger.v('len(crawl_param])', len(crawl_params[key]));
else:
for past_duration in dates:
start_date = DateTime.toString(
DateTime.getDaysAgo(days_to_crawl=-1,
datefrom=past_duration[0]))
end_date = DateTime.toString(
DateTime.getDaysAgo(days_to_crawl=1,
datefrom=past_duration[1]))
for sc in state_code:
for fc in facility_code[:limit_for_test]:
obj_ = {
'state_code':
sc,
'start_date':
start_date,
'end_date':
end_date,
'page_type':
key,
'facility_code':
fc,
'upid':
'_'.join([sc, start_date, end_date, fc]),
'url':
api_links[key].format(
sc, start_date.replace('-', ''),
end_date.replace('-', ''), fc)
}
if obj_ not in crawl_params[key]:
crawl_params[key].append(obj_)
# Logger.v('len(crawl_param])', len(crawl_params[key]));
for c in crawl_params:
# Logger.v('crawl_params', c, len(crawl_params[c]));
fn.writeExcelFile(filename='{0}/{1}'.format(test_folder, c),
data=crawl_params[c])
Logger.v('crawl_params', len(crawl_params))
Debug.show('Generate Crawl Params')
return crawl_params
#!/usr/bin/python3
from Crawl import Main as Crawl
from lib import fn
from lib import DateTime
try:
start_time = DateTime.now()
fn.writeTestFile(
filename='cron_start_{0}'.format(DateTime.now()),
data='successful cron tab, cron time:{0}'.format(start_time))
Crawl.run({
'process': ['schedule', 'crawl',
'update'], # 'schedule', 'crawl', 'update'
'schedule_params': {
'start_crawl': False,
'check_empty': True,
},
'interval': 1, # months between start date - end date
'keys': {
'option': ['state', 'ptj', 'facility', 'facility_type'],
'report': ['budget', 'procurement'], # 'budget', 'procurement'
},
'filter': {
'facility_code': False,
},
})
end_time = DateTime.now()
print('end_time', end_time)
fn.writeTestFile(
filename='cron_end_{0}'.format(end_time),
def generateExcelStructure(params, data):
result = []
metadata = {}
group_by_key = fn.getNestedElement(params, 'group_by_key')
column_to_add = {
'all': [
'state_name', 'drug_nondrug_name', 'code', 'packaging',
'quantity_by_month', 'sku'
],
'state': [
'facility_name', 'drug_nondrug_name', 'code', 'packaging',
'quantity_by_month', 'sku'
],
'facility': [
'requester_group_name', 'drug_nondrug_name', 'code', 'packaging',
'quantity_by_month', 'sku'
],
'requester': [
'drug_nondrug_name', 'code', 'packaging', 'batch_no',
'expiry_date', 'quantity_by_month', 'sku'
],
}
name_mapping = {
'state_name': 'state',
'facility_name': 'facility',
'requester_group_name': 'requester group',
'code': 'item code',
'drug_nondrug_name': 'drug/non-drug name',
'sku': 'sku',
'quantity': 'quantity',
'batch_no': 'batch no',
'expiry_date': 'expiry date',
'packaging': 'packaging description',
}
new_data = []
# Logger.v('asd', data);
for idx in range(0, len(data)):
row = data[idx]
for drug_code in row['summary']['detail']:
new_data += row['summary']['detail'][drug_code]
for idx in range(0, len(new_data)):
row = new_data[idx]
# Logger.v('row', row)
state_name = row['id'].split('|')[0].replace('_', ' ')
obj_ = {}
for col in column_to_add[group_by_key]:
if col == 'state_name':
obj_.update({
name_mapping[col]: state_name,
})
elif col == 'quantity_by_month':
for month in row[col]:
date = '{0}-01'.format(month)
month_string = DateTime.getDateCategoryName(
date=date, element='month')[:3].lower()
# Logger.v('month_string', month_string);
key_name = 'quantity ({0})'.format(month_string)
obj_.update({
key_name: row[col][month],
})
else:
obj_.update({
name_mapping[col]: row[col],
})
# Logger.v('group_by_key', group_by_key);
result.append(obj_)
metadata = generateExportMeta(params=params,
data={
'state_name': state_name,
'row': row
})
# Logger.v('result', result);
# Logger.v('metadata', metadata);
return {
'data': result,
'metadata': metadata,
}