def generateChildren(params, data): report_name = Report.getReportName(params) min_purchase_amount = fn.getNestedElement(params, 'filter.min_purchase_amount', 0) if type(min_purchase_amount) == str and min_purchase_amount: min_purchase_amount = int(min_purchase_amount) children = [] for d in data: # Logger.v('d', d); obj_ = {} # Logger.v('global_children_key',global_children_key) # Logger.v('report_name',report_name) for gck in global_children_key[report_name]: # Logger.v('gck', gck) value = fn.getNestedElement(d, gck) if value or value == 0: obj_[gck] = value else: if report_name == 'budget': # Logger.v('Report.generateChildren: {0} not found, sum up after data cleaning process.'.format(gck)); if gck == 'total_allocation': obj_[gck] = d['first_allocation'] + d[ 'additional_allocation'] elif gck == 'balance_amount': obj_[gck] = d['first_allocation'] + d[ 'additional_allocation'] - d['pending_amount'] - d[ 'liablity_amount'] - d['utilized_amount'] # Logger.v('obj_', obj_); if report_name == 'procurement': if obj_['purchase_amount'] > min_purchase_amount: children.append(obj_) else: children.append(obj_) return children
def update(data): global msia_tz, column_keymap, collection_name dbManager = SharedMemoryManager.getInstance() db = dbManager.query() state_facility_code = '_'.join( [str(data['state']), str(data['facility_code'])]) if state_facility_code not in list(set(unique_facility)): state_name = fn.getNestedElement(data, 'state') state_code = fn.getNestedElement(data, 'state') facility_name = fn.getNestedElement(data, 'facility_name') facility_code = fn.getNestedElement(data, 'facility_code') date = fn.getNestedElement(data, 'date') date_string = DateTime.toString(date) values = { 'state_name': state_name, 'state_code': state_code, 'facility_name': facility_name, 'facility_code': facility_code, 'state_updated_at': date_string, 'facility_updated_at': date_string, 'date': date_string, } dbManager.addBulkInsert(collection_name, values, batch=True) unique_facility.append(state_facility_code) dbManager.executeBulkOperations(collection_name)
def insertNthChild(params, data, is_last=False): po = fn.getNestedElement(params, 'po.po') gk = fn.getNestedElement(params, 'po.gk') naming_keymap = fn.getNestedElement(params, 'po.naming_keymap') info = data[data[po] == gk] if is_last: obj_ = { 'id': fn.convertToSnakecase(gk), 'name': info['item_desc'].values[0], 'code': info['item_code'].values[0], 'quantity': int(info['available_quantity'].values[0]), 'pku': round(float(info['pku'].values[0]), 2), } else: obj_ = { 'id': fn.convertToSnakecase(gk), 'name': info[naming_keymap[po]].unique().tolist()[0], 'code': info[po].unique().tolist()[0], } result = { 'obj_': obj_, 'info': info, } return result
def stockIntegrity(params, data): Debug = DebugManager.DebugManager() Debug.start() global msia_tz, date_retrieve_limit result = [] today = DateTime.now(tzinfo=msia_tz) start_date = DateTime.getDaysAgo(date_retrieve_limit, datefrom=today) durations = DateTime.getBetween([start_date, today], element='date', offset=24)['order'] # offset 24 hour to include today state_data = fn.getNestedElement(data, 'state') facility_data_by_state = fn.getNestedElement(data, 'state_facility') check_data = combinedFacilityList(data=facility_data_by_state) result = getIntegrity(params={ 'durations': durations, }, data={ 'facility': facility_data_by_state, 'state': state_data, 'to_update': result, 'check_data': check_data, }) updateStateData(result) result = list(sorted(result, key=lambda k: k['name'], reverse=False)) Debug.end() Debug.show('Model.Structure.stockIntegrity') return result
def generateItemDetail(params, data): result = {} report_name = Report.getReportName(params) split_uk = fn.getNestedElement(params, 'split_uk') for sm in global_group_order[report_name][:len(split_uk)]: if sm == 'facility_type': smk = 'facility' else: smk = sm if sm == 'facility_type': id_ = '{0}_type'.format(smk) name_ = '{0}_type'.format(smk) name_2 = '{0}_type'.format(smk) code_ = '{0}_type'.format(smk) else: id_ = '{0}_seq_no'.format(smk) name_ = '{0}_name'.format(smk) name_2 = '{0}_desc'.format(smk) code_ = '{0}_code'.format(smk) try: result['name'] = data[0][name_] except Exception as e: result['name'] = fn.getNestedElement( data[0], name_2, fn.getNestedElement(data[0], code_)) result['id'] = data[0][id_] result['code'] = data[0][code_] return result
def upload(params): Debug = DebugManager.DebugManager(); Debug.start(); Debug.trace('start'); dbManager = SharedMemoryManager.getInstance(); db = dbManager.query(); date = fn.getNestedElement(params, 'date'); path = fn.getNestedElement(params, 'path'); # url = fn.getNestedElement(params, 'callback_url'); # required params to handle callback_url paths, should_reset = ModelUpload.getPath(params); for idx in range(0, len(paths)): p = paths[idx]; processed_filename = File.converExcelFileToCsv(p, ignore_index=True); Logger.v('processed_filename', processed_filename); Debug.trace('convert to json : path {0}'.format( processed_filename ) ); if idx == 0 and should_reset: #reset once at the beginning Logger.v('Reset Database.'); reset(date); #reset stock_issue collection ModelSIIntegrity.reset(date); #reset stock_issue_datalog by date given File.readCsvFileInChunks(processed_filename, save, params, chunksize=chunksize); Debug.trace('uploaded to mongo.'); generateIndex(); ModelSIIntegrity.generateIndex(); Debug.trace('indexing mongo collection.'); saveIssueOption(); Debug.trace('save option to json.'); trigger_params = copy.deepcopy(params); trigger_params['result'] = 'data count: {0}'.format(params['data_count'][path]); # Logger.v('trigger_params', trigger_params); dbManager.executeBulkOperations(None); # Insert all the remaining job at once. ReportStock.triggerOnComplete(trigger_params); Debug.trace('trigger api on complete.'); Debug.end(); Debug.show('Stock.upload');
def save(params, chunk, chunks_info): global collection_name, column_keymap; upload_date = fn.getNestedElement(params, 'date'); data = File.readChunkData(chunk); dbManager = SharedMemoryManager.getInstance(); db = dbManager.query(); current_index = fn.getNestedElement(chunks_info, 'current', 0); total_index = fn.getNestedElement(chunks_info, 'total', len(data)); total_length = len(data); queue_info = chunks_info['queue'] # Logger.v('Running Index:', chunks_info['queue']['running']); chunks_info['queue']['current']+=1; # Logger.v('Saving from... {0}/{1}, current package: {2}'.format(current_index, total_index, total_length) ); fn.printProgressBar(queue_info['current'], queue_info['total'], 'Processing Chunk Insertion'); for idx in range(0, total_length): row = data[idx]; # Logger.v('row', row); obj_ = transformToLowercase(row); date_only = obj_['approved_date'].split(' ')[0]; # Logger.v('date_only', date_only); obj_.update({ 'approved_year_month': DateTime.getDateCategoryName(date=date_only, element='year_month_digit'), 'upload_date': upload_date, }); dbManager.addBulkInsert(collection_name, obj_, batch=True); ModelSIIntegrity.update(data=obj_); retrieveIssueOption(obj_); #ensure all data is save properly dbManager.executeBulkOperations(collection_name); return chunks_info;
def generateTemplate(params): result = {}; report_keys = fn.getNestedElement(params, 'keys.report', ['procurement', 'budget']); first_date = fn.getNestedElement(params, 'first_date'); last_date = fn.getNestedElement(params, 'last_date'); state_by = fn.getNestedElement(params, 'state_by'); states = fn.getNestedElement(params, 'states'); today = DateTime.now(tzinfo=msia_tz); # date only for rk in report_keys: if rk not in result: result[rk] = {}; for date in DateTime.getBetween([first_date, last_date], element='date')['order']: end_date_of_month = DateTime.getDaysAgo(days_to_crawl=1, datefrom=DateTime.getNextMonth(DateTime.convertDateTimeFromString(date))); year_month = date[:7]; day_diff = DateTime.getDifferenceBetweenDuration([today, end_date_of_month]); if day_diff >= 0: date_str = DateTime.toString(today); else: date_str = DateTime.toString(end_date_of_month); if date_str not in result[rk]: result[rk][date_str] = {}; result[rk][date_str].update({ 'date': date_str, }) for idx in range(0, len(states)): state = states[idx][state_by]; result[rk][date_str].update({ state: 0, }); return result;
def recordCrawledFile(data): global global_check_data; path = fn.getNestedElement(data, 'path'); report = fn.getNestedElement(data, 'report'); split_path = path.split('/'); file = split_path[-1]; date = file.split('.')[0]; year = date.split('-')[0]; if file == '.json': os.remove(path); Logger.v('Removed', path); indexes = { 'budget': { 'year': -4, 'state': -3, 'all_facility': -2, }, 'procurement': { 'year': -5, 'state': -4, 'all_facility': -2, }, }; state_idx = indexes[report]['state']; year_idx = indexes[report]['year']; all_facility_idx = indexes[report]['all_facility']; conditions = { 'budget': split_path[year_idx] == 'year_{0}'.format(year), 'procurement': True, }; if not file in ['.DS_Store', '.json'] and split_path[all_facility_idx] == 'all_facility' and conditions[report]: state_code = split_path[state_idx].replace('state_', ''); global_check_data[report].append('_'.join([date, state_code]));
def save(params, chunk, chunks_info): global latest_collection_name, history_collection_name data = File.readChunkData(chunk) dbManager = SharedMemoryManager.getInstance() db = dbManager.query() current_index = fn.getNestedElement(chunks_info, 'current', 0) total_index = fn.getNestedElement(chunks_info, 'total', len(data)) date = fn.getNestedElement(params, 'date') datetime = DateTime.convertDateTimeFromString(date) total_length = len(data) queue_info = chunks_info['queue'] # Logger.v('Running Index:', chunks_info['queue']['running']); chunks_info['queue']['current'] += 1 # Logger.v('Saving from... {0}/{1}, current package: {2}'.format(current_index, total_index, total_length) ); fn.printProgressBar(queue_info['current'], queue_info['total'], 'Processing Chunk Insertion') for idx in range(0, total_length): # insert stock_latest row = data[idx] obj_ = transformToLowercase(data=row, datetime=datetime) ModelStockIntegrity.update(data=obj_) dbManager.addBulkInsert(latest_collection_name, obj_, batch=True) # dbManager.addBulkInsert(history_collection_name, obj_, batch=True); # temporary off (need 7 day data only) # insert items # d = data[idx]; ModelItem.saveItem(row) # fn.printProgressBar(current_index+idx, total_index, 'Processing Item Insertion'); #ensure all data is save properly # dbManager.executeBulkOperations(history_collection_name); # temporary off (need 7 day data only) dbManager.executeBulkOperations(latest_collection_name) return chunks_info
def get(params): drug_codes = fn.getNestedElement(params, 'drug_nondrug_code', []); state = fn.getNestedElement(params, 'state'); requester_group = fn.getNestedElement(params, 'requester_group'); issue_type = fn.getNestedElement(params, 'issue_type'); dbManager = SharedMemoryManager.getInstance(); db = dbManager.query(); match_query = { 'state': state.replace('_', ' '), }; if drug_codes: match_query['drug_nondrug_code'] = {'$in': drug_codes}; if not requester_group == 'all' and requester_group: match_query['requester_group_name'] = requester_group.replace('_', ' '); if not issue_type == 'all' and issue_type: match_query['issue_type'] = issue_type.replace('_', ' '); data = list(db[collection_name].aggregate([ { '$match': match_query, }, { '$project': {'_id': 0, 'inserted_at': 0, 'updated_at': 0} } ])); data_length = len(data); # Logger.v('data length', data_length, data); return data;
def getTotalCount(params, data): filter_key = fn.getNestedElement(params, 'filter_key') key = fn.getNestedElement(params, 'key') row = fn.getNestedElement(data, 'row') facility = fn.getNestedElement(data, 'facility') if facility: if filter_key: state_code = filter_key facility_code = key # use this after confirm facility file structure # count = len(list(filter(lambda x: x['state_code'] == state_code and x['facility_code'] == facility_code, facility))); #XXX Temporary use state_name as state_code, follow stock data count = len( list( filter( lambda x: x['state_name'] == state_code and x[ 'facility_code'] == facility_code, facility))) else: state_code = key # use this after confirm facility file structure # count = len(list(filter(lambda x: x['state_code'] == state_code, facility))); #XXX Temporary use state_name as state_code, follow stock data count = len( list(filter(lambda x: x['state_name'] == state_code, facility))) else: count = fn.getNestedElement(row, 'count', 0) return count
def updateDropdownOptions(params): option_keys = fn.getNestedElement(params, 'keys.option', ['state']) today = fn.getNestedElement( params, 'schedule_params.today', DateTime.toString(DateTime.now(tzinfo=msia_tz))) data = {} crawled_data = {} # crawl from API URL (get options from API) # for key in keys: # url = api_links[key]; # # url = generateUrl(api_links[key]); # response = requests.get(url); # json_response = json.loads(response.text); # Logger.v('json_response', json_response); # crawled_data[key] = json_response; # Logger.v('Crawled', url); # Logger.v('Done crawling.'); # save(data); # read from file for key in option_keys: filename = api_files[key] crawled_data[key] = File.readJson(filename) # convert key to snakecase, value to lower for key in crawled_data: if key not in data: data[key] = [] for idx in range(0, len(crawled_data[key])): row = crawled_data[key][idx] obj_ = {} for row_key in row: row_value = row[row_key] new_key = fn.camelToSnakecase(str=row_key) if type(row_value) == str: new_value = row_value.lower() elif row_value is None: new_value = 'null' else: new_value = row_value obj_[new_key] = new_value data[key].append(obj_) for key in data: folder_path = '/'.join([crawl_folder, key]) if not os.path.exists(folder_path): os.makedirs(folder_path) filename = '{0}/{1}'.format(folder_path, today) Logger.v('Saving', filename) fn.writeJSONFile(filename='{0}.json'.format(filename), data=data[key]) for key in option_keys: directory = '/'.join([crawl_folder, key]) raw = File.readLatestFile(directory=directory) refresh_collection = refreshIsRequired(data=raw, collection_name=key) if refresh_collection: refreshCollection(data=raw, collection_name=key) Logger.v('refreshed', key)
def filterQuantity(params, data): filter_quantity = fn.getNestedElement(params, 'filter.quantity', None) main = fn.getNestedElement(data, 'main') sub = fn.getNestedElement(data, 'sub') qty = sub['quantity'] can_append = ModelStock.quantityWithinRange(params=params, quantity=qty) if can_append or not filter_quantity: main.append(sub)
def getMonthRange(params): start_month = fn.getNestedElement(params, 'start_month'); number_of_month = fn.getNestedElement(params, 'number_of_month', 1); month_range = [start_month]; new_month = '{0}-01'.format(start_month); for month_count in range(0, number_of_month - 1): # included start_month, so total month less 1 new_month = DateTime.getNextMonth(DateTime.convertDateTimeFromString(new_month)); year_month = DateTime.getDateCategoryName(new_month, element='year_month_digit'); month_range.append(year_month); return month_range;
def getCodeColumn(params): global extra_columns process_order = fn.getNestedElement(params, 'process_order') group_by_key = fn.getNestedElement(params, 'group_by_key') extra_col = copy.deepcopy(process_order[:-1]) extra_col += extra_columns[group_by_key] if 'item_code' not in extra_col: extra_col.append('item_code') if 'state' in extra_col: extra_col.remove('state') return extra_col
def createSchedules(args={}): #upid, page_type global filter_page_type Debug = DebugManager.DebugManager() Debug.start() dbManager = SharedMemoryManager.getInstance() # crawl_duration = fn.getNestedElement(fn.config,'CRAWL_DURATION', 12); incomplete_task, incomplete_task_count = checkRemaining() new_queue_count = 0 # Logger.v(incomplete_task_count, incomplete_task, filter_page_type); extra_params = { 'crawl_comment': fn.getNestedElement(args, 'crawl_comment', None) } extra_params = {k: v for k, v in extra_params.items() if v is not None} for platform in filter_page_type: if args and not platform in fn.getNestedElement( args, 'page_type', platform).split(','): Logger.v('Skip Platform:%s' % (platform)) continue # skip when page_type appear and not same pages = fn.getNestedElement(args, 'pages.{0}'.format(platform), []) Logger.v('platform', platform) # Logger.v('page', args['pages']['budget']); for page in pages: #Create queue for each # Logger.v('page', page); Queue.create(page, extra_params=extra_params, priority=fn.getNestedElement(args, 'priority', 'daily'), batch=True) new_queue_count += 1 Logger.v('new_queue_count', new_queue_count) # Debug.trace(); Logger.v('Incomplete:%s, New Queue: %s' % (incomplete_task_count, new_queue_count)) if incomplete_task_count > (new_queue_count * int(fn.config['DEBUG_CRAWL_WARNING']) / 100) or incomplete_task_count > int( fn.config['DEBUG_CRAWL_WARNING']): # Mail.send('[%s]Incomplete Crawl [%s], Current Schedule: [%s]'%(DateTime.getReadableDate(DateTime.now()), # incomplete_task_count, new_queue_count), # fn.dumps(incomplete_task, encode=False) # ); pass result = { 'pending_count': new_queue_count, 'incomplete_count': incomplete_task_count } dbManager.executeBulkOperations(None) # Debug.show('Create Schedule'); return Params.generate(True, result)
def checkEmpty(params): global global_check_data; dbManager = SharedMemoryManager.getInstance(); db = dbManager.query(); custom_params = copy.deepcopy(params); report_keys = fn.getNestedElement(params, 'keys.report', ['procurement', 'budget']); interval = fn.getNestedElement(params, 'interval', 1); past_dates = DateTime.getPastDate(count=12, duration=interval); # check previous 12 month data year = Crawl.extractYear(data=past_dates[0]); first_date = past_dates[0][-1][0]; last_date = past_dates[0][0][1]; # Logger.v('first_date', first_date, 'last_date', last_date); state_by = 'state_code'; states = list(db['state'].find({},{'_id': 0, state_by: 1})); result = {}; datetime = DateTime.toString(DateTime.now(tzinfo=msia_tz), date_format='%Y-%m-%d-%H-%M-%S'); custom_params['first_date'] = first_date; custom_params['last_date'] = last_date; custom_params['state_by'] = state_by; custom_params['states'] = states; temp_result = generateTemplate(params=custom_params); for rk in report_keys: if rk not in global_check_data: global_check_data[rk] = []; for y in year: root_path = '{0}/{1}/year_{2}'.format(crawl_folder, rk, y); openDir(root_path, rk); for gcd in global_check_data[rk]: date = gcd.split('_')[0]; state = gcd.split('_')[1]; if DateTime.inrange(date, [first_date, last_date]): try: temp_result[rk][date][state] += 1; except Exception as e: # Logger.v('Main.checkEmpty:', e); pass; for rk in temp_result: if rk not in result: result[rk] = []; for date in temp_result[rk]: result[rk].append(temp_result[rk][date]); filename = '{0}/{1}_check_moh_empty'.format(test_folder, rk); # filename = 'tests/{0}_{1}_check_moh_empty'.format(rk, datetime); fn.writeExcelFile(filename=filename, data=result[rk]); global_check_data = {}; return result;
def updateLog(params): global upload_log_collection_name dbManager = SharedMemoryManager.getInstance() db = dbManager.query() date = fn.getNestedElement(params, 'date') path = fn.getNestedElement(params, 'path') group = fn.getNestedElement(params, 'group') data_part = fn.getNestedElement(params, 'data_part', 'default') if type(group) == str: group = group.lower() if type(data_part) == str: data_part = data_part.lower() query = { 'date': date, 'collection': group, } stock_upload_log = list(db[upload_log_collection_name].find( query, { '_id': 0, 'inserted_at': 0, 'updated_at': 0 })) # Logger.v('stock_upload_log', stock_upload_log); if not stock_upload_log and group == 'stock': # Logger.v('backdate collection'); ModelStock.backdateCollection() # Logger.v('update upload_log collection'); values = {} if stock_upload_log: if 'part_of_the_day' not in values: values['part_of_the_day'] = [] for part in stock_upload_log[0]['part_of_the_day']: # Logger.v('part', part); values['part_of_the_day'].append(part) values[part] = stock_upload_log[0][part] if data_part not in stock_upload_log[0]['part_of_the_day']: values['part_of_the_day'].append(data_part) values[data_part] = path else: values['part_of_the_day'] = [data_part] values[data_part] = path # Logger.v('query', query, values) # exit(); dbManager.addBulkUpdate(upload_log_collection_name, query, values, upsert=True, batch=False)
def generateUniqueKeys(params, data): report_name = Report.getReportName(params) structure_keymap = fn.getNestedElement(params, 'structure_keymap') unique_keys = [] for idx in range(0, len(data)): row = data[idx] key = {} key['facility_type'] = fn.getNestedElement(row, 'facility_type') key['facility_code'] = fn.getNestedElement(row, 'facility_code') key['budget_type_code'] = fn.getNestedElement(row, 'budget_type_code') key['object_code'] = fn.getNestedElement(row, 'object_code') key['drug_code'] = fn.getNestedElement(row, 'drug_code') key['item_group_code'] = fn.getNestedElement(row, 'item_group_code') key['state_code'] = fn.getNestedElement(row, 'state_code') push_data = [] for sm in global_group_order[report_name]: mapped_key = global_group_order_kepmap[sm] push_data.append(key[mapped_key]) unique_keys.append('_'.join(push_data)) # Logger.v('unique_keys', len(unique_keys)); unique_keys = sorted(list(set(unique_keys))) # Logger.v('unique_keys', len(unique_keys)); return unique_keys
def addData(params, data): key = fn.getNestedElement(params, 'key') process_order = fn.getNestedElement(params, 'process_order') main = fn.getNestedElement(data, 'main') sub = fn.getNestedElement(data, 'sub') misc = fn.getNestedElement(data, 'misc') if sub: portion_key = renameDictKey(key=key, process_order=process_order) main.append({ 'id': misc['id'], 'name': misc['name'], 'code': misc['code'], portion_key: sub, })
def preprocessData(params, data): report_name = Report.getReportName(params) key_to_join = fn.getNestedElement(global_key_to_join, report_name) df = pd.DataFrame(data, dtype=str) # Logger.v('df', df); joined_key = [] joined_ = [] joined_columns_list = [key_to_join[0]] df['first_allocation'] = pd.to_numeric(df['first_allocation']) df['additional_allocation'] = pd.to_numeric(df['additional_allocation']) df['pending_amount'] = pd.to_numeric(df['pending_amount']) df['utilized_amount'] = pd.to_numeric(df['utilized_amount']) df['liablity_amount'] = pd.to_numeric(df['liablity_amount']) df['trans_in_amount'] = pd.to_numeric(df['trans_in_amount']) df['trans_out_amount'] = pd.to_numeric(df['trans_out_amount']) df['deduction_amount'] = pd.to_numeric(df['deduction_amount']) df['current_actual_amount'] = pd.to_numeric(df['current_actual_amount']) df['total_allocation'] = pd.to_numeric(df['total_allocation']) df['balance_amount'] = pd.to_numeric(df['balance_amount']) # df.info(); for idx in range(0, len(key_to_join)): ktj = key_to_join[idx] joined_key.append(ktj) if idx > 0: joined_.append(['_'.join(joined_key[:-1]), ktj]) columns = joined_[idx - 1] joined_columns = '_'.join(columns) joined_columns_list.append(joined_columns) df[joined_columns] = df[columns[0]].str.cat(df[columns[1]], sep="|") return df
def check(params): global msia_tz, date_retrieve_limit, date_count, collection_name dbManager = SharedMemoryManager.getInstance() db = dbManager.query() today = DateTime.now(tzinfo=msia_tz) start_date = DateTime.getDaysAgo(date_retrieve_limit, datefrom=today) durations = DateTime.getBetween([start_date, today], element='date', offset=24)['order'] # offset 24 to include today Logger.v('durations', durations) data = db[collection_name].aggregate([{ '$match': { 'state_updated_at': { '$in': durations }, 'facility_updated_at': { '$in': durations } } }, { '$project': { '_id': 0, 'inserted_at': 0, 'updated_at': 0 } }]) data = list(data) Logger.v('Total stock issue integrity in', date_retrieve_limit, 'days:', len(data)) state_data = {} facility_data_by_state = {} for idx in range(0, len(data)): row = data[idx] state_code = fn.getNestedElement(row, 'state_code') if state_code not in facility_data_by_state: facility_data_by_state[state_code] = {} state_data = addIntegrityData(data={ 'row': row, 'to_update': state_data }, category='state') facility_data_by_state[state_code] = addIntegrityData( data={ 'row': row, 'to_update': facility_data_by_state[state_code] }, category='facility') if date_count > date_retrieve_limit: # limit loop data/ show data in N days break date_count = 0 # reset to 0th day return { 'state': state_data, 'state_facility': facility_data_by_state, }
def generateQuery(params): report_name = Report.getReportName(params) filter_params = {} query = {} filter_keymap = { 'procurement': { 'year': 'txn_year', 'months': 'txn_month', 'facility_type': 'facility_type', 'facility': 'facility_code', 'ptj': 'ptj_code', 'state': 'state_code', 'procurement_type': 'procurement_type', }, 'budget': { 'year': 'financial_year', 'facility_type': 'facility_type', 'facility': 'facility_code', 'ptj': 'ptj_code', 'state': 'state_code', 'budget_type': 'budget_type_code', }, } filters = fn.getNestedElement(params, 'filter', {}) for f in list(filters.keys()): key = fn.getNestedElement(filter_keymap, '{0}.{1}'.format(report_name, f), None) if key: if key not in filter_params: filter_params[key] = [] values = fn.getNestedElement(filters, f, '').split(',') for val in values: if val: if key == 'txn_month': filter_params[key].append(val.zfill(2)) filter_params[key].append(val) for fp in filter_params: val = filter_params[fp] if val: query[fp] = { '$in': val } return query
def check(params): Debug = DebugManager.DebugManager() Debug.start() Debug.trace('start') global process_order global key_to_join global group_by group_by_list = fn.getNestedElement(params, 'group_by', []) filter_quantity = fn.getNestedElement(params, 'filter.quantity', []) export = fn.getNestedElement(params, 'export', None) custom_params = copy.deepcopy(params) result = {} # filter and read mongo db data = ModelStockIssue.get(params) Debug.trace('read mongo') # filtering by group if group_by_list: group_by = group_by_list[-1]['id'] else: group_by = 'state' custom_params['group_by_key'] = group_by custom_params['process_order'] = process_order[group_by] custom_params['key_to_join'] = key_to_join[group_by] custom_params['item_key_to_show'] = item_key_to_show[group_by] # processing data if data: temp_result = ModelStockIssue.calculateData(params=custom_params, data=data) Debug.trace('calculate data') result = toOutputStructure(params=custom_params, data=temp_result) Debug.trace('structure data') if result == {}: result = [] Debug.end() Debug.show('StockIssue.run') if export: export_result = generateExcelStructure(params=custom_params, data=result) return Params.generate(True, export_result) else: return Params.generate(True, result)
def groupDates(params, data): states = fn.getNestedElement(params, 'states'); state_by = fn.getNestedElement(params, 'state_by'); missing_list = []; crawled_list = []; for state in states: filtered_data = list(filter(lambda d: d[state[state_by]] == 0, data)); if filtered_data: missing_list += filtered_data; c_filtered_data = list(filter(lambda d: not d[state[state_by]] == 0, data)); if c_filtered_data: crawled_list += c_filtered_data; return { 'crawled': sorted(list(set([date['date'] for date in crawled_list])), reverse=True), 'missing': sorted(list(set([date['date'] for date in missing_list])), reverse=True), };
def organiseStructure(data, key): limit = 10; result = []; if data: if key in ['state', 'ptj', 'facility', 'facility_type']: # generate id for 'all' option #XXX # all_value = []; # for idx in range(0, len(data)): # row = data[idx]; # code = fn.getNestedElement(structure_map_name, '{0}.code'.format(key), ''); # val = fn.getNestedElement(row, code); # # Logger.v('code', code, 'val', val, 'key', key) # if val not in all_value: # all_value.append(val); # obj_ = { # 'id': ','.join(all_value), # 'name': 'all', # } # if obj_ not in result: # result.append(obj_); # generate id for each option for idx in range(0, len(data)): row = data[idx]; seq_no = fn.getNestedElement(structure_map_name, '{0}.seq_no'.format(key), ''); name = fn.getNestedElement(structure_map_name, '{0}.name'.format(key), ''); code = fn.getNestedElement(structure_map_name, '{0}.code'.format(key), ''); obj_ = { # use code as 'id': 'id': fn.getNestedElement(row, code), 'name': fn.getNestedElement(row, name), # use seq_no as 'id' # 'id': fn.getNestedElement(row, seq_no), # 'name': fn.getNestedElement(row, name), # 'code': fn.getNestedElement(row, code), } if obj_ not in result: result.append(obj_); return result[:limit];
def getQuery(params): duration = fn.getNestedElement(params, 'duration', ['2020-03-30', '2020-03-30']) item_codes = fn.getNestedElement(params, 'item_codes', []) item_desc = fn.getNestedElement(params, 'item_desc') group_by_list = fn.getNestedElement(params, 'group_by', []) facility_group = fn.getNestedElement(params, 'facility_group', []) dates = DateTime.getBetween(duration, element='date')['order'] query = {} if item_desc: # TEST wildcard search query.update({'item_desc': { '$regex': item_desc.lower() }}) if item_codes: query.update({ 'item_code': { '$in': [c.lower() for c in item_codes] }, }) if facility_group: facility_code_list = ModelFacility.getFacilityCodeList( facility_group=facility_group) query.update({ 'facility_code': { '$in': facility_code_list }, }) for gbl in group_by_list: gbl_id = gbl['id'] gbl_value = gbl['value'] val = gbl_value if type(val) == str: val = val.lower() if gbl_id == 'state': val = val.replace('_', ' ') query.update({ query_key[gbl_id]: val, }) Logger.v('query', query) return query
def generateExportMeta(params, data): metadata = {} group_by_key = fn.getNestedElement(params, 'group_by_key') state_name = fn.getNestedElement(data, 'state_name') row = fn.getNestedElement(data, 'row') if group_by_key == 'all': metadata['group_by'] = 'combine' elif group_by_key == 'state': metadata['group_by'] = group_by_key metadata['state_name'] = state_name elif group_by_key == 'facility': metadata['group_by'] = group_by_key metadata['state_name'] = state_name metadata['facility_name'] = row['facility_name'] elif group_by_key == 'requester': metadata['group_by'] = group_by_key metadata['state_name'] = state_name metadata['facility_name'] = row['facility_name'] metadata['requester_unit_desc'] = row['requester_unit_desc'] return metadata
def retrieveIssueOption(data): global stock_issue_options; if 'state' not in stock_issue_options: stock_issue_options['state'] = []; if 'requester_group' not in stock_issue_options: stock_issue_options['requester_group'] = []; if 'issue_type' not in stock_issue_options: stock_issue_options['issue_type'] = []; state = fn.getNestedElement(data, 'state'); if state: stock_issue_options['state'].append(state); requester_group = fn.getNestedElement(data, 'requester_group_name'); if requester_group: stock_issue_options['requester_group'].append(requester_group); issue_type = fn.getNestedElement(data, 'issue_type'); if issue_type: stock_issue_options['issue_type'].append(issue_type); return stock_issue_options;