def main(): """Main Module of this program""" args = args_fetch() logger = logger_fetch(args.get('log_level')) if args['test']: logger.info("Testing phase") if args['import_rejected_urls']: logger.info("Going to import Rejected URls") state_codes = [] objs = Location.objects.filter(location_type='state') for obj in objs: state_codes.append(obj.code) for state_code in state_codes: if state_code != '34': file_path = f"../import_data/rejected_url_data/{state_code}.csv" dataframe = pd.read_csv(file_path) for index, row in dataframe.iterrows(): location_code = row['code'] report_type = row['report_type'] finyear = row['finyear'] report_url = row['report_url'] my_location = Location.objects.filter( code=location_code).first() if my_location is not None: my_report = Report.objects.create( location=my_location, finyear=finyear, report_type=report_type, report_url=report_url) logger.info(f" Created report {my_report.id}") logger.info("...END PROCESSING")
def main(): """Main Module of this program""" args = args_fetch() logger = logger_fetch(args.get('log_level')) secret_key = ''.join(random.SystemRandom().choice( 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)') for i in range(50)) logger.info(secret_key) logger.info("...END PROCESSING")
def main(): """Main Module of this program""" args = args_fetch() logger = logger_fetch(args.get('log_level')) if args['test']: logger.info("Testing") objs = Context.objects.all() for obj in objs: logger.info(obj.id) logger.info("...END PROCESSING")
def main(): """Main Module of this program""" args = args_fetch() logger = logger_fetch(args.get('log_level')) if args['test']: logger.info("Testing") phone = '9845065241' app_name = '294685' PhoneCall.objects.create(phone=phone, exotel_app_no=app_name) logger.info("...END PROCESSING") if args['execute']: # Get Call Status exotel_status = ["completed", "busy", "no-answer", "failed"] objs = PhoneCall.objects.filter(in_progress=True) for obj in objs: sid = obj.sid res = exotel_call_status(logger, sid) res_dict = res["Call"] vendor_status = res_dict["Status"] if vendor_status in exotel_status: obj.vendor_status = vendor_status obj.retry = obj.retry + 1 obj.in_progress = False obj.extra_fields = res_dict if vendor_status == "completed": obj.status = "completed" obj.is_complete = True else: if obj.retry == 10: obj.status = "maxRetryFail" obj.is_complete = True obj.save() logger.info(res) #Place Calls objs = PhoneCall.objects.filter(is_complete=False, in_progress=False) for obj in objs: data = exotel_place_call(logger, obj.phone, obj.exotel_app_no, obj.id) try: sid = data["Call"]["Sid"] except: sid = None if sid is None: obj.is_complete = True obj.status = "DND" obj.save() else: obj.in_progress = True obj.sid = sid obj.save()
def main(): """Main Module of this program""" args = args_fetch() logger = logger_fetch(args.get('log_level')) if args['test']: logger.info("Testing") objs = Location.objects.all() for obj in objs: logger.info(obj.id) if args['import']: filename = "../import_data/census_state.csv" df = pd.read_csv(filename) logger.info(df.head()) for index, row in df.iterrows(): code = str(row.get('state_code', '')).lstrip().rstrip() name = row.get('name', None).lstrip().rstrip() logger.info(name) myLocation = Location.objects.filter(code=code).first() if myLocation is None: myLocation = Location.objects.create(code=code, name=name) myLocation.name = name myLocation.state_name = name myLocation.state_code = code myLocation.location_type = 'state' myLocation.save() filename = "../import_data/census_district.csv" df = pd.read_csv(filename) logger.info(df.head()) for index, row in df.iterrows(): code = str(row.get('district_code', '')).lstrip().rstrip() state_code = str(row.get('state_code', '')).lstrip().rstrip() name = row.get('name', None).lstrip().rstrip() logger.info(name) myLocation = Location.objects.filter(code=code).first() if myLocation is None: myLocation = Location.objects.create(code=code, name=name) myLocation.name = name myLocation.district_name = name myLocation.district_code = code myLocation.state_code = state_code myLocation.location_type = 'district' myLocation.save() logger.info("...END PROCESSING")
def main(): """Main Module of this program""" args = args_fetch() logger = logger_fetch(args.get('log_level')) if args['test']: logger.info("Testing phase") if args['sample']: sample_name = args.get("sampleName", None) scheme = args.get("scheme", None) percentage = int(args.get("percentage", 0)) location_type = args.get("locationType", None) logger.info("Sampling locations") itda_tag = LibtechTag.objects.filter(id=2).first() ltarray = [itda_tag] my_tag = LibtechTag.objects.filter(name=sample_name).first() if my_tag is None: my_tag = LibtechTag.objects.create(name=sample_name) ltarray = [my_tag] objs = Location.objects.filter(libtech_tag__in=ltarray) for obj in objs: logger.info(obj.code) exit(0) parent_objs = Location.objects.filter(libtech_tag__in=ltarray, scheme=scheme) for parent_obj in parent_objs: location_set = Location.objects.filter(parent_location=parent_obj) location_set_size = len(location_set) village_code_array = [] for location in location_set: village_code_array.append(location.code) # logger.info(village_code_array) to_be_sampled = round((percentage * location_set_size) / 100) village_code_sampled = random.sample(village_code_array, to_be_sampled) for village_code in village_code_sampled: my_location = Location.objects.filter(code=village_code, scheme=scheme).first() my_location.libtech_tag.add(my_tag) my_location.save() logger.info( f"{location_set_size}-{to_be_sampled}--{len(village_code_sampled)}" ) logger.info("...END PROCESSING")
def main(): """Main Module of this program""" args = args_fetch() logger = logger_fetch(args.get('log_level')) if args['test']: logger.info("Testing") objs = Entity.objects.all() for obj in objs: logger.info(obj.id) break if args['exportPledges']: export_pledges() if args['exportRequests']: export_requests() if args['export']: queryset = Entity.objects.filter(record_type='helpseekers') filename = '' export_entities(queryset, filename, bulk_export=True) if args['export1']: csv_array = [] columns = ["title", "state", "status", "urgency", "remarks"] columns = [ 'ID', 'Urgency', 'Status', 'Assigned to', 'Remarks', 'Connected with Govt Scheme', 'Organisation Name', 'Contact', 'Mobile', 'Help Expected', 'Stranded State', 'Stranded District', 'StrandedBlock', 'Stranded City/Panchayat', 'Stranded Address', 'Native State', 'Native District', 'People Stranded', 'Date Called', 'Contact with Govt Official', 'Contact with anyone', 'Accom for 14 days', 'If no', ' Where are you staying"', 'Ration', 'Ration Desc', 'Drinking Water', 'Health Issues', 'Health Issue Desc', 'Urgent Req', 'Need Cash', 'How Much', 'Urgent Req Desc', 'Donor Involved', 'Donor Name', 'Type Of donation', 'Donation Value(Rs))', 'Initial Date' ] objs = Entity.objects.filter(record_type='helpseekers') for obj in objs: logger.info(obj.id) a = [obj.title, obj.state, obj.status, obj.urgency, obj.remarks] if obj.assigned_to_user is not None: user_name = obj.assigned_to_user.name else: user_name = '' try: stranded_district = obj.prefill_json["data"]["contactForm"][ "data"]["district"] except: stranded_district = '' if obj.assigned_to_group is not None: org_name = obj.assigned_to_group.name else: org_name = '' #logger.info(f"Stranded District {stranded_district}") government_scheme = '' contact = '' stranded_state = obj.state # stranded_district = '' stranded_block = '' stranded_panchayat = '' stranded_address = obj.address native_state = '' native_district = '' how_many_people = obj.how_many_people date_called = '' govt_official = '' contact_with_anyone = '' accom_for_14_days = '' if_no = '' where_are_you_staying = '' ration = '' ration_desc = '' drinking_water = '' health_issues = '' health_issue_description = '' urgent_req = '' need_cash = '' how_much = '' urgent_req_desc = '' donor_involved = '' donar_name = '' donation_type = '' donation_value = '' initial_date = '' a = [ obj.id, obj.urgency, obj.status, user_name, obj.remarks, government_scheme, org_name, contact, obj.phone, obj.what_help, stranded_state, stranded_district, stranded_block, stranded_panchayat, stranded_address, native_state, native_district, how_many_people, date_called, govt_official, contact_with_anyone, accom_for_14_days, if_no, where_are_you_staying, ration, ration_desc, drinking_water, health_issues, health_issue_description, urgent_req, need_cash, how_much, urgent_req_desc, donor_involved, donar_name, donation_type, donation_value, initial_date ] csv_array.append(a) df = pd.DataFrame(csv_array, columns=columns) filename = "export/data.csv" file_url = upload_s3(logger, filename, df) logger.info(file_url) logger.info("...END PROCESSING")
def main(): """Main Module of this program""" args = args_fetch() logger = logger_fetch(args.get('log_level')) if args['importSwanUsers']: logger.info("Importing swan users") df = pd.read_csv("../import_data/swan_users.csv") for index, row in df.iterrows(): email = row['email'] name = row['name'] if isinstance(email, str): if "@" in email: password = User.objects.make_random_password() password = '******' logger.info(f"email is {email} and password is {password}") myuser = User.objects.filter(email=email).first() if myuser is None: myuser = User.objects.create(email=email) myteam = Team.objects.filter(name="swanteam").first() myuser.team = myteam myuser.name = name myuser.formio_usergroup = "swan" myuser.user_role = 'usergroupadmin' myuser.set_password(password) myuser.save() if args['test']: objs = Entity.objects.filter(extra_fields__volunteer="Navmee") for obj in objs: logger.info(obj.id) exit(0) facility_json = '{"_id":"5e8a81632776ff4e9ea73357","type":"resource","tags":["common"],"owner":"5e8a3d7bf1f3d54924170187","components":[{"autofocus":false,"input":true,"tableView":true,"inputType":"radio","label":"Is this facility functional?","key":"facilityFunctional","values":[{"value":"yes","label":"Yes","shortcut":"Y"},{"value":"no","label":"No","shortcut":"N"}],"defaultValue":"","protected":false,"fieldSet":false,"persistent":true,"hidden":false,"clearOnHide":true,"validate":{"required":false,"custom":"","customPrivate":false},"type":"radio","labelPosition":"top","optionsLabelPosition":"right","tags":[],"conditional":{"show":"","when":null,"eq":""},"properties":{},"lockKey":true},{"autofocus":false,"input":true,"label":"Submit","tableView":false,"key":"submit","size":"md","leftIcon":"","rightIcon":"","block":false,"action":"submit","disableOnInvalid":false,"theme":"primary","type":"button"}],"display":"form","submissionAccess":[{"roles":["5e8a3d73f1f3d5492417017a"],"type":"create_own"},{"roles":["5e8a3d73f1f3d5492417017a"],"type":"read_own"},{"roles":["5e8a3d73f1f3d5492417017a"],"type":"update_own"},{"roles":["5e8a3d73f1f3d5492417017a"],"type":"delete_own"}],"title":"Help facilities","name":"helpFacilities","path":"map/facility","access":[{"roles":["5e8a3d73f1f3d54924170179","5e8a3d73f1f3d5492417017a","5e8a3d73f1f3d5492417017b"],"type":"read_all"}],"created":"2020-04-06T01:09:55.726Z","modified":"2020-04-06T01:10:33.874Z","machineName":"helpFacilities"}' logger.info("Testing") objs = Entity.objects.all() for obj in objs: record_type = obj.record_type if record_type == "facility": obj.feedback_form_json = facility_json obj.save() exit(0) if args['import']: data_dir = "../import_data" failed_index_array = [] filename = args['filename'] filename = "import_workers.csv" filepath = f"{data_dir}/{filename}" df = pd.read_csv(filepath) logger.info(df.columns) record_type = "helpseekers" for index, row in df.iterrows(): name = row['name'] contact_numbers = row['contact_numbers'] volunteer = row['volunteer'] extra_fields = {} if isinstance(volunteer, str): extra_fields["volunteer"] = volunteer address = row['address'] how_many = row['how_many'] keywords = f"{name},{contact_numbers}" obj = Entity.objects.filter(title=name, record_type=record_type).first() if obj is None: obj = Entity.objects.create(title=name, record_type=record_type) obj.contact_numbers = contact_numbers obj.address = address obj.keywords = keywords obj.extra_fields = extra_fields obj.save() exit(0) filename = "volunteer.csv" filepath = f"{data_dir}/{filename}" df = pd.read_csv(filepath) logger.info(df.head()) record_type = "volunteer" icon_url = "./assets/blue-dot.png" record_subtype = "orgization NGO" for index, row in df.iterrows(): create_record = False place_status = row.get("places_status", None) if place_status == "OK": create_record = True name = row.get("Name", None) description = row.get("Work Willing To Do", "") latitude = row.get("lat", None) longitude = row.get("lng", None) information_source = row.get("Source of Information", None) contact_numbers = row.get("Phone", None) remarks = row.get("Comments", None) try: obj = Entity.objects.filter(name=name, record_type=record_type).first() if obj is None: obj = Entity.objects.create(name=name, description=description, record_type=record_type) obj.latitude = latitude obj.longitude = longitude obj.description = description obj.record_subtype = record_subtype obj.icon_url = icon_url obj.remarks = remarks obj.information_source = information_source obj.contact_numbers = contact_numbers obj.save() except: failed_index_array.append(index) logger.info(f"failed index is {failed_index_array}") record_type = "facility" filename = "indira.csv" filepath = f"{data_dir}/{filename}" df = pd.read_csv(filepath) icon_url = "http://maps.google.com/mapfiles/ms/icons/blue-dot.png" record_subtype = "Indira Food Canteen" for index, row in df.iterrows(): create_record = False place_status = row.get("places_status", None) if place_status == "OK": create_record = True name = row.get("Ward No and Loc", None) latitude = row.get("lat", None) longitude = row.get("lng", None) zone = row.get("Zone", "") constituency = row.get("Constituency", "") address = row.get("address_to_use", "") description = f"Zone:{zone} constituency:{constituency} ward:{name} addresss:{address}" try: obj = Entity.objects.filter(name=name, record_type=record_type).first() if obj is None: obj = Entity.objects.create(name=name, description=description, record_type=record_type) obj.latitude = latitude obj.longitude = longitude obj.description = description obj.record_subtype = record_subtype obj.icon_url = icon_url obj.save() except: failed_index_array.append(index) logger.info(f"failed index is {failed_index_array}") exit(0) record_subtype = "Delhi Govt Food Facility" description = "Delhi Government food facility" icon_url = "http://maps.google.com/mapfiles/ms/icons/blue-dot.png" for index, row in df.iterrows(): create_record = False name = row.get("Description", None) latitude = row.get("Where is it located.lat", None) longitude = row.get("Where is it located.lon", None) information_source = row.get("source", None) logger.info(name) logger.info(latitude) logger.info(longitude) logger.info(information_source) if (isinstance(latitude, float) and isinstance(longitude, float)): create_record = True try: obj = Entity.objects.filter(name=name, record_type=record_type).first() if obj is None: obj = Entity.objects.create(name=name, description=description, record_type=record_type) obj.latitude = latitude obj.longitude = longitude obj.information_source = information_source obj.record_subtype = record_subtype obj.icon_url = icon_url obj.save() except: failed_index_array.append(index) logger.info(f"failed index is {failed_index_array}") logger.info("...END PROCESSING")
def main(): """Main Module of this program""" args = args_fetch() logger = logger_fetch(args.get('log_level')) if args['importSwanUsers']: logger.info("Importing swan users") df = pd.read_csv("../import_data/swan_users_22may20.csv") swanteam = Team.objects.filter(name="swanteam").first() myorg = Organization.objects.filter(name="swan").first() for index, row in df.iterrows(): email = row['email'] name = row['name'] team = str(row['team']).lstrip().rstrip() if ((team != '') and (team != "nan")): myteam = Team.objects.filter(organization=myorg, name=team).first() if myteam is None: myteam = Team.objects.create(organization=myorg, name=team) else: myteam = swanteam if isinstance(email, str): if "@" in email: password = User.objects.make_random_password() password = '******' logger.info( f"{index} email is {email} and password is {password}") myuser = User.objects.filter(email=email).first() if myuser is None: myuser = User.objects.create(email=email, name=name) myuser.team = myteam myuser.name = name myuser.formio_usergroup = "swan" myuser.user_role = 'groupadmin' myuser.organization = myorg myuser.set_password(password) myuser.is_locked = False myuser.login_attempt_count = 0 myuser.save() if args['connectUsersEntity']: objs = Entity.objects.filter(record_type="helpseekers") myTeam = Team.objects.filter(id=10).first() for obj in objs: obj.assigned_to_group = myTeam logger.info(obj.id) obj.save() exit(0) for obj in objs: extra_fields = obj.extra_fields try: wasan_user_email = obj.extra_fields['common']['user_email'] except: wasan_user_id = None myUser = User.objects.filter(region=wasan_user_email).first() if myUser is not None: obj.assigned_to_user = myUser obj.save() if args['importUsers']: logger.info("Importing Users") role_dict = { '1': 'usergroupadmin', '2': 'groupadmin', '3': 'volunteer', '4': 'volunteer', '5': 'volunteer', '6': 'usergroupadmin' } df = pd.read_csv("../import_data/wassan_users_2may.csv") for index, row in df.iterrows(): # wasan_id = row['Id'] name = row['name'] email = row['email'] if "@" not in email: email = f"{email}@abcd.com" password = row['password'] user_role = row['role'] wassan_username = row['username'] group_name = row['group'] phone = row['mobile'] myTeam = Team.objects.filter(name=group_name).first() if myTeam is None: myTeam = Team.objects.create(name=group_name) myuser = User.objects.filter(email=email).first() if myuser is None: myuser = User.objects.create(email=email) myuser.group = myTeam myuser.name = name myuser.set_password(password) myuser.phone = phone myuser.wassan_username = wassan_username # myuser.region = wasan_id#Temporary # user_role = role_dict.get(str(role_id), "volunteer") myuser.user_role = user_role myuser.formio_usergroup = 'wassan' myuser.save() if args['importRequests']: with open('../import_data/support_requests_24may20.json', 'r') as f: records = json.load(f) for i, record in enumerate(records): logger.info(record) try: email = record['extra_fields']['email'] except: email = None try: amount_needed = record['prefill_json']['data']['totalcost'] except: amount_needed = '0' amount_needed = int(amount_needed.replace(",", "")) prefill_json = record['prefill_json'] extra_fields = record['extra_fields'] myuser = None if email is not None: myuser = User.objects.filter(email=email).first() if myuser is not None: myorg = myuser.organization logger.info( f"{i}-{myuser.email}-{myorg.name}--{amount_needed}") title = f"{i}-request from {myorg.name}" obj = Request.objects.filter(title=title).first() if obj is None: obj = Request.objects.create(title=title) obj.user = myuser obj.organization = myorg obj.amount_needed = amount_needed obj.amount_pending = amount_needed obj.amount_pledged = 0 obj.save() try: obj.prefill_json = prefill_json obj.data_json = prefill_json['data'] obj.extra_fields = extra_fields obj.save() except: logger.info("count not prefill") if args['importEntities']: superadminuser = User.objects.filter(id=1).first() #with open('../import_data/2020-05-22_SWAN_Data.json', 'r') as f: # #data = f.read().replace(': NaN,', ': "",').replace(': NaN',':""').replace(',NaN',',""') # data = f.read().replace('NaN','""') #with open('/tmp/a.json', 'w') as f: # f.write(data) #exit(0) with open('../import_data/swan_data_22may20.json', 'r') as f: records = json.load(f) backend_remarks = "Imported Swan Data on 22 May 2020" base_number = 200521 * 10000 email_array = [] #for i,record in enumerate(records): # logger.info(i) # try: # email = record["extra_fields"]["common"]["user_email"] # except: # email = None # if email is not None: # if email not in email_array: # email_array.append(email) #logger.info(email_array) #logger.info(len(email_array)) #exit(0) for i, record in enumerate(records): #logger.info(i) try: email = record["extra_fields"]["common"]["user_email"] volunteer = User.objects.filter(email=email).first() except: volunteer = None creator = superadminuser if volunteer is not None: creator = volunteer sr_no = base_number + i obj_id = create_entity(logger, record, creator, wasan_id=sr_no, wasan_org_id=500, usergroup='swan', backend_remarks=backend_remarks, assigned_to_user=volunteer) logger.info(f"{i}-{obj_id} creator-{creator} assigned-{volunteer}") exit(0) usergroup = "wassan" objs = Entity.objects.filter(record_type="helpseekers") for obj in objs: obj.status = STATUS_DICT.get(obj.status, None) obj.save() if args['populateDistricts']: objs = Entity.objects.filter(record_type="helpseekers").order_by("-id") # objs = Entity.objects.filter(id = 74533).order_by("-id") for obj in objs: try: district = obj.prefill_json['data']['contactForm']['data'][ 'district'] except: district = None logger.info(f"{obj.id} - {district}") if district is not None: obj.district = district obj.save() if args['importOrgs']: df = pd.read_csv("../import_data/requests_24may20.csv") df = df.fillna("") password = '******' for index, row in df.iterrows(): logger.info(index) org = row.get('org', None) phone = row.get('phone', None) try: phone = str(int(phone)) except: phone = '' email = row.get('email', None) name = row.get('name', None) if name is None: name = org logger.info(f"{index}-{org}-{name}-{email}-{phone}") if org is not None: myorg = Organization.objects.filter(name=org).first() if myorg is None: myorg = Organization.objects.create(name=org) myorg.contact_phone = phone myorg.contact_name = name phone_array = [phone] data_json = { "contactName": name, "organization": org, "mobile": phone_array } myorg.data_json = data_json myorg.save() myTeam = Team.objects.filter(name=org).first() if myTeam is None: myTeam = Team.objects.create(name=org, organization=myorg) myTeam.organization = myorg myTeam.save() if email is not None: myUser = User.objects.filter(email=email).first() if myUser is None: myUser = User.objects.create(email=email, name=name) myUser.name = name myUser.organization = myorg myUser.team = myTeam myUser.phone = phone myUser.user_role = "groupadmin" myUser.set_password(password) myUser.save() if args['test']: objs = Entity.objects.filter(information_source__icontains='APPI') i = 0 for obj in objs: i = i + 1 logger.info(f"{i}-{obj.record_type}-{obj.information_source}") exit(0) df = pd.read_csv("/tmp/requests.csv") logger.info(df.columns) password = "******" for index, row in df.iterrows(): logger.info(index) org = row.get('org', None) phone = row.get('phone', None) email = row.get('email', None) total = row.get('total', None) remarks = row.get('remarks', None) logger.info(remarks) if org is not None: myorg = Organization.objects.filter(name=org).first() if myorg is None: myorg = Organization.objects.create(name=org) myorg.contact_phone = phone myorg.save() if email is not None: myUser = User.objects.filter(email=email).first() if myUser is None: myUser = User.objects.create(email=email, name=org) myUser.organization = myorg myUser.user_role = "volunteer" myUser.set_password(password) myUser.save() logger.info(f"Created user {myUser.id}") try: total = int(total) except: total = '' if isinstance(total, int): req_title = f"request from {org}-{index}" my_req = Request.objects.filter(title=req_title).first() if my_req is None: my_req = Request.objects.create(title=req_title) my_req.amount_needed = total my_req.organization = myorg my_req.user = myUser my_req.amount_pending = total my_req.amount_pledged = 0 my_req.notes = remarks my_req.save() exit(0) users = User.objects.filter(formio_usergroup="swan") for user in users: logger.info(user.id) exit(0) states = Entity.objects.all().values('state').annotate(c=Count('id')) for state in states: logger.info(state) exit(0) org = Organization.objects.filter(id=1).first() objs = Team.objects.all() for obj in objs: obj.organization = org obj.save() exit(0) objs = User.objects.all() for obj in objs: obj.formio_usergroup = "wassan" logger.info(obj.id) obj.save() objs = Entity.objects.filter(record_type="helpseekers") for obj in objs: logger.info(obj.id) #logger.info(obj.prefill_json) obj.what_help = help_sought(obj.prefill_json) obj.status = get_status(obj.prefill_json) obj.remarks = get_remarks(obj.prefill_json) logger.info(obj.what_help) obj.save() exit(0) objs = User.objects.all() for obj in objs: obj.save() exit(0) objs = Entity.objects.filter(formio_usergroup="wassan") for obj in objs: try: status = obj.extra_fields["common"]["status"] except: status = None try: urgency = obj.extra_fields["needs"]["urgency"] except: urgency = None obj.status = slugify(status) obj.urgency = urgency logger.info(obj.urgency) obj.save() exit(0) csv_array = [] if args['db2csv']: logger.info("test") dbhost = "localhost" dbuser = "******" dbpasswd = "vivek123" #db = MySQLdb.connect(host=dbhost, user=dbuser, passwd=dbpasswd, charset='utf8') db = dbInitialize( db='wasan', charset="utf8" ) # The rest is updated automatically in the function cur = db.cursor() db.autocommit(True) query = "SET NAMES utf8" cur.execute(query) query = "use wasan" cur.execute(query) query = "show tables" cur.execute(query) results = cur.fetchall() table_names = [] for row in results: table_names.append(row[0]) logger.info(table_names) # table_names = ['blocks', 'districts', 'migrants', 'migrantshistory', 'roles', 'states', 'users'] # table_names = ["roles"] for table_name in table_names: csv_array = [] logger.info(table_name) query = f"SHOW COLUMNS from {table_name};" cur.execute(query) results = cur.fetchall() column_headers = [] for row in results: column_headers.append(row[0]) logger.info(column_headers) query = f"select * from {table_name};" cur.execute(query) results = cur.fetchall() for row in results: a = [] for item in row: item = str(item).replace(",", "") a.append(item) csv_array.append(a) df = pd.DataFrame(csv_array, columns=column_headers) df.to_csv(f"/tmp/wasan/{table_name}.csv") if args['importRegions']: logger.info("importing regions") df = pd.read_csv("../import_data/states.csv") for index, row in df.iterrows(): name = row['state'] myreg = Region.objects.filter(name=name).first() if myreg is None: Region.objects.create(name=name) logger.info("...END PROCESSING")
def main(): """Main Module of this program""" args = args_fetch() logger = logger_fetch(args.get('log_level')) if args['fixfilepath']: location_type = args['location_type'] logger.info("fixing broken file paths") objs = Location.objects.all() objs = Location.objects.filter(location_type=location_type) j = len(objs) for obj in objs: logger.info(f"processing j - {j}") j = j - 1 base_file_path = obj.parent_location.s3_filepath s3_filepath = f"{base_file_path[:-14]}/{obj.slug}/DATA/reports/" obj.s3_filepath = s3_filepath obj.save() if args['crawl']: scheme = 'nrega' location_type = args['location_type'] logger.info(f"Crawling data for {location_type}") copy_from_parent = [ "state_code", "state_name", "district_code", "district_name", "block_code", "block_name", "panchayat_code", "panchayat_name", "crawl_ip", "state_short_code" ] if location_type == 'state': parent_location_type = 'country' code_param = 'state_code' name_param = 'state_name' elif location_type == 'district': parent_location_type = 'state' code_param = 'district_code' name_param = 'district_name' elif location_type == 'block': parent_location_type = 'district' code_param = 'Block_Code' name_param = 'block_name' elif location_type == 'panchayat': parent_location_type = 'block' code_param = 'Panchayat_Code' name_param = 'Panchayat_name' queryset = Location.objects.filter(location_type=parent_location_type) queryset = Location.objects.filter(location_type=parent_location_type, id__gt=1395) queryset = Location.objects.filter(location_type=parent_location_type, id=500) #queryset = Location.objects.filter(location_type=parent_location_type)[:1] for parent_location in queryset: urls_to_process = [] base_url = parent_location.nic_url parent_file_path = parent_location.s3_filepath parent_display_name = parent_location.display_name logger.info(f"parent file path is {parent_file_path}") base_filepath = parent_file_path.rstrip('/DATA/reports/') logger.info(f"base_filepath is {base_filepath}") myhtml = None logger.info(f"Current Processing {base_url}") if 'http://nrega.nic.in' in base_url: base_url = base_url.replace("http", "https") res = requests.get(base_url) if res.status_code == 200: myhtml = res.content if myhtml is not None: mysoup = BeautifulSoup(myhtml, "lxml") links = mysoup.findAll("a") for link in links: href = link['href'] if code_param in href: urls_to_process.append(href) for href in urls_to_process: parsed = urlparse.urlparse(href) crawl_ip = parsed.netloc params = urlparse.parse_qs(parsed.query) # setattr(my_location, key.lower(), value) code = params[code_param][0] name = params[name_param][0] logger.info(f"{name}-{code}") if is_english(name): name_not_english = False english_name = name else: name_not_english = True #this part of script needs correction, currently it defaults to hindi english_name = ms_transliterate_word(logger, name) my_location = Location.objects.filter(code=code, scheme=scheme).first() if my_location is None: my_location = Location.objects.create(code=code, name=name, scheme=scheme) for item in copy_from_parent: value = getattr(parent_location, item) setattr(my_location, item, value) setattr(my_location, code_param.lower(), code) setattr(my_location, name_param.lower(), name) my_location.parent_location = parent_location if location_type == "state": logger.info(f"href is {href}") my_location.nic_url = href + "&lflag=eng" my_location.display_name = name.title() my_location.crawl_ip = crawl_ip my_location.state_short_code = STATE_SHORT_CODE_DICT.get( code, "") else: my_location.nic_url = urljoin(base_url, href) my_location.display_name = f"{parent_display_name}-{english_name.title()}" my_location.location_type = location_type my_location.name_not_english = name_not_english my_location.english_name = english_name slug = slugify(english_name) my_location.slug = slug if my_location.state_code in ['02', '36']: my_location.is_nic = False else: my_location.is_nic = True filepath = f"{base_filepath}/{slug}/DATA/reports/" logger.info(f"file paht is {filepath}") my_location.s3_filepath = filepath my_location.save() logger.info("...END PROCESSING")
def main(): """Main Module of this program""" args = args_fetch() logger = logger_fetch(args.get('log_level')) if args['test']: df = pd.read_csv("/tmp/facility.csv") for index, row in df.iterrows(): name = row.get("name", "") description = row.get("description", "") latitude = row.get("latitude", None) longitude = row.get("longitude", None) information_source = row.get("information_source", None) entity = Entity.objects.filter(name=name, description=description).first() if entity is None: entity = Entity.objects.create(name=name, description=description) entity.record_type = "facility" entity.record_subtype = "food" entity.latitude = latitude entity.longitude = longitude entity.information_source = information_source entity.save() exit(0) csv_array = [] column_headers = [ "name", "descript", "latitude", "longitude", "information_source" ] objs = Entity.objects.filter(record_type="facility") for obj in objs: a = [ obj.name, obj.description, obj.latitude, obj.longitude, obj.information_source ] csv_array.append(a) df = pd.DataFrame(csv_array, columns=column_headers) df.to_csv("/tmp/facility.csv") exit(0) objs = Covid.objects.all() for obj in objs: obj.delete() exit(0) facility_json = '{"_id":"5e8a81632776ff4e9ea73357","type":"resource","tags":["common"],"owner":"5e8a3d7bf1f3d54924170187","components":[{"autofocus":false,"input":true,"tableView":true,"inputType":"radio","label":"Is this facility functional?","key":"facilityFunctional","values":[{"value":"yes","label":"Yes","shortcut":"Y"},{"value":"no","label":"No","shortcut":"N"}],"defaultValue":"","protected":false,"fieldSet":false,"persistent":true,"hidden":false,"clearOnHide":true,"validate":{"required":false,"custom":"","customPrivate":false},"type":"radio","labelPosition":"top","optionsLabelPosition":"right","tags":[],"conditional":{"show":"","when":null,"eq":""},"properties":{},"lockKey":true},{"autofocus":false,"input":true,"label":"Submit","tableView":false,"key":"submit","size":"md","leftIcon":"","rightIcon":"","block":false,"action":"submit","disableOnInvalid":false,"theme":"primary","type":"button"}],"display":"form","submissionAccess":[{"roles":["5e8a3d73f1f3d5492417017a"],"type":"create_own"},{"roles":["5e8a3d73f1f3d5492417017a"],"type":"read_own"},{"roles":["5e8a3d73f1f3d5492417017a"],"type":"update_own"},{"roles":["5e8a3d73f1f3d5492417017a"],"type":"delete_own"}],"title":"Help facilities","name":"helpFacilities","path":"map/facility","access":[{"roles":["5e8a3d73f1f3d54924170179","5e8a3d73f1f3d5492417017a","5e8a3d73f1f3d5492417017b"],"type":"read_all"}],"created":"2020-04-06T01:09:55.726Z","modified":"2020-04-06T01:10:33.874Z","machineName":"helpFacilities"}' logger.info("Testing") objs = Entity.objects.all() for obj in objs: record_type = obj.record_type if record_type == "facility": obj.feedback_form_json = facility_json obj.save() exit(0) if args['import']: data_dir = "../import_data" failed_index_array = [] filename = args['filename'] filename = "volunteer.csv" filepath = f"{data_dir}/{filename}" df = pd.read_csv(filepath) logger.info(df.head()) record_type = "volunteer" icon_url = "./assets/blue-dot.png" record_subtype = "orgization NGO" for index, row in df.iterrows(): create_record = False place_status = row.get("places_status", None) if place_status == "OK": create_record = True name = row.get("Name", None) description = row.get("Work Willing To Do", "") latitude = row.get("lat", None) longitude = row.get("lng", None) information_source = row.get("Source of Information", None) contact_numbers = row.get("Phone", None) remarks = row.get("Comments", None) try: obj = Entity.objects.filter(name=name, record_type=record_type).first() if obj is None: obj = Entity.objects.create(name=name, description=description, record_type=record_type) obj.latitude = latitude obj.longitude = longitude obj.description = description obj.record_subtype = record_subtype obj.icon_url = icon_url obj.remarks = remarks obj.information_source = information_source obj.contact_numbers = contact_numbers obj.save() except: failed_index_array.append(index) logger.info(f"failed index is {failed_index_array}") record_type = "facility" filename = "indira.csv" filepath = f"{data_dir}/{filename}" df = pd.read_csv(filepath) icon_url = "http://maps.google.com/mapfiles/ms/icons/blue-dot.png" record_subtype = "Indira Food Canteen" for index, row in df.iterrows(): create_record = False place_status = row.get("places_status", None) if place_status == "OK": create_record = True name = row.get("Ward No and Loc", None) latitude = row.get("lat", None) longitude = row.get("lng", None) zone = row.get("Zone", "") constituency = row.get("Constituency", "") address = row.get("address_to_use", "") description = f"Zone:{zone} constituency:{constituency} ward:{name} addresss:{address}" try: obj = Entity.objects.filter(name=name, record_type=record_type).first() if obj is None: obj = Entity.objects.create(name=name, description=description, record_type=record_type) obj.latitude = latitude obj.longitude = longitude obj.description = description obj.record_subtype = record_subtype obj.icon_url = icon_url obj.save() except: failed_index_array.append(index) logger.info(f"failed index is {failed_index_array}") exit(0) record_subtype = "Delhi Govt Food Facility" description = "Delhi Government food facility" icon_url = "http://maps.google.com/mapfiles/ms/icons/blue-dot.png" for index, row in df.iterrows(): create_record = False name = row.get("Description", None) latitude = row.get("Where is it located.lat", None) longitude = row.get("Where is it located.lon", None) information_source = row.get("source", None) logger.info(name) logger.info(latitude) logger.info(longitude) logger.info(information_source) if (isinstance(latitude, float) and isinstance(longitude, float)): create_record = True try: obj = Entity.objects.filter(name=name, record_type=record_type).first() if obj is None: obj = Entity.objects.create(name=name, description=description, record_type=record_type) obj.latitude = latitude obj.longitude = longitude obj.information_source = information_source obj.record_subtype = record_subtype obj.icon_url = icon_url obj.save() except: failed_index_array.append(index) logger.info(f"failed index is {failed_index_array}") logger.info("...END PROCESSING")
def main(): """Main Module of this program""" args = args_fetch() logger = logger_fetch(args.get('log_level')) if args['test']: logger.info("Testing phase") scheme = "rayatubarosa" vskp_code = '520' vskp_location = Location.objects.filter(scheme=scheme, code=vskp_code).first() logger.info(vskp_location.id) vskp_block_url = "https://libtech-india-data.s3.ap-south-1.amazonaws.com/data/locations/rayatu_barosa/vskp_blocks.csv" vskp_village_url = "https://libtech-india-data.s3.ap-south-1.amazonaws.com/data/locations/rayatu_barosa/vskp_villages.csv" village_df = pd.read_csv(vskp_village_url) logger.info(village_df.columns) total = len(village_df) for index, row in village_df.iterrows(): block_code = str(int(row['mandal_code'])) village_code = str(int(row['village_code'])) logger.info(f"{index}-{total}-{block_code}-{village_code}") village_name = row['village_name_telugu'] english_name = ms_transliterate_word(logger, village_name, lang_code='te', script_code='Telu') parent_location = Location.objects.filter(scheme=scheme, code=block_code).first() my_location = Location.objects.filter(scheme=scheme, code=village_code).first() if my_location is None: my_location = Location.objects.create(scheme=scheme, code=village_code, name=village_name) my_location.parent_location = parent_location parent_display_name = parent_location.display_name my_location.location_type = "village" my_location.state_name = 'Andhra Pradesh' my_location.state_code = '28' my_location.district_code = parent_location.parent_location.code my_location.district_name = parent_location.parent_location.name my_location.block_code = parent_location.code my_location.block_name = parent_location.name my_location.name_not_english = True my_location.english_name = english_name my_location.display_name = f"{parent_display_name}-{english_name.title()}" slug = slugify(english_name) my_location.slug = slug my_location.save() block_df = pd.read_csv(vskp_block_url) logger.info(block_df.columns) for index, row in block_df.iterrows(): block_code = str(int(row['mandal_code'])) english_name = row['mandal_name_eng'] block_name = row["block_name_telugu"] logger.info(block_code) my_location = Location.objects.filter(scheme=scheme, code=block_code).first() if my_location is None: my_location = Location.objects.create(scheme=scheme, code=block_code, name=block_name) my_location.parent_location = vskp_location parent_display_name = vskp_location.display_name my_location.state_code = '28' my_location.state_name = 'Andhra Pradesh' my_location.district_code = vskp_location.code my_location.district_name = vskp_location.name my_location.location_type = "block" my_location.block_code = block_code my_location.block_name = block_name my_location.name_not_english = True my_location.english_name = english_name my_location.display_name = f"{parent_display_name}-{english_name.title()}" slug = slugify(english_name) my_location.slug = slug my_location.save() logger.info("...END PROCESSING")