def etl(): # Define our CLI arguments # example usage: `python etl.py -m rain raingardensTest.csv test.csv` parser = argparse.ArgumentParser(description="Manage various etl tasks") parser.add_argument('input', type=argparse.FileType('r'), help='input csv file') parser.add_argument('output', type=argparse.FileType('w'), help='output csv file') parser.add_argument('-m', '--method', dest="method", default="rain", help="Defines which method will process the file") args = parser.parse_args() print(args) if args.method == "rain": print("input:", args.input) print("output:", args.output) with args.input as readFile, args.output as writeFile: process_rain_gardens(readFile, writeFile) print("transformed csv from", args.output, "into", args.input) elif args.method == "geocode": from geocode import geocode with args.input as readFile, args.output as writeFile: geocode(readFile, writeFile) print("transformed csv from", args.output, "into", args.input)
def VIP(state, creds): name = state.replace('.csv', '') sheet = sheets.copySheet(name, vipTemplateKey, creds) client = sheets.getClient() stateData = read(vip_qa_data, state) exec('import ' + state) for row in stateData: address, county = getRowData(row) info = getVoterInfo(address) gppid, gaddress, gname = getVIPValues(info) sosppid, sosname, sosaddress = eval(state + ".run(row)") rowDict = { 'County': county, 'Address Run': address, 'SOS PP ID': sosppid, 'SOS PP NAME': sosname, 'SOS PP ADDRESS': sosaddress, 'Google PP ID': gppid, 'Google PP Name': gname, 'Google PP Address': gaddress } try: glocation = geocode.geocode(gaddress) soslocation = geocode.geocode(sosaddress) if glocation is not None and soslocation is not None: distance = geocode.haversine(glocation, soslocation) rowDict['SOS Location'] = str(soslocation) rowDict['Google Location'] = str(glocation) rowDict['Distance'] = str(distance) except Exception: pass while True: try: sheets.writeRow(rowDict, client, sheet['id']) break except Exception: pass return sheet['id']
def hello_world(bibcode): print("starting api-ing ads...") print(bibcode) paper_fai, paper_ai, cite_fai, cite_ai = papersearch(str(bibcode)) print(paper_fai) paper_fai_loc = [] for inst in paper_fai: geo = geocode(inst) print(geo) if geo: paper_fai_loc.append(geo) print('first author institutions', paper_fai_loc) paper_ai_loc = [] for inst in paper_ai: geo = geocode(inst) if geo: paper_ai_loc.append(geo) print('co-author institutions', paper_ai_loc) cite_fai_loc = [] for inst in cite_fai: geo = geocode(inst) if geo: cite_fai_loc.append(geo) print('citation first author institutions', cite_fai_loc) cite_ai_loc =[] for inst in cite_ai: geo = geocode(inst) if geo: cite_ai_loc.append(geo) print('citation coauthor institutions', cite_ai_loc) # #return "Hello world" # #'coauthors':paper_ai, 'citation_first_author':cite_fai, 'citation_coauthors':cite_ai}) # #return jsonify({'coords':[51.75952470000001, -1.2584172], 'coa_coords':[[47.3768866, 8.541694], [32.8797081, -117.2350537],[44.975331, -93.23461309999999]]}) # print('response', jsonify({'coords':paper_fai_loc, 'cao_coords':paper_ai_loc})) return jsonify({'coords':paper_fai_loc, 'cao_coords':paper_ai_loc, 'cite_fa':cite_fai_loc, 'cite_co': cite_ai_loc})
def ajout_ligne_peril(id, url, adresse, pathologies, date): db = ouverture_bdd() db[id] = [{ "categorie": "Arrêtés de péril", "adresse": adresse, "longitude": geo.geocode(adresse)[0], "lattitude": geo.geocode(adresse)[1], "pathologies": pathologies, "classification_pathologies": rec.classification_pathologie(pathologies), "classification_lieux": rec.classification_lieu(pathologies), "url": url, "date": date }] with open('data.json', 'w', encoding='utf-8') as f: json.dump(db, f, ensure_ascii=False) return None
def ajout_ligne_autre(categorie, id, url, adresse, date): db = ouverture_bdd() db[id] = [{ "categorie": categorie, "adresse": adresse, "longitude": geo.geocode(adresse)[0], "lattitude": geo.geocode(adresse)[1], "url": url, "date": date }] with open('data.json', 'w', encoding='utf-8') as f: json.dump(db, f, ensure_ascii=False) return None
def newevent(): json = request.json event = { 'host_name': json['host_name'], 'city': json['city'], 'address': json['location_address'], 'coordinates': geocode(json['location_address']), 'location_name': json['location_name'], 'event_name': json['event_name'], 'event_description': json['event_desc'], 'date_time': json['date_time'], 'duration': json['duration'], 'max_attendees': json['max_attendees'], 'attendees': [], 'fund_goal': json['fund_goal'], 'funds': 0, 'img_url': json['img_url'], 'tags': json['tags'], 'price': json['price'], 'leftovers': json['leftovers'], 'time_created': datetime.today() } event['desert'] = indesert(event['coordinates']) events.insert_one(event) msg = {'message': 'Event Created', 'code': 'Success'} resp = make_response(jsonify(msg), 201) return resp
def __geocodePlace(self, address): ''' get the lat/log/alt of the address ''' place = geocode.geocode(address, self.api_key) self.lat = place.lat self.long = place.long self.ele = place.ele
def new_message(request): r = twiml.Response() data = json.loads(str(request.POST['Body'])) if data[0] == UBER: typ, lng, lat, dest = data dest_coords = geocode(dest) ride_requests.append({'start': {'lng': lng, 'lat': lat}, 'end': dest_coords}) return HttpResponse(str(r)) elif data[0] == EXPLORE: typ, lng, lat, query = data places = json.dumps(encode_places(get_places(lng, lat, query)), separators=(',',':')) r.sms(places + '~') elif data[0] == DIRECTIONS: r.sms('directions') typ, lng, lat, end = data start = reverse_geocode(lng, lat) directions = get_directions(start, end) print(directions) r.sms(json.dumps(directions, separators=(',',':')) + '~') elif data[0] == HELP: places = get_places('-122.1430','37.4419') encoded_message = encode_places(places) print json.dumps(encoded_message, separators=(',',':')) r.sms(json.dumps(encoded_message, separators=(',',':'))) else: r.sms('not supported') return HttpResponse(str(r))
def parse_filial_info(html, place): """Parses html from UP filial details page and returns description, local address and phone number """ fullname = address = phone = "" town = city = village = coords = "" html = re.sub('"', '"', html) soup = BeautifulSoup(html, fromEncoding="utf-8") fullname = soup.find("table", id='ctl00_ContentPlaceHolder1_dw').find('td').nextSibling.string address = soup.find("table", id='ctl00_ContentPlaceHolder1_dw').findAll('tr')[1].find('td').nextSibling.string phone = soup.find("table", id='ctl00_ContentPlaceHolder1_dw').findAll('tr')[2].find('td').nextSibling.string coords = geocode(place, address) return {'address_full': fullname, 'street': address, 'phone': phone, 'place': place, 'coordinates': coords}
def distance(self, zip, dealerid): sql = "select * from funnel_dealer where id='%s' limit 1" % dealerid self.cursor.execute(sql) result = self.cursor.fetchone() dzip = result[common().getindex('funnel_dealer', 'fzip_id', self.cursor)] sql = "select * from funnel_zipcode where id= '%s' limit 1" % dzip self.cursor.execute(sql) result1 = self.cursor.fetchone() dzipcode = result1[common().getindex('funnel_zipcode', 'code', self.cursor)] sql = "select * from funnel_zipcode_distance where zip1='%s' and zip2='%s' limit 1" % ( dzipcode, zip) self.cursor.execute(sql) if self.cursor.rowcount == 0: distance = geocode(dzipcode, zip).distance() dict = {"zip1": dzipcode, "zip2": zip, "val": distance} common().insert('funnel_zipcode_distance', dict, self.cursor, self.db) return distance else: self.cursor.execute(sql) result = self.cursor.fetchone() return result[common().getindex('funnel_zipcode_distance', 'val', self.cursor)]
def handle_noargs(self, **options): import codecs sys.stdout = codecs.getwriter('utf8')(sys.stdout) geocoded_locations = {} for releaser_info in demozoo0.sceners_with_locations(): for releaser in Releaser.objects.filter(demozoo0_id=releaser_info['id']): if releaser.location: continue if releaser_info['location'] in geocoded_locations: location = geocoded_locations[releaser_info['location']] else: location = geocode(releaser_info['location']) geocoded_locations[releaser_info['location']] = location print "looked up %s, got %s as response" % (releaser_info['location'], location['location']) if location: releaser.location = location['location'] releaser.country_code = location['country_code'] or '' releaser.latitude = location['latitude'] releaser.longitude = location['longitude'] releaser.woe_id = location['woeid'] releaser.save() print "set location of %s to %s" % (releaser.name, releaser.location) sys.stdout.flush()
def __geocodePlace(self,address): ''' get the lat/log/alt of the address ''' place = geocode.geocode(address,self.api_key) self.lat = place.lat self.long = place.long self.ele = place.ele
def lambda_handler(event, context): print('Received event: {}'.format(event)) geocode.geocode(event['items']) dynamodb = boto3.resource('dynamodb') table = dynamodb.Table('scrapped_ads') # overwrites old values # TODO: have an array of changed values with table.batch_writer() as batch: for it in event['items']: batch.put_item(Item=it) return { 'statusCode': 200, 'event': event, 'body': json.dumps('Hello from Lambda!') }
def post(self): args = get_args(self.request) if 'address' in args: rsp = geocode.geocode(args['address']) if rsp: self.response.out.write(rsp) else: self.error(400)
def __init__(self, address, distance=2): self.location = geocode(address) self.params = dict( key = 'AIzaSyAehiOU6RPqcSc_sWh1e6gl8CSfOmi0EaM', types = 'bar|liquor_store|night_club', location = '%s,%s' % self.location[1], radius = int(distance) * 1609, # If passed as a argument, it might be a string. sensor = 'false') self.url = 'https://maps.googleapis.com/maps/api/place/search/json'
def test_geo(self): self.assertEqual(geocode(u"Дніпропетровськ"), {u'lat': 48.45, u'lng': 34.9833333}) self.assertEqual(geocode(u"Царичанка"), {u'lat': 48.9461111, u'lng': 34.4780556}) self.assertEqual(geocode(u"ичанка"), {}) with open(os.path.join(os.getcwd(), 'test/html/49069.html'), 'r') as f: html = f.read() parsed = parse_filial_info(html, u"Дніпропетровськ") self.assertEqual(geocode(parsed['place'], parsed['street']), {u'lat': 48.4451160, u'lng': 35.0259140})
def get_coordinate(address): """ координаты объекта по его адресу обращаемся к функции geocode из файла geocode.py""" toponym = geocode(address) if toponym: return toponym["Point"]["pos"] return None # print(get_coordinate('Красная пл-дь, 1'))
def get_ll_spn(address): toponym = geocode(address) toponym_coords = toponym["Point"]["pos"] lower_corner = toponym["boundedBy"]["Envelope"]["lowerCorner"] upper_corner = toponym["boundedBy"]["Envelope"]["upperCorner"] left, bottom = map(float, lower_corner.split()) right, up = map(float, upper_corner.split()) delta_x = abs(left - right) / 3 delta_y = abs(bottom - up) / 3 ll = toponym_coords.replace(' ', ',') spn = ','.join([str(delta_x), str(delta_y)]) return ll, spn
def main(): url = "http://www.map.com.tw/" headers = {'Referer': url, 'user-agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'} print('hi') # 使用chrome驅動器 browser = webdriver.Chrome() # 每一個網頁不超過10秒 browser.set_page_load_timeout(10) browser.get(url) print(geocode(browser,addr))
def hello_world(bibcode): print("starting api-ing ads...") print(bibcode) paper_fai, paper_ai, cite_fai, cite_ai = papersearch(str(bibcode)) print(paper_fai) paper_fai_loc = [] for inst in paper_fai: geo = geocode(inst) print(geo) if geo: paper_fai_loc.append(geo) print('first author institutions', paper_fai_loc) paper_ai_loc = [] for inst in paper_ai: geo = geocode(inst) if geo: paper_ai_loc.append(geo) print('co-author institutions', paper_ai_loc) cite_fai_loc = [] for inst in cite_fai: geo = geocode(inst) if geo: cite_fai_loc.append(geo) print('citation first author institutions', cite_fai_loc) cite_ai_loc = [] for inst in cite_ai: geo = geocode(inst) if geo: cite_ai_loc.append(geo) print('citation coauthor institutions', cite_ai_loc) # #return "Hello world" # #'coauthors':paper_ai, 'citation_first_author':cite_fai, 'citation_coauthors':cite_ai}) # #return jsonify({'coords':[51.75952470000001, -1.2584172], 'coa_coords':[[47.3768866, 8.541694], [32.8797081, -117.2350537],[44.975331, -93.23461309999999]]}) # print('response', jsonify({'coords':paper_fai_loc, 'cao_coords':paper_ai_loc})) return jsonify({ 'coords': paper_fai_loc, 'cao_coords': paper_ai_loc, 'cite_fa': cite_fai_loc, 'cite_co': cite_ai_loc })
def geocode_cust_addresses(self, caddresses): """Geocode all addresses from a customer dict.""" kml = KML() xmlroot = kml.xmlroot # geocode addresses for key, value in caddresses.items(): logging.info('Geocoding address..') gcaddr = geocode(value) logging.info('Appending row to kml..') xmlroot[0].append(kml.placemark(key, value, gcaddr)) self.wait(10) self.notifysubswrite(kml.serialize(xmlroot), '*.kml', 'Google Earth KML')
def geocode_cust_addresses(self, caddresses): """Geocode all addresses from a customer dict.""" kml = KML() xmlroot = kml.xmlroot # geocode addresses for key, value in caddresses.items(): logging.info('Geocoding address..') gcaddr = geocode(value) logging.info('Appending row to kml..') xmlroot[0].append(kml.placemark(key, value, gcaddr)) self.wait(10) self.notifysubswrite( kml.serialize(xmlroot), '*.kml', 'Google Earth KML')
def get(self): """HTTP get method.""" addr = unquote(self.request.get('address')) rsp = """ <html><body style="padding-top:4px;margin:0;font-size:10px; font-weight:bold;text-align:center;background-color:%s;"><div id="result">%s</div> """ if geocode.geocode(addr) == "": rsp = rsp % ("#ff3333", "BAD") else: rsp = rsp % ("#33ff33", "OK") self.response.out.write(rsp);
def post(self, request): datum = request.data['results'] outp = [] for row in datum: loc = gcd.geocode(row) if loc: no = {'lat': loc['lat'], 'lng': loc['lng']} for fn in row: no[fn] = row[fn] outp.append(no) body = {'geoData': outp} return Response(json.dumps(body))
def weatherResponse(self, keysFound, clientaddress, extra): ### Dominic Egginton ''' weatherResponse takes keysFound in client test, the client adrress and extra data and retunes a formatted string ''' locationStr = "" if 'location' in keysFound: from geocode import geocode geoCode = geocode() location = geoCode.getLocationCoords(extra.get('location')) if not location: return "Sorry, I can't fetch information about that location right now." locationStr = " in {}".format(extra.get('location').capitalize()) else: location = { 'latitude': clientaddress['lat'], 'longitude': clientaddress['lon'] } weatherData = self.forcastRequest(location) if 'time' not in keysFound: return 'It is currently {}{} and the temperature is {} Celsius'.format( weatherData['currently']['summary'], locationStr, str(weatherData['currently']['temperature'])) if extra.get('time') == 'daily': response = 'Daily Weather Forcast{}:\nSummary: {}\n\n'.format( locationStr, str(weatherData['daily']['summary'])) count = 0 for day in weatherData['daily']['data']: response = response + '{}:\nSummary: {}\nMax: {} @ {}\nMin: {} @ {}\n\n'.format( self.unixTimeToDateTime( day['time']), day['summary'], day['temperatureMax'], self.unixTimeToDateTime(day['temperatureMaxTime']), day['temperatureMin'], self.unixTimeToDateTime(day['temperatureMinTime'])) count += 1 if count == 8: break return response elif extra.get('time') == 'hourly': response = 'Hourly Weather Forcast{}:\nSummary: {}\n\n'.format( locationStr, str(weatherData['hourly']['summary'])) count = 0 for day in weatherData['hourly']['data']: response = response + '{}:\nSummary: {}\nTempature: {} Celsius\n\n'.format( self.unixTimeToDateTime(day['time']), day['summary'], day['temperature']) count += 1 if count == 8: break return response return None
def format_address(): houses = House.query.filter_by(formatted=False).all() print("Number of house(s) to process =>", len(houses)) for house in houses: try: formatted_address = geocode(house.address) if formatted_address: house.formattedAddress = formatted_address.get("formatted_address") house.latitude = formatted_address.get("lat") house.longitude = formatted_address.get("lng") house.formatted = True db.session.add(house) db.session.commit() except Exception as e: print(e)
def getzipcode(self, zip): sql = "select * from funnel_zipcode where code= '%s' limit 1" % zip self.cursor.execute(sql) if self.cursor.rowcount == 0: geo = geocode(zip) latlng = geo.getlatlng() dict = {} dict['lat'] = latlng[0] dict['lng'] = latlng[1] dict['code'] = zip return common().insert('funnel_zipcode', dict, self.cursor, self.db) else: self.cursor.execute(sql) result = self.cursor.fetchone() return result[0]
def bolder(address): """ координаты объекта по его адресу обращаемся к функции geocode из файла geocode.py""" toponym = geocode(address) if toponym: # Рамка вокруг объекта: envelope = toponym["boundedBy"]["Envelope"] # левая, нижняя, правая и верхняя границы из координат углов: l, b = envelope["lowerCorner"].split(" ") r, t = envelope["upperCorner"].split(" ") # Вычисляем полуразмеры по вертикали и горизонтали dx = abs(float(l) - float(r)) / 2.0 dy = abs(float(t) - float(b)) / 2.0 return str(max(dx, dy)) return None
def find_poi(data): streets = list(set(find_poi_from_street_id(data) + find_poi_from_streetnames(data))) for street in streets: if street not in GEOCODE_CACHE: GEOCODE_CACHE[street] = geocode(street) result = GEOCODE_CACHE.get(street) if result: if 'streets' not in data['meta']: data['meta']['streets'] = [] data['meta']['streets'].append({ 'raw_name': street, 'coordinates': result['geometry']['coordinates'], 'name': result['properties']['name'] }) return data
'Mytholmroyd', 'Whitby', 'Plymouth', 'Leeds', 'Dartmouth', 'London', 'Doncaster', 'Law', 'Scarborough', 'Sandy', 'California', 'Read', 'Sheffield', ,'Read', 'Lancaster', 'Houston', 'York', 'Oxford', 'Keith', 'Westminster', 'Read', 'Water', 'Leeds', 'Cardiff', 'Manchester', 'Street', 'Sandy'] locations = [] place_names = [] for i in test: places = test[i] for place in places: place_names.append(place) lat_lng = geocode.geocode(place) if lat_lng != False: locations.append(lat_lng) print(locations) ['Cardiff', 'Broadway', 'York', 'Walton', 'Eagle', 'Bridlington', 'Mytholmroyd', 'Whitby', 'Plymouth', 'Leeds', 'Dartmouth', 'London', 'Doncaster', 'Law', 'Scarborough', 'Sandy', 'California', 'Read', 'Sheffield', ,'Read', 'Lancaster', 'Houston', 'York', 'Oxford', 'Keith', 'Westminster', 'Read', 'Water', 'Leeds', 'Cardiff', 'Manchester', 'Street', 'Sandy']
""" ######### ###Il faut que ce soit décommenté pour Markers et commenté pour Points mcg = folium.plugins.MarkerCluster(control=False, icon_create_function=icon_create_function) c.add_child(mcg) ######## liste_adresses = carte.adresses() liste_messages = carte.message(liste_adresses, db_csv) for i in range(len(liste_adresses)): ########## ### Pour Marker: carte.creation_marker(mcg, geo.geocode(liste_adresses[i])[0], geo.geocode(liste_adresses[i])[1], liste_messages[i]) ### Pour Points: #carte.creation_marker(c, geo.geocode(liste_adresses[i])[0], geo.geocode(liste_adresses[i])[1], liste_messages[i]) ######### legend = carte.ajout_legend() c.get_root().add_child(legend) c.save('carte.html') webbrowser.open("file://" + os.getcwd() + '/carte.html')
def test_geocode_returns_tuple_when_passed_address(self): self.assertIs(type(geocode.geocode('496 Holly Grove School Road, ' 'West End, NC 27376')), type(('0', '0')))
def update_location(sender, instance, **kwargs): location = geocode.geocode(instance.address) instance.lng = location['longitude'] instance.lat = location['latitude']
import requests from geocode import geocode address = input() # обращаемся к функции geocode из файла geocode.py toponym = geocode(address) # Детали адреса топонима: toponym_address_details = toponym["metaDataProperty"]["GeocoderMetaData"][ "Address"]["Components"] print(address, 'входит в', toponym_address_details[1]["name"])
import ast from geocode import geocode import sys infile = sys.argv[1] outdir = sys.argv[2] shapefile = sys.argv[3] externalDEMFile = sys.argv[4] tmpdir = sys.argv[5] job_id = sys.argv[6] job_id_vsc = sys.argv[7] logdir = sys.argv[8] if "{" in shapefile: # This is a dict, convert from str shapefile = ast.literal_eval(shapefile) _ = geocode(infile, outdir, shapefile=shapefile, externalDEMFile=externalDEMFile, tmp_dir=tmpdir, job_id=job_id, job_id_vsc=job_id_vsc, logdir=logdir)
def geocode(self): return geocode.geocode(self.location)
def test_geocode_returns_zerozerotuple_when_passed_empty_string(self): self.assertEquals(geocode.geocode(''), (0, 0))
def test_geocode_returns_something_when_passed_address(self): self.assertIsNotNone(geocode.geocode('496 Holly Grove School Road, ' 'West End, NC 27376'))
import sys sys.path.append('../') import geocode _API_KEY = 'PUT YOUR KEY HERE' place = geocode.geocode('1 Main St, Pittsford, NY 14534',_API_KEY) print "Lat %s",place.lat print "Long %s",place.long
import sys sys.path.append('../') import geocode _API_KEY = 'PUT YOUR KEY HERE' place = geocode.geocode('1 Main St, Pittsford, NY 14534', _API_KEY) print "Lat %s", place.lat print "Long %s", place.long
import gdata.spreadsheet.service from geocode import geocode key = 'tM8UdRPe-6209NB-Hsx24rw' client = gdata.spreadsheet.service.SpreadsheetsService(source='yourCompany-YourAppName-v1') client.ClientLogin('*****@*****.**', 'password') list_feed = client.GetListFeed(key,'1') f = open('not_geocoded.txt', 'a') for i, entry in enumerate(list_feed.entry): address = entry.custom['address'].text if entry.custom['latitude'].text is None and entry.custom['longitude'].text is None: print 'geocoding: %s' % address try: lat, lng = geocode(address) except Exception as E: print 'EXCEPTION CAUGHT ... WRITING TO FILE' f.write(address) continue print 'results = %s, %s' % (lat, lng) client.UpdateCell(i+2, 2, str(lat), key) client.UpdateCell(i+2, 3, str(lng), key) else: print 'skipping address: %s' % address f.close()
def get_geocode(inst): rv = cache.get(inst) if rv is None: rv = geocode(inst) cache.set(inst, rv) return rv
# Pull out the primary key into a variable testKey = row["primary_key"] # Check to see if that distinctID is in the data base # See this post: https://stackoverflow.com/questions/6587879/how-to-elegantly-check-the-existence-of-an-object-instance-variable-and-simultan?utm_medium=organic&utm_source=google_rich_qa&utm_campaign=google_rich_qa entryExists = session.query(exists().where(Itenerary.distinctkey==testKey)).scalar() # If the entry is not in the database if not entryExists: # Get the string to be geocoded locationToGeocode = row["startFromLocation"] # Try to run the geocode function that returns a dictionary of information try: geocodeInfo = geocode(locationToGeocode) # If geocoding works, set valid to tre valid = True # If the geocode function doesn't work set valid to false except: valid = False # If valid is true create a database entry with information from the dataframe, and the returned geocode informaiton if valid: # Sometimes, such as when a generic city is sent to the geocode() function a geometric center # is returned, this means there is no postal code # In this case, we can set the postalCode to One if not 'postalCode' in geocodeInfo: