def return_latlng(enquiry):
address = enquiry
google_maps = GoogleMaps(api_key='AIzaSyCw6c6wq-VY06rOn401rwtv-Q7MU3-C89M')
location = google_maps.search(location=address) # sends search to Google Maps.
# print(location.all()) # returns all locations.
my_location = location.first() # returns only first location.
# route_name = my_location.route
# st = my_location.street_number
# place = my_location.formatted_address
# print(route_name)
# print(my_location.lng)
# print(my_location.street_number)
# print(my_location.formatted_address)
# lat = 37.751502
# lng = -122.386707
# my_location = google_maps.search(lat=lat, lng=lng).first()
# print (my_location.route)
return [my_location.lat, my_location.lng]
def __init__(self):
self.location_cache = {}
self.geolocator = Nominatim()
self.geolocation = None
self.backup_geolocator = GoogleMaps(api_key=google_map_api_key)
self.lat = None
self.long = None
def sedan():
form = BookingForm2(request.form)
try:
if request.method == 'POST':
start = form.start.data
destination = form.destination.data
start = request.form['start']
destination = request.form['destination']
gmaps = GoogleMaps('Your goolge key')
start = start
end = destination
dirs = gmaps.distance(start, end)
for step in dirs:
x = str(float(step.distance.kilometers))
z = float(x)
if z > 50.0:
flash(
"Distance is more than 50 km.Please change the credentials"
)
else:
y = str(z * float(4))
flash("Distance is:" + x)
flash("Total fare is :" + y)
except Exception as e:
flash("Please enter the credentials or check your internet connecton")
return render_template("sedan.html", form=form)
def geolocation(self):
self.maplist = []
self.buttonHeightCounter = .05
API_KEY = 'AIzaSyBPGAbevdKkeXaZT0ZsR0qbO30Bpqqm0Mc'
google_places = GooglePlaces(API_KEY)
self.query_result = google_places.nearby_search(
location=self.entry.get(),
radius=700,
types=[types.TYPE_RESTAURANT])
self.current_places = self.query_result
if self.query_result.has_attributions:
print(self.query_result.html_attributions)
for place in self.query_result.places:
place.get_details()
markers = "&markers=size:big|label:S|color:red|" + str(
place.details['geometry']['location']['lat']) + "," + str(
place.details['geometry']['location']['lng']) + "|"
self.maplist.append(markers)
print(place.name)
self.button_list.append(
Button(self,
text=place.name,
command=lambda pname=place.name: self.on_click(pname),
width=25))
self.button_list[-1].place(relx=.70,
rely=self.buttonHeightCounter,
anchor="c")
self.buttonHeightCounter += .035
print(place.formatted_address + "\n")
google_maps = GoogleMaps(
api_key='AIzaSyDlJqxwlOWWAPwf54ivrpAZw4R1Yb5j6Yk')
location = google_maps.search(
location=self.entry.get()) # sends search to Google Maps.
my_location = location.first() # returns only first location.
#MARKER = '&markers=color:blue' + '%' + str(7) + 'Clabel:S%' + str(7) + 'C' + str(my_location.lat) + ',' + str(my_location.lng)
#MARKER = "&markers=size:big|label:S|color:blue|" + str(my_location.lat) + "," + str(my_location.lng) + "|" + \
MARKER = self.maplist[1] + self.maplist[2] + self.maplist[3]
self.zoomlevel = ZOOM
maptype_index = 0
self.radiovar.set(maptype_index)
self.goompy = GooMPy(WIDTH, HEIGHT, my_location.lat, my_location.lng,
ZOOM, MAPTYPE, MARKER)
self.restart()
print(self.query_result)
print(str(my_location.lat))
print(str(my_location.lng))
def __init__(self, **kwargs):
super(ConferenceAlertsSpider, self).__init__(**kwargs)
self.init_month()
self.urls = self.get_dict('assets\conf_start_urls.json')
for url in self.urls:
self.start_urls = self.start_urls + [url['url']]
self.google_maps = GoogleMaps(api_key='AIzaSyCaxLzZ2r7AbCJiIy5RtJ4jOQXcOlDbeV0')
def get_latlng(address):
google_maps = GoogleMaps(api_key='AIzaSyCcuF3AbW0iS9QChjdMUsstXN7C5zOJ5gg')
location = google_maps.search(location=address)
my_location = location.first()
return (str(my_location.lat), str(my_location.lng))
def show_dist_delivery_time():
origins = [user_reverse_geocode_result]
destinations = [rest_reverse_geocode_result]
google_maps = GoogleMaps(api_key=’API_KEY’)
items = google_maps.distance(origins, destinations).all() # default mode parameter is DistanceMatrixApiClient.MODE_DRIVING.
for item in items:
print item.distance.kilometers
print item.distance.meters
print item.duration
def get_geolocation(address):
google_maps = GoogleMaps(api_key='your_geocoding_api_key')
location = google_maps.search(location=address)
my_location = location.first()
if my_location is not None:
coordinates = (my_location.lat, my_location.lng)
return coordinates
else:
return 'Not Found'
class GeoLocation(object):
def __init__(self, city):
self.mycity = city
self.gmap = GoogleMaps(api_key=KEY)
self.location = self.gmap.search(location=city).first()
print('Your lat,long: {},{}'.format(self.location.lat,
self.location.lng))
def get_distance_to(self, dest):
return self.gmap.distance(self.mycity,
dest).first().distance.kilometers
def display_details(self):
address = str(self.ui.lineEditLocation.text())
google_maps = GoogleMaps(
api_key='AIzaSyCqZcxCYBoZ4ui8ihg02MmR0CLEWzGhZp0')
location = google_maps.search(location=address)
my_location = location.first()
self.ui.labelCity.setText("City: " + str(my_location.city))
self.ui.labelPostalCode.setText("Postal Code: " +
str(my_location.postal_code))
self.ui.labelLongitude.setText("Longitude: " + str(my_location.lng))
self.ui.labelLatitude.setText("Latitude: " + str(my_location.lat))
def displayDetails(self):
address = str(self.ui.lineEditLocation.text())
google_maps = GoogleMaps(api_key='xxxxxxxxxxxxxxxxxxxxxxxxxxxx')
location = google_maps.search(location=address)
my_location = location.first()
self.ui.labelCity.setText("City:" + str(my_location.city))
self.ui.labelPostalCode.setText("Postal Code: " +
str(my_location.postal_code))
self.ui.labelLongitude.setText("Longitude:" + str(my_location.lng))
self.ui.labelLatitude.setText("Latitude:" + str(my_location.lat))
def dist():
origin = ['Santa Cruz']
destination = ['Seaside']
duration = []
google_maps = GoogleMaps(api_key='AIzaSyBTfMBsYy9fFaJ-XVhoHIz-VnKiN2DZGpg')
items = google_maps.distance(origin, destination).all()
for item in items:
duration.append('Start: ' + origin[0] + '\n' + 'Destination: ' +
destination[0] + '\n' +
'Travel time: %s' % item.duration)
return duration
def __init__(self):
self.location_cache = {}
self.geolocator = Nominatim()
self.geolocation = None
self.backup_geolocator = GoogleMaps(api_key=google_map_api_key)
self.lat = None
self.long = None
def send_data():
#addr = Entry1.get()
#print(addr)
addr = 'MNNIT Allhabad, Uttar Pradesh'
#quant = Scale1.get()
#print(quant)
'''
import urllib2
proxy = urllib2.ProxyHandler({'http':'edcguest:[email protected]:3128','https':'edcguest:[email protected]:3128'})
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
'''
address = addr
print(address)
google_maps = GoogleMaps(
api_key=' AIzaSyD4QiVK9WflR5GCkzuks27x55V2_v5CX2k ')
print(address)
location = google_maps.search(
location=address) # sends search to Google Maps.
print(location.all()) # returns all locations.
my_location = location.first() # returns only first location.
print(my_location.city)
print(my_location.route)
print(my_location.street_number)
print(my_location.postal_code)
for administrative_area in my_location.administrative_area:
print("{}: {} ({})".format(administrative_area.area_type,
administrative_area.name,
administrative_area.short_name))
print(my_location.country)
print(my_location.country_shortcut)
print(my_location.formatted_address)
print(my_location.lat)
print(my_location.lng)
'''
def __init__(self):
# Setup OSC
self.maxServer = OSC.OSCServer(('127.0.0.1', 7000))
self.maxServerThread = threading.Thread(target=self.maxServer.serve_forever)
self.maxServerThread.daemon = False
self.maxServerThread.start()
self.maxClient = OSC.OSCClient()
self.maxClient.connect(('127.0.0.1', 57121))
self.maxServer.addMsgHandler("/saveTweetPhrase", self.saveTweetPhraseResponder)
self.maxServer.addMsgHandler("/nextTweetPhrase", self.nextTweetPhraseResponder)
self.nodeClient = OSC.OSCClient()
self.nodeClient.connect(('127.0.0.1', 6000))
# Current tweets to be sonified
self.tweets = []
self.tweetIndex = 0
# Instantiate a Google Maps client for geocoding
self.maps = GoogleMaps(api_key=mapscredentials.api_key)
# Instantiate a Twitter client
self.twitterClient = twitter.Api(consumer_key=twittercredentials.consumer_key, consumer_secret=twittercredentials.consumer_secret,
access_token_key=twittercredentials.access_token_key, access_token_secret=twittercredentials.access_token_secret,
input_encoding=None, tweet_mode="extended")
# Instantiate a Google NLP client
cred = service_account.Credentials.from_service_account_file('TwittHear-a204ccf1b234.json')
cred = cred.with_scopes(
['https://www.googleapis.com/auth/cloud-platform'])
self.NLPClient = language.LanguageServiceClient(credentials=cred)
# Instantiate CMU Pronunciation Dictionary
try:
self.dict = cmudict.dict()
except:
print "Downloading cmudict..."
nltk.download('cmudict')
print "Downloaded cmudict."
self.dict = cmudict.dict()
# Attempt to create databases
self.setUpDatabase()
class Geo:
def __init__(self, API_KEY):
self.google_maps = GoogleMaps(api_key=API_KEY)
def processLocation(self, location):
"""Converts the given Address object into JSON"""
temp = location.__dict__
aa = temp['_administrative_area']
del temp['_administrative_area']
admin_areas = []
for admin_area in aa:
this_admin = {
'type': admin_area.area_type,
'name': admin_area.name,
'short_name': admin_area.short_name
}
admin_areas.append(this_admin)
temp["administrative_areas"] = admin_areas
return temp
def geo_from_address(self, address):
"""Gets full address details for the given address string. Returns JSON data"""
locations = self.google_maps.search(
location=address) # sends search to Google Maps.
result = []
for location in locations.all():
this_result = self.processLocation(location)
result.append(this_result)
return result
def address_from_geo(self, lat, lng):
"""Gets full address details for the given long/lat location. Returns JSON data"""
locations = self.google_maps.search(
lat=lat, lng=lng).all() # sends search to Google Maps.
result = []
for location in locations:
this_result = self.processLocation(location)
result.append(this_result)
return result
def inti():
try:
origins = raw_input("[*] Route Dari : ")
destinations = raw_input("[*] Route Tujuan : ")
time.sleep(1)
google_maps = GoogleMaps(
api_key='AIzaSyDGJVf8GBXp0UvNcF2hG_eyaSfEsobzHrA')
items = google_maps.distance(origins, destinations).all()
time.sleep(1)
print '[*] Searching location ...'
time.sleep(3)
for item in items:
print
print '[*] Route Dari : %s ' % item.origin
time.sleep(1)
print '[*] Route Tujuan : %s ' % item.destination
time.sleep(1)
print '[*] Jarak Kilometer : %s KM' % item.distance.kilometers
print '[*] Jarak Meter : %s M' % item.distance.meters
print '[*] Jarak Mil : %s Mil' % item.distance.miles
print '[*] Durasi Perjalanan : %s ' % item.duration
print '[*] Durasi Waktu : %s ' % item.duration.datetime
print '[*] Durasi Hari : %s ' % item.duration.days
print '[*] Durasi Jam : %s ' % item.duration.hours
print '[*] Durasi menit : %s ' % item.duration.minutes
print '[*] Durasi detik : %s \n' % item.duration.seconds
next = raw_input('[*] Tunjukan Mode Lain nya (yes/no) : ')
yes = "yes"
if next == yes:
time.sleep(2)
print '[*] Mode Lainnya Diterima\n '
else:
time.sleep(2)
print '[*] Mode lainnya Ditolak '
sys.exit()
except KeyboardInterrupt:
print "\n[*] System Control + C"
sys.exit()
except TypeError:
print "[*] System Error code: 0x6572726f722073797374656d"
sys.exit()
def on_data(self, data):
#Decode json data
tweet = json.loads(data)
username = tweet['user']['screen_name']
location_address = tweet['user']['location']
tweet_text = tweet['text'].encode('ascii', 'ignore')
processed_tweet = preprocess(tweet_text)
if processed_tweet != 1:
try:
if location_address and location_address is not None:
location_address.replace(" ", "+")
google_maps = GoogleMaps(
api_key='AIzaSyCoMZR7wii10qjJarohY7Ru__9cxC8Ftgw')
geo_location = google_maps.search(
location=location_address)
location = geo_location.first()
if location and location is not None:
latitude = location.lat
longitude = location.lng
print("Original token list:", tweet_text)
print("Location: ", location_address)
print("New token list:", processed_tweet)
print "Latitude: {}, Longitude= {} ".format(
latitude, longitude)
#print("Sentiment: ", sentiment)
tweet = " ".join(tweet_text.split())
processed_tweet = (processed_tweet + "\t" +
str(latitude) + "\t" +
str(longitude) + "\t" + tweet +
"\n")
conn.send(processed_tweet.encode('utf-8'))
else:
print("No location returned from the API!")
else:
print("No location data associated with the tweet!")
except:
print("Couldn't find location of the tweet!")
import sys
from multiprocessing import Pool
from functools import reduce
from geolocation.main import GoogleMaps
import pandas as pd
google_maps = GoogleMaps(api_key='your_google_maps_key')
key = 'Mailing Address'
def get_location_model(row):
address = row[key]
try:
print('Searching address: ', address)
return (row, google_maps.search(location=address).first())
except Exception:
print('Failed for: ', row['First Name'])
def to_df(lm):
if lm is None:
return
row, location = lm
street, city, state_zip, *country = location.formatted_address.split(", ")
state_zip = state_zip.split(" ")
postal_code = location.postal_code.decode(
# -*- coding: utf-8 -*-
from geolocation.main import GoogleMaps
from geolocation.distance_matrix.client import DistanceMatrixApiClient
if __name__ == "__main__":
origins = ['rybnik', 'oslo']
destinations = ['zagrzeb']
google_maps = GoogleMaps(api_key='your_google_maps_key')
items = google_maps.distance(origins, destinations).all() # default mode parameter is const.MODE_DRIVING
for item in items:
print('origin: %s' % item.origin)
print('destination: %s' % item.destination)
print('km: %s' % item.distance.kilometers)
print('m: %s' % item.distance.meters)
print('miles: %s' % item.distance.miles)
print('duration: %s' % item.duration) # it returns str
print('duration datetime: %s' % item.duration.datetime) # it returns datetime
# you can also get items from duration
print('duration days: %s' % item.duration.days)
print('duration hours: %s' % item.duration.hours)
print('duration minutes: %s' % item.duration.minutes)
print('duration seconds: %s' % item.duration.seconds)
items = google_maps.distance(origins, destinations, DistanceMatrixApiClient.MODE_BICYCLING).all()
for item in items:
print('origin: %s' % item.origin)
# -*- coding: utf-8 -*-
from geolocation.main import GoogleMaps
if __name__ == "__main__":
address = "New York City"
google_maps = GoogleMaps(api_key="your_google_maps_key")
lat = 40.7060008
lng = -74.0088189
location = google_maps.search(lat=lat, lng=lng)
print(location.all())
my_location = location.first()
print(my_location.city)
print(my_location.route)
print(my_location.street_number)
print(my_location.postal_code)
for administrative_area in my_location.administrative_area:
print("%s: %s" % (administrative_area.area_type, administrative_area.name))
print(my_location.country)
print(my_location.country_shortcut)
print(my_location.formatted_address)
print(my_location.lat)
def send_data():
global m1,m2
addr = Entry1.get()
print(addr)
quant = Scale1.get()
print(quant)
#top.withdraw()
#top1.deiconify()
'''
import urllib2
proxy = urllib2.ProxyHandler({'http':'edcguest:[email protected]:3128','https':'edcguest:[email protected]:3128'})
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
'''
address = addr
print(address)
google_maps = GoogleMaps(api_key=' AIzaSyD4QiVK9WflR5GCkzuks27x55V2_v5CX2k ')
print(address)
location = google_maps.search(location=address) # sends search to Google Maps.
print(location.all()) # returns all locations.
my_location = location.first() # returns only first location.
print(my_location.city)
print(my_location.route)
print(my_location.street_number)
print(my_location.postal_code)
for administrative_area in my_location.administrative_area:
print("{}: {} ({})".format(administrative_area.area_type,
administrative_area.name,
administrative_area.short_name))
print(my_location.country)
print(my_location.country_shortcut)
print(my_location.formatted_address)
print(my_location.lat)
print(my_location.lng)
#host1 = socket.gethostname()
host1 = '172.20.53.93'
port = 2004
BUFFER_SIZE = 2000
print(socket.gethostbyname(host1))
#host1 = '192.168.225.26'
m1 = str(my_location.lat) + ' '
m2 = str(my_location.lng)
MESSAGE = m1 + m2 + ' ' + str(quant)
print(MESSAGE)
tcpClientA = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcpClientA.connect((host1, port))
tcpClientA.send(MESSAGE)
tcpClientA.close()
top.withdraw()
top1.deiconify()
from geolocation.main import GoogleMaps
address = 'São Luís Maranhão'
google_maps = GoogleMaps(api_key='AIzaSyBCksh0B58c_C6k_Epm2k1ZQb-YF6kA6SE')
location = google_maps.search(location=address)
my_location = location.first()
if my_location.city:
print(my_location.city.decode('utf-8'))
if my_location.route:
print(my_location.route.decode('utf-8'))
if my_location.street_number:
print(my_location.street_number)
if my_location.postal_code:
print(my_location.postal_code)
for administrative_area in my_location.administrative_area:
print("%s: %s" % (administrative_area.area_type,
administrative_area.name.decode('utf-8')))
if my_location.country:
print(my_location.country.decode('utf-8'))
print(my_location.country_shortcut)
print(my_location.formatted_address)
print(my_location.lat)
print(my_location.lng)
# -*- coding: utf-8 -*-
from geolocation.main import GoogleMaps
if __name__ == "__main__":
address = "New York City"
google_maps = GoogleMaps(api_key='your_google_maps_key')
location = google_maps.search(location=address)
print(location.all())
my_location = location.first()
print(my_location.city)
print(my_location.route)
print(my_location.street_number)
print(my_location.postal_code)
for administrative_area in my_location.administrative_area:
print("%s: %s" %
(administrative_area.area_type, administrative_area.name))
print(my_location.country)
print(my_location.country_shortcut)
print(my_location.formatted_address)
print(my_location.lat)
print(my_location.lng)
class TwitterTransformer():
def __init__(self):
self.gender_detector = GenderDetector()
self.googlemaps_api = GoogleMaps(
api_key=app_settings.SERVICES_CREDENTIALS['google_api_key'])
def process(self, ds, **kwargs):
raw_records = self.__fetch_tweets()
print "{} new tweets have been analyzed".format(len(raw_records))
conn_db_lake = db_handler.get_connection('data_lake')
cur_db_lake = conn_db_lake.cursor()
for record in raw_records:
tweet = record[2]
clean_tweet = self.__tweet_cleaner(tweet['text'])
print clean_tweet
polarity, sentiment = self.__get_sentiment(clean_tweet)
coordinates = self.__get_go_points(tweet['user']['location'])
gender = self.__guess_gender(tweet['user']['name'].split()[0])
tweet_tokens = self.__tokenizer(clean_tweet)
processed_tweet = {
"author": tweet["user"]["screen_name"],
"tweet_geo": tweet['geo'],
"tweet_lang": tweet['lang'],
"tweet_place": tweet['place'],
"user_description": tweet['user']['description'],
"user_followers_count": tweet['user']['followers_count'],
"user_friends_count": tweet['user']['friends_count'],
"user_lang": tweet['user']['lang'],
"user_name": tweet['user']['name'],
"user_location_name": tweet['user']['location'],
"user_location_coordinate": {
"lat": coordinates[0],
"lon": coordinates[1]
} if coordinates else None,
"user_status_count": tweet['user']['statuses_count'],
"tweet_created_at": str(parser.parse(tweet['created_at'])),
"user_created_at":
str(parser.parse(tweet['user']['created_at'])),
"tweet_tokens": tweet_tokens,
'bigrams': ["_".join(x) for x in bigrams(tweet_tokens)],
'trigrams': ["_".join(x) for x in trigrams(tweet_tokens)],
"polarity": polarity,
"sentiment": sentiment,
"gender": gender,
}
try:
update_query = """
UPDATE records
SET is_analyzed=TRUE
WHERE id={};
""".format(record[0])
query = """INSERT INTO tweets (data, created_at) VALUES ('{}', '{}')""".format(
json.dumps(processed_tweet).replace("'", "''"), record[3])
cur_db_lake.execute(query)
cur_db_lake.execute(update_query)
conn_db_lake.commit()
except Exception as ex:
conn_db_lake.rollback()
raise ex
def __fetch_tweets(self):
try:
conn_db_lake = db_handler.get_connection('data_lake')
cur_db_lake = conn_db_lake.cursor()
query = """
SELECT * FROM records
WHERE type='tweet' AND is_analyzed = false
"""
cur_db_lake.execute(query)
return cur_db_lake.fetchall()
except Exception as ex:
conn_db_lake.rollback()
raise ex
def __guess_gender(self, name):
gender = None
try:
gender = self.gender_detector.guess(name)
return gender
except Exception as e:
print('error in gender detector')
def __get_go_points(self, address):
if not address:
return None
coordinate = None
try:
res = self.googlemaps_api.search(
address.strip(string.punctuation + ' ')).first()
if res:
coordinate = [res.lat, res.lng]
except Exception as ex:
print("Err in geo location convertor")
return coordinate
def __tweet_cleaner(self, tweet):
# Convert to lower case
tweet = tweet.lower()
# Convert www.* or https?://* to empty string
tweet = re.sub('((www\.[\s]+)|(https?://[^\s]+))', '', tweet)
# Convert @username to empty string
tweet = re.sub('@[^\s]+', '', tweet)
# Remove additional white spaces
tweet = re.sub('[\s]+', ' ', tweet)
# Replace #word with word
tweet = re.sub(r'#([^\s]+)', r'\1', tweet)
# trim
tweet = tweet.strip('\'"')
return tweet
def __get_sentiment(self, tweet):
res = TextBlob(tweet)
polarity = res.sentiment.polarity
if polarity < 0:
sentiment = 'negative'
elif polarity == 0:
sentiment = 'neutral'
else:
sentiment = 'positive'
return (polarity, sentiment)
def __tokenizer(self, tweet):
tokens = []
for word in tweet.split():
if len(word) > 3 and word not in stopwords.words(
'english') and wordnet.synsets(word):
tokens.append(word)
return list(set(tokens))
from geolocation.main import GoogleMaps
address = "Thane Wood Street Kenora "
google_maps = GoogleMaps(api_key='AIzaSyCIIOxzvcIvagBNbLs7xN_WR3rYPnEf-0A')
location = google_maps.search(location=address) # sends search to Google Maps.
print(location.all()) # returns all locations.
my_location = location.first() # returns only first location.
print(my_location.city)
print(my_location.route)
print(my_location.street_number)
print(my_location.postal_code)
for administrative_area in my_location.administrative_area:
print("{}: {} ({})".format(administrative_area.area_type,
administrative_area.name,
administrative_area.short_name))
print(my_location.country)
print(my_location.country_shortcut)
print(my_location.formatted_address)
print(my_location.lat)
print(my_location.lng)
# reverse geocode
import csv
from operator import itemgetter
from geolocation.main import GoogleMaps
ifile = open('crash.csv', 'r')
reader = csv.DictReader(ifile)
default = 0
google_maps = GoogleMaps(api_key='AIzaSyARs71g1TIgn9VYqK4xmdLN64AbwZxBXn0')
error = 0
breakvar = 0
total = 0
listLat = []
listLng = []
csvfile = open('sandend.csv', 'w')
fieldnames = ['slatitude', 'slongitude', 'elatitude', 'elongitude']
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
for row in reader:
try:
location1 = google_maps.search(location=((row['Route']).split('-'))[0])
my_location1 = location1.first()
location2 = google_maps.search(location=((row['Route']).split('-'))[1])
my_location2 = location2.first()
writer.writerow({
'slatitude': my_location1.lat,
'slongitude': my_location1.lng,
'elatitude': my_location2.lat,
class GeolocationTest(unittest.TestCase):
def setUp(self):
self.google_maps = GoogleMaps(api_key=TEST_API_KEY)
def test_query(self):
address = "New York City Wall Street 15"
location = self.google_maps.search(address)
self.assertIsNotNone(location.all())
def test_city(self):
address = "New York City Wall Street 14"
location = self.google_maps.search(address)
my_location = location.first()
self.assertEqual('New York', my_location.city.decode('utf-8'))
def test_route(self):
address = "New York City Wall Street 12"
location = self.google_maps.search(address)
my_location = location.first()
self.assertEqual('Wall Street', my_location.route.decode('utf-8'))
def test_country(self):
address = "New York City Wall Street 110"
location = self.google_maps.search(address)
my_location = location.first()
self.assertEqual('United States', my_location.country.decode('utf-8'))
def test_country_shortcut(self):
address = "New York City Wall Street 2"
location = self.google_maps.search(address)
my_location = location.first()
self.assertEqual('US', my_location.country_shortcut.decode('utf-8'))
def test_lat(self):
address = "New York City Wall Street 1"
location = self.google_maps.search(address)
my_location = location.first()
self.assertAlmostEqual(40.7060081, my_location.lat, 2)
def test_lng(self):
address = "New York City Wall Street 19"
location = self.google_maps.search(address)
my_location = location.first()
self.assertAlmostEqual(-74.0134436, my_location.lng, 2)
def test_formatted_address(self):
address = "New York City Wall Street 124"
location = self.google_maps.search(address)
my_location = location.first()
self.assertEqual('Charging Bull, Broadway, New York, NY 10004, USA',
my_location.formatted_address)
def test_administrative_area_level_1(self):
address = "New York City Wall Street 125"
location = self.google_maps.search(address)
my_location = location.first()
self.assertEqual(
'New York',
my_location.administrative_area[0].name.decode('utf-8'))
self.assertEqual(
'NY',
my_location.administrative_area[0].short_name.decode('utf-8'))
def test_administrative_area_level_2(self):
address = "New York City Wall Street 126"
location = self.google_maps.search(address)
my_location = location.first()
self.assertEqual(
'New York County',
my_location.administrative_area[1].name.decode('utf-8'))
self.assertEqual(
'New York County',
my_location.administrative_area[1].short_name.decode('utf-8'))
def test_coding(self):
address = "São Paulo"
my_location = self.google_maps.search(address).first()
self.assertEqual(u"São Paulo", my_location.city.decode('utf-8'))
def test_latlng(self):
lat = 37.4229210
lng = -122.0852112
my_location = self.google_maps.search(lat=lat, lng=lng).first()
self.assertEqual('New York', my_location.city.decode('utf-8'))
def test_administrative_area_resets(self):
address = "São Paulo"
sao_paulo = self.google_maps.search(address).first()
address = "Houston, TX"
houston = self.google_maps.search(address).first()
self.assertNotEqual(sao_paulo, houston)
from geolocation.main import GoogleMaps
address = "Thane Wood Street Kenora "
google_maps = GoogleMaps(api_key='AIzaSyCIIOxzvcIvagBNbLs7xN_WR3rYPnEf-0A')
location = google_maps.search(location=address) # sends search to Google Maps.
print(location.all()) # returns all locations.
my_location = location.first() # returns only first location.
print(my_location.city)
print(my_location.route)
print(my_location.street_number)
print(my_location.postal_code)
for administrative_area in my_location.administrative_area:
print("{}: {} ({})".format(administrative_area.area_type,
administrative_area.name,
administrative_area.short_name))
print(my_location.country)
print(my_location.country_shortcut)
print(my_location.formatted_address)
print(my_location.lat)
print(my_location.lng)
# reverse geocode
def setUp(self):
self.google_maps = GoogleMaps(api_key=TEST_API_KEY)
self.duration_regex = r'([0-9]*)d ([0-9]*)h ([0-9]*)m ([0-9]*)s'
self.delta_km = 25
self.delta_m = 25000
self.delta_miles = 25
class DistanceMatrixTest(unittest.TestCase):
def setUp(self):
self.google_maps = GoogleMaps(api_key=TEST_API_KEY)
self.duration_regex = r'([0-9]*)d ([0-9]*)h ([0-9]*)m ([0-9]*)s'
self.delta_km = 25
self.delta_m = 25000
self.delta_miles = 25
def test_distance_matrix(self):
origins = ['rybnik', 'oslo']
destinations = ['zagrzeb']
items = self.google_maps.distance(origins, destinations).all()
for item in items:
if item.origin == 'Rybnik, Poland':
self.assertEqual(item.destination, 'Zagreb, Croatia')
self.assertAlmostEqual(Decimal(709), item.distance.kilometers, delta=self.delta_km)
self.assertAlmostEqual(Decimal(713000), item.distance.meters, delta=self.delta_m)
self.assertAlmostEqual(Decimal(443.0368), item.distance.miles, delta=self.delta_miles)
self.assertRegexpMatches(str(item.duration), self.duration_regex)
if item.origin == 'Oslo, Norway':
self.assertEqual(item.destination, 'Zagreb, Croatia')
self.assertAlmostEqual(Decimal(2063), item.distance.kilometers, delta=self.delta_km)
self.assertAlmostEqual(Decimal(2063000), item.distance.meters, delta=self.delta_m)
self.assertAlmostEqual(Decimal(1281.8863), item.distance.miles, delta=self.delta_miles)
self.assertRegexpMatches(str(item.duration), self.duration_regex)
def test_distance_matrix_bicycling(self):
origins = ['rybnik']
destinations = ['oslo']
item = self.google_maps.distance(origins, destinations, DistanceMatrixApiClient.MODE_BICYCLING).first()
self.assertEqual(item.origin, 'Rybnik, Poland')
self.assertEqual(item.destination, 'Oslo, Norway')
self.assertAlmostEqual(Decimal(1596), item.distance.kilometers, delta=self.delta_km)
self.assertAlmostEqual(Decimal(1596000), item.distance.meters, delta=self.delta_m)
self.assertAlmostEqual(Decimal(991.7065), item.distance.miles, delta=self.delta_miles)
self.assertRegexpMatches(str(item.duration), self.duration_regex)
def test_distance_matrix_walking(self):
origins = ['rybnik']
destinations = ['oslo']
item = self.google_maps.distance(origins, destinations, DistanceMatrixApiClient.MODE_WALKING).first()
self.assertEqual(item.origin, 'Rybnik, Poland')
self.assertEqual(item.destination, 'Oslo, Norway')
self.assertAlmostEqual(Decimal(1380), item.distance.kilometers, delta=self.delta_km)
self.assertAlmostEqual(Decimal(1380000), item.distance.meters, delta=self.delta_m)
self.assertAlmostEqual(Decimal(857.4906), item.distance.miles, delta=self.delta_miles)
self.assertRegexpMatches(str(item.duration), self.duration_regex)
def test_distance_matrix_avoid_tolls(self):
origins = ['rybnik']
destinations = ['oslo']
item = self.google_maps.distance(origins, destinations, avoid=DistanceMatrixApiClient.AVOID_TOLLS).first()
self.assertEqual(item.origin, 'Rybnik, Poland')
self.assertEqual(item.destination, 'Oslo, Norway')
self.assertAlmostEqual(Decimal(1542), item.distance.kilometers, delta=self.delta_km)
self.assertAlmostEqual(Decimal(1542000), item.distance.meters, delta=self.delta_m)
self.assertAlmostEqual(Decimal(958.1525), item.distance.miles, delta=self.delta_miles)
self.assertRegexpMatches(str(item.duration), self.duration_regex)
def test_distance_matrix_avoid_highways(self):
origins = ['rybnik']
destinations = ['oslo']
item = self.google_maps.distance(origins, destinations, avoid=DistanceMatrixApiClient.AVOID_HIGHWAYS).first()
self.assertEqual(item.origin, 'Rybnik, Poland')
self.assertEqual(item.destination, 'Oslo, Norway')
self.assertAlmostEqual(Decimal(1542), item.distance.kilometers, delta=self.delta_km)
self.assertAlmostEqual(Decimal(1542000), item.distance.meters, delta=self.delta_m)
self.assertAlmostEqual(Decimal(958.1525), item.distance.miles, delta=self.delta_miles)
self.assertRegexpMatches(str(item.duration), self.duration_regex)
def test_distance_matrix_avoid_ferries(self):
origins = ['rybnik']
destinations = ['oslo']
item = self.google_maps.distance(origins, destinations, avoid=DistanceMatrixApiClient.AVOID_FERRIES).first()
self.assertEqual(item.origin, 'Rybnik, Poland')
self.assertEqual(item.destination, 'Oslo, Norway')
self.assertAlmostEqual(Decimal(1851), item.distance.kilometers, delta=self.delta_km)
self.assertAlmostEqual(Decimal(1851000), item.distance.meters, delta=self.delta_m)
self.assertAlmostEqual(Decimal(1150.1559), item.distance.miles, delta=self.delta_miles)
self.assertRegexpMatches(str(item.duration), self.duration_regex)
def setUp(self):
self.google_maps = GoogleMaps(api_key=TEST_API_KEY)
def testRequests(decrypt):
address = decrypt
google_maps = GoogleMaps(api_key="AIzaSyAEk_XVCNBK9RavS6KiXXfQW-fRvMzKt5o")
location = google_maps.search(location=address)
print(location.all())
class EventEyeSpider(scrapy.Spider):
google_maps = ''
month = []
name = "aiche"
allowed_domains = ["aiche.org"]
start_urls = ['http://www.aiche.org/resources/conferences']
def __init__(self, **kwargs):
super(EventEyeSpider, self).__init__(**kwargs)
self.init_Month()
self.google_maps = GoogleMaps(api_key='AIzaSyCaxLzZ2r7AbCJiIy5RtJ4jOQXcOlDbeV0')
def parse(self, response):
titles = response.css('div.view-content div.views-row article h3 a::text').extract()
urls = response.css('div.view-content div.views-row article h3 a::attr(href)').extract()
for num in range(len(urls)):
url = urls[num]
item = EventItem()
complete_url = url
item['url'] = 'http://www.aiche.org' + complete_url
item['title'] = titles[num]
request = scrapy.Request(item['url'], callback=self.parse_event, meta={'item': item})
yield request
# break
def parse_event(self, response):
sel = response.css('html')
content = sel.css('div.conference-lead-info')
item = response.meta['item']
item['source'] = response.request.url
item['event_website'] = ''
# item['title'] = content.css('div.conference-lead-info h1.title::text').extract()
try:
item['description'] = content.css('div.field-name-body div.field-item p:first-child::text').extract()[0].strip()
except Exception as e:
item['description'] = ''
pass
item['addressRegion'] = ''
item['postalCode'] = ''
item['price_range'] = ''
item['addressLocality'] = content.css('div.field-name-field-conf-venues a::text').extract()[0].strip()
item['location'] = ''
item['category'] = 'Conference'
item['industry'] = 'Event'
item['date'] = content.css('div.conference-dates div div::text').extract()[1]
self.get_event_date(item)
loc = self.google_maps.search(location=item['addressLocality'])
my_location = loc.first()
try:
item['country_code'] = my_location.country_shortcut
item['addressCountry'] = my_location.country_shortcut
item['city'] = my_location.city
item['longitude'] = my_location.lng
item['latitude'] = my_location.lat
item['streetAddress'] = my_location.formatted_address
except Exception as e:
item['country_code'] = ''
item['city'] = ''
item['longitude'] = ''
item['latitude'] = ''
item['streetAddress'] = ''
pass
yield item
def init_Month(self):
self.month = {'January': '01', 'February': '02', 'March': '03', 'April': '04', 'May': '05', 'June': '06',
'July': '07', 'August': '08', 'September': '09', 'October': '10', 'November': '11',
'December': '12'}
def get_event_date(self, item):
item['event_start'] = '0000-00-00 00:00:0'
item['event_end'] = '0000-00-00 00:00:0'
raw_date = item['date']
try:
year_date = raw_date.split(',')
month_days = (year_date[0]).split('-')
from_date = (month_days[0]).split(' ')
from_month = (from_date[0]).strip()
from_day = (from_date[1]).strip()
to_date = (month_days[1]).split(' ')
if len(to_date) == 1:
to_day = (to_date[0]).strip()
to_month = from_month
else:
to_month = (to_date[0]).strip()
to_day = (to_date[1]).strip()
year = (year_date[1]).strip()
from_month = self.month[from_month]
to_month = self.month[to_month]
from_date = year + '-' + from_month + '-' + from_day + 'T10:00:00'
to_date = year + '-' + to_month + '-' + to_day + 'T10:00:00'
print from_date + ' '
print to_date + ' '
event_start = datetime.strptime(from_date, "%Y-%m-%dT%H:%M:%S").strftime("%Y-%m-%d %H:%M:%S")
event_end = datetime.strptime(to_date, "%Y-%m-%dT%H:%M:%S").strftime("%Y-%m-%d %H:%M:%S")
item['event_start'] = event_start
item['event_end'] = event_end
except Exception as e:
item['event_start'] = '0000-00-00 00:00:0'
item['event_end'] = '0000-00-00 00:00:0'
pass
from datetime import datetime, timedelta
from pytz import timezone
from geolocation.main import GoogleMaps
from credentials import *
import threading
import time
import random
import requests
import tweepy
google_maps = GoogleMaps(api_key='')
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)
agencyids = [""]
TWEET_SLEEP_TIME = 10
PP_API_SLEEP = 5
MINUTES_REMOVE = 5
currtime = datetime.now()
incidents = []
def grab_pulsepoint(agencyid):
url = "https://api.pulsepoint.org/v1/incidents"
pp_json = None
while pp_json is None:
try:
pp_json = requests.get(url,
headers=random_headers(),
params=(
('apikey', ''),
class ConferenceAlertsSpider(scrapy.Spider):
urls = ''
google_maps = ''
month = []
name = "conf"
allowed_domains = ["conferencealerts.com"]
start_urls = []
def __init__(self, **kwargs):
super(ConferenceAlertsSpider, self).__init__(**kwargs)
self.init_month()
self.urls = self.get_dict('assets\conf_start_urls.json')
for url in self.urls:
self.start_urls = self.start_urls + [url['url']]
self.google_maps = GoogleMaps(api_key='AIzaSyCaxLzZ2r7AbCJiIy5RtJ4jOQXcOlDbeV0')
def parse(self, response):
sel = response.css('table#searchResultTable tr td')
title = sel.css('span#searchName a::text').extract()
source = sel.css('span#searchName a::attr(href)').extract()
location = sel.css('span#searchPlace::text').extract()
country = sel.css('span#searchPlace span::text').extract()
for num in range(0, len(title)):
item = EventItem()
item['url'] = response.request.url
item['source'] = 'http://conferencealerts.com/' + source[num]
item['title'] = title[num]
item['category'] = 'Conference'
item['industry'] = item['category']
item['location'] = location[num]
item['addressCountry'] = country[num]
item['addressLocality'] = location[num] + '' + country[num]
item['addressRegion'] = ''
item['price_range'] = ''
request = scrapy.Request(item['source'], callback=self.parse_event,
meta={'item': item})
yield request
def parse_event(self, response):
item = response.meta['item']
sel = response.css('div#eventInfoContainer table')
description = sel.css('span#eventDescription::text').extract()
event_website = sel.css('span#eventWebsite a::attr(href)').extract()
date = sel.css('span#eventDate::text').extract()
item['description'] = description[0]
item['event_website'] = event_website[0]
item['date'] = date[0]
item['postalCode'] = ''
self.get_datetime(item)
loc = self.google_maps.search(location=item['addressLocality'])
my_location = loc.first()
try:
item['country_code'] = my_location.country_shortcut
item['longitude'] = my_location.lng
item['latitude'] = my_location.lat
item['streetAddress'] = my_location.formatted_address
except Exception as e:
item['country_code'] = ''
item['longitude'] = ''
item['latitude'] = ''
item['streetAddress'] = ''
pass
yield item
def print_start_urls(self, response):
for href in response.css('div.topicTableColumn1Container table td, div.topicTableColumn2Container table td'):
url = href.css('a::attr(href)').extract()
if len(url) == 0 or url[0] == '#top':
continue
url = 'http://conferencealerts.com/' + url[0]
item = EventItem()
item['url'] = url
yield item
def get_dict(self, path):
with open(path, 'r') as f:
data = json.load(f)
return data
def init_month(self):
self.month = {'January': '01', 'February': '02', 'March': '03', 'April': '04', 'May': '05', 'June': '06',
'July': '07', 'August': '08', 'September': '09', 'October': '10', 'November': '11',
'December': '12'}
def get_datetime(self, item):
date = item['date']
try:
tokens = date.split(' ')
from_day = (tokens[0]).strip()
from_day = from_day[0:1]
to_day = (tokens[4]).strip()
to_day = to_day[0:1]
year = tokens[6]
month = tokens[5]
month = month.strip()
month = self.month[month]
print from_day + ' ' + to_day + ' ' + month + ' ' + year
from_date = year + '-' + month + '-' + from_day + 'T10:00:00'
to_date = year + '-' + month + '-' + to_day + 'T10:00:00'
event_start = datetime.strptime(from_date, "%Y-%m-%dT%H:%M:%S").strftime("%Y-%m-%d %H:%M:%S")
event_end = datetime.strptime(to_date, "%Y-%m-%dT%H:%M:%S").strftime("%Y-%m-%d %H:%M:%S")
item['event_start'] = event_start
item['event_end'] = event_end
except Exception as e:
item['event_start'] = '0000-00-00 00:00:0'
item['event_end'] = '0000-00-00 00:00:0'
pass
from geolocation.main import GoogleMaps
google_maps = GoogleMaps(api_key='AIzaSyBCksh0B58c_C6k_Epm2k1ZQb-YF6kA6SE')
lat = -2.500044
lng = -44.288093
location = google_maps.search(lat=lat, lng=lng)
my_location = location.first()
if my_location.city: print(my_location.city.decode('utf-8'))
if my_location.route: print(my_location.route.decode('utf-8'))
if my_location.street_number: print(my_location.street_number)
if my_location.postal_code: print(my_location.postal_code)
print(my_location.country)
print(my_location.country_shortcut)
print(my_location.formatted_address)
class GeolocationFinder:
"""
Class for retrieving geolocation when given an address.
Class creates a cache of geolocations to minimise calls to geopy library, reducing the risk of hitting rate limit
"""
def __init__(self):
self.location_cache = {}
self.geolocator = Nominatim()
self.geolocation = None
self.backup_geolocator = GoogleMaps(api_key=google_map_api_key)
self.lat = None
self.long = None
def get_location(self, location):
"""
gets location attributes give a location
:param location: string
:return: address:string, latitude:float, longitude:float
"""
if location is None:
return None, None, None
elif location in self.location_cache:
# Check cache for location
self.set_geolocation_from_cache(location)
address, latitude, longitude = self.get_addr_lat_long()
return address, latitude, longitude
else:
# Location not cached so fetch from geolocator
self.set_geolocation_from_geolocator(location)
if self.geolocation is not None or self.lat is not None:
address, latitude, longitude = self.get_addr_lat_long()
return address, latitude, longitude
return None, None, None
def set_geolocation_from_cache(self, location):
"""
sets class geolocation attribute to geolocation from the cache
:param location: string
"""
self.geolocation = self.location_cache[location]
def set_geolocation_from_geolocator(self, location):
"""
Sets class geolocation attribute to geolocation from the geopy library and adds new eolocation to cache
:param location: string
"""
self.geolocation = None
self.lat = None
self.long = None
try:
self.geolocation = self.geolocator.geocode(location, timeout=None)
except GeocoderTimedOut:
logger.logging.warning('GeolocationFinder: geolocator timeout')
print("geologger timeout")
except GeocoderUnavailable:
print("geologger unavailable")
except GeocoderInsufficientPrivileges:
# geopy doesnt work: go to backup geolocation finder
logger.logging.error("GeolocationFinder: Geocoder insufficient privileges")
self.getGeolocationInfoFromBackupGeolocator(location)
if self.geolocation is not None:
self.location_cache[location] = self.geolocation
def getGeolocationInfoFromBackupGeolocator(self, location):
try:
google_geolocation = self.backup_geolocator.search(location=location)
loc = google_geolocation.first()
self.lat = loc.lat
self.long = loc.lng
self.address = location
except:
self.geolocation = None
def get_addr_lat_long(self):
"""
Gets address, latitude, longitude from the class geolocation attribute
:return: address:string, latitude:float, longitude:float
"""
try:
if self.geolocation is None:
address = self.address
latitude = self.lat
longitude = self.long
else:
address = self.geolocation.address
latitude = self.geolocation.latitude
longitude = self.geolocation.longitude
except AttributeError:
logger.logging.exception('Attribute Error:GeolocationFinder.get_addr_lat_long in geolocation_finder')
address, latitude, longitude = None, None, None
self.geolocation = None
return address, latitude, longitude
def __init__(self):
self.gender_detector = GenderDetector()
self.googlemaps_api = GoogleMaps(
api_key=app_settings.SERVICES_CREDENTIALS['google_api_key'])
from webwhatsapi.objects.message import Message
from bs4 import BeautifulSoup
from geolocation.main import GoogleMaps
from csv import DictReader
# Create SQS client
sqs = boto3.client('sqs')
queue_url = 'https://us-west-2.queue.amazonaws.com/362759655961/sendQueue'
s_queue_url = 'https://sqs.us-west-2.amazonaws.com/362759655961/sendMessage.fifo'
r_queue_url = 'https://sqs.us-west-2.amazonaws.com/362759655961/recieveSQS.fifo'
conn = psycopg2.connect(database="postgres", user = "******", password = "******", host = "127.0.0.1", port = "5432")
print "Opened database successfully"
cur = conn.cursor()
google_maps = GoogleMaps(api_key='AIzaSyDNw4OLKRESwCrp1lLU6ojq9paZ85ZSqqs')
driver = WhatsAPIDriver(client="chrome")
print("Waiting for QR")
driver.wait_for_login()
print("Bot started")
def send():
msg = []
#pdb.set_trace()
try:
while(True):
pdb.set_trace()
response = sqs.receive_message(QueueUrl=s_queue_url,AttributeNames=['All'],MessageAttributeNames=['All'],MaxNumberOfMessages=1,VisibilityTimeout=10)['Messages']
try:
# -*- coding: utf-8 -*-
from geolocation.main import GoogleMaps
if __name__ == "__main__":
address = "Wall Street 12"
google_maps = GoogleMaps(api_key='your_google_maps_key')
location = google_maps.search(location=address)
print(location.all())
my_location = location.first()
print(my_location.city)
print(my_location.route)
print(my_location.street_number)
print(my_location.postal_code)
for administrative_area in my_location.administrative_area:
print("%s: %s" % (administrative_area.area_type, administrative_area.name))
print(my_location.country)
print(my_location.country_shortcut)
print(my_location.formatted_address)
print(my_location.lat)
print(my_location.lng)
import random import re import requests import sys auth_token = environ['EVENTBRITE_OAUTH_TOKEN'] pusher_app_id = environ['PUSHER_APP_ID'] pusher_key = environ['PUSHER_KEY'] pusher_secret = environ['PUSHER_SECRET'] mu_token = environ['MEETUP_API_KEY'] geo_code = environ['GEOCODE_API_KEY'] # Instantiate the Eventbrite and Meetup API clients. eventbrite = eventbrite.Eventbrite(auth_token) meetup = meetup.api.Client(mu_token) google_maps = GoogleMaps(api_key=geo_code) geo_api = geo_code compress = Compress() app = Flask(__name__) Compress(app) app.debug = False app.jinja_env.undefined = StrictUndefined app.secret_key = "leisure" MEETUP_IMG_URL = "static/assets/meetup_logo.png" EVBRTE_IMG_URL = "static/assets/eb_logo.jpg" # ----------------------------- ROUTES ---------------------------------- #
def __init__(self, **kwargs):
super(EventEyeSpider, self).__init__(**kwargs)
self.init_Month()
self.google_maps = GoogleMaps(api_key='AIzaSyCaxLzZ2r7AbCJiIy5RtJ4jOQXcOlDbeV0')
class EventEyeSpider(scrapy.Spider):
google_maps = ''
month = []
name = "eventeye"
allowed_domains = ["eventseye.com"]
# start_urls = ['http://www.eventseye.com/fairs/Upcoming_Trade_Shows.html']
start_urls = ['http://www.eventseye.com/fairs/d1_trade-shows_january_2016.html']
pagination_url = 'http://www.eventseye.com/fairs/d1_trade-shows_january_2016_%s.html'
pag_max_count = 13
# start_urls = []
def __init__(self, **kwargs):
super(EventEyeSpider, self).__init__(**kwargs)
self.init_Month()
self.url_per_month()
self.google_maps = GoogleMaps(api_key='AIzaSyCaxLzZ2r7AbCJiIy5RtJ4jOQXcOlDbeV0')
def parse(self, response):
urls = response.css('table table tr[bgcolor] td.mt a::attr(href)').extract()
dates = response.css('table table tr[bgcolor] td:first-child::text').extract()
for num in range(len(urls)):
url = urls[num]
raw_date = dates[num]
item = EventItem()
item['url'] = 'http://www.eventseye.com/fairs/' + url.strip()
item['date'] = raw_date.strip()
self.get_event_date(item)
request = scrapy.Request(item['url'], callback=self.parse_event, meta={'item': item})
yield request
def url_per_month(self):
for page_num in range(1, self.pag_max_count):
url = self.pagination_url % page_num
self.start_urls = self.start_urls + [url]
def print_start_urls(self, response):
urls = response.css('table td.mtb a::attr(href)').extract()
for url in urls:
item = EventItem()
item['url'] = url
yield item
def parse_event(self, response):
sel = response.css('html')
item = response.meta['item']
item['source'] = response.request.url
item['event_website'] = sel.css('table td.mt a[rel="nofollow"]::attr(href)').extract()[0].strip()
item['title'] = sel.css('table table h1 b::text').extract()[0]
item['description'] = ''
item['addressRegion'] = ''
item['postalCode'] = ''
item['price_range'] = ''
item['addressLocality'] = sel.css('table a font.etb::text').extract()[0]
item['location'] = sel.css('table table h1 b::text').extract()[0]
state_code = sel.css('table td.mt span a::attr(href)').extract()[0].strip()
state_code = (state_code[state_code.rindex('-') + 1:state_code.index('.')]).upper()
item['state_code'] = state_code
item['category'] = 'Cant Figure this out'
item['industry'] = 'Cant Figure this out'
loc = self.google_maps.search(location=item['addressLocality'])
my_location = loc.first()
try:
item['country_code'] = my_location.country_shortcut
item['addressCountry'] = my_location.country_shortcut
item['city'] = my_location.city
item['longitude'] = my_location.lng
item['latitude'] = my_location.lat
item['streetAddress'] = my_location.formatted_address
except Exception as e:
item['country_code'] = ''
item['city'] = ''
item['longitude'] = ''
item['latitude'] = ''
item['streetAddress'] = ''
pass
yield item
def get_dict(self, path):
with open(path, 'r') as f:
data = json.load(f)
return data
def get_event_date(self, item):
item['event_start'] = '0000-00-00 00:00:0'
item['event_end'] = '0000-00-00 00:00:0'
raw_date = item['date']
try:
raw_date = raw_date.split(' ')
from_day = ((raw_date[0]).split('.'))[0]
to_day = ((raw_date[2]).split('.'))[0]
from_month = ((raw_date[0]).split('.'))[1]
to_month = ((raw_date[2]).split('.'))[1]
year = raw_date[3]
from_date = year + '-' + from_month + '-' + from_day + 'T10:00:00'
to_date = year + '-' + to_month + '-' + to_day + 'T10:00:00'
event_start = datetime.strptime(from_date, "%Y-%m-%dT%H:%M:%S").strftime("%Y-%m-%d %H:%M:%S")
event_end = datetime.strptime(to_date, "%Y-%m-%dT%H:%M:%S").strftime("%Y-%m-%d %H:%M:%S")
item['event_start'] = event_start
item['event_end'] = event_end
except Exception as e:
item['event_start'] = '0000-00-00 00:00:0'
item['event_end'] = '0000-00-00 00:00:0'
pass
def init_Month(self):
self.month = {'Jan': '01', 'Feb': '02', 'March': '03', 'Apr': '04', 'May': '05', 'Jun': '06', 'Jly': '07',
'Aug': '08', 'Sep': '09', 'Oct': '10', 'Nov': '11', 'Dec': '12'}
def get_datetime(self, item):
item['event_start'] = '0000-00-00 00:00:0'
item['event_end'] = '0000-00-00 00:00:0'
date = item['date']
try:
check = date.split('.')
if len(check) == 2:
dot_index = date.index(".")
comma_index = date.index(",")
days = date[dot_index + 1:comma_index]
days = days.split('-')
from_day = (days[0]).strip()
to_day = (days[1]).strip()
year = (date[comma_index + 1:]).strip()
month_str = (date[0:dot_index]).strip()
month = self.month[month_str]
from_date = year + '-' + month + '-' + from_day + 'T10:00:00'
to_date = year + '-' + month + '-' + to_day + 'T10:00:00'
event_start = datetime.strptime(from_date, "%Y-%m-%dT%H:%M:%S").strftime("%Y-%m-%d %H:%M:%S")
event_end = datetime.strptime(to_date, "%Y-%m-%dT%H:%M:%S").strftime("%Y-%m-%d %H:%M:%S")
item['event_start'] = event_start
item['event_end'] = event_end
else:
comma_index = date.index(",")
year = (date[comma_index + 1:]).strip()
days = date.split('-')
from_month_str = (days[0].strip())[0:3]
to_month_str = (days[1].strip())[0:3]
for_days = date.split(',')
for_days = for_days[0]
for_days = for_days[for_days.index('.') + 1:]
for_days = for_days.split(to_month_str)
from_day = (for_days[0]).strip()
from_day = from_day[0:2]
to_day = (for_days[1]).strip()
to_day = to_day.split(' ')
to_day = to_day[1]
from_month = self.month[from_month_str]
to_month = self.month[to_month_str]
from_date = year + '-' + from_month + '-' + from_day + 'T10:00:00'
to_date = year + '-' + to_month + '-' + to_day + 'T10:00:00'
event_start = datetime.strptime(from_date, "%Y-%m-%dT%H:%M:%S").strftime("%Y-%m-%d %H:%M:%S")
event_end = datetime.strptime(to_date, "%Y-%m-%dT%H:%M:%S").strftime("%Y-%m-%d %H:%M:%S")
item['event_start'] = event_start
item['event_end'] = event_end
except Exception as e:
item['event_start'] = '0000-00-00 00:00:0'
item['event_end'] = '0000-00-00 00:00:0'
pass
