def radial_zip_data(self, out, session, **params):
if params.get('radius'):
try:
res = SearchEngine(
db_file_dir="/srv/reggie/data").by_coordinates(
self.center.lat,
self.center.lng,
radius=int(params['radius']),
returns=None)
except Exception as e:
log.error("Error calling SearchEngine: " + e)
else:
out.writerow([
'# of Attendees', 'City', 'State', 'Zipcode',
'Miles from Event', '% of Total Attendees'
])
if len(res) > 0:
keys = self.zips.keys()
center_coord = (self.center.lat, self.center.lng)
total_count = session.attendees_with_badges().count()
for x in res:
if x.zipcode in keys:
out.writerow([
self.zips_counter[x.zipcode], x.city, x.state,
x.zipcode,
VincentyDistance((x.lat, x.lng),
center_coord).miles,
"%.2f" % float(self.zips_counter[x.zipcode] /
total_count * 100)
])
def topTenUsers():
pq = PriorityQueue()
count = 0
reader = csv.reader(open('../utilities/data.csv', 'r'))
data = []
for row in reader:
data.append(row)
if(row[0] == "User_ID"):
pass
else:
pq.put((-int(row[3]), row))
count += 1
result = {"users" : []}
defaultzip = "Brooklyn"
for i in range(count):
result["users"].append((pq.get())[1])
for i in range(count):
zipcode = result["users"][i][4]
search = SearchEngine(simple_zipcode=True)
zipSearch = search.by_zipcode(zipcode)
if zipSearch.major_city != None: result["users"][i][4] = zipSearch.major_city
else: result["users"][i][4] = "Brooklyn"
return json.dumps(result)
def validate_location(self, field):
"""Check that the location given is valid using uszipcode module."""
search = SearchEngine(simple_zipcode=True)
z = search.by_zipcode(field.data)
if z.zipcode is None:
raise ValidationError('Invalid ZIP code.')
def execute(self, data: pd.DataFrame):
if self.command in self.valid_commands:
organized_dataframe: pd.DataFrame = data
# Doing a query is for filtering based on row values like location
query = ""
searched_by_zip = False
# Doing a group by is for filtering based on a column heading like # of male students
sort_by = []
for item in self.modifiers:
# All modifiers below are for the query function
if item.name == 'locationZip':
search = SearchEngine(simple_zipcode=True)
zipcode = search.by_zipcode(item.value)
query += f'locationState == "{zipcode.state_long.lower()}" and ' \
f'locationCity == "{zipcode.major_city.lower()}"'
searched_by_zip = True
elif item.name == 'locationCity' or item.name == 'locationState':
query += f'{item.name} == "{item.value}" and '
print(query)
# All modifiers below are for the sort_values function
elif item.name == 'minority':
sort_by.extend([
'American Indian/Alaska Native Students',
'Asian or Asian/Pacific Islander Students',
'Hispanic Students', 'Black Students',
'Hawaiian Nat./Pacific Isl. Students',
'Free & Reduced Lunch Students'
])
elif item.name == 'sex':
if item.value == 'Male':
sort_by.extend(['Male Students'])
else:
sort_by.extend(['Female Students'])
elif item.name == 'free&reducedLunch':
sort_by.extend(['Free & Reduced Lunch Students'
]) if item.value else item.value
if item.name == 'hasWebsite':
if item.value == 'true':
organized_dataframe = organized_dataframe.dropna(
subset=['Web Site URL'])
else:
organized_dataframe = organized_dataframe[
organized_dataframe['Web Site URL'].isnull()]
# Only query if there are actual modifiers given by user
if len(query) >= 1:
new_query = query
if not searched_by_zip:
new_query = query[:len(query) - 5]
print(new_query)
organized_dataframe = organized_dataframe.query(new_query)
if len(sort_by) >= 1:
organized_dataframe = organized_dataframe.sort_values(
by=sort_by, ascending=False, ignore_index=True)
# Only sort_values if there are actual modifier values given by user
elif len(sort_by) >= 1:
organized_dataframe = organized_dataframe.sort_values(
by=sort_by, ascending=False, ignore_index=True)
print(organized_dataframe.iloc[self.row_start:self.row_end, :])
def getCovidData(zipcode):
try:
search = SearchEngine(simple_zipcode=True)
zipcode = search.by_zipcode(str(zipcode))
zipcode = zipcode.to_dict()
state = states[zipcode["state"]].lower()
county = zipcode["county"].replace(" County", "").lower()
url = "https://covid-api.onrender.com/get?state={state}&county={county}".format(
state=state, county=county)
response = requests.get(url)
response_dict = json.loads(response.text)
if str(response_dict['Recovered']) == '0':
msg = "COVID UPDATE " + state.upper() + ", " + county.upper() + " COUNTY. DATE: " + str(response_dict['Date']) + "\n" \
"There are " + str(response_dict['Confirmed']) + " confirmed cases.\n" + \
"There are " + \
str(response_dict['Deaths']) + " confirmed deaths."
else:
msg = "COVID UPDATE " + state.upper() + ", " + county.upper() + " COUNTY. DATE: " + str(response_dict['Date']) + "\n" \
"There are " + str(response_dict['Confirmed']) + " confirmed cases.\n" + \
"There are " + str(response_dict['Deaths']) + " confirmed deaths.\n" + \
"There are " + \
str(response_dict['Recovered']) + " confirmed recoveries."
return msg
except:
return "error"
def __init__(self, region, county_info, redfin_cookies, redfin_headers,
redfin_params, interest_rate, borrowing_pct,
mortgage_term_years, insurance_cost):
Scrape.__init__(self)
self.region = region
self.insurance_cost = insurance_cost
self.county_info = county_info
self.interest_rate = interest_rate
self.borrowing_pct = borrowing_pct
self.redfin_headers = redfin_headers
self.redfin_params = redfin_params
self.redfin_cookies = redfin_cookies
self.mortgage_term_years = mortgage_term_years
self.housing_data = {}
self.data = []
self.exception_counties = {
"King County": "Kings County",
}
self.search = SearchEngine(simple_zipcode=True)
self.air_dna_headers = {
'Sec-Fetch-Mode': 'cors',
'Referer':
'https://www.airdna.co/vacation-rental-data/app/us/california/union-city/rentalizer',
'Origin': 'https://www.airdna.co',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36',
'DNT': '1',
}
self.air_dna_access_token = [
'MjkxMTI|8b0178bf0e564cbf96fc75b8518a5375',
'ODkwMTc|478ce2c743244a7eb3d1cfddc14909b3',
'MjA2Mjcw|69e663b4c51c4830a8bde0d3355be8ee',
'MjA2Mjcz|e35b14ebfb794d849f9484afcffced1d'
]
def get_ownership_info(self):
print "Owner:", self.cdm.case_list[
-9] # complainant name to search for
zip = self.cdm.case_list[-2] # gets complainant zip code
search = SearchEngine()
data = search.by_zipcode(zip)
# Get comp county; if not in Florida (or our list, ask for it manually)
if data.state != "FL":
print "The complainant lives in %s, not Florida." % data.state
zip = self.cdm.case_list[7] # try the respondent instead of comp
data = search.by_zipcode(zip)
if data.state != "FL":
print "The respondent is not in FL, either."
county = get_string("In what county is the unit located?")
county = county.title().replace("County", "").strip()
else:
county = data.county.replace("County", "").strip()
else:
county = data.county.replace("County", "").strip()
# Get the list of counties from the AppraiserSites table
results = self.dbm.query("SELECT * FROM AppraiserSites")
counties = [result[0] for result in results]
if county not in counties:
print "%s not found in the list of counties." % county
county = get_string("In what county is the unit located?")
county = county.title().replace("County", "").strip()
sql = " SELECT Site From AppraiserSites WHERE County = '%s' " % county
results = self.dbm.query(sql)
appraiser_site = results.fetchone()[0]
wb.open_new_tab(appraiser_site)
self.owner = get_bool("Is complainant a unit owner?")
self.owner = "unit owner" if self.owner is True else "not unit owner"
def test_zipcode_info(zipcodes):
df = zipcodes
df_comp = df.copy()
searcher = SearchEngine(simple_zipcode=True)
df_comp['state'] = ''
df_comp['county'] = ''
df_comp['city'] = ''
df_comp['lat'] = ''
df_comp['lng'] = ''
df_comp['timezone'] = ''
for zipcode in df_comp['zip_code'].unique():
zip_search = searcher.by_zipcode(zipcode)
df_comp.loc[df_comp['zip_code'] == zipcode,
'city'] = zip_search.major_city
df_comp.loc[df_comp['zip_code'] == zipcode,
'county'] = zip_search.county
df_comp.loc[df_comp['zip_code'] == zipcode, 'lat'] = zip_search.lat
df_comp.loc[df_comp['zip_code'] == zipcode, 'lng'] = zip_search.lng
df_comp.loc[df_comp['zip_code'] == zipcode, 'state'] = zip_search.state
df_comp.loc[df_comp['zip_code'] == zipcode,
'timezone'] = zip_search.timezone
zip_info = ZipCodeInfo.generate_feature(df, 'zip_code')
assert zip_info.equals(df_comp)
def attendees_by_state(self, out, session):
# Result of set(map(lambda x: x.state, SearchEngine(db_file_dir="/srv/reggie/data").ses.query(SimpleZipcode))) -- literally all the states uszipcode knows about
states = [
'SD', 'IL', 'WY', 'NV', 'NJ', 'NM', 'UT', 'OR', 'TX', 'NE', 'MS',
'FL', 'VA', 'HI', 'KY', 'MO', 'NY', 'WV', 'DC', 'AR', 'MT', 'MD',
'SC', 'NC', 'KS', 'OH', 'PR', 'CO', 'IN', 'VT', 'LA', 'ND', 'AZ',
'AK', 'AL', 'CT', 'TN', 'PA', 'IA', 'WA', 'ME', 'NH', 'MA', 'ID',
'OK', 'WI', 'GA', 'CA', 'DE', 'MN', 'MI', 'RI'
]
total_count = session.attendees_with_badges().count()
out.writerow(['# of Attendees', 'State', '% of Total Attendees'])
for state in states:
try:
zip_codes = list(
map(
lambda x: x.zipcode,
SearchEngine(db_file_dir="/srv/reggie/data").by_state(
state, returns=None)))
except Exception as e:
log.error("Error calling SearchEngine: " + e)
else:
current_count = session.attendees_with_badges().filter(
Attendee.zip_code.in_(zip_codes)).count()
if current_count:
out.writerow([
current_count, state,
"%.2f" % float(current_count / total_count * 100)
])
def getZipcode(city, state):
search = SearchEngine()
zipSearch = search.by_city_and_state(city, state)
zipcode = zipSearch[0]
zipcode = zipcode.zipcode
return zipcode
def inference(self, zip_code, population_density=0, median_home_value=0):
pd = population_density
mhv = median_home_value
zp = int(zip_code)
print(zp)
if pd + mhv == 0:
search = SearchEngine(simple_zipcode=False)
zip_dct = search.by_zipcode(zp).to_dict()
pd = zip_dct['population_density']
mhv = zip_dct['median_home_value']
print(mhv)
print(pd)
inp = np.asarray([float(mhv), float(zip_code), float(pd)])
print(inp)
print(self.reg_coef)
result = 0
# self.reg_coef= map(lambda x: float(x),self.reg_coef)
# inp = np.asarray([[mhv, zip_code, pd]])
# print(inp)
# result = self.reg.predict(inp)[0]
try:
result = np.dot(self.reg_coef, inp) + self.reg_intercept
print(result)
except:
print('failed')
print(result)
return result
def generate_feature(cls, data, column=None, **kwargs):
"""
:param data: dataframe containing zip codes
:param column: column label containing zip codes
:param kwargs: ignored
:return: dataframe with new columns for county, city, latitude and longitude
"""
if column is None:
raise ValueError('zipcode column must be given')
zip_searcher = SearchEngine(simple_zipcode=True)
data['state'] = ''
data['county'] = ''
data['city'] = ''
data['lat'] = ''
data['lng'] = ''
data['timezone'] = ''
for zipcode in data[column].unique():
zip_search = zip_searcher.by_zipcode(zipcode)
data.loc[data[column] == zipcode, 'city'] = zip_search.major_city
data.loc[data[column] == zipcode, 'county'] = zip_search.county
data.loc[data[column] == zipcode, 'lat'] = zip_search.lat
data.loc[data[column] == zipcode, 'lng'] = zip_search.lng
data.loc[data[column] == zipcode, 'state'] = zip_search.state
data.loc[data[column] == zipcode, 'timezone'] = zip_search.timezone
return data
def validate_zip_city_pfds(pfds_file_name):
correct = 0
incorrect = 0
code_city_dict = dict()
code_city_dict.clear()
search = SearchEngine(simple_zipcode=False)
with open(pfds_file_name, "r") as f:
lines = f.readlines()
for line in lines:
if line.startswith("==") or line.startswith("Coverage"):
continue
else:
s1 = line.rsplit('::', 1)
s2 = re.split('(\d*\.\d+|\W)', s1[0])
ss = [t for t in s2 if len(t) > 0]
city1 = s1[0].rsplit('\'', 1)
rec_city = city1[1]
for ii in ss:
if ii.isspace():
continue
if (ii[0].isdigit()):
Zip = ii
break
zipcode = search.by_zipcode(Zip)
true_city = zipcode.major_city
if not (true_city.lower() == rec_city.lower()):
print(Zip, rec_city, "|=", true_city)
incorrect += 1
else:
correct += 1
error_rate = incorrect / (correct + incorrect) * 100
accuracy = 100 - error_rate
print("Correct = ", correct, "\tIncorrect = ", incorrect,
"error rate = {0:.2f}".format(error_rate) + "%",
"\tAccuracy = {0:.2f}".format(accuracy), "%")
def create_crime_zip_codes(crime_df):
'''
Use the uszipcode library to identify a zip code for each unique
pair of latitude and longitude coordinates in the Crime Dataset.
Merges zip code information back into the Crime Dataset to later join with
ACS data.
NOTE: the uszipcode library can take a while to run (this is normal)
Input:
crime_df (dataframe): original crime dataframe
Output:
crime_df (dataframe): new crime dataframe including zip codes
'''
crime_df.loc[:, 'latitude'] = crime_df.latitude.astype(float)
crime_df.loc[:, 'longitude'] = crime_df.longitude.astype(float)
truncated = crime_df[['block', 'latitude',\
'longitude']].drop_duplicates(subset=['block'])
truncated = truncated.dropna()
search = SearchEngine(simple_zipcode=True)
truncated['zip_code'] = truncated.apply(lambda x: search.by_coordinates(
x['latitude'], x['longitude'])[0].zipcode,
axis=1)
merged_df = pd.merge(crime_df, truncated, on=['block', 'latitude',\
'longitude'], how='left')
merged_df.loc[:, 'zip_code'] = pd.to_numeric(merged_df['zip_code'],
errors='coerce')
return merged_df
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.transformer_name = self.__class__.__name__
if self.transformer_name.endswith("Transformer"):
self.transformer_name = self.transformer_name[:-len("Transformer")]
self.search = SearchEngine(simple_zipcode=False)
def main():
state2 = input("What" + '\x1b[1;31;40m' + ' state ' + '\x1b[0m' +
"do you want the temperature of?\n")
city2 = input("What" + '\x1b[1;31;40m' + ' city ' + '\x1b[0m' +
"do you want the temperature of?\n")
search = SearchEngine(simple_zipcode=True)
res = search.by_city_and_state(city2,
state2,
zipcode_type='Standard',
sort_by='zipcode',
ascending=True,
returns=5)
len(res)
try:
zipcode = res[0]
except IndexError:
print("Please type in a valid USA State/City\n")
main()
zipcode
city = zipcode.major_city
state = zipcode.state
urlend = zipcode.zipcode
URL = 'https://weather.com/weather/today/l/' + urlend
page = requests.get(URL)
soup = BeautifulSoup(page.content, 'html.parser')
temp = soup.find(class_='CurrentConditions--tempValue--3KcTQ').get_text()
print('\nThe temperature right now in ' + city + ', ' + state + ' is ' +
'\x1b[1;32;40m' + temp + '\x1b[0m' + "\n")
main()
def get_income_df(df):
"""
Attach zip code median income data to dataframe
In: DataFrame
Out: DataFrame w/ Median Income column
"""
search = SearchEngine()
zipcodes = list(df['ZIPCODE'].dropna().unique())
zipcodes = list(map(int, zipcodes))
zipcodes = sorted(zipcodes)
zipcodes.remove(12345)
zipcodes.remove(30339)
zipdf = pd.DataFrame(zipcodes)
income = []
for zip_ in zipcodes:
tosearch = search.by_zipcode(zip_)
income.append(tosearch.median_household_income)
zipdf['Median_Income'] = income
zipdf.set_index(0, inplace=True)
df.ZIPCODE = df.ZIPCODE.fillna(0)
df = df.astype({'ZIPCODE': int})
trim_inc = df.join(zipdf, on='ZIPCODE', how='left')
return trim_inc
def is_valid(int_zip):
search = SearchEngine()
zipcode = search.by_zipcode(int(int_zip))
if zipcode.zipcode and zipcode.lat and zipcode.lng:
return True
else:
return False
def add_zip_details(df):
"""Adds zip details retrieved with get_zip_details() to Dataframe
Args:
df (pandas dataframe obj): Pandas dataframe that must contain sender_zip and recipient_zip columns
Returns:
pandas dataframe obj: Pandas dataframe with new columns sender_pop, sender_pop_density, sender_houses,
recipient_pop, recipient_pop_density, recipient_houses
"""
print(f"Adding zipcode details, {len(df)} iterations expected...")
search = SearchEngine()
pop_list_s, pop_density_list_s, houses_list_s = [], [], []
pop_list_r, pop_density_list_r, houses_list_r = [], [], []
for row in tqdm(df.itertuples()):
pop_s, pop_density_s, houses_s = get_zip_details(
row.sender_zip, search)
pop_list_s.append(pop_s)
pop_density_list_s.append(pop_density_s)
houses_list_s.append(houses_s)
pop_r, pop_density_r, houses_r = get_zip_details(
row.recipient_zip, search)
pop_list_r.append(pop_r)
pop_density_list_r.append(pop_density_r)
houses_list_r.append(houses_r)
df['sender_pop'], df['sender_pop_density'], df[
'sender_houses'] = pop_list_s, pop_density_list_s, houses_list_s
df['recipient_pop'], df['recipient_pop_density'], df[
'recipient_houses'] = pop_list_r, pop_density_list_r, houses_list_r
return df
def date_localization(self):
if not self._date_localization:
try:
self._date_localization = 'TODAY!'
# The fast way.
if SearchEngine and self.zip:
with SearchEngine() as search:
zipcode = search.by_zipcode(
str(self.zip).split('-')[0])
if zipcode and zipcode.lat:
self.partner_latitude = zipcode.lat
self.partner_longitude = zipcode.lng
return self._date_localization
# The slow way.
geo_obj = self.env['base.geocoder']
search = geo_obj.geo_query_address(
city=self.city,
state=self.state_id.name,
country=self.country_id.name)
result = geo_obj.geo_find(search,
force_country=self.country_id.id)
if result:
self.partner_latitude = result[0]
self.partner_longitude = result[1]
except:
self._date_localization = 'ERROR'
return self._date_localization
def state_zip_by_popdense(state_str, num_returns):
search = SearchEngine()
res = search.query(state=state_str,
sort_by=Zipcode.population_density,
ascending=False,
returns=num_returns)
return [x.zipcode for x in res]
def findState(df):
search = SearchEngine()
for index, row in df.iterrows():
view = row['view']
zipcode = row['zipcode']
zipcode = search.by_zipcode(zipcode)
print(str(zipcode))
print("")
def find_zipcode_in_major_states():
major_states = ['NY', 'CA', 'WA', 'IL', 'DC', 'TX', 'GA', 'VA']
zipsearch = SearchEngine(simple_zipcode=False)
l = []
for i in range(500, 100000):
temp = zipsearch.by_zipcode(str(i).zfill(5))
if temp.zipcode and temp.state in major_states: l.append(temp.zipcode)
return l
def get_zipcode(self, df):
from uszipcode import SearchEngine
search = SearchEngine(simple_zipcode=True)
zipcode = search.by_coordinates(df['latitude'], df['longitude'])
if not zipcode:
return None
else:
return zipcode[0].values()[0]
def get_lat_long(apps, schema_editor):
Places = apps.get_model('webmanager', 'Place')
search = SearchEngine(simple_zipcode=True)
for place in Places.objects.all():
zipcode = search.by_zipcode(place.zip_code)
place.lng = zipcode.lng
place.lat = zipcode.lat
place.save(update_fields=['lat', 'lng'])
def states():
zipcode = request.args.get('zip')
statesvalues = pd.read_csv('stateslived.csv').to_dict('records')
if zipcode != '':
search = SearchEngine(simple_zipcode=True)
state = state_names[search.by_zipcode(int(zipcode)).state]
statesvalues.append({'state':state,'visited':1})
return json.dumps(statesvalues)
def is_valid_zip(zip_code: int):
"""See if the given value is a valid US zip code."""
search = SearchEngine(simple_zipcode=True)
origin_zip = search.by_zipcode(zip_code)
if origin_zip.zipcode is None:
return None
return (origin_zip.lat, origin_zip.lng)
def retrieve_zips(col):
search = SearchEngine(simple_zipcode=True)
states = []
for zipc in users['ZipCode']:
zipcode = search.by_zipcode(zipc)
state = zipcode.state
states = np.append(states, state)
return states
def get_regions(self, zipcode='22030', radius=100):
search = SearchEngine()
zipcode_info = self.zipcodeInfo(zipcode)
res = SearchEngine().query(
lat=zipcode_info.lat,
lng=zipcode_info.lng,
radius=radius,
# model.Zipcode.median_household_income,
sort_by=model.SimpleZipcode.population,
ascending=False,
returns=12,
) # get 12 biggest cities around 100 miles around the given zip code
counties = set([(r.post_office_city, r.state, r.county) for r in res])
return counties
def get_distance_in_miles(home_team_zip, away_team_zip):
#for extensive list of zipcodes, set simple_zipcode=False
search = SearchEngine(simple_zipcode=True)
zip1 = search.by_zipcode(home_team_zip)
zip2 = search.by_zipcode(away_team_zip)
return round(mpu.haversine_distance((zip1.lat, zip1.lng), (zip2.lat, zip2.lng)), 2)
