def reply_citystate(body):
"""reply method by City, State Code request"""
try:
city = body.split(",")[0].strip().title()
state = body.split(",")[1].strip().upper()
if state not in states_code_to_full.keys():
if state.title() not in states_full_to_code.keys():
return ERROR_MSG
else:
state = states_full_to_code[state.title()]
body = city + ", " + state
msg = str(datetime.date.today().strftime(
"%m/%d")) + " COVID-19 SMS Update for " + body + ": \n"
results = zipcodes.filter_by(city=city, state=state)
temp_list = []
for res in results:
rep_zip = reply_zipcode(res['zip_code'], False)
if rep_zip != ERROR_MSG:
temp_list.append(rep_zip)
msg += ''.join(
list(set(temp_list))
) + "\nSource: New York Times. Thanks for using COVID-19 SMS Update!" if len(
temp_list
) > 0 else "City data not found. Thanks for using COVID-19 SMS Update!"
return msg
except:
return ERROR_MSG
def find_near_zips(zipc, city, state):
x = zipcodes.similar_to(zipc[0],
zips=zipcodes.filter_by(zipcodes.list_all(), active=True, city= city, state = state))
zipps = []
for zips in x:
zipps.append(zips['zip_code'])
return zipps
def get_zip_code(state):
""" Validates if zip code is a real US zip code.
Uses the package zipcodes to verify if input
is a valid U.S Zip Code. See readMe.md for
instructions to install the zipcodes package.
Searches for the zip code in the State provided,
if no zip code is found - prompts for new zip
code.
Args:
state (String): The state provided by the User
"""
found = False
zip_code = input("What is your zipcode?: ").upper().strip()
exit_lab(zip_code)
state_zip_codes = zipcodes.filter_by(state=state)
for state_dict in state_zip_codes:
if zip_code == state_dict.get("zip_code"):
found = True
try:
if not zipcodes.is_real(zip_code) or not found:
print("Please enter a valid US Zip Code for", state + ".")
return get_zip_code(state)
except ValueError:
print("Please enter a valid US Zip Code for", state + ".")
return get_zip_code(state)
return zip_code
def contact_locations():
"""api-ish route: returns lat/long positions of all contacts texted"""
all_contacts = get_contacts()
chat_conn = get_db(CHAT_DATABASE)
year = int(request.args.get('year') or 2020)
cur_contacts = pd.read_sql(all_chats(year), chat_conn)
contacts = {
k: v
for (k, v) in all_contacts.items() if k in list(cur_contacts['name'])
}
people_map = {}
for number, name in contacts.items():
area_code = number[2:5]
if people_map.get(area_code) != None:
people_map[area_code]['people'].append(name)
continue
zipcode = zipcodes.filter_by(area_codes=[area_code])
if len(zipcode) == 0:
continue
else:
people_map[area_code] = {
"place": [zipcode[0]['lat'], zipcode[0]['long']],
"people": [name]
}
return people_map
def test_filter_by(self):
self.assertEqual(
zipcodes.filter_by(city="WINDSOR", state="CT"),
[
{
"zip_code": "06006",
"zip_code_type": "UNIQUE",
"city": "WINDSOR",
"state": "CT",
"lat": 41.85,
"long": -72.65,
"world_region": "NA",
"country": "US",
"active": True,
},
{
"zip_code": "06095",
"zip_code_type": "STANDARD",
"city": "WINDSOR",
"state": "CT",
"lat": 41.85,
"long": -72.65,
"world_region": "NA",
"country": "US",
"active": True,
},
],
)
def start_requests(self):
for zipcode in zipcodes.filter_by(zipcodes.list_all(), active=True):
for cat in self.categories:
url = self.search_url.format(
zipcode['city'].lower().replace(' ', '-'),
zipcode['state'].lower(), cat)
yield scrapy.Request(url, callback=self.parse)
def start_requests(self):
target_states = [state['abbr'] for state in states]
for zipcode in zipcodes.filter_by(zipcodes.list_all(), active=True):
if zipcode['state'] in target_states:
url = self.search_url.format(zipcode['zip_code'])
yield scrapy.Request(url,
callback=self.parse,
meta={'state': zipcode['state']})
def parse_address(address):
import usaddress
import zipcodes
zipcode = ""
address_ = dict(usaddress.tag(address)[0])
print(address_)
print(address_['StateName'])
state = address_['StateName'].split(",")[0]
address_['state'] = state
print(address_['PlaceName'])
print(zipcodes.filter_by(city=address_['PlaceName']))
for i in zipcodes.filter_by(city=address_['PlaceName']):
if i['state'] == state:
zipcode = i['zip_code']
print(i)
return address_, zipcode
def safe_postal_code(self):
"""
Returns a safe postal code as a string. Due to swagger fields being treated independently this may not match
the random state value in an address object.
"""
while True:
state = self.state_abbr(include_territories=False)
if state != "AK" and state != "HI":
return random.choice(zipcodes.filter_by(state=state)).get("zip_code")
def safe_city_state_zip(self):
"""
Returns a combination of a valid city, state, and postal code.
:return: tuple
"""
while True:
state = self.state_abbr(include_territories=False)
if state != "AK" and state != "HI":
location = random.choice(zipcodes.filter_by(state=state))
return {"city": location["city"], "state": location["state"], "postalCode": location["zip_code"]}
def chicago_zip_codes():
results = []
for index, zip_code in enumerate(
zipcodes.filter_by(zipcodes.list_all(),
active=True,
city='CHICAGO')):
url = f"{settings.V1_URL}/url-mediator/session-builder?zip_code={zip_code['zip_code']}"
results.append(
ZipCode(pk=index, zip_code=zip_code['zip_code'], url=url))
return results
def is_valid_vermont_zip(zip: str):
list_of_dicts_of_vermont_zips = zipcodes.filter_by(state="VT")
list_of_vermont_zips = [d["zip_code"] for d in list_of_dicts_of_vermont_zips]
if len(zip) > 10:
return False
elif type(zip) != str:
return False
elif zip in list_of_vermont_zips:
return True
else:
return False
def is_valid_south_carolina_zip(zip: str):
list_of_dicts_of_south_carolina_zips = zipcodes.filter_by(state="SC")
list_of_south_carolina_zips = [
d["zip_code"] for d in list_of_dicts_of_south_carolina_zips
]
if len(zip) > 10:
return False
elif type(zip) != str:
return False
elif zip in list_of_south_carolina_zips:
return True
else:
return False
def start_requests(self):
"""
yield scrapy.Request('https://www.realtor.com/realestateteam/89138', callback=self.parse, meta={'search_keyword': '89138'})
target_states = [state['abbr'] for state in states]
for zipcode in zipcodes.filter_by(zipcodes.list_all(), active=True):
if zipcode['state'] in target_states:
url = self.search_url.format(zipcode['city'], zipcode['state'], zipcode['zip_code'])
yield scrapy.Request(url, callback=self.parse)
for zipcode in zipcodes.filter_by(zipcodes.list_all(), active=True):
url = self.search_url.format(zipcode['zip_code'])
yield scrapy.Request(url, callback=self.parse, meta={'search_keyword': zipcode['zip_code']})
"""
for zipcode in zipcodes.filter_by(zipcodes.list_all(), active=True):
url = self.search_url.format(zipcode['zip_code'])
yield scrapy.Request(url, callback=self.parse, meta={'search_keyword': zipcode['zip_code']})
def start_requests(self):
with open(
os.path.dirname(os.path.realpath(__file__)) +
"/../external_data/output/findamortgagebrokercom.csv",
'w') as csvfile:
fieldnames = [
'Organization', 'Full Name', 'Zipcode', 'Address', 'Email',
'NMLS', 'Website', 'Phone'
]
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
for zipcode in zipcodes.filter_by(zipcodes.list_all(), active=True):
formData = {"Criteria": zipcode['zip_code']}
print(zipcode['zip_code'])
yield scrapy.FormRequest(self.search_url,
callback=self.parse,
formdata=formData)
def start_requests(self):
target_states = [state['abbr'] for state in states]
for zipcode in zipcodes.filter_by(zipcodes.list_all(), active=True):
if zipcode['state'] in target_states:
url = self.most_active_search_url.format(
'-'.join(zipcode['city'].split()).lower(),
zipcode['state'].lower(), zipcode['zip_code'])
yield scrapy.Request(url,
callback=self.parse,
meta={'dont_cache': True})
url = self.most_sales_search_url.format(
'-'.join(zipcode['city'].split()).lower(),
zipcode['state'].lower(), zipcode['zip_code'])
yield scrapy.Request(url,
callback=self.parse,
meta={'dont_cache': True})
url = self.most_listings_search_url.format(
'-'.join(zipcode['city'].split()).lower(),
zipcode['state'].lower(), zipcode['zip_code'])
yield scrapy.Request(url,
callback=self.parse,
meta={'dont_cache': True})
def buscar_latitude_longitude(dataframe):
dataframe.City = dataframe.City.str.strip()
dataframe.State = dataframe.State.str.strip()
lat_long = {"lat": [], "long": []}
for i, row in dataframe.iterrows():
city = row.City
state = row.State
cities = zipcodes.filter_by(city=city, state=state)
if cities:
lat_long["lat"].append(cities[0]['lat'])
lat_long["long"].append(cities[0]['long'])
continue
lat_long["lat"].append(np.nan)
lat_long["long"].append(np.nan)
df = pd.DataFrame(lat_long)
df.to_csv('lat_long.csv', index=False)
return df
def run(start_year: int, end_year: int, county: str, state: str):
print('Obtaining zip codes for ' + county + ' ' + state + '...')
locations = zipcodes.filter_by(county=county.capitalize() + ' County',
state=state.upper())
ssdi_scraper.begin(locations, start_year, end_year)
print('Finished!')
def start_requests(self):
for zipcode in zipcodes.filter_by(zipcodes.list_all(), active=True):
url = self.search_url.format(zipcode['zip_code'])
yield scrapy.Request(url, callback=self.parse, meta={'zip': zipcode['zip_code']})
def main():
# name of this stage, typically a name to reference the assertion
# assertion: lambda which returns unittest callable with self's (testcase's) context
# predicates: lambda or sequence of lambdas to call and pass to the assertion
unittests_schema = [
{
"name": "true",
"assertion": lambda self: self.assertTrue,
"predicates": [
lambda: zipcodes.is_real("06905"),
lambda: zipcodes._contains_nondigits("1234a"),
# bad length
lambda: callable_raise_exc(
lambda: zipcodes._clean("000000"), ValueError
),
# bad characters
lambda: callable_raise_exc(
lambda: zipcodes._clean("0000a"), ValueError
),
# ensure zips argument works
lambda: len(
zipcodes.similar_to(
"2", zips=zipcodes.filter_by(active=True, city="Windsor")
)
)
== 3,
],
},
{
"name": "false",
"assertion": lambda self: self.assertFalse,
"predicates": [
lambda: zipcodes.is_real("91239"),
# digits and "-" are acceptable
lambda: zipcodes._contains_nondigits("12345"),
lambda: zipcodes._contains_nondigits("1234-"),
],
},
{
"name": "equal",
"assertion": lambda self: self.assertEqual,
"predicates": [
# valid_zipcode_length parameter
(lambda: zipcodes._clean("0646", 4), lambda: "0646"),
# default behavior
(lambda: zipcodes._clean("06469"), lambda: "06469"),
(lambda: zipcodes.list_all(), lambda: zipcodes._zips),
(
lambda: zipcodes.filter_by(city="Old Saybrook"),
lambda: [
{
"zip_code": "06475",
"zip_code_type": "STANDARD",
"active": True,
"city": "Old Saybrook",
"acceptable_cities": [],
"unacceptable_cities": ["Fenwick"],
"state": "CT",
"county": "Middlesex County",
"timezone": "America/New_York",
"area_codes": ["860"],
"world_region": "NA",
"country": "US",
"lat": "41.3015",
"long": "-72.3879",
}
],
),
(
lambda: zipcodes.similar_to("1018"),
lambda: [
{
"acceptable_cities": [],
"active": False,
"area_codes": ["212"],
"city": "New York",
"country": "US",
"county": "New York County",
"lat": "40.71",
"long": "-74",
"state": "NY",
"timezone": "America/New_York",
"unacceptable_cities": ["J C Penney"],
"world_region": "NA",
"zip_code": "10184",
"zip_code_type": "UNIQUE",
},
{
"acceptable_cities": [],
"active": True,
"area_codes": ["212"],
"city": "New York",
"country": "US",
"county": "New York County",
"lat": "40.7143",
"long": "-74.0067",
"state": "NY",
"timezone": "America/New_York",
"unacceptable_cities": [],
"world_region": "NA",
"zip_code": "10185",
"zip_code_type": "PO BOX",
},
],
),
(
lambda: zipcodes.similar_to("1005"),
lambda: [
{
"zip_code": "10055",
"zip_code_type": "STANDARD",
"active": True,
"city": "New York",
"acceptable_cities": [],
"unacceptable_cities": ["Manhattan"],
"state": "NY",
"county": "New York County",
"timezone": "America/New_York",
"area_codes": ["212"],
"world_region": "NA",
"country": "US",
"lat": "40.7579",
"long": "-73.9743",
}
],
),
(
lambda: zipcodes.similar_to("10001"),
lambda: [
{
"zip_code": "10001",
"zip_code_type": "STANDARD",
"active": True,
"city": "New York",
"acceptable_cities": [],
"unacceptable_cities": [
"Empire State",
"G P O",
"Greeley Square",
"Macys Finance",
"Manhattan",
],
"state": "NY",
"county": "New York County",
"timezone": "America/New_York",
"area_codes": ["718", "917", "347", "646"],
"world_region": "NA",
"country": "US",
"lat": "40.7508",
"long": "-73.9961",
}
],
),
],
},
]
generate_unittests(unittests_schema)
logger.info("Zipcodes version: {}".format(zipcodes.__version__))
unittest.main()
]
ca_zip_ts['Lat'] = [
zip_latlong[row.ZipCode_str]['Lat'] for row in ca_zip_ts.itertuples()
]
ca_zip_ts['Long'] = [
zip_latlong[row.ZipCode_str]['Long'] for row in ca_zip_ts.itertuples()
]
# Intermediate Step: Pare down FIPS Mapping to only CA and then grab zipcodes using `zipcodes` package
fips_mapping_CA = fips_mapping.loc[fips_mapping['State'] == 'CA']
fips_mapping_CA.reset_index(drop=True, inplace=True)
zip_meta = dict()
for row in fips_mapping_CA.itertuples():
city_to_check = row.City
info = zipcodes.filter_by(city=city_to_check)
for i in info:
if row.County in i['county']:
zip_meta[i['zip_code']] = {
'Lat': float(i['lat']),
'Long': float(i['long']),
'City': i['city'],
'County': i['county'].replace(" County", "")
}
# Convert the dict to DataFrame
zip_meta_df = pd.DataFrame(zip_meta).T
zip_meta_df.reset_index(inplace=True)
city_uniqueCityID = dict(
zip(fips_mapping_CA['City'], fips_mapping_CA['Unique_City_ID']))
city_metroName = dict(
kind="bar",
data=data_4uf_plot,
height=8)
# Plota o gráfico a partir da reogarnização dos dados com o pivot_table
data_4uf_plot.pivot_table(index='state', columns='shape', values='views').\
plot.bar(stacked=True,figsize=(7,7))
data_4uf_plot.pivot_table(index='state', columns='shape', values='views')
# Importa biblioteca zipcodes
#!pip install zipcodes
import zipcodes
# Importa dados de geolocalização dos EUA
df_zipcodes = pd.DataFrame(zipcodes.filter_by(country="US"))
# Limpeza de linhas que não tem latitude e longitude
df_zipcodes_wna = df_zipcodes.dropna(subset=['lat', 'long'])
# Verifica os estados de USA com a função checkStatesUSA. Se for um estado do
# USA será retornado o nome do estado. Se não for, será retornado False:
StatesUSAboolTemp = \
list(map(lambda x: checkStatesUSA(x), df_zipcodes_wna["state"].tolist()))
# Converte os nomes dos estados em True
StatesUSAbool = list(map(lambda x: False if x else True, StatesUSAboolTemp))
# Cria novo dataframe realizando o drop StatesUSAbool, resultando num
# dataframe que contém somente os dados de estados do USA
df_zipcodesUSA = df_zipcodes_wna.drop(df_zipcodes_wna[StatesUSAbool].index)
legend_out=False,
palette=colors,
aspect=1.4)
sql3.pivot("State", "Shape",
"Views").plot(kind="bar",
stacked=True,
color=['#253031', '#315659', '#2978A0', '#C6E0FF'])
from ipywidgets.embed import embed_minimal_html
import gmaps
#!pip install gmaps
#!pip install zipcodes
import zipcodes
mapp = pd.DataFrame(zipcodes.filter_by(country='US')).dropna()
comparar = """
SELECT mapp.lat, mapp.long, COUNT(separados_usa.Posted) as Relatos FROM mapp, separados_usa
WHERE mapp.state = separados_usa.State
GROUP BY mapp.city;
"""
sql4 = pandasql.sqldf(comparar)
gmaps.configure(
api_key='AIza....'
) #A API KEY foi removida para não ser utilizado para outros fins
locations = sql4[['lat', 'long']]
relats = sql4['Relatos']
fig = gmaps.figure()
fig.add_layer(gmaps.heatmap_layer(locations, weights=relats))
import sys
import requests
from lxml import html
import zipcodes
import json
import sys
reload(sys)
sys.setdefaultencoding('utf8')
realtor_home_url = "https://www.realtor.com"
search_url = "https://www.realtor.com/realestateandhomes-search/{0}/type-single-family-home/price-150000-550000/nc-hide"
new_zipcodes_list = []
for zipcode in zipcodes.filter_by(zipcodes.list_all(), active=True):
if zipcode['state'] == "FL" and \
not os.path.isfile(os.path.dirname(os.path.realpath(__file__)) + "/../external_data/output/listing_searches_by_zip_codes/florida_{0}_listings_list.csv".format(zipcode['zip_code'])):
new_zipcodes_list.append(zipcode['zip_code'])
for zipcode in new_zipcodes_list:
retry_limit = 3
while retry_limit > 0:
try:
from six.moves.urllib import request
opener = request.build_opener(
request.ProxyHandler({'https': 'http://127.0.0.1:24000'}))
# html_content = opener.open(
# 'https://www.realtor.com/realestateandhomes-search/32615/type-single-family-home/price-150000-550000/nc-hide').read()
def _update_locations(self, geography_type, locations, community):
"""
Fill the locations for an updated geographic community
"""
# clean up in case there is garbage in there
if community.locations:
community.locations.clear()
# this is a list of zipcodes, towns, cities, counties, states
location_list = locations.replace(", ", ",").split(
",") # passed as comma separated list
print("Community includes the following locations :" +
str(location_list))
for location in location_list:
if geography_type == "ZIPCODE":
if location[0].isdigit():
location = location.replace(" ", "")
# looks like a zipcode. Check which towns it corresponds to
zipcode = zipcodes.matching(location)
if len(zipcode) > 0:
city = zipcode[0].get("city", None)
else:
raise Exception("No zip code entry found for zip=" +
location)
# get_or_create gives an error if multiple such locations exist (which can happen)
# loc, created = Location.objects.get_or_create(location_type='ZIP_CODE_ONLY', zipcode=location, city=city)
loc = Location.objects.filter(
location_type="ZIP_CODE_ONLY",
zipcode=location,
city=city)
if not loc:
loc = Location.objects.create(
location_type="ZIP_CODE_ONLY",
zipcode=location,
city=city)
print("Zipcode " + location + " created for town " +
city)
else:
loc = loc.first()
print("Zipcode " + location + " found for town " +
city)
self._check_geography_unique(community, geography_type,
location)
else:
# assume this is a town, see we can find the zip codes associated with it
ss = location.split("-")
town = ss[0]
if len(ss) == 2:
state = ss[1]
else:
state = "MA"
zips = zipcodes.filter_by(city=town,
state=state,
zip_code_type="STANDARD")
print("Number of zipcodes = " + str(len(zips)))
if len(zips) > 0:
for zip in zips:
print(zip)
zipcode = zip["zip_code"]
# get_or_create gives an error if multiple such locations exist (which can happen)
# loc, created = Location.objects.get_or_create(location_type='ZIP_CODE_ONLY', zipcode=location, city=city)
loc = Location.objects.filter(
location_type="ZIP_CODE_ONLY",
zipcode=location,
city=town,
)
if not loc:
loc = Location.objects.create(
location_type="ZIP_CODE_ONLY",
zipcode=location,
city=town,
)
print("Zipcode " + zipcode + " created")
else:
loc = loc.first()
print("Zipcode " + zipcode + " found")
self._check_geography_unique(
community, geography_type, zipcode)
else:
print("No zipcodes found corresponding to town " +
town + ", " + state)
raise Exception(
"No zipcodes found corresponding to city " + town +
", " + state)
elif geography_type == "CITY":
# check that this city is found in the zipcodes list
ss = location.split("-")
city = ss[0]
if len(ss) == 2:
state = ss[1]
else:
state = "MA"
zips = zipcodes.filter_by(city=city,
state=state,
zip_code_type="STANDARD")
print("Number of zipcodes = " + str(len(zips)))
if len(zips) > 0:
# get_or_create gives an error if multiple such locations exist (which can happen)
# loc, created = Location.objects.get_or_create(location_type='ZIP_CODE_ONLY', zipcode=location, city=city)
loc = Location.objects.filter(location_type="CITY_ONLY",
city=city,
state=state)
if not loc:
loc = Location.objects.create(
location_type="CITY_ONLY", city=city, state=state)
print("City " + city + " created")
else:
loc = loc.first()
print("City " + city + " found")
else:
print("No zipcodes found corresponding to city " + city +
", " + state)
raise Exception(
"No zipcodes found corresponding to city " + city +
", " + state)
self._check_geography_unique(community, geography_type, city)
elif geography_type == "COUNTY":
# check that this county is found in the zipcodes list
ss = location.split("-")
county = ss[0]
if len(ss) == 2:
state = ss[1]
else:
state = "MA"
zips = zipcodes.filter_by(county=town,
state=state,
zip_code_type="STANDARD")
print("Number of zipcodes = " + str(len(zips)))
if len(zips) > 0:
# get_or_create gives an error if multiple such locations exist (which can happen)
# loc, created = Location.objects.get_or_create(location_type='ZIP_CODE_ONLY', zipcode=location, city=city)
loc = Location.objects.filter(location_type="COUNTY_ONLY",
county=county,
state=state)
if not loc:
loc = Location.objects.create(
location_type="COUNTY_ONLY",
county=county,
state=state)
print("County " + county + " created")
else:
loc = loc.first()
print("County " + county + " found")
else:
print("No zipcodes found corresponding to county " +
county + ", " + state)
raise Exception(
"No zipcodes found corresponding to county " + county +
", " + state)
self._check_geography_unique(community, geography_type, county)
elif geography_type == "STATE":
# check that this state is found in the zipcodes list
state = location
zips = zipcodes.filter_by(state=state,
zip_code_type="STANDARD")
print("Number of zipcodes = " + str(len(zips)))
if len(zips) > 0:
# get_or_create gives an error if multiple such locations exist (which can happen)
# loc, created = Location.objects.get_or_create(location_type='ZIP_CODE_ONLY', zipcode=location, city=city)
loc = Location.objects.filter(location_type="STATE_ONLY",
state=state)
if not loc:
loc = Location.objects.create(
location_type="STATE_ONLY", state=state)
print("State " + state + " created")
else:
loc = loc.first()
print("State " + state + " found")
else:
print("No zipcodes found corresponding to state " +
location)
raise Exception(
"No zipcodes found corresponding to state " + location)
self._check_geography_unique(community, geography_type,
location)
elif geography_type == "COUNTRY":
# check that this state is found in the zipcodes list
country = location
zips = zipcodes.filter_by(country=country,
zip_code_type="STANDARD")
print("Number of zipcodes = " + str(len(zips)))
if len(zips) > 0:
# get_or_create gives an error if multiple such locations exist (which can happen)
# loc, created = Location.objects.get_or_create(location_type='ZIP_CODE_ONLY', zipcode=location, city=city)
loc = Location.objects.filter(location_type="COUNTRY_ONLY",
country=country)
if not loc:
loc = Location.objects.create(
location_type="COUNTRY_ONLY", country=country)
print("Country " + country + " created")
else:
loc = loc.first()
print("Country " + country + " found")
else:
print("No zipcodes found corresponding to country " +
location)
raise Exception(
"No zipcodes found corresponding to country " +
location)
self._check_geography_unique(community, geography_type,
location)
else:
raise Exception("Unexpected geography type: " +
str(geography_type))
# should be a five character string
community.locations.add(loc)