def plot_probability_map():
timeMap = ddict(list)
maps = []
for i in range(24):
maps.append(gmaps.Map())
with open('params/probmap', 'r') as data:
reader = csv.reader(data, delimiter=',')
for row in reader:
lat0 = row[2].replace("'", "")
lat1 = row[3].replace("'", "")
lon0 = row[0].replace("'", "")
lon1 = row[1].replace("'", "")
weight = float(row[-1].replace("'", ""))
time = int(row[4].replace("'", ""))
dot = centerize_grid(((lon0, lon1), (lat0, lat1)))
point = (dot[1], dot[0], weight)
timeMap[time].append(point)
overlapped_map = gmaps.Map()
for i in range(24):
data = timeMap[i]
heatmap_layer = gmaps.WeightedHeatmap(data=data)
heatmap_layer.max_intensity = 1
maps[i].add_layer(heatmap_layer)
overlapped_map.add_layer(heatmap_layer)
overlapped_map
def plot(locations):
m = gmaps.Map()
pts_layer = gmaps.symbol_layer(
locations, fill_color="red", stroke_color="red", scale=1
)
m = gmaps.Map()
m.add_layer(pts_layer)
return m
def generate_heatmap(flair_list):
comments_coords_list = []
mapping_dict = load_instituation_coords()
for institution in flair_list:
if mapping_dict.get(institution):
comments_coords_list.append(mapping_dict[institution])
gmaps.configure(api_key=GOOGLE_MAPS_API_KEY)
m = gmaps.Map()
heatmap_layer = gmaps.Heatmap(data=comments_coords_list)
heatmap_layer.max_intensity = 80
heatmap_layer.point_radius = 40
m.add_layer(heatmap_layer)
return m
def add_station_maps(API='AIzaSyASHzuwtrEHNRuadF-MhNbARUnSyFfRA9Q'):
"""
This function displays a google map of the
locations of the three stations with the
tidal data
"""
# Put in the locations of the tidal stations
NB = [48 + 22.2/60, -(124 + 36.1/60)]
PA = [48 + 7.5/60, -(123 + 26.5/60)]
PT = [48 + 6.8/60, -(122 + 45.6/60)]
latlon = [tuple(NB), tuple(PA), tuple(PT)]
# Generate the google map
try:
import gmaps
gmaps.configure(api_key=API)
m = gmaps.Map()
markers = gmaps.marker_layer(latlon)
m.add_layer(markers)
return m
except ImportError:
raise ImportError('Please install gmaps package')
else:
return None
def plot_dropoff_distribution():
timeLocMap = ddict(list)
with open('params/dropmap', 'r') as data:
reader = csv.reader(data, delimiter=',')
for row in reader:
dot = centerize_grid(((row[0], row[1]), (row[2], row[3])))
locs = row[5].split(' ')
for loc in locs:
coords = loc.split(':')
timeLocMap[(int(row[4]), dot)].append(
(float(coords[1]), float(coords[0])))
# lat, lon
gmap = gmaps.Map()
start = random.choice(timeLocMap.keys())
print start
print timeLocMap[start]
pickup_layer = gmaps.symbol_layer([(start[1][1], start[1][0])],
fill_color='green',
stroke_color='green',
scale=2)
dropoff_layer = gmaps.Heatmap(data=timeLocMap[start])
gmap.add_layer(pickup_layer)
gmap.add_layer(dropoff_layer)
gmap
switcher = {
0: 0,
1: 4,
2: 3,
3: 2,
4: 1,
}
return switcher[severity]
coordinate_lst = list(zip(chp_summary.LATITUDE, chp_summary.LONGITUDE))
weights = chp_df[chp_df.CASE_ID.isin(chp_summary.CASE_ID)].COLLISION_SEVERITY
sorted_weight = [severity_weights(i) for i in weights]
# Declare a google map object, add a heatmap layer and tune the parameters. Note that `heatmap_layer.gradient` changes the color. The default is from green to red.
get_ipython().run_line_magic('matplotlib', 'inline')
m = gmaps.Map()
heatmap_layer = gmaps.heatmap_layer(locations=coordinate_lst,
weights=sorted_weight)
heatmap_layer.max_intensity = 100
heatmap_layer.point_radius = 7
heatmap_layer.opacity = 1
heatmap_layer.gradient = [(0, 255, 255, 0), (0, 255, 255, 1), (0, 191, 255, 1),
(0, 127, 255, 1), (0, 63, 255, 1), (0, 0, 255, 1),
(0, 0, 223, 1), (0, 0, 191, 1), (0, 0, 159, 1),
(0, 0, 127, 1), (63, 0, 91, 1), (127, 0, 63, 1),
(191, 0, 31, 1), (255, 0, 0, 1)]
m.add_layer(heatmap_layer)
m
def steps(src, from_dt, to_dt):
import sys
MODULES_PATH = '../code/'
if MODULES_PATH not in sys.path:
sys.path.append(MODULES_PATH)
import mfuncs
import pandas as pd
import numpy as np
from tqdm import tqdm
tqdm.pandas()
pd.options.display.max_columns = 1000
import lightgbm as lgb
from sklearn.neighbors import NearestNeighbors
# start of step 01
df_train = pd.read_csv('../data/train_set.csv')
df_test = pd.read_csv('../data/test_set.csv')
rnm = {
'atm_address_lat': 'atm_lat',
'atm_address_lon': 'atm_lon',
'pos_adress_lat': 'pos_lat',
'pos_adress_lon': 'pos_lon',
'home_add_lat': 'home_lat',
'home_add_lon': 'home_lon',
'work_add_lat': 'work_lat',
'work_add_lon': 'work_lon',
}
df_train.rename(columns=rnm, inplace=True)
df_test.rename(columns=rnm, inplace=True)
# start of step 02
df_train['target_work'] = df_train.progress_apply(mfuncs.add_poswork_target, axis=1)
df_train['target_home'] = df_train.progress_apply(mfuncs.add_poshome_target, axis=1)
# start of step 03
df_train.to_csv('../data/train_1.csv', index=None)
# start of step 04
df_train.info()
# start of step 05
df_train.head()
# start of step 06
df_train.country.value_counts(normalize=True)[:10]
print(df_train.shape, df_test.shape)
df_train = df_train[df_train.country.isin(['RUS', 'RU'])]
df_test = df_test[df_test.country.isin(['RUS', 'RU'])]
print(df_train.shape, df_test.shape)
del df_train['country'], df_test['country']
# start of step 07
print(df_train.shape, df_train.currency.value_counts(normalize=True))
df_train = df_train[df_train.currency == 643]
print(df_train.shape)
del df_train['currency']
# start of step 08
print(df_train.shape, df_train.currency.value_counts(normalize=True))
df_train = df_train[df_train.currency == 643]
print(df_train.shape)
del df_train['currency']
# start of step 09
print(df_train.shape)
gb = df_train.groupby('customer_id')['work_lat'].agg('nunique')
cid_incorrect = gb[gb == 2].index
df_train = df_train[~df_train.customer_id.isin(cid_incorrect.values)]
print(df_train.shape)
gb = df_train.groupby('customer_id')['home_lat'].agg('nunique')
cid_incorrect = gb[gb == 2].index
df_train = df_train[~df_train.customer_id.isin(cid_incorrect.values)]
print(df_train.shape)
# start of step 10
print(df_train.shape)
df_train = df_train[df_train[['atm_lat', 'pos_lat']].isnull().sum(axis=1) == 1]
print(df_train.shape)
df_train['type'] = 'atm'
df_train.loc[~df_train['pos_lat'].isnull(), 'type'] = 'pos'
df_train['type'].value_counts()
# start of step 11
cid = df_train.sample(1)['customer_id'].values[0]
df_an = df_train[df_train.customer_id == cid]
df_point_dup = df_an.groupby(['pos_lat', 'pos_lon']).agg('size').reset_index()
df_point_dup.columns = ['pos_lat', 'pos_lon', 'pos_customer_freq']
df_an = pd.merge(df_an, df_point_dup, on=['pos_lat', 'pos_lon'], how='left')
df_an.head()
# start of step 12
df_train.head()
df_train[df_train.type == 'pos'].drop_duplicates(['pos_lat',
'pos_lon']).groupby(['terminal_id']).agg('size').value_counts()
df_train[df_train.type == 'atm'].drop_duplicates(['atm_lat',
'atm_lon']).groupby(['terminal_id']).agg('size').value_counts()
df_train[df_train.terminal_id == '1e15d02895068c3a864432f0c06f5ece']['atm_address'].unique()
df_train[df_train.type == 'atm'].drop_duplicates(['atm_lat',
'atm_lon']).groupby(['terminal_id']).agg('size')
import gmaps
API_KEY = 'AIzaSyCG_RL0_kavuEaJAqEN5xXbU4h0VJUbA9M'
gmaps.configure(api_key=API_KEY) # Your Google API key
cid = '0dc0137d280a2a82d2dc89282450ff1b'
cid = df_train.sample(1)['customer_id'].values[0]
df_an = df_train[df_train.customer_id == cid]
center_home = df_an[['home_lat', 'home_lon']].drop_duplicates().values
center_work = df_an[['work_lat', 'work_lon']].drop_duplicates().values
points_pos = df_an[['pos_lat', 'pos_lon']].dropna().values
points_atm = df_an[['atm_lat', 'atm_lon']].dropna().values
print(center_home.shape, center_work.shape, points_pos.shape, points_atm.shape)
gmap = gmaps.Map()
if len(points_pos) > 0:
gmap.add_layer(gmaps.symbol_layer(points_pos, hover_text='pos',
fill_color="blue", stroke_color="blue", scale=3))
if len(points_atm) > 0:
gmap.add_layer(gmaps.symbol_layer(points_atm, hover_text='atm',
fill_color="red", stroke_color="red", scale=3))
if not np.isnan(center_home)[0][0]:
gmap.add_layer(gmaps.marker_layer(center_home, label='home'))
if not np.isnan(center_work)[0][0]:
gmap.add_layer(gmaps.marker_layer(center_work, label='work'))
gmap
center_home = df_train[['home_lat', 'home_lon']].dropna().values
center_work = df_train[['work_lat', 'work_lon']].dropna().values
gmap = gmaps.Map()
gmap.add_layer(gmaps.symbol_layer(center_home, fill_color="red", stroke_color="red"))
gmap
np.isnan(center_home)
df_train.groupby(['customer_id']).agg('size').sort_values().value_counts()
df_test.customer_id.drop_duplicates().isin(df_train.customer_id.unique()).mean()
df_train['duplicated'] = df_train.duplicated()
df_pos = df_train[df_train['type'] == 'pos']
# target == pos in
df_pos['target_work'] = df_pos.progress_apply(mfuncs.add_poswork_target, axis=1)
df_pos['target_home'] = df_pos.progress_apply(mfuncs.add_poshome_target, axis=1)
df_pos['target_work'].mean(), df_pos['target_home'].mean()
df_pos.to_csv('../data/df_pos.csv', index=None)
df_pos = pd.read_csv('../data/df_pos.csv')
df_point_dup = df_pos.groupby(['customer_id', 'pos_lat', 'pos_lon']).agg('size').reset_index()
df_point_dup.columns = ['customer_id', 'pos_lat', 'pos_lon', 'pos_customer_freq']
df_pos = pd.merge(df_pos, df_point_dup, on=['customer_id', 'pos_lat', 'pos_lon'], how='left')
dfs = []
for cid in tqdm(df_pos.customer_id.unique()):
df_an = df_pos[df_pos.customer_id == cid]
df_an = mfuncs.add_dist_to_neighbours(df_an)
dfs.append(df_an)
df_pos['transaction_date'] = pd.to_datetime(df_pos['transaction_date'], format='%Y-%m-%d')
df_pos['month'] = df_pos.transaction_date.dt.month
df_pos['day'] = df_pos.transaction_date.dt.day
df_pos['dayofyear'] = df_pos.transaction_date.dt.dayofyear
df_pos['dayofweek'] = df_pos.transaction_date.dt.dayofweek
df_pos.transaction_date.dtype
df_gb = df_pos.groupby('customer_id')
coord_stat_df = df_gb[['amount', 'pos_lat', 'pos_lon']].agg(['mean', 'max', 'min'])
coord_stat_df['transactions_per_user'] = df_gb.agg('size')
coord_stat_df.columns = ['_'.join(col).strip() for col in coord_stat_df.columns.values]
coord_stat_df.reset_index(inplace=True)
df_pos = pd.merge(df_pos, coord_stat_df, on='customer_id', how='left')
cols = ['pos_lat', 'pos_lon']
types = ['min', 'max', 'mean']
for c in cols:
for t in types:
df_pos['{}_diff_{}'.format(c, t)] = np.abs(df_pos[c] - df_pos['{}_{}'.format(c, t)])
df_pos = pd.concat([df_pos, pd.get_dummies(df_pos['mcc'], prefix='mcc')], axis=1)
del df_pos['mcc']
df_pos.head()
drop_cols = ['customer_id', 'terminal_id', 'target_home', 'target_work', 'atm_address',
'pos_address', 'work_add_lat', 'work_add_lon', 'home_add_lat', 'home_add_lon',
'city', 'type', 'transaction_date']
drop_cols += ['atm_address', 'atm_address_lat', 'atm_address_lon']
df_pos.drop(drop_cols, 1, errors='ignore').head()
# drop_cols = ['pos_address', 'pos_address_lat', 'pos_address_lon']
from sklearn.model_selection import train_test_split, StratifiedKFold, KFold
df_pos_id = df_pos.customer_id.drop_duplicates().reset_index(drop=True)
skf_id = list(KFold(n_splits=5, shuffle=True, random_state=15).split(df_pos_id))
skf = []
for train_ind, test_ind in skf_id:
train_ind_ = df_pos[df_pos.customer_id.isin(df_pos_id.loc[train_ind].values)].index.values
test_ind_ = df_pos[df_pos.customer_id.isin(df_pos_id.loc[test_ind].values)].index.values
skf.append([train_ind_, test_ind_])
df_pos['target_work'].mean()
df_pos.head()
cid = '442fd7e3af4d8c3acd7807aa65bb5e85'
df_an = df_pos[df_pos.customer_id == cid]
df_an = mfuncs.add_dist_to_neighbours(df_an)
df_pos.customer_id.unique
if np.array([1]).size:
print(1)
lgb_train = lgb.Dataset(df_pos.drop(drop_cols, 1, errors='ignore'), df_pos['target_home'])
params = {
'objective': 'binary',
'num_leaves': 511,
'learning_rate': 0.05,
# 'metric' : 'error',
'feature_fraction': 0.8,
'bagging_fraction': 0.8,
'bagging_freq': 1,
'num_threads': 12,
'verbose': 0,
}
gbm = lgb.cv(params,
lgb_train,
num_boost_round=2000,
folds=skf,
verbose_eval=10,
early_stopping_rounds=500)
df_pos.loc[i2].shape
i1, i2 = skf[0]
df_pos[df_pos.loc[i1]]['customer_id'].unique
df_pos.loc[i1]
df_pos.dtypes
res = df_pos
return res
# ### Humidity Heatmap # * Configure gmaps. # * Use the Lat and Lng as locations and Humidity as the weight. # * Add Heatmap layer to map. # In[15]: gmaps.configure(api_key=g_key) # In[16]: locations = df1[["Lat", "Lng"]] humidity = df1["Humidity"] fig1 = gmaps.Map() heat = gmaps.heatmap_layer(locations, weights=humidity) fig1.add_layer(heat) # ### Create new DataFrame fitting weather criteria # * Narrow down the cities to fit weather conditions. # * Drop any rows will null values. # In[17]: idf = df1.loc[df1['Max Temp'] < 80, :] idf = idf.loc[idf['Max Temp'] > 70, :] idf = idf.loc[idf['Wind Speed'] < 10, :] idf = idf.loc[idf['Cloudiness'] == 0, :]
def main():
db_file = "RaMBLE_playstore_v35.14_20200621_2000.sqlite"
not_before_date = {"not_before_date": '2020-06-21'}
connection = sqlite3.connect(db_file)
connection.row_factory = sqlite3.Row
cursor = connection.cursor()
cursor.execute(
"""SELECT
datetime(locations.timestamp, 'unixepoch', 'localtime') as 'Timestamp',
devices.service_data as 'Rolling Proximity Identifier',
locations.rssi as 'RSSI',
locations.latitude as 'Latitude',
locations.longitude as 'Longitude',
locations.accuracy as 'Accuracy'
FROM devices
INNER JOIN locations ON devices.id = locations.device_id
WHERE
service_uuids = "fd6f" AND
datetime(locations.timestamp, 'unixepoch', 'localtime') > datetime( :not_before_date , 'localtime')
ORDER BY devices.service_data """, not_before_date)
result = cursor.fetchall()
connection.close()
if not result:
print("No exposure notification beacons found!")
return
locations = []
info_texts = []
colors = []
temp_color = 0
r_p_id = 0
for line in result:
if r_p_id != line['Rolling Proximity Identifier']:
rgb = hue_to_RGB(random.uniform(0.0, 1.0))
temp_color = 'rgba(' + str(round(rgb.r)) + ', ' + str(round(
rgb.g)) + ', ' + str(round(rgb.b)) + ', 1.0)'
r_p_id = line['Rolling Proximity Identifier']
lat, lon = destination_point(line["Latitude"], line["Longitude"],
random.uniform(0, line["Accuracy"]),
random.uniform(0, 360))
locations.append([lat, lon])
info_texts.append(line['Timestamp'] + "<br>" +
line['Rolling Proximity Identifier'])
colors.append(temp_color)
# gmaps.configure(api_key='API_KEY')
fig = gmaps.Map(width='100%', height='100vh', layout={'height': '98vh'})
heatmap_layer = gmaps.heatmap_layer(locations, point_radius=30)
dots = gmaps.symbol_layer(locations,
fill_color=colors,
stroke_color=colors,
scale=3,
info_box_content=info_texts,
display_info_box=True)
fig.add_layer(heatmap_layer)
fig.add_layer(dots)
embed_minimal_html('export.html', views=[fig], title="Corona App Export")
