def plot_probability_map(): timeMap = ddict(list) maps = [] for i in range(24): maps.append(gmaps.Map()) with open('params/probmap', 'r') as data: reader = csv.reader(data, delimiter=',') for row in reader: lat0 = row[2].replace("'", "") lat1 = row[3].replace("'", "") lon0 = row[0].replace("'", "") lon1 = row[1].replace("'", "") weight = float(row[-1].replace("'", "")) time = int(row[4].replace("'", "")) dot = centerize_grid(((lon0, lon1), (lat0, lat1))) point = (dot[1], dot[0], weight) timeMap[time].append(point) overlapped_map = gmaps.Map() for i in range(24): data = timeMap[i] heatmap_layer = gmaps.WeightedHeatmap(data=data) heatmap_layer.max_intensity = 1 maps[i].add_layer(heatmap_layer) overlapped_map.add_layer(heatmap_layer) overlapped_map
def plot(locations): m = gmaps.Map() pts_layer = gmaps.symbol_layer( locations, fill_color="red", stroke_color="red", scale=1 ) m = gmaps.Map() m.add_layer(pts_layer) return m
def generate_heatmap(flair_list): comments_coords_list = [] mapping_dict = load_instituation_coords() for institution in flair_list: if mapping_dict.get(institution): comments_coords_list.append(mapping_dict[institution]) gmaps.configure(api_key=GOOGLE_MAPS_API_KEY) m = gmaps.Map() heatmap_layer = gmaps.Heatmap(data=comments_coords_list) heatmap_layer.max_intensity = 80 heatmap_layer.point_radius = 40 m.add_layer(heatmap_layer) return m
def add_station_maps(API='AIzaSyASHzuwtrEHNRuadF-MhNbARUnSyFfRA9Q'): """ This function displays a google map of the locations of the three stations with the tidal data """ # Put in the locations of the tidal stations NB = [48 + 22.2/60, -(124 + 36.1/60)] PA = [48 + 7.5/60, -(123 + 26.5/60)] PT = [48 + 6.8/60, -(122 + 45.6/60)] latlon = [tuple(NB), tuple(PA), tuple(PT)] # Generate the google map try: import gmaps gmaps.configure(api_key=API) m = gmaps.Map() markers = gmaps.marker_layer(latlon) m.add_layer(markers) return m except ImportError: raise ImportError('Please install gmaps package') else: return None
def plot_dropoff_distribution(): timeLocMap = ddict(list) with open('params/dropmap', 'r') as data: reader = csv.reader(data, delimiter=',') for row in reader: dot = centerize_grid(((row[0], row[1]), (row[2], row[3]))) locs = row[5].split(' ') for loc in locs: coords = loc.split(':') timeLocMap[(int(row[4]), dot)].append( (float(coords[1]), float(coords[0]))) # lat, lon gmap = gmaps.Map() start = random.choice(timeLocMap.keys()) print start print timeLocMap[start] pickup_layer = gmaps.symbol_layer([(start[1][1], start[1][0])], fill_color='green', stroke_color='green', scale=2) dropoff_layer = gmaps.Heatmap(data=timeLocMap[start]) gmap.add_layer(pickup_layer) gmap.add_layer(dropoff_layer) gmap
switcher = { 0: 0, 1: 4, 2: 3, 3: 2, 4: 1, } return switcher[severity] coordinate_lst = list(zip(chp_summary.LATITUDE, chp_summary.LONGITUDE)) weights = chp_df[chp_df.CASE_ID.isin(chp_summary.CASE_ID)].COLLISION_SEVERITY sorted_weight = [severity_weights(i) for i in weights] # Declare a google map object, add a heatmap layer and tune the parameters. Note that `heatmap_layer.gradient` changes the color. The default is from green to red. get_ipython().run_line_magic('matplotlib', 'inline') m = gmaps.Map() heatmap_layer = gmaps.heatmap_layer(locations=coordinate_lst, weights=sorted_weight) heatmap_layer.max_intensity = 100 heatmap_layer.point_radius = 7 heatmap_layer.opacity = 1 heatmap_layer.gradient = [(0, 255, 255, 0), (0, 255, 255, 1), (0, 191, 255, 1), (0, 127, 255, 1), (0, 63, 255, 1), (0, 0, 255, 1), (0, 0, 223, 1), (0, 0, 191, 1), (0, 0, 159, 1), (0, 0, 127, 1), (63, 0, 91, 1), (127, 0, 63, 1), (191, 0, 31, 1), (255, 0, 0, 1)] m.add_layer(heatmap_layer) m
def steps(src, from_dt, to_dt): import sys MODULES_PATH = '../code/' if MODULES_PATH not in sys.path: sys.path.append(MODULES_PATH) import mfuncs import pandas as pd import numpy as np from tqdm import tqdm tqdm.pandas() pd.options.display.max_columns = 1000 import lightgbm as lgb from sklearn.neighbors import NearestNeighbors # start of step 01 df_train = pd.read_csv('../data/train_set.csv') df_test = pd.read_csv('../data/test_set.csv') rnm = { 'atm_address_lat': 'atm_lat', 'atm_address_lon': 'atm_lon', 'pos_adress_lat': 'pos_lat', 'pos_adress_lon': 'pos_lon', 'home_add_lat': 'home_lat', 'home_add_lon': 'home_lon', 'work_add_lat': 'work_lat', 'work_add_lon': 'work_lon', } df_train.rename(columns=rnm, inplace=True) df_test.rename(columns=rnm, inplace=True) # start of step 02 df_train['target_work'] = df_train.progress_apply(mfuncs.add_poswork_target, axis=1) df_train['target_home'] = df_train.progress_apply(mfuncs.add_poshome_target, axis=1) # start of step 03 df_train.to_csv('../data/train_1.csv', index=None) # start of step 04 df_train.info() # start of step 05 df_train.head() # start of step 06 df_train.country.value_counts(normalize=True)[:10] print(df_train.shape, df_test.shape) df_train = df_train[df_train.country.isin(['RUS', 'RU'])] df_test = df_test[df_test.country.isin(['RUS', 'RU'])] print(df_train.shape, df_test.shape) del df_train['country'], df_test['country'] # start of step 07 print(df_train.shape, df_train.currency.value_counts(normalize=True)) df_train = df_train[df_train.currency == 643] print(df_train.shape) del df_train['currency'] # start of step 08 print(df_train.shape, df_train.currency.value_counts(normalize=True)) df_train = df_train[df_train.currency == 643] print(df_train.shape) del df_train['currency'] # start of step 09 print(df_train.shape) gb = df_train.groupby('customer_id')['work_lat'].agg('nunique') cid_incorrect = gb[gb == 2].index df_train = df_train[~df_train.customer_id.isin(cid_incorrect.values)] print(df_train.shape) gb = df_train.groupby('customer_id')['home_lat'].agg('nunique') cid_incorrect = gb[gb == 2].index df_train = df_train[~df_train.customer_id.isin(cid_incorrect.values)] print(df_train.shape) # start of step 10 print(df_train.shape) df_train = df_train[df_train[['atm_lat', 'pos_lat']].isnull().sum(axis=1) == 1] print(df_train.shape) df_train['type'] = 'atm' df_train.loc[~df_train['pos_lat'].isnull(), 'type'] = 'pos' df_train['type'].value_counts() # start of step 11 cid = df_train.sample(1)['customer_id'].values[0] df_an = df_train[df_train.customer_id == cid] df_point_dup = df_an.groupby(['pos_lat', 'pos_lon']).agg('size').reset_index() df_point_dup.columns = ['pos_lat', 'pos_lon', 'pos_customer_freq'] df_an = pd.merge(df_an, df_point_dup, on=['pos_lat', 'pos_lon'], how='left') df_an.head() # start of step 12 df_train.head() df_train[df_train.type == 'pos'].drop_duplicates(['pos_lat', 'pos_lon']).groupby(['terminal_id']).agg('size').value_counts() df_train[df_train.type == 'atm'].drop_duplicates(['atm_lat', 'atm_lon']).groupby(['terminal_id']).agg('size').value_counts() df_train[df_train.terminal_id == '1e15d02895068c3a864432f0c06f5ece']['atm_address'].unique() df_train[df_train.type == 'atm'].drop_duplicates(['atm_lat', 'atm_lon']).groupby(['terminal_id']).agg('size') import gmaps API_KEY = 'AIzaSyCG_RL0_kavuEaJAqEN5xXbU4h0VJUbA9M' gmaps.configure(api_key=API_KEY) # Your Google API key cid = '0dc0137d280a2a82d2dc89282450ff1b' cid = df_train.sample(1)['customer_id'].values[0] df_an = df_train[df_train.customer_id == cid] center_home = df_an[['home_lat', 'home_lon']].drop_duplicates().values center_work = df_an[['work_lat', 'work_lon']].drop_duplicates().values points_pos = df_an[['pos_lat', 'pos_lon']].dropna().values points_atm = df_an[['atm_lat', 'atm_lon']].dropna().values print(center_home.shape, center_work.shape, points_pos.shape, points_atm.shape) gmap = gmaps.Map() if len(points_pos) > 0: gmap.add_layer(gmaps.symbol_layer(points_pos, hover_text='pos', fill_color="blue", stroke_color="blue", scale=3)) if len(points_atm) > 0: gmap.add_layer(gmaps.symbol_layer(points_atm, hover_text='atm', fill_color="red", stroke_color="red", scale=3)) if not np.isnan(center_home)[0][0]: gmap.add_layer(gmaps.marker_layer(center_home, label='home')) if not np.isnan(center_work)[0][0]: gmap.add_layer(gmaps.marker_layer(center_work, label='work')) gmap center_home = df_train[['home_lat', 'home_lon']].dropna().values center_work = df_train[['work_lat', 'work_lon']].dropna().values gmap = gmaps.Map() gmap.add_layer(gmaps.symbol_layer(center_home, fill_color="red", stroke_color="red")) gmap np.isnan(center_home) df_train.groupby(['customer_id']).agg('size').sort_values().value_counts() df_test.customer_id.drop_duplicates().isin(df_train.customer_id.unique()).mean() df_train['duplicated'] = df_train.duplicated() df_pos = df_train[df_train['type'] == 'pos'] # target == pos in df_pos['target_work'] = df_pos.progress_apply(mfuncs.add_poswork_target, axis=1) df_pos['target_home'] = df_pos.progress_apply(mfuncs.add_poshome_target, axis=1) df_pos['target_work'].mean(), df_pos['target_home'].mean() df_pos.to_csv('../data/df_pos.csv', index=None) df_pos = pd.read_csv('../data/df_pos.csv') df_point_dup = df_pos.groupby(['customer_id', 'pos_lat', 'pos_lon']).agg('size').reset_index() df_point_dup.columns = ['customer_id', 'pos_lat', 'pos_lon', 'pos_customer_freq'] df_pos = pd.merge(df_pos, df_point_dup, on=['customer_id', 'pos_lat', 'pos_lon'], how='left') dfs = [] for cid in tqdm(df_pos.customer_id.unique()): df_an = df_pos[df_pos.customer_id == cid] df_an = mfuncs.add_dist_to_neighbours(df_an) dfs.append(df_an) df_pos['transaction_date'] = pd.to_datetime(df_pos['transaction_date'], format='%Y-%m-%d') df_pos['month'] = df_pos.transaction_date.dt.month df_pos['day'] = df_pos.transaction_date.dt.day df_pos['dayofyear'] = df_pos.transaction_date.dt.dayofyear df_pos['dayofweek'] = df_pos.transaction_date.dt.dayofweek df_pos.transaction_date.dtype df_gb = df_pos.groupby('customer_id') coord_stat_df = df_gb[['amount', 'pos_lat', 'pos_lon']].agg(['mean', 'max', 'min']) coord_stat_df['transactions_per_user'] = df_gb.agg('size') coord_stat_df.columns = ['_'.join(col).strip() for col in coord_stat_df.columns.values] coord_stat_df.reset_index(inplace=True) df_pos = pd.merge(df_pos, coord_stat_df, on='customer_id', how='left') cols = ['pos_lat', 'pos_lon'] types = ['min', 'max', 'mean'] for c in cols: for t in types: df_pos['{}_diff_{}'.format(c, t)] = np.abs(df_pos[c] - df_pos['{}_{}'.format(c, t)]) df_pos = pd.concat([df_pos, pd.get_dummies(df_pos['mcc'], prefix='mcc')], axis=1) del df_pos['mcc'] df_pos.head() drop_cols = ['customer_id', 'terminal_id', 'target_home', 'target_work', 'atm_address', 'pos_address', 'work_add_lat', 'work_add_lon', 'home_add_lat', 'home_add_lon', 'city', 'type', 'transaction_date'] drop_cols += ['atm_address', 'atm_address_lat', 'atm_address_lon'] df_pos.drop(drop_cols, 1, errors='ignore').head() # drop_cols = ['pos_address', 'pos_address_lat', 'pos_address_lon'] from sklearn.model_selection import train_test_split, StratifiedKFold, KFold df_pos_id = df_pos.customer_id.drop_duplicates().reset_index(drop=True) skf_id = list(KFold(n_splits=5, shuffle=True, random_state=15).split(df_pos_id)) skf = [] for train_ind, test_ind in skf_id: train_ind_ = df_pos[df_pos.customer_id.isin(df_pos_id.loc[train_ind].values)].index.values test_ind_ = df_pos[df_pos.customer_id.isin(df_pos_id.loc[test_ind].values)].index.values skf.append([train_ind_, test_ind_]) df_pos['target_work'].mean() df_pos.head() cid = '442fd7e3af4d8c3acd7807aa65bb5e85' df_an = df_pos[df_pos.customer_id == cid] df_an = mfuncs.add_dist_to_neighbours(df_an) df_pos.customer_id.unique if np.array([1]).size: print(1) lgb_train = lgb.Dataset(df_pos.drop(drop_cols, 1, errors='ignore'), df_pos['target_home']) params = { 'objective': 'binary', 'num_leaves': 511, 'learning_rate': 0.05, # 'metric' : 'error', 'feature_fraction': 0.8, 'bagging_fraction': 0.8, 'bagging_freq': 1, 'num_threads': 12, 'verbose': 0, } gbm = lgb.cv(params, lgb_train, num_boost_round=2000, folds=skf, verbose_eval=10, early_stopping_rounds=500) df_pos.loc[i2].shape i1, i2 = skf[0] df_pos[df_pos.loc[i1]]['customer_id'].unique df_pos.loc[i1] df_pos.dtypes res = df_pos return res
# ### Humidity Heatmap # * Configure gmaps. # * Use the Lat and Lng as locations and Humidity as the weight. # * Add Heatmap layer to map. # In[15]: gmaps.configure(api_key=g_key) # In[16]: locations = df1[["Lat", "Lng"]] humidity = df1["Humidity"] fig1 = gmaps.Map() heat = gmaps.heatmap_layer(locations, weights=humidity) fig1.add_layer(heat) # ### Create new DataFrame fitting weather criteria # * Narrow down the cities to fit weather conditions. # * Drop any rows will null values. # In[17]: idf = df1.loc[df1['Max Temp'] < 80, :] idf = idf.loc[idf['Max Temp'] > 70, :] idf = idf.loc[idf['Wind Speed'] < 10, :] idf = idf.loc[idf['Cloudiness'] == 0, :]
def main(): db_file = "RaMBLE_playstore_v35.14_20200621_2000.sqlite" not_before_date = {"not_before_date": '2020-06-21'} connection = sqlite3.connect(db_file) connection.row_factory = sqlite3.Row cursor = connection.cursor() cursor.execute( """SELECT datetime(locations.timestamp, 'unixepoch', 'localtime') as 'Timestamp', devices.service_data as 'Rolling Proximity Identifier', locations.rssi as 'RSSI', locations.latitude as 'Latitude', locations.longitude as 'Longitude', locations.accuracy as 'Accuracy' FROM devices INNER JOIN locations ON devices.id = locations.device_id WHERE service_uuids = "fd6f" AND datetime(locations.timestamp, 'unixepoch', 'localtime') > datetime( :not_before_date , 'localtime') ORDER BY devices.service_data """, not_before_date) result = cursor.fetchall() connection.close() if not result: print("No exposure notification beacons found!") return locations = [] info_texts = [] colors = [] temp_color = 0 r_p_id = 0 for line in result: if r_p_id != line['Rolling Proximity Identifier']: rgb = hue_to_RGB(random.uniform(0.0, 1.0)) temp_color = 'rgba(' + str(round(rgb.r)) + ', ' + str(round( rgb.g)) + ', ' + str(round(rgb.b)) + ', 1.0)' r_p_id = line['Rolling Proximity Identifier'] lat, lon = destination_point(line["Latitude"], line["Longitude"], random.uniform(0, line["Accuracy"]), random.uniform(0, 360)) locations.append([lat, lon]) info_texts.append(line['Timestamp'] + "<br>" + line['Rolling Proximity Identifier']) colors.append(temp_color) # gmaps.configure(api_key='API_KEY') fig = gmaps.Map(width='100%', height='100vh', layout={'height': '98vh'}) heatmap_layer = gmaps.heatmap_layer(locations, point_radius=30) dots = gmaps.symbol_layer(locations, fill_color=colors, stroke_color=colors, scale=3, info_box_content=info_texts, display_info_box=True) fig.add_layer(heatmap_layer) fig.add_layer(dots) embed_minimal_html('export.html', views=[fig], title="Corona App Export")