def csvUpdate(self , devicePairedEach ):
self.D = devicePairedEach
resultQuery1 = wifiProject.resultQuery1(self.D[0] , self.D[1] , self.time_now , self.time_past)
cur.execute(resultQuery1)
#data = cur.fetchall()
data1 = cur.fetchall()
resultQuery2 = wifiProject.resultQuery2(self.D[0] , self.D[1] , self.time_now , self.time_past)
#print(resultQuery2)
cur.execute(resultQuery2)
data2 = cur.fetchall()
data = data1 + data2
#print(data)
df = pd.DataFrame ( data , columns = ['mac' , 'logtimeLeft' , 'logtimeRight'])
#print(df)
func = lambda x : int(x)
df['logtimeLeft'] , df['logtimeRight'] = df['logtimeLeft'].map(func) , df['logtimeRight'].map(func)
df['route_id'] , df['travel_time'] = [self.D[2] if index else self.D[3] for index in df.logtimeLeft<df.logtimeRight] , [abs(index) for index in df.logtimeLeft - df.logtimeRight]
datetimeMap = lambda x : datetime.strftime(datetime.fromtimestamp(x) , "%a %b %d %H:%M:%S %Y" )
dfLeftMap = lambda x : df.loc[x,'logtimeLeft']
dfRightMap = lambda x : df.loc[x , 'logtimeRight']
df['datetime'] = [ datetimeMap(dfLeftMap(index)) if dfLeftMap(index) > dfRightMap(index) else datetimeMap(dfRightMap(index)) for index in df.index ]
df = df.drop(['logtimeLeft' ], axis = 1 )
df = df[['logtimeRight' , 'datetime' , 'route_id' , 'mac','travel_time']]
df.columns = ['timestamp','datetime' , 'route_id' ,'mac','travel_time']
groups = df.groupby("route_id")
path = wifiProject.csvFilePath()
left1 = path + f"{self.D[2]}-"+datetime.strftime(datetime.fromtimestamp(self.time_now) , "%d-%m-%y")+".csv"
right1 = path + f"{self.D[3]}-" + datetime.strftime(datetime.fromtimestamp(self.time_now) , "%d-%m-%y") + ".csv"
try:
if os.path.isfile(left1):
#print(groups.get_group(self.D[2]))
groups.get_group(self.D[2]) .to_csv ( left1 ,mode= 'a' , index= False , header= False )
else:
groups.get_group(self.D[2]) .to_csv ( left1 ,mode= 'a' , index= False , header= True )
except:
print('None')
try:
if os.path.isfile(right1):
groups.get_group(self.D[3]).to_csv(right1, mode='a', index=False, header=False)
else:
groups.get_group(self.D[3]).to_csv(right1, mode='a', index=False, header=True)
except:
print('None')
conn.commit()
def load_data_feature_selection():
energy = load_energy()
del energy['raw_energy']
weather = read_csv(
'/Users/kevin/PycharmProjects/TimeSeries/data/apartment2016.csv',
usecols=[
'temperature', 'icon', 'humidity', 'visibility', 'summary',
'apparentTemperature', 'pressure', 'windSpeed', 'cloudCover',
'time', 'windBearing', 'precipIntensity', 'dewPoint',
'precipProbability'
],
index_col=9,
squeeze=True,
date_parser=lambda x: datetime.fromtimestamp(int(x)) - timedelta(hours=
4),
parse_dates=['time'])
data = weather.join(pd.DataFrame(energy), how='outer').bfill()
data = data.dropna()
data = generate_dt_feature(data)
data['icon'] = data['icon'].astype('category')
data['summary'] = data['summary'].astype('category')
data['is_weekend'] = data['is_weekend'].astype('category')
cat_columns = data.select_dtypes(['category']).columns
data[cat_columns] = data[cat_columns].apply(lambda x: x.cat.codes)
train = data[TRAIN_START:TRAIN_END]
return train
def load_data(apt_fname, freq):
energy = None
# 1) read data
if basename(apt_fname).startswith('Apt'):
energy = read_csv(
apt_fname,
header=None,
names=['energy'],
index_col=0,
squeeze=True,
date_parser=lambda x: datetime.strptime(x, '%Y-%m-%d %H:%M:%S'))
energy = energy.resample(freq).mean()
elif basename(apt_fname).startswith('Mean_seed') \
or basename(apt_fname).startswith('SUM_'):
energy = pd.read_pickle(apt_fname)
energy = energy.rename('energy')
log_energy = np.log(energy + 1)
# s_series = s_series.diff()
df_energy = pd.DataFrame({'raw_energy': energy, 'energy': log_energy})
df_weather = read_csv(DATA_SET_DIR + 'apartment2016.csv',
usecols=[
'temperature', 'icon', 'humidity', 'visibility',
'summary', 'apparentTemperature', 'pressure',
'windSpeed', 'cloudCover', 'time', 'windBearing',
'precipIntensity', 'dewPoint',
'precipProbability'
],
index_col=9,
squeeze=True,
date_parser=lambda x: datetime.fromtimestamp(int(x))
- timedelta(hours=5),
parse_dates=['time'])
df_weather = df_weather.dropna()
df = df_energy.join(df_weather, how='inner').bfill()
df = df.dropna()
# 2) pre-processing
df = preprocessing(df)
return df
s = requests.get(
url=
'https://poloniex.com/public?command=returnChartData¤cyPair=USDT_ETC&end=9999999999&period=14400&start=1405699200'
)
# In[3]:
dfETC = pd.DataFrame.from_records(s.json())
# In[4]:
start = time.time() #works but first create a column wiht DateTime
dfETC['DateTime'] = pd.Series(np.random.randn(len(dfETC)), index=dfETC.index)
for i, row in dfETC.iterrows():
dfETC.loc[i, 'DateTime'] = datetime.fromtimestamp(row['date']).strftime(
"%m-%d-%Y %H:%M:%S"
) #instead of .loc you can also use at for future purposes
dfETC['DateTime'] = pd.to_datetime(dfETC['DateTime'],
format='%m-%d-%Y %H:%M:%S')
end = time.time()
print(end - start)
# In[5]:
#Using Roller to generate Moving Averages(Simple). Here we re multiplying by 6 cos each day has 6 recordings
#roller = dfETC['close'].rolling(10*6)
#dfETC['10DayMA'] = roller.mean()
#roller = dfETC['close'].rolling(30*6)
#dfETC['30DayMA'] = roller.mean()
def dateparse(time_in_secs):
return datetime.fromtimestamp(float(time_in_secs))
def parser(x):
dateout = []
for time in x:
dateout.append(datetime.fromtimestamp(int(time)))
return dateout
def parser(x):
y = datetime.fromtimestamp(float(x))
return (
y.strftime('%d %H:%M:%S')
) #datetime.fromtimestamp(float(x))#strptime(x, '%Y-%m-%d %H:%M:%S')
def parser(x):
return datetime.fromtimestamp(int(x)) - timedelta(hours=4)
