def query_concurrent(last_hour_date_time):
REDIS_PASSWORD = os.popen("grep REDIS_PASSWORD /home/xIDM-SSO-Cent8/sso/idp/config/config.php |cut -f4 -d\"'\"").read().replace('\n','')
client = InfluxDBClient('127.0.0.1', 8086, 'admin', REDIS_PASSWORD, 'xidmsso')
select_clause = 'select * from idpnode ORDER BY time DESC LIMIT 1'
result = client.query(select_clause)
for point in result.get_points():
time = point['time']
#print(time)
datetime = []
data = {}
select_clause = "select * from idpnode where time = '"+ time +"'"
result = client.query(select_clause)
for point in result.get_points():
scur=[]
#print(point['idp'])
result_idp = client.query("select * from idpnode where idp='"+ point['idp'] +"' and time > '"+last_hour_date_time+"'")
for point_idp in result_idp.get_points():
if point_idp['time'] not in datetime :
datetime.append(point_idp['time'])
scur.append(point_idp['scur'])
#print(point_idp)
data[point['idp']] = scur
data["datetime"] = datetime
result_json = json.dumps(data)
print("ok#%s" % result_json)
def parse_profiles(pfs):
"""Lat, Long, datetime, and temperature, pH, etc. at
shallowest depth (surface) for all profiles in pfs."""
Npfs = len(pfs)
lat = []
lon = []
datetime = []
temp = []
phosphate = []
pH = []
for ipf in range(Npfs):
nlevels = pfs[ipf].n_levels()
if nlevels > 0:
lat.append(pfs[ipf].latitude())
lon.append(pfs[ipf].longitude())
datetime.append(pfs[ipf].datetime())
temp.append(pfs[ipf].t()[0])
phosphate.append(pfs[ipf].phosphate()[0])
pH.append(pfs[ipf].pH()[0])
t = tab.Table()
t['lat'] = tab.Column(lat)
t['lon'] = tab.Column(lon)
t['datetime'] = tab.Column(datetime)
t['temp'] = tab.Column(temp)
t['phosphate'] = tab.Column(phosphate)
t['pH'] = tab.Column(pH)
return t
def read_drifter_file(filename):
# This needed to be redone completely
# time output needs to be in "datetime" form
dataReader = csv.reader(open(filename,'rb'))
verts=[]
for row in dataReader:
if row[0][0] !='%': #skip the command lines
verts.append(row)
id,lat_num,lon_num,yearday=[],[],[],[]
for vert in verts:
lat_num.append(vert[0].split()[8]) #latitude
lon_num.append(vert[0].split()[7]) #longtitude
yearday.append(vert[0].split()[6]) #yearday
id.append(vert[0].split()[0])
yeartime="20"+id[0][0:2] #get the year frome the id
lat_float=[float(i) for i in lat_num]
lon=[float(i) for i in lon_num]
lat=[]
for lati in lat_float:
lat.append(str(lati))
#delete "nan" form lat
lat_array=scipy.array(lat)#convert list to array
lat_nan_list=list(np.nonzero(lat_array=='nan'))#get location of the 'nan'
lat_nan_id=list(lat_nan_list[0])#convert dt_zero_list to list format
lat_nan_id.reverse()# reverse the lat_nan_id
for locationi in lat_nan_id:
del lat[locationi],lon[locationi],yearday[locationi]#del the lines which lat="nan"
yearday_number = [float(i) for i in yearday]
dti = list(np.diff(yearday_number)*24*60*60)#convert yearday_difference to seconds
dt1 = []
for dti in dti:
dt.append(str(dti))
lat=[float(i) for i in lat] #the lat is number for calculate and plot
jd=[float(i) for i in yearday] #the lon is number for calculate and plot
dt1=[float(i) for i in dt1]
dt_zero=scipy.array(dt)#convert list to array
dt_zero_list=list(np.nonzero(dt_zero==0.0))#get location of the 0.0
dt_zero_id=list(dt_zero_list[0])#convert dt_zero_list to list formate
zero=[]
for zero_id in dt_zero_id:
zero.append(dt[zero_id])#just get 0.0
dt1=[val for val in dt1 if val not in zero]#if dt==0.0,del
dtime=[]
for i in range(0,len(jd)):
jd_date=num2date(jd[i])
jd_year_date=jd_date.replace(tzinfo=None)
jd_year= jd_year_date.replace(year=int(yeartime))
dtime.append(jd_year)
return dtime,lat,lon
def drop_hours(df):
"""
Drop l'heure et les minutes des dates (cdate_train/test, datetime, cdate_member, cdate_restaurant) pour une question de simplicité
"""
cdate_train = []
cdate_test = []
datetime = []
cdate_member = []
cdate_restaurant = []
try:
for elt in df["cdate_train"]:
if type(elt) != float:
cdate_train.append(elt.split(" ")[0])
else:
cdate_train.append(np.nan)
except:
for elt in df["cdate_test"]:
if type(elt) != float:
cdate_test.append(elt.split(" ")[0])
else:
cdate_test.append(np.nan)
for elt in df["datetime"]:
if type(elt) != float:
datetime.append(elt.split(" ")[0])
else:
datetime.append(np.nan)
for elt in df["cdate_member"]:
if type(elt) != float:
cdate_member.append(elt.split(" ")[0])
else:
cdate_member.append(np.nan)
for elt in df["cdate_restaurant"]:
if type(elt) != float:
cdate_restaurant.append(elt.split(" ")[0])
else:
cdate_restaurant.append(np.nan)
try:
df["cdate_train"] = cdate_train
except:
df["cdate_test"] = cdate_test
df["datetime"] = datetime
df["cdate_member"] = cdate_member
df["cdate_restaurant"] = cdate_restaurant
return df
def read_lines():
with open("alert_log_file.csv") as file:
df = pd.read_csv(file, header=None)
df.set_axis([
"summons", "plate", "state", "date", "time", "violation",
"car make", "latitude", "longitude", "battery", "drone id."
],
axis=1,
inplace=True)
datetime = []
for date, time in zip(df.date, df.time):
datetime.append(date + " " + time)
df["datetime"] = datetime
pd.to_datetime(df.datetime)
#df.sort_values(by='datetime', inplace=True)
df.drop(axis=1,
labels=["summons", "date", "time", "violation", "battery"],
inplace=True)
return df
def findUserImgs(uid):
entries = Entry.objects(uid = unicode(uid)).as_pymongo()
uuids = []
datetime = []
diag = []
for entry in entries:
uuid = str(entry['uuid']) # find all filenames of images
# Check if filenmae exists and clear out in database if not?
uuids.append(str(uuid))
datetime.append(entry['datetime']) # times of upload
diag.append(entry['diag']) # diagnoses
b64imgs = []
for uuid in uuids:
# for some reason, won't load these
filename = app.config['UPLOAD_FOLDER2'] + uuid + '_thumb.png'
b64img = encodeB64(filename)
b64imgs.append(b64img) # specify filepath (change for computer)
return datetime, diag, b64imgs # received by client
def view_chat(request, user1_id, user2_id):
list_of_messages = []
list_of_messages = Messages.objects.filter(
(Q(suppliant=user1_id) & Q(accepter=user2_id))
| (Q(suppliant=user2_id) & Q(accepter=user1_id))).order_by('datetime')
messages = []
suppliant = []
accepter = []
datetime = []
for element in list_of_messages:
messages.append(element.text)
if (element.suppliant == request.user):
suppliant.append('me')
else:
suppliant.append(element.suppliant)
accepter.append(element.accepter)
datetime.append(element.datetime)
dictionary = {
'suppliant': suppliant,
'accepter': accepter,
'messages': messages,
'datetime': datetime
}
return dictionary
def main(host='localhost', port=8086):
"""Instantiate a connection to the InfluxDB."""
user = '******'
password = '******'
"""This is the database name if you want to specify it"""
dbname = 'taxi0228'
client = InfluxDBClient(host, port, user, password, dbname)
"""Drop previous database (if not needed comment out following two lines) and create new database"""
print("Drop database: " + dbname)
client.drop_database(dbname)
print("Create database: " + dbname)
client.create_database(dbname)
""" Collections to hold csv data"""
id = list()
taxiID = list()
longitude = list()
latitude = list()
speed = list()
angle = list()
datetime = list()
status = list()
extendedStatus = list()
reversed = list()
query = 'SELECT * FROM "' + dbname + '"."autogen"."' + dbname + '" GROUP BY id ORDER BY DESC LIMIT 100'
"""This is the filePath you have to specify"""
filePath = '../data/taxi0228.csv'
print("Writing data to InfluxDB.......")
with open(filePath, newline='') as f:
reader = csv.reader(f)
rownum = 0
for row in reader:
'''
# remove comment brackets if you just want to read in certain lines, e.g. the first 1000
if rownum >= 200:
break
'''
column = 0
for col in row:
if column == 0:
id.append(col)
if column == 1:
taxiID.append(col)
if column == 2:
longitude.append(col)
if column == 3:
latitude.append(col)
if column == 4:
speed.append(col)
if column == 5:
angle.append(col)
if column == 6:
datetime.append(col)
if column == 7:
status.append(col)
if column == 8:
extendedStatus.append(col)
if column == 9:
reversed.append(col)
column = column + 1
geohash1 = geohash.encode(float(latitude[rownum]), float(longitude[rownum]))
json_body = [
{
"measurement": "taxi0228",
"tags": {
"region": "shanghai",
"geohash": geohash1
},
"time": int(timeChanger(datetime[rownum])),
"fields": {
"id": int(id[rownum]),
"taxiID": taxiID[rownum],
"longitude": longitude[rownum],
"latitude": latitude[rownum],
"speed": int(speed[rownum]),
"angle": int(angle[rownum]),
"datetime": datetime[rownum],
"status": int(status[rownum]),
"extendedStatus": int(extendedStatus[rownum]),
"reversed": int(reversed[rownum])
}
}
]
client.write_points(json_body)
"""remove " # " if you want to print out every line"""
# print("Write points: {0}".format(json_body))
rownum = rownum + 1
""" print out last 100 lines to check if data is in InfluxDB"""
print("Data was successfully commited!\n\n")
print("Get the last 100 rows in database: \n")
print("Querying data: " + query)
result = client.query(query)
for row in result:
for col in row:
print(col)
def calc_similarity(self, session_items, sessions, dwelling_times,
timestamp):
'''
Calculates the configured similarity for the items in session_items and each session in sessions.
Parameters
--------
session_items: set of item ids
sessions: list of session ids
Returns
--------
out : list of tuple (session_id,similarity)
'''
pos_map = {}
length = len(session_items)
count = 1
for item in session_items:
if self.weighting is not None:
pos_map[item] = getattr(self, self.weighting)(count, length)
count += 1
else:
pos_map[item] = 1
if self.dwelling_time:
dt = dwelling_times.copy()
dt.append(0)
dt = pd.Series(dt, index=session_items)
dt = dt / dt.max()
# dt[session_items[-1]] = dt.mean() if len(session_items) > 1 else 1
dt[session_items[-1]] = 1
# print(dt)
for i in range(len(dt)):
pos_map[session_items[i]] *= dt.iloc[i]
# print(pos_map)
if self.idf_weighting_session:
max = -1
for item in session_items:
pos_map[item] = self.idf[item] if item in self.idf else 0
# if pos_map[item] > max:
# max = pos_map[item]
# for item in session_items:
# pos_map[item] = pos_map[item] / max
# print 'nb of sessions to test ', len(sessionsToTest), ' metric: ', self.metric
items = set(session_items)
neighbors = []
cnt = 0
for session in sessions:
cnt = cnt + 1
# get items of the session, look up the cache first
n_items = self.items_for_session(session)
sts = self.session_time[session]
# dot product
similarity = self.vec(items, n_items, pos_map)
if similarity > 0:
if self.weighting_time:
diff = timestamp - sts
days = round(diff / 60 / 60 / 24)
decay = pow(7 / 8, days)
similarity *= decay
# print("days:",days," => ",decay)
neighbors.append((session, similarity))
return neighbors
'class': 'story-title'
}).get_text().strip()
except:
titlelink = "None"
try:
datetimelink = link.find("span", attrs={
'class': 'p1_home-date'
}).get_text().strip()
except:
datetimelink = "None"
try:
imglink = link.find("img", attrs={'img-fill divider'})
img = imglink.get("src")
except:
img = "None"
datetime.append(datetimelink)
pagelinks.append(linker)
title.append(titlelink)
imglinks.append(img)
url = linker
try:
page = requests.get(url)
except MissingSchema:
continue
soup = BeautifulSoup(page.text, 'html.parser')
try:
authordetail = soup.find("div", attrs={
'class': 'post-author'
}).find("a").get_text().strip()
except:
prediction_num = 0
list_pred_num = []
datetime = []
timezone = []
Temp_Hi = []
timezone = []
Temp_Low = []
Solar = []
Cloud = []
Rainfall = []
#Add json data to the list
for num_forecasts in json_data['daily']:
weather_solar['prediction_num'] = prediction_num
list_pred_num.append(prediction_num)
datetime.append(json_data['daily'][prediction_num]['dt'])
Temp_Hi.append(json_data['daily'][prediction_num]['temp']['max'])
Temp_Low.append(json_data['daily'][prediction_num]['temp']['min'])
Cloud.append(json_data['daily'][prediction_num]['clouds'])
Solar.append(json_data['daily'][prediction_num]['uvi'])
#Rainfall.append(json_data['daily'][prediction_num]['rain'])
prediction_num += 1
#Write list to DF
weather_solar['prediction_num'] = list_pred_num
weather_solar['datetime'] = datetime
weather_solar['Temp_Hi'] = Temp_Hi
weather_solar['Temp_Low'] = Temp_Low
weather_solar['Cloud'] = Cloud
weather_solar['Solar'] = Solar
#weather_solar['Rainfall'] = Rainfall
attrs={'class': 'ArticleListing__author'})
author = authorlink.get_text()
except:
author = 'None'
authorlinks.append(author)
imglink = link.find(
"img", attrs={'class': 'ArticleListing__image wp-post-image'})
img = imglink.get('src')
imglinks.append(img)
try:
datetimelinks = link.find("time",
attrs={'class': 'ArticleListing__time'})
dateandtime = datetimelinks.get_text()
except:
dateandtime = 'None'
datetime.append(dateandtime)
url = pagelinker
page = requests.get(url)
soup = BeautifulSoup(page.text, 'html.parser')
paragraphcontent = soup.find("div", attrs={'class': 'article-content'})
maincontent = soup.find_all("p")[2:9]
for p in maincontent:
paragraphtext = p.get_text().strip()
PageContent.append(paragraphtext)
articletext.append(PageContent)
articlecontent = [''.join(article) for article in articletext]
content.append(articlecontent)
PageContent.clear()
articletext.clear()
def average(self, period):
"""
Average the residuals over a given period.
Parameters
----------
period : str
Choose 'daily' or 'monthly' for the averaging period.
Adds a new object (`daily' or `monthly') with `speed', `direction',
`u_res' and `v_res' arrays to match the main Residual object.
"""
# Get the necessary bits of time.
daily_inc = int(24.0 / self._inc) # get model steps per day
monthly_inc = daily_inc * 30 # get model steps per 30 days (~month)
nt = self.time.datetime.shape[0]
dnt = self.time.datetime[::daily_inc].shape[0]
# Monthly increment might mean we end up trying to skip over more days than we have in the input. So,
# set the number of montly times to be 1 (only one month of data) or the number of times when subsampled at
# the monthly increment, whichever is larger.
mnt = np.max((self.time.datetime[::monthly_inc].shape[0], 1))
nx = self.u_diff.shape[-1]
# Prepare the arrays if we're doing averaging
if period == 'daily':
self.daily = type('daily', (object, ), {})()
if self._noisy:
print('Compute daily residuals')
for var in ('u_diff', 'v_diff'):
datetime = [
] # has to be a list so we can insert a datetime object.
daily = np.empty((dnt, nx))
# This could be done with a neat reshaping, but I can't be bothered to figure it out, so we'll just
# do it the old-fashioned way.
for ti, t in enumerate(
np.arange(0, nt, daily_inc).astype(int)):
daily[ti, :] = np.median(getattr(self,
var)[t:t + daily_inc, :],
axis=0)
datetime.append(self.time.datetime[np.min((t, nt - 1))])
if 'instantaneous' in self._max_speed:
daily = self._clip(daily, self._max_speed['instantaneous'])
setattr(self.daily, var, daily)
setattr(self.daily, 'datetime', np.asarray(datetime))
# Now create the speed and direction arrays.
setattr(
self.daily, 'speed',
np.sqrt(
getattr(self.daily, 'u_diff')**2 +
getattr(self.daily, 'v_diff')**2))
setattr(
self.daily, 'direction',
np.rad2deg(
np.arctan2(getattr(self.daily, 'u_diff'),
getattr(self.daily, 'v_diff'))))
# Make the components after we've clipped so our plots look nice.
self.daily.u_res = np.sin(np.deg2rad(
self.daily.direction)) * self.daily.speed
self.daily.v_res = np.cos(np.deg2rad(
self.daily.direction)) * self.daily.speed
elif period == 'monthly':
self.monthly = type('monthly', (object, ), {})()
if self._noisy:
print('Compute monthly residuals')
for var in ('u_diff', 'v_diff'):
datetime = [
] # has to be a list so we can insert a datetime object.
monthly = np.empty((mnt, nx))
# This could be done with a neat reshaping, but I can't be bothered to figure it out, so we'll just
# do it the old-fashioned way.
for ti, t in enumerate(
np.arange(0, nt / 60 / 30, monthly_inc).astype(int)):
monthly[ti, :] = np.median(getattr(
self, var)[t:t + monthly_inc, :],
axis=0)
datetime.append(self.time.datetime[np.min(
((t + monthly_inc) // 2, nt - 1))]) # mid-point
setattr(self.monthly, var, monthly)
setattr(self.monthly, 'datetime', np.asarray(datetime))
# Now create the speed and direction arrays.
setattr(
self.monthly, 'speed',
np.sqrt(
getattr(self.monthly, 'u_diff')**2 +
getattr(self.monthly, 'v_diff')**2))
setattr(
self.monthly, 'direction',
np.rad2deg(
np.arctan2(getattr(self.monthly, 'u_diff'),
getattr(self.monthly, 'v_diff'))))
if 'monthly' in self._max_speed:
if self._noisy:
print('Capping monthly residuals to {} m/s'.format(
self._max_speed['monthly']))
self.monthly.speed = self._clip(self.monthly.speed,
self._max_speed['monthly'])
# Make the components after we've clipped so our plots look nice.
self.monthly.u_res = np.sin(np.deg2rad(
self.monthly.direction)) * self.monthly.speed
self.monthly.v_res = np.cos(np.deg2rad(
self.monthly.direction)) * self.monthly.speed
if period == 'monthly':
# We need to add a pseudo-time dimension to the monthly data so we can still use the plot_var function.
if np.ndim(self.monthly.speed) == 1:
self.monthly.speed = self.monthly.speed[np.newaxis, :]
self.monthly.direction = self.monthly.direction[np.newaxis, :]
self.monthly.u_res = self.monthly.u_res[np.newaxis, :]
self.monthly.v_res = self.monthly.v_res[np.newaxis, :]
print dir(fake) # gives you all the attributes of fake # for example: for _ in range(0,10): print fake.military_state() for _ in range(0,10): print fake.name() # creating lots of fake data email = [] for _ in range(0,10): email.append(fake.email()) datetime = [] for _ in range(0,10): datetime.append(fake.date_time()) address = [] for _ in range(0,10): address.append(fake.address()) city = [] for _ in range(0,10): city.append(fake.city()) state = [] for _ in range(0,10): state.append(fake.state()) mylist = [list(a) for a in zip(email, datetime, address, city, state)]
'Title', 'Genres'
])
DF['Title'].unique()
# In[15]:
from datetime import datetime
import time
# In[16]:
timestamp = DF['Timestamp'].values
datetime = []
for j in timestamp:
z = time.strftime("%Y-%m-%d", time.localtime(int(j)))
datetime.append(z)
DF['Date'] = datetime
# In[17]:
DF[:5]
# In[18]:
DF.Genres = DF.Genres.str.replace("'", "")
DF[:5]
# In[19]:
genres = pd.DataFrame(DF.Genres.str.split('|').tolist()).stack().unique()
genres = pd.DataFrame(genres, columns=['Genre'])
trows = trows[0::2]
def clean_tweet(tweet):
return ' '.join(
re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)", " ",
tweet).split())
words = []
tweets = []
datetime = []
for elem in trows:
dt = elem[3]
datetime.append(dt)
elem = str(elem)
elemclean = clean_tweet(elem)
tweets.append(elemclean)
elemtb = tb(elemclean)
elemsent = elemtb.sentiment
sentiment.append(elemsent)
elempol = elemsent.polarity
polarity.append(elempol)
date = []
days = []
for dtime in datetime:
date.append(dtime[0:10])
days.append(dtime[8:10])
