def master():
cur_path = os.getcwd()
for fol in range(0, 182):
folder = get_folder(fol)
folder += "/Trajectory/"
path = "/home/zishan/development/minor/data/"
path += folder
os.chdir(path)
files = os.listdir()
total_content = ""
print("Number of files: ", len(files))
for n in range(1, len(files)+1):
with open(str(n)+".plt", 'r') as f:
for i in range(0, 6):
f.readline()
for line in f:
line = line.split(',')
line = line[:2] + line[5:]
content = ','.join(line)
# content += '\n'
print(content)
total_content += content
folder = get_folder(fol)
path = "/home/zishan/development/minor/data/"
path += folder
os.chdir(path)
with open("master.csv", 'w') as f:
f.write(total_content)
# print(total_content)
print("Total records: ", len(total_content))
os.chdir(cur_path)
def create_master():
cur_path = os.getcwd()
for fol in range(0, 182):
folder = get_folder(fol)
folder += "/Trajectory/"
path = "/home/evamy/SignificantLocations/Data/"
path += folder
os.chdir(path)
files = os.listdir('.')
total_content = ""
print("Number of files in",folder,":", len(files))
for n in files:
with open(str(n), 'r') as f:
for i in range(0, 6):
f.readline()
for line in f:
line = line.split(',')
line = line[:2] + line[5:]
content = ','.join(line)
total_content += content
folder = get_folder(fol)
path = "/home/evamy/SignificantLocations/Data/"
path += folder
os.chdir(path)
with open("master.csv", 'w') as f:
f.write(total_content)
print("Total records written :", len(total_content))
os.chdir(cur_path)
def master():
cur_path = os.getcwd()
for fol in range(0, 182):
folder = get_folder(fol)
folder += "/Trajectory/"
path = "/home/zishan/development/minor/data/"
path += folder
os.chdir(path)
files = os.listdir()
total_content = ""
print("Number of files: ", len(files))
for n in range(1, len(files) + 1):
with open(str(n) + ".plt", 'r') as f:
for i in range(0, 6):
f.readline()
for line in f:
line = line.split(',')
line = line[:2] + line[5:]
content = ','.join(line)
# content += '\n'
print(content)
total_content += content
folder = get_folder(fol)
path = "/home/zishan/development/minor/data/"
path += folder
os.chdir(path)
with open("master.csv", 'w') as f:
f.write(total_content)
# print(total_content)
print("Total records: ", len(total_content))
os.chdir(cur_path)
def create_master():
cur_path = os.getcwd()
for fol in range(0, 182):
folder = get_folder(fol)
folder += "/Trajectory/"
path = "/home/evamy/SignificantLocations/Data/"
path += folder
os.chdir(path)
files = os.listdir('.')
total_content = ""
print("Number of files in", folder, ":", len(files))
for n in files:
with open(str(n), 'r') as f:
for i in range(0, 6):
f.readline()
for line in f:
line = line.split(',')
line = line[:2] + line[5:]
content = ','.join(line)
total_content += content
folder = get_folder(fol)
path = "/home/evamy/SignificantLocations/Data/"
path += folder
os.chdir(path)
with open("master.csv", 'w') as f:
f.write(total_content)
print("Total records written :", len(total_content))
os.chdir(cur_path)
def master():
cur_path = os.getcwd()
path = cur_path[:-7] + "data/"
os.chdir(path)
conn = sqlite3.connect("data.db")
c = conn.cursor()
for fol in range(0, 182):
folder = get_folder(fol)
path = "/home/zishan/development/minor/data/"
path += folder
os.chdir(path)
table_name = "master" + folder
query = (
"CREATE TABLE IF NOT EXISTS "
+ table_name
+ """ (id integer primary key autoincrement not null,
latitude real not null, longitude real not null, dated date not null, timed time not null)"""
)
c.execute(query)
with open("master.csv", "r") as f:
for line in f:
line = line.split(",")
c.execute("INSERT INTO " + table_name + "(latitude, longitude, dated, timed) values (?, ?, ?, ?)", line)
print(line)
conn.commit()
conn.close()
os.chdir(cur_path)
def master():
cur_path = os.getcwd()
path = cur_path[:-7] + "Data/"
os.chdir(path)
conn = sqlite3.connect('data.db')
c = conn.cursor()
for fol in range(153, 182):
folder = get_folder(fol)
path = "/home/evamy/SignificantLocations/Data/"
path += folder
os.chdir(path)
table_name = "master" + folder
c.execute("DROP TABLE IF EXISTS " + table_name)
query = "CREATE TABLE IF NOT EXISTS " + table_name + ''' (id integer primary key\
autoincrement not null,latitude real not null, longitude real not null,\
dated date not null, timed time not null)'''
c.execute(query)
with open("master.csv", "r") as f:
for line in f:
line = line.split(",")
# print line, folder
c.execute(
"INSERT INTO " + table_name +
"(latitude, longitude, dated, timed)\
values (?, ?, ?, ?)", line)
# print(line)
print "in folder :", folder
conn.commit()
conn.close()
os.chdir(cur_path)
def master():
cur_path = os.getcwd()
path = cur_path[:-7] + "data/"
os.chdir(path)
conn = sqlite3.connect('data.db')
c = conn.cursor()
for fol in range(0, 182):
folder = get_folder(fol)
path = "/home/zishan/development/minor/data/"
path += folder
os.chdir(path)
table_name = "master" + folder
query = "CREATE TABLE IF NOT EXISTS " + table_name + ''' (id integer primary key autoincrement not null,
latitude real not null, longitude real not null, dated date not null, timed time not null)'''
c.execute(query)
with open("master.csv", "r") as f:
for line in f:
line = line.split(",")
c.execute(
"INSERT INTO " + table_name +
"(latitude, longitude, dated, timed) values (?, ?, ?, ?)",
line)
print(line)
conn.commit()
conn.close()
os.chdir(cur_path)
def master():
cur_path = os.getcwd()
path = cur_path[:-7] + "Data/"
os.chdir(path)
conn = sqlite3.connect('data.db')
c = conn.cursor()
for fol in range(153, 182):
folder = get_folder(fol)
path = "/home/evamy/SignificantLocations/Data/"
path += folder
os.chdir(path)
table_name = "master" + folder
c.execute("DROP TABLE IF EXISTS "+table_name)
query = "CREATE TABLE IF NOT EXISTS " + table_name + ''' (id integer primary key\
autoincrement not null,latitude real not null, longitude real not null,\
dated date not null, timed time not null)'''
c.execute(query)
with open("master.csv", "r") as f:
for line in f:
line = line.split(",")
# print line, folder
c.execute("INSERT INTO " + table_name + "(latitude, longitude, dated, timed)\
values (?, ?, ?, ?)", line)
# print(line)
print "in folder :",folder
conn.commit()
conn.close()
os.chdir(cur_path)
def get_places():
cur_path = os.getcwd()
path = cur_path[:-7] + "Data/"
os.chdir(path)
conn = sqlite3.connect("data.db")
c = conn.cursor()
places = []
query = "SELECT * FROM places"
for line in c.execute(query):
line = list(line[1:5])
if len(places) == 0:
line.append(0)
else:
line.append(get_distance(line, places[0]))
places.append(line)
for u in range(0, 182):
user = get_folder(u)
query = "SELECT * FROM master" + user
lines = []
now = time.time()
for line in c.execute(query):
line = list(line[1:])
if len(places) == 0:
line.append(0)
else:
line.append(get_distance(line, places[0]))
lines.append(line)
for line in lines:
if line not in places:
if not find_place(places, line):
if line[4] == 0 and len(places) != 0:
line[4] = get_distance(line, places[0])
places.append(line)
# print(line)
if time.time() - now > 60:
print("Processing user: "******"Line: ", line)
now = time.time()
print("User: "******"Num Places: ", len(places))
print("Num Lines", len(lines))
places.sort(key=lambda place: place[4])
query = "DELETE FROM places"
c.execute(query)
query = "DELETE FROM sqlite_sequence WHERE NAME = 'places'"
c.execute(query)
for place in places:
c.execute(
"INSERT INTO places(latitude, longitude, dated, timed) VALUES \
(?, ?, ?, ?)", place[0:4])
print 'Entered', user
conn.commit()
conn.close()
os.chdir(cur_path)
def get_places():
cur_path = os.getcwd()
path = cur_path[:-7] + "Data/"
os.chdir(path)
conn = sqlite3.connect("data.db")
c = conn.cursor()
places = []
query = "SELECT * FROM places"
for line in c.execute(query):
line = list(line[1:5])
if len(places) == 0:
line.append(0)
else:
line.append(get_distance(line, places[0]))
places.append(line)
for u in range(0, 182):
user = get_folder(u)
query = "SELECT * FROM master" + user
lines = []
now = time.time()
for line in c.execute(query):
line = list(line[1:])
if len(places) == 0:
line.append(0)
else:
line.append(get_distance(line, places[0]))
lines.append(line)
for line in lines:
if line not in places:
if not find_place(places, line):
if line[4] == 0 and len(places) != 0:
line[4] = get_distance(line, places[0])
places.append(line)
# print(line)
if time.time() - now > 60:
print("Processing user: "******"Line: ", line)
now = time.time()
print("User: "******"Num Places: ", len(places))
print("Num Lines", len(lines))
places.sort(key=lambda place: place[4])
query = "DELETE FROM places"
c.execute(query)
query = "DELETE FROM sqlite_sequence WHERE NAME = 'places'"
c.execute(query)
for place in places:
c.execute("INSERT INTO places(latitude, longitude, dated, timed) VALUES \
(?, ?, ?, ?)", place[0:4])
print 'Entered',user
conn.commit()
conn.close()
os.chdir(cur_path)
def rename():
cur_path = os.getcwd()
for f in range(0, 182):
folder = get_folder(f)
folder += "/Trajectory/"
path = "/home/zishan/development/minor/data/"
path += folder
os.chdir(path)
files = os.listdir()
files.sort()
j = 1
for i in files:
os.rename(i, str(j) + ".plt")
j += 1
print("File rename complete in folder {0}. Number of files renamed: {1}".format(folder, len(files)))
os.chdir(cur_path)
def rename():
cur_path = os.getcwd()
for f in range(0, 182):
folder = get_folder(f)
folder += "/Trajectory/"
path = "/home/evamy/SignificantLocations/Data/"
path += folder
os.chdir(path)
files = os.listdir('.')
files.sort()
j = 1
for i in files:
os.rename(i, str(j)+".plt")
j += 1
print("File rename complete in folder {0}. Number of files renamed: {1}".format(folder, len(files)))
os.chdir(cur_path)
def rename():
cur_path = os.getcwd()
for f in range(0, 182):
folder = get_folder(f)
folder += "/Trajectory/"
path = "/home/evamy/SignificantLocations/Data/"
path += folder
os.chdir(path)
files = os.listdir('.')
files.sort()
j = 1
for i in files:
os.rename(i, str(j) + ".plt")
j += 1
print(
"File rename complete in folder {0}. Number of files renamed: {1}".
format(folder, len(files)))
os.chdir(cur_path)
def rename():
cur_path = os.getcwd()
for f in range(0, 182):
folder = get_folder(f)
folder += "/Trajectory/"
path = "/home/zishan/development/minor/data/"
path += folder
os.chdir(path)
files = os.listdir()
files.sort()
j = 1
for i in files:
os.rename(i, str(j) + ".plt")
j += 1
print(
"File rename complete in folder {0}. Number of files renamed: {1}".
format(folder, len(files)))
os.chdir(cur_path)
def places_using_time():
cur_path = change_path_to_data()
conn = sqlite3.connect('data.db')
c = conn.cursor()
initial = []
final = []
added_to_final = True
added_to_initial = False
for u in range(0, 1):
user = get_folder(u)
query = "SELECT * FROM master" + user
last = []
for line in c.execute(query):
line = list(line)
line[4] = line[4][:-1]
# print(line)
if added_to_initial:
if get_minutes(line[4], last[4]) > 10:
# print("adding to final: ", line)
final.append(line)
added_to_final = True
added_to_initial = False
continue
if added_to_final:
# print("adding to initial: ", line)
initial.append(line)
added_to_initial = True
added_to_final = False
last = line
if added_to_initial:
final.append(last)
# print(len(initial))
# print(len(final))
for i in range(0, len(initial) - 1):
values = [initial[i][0], final[i][0]]
query = "INSERT INTO finit VALUES({},{})".format(
initial[i][0], final[i][0])
c.execute(query)
conn.commit()
conn.close()
back_to_path(cur_path)
def weights():
print("Assigning weights to places. Process started at: " + time.ctime())
start_time = time.time()
cur_path = os.getcwd()
path = cur_path[:-7] + "data/"
os.chdir(path)
conn = sqlite3.connect("data.db")
c = conn.cursor()
query = "SELECT * FROM places"
places = []
t_places = []
for place in c.execute(query):
place = list(place)
places.append(place)
t_places.append(place[1:5])
for u in range(0, 120):
user = get_folder(u)
query = "SELECT * FROM master" + user
for location in c.execute(query):
location = list(location[1:])
if location in t_places:
index = t_places.index(location)
places[index][5] += 1
print("Processed user: "******"Processed all users. Updating the database...")
for place in places:
if place[5] != 0:
query = "UPDATE places SET weight = " + str(
place[5]) + " WHERE id = " + str(place[0])
c.execute(query)
conn.commit()
conn.close()
os.chdir(cur_path)
print("Database updated.")
end_time = time.time()
total_time = (end_time - start_time) / 60
print("Process completed at " + time.ctime())
print("Total running time: " + str(total_time) + " minutes")
def places_using_time():
cur_path = change_path_to_data()
conn = sqlite3.connect('data.db')
c = conn.cursor()
initial = []
final = []
added_to_final = True
added_to_initial = False
for u in range(0, 1):
user = get_folder(u)
query = "SELECT * FROM master" + user
last = []
for line in c.execute(query):
line = list(line)
line[4] = line[4][:-1]
# print(line)
if added_to_initial:
if get_minutes(line[4], last[4]) > 10:
# print("adding to final: ", line)
final.append(line)
added_to_final = True
added_to_initial = False
continue
if added_to_final:
# print("adding to initial: ", line)
initial.append(line)
added_to_initial = True
added_to_final = False
last = line
if added_to_initial:
final.append(last)
# print(len(initial))
# print(len(final))
for i in range(0, len(initial) - 1):
values = [initial[i][0], final[i][0]]
query = "INSERT INTO finit VALUES({},{})".format(initial[i][0], final[i][0])
c.execute(query)
conn.commit()
conn.close()
back_to_path(cur_path)
def weights():
print("Assigning weights to places. Process started at: " + time.ctime())
start_time = time.time()
cur_path = os.getcwd()
path = cur_path[:-7] + "data/"
os.chdir(path)
conn = sqlite3.connect("data.db")
c = conn.cursor()
query = "SELECT * FROM places"
places = []
t_places = []
for place in c.execute(query):
place = list(place)
places.append(place)
t_places.append(place[1:5])
for u in range(0, 120):
user = get_folder(u)
query = "SELECT * FROM master" + user
for location in c.execute(query):
location = list(location[1:])
if location in t_places:
index = t_places.index(location)
places[index][5] += 1
print("Processed user: "******"Processed all users. Updating the database...")
for place in places:
if place[5] != 0:
query = "UPDATE places SET weight = " + str(place[5]) + " WHERE id = " + str(place[0])
c.execute(query)
conn.commit()
conn.close()
os.chdir(cur_path)
print("Database updated.")
end_time = time.time()
total_time = (end_time - start_time) / 60
print("Process completed at " + time.ctime())
print("Total running time: " + str(total_time) + " minutes")
def get_places_by_date_time():
cur_path = change_path_to_data()
conn = sqlite3.connect('data.db')
c = conn.cursor()
new_places = []
for u in range(0, 10):
user = get_folder(u)
# Get all dates from the database
query = "SELECT DISTINCT dated FROM master" + user # 6367 for 000, 256 for 001
dates = []
for line in c.execute(query):
dates.append(line[0])
places = []
# places.append([172042, 39.955832, 116.329224, '2009-07-05', '04:44:31'])
# For each date in the database, get all the locations and extract places based on 10 minute difference
for d in dates:
query = "SELECT * FROM master" + user + " WHERE dated = '" + str(d) + "'"
# print(query)
prev = 0
for line in c.execute(query):
line = list(line)
line[4] = line[4][:-1]
if prev == 0:
# print(line, prev)
places.append(line)
prev += 1
else:
minutes = get_minutes(places[prev-1][4], line[4])
# print(minutes)
if minutes > 15:
distance = get_distance(places[prev-1][1:3], line[1:3])
if distance > 20:
if line not in places:
# line.append(1)
# print(line, minutes, distance)
places.append(line)
prev += 1
# Add entries to new_places
initial = len(new_places)
if len(new_places) == 0:
for p in places:
p.append(1)
new_places.append(p)
else:
for p in places:
index = find_place_linear(new_places, p)
if index == -1:
p.append(1)
new_places.append(p)
else:
new_places[index][5] += 1
print("User: "******"Number of places: ", len(places))
print("Number of places added: ", len(new_places) - initial)
# print(new_places)
print("Total new places: ", len(new_places))
# Remove entries from new_places table
query = "DELETE FROM new_places"
c.execute(query)
query = "DELETE FROM sqlite_sequence WHERE NAME = 'new_places'"
c.execute(query)
for place in new_places:
c.execute("INSERT INTO new_places(latitude, longitude, dated, timed, weight) VALUES (?, ?, ?, ?, ?)", place[1:])
# print(place[0:4])
conn.commit()
conn.close()
back_to_path(cur_path)
def get_places_by_date_time():
cur_path = change_path_to_data()
conn = sqlite3.connect('data.db')
c = conn.cursor()
new_places = []
for u in range(0, 10):
user = get_folder(u)
# Get all dates from the database
query = "SELECT DISTINCT dated FROM master" + user # 6367 for 000, 256 for 001
dates = []
for line in c.execute(query):
dates.append(line[0])
places = []
# places.append([172042, 39.955832, 116.329224, '2009-07-05', '04:44:31'])
# For each date in the database, get all the locations and extract places based on 10 minute difference
for d in dates:
query = "SELECT * FROM master" + user + " WHERE dated = '" + str(
d) + "'"
# print(query)
prev = 0
for line in c.execute(query):
line = list(line)
line[4] = line[4][:-1]
if prev == 0:
# print(line, prev)
places.append(line)
prev += 1
else:
minutes = get_minutes(places[prev - 1][4], line[4])
# print(minutes)
if minutes > 15:
distance = get_distance(places[prev - 1][1:3],
line[1:3])
if distance > 20:
if line not in places:
# line.append(1)
# print(line, minutes, distance)
places.append(line)
prev += 1
# Add entries to new_places
initial = len(new_places)
if len(new_places) == 0:
for p in places:
p.append(1)
new_places.append(p)
else:
for p in places:
index = find_place_linear(new_places, p)
if index == -1:
p.append(1)
new_places.append(p)
else:
new_places[index][5] += 1
print("User: "******"Number of places: ", len(places))
print("Number of places added: ", len(new_places) - initial)
# print(new_places)
print("Total new places: ", len(new_places))
# Remove entries from new_places table
query = "DELETE FROM new_places"
c.execute(query)
query = "DELETE FROM sqlite_sequence WHERE NAME = 'new_places'"
c.execute(query)
for place in new_places:
c.execute(
"INSERT INTO new_places(latitude, longitude, dated, timed, weight) VALUES (?, ?, ?, ?, ?)",
place[1:])
# print(place[0:4])
conn.commit()
conn.close()
back_to_path(cur_path)
