def plotMap(host, port, db_name):
# Get maps coordinate
dao = Dao(host, port)
dao.connect(db_name)
c_list = list(dao.query('globals', ''))
c_dict = dict(c_list[0])
dao.close()
# Select the map
m = Basemap(projection='mill',
llcrnrlat=int(c_dict['lat_min']),
urcrnrlat=int(c_dict['lat_max'] + 1),
llcrnrlon=int(c_dict['lon_min']),
urcrnrlon=int(c_dict['lon_max'] + 1),
resolution='i')
m.drawcoastlines()
m.drawcountries()
m.drawstates()
m.fillcontinents(color='#04BAE3', lake_color='#FFFFFF')
m.drawmapboundary(fill_color='#FFFFFF')
# Get maps coordinate
dao = Dao(host, port)
dao.connect(db_name)
it = dao.query('clicks', '')
counter = 0
hasNext = True
while hasNext and counter < 20000:
#x,y = m(lon,lat) #m.plot(x,y, 'ro')
try:
url = next(it, None)
except StopIteration:
hasNext = False
if hasNext:
loc = url['loc']
lat, lon = float(loc[0]), float(loc[1])
x, y = m(lon, lat)
m.plot(x, y, 'ro')
counter += 1
if counter % 1000 == 0:
print(counter)
dao.close()
plt.title("Geo Plotting")
plt.show()
def run(db_host_name, db_port, directory, db_name, collection_name,
collection_name_dbstat, n_threads):
print('GEO INDEXING :::::::::::::::::::::::::::')
start_time = time.time()
# get files names
file_list = getFilesList(directory)
file_list.sort()
print('dataset files : ')
for f in file_list:
print(str(f))
print('')
print('I\'m working...')
if len(file_list) < n_threads:
n_threads = len(file_list)
for file_name in file_list:
if os.path.isfile(file_name):
t = GeoIndexingThread(file_name, db_host_name, int(db_port),
db_name, collection_name,
collection_name_dbstat)
t.start()
if threading.active_count() > n_threads:
while threading.active_count() > n_threads:
time.sleep(0.1)
while threading.active_count() > 1:
time.sleep(0.1)
# if there are more than row in 'globals' merge them
dao = Dao(db_host_name, int(db_port))
dao.connect(db_name)
generals_list = list(dao.query(collection_name_dbstat, ''))
print(generals_list)
db_stat = None
if len(generals_list) > 1:
for e in generals_list:
if db_stat == None:
db_stat = Db_stat(e['lat_max'], e['lon_max'], e['lat_min'],
e['lon_min'])
else:
db_stat.merge(e)
dao.removeAll(collection_name_dbstat)
dao.addOne(collection_name_dbstat, db_stat.__dict__)
dao.close()
end_time = time.time()
logs = {}
logs['time'] = end_time - start_time
return logs
def getBoundaries(host, port, db_name):
# Get maps coordinate
dao = Dao(host, port)
dao.connect(db_name)
c_list = list(dao.query('globals', ''))
c_dict = dict(c_list[0])
dao.close()
return (float(c_dict['lat_max']),float(c_dict['lon_max'])),(float(c_dict['lat_min']),float(c_dict['lon_min']))
def getPlotsMap(host, port, db_name, collection):
# Get maps coordinate
dao = Dao(host, port)
dao.connect(db_name)
c_list = list(dao.query(collection, ''))
c_dict = dict(c_list[0])
dao.close()
# Select the map
m = Basemap(projection='mill',
llcrnrlat=int(c_dict['lat_min']),
urcrnrlat=int(c_dict['lat_max'] + 1),
llcrnrlon=int(c_dict['lon_min']),
urcrnrlon=int(c_dict['lon_max'] + 1),
resolution='i')
m.drawcoastlines()
m.drawcountries()
m.drawstates()
m.fillcontinents(color='#04BAE3', lake_color='#FFFFFF')
m.drawmapboundary(fill_color='#FFFFFF')
return m
def persist(filename, host_name, port, db_name, collection_name,
collection_name_dbstat):
# Definition of max lat and lon
lat_max = None
lat_min = None
lon_max = None
lon_min = None
# Used like a hinge
first_url = None
# Check if the file is a txt
if filename.endswith('.txt'):
with open(filename, 'r') as f:
text = f.read()
# Open db
dao = Dao(host_name, port)
dao.connect(db_name)
rows = text.split('\n')
counter = 0
size = len(rows)
#print('# Number of rows in \"'+filename+'\": ' + str(size)+'\n')
# Get the maximum coordinates from the db
stat = list(dao.query(collection_name_dbstat, ''))
l_db = None
if len(stat) > 0:
stat_dict = dict(stat[0])
lat_max = stat_dict['lat_max']
lat_min = stat_dict['lat_min']
lon_max = stat_dict['lon_max']
lon_min = stat_dict['lon_min']
l_db = Db_stat(lat_max, lon_max, lat_min, lon_min)
for row in rows:
url = parse_row(row)
if url != None:
loc = url.getCoordinates()
if l_db == None:
l_db = Db_stat(loc[0], loc[1], loc[0], loc[1])
l_db.setModified()
else:
l_db.updateLat(loc[0])
l_db.updateLon(loc[1])
res = dao.addOne(collection_name, url.__dict__)
'''counter = counter + 1
if counter % (size // 15) == 0:
print(str(100 // (size / counter)) + ' % Done of \"'+ filename+'\"')'''
# add lat_max, lon_max, lat_min and lon_min to db if are better
stat = list(dao.query(collection_name_dbstat, ''))
if len(stat) > 0:
stat_dict = dict(stat[0])
if l_db.isModify() == True:
doc_id = stat_dict["_id"]
if l_db.lat_max > float(stat_dict['lat_max']):
dao.updateOne(collection_name_dbstat, {'_id': doc_id},
{'$set': {
"lat_max": l_db.lat_max
}})
if l_db.lat_min < float(stat_dict['lat_min']):
dao.updateOne(collection_name_dbstat, {'_id': doc_id},
{'$set': {
"lat_min": l_db.lat_min
}})
if l_db.lon_max > float(stat_dict['lon_max']):
dao.updateOne(collection_name_dbstat, {'_id': doc_id},
{'$set': {
"lon_max": l_db.lon_max
}})
if l_db.lon_min < float(stat_dict['lon_min']):
dao.updateOne(collection_name_dbstat, {'_id': doc_id},
{'$set': {
"lon_min": l_db.lon_min
}})
else:
dao.addOne(collection_name_dbstat, l_db.__dict__)
dao.close()
print('100 % Done of \"' + filename + '\"')