def make_output_csvs(dir, metadata):
metadataframe = metadata
# taking meta data file to list
if isinstance(metadata, pd.DataFrame):
metadata = bl.df2list(metadata)
# getting header value
header = metadata[0]
count = 0
for row in metadata[1:]:
count += 1
# getting the extrema values in each row
extrema = get_extrema_metadata(header, row)
# getting filename
filename = dir + '/' + row[0] + '.tif'
# from the arguments compiles makes the csv output
try:
make_image_output(filename,
metadataframe,
extrema,
outputcsvfile=dir + '/' + row[0] + '.csv')
except IOError:
print 'out'
print 'Task:%s,Completed:%s' % (count, filename)
def generate_corners_metadata(filename,csvmetadata): # getting only the filename column poriton of the metadata file filename = str.split(filename,'.')[0] if '/' in str(filename): filename = str.split(filename,'/')[-1] # getting only the row associated with filename metarow = csvmetadata[csvmetadata.FILENAME == filename] # taking the data frame row to list metarow = metarow.values.tolist()[0] # getting the metadata header header = bl.df2list(csvmetadata)[0] # iterating through both the header and row to collect # both lat and longs lats = [] longs = [] for headval,rowval in itertools.izip(header,metarow): if 'lat' in str(headval).lower(): lats.append(rowval) if 'long' in str(headval).lower(): longs.append(rowval) # iterating through lat/longs and putting into list corners = [['LONG','LAT']] for long,lat in itertools.izip(longs,lats): corners.append([long,lat]) return corners
def generate_corners_metadata(filename, csvmetadata):
# getting only the filename column poriton of the metadata file
filename = str.split(filename, '.')[0]
if '/' in str(filename):
filename = str.split(filename, '/')[-1]
# getting only the row associated with filename
metarow = csvmetadata[csvmetadata.FILENAME == filename]
# taking the data frame row to list
metarow = metarow.values.tolist()[0]
# getting the metadata header
header = bl.df2list(csvmetadata)[0]
# iterating through both the header and row to collect
# both lat and longs
lats = []
longs = []
for headval, rowval in itertools.izip(header, metarow):
if 'lat' in str(headval).lower():
lats.append(rowval)
if 'long' in str(headval).lower():
longs.append(rowval)
# iterating through lat/longs and putting into list
corners = [['LONG', 'LAT']]
for long, lat in itertools.izip(longs, lats):
corners.append([long, lat])
return corners
def make_output_csvs(dir,metadata):
metadataframe = metadata
# taking meta data file to list
if isinstance(metadata,pd.DataFrame):
metadata = bl.df2list(metadata)
# getting header value
header = metadata[0]
count = 0
for row in metadata[1:]:
count += 1
# getting the extrema values in each row
extrema = get_extrema_metadata(header,row)
# getting filename
filename = dir + '/' + row[0] + '.tif'
# from the arguments compiles makes the csv output
try:
make_image_output(filename,metadataframe,extrema,outputcsvfile=dir+'/'+row[0]+'.csv')
except IOError:
print 'out'
print 'Task:%s,Completed:%s' % (count,filename)
count=0
ind=0
while not count==8:
count+=1
# making table associated with a specific presicion
newtable=bl.map_table(data,count,list=True)
#taking output to table and reading back into memory
newtable.to_csv('table'+str(count)+'.csv')
newtable=pd.read_csv('table'+str(count)+'.csv')
# taking new table from a dataframe to a list
newtable=bl.df2list(newtable)
# reading table to test into memory
testtable=pd.read_csv('table_datum/table'+str(count)+'.csv')
# taking test table to list
testtable=bl.df2list(testtable)
# testing every row in tables for
for a,b in itertools.izip(newtable,testtable):
if not a==b:
ind=1
print 'Row New: %s, Row Old: %s' % (a,b)
if ind==0:
print 'Table size: %s Test Passed' % count
def make_squares_table(table): checkrow = bl.df2list(table)[1] if not checkrow[-5] == 1 or checkrow[-5] == 0: startx = checkrow[0] starty = checkrow[1] # getting the first and second position point point1 = table[(table.X == startx)&(table.Y == starty)] point2 = table[(table.X == (startx + 1))&(table.Y == (starty + 1))] else: # getting the first and second position point point1 = table[(table.X == 1)&(table.Y == 1)] point2 = table[(table.X == 2)&(table.Y == 2)] # taking table to list table = bl.df2list(table) # getting header header = table[0] # getting first row firstrow = bl.df2list(point1)[1] # getting second row secondrow = bl.df2list(point2)[1] # getting point1 and point2 point1 = getlatlong(header,firstrow) point2 = getlatlong(header,secondrow) # settin up newlist with header header newlist = [['GEOHASH','LAT1','LONG1','LAT2','LONG2','LAT3','LONG3','LAT4','LONG4','X','Y','RED','GREEN','BLUE','COLORKEY']] # getting distance distance = (((point1[0] - point2[0]) ** 2) + ((point1[1] - point2[1]) ** 2)) ** .5 distance_vert = abs(point1[1] - point2[1]) / 2.0 distance_horz = abs(point1[0] - point2[0]) / 2.0 # iterating through each point to make squares for row in table[1:]: # getting point info to be added to square pointinfo = row[-5:] # getting point for each row point = getlatlong(header,row) # adding distance to get each corner point ul_point = [point[0] - distance_horz,point[1] + distance_vert] ur_point = [point[0] + distance_horz,point[1] + distance_vert] bl_point = [point[0] - distance_horz,point[1] - distance_vert] br_point = [point[0] + distance_horz,point[1] - distance_vert] # making newrow newrow = [pointinfo[0]] + [bl_point[1],bl_point[0]] + [br_point[1],br_point[0]] + [ul_point[1],ul_point[0]] + [ur_point[1],ur_point[0]] + [row[0],row[1]] +pointinfo[1:] newlist.append(newrow) # taking newlist to dataframe again newlist = bl.list2df(newlist) return newlist
# reading csv file to pandas
data=pd.read_csv('sharks.csv')
# mapping table to precision 4 geohashs
data=bl.map_table(data,3,list=True) # this creates a csv file with a density block table
squares=pd.read_csv('squares3.csv')
squares=squares[:10] # getting the top 10 densest geohash squares
# getting each unique geohash
geohashs=np.unique(squares['GEOHASH']).tolist()
# constructing list of all top shark attack incidents
total=[data.columns.values.tolist()]
for row in geohashs:
temp=data[data.GEOHASH==str(row)] # getting all the geohashs within the entire table
temp=bl.df2list(temp) # taking dataframe to list
total+=temp[1:] #ignoring the header on each temporary list
# taking list back to dataframe
total=bl.list2df(total)
# getting the unique activities that occured
uniques=np.unique(total['Activity']).tolist()
# we now have a list with each top geohash in a aggregated table
# we can now style each color icon based on each activity being performed during the attack
count=0
filenames=[]
for unique,color in itertools.izip(uniques,colors):
count+=1
key='pk.eyJ1IjoibXVycGh5MjE0IiwiYSI6ImNpam5kb3puZzAwZ2l0aG01ZW1uMTRjbnoifQ.5Znb4MArp7v3Wwrn6WFE6A'
# reading into memory
points=pd.read_csv('points_example.csv')
line=pd.read_csv('line_example.csv')
# geohashing each table
points=bl.map_table(points,7,list=True)
line=bl.map_table(line,7,list=True)
# getting unique geohashs
uniquepoints=np.unique(points['GEOHASH']).tolist()
uniqueline=np.unique(line['GEOHASH']).tolist()
newpoints=[points.columns.values.tolist()]
# we know if a unique point is in any unique line its on the route
for row in uniquepoints:
oldrow=row
for row in uniqueline:
if row==oldrow:
temp=points[points.GEOHASH==oldrow]
temp=bl.df2list(temp)
newpoints+=temp[1:] # getting all the points within this geohashs
# making the new points, line, and blocks along line
bl.make_points(newpoints,list=True,filename='points.geojson')
bl.make_blocks('squares7.csv',filename='blocks_on_line.geojson')
bl.make_line(line,list=True,filename='line.geojson')
bl.loadparsehtml(bl.collect(),key)
def make_squares_table(table):
checkrow = bl.df2list(table)[1]
if not checkrow[-5] == 1 or checkrow[-5] == 0:
startx = checkrow[0]
starty = checkrow[1]
# getting the first and second position point
point1 = table[(table.X == startx) & (table.Y == starty)]
point2 = table[(table.X == (startx + 1)) & (table.Y == (starty + 1))]
else:
# getting the first and second position point
point1 = table[(table.X == 1) & (table.Y == 1)]
point2 = table[(table.X == 2) & (table.Y == 2)]
# taking table to list
table = bl.df2list(table)
# getting header
header = table[0]
# getting first row
firstrow = bl.df2list(point1)[1]
# getting second row
secondrow = bl.df2list(point2)[1]
# getting point1 and point2
point1 = getlatlong(header, firstrow)
point2 = getlatlong(header, secondrow)
# settin up newlist with header header
newlist = [[
'GEOHASH', 'LAT1', 'LONG1', 'LAT2', 'LONG2', 'LAT3', 'LONG3', 'LAT4',
'LONG4', 'X', 'Y', 'RED', 'GREEN', 'BLUE', 'COLORKEY'
]]
# getting distance
distance = (((point1[0] - point2[0])**2) +
((point1[1] - point2[1])**2))**.5
distance_vert = abs(point1[1] - point2[1]) / 2.0
distance_horz = abs(point1[0] - point2[0]) / 2.0
# iterating through each point to make squares
for row in table[1:]:
# getting point info to be added to square
pointinfo = row[-5:]
# getting point for each row
point = getlatlong(header, row)
# adding distance to get each corner point
ul_point = [point[0] - distance_horz, point[1] + distance_vert]
ur_point = [point[0] + distance_horz, point[1] + distance_vert]
bl_point = [point[0] - distance_horz, point[1] - distance_vert]
br_point = [point[0] + distance_horz, point[1] - distance_vert]
# making newrow
newrow = [pointinfo[0]] + [bl_point[1], bl_point[0]] + [
br_point[1], br_point[0]
] + [ul_point[1], ul_point[0]] + [ur_point[1], ur_point[0]
] + [row[0], row[1]] + pointinfo[1:]
newlist.append(newrow)
# taking newlist to dataframe again
newlist = bl.list2df(newlist)
return newlist
data = pd.read_csv('points_example.csv')
count = 0
ind = 0
while not count == 8:
count += 1
# making table associated with a specific presicion
newtable = bl.map_table(data, count, list=True)
#taking output to table and reading back into memory
newtable.to_csv('table' + str(count) + '.csv')
newtable = pd.read_csv('table' + str(count) + '.csv')
# taking new table from a dataframe to a list
newtable = bl.df2list(newtable)
# reading table to test into memory
testtable = pd.read_csv('table_datum/table' + str(count) + '.csv')
# taking test table to list
testtable = bl.df2list(testtable)
# testing every row in tables for
for a, b in itertools.izip(newtable, testtable):
if not a == b:
ind = 1
print 'Row New: %s, Row Old: %s' % (a, b)
if ind == 0:
print 'Table size: %s Test Passed' % count