def ProcessFiles(dir): log('Getting del and ny data from files:') flist = os.listdir(dir) res = [] for fname in flist: if '.data' in fname: print 'processing file: ' + fname tot_lines =0 loc_lines_del=0 loc_lines_ny=0 delfname = OUT_DIR + fname + '.delhi' nyfname = OUT_DIR + fname + '.ny' delf = open(delfname, 'w') nyf = open(nyfname, 'w') inputf = open(IN_DIR + fname) line = inputf.readline() while line: rec = loads(line) coord = rec['coordinates'] type = coord['type'] coordinates = coord['coordinates'] lat1 = coordinates[1] long1 = coordinates[0] near_to_city(del_lat,del_long, lat1, long1, 'del') del_dist = distance_on_unit_sphere(lat1,long1,del_lat,del_long) * MF ny_dist = distance_on_unit_sphere(lat1,long1,ny_lat,ny_long) * MF if del_dist< CITY_RADIUS: delf.write(line) loc_lines_del += 1 printcStat(loc_lines_del,tot_lines,'del', 100) #print the status if ny_dist < CITY_RADIUS: nyf.write(line) loc_lines_ny +=1 printcStat(loc_lines_ny, tot_lines,'ny',1000) tot_lines+=1 line = inputf.readline() log_file_stats(delfname, tot_lines, loc_lines_del) log_file_stats(nyfname, tot_lines, loc_lines_ny) res.append({'input_file':fname,'del_file':delfname, 'ny_file': nyfname, 'tot_lines':tot_lines, 'loc_lines_del': loc_lines_del, 'loc_lines_ny': loc_lines_ny}) delf.close() nyf.close() inputf.close() log_final_stats(res)
def near_to_city(city_lat, city_long, lat, long, name): del_dist = distance_on_unit_sphere(lat,long,city_lat,city_long) * MF if del_dist<CITY_RADIUS: return True else: return False