def read_source_file(file_name, weather_file_name): with open(file_name, 'r') as readCollision: # r represent read model print("Start to read file: " + file_name + ". This may take a while...") file = csv.reader(readCollision) for row in file: if "COLLISION_ID" not in row[0]: collision = Collision() collision.collision_id = row[0] collision.location_key = row[1] collision.hour_key = row[2] collision.environment = row[3] collision.light = row[4] collision.surface_condition = row[5] collision.traffic_control = row[6] collision.traffic_control_condition = row[7] collision.collision_classification = row[8] collision.impace_type = row[9] collision.no_of_pedestrians = row[10] collision.date = row[11] collision.location = row[12] collision.is_intersection = row[13] collisions.append(collision) readCollision.close() with open(weather_file_name, 'r') as readWeather: # r represent read model print("Start to read file: " + weather_file_name + ". This may take a while...") file = csv.reader(readWeather) key_ctr = 0 for row in file: weathers.append(row) # print(row[0]+" @ "+row[24]) readWeather.close()
def accident_processor(collision_file_name, hour_file_name, location_file_name): """ Retrieve data about the time, in hour, of the ottawa's collision record from a csv file. :param collision_file_name: The name of the collision file, which has been generated from preprocessor :param hour_file_name: The processed hour table file :param location_file_name: the processed location table file """ global total_record global total_valid_record hours = [] locations = [] collisions = [] with open(hour_file_name, 'r') as readHour: # r represent read model print("Start to read file: " + hour_file_name + ". This may take a while...") file = csv.reader(readHour) for row in file: if "HOUR_ID" not in row[0]: hour_id = row[0] hours.append(hour_id) readHour.close() print("Finished reading data from hour table") with open(location_file_name, 'r') as readLocation: # r represent read model print("Start to read file: " + location_file_name + ". This may take a while...") file = csv.reader(readLocation) for row in file: if "LOCATION_ID" not in row[0]: location_id = row[0] locations.append(location_id) readLocation.close() print("Finished reading data from location table") with open(collision_file_name, 'r') as readCollision: # r represent read model print("Start to read file: " + collision_file_name + ". This may take a while...") file = csv.reader(readCollision) ptr = 0 for row in file: if "COLLISION_ID" not in row[0]: collision = Collision() collision.collision_id = row[0] collision.location_id = locations[ptr] # append corresponding id collision.hour_id = hours[ptr] # append corresponding id collision.environment = row[6] collision.light = row[7] collision.surface_condition = row[8] collision.traffic_control = row[9] collision.traffic_control_condition = row[10] collision.collision_classification = row[11] collision.impace_type = row[12] collision.no_of_pedestrians = row[13] ptr = ptr + 1 collisions.append(collision) readCollision.close() print("Finished processing collision table") return collisions
def read_source_file(file_name, hour_file_name): with open(file_name, 'r') as readCollision: # r represent read model print("Start to read file: " + file_name + ". This may take a while...") file = csv.reader(readCollision) for row in file: if "COLLISION_ID" not in row[0]: collision = Collision() collision.collision_id = row[0] collision.location_key = row[1] collision.hour_key = row[2] collision.weather_key = row[3] collision.environment = row[4] collision.light = row[5] collision.surface_condition = row[6] collision.traffic_control = row[7] collision.traffic_control_condition = row[8] collision.collision_classification = row[9] collision.impace_type = row[10] collision.no_of_pedestrians = row[11] collision.date = row[12] collision.location = row[13] collision.is_intersection = row[14] collisions.append(collision) readCollision.close() with open(hour_file_name, 'r') as readHour: # r represent read model print("Start to read file: " + hour_file_name + ". This may take a while...") file = csv.reader(readHour) for row in file: if "EVENT_ID" not in row[0]: event = Event() event.event_id = row[0] event.event_name = row[1] event.event_start_date = row[2] event.event_end_date = row[3] if int(event.event_start_date.split("-")[0]) > int( event.event_end_date.split("-")[0]): raise Exception( "Wrong event time - event starting date must be earlier than event ending date: " "START:" + event.event_start_date + " compare to END:" + event.event_end_date) if int(event.event_start_date.split("-")[1]) > int( event.event_end_date.split("-")[1]): raise Exception( "Wrong event time - event starting date must be earlier than event ending date: " "START:" + event.event_start_date + " compare to END:" + event.event_end_date) events.append(event) readHour.close()
def read_source_file(file_name, hour_file_name): with open(file_name, 'r') as readCollision: # r represent read model print("Start to read file: " + file_name + ". This may take a while...") file = csv.reader(readCollision) for row in file: if "COLLISION_ID" not in row[0]: collision = Collision() collision.collision_id = row[0] collision.location_id = row[1] collision.hour_id = row[2] collision.environment = row[3] collision.light = row[4] collision.surface_condition = row[5] collision.traffic_control = row[6] collision.traffic_control_condition = row[7] collision.collision_classification = row[8] collision.impace_type = row[9] collision.no_of_pedestrians = row[10] collisions.append(collision) readCollision.close() with open(hour_file_name, 'r') as readHour: # r represent read model print("Start to read file: " + hour_file_name + ". This may take a while...") file = csv.reader(readHour) for row in file: if "HOUR_ID" not in row[0]: hour = Hour() hour.hour_id = row[0] hour.hour_start = row[1] hour.hour_end = row[2] hour.date = row[3] hour.day_of_week = row[4] hour.day = row[5] hour.month = row[6] hour.year = row[7] hour.weekend = row[8] hour.holiday = row[9] hour.holiday_name = row[10] hours.append(hour) readHour.close()
def read_source_file(file_name, locoation_file_name): with open(file_name, 'r') as readCollision: # r represent read model print("Start to read file: " + file_name + ". This may take a while...") file = csv.reader(readCollision) for row in file: if "COLLISION_ID" not in row[0]: collision = Collision() collision.collision_id = row[0] collision.location_id = row[1] collision.hour_key = row[2] collision.environment = row[3] collision.light = row[4] collision.surface_condition = row[5] collision.traffic_control = row[6] collision.traffic_control_condition = row[7] collision.collision_classification = row[8] collision.impace_type = row[9] collision.no_of_pedestrians = row[10] collision.date = row[11] collisions.append(collision) readCollision.close() with open(locoation_file_name, 'r') as readLocation: # r represent read model print("Start to read file: " + locoation_file_name + ". This may take a while...") file = csv.reader(readLocation) for row in file: if "LOCATION_ID" not in row[0]: location = Location() location.location_id = row[0] location.street_name = row[1] location.intersection_one = row[2] location.intersection_two = row[3] location.longitude = row[4] location.latitude = row[5] location.neighborhood = row[6] location.closest_weather_station = row[7] locations.append(location) readLocation.close()
def process_collision_table(): list = collision_processor("2014collisionsfinal.xls.csv") list1 = collision_processor("2016collisionsfinal.xls.csv") list2 = collision_processor("2015collisionsfinal.xls.csv") collisions = [] with open("h2017collisionsfinal.csv", 'r') as readData: # r represent read model print( "Start to read file: h2017collisionsfinal.csv. This may take a while..." ) file = csv.reader(readData) key_ctr = 0 for row in file: if "Record" not in row[0]: collision = Collision() collision.collision_id = key_ctr key_ctr = key_ctr + 1 collision.location = row[1] collision.longtitude = row[4] collision.latitude = row[5] collision.date = row[7] collision.time = row[8] environment = row[9] if row[9] == "": environment = "Unknown" collision.environment = remove_prefix(environment, "Unknown") light = row[13] if row[13] == "": light = "Unknown" collision.light = remove_prefix(light, "Unknown") surface_condition = row[10] if row[10] == "": surface_condition = "Unknown" collision.surface_condition = remove_prefix( surface_condition, "Unknown") traffic_control = row[11] if row[11] == "": traffic_control = "Unknown" collision.traffic_control = remove_prefix( traffic_control, "Unknown") collision.traffic_control_condition = "N/A" collision_classification = row[14] if row[14] == "": collision_classification = "Unknown" collision.collision_classification = remove_prefix( collision_classification, "Unknown") impact_type = row[15] if row[15] == "": impact_type = "Unknown" collision.impace_type = remove_prefix(impact_type, "Unknown") collision.no_of_pedestrians = "N/A" collisions.append(collision) readData.close() # with open("2017collisionsfinal.xls.csv", 'w', newline='') as csvFile: # print("Prepare to write the data into the file: h2017collisionsfinal.csv. It might take a while...") # writer = csv.writer(csvFile) # writer.writerow(["COLLISION_ID", "LOCATION", "LONGITUDE", "LATITUDE", "DATE", "TIME", "ENVIRONMENT", # "LIGHT", "SURFACE_CONDITION", "TRAFFIC_CONTROL", "TRAFFIC_CONTROL_CONDITION", # "COLLISION_CLASSIFICATION", "IMPACT_TYPE", "NO_OF_PEDESTRIANS"]) # for collision in collisions: # writer.writerow([collision.collision_id, collision.location, collision.longtitude, collision.latitude, # collision.date, collision.time, collision.environment, collision.light, # collision.surface_condition, collision.traffic_control, # collision.traffic_control_condition, collision.collision_classification, # collision.impace_type, collision.no_of_pedestrians]) # csvFile.close() list.extend(list1) list.extend(list2) list.extend(collisions) output_collision_data_from_list_to_new_csv("Collision_Table", list)
def collision_processor(file_name): """ Retrieve data about the time, in hour, of the ottawa's collision record from a csv file. :param file_name: the file to retrieve data """ global total_record global total_valid_record collisions = [] key_ctr = 0 with open(file_name, 'r') as readData: # r represent read model print("Start to read file: " + file_name + ". This may take a while...") file = csv.reader(readData) canada_holiday = holidays.CA() for row in file: if "COLLISION_ID" not in row[0]: collision = Collision() collision.collision_id = key_ctr key_ctr = key_ctr + 1 collision.location = row[1] collision.longtitude = row[4] collision.latitude = row[5] collision.date = row[6] collision.time = row[7] environment = row[8] if row[8] == "": environment = "Unknown" collision.environment = remove_prefix(environment, "Unknown") light = row[9] if row[9] == "": light = "Unknown" collision.light = remove_prefix(light, "Unknown") surface_condition = row[10] if row[10] == "": surface_condition = "Unknown" collision.surface_condition = remove_prefix( surface_condition, "Unknown") traffic_control = row[11] if row[11] == "": traffic_control = "Unknown" collision.traffic_control = remove_prefix( traffic_control, "Unknown") traffic_control_condition = row[12] if row[12] == "": if collision.traffic_control == "No control": traffic_control_condition = "N/A" collision.traffic_control_condition = traffic_control_condition else: traffic_control_condition = "Unknown" collision.traffic_control_condition = traffic_control_condition else: collision.traffic_control_condition = remove_prefix( traffic_control_condition, "Unknown") collision_classification = row[13] if row[13] == "": collision_classification = "Unknown" collision.collision_classification = remove_prefix( collision_classification, "Unknown") impact_type = row[14] if row[14] == "": impact_type = "Unknown" collision.impace_type = remove_prefix(impact_type, "Unknown") no_of_pedestrians = row[15] if row[14] == "": no_of_pedestrians = -1 collision.no_of_pedestrians = no_of_pedestrians collisions.append(collision) return collisions