def __init__(self,
mmsi,
imo,
vessel_name,
vessel_type,
length,
width,
country,
longitude,
latitude,
draft,
speed,
str_time,
mark=0):
self.mmsi = mmsi
self.mark = mark
self.imo = imo
if vessel_name is not None:
self.vessel_name = vessel_name.rstrip()
else:
self.vessel_name = None
self.vessel_type = vessel_type
self.length = length
self.width = width
self.country = country
self.longitude = longitude
self.latitude = latitude
self.draft = draft
self.speed = speed
self.date = Utils.convert_utc_to_str_time(str(str_time))
self.utc = Utils.convert_str_time_to_utc(str(str_time))
def get_single_year_oil_tanker(source_path,
source_table,
target_db,
ais_table,
ais_rol_list,
create_table=True,
is_need_cleaning=False,
clean_speed_threshold=0,
clean_draft_threshold=0):
Utils.check_file_path(target_db)
ais = AISService(target_db)
# 数据导入
ais.import_data_from_path(
source_path,
source_table,
ais_table,
ais_rol_list,
create_table=create_table,
filter_query="WHERE Vessel_type_sub='Crude Oil Tanker'")
# 数据清理
if is_need_cleaning:
ais.clean_dirty_data(ais_table, clean_speed_threshold,
clean_draft_threshold)
def split_source_csv_file(source_csv_file,
target_path,
split_csv_file_name,
line_cover=100000):
with open(source_csv_file) as source_csv:
source_csv_reader = csv.reader(source_csv)
header = source_csv_reader.next()
target_index = 0
Utils.check_path(os.path.join(target_path, str(target_index)))
print(os.path.join(target_path, str(target_index)))
os.chdir(os.path.join(target_path, str(target_index)))
split_csv_file = open(split_csv_file_name, 'wb')
split_csv_writer = csv.writer(split_csv_file)
split_csv_writer.writerow(header)
for row in source_csv_reader:
if int(row[15]) / line_cover != target_index:
split_csv_file.close()
target_index += 1
os.chdir(os.path.join(target_path, str(target_index)))
split_csv_file = open(split_csv_file_name, 'wb')
split_csv_writer = csv.writer(split_csv_file)
split_csv_writer.writerow(header)
split_csv_writer.writerow(row)
else:
split_csv_writer.writerow(row)
split_csv_file.close()
return target_index + 1
def dirty_trajectory_clean(
self,
clean_file_name,
time_threshold,
distance_threshold,
):
Utils.check_file_path(clean_file_name)
clean_file = open(clean_file_name, 'wb')
file_writer = csv.writer(clean_file)
file_writer.writerow(next(self.trajectory_reader)) # write head
trajectory_first = self.trajectory_reader.next()
trajectory_list = [trajectory_first]
for after_point in self.trajectory_reader:
if not self.judge_same_line(trajectory_first, after_point):
if not self.judge_dirty(trajectory_list, time_threshold,
distance_threshold):
self.export_clean_trajecotry(trajectory_list, file_writer)
trajectory_first = after_point
trajectory_list = [trajectory_first]
trajectory_list.append(after_point)
if not self.judge_dirty(trajectory_list, time_threshold,
distance_threshold):
self.export_clean_trajecotry(trajectory_list, file_writer)
clean_file.close()
def step4_extract_china_line():
# 1. 获取港口
port_service = PortService(Config.port_name)
Utils.extract_country_trajectories(
port_service, Config.trajectory_output_file,
Config.china_trajectory_output_file,
Config.trajectory_output_header + ['inputOrOutput'], Const.CHINA_NAME)
def get_draft_count_and_load_identify(source_db, ais_table, target_db,
draft_table, draft_rol_list,
draft_state_table, draft_state_rol_list):
Utils.check_file_path(target_db)
draft = DraftDB(target_db)
draft.import_data(source_db, ais_table, draft_table, draft_rol_list)
draft.ships_draft_state_identify(draft_table, draft_state_table,
draft_state_rol_list)
return
def get_use_time(db_name, csv_name):
db_file = CommonDB(db_name)
db_file.run_sql("SELECT * FROM china_trajectory_cn WHERE load_state = 1 AND input_or_output = 'Input' AND "
"vessel_type_sub = 'Crude Oil Tanker' ORDER BY arrive_Time;")
with open(csv_name, 'wb') as csv_file:
csv_writer = csv.writer(csv_file)
for row in db_file.db_cursor:
use_time = Utils.convert_str_time_to_utc(str(row[10])) - Utils.convert_str_time_to_utc(str(row[9]))
csv_writer.writerow(list(row) + [use_time])
def extract_country_relation_trajectory(port_shp_file, source_trajectory_file,
country_trajectory_file,
trajectory_header, country_name):
Utils.check_file_path(country_trajectory_file)
# 1. 获取港口
port_service = PortService(port_shp_file)
Utils.extract_country_trajectories(port_service, source_trajectory_file,
country_trajectory_file,
trajectory_header, country_name)
def get_trajectory_info(source_csv_file, static_info_file_header,
static_info_file_for_analysis, deadweight_db,
deadweight_table, port_shp_file):
# ['line_index', 'mmsi', 'mark', 'imo', 'vessel_name', 'vessel_type', 'length', 'width', 'deadweight',
# 'start_time', 'arrive_time', 'source_port', 'target_port', 'load_state', 'input_or_output']
Utils.check_file_path(static_info_file_for_analysis)
# 1.acquire deadweight info and port info
ships_deadweight = DeadweightDB(deadweight_db)
ships_deadweight.init_ships_deadweight(deadweight_table)
port_service = PortService(port_shp_file)
static_info_file = open(static_info_file_for_analysis, 'wb')
static_info_file_writer = csv.writer(static_info_file)
static_info_file_writer.writerow(static_info_file_header)
with open(source_csv_file) as source_file:
source_reader = csv.reader(source_file)
next(source_reader)
before_line = source_reader.next()
print(before_line)
trajectory_info = start_init(before_line, ships_deadweight,
port_service)
for line in source_reader:
if line[0] == before_line[0]:
# print("!!!!!!!!!!!!!!1")
pass
elif line[1] == before_line[1]:
# print(line)
trajectory_info = same_ship_deal(before_line, line,
trajectory_info, port_service,
static_info_file_writer,
ships_deadweight, True)
else:
print(line)
trajectory_info = same_ship_deal(before_line, line,
trajectory_info, port_service,
static_info_file_writer,
ships_deadweight, True)
before_line = line
final_deal(before_line, trajectory_info, port_service,
static_info_file_writer)
def output_info_init(self, info):
mmsi = info[TrajectoryTime.MMSI_INDEX]
arrive_time = Utils.convert_str_time_to_utc(
info[TrajectoryTime.ARRIVE_TIME_INDEX])
deadweight = info[TrajectoryTime.DEADWEIGHT_INDEX]
ship_static_info = self.fetch_ship_static_info(mmsi)
return [str(mmsi), arrive_time, deadweight] + ship_static_info
def start_init(self, before_row):
output_info = None
if before_row[TrajectoryTime.INPUT_OR_OUTPUT_INDEX] == 'Input':
mmsi = before_row[TrajectoryTime.MMSI_INDEX]
arrive_time = Utils.convert_str_time_to_utc(before_row[TrajectoryTime.ARRIVE_TIME_INDEX])
output_info = [mmsi, arrive_time]
return output_info
def step3_format_line():
# 1. 读取数据库
draft = DraftDB(Config.target_db)
ais = AISService(Config.target_db)
trajectory_file = open(Config.trajectory_output_file, 'wb')
trajectory_writer = csv.writer(trajectory_file)
trajectory_writer.writerow(Config.trajectory_output_header)
port_service = PortService(Config.port_name)
# port_service = ''
# 2. 开始读取
draft_state_dict = draft.start_fetch_transaction(Config.draft_state_table)
ais.start_fetch_data_transaction(Config.ais_table)
line_index = 0
while draft.has_next_draft_state() and ais.has_next_ais_ship():
compare_result = Utils.compare_mmsi(ais.ais_point, draft_state_dict)
if compare_result < 0:
ais.skip_useless_trajectory()
elif compare_result > 0:
draft_state_dict = draft.fetch_draft_state()
else:
line_index = ais.form_trajectory(
draft_state_dict, trajectory_writer, line_index, port_service,
Config.search_distance, Config.trajectory_distance_threshold,
Config.trajectory_speed_threshold)
# 3. 关闭
trajectory_file.close()
draft.close()
ais.close()
def check_file_by_creating_trajectory():
mmsi = 215153000
filename = r"D:\graduation\data\step_result\check\step4\trajectory_{}.csv".format(
mmsi)
target_file_name = r"D:\graduation\data\step_result\check\step4\trajectory_{}.txt".format(
mmsi)
port_name = r"D:\GeoData\Port\WPI.shp"
degree_threshold = Const.LINE_NO_SPLIT_DEGREE
port_service = PortService(port_name)
Utils.convert_csv_to_format_txt(filename, target_file_name, port_service,
degree_threshold)
shp_file_name = r"D:\graduation\data\step_result\check\step4\trajectory_{}.shp".format(
mmsi)
Utils.create_shp(target_file_name, shp_file_name)
def deal_situation(self, info, output_saver):
start_time = info[TrajectoryTime.START_TIME_INDEX]
arrive_time = info[TrajectoryTime.ARRIVE_TIME_INDEX]
line_index = info[TrajectoryTime.LINE_INDEX]
mmsi = info[TrajectoryTime.MMSI_INDEX]
use_time = Utils.convert_str_time_to_utc(
arrive_time) - Utils.convert_str_time_to_utc(start_time)
source = info[TrajectoryTime.SOURCE_INDEX]
load_state = info[TrajectoryTime.LOAD_STATE_INDEX]
ship_static_info = self.fetch_ship_static_info(mmsi)
deadweight = info[TrajectoryTime.DEADWEIGHT_INDEX]
output_saver.writerow([
line_index, use_time, arrive_time, source, load_state, deadweight
] + ship_static_info)
self.update_ship_level_info(
str(mmsi) + "-" + str(deadweight), use_time)
def get_for_analysis(
deadweight_db,
deadweight_table,
source_csv_file,
static_info_file_for_analysis,
):
Utils.check_file_path(static_info_file_for_analysis)
# get trajectory _info
deadweights = DeadweightDB(deadweight_db)
deadweights.init_ships_deadweight(deadweight_table)
Utils.get_trajectory_static_info_file(source_csv_file, deadweights,
static_info_file_for_analysis,
static_info_file_header,
Const.LINE_NO_SPLIT_DEGREE)
return
def split_ship_trajectory(target_db, draft_db, draft_state_table, ais_table,
port_name, trajectory_output_file,
trajectory_output_header, search_distance,
trajectory_distance_threshold,
trajectory_speed_threshold, outliers_output_file):
Utils.check_file_path(trajectory_output_file)
Utils.check_file_path(outliers_output_file)
# 1. 读取数据库
draft = DraftDB(draft_db)
ais = AISService(target_db)
port_service = PortService(port_name)
trajectory_file = open(trajectory_output_file, 'wb')
trajectory_writer = csv.writer(trajectory_file)
trajectory_writer.writerow(trajectory_output_header)
outliers_file = open(outliers_output_file, 'wb')
outliers_writer = csv.writer(outliers_file)
outliers_writer.writerow(['mmsi', 'mark', 'count'])
# 2. 开始读取
draft_state_dict = draft.start_fetch_transaction(draft_state_table)
ais.start_fetch_data_transaction(ais_table)
line_index = 0
while draft.has_next_draft_state() and ais.has_next_ais_ship():
compare_result = Utils.compare_mmsi(ais.ais_point, draft_state_dict)
mmsi = ais.ais_point.mmsi
mark = ais.ais_point.mark
if compare_result < 0:
ais.skip_useless_trajectory()
elif compare_result > 0:
draft_state_dict = draft.fetch_draft_state()
else:
line_index, outliers_count = ais.form_trajectory(
draft_state_dict, trajectory_writer, line_index, port_service,
search_distance, trajectory_distance_threshold,
trajectory_speed_threshold)
outliers_writer.writerow([mmsi, mark, outliers_count])
# 3. 关闭
trajectory_file.close()
draft.close()
ais.close()
def delete_data(self, table_name, filter_list, connect_word):
if not filter_list:
self.clean_table(table_name)
return
filter_query = Utils.parse_filter_query(filter_list, connect_word)
self.db_cursor.execute('DELETE FROM {} Where {}'.format(
table_name, filter_query))
self.db_file.commit()
def extract_country_crude_oil(self, port_shp_name, deadweight_db,
deadweight_table, import_file_name,
export_file_name, file_header,
need_full_load):
Utils.check_file_path(import_file_name)
Utils.check_file_path(export_file_name)
import_file = open(import_file_name, "wb")
import_writer = csv.writer(import_file)
import_writer.writerow(file_header)
export_file = open(export_file_name, "wb")
export_writer = csv.writer(export_file)
export_writer.writerow(file_header)
port_service = PortService(port_shp_name)
# get trajectory _info
deadweights = DeadweightDB(deadweight_db)
deadweights.init_ships_deadweight(deadweight_table)
# skip header
next(self.trajectory_reader)
before = self.trajectory_reader.next()
source_port_name = before[TrajectoryStep.SOURCE_PORT_INDEX]
self.export_country_info(before, source_port_name, port_service,
deadweights, export_writer, need_full_load)
for after in self.trajectory_reader:
if not self.judge_same_line(before, after):
target_port_name = before[TrajectoryStep.TARGET_PORT_INDEX]
self.export_country_info(before, target_port_name,
port_service, deadweights,
import_writer, need_full_load)
source_port_name = after[TrajectoryStep.SOURCE_PORT_INDEX]
self.export_country_info(after, source_port_name, port_service,
deadweights, export_writer,
need_full_load)
before = after
target_port_name = before[TrajectoryStep.TARGET_PORT_INDEX]
self.export_country_info(before, target_port_name, port_service,
deadweights, import_writer, need_full_load)
def run(self):
self.startTime = datetime.datetime.now()
Utils.exeCmdWithTimeout("adb logcat -c",5)
self.runTimer()
try:
print "Begin Listening"
self.process = subprocess.Popen(self.adbCmd.split(' '),stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=False)
while self.requestingQuit==False and self.process.poll() is None:
line = self.process.stdout.readline()
if line.startswith('{'):
self.startTime = datetime.datetime.now()
# print(line)
data = json.loads(line)
self.logListener.onRead(data)
except Exception as e:
print("Catch an exception:"+e.message)
finally:
print "Stop Listening"
pass
def deal_same_ship_case(self, after_row, output_info, csv_writer):
if output_info is not None:
next_start_time = Utils.convert_str_time_to_utc(after_row[TrajectoryTime.START_TIME_INDEX])
output_info[1] = next_start_time - output_info[1]
mmsi = output_info[0]
dead_weight = after_row[TrajectoryTime.MMSI_INDEX]
csv_writer.writerow(output_info)
self.update_ship_level_info(mmsi, output_info[1])
return self.start_init(after_row)
def deal_second_situation(self, output_info, info, output_saver):
next_arrive_time = Utils.convert_str_time_to_utc(
info[TrajectoryTime.ARRIVE_TIME_INDEX])
output_info[1] = next_arrive_time - output_info[1]
mmsi = output_info[0]
deadweight = output_info[2]
output_saver.writerow(output_info)
self.update_ship_level_info(
str(mmsi) + "-" + str(deadweight), output_info[1])
return self.output_info_init(info)
def export_trajectory_to_csv(ais_points, load_state, port_service,
distance_threshold, csv_writer, line_index):
if len(ais_points) == 0:
return
first_point = ais_points[0]
source_port, source_distance = port_service.get_nearest_port(
arcpy.PointGeometry(
arcpy.Point(first_point.longitude, first_point.latitude)),
distance_threshold)
last_point = ais_points[-1]
target_port, target_distance = port_service.get_nearest_port(
arcpy.PointGeometry(
arcpy.Point(last_point.longitude, last_point.latitude)),
distance_threshold)
Utils.export_to_csv(ais_points, csv_writer, [
source_port.name, source_distance, target_port.name,
target_distance, load_state, line_index
])
def final_deal(ship_info, trajectory_info, port_service, csv_writer):
arrive_time = Utils.convert_utc_to_str_time(int(ship_info[12]))
trajectory_info[10] = arrive_time
# input_or_output = trajectory_info[-1]
target_country = port_service.get_port_by_name(ship_info[14]).country
input_or_output = get_input_or_output_state(trajectory_info[-1],
target_country, False)
trajectory_info[-1] = input_or_output
# print(trajectory_info)
csv_writer.writerow(trajectory_info)
def create_shp_with_info(port_shp_file, deadweight_db, deadweight_table,
source_csv_file, target_path, split_csv_file,
format_txt_file, static_info_file_for_shp, shp_file,
split_trajectory_degree):
split_count = split_source_csv_file(source_csv_file, target_path,
split_csv_file)
port_service = PortService(port_shp_file)
deadweights = DeadweightDB(deadweight_db)
deadweights.init_ships_deadweight(deadweight_table)
for i in range(split_count):
os.chdir(os.path.join(target_path, str(i)))
Utils.convert_csv_to_format_txt(split_csv_file, format_txt_file,
port_service, split_trajectory_degree)
Utils.get_trajectory_static_info_file(split_csv_file, deadweights,
static_info_file_for_shp,
static_info_file_header,
split_trajectory_degree)
Utils.create_shp(format_txt_file, shp_file)
return
def get_count(self,
source_table,
count_target='*',
filter_list=None,
connect_word=''):
if filter_list is None:
self.db_cursor.execute("SELECT count({}) FROM {}".format(
count_target, source_table))
else:
filter_query = Utils.parse_filter_query(filter_list, connect_word)
self.db_cursor.execute("SELECT count({}) FROM {} WHERE {}".format(
count_target, source_table, filter_query))
row = next(self.db_cursor)
return row[0]
def shared_mmsi_identify(source_db, source_table, speed_threshold,
distance_threshold, point_percent, output_ais_csv,
output_header):
"""
identify the situation of the shared MMSI, consider the speed, distance threshold, and delete the the ship which
point is less than the point_threshold
:param source_db: the database file of the original data
:param source_table: the table name of the original data in the database file
:param speed_threshold: the threshold of speed to identify whether the same ship
:param distance_threshold: the threshold of distance to identify whether the same ship
:param point_percent: delete the ship with the point less than the percent threshold
:param output_ais_csv: the .csv file of the output data
:param output_header: the header in the .csv file
:return: None
"""
Utils.check_file_path(output_ais_csv)
ais = AISService(source_db)
result_file = open(output_ais_csv, 'wb')
csv_writer = csv.writer(result_file)
csv_writer.writerow(output_header)
ais.start_fetch_original_data_transaction(source_table)
while ais.has_next_ais_ship():
print(ais.ais_point.mmsi)
ais.same_mmsi_identify(
csv_writer,
speed_threshold,
distance_threshold,
point_percent,
)
result_file.close()
ais.close()
return
def start_init(ship_info, ships_deadweight, port_service):
# ["line_index", "mmsi", "mark", "imo", "vessel_name", "vessel_type", "length", "width", "longitude", "latitude",
# "draft", "speed", "date", "utc", "source_port", "target_port", "load_state", ]
deadweight_of_mmsi = ships_deadweight.get_deadweight_by_mmsi(ship_info[1])
start_time = Utils.convert_utc_to_str_time(int(ship_info[12]))
source_country = port_service.get_port_by_name(ship_info[13]).country
input_or_output = get_input_or_output_state(None, source_country, True)
trajectory_info = ship_info[:8] + [
deadweight_of_mmsi,
start_time,
None,
] + ship_info[13:] + [input_or_output]
# print(trajectory_info)
return trajectory_info
def doTest(self):
if Utils.exeInstrument(self.instrumentCmd):
print "test {} ok".format(self.current)
else:
print "test {} failure".format(self.current)
self.logcatMonitor.stop(True)
def __lateRentals(self):
Utils.print_all_indexed(self.__statisticsControler.lateRentedMovies())
def __currentlyRentedMovies(self):
Utils.print_all_indexed(self.__statisticsControler.allRentedMovies())
def __mostActiveClients(self):
Utils.print_all_indexed(self.__statisticsControler.mostActiveClients())
def __mostRentedMovies(self):
Utils.print_all_indexed(self.__statisticsControler.mostRentedMovies())