def visualize_community(date):
s1 = time.time()
cow_id_list = my_utility.get_existing_cow_list(date, cows_record_file)
com_creater = community_creater.CommunityCreater(date, cow_id_list)
cow_id_list = com_creater.cow_id_list
# --- 行動同期を計測する ---
t = date + datetime.timedelta(
hours=12) # 正午12時を始まりとするが.......ときに9時始まりのときもある
t_start = date + datetime.timedelta(hours=12) # 正午12時を始まりとする
t_end = date + datetime.timedelta(days=1) + datetime.timedelta(
hours=9) # 翌午前9時を終わりとする
while (t < t_end):
interaction_graph = com_creater.make_interaction_graph(t, t+datetime.timedelta(minutes=delta_c), method="behavior", delta=delta_s, epsilon=epsilon, dzeta=dzeta) \
if (t_start <= t) else np.array([[]]) # 重み付きグラフを作成
community = com_creater.create_community(t, t+datetime.timedelta(minutes=delta_c), interaction_graph, delta=delta_s, leng=leng) \
if (t_start <= t) else [[]] # コミュニティを決定
com_creater.visualize_position(t,
t + datetime.timedelta(minutes=delta_c),
community,
target_cow_id=None,
delta=delta_s) # 位置情報とコミュニティをプロット
t += datetime.timedelta(minutes=delta_c)
e1 = time.time()
print("処理時間", (e1 - s1) / 60, "[min]")
return
def fetch_information(date):
""" 1日分の牛のデータを読み込む """
date = datetime.datetime(date.year, date.month, date.day, 0, 0, 0)
cows_record_file = os.path.abspath('../') + "/CowTagOutput/csv/" # 分析用のファイル
cow_id_list = my_utility.get_existing_cow_list(date, cows_record_file)
com_creater = community_creater.CommunityCreater(date, cow_id_list)
cow_id_list = com_creater.cow_id_list
return cow_id_list, com_creater
def create_corpus(repeat_num = 100):
""" コーパスを作成し，ファイルに書き込む """
corpus = []
start = datetime.datetime(2018, 6, 1, 0, 0, 0)
end = datetime.datetime(2018, 12, 31, 0, 0, 0)
log_file = "./synchronization/topic_model/log_for_making_session.txt"
my_utility.delete_file(log_file) # 一度既存のログファイルをクリーンする
for cow_id in target_list:
my_utility.write_logs(log_file, [str(cow_id)])
for i in range(repeat_num):
date = start + datetime.timedelta(days=np.random.randint(0, 200)) # startからランダムな日数を経過させる
if (date < end): # 期間内であれば
s1 = time.time()
communities_list = []
community_graph_list = []
interaction_graph_list = []
t_list = []
cow_id_list = my_utility.get_existing_cow_list(date, cows_record_file)
com_creater = community_creater.CommunityCreater(date, cow_id_list)
cow_id_list = com_creater.cow_id_list
if (cow_id in cow_id_list): # 対象牛の位置情報があれば
# --- コミュニティ構造を分析する ---
t = date + datetime.timedelta(hours=12) # 正午12時を始まりとするが.......ときに9時始まりのときもある
t_start = date + datetime.timedelta(hours=12) # 正午12時を始まりとする
t_end = date + datetime.timedelta(days=1) + datetime.timedelta(hours=9) # 翌午前9時を終わりとする
while (t < t_end):
t_list.append(t)
interaction_graph = com_creater.make_interaction_graph(t, t+datetime.timedelta(minutes=delta_c), method="position", delta=delta_s, epsilon=epsilon, dzeta=dzeta) \
if (t_start <= t) else np.array([[]]) # 重み付きグラフを作成
community = com_creater.create_community(t, t+datetime.timedelta(minutes=delta_c), interaction_graph, delta=delta_s, leng=leng) \
if (t_start <= t) else [[]] # コミュニティを決定
# com_creater.visualize_position(t, t+datetime.timedelta(minutes=delta_c), community, target_cow_id='20170', delta=delta_s) # 位置情報とコミュニティをプロット1
community_graph = com_creater.get_community_graph(community)
interaction_graph_list.append(interaction_graph)
communities_list.append(community)
community_graph_list.append(community_graph)
t += datetime.timedelta(minutes=delta_c)
e1 = time.time()
print("処理時間", (e1-s1)/60, "[min]")
# --- 変化点を検知し，セッションを作る ---
s2 = time.time()
behavior_synch = com_creater.get_behavior_synch()
graph_analyzer = GraphSeriesAnalysis(cow_id_list, interaction_graph_list, "Poisson")
# graph_analyzer.visualize_graph(cow_id, t_list) # グラフをまとめて可視化
change_points, score_list = graph_analyzer.detect_change_point(cow_id, 5, 5, threshold=400) # 変化点検知
# df = pd.concat([pd.Series(t_list), pd.Series(score_list), pd.Series(change_points)], axis=1, names=["time", "score", "change_flag"])
# df.to_csv("./synchronization/graph_operation/"+ str(cow_id) + ".csv") # csvで保存
community_list = make_session.get_focused_community(communities_list, cow_id) # セッションを作成するために対象牛の所属するコミュニティを抽出
cow_id_session = make_session.process_time_series(t_list, community_list, change_points) # 牛IDでセッションを作成
space_session = make_session.exchange_cowid_to_space(cow_id_session, behavior_synch, delta_c, delta_s) # 特徴表現でセッションを作成
corpus.extend(space_session)
session_io.write_session(space_session, corpus_file+cow_id + "/" + date.strftime("%Y%m%d/"))
my_utility.write_logs(log_file, [str(i+1) + "番目の試行 Date: " + date.strftime("%Y/%m/%d") + " Success"])
e2 = time.time()
print("処理時間", (e2-s2)/60, "[min]")
else:
my_utility.write_logs(log_file, [str(i+1) + "番目の試行 Date: " + date.strftime("%Y/%m/%d") + " Failure"])
return
def predict_session(gaussian_lda, theta):
""" 新にセッションを作り，インタラクション列を追加する """
start = datetime.datetime(2018, 5, 16, 0, 0, 0)
end = datetime.datetime(2018, 7, 31, 0, 0, 0)
date = start
while (date < end):
s1 = time.time()
communities_list = []
community_graph_list = []
interaction_graph_list = []
t_list = []
cow_id_list = my_utility.get_existing_cow_list(date, cows_record_file)
com_creater = community_creater.CommunityCreater(date, cow_id_list)
cow_id_list = com_creater.cow_id_list
# --- 行動同期を計測する ---
t = date + datetime.timedelta(hours=12) # 正午12時を始まりとするが.......ときに9時始まりのときもある
t_start = date + datetime.timedelta(hours=12) # 正午12時を始まりとする
t_end = date + datetime.timedelta(days=1) + datetime.timedelta(hours=9) # 翌午前9時を終わりとする
while (t < t_end):
t_list.append(t)
interaction_graph = com_creater.make_interaction_graph(t, t+datetime.timedelta(minutes=delta_c), method="position", delta=delta_s, epsilon=epsilon, dzeta=dzeta) \
if (t_start <= t) else np.array([[]]) # 重み付きグラフを作成
community = com_creater.create_community(t, t+datetime.timedelta(minutes=delta_c), interaction_graph, delta=delta_s, leng=leng) \
if (t_start <= t) else [[]] # コミュニティを決定
com_creater.visualize_position(t, t+datetime.timedelta(minutes=delta_c), community, target_cow_id='20113', delta=delta_s) # 位置情報とコミュニティをプロット1
community_graph = com_creater.get_community_graph(community)
interaction_graph_list.append(interaction_graph)
communities_list.append(community)
community_graph_list.append(community_graph)
t += datetime.timedelta(minutes=delta_c)
e1 = time.time()
print("処理時間", (e1-s1)/60, "[min]")
# --- 変化点を検知し，セッションを作る ---
s2 = time.time()
behavior_synch = com_creater.get_behavior_synch()
for cow_id in target_list:
if (cow_id in cow_id_list):
graph_analyzer = GraphSeriesAnalysis(cow_id_list, interaction_graph_list, "Poisson")
# graph_analyzer.visualize_graph(cow_id, t_list) # グラフをまとめて可視化
change_points, score_list = graph_analyzer.detect_change_point(cow_id, 5, 5, threshold=400) # 変化点検知
df = pd.concat([pd.Series(t_list), pd.Series(score_list), pd.Series(change_points)], axis=1, names=["time", "score", "change_flag"])
# df.to_csv("./synchronization/graph_operation/"+ str(cow_id) + ".csv") # csvで保存
community_list = make_session.get_focused_community(communities_list, cow_id) # セッションを作成するために対象牛の所属するコミュニティを抽出
cow_id_session = make_session.process_time_series(t_list, community_list, change_points) # 牛IDでセッションを作成
space_session = make_session.exchange_cowid_to_space(cow_id_session, behavior_synch, delta_c, delta_s, dim=2) # 特徴表現でセッションを作成
topic_dist = gaussian_lda.predict(space_session, theta) # 予測結果．各トピックの分布で現れる
result = [np.where(p==max(p))[0][0] for p in topic_dist] # 最も予測確率が高いものをトピックに据える
time_series_result = make_session.restore_time_series(t_list, change_points, [topic_dist, result]) # 結果を時系列データに直す
df = pd.concat([df, pd.DataFrame(time_series_result[0]), pd.Series(time_series_result[1])], axis=1)
my_utility.confirm_dir(change_point_file+str(cow_id)) # ディレクトリのパスを作る
df.to_csv(change_point_file+str(cow_id)+date.strftime("/%Y%m%d.csv"))
e2 = time.time()
print("処理時間", (e2-s2)/60, "[min]")
date += datetime.timedelta(days=1)
return
def visualize_adjectory(dt, row, target_list, delta=5):
s = time.time()
tmp_time = dt - datetime.timedelta(hours=9) # 時差の分巻き戻しておく（翌朝になっている可能性もあるので）
date = datetime.datetime(tmp_time.year, tmp_time.month, tmp_time.day, 0, 0, 0)
cow_id_list = my_utility.get_existing_cow_list(date, cows_record_file)
com_creater = community_creater.CommunityCreater(date, cow_id_list)
for i, cow_data in enumerate(row): # 各牛ごとに見ていく
if (cow_data is not None and cow_data is not np.nan):
target_cow_id = target_list[i]
try:
opponent_cow_id = str(int(cow_data[7]))
except:
pdb.set_trace()
start_t = datetime.datetime.strptime(cow_data[0], '%Y-%m-%d %H:%M:%S')
end_t = datetime.datetime.strptime(cow_data[1], '%Y-%m-%d %H:%M:%S')
com_creater.visualize_adjectory(start_t, end_t, [target_cow_id, opponent_cow_id], target_cow_id=target_cow_id, delta=delta) # 軌跡をプロット
print("軌跡動画と画像を作成しました: ", target_cow_id, " -> ", start_t.strftime('%Y/%m/%d %H:%M:%S'))
e = time.time()
print("処理時間", (e-s)/60, "[min]")
return
def visualize_adjectory(target_cow_id, detected_cow_list, start, end, delta=5):
s = time.time()
cows_record_file = os.path.abspath(
'../') + "/CowTagOutput/csv/" # 分析用のファイル
tmp_time = start - datetime.timedelta(
hours=9) # 時差の分巻き戻しておく（翌朝になっている可能性もあるので）
date = datetime.datetime(tmp_time.year, tmp_time.month, tmp_time.day, 0, 0,
0)
cow_id_list = my_utility.get_existing_cow_list(date, cows_record_file)
com_creater = community_creater.CommunityCreater(date, cow_id_list)
com_creater.visualize_adjectory(start,
end, [target_cow_id] + detected_cow_list,
target_cow_id=target_cow_id,
dirname='test/',
delta=delta) # 軌跡をプロット
print("軌跡動画と画像を作成しました: ", target_cow_id, " -> ",
start.strftime('%Y/%m/%d %H:%M:%S'))
e = time.time()
print("処理時間", (e - s) / 60, "[min]")
return
if __name__ == "__main__":
os.chdir('../') # カレントディレクトリを一階層上へ
print(os.getcwd())
sys.path.append(os.path.join(os.path.dirname(__file__))) #パスの追加
import synchronization.functions.utility as my_utility
import synchronization.community_creater as community_creater
delta_c = 2 # コミュニティの抽出間隔 [minutes]
delta_s = 5 # データのスライス間隔 [seconds]
epsilon = 12 # コミュニティ決定のパラメータ
dzeta = 12 # コミュニティ決定のパラメータ
leng = 1 # コミュニティ決定のパラメータ
cows_record_file = os.path.abspath(
'../') + "/CowTagOutput/csv/" # 分析用のファイル
date = datetime.datetime(2018, 6, 17, 0, 0, 0)
start_t = datetime.datetime(2018, 6, 18, 4, 0, 0)
end_t = datetime.datetime(2018, 6, 18, 6, 50, 0)
target_cow_id = '20122'
cow_id_list = my_utility.get_existing_cow_list(date, cows_record_file)
com_creater = community_creater.CommunityCreater(date, cow_id_list)
# interaction_graph = com_creater.make_interaction_graph(start_t, end_t, method="position", delta=delta_s, epsilon=epsilon, dzeta=dzeta)
# community_list = com_creater.create_community(start_t, end_t, interaction_graph, delta=delta_s, leng=leng)
# community = [com for com in community_list if target_cow_id in com][0]
com_creater.visualize_adjectory(start_t,
end_t, ['20122', '20192'],
target_cow_id=target_cow_id,
delta=delta_s) # 軌跡をプロット
# com_creater.visualize_position(start_t, end_t, community, target_cow_id=target_cow_id, delta=delta_s) # 位置情報とコミュニティをプロット
def detect_interaction():
""" 行動同期を検出する """
behavior_model = pickle.load(open('./synchronization/detection_model/model.pkl', 'rb'))
start = datetime.datetime(2018, 6, 17, 0, 0, 0)
end = datetime.datetime(2018, 6, 22, 0, 0, 0)
# target_list = ['20122', '20158', '20192', '20215']
target_list = ['20122', '20126', '20129', '20158' ,'20192', '20197', '20215', '20220', '20267', '20283'] # 2018/5/1 - 2018/7/31
# target_list = ['20113', '20118', '20126', '20170', '20255', '20295', '20299'] # 2018/9/10 - 2018/12/25
# target_list = ['20115', '20117', '20127', '20131', '20171', '20220', '20256', '20283', '20303'] # 2019/3/20 - 2019/7/3
date = start
while (date < end):
s1 = time.time()
interaction_graph_list = []
t_list = []
cow_id_list = my_utility.get_existing_cow_list(date, cows_record_file)
com_creater = community_creater.CommunityCreater(date, cow_id_list)
cow_id_list = com_creater.cow_id_list
# --- インタラクショングラフを作成する ---
t = date + datetime.timedelta(hours=12) # 正午12時を始まりとするが.......ときに9時始まりのときもある
t_start = date + datetime.timedelta(hours=12) # 正午12時を始まりとする
t_end = date + datetime.timedelta(days=1) + datetime.timedelta(hours=9) # 翌午前9時を終わりとする
while (t < t_end):
print(t.strftime("%Y/%m/%d %H:%M:%S"))
t_list.append(t)
interaction_graph = com_creater.make_interaction_graph(t, t+datetime.timedelta(minutes=delta_c), method="behavior", delta=delta_s, epsilon=epsilon, dzeta=dzeta) \
if (t_start <= t) else np.array([[]]) # 重み付きグラフを作成
interaction_graph_list.append(interaction_graph)
t += datetime.timedelta(minutes=delta_c)
e1 = time.time()
print("処理時間", (e1-s1)/60, "[min]")
# --- 対象牛のグラフ変化点を検知し，セッションを作る ---
s2 = time.time()
behavior_synch = com_creater.get_behavior_synch()
position_synch = com_creater.get_position_synch()
graph_analyzer = GraphSeriesAnalysis(cow_id_list, interaction_graph_list, "Poisson")
for cow_id in target_list:
if (cow_id in cow_id_list):
change_points, _ = graph_analyzer.detect_change_point(cow_id, 5, 5, threshold=400) # 変化点検知
session_times = get_change_time(t_list, change_points)
# --- 1つのセッションに対して，行動分岐点を探す ---
for i, t_idx in enumerate(session_times):
ses_start, ses_end = t_idx[0], t_idx[1]
interaction_graph = com_creater.make_interaction_graph(ses_start, ses_end, method="behavior", delta=delta_s, epsilon=epsilon, dzeta=dzeta)
community = com_creater.create_community(ses_start, ses_end, interaction_graph, delta=delta_s, leng=leng)
community = [com for com in community if str(cow_id) in com][0]
# density = calculate_graph_density(community, interaction_graph, cow_id_list)
behaviors = behavior_synch.extract_df(ses_start, ses_end, delta_s) [community]
positions = position_synch.extract_df(ses_start, ses_end, delta_s) [community]
# --- セッション内の行動分岐点を探索する ---
change_point_series = cut_point_search(behaviors[str(cow_id)].values.tolist())
b_segments, p_segments = cut_data(behaviors, positions, change_point_series)
score_dict = {}
for c in cow_id_list:
score_dict[c] = 0 # 牛のIDをキーにスコアを格納する
for b_seg, p_seg in zip(b_segments, p_segments):
theta = estimate_parameters(b_seg[str(cow_id)])
# 条件を満たしたセグメントは同期度をチェックする
pattern = np.argmax(behavior_model.predict([theta * len(b_seg[str(cow_id)])]))
if (not (pattern == 0 or pattern == 1)):
behavior_seg_df = behavior_synch.extract_df(b_seg[str(cow_id)].index[0], b_seg[str(cow_id)].index[-1], delta_s)
score_matrix = get_score_matrix(behavior_seg_df)
scores = score_synchro(b_seg, p_seg, cow_id, community, score_matrix)
for key in scores.keys():
score_dict[key] += scores[key]
try:
my_utility.write_values(detection_record_file + str(cow_id) + start.strftime('_%Y%m%d') + '.csv', [[ses_start, ses_end, score_dict, max(score_dict.values()), (ses_end - ses_start).total_seconds(), i]])
except:
print('error')
continue
e2 = time.time()
print("処理時間", (e2-s2)/60, "[min]")
date += datetime.timedelta(days=1)
return
def make_features():
""" 特徴を作成し各セッションを分析する """
global start, end
date = start
while (date < end):
s1 = time.time()
communities_list = []
community_graph_list = []
interaction_graph_list = []
t_list = []
cow_id_list = my_utility.get_existing_cow_list(date, cows_record_file)
com_creater = community_creater.CommunityCreater(date, cow_id_list)
cow_id_list = com_creater.cow_id_list
# --- 行動同期を計測する ---
t = date + datetime.timedelta(hours=12) # 正午12時を始まりとするが.......ときに9時始まりのときもある
t_start = date + datetime.timedelta(hours=12) # 正午12時を始まりとする
t_end = date + datetime.timedelta(days=1) + datetime.timedelta(hours=9) # 翌午前9時を終わりとする
while (t < t_end):
t_list.append(t)
interaction_graph = com_creater.make_interaction_graph(t, t+datetime.timedelta(minutes=delta_c), method="position", delta=delta_s, epsilon=epsilon, dzeta=dzeta) \
if (t_start <= t) else np.array([[]]) # 重み付きグラフを作成
community = com_creater.create_community(t, t+datetime.timedelta(minutes=delta_c), interaction_graph, delta=delta_s, leng=leng) \
if (t_start <= t) else [[]] # コミュニティを決定
com_creater.visualize_position(t, t+datetime.timedelta(minutes=delta_c), community, target_cow_id='20170', delta=delta_s) # 位置情報とコミュニティをプロット1
community_graph = com_creater.get_community_graph(community)
interaction_graph_list.append(interaction_graph)
communities_list.append(community)
community_graph_list.append(community_graph)
t += datetime.timedelta(minutes=delta_c)
e1 = time.time()
print("処理時間", (e1-s1)/60, "[min]")
# --- 変化点を検知し，セッションを作る ---
s2 = time.time()
behavior_synch = com_creater.get_behavior_synch()
position_synch = com_creater.get_position_synch()
graph_analyzer = GraphSeriesAnalysis(cow_id_list, interaction_graph_list, "Poisson")
for cow_id in target_list:
if (cow_id in cow_id_list):
# graph_analyzer.visualize_graph(cow_id, t_list) # グラフをまとめて可視化
change_points, score_list = graph_analyzer.detect_change_point(cow_id, 5, 5, threshold=400) # 変化点検知
# df = pd.concat([pd.Series(t_list), pd.Series(score_list), pd.Series(change_points)], axis=1, names=["time", "score", "change_flag"])
# df.to_csv("./synchronization/graph_operation/"+ str(cow_id) + ".csv") # csvで保存
community_list = make_session.get_focused_community(communities_list, cow_id) # セッションを作成するために対象牛の所属するコミュニティを抽出
inte_analyzer = interaction_analyzer.InteractionAnalyzer(cow_id, behavior_synch, position_synch) # 特徴量を作成するクラス
start_end_list = _get_start_end(t_list, change_points)
feature_list = []
for (start_point, end_point) in start_end_list:
try:
community_series, graph_series = _extract_community(t_list, community_list, interaction_graph_list, start_point, end_point)
features = inte_analyzer.extract_feature(start_point, end_point, community_series, delta_c=delta_c)
ave_dence = calculate_average_graph_density(community_series, graph_series, cow_id_list)
ave_iso = calculate_average_graph_isolation(community_series, graph_series, cow_id_list)
max_cow, same_time = look_for_cow_having_max_weight(community_series, graph_series, cow_id_list, cow_id)
# 開始時刻, 平均密度, 平均孤立度, 非休息割合, セッション長, 同じコミュニティになった回数, その牛
feature_list.append([start_point, end_point, ave_dence, ave_iso, 1 - features[2], features[0], same_time, max_cow])
except KeyError:
pdb.set_trace()
my_utility.write_values(change_point_file + str(cow_id) + ".csv", feature_list)
else:
my_utility.write_values(change_point_file + str(cow_id) + ".csv", [[t_start, t_end]])
e2 = time.time()
print("処理時間", (e2-s2)/60, "[min]")
date += datetime.timedelta(days=1)
return