def main():
argc = len(sys.argv)
if(argc == 2):
path_to_data = sys.argv[1]
else:
print("(!) ERROR: Invalid args, see usage.\nUsage: ./saveTrials.py path_to_data")
return
# trim '/' off path
end = len(path_to_data)-1
if(path_to_data[end] == '/'):
path_to_data = path_to_data[:end]
print("Please specify the moth and trial conditions you want to save:")
mid = input("'moth_id': ")
obs = input("'obstacle': ")
speed = float(input("'flight_speed': "))
fmin = float(input("'fog_min': "))
fmax = float(input("'fog_max': "))
# read moth data
dmoth = load_dataframe("h5","/media/usb/Input/moth_data.h5")
# extract trials
trajs = get_trajs(dmoth,obs,speed,fmin,fmax,mid)
if(len(trajs) == 0):
print("(!) ERROR: can't save trajs")
return
# save trials
verbose_name = mid+'_'+str(int(speed))+str(int(fmin))+str(int(fmax))
for ff in trajs:
tt = dmoth[trajs[ff][0]:trajs[ff][1]]
save_dataframe(tt,'h5',path_to_data+'/'+verbose_name+ff+".h5")
return
def main():
DATA_LOC = "../data/masks/original_set"
FOREST_LOC = "../data/forests/forest.csv"
import sys
# if a new forest path is specified, then save masks as a random set
if len(sys.argv) == 2:
FOREST_LOC = sys.argv[1]
DATA_LOC = DATA_LOC.replace("original_set", "random_sets")
DATA_LOC += "/" + FOREST_LOC.split("/")[-1]
# make sure directory exists for saving data
if not os.path.exists(DATA_LOC):
os.makedirs(DATA_LOC)
""" LOAD & INITIALIZE DATA """
# read tree data
forest = load_dataframe("csv", FOREST_LOC)
print("Forest size: " + str(len(forest.values)))
# terminate early if forest data is empty
if len(forest) == 0:
print("(!) generate_trail_masks.main: No tree data loaded.")
return
# prepare to log processed trials in Notes file and record datetime
if os.path.exists(DATA_LOC + "/trial_log"):
LOG = open(DATA_LOC + "/trial_log", "ab") # append to file
else:
LOG = open(DATA_LOC + "/trial_log", "wb")
import time, datetime
writeToFile(LOG, str(datetime.datetime.fromtimestamp(time.time())))
# gather single trial data stored in hdf format
single_trials = glob.glob("/home/bilkit/Dropbox/moth_nav_analysis/data/single_trials/*.h5")
if len(single_trials) < 1:
print("No trials to process.\n~~Done.")
return
single_trials.sort()
print("Computing masks for:")
# print file names that will be processed
for t in single_trials:
print("\t" + t + "\n")
# start processing trial data
processTrials(single_trials, forest, DATA_LOC, LOG)
# cleanup files
LOG.close()
print("~~Done :)")
return
def processTrials(batch_o_trials, forest, filepath_prefix, logfile):
"""
(array of str,pandas.dataframe,str,str) -> None
Loads a data frame from each file path in the list and transforms it
into an array of discretized frames. These frames are saved into a
dictionary of the corresponding mothid. The dictionary of trials is
saved into a pickle labeled with the trial conditions.
>>> import os
>>> dump = os.getcwd()+"/test"
>>> trials = [dump+"/moth1_448f0.h5"]
>>> trees = load_dataframe("csv",dump+"/trees.csv")
>>> directory_path = dump+"/throwaway_masks"
>>> logfile = open(dump+"throwaway_logfile.txt","wb+")
>>> processTrials(trials,trees,directory_path,logfile)
loading: /home/bilkit/Dropbox/moth_nav_analysis/scripts/test/moth1_448f0.h5
Processing points: 3454
t0
len: 3454
moth1: saving 1 trajs in /home/bilkit/Dropbox/moth_nav_analysis/scripts/test/throwaway_masks/moth1/4_4_8.pickle
>>> trials_string = trials[0]
>>> with open(dump+"fancy_logfile.txt","wb+") as fancy_logfile:
>>> ... processTrials([trials_string,],trees,directory_path,fancy_logfile)
loading: /home/bilkit/Dropbox/moth_nav_analysis/scripts/test/moth1_448f0.h5
Processing points: 3454
t0
len: 3454
moth1: saving 1 trajs in /home/bilkit/Dropbox/moth_nav_analysis/scripts/test/throwaway_masks/moth1/4_4_8.pickle
>>> trial_moth6 = "../data/single_trials/moth6_448f2.h5"
>>> processTrials([trial_moth6],trees,directory_path,logfile)
loading: ../data/single_trials/moth6_448f2.h5
Processing points: 3455
t0
len: 3455
moth6: saving 1 trajs in /home/bilkit/Dropbox/moth_nav_analysis/scripts/test/throwaway_masks/moth6/4_4_8.pickle
"""
# trial and current mothid
trial_cnt = 0
mothname = ""
# conditions of the experiment are embedded in the pickle file name
# i.e., desc = 'flightspeed_fogmin_fogmax'
filepath, desc = "", ""
# initialize trial dictionary
# key = trialid_datetime, val = [trial,block_size]
trial_hash = {}
""" PROCESS EACH TRIAL INTO STACK OF MASKS """
for st in batch_o_trials:
raw_data = load_dataframe("h5", st)
print("Processing points: " + str(len(raw_data.values)))
# skip processing if dataframe is empty
if len(raw_data) == 0:
print("(!) generate_trail_masks.processTrials: No moth data loaded for " + st)
continue
new_moth = raw_data.moth_id.iloc[0]
# get a description of conditions
desc = (
str(int(raw_data.flight_speed.iloc[0]))
+ "_"
+ str(int(raw_data.fog_min.iloc[0]))
+ "_"
+ str(int(raw_data.fog_max.iloc[0]))
)
# setup a directory (named after mothid) to store pickle files
filepath = filepath_prefix + "/" + new_moth
if not os.path.exists(filepath):
os.makedirs(filepath)
# We save a group of trials into a pickle file for each mothid
# This condition only occurs when the mothid changes (i.e., the
# previous set of trials is saved).
if mothname != "" and mothname != new_moth:
print(mothname + ": saving " + str(trial_cnt) + " trajs in " + filepath + "/" + desc + ".pickle")
with open(filepath + "/" + desc + ".pickle", "wb") as handle:
pickle.dump(trial_hash, handle)
# reset count and dictionary!
trial_cnt = 0
trial_hash = {}
# record the mothid, trial count, and trial length in a log file
# --(!) Logging must be done after we determine whether this set
# of trials is for a new moth.
if trial_cnt == 0:
writeToFile(logfile, str(new_moth) + ":")
# log trial length
writeToFile(logfile, "\tt" + str(trial_cnt) + " = " + str(len(raw_data.values)))
# process x,y,hx,hy slice of raw data into the dictionary of discretized frames
traj = raw_data[["pos_x", "pos_y", "head_x", "head_y"]]
trial_datetime = raw_data.datetime.iloc[0]
discretize_and_save(trial_hash, forest, traj, "t" + str(trial_cnt), trial_datetime)
# update the current mothid
mothname = new_moth
# update trial count
trial_cnt += 1
# save the last trial into trial dictionary
print(mothname + ": saving " + str(trial_cnt) + " trajs in " + filepath + "/" + desc + ".pickle")
with open(filepath + "/" + desc + ".pickle", "wb") as handle:
pickle.dump(trial_hash, handle)
return
