def main(): argc = len(sys.argv) if(argc == 2): path_to_data = sys.argv[1] else: print("(!) ERROR: Invalid args, see usage.\nUsage: ./saveTrials.py path_to_data") return # trim '/' off path end = len(path_to_data)-1 if(path_to_data[end] == '/'): path_to_data = path_to_data[:end] print("Please specify the moth and trial conditions you want to save:") mid = input("'moth_id': ") obs = input("'obstacle': ") speed = float(input("'flight_speed': ")) fmin = float(input("'fog_min': ")) fmax = float(input("'fog_max': ")) # read moth data dmoth = load_dataframe("h5","/media/usb/Input/moth_data.h5") # extract trials trajs = get_trajs(dmoth,obs,speed,fmin,fmax,mid) if(len(trajs) == 0): print("(!) ERROR: can't save trajs") return # save trials verbose_name = mid+'_'+str(int(speed))+str(int(fmin))+str(int(fmax)) for ff in trajs: tt = dmoth[trajs[ff][0]:trajs[ff][1]] save_dataframe(tt,'h5',path_to_data+'/'+verbose_name+ff+".h5") return
def main(): DATA_LOC = "../data/masks/original_set" FOREST_LOC = "../data/forests/forest.csv" import sys # if a new forest path is specified, then save masks as a random set if len(sys.argv) == 2: FOREST_LOC = sys.argv[1] DATA_LOC = DATA_LOC.replace("original_set", "random_sets") DATA_LOC += "/" + FOREST_LOC.split("/")[-1] # make sure directory exists for saving data if not os.path.exists(DATA_LOC): os.makedirs(DATA_LOC) """ LOAD & INITIALIZE DATA """ # read tree data forest = load_dataframe("csv", FOREST_LOC) print("Forest size: " + str(len(forest.values))) # terminate early if forest data is empty if len(forest) == 0: print("(!) generate_trail_masks.main: No tree data loaded.") return # prepare to log processed trials in Notes file and record datetime if os.path.exists(DATA_LOC + "/trial_log"): LOG = open(DATA_LOC + "/trial_log", "ab") # append to file else: LOG = open(DATA_LOC + "/trial_log", "wb") import time, datetime writeToFile(LOG, str(datetime.datetime.fromtimestamp(time.time()))) # gather single trial data stored in hdf format single_trials = glob.glob("/home/bilkit/Dropbox/moth_nav_analysis/data/single_trials/*.h5") if len(single_trials) < 1: print("No trials to process.\n~~Done.") return single_trials.sort() print("Computing masks for:") # print file names that will be processed for t in single_trials: print("\t" + t + "\n") # start processing trial data processTrials(single_trials, forest, DATA_LOC, LOG) # cleanup files LOG.close() print("~~Done :)") return
def processTrials(batch_o_trials, forest, filepath_prefix, logfile): """ (array of str,pandas.dataframe,str,str) -> None Loads a data frame from each file path in the list and transforms it into an array of discretized frames. These frames are saved into a dictionary of the corresponding mothid. The dictionary of trials is saved into a pickle labeled with the trial conditions. >>> import os >>> dump = os.getcwd()+"/test" >>> trials = [dump+"/moth1_448f0.h5"] >>> trees = load_dataframe("csv",dump+"/trees.csv") >>> directory_path = dump+"/throwaway_masks" >>> logfile = open(dump+"throwaway_logfile.txt","wb+") >>> processTrials(trials,trees,directory_path,logfile) loading: /home/bilkit/Dropbox/moth_nav_analysis/scripts/test/moth1_448f0.h5 Processing points: 3454 t0 len: 3454 moth1: saving 1 trajs in /home/bilkit/Dropbox/moth_nav_analysis/scripts/test/throwaway_masks/moth1/4_4_8.pickle >>> trials_string = trials[0] >>> with open(dump+"fancy_logfile.txt","wb+") as fancy_logfile: >>> ... processTrials([trials_string,],trees,directory_path,fancy_logfile) loading: /home/bilkit/Dropbox/moth_nav_analysis/scripts/test/moth1_448f0.h5 Processing points: 3454 t0 len: 3454 moth1: saving 1 trajs in /home/bilkit/Dropbox/moth_nav_analysis/scripts/test/throwaway_masks/moth1/4_4_8.pickle >>> trial_moth6 = "../data/single_trials/moth6_448f2.h5" >>> processTrials([trial_moth6],trees,directory_path,logfile) loading: ../data/single_trials/moth6_448f2.h5 Processing points: 3455 t0 len: 3455 moth6: saving 1 trajs in /home/bilkit/Dropbox/moth_nav_analysis/scripts/test/throwaway_masks/moth6/4_4_8.pickle """ # trial and current mothid trial_cnt = 0 mothname = "" # conditions of the experiment are embedded in the pickle file name # i.e., desc = 'flightspeed_fogmin_fogmax' filepath, desc = "", "" # initialize trial dictionary # key = trialid_datetime, val = [trial,block_size] trial_hash = {} """ PROCESS EACH TRIAL INTO STACK OF MASKS """ for st in batch_o_trials: raw_data = load_dataframe("h5", st) print("Processing points: " + str(len(raw_data.values))) # skip processing if dataframe is empty if len(raw_data) == 0: print("(!) generate_trail_masks.processTrials: No moth data loaded for " + st) continue new_moth = raw_data.moth_id.iloc[0] # get a description of conditions desc = ( str(int(raw_data.flight_speed.iloc[0])) + "_" + str(int(raw_data.fog_min.iloc[0])) + "_" + str(int(raw_data.fog_max.iloc[0])) ) # setup a directory (named after mothid) to store pickle files filepath = filepath_prefix + "/" + new_moth if not os.path.exists(filepath): os.makedirs(filepath) # We save a group of trials into a pickle file for each mothid # This condition only occurs when the mothid changes (i.e., the # previous set of trials is saved). if mothname != "" and mothname != new_moth: print(mothname + ": saving " + str(trial_cnt) + " trajs in " + filepath + "/" + desc + ".pickle") with open(filepath + "/" + desc + ".pickle", "wb") as handle: pickle.dump(trial_hash, handle) # reset count and dictionary! trial_cnt = 0 trial_hash = {} # record the mothid, trial count, and trial length in a log file # --(!) Logging must be done after we determine whether this set # of trials is for a new moth. if trial_cnt == 0: writeToFile(logfile, str(new_moth) + ":") # log trial length writeToFile(logfile, "\tt" + str(trial_cnt) + " = " + str(len(raw_data.values))) # process x,y,hx,hy slice of raw data into the dictionary of discretized frames traj = raw_data[["pos_x", "pos_y", "head_x", "head_y"]] trial_datetime = raw_data.datetime.iloc[0] discretize_and_save(trial_hash, forest, traj, "t" + str(trial_cnt), trial_datetime) # update the current mothid mothname = new_moth # update trial count trial_cnt += 1 # save the last trial into trial dictionary print(mothname + ": saving " + str(trial_cnt) + " trajs in " + filepath + "/" + desc + ".pickle") with open(filepath + "/" + desc + ".pickle", "wb") as handle: pickle.dump(trial_hash, handle) return