def load_System_masks_from_clickpoints(self, path): self.system_db = DataFileExtended(path) # self.System_Masks.extend([str(m.image.timestamp) for m in self.system_db.getMasks()]) timestamps = self.system_db.db.execute_sql( "select image.timestamp from image inner join mask on mask.image_id=image.id where mask.data is not null" ).fetchall() self.System_Masks.extend([str(t[0]) for t in timestamps])
def CreateDB(self, Folder): Files = self.MatchedFiles[Folder] db_path = os.path.sep.join(Folder.split(os.path.sep)[:-1] + [""]) db_name = self.name_from_path(Folder) db = DataFileExtended(db_path + db_name + ".cdb", "w") path = db.setPath(Folder) idx_dict = {} for file in Files: # self.Bar2.setValue(self.Bar2.value()+1) self.Bar2.increase() # QtGui.QApplication.processEvents() QtGui.QGuiApplication.processEvents() layer = self.layer_dict[[ k for k in self.layer_dict if file.count(k) ][0]] time = datetime.datetime.strptime( file.split("_")[0], "%Y%m%d-%H%M%S") idx = int([k[3:] for k in file.split("_") if k.count("rep")][0]) if len(idx_dict) < 1: idx_dict.update({idx: 0}) elif idx not in idx_dict: idx_dict.update({idx: max(idx_dict.values()) + 1}) image = db.setImage(filename=file, path=path, layer=layer, timestamp=time) # , frame=int(idx)) image.sort_index = idx #idx_dict[idx] image.save()
def Track(self, Folder): db_name = self.name_from_path(Folder) db_path = os.path.sep.join(Folder.split(os.path.sep)[:-1] + [""]) if not os.path.exists(db_path + db_name + ".cdb"): self.CreateDB(Folder, self.MatchedFiles[Folder]) db = DataFileExtended(db_path + db_name + ".cdb") print([i.sort_index for i in db.getImageIterator()]) TCell_Analysis.Track(db, progress_bar=self.Bar3)
def save_marker_to_System_db(self, markers, path, type): if function is None: function = lambda x : x self.system_db = DataFileExtended(path) with self.system_db.db.atomic() as transaction: self.system_db.deleteMarkers(type=type) for t in markers: self.system_db.setMarkers(image=self.system_db.getImage(frame=t),x=markers[t].T[0],y=markers[t].T[1], type=type)
def load_tracks(path, type): database = DataFileExtended(path) Tracks = {} if database.getTracks(type=type)[0].markers[0].measurement is not None: for track in database.getTracks(type=type): Tracks.update({track.id: Filter(VariableSpeed(dim=3))}) for m in track.markers: meas = Measurement(1., [m.measurement[0].x, m.measurement[0].y, m.measurement[0].z]) Tracks[track.id].update(z=meas, i=m.image.sort_index) else: for track in database.getTracks(type=type): Tracks.update({track.id: Filter(VariableSpeed(dim=3))}) for m in track.markers: meas = Measurement(1., [m.x, m.y, 0]) Tracks[track.id].update(z=meas, i=m.image.sort_index) return Tracks
def CreateDB(path, files, progress_bar = None): Folder = os.path.normpath(path) Files = [os.path.normpath(f) for f in files] db_path = os.path.sep.join(Folder.split(os.path.sep)[:-1]+[""]) db_name = name_from_path(Folder) # db_path = "/home/alex/2017-03-10_Tzellen_microwells_bestdata/T-CellMotility/2017-10-17/1_2Gel/24hnachMACS/24himGel/Kontrolle/RB/" db = DataFileExtended(db_path+db_name+".cdb", "w") path = db.setPath(Folder) idx_dict = {} for file in Files: if progress_bar is not None: progress_bar.increase() layer = LAYER_DICT[[k for k in LAYER_DICT if file.count(k)][0]] time = datetime.datetime.strptime(file.split("_")[0], "%Y%m%d-%H%M%S") idx = int([k[3:] for k in file.split("_") if k.count("rep")][0]) if len(idx_dict)<1: idx_dict.update({idx: 0}) elif idx not in idx_dict: idx_dict.update({idx: max(idx_dict.values())+1}) image = db.setImage(filename=file, path=path, layer=layer, timestamp=time) # , frame=int(idx)) image.sort_index = idx#idx_dict[idx] image.save() return db_path+db_name+".cdb"
def AnalyzeDB(db_str): db = DataFileExtended(db_str) time_step = 110 v_fac = 0.645 / (time_step / 60.) perc = 30 step = 20 type = 'PT_Track_Marker' Frame_list = [] for f in db.getImageIterator(): Frame_list.append(f.sort_index) Frames = np.max(Frame_list) timer() Tracks = TCell_Analysis.load_tracks(db_str, type) timer("Loading Tracks") Z_posis = TCell_Analysis.getZpos(Tracks, v_fac) timer("Fetching Z") Z_posis = np.asarray(Z_posis) tracks_to_delete = TCell_Analysis.get_Tracks_to_delete(Z_posis, perc) timer("Getting Tracks to delete") ### list2 = TCell_Analysis.create_list2(db) # Create List for true dist timer("Creating List2") drift, drift_list, missing_frame = TCell_Analysis.Drift( Frames, list2, 5) # Create List with offsets timer("Getting Drift") list2 = TCell_Analysis.list2_with_drift( db, drift, tracks_to_delete, del_track=True) # Create list for true dist with drift_cor timer("List2 with drift") list = TCell_Analysis.create_list( Frames, db, drift=drift, Drift=True, Missing=missing_frame) # Create List for analysis timer("Create List") ### For Deleting Tracks above and below list_copy = list[:] for l, m in enumerate(list): keys = m.keys() for k, j in enumerate(keys): if j in tracks_to_delete: del list_copy[l][j] timer("Stuff") ### print("bla") directions, velocities, dirt, alternative_vel, vel_mean, dir_mean, alt_vel_mean = TCell_Analysis.measure( step, time_step, list, Frames) # Calculate directions and velocities timer("Measure") motile_percentage, mean_v, mean_dire, number, len_count, mo_p_al, me_v_al, me_d_al = TCell_Analysis.values( directions, velocities, db, dirt, alternative_vel, tracks_to_delete, del_Tracks=True) timer("Values") motile_per_true_dist, real_dirt = TCell_Analysis.motiletruedist(list2) timer("Motile True Dist") if not os.path.exists(db_str[:-4] + "_analyzed.txt"): with open(db_str[:-4] + "_analyzed.txt", "w") as f: f.write( 'Day \t\t\tData \t\t\tMotile % true dist\t\t\tMotile in %\t\t\tMean velocity\t\t\tMean Directionality\t\t\tMean vel dt1\t\t\t#Tracks\t\t\t#Evaluated Tracks\t\t\t#Dirt\n' ) Day = getDateFromPath(db_str).strftime("%Y-%M-%d") if db_str.count('TCell') or db_str.count('T-Cell'): TCell_Analysis.Colorplot( directions, velocities, db_str, path=os.path.sep.join(db_str.split(os.path.sep)[:-1]), Save=True) # Save the velocity vs directionality picture with open(db_str[:-4] + "_analyzed.txt", "ab") as f: f.write( '%s\t\t\t%34s\t\t\t%18f\t\t\t%11f\t\t\t%13f\t\t\t%19f\t\t\t%12f\t\t\t%7d\t\t\t%17d\t\t\t%5d\n' % (Day, db_str, motile_per_true_dist, motile_percentage, mean_v, mean_dire, me_v_al, number, len_count, real_dirt)) elif db_str.count('NKCell'): TCell_Analysis.Colorplot(directions, velocities, db_str, path=os.path.sep.join( db_str.split(os.path.sep)[:-1]), Save=True) with open(db_str[:-4] + "_analyzed.txt", 'ab') as f: f.write( '%s\t\t\t%34s\t\t\t%18f\t\t\t%11f\t\t\t%13f\t\t\t%19f\t\t\t%12f\t\t\t%7d\t\t\t%17d\t\t\t%5d\n' % (Day, db_str, motile_per_true_dist, motile_percentage, mean_v, mean_dire, me_v_al, number, len_count, real_dirt)) db.db.close() timer("Write")
cov=Q) # Initialize Distributions for Filter Meas_Dist = ss.multivariate_normal( cov=R) # Initialize Distributions for Filter # Initialize Filter/Tracker MultiKal = MultiFilter(KalmanFilter, model, np.diag(Q), np.diag(R), meas_dist=Meas_Dist, state_dist=State_Dist) MultiKal.LogProbabilityThreshold = log_prob_threshold # Extended Clickpoints Database for usage with pengutack from PenguTrack.DataFileExtended import DataFileExtended # Open ClickPoints Database db = DataFileExtended("./ExampleData/sim_data.cdb", "w") # Define ClickPoints Marker detection_marker_type = db.setMarkerType(name="Detection_Marker", color="#FF0000", style='{"scale":1.2}') db.deleteMarkers(type=detection_marker_type) track_marker_type = db.setMarkerType(name="Track_Marker", color="#00FF00", mode=db.TYPE_Track) db.deleteMarkers(type=track_marker_type) prediction_marker_type = db.setMarkerType(name="Prediction_Marker", color="#0000FF") db.deleteMarkers(type=prediction_marker_type) # Delete Old Tracks db.deleteTracks(type=track_marker_type)
# from PenguTrack.Detectors import SimpleAreaDetector as AreaDetector from skimage.morphology import binary_closing, binary_dilation, binary_opening, binary_erosion from skimage.morphology import disk SELEM = disk(2,dtype=bool) import scipy.stats as ss # Load Database # file_path = "/home/birdflight/Desktop/PT_Test.cdb" file_path = "/mnt/mmap/Starter_Full.cdb" # file_path = "/mnt/mmap/PT_Test3.cdb" # file_path = "/mnt/mmap/PT_Test4.cdb" global db db = DataFileExtended(file_path,"w") db_start = DataFileExtended("/home/birdflight/Desktop/Starter.cdb") # images = db_start.getImageIterator(start_frame=2490-30, end_frame=2600) # images = db_start.getImageIterator(start_frame=1936-210, end_frame=2600) images = db_start.getImageIterator(start_frame=700) # images = db_start.getImageIterator(start_frame=1936-20-90, end_frame=2600) # images = db_start.getImageIterator(start_frame=1500, end_frame=2600) # images = db_start.getImageIterator() def getImage(): im = images.next() fname = im.filename from datetime import datetime d = datetime.strptime(fname[0:15], '%Y%m%d-%H%M%S') time_unix = np.uint32(time.mktime(d.timetuple()))
# Initialize Filter/Tracker # MultiKal = MultiFilter(KalmanFilter, model, np.diag(Q), # np.diag(R), meas_dist=Meas_Dist, state_dist=State_Dist) from PenguTrack.Filters import HungarianTracker MultiKal = HungarianTracker(KalmanFilter, model, np.diag(Q), np.diag(R), meas_dist=Meas_Dist, state_dist=State_Dist) MultiKal.LogProbabilityThreshold = log_prob_threshold # Extended Clickpoints Database for usage with pengutack from PenguTrack.DataFileExtended import DataFileExtended # Open ClickPoints Database db = DataFileExtended("./ExampleData/cell_data.cdb") # Define ClickPoints Marker detection_marker_type = db.setMarkerType(name="Detection_Marker", color="#FF0000", style='{"scale":1.2}') db.deleteMarkers(type=detection_marker_type) track_marker_type = db.setMarkerType(name="Track_Marker", color="#00FF00", mode=db.TYPE_Track) db.deleteMarkers(type=track_marker_type) prediction_marker_type = db.setMarkerType(name="Prediction_Marker", color="#0000FF") db.deleteMarkers(type=prediction_marker_type) # Delete Old Tracks db.deleteTracks(type=track_marker_type)
def load_GT_marker_from_clickpoints(self, path, type): self.gt_db = DataFileExtended(path) markers = np.asarray([[m.image.sort_index, m.x, m.y] for m in self.gt_db.getMarkers(type=type)]) self.GT_Markers.update(dict([[t, markers[markers.T[0]==t].T[1:]] for t in set(markers.T[0])]))
from skimage.morphology import binary_closing, binary_dilation, binary_opening, binary_erosion from skimage.morphology import disk SELEM = disk(2, dtype=bool) import scipy.stats as ss # Load Database # file_path = "/home/birdflight/Desktop/PT_Test.cdb" file_path = "/home/user/Desktop/PT_Test_full_n3_r7_A20.cdb" c = 20 # file_path = "/mnt/mmap/PT_Test3.cdb" # file_path = "/mnt/mmap/PT_Test4.cdb" global db db = DataFileExtended(file_path, "w") db_start = DataFileExtended(input_file) # images = db_start.getImageIterator(start_frame=2490-30, end_frame=2600) # images = db_start.getImageIterator(start_frame=1936-210, end_frame=2600) # images = db_start.getImageIterator(end_frame=52) # images = db_start.getImageIterator(start_frame=1936-20-90, end_frame=2600) # images = db_start.getImageIterator(start_frame=1500, end_frame=2600) images = db_start.getImageIterator() def getImage(): try: im = images.next() except StopIteration:
Save=True) # Save the velocity vs directionality picture with open(db_str[:-4] + "_analyzed.txt", "a") as f: f.write( '%s\t\t\t%34s\t\t\t%18f\t\t\t%11f\t\t\t%13f\t\t\t%19f\t\t\t%12f\t\t\t%7d\t\t\t%17d\t\t\t%5d\n' % ( Day, db_str, motile_per_true_dist, motile_percentage, mean_v, mean_dire, me_v_al, number, len_count, real_dirt)) elif db_str.count('NKCell'): Analysis_Tools.Colorplot(directions, velocities, db_str.split(os.path.sep)[-1][::-4], path=os.path.sep.join(db_str.split(os.path.sep)[:-1]), Save=True) with open(db_str[:-4] + "_analyzed.txt", 'a') as f: f.write('%s\t\t\t%34s\t\t\t%18f\t\t\t%11f\t\t\t%13f\t\t\t%19f\t\t\t%12f\t\t\t%7d\t\t\t%17d\t\t\t%5d\n' % ( Day, db_str, motile_per_true_dist, motile_percentage, mean_v, mean_dire, me_v_al, number, len_count, real_dirt)) db.db.close() if __name__ == "__main__": # AnalyzeDB("/home/alex/2017-03-10_Tzellen_microwells_bestdata/1T-Cell-Motility_2017-10-17_1_2Gel_24hnachMACS_24himGel_Kontrolle_RB_1.cdb") # AnalyzeDB(r"Z:\T-Cell-Motility\2017-10-17\1_2Gel\24hnachMACS\1himGel\Kontrolle\RB\4T-Cell-Motility_2017-10-17_1_2Gel_24hnachMACS_1himGel_Kontrolle_RB_4_stitched2.cdb") Matches, Matched_Files = Crawl_Folder("/mnt/cmark2/T-Cell-Motility/2017-10-17/1_2Gel/24hnachMACS/24himGel/Kontrolle/RB/") Database_paths =[] for m in Matches: Database_paths.append(CreateDB(m, Matched_Files[m])) for db_path in Database_paths: db = DataFileExtended(db_path) Track(db, None) for db_path in Database_paths: Stitch(db_path) for db_path in Database_paths: AnalyzeDB(db_path)
if __name__ == '__main__': import my_plot import numpy as np import clickpoints import matplotlib.pyplot as plt from PenguTrack.DataFileExtended import DataFileExtended # Load Data from Databases db = DataFileExtended( "/home/birdflight/Desktop/PT_Test_full_n3_r7_A20_filtered.cdb") db2 = DataFileExtended("/home/birdflight/Desktop/252_GT_Detections.cdb") LogType = db.getMarkerType(name="PT_Detection_Marker") ImgType = db.getMarkerType(name="PT_Track_Marker") # Positions_Log = np.asarray([[m.x, m.y, m.image.sort_index] for m in db.getMarkers(type=LogType) if not m.text.count("inf")]) Positions_Img = np.asarray([[m.x, m.y, m.image.sort_index] for m in db.getMarkers(type=ImgType) if m.track.markers.count() > 3]) # Do Correction for Position Data from CameraTransform import CameraTransform CT = CameraTransform(14, [17, 9], [4608, 2592], observer_height=31., angel_to_horizon=(np.pi / 2. - 0.24) * 180 / np.pi) orth_x, orth_y, orth_z = CT.transCamToWorld(Positions_Img.T[:2], Z=0.525) # Calculate Histogramms cx = cy = 2 times = np.asarray(sorted([i.timestamp for i in db.getImages()])) scale = 1. / (cx * cy) / ((times[-1] - times[0]).seconds / 3600.) hist, binx, biny = np.histogram2d(orth_x, orth_y,
import numpy as np import cv2 from PenguTrack.Detectors import FlowDetector, EmperorDetector, rgb2gray from PenguTrack.Filters import HungarianTracker, KalmanFilter from PenguTrack.Models import BallisticWSpeed from PenguTrack.DataFileExtended import DataFileExtended import matplotlib.pyplot as plt import scipy.stats as ss db = DataFileExtended( r"D:\User\Alex\Documents\Promotion\RotatingHuddles\DataBases\data1980.cdb") db.deletetOld() # FD = FlowDetector() object_size = 3 # Object diameter (smallest) q = 1. # Variability of object speed relative to object size r = 1. # Error of object detection relative to object size log_prob_threshold = -20. # Threshold for track stopping model = BallisticWSpeed(dim=2) # Set up Kalman filter X = np.zeros(model.State_dim).T # Initial Value for Position # Q = np.diag([q * object_size * np.ones(model.Evolution_dim)]) # Prediction uncertainty Q = np.diag([q * object_size * np.array([0.1, 1.0, 0.1, 1.0]) ]) # Prediction uncertainty # R = np.diag([r * object_size * np.ones(model.Meas_dim)]) # Measurement uncertainty R = np.diag([r * object_size * np.array([0.1, 1.0, 0.1, 1.0]) ]) # Measurement uncertainty
from PenguTrack.DataFileExtended import DataFileExtended import numpy as np db = DataFileExtended( "/home/alex/Masterarbeit/Data/Cells/DataBases/PT_Cell_T850_A75_inf_3d_backup.cdb" ) tracks = db.getTracks(type="PT_Track_Marker") # db = DataFileExtended("/home/alex/Desktop/PT_Cell_GT_Track.cdb") # tracks = db.getTracks(type="GroundTruth") V = [] vals = {} for track in tracks: if track.markers.count() < 4: continue x, y, z, t = np.array([[ m.measurement[0].x * 0.646, m.measurement[0].y * 0.645, m.measurement[0].z * 0.645, m.image.timestamp ] for m in track.markers]).T # x,y,z,t = np.array([[m.x*0.646, m.y*0.645, 0, m.image.timestamp] for m in track.markers]).T n = 10. # z = np.convolve(z, np.ones(n)/n, mode="same") if len(z)>n else z t = (np.array([tt.total_seconds() for tt in t[1:] - t[:-1]], dtype=float)) vx = ((x[1:] - x[:-1]) / t).astype(float) vy = ((y[1:] - y[:-1]) / t).astype(float) vz = ((z[1:] - z[:-1]) / t).astype(float) # vz = np.convolve(vz, np.ones(10)/10., mode="same") if len(vz)>10 else vz # v = (vx**2+vy**2+vz**2)**0.5 v = (vx**2 + vy**2)**0.5 print(np.nanmean(v), np.mean(np.abs(vz)), np.mean(vx), np.mean(vy))
def AnalyzeDB(db_str): db = DataFileExtended(db_str) time_step = 110 v_fac = 0.645 / (time_step / 60.) perc = 30 step = 20 type = 'PT_Track_Marker' Frame_list = [f.sort_index for f in db.getImages()] Frames = np.amax(Frame_list) timer() Tracks = db.PT_tracks_from_db(type) timer("Normal Tracks") Tracks_woMeasurements = db.PT_tracks_from_db(type, get_measurements=False) timer("Tracks WO") Z_posis = Z_from_PT_Track(Tracks, v_fac) timer("Z_posis") tracks_to_delete = deletable_tracks(Z_posis, perc) timer("Trackstodelete") list2 = getXY(Tracks_woMeasurements) # Create List for true dist timer("GETXY") drift, drift_list, missing_frame = TCell_Analysis.Drift( Frames, list2, 5) # Create List with offsets timer("Drift") list2 = TCell_Analysis.list2_with_drift( db, drift, tracks_to_delete, del_track=True) # Create list for true dist with drift_cor list2 = getXY_drift_corrected(Tracks_woMeasurements, np.vstack([[0, 0], drift])) timer("CorrectXY") list1 = analysis_list_from_tracks( Frames, Tracks_woMeasurements, drift=drift, Drift=True, Missing=missing_frame) # Create List for analysis ### For Deleting Tracks above and below list_copy = list1[:] for l, m in enumerate(list1): keys = m.keys() for k, j in enumerate(keys): if j in tracks_to_delete: del list_copy[l][j] directions, velocities, dirt, alternative_vel, vel_mean, dir_mean, alt_vel_mean = TCell_Analysis.measure( step, time_step, list1, Frames) # Calculate directions and velocities timer("Measure") motile_percentage, mean_v, mean_dire, number, len_count, mo_p_al, me_v_al, me_d_al = TCell_Analysis.values( directions, velocities, db, dirt, alternative_vel, tracks_to_delete, del_Tracks=True) timer("Values") motile_per_true_dist, real_dirt = TCell_Analysis.motiletruedist(list2) timer("Motile True Dist")
def load_neg_System_marker_from_clickpoints(self, path, type): self.system_db = DataFileExtended(path) markers = np.asarray([[self.gt_db.getImage(filename=m.image.filename).sort_index, m.x, m.y] for m in self.system_db.getMarkers(type=type) if self.gt_db.getImage(filename=m.image.filename) is not None]) self.neg_System_Markers.update(dict([[t, markers[markers.T[0]==t].T[1:]] for t in set(markers.T[0])]))
import numpy as np import seaborn as sn import matplotlib.pyplot as plt from PenguTrack.DataFileExtended import DataFileExtended file_path = "/home/user/Desktop/Birdflight.cdb" db = DataFileExtended(file_path) PT_Track_Type = db.getMarkerType(name="PT_Track_Marker") PT_Detection_Type = db.getMarkerType(name="PT_Detection_Marker") PT_Prediction_Type = db.getMarkerType(name="PT_Prediction_Marker") GT_Type = db.getMarkerType(name="GT") Tracks = db.getTracks(type=PT_Track_Type) mean_phi = [] phies = [] std_phi = [] mean_len = [] lens = [] for track in Tracks: if track.markers.count() < 3: continue x = np.asarray([[m.x, m.y] for m in track.markers]) del_x = x[1:] - x[:-1] del_x_norm = np.linalg.norm(del_x, axis=1) try: # phi = np.tensordot(del_x[1:],del_x[:-1].T, axes=1)/del_x_norm[1:]/del_x_norm[:-1] phi = np.diag(np.tensordot(del_x[1:], del_x[:-1].T, axes=1)) / del_x_norm[1:] / del_x_norm[:-1]
def load_System_marker_from_clickpoints(self, path, type): self.system_db = DataFileExtended(path) image_filenames = set([img.filename for img in self.gt_db.getImages()]).intersection([img.filename for img in self.system_db.getImages()]) markers = np.asarray([[self.gt_db.getImage(filename=m.image.filename).sort_index, m.x, m.y] for m in self.system_db.getMarkers(type=type) if m.image.filename in image_filenames]) self.System_Markers.update(dict([[t, markers[markers.T[0]==t].T[1:]] for t in set(markers.T[0])]))