def generate_Data_5_7(path, sum=7500, invalid_value=80): trainbundles = [] testbundle = None print("generiere Samples von 2004-2017 ...") for year in range(2004, 2018): sb = sample_bundle.load_Sample_Bundle(path.format(year)) sys.stdout.write(" clear") sys.stdout.flush() b = sb.clear_by_sum(sum) # 2017 hat anderen Grenzwert #plt.hist(b,bins=100) #plt.show() #sb.replace_borders(invalid_value) #entferne Radarfreie Zonen sys.stdout.write(" normalize") sys.stdout.flush() sb.normalize() # normalisieren zwischen [0;1] sys.stdout.write("\r|{}{}|".format('##' * (year - 2003), ' ' * (13 - (year - 2003)))) sys.stdout.flush() trainbundles.append(sb) print("\tfinished") return trainbundles, testbundle
def default_values_UNet64_output_2016(): learningcurve = [ 0.0115894, 0.00958822, 0.00796455, 0.00732153, 0.00727009, 0.00711318, 0.00692073, 0.00675846, 0.00665379, 0.00645629, 0.00637198, 0.00650981, 0.0064181, 0.00635487, 0.00624361, 0.00618663, 0.00612047, 0.00611733, 0.00620965, 0.00614361, 0.00616507, 0.00612534, 0.00606998, 0.00603275, 0.00605507, 0.00602762, 0.00608261, 0.00606057, 0.00606904, 0.00604447, 0.00601669, 0.00601813, 0.00598835, 0.00599038, 0.00603436, 0.00598364, 0.00598707, 0.00600294, 0.00601596, 0.00600256, 0.0059786, 0.00595138, 0.00595035, 0.00595839, 0.00597264, 0.00595936, 0.00598617, 0.00596159, 0.00599627, 0.00592671, 0.00591463, 0.00597174, 0.00601303, 0.00591218, 0.00599138, 0.00586511, 0.00585967, 0.0058672, 0.00586569, 0.00587476, 0.00590325, 0.00586857, 0.00587302, 0.00583622, 0.00586059, 0.00585119, 0.00583102, 0.00580262, 0.00583791, 0.00588642, 0.00589478, 0.00580198, 0.00578195, 0.00577411, 0.00583903, 0.00592595, 0.00573809, 0.00575651, 0.00580867, 0.0057976, 0.00574949, 0.00574277, 0.00572543, 0.0057714, 0.00575325, 0.00593839, 0.00577013, 0.00578963, 0.00575294, 0.005784, 0.00582843, 0.0057558, 0.00588208, 0.00589379, 0.00601915, 0.00569832, 0.00571483, 0.00572478, 0.00578767, 0.00569442 ] prediction_loss = [ 1.86470236e-06, 2.8264612e-06, 2.57799939e-06, 2.24289355e-05, 0.00015157, 0.00026531, 0.00033583, 0.00037115, 0.0004648, 0.0007185, 0.00069467, 0.00075835, 0.00068323, 0.00084506, 0.00081046, 0.00067252, 0.00044094, 0.00037288, 0.00024517, 0.00016385, 0.00019954, 0.00014589, 9.01579817e-05, 2.38962804e-09, 8.98763642e-07, 1.22362092e-05, 3.10096528e-05, 3.37138407e-06, 2.8856611e-06, 5.61989257e-05, 1.71629722e-05, 5.08496547e-09, 4.94259982e-06, 3.37298162e-05, 8.39688232e-05, 0.00017017, 0.00014647, 0.00017797, 0.00022459, 0.00023736, 0.0002829, 0.00024595, 0.00019492, 5.99015221e-05, 3.99775451e-05, 4.63488214e-05, 9.46813496e-06, 2.8748552e-05, 7.44204943e-05, 5.18611243e-06 ] baseline_loss = [ 7.396494136870434e-07, 2.2677575932333715e-06, 3.161344194540561e-06, 3.26459474721261e-05, 0.0002704300929930796, 0.0005964502835447904, 0.0006755326977604766, 0.0008103178465013457, 0.001028285395040369, 0.0014358960495963091, 0.0013702099252691272, 0.0013215620194156094, 0.0014910693903787005, 0.0014552545895809305, 0.0016509913554882737, 0.001446596561418685, 0.0009419228782199154, 0.0006850129757785468, 0.0004638991013071895, 0.000308084630911188, 0.0003598450716070742, 0.000374457840734333, 0.00034752258746635915, 0.0, 4.8997080449827e-06, 1.4049584294502115e-05, 8.270181540753557e-05, 7.411512399077278e-06, 3.195135284505959e-06, 4.742391748366013e-05, 5.453881920415225e-05, 1.87728277585544e-06, 9.476523452518261e-06, 5.482041162053057e-05, 0.00013154871323529412, 0.0002496222907054979, 0.00020320459678969626, 0.00028636822376009233, 0.00028379259179161864, 0.0003579677888312187, 0.0004275173611111111, 0.00031476400302768166, 0.00024243229767397157, 8.324622741253364e-05, 5.39718798058439e-05, 8.502964604959632e-05, 1.7590139609765473e-05, 5.9633764657823906e-05, 0.00017642328070934256, 8.902074923106497e-06 ] statistical_evaluation_lr_baseline(baseline_loss, prediction_loss, learningcurve) samplebundle = sample_bundle.load_Sample_Bundle("RegenTage2016") samplebundle.normalize() blul = (([0, 63]), ([35, 35], "blueviolet")) purl = (([0, 63]), ([41, 41], "purple")) skyl = (([3, 32]), ([35, 17], "skyblue")) orl = (([32, 32]), ([0, 63], "orange")) net_model = load_model( "C:\\Users\\TopSecret!\\Documents\\aMSI1\\Teamprojekt\\DeepRain\\NetworkTypes\\UNet64_output3x3\\UNet64_outputexpansed_2016_100.h5", verbose=False) data, label = samplebundle.get_all_data_label(channels_Last=True) prediction = net_model.predict(np.expand_dims(data[14], axis=0)) plt.imshow(prediction.reshape((64, 64)).T, cmap="gray", vmax=0.85) plt.plot(blul[0], *blul[1]) plt.plot(purl[0], *purl[1]) plt.plot(skyl[0], *skyl[1]) plt.plot(orl[0], *orl[1]) plt.title("prediction") plt.show() return
def default_values_UNet64_2016(): learningcurve = [ 0.0125825, 0.01090752, 0.00996938, 0.00877155, 0.00785926, 0.00749076, 0.00738954, 0.00710216, 0.00695553, 0.00685954, 0.00685263, 0.00683551, 0.0067013, 0.00669573, 0.00663494, 0.0065361, 0.00656253, 0.00655892, 0.00641627, 0.00643257, 0.00642143, 0.00643228, 0.00638859, 0.00635681, 0.00639316, 0.00631695, 0.0062926, 0.00633417, 0.00622689, 0.00616801, 0.00625138, 0.00621941, 0.00615199, 0.00620083, 0.00616311, 0.00617193, 0.00621265, 0.00614376, 0.0061507, 0.00614031, 0.00610141, 0.00613152, 0.00613031, 0.00610272, 0.00609479, 0.0061553, 0.0061067, 0.00605778, 0.00606069, 0.00603255, 0.00605208, 0.00601227, 0.00600995, 0.00601041, 0.00601996, 0.0060082, 0.00602065, 0.00619004, 0.00603388, 0.00603091, 0.00607781, 0.00611581, 0.00606192, 0.00599001, 0.00598812, 0.00596377, 0.00595005, 0.00597897, 0.00595829, 0.00594891, 0.00595861, 0.00597969, 0.00591869, 0.00593133, 0.00592642, 0.00592538, 0.00590671, 0.00598633, 0.0060014, 0.00590481, 0.00590337, 0.00598106, 0.00590425, 0.00589903, 0.00587542, 0.00588902, 0.0059049, 0.00590722, 0.00590535, 0.00589348, 0.00586809, 0.00586199, 0.00587707, 0.00585575, 0.00584643, 0.00583368, 0.00580768, 0.00581735, 0.00581284, 0.00581762 ] prediction_loss = [ 2.05482041e-06, 2.16494586e-06, 2.3332747e-06, 2.07652893e-05, 0.00015295, 0.00028465, 0.00031767, 0.00037503, 0.00047629, 0.00070817, 0.00069556, 0.00071058, 0.00068717, 0.00085751, 0.00080836, 0.00068803, 0.0004593, 0.00038036, 0.00023775, 0.00016827, 0.00018715, 0.0001683, 9.01244588e-05, 1.11477099e-08, 6.33360328e-07, 1.02075804e-05, 3.32653464e-05, 3.32339511e-06, 2.88497277e-06, 5.62943246e-05, 1.4668768e-05, 5.08214763e-08, 5.38758653e-06, 3.22183944e-05, 8.69566071e-05, 0.00017487, 0.00013233, 0.00017856, 0.00021881, 0.0002324, 0.00027731, 0.00024965, 0.00020651, 5.31775661e-05, 4.04651739e-05, 4.03603105e-05, 1.04818476e-05, 2.91035737e-05, 7.37178008e-05, 5.13091154e-06 ] baseline_loss = [ 7.396494136870434e-07, 2.2677575932333715e-06, 3.161344194540561e-06, 3.26459474721261e-05, 0.0002704300929930796, 0.0005964502835447904, 0.0006755326977604766, 0.0008103178465013457, 0.001028285395040369, 0.0014358960495963091, 0.0013702099252691272, 0.0013215620194156094, 0.0014910693903787005, 0.0014552545895809305, 0.0016509913554882737, 0.001446596561418685, 0.0009419228782199154, 0.0006850129757785468, 0.0004638991013071895, 0.000308084630911188, 0.0003598450716070742, 0.000374457840734333, 0.00034752258746635915, 0.0, 4.8997080449827e-06, 1.4049584294502115e-05, 8.270181540753557e-05, 7.411512399077278e-06, 3.195135284505959e-06, 4.742391748366013e-05, 5.453881920415225e-05, 1.87728277585544e-06, 9.476523452518261e-06, 5.482041162053057e-05, 0.00013154871323529412, 0.0002496222907054979, 0.00020320459678969626, 0.00028636822376009233, 0.00028379259179161864, 0.0003579677888312187, 0.0004275173611111111, 0.00031476400302768166, 0.00024243229767397157, 8.324622741253364e-05, 5.39718798058439e-05, 8.502964604959632e-05, 1.7590139609765473e-05, 5.9633764657823906e-05, 0.00017642328070934256, 8.902074923106497e-06 ] learningcurve = [ 0.00876916, 0.00763574, 0.00775013, 0.00728725, 0.00706771, 0.00690446, 0.00679859, 0.00685016, 0.0083208, 0.00674477, 0.00669052, 0.0065954, 0.00673142, 0.00698619, 0.00660383, 0.00655915, 0.00652131, 0.00663586, 0.00654554, 0.00680054, 0.00727, 0.00677988, 0.00748705, 0.00646014, 0.00644975, 0.00646124, 0.00648091, 0.00649791, 0.0063754, 0.0064263, 0.00636619, 0.00636916, 0.00642287, 0.00659642, 0.00636527, 0.006555, 0.00633962, 0.00630069, 0.00631354, 0.0062959, 0.00634053, 0.00628647, 0.00631652, 0.00621621, 0.00622429, 0.00662466, 0.00626726, 0.00645528, 0.00620124, 0.00660065, 0.00620791, 0.00617543, 0.00616647, 0.00620594, 0.00616584, 0.00615409, 0.0061273, 0.0061772, 0.00614339, 0.0061334, 0.00616816, 0.00613709, 0.00609759, 0.00638806, 0.00613235, 0.00612941, 0.00612209, 0.00615991, 0.006145, 0.00615401, 0.00635393, 0.00611096, 0.00615352, 0.0060822, 0.00610717, 0.00606298, 0.00606257, 0.00604071, 0.00606661, 0.00603798 ] prediction_loss = [ 1.71875397e-06, 1.65552961e-06, 2.51839882e-06, 2.19947249e-05, 0.00014833, 0.00027089, 0.00031949, 0.00036634, 0.00046509, 0.00072178, 0.00068672, 0.00071651, 0.00068261, 0.00081278, 0.0008001, 0.00070923, 0.00047983, 0.00037889, 0.00024699, 0.00016343, 0.00020333, 0.00018502, 0.00010807, 4.36043915e-07, 6.65378881e-07, 1.43982297e-05, 3.11619439e-05, 3.33887501e-06, 2.96609177e-06, 5.79021047e-05, 1.96858796e-05, 1.84167894e-07, 4.46581796e-06, 2.94263092e-05, 8.62778783e-05, 0.00016357, 0.0001413, 0.00017918, 0.00021462, 0.00023868, 0.00028542, 0.00024291, 0.00019439, 5.60679626e-05, 4.105311e-05, 4.99470187e-05, 1.29807418e-05, 3.14061303e-05, 8.14463202e-05, 5.68513118e-06 ] baseline_loss = [ 7.396494136870434e-07, 2.2677575932333715e-06, 3.161344194540561e-06, 3.26459474721261e-05, 0.0002704300929930796, 0.0005964502835447904, 0.0006755326977604766, 0.0008103178465013457, 0.001028285395040369, 0.0014358960495963091, 0.0013702099252691272, 0.0013215620194156094, 0.0014910693903787005, 0.0014552545895809305, 0.0016509913554882737, 0.001446596561418685, 0.0009419228782199154, 0.0006850129757785468, 0.0004638991013071895, 0.000308084630911188, 0.0003598450716070742, 0.000374457840734333, 0.00034752258746635915, 0.0, 4.8997080449827e-06, 1.4049584294502115e-05, 8.270181540753557e-05, 7.411512399077278e-06, 3.195135284505959e-06, 4.742391748366013e-05, 5.453881920415225e-05, 1.87728277585544e-06, 9.476523452518261e-06, 5.482041162053057e-05, 0.00013154871323529412, 0.0002496222907054979, 0.00020320459678969626, 0.00028636822376009233, 0.00028379259179161864, 0.0003579677888312187, 0.0004275173611111111, 0.00031476400302768166, 0.00024243229767397157, 8.324622741253364e-05, 5.39718798058439e-05, 8.502964604959632e-05, 1.7590139609765473e-05, 5.9633764657823906e-05, 0.00017642328070934256, 8.902074923106497e-06 ] statistical_evaluation_lr_baseline(baseline_loss, prediction_loss, learningcurve) samplebundle = sample_bundle.load_Sample_Bundle("RegenTage2016") samplebundle.normalize() blul = (([0, 63]), ([35, 35], "blueviolet")) purl = (([0, 63]), ([41, 41], "purple")) skyl = (([3, 32]), ([35, 17], "skyblue")) orl = (([32, 32]), ([0, 63], "orange")) display_one_img( samplebundle, 14, [[], [], [], [], [skyl, purl, blul, orl], [skyl, purl, blul, orl]], vmax=0.85) # net_model = load_model("C:\\Users\\TopSecret!\\Documents\\aMSI1\\Teamprojekt\\DeepRain\\NetworkTypes\\UNet64\\UNet64_2016_100.h5", verbose=False) net_model = load_model( "C:\\Users\\TopSecret!\\Documents\\aMSI1\\Teamprojekt\\DeepRain\\NetworkTypes\\UNet64_sigmoid_tanh_2016_80.h5", verbose=False) data, label = samplebundle.get_all_data_label(channels_Last=True) prediction = net_model.predict(np.expand_dims(data[14], axis=0)) plt.imshow(prediction.reshape((64, 64)).T, cmap="gray", vmax=0.85) plt.plot(blul[0], *blul[1]) plt.plot(purl[0], *purl[1]) plt.plot(skyl[0], *skyl[1]) plt.plot(orl[0], *orl[1]) plt.title("prediction") plt.show() return
plt.legend() plt.savefig(logfilepath + '.png') plt.show() return learning_curve, learning_curve_traindata if __name__ == '__main__': loadAndEval = True val_curve, train_curve = eval_trainlogfile( "C:\\Users\\TopSecret!\\Documents\\aMSI1\\Teamprojekt\\DeepRain\\Final_Networks\\trainphase.log", plot=True) if loadAndEval: sb = sample_bundle.load_Sample_Bundle("RegenTage2016") sb.normalize() data, label = sb.get_all_data_label(channels_Last=True, flatten_output=True) test_samples = data[:50] test_labels = label[:50] train_s = data[50:] train_l = data[50:] lcd = data[896] lcl = label[896] path = "C:\\Users\\TopSecret!\\Documents\\aMSI1\\Teamprojekt\\DeepRain\\NetworkTypes\\UNet64\\UNet64_2016_" load_and_eval_network(plotit=True, testdata=test_samples, testlabel=test_labels, learncurvesample=lcd, learncurvelabel=lcl,
### vergleiche Skalierungsfaktoren von 2016 #print("beginne Auswertung:") #dataset = ["TestData2016", "TestData2016EDIT", "TestData2016MAL20", "RegenTage2017_5_7_kn_centered"] #for d in dataset: # print("\n",d) # sb = sample_bundle.load_Sample_Bundle(d) # #print(sb.info()) # all_m = quick_eval(sb, get_all_max=True) # plt.figure(d) # plt.title(d) # plt.hist(all_m, bins=25, log=True) #plt.show() ## Einzelbildauswertung: sb = sample_bundle.load_Sample_Bundle("samplebundles\\2008_5in_7out_64x64_without_border") print(sb.info()) b = sb.clear_by_sum(76100) dl = sb.get_item_at(1) plt.imshow(dl[0][:,:,0]) plt.show() sb.replace_borders(80) dl = sb.get_item_at(1) plt.imshow(dl[0][:,:,0]) plt.show() #plt.hist(b, bins=100) #plt.show() print(sb.info())
## UNet to predict 1 timestep (5min) #model = tfM.UNet64_sigmoid_tanh(input_shape) #sb = sample_bundle.load_Sample_Bundle("C:/Users/TopSecret!/Documents/aMSI1/Teamprojekt/DeepRain/Data/RegenTage2016") #print(sb.info()) #train_realdata(model, sb, n_epoch=80, savename="UNet64_sigmoid_tanh_2016", channelsLast=True, use_logfile=True, # load_last_state=True) ## UNet to predict 2 timesteps (10min) # zwei Zeitschritte: #model = tfM.UNet64x2(input_shape) #sb = sample_bundle.load_Sample_Bundle("C:/Users/TopSecret!/Documents/aMSI1/Teamprojekt/DeepRain/Data/RegenTage2016_5_2") #ein Zeitschritt model = tfM.UNet64(input_shape=(64, 64, 5), lossfunction=tfL.mean_squared_error_kopie) sb = sample_bundle.load_Sample_Bundle( "C:/Users/TopSecret!/Documents/aMSI1/Teamprojekt/DeepRain/Data/RegenTage2016" ) print(sb.info()) train_realdata(model, sb, n_epoch=10, savename="UNet64", channelsLast=True, use_logfile=True, load_last_state=True, n_testsamples=365, prediction_shape=(64, 64), PREDICTION_IMG_ID=6) # eval_trainingsphase(model, n_epoch=100, diffToLabel=DIFF_TO_LABEL, n_train=1000, # savename="twoUPSamplings", channelsLast=True, n_inputs=N_INPUTS, use_logfile=True, load_last_state=True)