def complete_configuration(self): self.data = {} dir = './' #dir = '/export/karoly2/rhuva/phd/ACCESS/muriel/access_2month_optim/' file = 'CoV_wind_station_output_prox_penalty.nc' #file with _II has smaller exclusion zone infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['CoV_wind'][:,:] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_wind'] = numpy.array(temp, dtype=float) else: self.data['ts_wind'] = temp file = 'CoV_dsr_station_output_prox_penalty.nc' infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['CoV_dsr'][:,:] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_solar'] = numpy.array(temp, dtype=float) else: self.data['ts_solar'] = temp file = 'Aus_demand_sample_raw.nc' infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['ts_demand'][:] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_demand'] = numpy.array(temp, dtype=float) else: self.data['ts_demand'] = temp wind_nan = numpy.isnan(self.data['ts_wind']) solar_nan = numpy.isnan(self.data['ts_solar']) demand_nan = numpy.isnan(self.data['ts_demand']) wind_row = wind_nan.any(1) solar_row = solar_nan.any(1) combo = numpy.array([wind_row, solar_row, demand_nan]) combo_flat = combo.any(0) self.data['ts_wind'] = self.data['ts_wind'][combo_flat == False, :] self.data['ts_solar'] = self.data['ts_solar'][combo_flat == False, :] self.data['ts_demand'] = self.data['ts_demand'][combo_flat == False] print self.data['ts_wind'].shape print self.data['ts_solar'].shape print self.data['ts_demand'].shape self.ts_length = self.data['ts_wind'].shape[0] return None
def set_data(self, data): """Set the data dict with the data series required for the generator. Inputs: data: dict - with keys matching those requested by get_data_types. """ VariableGeneratorBasic.set_data(self, data) self.distances = data[self.config['data_type'] + '_distances'] mureiltypes.check_ndarray_float(self.distances)
def calc_cost(self, gene, save_result=False): """Calculate the total system cost for this gene. This function is called by the algorithm from a callback. The algorithm may set up multi-processing and so this calc_cost function (and all functions it calls) must be thread-safe when save_result=False. This means that the function must not modify any of the internal data of the objects. """ params = np.array(gene) if self.config['optim_type'] == 'match_demand': rem_demand = np.array(self.data.get_timeseries('ts_demand'), dtype=float) mureiltypes.check_ndarray_float(rem_demand) (solar_cost, solar_ts) = self.gen_list['solar'].calculate_cost_and_output( params[self.gen_params['solar'][0]:self. gen_params['solar'][1]], rem_demand, save_result) rem_demand -= solar_ts (wind_cost, wind_ts) = self.gen_list['wind'].calculate_cost_and_output( params[self.gen_params['wind'][0]:self.gen_params['wind'][1]], rem_demand, save_result) rem_demand -= wind_ts cost = abs(rem_demand).sum() / 1000.0 #now in GW elif self.config['optim_type'] == 'missed_supply': # rem_demand is the running total, modified here if 'demand' in self.dispatch_order: rem_demand = np.zeros(self.data.get_ts_length(), dtype=float) else: rem_demand = np.array(self.data.get_timeseries('ts_demand'), dtype=float) cost = 0 for gen_type in self.dispatch_order: gen = self.gen_list[gen_type] gen_ptr = self.gen_params[gen_type] (this_cost, this_ts) = gen.calculate_cost_and_output( params[gen_ptr[0]:gen_ptr[1]], rem_demand, save_result) cost += this_cost rem_demand -= this_ts return cost
def set_data(self, data): """Set the data dict with the data series required for the generator. Inputs: data: dict - with keys matching those requested by get_data_types. """ self.ts_cap_fac = data[self.config['data_type']] # For best speed performance, require numpy.array with dtype=float64. # This should have been converted in the Data module. mureiltypes.check_ndarray_float(self.ts_cap_fac)
def calc_cost(self, gene, save_result=False): """Calculate the total system cost for this gene. This function is called by the algorithm from a callback. The algorithm may set up multi-processing and so this calc_cost function (and all functions it calls) must be thread-safe when save_result=False. This means that the function must not modify any of the internal data of the objects. """ params = np.array(gene) if self.config["optim_type"] == "match_demand": rem_demand = np.array(self.data.get_timeseries("ts_demand"), dtype=float) mureiltypes.check_ndarray_float(rem_demand) (solar_cost, solar_ts) = self.gen_list["solar"].calculate_cost_and_output( params[self.gen_params["solar"][0] : self.gen_params["solar"][1]], rem_demand, save_result ) rem_demand -= solar_ts (wind_cost, wind_ts) = self.gen_list["wind"].calculate_cost_and_output( params[self.gen_params["wind"][0] : self.gen_params["wind"][1]], rem_demand, save_result ) rem_demand -= wind_ts cost = abs(rem_demand).sum() / 1000.0 # now in GW elif self.config["optim_type"] == "missed_supply": # rem_demand is the running total, modified here if "demand" in self.dispatch_order: rem_demand = np.zeros(self.data.get_ts_length(), dtype=float) else: rem_demand = np.array(self.data.get_timeseries("ts_demand"), dtype=float) cost = 0 for gen_type in self.dispatch_order: gen = self.gen_list[gen_type] gen_ptr = self.gen_params[gen_type] (this_cost, this_ts) = gen.calculate_cost_and_output( params[gen_ptr[0] : gen_ptr[1]], rem_demand, save_result ) cost += this_cost rem_demand -= this_ts return cost
def set_data(self, data): """The demand timeseries is supplied for use in forecasting the required demand. """ self.ts_demand = data['ts_demand'] mureiltypes.check_ndarray_float(self.ts_demand)
def complete_configuration(self): self.data = {} dir = "/export/karoly2/Roger/ACCESS-A/" file = "RegGrid_wind_output_11point_gap.nc" # file with _II has smaller exclusion zone infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables["wind_power"][:, :] if not mureiltypes.check_ndarray_float(temp, True): self.data["ts_wind"] = numpy.array(temp, dtype=float) else: self.data["ts_wind"] = temp file = "RegGrid_dsr_output_19point_gap.nc" infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables["dsr"][:, :] if not mureiltypes.check_ndarray_float(temp, True): self.data["ts_solar"] = numpy.array(temp, dtype=float) else: self.data["ts_solar"] = temp dir = "/export/karoly2/rhuva/phd/ACCESS/muriel/access_2month_optim/" file = "Aus_demand_2010_2011.nc" infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables["ts_demand"][:] if not mureiltypes.check_ndarray_float(temp, True): self.data["ts_demand"] = numpy.array(temp, dtype=float) else: self.data["ts_demand"] = temp wind_nan = numpy.isnan(self.data["ts_wind"]) solar_nan = numpy.isnan(self.data["ts_solar"]) demand_nan = numpy.isnan(self.data["ts_demand"]) wind_row = wind_nan.any(1) solar_row = solar_nan.any(1) combo = numpy.array([wind_row, solar_row, demand_nan]) combo_flat = combo.any(0) self.data["ts_wind"] = self.data["ts_wind"][combo_flat == False, :] self.data["ts_solar"] = self.data["ts_solar"][combo_flat == False, :] self.data["ts_demand"] = self.data["ts_demand"][combo_flat == False] print self.data["ts_wind"].shape print self.data["ts_solar"].shape print self.data["ts_demand"].shape self.ts_length = self.data["ts_wind"].shape[0] self.data["ts_solar_distances"] = numpy.array( [ 668730.4375, 909338.1875, 372287.25, 1125613.25, 211878.890625, 68551.4609375, 768108.3125, 513450.8125, 1239141.625, 242710.109375, 221302.75, 835822.375, 641565.125, 1371163.875, 404052.09375, 449818.1875, 949940.75, 1176451.625, 1239225.875, 811838.8125, 610120.125, 679184.1875, 1126859.125, 926546.0, 987652.1875, 1093136.0, 923490.5625, 605613.3125, 681002.75, 1170811.375, 1214910.625, 881211.1875, 1177105.25, 1020829.875, 374478.46875, 884468.125, 1147496.5, 1412441.0, 557196.4375, 111162.4375, 1129901.25, 901584.5625, 207429.328125, 830748.625, 1097925.0, 1311133.125, 474638.53125, 231650.921875, 1030239.5625, 784340.375, 599865.75, 355621.71875, 801379.6875, 1050643.0, 1298696.375, 535831.75, 1012697.625, 282052.0, 738578.625, 468747.75, 587530.1875, 874383.75, 1085080.5, 1165477.5, 739965.6875, 1043011.9375, 133451.609375, 466417.03125, 887445.1875, 843954.375, 882972.9375, 911230.0625, 256325.21875, 196988.21875, 722306.0625, 494064.375, 656777.1875, 590096.75, 838475.0, 576432.125, 1038951.0625, 709157.1875, 329522.5625, 246577.59375, 866532.3125, 684706.0625, 507050.0625, 353760.5, 528336.75, 308515.5, 294442.8125, 38012.9140625, 354666.53125, 473246.90625, 815155.4375, 729953.8125, 211116.015625, 548792.25, 134444.53125, 367146.65625, 252677.890625, 404498.71875, 615562.1875, 426158.125, 478903.125, 478492.6875, 204912.421875, 186742.734375, 449120.125, 467410.25, 242252.140625, 325181.5625, 223081.09375, 109695.5546875, 159941.5, ], dtype=float, ) self.data["ts_wind_distances"] = numpy.array( [ 668730.4375, 909338.1875, 1058640.875, 516586.90625, 754774.625, 909874.625, 372287.25, 1125613.25, 212450.59375, 211878.890625, 611953.6875, 68551.4609375, 768108.3125, 938604.9375, 475540.34375, 341444.6875, 1206778.375, 1072823.25, 209707.75, 161579.765625, 637232.6875, 111058.5234375, 353789.75, 811775.375, 946141.8125, 513450.8125, 383905.8125, 1239141.625, 1106075.25, 274558.0, 157525.8125, 242710.109375, 694877.25, 221302.75, 387798.59375, 835822.375, 992085.4375, 585522.375, 465096.75, 1306448.625, 1127716.375, 376003.46875, 262230.59375, 309341.46875, 734677.625, 341611.15625, 897978.625, 1027317.9375, 641565.125, 533193.8125, 1371163.875, 1242591.625, 464625.5, 374925.21875, 404052.09375, 811223.6875, 449818.1875, 949940.75, 1152356.75, 731487.6875, 632564.3125, 1328731.375, 1318661.25, 575718.375, 489012.5, 503957.4375, 877959.625, 567724.0, 1176451.625, 1085111.125, 1239225.875, 811838.8125, 723001.8125, 1228210.0, 1317506.875, 1061301.5, 678183.0, 604640.75, 610120.125, 1019876.0625, 679184.1875, 1064924.875, 1183351.125, 957665.0, 869637.5625, 1125113.625, 1221260.5, 1323786.875, 816667.5625, 719099.3125, 722332.8125, 908779.125, 1126859.125, 812075.8125, 926546.0, 1267433.125, 1071478.125, 987652.1875, 992131.5625, 1093136.0, 1200561.75, 944202.625, 841807.75, 831524.5625, 766421.25, 923490.5625, 797534.4375, 1268186.625, 1294879.625, 1179975.25, 1100428.25, 605613.3125, 871203.0625, 978584.5625, 1091826.375, 1060980.625, 956020.0625, 637515.875, 1057804.5, 681002.75, 1170811.375, 1295239.375, 1214910.625, 457551.96875, 766078.25, 881211.1875, 1001023.3125, 1178623.0, 1083117.625, 1063405.75, 511700.875, 1177105.25, 562428.6875, 660418.0625, 783159.8125, 1081792.375, 1020829.875, 374478.46875, 1181675.5, 444261.125, 884468.125, 1015654.0, 1147496.5, 1279800.0, 1412441.0, 557196.4375, 686802.875, 1046372.1875, 969600.625, 224758.90625, 1270513.0, 111162.4375, 1129901.25, 321961.25, 829833.5625, 963491.6875, 1097242.375, 1231062.0, 1364937.25, 450944.96875, 584710.375, 989115.9375, 901584.5625, 60845.32421875, 1226836.625, 130785.546875, 1098943.5, 207429.328125, 830748.625, 964274.5, 1097925.0, 1231665.75, 1311133.125, 345572.96875, 474638.53125, 605817.375, 953379.625, 853869.9375, 709148.6875, 144821.765625, 1161816.75, 231650.921875, 1030239.5625, 281000.03125, 757300.375, 887036.6875, 1017876.0625, 1149449.375, 1281537.875, 405575.3125, 519712.8125, 641537.125, 784340.375, 599865.75, 263454.5625, 1193936.0, 1058048.625, 938030.5, 648269.0625, 355621.71875, 801379.6875, 924709.125, 1050643.0, 1178353.875, 1298696.375, 448873.90625, 535831.75, 639195.5625, 778615.625, 486159.1875, 1150266.875, 432368.78125, 1012697.625, 887173.875, 635939.25, 370392.96875, 475697.8125, 802408.9375, 914610.6875, 1042862.375, 1164123.875, 1287900.625, 551048.75, 625359.1875, 713181.125, 282052.0, 738578.625, 468747.75, 1176506.0, 1038450.75, 907944.3125, 594211.4375, 331333.875, 587530.1875, 874383.75, 976254.0625, 1085080.5, 1127779.875, 1165477.5, 739965.6875, 196089.296875, 762503.6875, 446178.5625, 1156598.25, 1043011.9375, 608477.6875, 302244.0, 891794.3125, 974806.9375, 971028.9375, 981571.0, 1009794.625, 133451.609375, 114606.5859375, 466417.03125, 818209.125, 1002819.125, 1039978.75, 683042.9375, 329406.875, 887445.1875, 855617.0625, 843954.375, 853292.25, 882972.9375, 1000176.8125, 196278.171875, 64264.375, 88714.125, 552421.75, 869001.5625, 870218.0, 911230.0625, 429908.59375, 256325.21875, 740118.0625, 718609.1875, 721628.5625, 748889.0625, 978856.625, 310406.625, 196988.21875, 722306.0625, 161488.421875, 599724.625, 934020.25, 737569.25, 814771.6875, 494064.375, 251381.703125, 336539.0, 656777.1875, 609839.375, 590096.75, 600250.25, 893760.0625, 391393.0625, 297513.59375, 823148.3125, 838475.0, 704558.125, 686599.5625, 948955.6875, 964129.25, 294521.5625, 576432.125, 645428.0625, 1038951.0625, 709157.1875, 594892.125, 329522.5625, 258209.796875, 246577.59375, 537336.3125, 483168.28125, 462575.5625, 798845.6875, 503718.4375, 418395.09375, 866532.3125, 777922.0625, 670250.3125, 762942.75, 881406.8125, 884783.375, 413056.8125, 413987.5, 473308.65625, 535773.375, 684706.0625, 301884.875, 191727.3125, 135755.625, 507050.0625, 416187.03125, 353760.5, 629375.8125, 604659.375, 528336.75, 923294.9375, 236056.421875, 848792.3125, 723014.9375, 432805.5, 588751.75, 266921.25, 308515.5, 371523.1875, 745499.75, 294442.8125, 161392.625, 38012.9140625, 354666.53125, 442686.21875, 330582.15625, 241744.234375, 473246.90625, 653481.125, 585862.0, 815155.4375, 771210.125, 489359.5625, 579454.375, 453331.5, 139264.03125, 201134.484375, 298521.34375, 729953.8125, 327048.5625, 199195.953125, 91859.6015625, 115152.90625, 211116.015625, 421740.375, 632862.875, 548792.25, 697630.4375, 659870.3125, 412630.8125, 548571.125, 439476.25, 543691.0, 11564.77832031, 134444.53125, 268900.28125, 598547.3125, 367146.65625, 358178.9375, 252677.890625, 184504.65625, 404498.71875, 600163.5, 535114.375, 615562.1875, 543468.25, 426158.125, 545614.5625, 401627.4375, 567999.5625, 136595.25, 255334.890625, 478903.125, 288917.75, 254081.71875, 285744.96875, 301214.53125, 381815.09375, 484310.0625, 478492.6875, 505826.21875, 425950.34375, 526906.8125, 336393.5625, 360000.46875, 204912.421875, 142902.09375, 186742.734375, 292174.5625, 375563.59375, 433015.375, 449120.125, 467410.25, 341196.125, 534745.5625, 289708.96875, 242252.140625, 125803.328125, 32823.0625, 242620.25, 325181.5625, 379624.03125, 253196.859375, 344051.40625, 129340.453125, 223081.09375, 129551.390625, 239891.03125, 261754.0625, 356155.5, 109695.5546875, 32649.04492188, 159941.5, ], dtype=float, ) return None
def complete_configuration(self): self.data = {} dir = '/export/karoly2/Roger/ACCESS-A/' file = 'RegGrid_wind_output_11point_gap.nc' #file with _II has smaller exclusion zone infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['wind_power'][:, :] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_wind'] = numpy.array(temp, dtype=float) else: self.data['ts_wind'] = temp file = 'RegGrid_dsr_output_19point_gap.nc' infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['dsr'][:, :] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_solar'] = numpy.array(temp, dtype=float) else: self.data['ts_solar'] = temp dir = '/export/karoly2/rhuva/phd/ACCESS/muriel/access_2month_optim/' file = 'Aus_demand_2010_2011.nc' infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['ts_demand'][:] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_demand'] = numpy.array(temp, dtype=float) else: self.data['ts_demand'] = temp wind_nan = numpy.isnan(self.data['ts_wind']) solar_nan = numpy.isnan(self.data['ts_solar']) demand_nan = numpy.isnan(self.data['ts_demand']) wind_row = wind_nan.any(1) solar_row = solar_nan.any(1) combo = numpy.array([wind_row, solar_row, demand_nan]) combo_flat = combo.any(0) self.data['ts_wind'] = self.data['ts_wind'][combo_flat == False, :] self.data['ts_solar'] = self.data['ts_solar'][combo_flat == False, :] self.data['ts_demand'] = self.data['ts_demand'][combo_flat == False] print self.data['ts_wind'].shape print self.data['ts_solar'].shape print self.data['ts_demand'].shape self.ts_length = self.data['ts_wind'].shape[0] self.data['ts_solar_distances'] = numpy.array([ 668730.4375, 909338.1875, 372287.25, 1125613.25, 211878.890625, 68551.4609375, 768108.3125, 513450.8125, 1239141.625, 242710.109375, 221302.75, 835822.375, 641565.125, 1371163.875, 404052.09375, 449818.1875, 949940.75, 1176451.625, 1239225.875, 811838.8125, 610120.125, 679184.1875, 1126859.125, 926546., 987652.1875, 1093136., 923490.5625, 605613.3125, 681002.75, 1170811.375, 1214910.625, 881211.1875, 1177105.25, 1020829.875, 374478.46875, 884468.125, 1147496.5, 1412441., 557196.4375, 111162.4375, 1129901.25, 901584.5625, 207429.328125, 830748.625, 1097925., 1311133.125, 474638.53125, 231650.921875, 1030239.5625, 784340.375, 599865.75, 355621.71875, 801379.6875, 1050643., 1298696.375, 535831.75, 1012697.625, 282052., 738578.625, 468747.75, 587530.1875, 874383.75, 1085080.5, 1165477.5, 739965.6875, 1043011.9375, 133451.609375, 466417.03125, 887445.1875, 843954.375, 882972.9375, 911230.0625, 256325.21875, 196988.21875, 722306.0625, 494064.375, 656777.1875, 590096.75, 838475., 576432.125, 1038951.0625, 709157.1875, 329522.5625, 246577.59375, 866532.3125, 684706.0625, 507050.0625, 353760.5, 528336.75, 308515.5, 294442.8125, 38012.9140625, 354666.53125, 473246.90625, 815155.4375, 729953.8125, 211116.015625, 548792.25, 134444.53125, 367146.65625, 252677.890625, 404498.71875, 615562.1875, 426158.125, 478903.125, 478492.6875, 204912.421875, 186742.734375, 449120.125, 467410.25, 242252.140625, 325181.5625, 223081.09375, 109695.5546875, 159941.5 ], dtype=float) self.data['ts_wind_distances'] = numpy.array([ 668730.4375, 909338.1875, 1058640.875, 516586.90625, 754774.625, 909874.625, 372287.25, 1125613.25, 212450.59375, 211878.890625, 611953.6875, 68551.4609375, 768108.3125, 938604.9375, 475540.34375, 341444.6875, 1206778.375, 1072823.25, 209707.75, 161579.765625, 637232.6875, 111058.5234375, 353789.75, 811775.375, 946141.8125, 513450.8125, 383905.8125, 1239141.625, 1106075.25, 274558., 157525.8125, 242710.109375, 694877.25, 221302.75, 387798.59375, 835822.375, 992085.4375, 585522.375, 465096.75, 1306448.625, 1127716.375, 376003.46875, 262230.59375, 309341.46875, 734677.625, 341611.15625, 897978.625, 1027317.9375, 641565.125, 533193.8125, 1371163.875, 1242591.625, 464625.5, 374925.21875, 404052.09375, 811223.6875, 449818.1875, 949940.75, 1152356.75, 731487.6875, 632564.3125, 1328731.375, 1318661.25, 575718.375, 489012.5, 503957.4375, 877959.625, 567724., 1176451.625, 1085111.125, 1239225.875, 811838.8125, 723001.8125, 1228210., 1317506.875, 1061301.5, 678183., 604640.75, 610120.125, 1019876.0625, 679184.1875, 1064924.875, 1183351.125, 957665., 869637.5625, 1125113.625, 1221260.5, 1323786.875, 816667.5625, 719099.3125, 722332.8125, 908779.125, 1126859.125, 812075.8125, 926546., 1267433.125, 1071478.125, 987652.1875, 992131.5625, 1093136., 1200561.75, 944202.625, 841807.75, 831524.5625, 766421.25, 923490.5625, 797534.4375, 1268186.625, 1294879.625, 1179975.25, 1100428.25, 605613.3125, 871203.0625, 978584.5625, 1091826.375, 1060980.625, 956020.0625, 637515.875, 1057804.5, 681002.75, 1170811.375, 1295239.375, 1214910.625, 457551.96875, 766078.25, 881211.1875, 1001023.3125, 1178623., 1083117.625, 1063405.75, 511700.875, 1177105.25, 562428.6875, 660418.0625, 783159.8125, 1081792.375, 1020829.875, 374478.46875, 1181675.5, 444261.125, 884468.125, 1015654., 1147496.5, 1279800., 1412441., 557196.4375, 686802.875, 1046372.1875, 969600.625, 224758.90625, 1270513., 111162.4375, 1129901.25, 321961.25, 829833.5625, 963491.6875, 1097242.375, 1231062., 1364937.25, 450944.96875, 584710.375, 989115.9375, 901584.5625, 60845.32421875, 1226836.625, 130785.546875, 1098943.5, 207429.328125, 830748.625, 964274.5, 1097925., 1231665.75, 1311133.125, 345572.96875, 474638.53125, 605817.375, 953379.625, 853869.9375, 709148.6875, 144821.765625, 1161816.75, 231650.921875, 1030239.5625, 281000.03125, 757300.375, 887036.6875, 1017876.0625, 1149449.375, 1281537.875, 405575.3125, 519712.8125, 641537.125, 784340.375, 599865.75, 263454.5625, 1193936., 1058048.625, 938030.5, 648269.0625, 355621.71875, 801379.6875, 924709.125, 1050643., 1178353.875, 1298696.375, 448873.90625, 535831.75, 639195.5625, 778615.625, 486159.1875, 1150266.875, 432368.78125, 1012697.625, 887173.875, 635939.25, 370392.96875, 475697.8125, 802408.9375, 914610.6875, 1042862.375, 1164123.875, 1287900.625, 551048.75, 625359.1875, 713181.125, 282052., 738578.625, 468747.75, 1176506., 1038450.75, 907944.3125, 594211.4375, 331333.875, 587530.1875, 874383.75, 976254.0625, 1085080.5, 1127779.875, 1165477.5, 739965.6875, 196089.296875, 762503.6875, 446178.5625, 1156598.25, 1043011.9375, 608477.6875, 302244., 891794.3125, 974806.9375, 971028.9375, 981571., 1009794.625, 133451.609375, 114606.5859375, 466417.03125, 818209.125, 1002819.125, 1039978.75, 683042.9375, 329406.875, 887445.1875, 855617.0625, 843954.375, 853292.25, 882972.9375, 1000176.8125, 196278.171875, 64264.375, 88714.125, 552421.75, 869001.5625, 870218., 911230.0625, 429908.59375, 256325.21875, 740118.0625, 718609.1875, 721628.5625, 748889.0625, 978856.625, 310406.625, 196988.21875, 722306.0625, 161488.421875, 599724.625, 934020.25, 737569.25, 814771.6875, 494064.375, 251381.703125, 336539., 656777.1875, 609839.375, 590096.75, 600250.25, 893760.0625, 391393.0625, 297513.59375, 823148.3125, 838475., 704558.125, 686599.5625, 948955.6875, 964129.25, 294521.5625, 576432.125, 645428.0625, 1038951.0625, 709157.1875, 594892.125, 329522.5625, 258209.796875, 246577.59375, 537336.3125, 483168.28125, 462575.5625, 798845.6875, 503718.4375, 418395.09375, 866532.3125, 777922.0625, 670250.3125, 762942.75, 881406.8125, 884783.375, 413056.8125, 413987.5, 473308.65625, 535773.375, 684706.0625, 301884.875, 191727.3125, 135755.625, 507050.0625, 416187.03125, 353760.5, 629375.8125, 604659.375, 528336.75, 923294.9375, 236056.421875, 848792.3125, 723014.9375, 432805.5, 588751.75, 266921.25, 308515.5, 371523.1875, 745499.75, 294442.8125, 161392.625, 38012.9140625, 354666.53125, 442686.21875, 330582.15625, 241744.234375, 473246.90625, 653481.125, 585862., 815155.4375, 771210.125, 489359.5625, 579454.375, 453331.5, 139264.03125, 201134.484375, 298521.34375, 729953.8125, 327048.5625, 199195.953125, 91859.6015625, 115152.90625, 211116.015625, 421740.375, 632862.875, 548792.25, 697630.4375, 659870.3125, 412630.8125, 548571.125, 439476.25, 543691., 11564.77832031, 134444.53125, 268900.28125, 598547.3125, 367146.65625, 358178.9375, 252677.890625, 184504.65625, 404498.71875, 600163.5, 535114.375, 615562.1875, 543468.25, 426158.125, 545614.5625, 401627.4375, 567999.5625, 136595.25, 255334.890625, 478903.125, 288917.75, 254081.71875, 285744.96875, 301214.53125, 381815.09375, 484310.0625, 478492.6875, 505826.21875, 425950.34375, 526906.8125, 336393.5625, 360000.46875, 204912.421875, 142902.09375, 186742.734375, 292174.5625, 375563.59375, 433015.375, 449120.125, 467410.25, 341196.125, 534745.5625, 289708.96875, 242252.140625, 125803.328125, 32823.0625, 242620.25, 325181.5625, 379624.03125, 253196.859375, 344051.40625, 129340.453125, 223081.09375, 129551.390625, 239891.03125, 261754.0625, 356155.5, 109695.5546875, 32649.04492188, 159941.5 ], dtype=float) return None
def complete_configuration(self): self.data = {} for list_type in ['ts_float_list', 'ts_int_list', 'other_float_list', 'other_int_list']: for series_name in self.config[list_type]: infile = self.config['dir'] + self.config[series_name + '_file'] try: f = nc.NetCDFFile(infile) except: msg = ('File ' + infile + ' for data series ' + series_name + ' was not opened.') raise mureilexception.ConfigException(msg, {}) try: vbl = f.variables[self.config[series_name + '_vbl']] except: msg = ('Variable ' + self.config[series_name + '_vbl'] + ' not found in file ' + infile) raise mureilexception.ConfigException(msg, {}) dims = len(vbl.shape) if (dims == 1): temp = vbl[:] elif (dims == 2): temp = vbl[:,:] else: msg = 'Data series ' + series_name + ' has more than 2 dimensions, so is not handled.' raise mureilexception.ConfigException(msg, {}) if 'float' in list_type: if not mureiltypes.check_ndarray_float(temp, True): self.data[series_name] = numpy.array(temp, dtype=float) else: self.data[series_name] = temp else: if not mureiltypes.check_ndarray_int(temp, True): self.data[series_name] = numpy.array(temp, dtype=int) else: self.data[series_name] = temp for list_type in ['ts_csv_list']: for series_name in self.config[list_type]: infile = self.config['dir'] + self.config[series_name + '_file'] temp = [] try: with open(infile, 'rU') as n: reader = csv.reader(n) for row in reader: if reader.line_num == 1: self.data[series_name + '_hdr'] = row[1:] else: if reader.line_num == 2: temp = [map(float, (row[1:]))] else: temp.append(map(float, (row[1:]))) self.data[series_name] = numpy.array(temp, dtype=float) if not mureiltypes.check_ndarray_float(temp, True): self.data[series_name] = numpy.array(temp, dtype=float) else: self.data[series_name] = temp except: msg = ('File ' + infile + ' for data series ' + series_name + ' was not opened or had an error in reading.') raise mureilexception.ConfigException(msg, {}) # Now apply the NaN filter to the ts lists, but note that the integer # ones are not identified as nan. all_ts = self.config['ts_float_list'] + self.config['ts_int_list'] + self.config['ts_csv_list'] if len(all_ts) == 0: self.ts_length = 0 logger.warning('No timeseries data defined') else: self.ts_length = self.data[all_ts[0]].shape[0] # Start with an array of 'False' nan_acc = (numpy.ones(self.ts_length) == 0) # Accumulate 'True' entries in nan_acc where NaN found in timeseries for ts_name in all_ts: ts_nan = numpy.isnan(self.data[ts_name]) if ts_nan.ndim > 1: ts_nan = ts_nan.any(1) # Check all the timeseries are the same length if not (len(ts_nan) == self.ts_length): msg = ('Data series ' + ts_name + ' is length {:d}, not matching {:d} of '.format( len(ts_nan), self.ts_length) + all_ts[0]) raise mureilexception.ConfigException(msg, {}) nan_acc = numpy.logical_or(nan_acc, ts_nan) # Clean up the timeseries using slices nan_acc = numpy.logical_not(nan_acc) for ts_name in all_ts: if self.data[ts_name].ndim == 1: self.data[ts_name] = self.data[ts_name][nan_acc] else: self.data[ts_name] = self.data[ts_name][nan_acc, :] self.ts_length = self.data[all_ts[0]].shape[0] self.is_configured = True return None
def complete_configuration(self): self.data = {} wind_gap = '11' solar_gap = '21' dir = '/mnt/meteo0/data/dargaville/rhuva/' file = 'RegGrid_wind_output_'+wind_gap+'point_gap.nc' infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['ts_wind'][:,:] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_wind'] = numpy.array(temp, dtype=float) else: self.data['ts_wind'] = temp file = 'RegGrid_dsr_output_'+solar_gap+'point_gap.nc' infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['ts_solar'][:,:] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_solar'] = numpy.array(temp, dtype=float) else: self.data['ts_solar'] = temp file = 'Aus_demand_2010-2011.nc' infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['ts_demand'][:] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_demand'] = numpy.array(temp, dtype=float) else: self.data['ts_demand'] = temp wind_nan = numpy.isnan(self.data['ts_wind']) solar_nan = numpy.isnan(self.data['ts_solar']) demand_nan = numpy.isnan(self.data['ts_demand']) wind_row = wind_nan.any(1) solar_row = solar_nan.any(1) combo = numpy.array([wind_row, solar_row, demand_nan]) combo_flat = combo.any(0) self.data['ts_wind'] = self.data['ts_wind'][combo_flat == False, :] self.data['ts_solar'] = self.data['ts_solar'][combo_flat == False, :] self.data['ts_demand'] = self.data['ts_demand'][combo_flat == False] print self.data['ts_wind'].shape print self.data['ts_solar'].shape print self.data['ts_demand'].shape self.ts_length = self.data['ts_wind'].shape[0] file = 'Dist-to-nearest-cap_wind_'+wind_gap+'point_gap.nc' infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['ts_wind_distances'][:] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_wind_distances'] = numpy.array(temp, dtype=float) else: self.data['ts_wind_distances'] = temp file = 'Dist-to-nearest-cap_dsr_'+solar_gap+'point_gap.nc' infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['ts_solar_distances'][:] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_solar_distances'] = numpy.array(temp, dtype=float) else: self.data['ts_solar_distances'] = temp return None
def complete_configuration(self): self.data = {} wind_gap = '11' solar_gap = '21' time_option = 1 #1 is all data, 2 is winter-only, 3 is summer-only if time_option == 1: time_str = '_' if time_option == 2: time_str = '_winter-only_' if time_option == 3: time_str = '_summer-only_' dir = '/mnt/meteo0/data/dargaville/rhuva/ACCESS/' file = 'RegGrid_wind'+time_str+'output_'+wind_gap+'point_gap_east-states.nc' infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['ts_wind'][:,:] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_wind'] = numpy.array(temp, dtype=float) else: self.data['ts_wind'] = temp file = 'RegGrid_dsr'+time_str+'output_'+solar_gap+'point_gap_east-states.nc' infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['ts_solar'][:,:] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_solar'] = numpy.array(temp, dtype=float) else: self.data['ts_solar'] = temp file = 'NEM_demand'+time_str+'2010-2011.nc' infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['ts_demand'][:] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_demand'] = numpy.array(temp, dtype=float) else: self.data['ts_demand'] = temp wind_nan = numpy.isnan(self.data['ts_wind']) solar_nan = numpy.isnan(self.data['ts_solar']) demand_nan = numpy.isnan(self.data['ts_demand']) wind_row = wind_nan.any(1) solar_row = solar_nan.any(1) combo = numpy.array([wind_row, solar_row, demand_nan]) combo_flat = combo.any(0) self.data['ts_wind'] = self.data['ts_wind'][combo_flat == False, :] self.data['ts_solar'] = self.data['ts_solar'][combo_flat == False, :] self.data['ts_demand'] = self.data['ts_demand'][combo_flat == False] print self.data['ts_wind'].shape print self.data['ts_solar'].shape print self.data['ts_demand'].shape self.ts_length = self.data['ts_wind'].shape[0] file = 'Dist-to-nearest-cap_wind_'+wind_gap+'point_gap_NEM-only_east-states.nc' infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['ts_wind_distances'][:] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_wind_distances'] = numpy.array(temp, dtype=float) else: self.data['ts_wind_distances'] = temp file = 'Dist-to-nearest-cap_dsr_'+solar_gap+'point_gap_NEM-only_east-states.nc' infile = dir + file f = nc.NetCDFFile(infile) temp = f.variables['ts_solar_distances'][:] if not mureiltypes.check_ndarray_float(temp, True): self.data['ts_solar_distances'] = numpy.array(temp, dtype=float) else: self.data['ts_solar_distances'] = temp return None
def complete_configuration(self): self.data = {} for list_type in [ 'ts_float_list', 'ts_int_list', 'other_float_list', 'other_int_list' ]: for series_name in self.config[list_type]: infile = self.config['dir'] + self.config[series_name + '_file'] try: f = nc.NetCDFFile(infile) except: msg = ('File ' + infile + ' for data series ' + series_name + ' was not opened.') raise mureilexception.ConfigException(msg, {}) try: vbl = f.variables[self.config[series_name + '_vbl']] except: msg = ('Variable ' + self.config[series_name + '_vbl'] + ' not found in file ' + infile) raise mureilexception.ConfigException(msg, {}) dims = len(vbl.shape) if (dims == 1): temp = vbl[:] elif (dims == 2): temp = vbl[:, :] else: msg = 'Data series ' + series_name + ' has more than 2 dimensions, so is not handled.' raise mureilexception.ConfigException(msg, {}) if 'float' in list_type: if not mureiltypes.check_ndarray_float(temp, True): self.data[series_name] = numpy.array(temp, dtype=float) else: self.data[series_name] = temp else: if not mureiltypes.check_ndarray_int(temp, True): self.data[series_name] = numpy.array(temp, dtype=int) else: self.data[series_name] = temp for list_type in ['ts_csv_list']: for series_name in self.config[list_type]: infile = self.config['dir'] + self.config[series_name + '_file'] temp = [] try: with open(infile, 'rU') as n: reader = csv.reader(n) for row in reader: if reader.line_num == 1: self.data[series_name + '_hdr'] = row[1:] else: if reader.line_num == 2: temp = [map(float, (row[1:]))] else: temp.append(map(float, (row[1:]))) self.data[series_name] = numpy.array(temp, dtype=float) if not mureiltypes.check_ndarray_float(temp, True): self.data[series_name] = numpy.array(temp, dtype=float) else: self.data[series_name] = temp except: msg = ('File ' + infile + ' for data series ' + series_name + ' was not opened or had an error in reading.') raise mureilexception.ConfigException(msg, {}) # Now apply the NaN filter to the ts lists, but note that the integer # ones are not identified as nan. all_ts = self.config['ts_float_list'] + self.config[ 'ts_int_list'] + self.config['ts_csv_list'] if len(all_ts) == 0: self.ts_length = 0 logger.warning('No timeseries data defined') else: self.ts_length = self.data[all_ts[0]].shape[0] # Start with an array of 'False' nan_acc = (numpy.ones(self.ts_length) == 0) # Accumulate 'True' entries in nan_acc where NaN found in timeseries for ts_name in all_ts: ts_nan = numpy.isnan(self.data[ts_name]) if ts_nan.ndim > 1: ts_nan = ts_nan.any(1) # Check all the timeseries are the same length if not (len(ts_nan) == self.ts_length): msg = ('Data series ' + ts_name + ' is length {:d}, not matching {:d} of '.format( len(ts_nan), self.ts_length) + all_ts[0]) raise mureilexception.ConfigException(msg, {}) nan_acc = numpy.logical_or(nan_acc, ts_nan) # Clean up the timeseries using slices nan_acc = numpy.logical_not(nan_acc) for ts_name in all_ts: if self.data[ts_name].ndim == 1: self.data[ts_name] = self.data[ts_name][nan_acc] else: self.data[ts_name] = self.data[ts_name][nan_acc, :] self.ts_length = self.data[all_ts[0]].shape[0] self.is_configured = True return None