entries = ['db', 'metric', 'run_name', 'folder_results'] tasks = pd.DataFrame(columns=entries) for db in dbs: # metrics available # metrics = ['costs_yearly', 'emissions_yearly', 'activity_by_fuel', 'activity_by_tech', 'capacity_by_fuel', # 'capacity_by_tech'] # For our analysis we only use the following metrics if db == "T_0.sqlite" or db == "U_0.sqlite" or db == "V_0.sqlite": metrics = [ 'costs_yearly', 'emissions_yearly', 'activity_by_fuel', 'activity_by_tech', 'capacity_by_fuel', 'capacity_by_tech' ] else: metrics = ['costs_yearly', 'emissions_yearly'] for metric in metrics: t = pd.Series(index=entries) t['db'] = db t['metric'] = metric t['run_name'] = tt.remove_ext(db) t['folder_results'] = os.path.join(result_folder, t['run_name']) tasks = tasks.append(t, ignore_index=True) # Perform simulations in parallel with parallel_backend('multiprocessing', n_jobs=ncpus): Parallel(n_jobs=ncpus, verbose=5)(delayed(analyze_results)( task, db_folder, all_dbs_dict, db_shift, node_prob, tech_group_dict, prob_type_dict, infra_dict, carbon_tax_dict) for index, task in tasks.iterrows())
os.mkdir(stochdir) os.chdir(stochdir) n_cases = 1 for case in range(n_cases): # historical probabilities if case == 0: probabilities = probabilities_hist # climate change probabilities else: # case == 1: probabilities = probabilities_climate_change # Iterate through each database for each case for db in dbs: db_name = tt.remove_ext(db) # ==================================== # Stochastic input file # ==================================== # Write File filename = "stoch_" + db_name + "_" + str(case) + ".py" # Open File f = open(filename, "w") f.write( "# Automatically generated stochastic input file from temoatools github.com/EnergyModels/temoatools\n\n" ) f.write("verbose = True\n") f.write("force = True\n") f.write("\n") f.write("dirname = '" + db_name + "_" + str(case) + "'\n") # Update
def CreateConfigFile(model_directory, model_filename, saveEXCEL=False, saveTEXTFILE=False, keep_pyomo_lp_file=False, debug=False, solver=''): # Locate Database full_filename = tt.remove_ext(model_filename) + '.sqlite' dBpath = os.path.join(model_directory, full_filename) # Write Config File config_file = "config_" + tt.remove_ext(model_filename) + ".txt" if debug == True: print("config_file: " + str(config_file)) f = open(config_file, "w") # --- f.write("#-----------------------------------------------------\n") f.write( "# This is an automatically generated configuration file for Temoa using" ) f.write(" temoatools github.com/EnergyModels/temoatools\n") f.write( "# It allows you to specify (and document) all run-time model options\n" ) f.write("# Legal chars in path: a-z A-Z 0-9 - _ \' / . :\n") f.write("# Comment out non-mandatory options to omit them\n") f.write("#-----------------------------------------------------\n") f.write("\n") f.write("# Input File (Mandatory)\n") f.write("# Input can be a .sqlite or .dat file\n") f.write("# Both relative path and absolute path are accepted\n") f.write("--input=" + dBpath + "\n") f.write("\n") f.write("# Output File (Mandatory)\n") f.write("# The output file must be a existing .sqlite file\n") f.write("--output=" + dBpath + "\n") f.write("\n") f.write("# Scenario Name (Mandatory)\n") f.write( "# This scenario name is used to store results within the output .sqlite file\n" ) f.write("--scenario=" + "solve" + "\n") f.write("\n") f.write("# Path to the \"db_io\" folder (Mandatory)\n") f.write("# This is the location where database files reside\n") f.write("--path_to_db_io=" + model_directory + "\n") f.write("\n") f.write("# Spreadsheet Output (Optional)\n") f.write("# Direct model output to a spreadsheet\n") f.write( "# Scenario name specified above is used to name the spreadsheet\n") # --- # Option - saveExcel file, Turn off to save HD space # --- if saveEXCEL: f.write("--saveEXCEL\n") else: f.write("#--saveEXCEL\n") # --- f.write("\n") f.write("# Save the log file output (Optional)\n") f.write("# This is the same output provided to the shell\n") # --- # Option - saveTEXTFILE file, Turn off to save HD space # --- if saveTEXTFILE == True: # Turn off to save HD space f.write("--saveTEXTFILE\n") else: f.write("#--saveTEXTFILE\n") # --- f.write("\n") f.write("# Solver-related arguments (Optional)\n") if len(solver) > 0: f.write("--solver=" + solver + " # Optional, indicate the solver\n") else: f.write( "#--solver=cplex # Optional, indicate the solver\n" ) # --- # Option - keep_pyomo_lp_file file, Turn off to save HD space # --- if keep_pyomo_lp_file: # Turn off to save HD space f.write( "--keep_pyomo_lp_file # Optional, generate Pyomo-compatible LP file\n" ) else: f.write( "#--keep_pyomo_lp_file # Optional, generate Pyomo-compatible LP file\n" ) # --- f.write("\n") f.write("# Modeling-to-Generate Alternatives (Optional)\n") f.write( "# Run name will be automatically generated by appending '_mga_' and iteration number to scenario name\n" ) f.write("#--mga {\n") f.write( "# slack=0.1 # Objective function slack value in MGA runs\n" ) f.write("# iteration=4 # Number of MGA iterations\n") f.write( "# weight=integer # MGA objective function weighting method, currently)" ) f.write("'integer' or 'normalized'\n") f.write("#}\n") f.write("\n") f.close() # --- return config_file
def getActivityTOD(folders, dbs, switch='fuel', sector_name='electric', save_data='N', create_plots='N', conversion=277.777778, run_name=''): # inputs: # 1) folders - paths containing dbs (list or single string if all in the same path) # 2) dbs - names of databases (list) # 3) switch - 'fuel' or 'tech', basis of categorization # 4) sectorName - name of temoa sector to be analyzed # 5) saveData - 'Y' or 'N', default is 'N' # 6) createPlots - 'Y' or 'N', default is 'N' # 7) conversion - conversion to GWh, default is 277.778 (from PJ) # 8) run_name - Used for saving results in dedicated folder # outputs: # 1) activity # 2) plots - optional # 3) Data - optional # ============================================================================== print("Analyzing activity by time of day (TOD)") # Save original directory wrkdir = os.getcwd() # If only a single db and folder provided, change to a list if type(dbs) == str and type(folders) == str: dbs = [dbs] folders = [folders] # If a list of folders is provided with one database, only use first folder elif type(dbs) == str: dbs = [dbs] folders = [folders[0]] # If only a single folder provided, create a list of the same folder elif type(folders) == str: fldrs = [] for db in dbs: fldrs.append(folders) folders = fldrs # Create dataframe to hold each capacity_single series activity = pd.DataFrame(dtype='float64') # Iterate through each db for folder, db in zip(folders, dbs): activity_single = SingleDB(folder, db, switch=switch, sector_name=sector_name, conversion=conversion) activity = pd.concat([activity, activity_single]) # Reset index (remove multi-level indexing, easier to use in Excel) activity = activity.reset_index() # Directory to hold results if save_data == 'Y' or create_plots == 'Y': tt.create_results_dir(wrkdir=wrkdir, run_name=run_name) # Save results to CSV if save_data == 'Y': # Create savename based on switch if switch == 'fuel': savename = 'activityTOD_by_fuel.csv' else: savename = 'activityTOD_by_tech.csv' activity.to_csv(savename) if create_plots == 'Y': df = activity.reset_index() import matplotlib.pyplot as plt import seaborn as sns plt.rcParams.update({'figure.max_open_warning': 0}) # ignore warning for database in df.database.unique(): # new figure plt.figure() # set aesthetics sns.set_style( "white", { "font.family": "serif", "font.serif": ["Times", "Palatino", "serif"] }) sns.set_context("talk") # select relevant database df2 = df[(df.database == database)] # plot sns.relplot(x='tod', y='value', hue='fuelOrTech', row='year', col='season', data=df2, kind='line') # save if switch == 'fuel': savename = 'yearlyActivityTOD_byFuel' + tt.remove_ext( database) + '.pdf' else: savename = 'yearlyActivityTOD_byTech' + tt.remove_ext( database) + '.pdf' plt.savefig(savename, dpi=resolution) # close the figure plt.close() # Return to original directory os.chdir(wrkdir) return activity
def move_data_to_db(XLSX, path=os.path.normcase('.')): # ============================================================================= # Begin Function # ============================================================================= data_path = os.path.join(path, 'data') print(data_path) workDir = os.getcwd() os.chdir(data_path) # Empty db with set schema (expected to be within the same folder) emptydB = os.path.join(tt.resource_path, "db_schema_universal.db") # Create output filename using inputfilename outputdB = tt.remove_ext(XLSX) + ".db" # Keep track of sheet_names and corresponding number of columns to read-in sheets = [("representativeDays", 3), ("timesOfDay", 2), ("Connections", 18), ("ConnectionsExisting", 4), ("Demand", 4), ("DemandTOD", 3), ("DiscountRateGlobal", 2), ("Emission", 5), ("Fuels", 19), ("FuelsExisting", 4), ("PowerPlants", 12), ("PowerPlantsPerformance", 10), ("PowerPlantsCosts", 12), ("PowerPlantsConstraints", 10), ("PowerPlantsExisting", 4), ("MinCapacity", 4), ("ReserveMargin", 2), ("capacityFactorTOD", 5), ("ref", 6)] # ---------- # sqlite file prep # ---------- # Delete old *.sqlite file (if it already exists) and copy/rename copy of temoa_schema.sqlite if os.path.isfile(outputdB): os.remove(outputdB) shutil.copyfile(emptydB, outputdB) # Set-up sqlite connection conn = sqlite3.connect(outputdB) c = conn.cursor() # ---------- # sqlite file prep # ---------- for sheet in sheets: # Extract sheet_name and number of columns for each sheet: sheet_name = sheet[0] sheet_col = sheet[1] # Read XLS sheet df = pd.read_excel(XLSX, sheet_name=sheet_name) df = df.drop([0]) # Remove first row (units) # Create SQL command based on number of entries command = 'INSERT INTO ' + sheet_name + ' VALUES (?' for i in range(sheet_col - 1): command = command + ',?' command = command + ')' # Execute SQL command try: c.executemany(command, np.array(df)) except: print(command) print(np.array(df)) c.executemany(command, np.array(df)) # ---------- # Save(commit) the changes and close sqlite file # ---------- conn.commit() conn.close() os.chdir(workDir) return outputdB
# new figure plt.figure() # set aesthetics sns.set_style("white", { "font.family": "serif", "font.serif": ["Times", "Palatino", "serif"] }) sns.set_context("talk") # select relevant database df2 = df[(df.database == database)] # plot sns.relplot(x='Hour (-)', y='Activity (GWh)', hue='Fuel', row='Year', col='Season', data=df2, kind='line', palette='bright') # save savename = 'plot_yearlyActivityTOD_byFuel_' + tt.remove_ext( database) + '.pdf' plt.savefig(savename, dpi=600) # return to original directory os.chdir(wrkdir)
def test_remove_ext(self): filename = "longFileName1234.sqlite" result = tt.remove_ext(filename) expected = "longFileName1234" self.assertEqual(result, expected)