def main():
# Set up logging
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(message)s',
datefmt='%Y-%d-%m %I:%M:%S %p')
# Get parameters from the provided parameter file
param_file_path = sys.argv[1]
params = parameters.get_params(param_file_path)
# From parameter file, calculate intensity map + other outputs
t0 = time.time()
imgrid = get_grid(
params
) # Create the grid object that will contain the 3D brightness cube
halos = get_halos(params) # Load and pre-process halo data
halos.lum = get_lum(halos, params) # Calculate halo line luminosities
imgrid.tcube = get_tcube(
halos, imgrid, params) # From halos, make brightness temperature cube
##################################################
# Calculate power spectra from temperature cube
ksph, psph = get_powersph(imgrid)
kprp, kpar, pcyl = get_powercyl(imgrid)
# Calculate error bars
errsph, noise_power, nmodes, fres = get_powersph_errorbars(
ksph, psph, params)
# TODO: get cylindrical power spectrum error bars
##################################################
# Write temperature cube and other stuff to file
save_tcube(imgrid, params)
save_powersph(ksph, psph, errsph, noise_power, nmodes, fres, params)
save_powercyl(kprp, kpar, pcyl, params)
save_paramfile(params) # Copy parameter file to output folder
##################################################
# Summarize timing
t1 = time.time()
tpass = t1 - t0
logging.info("Done!")
logging.info("")
logging.info("Total time : {:.4f}\n".format(tpass))
def free_energy_gap(x, sequence_structure_pairs, apply_params):
dG_gap = 0.0
params = get_params(suppress_all_output=True)
apply_params(params, x)
print
print x
sequence_structure_params_tuples = map(lambda y: y + tuple([params]),
sequence_structure_pairs)
#all_dG_gap = map( calc_dG_gap, sequence_structure_params_tuples )
all_dG_gap = pool.map(calc_dG_gap, sequence_structure_params_tuples)
return sum(all_dG_gap)
def main():
# Set up logging
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s %(message)s',
datefmt='%Y-%d-%m %I:%M:%S %p')
# Get parameters from the provided parameter file
param_file_path = sys.argv[1]
params = parameters.get_params(param_file_path)
# From parameter file, calculate intensity map + other outputs
t0 = time.time()
for lc_idx, lc_path in enumerate(params['io']['lightcone_path']):
imgrid = get_grid(params) # Create the grid object that will contain the 3D brightness cube
halos = get_halos(params, lc_path=lc_path) # Load and pre-process halo data
halos.lum = get_lum(halos, params) # Calculate halo line luminosities
imgrid.tcube = get_tcube(halos, imgrid, params) # From halos, make brightness temperature cube
##################################################
# Calculate power spectra from temperature cube
ksph, psph = get_powersph(imgrid)
kprp, kpar, pcyl = get_powercyl(imgrid)
# Calculate error bars
errsph, noise_power, nmodes, fres = get_powersph_errorbars(ksph, psph, params)
# TODO: get cylindrical power spectrum error bars
##################################################
# Write temperature cube and other stuff to file
save_tcube(imgrid, params, idx=lc_idx)
save_powersph(ksph, psph, errsph, noise_power, nmodes, fres, params, idx=lc_idx)
save_powercyl(kprp, kpar, pcyl, params, idx=lc_idx)
save_paramfile(params) # Copy parameter file to output folder
##################################################
# Summarize timing
t1 = time.time()
tpass = t1-t0
logging.info("Done!")
logging.info("")
logging.info("Total time : {:.4f}\n".format(tpass))
def asset_cost_of_capital(fixed_assets): # grabs the constant values from the parameters dictionary params = param.get_params() inflation_rate = params['inflation rate'] stat_tax = params['tax rate'] discount_rate = params['discount rate'] save_rate = params['return to savers'] delta = params['econ depreciation'] r_prime = params['after-tax rate'] inv_credit = params['inv_credit'] w = params['prop tax'] delta = np.tile(np.reshape(delta,(delta.shape[0],1,1)),(1,discount_rate.shape[0],discount_rate.shape[1])) z = params['depr allow'] rho = ((discount_rate - inflation_rate) + delta) * (1- inv_credit- stat_tax * z) / (1- stat_tax) + w - delta metr = (rho - (r_prime - inflation_rate)) / rho mettr = ((rho-save_rate)/rho) return rho, metr, mettr
if version >= 11.0 and version <= 11.9:
codename = 'Eden'
if version >= 12.0 and version <= 12.9:
codename = 'Frodo'
if version >= 13.0 and version <= 13.9:
codename = 'Gotham'
if version >= 14.0 and version <= 14.9:
codename = 'Helix'
if version >= 15.0 and version <= 15.9:
codename = 'Isengard'
if version >= 16.0 and version <= 16.9:
codename = 'Jarvis'
if version >= 17.0 and version <= 17.9:
codename = 'Krypton'
params=parameters.get_params()
#######################################################################
# ROOT MENU
#######################################################################
def INDEX():
#if plugintools.get_setting("username")=="":
# settings(params)
#login_request = login(LoginServer , plugintools.get_setting("username") , plugintools.get_setting("password") )
#if login_request!="ACCESS GRANTED":
Common.addItem('[COLOR cyan]SERVER STATUS [/COLOR]' + CommunityStatus + JarvisWizStatus + JarvisUpdateStatus + KryptonWizStatus + KryptonUpdateStatus + SpeedTestStatus,BASEURL,92,ICON,FANART,'')
Common.addDir('[COLOR cyan]Imperio Tuga WIZARD BUILDS[/COLOR] - [COLOR powderblue]J v16[/COLOR] | ' + JarvisWizStatusMain + ' - [COLOR powderblue]K v17[/COLOR] | ' + KryptonWizStatusMain,BASEURL,88,ICON,FANART,'')
if version >= 11.0 and version <= 11.9:
codename = 'Eden'
if version >= 12.0 and version <= 12.9:
codename = 'Frodo'
if version >= 13.0 and version <= 13.9:
codename = 'Gotham'
if version >= 14.0 and version <= 14.9:
codename = 'Helix'
if version >= 15.0 and version <= 15.9:
codename = 'Isengard'
if version >= 16.0 and version <= 16.9:
codename = 'Jarvis'
if version >= 17.0 and version <= 17.9:
codename = 'Krypton'
params = parameters.get_params()
#######################################################################
# ROOT MENU
#######################################################################
def INDEX():
#if plugintools.get_setting("username")=="":
# settings(params)
#login_request = login(LoginServer , plugintools.get_setting("username") , plugintools.get_setting("password") )
#if login_request!="ACCESS GRANTED":
Common.addItem(
def get_parameters(self, config_path): params = parameters.get_params(config_path) return params
dtype=np.float32)
wcs_1 = wcs.WCS(self.original_map)
pys, pxs = wcs_1.wcs_world2pix(self.RAs, self.DECs, 0)
beam_FWHM = 17.6
beam_sigma = beam_FWHM / (2 * np.sqrt(2 * np.log(2)))
#CREATE THE IMAGE FILE
w = wcs.WCS(naxis=2)
#print original_header
w.wcs.crpix = [original_header['CRPIX1'], original_header['CRPIX2']]
w.wcs.cdelt = np.array([-pixsize / 3600, pixsize / 3600])
w.wcs.crval = [original_header['CRVAL1'], original_header['CRVAL2']]
w.wcs.ctype = [original_header['CTYPE1'], original_header['CTYPE2']]
header_w = w.to_header()
hdu = fits.PrimaryHDU(data=new_image_data, header=header_w)
file_path = self.new_map_location + '/FAKE_MAP_NO_SOURCES.FITS'
if os.path.exists(file_path) == True:
os.remove(file_path)
hdu.writeto(file_path)
else:
hdu.writeto(file_path)
map_path_original = '/home/sam/Desktop/LSB/data/maps/HATLAS_NGP_DR2_RAW250_CUTOUT.FITS'
noise_path_original = '/home/sam/Desktop/LSB/data/maps/HATLAS_NGP_DR2_SIGMA250_CUTOUT.FITS'
param_file_path = '/home/sam/Desktop/LSB/simstack-master/Parameters1.cfg'
params = parameters.get_params(param_file_path)
#map_basis = false_map(map_path_original,noise_path_original,params)
#false_map.create_fake_matrix()
import os
import sys
import torch
import utils
from model_functions import eval
from parameters import get_params
if __name__ == '__main__':
# Get all parameters
args = get_params(eval_mode=True)
args.command = 'python ' + ' '.join(sys.argv)
assert args.pth
assert os.path.exists(args.pth)
assert args.data_path
assert os.path.exists(args.data_path)
assert args.out_path
assert os.path.exists(args.out_path)
# CUDA
utils.check_for_CUDA(args)
# Load pth
pth_dir_name = os.path.dirname(args.pth)
print("Loading model", os.path.join(pth_dir_name, 'model.pth'))
model = torch.load(os.path.join(pth_dir_name, 'model.pth'))
print("Loading model state dict", args.pth)
model.load_state_dict(torch.load(args.pth))
model = model.to(args.device)
def score_structure( sequences, structure, circle = False, params = '', test_mode = False ):
# What we get if we parse out motifs
structure = secstruct_util.get_structure_string( structure )
bps_list = secstruct_util.bps( structure )
motifs = secstruct_util.parse_motifs( structure )
sequence, ligated, sequences = sequence_util.initialize_sequence_and_ligated( sequences, circle )
params = get_params( params, suppress_all_output = True )
Kd_ref = params.base_pair_types[0].Kd # Kd[G-C], a la Turner rule convention
C_std = params.C_std
# Now go through each motif parsed out of the target structure
Z = 1.0
for motif in motifs:
motif_res = []
motif_sequences = []
for strand in motif:
strand_sequence = ''
for i in strand:
motif_res.append( i )
strand_sequence += sequence[i]
if not ligated[i]:
motif_sequences.append( strand_sequence )
strand_sequence = ''
if len( strand_sequence ) > 0: motif_sequences.append( strand_sequence )
motif_circle = ligated[ motif_res[-1] ] and ( (motif_res[0] - motif_res[-1]) % len(sequence) == 1 )
motif_sequence = ''.join( motif_sequences )
# each motif res better show up only once
assert( len( set( motif_res ) ) == len( motif_res ) )
motif_bps_list = []
for i,j in bps_list:
if motif_res.count( i ) == 0: continue
if motif_res.count( j ) == 0: continue
motif_bps_list.append( (motif_res.index(i), motif_res.index(j)) )
motif_structure = secstruct_util.secstruct( motif_bps_list, len( motif_res ) )
p = partition( motif_sequences, circle = motif_circle, structure = motif_structure, params = params, suppress_all_output = True )
Z_motif = p.Z
# Need to 'correct' for half-terminal penalties (a la Turner rules) and also remove extra costs
# for connecting these 'sub-strands' together.
Z_motif *= ( Kd_ref / C_std ) ** sequence_util.get_num_strand_connections( motif_sequences, motif_circle )
for i_motif, j_motif in motif_bps_list:
# what kind of base pair is this?
for base_pair_type in p.params.base_pair_types:
if base_pair_type.is_match( motif_sequence[ i_motif ], motif_sequence[ j_motif ] ):
Z_motif *= ( base_pair_type.Kd / Kd_ref )**(0.5)
break
if test_mode: print "Motif: ", Z_motif, motif, motif_sequences, motif_structure
Z *= Z_motif
# Compute cost of connecting the strands into a complex
Z_connect = ( C_std / Kd_ref ) ** sequence_util.get_num_strand_connections( sequences, circle )
Z *= Z_connect
if test_mode:
print "Connect strands: ", Z_connect
print 'Product of motif Z :', Z
if test_mode:
# Reference value from 'hacked' dynamic programming, which takes a while.
p = partition( sequences, circle = circle, structure = structure, params = params, suppress_all_output = True )
print 'From dynamic programming:', p.Z
assert( abs( p.Z - Z )/Z < 1.0e-5 )
dG = -KT_IN_KCAL * log( Z )
return dG
from parameters import get_params from utils import beamforming import os.path import soundfile as sf import csv import numpy as np import matplotlib.pyplot as plt # new imports import pandas as pd # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% preset = '0' # Default preset params = get_params(preset) # Get path to data to be processed dataset_dir = params['dataset_dir'] dataset_type_folder = params['dataset'] + "_" + params['mode'] data_folder_path = os.path.join(dataset_dir, dataset_type_folder) # Get path to groundtruth metadata gt_folder = 'metadata' + "_" + params['mode'] gt_folder_path = os.path.join(dataset_dir, gt_folder) # store ground truth info for classification files = [] labels = [] splits = [] irs = []
def main():
# Set up logging
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s %(message)s',
datefmt='%Y-%d-%m %I:%M:%S %p')
# Get parameters from the provided parameter file
param_file_path = sys.argv[1]
params = parameters.get_params(param_file_path)
zkey = params['zkey']
mkey = params['mkey']
rkey = params['ra_key']
dkey = params['dec_key']
t0 = time.time()
if params['bins']['bin_in_lookback_time'] == True:
z_pref = 'lookt'
else:
z_pref = 'z'
# Stack in Slices or ALL AT ONCE Choice made here
if params['bins']['stack_all_z_at_once'] == True: n_slices = 1
else: n_slices = len(params['bins']['z_nodes']) - 1
#Save Parameter file in folder
save_paramfile(params)
for i in range(n_slices):
if params['bins']['stack_all_z_at_once'] == True:
j = None
stacked_flux_density_key = 'all_'+z_pref
else:
j = i
if params['bins']['bin_in_lookback_time'] == True:
stacked_flux_density_key = '{:.2f}'.format(params['bins']['t_nodes'][j])+'-'+'{:.2f}'.format(params['bins']['t_nodes'][j+1])
else:
stacked_flux_density_key = str(params['bins']['t_nodes'][j])+'-'+str(params['bins']['t_nodes'][j+1])
print stacked_flux_density_key
# From parameter file read maps, psfs, cats, and divide them into bins
sky_library = get_maps(params)
cats = get_catalogs(params)
if params['bootstrap'] == True:
pcat = Bootstrap(cats.table)
# Bootstrap Loop Starts here
for iboot in np.arange(params['number_of_boots'])+params['boot0']:
#stacked_flux_densities = {}
if params['bootstrap'] == True:
print 'Running ' +str(int(iboot))+' of '+ str(int(params['boot0'])) +'-'+ str(int(params['boot0']+params['number_of_boots']-1)) + ' bootstraps'
pcat.perturb_catalog(perturb_z = params['perturb_z'])
bootcat = Field_catalogs(pcat.pseudo_cat,zkey=zkey,mkey=mkey,rkey=rkey,dkey=dkey)
binned_ra_dec = get_bin_radec(params, bootcat, single_slice = j)
if params['save_bin_ids'] == False:
bin_ids = None
else:
bin_ids = get_bin_ids(params, bootcat, single_slice = j)
out_file_path = params['io']['output_folder']+'/bootstrapped_fluxes/'+params['io']['shortname']
out_file_suffix = '_'+stacked_flux_density_key+'_boot_'+str(int(iboot))
else:
binned_ra_dec = get_bin_radec(params, cats, single_slice = j)
if params['save_bin_ids'] == False:
bin_ids = None
else:
bin_ids = get_bin_ids(params, cats, single_slice = j)
out_file_path = params['io']['output_folder'] + '/simstack_fluxes/' + params['io']['shortname']
out_file_suffix = '_'+stacked_flux_density_key
# Do simultaneous stacking
if params['float_background'] == True:
stacked_flux_densities = stack_libraries_in_layers_w_background(sky_library,binned_ra_dec)
else:
stacked_flux_densities = stack_libraries_in_layers(sky_library,binned_ra_dec)
save_stacked_fluxes(stacked_flux_densities,params, out_file_path,out_file_suffix, IDs=bin_ids)
#pdb.set_trace()
# Summarize timing
t1 = time.time()
tpass = t1-t0
logging.info("Done!")
logging.info("")
logging.info("Total time : {:.4f} minutes\n".format(tpass/60.))
def run_btax(user_params):
# get parameters
parameters = params.get_params()
# make calculations
rho, metr, mettr = asset_cost_of_capital(parameters)
# format output
numberOfRows = (parameters['econ depreciation']).shape[0]
column_list = ('Asset Type', 'delta', 'z_c', 'z_c_d', 'z_c_e', 'z_nc',
'rho_c', 'rho_c_d', 'rho_c_e', 'rho_nc', 'metr_c', 'metr_c_d', 'metr_c_e',
'metr_nc', 'mettr_c', 'mettr_c_d', 'mettr_c_e', 'mettr_nc')
vars_by_asset = pd.DataFrame(index=np.arange(0, numberOfRows), columns=column_list)
tax_depr = pd.read_csv(_TAX_DEPR_IN_PATH)
vars_by_asset['Asset Type'] = tax_depr['Asset Type']
vars_by_asset['delta'] = parameters['econ depreciation']
suffix_list = ['', '_d', '_e']
vars_by_asset['z_nc'] = parameters['depr allow'][:,0,1]
vars_by_asset['rho_nc'] = rho[:,0,1]
vars_by_asset['metr_nc'] = metr[:,0,1]
vars_by_asset['mettr_nc'] = mettr[:,0,1]
for i in range(rho.shape[1]):
vars_by_asset['z_c'+suffix_list[i]] = parameters['depr allow'][:,i,0]
vars_by_asset['rho_c'+suffix_list[i]] = rho[:,i,0]
vars_by_asset['metr_c'+suffix_list[i]] = metr[:,i,0]
vars_by_asset['mettr_c'+suffix_list[i]] = mettr[:,i,0]
# save to csv for comparison to CBO
vars_by_asset.to_csv(_OUT_DIR+'/calculations_by_asset.csv')
# read in CBO file
CBO_data = pd.read_excel(os.path.join(_REF_DIR, 'effective_taxrates.xls'),
sheetname='Full detail', header=1, skiprows=0, skip_footer=8)
CBO_data.columns = [col.encode('ascii', 'ignore') for col in CBO_data]
CBO_data.rename(columns = {'Top page (Rows 3-35): Equipment Bottom page (Rows 36-62): All Other ':'Asset Type'}, inplace = True)
CBO_data.to_csv(_OUT_DIR+'/CBO_data.csv',encoding='utf-8')
# join CBO data to ours
# import difflib
# df2 = vars_by_asset.copy()
# df1 = CBO_data.copy()
# df2['Asset Type'] = df2['Asset Type'].apply(lambda x: difflib.get_close_matches(x, df1['Asset Type'])[0])
# df1.merge(df2)
# print df1.head(n=5)
CBO_v_OSPC = vars_by_asset.merge(CBO_data,how='inner',on='Asset Type')
OSPC_list = ['delta','z_c','z_c_d','z_c_e','z_nc','rho_c','rho_c_d','rho_c_e','rho_nc',
'metr_c', 'metr_c_d', 'metr_c_e', 'mettr_c', 'mettr_c_d', 'mettr_c_e',
'mettr_nc']
CBO_list = ['Economic deprecia- tion rate []','Corporate: total [z(c)]',
'Corporate: debt-financed [z(c,d)]', 'Corporate: equity-financed [z(c,e)]',
'Non-corporate [z(n)]', 'Corporate: total [(c)]', 'Corporate: debt-financed [(c,d)]',
'Corporate: equity-financed [(c,e)]', 'Non-corporate [(n)]', 'Corporate: total [ETR(c)]',
'Corporate: debt-financed [ETR(c,d)]', 'Corporate: equity-financed [ETR(c,e)]',
'Corporate: total [ETTR(c)]', 'Corporate: debt-financed [ETTR(c,d)]',
'Corporate: equity-financed [ETTR(c,e)]', 'Non-corporate [ETTR(n)]']
#print CBO_v_OSPC.index
# print CBO_v_OSPC.columns
# print CBO_v_OSPC.shape
# print CBO_v_OSPC.T.drop_duplicates().T
# quit()
CBO_v_OSPC = CBO_v_OSPC.T.drop_duplicates().T
diff_list = [None] * len(OSPC_list)
for i in xrange(0,len(OSPC_list)):
CBO_v_OSPC[OSPC_list[i]+'_diff'] = CBO_v_OSPC[OSPC_list[i]] - CBO_v_OSPC[CBO_list[i]]
diff_list[i] = OSPC_list[i]+'_diff'
cols_to_print = ['Asset Type']+OSPC_list + CBO_list + diff_list
CBO_v_OSPC[cols_to_print].to_csv(_OUT_DIR+'/CBO_v_OSPC.csv',encoding='utf-8')
with open(os.path.join(_OUT_DIR, 'final_output.pkl'), 'wb') as handle:
pickle.dump(vars_by_asset, handle)
return vars_by_asset
debug = False
if len(sys.argv) > 1:
for command in range(1, len(sys.argv)):
if sys.argv[command] == "help":
dse.help()
sys.exit()
if sys.argv[command] == "debug":
debug = True
elif sys.argv[command] == "list":
dse.command_list()
sys.exit()
elif sys.argv[command] == "single-point":
single_point = True
elif sys.argv[command] == "complexity":
params = user.get_params()
assert dse.check(params),\
"Parameters error. Please see the log"
print "Expected number of tests: %d" \
%(dse.ds_size(params))
print "For the real number of test, use\
[complexity-hard]"
sys.exit()
elif sys.argv[command] == "complexity-hard":
complexity_hard_mode = True
else:
print "[%s] is not a recongnized command. try [help]" \
%(sys.argv[1])
sys.exit()
import sys
import torch
import torch.nn as nn
import torch.optim as optim
import time
import utils
from models import *
from model_functions import *
from parameters import get_params
if __name__ == '__main__':
# Get all parameters
args = get_params()
args.command = 'python ' + ' '.join(sys.argv)
# CUDA
utils.check_for_CUDA(args)
# Seed
torch.manual_seed(args.seed)
# IMAGES DATALOADER
train_loader, valid_loader = utils.make_dataloader(args)
print(args)
# OUT PATH
if not os.path.exists(args.out_path):
print('\n====================Starting metrics for challenge with REAL frontend=====================================')
print('====================Starting metrics for challenge with REAL frontend=====================================')
print('====================Starting metrics for challenge with REAL frontend=====================================\n')
data_folder_path = '../data/foa_dev/'
# Iterate over all audio files from the dev set, some are from split 1234
audio_files = [f for f in os.listdir(data_folder_path) if not f.startswith('.')]
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# Path stuff
# This parameter will define the algorithm type
preset_string = 'Q'
# Default preset: contains path to folders
params = get_params(preset_string)
# Dataset type:
dataset_type_folder = params['dataset'] + "_" + params['mode']
dataset_preset_folder = dataset_type_folder + '_' + preset_string
# Get folder names before and after classification
doa_folder = params['before_classification_folder_name']
classif_folder = params['after_classification_folder_name']
# Path to audio folder
dataset_dir = '../data'
data_folder_path = os.path.join(dataset_dir, dataset_type_folder)
# Path to results_metadata folder _before classification_; it should exist
results_metadata_doa_folder = os.path.join('.' + params['metadata_result_folder_path'],
