def output_T (samples, dic, outputfile):
fout = open(outputfile, 'w')
index = dic.keys()
fout.write('cell\t' + string.join(index,'\t') +'\n')
for i in range(0, len(samples)):
sample = samples[i]
fout.write(sample)
for key in index:
try:
fout.write('\t' + numpy.format_float_positional(dic[key][i]))
except:
fout.write('\t' + dic[key][i])
fout.write('\n')
fout.close()
def format_float(num):
return np.format_float_positional(num, trim='-')
def standarddeviation(value):
print(str(np.format_float_positional(np.std(np.array(value)), 2)))
def num2str(num, width, dec=False, space=0, eng=False):
''' Converts a number to a string using F70-like formatting.
Purpose
-------
To easily convert numbers to strings while complying with F70/F90 string
requirements.
Example: for 'F10.0' use: width = 10, dec = False
for 'F5.2' use: width = 5, dec = 2
Parameters
----------
num : floats
number to be transformed to string.
width : int
number of characters that number will occupy.
dec : int (optional)
number of decimals to include. When none passed, defaults to returning
as many decimals as fit within specified width.
space : int (optional)
blank characters to keep between adjecent numbers
THIS ARGUMENT WILL ONLY BE USED IF DEC = FALSE.
eng : bool (optional)
if true, it will use scientific format instead of floating point format
defaults to false
Returns
-------
str_out : str
number transformed into a string based on specified formatting.
Notes
-----
* Careful that the width and decimals specified work for the given number,
otherwise, functions will raise exceptions.
* Probably not the smartest way of doing things, but I'm tired and grumpy.
'''
# If engineering, format as such
if eng:
kwargs = {'trim': '.', 'sign': False, 'exp_digits': 0}
len_exp = 2 + len(str(int(abs(np.log(num))))) # length of exponent
if dec:
# Raise error if there isnt enough space for the number
if width - dec - len_exp - 2 < 0:
raise Exception('cant format eng number; does not fit :( ')
kwargs.update({'precision': dec})
else:
# Raise error if there isnt enough space for the number
if width - space - len_exp - 2 < 0:
raise Exception('cant format eng number; does not fit :( ')
# Format number
len_dec = width - len_exp - space - 2 # Available space for decimal
if num < 0: len_dec -= 1
kwargs.update({'precision': len_dec})
len_all = len(np.format_float_scientific(num, **kwargs))
# Correct for missing padding
pad = width - len_all
kwargs.update({'pad_left': pad})
str_out = np.format_float_scientific(num, **kwargs)
# Otherwise, format as floating point number
else:
# Round if required
if dec:
num = np.round(num, dec)
# Determine number of digits
digits = len(str(int(num)))
# Raise error if digits don't fit in specified width
if digits > width:
raise Exception('Digits > width | Use scientific?')
# Raise error if digits + dec + space don't fit in specified width
if digits + dec + space + 1 > width:
raise Exception('Digits + dec + space > width | Use scientific?')
# Specify lpad and prec if exact digits were required
if dec:
prec = dec
lpad = max(0, width - prec - 1)
# Specify lpad and prec if there are no decimals in digit
elif digits == len(str(num)):
lpad = width - space
prec = 0
# Specify lapd and prec if there are decimals
else:
prec = max(0, width - space - digits - 1)
lpad = max(0, width - prec - 1)
# Convert digit to string (note that pad includes digits!!) and output
frmt = {
'precision': prec,
'pad_left': lpad,
'unique': True,
'trim': '-'
}
str_out = np.format_float_positional(num, **frmt)
lpad += width - len(str_out)
frmt.update({'pad_left': lpad})
str_out = np.format_float_positional(num, **frmt)
return (str_out)
def format_float(value):
return np.format_float_positional(value, unique=True, trim="0")
def tls_search(time,
flux,
flux_err,
known_transits=None,
tls_kwargs=None,
wotan_kwargs=None,
options=None):
'''
Summary:
-------
This runs TLS on these data with the given infos
Inputs:
-------
time : array of flaot
time stamps of observations
flux : array of flaot
normalized flux
flux_err : array of flaot
error of normalized flux
Optional Inputs:
----------------
tls_kwargs : None or dict, keywords:
R_star : float
radius of the star (e.g. median)
default 1 R_sun (from TLS)
R_star_min : float
minimum radius of the star (e.g. 1st percentile)
default 0.13 R_sun (from TLS)
R_star_max : float
maximum radius of the star (e.g. 99th percentile)
default 3.5 R_sun (from TLS)
M_star : float
mass of the star (e.g. median)
default 1. M_sun (from TLS)
M_star_min : float
minimum mass of the star (e.g. 1st percentile)
default 0.1 M_sun (from TLS)
M_star_max : float
maximum mass of the star (e.g. 99th percentile)
default 1. M_sun (from TLS)
u : list
quadratic limb darkening parameters
default [0.4804, 0.1867]
...
SNR_threshold : float
the SNR threshold at which to stop the TLS search
known_transits : None or dict
if dict and one transit is already known:
known_transits = {'period':[1.3], 'duration':[2.1], 'epoch':[245800.0]}
if dict and multiple transits are already known:
known_transits = {'name':['b','c'], 'period':[1.3, 21.0], 'duration':[2.1, 4.1], 'epoch':[245800.0, 245801.0]}
'period' is the period of the transit
'duration' must be the total duration, i.e. from first ingress point to last egrees point, in days
'epoch' is the epoch of the transit
options : None or dict, keywords:
show_plot : bool
show a plot of each phase-folded transit candidate and TLS model in the terminal
default is False
save_plot : bool
save a plot of each phase-folded transit candidate and TLS model into outdir
default is False
outdir : string
if None, use the current working directory
default is ""
Returns:
-------
List of all TLS results
'''
#::: seeed
np.random.seed(42)
#::: handle inputs
if flux_err is None:
ind = np.where(~np.isnan(time * flux))[0]
time = time[ind]
flux = flux[ind]
else:
ind = np.where(~np.isnan(time * flux * flux_err))[0]
time = time[ind]
flux = flux[ind]
flux_err = flux_err[ind]
time_input = 1. * time
flux_input = 1. * flux #for plotting
if wotan_kwargs is None:
detrend = False
else:
detrend = True
if 'slide_clip' not in wotan_kwargs: wotan_kwargs['slide_clip'] = {}
if 'window_length' not in wotan_kwargs['slide_clip']:
wotan_kwargs['slide_clip']['window_length'] = 1.
if 'low' not in wotan_kwargs['slide_clip']:
wotan_kwargs['slide_clip']['low'] = 20.
if 'high' not in wotan_kwargs['slide_clip']:
wotan_kwargs['slide_clip']['high'] = 3.
if 'flatten' not in wotan_kwargs: wotan_kwargs['flatten'] = {}
if 'method' not in wotan_kwargs['flatten']:
wotan_kwargs['flatten']['method'] = 'biweight'
if 'window_length' not in wotan_kwargs['flatten']:
wotan_kwargs['flatten']['window_length'] = 1.
#the rest is filled automatically by Wotan
if tls_kwargs is None: tls_kwargs = {}
if 'show_progress_bar' not in tls_kwargs:
tls_kwargs['show_progress_bar'] = False
if 'SNR_threshold' not in tls_kwargs: tls_kwargs['SNR_threshold'] = 5.
if 'SDE_threshold' not in tls_kwargs: tls_kwargs['SDE_threshold'] = 5.
if 'FAP_threshold' not in tls_kwargs: tls_kwargs['FAP_threshold'] = 0.05
tls_kwargs_original = {
key: tls_kwargs[key]
for key in tls_kwargs.keys()
if key not in ['SNR_threshold', 'SDE_threshold', 'FAP_threshold']
} #for the original tls
#the rest is filled automatically by TLS
if options is None: options = {}
if 'show_plot' not in options: options['show_plot'] = False
if 'save_plot' not in options: options['save_plot'] = False
if 'outdir' not in options: options['outdir'] = ''
#::: init
SNR = 1e12
SDE = 1e12
FAP = 0
FOUND_SIGNAL = False
results_all = []
if len(options['outdir']) > 0 and not os.path.exists(options['outdir']):
os.makedirs(options['outdir'])
#::: logprint
with open(os.path.join(options['outdir'], 'logfile.log'), 'w') as f:
f.write('TLS search, UTC ' +
datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') + '\n')
logprint('\nWotan kwargs:', options=options)
logpprint(wotan_kwargs, options=options)
logprint('\nTLS kwargs:', options=options)
logpprint(tls_kwargs, options=options)
logprint('\nOptions:', options=options)
logpprint(options, options=options)
#::: apply a mask (if wished so)
if known_transits is not None:
for period, duration, T0 in zip(known_transits['period'],
known_transits['duration'],
known_transits['epoch']):
time, flux, flux_err = mask(time, flux, flux_err, period, duration,
T0)
#::: global sigma clipping
flux = sigma_clip(flux, sigma_upper=3, sigma_lower=20)
#::: detrend (if wished so)
if detrend:
flux = slide_clip(time, flux, **wotan_kwargs['slide_clip'])
flux, trend = flatten(time,
flux,
return_trend=True,
**wotan_kwargs['flatten'])
if options['show_plot'] or options['save_plot']:
fig, axes = plt.subplots(2, 1, figsize=(40, 8))
axes[0].plot(time, flux_input, 'b.', rasterized=True)
axes[0].plot(time, trend, 'r-', lw=2)
axes[0].set(ylabel='Flux (input)', xticklabels=[])
axes[1].plot(time, flux, 'b.', rasterized=True)
axes[1].set(ylabel='Flux (detrended)', xlabel='Time (BJD)')
if options['save_plot']:
fig.savefig(os.path.join(
options['outdir'],
'flux_' + wotan_kwargs['flatten']['method'] + '.pdf'),
bbox_inches='tight')
if options['show_plot']:
plt.show(fig)
else:
plt.close(fig)
X = np.column_stack((time, flux, flux_err, trend))
np.savetxt(os.path.join(
options['outdir'],
'flux_' + wotan_kwargs['flatten']['method'] + '.csv'),
X,
delimiter=',',
header='time,flux_detrended,flux_err,trend')
time_detrended = 1. * time
flux_detrended = 1. * flux #for plotting
#::: search for the rest
i = 0
ind_trs = []
while (SNR >= tls_kwargs['SNR_threshold']
) and (SDE >= tls_kwargs['SDE_threshold']) and (
FAP <= tls_kwargs['FAP_threshold']) and (FOUND_SIGNAL == False):
model = tls(time, flux, flux_err)
results = model.power(**tls_kwargs_original)
if (results.snr >= tls_kwargs['SNR_threshold']) and (
results.SDE >= tls_kwargs['SDE_threshold']) and (
results.FAP <= tls_kwargs['FAP_threshold']):
#::: calculcate the correct_duration, as TLS sometimes returns unreasonable durations
ind_tr_phase = np.where(results['model_folded_model'] < 1.)[0]
correct_duration = results['period'] * (
results['model_folded_phase'][ind_tr_phase[-1]] -
results['model_folded_phase'][ind_tr_phase[0]])
#::: mark transit
ind_tr, ind_out = index_transits(time_input, results['T0'],
results['period'],
correct_duration)
ind_trs.append(ind_tr)
#::: mask out detected transits and append results
time1, flux1 = time, flux #for plotting
time, flux, flux_err = mask(time, flux, flux_err, results.period,
np.max((1.5 * correct_duration)),
results.T0)
results_all.append(results)
#::: write TLS stats to file
with open(
os.path.join(options['outdir'],
'tls_signal_' + str(i) + '.txt'),
'wt') as out:
pprint(results, stream=out)
#::: individual TLS plots
if options['show_plot'] or options['save_plot']:
fig = plt.figure(figsize=(20, 8), tight_layout=True)
gs = fig.add_gridspec(2, 3)
ax = fig.add_subplot(gs[0, :])
ax.plot(time1, flux1, 'b.', rasterized=True)
ax.plot(results['model_lightcurve_time'],
results['model_lightcurve_model'],
'r-',
lw=3)
ax.set(xlabel='Time (BJD)', ylabel='Flux')
ax = fig.add_subplot(gs[1, 0])
ax.plot(results['folded_phase'],
results['folded_y'],
'b.',
rasterized=True)
ax.plot(results['model_folded_phase'],
results['model_folded_model'],
'r-',
lw=3)
ax.set(xlabel='Phase', ylabel='Flux')
ax = fig.add_subplot(gs[1, 1])
ax.plot(
(results['folded_phase'] - 0.5) * results['period'] * 24,
results['folded_y'],
'b.',
rasterized=True)
ax.plot((results['model_folded_phase'] - 0.5) *
results['period'] * 24,
results['model_folded_model'],
'r-',
lw=3)
ax.set(xlim=[
-1.5 * correct_duration * 24, +1.5 * correct_duration * 24
],
xlabel='Time (h)',
yticks=[])
ax = fig.add_subplot(gs[1, 2])
ax.text(.02,
0.95,
'P = ' +
np.format_float_positional(results['period'], 4) +
' d',
ha='left',
va='center',
transform=ax.transAxes)
ax.text(
.02,
0.85,
'Depth = ' +
np.format_float_positional(1e3 *
(1. - results['depth']), 4) +
' ppt',
ha='left',
va='center',
transform=ax.transAxes)
ax.text(.02,
0.75,
'Duration = ' +
np.format_float_positional(24 * correct_duration, 4) +
' h',
ha='left',
va='center',
transform=ax.transAxes)
ax.text(.02,
0.65,
'T_0 = ' +
np.format_float_positional(results['T0'], 4) + ' d',
ha='left',
va='center',
transform=ax.transAxes)
ax.text(.02,
0.55,
'SNR = ' +
np.format_float_positional(results['snr'], 4),
ha='left',
va='center',
transform=ax.transAxes)
ax.text(.02,
0.45,
'SDE = ' +
np.format_float_positional(results['SDE'], 4),
ha='left',
va='center',
transform=ax.transAxes)
ax.text(.02,
0.35,
'FAP = ' +
np.format_float_scientific(results['FAP'], 4),
ha='left',
va='center',
transform=ax.transAxes)
ax.set_axis_off()
if options['save_plot']:
fig.savefig(os.path.join(options['outdir'],
'tls_signal_' + str(i) + '.pdf'),
bbox_inches='tight')
if options['show_plot']:
plt.show(fig)
else:
plt.close(fig)
SNR = results.snr
SDE = results.SDE
FAP = results.FAP
i += 1
#::: full lightcurve plot
if options['show_plot'] or options['save_plot']:
if detrend:
fig, axes = plt.subplots(2, 1, figsize=(40, 8), tight_layout=True)
ax = axes[0]
ax.plot(time_input,
flux_input,
'k.',
color='grey',
rasterized=True)
ax.plot(time_input, trend, 'r-', lw=2)
for number, ind_tr in enumerate(ind_trs):
ax.plot(time_input[ind_tr],
flux_input[ind_tr],
marker='.',
linestyle='none',
label='signal ' + str(number))
ax.set(ylabel='Flux (input)', xticklabels=[])
ax.legend()
ax = axes[1]
ax.plot(time_detrended,
flux_detrended,
'k.',
color='grey',
rasterized=True)
for number, ind_tr in enumerate(ind_trs):
ax.plot(time_detrended[ind_tr],
flux_detrended[ind_tr],
marker='.',
linestyle='none',
label='signal ' + str(number))
ax.set(ylabel='Flux (detrended)', xlabel='Time (BJD)')
ax.legend()
else:
fig = plt.figure(figsize=(20, 4), tight_layout=True)
fig, ax = plt.subplots(1, 1, figsize=(40, 4))
ax.plot(time_input,
flux_input,
'k.',
color='grey',
rasterized=True)
ax.set(ylabel='Flux (input)', xlabel='Time (BJD)')
for number, ind_tr in enumerate(ind_trs):
ax.plot(time_input[ind_tr],
flux_input[ind_tr],
marker='.',
linestyle='none',
label='signal ' + str(number))
ax.legend()
if options['save_plot']:
fig.savefig(os.path.join(options['outdir'], 'tls_signal_all.pdf'),
bbox_inches='tight')
if options['show_plot']:
plt.show(fig)
else:
plt.close(fig)
return results_all
def match_ztf_message(job_info,
message_body,
message_time_epoch,
time_delta=10,
new_email_matching=False,
angular_separation=2):
'''
Check if the given email matches the information passed from the log file via job_info.
If a similar request was passed prior to the submission, you may need to use the
only_require_new_email parameter because the information won't exactly match.
In this case, look only for a close position, the relevant body text, and ensure that the
email was sent after the request.
'''
match = False
#
# Only continue if the message was received AFTER the job was submitted
#
if message_time_epoch < job_info['cdatetime'].to_list()[0]:
return match
message_lines = message_body.splitlines()
for line in message_lines:
#
# Incomplete data product
#
if re.search("A request similar to yours is waiting to be processed",
line):
match = False
break # Stop early if this isn't a finished data product
if re.search("reqid", line):
inputs = line.split('(')[-1]
# Two ways
# Processing has completed for reqid=XXXX ()
test_ra = inputs.split('ra=')[-1].split(',')[0]
test_decl = inputs.split('dec=')[-1].split(')')[0]
if re.search('minJD', line) and re.search('maxJD', line):
test_minjd = inputs.split('minJD=')[-1].split(',')[0]
test_maxjd = inputs.split('maxJD=')[-1].split(',')[0]
else:
test_minjd = inputs.split('startJD=')[-1].split(',')[0]
test_maxjd = inputs.split('endJD=')[-1].split(',')[0]
if new_email_matching:
# Call this a match only if parameters match
if np.format_float_positional(float(test_ra), precision=6, pad_right=6).replace(' ','0') == job_info['ra'].to_list()[0] and \
np.format_float_positional(float(test_decl), precision=6, pad_right=6).replace(' ','0') == job_info['dec'].to_list()[0] and \
(np.format_float_positional(float(test_minjd), precision=6, pad_right=6).replace(' ','0') == job_info['jdstart'].to_list()[0] and \
np.format_float_positional(float(test_maxjd), precision=6, pad_right=6).replace(' ','0') == job_info['jdend'].to_list()[0]) or ( \
float(test_minjd) - time_delta < float(job_info['jdstart'].to_list()[0]) and float(test_minjd) + time_delta > float(job_info['jdstart'].to_list()[0]) and \
float(test_maxjd) - time_delta < float(job_info['jdend'].to_list()[0]) and float(test_maxjd) + time_delta > float(job_info['jdend'].to_list()[0])):
match = True
else:
# Check if new and positions are similar
submitted_skycoord = SkyCoord(job_info["ra"],
job_info["dec"],
frame='icrs',
unit='deg')
email_skycoord = SkyCoord(test_ra,
test_decl,
frame='icrs',
unit='deg')
if submitted_skycoord.separation(email_skycoord).arcsecond < angular_separation and \
message_time_epoch > job_info['cdatetime'].to_list()[0]:
match = True
return match
def get_avg_daily_ac_power(self):
total = self.ac_power.values.sum()
avg = total / self.forecast_length
return np.format_float_positional(avg, precision=3)
def draw_basemap(fig, ax, df_lon, df_lat, list_season, seasons, bathy, gs, ss,
milo, malo, mila, mala, cont, which_method, gridsize):
# if bathy=True, plot bathy
ax = fig.sca(ax)
m = Basemap(projection='stere',lon_0=(milo+malo)/2,lat_0=(mila+mala)/2,lat_ts=0,llcrnrlat=mila,urcrnrlat=mala,\
llcrnrlon=milo,urcrnrlon=malo,rsphere=6371200.,resolution='l',area_thresh=100)
#m.drawcoastlines()
m.fillcontinents(color='grey', lake_color='grey')
if (ax == ax1) or (ax == ax3): #draw parallels
parallels = np.arange(mila, mala, 3)
m.drawparallels(parallels,
labels=[1, 0, 0, 0],
fontsize=10,
linewidth=0.0)
if (ax == ax3) or (ax == ax4): # draw meridians
meridians = np.arange(milo + 2, malo, 4)
m.drawmeridians(meridians,
labels=[0, 0, 0, 1],
fontsize=10,
linewidth=0.0)
if which_method == 'scatter':
a = m.scatter(df_lon,
df_lat,
c=list_season,
cmap='coolwarm',
marker='o',
linewidths=0.01,
latlon=True)
else:
print('binning data')
xi = np.arange(milo, malo, gridsize) # longitude grid
yi = np.arange(mila, mala, gridsize) # latitude grid
X, Y, Z = sh_bindata(df_lon, df_lat, list_season, xi, yi)
X, Y = m(X.flatten(), Y.flatten())
print('finally adding scatter plot ')
a = m.scatter(X, Y, c=Z, cmap='coolwarm', marker='o', linewidths=0.01)
a.set_clim(-1 * maxdiff_ignore, maxdiff_ignore)
if bathy == True: # get some detail bathymetry from USGS
add_isobath(m, gs, ss, cont)
mx, my = m(milo + .2, mala - .5)
#plt.text(mx+50000,my-110000,seasons,color='w',fontsize=18,fontweight='bold')
plt.text(mx, my, seasons, color='w', fontsize=18, fontweight='bold')
plt.text(mx,
my - 100000,
'mean=' + str(
np.format_float_positional(np.mean(list_season),
precision=2,
unique=False,
fractional=False)),
color='w',
fontsize=14,
fontweight='bold')
plt.text(mx,
my - 180000,
'RMS = ' + str(
np.format_float_positional(np.sqrt(
np.mean(np.square(list_season))),
precision=2,
unique=False,
fractional=False)),
color='w',
fontsize=14,
fontweight='bold')
return a
def test_format_float_positional_python_scalar(self):
x = 1.0
assert cupy.format_float_positional(
x) == numpy.format_float_positional(x)
def test_format_float_positional(self):
a = testing.shaped_arange((1, ), cupy)
b = testing.shaped_arange((1, ), numpy)
assert cupy.format_float_positional(
a) == numpy.format_float_positional(b)
def format_float(x):
formatted = numpy.format_float_positional(x, unique=True)
if formatted[-1] == '.':
return formatted + '0'
return formatted
positive_scalar_factor=1,
negative_scalar_factor=1)
grid.run_value_iteration()
reward = grid.reward[1:4, 1:5]
value = grid.state_value_function[1:4, 1:5]
policy = grid.policy[1:4, 1:5]
print("State Value function = \n", value)
print("Action Value function = \n", grid.action_value_function[1:4,
1:5, :])
print("Policy = \n", policy)
fig1 = plt.figure(1, figsize=(16, 6))
ax1 = fig1.add_subplot(121)
im1 = ax1.imshow(policy, cmap='rainbow')
for (j, i), label in np.ndenumerate(policy):
if label == -1:
label_text = np.format_float_positional(reward[j, i], precision=3)
elif label == 0:
label_text = '\u27a1'
elif label == 1:
label_text = '\u2b07'
elif label == 2:
label_text = '\u2b05'
else:
label_text = '\u2b06'
ax1.text(i, j, label_text, ha='center', va='center', fontsize=28)
ax2 = fig1.add_subplot(122)
im2 = ax2.imshow(value, cmap='rainbow')
for (j, i), label in np.ndenumerate(value):
label_text = np.format_float_positional(label, precision=3)
ax2.text(i, j, label_text, ha='center', va='center', fontsize=28)
# plt.title('Reward and state value function', x=0, y=1.1, fontdict = {'fontsize' : 15})
drawnow(makeFig) # plot average after window
ser = None # Closes port
print('Returning to Main Menu...')
state = 0
elif state == 3:
print('-------------Help-------------')
print(
'This program can receive/send at a certain baudrate information received using serial protocol'
)
print(
'To change baudrate and port, please change BAUD and COM respectively in code'
)
print('Returning to Main Menu...')
state = 0
elif state == 1:
coeff = list(filterDesign())
for i in range(Order):
coeff[i] = np.format_float_positional(coeff[i])
state = 5
elif state == 4:
state = -1
ser = None # Closes port
else:
print('Command not recognized')
print('Returning to Main Menu...')
# rename (year rpp_all_items) (rpp_all_items year)
# save "$int_data/rpp_msa_long.dta", replace
bea_msa_year_list = [int(x) for x in bea_rpp_msa.columns.to_list() if x.isdigit()]
bea_rpp_msa_processed = pd.DataFrame([], columns=['MSATitle', 'year', 'rpp_all_items', 'met2013'], dtype='object')
for geo_name in bea_rpp_msa.loc[:, 'GeoName'].unique():
for year in bea_msa_year_list:
raw_val = bea_rpp_msa[bea_rpp_msa['GeoName']==geo_name][str(year)].values[0]
val = None
if type(raw_val) == str:
try:
val = float(raw_val)
except:
bea_rpp_msa_processed = bea_rpp_msa_processed[bea_rpp_msa_processed['MSATitle'] != geo_name]
break
else:
val = np.format_float_positional(raw_val)
met2013 = bea_rpp_msa[bea_rpp_msa['GeoName']==geo_name]['GeoFIPS'].values[0].replace('"', '')
bea_rpp_msa_processed = bea_rpp_msa_processed.append({
"MSATitle": geo_name,
"year": year,
"rpp_all_items": val,
"met2013": met2013
}, ignore_index=True)
bea_rpp_msa_processed.loc[:, 'year'] = bea_rpp_msa_processed['year'].astype('int32')
bea_rpp_msa_processed.loc[:, 'met2013'] = bea_rpp_msa_processed['met2013'].astype('int32')
bea_rpp_msa_processed.loc[:, 'rpp_all_items'] = bea_rpp_msa_processed['rpp_all_items'].astype('float64')
bea_rpp_msa_processed.to_stata(os.path.join(int_data, 'rpp_msa_long_test.dta'), write_index=False)
# /*******************
# * Main Analysis
def output_edges_(edge, length, resolution):
str_edges = '<edges>\n'
r = length / (2 * pi)
edgelen = length / 4.
edges = [{
"id":
"bottom",
"type":
"edgeType",
"from":
"bottom",
"to":
"right",
"length":
edgelen,
"shape":
[(r * cos(t), r * sin(t)) for t in linspace(-pi / 2, 0, resolution)]
}, {
"id":
"right",
"type":
"edgeType",
"from":
"right",
"to":
"top",
"length":
edgelen,
"shape":
[(r * cos(t), r * sin(t)) for t in linspace(0, pi / 2, resolution)]
}, {
"id":
"top",
"type":
"edgeType",
"from":
"top",
"to":
"left",
"length":
edgelen,
"shape":
[(r * cos(t), r * sin(t)) for t in linspace(pi / 2, pi, resolution)]
}, {
"id":
"left",
"type":
"edgeType",
"from":
"left",
"to":
"bottom",
"length":
edgelen,
"shape": [(r * cos(t), r * sin(t))
for t in linspace(pi, 3 * pi / 2, resolution)]
}]
for e in edges:
shape = ""
for t in range(len(e['shape'])):
shape += '%s,%s' % (
np.format_float_positional(e['shape'][t][0], trim='-'),
np.format_float_positional(e['shape'][t][1], trim='-'))
if t < len(e['shape']) - 1:
shape += ' '
str_edges += get_edge_str_(edge, e['id'], e['from'], e['to'], 'a',
shape)
str_edges += '</edges>\n'
print('Number of edges: ', len(edges))
return str_edges
def obtusify(val, idims, SI, pref, loops = 2, maxvalord = None, minvalord = None, maxprefs = None, spread = None):
'''
Given a quantity in SI base units, returns an equivalent value in an obtuse combination of
derived SI units and prefixes
Args:
val -- float : numerical value of quantity in base units
idims -- (4,) : tuple corresponding to dimensionality of input quantity (M,L,T,I)
SI -- (n,5) struct ndarray : information on SI derived units, see config
pref -- (n,2) struct ndarray : indicating a set of SI prefixes to superfluously add
Kwargs:
loops -- int : Number of derived units to be present in the obfuscation. Set it irresponsibly high!
maxvalord -- int : If not None, maximum order of output numeric value
minvalord -- int : If not None, minimum order of output numeric value
maxprefs -- int : If not None, maximum number of SI prefixes to include in output
spread -- float [0,1) : If not None, creates a bias towards including more individual unit names
in the output (vs. less unique names with higher powers). Strong interaction with loops.
Returns:
ostring -- string : quantity equal to input, expressed obtusely
Sample Usage:
# Obtusification of 8024 metres
obtusify(8024, (0,1,0,0), config.SI, config.pref) --> "8.024 microsievert-kilograms per nanonewton"
'''
#Basic input checks
if maxvalord == None: maxvalord = np.inf
if minvalord == None: minvalord = -np.inf
if maxvalord - minvalord < 0:
print("You set the output value order minimum higher than the maximum. Don't think I don't see you trying to crash the prefix algorithm.")
raise Exception
#Pick a random non-zero positive dimension
ind = random.choice([i for i,x in enumerate(idims) if x > 0])
#Pick a random SI derived unit which is nonzero positive in this dimension (ensures at least one unit on numerator)
SIind = random.choice([i for i,x in enumerate(SI['dims']) if x[ind] > 0])
SI['count'][SIind] += 1
#Check which dimensions remain to be filled
wSI = [np.array(x)*y for x,y in zip(SI['dims'], SI['count'])]
remainder = idims - np.sum(wSI, axis=0)
#Loop while dimensions are not matched
i = 0
while i < loops:
i += 1
#Find the derived unit which would most decrease the remainder if added to the numerator
diffs = np.sum(np.abs([list(x) - remainder for x in SI['dims']]), axis=1).astype(np.float32)
diffs[SI['count'] < 0] = np.nan #Ignore options which would simply cancel out existing units
n,ndiff = np.where(diffs == np.nanmin(diffs))[0][0], np.nanmin(diffs)
#Find the derived unit which would most decrease the remainder if added to the denominator
diffs = np.sum(np.abs([list(x) + remainder for x in SI['dims']]), axis=1).astype(np.float32)
diffs[SI['count'] > 0] = np.nan
d,ddiff = np.where(diffs == np.nanmin(diffs))[0][0], np.nanmin(diffs) #when added to the denominator
#Sometimes selecting the unit which best moves the dimensionality towards the target actually produces
# lackluster results, e.g. feedback loops of two units undoing eachother around the target.
#These options are ways to introduce bias for more interesting results
if spread is not None and loops - i > 3:
#Adds a chance to instead select a random unused unit if possible
if np.random.rand() > spread and 0 in SI['count']:
uu = random.choice([i for i,x in enumerate(SI['count']) if x == 0])
if np.random.rand() > .5:
SI['count'][uu] += 1 #Slap it on the numerator
else:
SI['count'][uu] -= 1 #Slap it on the denominator
else:
if ndiff <= ddiff:
SI['count'][n] += 1
else:
SI['count'][d] -= 1
#Check which dimensions remain to be filled
wSI = [np.array(x)*y for x,y in zip(SI['dims'], SI['count'])]
remainder = idims - np.sum(wSI, axis=0)
#Assign SI prefixes
nslots = np.sum(SI['count'] > 0) + np.sum(remainder > 0) #Number of slots available on numerator
dslots = np.sum(SI['count'] < 0) + np.sum(remainder < 0) #Number of slots available on denominator
pn = np.full(nslots, None)
pd = np.full(dslots, None)
#Bound the number of prefixes to be added
if maxprefs is not None:
nempty, dempty = np.sum(pn == None), np.sum(pd == None)
while nempty + dempty > maxprefs:
#Fill a random slot with an empty prefix (uniform selection across all slots)
ratio = nempty/(nempty + dempty)
if np.random.rand() < ratio:
i = [i for i,x in enumerate(pn) if x == None]
pn[random.choice(i)] = 0
else:
i = [i for i,x in enumerate(pd) if x == None]
pd[random.choice(i)] = 0
nempty, dempty = np.sum(pn == None), np.sum(pd == None)
#Assign arbitrary prefixes to open slots slots, avoiding dupes
for i in [i for i,x in enumerate(pn) if x == None]:
pn[i] = random.choice(pref['mag'])
for i in [i for i,x in enumerate(pd) if x == None]:
pd[i] = random.choice(pref['mag'])
#Unit exponents (and thus prefix powers)
en = [ i for i in SI['count'][SI['count'] > 0]] #Derived units
en += [ i for i in remainder if i > 0] #Base units
ed = [-i for i in SI['count'][SI['count'] < 0]] #Derived units
ed += [-i for i in remainder if i < 0] #Base units
#Lower/raise prefixes until within user tolerance
current_ord = sum([p*e for p,e in zip(pn,en)]) - sum([p*e for p,e in zip(pd,ed)]) - np.floor(np.log10(val))
ratio = len(pn)/ (len(pn) + len(pd))
numprobs = [e if p!=0 else 0 for e,p in zip(en,pn)] #Favour modifying high power values (no one wnats to see "yoctometres octed" (I kinda do though))
numprobs = np.array(numprobs)/np.sum(np.abs(numprobs)) # Additionally, don't select empty prefixes for change to prevent conflict with maxprefs
denprobs = [e if p!=0 else 0 for e,p in zip(ed,pd)] #Favour modifying high power values (no one wnats to see "yoctometres octed" (I kinda do though))
denprobs = np.array(denprobs)/np.sum(np.abs(denprobs)) # Additionally, don't select empty prefixes for change to prevent conflict with maxprefs
t = 0
while current_ord > maxvalord or current_ord < minvalord:
t+=1
if t>100:
print("Error, unable to meet user tolerance for output value magnitude within %i iteration attempts"%t)
print("Keywords minvalord/maxvalord most likely define an extremely narrow range")
raise Exception()
if current_ord > maxvalord:
if np.random.rand() < ratio:
#Go down one prefix in a numerator slot
i = np.random.choice(range(len(pn)), p=numprobs)
newprefs = pref['mag'][pref['mag'] < pn[i]]
if len(newprefs) == 0: continue #If there is no such prefix, reiterate
pn[i] = min(newprefs, key=lambda x:abs(x-pn[i]))
else:
#Go up on prefix in a denominator slot
i = np.random.choice(range(len(pd)), p=denprobs)
newprefs = pref['mag'][pref['mag'] > pd[i]]
if len(newprefs) == 0: continue
pd[i] = min(newprefs, key=lambda x:abs(x-pd[i]))
else:
if np.random.rand() < ratio:
#Go up one prefix in a numerator slot
i = np.random.choice(range(len(pn)), p=numprobs)
newprefs = pref['mag'][pref['mag'] > pn[i]]
if len(newprefs) == 0: continue
pn[i] = min(newprefs, key=lambda x:abs(x-pn[i]))
else:
#Go down on prefix in a denominator slot
i = np.random.choice(range(len(pd)), p=denprobs)
newprefs = pref['mag'][pref['mag'] < pd[i]]
if len(newprefs) == 0: continue
pd[i] = min(newprefs, key=lambda x:abs(x-pd[i]))
current_ord = sum([p*e for p,e in zip(pn,en)]) - sum([p*e for p,e in zip(pd,ed)]) - np.floor(np.log10(val))
#Calculate new numerical value of quantity
oval = val / (10**np.floor(np.log10(val))) * 10**-current_ord
#Convert numerical prefix orders to their corresponding strings
pn = [pref['name'][pref['mag'] == i][0] for i in pn]
pd = [pref['name'][pref['mag'] == i][0] for i in pd]
if len(pd) > 0: pd[0] = ' per ' + pd[0]
#Convert exponent values to their corresponding strings
enames = {1:'', 2:'s-squared', 3:'s-cubed', 4:'s-quarted', 5: 's-quinted', 6: 's-sexted', 7: 's-hepted', 8: 's-octed',
9: 'nonned????', 10: 'dude', 11: 'stop', 12: 'the', 13 : 'words', 14: ' don\'t', 15: 'even', 16: 'go', 17: 'this', 18:'high'} #If you get a key error on this you requested a BEEG obfuscation
en = [enames[i] for i in en]
ed = [enames[i] for i in ed]
#Unit strings
bnames = {0:'kilogram', 1: 'metre', 2: 'second', 3: 'ampere'}
un = list(SI['uname'][SI['count'] > 0]) #Derived units
un += [bnames[i] for i,x in enumerate(remainder) if x > 0] #Base units
ud = list(SI['uname'][SI['count'] < 0])
ud += [bnames[i] for i,x in enumerate(remainder) if x < 0]
#Construct final value string
ustring = ''
for p,u,e in zip(pn, un, en):
ustring += '-%s%s%s'%(p,u,e)
ustring += 's' #Pluralize
for p,u,e in zip(pd, ud, ed):
ustring += '%s%s%s-'%(p,u,e)
ustring = ustring[1:] #Chop off superfluous hyphens
if len(pd) != 0:
ustring = ustring[:-1]
#Round oval for readability
oval = np.format_float_positional(oval, precision=4, unique=False, fractional=False, trim='k')
ostring = str(oval) + ' ' + ustring
#Cleanup plurals
ostring = ostring.replace('hertzs', 'hertz')
ostring = ostring.replace('henrys', 'henries')
return ostring
def run(self):
model_name = self.transaction.persistent_model_metadata.model_name
self.train_start_time = time.time()
self.session.logging.info(
'Training: model {model_name}, epoch 0'.format(
model_name=model_name))
self.last_time = time.time()
# We moved everything to a worker so we can run many of these in parallel
# Todo: use Ray https://github.com/ray-project/tutorial
ret_diffs = PredictWorker.start(self.transaction.model_data,
model_name=model_name)
confusion_matrices = self.transaction.persistent_ml_model_info.confussion_matrices
self.transaction.output_data.columns = self.transaction.input_data.columns
# TODO: This may be inneficient, try to work on same pointer
self.transaction.output_data.data_array = self.transaction.input_data.data_array
self.transaction.output_data.predicted_columns = self.transaction.metadata.model_predict_columns
for diff in ret_diffs:
for col in diff['ret_dict']:
confusion_matrix = confusion_matrices[col]
col_index = self.transaction.input_data.columns.index(col)
self.transaction.output_data.columns.insert(
col_index + 1, KEY_CONFIDENCE)
offset = diff['start_pointer']
group_pointer = diff['group_pointer']
column_pointer = diff['column_pointer']
for j, cell in enumerate(diff['ret_dict'][col]):
#TODO: This may be calculated just as j+offset
if not cell:
continue
actual_row = self.transaction.model_data.predict_set_map[
group_pointer][j + offset]
if not self.transaction.output_data.data_array[actual_row][
col_index] or self.transaction.output_data.data_array[
actual_row][col_index] == '':
if self.transaction.persistent_model_metadata.column_stats[
col][KEYS.DATA_TYPE] == DATA_TYPES.NUMERIC:
target_val = np.format_float_positional(
cell, precision=2)
else:
target_val = cell
self.transaction.output_data.data_array[actual_row][
col_index] = target_val
confidence = self.getConfidence(cell, confusion_matrix)
#randConfidence = random.uniform(0.85, 0.93)
self.transaction.output_data.data_array[
actual_row].insert(col_index + 1, confidence)
else:
self.transaction.output_data.data_array[
actual_row].insert(col_index + 1, 1.0)
total_time = time.time() - self.train_start_time
self.session.logging.info(
'Trained: model {model_name} [OK], TOTAL TIME: {total_time:.2f} seconds'
.format(model_name=model_name, total_time=total_time))
pass
def cross_validation(
estimator,
estimator_name,
X,
y,
metrics,
) -> List[str]:
"""Run cross validation to the chosen estimator.
:param estimator: object to which apply fit and predict, to obtain results.
:param estimator_name: name of the chosen estimator.
:param X: dataset values without the labels.
:param y: dataset labels.
:param metrics: metrics which will be applied to the estimator.
:returns: a dictionary with the obtained results.
"""
# pylint: disable=too-many-locals,invalid-name,redefined-outer-name
DGPLOGGER.debug("Training %s...", estimator_name)
DGPLOGGER.debug("Estimator: %s", str(estimator))
result = {"estimator": estimator_name}
result["fit_times"] = np.zeros((N_SPLITS, ))
for metric_name in metrics.keys():
result[f"train_{metric_name}"] = np.zeros((N_SPLITS, ))
result[f"test_{metric_name}"] = np.zeros((N_SPLITS, ))
for fold, (train_idx, test_idx) in enumerate(K_FOLD.split(X=X, y=y)):
DGPLOGGER.info("Fold %d/%d", fold + 1, N_SPLITS)
X_train, y_train = X[train_idx], y[train_idx]
X_test, y_test = X[test_idx], y[test_idx]
DGPLOGGER.debug("Estimator fit...")
start_time = time.perf_counter()
estimator.fit(X_train, y_train)
fit_time = time.perf_counter() - start_time
DGPLOGGER.debug("Estimator fit... Done in %.5f seconds", fit_time)
result["fit_times"][fold] = fit_time
DGPLOGGER.debug("Predicting train labels...")
y_pred_train = estimator.predict(X_train)
DGPLOGGER.debug("Predicting test labels...")
y_pred_test = estimator.predict(X_test)
for metric_name, (func, kwargs) in metrics.items():
DGPLOGGER.debug("Obtaining train metric: %s...", metric_name)
result[f"train_{metric_name}"][fold] = func(
y_train, y_pred_train, **kwargs)
DGPLOGGER.debug(
"Obtaining train metric: %s... Done: %s",
metric_name,
result[f"train_{metric_name}"][fold],
)
DGPLOGGER.debug("Obtaining test metric: %s...", metric_name)
result[f"test_{metric_name}"][fold] = func(y_test, y_pred_test,
**kwargs)
DGPLOGGER.debug(
"Obtaining test metric: %s... Done: %s",
metric_name,
result[f"test_{metric_name}"][fold],
)
DGPLOGGER.debug("Training %s... Done", estimator_name)
output_list = [
result.pop("estimator"),
np.format_float_positional(result.pop("fit_times").mean(),
precision=2,
unique=False),
]
for metric_name in sorted(metrics.keys()):
output_list.append(
np.format_float_positional(
result[f"train_{metric_name}"].mean(),
precision=6,
unique=False,
))
output_list.append(
np.format_float_positional(result[f"train_{metric_name}"].std(),
precision=6,
unique=False))
output_list.append(
np.format_float_positional(result[f"test_{metric_name}"].mean(),
precision=6,
unique=False))
output_list.append(
np.format_float_positional(result[f"test_{metric_name}"].std(),
precision=6,
unique=False))
return output_list
def test_dragon4(self):
# these tests are adapted from Ryan Juckett's dragon4 implementation,
# see dragon4.c for details.
fpos32 = lambda x, **k: np.format_float_positional(np.float32(x), **k)
fsci32 = lambda x, **k: np.format_float_scientific(np.float32(x), **k)
fpos64 = lambda x, **k: np.format_float_positional(np.float64(x), **k)
fsci64 = lambda x, **k: np.format_float_scientific(np.float64(x), **k)
preckwd = lambda prec: {"unique": False, "precision": prec}
assert_equal(fpos32("1.0"), "1.")
assert_equal(fsci32("1.0"), "1.e+00")
assert_equal(fpos32("10.234"), "10.234")
assert_equal(fpos32("-10.234"), "-10.234")
assert_equal(fsci32("10.234"), "1.0234e+01")
assert_equal(fsci32("-10.234"), "-1.0234e+01")
assert_equal(fpos32("1000.0"), "1000.")
assert_equal(fpos32("1.0", precision=0), "1.")
assert_equal(fsci32("1.0", precision=0), "1.e+00")
assert_equal(fpos32("10.234", precision=0), "10.")
assert_equal(fpos32("-10.234", precision=0), "-10.")
assert_equal(fsci32("10.234", precision=0), "1.e+01")
assert_equal(fsci32("-10.234", precision=0), "-1.e+01")
assert_equal(fpos32("10.234", precision=2), "10.23")
assert_equal(fsci32("-10.234", precision=2), "-1.02e+01")
assert_equal(
fsci64("9.9999999999999995e-08", **preckwd(16)), "9.9999999999999995e-08"
)
assert_equal(
fsci64("9.8813129168249309e-324", **preckwd(16)), "9.8813129168249309e-324"
)
assert_equal(
fsci64("9.9999999999999694e-311", **preckwd(16)), "9.9999999999999694e-311"
)
# test rounding
# 3.1415927410 is closest float32 to np.pi
assert_equal(fpos32("3.14159265358979323846", **preckwd(10)), "3.1415927410")
assert_equal(
fsci32("3.14159265358979323846", **preckwd(10)), "3.1415927410e+00"
)
assert_equal(fpos64("3.14159265358979323846", **preckwd(10)), "3.1415926536")
assert_equal(
fsci64("3.14159265358979323846", **preckwd(10)), "3.1415926536e+00"
)
# 299792448 is closest float32 to 299792458
assert_equal(fpos32("299792458.0", **preckwd(5)), "299792448.00000")
assert_equal(fsci32("299792458.0", **preckwd(5)), "2.99792e+08")
assert_equal(fpos64("299792458.0", **preckwd(5)), "299792458.00000")
assert_equal(fsci64("299792458.0", **preckwd(5)), "2.99792e+08")
assert_equal(
fpos32("3.14159265358979323846", **preckwd(25)),
"3.1415927410125732421875000",
)
assert_equal(
fpos64("3.14159265358979323846", **preckwd(50)),
"3.14159265358979311599796346854418516159057617187500",
)
assert_equal(fpos64("3.14159265358979323846"), "3.141592653589793")
# smallest numbers
assert_equal(
fpos32(0.5 ** (126 + 23), unique=False, precision=149),
"0.00000000000000000000000000000000000000000000140129846432"
"4817070923729583289916131280261941876515771757068283889791"
"08268586060148663818836212158203125",
)
assert_equal(
fpos64(0.5 ** (1022 + 52), unique=False, precision=1074),
"0.00000000000000000000000000000000000000000000000000000000"
"0000000000000000000000000000000000000000000000000000000000"
"0000000000000000000000000000000000000000000000000000000000"
"0000000000000000000000000000000000000000000000000000000000"
"0000000000000000000000000000000000000000000000000000000000"
"0000000000000000000000000000000000049406564584124654417656"
"8792868221372365059802614324764425585682500675507270208751"
"8652998363616359923797965646954457177309266567103559397963"
"9877479601078187812630071319031140452784581716784898210368"
"8718636056998730723050006387409153564984387312473397273169"
"6151400317153853980741262385655911710266585566867681870395"
"6031062493194527159149245532930545654440112748012970999954"
"1931989409080416563324524757147869014726780159355238611550"
"1348035264934720193790268107107491703332226844753335720832"
"4319360923828934583680601060115061698097530783422773183292"
"4790498252473077637592724787465608477820373446969953364701"
"7972677717585125660551199131504891101451037862738167250955"
"8373897335989936648099411642057026370902792427675445652290"
"87538682506419718265533447265625",
)
# largest numbers
assert_equal(
fpos32(np.finfo(np.float32).max, **preckwd(0)),
"340282346638528859811704183484516925440.",
)
assert_equal(
fpos64(np.finfo(np.float64).max, **preckwd(0)),
"1797693134862315708145274237317043567980705675258449965989"
"1747680315726078002853876058955863276687817154045895351438"
"2464234321326889464182768467546703537516986049910576551282"
"0762454900903893289440758685084551339423045832369032229481"
"6580855933212334827479782620414472316873817718091929988125"
"0404026184124858368.",
)
# Warning: In unique mode only the integer digits necessary for
# uniqueness are computed, the rest are 0. Should we change this?
assert_equal(
fpos32(np.finfo(np.float32).max, precision=0),
"340282350000000000000000000000000000000.",
)
# test trailing zeros
assert_equal(fpos32("1.0", unique=False, precision=3), "1.000")
assert_equal(fpos64("1.0", unique=False, precision=3), "1.000")
assert_equal(fsci32("1.0", unique=False, precision=3), "1.000e+00")
assert_equal(fsci64("1.0", unique=False, precision=3), "1.000e+00")
assert_equal(fpos32("1.5", unique=False, precision=3), "1.500")
assert_equal(fpos64("1.5", unique=False, precision=3), "1.500")
assert_equal(fsci32("1.5", unique=False, precision=3), "1.500e+00")
assert_equal(fsci64("1.5", unique=False, precision=3), "1.500e+00")
# gh-10713
assert_equal(
fpos64("324", unique=False, precision=5, fractional=False), "324.00"
)
def ztf_forced_photometry(ra,
decl,
jdstart=None,
jdend=None,
days=60,
send=True,
verbose=True):
# Wget is required for the ZTF forced photometry request submission
wget_installed = wget_check()
if wget_installed == False:
return None
#
# Set dates
#
if jdend is None:
jdend = Time(datetime.utcnow(), scale='utc').jd
if jdstart is None:
jdstart = jdend - days
if ra is not None and decl is not None:
# Check if ra is a decimal
try:
# These will trigger the exception if they aren't float
float(ra)
float(decl)
skycoord = SkyCoord(ra, decl, frame='icrs', unit='deg')
# Else assume sexagesimal
except Exception:
skycoord = SkyCoord(ra,
decl,
frame='icrs',
unit=(u.hourangle, u.deg))
# Convert to string to keep same precision. This will make matching easier in the case of submitting multiple jobs.
jdend_str = np.format_float_positional(float(jdend), precision=6)
jdstart_str = np.format_float_positional(float(jdstart), precision=6)
ra_str = np.format_float_positional(float(skycoord.ra.deg),
precision=6)
decl_str = np.format_float_positional(float(skycoord.dec.deg),
precision=6)
log_file_name = random_log_file_name() # Unique file name
if verbose:
print(f"Sending ZTF request for (R.A.,Decl)=({ra},{decl})")
wget_command = f"wget --http-user={_ztfuser} --http-passwd={_ztfinfo} -O {log_file_name} \"https://ztfweb.ipac.caltech.edu/cgi-bin/requestForcedPhotometry.cgi?" + \
f"ra={ra_str}&" + \
f"dec={decl_str}&" + \
f"jdstart={jdstart_str}&" +\
f"jdend={jdend_str}&" + \
f"email={_ztffp_user_address}&userpass={_ztffp_user_password}\""
if verbose:
print(wget_command)
if send:
p = subprocess.Popen(wget_command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
stdout, stderr = p.communicate()
if verbose:
print(stdout.decode('utf-8'))
return log_file_name
else:
if verbose:
print("Missing necessary R.A. or declination.")
return None
def format_raw(raw):
output = np.format_float_positional(raw, precision=4, trim='-')
# doesn't seem to trim properly
if output[-1] == '.':
output = output[:-1]
return output
def pretty_print_decimal_places(lr):
return str(np.format_float_positional(lr).split('.')[1])
def numforce_float(num) -> float:
return enforce_float(
np.format_float_positional(num, unique=True, trim='-', precision=10))
def predict(self):
args = self.args
if args.verbosity:
print("Preparing model for prediction")
self.model_id = self.experiment_config["experiment"]["name"]
if not args.trials:
args.trials = os.path.join(self.cache_dir, self.model_id, "predictions", "trials")
if not args.scores:
args.scores = os.path.join(self.cache_dir, self.model_id, "predictions", "scores")
self.make_named_dir(os.path.dirname(args.trials))
self.make_named_dir(os.path.dirname(args.scores))
training_config = self.experiment_config["experiment"]
feat_config = self.experiment_config["features"]
if args.verbosity > 1:
print("Using model parameters:")
yaml_pprint(training_config)
print()
if args.verbosity > 1:
print("Using feature extraction parameters:")
yaml_pprint(feat_config)
print()
model = self.create_model(dict(training_config), skip_training=True)
if args.verbosity > 1:
print("Preparing model")
labels = self.experiment_config["dataset"]["labels"]
model.prepare(labels, training_config)
checkpoint_dir = self.get_checkpoint_dir()
if args.checkpoint:
checkpoint_path = os.path.join(checkpoint_dir, args.checkpoint)
elif "best_checkpoint" in self.experiment_config.get("prediction", {}):
checkpoint_path = os.path.join(checkpoint_dir, self.experiment_config["prediction"]["best_checkpoint"])
else:
checkpoints = os.listdir(checkpoint_dir) if os.path.isdir(checkpoint_dir) else []
if not checkpoints:
print("Error: Cannot evaluate with a model that has no checkpoints, i.e. is not trained.")
return 1
if "checkpoints" in training_config:
monitor_value = training_config["checkpoints"]["monitor"]
monitor_mode = training_config["checkpoints"].get("mode")
else:
monitor_value = "epoch"
monitor_mode = None
checkpoint_path = os.path.join(checkpoint_dir, models.get_best_checkpoint(checkpoints, key=monitor_value, mode=monitor_mode))
if args.verbosity:
print("Loading model weights from checkpoint file '{}'".format(checkpoint_path))
model.load_weights(checkpoint_path)
if args.verbosity:
print("\nEvaluating testset with model:")
print(str(model))
print()
ds = "test"
if args.verbosity > 2:
print("Dataset config for '{}'".format(ds))
yaml_pprint(training_config[ds])
ds_config = dict(training_config, **training_config[ds])
del ds_config["train"], ds_config["validation"]
if args.verbosity and "dataset_logger" in ds_config:
print("Warning: dataset_logger in the test datagroup has no effect.")
datagroup_key = ds_config.pop("datagroup")
datagroup = self.experiment_config["dataset"]["datagroups"][datagroup_key]
utt2path_path = os.path.join(datagroup["path"], datagroup.get("utt2path", "utt2path"))
utt2label_path = os.path.join(datagroup["path"], datagroup.get("utt2label", "utt2label"))
utt2path = collections.OrderedDict(
row[:2] for row in parse_space_separated(utt2path_path)
)
utt2label = collections.OrderedDict(
row[:2] for row in parse_space_separated(utt2label_path)
)
utterance_list = list(utt2path.keys())
if args.file_limit:
utterance_list = utterance_list[:args.file_limit]
if args.verbosity > 3:
print("Using utterance ids:")
yaml_pprint(utterance_list)
int2label = self.experiment_config["dataset"]["labels"]
label2int, OH = make_label2onehot(int2label)
def label2onehot(label):
return OH[label2int.lookup(label)]
labels_set = set(int2label)
paths = []
paths_meta = []
for utt in utterance_list:
label = utt2label[utt]
if label not in labels_set:
continue
paths.append(utt2path[utt])
paths_meta.append((utt, label))
if args.verbosity:
print("Extracting test set features for prediction")
features = self.extract_features(
feat_config,
"test",
trim_audio=False,
debug_squeeze_last_dim=(ds_config["input_shape"][-1] == 1),
)
conf_json, conf_checksum = config_checksum(self.experiment_config, datagroup_key)
features = tf_data.prepare_dataset_for_training(
features,
ds_config,
feat_config,
label2onehot,
self.model_id,
verbosity=args.verbosity,
conf_checksum=conf_checksum,
)
# drop meta wavs required only for vad
features = features.map(lambda *t: t[:3])
if ds_config.get("persistent_features_cache", True):
features_cache_dir = os.path.join(self.cache_dir, "features")
else:
features_cache_dir = "/tmp/tensorflow-cache"
features_cache_path = os.path.join(
features_cache_dir,
self.experiment_config["dataset"]["key"],
ds,
feat_config["type"],
conf_checksum,
)
self.make_named_dir(os.path.dirname(features_cache_path), "features cache")
if not os.path.exists(features_cache_path + ".md5sum-input"):
with open(features_cache_path + ".md5sum-input", "w") as f:
print(conf_json, file=f, end='')
if args.verbosity:
print("Writing features into new cache: '{}'".format(features_cache_path))
else:
if args.verbosity:
print("Loading features from existing cache: '{}'".format(features_cache_path))
features = features.cache(filename=features_cache_path)
if args.verbosity:
print("Gathering all utterance ids from features dataset iterator")
# Gather utterance ids, this also causes the extraction pipeline to be evaluated
utterance_ids = []
i = 0
if args.verbosity > 1:
print(now_str(date=True), "- 0 samples done")
for _, _, uttids in features.as_numpy_iterator():
for uttid in uttids:
utterance_ids.append(uttid.decode("utf-8"))
i += 1
if args.verbosity > 1 and i % 10000 == 0:
print(now_str(date=True), "-", i, "samples done")
if args.verbosity > 1:
print(now_str(date=True), "- all", i, "samples done")
if args.verbosity:
print("Features extracted, writing target and non-target language information for each utterance to '{}'.".format(args.trials))
with open(args.trials, "w") as trials_f:
for utt, target in utt2label.items():
for lang in int2label:
print(lang, utt, "target" if target == lang else "nontarget", file=trials_f)
if args.verbosity:
print("Starting prediction with model")
predictions = model.predict(features.map(lambda *t: t[0]))
if args.verbosity > 1:
print("Done predicting, model returned predictions of shape {}. Writing them to '{}'.".format(predictions.shape, args.scores))
num_predictions = 0
with open(args.scores, "w") as scores_f:
print(*int2label, file=scores_f)
for utt, pred in zip(utterance_ids, predictions):
pred_scores = [np.format_float_positional(x, precision=args.score_precision) for x in pred]
print(utt, *pred_scores, sep=args.score_separator, file=scores_f)
num_predictions += 1
if args.verbosity:
print("Wrote {} prediction scores to '{}'.".format(num_predictions, args.scores))
start = timeit.default_timer()
# combined function
def f(x):
return (x**2 + (np.sqrt(3) * x)**2)**3 - 4 * x**2 * (np.sqrt(3) * x)**2
# roots are labeled from x_0 to x_2 from left to right
# root of x_0 using bisection method
x0_root = optimize.bisect(f, -1, -0.4)
x0_root = np.format_float_positional(x0_root,
precision=4,
unique=False,
fractional=False,
trim='k')
y = np.sqrt(3) * float(x0_root)
int0 = (x0_root, y)
print("Intersection 1 is {}".format(int0))
# root of x_1 using newton method
x1_root = optimize.newton(f, -0.1, tol=1.48e-06)
if (x1_root < 0.0001) | (x1_root > -0.0001):
x1_root = 0
y = np.sqrt(3) * float(x1_root)
int1 = (x1_root, y)
print("Root of x_1 is {}".format(int1))
# root of x_2 using bisection method
def test_dragon4(self):
# these tests are adapted from Ryan Juckett's dragon4 implementation,
# see dragon4.c for details.
fpos32 = lambda x, **k: np.format_float_positional(np.float32(x), **k)
fsci32 = lambda x, **k: np.format_float_scientific(np.float32(x), **k)
fpos64 = lambda x, **k: np.format_float_positional(np.float64(x), **k)
fsci64 = lambda x, **k: np.format_float_scientific(np.float64(x), **k)
preckwd = lambda prec: {'unique': False, 'precision': prec}
assert_equal(fpos32('1.0'), "1.")
assert_equal(fsci32('1.0'), "1.e+00")
assert_equal(fpos32('10.234'), "10.234")
assert_equal(fpos32('-10.234'), "-10.234")
assert_equal(fsci32('10.234'), "1.0234e+01")
assert_equal(fsci32('-10.234'), "-1.0234e+01")
assert_equal(fpos32('1000.0'), "1000.")
assert_equal(fpos32('1.0', precision=0), "1.")
assert_equal(fsci32('1.0', precision=0), "1.e+00")
assert_equal(fpos32('10.234', precision=0), "10.")
assert_equal(fpos32('-10.234', precision=0), "-10.")
assert_equal(fsci32('10.234', precision=0), "1.e+01")
assert_equal(fsci32('-10.234', precision=0), "-1.e+01")
assert_equal(fpos32('10.234', precision=2), "10.23")
assert_equal(fsci32('-10.234', precision=2), "-1.02e+01")
assert_equal(fsci64('9.9999999999999995e-08', **preckwd(16)),
'9.9999999999999995e-08')
assert_equal(fsci64('9.8813129168249309e-324', **preckwd(16)),
'9.8813129168249309e-324')
assert_equal(fsci64('9.9999999999999694e-311', **preckwd(16)),
'9.9999999999999694e-311')
# test rounding
# 3.1415927410 is closest float32 to np.pi
assert_equal(fpos32('3.14159265358979323846', **preckwd(10)),
"3.1415927410")
assert_equal(fsci32('3.14159265358979323846', **preckwd(10)),
"3.1415927410e+00")
assert_equal(fpos64('3.14159265358979323846', **preckwd(10)),
"3.1415926536")
assert_equal(fsci64('3.14159265358979323846', **preckwd(10)),
"3.1415926536e+00")
# 299792448 is closest float32 to 299792458
assert_equal(fpos32('299792458.0', **preckwd(5)), "299792448.00000")
assert_equal(fsci32('299792458.0', **preckwd(5)), "2.99792e+08")
assert_equal(fpos64('299792458.0', **preckwd(5)), "299792458.00000")
assert_equal(fsci64('299792458.0', **preckwd(5)), "2.99792e+08")
assert_equal(fpos32('3.14159265358979323846', **preckwd(25)),
"3.1415927410125732421875000")
assert_equal(fpos64('3.14159265358979323846', **preckwd(50)),
"3.14159265358979311599796346854418516159057617187500")
assert_equal(fpos64('3.14159265358979323846'), "3.141592653589793")
# smallest numbers
assert_equal(fpos32(0.5**(126 + 23), unique=False, precision=149),
"0.00000000000000000000000000000000000000000000140129846432"
"4817070923729583289916131280261941876515771757068283889791"
"08268586060148663818836212158203125")
assert_equal(fpos64(5e-324, unique=False, precision=1074),
"0.00000000000000000000000000000000000000000000000000000000"
"0000000000000000000000000000000000000000000000000000000000"
"0000000000000000000000000000000000000000000000000000000000"
"0000000000000000000000000000000000000000000000000000000000"
"0000000000000000000000000000000000000000000000000000000000"
"0000000000000000000000000000000000049406564584124654417656"
"8792868221372365059802614324764425585682500675507270208751"
"8652998363616359923797965646954457177309266567103559397963"
"9877479601078187812630071319031140452784581716784898210368"
"8718636056998730723050006387409153564984387312473397273169"
"6151400317153853980741262385655911710266585566867681870395"
"6031062493194527159149245532930545654440112748012970999954"
"1931989409080416563324524757147869014726780159355238611550"
"1348035264934720193790268107107491703332226844753335720832"
"4319360923828934583680601060115061698097530783422773183292"
"4790498252473077637592724787465608477820373446969953364701"
"7972677717585125660551199131504891101451037862738167250955"
"8373897335989936648099411642057026370902792427675445652290"
"87538682506419718265533447265625")
# largest numbers
f32x = np.finfo(np.float32).max
assert_equal(fpos32(f32x, **preckwd(0)),
"340282346638528859811704183484516925440.")
assert_equal(fpos64(np.finfo(np.float64).max, **preckwd(0)),
"1797693134862315708145274237317043567980705675258449965989"
"1747680315726078002853876058955863276687817154045895351438"
"2464234321326889464182768467546703537516986049910576551282"
"0762454900903893289440758685084551339423045832369032229481"
"6580855933212334827479782620414472316873817718091929988125"
"0404026184124858368.")
# Warning: In unique mode only the integer digits necessary for
# uniqueness are computed, the rest are 0.
assert_equal(fpos32(f32x),
"340282350000000000000000000000000000000.")
# Further tests of zero-padding vs rounding in different combinations
# of unique, fractional, precision, min_digits
# precision can only reduce digits, not add them.
# min_digits can only extend digits, not reduce them.
assert_equal(fpos32(f32x, unique=True, fractional=True, precision=0),
"340282350000000000000000000000000000000.")
assert_equal(fpos32(f32x, unique=True, fractional=True, precision=4),
"340282350000000000000000000000000000000.")
assert_equal(fpos32(f32x, unique=True, fractional=True, min_digits=0),
"340282346638528859811704183484516925440.")
assert_equal(fpos32(f32x, unique=True, fractional=True, min_digits=4),
"340282346638528859811704183484516925440.0000")
assert_equal(fpos32(f32x, unique=True, fractional=True,
min_digits=4, precision=4),
"340282346638528859811704183484516925440.0000")
assert_raises(ValueError, fpos32, f32x, unique=True, fractional=False,
precision=0)
assert_equal(fpos32(f32x, unique=True, fractional=False, precision=4),
"340300000000000000000000000000000000000.")
assert_equal(fpos32(f32x, unique=True, fractional=False, precision=20),
"340282350000000000000000000000000000000.")
assert_equal(fpos32(f32x, unique=True, fractional=False, min_digits=4),
"340282350000000000000000000000000000000.")
assert_equal(fpos32(f32x, unique=True, fractional=False,
min_digits=20),
"340282346638528859810000000000000000000.")
assert_equal(fpos32(f32x, unique=True, fractional=False,
min_digits=15),
"340282346638529000000000000000000000000.")
assert_equal(fpos32(f32x, unique=False, fractional=False, precision=4),
"340300000000000000000000000000000000000.")
# test that unique rounding is preserved when precision is supplied
# but no extra digits need to be printed (gh-18609)
a = np.float64.fromhex('-1p-97')
assert_equal(fsci64(a, unique=True), '-6.310887241768095e-30')
assert_equal(fsci64(a, unique=False, precision=15),
'-6.310887241768094e-30')
assert_equal(fsci64(a, unique=True, precision=15),
'-6.310887241768095e-30')
assert_equal(fsci64(a, unique=True, min_digits=15),
'-6.310887241768095e-30')
assert_equal(fsci64(a, unique=True, precision=15, min_digits=15),
'-6.310887241768095e-30')
# adds/remove digits in unique mode with unbiased rnding
assert_equal(fsci64(a, unique=True, precision=14),
'-6.31088724176809e-30')
assert_equal(fsci64(a, unique=True, min_digits=16),
'-6.3108872417680944e-30')
assert_equal(fsci64(a, unique=True, precision=16),
'-6.310887241768095e-30')
assert_equal(fsci64(a, unique=True, min_digits=14),
'-6.310887241768095e-30')
# test min_digits in unique mode with different rounding cases
assert_equal(fsci64('1e120', min_digits=3), '1.000e+120')
assert_equal(fsci64('1e100', min_digits=3), '1.000e+100')
# test trailing zeros
assert_equal(fpos32('1.0', unique=False, precision=3), "1.000")
assert_equal(fpos64('1.0', unique=False, precision=3), "1.000")
assert_equal(fsci32('1.0', unique=False, precision=3), "1.000e+00")
assert_equal(fsci64('1.0', unique=False, precision=3), "1.000e+00")
assert_equal(fpos32('1.5', unique=False, precision=3), "1.500")
assert_equal(fpos64('1.5', unique=False, precision=3), "1.500")
assert_equal(fsci32('1.5', unique=False, precision=3), "1.500e+00")
assert_equal(fsci64('1.5', unique=False, precision=3), "1.500e+00")
# gh-10713
assert_equal(fpos64('324', unique=False, precision=5,
fractional=False), "324.00")
def out_clusters_pdf(data, d, sp_labels, km_labels, true_labels, k, K):
"""
output a visualization of the result of the
spectral clustering and k-means clustering
algorithms, and the jaccard score for each.
this function sometimes uses separate commands
for 2d and 3d data.
:param data: Data of (n*d) generaterd from sklearn
:param d: dimension of data (2 or 3)
:param sp_labels: lables based on spectral clustering
:param km_labels: lables based on k-means clustering
:param true_labels: Original lables based on make-blobs
:param k: if Random is false number of Clusters based on make-blobs
:param K: number of Clusters by user
:return: NULL
"""
# figure setup
fig = plt.figure(figsize=plt.figaspect(0.5))
# prepare the plot data
x = data[:, 0]
y = data[:, 1]
if d == 3:
z = data[:, 2]
# create a subplot: spectral clustering
if d == 2:
plt.subplot(221)
plt.scatter(x, y, c=sp_labels)
else:
ax = fig.add_subplot(2, 2, 1, projection='3d')
ax.scatter3D(x, y, z, c=sp_labels)
plt.title('Normalized Spectral Clustering')
# create a subplot: kmeans clustering
if d == 2:
plt.subplot(222)
plt.scatter(x, y, c=km_labels)
else:
ax = fig.add_subplot(2, 2, 2, projection='3d')
ax.scatter3D(x, y, z, c=km_labels)
plt.title('K-means')
# prepare the plot text
# jaccard scores for each algorithm's result
sp_jaccard = jaccard_measure(sp_labels, true_labels)
sp_jaccard_str = np.format_float_positional(sp_jaccard, precision=2)
km_jaccard = jaccard_measure(km_labels, true_labels)
km_jaccard_str = np.format_float_positional(km_jaccard, precision=2)
# program parameters
n = str(data.shape[0])
K = str(K)
k = str(k)
# text to be displayed
text = ("Data was generated from the values:\n" + "n = " + n + " , k = " +
K + "\n" + "The k that was used for both algorithms was " +
k + "\n" + "The jaccard measure for Spectral Clustering: " +
sp_jaccard_str + "\n" + "The jaccard measure for K-means: " +
km_jaccard_str)
# add the text in place of a subplot
fig.add_subplot(2, 2, (3, 4), frameon=False)
plt.text(0.5, 0.5, text, ha='center', wrap=False)
plt.axis('off')
# save to pdf
plt.savefig("clusters.pdf")
def min(value):
print(str(np.format_float_positional(np.min(np.array(value)), 2)))
def format_performance_info(performance_values):
performance_text = ""
for counter, value in enumerate(performance_values):
performance_text += str(counter) + ": " + str(np.format_float_positional(value, 2))
return performance_text
def test_dragon4(self):
# these tests are adapted from Ryan Juckett's dragon4 implementation,
# see dragon4.c for details.
fpos32 = lambda x, **k: np.format_float_positional(np.float32(x), **k)
fsci32 = lambda x, **k: np.format_float_scientific(np.float32(x), **k)
fpos64 = lambda x, **k: np.format_float_positional(np.float64(x), **k)
fsci64 = lambda x, **k: np.format_float_scientific(np.float64(x), **k)
preckwd = lambda prec: {'unique': False, 'precision': prec}
assert_equal(fpos32('1.0'), "1.")
assert_equal(fsci32('1.0'), "1.e+00")
assert_equal(fpos32('10.234'), "10.234")
assert_equal(fpos32('-10.234'), "-10.234")
assert_equal(fsci32('10.234'), "1.0234e+01")
assert_equal(fsci32('-10.234'), "-1.0234e+01")
assert_equal(fpos32('1000.0'), "1000.")
assert_equal(fpos32('1.0', precision=0), "1.")
assert_equal(fsci32('1.0', precision=0), "1.e+00")
assert_equal(fpos32('10.234', precision=0), "10.")
assert_equal(fpos32('-10.234', precision=0), "-10.")
assert_equal(fsci32('10.234', precision=0), "1.e+01")
assert_equal(fsci32('-10.234', precision=0), "-1.e+01")
assert_equal(fpos32('10.234', precision=2), "10.23")
assert_equal(fsci32('-10.234', precision=2), "-1.02e+01")
assert_equal(fsci64('9.9999999999999995e-08', **preckwd(16)),
'9.9999999999999995e-08')
assert_equal(fsci64('9.8813129168249309e-324', **preckwd(16)),
'9.8813129168249309e-324')
assert_equal(fsci64('9.9999999999999694e-311', **preckwd(16)),
'9.9999999999999694e-311')
# test rounding
# 3.1415927410 is closest float32 to np.pi
assert_equal(fpos32('3.14159265358979323846', **preckwd(10)),
"3.1415927410")
assert_equal(fsci32('3.14159265358979323846', **preckwd(10)),
"3.1415927410e+00")
assert_equal(fpos64('3.14159265358979323846', **preckwd(10)),
"3.1415926536")
assert_equal(fsci64('3.14159265358979323846', **preckwd(10)),
"3.1415926536e+00")
# 299792448 is closest float32 to 299792458
assert_equal(fpos32('299792458.0', **preckwd(5)), "299792448.00000")
assert_equal(fsci32('299792458.0', **preckwd(5)), "2.99792e+08")
assert_equal(fpos64('299792458.0', **preckwd(5)), "299792458.00000")
assert_equal(fsci64('299792458.0', **preckwd(5)), "2.99792e+08")
assert_equal(fpos32('3.14159265358979323846', **preckwd(25)),
"3.1415927410125732421875000")
assert_equal(fpos64('3.14159265358979323846', **preckwd(50)),
"3.14159265358979311599796346854418516159057617187500")
assert_equal(fpos64('3.14159265358979323846'), "3.141592653589793")
# smallest numbers
assert_equal(fpos32(0.5**(126 + 23), unique=False, precision=149),
"0.00000000000000000000000000000000000000000000140129846432"
"4817070923729583289916131280261941876515771757068283889791"
"08268586060148663818836212158203125")
assert_equal(fpos64(0.5**(1022 + 52), unique=False, precision=1074),
"0.00000000000000000000000000000000000000000000000000000000"
"0000000000000000000000000000000000000000000000000000000000"
"0000000000000000000000000000000000000000000000000000000000"
"0000000000000000000000000000000000000000000000000000000000"
"0000000000000000000000000000000000000000000000000000000000"
"0000000000000000000000000000000000049406564584124654417656"
"8792868221372365059802614324764425585682500675507270208751"
"8652998363616359923797965646954457177309266567103559397963"
"9877479601078187812630071319031140452784581716784898210368"
"8718636056998730723050006387409153564984387312473397273169"
"6151400317153853980741262385655911710266585566867681870395"
"6031062493194527159149245532930545654440112748012970999954"
"1931989409080416563324524757147869014726780159355238611550"
"1348035264934720193790268107107491703332226844753335720832"
"4319360923828934583680601060115061698097530783422773183292"
"4790498252473077637592724787465608477820373446969953364701"
"7972677717585125660551199131504891101451037862738167250955"
"8373897335989936648099411642057026370902792427675445652290"
"87538682506419718265533447265625")
# largest numbers
assert_equal(fpos32(np.finfo(np.float32).max, **preckwd(0)),
"340282346638528859811704183484516925440.")
assert_equal(fpos64(np.finfo(np.float64).max, **preckwd(0)),
"1797693134862315708145274237317043567980705675258449965989"
"1747680315726078002853876058955863276687817154045895351438"
"2464234321326889464182768467546703537516986049910576551282"
"0762454900903893289440758685084551339423045832369032229481"
"6580855933212334827479782620414472316873817718091929988125"
"0404026184124858368.")
# Warning: In unique mode only the integer digits necessary for
# uniqueness are computed, the rest are 0. Should we change this?
assert_equal(fpos32(np.finfo(np.float32).max, precision=0),
"340282350000000000000000000000000000000.")
# test trailing zeros
assert_equal(fpos32('1.0', unique=False, precision=3), "1.000")
assert_equal(fpos64('1.0', unique=False, precision=3), "1.000")
assert_equal(fsci32('1.0', unique=False, precision=3), "1.000e+00")
assert_equal(fsci64('1.0', unique=False, precision=3), "1.000e+00")
assert_equal(fpos32('1.5', unique=False, precision=3), "1.500")
assert_equal(fpos64('1.5', unique=False, precision=3), "1.500")
assert_equal(fsci32('1.5', unique=False, precision=3), "1.500e+00")
assert_equal(fsci64('1.5', unique=False, precision=3), "1.500e+00")
def map_df_to_str(df: pd.DataFrame) -> pd.DataFrame:
return df.applymap(lambda x: np.format_float_positional(x, trim="-")
if isinstance(x, float) else x).astype(str)