def main(): saved_handler = sp.seterrcall(err_handler) saved_err = sp.seterr(all='call') print('============ Part 1: Plotting =============================') x, y = load_data('ex2/ex2data1.txt') plot_data(x, y) pl.show() print('============ Part 2: Compute Cost and Gradient ============') m, n = x.shape x = sp.column_stack((sp.ones((m, 1)), x)) init_theta = sp.asmatrix(sp.zeros((n + 1, 1))) cost, grad = cost_function(init_theta, x, y) print('Cost at initial theta: %s' % cost) print('Gradient at initial theta:\n %s' % grad) print('============ Part 3: Optimizing minimize ====================') # res = op.minimize(cost_function, init_theta, args=(x, y), jac=True, method='Newton-CG') res = op.minimize(cost_function_without_grad, init_theta, args=(x, y), method='Powell') # print('Cost at theta found by fmin: %s' % cost) print('Result by minimize:\n%s' % res) plot_decision_boundary(res.x, x, y) pl.show() print('============ Part 4: Optimizing fmin ====================') res = op.fmin(cost_function_without_grad, init_theta, args=(x, y)) # print('Cost at theta found by fmin: %s' % cost) print('Result by fmin:\n%s' % res) plot_decision_boundary(res, x, y) pl.show() sp.seterrcall(saved_handler) sp.seterr(**saved_err)
def cumulative_state_probability(threshold, beta, value): """Fragility curve calculation EQRM manual 7.8 (Note eqn in manual is wrong. Is is correct here.) Calcultate the cumulative state probability. P_cumulative(slight) = P(slight)+P(mid)+P(high)... Note that value/threshold is the median, not the mean. Since this is cumulative and the values are descending; column 0 is Slight column 1 is Moderate column 2 is Extreme column 3 is Complete parameters threshold - median damage state threshold beta - represents uncertainty in the damage state value - motion of the structure for buildings - either peak spectral displacement , peak spectrial acceleration or for bridges - spectral acceleration at t = 1 sec """ oldsettings = seterr(divide='ignore') temp = (1 / beta) * log(value / threshold) seterr(**oldsettings) return norm.cdf(temp)
def relative_bin_deviation(h1, h2): # 79 us @array, 104 us @list \w 100 bins r""" Calculate the bin-wise deviation between two histograms. The relative bin deviation between two histograms :math:`H` and :math:`H'` of size :math:`m` is defined as: .. math:: d_{rbd}(H, H') = \sum_{m=1}^M \frac{ \sqrt{(H_m - H'_m)^2} }{ \frac{1}{2} \left( \sqrt{H_m^2} + \sqrt{{H'}_m^2} \right) } *Attributes:* - semimetric (triangle equation satisfied?) *Attributes for normalized histograms:* - :math:`d(H, H')\in[0, \infty)` - :math:`d(H, H) = 0` - :math:`d(H, H') = d(H', H)` *Attributes for not-normalized histograms:* - :math:`d(H, H')\in[0, \infty)` - :math:`d(H, H) = 0` - :math:`d(H, H') = d(H', H)` *Attributes for not-equal histograms:* - not applicable Parameters ---------- h1 : sequence The first histogram. h2 : sequence The second histogram, same bins as ``h1``. Returns ------- relative_bin_deviation : float Relative bin deviation between the two histograms. """ h1, h2 = __prepare_histogram(h1, h2) numerator = scipy.sqrt(scipy.square(h1 - h2)) denominator = (scipy.sqrt(scipy.square(h1)) + scipy.sqrt(scipy.square(h2))) / 2. old_err_state = scipy.seterr(invalid='ignore') # divide through zero only occurs when the bin is zero in both histograms, in which case the division is 0/0 and leads to (and should lead to) 0 result = numerator / denominator scipy.seterr(**old_err_state) result[scipy.isnan(result)] = 0 # faster than scipy.nan_to_num, which checks for +inf and -inf also return scipy.sum(result)
def calculate_corner_periods(periods, ground_motion, magnitude): """ periods - 1D array, dimension # of periods ground_motion - XD array, with last dimension periods. magnitude - 1D array, dimension # of events from Newmark and Hall 1982 returns TAV,TVD - the acceleration and velocity dependent corner periods TAV and TVD have the same dimensions as ground_motion, except the last axis (periods) is dropped. """ #reference_periods = 0.3,1.0 S03 = interp(array(0.3), ground_motion, periods, axis=-1) S10 = interp(array(1.0), ground_motion, periods, axis=-1) # print 'S03',S03[:,0:5] # print 'S10',S10[:,0:5] # interpolate the ground motion at reference periods oldsettings = seterr(invalid='ignore') acceleration_dependent = (S10 / S03) seterr(**oldsettings) acceleration_dependent[where(S03 < 0.00000000000001)] = 0.0 acceleration_dependent = acceleration_dependent[..., 0] # and collapse # This assumes ground_motion.shape = (1, events) velocity_dependent = (10**((magnitude-5.0)/2))[newaxis,:] assert len(acceleration_dependent.shape) == 2 assert len(velocity_dependent.shape) == 2 # removed this assert so test_cadell_damage passes. # This test passes 4660 sites, instead of the usual 1. # Fixing/ getting rid of the test is another option... #assert velocity_dependent.shape == acceleration_dependent.shape return acceleration_dependent, velocity_dependent
def FillFD(En_A,T): """ fill an array with Fermi-Dirac distribution """ N = int((len(En_A)-1)/2) sp.seterr(over='ignore') ## ignore overflow in exp, not important in this calculation if T == 0.0: FD_A = 1.0*sp.concatenate([sp.ones(N),[0.5],sp.zeros(N)]) else: FD_A = FermiDirac(En_A,T) sp.seterr(over='warn') return FD_A
def solve(SA, SD, SAcap, update_function, rtol=0.05, maxits=100): """ #FIXME DSG-EQRM what is the dimensions of these, and the return value? SA = demand curve (g) SAcap = capacity curve (g) SD = x axis (for both capcity and demand) (mm) update_function(intersection_point.x) makes a new SA,SD,SAcap. it also returns an exit flag it is usaully (always?) eqrm_code.capacity_spectrum_model.Capacity_spectrum_model.updated_responce rtol of 0.05 process will halt if intersection_x moved by less than 5% in the last iteration. All points are deemed to have converged. maxits is the maximum iterations. If maxits is exceeded, then any points that are not yet deemed to have converged are set to (intersection_x + old_intersection_x)/2 """ # old terminology, SDcr was intersection_x iters = 0 intersection_x = find_intersection(SD, SA, SAcap) exit_flag = False while ((iters <= maxits) & (not exit_flag)): # if 1: # if iters>0: # print 'iter' # print iters iters += 1 # update number of iterations old_intersection_x = intersection_x.copy() # copy old intersection # update curves SA, SD, SAcap, exit_flag = update_function(intersection_x) # get new intersection intersection_x = find_intersection(SD, SA, SAcap) oldsettings = seterr(invalid='ignore') diff = abs(intersection_x - old_intersection_x) / \ old_intersection_x # diff seterr(**oldsettings) # This is needed in windows to stop nan's setting the diff to -1.#IND diff = nan_to_num(diff) max_diff = diff.max() # find the relative change in intersection_x if max_diff < rtol: exit_flag = True # check for convergence if (iters >= maxits): # if iteration doesn't converge, take the average value # use average values non_convergent = where(diff >= rtol) # find non_convergent cases # x = (x+x_old)/2 intersection_x[non_convergent] += old_intersection_x[non_convergent] intersection_x[non_convergent] *= 0.5 else: non_convergent = array([]) return intersection_x, non_convergent
def FillBE(En_A,T): """ fill an array with Bose-Einstein distribution """ N = int((len(En_A)-1)/2) sp.seterr(over='ignore') ## ignore overflow in exp, not important in this calculation if T == 0.0: BE_A = -1.0*sp.concatenate([sp.ones(N),[0.5],sp.zeros(N)]) else: BE_A = BoseEinstein(En_A,T) BE_A[N] = -0.5 sp.seterr(over='warn') return BE_A
def chi_square(h1, h2): # 23 us @array, 49 us @list \w 100 r""" Chi-square distance. Measure how unlikely it is that one distribution (histogram) was drawn from the other. The Chi-square distance between two histograms :math:`H` and :math:`H'` of size :math:`m` is defined as: .. math:: d_{\chi^2}(H, H') = \sum_{m=1}^M \frac{ (H_m - H'_m)^2 }{ H_m + H'_m } *Attributes:* - semimetric *Attributes for normalized histograms:* - :math:`d(H, H')\in[0, 2]` - :math:`d(H, H) = 0` - :math:`d(H, H') = d(H', H)` *Attributes for not-normalized histograms:* - :math:`d(H, H')\in[0, \infty)` - :math:`d(H, H) = 0` - :math:`d(H, H') = d(H', H)` *Attributes for not-equal histograms:* - not applicable Parameters ---------- h1 : sequence The first histogram. h2 : sequence The second histogram. Returns ------- chi_square : float Chi-square distance. """ h1, h2 = __prepare_histogram(h1, h2) old_err_state = scipy.seterr(invalid='ignore') # divide through zero only occurs when the bin is zero in both histograms, in which case the division is 0/0 and leads to (and should lead to) 0 result = scipy.square(h1 - h2) / (h1 + h2) scipy.seterr(**old_err_state) result[scipy.isnan(result)] = 0 # faster than scipy.nan_to_num, which checks for +inf and -inf also return scipy.sum(result)
def kullback_leibler(h1, h2): # 83 us @array, 109 us @list \w 100 bins r""" Kullback-Leibler divergence. Compute how inefficient it would to be code one histogram into another. Actually computes :math:`\frac{d_{KL}(h1, h2) + d_{KL}(h2, h1)}{2}` to achieve symmetry. The Kullback-Leibler divergence between two histograms :math:`H` and :math:`H'` of size :math:`m` is defined as: .. math:: d_{KL}(H, H') = \sum_{m=1}^M H_m\log\frac{H_m}{H'_m} *Attributes:* - quasimetric (but made symetric) *Attributes for normalized histograms:* - :math:`d(H, H')\in[0, \infty)` - :math:`d(H, H) = 0` - :math:`d(H, H') = d(H', H)` *Attributes for not-normalized histograms:* - not applicable *Attributes for not-equal histograms:* - not applicable Parameters ---------- h1 : sequence The first histogram, where h1[i] > 0 for any i such that h2[i] > 0, normalized. h2 : sequence The second histogram, where h2[i] > 0 for any i such that h1[i] > 0, normalized, same bins as ``h1``. Returns ------- kullback_leibler : float Kullback-Leibler divergence. """ old_err_state = scipy.seterr(divide='raise') try: h1, h2 = __prepare_histogram(h1, h2) result = (__kullback_leibler(h1, h2) + __kullback_leibler(h2, h1)) / 2. scipy.seterr(**old_err_state) return result except FloatingPointError: scipy.seterr(**old_err_state) raise ValueError('h1 can only contain zero values where h2 also contains zero values and vice-versa')
def nonlin_damp(capacity_parameters, kappa, acceleration, displacement, csm_hysteretic_damping): """ Calculate the damping correction (Fulford 02) using the exact capacity curve. """ # print "csm_hysteretic_damping", csm_hysteretic_damping DyV, AyV, DuV, AuV, a, b, c = capacity_parameters Harea = hyst_area_rand(displacement, acceleration, DyV, AyV, DuV, AuV, csm_hysteretic_damping) oldsettings = seterr(invalid="ignore") BH = kappa * Harea / (2 * pi * displacement * acceleration) seterr(**oldsettings) return BH
def relative_bin_deviation(h1, h2): # 79 us @array, 104 us @list \w 100 bins """ Calculate the bin-wise deviation between two histograms. The relative bin deviation between two histograms \f$H\f$ and \f$H'\f$ of size \f$m\f$ is defined as \f[ d_{rbd}(H, H') = \sum_{m=1}^M \frac{ \sqrt{(H_m - H'_m)^2} }{ \frac{1}{2} \left( \sqrt{H_m^2} + \sqrt{{H'}_m^2} \right) } \f] Attributes: - semimetric (triangle equation satisfied?) Attributes for normalized histograms: - \f$d(H, H')\in[0, \infty)\f$ - \f$d(H, H) = 0\f$ - \f$d(H, H') = d(H', H)\f$ Attributes for not-normalized histograms: - \f$d(H, H')\in[0, \infty)\f$ - \f$d(H, H) = 0\f$ - \f$d(H, H') = d(H', H)\f$ Attributes for not-equal histograms: - not applicable @param h1 the first histogram @type h1 array-like sequence @param h2 the second histogram, same bins as h1 @type h2 array-like sequence @return relative bin deviation @rtype float """ h1, h2 = __prepare_histogram(h1, h2) numerator = scipy.sqrt(scipy.square(h1 - h2)) denominator = (scipy.sqrt(scipy.square(h1)) + scipy.sqrt(scipy.square(h2))) / 2.0 old_err_state = scipy.seterr( invalid="ignore" ) # divide through zero only occurs when the bin is zero in both histograms, in which case the division is 0/0 and leads to (and should lead to) 0 result = numerator / denominator scipy.seterr(**old_err_state) result[scipy.isnan(result)] = 0 # faster than scipy.nan_to_num, which checks for +inf and -inf also return scipy.sum(result)
def compute_sf_entropy(self, sf_group, theoryDotTheory, theoryDotData, T): """ Compute the entropy for a given scale factor. """ try: prior_type, prior_params = self.sf_priors[sf_group] except KeyError: prior_type, prior_params = 'uniform in sf', None if prior_type == 'uniform in sf': if theoryDotTheory != 0: entropy = scipy.log(scipy.sqrt(2*scipy.pi*T/theoryDotTheory)) else: entropy = 0 elif prior_type == 'gaussian in log sf': # This is the integrand for the prior. Note that it's important # that u = 0 corresponds to B_best. This ensures that the # integration doesn't miss the (possibly sharp) peak there. try: import SloppyCell.misc_c integrand = SloppyCell.misc_c.log_gaussian_prior_integrand except ImportError: logger.warn('Falling back to python integrand on log gaussian ' 'prior integration.') exp = scipy.exp def integrand(u, ak, bk, mulB, siglB, T, B_best, lB_best): B = exp(u) * B_best lB = u + lB_best ret = exp(-ak/(2*T) * (B-B_best)**2 - (lB-mulB)**2/(2 * siglB**2)) return ret mulB, siglB = prior_params B_best = theoryDotData/theoryDotTheory lB_best = scipy.log(B_best) # Often we get overflow errors in the integration, but they # don't actually cause a problem. (exp(-inf) is still 0.) This # prevents scipy from printing annoying error messages in this # case. prev_err = scipy.seterr(over='ignore') int_args = (theoryDotTheory, theoryDotData, mulB, siglB, T, B_best, lB_best) ans, temp = scipy.integrate.quad(integrand, -scipy.inf, scipy.inf, args = int_args, limit=1000) scipy.seterr(**prev_err) entropy = scipy.log(ans) else: raise ValueError('Unrecognized prior type: %s.' % prior_type) return entropy
def get_interpolated_SD(SD, SAdiff, id0, id1): # Select points for interpolation: x0 = SD[id0] y0 = SAdiff[id0] x1 = SD[id1] y1 = SAdiff[id1] dx = x1 - x0 dy = y1 - y0 # interpolate SD oldsettings = seterr(invalid='ignore') SDcr = x1 - y1 * (dx / dy) seterr(**oldsettings) return SDcr
def _monte_carlo(self, mean, sigma, var_in_last_axis): """ Perform random sampling about mean with sigma. self.sample_shape and self._vs() controls the shape of the result. """ assert sigma.shape == mean.shape variate_site = self._vs(sigma, var_in_last_axis) oldsettings = seterr(over='ignore') # self.sample_shape and variate_site will have compatible dims sample_values = self.val_func(mean[self.sample_shape] + variate_site * sigma[self.sample_shape]) seterr(**oldsettings) return sample_values
def test_array_ab_diff(self): a = array([7.35287023, 3.98947559, 0.]) b = array([7.38625883, 3.98947559, 0.]) oldsettings = seterr(invalid='ignore') diff = abs(a - b) / b seterr(**oldsettings) # diff [ 0.00452037 0. NaN] # Windows can't handle the NaN, # so it has to be set to zero diff = nan_to_num(diff) # print "diff", diff # this would return max_diff -1.#IND if NaN's aren't removed, # in windows. max_diff = diff.max() # print "max_diff", max_diff assert max_diff == diff[0]
def chi_square(h1, h2): # 23 us @array, 49 us @list \w 100 """ Measure how unlikely it is that one distribution (histogram) was drawn from the other. The Chi-square distance between two histograms \f$H\f$ and \f$H'\f$ of size \f$m\f$ is defined as \f[ d_{\chi^2}(H, H') = \sum_{m=1}^M \frac{ (H_m - H'_m)^2 }{ H_m + H'_m } \f] Attributes: - semimetric Attributes for normalized histograms: - \f$d(H, H')\in[0, 2]\f$ - \f$d(H, H) = 0\f$ - \f$d(H, H') = d(H', H)\f$ Attributes for not-normalized histograms: - \f$d(H, H')\in[0, \infty)\f$ - \f$d(H, H) = 0\f$ - \f$d(H, H') = d(H', H)\f$ Attributes for not-equal histograms: - not applicable @param h1 the first histogram @type h1 array-like sequence @param h2 the second histogram @type h2 array-like sequence @return chi-square distance @rtype float """ h1, h2 = __prepare_histogram(h1, h2) old_err_state = scipy.seterr( invalid="ignore" ) # divide through zero only occurs when the bin is zero in both histograms, in which case the division is 0/0 and leads to (and should lead to) 0 result = scipy.square(h1 - h2) / (h1 + h2) scipy.seterr(**old_err_state) result[scipy.isnan(result)] = 0 # faster than scipy.nan_to_num, which checks for +inf and -inf also return scipy.sum(result)
def FillFDplusBE(En_A,T): """ fill an array with a som of Bose-Einstein and Fermi-Dirac numerically more precise than the sum of the above functions due to a pole in BE """ N = int((len(En_A)-1)/2) sp.seterr(over= 'ignore') ## ignore overflow in sinh sp.seterr(divide='ignore') ## we deal with the pole ourselves if T == 0.0: FB_A = sp.zeros(len(En_A)) else: FB_A = 1.0/sp.sinh(En_A/T) FB_A[N] = 0.0 sp.seterr(divide='warn') sp.seterr(over= 'warn') return FB_A
def test_DLN_monte_carlo2(self): # dimensions (2,1,3,4) = 24 elements dim = (2,1,3,4) count_up = arange(1,24,1) log_mean = resize(count_up*10, dim) log_sigma = resize(count_up, dim) count_up_2 = arange(1,48,2) variate = resize(count_up_2, dim) var_method = 2 dist = Distribution_Log_Normal(var_method) # Provide a predictable sample dist.rvs = lambda size: count_up_2 sample_values = dist._monte_carlo(log_mean, log_sigma, True) oldsettings = seterr(over='ignore') actual = exp(log_mean + variate*log_sigma) seterr(**oldsettings) self.assert_(allclose(sample_values, actual)) self.assert_(sample_values.shape == dim)
def kullback_leibler(h1, h2): # 83 us @array, 109 us @list \w 100 bins """ Compute how inefficient it would to be code one histogram into another. Actually computes \f$\frac{d_{KL}(h1, h2) + d_{KL}(h2, h1)}{2}\f$ to achieve symmetry. The Kullback-Leibler divergence between two histograms \f$H\f$ and \f$H'\f$ of size \f$m\f$ is defined as \f[ d_{KL}(H, H') = \sum_{m=1}^M H_m\log\frac{H_m}{H'_m} \f] Attributes: - quasimetric (but made symetric) Attributes for normalized histograms: - \f$d(H, H')\in[0, \infty)\f$ - \f$d(H, H) = 0\f$ - \f$d(H, H') = d(H', H)\f$ Attributes for not-normalized histograms: - not applicable Attributes for not-equal histograms: - not applicable @param h1 the first histogram, where h1[i] > 0 for any i such that h2[i] > 0, normalized @type h1 array-like sequence @param h2 the second histogram, where h2[i] > 0 for any i such that h1[i] > 0, normalized, same bins as h1 @type h2 array-like sequence @return Kullback-Leibler divergence @rtype float """ old_err_state = scipy.seterr(divide="raise") try: h1, h2 = __prepare_histogram(h1, h2) result = (__kullback_leibler(h1, h2) + __kullback_leibler(h2, h1)) / 2.0 scipy.seterr(**old_err_state) return result except FloatingPointError: scipy.seterr(**old_err_state) raise ValueError("h1 can only contain zero values where h2 also contains zero values and vice-versa")
def _propagate_wavenumbers(self, wavenumbers, distances): """Calculate the wavenumber offset, delta. Arguments --------- wavenumbers : array The wavevector distances : array An array of thicknesses, ordered from source to terminator Returns ------- delta : array The phase difference """ # Turn off 'invalid multiplication' error; # It's just the 'inf' boundaries. olderr = sp.seterr(invalid='ignore') deltas = wavenumbers*distances # Now turn the error back on sp.seterr(**olderr) return deltas
def test_cumulative_state_probability(self): blocking_block_comments = True """Test that cumulative_state_probability works the same way as matlab function. Test that reduce_cumulative_to_pdf 'looks' right - it should look a bit like a bell curve - zero at ends with a max somewhere in the middle, and sum=one. """ # test against matlab implementation: beta = 0.4 value = 5.0 threshold = array((0.000001, 0.00001, 1, 1.5, 2, 3, 4, 5, 10, 100, 1000)) oldsettings = seterr(divide='ignore') x = (1/beta)*log(value/threshold) seterr(**oldsettings) # matlab: # Pr11 = normcdf2(1/THE_VUN_T.('beta_nsd_d')*log(SDcrAll./Thresh)) root2 = sqrt(2) y = 0.5*(1 + erf(x/root2)) # matlab: # root2 = sqrt(2); # y = 0.5*(1+erf(x/root2)) y2 = cumulative_state_probability(threshold, beta, value) assert allclose(y, y2) reduce_cumulative_to_pdf(y) # y should now look a bit like a bell curve - zero at ends # increasing to the middle, sum=one. assert (y.sum() == 1.0) assert y[0] == 0 assert y[1] < y[2] assert y[-3] > y[-2]
def bic(self): '''() -> ndarray Calculates the Bayesian Information Criteria for all models in range. ''' seterr('ignore') temp = [] #set minimization funtion to work with powell method f_temp = lambda x :-self.lik(x) #find maximum likelihood for all models for i in range(self._min_order,self._max_order): #gen params param = self.initalize_param(i)[0] param = fmin_powell(f_temp, param ,disp=False) temp.append([i,nu.copy(param),self.lik(param)]) #calculate bic #BIC, number of gausians out = nu.zeros((len(temp),2)) for i,j in enumerate(temp): out[i,1] = j[2] + j[0]**2*nu.log(self.data.shape[0]) out[i,0] = j[0] return out
def resolve_collision(A, B, time): """Detects and acts upon any collisions in the specified time interval between two entities :param A: First entity :param B: Second entity :param time: time interval over which to check if any collisions occur :return: None if no collision will occur in the timeframe, the two collided entities if there is a collision """ # general overview of this function: # 1. find if the objects will collide in the given time # 2. if yes, calculate collision: # 2.1 represent velocities as normal velocity and tangential velocity # 2.2 do a 1D collision using the normal veloctiy # 2.3 add the normal and tangential velocities to get the new velocity scipy.seterr(divide="raise", invalid="raise") # for this function I make one of the objects the frame of reference # which means my calculations are much simplified displacement = A.displacement - B.displacement velocity = A.velocity - B.velocity acceleration = A.acceleration - B.acceleration radius_sum = A.radius + B.radius # this code finds when the the two entities will collide. See # http://www.gvu.gatech.edu/people/official/jarek/graphics/material/collisionsDeshpandeKharsikarPrabhu.pdf # for how I got the algorithm a = magnitude(velocity, m / s)**2 b = m**2 / s * 2 * scipy.dot(displacement.asNumber(m), velocity.asNumber(m / s)) c = magnitude(displacement, m)**2 - radius_sum**2 try: t_to_impact = \ (-b - m**2/s * math.sqrt((b**2 - 4*a*c).asNumber(m**4/s**2))) / (2 * a) except: return if not scipy.isfinite(t_to_impact.asNumber(s)): return if t_to_impact > time or t_to_impact < 0 * s: return # at this point, we know there is a collision print("Collision:", A.name, "and", B.name, "in", t_to_impact) # for this section, basically turn the vectors into normal velocity and tangential velocity, # then do a 1D collision calculation, using the normal velocities # since a ' (prime symbol) wouldn't work, I've replaced it with a _ in variable names n = displacement # normal vector un = n / magnitude(n, m) # normal unit vector unt = copy.deepcopy(un) # normal tangent vector unt[0], unt[1] = -unt[1], unt[ 0] # ofc the tangent is orthogonal to the normal # A's centripetal velocity vAn = m / s * scipy.dot(un.asNumber(), A.velocity.asNumber(m / s)) # A's tangential velocity vAt = m / s * scipy.dot(unt.asNumber(), A.velocity.asNumber(m / s)) # B's centripetal velocity vBn = m / s * scipy.dot(un.asNumber(), B.velocity.asNumber(m / s)) # B's tangential velocity vBt = m / s * scipy.dot(unt.asNumber(), B.velocity.asNumber(m / s)) # tangent velocities are unchanged, nothing happens to them vAt_ = vAt vBt_ = vBt # centripetal velocities are calculated with a simple 1D collision formula R = 0.1 vAn_ = \ (A.mass_fun() * vAn + B.mass_fun() * vBn + R * B.mass_fun() * (B.velocity - A.velocity)) / \ (A.mass_fun() + B.mass_fun()) vBn_ = \ (A.mass_fun() * vAn + B.mass_fun() * vBn + R * A.mass_fun() * (A.velocity - B.velocity)) / \ (A.mass_fun() + B.mass_fun()) # convert scalar normal and tangent velocities to vector quantities VAn = vAn_ * un VAt = vAt_ * unt VBn = vBn_ * un VBt = vBt_ * unt # move until the point of impact A.move(t_to_impact) B.move(t_to_impact) # add em up to get v' A.velocity = VAn + VAt B.velocity = VBn + VBt # move for the rest of the frame A.move(time - t_to_impact) B.move(time - t_to_impact) return [A.name, B.name]
def plot_params(args): """Plot alpha, theta, and the emission probabilities""" old_err = sp.seterr(under='ignore') oldsize = matplotlib.rcParams['font.size'] K, L = args.emit_probs.shape if not args.continuous_observations else args.means.shape # alpha #matplotlib.rcParams['font.size'] = 12 pyplot.figure() _, xedges, yedges = sp.histogram2d([0, K], [0, K], bins=[K, K]) extent = [xedges[0], xedges[-1], yedges[0], yedges[-1]] pyplot.imshow(args.alpha.astype(sp.float64), extent=extent, interpolation='nearest', vmin=0, vmax=1, cmap='OrRd', origin='lower') pyplot.xticks(sp.arange(K) + .5, sp.arange(K) + 1) pyplot.gca().set_xticks(sp.arange(K) + 1, minor=True) pyplot.yticks(sp.arange(K) + .5, sp.arange(K) + 1) pyplot.gca().set_yticks(sp.arange(K) + 1, minor=True) pyplot.grid(which='minor', alpha=.2) for line in pyplot.gca().yaxis.get_ticklines() + pyplot.gca( ).xaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines( minor=True) + pyplot.gca().yaxis.get_ticklines(minor=True): # label is a Text instance line.set_markersize(0) pyplot.ylabel('Horizontal parent state') pyplot.xlabel('Node state') pyplot.title( r"Top root transition ($\alpha$) for {approx} iteration {iteration}". format(approx=args.approx, iteration=args.iteration)) b = pyplot.colorbar(shrink=.9) b.set_label("Probability") outfile = (args.out_params + '_it{iteration}.png').format(param='alpha', **args.__dict__) pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240) # beta pyplot.figure() _, xedges, yedges = sp.histogram2d([0, K], [0, K], bins=[K, K]) extent = [xedges[0], xedges[-1], yedges[0], yedges[-1]] pyplot.clf() pyplot.imshow(args.beta.astype(sp.float64), extent=extent, interpolation='nearest', vmin=0, vmax=1, cmap='OrRd', origin='lower') pyplot.xticks(sp.arange(K) + .5, sp.arange(K) + 1) pyplot.gca().set_xticks(sp.arange(K) + 1, minor=True) pyplot.yticks(sp.arange(K) + .5, sp.arange(K) + 1) pyplot.gca().set_yticks(sp.arange(K) + 1, minor=True) pyplot.grid(which='minor', alpha=.2) for line in pyplot.gca().yaxis.get_ticklines() + pyplot.gca( ).xaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines( minor=True) + pyplot.gca().yaxis.get_ticklines(minor=True): # label is a Text instance line.set_markersize(0) pyplot.ylabel('Vertical parent state') pyplot.xlabel('Node state') pyplot.title( r"Left root transition ($\beta$) for {approx} iteration {iteration}". format(approx=args.approx, iteration=args.iteration)) b = pyplot.colorbar(shrink=.9) b.set_label("Probability") outfile = (args.out_params + '_it{iteration}.png').format(param='beta', **args.__dict__) pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240) # theta if args.separate_theta: theta_tmp = args.theta for i in range((args.theta.shape)[0]): setattr(args, 'theta_%s' % (i + 1), args.theta[i, :, :, :]) for theta_name in ['theta'] + ['theta_%s' % i for i in range(20)]: #print 'trying', theta_name if not hasattr(args, theta_name): #print 'missing', theta_name continue _, xedges, yedges = sp.histogram2d([0, K], [0, K], bins=[K, K]) extent = [xedges[0], xedges[-1], yedges[0], yedges[-1]] if K == 18: numx_plots = 6 numy_plots = 3 elif K == 15: numx_plots = 5 numy_plots = 3 else: numx_plots = int(ceil(sp.sqrt(K))) numy_plots = int(ceil(sp.sqrt(K))) matplotlib.rcParams['font.size'] = 8 fig, axs = pyplot.subplots(numy_plots, numx_plots, sharex=True, sharey=True, figsize=(numx_plots * 2.5, numy_plots * 2.5)) for k in xrange(K): pltx, plty = k // numx_plots, k % numx_plots #axs[pltx,plty].imshow(args.theta[k,:,:], extent=extent, interpolation='nearest', axs[pltx, plty].imshow(getattr(args, theta_name)[:, k, :].astype(sp.float64), extent=extent, interpolation='nearest', vmin=0, vmax=1, cmap='OrRd', aspect='auto', origin='lower') #if k < numx_plots: #axs[pltx,plty].text(0 + .5, K - .5, 'vp=%s' % (k+1), horizontalalignment='left', verticalalignment='top', fontsize=10) axs[pltx, plty].text(0 + .5, K - .5, 'hp=%s' % (k + 1), horizontalalignment='left', verticalalignment='top', fontsize=10) #axs[pltx,plty].xticks(sp.arange(K) + .5, sp.arange(K)) #axs[pltx,plty].yticks(sp.arange(K) + .5, sp.arange(K)) axs[pltx, plty].set_xticks(sp.arange(K) + .5) axs[pltx, plty].set_xticks(sp.arange(K) + 1, minor=True) axs[pltx, plty].set_xticklabels(sp.arange(K) + 1) axs[pltx, plty].set_yticks(sp.arange(K) + .5) axs[pltx, plty].set_yticks(sp.arange(K) + 1, minor=True) axs[pltx, plty].set_yticklabels(sp.arange(K) + 1) for line in axs[pltx, plty].yaxis.get_ticklines() + axs[ pltx, plty].xaxis.get_ticklines() + axs[ pltx, plty].yaxis.get_ticklines( minor=True) + axs[pltx, plty].xaxis.get_ticklines( minor=True): line.set_markersize(0) axs[pltx, plty].grid(True, which='minor', alpha=.2) #fig.suptitle(r"$\Theta$ with fixed parents for {approx} iteration {iteration}". # format(approx=args.approx, iteration=args.iteration), # fontsize=14, verticalalignment='top') fig.suptitle('Node state', y=.03, fontsize=14, verticalalignment='center') #fig.suptitle('Horizontal parent state', y=.5, x=.02, rotation=90, fig.suptitle('Vertical parent state', y=.5, x=.02, rotation=90, verticalalignment='center', fontsize=14) matplotlib.rcParams['font.size'] = 6.5 fig.subplots_adjust(wspace=.05, hspace=.05, left=.05, right=.95) #b = fig.colorbar(shrink=.9) #b.set_label("Probability") outfile = (args.out_params + '_vertparent_it{iteration}.png').format( param=theta_name, **args.__dict__) pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240) fig, axs = pyplot.subplots(numy_plots, numx_plots, sharex=True, sharey=True, figsize=(numx_plots * 2.5, numy_plots * 2.5)) for k in xrange(K): pltx, plty = k // numx_plots, k % numx_plots axs[pltx, plty].imshow( getattr(args, theta_name)[k, :, :].astype(sp.float64), extent=extent, interpolation='nearest', #axs[pltx,plty].imshow(args.theta[:,k,:], extent=extent, interpolation='nearest', vmin=0, vmax=1, cmap='OrRd', aspect='auto', origin='lower') #if k < numx_plots: axs[pltx, plty].text(0 + .5, K - .5, 'vp=%s' % (k + 1), horizontalalignment='left', verticalalignment='top', fontsize=10) #axs[pltx,plty].xticks(sp.arange(K) + .5, sp.arange(K)) #axs[pltx,plty].yticks(sp.arange(K) + .5, sp.arange(K)) axs[pltx, plty].set_xticks(sp.arange(K) + .5) axs[pltx, plty].set_xticks(sp.arange(K) + 1, minor=True) axs[pltx, plty].set_xticklabels(sp.arange(K) + 1) axs[pltx, plty].set_yticks(sp.arange(K) + .5) axs[pltx, plty].set_yticks(sp.arange(K) + 1, minor=True) axs[pltx, plty].set_yticklabels(sp.arange(K) + 1) for line in axs[pltx, plty].yaxis.get_ticklines() + axs[ pltx, plty].xaxis.get_ticklines() + axs[ pltx, plty].yaxis.get_ticklines( minor=True) + axs[pltx, plty].xaxis.get_ticklines( minor=True): line.set_markersize(0) axs[pltx, plty].grid(True, which='minor', alpha=.2) #fig.suptitle(r"$\Theta$ with fixed parents for {approx} iteration {iteration}". # format(approx=args.approx, iteration=args.iteration), # fontsize=14, verticalalignment='top') fig.suptitle('Node state', y=.03, fontsize=14, verticalalignment='center') fig.suptitle( 'Horizontal parent state', y=.5, x=.02, rotation=90, #fig.suptitle('Vertical parent state', y=.5, x=.02, rotation=90, verticalalignment='center', fontsize=14) matplotlib.rcParams['font.size'] = 6.5 fig.subplots_adjust(wspace=.05, hspace=.05, left=.05, right=.95) #b = fig.colorbar(shrink=.9) #b.set_label("Probability") outfile = (args.out_params + '_it{iteration}.png').format( param=theta_name, **args.__dict__) pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240) # emission probabilities if args.continuous_observations: # plot mean values matplotlib.rcParams['font.size'] = 8 pyplot.figure(figsize=(max(1, round(L / 3.)), max(1, round(K / 3.)))) print(max(1, round(L / 3.)), max(1, round(K / 3.))) pyplot.imshow(args.means.astype(sp.float64), interpolation='nearest', aspect='auto', vmin=0, vmax=args.means.max(), cmap='OrRd', origin='lower') for k in range(K): for l in range(L): pyplot.text(l, k, '%.1f' % (args.means[k, l]), horizontalalignment='center', verticalalignment='center', fontsize=5) pyplot.yticks(sp.arange(K), sp.arange(K) + 1) pyplot.gca().set_yticks(sp.arange(K) + .5, minor=True) pyplot.xticks(sp.arange(L), valid_marks, rotation=30, horizontalalignment='right') pyplot.gca().set_xticks(sp.arange(L) + .5, minor=True) pyplot.grid(which='minor', alpha=.2) for line in pyplot.gca().yaxis.get_ticklines() + pyplot.gca( ).xaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines( minor=True) + pyplot.gca().yaxis.get_ticklines(minor=True): # label is a Text instance line.set_markersize(0) pyplot.ylabel('Hidden State') pyplot.title("Emission Mean") #b = pyplot.colorbar(shrink=.7) #b.set_label("Probability") outfile = (args.out_params + '_it{iteration}.png').format( param='emission_means', **args.__dict__) pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240) # plot variances pyplot.figure(figsize=(max(1, round(L / 3.)), max(1, round(K / 3.)))) print(L / 3, K / 3.) pyplot.imshow(args.variances.astype(sp.float64), interpolation='nearest', aspect='auto', vmin=0, vmax=args.variances.max(), cmap='OrRd', origin='lower') for k in range(K): for l in range(L): pyplot.text(l, k, '%.1f' % (args.variances[k, l]), horizontalalignment='center', verticalalignment='center', fontsize=5) pyplot.yticks(sp.arange(K), sp.arange(K) + 1) pyplot.gca().set_yticks(sp.arange(K) + .5, minor=True) pyplot.xticks(sp.arange(L), valid_marks, rotation=30, horizontalalignment='right') pyplot.gca().set_xticks(sp.arange(L) + .5, minor=True) pyplot.grid(which='minor', alpha=.2) for line in pyplot.gca().yaxis.get_ticklines() + pyplot.gca( ).xaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines( minor=True) + pyplot.gca().yaxis.get_ticklines(minor=True): # label is a Text instance line.set_markersize(0) pyplot.ylabel('Hidden State') pyplot.title("Emission Variance") #b = pyplot.colorbar(shrink=.7) #b.set_label("Probability") outfile = (args.out_params + '_it{iteration}.png').format( param='emission_variances', **args.__dict__) pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240) else: matplotlib.rcParams['font.size'] = 8 pyplot.figure(figsize=(max(1, round(L / 3.)), max(1, round(K / 3.)))) print(L / 3, K / 3.) pyplot.imshow(args.emit_probs.astype(sp.float64), interpolation='nearest', aspect='auto', vmin=0, vmax=1, cmap='OrRd', origin='lower') for k in range(K): for l in range(L): pyplot.text(l, k, '%2.0f' % (args.emit_probs[k, l] * 100), horizontalalignment='center', verticalalignment='center') pyplot.yticks(sp.arange(K), sp.arange(K) + 1) pyplot.gca().set_yticks(sp.arange(K) + .5, minor=True) pyplot.xticks(sp.arange(L), valid_marks, rotation=30, horizontalalignment='right') pyplot.gca().set_xticks(sp.arange(L) + .5, minor=True) pyplot.grid(which='minor', alpha=.2) for line in pyplot.gca().yaxis.get_ticklines() + pyplot.gca( ).xaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines( minor=True) + pyplot.gca().yaxis.get_ticklines(minor=True): # label is a Text instance line.set_markersize(0) pyplot.ylabel('Hidden State') pyplot.title("Emission probabilities") #b = pyplot.colorbar(shrink=.7) #b.set_label("Probability") outfile = (args.out_params + '_it{iteration}.png').format( param='emission', **args.__dict__) pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240) #broad_paper_enrichment = sp.array([[16,2,2,6,17,93,99,96,98,2], # [12,2,6,9,53,94,95,14,44,1], # [13,72,0,9,48,78,49,1,10,1], # [11,1,15,11,96,99,75,97,86,4], # [5,0,10,3,88,57,5,84,25,1], # [7,1,1,3,58,75,8,6,5,1], # [2,1,2,1,56,3,0,6,2,1], # [92,2,1,3,6,3,0,0,1,1], # [5,0,43,43,37,11,2,9,4,1], # [1,0,47,3,0,0,0,0,0,1], # [0,0,3,2,0,0,0,0,0,0], # [1,27,0,2,0,0,0,0,0,0], # [0,0,0,0,0,0,0,0,0,0], # [22,28,19,41,6,5,26,5,13,37], # [85,85,91,88,76,77,91,73,85,78], # [float('nan'), float('nan'),float('nan'),float('nan'),float('nan'),float('nan'),float('nan'),float('nan'),float('nan'),float('nan')] # ]) / 100. #mapping_from_broad = dict(zip(range(K), (5,2,0,14,4,6,9,1,12,-1,3,12,8,7,10,12,11,13))) #broad_paper_enrichment = broad_paper_enrichment[tuple(mapping_from_broad[i] for i in range(K)), :] #broad_names = ['Active promoter', 'Weak promoter', 'Inactive/poised promoter', 'Strong enhancer', # 'Strong enhancer', 'weak/poised enhancer', 'Weak/poised enhancer', 'Insulator', # 'Transcriptional transition', 'Transcriptional elongation', 'Weak transcribed', # 'Polycomb repressed', 'Heterochrom; low signal', 'Repetitive/CNV', 'Repetitive/CNV', # 'NA', 'NA', 'NA'] #pyplot.figure(figsize=(L/3,K/3.)) #print (L/3,K/3.) #pyplot.imshow(broad_paper_enrichment, interpolation='nearest', aspect='auto', # vmin=0, vmax=1, cmap='OrRd', origin='lower') #for k in range(K): # for l in range(L): # pyplot.text(l, k, '%2.0f' % (broad_paper_enrichment[k,l] * 100), horizontalalignment='center', verticalalignment='center') # pyplot.text(L, k, broad_names[mapping_from_broad[k]], horizontalalignment='left', verticalalignment='center', fontsize=6) #pyplot.yticks(sp.arange(K), sp.arange(K)+1) #pyplot.gca().set_yticks(sp.arange(K)+.5, minor=True) #pyplot.xticks(sp.arange(L), valid_marks, rotation=30, horizontalalignment='right') #pyplot.gca().set_xticks(sp.arange(L)+.5, minor=True) #pyplot.grid(which='minor', alpha=.2) #for line in pyplot.gca().yaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines(minor=True) + pyplot.gca().yaxis.get_ticklines(minor=True): ## label is a Text instance # line.set_markersize(0) #pyplot.ylabel('Hidden State') #pyplot.title("Broad paper Emission probabilities") ##b = pyplot.colorbar(shrink=.7) ##b.set_label("Probability") #pyplot.subplots_adjust(right=.7) #outfile = (args.out_params + '_broadpaper.png').format(param='emission', **args.__dict__) #pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240) pyplot.close('all') sp.seterr(**old_err) matplotlib.rcParams['font.size'] = oldsize
def herm_sqrt_inv(x, zero_tol=1E-15, sanity_checks=False, return_rank=False, sc_data=''): if isinstance(x, mm.eyemat): x_sqrt = x x_sqrt_i = x rank = x.shape[0] else: try: ev = x.diag #simple_diag_matrix EV = None except AttributeError: ev, EV = la.eigh(x) zeros = ev <= zero_tol #throw away negative results too! ev_sqrt = sp.sqrt(ev) err = sp.seterr(divide='ignore', invalid='ignore') try: ev_sqrt_i = 1 / ev_sqrt ev_sqrt[zeros] = 0 ev_sqrt_i[zeros] = 0 finally: sp.seterr(divide=err['divide'], invalid=err['invalid']) if EV is None: x_sqrt = mm.simple_diag_matrix(ev_sqrt, dtype=x.dtype) x_sqrt_i = mm.simple_diag_matrix(ev_sqrt_i, dtype=x.dtype) else: B = mm.mmul_diag(ev_sqrt, EV.conj().T) x_sqrt = EV.dot(B) B = mm.mmul_diag(ev_sqrt_i, EV.conj().T) x_sqrt_i = EV.dot(B) rank = x.shape[0] - np.count_nonzero(zeros) if sanity_checks: if ev.min() < -zero_tol: log.warning("Sanity Fail in herm_sqrt_inv(): Throwing away negative eigenvalues! %s %s", ev.min(), sc_data) if not np.allclose(x_sqrt.dot(x_sqrt), x): log.warning("Sanity Fail in herm_sqrt_inv(): x_sqrt is bad! %s %s", la.norm(x_sqrt.dot(x_sqrt) - x), sc_data) if EV is None: nulls = sp.zeros(x.shape[0]) nulls[zeros] = 1 nulls = sp.diag(nulls) else: #if we did an EVD then we use the eigenvectors nulls = EV.copy() nulls[:, sp.invert(zeros)] = 0 nulls = nulls.dot(nulls.conj().T) eye = np.eye(x.shape[0]) if not np.allclose(x_sqrt.dot(x_sqrt_i), eye - nulls): log.warning("Sanity Fail in herm_sqrt_inv(): x_sqrt_i is bad! %s %s", la.norm(x_sqrt.dot(x_sqrt_i) - eye + nulls), sc_data) if return_rank: return x_sqrt, x_sqrt_i, rank else: return x_sqrt, x_sqrt_i
def main(): # This configures numpy/scipy to raise an exception in case of errors, instead of printing a warning and going ahead. numpy.seterr(all='raise') scipy.seterr(all='raise') parser = argparse.ArgumentParser( description='Runs a set of benchmarks defined in a YAML file.') parser.add_argument( '--fruit-benchmark-sources-dir', help='Path to the fruit sources (used for benchmarking code only)') parser.add_argument('--fruit-sources-dir', help='Path to the fruit sources') parser.add_argument('--boost-di-sources-dir', help='Path to the Boost.DI sources') parser.add_argument( '--output-file', help= 'The output file where benchmark results will be stored (1 per line, with each line in JSON format). These can then be formatted by e.g. the format_bench_results script.' ) parser.add_argument( '--benchmark-definition', help= 'The YAML file that defines the benchmarks (see fruit_wiki_benchs_fruit.yml for an example).' ) parser.add_argument( '--continue-benchmark', help= 'If this is \'true\', continues a previous benchmark run instead of starting from scratch (taking into account the existing benchmark results in the file specified with --output-file).' ) args = parser.parse_args() if args.output_file is None: raise Exception('You must specify --output_file') if args.continue_benchmark == 'true': try: with open(args.output_file, 'r') as f: previous_run_completed_benchmarks = [ json.loads(line)['benchmark'] for line in f.readlines() ] except FileNotFoundError: previous_run_completed_benchmarks = [] else: previous_run_completed_benchmarks = [] run_command('rm', args=['-f', args.output_file]) fruit_build_dir = tempfile.gettempdir() + '/fruit-benchmark-build-dir' with open(args.benchmark_definition, 'r') as f: yaml_file_content = yaml.full_load(f) global_definitions = yaml_file_content['global'] benchmark_definitions = expand_benchmark_definitions( yaml_file_content['benchmarks']) benchmark_index = 0 for (compiler_executable_name, additional_cmake_args), benchmark_definitions_with_current_config \ in group_by(benchmark_definitions, lambda benchmark_definition: (benchmark_definition['compiler'], tuple(benchmark_definition['additional_cmake_args']))): print( 'Preparing for benchmarks with the compiler %s, with additional CMake args %s' % (compiler_executable_name, additional_cmake_args)) try: # We compute this here (and memoize the result) so that the benchmark's describe() will retrieve the cached # value instantly. determine_compiler_name(compiler_executable_name) # Build Fruit in fruit_build_dir, so that fruit_build_dir points to a built Fruit (useful for e.g. the config header). shutil.rmtree(fruit_build_dir, ignore_errors=True) os.makedirs(fruit_build_dir) modified_env = os.environ.copy() modified_env['CXX'] = compiler_executable_name run_command('cmake', args=[ args.fruit_sources_dir, '-DCMAKE_BUILD_TYPE=Release', *additional_cmake_args, ], cwd=fruit_build_dir, env=modified_env) run_command('make', args=make_args, cwd=fruit_build_dir) except Exception as e: print( 'Exception while preparing for benchmarks with the compiler %s, with additional CMake args %s.\n%s\nGoing ahead with the rest.' % (compiler_executable_name, additional_cmake_args, traceback.format_exc())) continue for benchmark_definition in benchmark_definitions_with_current_config: benchmark_index += 1 print('%s/%s: %s' % (benchmark_index, len(benchmark_definitions), benchmark_definition)) benchmark_name = benchmark_definition['name'] if (benchmark_name in { 'boost_di_compile_time', 'boost_di_run_time', 'boost_di_executable_size' } and args.boost_di_sources_dir is None): raise Exception( 'Error: you need to specify the --boost-di-sources-dir flag in order to run Boost.DI benchmarks.' ) if benchmark_name == 'new_delete_run_time': benchmark = SimpleNewDeleteRunTimeBenchmark( benchmark_definition, fruit_benchmark_sources_dir=args. fruit_benchmark_sources_dir) elif benchmark_name == 'fruit_single_file_compile_time': benchmark = FruitSingleFileCompileTimeBenchmark( benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_benchmark_sources_dir=args. fruit_benchmark_sources_dir, fruit_build_dir=fruit_build_dir) elif benchmark_name.startswith('fruit_'): benchmark_class = { 'fruit_compile_time': FruitCompileTimeBenchmark, 'fruit_incremental_compile_time': FruitIncrementalCompileTimeBenchmark, 'fruit_compile_memory': FruitCompileMemoryBenchmark, 'fruit_run_time': FruitRunTimeBenchmark, 'fruit_startup_time': FruitStartupTimeBenchmark, 'fruit_startup_time_with_normalized_component': FruitStartupTimeWithNormalizedComponentBenchmark, 'fruit_executable_size': FruitExecutableSizeBenchmark, 'fruit_executable_size_without_exceptions_and_rtti': FruitExecutableSizeBenchmarkWithoutExceptionsAndRtti, }[benchmark_name] benchmark = benchmark_class( benchmark_definition=benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_build_dir=fruit_build_dir) elif benchmark_name.startswith('boost_di_'): benchmark_class = { 'boost_di_compile_time': BoostDiCompileTimeBenchmark, 'boost_di_incremental_compile_time': BoostDiIncrementalCompileTimeBenchmark, 'boost_di_compile_memory': BoostDiCompileMemoryBenchmark, 'boost_di_run_time': BoostDiRunTimeBenchmark, 'boost_di_startup_time': BoostDiStartupTimeBenchmark, 'boost_di_executable_size': BoostDiExecutableSizeBenchmark, 'boost_di_executable_size_without_exceptions_and_rtti': BoostDiExecutableSizeBenchmarkWithoutExceptionsAndRtti, }[benchmark_name] benchmark = benchmark_class( benchmark_definition=benchmark_definition, boost_di_sources_dir=args.boost_di_sources_dir) elif benchmark_name.startswith('simple_di_'): benchmark_class = { 'simple_di_compile_time': SimpleDiCompileTimeBenchmark, 'simple_di_incremental_compile_time': SimpleDiIncrementalCompileTimeBenchmark, 'simple_di_compile_memory': SimpleDiCompileMemoryBenchmark, 'simple_di_run_time': SimpleDiRunTimeBenchmark, 'simple_di_startup_time': SimpleDiStartupTimeBenchmark, 'simple_di_executable_size': SimpleDiExecutableSizeBenchmark, 'simple_di_executable_size_without_exceptions_and_rtti': SimpleDiExecutableSizeBenchmarkWithoutExceptionsAndRtti, 'simple_di_with_interfaces_compile_time': SimpleDiWithInterfacesCompileTimeBenchmark, 'simple_di_with_interfaces_incremental_compile_time': SimpleDiWithInterfacesIncrementalCompileTimeBenchmark, 'simple_di_with_interfaces_compile_memory': SimpleDiWithInterfacesCompileMemoryBenchmark, 'simple_di_with_interfaces_run_time': SimpleDiWithInterfacesRunTimeBenchmark, 'simple_di_with_interfaces_startup_time': SimpleDiWithInterfacesStartupTimeBenchmark, 'simple_di_with_interfaces_executable_size': SimpleDiWithInterfacesExecutableSizeBenchmark, 'simple_di_with_interfaces_executable_size_without_exceptions_and_rtti': SimpleDiWithInterfacesExecutableSizeBenchmarkWithoutExceptionsAndRtti, 'simple_di_with_interfaces_and_new_delete_compile_time': SimpleDiWithInterfacesAndNewDeleteCompileTimeBenchmark, 'simple_di_with_interfaces_and_new_delete_incremental_compile_time': SimpleDiWithInterfacesAndNewDeleteIncrementalCompileTimeBenchmark, 'simple_di_with_interfaces_and_new_delete_compile_memory': SimpleDiWithInterfacesAndNewDeleteCompileMemoryBenchmark, 'simple_di_with_interfaces_and_new_delete_run_time': SimpleDiWithInterfacesAndNewDeleteRunTimeBenchmark, 'simple_di_with_interfaces_and_new_delete_startup_time': SimpleDiWithInterfacesAndNewDeleteStartupTimeBenchmark, 'simple_di_with_interfaces_and_new_delete_executable_size': SimpleDiWithInterfacesAndNewDeleteExecutableSizeBenchmark, 'simple_di_with_interfaces_and_new_delete_executable_size_without_exceptions_and_rtti': SimpleDiWithInterfacesAndNewDeleteExecutableSizeBenchmarkWithoutExceptionsAndRtti, }[benchmark_name] benchmark = benchmark_class( benchmark_definition=benchmark_definition) else: raise Exception("Unrecognized benchmark: %s" % benchmark_name) if benchmark.describe() in previous_run_completed_benchmarks: print( "Skipping benchmark that was already run previously (due to --continue-benchmark):", benchmark.describe()) continue try: run_benchmark(benchmark, output_file=args.output_file, max_runs=global_definitions['max_runs'], timeout_hours=global_definitions[ 'max_hours_per_combination']) except Exception as e: print( 'Exception while running benchmark: %s.\n%s\nGoing ahead with the rest.' % (benchmark.describe(), traceback.format_exc()))
def coh_tmm(pol, n_list, d_list, th_0, lam_vac): """ Main "coherent transfer matrix method" calc. Given parameters of a stack, calculates everything you could ever want to know about how light propagates in it. (If performance is an issue, you can delete some of the calculations without affecting the rest.) pol is light polarization, "s" or "p". n_list is the list of refractive indices, in the order that the light would pass through them. The 0'th element of the list should be the semi-infinite medium from which the light enters, the last element should be the semi- infinite medium to which the light exits (if any exits). th_0 is the angle of incidence: 0 for normal, pi/2 for glancing. Remember, for a dissipative incoming medium (n_list[0] is not real), th_0 should be complex so that n0 sin(th0) is real (intensity is constant as a function of lateral position). d_list is the list of layer thicknesses (front to back). Should correspond one-to-one with elements of n_list. First and last elements should be "inf". lam_vac is vacuum wavelength of the light. Outputs the following as a dictionary (see manual for details) * r--reflection amplitude * t--transmission amplitude * R--reflected wave power (as fraction of incident) * T--transmitted wave power (as fraction of incident) * power_entering--Power entering the first layer, usually (but not always) equal to 1-R (see manual). * vw_list-- n'th element is [v_n,w_n], the forward- and backward-traveling amplitudes, respectively, in the n'th medium just after interface with (n-1)st medium. * kz_list--normal component of complex angular wavenumber for forward-traveling wave in each layer. * th_list--(complex) propagation angle (in radians) in each layer * pol, n_list, d_list, th_0, lam_vac--same as input """ # Convert lists to numpy arrays if they're not already. n_list = array(n_list) d_list = array(d_list, dtype=float) # Input tests if ((hasattr(lam_vac, 'size') and lam_vac.size > 1) or (hasattr(th_0, 'size') and th_0.size > 1)): raise ValueError('This function is not vectorized; you need to run one ' 'calculation at a time (1 wavelength, 1 angle, etc.)') if (n_list.ndim != 1) or (d_list.ndim != 1) or (n_list.size != d_list.size): raise ValueError("Problem with n_list or d_list!") assert d_list[0] == d_list[-1] == inf, 'd_list must start and end with inf!' assert abs((n_list[0]*np.sin(th_0)).imag) < 100*EPSILON, 'Error in n0 or th0!' assert is_forward_angle(n_list[0], th_0), 'Error in n0 or th0!' num_layers = n_list.size # th_list is a list with, for each layer, the angle that the light travels # through the layer. Computed with Snell's law. Note that the "angles" may be # complex! th_list = list_snell(n_list, th_0) # kz is the z-component of (complex) angular wavevector for forward-moving # wave. Positive imaginary part means decaying. kz_list = 2 * np.pi * n_list * cos(th_list) / lam_vac # delta is the total phase accrued by traveling through a given layer. # Ignore warning about inf multiplication olderr = sp.seterr(invalid='ignore') delta = kz_list * d_list sp.seterr(**olderr) # For a very opaque layer, reset delta to avoid divide-by-0 and similar # errors. The criterion imag(delta) > 35 corresponds to single-pass # transmission < 1e-30 --- small enough that the exact value doesn't # matter. for i in range(1, num_layers-1): if delta[i].imag > 35: delta[i] = delta[i].real + 35j if 'opacity_warning' not in globals(): global opacity_warning opacity_warning = True print("Warning: Layers that are almost perfectly opaque " "are modified to be slightly transmissive, " "allowing 1 photon in 10^30 to pass through. It's " "for numerical stability. This warning will not " "be shown again.") # t_list[i,j] and r_list[i,j] are transmission and reflection amplitudes, # respectively, coming from i, going to j. Only need to calculate this when # j=i+1. (2D array is overkill but helps avoid confusion.) t_list = zeros((num_layers, num_layers), dtype=complex) r_list = zeros((num_layers, num_layers), dtype=complex) for i in range(num_layers-1): t_list[i,i+1] = interface_t(pol, n_list[i], n_list[i+1], th_list[i], th_list[i+1]) r_list[i,i+1] = interface_r(pol, n_list[i], n_list[i+1], th_list[i], th_list[i+1]) # At the interface between the (n-1)st and nth material, let v_n be the # amplitude of the wave on the nth side heading forwards (away from the # boundary), and let w_n be the amplitude on the nth side heading backwards # (towards the boundary). Then (v_n,w_n) = M_n (v_{n+1},w_{n+1}). M_n is # M_list[n]. M_0 and M_{num_layers-1} are not defined. # My M is a bit different than Sernelius's, but Mtilde is the same. M_list = zeros((num_layers, 2, 2), dtype=complex) for i in range(1, num_layers-1): M_list[i] = (1/t_list[i,i+1]) * np.dot( make_2x2_array(exp(-1j*delta[i]), 0, 0, exp(1j*delta[i]), dtype=complex), make_2x2_array(1, r_list[i,i+1], r_list[i,i+1], 1, dtype=complex)) Mtilde = make_2x2_array(1, 0, 0, 1, dtype=complex) for i in range(1, num_layers-1): Mtilde = np.dot(Mtilde, M_list[i]) Mtilde = np.dot(make_2x2_array(1, r_list[0,1], r_list[0,1], 1, dtype=complex)/t_list[0,1], Mtilde) # Net complex transmission and reflection amplitudes r = Mtilde[1,0]/Mtilde[0,0] t = 1/Mtilde[0,0] # vw_list[n] = [v_n, w_n]. v_0 and w_0 are undefined because the 0th medium # has no left interface. # vw_list = zeros((num_layers, 2), dtype=complex) # vw = array([[t],[0]]) # vw_list[-1,:] = np.transpose(vw) # for i in range(num_layers-2, 0, -1): # vw = np.dot(M_list[i], vw) # vw_list[i,:] = np.transpose(vw) # Net transmitted and reflected power, as a proportion of the incoming light # power. R = R_from_r(r) # T = T_from_t(pol, t, n_list[0], n_list[-1], th_0, th_list[-1]) # power_entering = power_entering_from_r(pol, r, n_list[0], th_0) # return {'r': r, 't': t, 'R': R, 'T': T, 'power_entering': power_entering, # 'vw_list': vw_list, 'kz_list': kz_list, 'th_list': th_list, # 'pol': pol, 'n_list': n_list, 'd_list': d_list, 'th_0': th_0, # 'lam_vac':lam_vac} return {'R': R}
def coh_tmm(pol, n_list, d_list, th_0, lam_vac, pame_output=False): """ ------- IMPORTANT ----- pame_output IS FOR USE WITH vector_coh_tmm() AND PLASMONIC ASSAY MODELING ENVIRONMENT. If left as False, will return data unmolested from original implementation of tmm module. There are several calls to coh_tmm from other utilitles herein, such as getting aborbed power in each layer, so this keyword maintains compatibility with both applications. ---------------------------------------------- Main "coherent transfer matrix method" calc. Given parameters of a stack, calculates everything you could ever want to know about how light propagates in it. (If performance is an issue, you can delete some of the calculations without affecting the rest.) pol is light polarization, "s" or "p". n_list is the list of refractive indices, in the order that the light would pass through them. The 0'th element of the list should be the semi-infinite medium from which the light enters, the last element should be the semi- infinite medium to which the light exits (if any exits). th_0 is the angle of incidence: 0 for normal, pi/2 for glancing. Remember, for a dissipative incoming medium (n_list[0] is not real), th_0 should be complex so that n0 sin(th0) is real (intensity is constant as a function of lateral position). d_list is the list of layer thicknesses (front to back). Should correspond one-to-one with elements of n_list. First and last elements should be "inf". lam_vac is vacuum wavelength of the light. Outputs the following as a dictionary (see manual for details) * r--reflection amplitude * t--transmission amplitude * R--reflected wave power (as fraction of incident) * T--transmitted wave power (as fraction of incident) * A--total absorbed power (as fraction of incident) = (1 - (R+T) ) * power_entering--Power entering the first layer, usually (but not always) equal to 1-R (see manual). * vw_list-- n'th element is [v_n,w_n], the forward- and backward-traveling amplitudes, respectively, in the n'th medium just after interface with (n-1)st medium. * kz_list--normal component of complex angular wavenumber for forward-traveling wave in each layer. * th_list--(complex) propagation angle (in radians) in each layer * pol, n_list, d_list, th_0, lam_vac--same as input """ #convert lists to numpy arrays if they're not already. n_list = array(n_list) d_list = array(d_list, dtype=float) #input tests if ((hasattr(lam_vac, 'size') and lam_vac.size > 1) or (hasattr(th_0, 'size') and th_0.size > 1)): raise ValueError( 'This function is not vectorized; you need to run one ' 'calculation at a time (1 wavelength, 1 angle, etc.)') if (n_list.ndim != 1) or (d_list.ndim != 1) or (n_list.size != d_list.size): raise ValueError("Problem with n_list or d_list!") if (d_list[0] != inf) or (d_list[-1] != inf): raise ValueError('d_list must start and end with inf!') # Should this case exist? Light couldn't get down a semi-infinite absorbing # media to begin with! if abs((n_list[0] * np.sin(th_0)).imag) > 100 * EPSILON: raise ValueError('Error in n0 or th0! Semi-infinite media cannot' ' be absorbing (have imaginary component).') num_layers = n_list.size #th_list is a list with, for each layer, the angle that the light travels #through the layer. Computed with Snell's law. Note that the "angles" may be #complex! th_list = list_snell(n_list, th_0) #kz is the z-component of (complex) angular wavevector for forward-moving #wave. Positive imaginary part means decaying. kz_list = 2 * np.pi * n_list * cos(th_list) / lam_vac #delta is the total phase accrued by traveling through a given layer. #ignore warning about inf multiplication olderr = sp.seterr(invalid='ignore') delta = kz_list * d_list sp.seterr(**olderr) # For a very opaque layer, reset delta to avoid divide-by-0 and similar # errors. The criterion imag(delta) > 35 corresponds to single-pass # transmission < 1e-30 --- small enough that the exact value doesn't # matter. for i in range(1, num_layers - 1): if delta[i].imag > 35: delta[i] = delta[i].real + 35j if 'opacity_warning' not in globals(): global opacity_warning opacity_warning = True print("Warning: Layers that are almost perfectly opaque " "are modified to be slightly transmissive, " "allowing 1 photon in 10^30 to pass through. It's " "for numerical stability. This warning will not " "be shown again.") #t_list[i,j] and r_list[i,j] are transmission and reflection amplitudes, #respectively, coming from i, going to j. Only need to calculate this when #j=i+1. (2D array is overkill but helps avoid confusion.) t_list = zeros((num_layers, num_layers), dtype=complex) r_list = zeros((num_layers, num_layers), dtype=complex) for i in range(num_layers - 1): t_list[i, i + 1] = interface_t(pol, n_list[i], n_list[i + 1], th_list[i], th_list[i + 1]) r_list[i, i + 1] = interface_r(pol, n_list[i], n_list[i + 1], th_list[i], th_list[i + 1]) #At the interface between the (n-1)st and nth material, let v_n be the #amplitude of the wave on the nth side heading forwards (away from the #boundary), and let w_n be the amplitude on the nth side heading backwards #(towards the boundary). Then (v_n,w_n) = M_n (v_{n+1},w_{n+1}). M_n is #M_list[n]. M_0 and M_{num_layers-1} are not defined. #My M is a bit different than Sernelius's, but Mtilde is the same. M_list = zeros((num_layers, 2, 2), dtype=complex) for i in range(1, num_layers - 1): M_list[i] = (1 / t_list[i, i + 1]) * np.dot( make_2x2_array( exp(-1j * delta[i]), 0, 0, exp(1j * delta[i]), dtype=complex), make_2x2_array( 1, r_list[i, i + 1], r_list[i, i + 1], 1, dtype=complex)) Mtilde = make_2x2_array(1, 0, 0, 1, dtype=complex) for i in range(1, num_layers - 1): Mtilde = np.dot(Mtilde, M_list[i]) Mtilde = np.dot( make_2x2_array(1, r_list[0, 1], r_list[0, 1], 1, dtype=complex) / t_list[0, 1], Mtilde) #Net complex transmission and reflection amplitudes r = Mtilde[1, 0] / Mtilde[0, 0] t = 1 / Mtilde[0, 0] #vw_list[n] = [v_n, w_n]. v_0 and w_0 are undefined because the 0th medium #has no left interface. vw_list = zeros((num_layers, 2), dtype=complex) vw = array([[t], [0]]) vw_list[-1, :] = np.transpose(vw) for i in range(num_layers - 2, 0, -1): vw = np.dot(M_list[i], vw) vw_list[i, :] = np.transpose(vw) #vn, wn ADDED MYSEFL vn = vw_list[:, 0] wn = vw_list[:, 1] #Net transmitted and reflected power, as a proportion of the incoming light #power. R = R_from_r(r) T = T_from_t(pol, t, n_list[0], n_list[-1], th_0, th_list[-1]) A = 1.0 - (R + T) #<--- ADDED MYSEFLF power_entering = power_entering_from_r(pol, r, n_list[0], th_0) # Need unaltered output to comput absorb_in_each_layer # Unchanged return from original source # https://github.com/sbyrnes321/tmm/blob/master/tmm_core.py#L304 unaltered_out = { 'r': r, 't': t, 'R': R, 'T': T, 'power_entering': power_entering, 'vw_list': vw_list, 'kz_list': kz_list, 'th_list': th_list, 'pol': pol, 'n_list': n_list, 'd_list': d_list, 'th_0': th_0, 'lam_vac': lam_vac } absorp_in_layers = absorp_in_each_layer(unaltered_out) if pame_output: # pame /vector_coh_tmm return out = { 'r_amp': r, 't_amp': t, 'R': R, 'T': T, 'A': A, 'pe': power_entering, } # Add quantities that have a value in each layer out.update(_flatten('vn', vn)) out.update(_flatten('wn', wn)) out.update(_flatten('kz', kz_list)) #adsorption in each layer (special) out.update(_flatten('absorb', absorp_in_layers)) out.update(_flatten('ang_prop', th_list)) return out else: return unaltered_out
def main(): parser = argparse.ArgumentParser() parser.add_argument('--path', default='trajdata', help='directory of data to test') parser.add_argument('--output', required=True, nargs='+', help='relative path to saved model') parser.add_argument('--obs_length', default=9, type=int, help='observation length') parser.add_argument('--pred_length', default=12, type=int, help='prediction length') parser.add_argument('--disable-write', action='store_true', help='disable writing new files') parser.add_argument('--disable-collision', action='store_true', help='disable collision metrics') parser.add_argument('--labels', required=False, nargs='+', help='labels of models') parser.add_argument('--normalize_scene', action='store_true', help='augment scenes') parser.add_argument('--goals', action='store_true', help='Considers goals during prediction') parser.add_argument('--unimodal', action='store_true', help='provide unimodal evaluation') parser.add_argument('--topk', action='store_true', help='provide topk evaluation') parser.add_argument('--multimodal', action='store_true', help='provide multimodal nll evaluation') parser.add_argument('--modes', default=1, type=int, help='number of modes to predict') args = parser.parse_args() scipy.seterr('ignore') ## Path to the data folder name to predict args.path = 'DATA_BLOCK/' + args.path + '/' ## Test_pred: Folders for saving model predictions args.path = args.path + 'test_pred/' if (not args.unimodal) and (not args.topk) and (not args.multimodal): args.unimodal = True # Compute unimodal metrics by default if args.multimodal: args.modes = 20 ## Writes to Test_pred ## Does this overwrite existing predictions? No. ### datasets = sorted([ f.split('.')[-2] for f in os.listdir(args.path.replace('_pred', '')) if not f.startswith('.') and f.endswith('.ndjson') ]) ## Model names are passed as arguments for model in args.output: model_name = model.split('/')[-1].replace('.pkl', '') # Loading the appropriate model (functionality only for SGAN and LSTM) print("Model Name: ", model_name) if 'sgan' in model_name: predictor = trajnetbaselines.sgan.SGANPredictor.load(model) else: predictor = trajnetbaselines.lstm.LSTMPredictor.load(model) # On CPU device = torch.device('cpu') predictor.model.to(device) total_scenes = 0 average = 0 final = 0 gt_col = 0. pred_col = 0. neigh_scenes = 0 topk_average = 0 topk_final = 0 all_goals = {} average_nll = 0 ## Start writing in dataset/test_pred for dataset in datasets: # Model's name name = dataset.replace( args.path.replace('_pred', '') + 'test/', '') # Copy file from test into test/train_pred folder print('processing ' + name) if 'collision_test' in name: continue # Read file from 'test' reader = trajnetplusplustools.Reader( args.path.replace('_pred', '') + dataset + '.ndjson', scene_type='paths') ## Necessary modification of train scene to add filename (for goals) scenes = [(dataset, s_id, s) for s_id, s in reader.scenes()] ## Consider goals ## Goal file must be present in 'goal_files/test_private' folder ## Goal file must have the same name as corresponding test file if args.goals: goal_dict = pickle.load( open('goal_files/test_private/' + dataset + '.pkl', "rb")) all_goals[dataset] = { s_id: [goal_dict[path[0].pedestrian] for path in s] for _, s_id, s in scenes } reader_gt = trajnetplusplustools.Reader( args.path.replace('_pred', '_private') + dataset + '.ndjson', scene_type='paths') scenes_gt = [s for _, s in reader_gt.scenes()] total_scenes += len(scenes_gt) for i, (filename, scene_id, paths) in enumerate(scenes): if i % 100 == 0: print("Scenes evaluated: ", '{}/{}'.format(i, len(scenes_gt))) ground_truth = scenes_gt[i] ## Convert numpy array to Track Rows ## ## Extract 1) first_frame, 2) frame_diff 3) ped_ids for writing predictions observed_path = paths[0] frame_diff = observed_path[1].frame - observed_path[0].frame first_frame = observed_path[args.obs_length - 1].frame + frame_diff ped_id = observed_path[0].pedestrian goals = get_goals(paths, all_goals, filename, scene_id) ## Zeros if no goals utilized predictions = predictor(paths, goals, n_predict=args.pred_length, obs_length=args.obs_length, modes=args.modes, args=args) if args.unimodal: ## Unimodal ## ADE / FDE prediction, neigh_predictions = predictions[0] prediction = np.round(prediction, 2) ## make Track Rows # primary prediction = [ trajnetplusplustools.TrackRow( first_frame + i * frame_diff, ped_id, prediction[i, 0], prediction[i, 1], 0) for i in range(len(prediction)) ] primary_tracks = [ t for t in prediction if t.prediction_number == 0 ] frame_gt = [ t.frame for t in ground_truth[0] ][args.obs_length:args.obs_length + args.pred_length] frame_pred = [t.frame for t in primary_tracks] ## To verify if same scene if frame_gt != frame_pred: raise Exception('frame numbers are not consistent') average_l2 = trajnetplusplustools.metrics.average_l2( ground_truth[0][args.obs_length:args.obs_length + args.pred_length], primary_tracks, n_predictions=args.pred_length) final_l2 = trajnetplusplustools.metrics.final_l2( ground_truth[0][args.obs_length:args.obs_length + args.pred_length], primary_tracks) # aggregate FDE and ADE average += average_l2 final += final_l2 ## Collision Metrics for j in range(1, len(ground_truth)): if trajnetplusplustools.metrics.collision( primary_tracks, ground_truth[j], n_predictions=args.pred_length): gt_col += 1 break # ## neighbours (if not empty) [Col-I] if neigh_predictions.shape[1]: neigh_scenes += 1 for n in range(neigh_predictions.shape[1]): neigh = neigh_predictions[:, n] neigh = np.round(neigh, 2) neigh_track = [ trajnetplusplustools.TrackRow( first_frame + j * frame_diff, n, neigh[j, 0], neigh[j, 1], 0) for j in range(len(neigh)) ] if trajnetplusplustools.metrics.collision( primary_tracks, neigh_track, n_predictions=args.pred_length): pred_col += 1 break primary_tracks_all = [ trajnetplusplustools.TrackRow(first_frame + i * frame_diff, ped_id, x, y, m) for m, (prim, neighs) in predictions.items() for i, (x, y) in enumerate(prim) ] if args.topk: topk_ade, topk_fde = trajnetplusplustools.metrics.topk( primary_tracks_all, ground_truth[0][args.obs_length:args.obs_length + args.pred_length], n_predictions=args.pred_length) topk_average += topk_ade topk_final += topk_fde if args.multimodal: nll_val = trajnetplusplustools.metrics.nll( primary_tracks_all, ground_truth[0], n_predictions=args.pred_length, n_samples=20) average_nll += nll_val if args.unimodal: ## Average ADE and FDE average /= total_scenes final /= total_scenes gt_col /= (total_scenes * 0.01) if neigh_scenes == 0: pred_col = -1 else: pred_col /= (neigh_scenes * 0.01) print('ADE: ', average) print('FDE: ', final) print("Col-I: ", pred_col) print("Col-II: ", gt_col) if args.topk: topk_average /= total_scenes topk_final /= total_scenes print('Topk_ADE: ', topk_average) print('Topk_FDE: ', topk_final) if args.multimodal: average_nll /= total_scenes print('Average NLL: ', average_nll)
# 1500K p['S'] = 1.5852424435 # p.model.chem_eq.DEBUG=True p.run_model() # [0.00036, 0.99946, 0.00018]) expected_concentrations = np.array( [3.58768646e-04, 9.99461847e-01, 1.79384323e-04]) n = p['n'] n_moles = p['n_moles'] concentrations = n / n_moles assert_rel_error(self, concentrations, expected_concentrations, 1e-4) expected_n_moles = 0.022726185333 assert_rel_error(self, n_moles, expected_n_moles, 1e-4) assert_rel_error(self, p['gamma'], 1.16381209181, 1e-4) # check = p.check_partial_derivatives() if __name__ == "__main__": import numpy as np import scipy as sp np.seterr(all='raise') sp.seterr(all='raise') unittest.main()
def enable_warnings(): logging.root.setLevel(logging.WARNING) scipy.seterr(over='print', divide='print', invalid='print', under='ignore')
def main(): scipy.seterr(all='ignore') ################################################# ########## CALCULATE RAW EVENT VECTORS ########## ################################################# db = mysql.connector.connect( host="localhost", user="******", database="monitor_db", passwd="D@1syKn0ws" ) # LOAD RAW VECTORS query = "SELECT name, start, end FROM raw_vectors" cursor = db.cursor(buffered=True) cursor.execute(query) raw_vectors = [] for row in cursor: raw_vectors.append({ 'stream_name': row[0], 'start': row[1], 'end': row[2] }) for raw_vector in tqdm(raw_vectors, desc='Creating Raw Vectors'): stream_name, start, end = raw_vector['stream_name'], raw_vector['start'], raw_vector['end'] query = "SELECT timestamp, value FROM stream_data WHERE name='%s' and timestamp>=%s and timestamp<=%s ORDER BY timestamp ASC;" % (stream_name, start, end) cursor.execute(query) vector = [] for row in cursor: vector.append(row[1]) vector = np.asarray(vector) raw_vector['vector'] = vector # LOAD EVENTS query = "SELECT event, stream, timeStart, timeEnd FROM event" cursor = db.cursor() cursor.execute(query) events = [] for row in cursor: events.append({ 'event_name': row[0], 'stream_name': row[1], 'start': row[2], 'end': row[3] }) all_corr_data = [] for event in tqdm(events, desc='Comparing Events'): event_name, stream_name, start, end = event['event_name'], event['stream_name'], event['start'], event['end'] query = "SELECT name, timestamp, value FROM stream_data WHERE name='%s' and timestamp >= '%s' and timestamp <= '%s' ORDER BY timestamp ASC;" % (stream_name, start, end) cursor.execute(query) event_vector = [] for index, row in enumerate(cursor): event_vector.append(row[2]) event_vector = np.asarray(event_vector) corr_data = [] for raw_vector_data in tqdm(raw_vectors, desc='Comparing to Event %s' % event_name): rstream_name, rstart, rend, rvector = raw_vector_data['stream_name'], raw_vector_data['start'], raw_vector_data['end'], raw_vector_data['vector'] if rvector.shape[0] > event_vector.shape[0]: rvector = rvector[:event_vector.shape[0]] if rvector.shape[0] == event_vector.shape[0]: corr = pearsonr(event_vector, rvector)[0] corr_data.append([event_name, rstream_name, rstart, rend, corr]) corr_data = pd.DataFrame(corr_data) corr_data = corr_data.rename(columns={0: 'event', 1: 'stream', 2: 'start', 3: 'end', 4:'corr'}) corr_data = corr_data.dropna(subset=['corr']) corr_data = corr_data.sort_values('corr', ascending=False).reset_index(drop=True) corr_data['rank'] = corr_data.index + 1 corr_data = corr_data[['event', 'rank', 'stream', 'start', 'end', 'corr']] all_corr_data.append(corr_data.iloc[:99, ]) all_corr_data = pd.concat(all_corr_data) all_corr_data.to_csv(DATA_OUTPUT, index=False, header=False) ####################################### ########## CREATE SQL SCRIPT ########## ####################################### sqlstr = 'USE monitor_db;\n\n' sqlstr += 'DROP TABLE IF EXISTS raw_event_vectors;\n\n' sqlstr += 'CREATE TABLE raw_event_vectors(\n' sqlstr += '\tevent VARCHAR(100),\n' sqlstr += '\trank INTEGER,\n' sqlstr += '\tstream VARCHAR(100),\n' sqlstr += '\ttimeStart INTEGER,\n' sqlstr += '\ttimeEnd INTEGER,\n' sqlstr += '\tcorrelation FLOAT\n' sqlstr += ');\n\n' sqlstr += f'LOAD DATA LOCAL INFILE "{DATA_OUTPUT}" INTO TABLE raw_event_vectors FIELDS TERMINATED BY ",";\n\n' with open(SCRIPT_OUTPUT, 'w') as w: w.write(sqlstr)
def coh_tmm(pol, n_list, d_list, th_0, lam_vac): """ This function is vectorized. Main "coherent transfer matrix method" calc. Given parameters of a stack, calculates everything you could ever want to know about how light propagates in it. (If performance is an issue, you can delete some of the calculations without affecting the rest.) pol is light polarization, "s" or "p". n_list is the list of refractive indices, in the order that the light would pass through them. The 0'th element of the list should be the semi-infinite medium from which the light enters, the last element should be the semi- infinite medium to which the light exits (if any exits). th_0 is the angle of incidence: 0 for normal, pi/2 for glancing. Remember, for a dissipative incoming medium (n_list[0] is not real), th_0 should be complex so that n0 sin(th0) is real (intensity is constant as a function of lateral position). d_list is the list of layer thicknesses (front to back). Should correspond one-to-one with elements of n_list. First and last elements should be "inf". lam_vac is vacuum wavelength of the light. Outputs the following as a dictionary (see manual for details) * r--reflection amplitude * t--transmission amplitude * R--reflected wave power (as fraction of incident) * T--transmitted wave power (as fraction of incident) * power_entering--Power entering the first layer, usually (but not always) equal to 1-R (see manual). * vw_list-- n'th element is [v_n,w_n], the forward- and backward-traveling amplitudes, respectively, in the n'th medium just after interface with (n-1)st medium. * kz_list--normal component of complex angular wavenumber for forward-traveling wave in each layer. * th_list--(complex) propagation angle (in radians) in each layer * pol, n_list, d_list, th_0, lam_vac--same as input """ # convert lists to numpy arrays if they're not already. n_list = array(n_list) d_list = array(d_list, dtype=float)[:, None] # input tests if hasattr(th_0, 'size') and th_0.size > 1 and th_0.size != lam_vac.size: raise ValueError( 'This function is not vectorized for angles; you need to run one angle calculation at a time.' ) if n_list.shape[0] != d_list.shape[0]: raise ValueError("Problem with n_list or d_list!") if (d_list[0] != inf) or (d_list[-1] != inf): raise ValueError('d_list must start and end with inf!') if any(abs((n_list[0] * np.sin(th_0)).imag) > 100 * EPSILON): raise ValueError('Error in n0 or th0!') if hasattr(th_0, 'size'): th_0 = array(th_0) num_layers = n_list.shape[0] num_wl = n_list.shape[1] # th_list is a list with, for each layer, the angle that the light travels # through the layer. Computed with Snell's law. Note that the "angles" may be # complex! th_list = list_snell(n_list, th_0) # kz is the z-component of (complex) angular wavevector for forward-moving # wave. Positive imaginary part means decaying. kz_list = 2 * np.pi * n_list * cos(th_list) / lam_vac # delta is the total phase accrued by traveling through a given layer. # ignore warning about inf multiplication olderr = sp.seterr(invalid='ignore') delta = kz_list * d_list sp.seterr(**olderr) # For a very opaque layer, reset delta to avoid divide-by-0 and similar # errors. The criterion imag(delta) > 35 corresponds to single-pass # transmission < 1e-30 --- small enough that the exact value doesn't # matter. # It DOES matter (for depth-dependent calculations!) delta[1:num_layers - 1, :] = np.where( delta[1:num_layers - 1, :].imag > 100, delta[1:num_layers - 1, :].real + 100j, delta[1:num_layers - 1, :]) # t_list[i,j] and r_list[i,j] are transmission and reflection amplitudes, # respectively, coming from i, going to j. Only need to calculate this when # j=i+1. (2D array is overkill but helps avoid confusion.) t_list = zeros((num_wl, num_layers, num_layers), dtype=complex) r_list = zeros((num_wl, num_layers, num_layers), dtype=complex) for i in range(num_layers - 1): t_list[:, i, i + 1] = interface_t(pol, n_list[i], n_list[i + 1], th_list[i], th_list[i + 1]) r_list[:, i, i + 1] = interface_r(pol, n_list[i], n_list[i + 1], th_list[i], th_list[i + 1]) # At the interface between the (n-1)st and nth material, let v_n be the # amplitude of the wave on the nth side heading forwards (away from the # boundary), and let w_n be the amplitude on the nth side heading backwards # (towards the boundary). Then (v_n,w_n) = M_n (v_{n+1},w_{n+1}). M_n is # M_list[n]. M_0 and M_{num_layers-1} are not defined. # My M is a bit different than Sernelius's, but Mtilde is the same. M_list = zeros((num_layers, num_wl, 2, 2), dtype=complex) for i in range(1, num_layers - 1): A = make_2x2_array(exp(-1j * delta[i]), np.zeros_like(delta[i]), np.zeros_like(delta[i]), exp(1j * delta[i]), dtype=complex) B = make_2x2_array(np.ones_like(delta[i]), r_list[:, i, i + 1], r_list[:, i, i + 1], np.ones_like(delta[i]), dtype=complex) d = (1 / t_list[:, i, i + 1]) M_list[i] = np.transpose( d * np.transpose(np.matmul(A, B))) # , (1, 2, 0)), (2, 0, 1)) Mtilde = make_2x2_array(np.ones_like(delta[i]), np.zeros_like(delta[i]), np.zeros_like(delta[i]), np.ones_like(delta[i]), dtype=complex) for i in range(1, num_layers - 1): Mtilde = np.matmul(Mtilde, M_list[i]) A = make_2x2_array(np.ones_like(delta[i]), r_list[:, 0, 1], r_list[:, 0, 1], np.ones_like(delta[i]), dtype=complex) d = 1 / t_list[:, 0, 1] Mtilde = np.matmul(np.transpose(d * np.transpose(A, (1, 2, 0)), (2, 0, 1)), Mtilde) # Net complex transmission and reflection amplitudes r = Mtilde[:, 1, 0] / Mtilde[:, 0, 0] t = np.ones_like(Mtilde[:, 0, 0]) / Mtilde[:, 0, 0] # vw_list[n] = [v_n, w_n]. v_0 and w_0 are undefined because the 0th medium # has no left interface. vw_list = zeros((num_layers, num_wl, 2), dtype=complex) vw = zeros((num_wl, 2, 2), dtype=complex) I = np.identity(2) vw[:, 0, 0] = t vw[:, 0, 1] = t vw_list[-1] = vw[:, 0, :] for i in range(num_layers - 2, 0, -1): vw = np.matmul(M_list[i], vw) vw_list[i, :, :] = vw[:, :, 1] vw_list[-1, :, 1] = 0 # Net transmitted and reflected power, as a proportion of the incoming light # power. R = R_from_r(r) T = T_from_t(pol, t, n_list[0], n_list[-1], th_0, th_list[-1]) power_entering = power_entering_from_r(pol, r, n_list[0], th_0) return { 'r': r, 't': t, 'R': R, 'T': T, 'power_entering': power_entering, 'vw_list': vw_list, 'kz_list': kz_list, 'th_list': th_list, 'pol': pol, 'n_list': n_list, 'd_list': d_list, 'th_0': th_0, 'lam_vac': lam_vac }
#!/usr/bin/env python """ fitting Phenome library biolog data fitting functions """ from ductape.common.utils import compress, smooth from scipy.optimize.minpack import curve_fit import numpy as np import logging # No country for warnings import scipy as sp sp.seterr(all='ignore') # __author__ = "Marco Galardini" logger = logging.getLogger('ductape.fitting') def logistic(x, A, u, d, v, y0): ''' Logistic growth model Taken from: "Modeling of the bacterial growth curve." (Zwietering et al., 1990) PMID: 16348228 ''' y = (A / (1 + np.exp( ( ((4 * u)/A) * (d - x) ) + 2 ))) + y0 return y
else: pcov = inf if return_full: return popt, pcov, infodict, errmsg, ier else: return popt, pcov # End of borrowed scipy fix import numpy as np import logging # No country for warnings import scipy as sp sp.seterr(all='ignore') # __author__ = "Marco Galardini" logger = logging.getLogger('ductape.fitting') def logistic(x, A, u, d, v, y0): ''' Logistic growth model Taken from: "Modeling of the bacterial growth curve." (Zwietering et al., 1990) PMID: 16348228 ''' y = (A / (1 + np.exp((((4 * u) / A) * (d - x)) + 2))) + y0
from unum.units import m, s, N, kg import copy import scipy import scipy.linalg import scipy.spatial import math scipy.seterr(divide="raise", invalid="raise") G = 6.673 * 10**-11 * N * (m / kg)**2 def magnitude(vect, unit): # shorthand to work around scipy not working with units return unit * scipy.linalg.norm(vect.asNumber(unit)) def distance(A, B): return m * scipy.spatial.distance.cdist([A.displacement.asNumber(m)], [B.displacement.asNumber(m)])[0][0] def speed(A, B): return magnitude(A.velocity - B.velocity, m / s) def velocity(A, B): return A.velocity - B.velocity def altitude(A, B): return distance(A, B) - A.radius - B.radius
def hyst_area_rand(D, A, DyV, AyV, DuV, AuV, csm_hysteretic_damping): """ function to calculate hystereris loop area used in the building damage calcs this version works for where the building cap curves are chasen randomly (which means Ay needs to be rescaled by Rcap;) In: DyV, AyV = yield point (vectors(nev,1)) DuV, AuV = ultimate point (vectors(nev,1)) D, A = peak dispacement and accel (vector(nev,1) of events) Out: Area = vector of events Glenn Fulford: 21/4/02. This is a diagram, trying to show how the hysteresis area is calculated if csm_hysteretic_damping = 'curve'. The diagram shows half the hysteresis area. y is the yield point PP is the performance point x2 is the translation along the displacement axis. A2 is the area from (y-x2) to PP on the top curve. A2 is also the area from y to (PP-x2) on the bottom curve. A1 and A3 are triaglular areas. The hysteresis area = 2(A1+A2-A3) y-x2 y PP | | | | | | | | | | | | ___.......--=----.------''' | | _,.,-':' __..--'' | |.-'' |_.-'' | _,'| ,-+ | ,' | ,-' /| | / A2 | ,-' / | |/ |,' / | :i______|______/ | + / / | /| / / | / | / / | / | / / | / | / / A3 | / | / / | / A1 |/ / | / + / | / '| / | """ # use a coord system with the orign are at the beginning # of the hysterisis curve # disp('hyst: caprand='); disp(SAcapR(:,1)); ky = (AyV / DyV) # slope of linear part of capacity curve # x0=D-DyV # x distance from linear part (<=0 implies point is linear) linear_region = where(D <= DyV) x2 = D - A / ky # translation along displacement axis if csm_hysteretic_damping is 'trapezoidal': # y distance from linear part (<=0 implies point is linear) y1 = (A - AyV) y1[linear_region] = -1 # avoid NaNs - only Harea3 has any impact there x1 = 2 * x2 + y1 / ky cc = A - AyV bb = ky / (A - AyV) aa = -ky / bb oldsettings = seterr(under='ignore') Harea1 = cc * x1 + aa / bb * (1 - exp(-bb * x1)) seterr(**oldsettings) Harea2 = 0.5 * y1 * y1 / ky Harea3 = 2 * x2 * AyV Harea = 2 * (Harea1 - Harea2 + Harea3) Harea[linear_region] = 0 elif csm_hysteretic_damping is 'curve': cc = AuV bb = ky / (AuV - AyV) aa = (AyV - AuV) * exp(bb * DyV) Harea1 = 0.5 * AyV * DyV Harea2 = aa / bb * \ (exp(-bb * DyV) - exp(-bb * (D + x2))) + cc * (D + x2 - DyV) Harea3 = 0.5 * A * (D - x2) Harea = 2 * (Harea1 + Harea2 - Harea3) Harea[linear_region] = 0 # Area calculation from # ATC 40 Seismic Evaluation and Retrofit of Concrete Buildings 8-15 # Ed = 4(Ay*Dpi - Dy*Api) elif csm_hysteretic_damping is 'parallelogram': Harea = 4 * (D * AyV - DyV * A) Harea[linear_region] = 0 else: print "csm_hysteretic_damping", csm_hysteretic_damping return Harea
def herm_sqrt_inv(x, zero_tol=1E-15, sanity_checks=False, return_rank=False, sc_data=''): if isinstance(x, mm.eyemat): x_sqrt = x x_sqrt_i = x rank = x.shape[0] else: try: ev = x.diag #simple_diag_matrix EV = None except AttributeError: ev, EV = la.eigh(x) zeros = ev <= zero_tol #throw away negative results too! ev_sqrt = sp.sqrt(ev) err = sp.seterr(divide='ignore', invalid='ignore') try: ev_sqrt_i = 1 / ev_sqrt ev_sqrt[zeros] = 0 ev_sqrt_i[zeros] = 0 finally: sp.seterr(divide=err['divide'], invalid=err['invalid']) if EV is None: x_sqrt = mm.simple_diag_matrix(ev_sqrt, dtype=x.dtype) x_sqrt_i = mm.simple_diag_matrix(ev_sqrt_i, dtype=x.dtype) else: B = mm.mmul_diag(ev_sqrt, EV.conj().T) x_sqrt = EV.dot(B) B = mm.mmul_diag(ev_sqrt_i, EV.conj().T) x_sqrt_i = EV.dot(B) rank = x.shape[0] - np.count_nonzero(zeros) if sanity_checks: if ev.min() < -zero_tol: log.warning( "Sanity Fail in herm_sqrt_inv(): Throwing away negative eigenvalues! %s %s", ev.min(), sc_data) if not np.allclose(x_sqrt.dot(x_sqrt), x): log.warning( "Sanity Fail in herm_sqrt_inv(): x_sqrt is bad! %s %s", la.norm(x_sqrt.dot(x_sqrt) - x), sc_data) if EV is None: nulls = sp.zeros(x.shape[0]) nulls[zeros] = 1 nulls = sp.diag(nulls) else: #if we did an EVD then we use the eigenvectors nulls = EV.copy() nulls[:, sp.invert(zeros)] = 0 nulls = nulls.dot(nulls.conj().T) eye = np.eye(x.shape[0]) if not np.allclose(x_sqrt.dot(x_sqrt_i), eye - nulls): log.warning( "Sanity Fail in herm_sqrt_inv(): x_sqrt_i is bad! %s %s", la.norm(x_sqrt.dot(x_sqrt_i) - eye + nulls), sc_data) if return_rank: return x_sqrt, x_sqrt_i, rank else: return x_sqrt, x_sqrt_i
def main(image, quantization_level): sp.seterr(all="ignore") used_qm = MIDDLE_COMPRESSION_QT if quantization_level == 1: used_qm = LOW_COMPRESSION_QT elif quantization_level == 2: used_qm = MIDDLE_COMPRESSION_QT elif quantization_level == 3: used_qm = HIGH_COMPRESSION_QT else: print("Quantization level can only be between 1 and 3") exit(1) quantized_image, shape, ac_coefs, dc_coefs = quantization(used_qm, image) encoded_qt = encode_quantization_table(used_qm) output_file = image.split("/")[-1].split(".")[0] + ".jpg" jpg_fd = open(output_file, "wb") #write SOI(Start of Image) jpg_fd.write(hex_to_bytes("FFD8")) #APP0 segment jpg_fd.write(hex_to_bytes('FFE000104A46494600010100000100010000')) # write Define Quantization Table(DQT) jpg_fd.write(hex_to_bytes("FFDB")) # write the size of DQT b = binary_to_bytes(encoded_qt) jpg_fd.write(int_to_bytes(len(b) + 3)) print(int_to_bytes(len(b) + 3)) # mode of the DQT: 00h for Y jpg_fd.write(hex_to_bytes("00")) # write encoded quantization table jpg_fd.write(binary_to_bytes(encoded_qt)) jpg_fd.write( hex_to_bytes( 'FFC401A20000000701010101010000000000000000040503020601000708090A0B0100020203010101010100000000000000010002030405060708090A0B1000020103030204020607030402060273010203110400052112314151061361227181143291A10715B14223C152D1E1331662F0247282F12543345392A2B26373C235442793A3B33617546474C3D2E2082683090A181984944546A4B456D355281AF2E3F3C4D4E4F465758595A5B5C5D5E5F566768696A6B6C6D6E6F637475767778797A7B7C7D7E7F738485868788898A8B8C8D8E8F82939495969798999A9B9C9D9E9F92A3A4A5A6A7A8A9AAABACADAEAFA110002020102030505040506040803036D0100021103042112314105511361220671819132A1B1F014C1D1E1234215526272F1332434438216925325A263B2C20773D235E2448317549308090A18192636451A2764745537F2A3B3C32829D3E3F38494A4B4C4D4E4F465758595A5B5C5D5E5F5465666768696A6B6C6D6E6F6475767778797A7B7C7D7E7F738485868788898A8B8C8D8E8F839495969798999A9B9C9D9E9F92A3A4A5A6A7A8A9AAABACADAEAFA' )) # write Start of Frame (SOF) # write the length of the segment (fixed) jpg_fd.write(hex_to_bytes("FFC0000B08")) # write height of the image jpg_fd.write(int_to_bytes(shape[0])) # write width of the image jpg_fd.write(int_to_bytes(shape[1])) # write number of components and component ids # Note: probably fixed jpg_fd.write(hex_to_bytes("01011100")) #SOS jpg_fd.write(hex_to_bytes("FFDA0008010100003F00")) a = encode_quantized_image(quantized_image) jpg_fd.write( binary_to_bytes(encode_quantized_image(quantized_image), add=1)) #EOI jpg_fd.write(hex_to_bytes("FFD9")) jpg_fd.close()
# -*- coding: utf-8 -*- """ Created on Sun Jan 04 16:42:15 2015 @author: Jan Øye Lindroos IO library methods for reading and writing to file and printing to screen """ import os, sys import scipy as sp sp.seterr(divide="ignore") # Inore errors related to log giving inf import pickle import time # Writer class for writing models to file class writer(object): def __init__(self, rank, opt): self.rank = rank self.path = opt.path self.format = opt.fileformat if self.format == "hdf5": import h5py self.filename = opt.run_name + "_" + str(rank) + ".hdf5" else: self.filename = opt.run_name + "_" + str(rank) + ".dat" self.f = open(os.path.join(opt.path, opt.run_name, self.filename), "w", 0) self.header = True # Determines weather the header shoul be written to file
# V_SI = cu(V, 'ft/s', 'm/s') # A_SI = cu(A, 'inch**2', 'm**2') # print(p['T'], p['P']) # print("Ps", Ps_computed, Ps) # print("Ts", Ts_computed, Ts) # print("gamma", gams_computed, gams) # print("V", V_computed, V) # print("A", A_computed, A) # print("MN", MN_computed, MN) # print("rhos", rhos_computed, rhos) # print() tol = 1.0e-4 assert_rel_error(self, gams_computed, gams, tol) assert_rel_error(self, MN_computed, MN, tol) assert_rel_error(self, Ps_computed, Ps, tol) assert_rel_error(self, Ts_computed, Ts, tol) assert_rel_error(self, hs_computed, hs, tol) assert_rel_error(self, rhos_computed, rhos, tol) assert_rel_error(self, gams_computed, gams, tol) assert_rel_error(self, V_computed, V, tol) assert_rel_error(self, A_computed, A, tol) if __name__ == "__main__": import scipy np.seterr(all='raise') scipy.seterr(all='raise') unittest.main()
#% in this example to show that the theoretical prices, volatilities, and correlations match up with the observed market data. #% #% There is not, to my knowledge, a commodities methodology that incorporates so many market factors across multiple commodities #% into one simulation. The advantages of such a model allows for more accurate modeling of spark spreads and pricing of deals #% that are dependent on multiple commodities prices. I have included all files, including excel, associated with this calibration #% and simulation. from quantdsl.priceprocess.base import PriceProcess try: from matplotlib import pylab as plt, pylab except RuntimeError: pass import scipy as np from scipy.optimize import basinhopping np.seterr(over='raise') class SchwartzSmithFromFuturesAndImpliedVols(PriceProcess): def simulate_future_prices(self, observation_date, requirements, path_count, calibration_params): fixing_dates = set() for requirement in requirements: fixing_dates.add(requirement[1]) fixing_dates = sorted(list(fixing_dates)) allMarketNames = [] allOptimizedParams = [] allSeasonalParams = []
def main(): parser = argparse.ArgumentParser() parser.add_argument('--path', default='trajdata', help='directory of data to test') parser.add_argument('--output', required=True, nargs='+', help='relative path to saved model') parser.add_argument('--obs_length', default=9, type=int, help='observation length') parser.add_argument('--pred_length', default=12, type=int, help='prediction length') parser.add_argument('--disable-write', action='store_true', help='disable writing new files') parser.add_argument('--disable-collision', action='store_true', help='disable collision metrics') parser.add_argument('--labels', required=False, nargs='+', help='labels of models') parser.add_argument('--sf', action='store_true', help='consider socialforce in evaluation') parser.add_argument('--orca', action='store_true', help='consider orca in evaluation') parser.add_argument('--kf', action='store_true', help='consider kalman in evaluation') parser.add_argument('--normalize_scene', action='store_true', help='augment scenes') parser.add_argument('--goals', action='store_true', help='Considers goals during prediction') parser.add_argument('--modes', default=1, type=int, help='number of modes to predict') args = parser.parse_args() scipy.seterr('ignore') ## Path to the data folder name to predict args.path = 'DATA_BLOCK/' + args.path + '/' ## Test_pred : Folders for saving model predictions args.path = args.path + 'test_pred/' ## Writes to Test_pred ### Does this overwrite existing predictions? No. ### if not args.disable_write: write.main(args) ## Evaluates test_pred with test_private names = [] for model in args.output: model_name = model.split('/')[-1].replace('.pkl', '') model_name = model_name + '_modes' + str(args.modes) names.append(model_name) ## labels if args.labels: labels = args.labels else: labels = names # Initiate Result Table table = Table() for num, name in enumerate(names): print(name) result_file = args.path.replace('pred', 'results') + name ## If result was pre-calculated and saved, Load if os.path.exists(result_file + '/results.pkl'): print("Loading Saved Results") with open(result_file + '/results.pkl', 'rb') as handle: [final_result, sub_final_result, col_result] = pickle.load(handle) table.add_result(labels[num], final_result, sub_final_result) table.add_collision_entry(labels[num], col_result) # ## Else, Calculate results and save else: list_sub = sorted([ f for f in os.listdir(args.path + name) if not f.startswith('.') ]) ## Simple Collision Test col_result = collision_test(list_sub, name, args) table.add_collision_entry(labels[num], col_result) submit_datasets = [ args.path + name + '/' + f for f in list_sub if 'collision_test.ndjson' not in f ] true_datasets = [ args.path.replace('pred', 'private') + f for f in list_sub if 'collision_test.ndjson' not in f ] ## Evaluate submitted datasets with True Datasets [The main eval function] # results = {submit_datasets[i].replace(args.path, '').replace('.ndjson', ''): # eval(true_datasets[i], submit_datasets[i], args) # for i in range(len(true_datasets))} results_list = Parallel(n_jobs=4)( delayed(eval)(true_datasets[i], submit_datasets[i], args) for i in range(len(true_datasets))) results = { submit_datasets[i].replace(args.path, '').replace('.ndjson', ''): results_list[i] for i in range(len(true_datasets)) } # print(results) ## Generate results final_result, sub_final_result = table.add_entry( labels[num], results) ## Save results as pkl (to avoid computation again) os.makedirs(result_file) with open(result_file + '/results.pkl', 'wb') as handle: pickle.dump([final_result, sub_final_result, col_result], handle, protocol=pickle.HIGHEST_PROTOCOL) ## Make Result Table table.print_table()
def greentensor(Freq, EpsB, Cell, NX, NY, NZ): """Returns the Fourier transform GF of the circular extension of the Green's tensor array """ c0 = 299792458.0 # speed of light in vacuum Mu0 = 4.0 * sci.pi * 1.0e-7 # vacuum permeability Eps0 = 1.0 / (Mu0 * c0 * c0) # vacuum permittivity Omega = 2.0 * sci.pi * Freq EtaB = -1.0j * Omega * Eps0 * EpsB ZetaB = -1.0j * Omega * Mu0 KB = Omega * sci.sqrt(Eps0 * EpsB * Mu0) G = sci.zeros((NX, NY, NZ, 3, 3), complex) GC = sci.zeros((NX * 2, NY * 2, NZ * 2, 3, 3), complex) GF = sci.zeros((NX * 2, NY * 2, NZ * 2, 3, 3), complex) # 3D arrays of x,y,z coordinates xx, yy, zz = sci.mgrid[0:NX * Cell:Cell, 0:NY * Cell:Cell, 0:NZ * Cell:Cell] dd = sci.zeros((NX, NY, NZ), complex) alpha = sci.zeros((NX, NY, NZ), complex) beta = sci.zeros((NX, NY, NZ), complex) Q11 = sci.zeros((NX, NY, NZ), complex) Q12 = sci.zeros((NX, NY, NZ), complex) Q13 = sci.zeros((NX, NY, NZ), complex) Q21 = sci.zeros((NX, NY, NZ), complex) Q22 = sci.zeros((NX, NY, NZ), complex) Q23 = sci.zeros((NX, NY, NZ), complex) Q31 = sci.zeros((NX, NY, NZ), complex) Q32 = sci.zeros((NX, NY, NZ), complex) Q33 = sci.zeros((NX, NY, NZ), complex) # 3D arrays of distances dd = sci.sqrt((xx)**2 + (yy)**2 + (zz)**2) dd2 = dd * dd # 3D arrays of components of the Q-matrix sci.seterr(divide='ignore', invalid='ignore') Q11 = sci.divide(xx * xx, dd2) Q12 = sci.divide(xx * yy, dd2) Q13 = sci.divide(xx * zz, dd2) Q22 = sci.divide(yy * yy, dd2) Q23 = sci.divide(yy * zz, dd2) Q33 = sci.divide(zz * zz, dd2) Q21 = Q12 Q31 = Q13 Q32 = Q23 # alpha and beta scalar multipliers alpha = sci.divide(sci.exp(1.0j*KB*dd),4.0*sci.pi*dd)*\ (-KB**2.0 - sci.divide(1.0j*3.0*KB,dd) + sci.divide(3.0,dd2)) beta = sci.divide(sci.exp(1.0j*KB*dd),4.0*sci.pi*dd)*\ (KB**2.0 + sci.divide(1.0j*KB,dd) - sci.divide(1.0,dd2)) # Green's tensor without self-patch G[:, :, :, 0, 0] = Q11 * alpha + beta G[:, :, :, 0, 1] = Q12 * alpha G[:, :, :, 0, 2] = Q13 * alpha G[:, :, :, 1, 0] = Q21 * alpha G[:, :, :, 1, 1] = Q22 * alpha + beta G[:, :, :, 1, 2] = Q23 * alpha G[:, :, :, 2, 0] = Q31 * alpha G[:, :, :, 2, 1] = Q32 * alpha G[:, :, :, 2, 2] = Q33 * alpha + beta G = G * (Cell**3) # multiplying by the elementary volume # self-patch G[0,0,0,0,0] = (2./3.)*(1.-1.j*KB*Cell*((3./(4.*sci.pi))**(1./3.)))*\ sci.exp(1.j*KB*Cell*((3./(4.*sci.pi))**(1./3.)))-1.0 G[0, 0, 0, 0, 1] = 0. G[0, 0, 0, 0, 2] = 0. G[0, 0, 0, 1, 0] = 0. G[0, 0, 0, 1, 1] = G[0, 0, 0, 0, 0] G[0, 0, 0, 1, 2] = 0. G[0, 0, 0, 2, 0] = 0. G[0, 0, 0, 2, 1] = 0. G[0, 0, 0, 2, 2] = G[0, 0, 0, 0, 0] #Circular extension of G GC[0:NX, 0:NY, 0:NZ, :, :] = G DeltaOp = sci.eye(3, 3) s = 0 while s <= 2: ss = 0 while ss <= 2: GC[NX + 1:, 0:NY, 0:NZ, s, ss] = (1 - 2 * DeltaOp[0, s]) * ( 1 - 2 * DeltaOp[0, ss]) * G[:0:-1, :, :, s, ss] GC[NX+1:,NY+1:,0:NZ,s,ss] = (1-2*DeltaOp[0,s])*(1-2*DeltaOp[0,ss])*\ (1-2*DeltaOp[1,s])*(1-2*DeltaOp[1,ss])*G[:0:-1,:0:-1,:,s,ss] GC[NX+1:,NY+1:,NZ+1:,s,ss] = (1-2*DeltaOp[0,s])*(1-2*DeltaOp[0,ss])*\ (1-2*DeltaOp[1,s])*(1-2*DeltaOp[1,ss])*\ (1-2*DeltaOp[2,s])*(1-2*DeltaOp[2,ss])*G[:0:-1,:0:-1,:0:-1,s,ss] GC[0:NX, NY + 1:, 0:NZ, s, ss] = (1 - 2 * DeltaOp[1, s]) * ( 1 - 2 * DeltaOp[1, ss]) * G[:, :0:-1, :, s, ss] GC[0:NX,NY+1:,NZ+1:,s,ss] = (1-2*DeltaOp[1,s])*(1-2*DeltaOp[1,ss])*\ (1-2*DeltaOp[2,s])*(1-2*DeltaOp[2,ss])*G[:,:0:-1,:0:-1,s,ss] GC[0:NX, 0:NY, NZ + 1:, s, ss] = (1 - 2 * DeltaOp[2, s]) * ( 1 - 2 * DeltaOp[2, ss]) * G[:, :, :0:-1, s, ss] GC[NX+1:,0:NY,NZ+1:,s,ss] = (1-2*DeltaOp[0,s])*(1-2*DeltaOp[0,ss])*\ (1-2*DeltaOp[2,s])*(1-2*DeltaOp[2,ss])*G[:0:-1,:,:0:-1,s,ss] ss = ss + 1 s = s + 1 # FFT of the Green's tensor array s = 0 while s <= 2: ss = 0 while ss <= 2: GF[:, :, :, s, ss] = fft.fftn(sci.squeeze(GC[:, :, :, s, ss])) ss = ss + 1 s = s + 1 return GF
def plot_params(args): """Plot alpha, theta, and the emission probabilities""" old_err = sp.seterr(under='ignore') oldsize = matplotlib.rcParams['font.size'] K, L = args.emit_probs.shape if not args.continuous_observations else args.means.shape # alpha #matplotlib.rcParams['font.size'] = 12 pyplot.figure() _, xedges, yedges = sp.histogram2d([0,K], [0,K], bins=[K,K]) extent = [xedges[0], xedges[-1], yedges[0], yedges[-1]] pyplot.imshow(args.alpha.astype(sp.float64), extent=extent, interpolation='nearest', vmin=0, vmax=1, cmap='OrRd', origin='lower') pyplot.xticks(sp.arange(K) + .5, sp.arange(K)+1) pyplot.gca().set_xticks(sp.arange(K)+1, minor=True) pyplot.yticks(sp.arange(K) + .5, sp.arange(K)+1) pyplot.gca().set_yticks(sp.arange(K)+1, minor=True) pyplot.grid(which='minor', alpha=.2) for line in pyplot.gca().yaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines(minor=True) + pyplot.gca().yaxis.get_ticklines(minor=True): # label is a Text instance line.set_markersize(0) pyplot.ylabel('Horizontal parent state') pyplot.xlabel('Node state') pyplot.title(r"Top root transition ($\alpha$) for {approx} iteration {iteration}". format(approx=args.approx, iteration=args.iteration)) b = pyplot.colorbar(shrink=.9) b.set_label("Probability") outfile = (args.out_params + '_it{iteration}.png').format(param='alpha', **args.__dict__) pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240) # beta pyplot.figure() _, xedges, yedges = sp.histogram2d([0,K], [0,K], bins=[K,K]) extent = [xedges[0], xedges[-1], yedges[0], yedges[-1]] pyplot.clf() pyplot.imshow(args.beta.astype(sp.float64), extent=extent, interpolation='nearest', vmin=0, vmax=1, cmap='OrRd', origin='lower') pyplot.xticks(sp.arange(K) + .5, sp.arange(K)+1) pyplot.gca().set_xticks(sp.arange(K)+1, minor=True) pyplot.yticks(sp.arange(K) + .5, sp.arange(K)+1) pyplot.gca().set_yticks(sp.arange(K)+1, minor=True) pyplot.grid(which='minor', alpha=.2) for line in pyplot.gca().yaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines(minor=True) + pyplot.gca().yaxis.get_ticklines(minor=True): # label is a Text instance line.set_markersize(0) pyplot.ylabel('Vertical parent state') pyplot.xlabel('Node state') pyplot.title(r"Left root transition ($\beta$) for {approx} iteration {iteration}". format(approx=args.approx, iteration=args.iteration)) b = pyplot.colorbar(shrink=.9) b.set_label("Probability") outfile = (args.out_params + '_it{iteration}.png').format(param='beta', **args.__dict__) pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240) # theta if args.separate_theta: theta_tmp = args.theta for i in range((args.theta.shape)[0]): setattr(args, 'theta_%s'%(i+1), args.theta[i,:,:,:]) for theta_name in ['theta'] + ['theta_%s' % i for i in range(20)]: #print 'trying', theta_name if not hasattr(args, theta_name): #print 'missing', theta_name continue _, xedges, yedges = sp.histogram2d([0,K], [0,K], bins=[K,K]) extent = [xedges[0], xedges[-1], yedges[0], yedges[-1]] if K == 18: numx_plots = 6 numy_plots = 3 elif K == 15: numx_plots = 5 numy_plots = 3 else: numx_plots = int(ceil(sp.sqrt(K))) numy_plots = int(ceil(sp.sqrt(K))) matplotlib.rcParams['font.size'] = 8 fig, axs = pyplot.subplots(numy_plots, numx_plots, sharex=True, sharey=True, figsize=(numx_plots*2.5,numy_plots*2.5)) for k in xrange(K): pltx, plty = k // numx_plots, k % numx_plots #axs[pltx,plty].imshow(args.theta[k,:,:], extent=extent, interpolation='nearest', axs[pltx,plty].imshow(getattr(args, theta_name)[:,k,:].astype(sp.float64), extent=extent, interpolation='nearest', vmin=0, vmax=1, cmap='OrRd', aspect='auto', origin='lower') #if k < numx_plots: #axs[pltx,plty].text(0 + .5, K - .5, 'vp=%s' % (k+1), horizontalalignment='left', verticalalignment='top', fontsize=10) axs[pltx,plty].text(0 + .5, K - .5, 'hp=%s' % (k+1), horizontalalignment='left', verticalalignment='top', fontsize=10) #axs[pltx,plty].xticks(sp.arange(K) + .5, sp.arange(K)) #axs[pltx,plty].yticks(sp.arange(K) + .5, sp.arange(K)) axs[pltx,plty].set_xticks(sp.arange(K) + .5) axs[pltx,plty].set_xticks(sp.arange(K)+1, minor=True) axs[pltx,plty].set_xticklabels(sp.arange(K) + 1) axs[pltx,plty].set_yticks(sp.arange(K) + .5) axs[pltx,plty].set_yticks(sp.arange(K)+1, minor=True) axs[pltx,plty].set_yticklabels(sp.arange(K) + 1) for line in axs[pltx,plty].yaxis.get_ticklines() + axs[pltx,plty].xaxis.get_ticklines() + axs[pltx,plty].yaxis.get_ticklines(minor=True) + axs[pltx,plty].xaxis.get_ticklines(minor=True): line.set_markersize(0) axs[pltx,plty].grid(True, which='minor', alpha=.2) #fig.suptitle(r"$\Theta$ with fixed parents for {approx} iteration {iteration}". # format(approx=args.approx, iteration=args.iteration), # fontsize=14, verticalalignment='top') fig.suptitle('Node state', y=.03, fontsize=14, verticalalignment='center') #fig.suptitle('Horizontal parent state', y=.5, x=.02, rotation=90, fig.suptitle('Vertical parent state', y=.5, x=.02, rotation=90, verticalalignment='center', fontsize=14) matplotlib.rcParams['font.size'] = 6.5 fig.subplots_adjust(wspace=.05, hspace=.05, left=.05, right=.95) #b = fig.colorbar(shrink=.9) #b.set_label("Probability") outfile = (args.out_params + '_vertparent_it{iteration}.png').format(param=theta_name, **args.__dict__) pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240) fig, axs = pyplot.subplots(numy_plots, numx_plots, sharex=True, sharey=True, figsize=(numx_plots*2.5,numy_plots*2.5)) for k in xrange(K): pltx, plty = k // numx_plots, k % numx_plots axs[pltx,plty].imshow(getattr(args, theta_name)[k,:,:].astype(sp.float64), extent=extent, interpolation='nearest', #axs[pltx,plty].imshow(args.theta[:,k,:], extent=extent, interpolation='nearest', vmin=0, vmax=1, cmap='OrRd', aspect='auto', origin='lower') #if k < numx_plots: axs[pltx,plty].text(0 + .5, K - .5, 'vp=%s' % (k+1), horizontalalignment='left', verticalalignment='top', fontsize=10) #axs[pltx,plty].xticks(sp.arange(K) + .5, sp.arange(K)) #axs[pltx,plty].yticks(sp.arange(K) + .5, sp.arange(K)) axs[pltx,plty].set_xticks(sp.arange(K) + .5) axs[pltx,plty].set_xticks(sp.arange(K)+1, minor=True) axs[pltx,plty].set_xticklabels(sp.arange(K) + 1) axs[pltx,plty].set_yticks(sp.arange(K) + .5) axs[pltx,plty].set_yticks(sp.arange(K)+1, minor=True) axs[pltx,plty].set_yticklabels(sp.arange(K) + 1) for line in axs[pltx,plty].yaxis.get_ticklines() + axs[pltx,plty].xaxis.get_ticklines() + axs[pltx,plty].yaxis.get_ticklines(minor=True) + axs[pltx,plty].xaxis.get_ticklines(minor=True): line.set_markersize(0) axs[pltx,plty].grid(True, which='minor', alpha=.2) #fig.suptitle(r"$\Theta$ with fixed parents for {approx} iteration {iteration}". # format(approx=args.approx, iteration=args.iteration), # fontsize=14, verticalalignment='top') fig.suptitle('Node state', y=.03, fontsize=14, verticalalignment='center') fig.suptitle('Horizontal parent state', y=.5, x=.02, rotation=90, #fig.suptitle('Vertical parent state', y=.5, x=.02, rotation=90, verticalalignment='center', fontsize=14) matplotlib.rcParams['font.size'] = 6.5 fig.subplots_adjust(wspace=.05, hspace=.05, left=.05, right=.95) #b = fig.colorbar(shrink=.9) #b.set_label("Probability") outfile = (args.out_params + '_it{iteration}.png').format(param=theta_name, **args.__dict__) pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240) # emission probabilities if args.continuous_observations: # plot mean values matplotlib.rcParams['font.size'] = 8 pyplot.figure(figsize=(max(1,round(L/3.)),max(1, round(K/3.)))) print (max(1,round(L/3.)),max(1, round(K/3.))) pyplot.imshow(args.means.astype(sp.float64), interpolation='nearest', aspect='auto', vmin=0, vmax=args.means.max(), cmap='OrRd', origin='lower') for k in range(K): for l in range(L): pyplot.text(l, k, '%.1f' % (args.means[k,l]), horizontalalignment='center', verticalalignment='center', fontsize=5) pyplot.yticks(sp.arange(K), sp.arange(K)+1) pyplot.gca().set_yticks(sp.arange(K)+.5, minor=True) pyplot.xticks(sp.arange(L), valid_marks, rotation=30, horizontalalignment='right') pyplot.gca().set_xticks(sp.arange(L)+.5, minor=True) pyplot.grid(which='minor', alpha=.2) for line in pyplot.gca().yaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines(minor=True) + pyplot.gca().yaxis.get_ticklines(minor=True): # label is a Text instance line.set_markersize(0) pyplot.ylabel('Hidden State') pyplot.title("Emission Mean") #b = pyplot.colorbar(shrink=.7) #b.set_label("Probability") outfile = (args.out_params + '_it{iteration}.png').format(param='emission_means', **args.__dict__) pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240) # plot variances pyplot.figure(figsize=(max(1,round(L/3.)),max(1, round(K/3.)))) print (L/3,K/3.) pyplot.imshow(args.variances.astype(sp.float64), interpolation='nearest', aspect='auto', vmin=0, vmax=args.variances.max(), cmap='OrRd', origin='lower') for k in range(K): for l in range(L): pyplot.text(l, k, '%.1f' % (args.variances[k,l]), horizontalalignment='center', verticalalignment='center', fontsize=5) pyplot.yticks(sp.arange(K), sp.arange(K)+1) pyplot.gca().set_yticks(sp.arange(K)+.5, minor=True) pyplot.xticks(sp.arange(L), valid_marks, rotation=30, horizontalalignment='right') pyplot.gca().set_xticks(sp.arange(L)+.5, minor=True) pyplot.grid(which='minor', alpha=.2) for line in pyplot.gca().yaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines(minor=True) + pyplot.gca().yaxis.get_ticklines(minor=True): # label is a Text instance line.set_markersize(0) pyplot.ylabel('Hidden State') pyplot.title("Emission Variance") #b = pyplot.colorbar(shrink=.7) #b.set_label("Probability") outfile = (args.out_params + '_it{iteration}.png').format(param='emission_variances', **args.__dict__) pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240) else: matplotlib.rcParams['font.size'] = 8 pyplot.figure(figsize=(max(1,round(L/3.)),max(1, round(K/3.)))) print (L/3,K/3.) pyplot.imshow(args.emit_probs.astype(sp.float64), interpolation='nearest', aspect='auto', vmin=0, vmax=1, cmap='OrRd', origin='lower') for k in range(K): for l in range(L): pyplot.text(l, k, '%2.0f' % (args.emit_probs[k,l] * 100), horizontalalignment='center', verticalalignment='center') pyplot.yticks(sp.arange(K), sp.arange(K)+1) pyplot.gca().set_yticks(sp.arange(K)+.5, minor=True) pyplot.xticks(sp.arange(L), valid_marks, rotation=30, horizontalalignment='right') pyplot.gca().set_xticks(sp.arange(L)+.5, minor=True) pyplot.grid(which='minor', alpha=.2) for line in pyplot.gca().yaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines(minor=True) + pyplot.gca().yaxis.get_ticklines(minor=True): # label is a Text instance line.set_markersize(0) pyplot.ylabel('Hidden State') pyplot.title("Emission probabilities") #b = pyplot.colorbar(shrink=.7) #b.set_label("Probability") outfile = (args.out_params + '_it{iteration}.png').format(param='emission', **args.__dict__) pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240) #broad_paper_enrichment = sp.array([[16,2,2,6,17,93,99,96,98,2], # [12,2,6,9,53,94,95,14,44,1], # [13,72,0,9,48,78,49,1,10,1], # [11,1,15,11,96,99,75,97,86,4], # [5,0,10,3,88,57,5,84,25,1], # [7,1,1,3,58,75,8,6,5,1], # [2,1,2,1,56,3,0,6,2,1], # [92,2,1,3,6,3,0,0,1,1], # [5,0,43,43,37,11,2,9,4,1], # [1,0,47,3,0,0,0,0,0,1], # [0,0,3,2,0,0,0,0,0,0], # [1,27,0,2,0,0,0,0,0,0], # [0,0,0,0,0,0,0,0,0,0], # [22,28,19,41,6,5,26,5,13,37], # [85,85,91,88,76,77,91,73,85,78], # [float('nan'), float('nan'),float('nan'),float('nan'),float('nan'),float('nan'),float('nan'),float('nan'),float('nan'),float('nan')] # ]) / 100. #mapping_from_broad = dict(zip(range(K), (5,2,0,14,4,6,9,1,12,-1,3,12,8,7,10,12,11,13))) #broad_paper_enrichment = broad_paper_enrichment[tuple(mapping_from_broad[i] for i in range(K)), :] #broad_names = ['Active promoter', 'Weak promoter', 'Inactive/poised promoter', 'Strong enhancer', # 'Strong enhancer', 'weak/poised enhancer', 'Weak/poised enhancer', 'Insulator', # 'Transcriptional transition', 'Transcriptional elongation', 'Weak transcribed', # 'Polycomb repressed', 'Heterochrom; low signal', 'Repetitive/CNV', 'Repetitive/CNV', # 'NA', 'NA', 'NA'] #pyplot.figure(figsize=(L/3,K/3.)) #print (L/3,K/3.) #pyplot.imshow(broad_paper_enrichment, interpolation='nearest', aspect='auto', # vmin=0, vmax=1, cmap='OrRd', origin='lower') #for k in range(K): # for l in range(L): # pyplot.text(l, k, '%2.0f' % (broad_paper_enrichment[k,l] * 100), horizontalalignment='center', verticalalignment='center') # pyplot.text(L, k, broad_names[mapping_from_broad[k]], horizontalalignment='left', verticalalignment='center', fontsize=6) #pyplot.yticks(sp.arange(K), sp.arange(K)+1) #pyplot.gca().set_yticks(sp.arange(K)+.5, minor=True) #pyplot.xticks(sp.arange(L), valid_marks, rotation=30, horizontalalignment='right') #pyplot.gca().set_xticks(sp.arange(L)+.5, minor=True) #pyplot.grid(which='minor', alpha=.2) #for line in pyplot.gca().yaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines() + pyplot.gca().xaxis.get_ticklines(minor=True) + pyplot.gca().yaxis.get_ticklines(minor=True): ## label is a Text instance # line.set_markersize(0) #pyplot.ylabel('Hidden State') #pyplot.title("Broad paper Emission probabilities") ##b = pyplot.colorbar(shrink=.7) ##b.set_label("Probability") #pyplot.subplots_adjust(right=.7) #outfile = (args.out_params + '_broadpaper.png').format(param='emission', **args.__dict__) #pyplot.savefig(os.path.join(args.out_dir, outfile), dpi=240) pyplot.close('all') sp.seterr(**old_err) matplotlib.rcParams['font.size'] = oldsize
import numpy as np import scipy as sp import matplotlib.pyplot as plt import pandas as pd import time from sklearn.model_selection import KFold from sklearn.metrics import r2_score from sklearn.kernel_ridge import KernelRidge import random import warnings as ws # Ignore all Underflow Warnings : Value close to Zero - No Accuracy Errors np.seterr(under='ignore') sp.seterr(under='ignore') ws.simplefilter('ignore') def kfold(dataset, k): kf = KFold(n_splits=k, shuffle=True, random_state=random.randint(1, 10)) return kf # Load the CSV file dataset = pd.read_csv('wineQualityReds.csv') percent = 0.80 dataset_cv = dataset[:int(percent * dataset.shape[0])] dataset_main = dataset[int(percent * dataset.shape[0]):] # Get Cross Validation K-Fold folds = kfold(dataset, 5)
#History (version,date, change author) # # # """ creates Likelihood classes for testing of MCMC,RJMCMC and other fitting methods. Also servers as an example on how to write functions needed to run programs""" _module_name = 'gauss_param_space' import numpy as nu #import pylab as lab import scipy.stats as stat_dist from scipy.special import expn, erfinv from scipy.optimize import fmin_powell from scipy import seterr a = seterr('ignore') #template class things needed for programs to run class template_class(object): #things needed for mcmc or rjmcmc to run def __init__(self): #initalize and put input data pass def sampler(self): #sampler or proposial distribution for drawing numbers pass def prior(self): #prior, should be a distribution, return values between [0,1] and also returns 0's for out of bounds parameters
def relative_bin_deviation(h1, h2): # 79 us @array, 104 us @list \w 100 bins r""" Calculate the bin-wise deviation between two histograms. The relative bin deviation between two histograms :math:`H` and :math:`H'` of size :math:`m` is defined as: .. math:: d_{rbd}(H, H') = \sum_{m=1}^M \frac{ \sqrt{(H_m - H'_m)^2} }{ \frac{1}{2} \left( \sqrt{H_m^2} + \sqrt{{H'}_m^2} \right) } *Attributes:* - a real metric *Attributes for normalized histograms:* - :math:`d(H, H')\in[0, \infty)` - :math:`d(H, H) = 0` - :math:`d(H, H') = d(H', H)` *Attributes for not-normalized histograms:* - :math:`d(H, H')\in[0, \infty)` - :math:`d(H, H) = 0` - :math:`d(H, H') = d(H', H)` *Attributes for not-equal histograms:* - not applicable Parameters ---------- h1 : sequence The first histogram. h2 : sequence The second histogram, same bins as ``h1``. Returns ------- relative_bin_deviation : float Relative bin deviation between the two histograms. """ h1, h2 = __prepare_histogram(h1, h2) numerator = scipy.sqrt(scipy.square(h1 - h2)) denominator = (scipy.sqrt(scipy.square(h1)) + scipy.sqrt(scipy.square(h2))) / 2. old_err_state = scipy.seterr( invalid='ignore' ) # divide through zero only occurs when the bin is zero in both histograms, in which case the division is 0/0 and leads to (and should lead to) 0 result = numerator / denominator scipy.seterr(**old_err_state) result[scipy.isnan( result )] = 0 # faster than scipy.nan_to_num, which checks for +inf and -inf also return scipy.sum(result)
from ..physics.Constants import erg_per_ev from ..physics.SecondaryElectrons import * import os, re, scipy, itertools, math, copy from scipy.integrate import quad, trapz, simps try: from mpi4py import MPI rank = MPI.COMM_WORLD.rank size = MPI.COMM_WORLD.size except ImportError: rank = 0 size = 1 E_th = [13.6, 24.6, 54.4] scipy.seterr(all='ignore') class IntegralTable: def __init__(self, pf, source, grid, logN=None): """ Initialize a table of integral quantities. """ self.pf = pf self.src = source self.grid = grid # need to not do this stuff if a table was supplied via source_table # best way: save pf in hdf5 file, use it # just override table_* parameters...what else?
def main(): parser = argparse.ArgumentParser() parser.add_argument('--path', default='trajdata', help='directory of data to test') parser.add_argument('--output', required=True, nargs='+', help='relative path to saved model') parser.add_argument('--obs_length', default=9, type=int, help='observation length') parser.add_argument('--pred_length', default=12, type=int, help='prediction length') parser.add_argument('--disable-write', action='store_true', help='disable writing new files') parser.add_argument('--disable-collision', action='store_true', help='disable collision metrics') parser.add_argument('--labels', required=False, nargs='+', help='labels of models') parser.add_argument('--normalize_scene', action='store_true', help='augment scenes') parser.add_argument('--unimodal', action='store_true', help='provide unimodal evaluation') parser.add_argument('--topk', action='store_true', help='provide topk evaluation') parser.add_argument('--multimodal', action='store_true', help='provide multimodal nll evaluation') parser.add_argument('--modes', default=1, type=int, help='number of modes to predict') parser.add_argument('--scene_type', default=0, type=int, choices=(0, 1, 2, 3, 4), help='type of scene to evaluate') parser.add_argument('--thresh', default=0.0, type=float, help='noise thresh') parser.add_argument('--ped_type', default='primary', help='type of ped to add noise to') args = parser.parse_args() scipy.seterr('ignore') ## Path to the data folder name to predict args.path = 'DATA_BLOCK/' + args.path + '/' ## Test_pred: Folders for saving model predictions args.path = args.path + 'test_pred/' if (not args.unimodal) and (not args.topk) and (not args.multimodal): args.unimodal = True # Compute unimodal metrics by default if args.topk: args.modes = 3 if args.multimodal: args.modes = 20 enable_col1 = True ## drop pedestrians that appear post observation def drop_post_obs(ground_truth, obs_length): obs_end_frame = ground_truth[0][obs_length].frame ground_truth = [ track for track in ground_truth if track[0].frame < obs_end_frame ] return ground_truth ## Writes to Test_pred ## Does this overwrite existing predictions? No. ### datasets = sorted([ f.split('.')[-2] for f in os.listdir(args.path.replace('_pred', '')) if not f.startswith('.') and f.endswith('.ndjson') ]) ## Model names are passed as arguments for model in args.output: model_name = model.split('/')[-1].replace('.pkl', '') # Loading the appropriate model (functionality only for SGAN and LSTM) print("Model Name: ", model_name) if 'sgan' in model_name: predictor = trajnetbaselines.sgan.SGANPredictor.load(model) goal_flag = predictor.model.generator.goal_flag elif 'vae' in model_name: predictor = trajnetbaselines.vae.VAEPredictor.load(model) goal_flag = predictor.model.goal_flag else: predictor = trajnetbaselines.lstm.LSTMPredictor.load(model) goal_flag = predictor.model.goal_flag # On CPU device = torch.device('cpu') predictor.model.to(device) total_scenes = 0 average = 0 final = 0 gt_col = 0. pred_col = 0. neigh_scenes = 0 topk_average = 0 topk_final = 0 all_goals = {} average_nll = 0 ## Start writing in dataset/test_pred for dataset in datasets: # Model's name name = dataset.replace( args.path.replace('_pred', '') + 'test/', '') # Copy file from test into test/train_pred folder print('processing ' + name) if 'collision_test' in name: continue ## Filter for Scene Type reader_tag = trajnetplusplustools.Reader( args.path.replace('_pred', '_private') + dataset + '.ndjson', scene_type='tags') if args.scene_type != 0: filtered_scene_ids = [ s_id for s_id, tag, s in reader_tag.scenes() if tag[0] == args.scene_type ] else: filtered_scene_ids = [ s_id for s_id, _, _ in reader_tag.scenes() ] # Read file from 'test' reader = trajnetplusplustools.Reader( args.path.replace('_pred', '') + dataset + '.ndjson', scene_type='paths') ## Necessary modification of train scene to add filename (for goals) scenes = [(dataset, s_id, s) for s_id, s in reader.scenes() if s_id in filtered_scene_ids] ## Consider goals ## Goal file must be present in 'goal_files/test_private' folder ## Goal file must have the same name as corresponding test file if goal_flag: goal_dict = pickle.load( open('goal_files/test_private/' + dataset + '.pkl', "rb")) all_goals[dataset] = { s_id: [goal_dict[path[0].pedestrian] for path in s] for _, s_id, s in scenes } ## Get Goals if goal_flag: scene_goals = [ np.array(all_goals[filename][scene_id]) for filename, scene_id, _ in scenes ] else: scene_goals = [ np.zeros((len(paths), 2)) for _, scene_id, paths in scenes ] print("Getting Predictions") scenes = tqdm(scenes) ## Get all predictions in parallel. Faster! pred_list = Parallel(n_jobs=12)( delayed(process_scene)(predictor, model_name, paths, scene_goal, args) for (_, _, paths), scene_goal in zip(scenes, scene_goals)) ## GT Scenes reader_gt = trajnetplusplustools.Reader( args.path.replace('_pred', '_private') + dataset + '.ndjson', scene_type='paths') scenes_gt = [ s for s_id, s in reader_gt.scenes() if s_id in filtered_scene_ids ] total_scenes += len(scenes_gt) print("Evaluating Predictions") scenes = tqdm(scenes) for (predictions, (_, scene_id, paths), ground_truth) in zip(pred_list, scenes, scenes_gt): ## Extract 1) first_frame, 2) frame_diff 3) ped_ids for writing predictions observed_path = paths[0] frame_diff = observed_path[1].frame - observed_path[0].frame first_frame = observed_path[args.obs_length - 1].frame + frame_diff ped_id = observed_path[0].pedestrian ped_id_ = [] for j, _ in enumerate(paths[1:]): ## Only need neighbour ids ped_id_.append(paths[j + 1][0].pedestrian) if args.unimodal: ## Unimodal ## ADE / FDE prediction, neigh_predictions = predictions[0] prediction = np.round(prediction, 2) ## make Track Rows # primary prediction = [ trajnetplusplustools.TrackRow( first_frame + i * frame_diff, ped_id, prediction[i, 0], prediction[i, 1], 0) for i in range(len(prediction)) ] primary_tracks = [ t for t in prediction if t.prediction_number == 0 ] frame_gt = [ t.frame for t in ground_truth[0] ][args.obs_length:args.obs_length + args.pred_length] frame_pred = [t.frame for t in primary_tracks] ## To verify if same scene if frame_gt != frame_pred: raise Exception('frame numbers are not consistent') average_l2 = trajnetplusplustools.metrics.average_l2( ground_truth[0][args.obs_length:args.obs_length + args.pred_length], primary_tracks, n_predictions=args.pred_length) final_l2 = trajnetplusplustools.metrics.final_l2( ground_truth[0][args.obs_length:args.obs_length + args.pred_length], primary_tracks) # aggregate FDE and ADE average += average_l2 final += final_l2 ground_truth = drop_post_obs(ground_truth, args.obs_length) ## Collision Metrics for j in range(1, len(ground_truth)): if trajnetplusplustools.metrics.collision( primary_tracks, ground_truth[j], n_predictions=args.pred_length): gt_col += 1 break num_gt_neigh = len(ground_truth) - 1 num_predicted_neigh = neigh_predictions.shape[1] if num_gt_neigh != num_predicted_neigh: enable_col1 = False # [Col-I] only if neighs in gt = neighs in prediction if enable_col1: neigh_scenes += 1 for n in range(neigh_predictions.shape[1]): neigh = neigh_predictions[:, n] neigh = np.round(neigh, 2) neigh_track = [ trajnetplusplustools.TrackRow( first_frame + j * frame_diff, n, neigh[j, 0], neigh[j, 1], 0) for j in range(len(neigh)) ] if trajnetplusplustools.metrics.collision( primary_tracks, neigh_track, n_predictions=args.pred_length): pred_col += 1 break primary_tracks_all = [ trajnetplusplustools.TrackRow(first_frame + i * frame_diff, ped_id, x, y, m) for m, (prim, neighs) in predictions.items() for i, (x, y) in enumerate(prim) ] if args.topk: topk_ade, topk_fde = trajnetplusplustools.metrics.topk( primary_tracks_all, ground_truth[0][args.obs_length:args.obs_length + args.pred_length], n_predictions=args.pred_length) topk_average += topk_ade topk_final += topk_fde if args.multimodal: nll_val = trajnetplusplustools.metrics.nll( primary_tracks_all, ground_truth[0], n_predictions=args.pred_length, n_samples=20) average_nll += nll_val if args.unimodal: ## Average ADE and FDE average /= total_scenes final /= total_scenes gt_col /= (total_scenes * 0.01) if not enable_col1: pred_col = -1 else: pred_col /= (neigh_scenes * 0.01) print('ADE: ', np.round(average, 3)) print('FDE: ', np.round(final, 3)) print("Col-I: ", np.round(pred_col, 2)) print("Col-II: ", np.round(gt_col, 2)) if args.topk: topk_average /= total_scenes topk_final /= total_scenes print('Topk_ADE: ', topk_average) print('Topk_FDE: ', topk_final) if args.multimodal: average_nll /= total_scenes print('Average NLL: ', average_nll)
def disable_warnings(): scipy.seterr(over='ignore', divide='ignore', invalid='ignore', under='ignore') logging.root.setLevel(logging.CRITICAL)
####################################################################### # This is the main part of the program, which calls the above functions ####################################################################### # First, initialise some of our variables to be empty coords_array = scipy.array([]) point_handles_array = scipy.array([]) handle_of_regression_line_plot = [] ### Set up an initial space to click inside axis_range = 10 ### Make the figure window pylab.figure() ### Clear the figure window pylab.clf() # clf means "clear the figure" ### In order to keep the boundaries of the figure fixed in place, ### we will draw a white box around the region that we want. pylab.plot(axis_range*scipy.array([-1, 1, 1, -1]), axis_range*scipy.array([-1, -1, 1, 1]),'w-') pylab.axis('equal') # Make the tick-marks equally spaced on x- and y-axes pylab.axis(axis_range*scipy.array([-1, 1, -1, 1])) ### Python issues a warning when we try to calculate ### the correlation when there are just two points, ### as the p-value is zero. This next line hides that warning scipy.seterr(invalid="ignore") ### Tell Python to call a function every time ### when the mouse is pressed in this figure pylab.connect('button_press_event', do_this_when_the_mouse_is_clicked) clear_the_figure_and_empty_points_list() pylab.show() # This shows the figure window onscreen
def main(): # This configures numpy/scipy to raise an exception in case of errors, instead of printing a warning and going ahead. numpy.seterr(all='raise') scipy.seterr(all='raise') parser = argparse.ArgumentParser( description='Runs a set of benchmarks defined in a YAML file.') parser.add_argument( '--fruit-benchmark-sources-dir', help='Path to the fruit sources (used for benchmarking code only)') parser.add_argument('--fruit-sources-dir', help='Path to the fruit sources') parser.add_argument('--boost-di-sources-dir', help='Path to the Boost.DI sources') parser.add_argument( '--output-file', help= 'The output file where benchmark results will be stored (1 per line, with each line in JSON format). These can then be formatted by e.g. the format_bench_results script.' ) parser.add_argument( '--benchmark-definition', help= 'The YAML file that defines the benchmarks (see fruit_wiki_benchs.yml for an example).' ) args = parser.parse_args() if args.output_file is None: raise Exception('You must specify --output_file') sh.rm('-f', args.output_file) fruit_build_tmpdir = tempfile.gettempdir() + '/fruit-benchmark-build-dir' with open(args.benchmark_definition, 'r') as f: yaml_file_content = yaml.load(f) global_definitions = yaml_file_content['global'] benchmark_definitions = expand_benchmark_definitions( yaml_file_content['benchmarks']) benchmark_index = 0 all_compilers = { benchmarks_definition['compiler'] for benchmarks_definition in benchmark_definitions } for compiler_executable_name in all_compilers: print('Preparing for benchmarks with the compiler %s' % compiler_executable_name) compiler_name = determine_compiler_name(compiler_executable_name) # Build Fruit in fruit_build_tmpdir, so that fruit_build_tmpdir points to a built Fruit (useful for e.g. the config header). shutil.rmtree(fruit_build_tmpdir, ignore_errors=True) os.makedirs(fruit_build_tmpdir) modified_env = os.environ.copy() modified_env['CXX'] = compiler_executable_name sh.cmake(args.fruit_sources_dir, '-DCMAKE_BUILD_TYPE=Release', _cwd=fruit_build_tmpdir, _env=modified_env) make_command(_cwd=fruit_build_tmpdir) for benchmark_definition in benchmark_definitions: if benchmark_definition['compiler'] != compiler_executable_name: continue # The 'compiler_name' dimension is synthesized automatically from the 'compiler' dimension. # We put the compiler name/version in the results because the same 'compiler' value might refer to different compiler versions # (e.g. if GCC 6.0.0 is installed when benchmarks are run, then it's updated to GCC 6.0.1 and finally the results are formatted, we # want the formatted results to say "GCC 6.0.0" instead of "GCC 6.0.1"). benchmark_definition = benchmark_definition.copy() benchmark_definition['compiler_name'] = compiler_name benchmark_index += 1 print('%s/%s: %s' % (benchmark_index, len(benchmark_definitions), benchmark_definition)) benchmark_name = benchmark_definition['name'] if (benchmark_name in { 'boost_di_compile_time', 'boost_di_run_time', 'boost_di_executable_size' } and args.boost_di_sources_dir is None): raise Exception( 'Error: you need to specify the --boost-di-sources-dir flag in order to run Boost.DI benchmarks.' ) if benchmark_name == 'new_delete_run_time': benchmark = NewDeleteRunTimeBenchmark( benchmark_definition, fruit_benchmark_sources_dir=args. fruit_benchmark_sources_dir) elif benchmark_name == 'fruit_single_file_compile_time': benchmark = FruitSingleFileCompileTimeBenchmark( benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_benchmark_sources_dir=args. fruit_benchmark_sources_dir, fruit_build_tmpdir=fruit_build_tmpdir) elif benchmark_name == 'fruit_compile_time': benchmark = FruitCompileTimeBenchmark( benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_build_tmpdir=fruit_build_tmpdir) elif benchmark_name == 'fruit_run_time': benchmark = FruitRunTimeBenchmark( benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_build_tmpdir=fruit_build_tmpdir) elif benchmark_name == 'fruit_executable_size': benchmark = FruitExecutableSizeBenchmark( benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_build_tmpdir=fruit_build_tmpdir) elif benchmark_name == 'boost_di_compile_time': benchmark = BoostDiCompileTimeBenchmark( benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_build_tmpdir=fruit_build_tmpdir, boost_di_sources_dir=args.boost_di_sources_dir) elif benchmark_name == 'boost_di_run_time': benchmark = BoostDiRunTimeBenchmark( benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_build_tmpdir=fruit_build_tmpdir, boost_di_sources_dir=args.boost_di_sources_dir) elif benchmark_name == 'boost_di_executable_size': benchmark = BoostDiExecutableSizeBenchmark( benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_build_tmpdir=fruit_build_tmpdir, boost_di_sources_dir=args.boost_di_sources_dir) else: raise Exception("Unrecognized benchmark: %s" % benchmark_name) run_benchmark(benchmark, output_file=args.output_file, max_runs=global_definitions['max_runs'])
def main(): # This configures numpy/scipy to raise an exception in case of errors, instead of printing a warning and going ahead. numpy.seterr(all='raise') scipy.seterr(all='raise') parser = argparse.ArgumentParser(description='Runs a set of benchmarks defined in a YAML file.') parser.add_argument('--fruit-benchmark-sources-dir', help='Path to the fruit sources (used for benchmarking code only)') parser.add_argument('--fruit-sources-dir', help='Path to the fruit sources') parser.add_argument('--boost-di-sources-dir', help='Path to the Boost.DI sources') parser.add_argument('--output-file', help='The output file where benchmark results will be stored (1 per line, with each line in JSON format). These can then be formatted by e.g. the format_bench_results script.') parser.add_argument('--benchmark-definition', help='The YAML file that defines the benchmarks (see fruit_wiki_benchs_fruit.yml for an example).') parser.add_argument('--continue-benchmark', help='If this is \'true\', continues a previous benchmark run instead of starting from scratch (taking into account the existing benchmark results in the file specified with --output-file).') args = parser.parse_args() if args.output_file is None: raise Exception('You must specify --output_file') if args.continue_benchmark == 'true': try: with open(args.output_file, 'r') as f: previous_run_completed_benchmarks = [json.loads(line)['benchmark'] for line in f.readlines()] except FileNotFoundError: previous_run_completed_benchmarks = [] else: previous_run_completed_benchmarks = [] run_command('rm', args=['-f', args.output_file]) fruit_build_tmpdir = tempfile.gettempdir() + '/fruit-benchmark-build-dir' with open(args.benchmark_definition, 'r') as f: yaml_file_content = yaml.load(f) global_definitions = yaml_file_content['global'] benchmark_definitions = expand_benchmark_definitions(yaml_file_content['benchmarks']) benchmark_index = 0 for (compiler_executable_name, additional_cmake_args), benchmark_definitions_with_current_config \ in group_by(benchmark_definitions, lambda benchmark_definition: (benchmark_definition['compiler'], tuple(benchmark_definition['additional_cmake_args']))): print('Preparing for benchmarks with the compiler %s, with additional CMake args %s' % (compiler_executable_name, additional_cmake_args)) # We compute this here (and memoize the result) so that the benchmark's describe() will retrieve the cached # value instantly. determine_compiler_name(compiler_executable_name) # Build Fruit in fruit_build_tmpdir, so that fruit_build_tmpdir points to a built Fruit (useful for e.g. the config header). shutil.rmtree(fruit_build_tmpdir, ignore_errors=True) os.makedirs(fruit_build_tmpdir) modified_env = os.environ.copy() modified_env['CXX'] = compiler_executable_name run_command('cmake', args=[ args.fruit_sources_dir, '-DCMAKE_BUILD_TYPE=Release', *additional_cmake_args, ], cwd=fruit_build_tmpdir, env=modified_env) run_command('make', args=make_args, cwd=fruit_build_tmpdir) for benchmark_definition in benchmark_definitions_with_current_config: benchmark_index += 1 print('%s/%s: %s' % (benchmark_index, len(benchmark_definitions), benchmark_definition)) benchmark_name = benchmark_definition['name'] if (benchmark_name in {'boost_di_compile_time', 'boost_di_run_time', 'boost_di_executable_size'} and args.boost_di_sources_dir is None): raise Exception('Error: you need to specify the --boost-di-sources-dir flag in order to run Boost.DI benchmarks.') if benchmark_name == 'new_delete_run_time': benchmark = NewDeleteRunTimeBenchmark( benchmark_definition, fruit_benchmark_sources_dir=args.fruit_benchmark_sources_dir) elif benchmark_name == 'fruit_single_file_compile_time': benchmark = FruitSingleFileCompileTimeBenchmark( benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_benchmark_sources_dir=args.fruit_benchmark_sources_dir, fruit_build_tmpdir=fruit_build_tmpdir) elif benchmark_name == 'fruit_compile_time': benchmark = FruitCompileTimeBenchmark( benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_build_tmpdir=fruit_build_tmpdir) elif benchmark_name == 'fruit_run_time': benchmark = FruitRunTimeBenchmark( benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_build_tmpdir=fruit_build_tmpdir) elif benchmark_name == 'fruit_executable_size': benchmark = FruitExecutableSizeBenchmark( benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_build_tmpdir=fruit_build_tmpdir) elif benchmark_name == 'boost_di_compile_time': benchmark = BoostDiCompileTimeBenchmark( benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_build_tmpdir=fruit_build_tmpdir, boost_di_sources_dir=args.boost_di_sources_dir) elif benchmark_name == 'boost_di_run_time': benchmark = BoostDiRunTimeBenchmark( benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_build_tmpdir=fruit_build_tmpdir, boost_di_sources_dir=args.boost_di_sources_dir) elif benchmark_name == 'boost_di_executable_size': benchmark = BoostDiExecutableSizeBenchmark( benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_build_tmpdir=fruit_build_tmpdir, boost_di_sources_dir=args.boost_di_sources_dir) else: raise Exception("Unrecognized benchmark: %s" % benchmark_name) if benchmark.describe() in previous_run_completed_benchmarks: print("Skipping benchmark that was already run previously (due to --continue-benchmark):", benchmark.describe()) continue run_benchmark(benchmark, output_file=args.output_file, max_runs=global_definitions['max_runs'])
# moduleNames is used for do_coverage # test_result is used in distribution.py return moduleNames, test_result if __name__ == '__main__': """ Note, the parameter is currently obsolete The eqrm_code directory that will be tested can be passed in as a parameter. This is usefull to test one sandpit from another sandpit. Used in distribution.py """ from os import access, F_OK import sys seterr(all='warn') if len(sys.argv) > 1 and access(sys.argv[1], F_OK): path = sys.argv[1] path = None else: path = None _, test_result = main(path=path) try: c_errors_failures = len(test_result.errors) + len(test_result.failures) except: print "WARNING TEST RESULTS UNKNOWN" c_errors_failures = 1 sys.exit(c_errors_failures)
def ypred_leave_one_out (train, test, percent) : #split to ground truth (y) and feature values (X) ytrain, Xtrain = np.hsplit(train, np.array([1])) ytest, Xtest = np.hsplit(test, np.array([1])) ytrain = np.concatenate(ytrain) ytest = np.concatenate(ytest) #normalize feature matrices (mins,maxs) = normalize_by_columns(Xtrain) #reduce features a = maxs-mins boolean_array = np.bool_(maxs-mins < 2.2204460492503131e-16) #print np.where(boolean_array) Xtrain = Xtrain.compress(np.logical_not(boolean_array), axis=1) training_n, reduced_feature_n = Xtrain.shape seterr(invalid='print',divide='print') idx = 0 C = [(0), (0), (0)] E = np.zeros(training_n) for idx in range(0, reduced_feature_n): xi = Xtrain.T[idx] #print xi.shape, idx #print ytrain #print xi ytrain, xi slope, intercept, r_value, p_value, std_err = stats.linregress(xi,ytrain) B = [(slope), (intercept), (r_value)] #print 'r value', r_value #print 'slope', slope #print 'intercept', intercept C = np.c_[ C, B ] #plot(xi,line,'r-',xi,y,'o') #show() count = 0 D = [[(5)]] for count in range(0,training_n): y_pred = slope*xi[count]+intercept D = np.append(D, np.array(y_pred)) D = np.delete(D,0,0) E = np.vstack([E, D]) idx =+ 1 C = np.delete(C,0,1) E = np.delete(E,0,0) E = E.T C[2] = np.square(C[2]) #reduce features based on R new_feature_number = math.ceil((percent/100)*reduced_feature_n) #new_feature_number = 60 number_of_features_cut = reduced_feature_n - new_feature_number index = C[2].argsort()[:number_of_features_cut] C = np.delete(C, index, 1) #normalize test matrices normalize_by_columns(Xtest, mins, maxs) Xtest = Xtest.compress(np.logical_not(boolean_array), axis=1) Xtest = np.delete(Xtest, index, 1) Xtrain = np.delete(Xtrain, index, 1) ############################## #weight the train and test matrices w = C[2] Xtrain = w*Xtrain Xtest = w*Xtest #add ones to feature matrix for least squares Xtrain = np.hstack([np.ones((training_n, 1)), Xtrain]) Xtest = np.hstack([np.ones((test_n, 1)), Xtest]) #least squares linear regression a = np.linalg.lstsq(Xtrain, ytrain)[0] ypred = np.array(np.dot(Xtest,a)) ytest = np.array(ytest) return ypred, new_feature_number
def main(): # This configures numpy/scipy to raise an exception in case of errors, instead of printing a warning and going ahead. numpy.seterr(all='raise') scipy.seterr(all='raise') parser = argparse.ArgumentParser(description='Runs a set of benchmarks defined in a YAML file.') parser.add_argument('--fruit-benchmark-sources-dir', help='Path to the fruit sources (used for benchmarking code only)') parser.add_argument('--fruit-sources-dir', help='Path to the fruit sources') parser.add_argument('--boost-di-sources-dir', help='Path to the Boost.DI sources') parser.add_argument('--output-file', help='The output file where benchmark results will be stored (1 per line, with each line in JSON format). These can then be formatted by e.g. the format_bench_results script.') parser.add_argument('--benchmark-definition', help='The YAML file that defines the benchmarks (see fruit_wiki_benchs_fruit.yml for an example).') parser.add_argument('--continue-benchmark', help='If this is \'true\', continues a previous benchmark run instead of starting from scratch (taking into account the existing benchmark results in the file specified with --output-file).') args = parser.parse_args() if args.output_file is None: raise Exception('You must specify --output_file') if args.continue_benchmark == 'true': with open(args.output_file, 'r') as f: previous_run_completed_benchmarks = [json.loads(line)['benchmark'] for line in f.readlines()] else: previous_run_completed_benchmarks = [] sh.rm('-f', args.output_file) fruit_build_tmpdir = tempfile.gettempdir() + '/fruit-benchmark-build-dir' with open(args.benchmark_definition, 'r') as f: yaml_file_content = yaml.load(f) global_definitions = yaml_file_content['global'] benchmark_definitions = expand_benchmark_definitions(yaml_file_content['benchmarks']) benchmark_index = 0 all_compilers = {benchmarks_definition['compiler'] for benchmarks_definition in benchmark_definitions} for compiler_executable_name in all_compilers: print('Preparing for benchmarks with the compiler %s' % compiler_executable_name) # We compute this here (and memoize the result) so that the benchmark's describe() will retrieve the cached # value instantly. determine_compiler_name(compiler_executable_name) # Build Fruit in fruit_build_tmpdir, so that fruit_build_tmpdir points to a built Fruit (useful for e.g. the config header). shutil.rmtree(fruit_build_tmpdir, ignore_errors=True) os.makedirs(fruit_build_tmpdir) modified_env = os.environ.copy() modified_env['CXX'] = compiler_executable_name sh.cmake(args.fruit_sources_dir, '-DCMAKE_BUILD_TYPE=Release', _cwd=fruit_build_tmpdir, _env=modified_env) make_command(_cwd=fruit_build_tmpdir) for benchmark_definition in benchmark_definitions: if benchmark_definition['compiler'] != compiler_executable_name: continue benchmark_index += 1 print('%s/%s: %s' % (benchmark_index, len(benchmark_definitions), benchmark_definition)) benchmark_name = benchmark_definition['name'] if (benchmark_name in {'boost_di_compile_time', 'boost_di_run_time', 'boost_di_executable_size'} and args.boost_di_sources_dir is None): raise Exception('Error: you need to specify the --boost-di-sources-dir flag in order to run Boost.DI benchmarks.') if benchmark_name == 'new_delete_run_time': benchmark = NewDeleteRunTimeBenchmark( benchmark_definition, fruit_benchmark_sources_dir=args.fruit_benchmark_sources_dir) elif benchmark_name == 'fruit_single_file_compile_time': benchmark = FruitSingleFileCompileTimeBenchmark( benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_benchmark_sources_dir=args.fruit_benchmark_sources_dir, fruit_build_tmpdir=fruit_build_tmpdir) elif benchmark_name == 'fruit_compile_time': benchmark = FruitCompileTimeBenchmark( benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_build_tmpdir=fruit_build_tmpdir) elif benchmark_name == 'fruit_run_time': benchmark = FruitRunTimeBenchmark( benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_build_tmpdir=fruit_build_tmpdir) elif benchmark_name == 'fruit_executable_size': benchmark = FruitExecutableSizeBenchmark( benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_build_tmpdir=fruit_build_tmpdir) elif benchmark_name == 'boost_di_compile_time': benchmark = BoostDiCompileTimeBenchmark( benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_build_tmpdir=fruit_build_tmpdir, boost_di_sources_dir=args.boost_di_sources_dir) elif benchmark_name == 'boost_di_run_time': benchmark = BoostDiRunTimeBenchmark( benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_build_tmpdir=fruit_build_tmpdir, boost_di_sources_dir=args.boost_di_sources_dir) elif benchmark_name == 'boost_di_executable_size': benchmark = BoostDiExecutableSizeBenchmark( benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_build_tmpdir=fruit_build_tmpdir, boost_di_sources_dir=args.boost_di_sources_dir) else: raise Exception("Unrecognized benchmark: %s" % benchmark_name) if benchmark.describe() in previous_run_completed_benchmarks: print("Skipping benchmark that was already run previously (due to --continue-benchmark):", benchmark.describe()) continue run_benchmark(benchmark, output_file=args.output_file, max_runs=global_definitions['max_runs'])