def test_init(self):
import numpy as np
import math
import sys
assert np.intp() == np.intp(0)
assert np.intp("123") == np.intp(123)
raises(TypeError, np.intp, None)
assert np.float64() == np.float64(0)
assert math.isnan(np.float64(None))
assert np.bool_() == np.bool_(False)
assert np.bool_("abc") == np.bool_(True)
assert np.bool_(None) == np.bool_(False)
assert np.complex_() == np.complex_(0)
# raises(TypeError, np.complex_, '1+2j')
assert math.isnan(np.complex_(None))
for c in ["i", "I", "l", "L", "q", "Q"]:
assert np.dtype(c).type().dtype.char == c
for c in ["l", "q"]:
assert np.dtype(c).type(sys.maxint) == sys.maxint
for c in ["L", "Q"]:
assert np.dtype(c).type(sys.maxint + 42) == sys.maxint + 42
assert np.float32(np.array([True, False])).dtype == np.float32
assert type(np.float32(np.array([True]))) is np.ndarray
assert type(np.float32(1.0)) is np.float32
a = np.array([True, False])
assert np.bool_(a) is a
def total_production(df):
"""
Takes DataFrame and finds generation totals for each hour.
Returns a dictionary.
"""
vals = []
# The Dominican Republic does not observe daylight savings time.
for hour in df.index.values:
dt = hour
current = df.loc[[hour]]
hydro = current.iloc[0]['Hydro']
wind = current.iloc[0]['Wind']
if wind > -10:
wind = max(wind, 0)
# Wind and hydro totals do not always update exactly on the new hour.
# In this case we set them to None because they are unknown rather than zero.
if isnan(wind):
wind = None
if isnan(hydro):
hydro = None
prod = {'wind': wind, 'hydro': hydro, 'datetime': dt}
vals.append(prod)
return vals
def fitToData(self, data):
'''
param data: numpy array where [:,0] is x and [:,1] is y
'''
x = data[:, 0][:, np.newaxis]
y = data[:, 1][:, np.newaxis]
D = np.hstack((x*x, x*y, y*y, x, y, np.ones_like(x)))
S = np.dot(D.T, D)
C = np.zeros([6, 6])
C[0, 2] = C[2, 0] = 2; C[1, 1] = -1
E, V = eig(np.dot(inv(S), C))
n = np.argmax(np.abs(E))
self.parameters = V[:, n]
axes = self.ellipse_axis_length()
self.a = axes[0]
self.b = axes[1]
self.angle = self.ellipse_angle_of_rotation()
if not self.a or not self.b or self.parameters == None or np.iscomplexobj(self.parameters) or \
math.isnan(self.a) or math.isnan(self.b) or math.isnan(self.ellipse_center()[0]) or \
np.iscomplex(self.ellipse_center()[0]) or np.iscomplex(self.a) or np.iscomplex(self.b) or \
np.iscomplexobj(self.angle):
self.a = 0
self.b = 0
self.parameters = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
self.angle = 0
self.error = True
def test_good(x):
"""Tests if scalar is infinity, NaN, or None.
Parameters
----------
x : scalar
Input to test.
Results
-------
good : logical
False if x is inf, NaN, or None; True otherwise."""
good = False
#DEBUG
return True
if x.ndim==0:
if x==np.inf or x==-np.inf or x is None or math.isnan(x):
good = False
else:
good = True
else:
x0 = x.flatten()
if any(x0==np.inf) or any(x==-np.inf) or any(x is None) or math.isnan(x0):
good = False
else:
good = True
return good
def load(cls, filename):
""" @brief Loads a png from a file as a 16-bit heightmap.
@param filename Name of target .png image
@returns A 16-bit, 1-channel image.
"""
# Get various png parameters so that we can allocate the
# correct amount of storage space for the new image
dx, dy, dz = ctypes.c_float(), ctypes.c_float(), ctypes.c_float()
ni, nj = ctypes.c_int(), ctypes.c_int()
libfab.load_png_stats(filename, ni, nj, dx, dy, dz)
# Create a python image data structure
img = cls(ni.value, nj.value, channels=1, depth=16)
# Add bounds to the image
if math.isnan(dx.value):
print 'Assuming 72 dpi for x resolution.'
img.xmin, img.xmax = 0, 72*img.width/25.4
else: img.xmin, img.xmax = 0, dx.value
if math.isnan(dy.value):
print 'Assuming 72 dpi for y resolution.'
img.ymin, img.ymax = 0, 72*img.height/25.4
else: img.ymin, img.ymax = 0, dy.value
if not math.isnan(dz.value): img.zmin, img.zmax = 0, dz.value
# Load the image data from the file
libfab.load_png(filename, img.pixels)
img.filename = filename
return img
def _align_top_level(self):
"""Temporary function to plot data. Expected to be
implemented in plotter
"""
result_1 = self._first_agg.aggregate(level="all")
result_2 = self._second_agg.aggregate(level="all")
s_x = self._resample(result_1[0])
s_y = self._resample(result_2[0])
front_x, front_y, front_shift = align(s_x, s_y, mode="front")
front_corr = self._correlate(front_x, front_y)
back_x, back_y, back_shift = align(s_x, s_y, mode="back")
back_corr = self._correlate(back_x, back_y)
if math.isnan(back_corr):
back_corr = 0
if math.isnan(front_corr):
front_corr = 0
if front_corr >= back_corr:
return front_shift
else:
return back_shift
def test_stdev(self):
Case = namedtuple("Case", "array stdev")
cases = [
Case([], float("nan")),
Case([1], float("nan")),
Case([1, 1], 0),
Case([1, 1, 1], 0),
Case([1, 2, 3, 4, 5], 1.5811388300841898),
Case([1.5, 4.7, 24, 8.5], 9.969411550671717),
Case([-34, 5, 3.6, -104.95, 67.4], 63.16384)
]
# Calculating the stdev using the pystat module.
for case in cases:
runningStat = RunningStat()
# Push all elements.
for element in case.array:
runningStat.push(element)
# Ensures that both numbers are either NaNs or not NaNs.
self.assertEqual(math.isnan(runningStat.stdev()),
math.isnan(case.stdev))
# If the stdev is not a NaN compare it to the expected stdev.
if not math.isnan(runningStat.stdev()):
precision = 0
if "." in str(case.stdev):
precision = len(str(case.stdev).split(".")[1])
self.assertEqual(round(runningStat.stdev(), precision),
round(case.stdev, precision))
def _tabulate(self, tablefmt="simple", rollups=False, rows=10):
"""Pretty tabulated string of all the cached data, and column names"""
if not self.is_valid(): self.fill(rows=rows)
# Pretty print cached data
d = collections.OrderedDict()
# If also printing the rollup stats, build a full row-header
if rollups:
col = next(iter(viewvalues(self._data))) # Get a sample column
lrows = len(col['data']) # Cached rows being displayed
d[""] = ["type", "mins", "mean", "maxs", "sigma", "zeros", "missing"] + list(map(str, range(lrows)))
# For all columns...
for k, v in viewitems(self._data):
x = v['data'] # Data to display
t = v["type"] # Column type
if t == "enum":
domain = v['domain'] # Map to cat strings as needed
x = ["" if math.isnan(idx) else domain[int(idx)] for idx in x]
elif t == "time":
x = ["" if math.isnan(z) else time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(z / 1000)) for z in x]
if rollups: # Rollups, if requested
mins = v['mins'][0] if v['mins'] and v["type"] != "enum" else None
maxs = v['maxs'][0] if v['maxs'] and v["type"] != "enum" else None
#Cross check type with mean and sigma. Set to None if of type enum.
if v['type'] == "enum":
v['mean'] = v['sigma'] = v['zero_count'] = None
x = [v['type'], mins, v['mean'], maxs, v['sigma'], v['zero_count'], v['missing_count']] + x
d[k] = x # Insert into ordered-dict
return tabulate.tabulate(d, headers="keys", tablefmt=tablefmt)
def test_mean(self):
Case = namedtuple("Case", "array mean")
cases = [
Case([], float("nan")),
Case([1], 1),
Case([1, 1], 1),
Case([1, 1, 1], 1),
Case([1, 2, 3, 4, 5], 3),
Case([1.5, 4.7, 24, 8.5], 9.675),
Case([-34, 5, 3.6, -104.95, 67.4], -12.59)
]
# Calculating the mean using the pystat module.
for case in cases:
runningStat = RunningStat()
# Push all elements.
for element in case.array:
runningStat.push(element)
# Ensures that both numbers are either NaNs or not NaNs.
self.assertEqual(math.isnan(runningStat.mean()),
math.isnan(case.mean))
# If the mean is not a NaN compare it to the expected mean.
if not math.isnan(runningStat.mean()):
precision = 0
if "." in str(case.mean):
precision = len(str(case.mean).split(".")[1])
self.assertEqual(round(runningStat.mean(), precision),
round(case.mean, precision))
def test_variance(self):
Case = namedtuple("Case", "array variance")
cases = [
Case([], float("nan")),
Case([1], float("nan")),
Case([1, 1], 0),
Case([1, 1, 1], 0),
Case([1, 2, 3, 4, 5], 2.5),
Case([1.5, 4.7, 24, 8.5], 99.38916666666665),
Case([-34, 5, 3.6, -104.95, 67.4], 3989.6705)
]
# Calculating the variance using the pystat module.
for case in cases:
runningStat = RunningStat()
# Push all elements.
for element in case.array:
runningStat.push(element)
# Ensures that both numbers are either NaNs or not NaNs.
self.assertEqual(math.isnan(runningStat.variance()),
math.isnan(case.variance))
# If the variance is not a NaN compare it to the expected variance.
if not math.isnan(runningStat.variance()):
precision = 0
if "." in str(case.variance):
precision = len(str(case.variance).split(".")[1])
self.assertEqual(round(runningStat.variance(), precision),
round(case.variance, precision))
def learn(self): #print self.cs, self.ts learning = learn.learnCurve(self.cs, self.ts) #print learning if not math.isnan(learning['r']): #if it's too huge, just set it to 1000 if learning['r'] > 5000.0: self.er = 5000.0 else: if learning['r'] <= 0: self.er = 5000.0 else: self.er = learning['r'] self.epv = learning['pv'] self.erv = learning['rv'] else: self.er = 5000.0 self.epv = 1.0 self.erv = 1.0 if not math.isnan(learning['p']): if learning['p'] < 0: self.ep = 0.0 else: self.ep = learning['p'] else: self.ep = 0.0
def test_max(self):
Case = namedtuple("Case", "array max")
cases = [
Case([], float("nan")),
Case([1], 1),
Case([1, 1], 1),
Case([1, 1, 1], 1),
Case([1, 2, 3, 4, 5], 5),
Case([1.5, 4.7, 24, 8.5], 24),
Case([-34, 5, 3.6, -104.95, 67.4], 67.4)
]
# Calculating the max using the pystat module.
for case in cases:
runningStat = RunningStat()
# Push all elements.
for element in case.array:
runningStat.push(element)
# Ensures that both numbers are either NaNs or not NaNs.
self.assertEqual(math.isnan(runningStat.max()),
math.isnan(case.max))
# If the max is not a NaN compare it to the expected max.
if not math.isnan(runningStat.max()):
self.assertEqual(runningStat.max(), case.max)
def testCopysign(self):
self.assertEqual(math.copysign(1, 42), 1.0)
self.assertEqual(math.copysign(0., 42), 0.0)
self.assertEqual(math.copysign(1., -42), -1.0)
self.assertEqual(math.copysign(3, 0.), 3.0)
self.assertEqual(math.copysign(4., -0.), -4.0)
self.assertRaises(TypeError, math.copysign)
# copysign should let us distinguish signs of zeros
self.assertEquals(math.copysign(1., 0.), 1.)
self.assertEquals(math.copysign(1., -0.), -1.)
self.assertEquals(math.copysign(INF, 0.), INF)
self.assertEquals(math.copysign(INF, -0.), NINF)
self.assertEquals(math.copysign(NINF, 0.), INF)
self.assertEquals(math.copysign(NINF, -0.), NINF)
# and of infinities
self.assertEquals(math.copysign(1., INF), 1.)
self.assertEquals(math.copysign(1., NINF), -1.)
self.assertEquals(math.copysign(INF, INF), INF)
self.assertEquals(math.copysign(INF, NINF), NINF)
self.assertEquals(math.copysign(NINF, INF), INF)
self.assertEquals(math.copysign(NINF, NINF), NINF)
self.assertTrue(math.isnan(math.copysign(NAN, 1.)))
self.assertTrue(math.isnan(math.copysign(NAN, INF)))
self.assertTrue(math.isnan(math.copysign(NAN, NINF)))
self.assertTrue(math.isnan(math.copysign(NAN, NAN)))
# copysign(INF, NAN) may be INF or it may be NINF, since
# we don't know whether the sign bit of NAN is set on any
# given platform.
self.assertTrue(math.isinf(math.copysign(INF, NAN)))
# similarly, copysign(2., NAN) could be 2. or -2.
self.assertEquals(abs(math.copysign(2., NAN)), 2.)
def testLdexp(self):
self.assertRaises(TypeError, math.ldexp)
self.ftest('ldexp(0,1)', math.ldexp(0,1), 0)
self.ftest('ldexp(1,1)', math.ldexp(1,1), 2)
self.ftest('ldexp(1,-1)', math.ldexp(1,-1), 0.5)
self.ftest('ldexp(-1,1)', math.ldexp(-1,1), -2)
self.assertRaises(OverflowError, math.ldexp, 1., 1000000)
self.assertRaises(OverflowError, math.ldexp, -1., 1000000)
self.assertEquals(math.ldexp(1., -1000000), 0.)
self.assertEquals(math.ldexp(-1., -1000000), -0.)
self.assertEquals(math.ldexp(INF, 30), INF)
self.assertEquals(math.ldexp(NINF, -213), NINF)
self.assert_(math.isnan(math.ldexp(NAN, 0)))
# large second argument
for n in [10**5, 10L**5, 10**10, 10L**10, 10**20, 10**40]:
self.assertEquals(math.ldexp(INF, -n), INF)
self.assertEquals(math.ldexp(NINF, -n), NINF)
self.assertEquals(math.ldexp(1., -n), 0.)
self.assertEquals(math.ldexp(-1., -n), -0.)
self.assertEquals(math.ldexp(0., -n), 0.)
self.assertEquals(math.ldexp(-0., -n), -0.)
self.assert_(math.isnan(math.ldexp(NAN, -n)))
self.assertRaises(OverflowError, math.ldexp, 1., n)
self.assertRaises(OverflowError, math.ldexp, -1., n)
self.assertEquals(math.ldexp(0., n), 0.)
self.assertEquals(math.ldexp(-0., n), -0.)
self.assertEquals(math.ldexp(INF, n), INF)
self.assertEquals(math.ldexp(NINF, n), NINF)
self.assert_(math.isnan(math.ldexp(NAN, n)))
def __eq__(self, actual):
"""
Return true if the given value is equal to the expected value within
the pre-specified tolerance.
"""
if _is_numpy_array(actual):
# Call ``__eq__()`` manually to prevent infinite-recursion with
# numpy<1.13. See #3748.
return all(self.__eq__(a) for a in actual.flat)
# Short-circuit exact equality.
if actual == self.expected:
return True
# Allow the user to control whether NaNs are considered equal to each
# other or not. The abs() calls are for compatibility with complex
# numbers.
if math.isnan(abs(self.expected)):
return self.nan_ok and math.isnan(abs(actual))
# Infinity shouldn't be approximately equal to anything but itself, but
# if there's a relative tolerance, it will be infinite and infinity
# will seem approximately equal to everything. The equal-to-itself
# case would have been short circuited above, so here we can just
# return false if the expected value is infinite. The abs() call is
# for compatibility with complex numbers.
if math.isinf(abs(self.expected)):
return False
# Return true if the two numbers are within the tolerance.
return abs(self.expected - actual) <= self.tolerance
def branch_calculte(filename):
f=open(WORKPATH+filename)
f.readline()
paml_rate = {}
for line in f:
line = line.strip()
paml_out = paml.read_line(line)
sp0 = round(((paml_out.sp1_sp0 + paml_out.sp2_sp0) - paml_out.sp2_sp1)/2,4)
sp1 = round(((paml_out.sp2_sp1 + paml_out.sp1_sp0) - paml_out.sp2_sp0)/2,4)
sp2 = round(((paml_out.sp2_sp1 + paml_out.sp2_sp0) - paml_out.sp1_sp0)/2,4)
if math.isnan(sp0) or math.isnan(sp1) or math.isnan(sp2):
pass
else:
sp0 = sp0 if sp0 > 0 else 0.0001
sp1 = sp1 if sp1 > 0 else 0.0001
sp0_devide_sp1 = sp0/sp1
sp0_devide_sp1 = 16 if sp0_devide_sp1 > 16 else sp0_devide_sp1
sp0_devide_sp1 = 0.0625 if sp0_devide_sp1 < 0.0625 else sp0_devide_sp1
paml_rate[paml_out.sp0name]=np.log2(sp0_devide_sp1) #paml-rate structure
'''
f = open(branch_file,"w")
f.write("sp0\tB_sp0\tB_sp1\tB_sp2\n")
for name in paml_rate:
f.write(name+"\t"+str(paml_rate[name][0])+\
"\t"+str(paml_rate[name][1])+\
"\t"+str(paml_rate[name][2])+"\n")
f.close()
'''
return paml_rate
def testMinMaxNan(self):
"""Test C minMax and min positive with NaN.
"""
self._log("testMinMaxNan")
for dtype in self.FLOATING_DTYPES:
# All NaN
data = np.array((float("nan"), float("nan")), dtype=dtype)
min_, minPositive, max_ = ctools.minMax(data, minPositive=True)
self.assertTrue(math.isnan(min_))
self.assertEqual(minPositive, None)
self.assertTrue(math.isnan(max_))
# NaN first and positive
data = np.array((float("nan"), 1.0), dtype=dtype)
self._testMinMaxVsNumpy(data)
# NaN first and negative
data = np.array((float("nan"), -1.0), dtype=dtype)
self._testMinMaxVsNumpy(data)
# NaN last and positive
data = np.array((1.0, 2.0, float("nan")), dtype=dtype)
self._testMinMaxVsNumpy(data)
# NaN last and negative
data = np.array((-1.0, -2.0, float("nan")), dtype=dtype)
self._testMinMaxVsNumpy(data)
# Some NaN
data = np.array((1.0, float("nan"), -1.0), dtype=dtype)
self._testMinMaxVsNumpy(data)
def _assert_sarray_equal(self, sa1, sa2):
l1 = list(sa1)
l2 = list(sa2)
assert len(l1) == len(l2)
for i in range(len(l1)):
v1 = l1[i]
v2 = l2[i]
if v1 == None:
assert v2 == None
else:
if type(v1) == dict:
assert len(v1) == len(v2)
for key in v1:
assert v1.has_key(key)
assert v1[key] == v2[key]
elif (hasattr(v1, "__iter__")):
assert len(v1) == len(v2)
for j in range(len(v1)):
t1 = v1[j]; t2 = v2[j]
if (type(t1) == float):
if (math.isnan(t1)):
assert math.isnan(t2)
else:
assert t1 == t2
else:
assert t1 == t2
else:
assert v1 == v2
def calculate_threshold_hits(data, nr_bins=11):
bin_centers, bin_edges, _ = get_bins(nr_bins, -0.05, 1.05)
# TODO Hard-coded single threshold
threshold = (op.le, 273.15 + constants.THRESHOLD)
# Counting tables
prob_count = np.zeros(nr_bins)
prob_hits = np.zeros(nr_bins)
nr_rows = 0
# For each record
for _, row in data.iterrows():
observation = row['2T_OBS']
threshold_prob = row['2T_BELOW_DEGREE_PROB']
# Check quality
if isnan(threshold_prob) or isnan(observation):
continue
# Bin probability
threshold_prob_bin = \
np.digitize(threshold_prob, bin_edges, right=True) - 1
prob_count[threshold_prob_bin] += 1
# Threshold is fulfilled.
if threshold[0](observation, threshold[1]):
prob_hits[threshold_prob_bin] += 1
nr_rows += 1
prob_hits /= nr_rows
return bin_centers, prob_hits, prob_count
def main():
trainset = np.genfromtxt(open('train.csv','r'), delimiter=',')[1:]
X = np.array([x[1:8] for x in trainset])
y = np.array([x[8] for x in trainset])
#print X,y
import math
for i, x in enumerate(X):
for j, xx in enumerate(x):
if(math.isnan(xx)):
X[i][j] = 26.6
testset = np.genfromtxt(open('test.csv','r'), delimiter = ',')[1:]
test = np.array([x[1:8] for x in testset])
for i, x in enumerate(test):
for j, xx in enumerate(x):
if(math.isnan(xx)):
test[i][j] = 26.6
X, test = decomposition_pca(X, test)
bdt = AdaBoostClassifier(base_estimator = KNeighborsClassifier(n_neighbors=20, algorithm = 'auto'), algorithm="SAMME", n_estimators = 200)
bdt.fit(X, y)
print 'PassengerId,Survived'
for i, t in enumerate(test):
print '%d,%d' % (i + 892, int(bdt.predict(t)[0]))
def assertDataAlmostEqual(self, data, reference_filename, **kwargs):
reference_path = self.get_result_path(reference_filename)
if self._check_reference_file(reference_path):
kwargs.setdefault('err_msg', 'Reference file %s' % reference_path)
with open(reference_path, 'r') as reference_file:
stats = json.load(reference_file)
self.assertEqual(stats.get('shape', []), list(data.shape))
self.assertEqual(stats.get('masked', False),
ma.is_masked(data))
nstats = np.array((stats.get('mean', 0.), stats.get('std', 0.),
stats.get('max', 0.), stats.get('min', 0.)),
dtype=np.float_)
if math.isnan(stats.get('mean', 0.)):
self.assertTrue(math.isnan(data.mean()))
else:
data_stats = np.array((data.mean(), data.std(),
data.max(), data.min()),
dtype=np.float_)
self.assertArrayAllClose(nstats, data_stats, **kwargs)
else:
self._ensure_folder(reference_path)
stats = collections.OrderedDict([
('std', np.float_(data.std())),
('min', np.float_(data.min())),
('max', np.float_(data.max())),
('shape', data.shape),
('masked', ma.is_masked(data)),
('mean', np.float_(data.mean()))])
with open(reference_path, 'w') as reference_file:
reference_file.write(json.dumps(stats))
def explorer():
(metadata,pose_origin,pose_robot,pose_in_im,pose_in_map,copyData,image_array)=get_map_data()
remplissage_diff(metadata,pose_in_im)
scipy.misc.imsave('map.png', image_array)
(x_im,y_im)=find_ppv()
while(not (isnan(x_im) and isnan(y_im)) and rospy.is_shutdown()):
(x,y,theta)=pix_to_pose((x_im,y_im,0), pose_origin, metadata)
print(x,y,theta)
# Plotting the location of the robot
for i in range(-3,4):
for j in range(-3,4):
image_array[pose_in_im[0]+i, pose_in_im[1]+j] = (255, 0, 0)
# Plotting its orientation
for i in range(10):
image_array[int(pose_in_im[0]+i*sin(-pose_in_im[2])), int(pose_in_im[1]+i*cos(-pose_in_im[2]))] = (0, 0, 255)
print("Enregistrement de l'image")
scipy.misc.imsave('map.png', image_array)
reach_goal(x,y,theta)
(metadata,pose_origin,pose_robot,pose_in_im,pose_in_map,copyData,image_array)=get_map_data()
remplissage_diff()
scipy.misc.imsave('map.png', image_array)
(x_im,y_im)=find_ppv()
def verify_counters(self, opcounts, expected_elements, expected_size=None):
self.assertEqual(expected_elements, opcounts.element_counter.value())
if expected_size is not None:
if math.isnan(expected_size):
self.assertTrue(math.isnan(opcounts.mean_byte_counter.value()))
else:
self.assertEqual(expected_size, opcounts.mean_byte_counter.value())
def write_log(task, data, in_rows, question, out_rows, out_cols, solution, version, git, fun, run, time_sec, mem_gb, cache, chk, chk_time_sec):
batch = os.getenv('BATCH', "")
timestamp = time.time()
csv_file = os.getenv('CSV_TIME_FILE', "time.csv")
nodename = platform.node()
comment = "" # placeholder for updates to timing data
time_sec = round(time_sec, 3)
chk_time_sec = round(chk_time_sec, 3)
mem_gb = round(mem_gb, 3)
if math.isnan(time_sec):
time_sec = ""
if math.isnan(mem_gb):
mem_gb = ""
log_row = [nodename, batch, timestamp, task, data, in_rows, question, out_rows, out_cols, solution, version, git, fun, run, time_sec, mem_gb, cache, chk, chk_time_sec, comment]
log_header = ["nodename","batch","timestamp","task","data","in_rows","question","out_rows","out_cols","solution","version","git","fun","run","time_sec","mem_gb","cache","chk","chk_time_sec","comment"]
append = os.path.isfile(csv_file)
csv_verbose = os.getenv('CSV_VERBOSE', "true")
if csv_verbose.lower()=="true":
print('# ' + ','.join(str(x) for x in log_row))
if append:
with open(csv_file, 'a') as f:
w = csv.writer(f, lineterminator='\n')
w.writerow(log_row)
else:
with open(csv_file, 'w+') as f:
w = csv.writer(f, lineterminator='\n')
w.writerow(log_header)
w.writerow(log_row)
return True
def tradeRuleNVI(nvit1, nvi, sma):
if math.isnan(nvit1) or math.isnan(nvi) or math.isnan(sma):
return np.nan
if nvit1 <= sma and nvi > sma:
return 1 # buy
else:
return 0 # hold
def assertFloatsAreIdentical(self, x, y):
"""assert that floats x and y are identical, in the sense that:
(1) both x and y are nans, or
(2) both x and y are infinities, with the same sign, or
(3) both x and y are zeros, with the same sign, or
(4) x and y are both finite and nonzero, and x == y
"""
msg = 'floats {!r} and {!r} are not identical'
if isnan(x) or isnan(y):
if isnan(x) and isnan(y):
return
elif x == y:
if x != 0.0:
return
# both zero; check that signs match
elif copysign(1.0, x) == copysign(1.0, y):
return
else:
msg += ': zeros have different signs'
if test_support.due_to_ironpython_bug("http://ironpython.codeplex.com/workitem/28352"):
print msg.format(x, y)
return
self.fail(msg.format(x, y))
def bv2rgb(bv):
if math.isnan(bv):
bc = 0
if bv < -0.4: bv = -0.4
if bv > 2.0: bv = 2.0
if bv >= -0.40 and bv < 0.00:
t = (bv + 0.40) / (0.00 + 0.40)
r = 0.61 + 0.11 * t + 0.1 * t * t
g = 0.70 + 0.07 * t + 0.1 * t * t
b = 1.0
elif bv >= 0.00 and bv < 0.40:
t = (bv - 0.00) / (0.40 - 0.00)
r = 0.83 + (0.17 * t)
g = 0.87 + (0.11 * t)
b = 1.0
elif bv >= 0.40 and bv < 1.60:
t = (bv - 0.40) / (1.60 - 0.40)
r = 1.0
g = 0.98 - 0.16 * t
else:
t = (bv - 1.60) / (2.00 - 1.60)
r = 1.0
g = (0.82 - 0.5) * t * t
if bv >= 0.40 and bv < 1.50:
t = (bv - 0.40) / (1.50 - 0.40)
b = 1.00 - 0.47 * t + 0.1 * t * t
elif bv >= 1.50 and bv < 1.951:
t = (bv - 1.50) / (1.94 - 1.50)
b = 0.63 - 0.6 * t * t
else:
b = 0.0
if bv == 'nan' or math.isnan(bv):
g = 0.5
g /= 1.3
return "rgb({0}, {1}, {2})".format(int(r*255),int(g*255),int(b*255))
def arith_min_same(self, other):
assert isinstance(other, values.W_Flonum)
if math.isnan(self.value):
return self
if math.isnan(other.value):
return other
return values.W_Number.arith_min_same(self, other)
def adapt_canopsis_data_to_ember(data):
"""
Transform canopsis data to ember data (in changing ``id`` to ``cid``).
:param data: data to transform
"""
if isinstance(data, dict):
for key, item in data.iteritems():
if isinstance(item, float) and (isnan(item) or isinf(item)):
data[key] = None
else:
if isinstance(item, (tuple, frozenset)):
item = list(item)
data[key] = item
adapt_canopsis_data_to_ember(item)
elif isiterable(data, is_str=False):
for i in range(len(data)):
item = data[i]
if isinstance(item, float) and (isnan(item) or isinf(item)):
data[i] = None
else:
if isinstance(item, (tuple, frozenset)):
item = list(item)
data[i] = item
adapt_canopsis_data_to_ember(item)
def test_pow(self):
import math
def pw(x, y):
return x ** y
def espeq(x, y):
return not abs(x-y) > 1e05
raises(ZeroDivisionError, pw, 0.0, -1)
assert pw(0, 0.5) == 0.0
assert espeq(pw(4.0, 0.5), 2.0)
assert pw(4.0, 0) == 1.0
assert pw(-4.0, 0) == 1.0
assert type(pw(-1.0, 0.5)) == complex
assert pw(-1.0, 2.0) == 1.0
assert pw(-1.0, 3.0) == -1.0
assert pw(-1.0, 1e200) == 1.0
if self.py26:
assert pw(0.0, float("-inf")) == float("inf")
assert math.isnan(pw(-3, float("nan")))
assert math.isnan(pw(-3., float("nan")))
assert pw(-1.0, -float('inf')) == 1.0
assert pw(-1.0, float('inf')) == 1.0
assert pw(float('inf'), 0) == 1.0
assert pw(float('nan'), 0) == 1.0
assert math.isinf(pw(-0.5, float('-inf')))
assert math.isinf(pw(+0.5, float('-inf')))
assert pw(-1.5, float('-inf')) == 0.0
assert pw(+1.5, float('-inf')) == 0.0
assert str(pw(float('-inf'), -0.5)) == '0.0'
assert str(pw(float('-inf'), -2.0)) == '0.0'
assert str(pw(float('-inf'), -1.0)) == '-0.0'
assert str(pw(float('-inf'), 1.0)) == '-inf'
assert str(pw(float('-inf'), 2.0)) == 'inf'
def get_z_max_z_min(self):
QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
roi = QgsRectangle(self.roi_x_min, self.roi_y_min, self.roi_x_max,
self.roi_y_max)
source = self.map_crs
target = self.layer.crs()
transform = QgsCoordinateTransform(source, target)
rec = transform.transform(roi)
x_max = rec.xMaximum()
x_min = rec.xMinimum()
y_max = rec.yMaximum()
y_min = rec.yMinimum()
x_off = int(
math.floor((x_min - self.raster_x_min) * self.cols /
(self.raster_x_max - self.raster_x_min)))
y_off = int(
math.floor((self.raster_y_max - y_max) * self.rows /
(self.raster_y_max - self.raster_y_min)))
col_size = int(math.floor((x_max - x_min) / self.cell_size))
row_size = int(math.floor((y_max - y_min) / self.cell_size))
if x_off < 0:
x_off = 0
if y_off < 0:
y_off = 0
if x_off >= self.cols:
x_off = self.cols - 1
if y_off >= self.rows:
y_off = self.rows - 1
if x_off + col_size > self.cols:
col_size = self.cols - x_off
if row_size + row_size > self.rows:
row_size = self.rows - y_off
provider = self.layer.dataProvider()
path = provider.dataSourceUri()
path_layer = path.split('|')
dem_dataset = gdal.Open(path_layer[0])
data = self.get_dem_z(dem_dataset, x_off, y_off, col_size, row_size)
if data is not None:
self.z_max = max(data)
self.z_min = self.z_max
no_data = dem_dataset.GetRasterBand(1).GetNoDataValue()
if min(data) == no_data:
for z_cell in data:
if z_cell != no_data and z_cell < self.z_min:
self.z_min = z_cell
elif math.isnan(min(data)):
self.z_max = 0
self.z_min = 0
for z_cell in data:
if not math.isnan(z_cell):
if self.z_min > z_cell:
self.z_min = z_cell
if self.z_max < z_cell:
self.z_max = z_cell
else:
self.z_min = min(data)
if self.z_min < 0:
self.z_min = 0
self.z_max = round(self.z_max, 3)
self.z_min = round(self.z_min, 3)
self.ui.ZMaxLabel.setText(str(self.z_max) + ' m')
self.ui.ZMinLabel.setText(str(self.z_min) + ' m')
dem_dataset = None
band = None
QApplication.restoreOverrideCursor()
def main(argv):
if len(argv) < 3:
raise Exception("Missing parameter")
path_in = argv[1]
path_out = argv[2]
threshold = float(argv[3]) if len(argv) > 3 else 3.0
sys.stderr.write("path_in:%s path_out:%s thr:%s\n" %
(path_in, path_out, threshold))
with open(os.path.join(path_in, "train.score")) as score_stream:
scores = {}
line = 0
while (True):
score_line = score_stream.readline()
if not score_line:
break
score = score_line.strip().split()
s = float(score[0])
if math.isinf(s) or math.isnan(s):
scores[line] = -1000.0
else:
scores[line] = s
line += 1
selected_scores, z_scores = select(scores, threshold)
#print selected sentence pairs
with open(os.path.join(path_in, "train.src"), "r") as source_stream:
with open(os.path.join(path_in, "train.trg"), "r") as target_stream:
with open(os.path.join(path_out, "train_filtered.src"),
"w") as source_filtered_stream:
with open(os.path.join(path_out, "train_filtered.trg"),
"w") as target_filtered_stream:
with open(os.path.join(path_out, "train_filtered.score"),
"w") as score_filtered_stream:
with open(os.path.join(path_out, "train_removed.src"),
"w") as source_removed_stream:
with open(
os.path.join(path_out,
"train_removed.trg"),
"w") as target_removed_stream:
with open(
os.path.join(path_out,
"train_removed.score"),
"w") as score_removed_stream:
with open(
os.path.join(
path_out, "train.label"),
"w") as output_stream:
line = 0
selected = 0
while (True):
source_line = source_stream.readline(
)
target_line = target_stream.readline(
)
if not source_line or not target_line:
break
if selected_scores[line] == True:
source_filtered_stream.write(
source_line)
target_filtered_stream.write(
target_line)
output_stream.write(
"1\n"
) # TMOP value for BAD
if scores[line] > -1000.0:
score_filtered_stream.write(
"%f %f\n" %
(scores[line],
z_scores[line]))
else:
score_filtered_stream.write(
"MY_NAN %f\n" %
(z_scores[line]))
selected += 1
else:
source_removed_stream.write(
source_line)
target_removed_stream.write(
target_line)
output_stream.write(
"3\n"
) # TMOP value for BAD
if scores[line] > -1000.0:
score_removed_stream.write(
"%f %f\n" %
(scores[line],
z_scores[line]))
else:
score_removed_stream.write(
"MY_NAN %f\n" %
(z_scores[line]))
line += 1
sys.stderr.write("%s selected among %s sentence pairs\n" %
(selected, len(scores)))
def total_time() -> Optional[int]:
"""Total play time in seconds."""
duration = state.metadata_field("duration")
if duration is None or math.isnan(duration):
return None
return int(duration)
def train_save_model(filelists, num_input, mhidden, timesteps, moptions):
training_steps = 4
#training_steps = 40
init, init_l, loss_op, accuracy, train_op, X, Y, saver, auc_op, mpre, mspf, mfpred = mCreateSession(
num_input, mhidden, timesteps, moptions)
# display step
desplay_files = len(filelists[0]) / 100
if desplay_files < 2: desplay_files = 2
if desplay_files > 10:
desplay_files = int(desplay_files / 10) * 10
#desplay_files=2
if desplay_files > 100: desplay_files = 100
file_group_id = [0 for _ in range(len(filelists))]
sumpsize = 25
# for configuration
config = tf.ConfigProto()
if (timesteps > 61 and num_input > 50):
config.gpu_options.per_process_gpu_memory_fraction = 0.5
else:
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
# initialization
sess.run(init)
sess.run(init_l)
start_time = time.time()
start_c_time = time.time()
io_time = 0
# for each epoch
for step in range(1, training_steps + 1):
print(
'===%d=====================step========================%d/%d' %
(desplay_files, step, training_steps))
sys.stdout.flush()
last_desplay_files_num = -1
file_group_id[0] = 0
while file_group_id[0] < len(filelists[0]):
io_start_time = time.time()
# for each input groups.
# usually two groups: one positive group and one negative group
# might also one group containing both positive and negative labelling data
featurelist = [[[], []] for _ in range(len(filelists))]
minsize = None
cur_batch_num = None
# get data from all groups until 'minsize' data is loaded.
for ifl in range(len(filelists)):
if ifl == 0:
minsize = batchsize * sumpsize
else:
minsize = batchsize * cur_batch_num
while len(featurelist[ifl][0]) < minsize:
if not file_group_id[ifl] < len(filelists[ifl]):
if ifl == 0: break
else: file_group_id[ifl] = 0
# get more data
batch_2_x, batch_2_y, _ = getDataFromFile_new(
filelists[ifl][file_group_id[ifl]], moptions)
if len(batch_2_y) > 0:
if len(featurelist[ifl][0]) == 0:
featurelist[ifl][0] = batch_2_x
featurelist[ifl][1] = batch_2_y
else:
# merge current loading data with previously loading data
featurelist[ifl][0] = np.concatenate(
(featurelist[ifl][0], batch_2_x), axis=0)
featurelist[ifl][1] = np.concatenate(
(featurelist[ifl][1], batch_2_y), axis=0)
file_group_id[ifl] += 1
# split for small groups for training
if ifl == 0:
featurelist[ifl][0] = np.array_split(
featurelist[ifl][0],
int(len(featurelist[ifl][0]) / batchsize))
featurelist[ifl][1] = np.array_split(
featurelist[ifl][1],
int(len(featurelist[ifl][1]) / batchsize))
cur_batch_num = len(featurelist[ifl][0])
if len(featurelist[0][0]) < sumpsize * 0.8:
for ifl in range(1, len(filelists)):
if len(featurelist[0][0]) * batchsize * 1.2 < len(
featurelist[ifl][0]):
featurelist[ifl][0] = featurelist[ifl][0][:int(
len(featurelist[0][0]) * batchsize * 1.2)]
featurelist[ifl][1] = featurelist[ifl][1][:int(
len(featurelist[0][0]) * batchsize * 1.2)]
if len(featurelist[0][0]) < 1: continue
#
if len(filelists) > 1:
for ifl in range(1, len(filelists)):
#if (file_group_id[0]+1) - last_desplay_files_num >= desplay_files: msizeprint.append(str(len(featurelist[ifl][0])))
featurelist[ifl][0] = np.array_split(
featurelist[ifl][0], len(featurelist[0][0]))
featurelist[ifl][1] = np.array_split(
featurelist[ifl][1], len(featurelist[0][0]))
io_time += (time.time() - io_start_time)
ifl = 3 if len(featurelist) > 3 else len(featurelist) - 1
if (file_group_id[0] +
1) - last_desplay_files_num >= desplay_files:
sess.run(init_l)
try:
# print some testing information as progress indicators
loss, aucm, acc, p, r = sess.run(
[loss_op, auc_op[1], accuracy, mpre[1], mspf[1]],
feed_dict={
X: featurelist[ifl][0][0],
Y: featurelist[ifl][1][0]
})
print(">>>Tratin#files " + str(file_group_id[0] + 1) +
",loss=" + "{:.3f}".format(loss) + ",AUC=" +
"{:.3f}".format(aucm) + ",acc=" +
"{:.3f}".format(acc) + ",p=" +
"{:.3f}".format(p) + ",r=" + "{:.3f}".format(r) +
(" Comsuming time: %d(current=%d) IO=%d(%.3f)" %
(time.time() - start_time,
time.time() - start_c_time, io_time,
io_time / float(time.time() - start_time))))
except:
print(">>>Tratin#filesError " +
str(file_group_id[0] + 1) +
(" Comsuming time: %d(current=%d) IO=%d(%.3f)" %
(time.time() - start_time,
time.time() - start_c_time, io_time,
io_time / float(time.time() - start_time))))
sys.stdout.flush()
start_c_time = time.time()
# using each subgroup of data for training
for subi in range(len(featurelist[0][0])):
for ifl in range(len(filelists)):
to = sess.run(
[train_op, loss_op],
feed_dict={
X: featurelist[ifl][0][subi],
Y: featurelist[ifl][1][subi]
})
if len(featurelist) == 1:
# print some detail if nan issue happens
if math.isnan(to[1]):
for toj in range(len(
featurelist[ifl][0][subi])):
print('{} vs {}'.format(
featurelist[ifl][1][subi][toj][0],
featurelist[ifl][1][subi][toj][1]))
for tok in featurelist[ifl][0][subi][toj]:
opstr = []
for tol in tok:
opstr.append(str(round(tol, 2)))
print("\t\t\t" + ','.join(opstr))
sys.exit(1)
# adjust progress output information
ifl = 3 if len(featurelist) > 3 else len(featurelist) - 1
if (file_group_id[0] +
1) - last_desplay_files_num >= desplay_files:
last_desplay_files_num = (file_group_id[0] + 1) - (
(file_group_id[0] + 1) % desplay_files)
# store more models
if 49.5 < int(file_group_id[0] * 100 /
float(len(filelists[0]))) < 50.5:
savp = '.50'
if (not os.path.isdir(moptions['outFolder'] +
str(step - 1) + savp)):
os.system('mkdir -p ' + moptions['outFolder'] +
str(step - 1) + savp)
saver.save(
sess, moptions['outFolder'] + str(step - 1) + savp +
'/' + moptions['FileID'])
if len(featurelist) == 1:
cur_per = int(file_group_id[0] * 100 /
float(len(filelists[0])))
if cur_per in [10, 20, 30, 40, 60, 70, 80, 90]:
savp = str(round(cur_per / 100.0, 2))
if (not os.path.isdir(moptions['outFolder'] +
str(step - 1) + savp)):
os.system('mkdir -p ' + moptions['outFolder'] +
str(step - 1) + savp)
saver.save(
sess, moptions['outFolder'] + str(step - 1) +
savp + '/' + moptions['FileID'])
# for each epoch, store the trained model
if (not os.path.isdir(moptions['outFolder'] + str(step))):
os.system('mkdir -p ' + moptions['outFolder'] + str(step))
saver.save(
sess,
moptions['outFolder'] + str(step) + '/' + moptions['FileID'])
print("Training Finished!")
return (accuracy, X, Y, auc_op, mpre, mspf, init_l, mfpred)
def suggest_targets(self, timestamp, deepdrilling_target,
constrained_filter, cloud, seeing):
Proposal.suggest_targets(self, timestamp)
if self.ignore_clouds:
cloud = 0.0
if self.ignore_seeing:
seeing = 0.0
if cloud > self.params.max_cloud:
self.log.debug("suggest_targets: cloud=%.2f > max_cloud=%.2f" %
(cloud, self.params.max_cloud))
return []
self.clear_evaluated_target_list()
if deepdrilling_target is None:
fields_evaluation_list = self.tonight_fields_list
num_targets_to_propose = self.params.max_num_targets
else:
if deepdrilling_target.fieldid in self.tonight_fields_list:
fields_evaluation_list = [
self.survey_fields_dict[deepdrilling_target.fieldid]
]
else:
fields_evaluation_list = []
num_targets_to_propose = 0
if constrained_filter is None:
filters_evaluation_list = self.tonight_filters_list
else:
filters_evaluation_list = [constrained_filter]
# compute sky brightness for all targets
id_list = []
ra_rad_list = []
dec_rad_list = []
mags_dict = {}
airmass_dict = {}
hour_angle_dict = {}
moon_distance_dict = {}
for field in fields_evaluation_list:
# create coordinates arrays
id_list.append(field.fieldid)
ra_rad_list.append(field.ra_rad)
dec_rad_list.append(field.dec_rad)
if (len(id_list) > 0) and (not self.ignore_sky_brightness
or not self.ignore_airmass):
self.sky.update(timestamp)
sky_mags = self.sky.get_sky_brightness(numpy.array(id_list))
airmass = self.sky.get_airmass(numpy.array(id_list))
nra = numpy.array(ra_rad_list)
moon_distance = self.sky.get_separation("moon", nra,
numpy.array(dec_rad_list))
hour_angle = self.sky.get_hour_angle(nra)
for ix, fieldid in enumerate(id_list):
mags_dict[fieldid] = {k: v[ix] for k, v in sky_mags.items()}
airmass_dict[fieldid] = airmass[ix]
moon_distance_dict[fieldid] = moon_distance[ix]
hour_angle_dict[fieldid] = hour_angle[ix]
evaluated_fields = 0
discarded_fields_airmass = 0
discarded_fields_notargets = 0
discarded_targets_consecutive = 0
discarded_targets_nanbrightness = 0
discarded_targets_lowbrightness = 0
discarded_targets_highbrightness = 0
discarded_targets_seeing = 0
discarded_moon_distance = 0
evaluated_targets = 0
groups_to_close_list = []
# compute target value
for field in fields_evaluation_list:
fieldid = field.fieldid
# discard fields with no more targets tonight
if fieldid not in self.tonight_targets_dict:
discarded_fields_notargets += 1
continue
# discard fields beyond airmass limit
if self.ignore_airmass:
airmass = 1.0
else:
airmass = airmass_dict[fieldid]
if airmass > self.params.max_airmass:
discarded_fields_airmass += 1
continue
moon_distance = moon_distance_dict[fieldid]
if moon_distance < self.params.min_distance_moon_rad:
discarded_moon_distance += 1
continue
airmass_rank = self.params.airmass_bonus * \
(self.params.max_airmass - airmass) / (self.params.max_airmass - 1.0)
hour_angle_rank = self.params.hour_angle_bonus * \
(1.0 - numpy.abs(hour_angle_dict[fieldid]) / self.params.hour_angle_max_rad)
for filter in self.tonight_targets_dict[fieldid]:
if filter not in filters_evaluation_list:
continue
# discard target beyond seeing limit
if seeing > self.params.filter_max_seeing_dict[filter]:
discarded_targets_seeing += 1
continue
target = self.tonight_targets_dict[fieldid][filter]
target.time = timestamp
target.airmass = airmass
# discard target if consecutive
if self.last_observation_was_for_this_proposal:
if (self.observation_fulfills_target(
self.last_observation, target)
and not self.params.accept_consecutive_visits):
discarded_targets_consecutive += 1
continue
if self.ignore_sky_brightness:
sky_brightness = 0.0
else:
# discard target beyond sky brightness limits
sky_brightness = mags_dict[fieldid][filter]
if math.isnan(sky_brightness):
discarded_targets_nanbrightness += 1
continue
if sky_brightness < self.params.filter_min_brig_dict[
filter]:
discarded_targets_lowbrightness += 1
continue
if sky_brightness > self.params.filter_max_brig_dict[
filter]:
discarded_targets_highbrightness += 1
continue
# target is accepted
# compute value for available targets
target.sky_brightness = sky_brightness
target.cloud = cloud
target.seeing = seeing
if self.survey_targets_progress < 1.0:
area_rank = (1.0 - target.progress) / (
1.0 - self.survey_targets_progress)
if (self.params.filter_num_grouped_visits_dict[filter] >
1) and (target.groupix > 1):
time_rank = self.time_window(timestamp -
target.last_visit_time)
if time_rank > 0.0:
need_ratio = area_rank + time_rank * self.params.time_weight
else:
need_ratio = time_rank
else:
need_ratio = area_rank
else:
need_ratio = 0.0
if need_ratio > 0.0:
# target is needed
target.need = need_ratio
target.bonus = airmass_rank + hour_angle_rank
target.value = target.need + target.bonus
self.add_evaluated_target(target)
elif need_ratio < 0.0:
# target group lost
groups_to_close_list.append(target)
evaluated_targets += 1
evaluated_fields += 1
for target in groups_to_close_list:
self.close_group(target, "group lost")
self.log.log(
EXTENSIVE, "suggest_targets: fields=%i, evaluated=%i, "
"discarded airmass=%i notargets=%i" %
(len(id_list), evaluated_fields, discarded_fields_airmass,
discarded_fields_notargets))
self.log.log(
EXTENSIVE,
"suggest_targets: evaluated targets=%i, discarded consecutive=%i "
"seeing=%i "
"lowbright=%i highbright=%i nanbright=%i moondistance=%i" %
(evaluated_targets, discarded_targets_consecutive,
discarded_targets_seeing, discarded_targets_lowbrightness,
discarded_targets_highbrightness, discarded_targets_nanbrightness,
discarded_moon_distance))
return self.get_evaluated_target_list(num_targets_to_propose)
def serialize(self):
if math.isnan(self.value) or math.isinf(self.value):
return 'float("{}")'.format(self.value), set()
return super().serialize()
def search_engine_3(query):
query = remove_step(query)
query = list(set(query.split(' ')))
heap = [] #heap._heapify_max()
lst_of_lst = []
vocabulary = load_obj('vocabulary')
ii1 = load_obj('inverted_index_1')
for w in query:
if w not in vocabulary:
print('No results')
return
i = vocabulary[w]
lst_of_lst.append(ii1[i])
doc_list = set.intersection(*[set(sublist) for sublist in lst_of_lst])
doc_list = list(doc_list)
d_len = len(doc_list)
if d_len == 0:
print('No results')
return
# user's informations for the ranks
print("target price: ")
target_price = int(input())
print("n rooms: ")
rooms = int(input())
print("location: ")
location = input()
max_distance = int(input('max distance from your location(km): '))
if max_distance <= 0:
max_distance = 10 # default value 10 km
# loading location's informations from geopy
geolocator = Nominatim(user_agent="specify_your_app_name_here")
location = geolocator.geocode(location, timeout=None)
center = (location.latitude, location.longitude)
#if len(str(rmin)) == 0:
# rmin = 1
for doc in doc_list:
with open('data/docs/' + doc + '.tsv', encoding='utf-8') as f:
row = f.read()
row = row.split('\t')
dp = int(row[0][1:])
nrooms = row[1]
if math.isnan(float(row[5])):
continue
target_location = (float(row[5]), float(row[6]))
rank_dist = distance_rank(center, target_location)
score_price = 0.2 * price_rank(target_price, dp)
ratio_rooms = 0.1 * rooms_rank(rooms, nrooms)
score_location = 0.7 * rank_dist
heap.append(
tuple([score_price + ratio_rooms + score_location, doc]))
#heapq.heappush(heap,tuple([score_price+ratio_rooms,doc]))
list_for_df = []
k = 5
l = len(heap)
for i in range(l):
heapq._heapify_max(heap)
tup = heappop(heap)
#print(tup)
with open("data/docs/" + tup[1] + '.tsv', encoding='utf-8') as doc:
row = doc.read()
lst = row.split('\t')
lst = [i + 1, lst[7], lst[4], lst[2], lst[8]]
list_for_df.append(lst)
df = pd.DataFrame(
list_for_df,
columns=['Ranking', 'Title', 'Description', 'City', 'Url'])
print('Found %i results' % d_len)
df.set_index('Ranking')
return df.head(k)
def run(self):
# -------- Main Program Loop -----------
while not self.done:
# --- Main event loop
if self.gameOver:
break
for event in pygame.event.get(): # User did something
if event.type == pygame.QUIT: # If user clicked close
self.done = True # Flag that we are done so exit this loop
# section (4 lines) from github.com/fletcher-marsh
# have a color frame. Fill out back buffer surface with frame's data
if self.kinect.has_new_color_frame(): #is the kinect working
frame = self.kinect.get_last_color_frame()
self.drawColorFrame(frame, self.frameSurface)
frame = None #saving memory, be clean
if self.roundOngoing:
# section (7 lines) from github.com/fletcher-marsh
# We have a body frame, so can get skeletons
if self.kinect.has_new_body_frame(): # is IR camera working
self.bodies = self.kinect.get_last_body_frame()
if self.bodies is not None:
for i in range(0, self.kinect.max_body_count):
body = self.bodies.bodies[i] #extract body object
if not body.is_tracked: # where is body
continue
joints = body.joints # access data from IR camera
handRight = PyKinectV2.JointType_HandRight
handLeft = PyKinectV2.JointType_HandLeft
#access joints at the joint type
if joints[handRight].TrackingState != \
PyKinectV2.TrackingState_NotTracked:
# update current right hand positions
self.currRightHandX = \
joints[PyKinectV2.JointType_HandRight].Position.x
self.currRightHandY = \
joints[PyKinectV2.JointType_HandRight].Position.y
self.currRightHandZ = \
joints[PyKinectV2.JointType_HandRight].Position.z
# punch starts when right hand closes
self.isRightHandClosed = \
(PyKinectV2.HandState_Closed==body.hand_right_state)
if self.isRightHandClosed and \
self.rightPunchStartX == 0:
self.isRightPunching = True
self.rightPunchStartX = self.currRightHandX
self.rightPunchStartY = self.currRightHandY
self.rightPunchStartZ = self.currRightHandZ
elif not self.isRightHandClosed and \
self.rightPunchStartX != 0:
#not currently punching
self.isRightPunching = False
# save positions at end of punch
if self.rightPunchStartX != 0 and \
self.rightPunchEndX == 0:
self.rightPunchEndX = \
self.currRightHandX
self.rightPunchDX = \
(self.rightPunchEndX - \
self.rightPunchStartX)
if self.rightPunchStartY != 0 and \
self.rightPunchEndY == 0:
self.rightPunchEndY = \
self.currRightHandY
self.rightPunchDY = \
(self.rightPunchEndY - \
self.rightPunchStartY)
if self.rightPunchStartZ != 0 and \
self.rightPunchEndZ == 0:
self.rightPunchEndZ = \
self.currRightHandZ
self.rightPunchDZ = \
(self.rightPunchEndZ - \
self.rightPunchStartZ)
# reset punch start and end values
self.rightPunchStartX = 0
self.rightPunchStartY = 0
self.rightPunchStartZ = 0
self.rightPunchEndX = 0
self.rightPunchEndY = 0
self.rightPunchEndZ = 0
if BagSeshRuntime.isRightHook(self):
self.totalPunches += 1
elif BagSeshRuntime.isRightUppercut(self):
self.totalPunches += 1
elif BagSeshRuntime.isJab(self):
self.totalPunches += 1
# right jabs only for southpaw boxers
elif BagSeshRuntime.isCross(self):
self.totalPunches += 1
# right crosses only for orthodox boxers
# reset delta's after calculating punch
self.rightPunchDX = 0
self.rightPunchDY = 0
self.rightPunchDZ = 0
if joints[handLeft].TrackingState != \
PyKinectV2.TrackingState_NotTracked:
# update current left hand positions
self.currLeftHandX = \
joints[PyKinectV2.JointType_HandLeft].Position.x
self.currLeftHandY = \
joints[PyKinectV2.JointType_HandLeft].Position.y
self.currLeftHandZ = \
joints[PyKinectV2.JointType_HandLeft].Position.z
self.isLeftHandClosed = \
(PyKinectV2.HandState_Closed==\
body.hand_left_state)
if self.isLeftHandClosed and \
self.leftPunchStartX == 0:
self.isLeftPunching = True
self.leftPunchStartX = self.currLeftHandX
self.leftPunchStartY = self.currLeftHandY
self.leftPunchStartZ = self.currLeftHandZ
elif not self.isLeftHandClosed and \
self.leftPunchStartX != 0:
#not currently punching
self.isLeftPunching = False
# save positions at end of punch (hand opens)
if self.leftPunchStartX != 0 and \
self.leftPunchEndX == 0:
self.leftPunchEndX = self.currLeftHandX
self.leftPunchDX = \
(self.leftPunchEndX - \
self.leftPunchStartX)
if self.leftPunchStartY != 0 and \
self.leftPunchEndY == 0:
self.leftPunchEndY = self.currLeftHandY
self.leftPunchDY = \
(self.leftPunchEndY - \
self.leftPunchStartY)
if self.leftPunchStartZ != 0 and \
self.leftPunchEndZ == 0:
self.leftPunchEndZ = self.currLeftHandZ
self.leftPunchDZ = \
(self.leftPunchEndZ - \
self.leftPunchStartZ)
# reset punch start and end values
self.leftPunchStartX = 0
self.leftPunchStartY = 0
self.leftPunchStartZ = 0
self.leftPunchEndX = 0
self.leftPunchEndY = 0
self.leftPunchEndZ = 0
if BagSeshRuntime.isLeftHook(self):
self.totalPunches += 1
elif BagSeshRuntime.isLeftUppercut(self):
self.totalPunches += 1
elif BagSeshRuntime.isJab(self):
self.totalPunches += 1
# left jabs only for orthodox boxers
elif BagSeshRuntime.isCross(self):
self.totalPunches += 1
# left crosses only for southpaw boxers
# reset delta's after calculating punch
self.leftPunchDX = 0
self.leftPunchDY = 0
self.leftPunchDZ = 0
if math.isnan(self.rightPunchDX):
self.rightPunchDX = 0
if math.isnan(self.rightPunchDY):
self.rightPunchDY = 0
if math.isnan(self.rightPunchDZ):
self.rightPunchDZ = 0
if math.isnan(self.leftPunchDX):
self.leftPunchDX = 0
if math.isnan(self.leftPunchDY):
self.leftPunchDY = 0
if math.isnan(self.leftPunchDZ):
self.leftPunchDZ = 0
# cycle previous and current positions for next time
self.prevRightHandX = self.currRightHandX
self.prevRightHandY = self.currRightHandY
self.prevRightHandZ = self.currRightHandZ
self.prevLeftHandX = self.currLeftHandX
self.prevLeftHandY = self.currLeftHandY
self.prevLeftHandZ = self.currLeftHandZ
# --- Game logic
# Draw graphics
self.drawPunchingBag()
self.drawTimerBar()
# update user's accuracy so far
self.getUserAccuracy()
self.suggestPunches()
self.displaySuggestion()
self.displayWarning()
# text displaying round time left
pygame.draw.rect(self.frameSurface,(189, 195, 199),
[75,self.screenHeight-425,500,300])
roundTimeFont = pygame.font.Font(None, 80)
if self.roundTimeLeft%60 != 0:
if self.roundTimeLeft%60 >= 10:
roundText = roundTimeFont.render("Round Time: "+\
str(self.roundTimeLeft//60)+":"+\
"%d"%(self.roundTimeLeft%60),
1, (192, 57, 43))
else:
roundText = roundTimeFont.render("Round Time: "+\
str(self.roundTimeLeft//60)+":"+"0"+\
"%d"%(self.roundTimeLeft%60),
1, (192, 57, 43))
else:
roundText = roundTimeFont.render("Round Time: "+\
str(self.roundTimeLeft//60)+":"+"00",
1, (192, 57, 43))
self.frameSurface.blit(roundText, (100,self.screenHeight-400))
currRoundFont = pygame.font.Font(None, 80)
currRoundText = currRoundFont.render("Round "+\
str(self.currRound),1, (0, 110, 27))
self.frameSurface.blit(currRoundText, (100,self.screenHeight-200))
# text displaying rest time left
restTimeFont = pygame.font.Font(None, 80)
if self.restTimeLeft%60 != 0:
if self.restTimeLeft%60 >= 10:
restText = restTimeFont.render("Rest Time: "+\
str(self.restTimeLeft//60)+":"+\
"%d"%(self.restTimeLeft%60),
1, (39, 174, 96))
else:
restText = restTimeFont.render("Rest Time: "+\
str(self.restTimeLeft//60)+":"+"0"+\
"%d"%(self.restTimeLeft%60),
1, (39, 174, 96))
else:
restText = restTimeFont.render("Rest Time: "+\
str(self.restTimeLeft//60)+":"+"00",
1, (39, 174, 96))
self.frameSurface.blit(restText, (100,self.screenHeight-300))
if self.roundTimeLeft == 10:
tenSecWarning = roundTimeFont.render("10 seconds!",
1, (170, 218, 255))
self.frameSurface.blit(tenSecWarning, (self.screenWidth//2,200))
# section (6 lines) from github.com/fletcher-marsh
# --- copy back buffer surface pixels to the screen
# resize it if needed and keep aspect ratio
# --- (screen size may be different from Kinect's color frame size)
hToW = float(self.frameSurface.get_height()) / \
self.frameSurface.get_width()
targetHeight = int(hToW * self.screen.get_width())
surfaceToDraw = pygame.transform.scale(self.frameSurface,
(self.screen.get_width(), targetHeight));
self.screen.blit(surfaceToDraw, (0,0))
surfaceToDraw = None
pygame.display.update()
# play start bell
if self.gameTimeLeft == self.preGameBuffer + self.postGameBuffer +\
self.numRounds*(self.roundTimeLeft+self.restTimeLeft) and\
self.threeSecWarningPlayed == False:#188:
pygame.mixer.Sound.play(self.bellSound)
self.threeSecWarningPlayed = True
if self.gameTimeLeft == self.postGameBuffer +\
self.numRounds*(self.roundTimeLeft+self.restTimeLeft):
self.roundOngoing = True
# --- 60 frames per second
self.clock.tick(60)
if self.gameStartTime == None:
self.gameStartTime = pygame.time.get_ticks()
self.gameTimeLeft = (self.preGameBuffer+self.postGameBuffer\
+ self.numRounds*(self.roundLength+self.restLength))\
- (pygame.time.get_ticks()-self.gameStartTime)//1000
modGameTime = ((self.gameTimeLeft-self.postGameBuffer-1) %\
(self.roundLength+self.restLength))
if self.roundOngoing:
self.roundTimeLeft = modGameTime - self.restLength
self.tenSecWarningPlayed = False
self.threeSecWarningPlayed = False
if self.restOngoing:
self.restTimeLeft = modGameTime
self.tenSecWarningPlayed = False
self.threeSecWarningPlayed = False
# time's up for the round and game
if self.roundTimeLeft == 10 and not self.tenSecWarningPlayed:
pygame.mixer.Sound.play(self.warningSound)
self.tenSecWarningPlayed = True
elif self.roundTimeLeft == 3 and not self.threeSecWarningPlayed:
# three second warning that the round will end
pygame.mixer.Sound.play(self.bellSound)
self.threeSecWarningPlayed = True
if self.restTimeLeft == 10 and not self.tenSecWarningPlayed:
pygame.mixer.Sound.play(self.warningSound)
self.tenSecWarningPlayed = True
elif self.restTimeLeft == 3 and not self.threeSecWarningPlayed:
# three second warning that the round will end
pygame.mixer.Sound.play(self.bellSound)
self.threeSecWarningPlayed = True
if self.roundOngoing and self.roundTimeLeft == 0:
self.roundOngoing = False
self.tenSecWarningPlayed = False
self.threeSecWarningPlayed = False
self.restOngoing = True
self.restTimeLeft = self.restLength
if self.restOngoing and self.restTimeLeft == 0 and \
self.currRound<self.numRounds:
self.restOngoing = False
self.tenSecWarningPlayed = False
self.threeSecWarningPlayed = False
self.roundOngoing = True
self.roundTimeLeft = self.roundLength
if modGameTime == 0:
self.currRound = self.numRounds-self.gameTimeLeft //\
(self.roundLength+self.restLength)+1
if self.gameTimeLeft == 0:
self.gameOver = True
# Close Kinect sensor, close the window and quit.
self.kinect.close()
pygame.quit()
def akde(X,
grid=None,
gam=None,
errtol=10**-5,
maxiter=100,
seed=0,
verbose=False):
"""
Adaptive Kernel Density Estimate (AKDE)
Provides optimal accuracy/speed tradeoff, controlled with parameter gam
---------------INPUT---------------
- x : event points
- grid : points at which the density rate is estimated
- gam : scale parameter
- errtol : convergence tolerance (default: 10^-5)
- maxiter : maximum iterations (default: 200)
- seed : random number seed
---------------OUTPUT---------------
- pdf : estimated density
- grid : grid points at which density is estimated
Usage:
import numpy as np
mu, sigma = 3., 1.
X = np.random.lognormal(mu, sigma, 1000)
pdf, grid = akde(X)
Reference:
Kernel density estimation via diffusion
Z. I. Botev, J. F. Grotowski, and D. P. Kroese (2010)
Annals of Statistics, Volume 38, Number 5, pages 2916-2957.
"""
np.seterr(divide='raise')
n = X.shape[0]
assert (n > 1)
if len(X.shape) > 1:
d = X.shape[1]
else:
d = 1
X = X.reshape((n, d))
zd = np.where(np.diff(X[:, 0]) >= 1e-3)[0] + 1
X = np.vstack([np.asarray([X[0, :]]).reshape((-1, d)), X[zd, :]])
n = X.shape[0]
if n < 2:
if grid is None:
return (np.zeros((1, 1)), X[0, 0])
else:
return (np.zeros(grid.shape), grid)
xmax = np.max(X, axis=0)
xmin = np.min(X, axis=0)
r = xmax - xmin
smax = xmax + r / 10.
smin = xmin - r / 10.
scaling = smax - smin
X = X - smin
X = np.divide(X, scaling)
if gam is None:
gam = int(math.ceil(math.sqrt(n)))
if grid is None:
step = scaling / (2**12 - 1)
npts = int(math.ceil(scaling / step)) + 1
grid = np.linspace(smin, smax, npts)
grid = grid.reshape((-1, 1))
mesh = np.subtract(grid, smin)
mesh = np.divide(mesh, scaling)
## algorithm initialization
local_random = np.random.RandomState(seed=seed)
bw = 0.2 / (n**(d / (d + 4.)))
perm = local_random.permutation(n)
##perm = list(xrange(0, n))
mus = X[perm[0:gam], :]
w = local_random.rand(gam)
# w = np.linspace(0., 1., gam) + 0.001
w = np.divide(w, np.sum(w))
sigmas = (bw**2.) * local_random.rand(gam, d)
##sigmas = (bw ** 2.) * (np.linspace(0., 1., gam).reshape((-1,1)) + 0.01)
ent = float("-inf")
for i in range(maxiter):
Eold = ent
(w, mus, sigmas, bw, ent) = akde_reg_EM(w, mus, sigmas, bw, X)
err = abs((ent - Eold) / ent)
if verbose:
print('Iter. Err. Bandwidth \n')
print('%4i %8.2e %8.2e\n' % (i, err, bw))
assert (not math.isnan(bw))
assert (not math.isnan(err))
if (err < errtol):
break
pdf = akde_probfun(mesh, w, mus, sigmas) / scaling
return pdf, grid
def addFact(self, f):
if f.id is None:
f.set("id","ixv-%d" % (self.idGen))
self.idGen += 1
conceptName = self.nsmap.qname(f.qname)
scheme, ident = f.context.entityIdentifier
aspects = {
"c": conceptName,
"e": self.nsmap.qname(QName(self.nsmap.getPrefix(scheme,"e"), scheme, ident)),
}
factData = {
"a": aspects,
}
if f.isNil:
factData["v"] = None
elif f.concept is not None and f.concept.isEnumeration:
qnEnums = f.xValue
if not isinstance(qnEnums, list):
qnEnums = (qnEnums,)
factData["v"] = " ".join(self.nsmap.qname(qn) for qn in qnEnums)
for qn in qnEnums:
self.addConcept(self.dts.qnameConcepts.get(qn))
else:
factData["v"] = f.value
if f.value == INVALIDixVALUE:
factData["err"] = 'INVALID_IX_VALUE'
if f.format is not None:
factData["f"] = str(f.format)
if f.isNumeric:
if f.unit is not None and len(f.unit.measures[0]):
# XXX does not support complex units
unit = self.nsmap.qname(f.unit.measures[0][0])
aspects["u"] = unit
else:
# The presence of the unit aspect is used by the viewer to
# identify numeric facts. If the fact has no unit (invalid
# XBRL, but we want to support it for draft documents),
# include the unit aspect with a null value.
aspects["u"] = None
d = inferredDecimals(f)
if d != float("INF") and not math.isnan(d):
factData["d"] = d
for d, v in f.context.qnameDims.items():
if v.memberQname is not None:
aspects[self.nsmap.qname(v.dimensionQname)] = self.nsmap.qname(v.memberQname)
self.addConcept(v.member)
self.addConcept(v.dimension, dimensionType = "e")
elif v.typedMember is not None:
aspects[self.nsmap.qname(v.dimensionQname)] = v.typedMember.text
self.addConcept(v.dimension, dimensionType = "t")
if f.context.isForeverPeriod:
aspects["p"] = "f"
elif f.context.isInstantPeriod and f.context.instantDatetime is not None:
aspects["p"] = self.dateFormat(f.context.instantDatetime.isoformat())
elif f.context.isStartEndPeriod and f.context.startDatetime is not None and f.context.endDatetime is not None:
aspects["p"] = "%s/%s" % (
self.dateFormat(f.context.startDatetime.isoformat()),
self.dateFormat(f.context.endDatetime.isoformat())
)
frels = self.footnoteRelationshipSet.fromModelObject(f)
if frels:
for frel in frels:
if frel.toModelObject is not None:
factData.setdefault("fn", []).append(frel.toModelObject.id)
self.taxonomyData["facts"][f.id] = factData
self.addConcept(f.concept)
def _isinvalid(x):
return isnan(x) or isinf(x)
def fmriRDM_roi(fmri_data, mask_data):
"""
Calculate the Representational Dissimilarity Matrix - RDM(s) for fMRI data (for ROI)
Parameters
----------
fmri_data : array [nx, ny, nz].
The fmri data.
The shape of fmri_data must be [n_cons, n_chls, nx, ny, nz].
n_cons, n_chls, nx, ny, nz represent the number of conidtions, the number of channels &
the size of fMRI-img, respectively.
mask_data : array [nx, ny, nz].
The mask data for region of interest (ROI)
The size of the fMRI-img. nx, ny, nz represent the number of voxels along the x, y, z axis.
Returns
-------
RDM : array
The fMRI-ROI RDM.
The shape of RDM is [n_cons, n_cons].
"""
# get the number of conditions, subjects, the size of the fMRI-img
ncons, nsubs, nx, ny, nz = fmri_data.shape
# record the the number of voxels that is not 0 or NaN
n = 0
for i in range(nx):
for j in range(ny):
for k in range(nz):
# not 0 or NaN
if (mask_data[i, j, k] != 0) and (math.isnan(mask_data[i, j, k]) == False):
n = n + 1
# initialize the data for calculating the RDM
data = np.zeros([ncons, nsubs, n], dtype=np.float)
# assignment
for p in range(ncons):
for q in range(nsubs):
n = 0
for i in range(nx):
for j in range(ny):
for k in range(nz):
# not 0 or NaN
if (mask_data[i, j, k] != 0) and (math.isnan(mask_data[i, j, k]) == False):
data[p, q, n] = fmri_data[p, q, i, j, k]
n = n + 1
# initialize the RDMs
rdm = np.zeros([ncons, ncons], dtype=np.float)
# flatten the data for different calculating conditions
data = np.reshape(data, [ncons, nsubs*n])
# calculate the values in RDM
for i in range(ncons):
for j in range(ncons):
if (np.isnan(data[i]).any() == False) and (np.isnan(data[j]).any() == False):
# calculate the Pearson Coefficient
r = pearsonr(data[i], data[j])[0]
# calculate the dissimilarity
rdm[i, j] = limtozero(1 - abs(r))
return rdm
def read_cb(data):
global ang_min
global a_inc
global radius
global rmin
global rmax
global prev_x
global prev_y
global prev_z
# define constants
leg_size_min = 0.05
leg_size_max = 0.15
depth_min = 0.01
depth_max = 0.07
connect_dist = 0.05
# reset feature variables
angle = []
far = 0
vector = Vector3()
temp_i = 0
temp_f = 0
clear_flag = 0
leg = []
leg_c = 0
# ensure laser variables are up to date
ang_min = data.angle_min
ang_max = data.angle_max
a_inc = data.angle_increment
# initialise scan data to be mutable object
new_scan = list(data.ranges)
print('New Leg')
# reject invalid data and initialise new data to be processed
for d, data_l in enumerate(data.ranges):
angle.append(ang_min + d * a_inc)
if data_l < rmin or data_l > rmax or math.isnan(data_l):
new_scan[d] = float('nan')
# attempt to algorithm
for c, scan in enumerate(new_scan):
if c == (len(new_scan) - 1):
break
if abs(new_scan[c] - new_scan[c + 1]) < connect_dist:
if temp_i == 0:
temp_i = c
continue
elif temp_i != 0:
temp_f = c
clear_flag = 1
if temp_f - temp_i > 5:
print('Entry {:f} and {:f}'.format(angle[temp_i], angle[temp_f]))
if leg_size_min < polar_dist(new_scan[temp_i], angle[temp_i],
new_scan[temp_f],
angle[temp_f]) < leg_size_max:
mid = int(math.ceil((temp_f - temp_i) / 2) + temp_i)
if depth_min < (new_scan[temp_i] -
new_scan[mid]) < depth_max and depth_min < (
new_scan[temp_f] -
new_scan[mid]) < depth_max:
ini_angle = round(180 / math.pi * (angle[temp_i]), 3)
fin_angle = round(180 / math.pi * (angle[temp_f]), 3)
leg.append([angle[mid], new_scan[mid]])
print('Leg between {:f} and {:f}'.format(
ini_angle, fin_angle))
if clear_flag == 1:
temp_i = 0
temp_f = 0
clear_flag = 0
# convert legs to useful information
# leg[0][i] is the first leg entry discovered starting from negative angles
# leg[i][0] is the angle of corresponding leg
# leg[i][1] is the distance of corresponding leg
if len(leg) == 0:
if prev_x != 0 and prev_y != 0:
x_leg = prev_x
y_leg = prev_y
z_leg = prev_z + 1
else:
x_leg = 0
y_leg = 0
z_leg = 0
elif len(leg) == 1:
x_leg = leg[0][1] * math.cos(leg[0][0])
y_leg = leg[0][1] * math.sin(leg[0][0])
z_leg = 1
elif len(leg) == 2:
if leg[1][0] - leg[0][0] < 15:
x_leg = (leg[0][1] + leg[1][1]) / 2 * math.cos(
(leg[0][0] + leg[1][0]) / 2)
y_leg = (leg[0][1] + leg[1][1]) / 2 * math.sin(
(leg[0][0] + leg[1][0]) / 2)
z_leg = 1
elif abs(leg[0][0]) < abs(leg[1][0]):
x_leg = leg[0][1] * math.cos(leg[0][0])
y_leg = leg[0][1] * math.sin(leg[0][0])
z_leg = 1
else:
x_leg = leg[1][1] * math.cos(leg[1][0])
y_leg = leg[1][1] * math.sin(leg[1][0])
z_leg = 1
else:
if prev_x != 0 and prev_y != 0:
x_leg = prev_x
y_leg = prev_y
z_leg = prev_z + 1
else:
x_leg = 0
y_leg = 0
z_leg = 0
vector.x = x_leg
vector.y = y_leg
vector.z = z_leg
prev_x = x_leg
prev_y = y_leg
prev_z = z_leg
leg_pub.publish(vector)
def value(self):
if self.variable.is_discrete:
return Unknown if isnan(self) else self.variable.values[int(self)]
if self.variable.is_string:
return self._value
return float(self)
def track_footprint_identifiers(json_dir,
out_dir,
min_iou=0.25,
iou_field='iou_score',
id_field='Id',
reverse_order=False,
verbose=True,
super_verbose=False):
'''
Track footprint identifiers in the deep time stack.
We need to track the global gdf instead of just the gdf of t-1.
'''
os.makedirs(out_dir, exist_ok=True)
# set columns for master gdf
gdf_master_columns = [id_field, iou_field, 'area', 'geometry']
json_files = sorted([
f for f in os.listdir(os.path.join(json_dir))
if f.endswith('.geojson') and os.path.exists(os.path.join(json_dir, f))
])
# start at the end and work backwards?
if reverse_order:
json_files = json_files[::-1]
# check if only partical matching has been done (this will cause errors)
out_files_tmp = sorted(
[z for z in os.listdir(out_dir) if z.endswith('.geojson')])
if len(out_files_tmp) > 0:
if len(out_files_tmp) != len(json_files):
raise Exception(
"\nError in:", out_dir, "with N =", len(out_files_tmp),
"files, need to purge this folder and restart matching!\n")
return
elif len(out_files_tmp) == len(json_files):
print("\nDir:", os.path.basename(out_dir), "N files:",
len(json_files), "directory matching completed, skipping...")
return
else:
print("\nMatching json_dir: ", os.path.basename(json_dir), "N json:",
len(json_files))
gdf_dict = {}
for j, f in enumerate(json_files):
name_root = f.split('.')[0]
json_path = os.path.join(json_dir, f)
output_path = os.path.join(out_dir, f)
if verbose and ((j % 1) == 0):
print(" ", j, "/", len(json_files), "for",
os.path.basename(json_dir), "=", name_root)
# gdf
gdf_now = gpd.read_file(json_path)
# drop value if it exists
gdf_now = gdf_now.drop(columns=['value'])
# get area
gdf_now['area'] = gdf_now['geometry'].area
# initialize iou, id
gdf_now[iou_field] = -1
gdf_now[id_field] = -1
# sort by reverse area
gdf_now.sort_values(by=['area'], ascending=False, inplace=True)
gdf_now = gdf_now.reset_index(drop=True)
# reorder columns (if needed)
gdf_now = gdf_now[gdf_master_columns]
id_set = set([])
if verbose:
print("\n")
print("", j, "file_name:", f)
print(" ", "gdf_now.columns:", gdf_now.columns)
if j == 0:
# Establish initial footprints at Epoch0
# set id
gdf_now[id_field] = gdf_now.index.values
gdf_now[iou_field] = 0
n_new = len(gdf_now)
n_matched = 0
id_set = set(gdf_now[id_field].values)
gdf_master_Out = gdf_now.copy(deep=True)
# gdf_dict[f] = gdf_now
else:
# match buildings in epochT to epochT-1
# see: https://github.com/CosmiQ/solaris/blob/master/solaris/eval/base.py
# print("gdf_master;", gdf_dict['master']) #gdf_master)
gdf_master_Out = gdf_dict['master'].copy(deep=True)
gdf_master_Edit = gdf_dict['master'].copy(deep=True)
if verbose:
print(" len gdf_now:", len(gdf_now), "len(gdf_master):",
len(gdf_master_Out), "max master id:",
np.max(gdf_master_Out[id_field]))
print(" gdf_master_Edit.columns:", gdf_master_Edit.columns)
new_id = np.max(gdf_master_Edit[id_field]) + 1
# if verbose:
# print("new_id:", new_id)
idx = 0
n_new = 0
n_matched = 0
for pred_idx, pred_row in gdf_now.iterrows():
if verbose:
if (idx % 1000) == 0:
print(" ", name_root, idx, "/", len(gdf_now))
if super_verbose:
# print(" ", i, j, idx, "/", len(gdf_now))
print(" ", idx, "/", len(gdf_now))
idx += 1
pred_poly = pred_row.geometry
# if super_verbose:
# print(" pred_poly.exterior.coords:", list(pred_poly.exterior.coords))
# get iou overlap
iou_GDF = calculate_iou(pred_poly, gdf_master_Edit)
# iou_GDF = iou.calculate_iou(pred_poly, gdf_master_Edit)
# print("iou_GDF:", iou_GDF)
# Get max iou
if not iou_GDF.empty:
max_iou_row = iou_GDF.loc[iou_GDF['iou_score'].idxmax(
axis=0, skipna=True)]
# sometimes we are get an erroneous id of 0, caused by nan area,
# so check for this
max_area = max_iou_row.geometry.area
if max_area == 0 or math.isnan(max_area):
# print("nan area!", max_iou_row, "returning...")
raise Exception("\n Nan area!:", max_iou_row,
"returning...")
return
id_match = max_iou_row[id_field]
if id_match in id_set:
print("Already seen id! returning...")
raise Exception("\n Already seen id!", id_match,
"returning...")
return
# print("iou_GDF:", iou_GDF)
if max_iou_row['iou_score'] >= min_iou:
if super_verbose:
print(" pred_idx:", pred_idx, "match_id:",
max_iou_row[id_field], "max iou:",
max_iou_row['iou_score'])
# we have a successful match, so set iou, and id
gdf_now.loc[pred_row.name,
iou_field] = max_iou_row['iou_score']
gdf_now.loc[pred_row.name, id_field] = id_match
# drop matched polygon in ground truth
gdf_master_Edit = gdf_master_Edit.drop(
max_iou_row.name, axis=0)
n_matched += 1
# # update gdf_master geometry?
# # Actually let's leave the geometry the same so it doesn't move around...
# gdf_master_Out.at[max_iou_row['gt_idx'], 'geometry'] = pred_poly
# gdf_master_Out.at[max_iou_row['gt_idx'], 'area'] = pred_poly.area
# gdf_master_Out.at[max_iou_row['gt_idx'], iou_field] = max_iou_row['iou_score']
else:
# no match,
if super_verbose:
print(" Minimal match! - pred_idx:", pred_idx,
"match_id:", max_iou_row[id_field],
"max iou:", max_iou_row['iou_score'])
print(" Using new id:", new_id)
if (new_id in id_set) or (new_id == 0):
raise Exception(
"trying to add an id that already exists, returning!"
)
return
gdf_now.loc[pred_row.name, iou_field] = 0
gdf_now.loc[pred_row.name, id_field] = new_id
id_set.add(new_id)
# update master, cols = [id_field, iou_field, 'area', 'geometry']
gdf_master_Out.loc[new_id] = [
new_id, 0, pred_poly.area, pred_poly
]
new_id += 1
n_new += 1
else:
# no match (same exact code as right above)
if super_verbose:
print(" pred_idx:", pred_idx, "no overlap, new_id:",
new_id)
if (new_id in id_set) or (new_id == 0):
raise Exception(
"trying to add an id that already exists, returning!"
)
return
gdf_now.loc[pred_row.name, iou_field] = 0
gdf_now.loc[pred_row.name, id_field] = new_id
id_set.add(new_id)
# update master, cols = [id_field, iou_field, 'area', 'geometry']
gdf_master_Out.loc[new_id] = [
new_id, 0, pred_poly.area, pred_poly
]
new_id += 1
n_new += 1
# print("gdf_now:", gdf_now)
gdf_dict[f] = gdf_now
gdf_dict['master'] = gdf_master_Out
# save!
if len(gdf_now) > 0:
gdf_now.to_file(output_path, driver="GeoJSON")
else:
print("Empty dataframe, writing empty gdf", output_path)
open(output_path, 'a').close()
if verbose:
print(" ", "N_new, N_matched:", n_new, n_matched)
return
def isnan(value):
try:
import math
return math.isnan(float(value))
except:
return False
def gps_insar_ts(data_all, titles, region, colors, fileDir, output_name_compare, save, gps_filename, data_format, component):
dates_insar = []
range_change = []
fig = plt.figure(figsize=(25, 10))
# Plot deformation map with selected region/pixels overlain
ax1 = plt.subplot(121)
im = ax1.imshow(data_all[-1], cmap=colors, vmin=-0.05, vmax=0.05, aspect=0.75)
ax1.plot([region[0], region[0], region[1], region[1], region[0]], [region[2], region[3], region[3], region[2], region[2]], color='blue', zorder=10000)
ax1.set_title(titles[-1], fontsize=12, color='black')
ax1.invert_yaxis()
ax1.invert_xaxis()
cbar = fig.colorbar(im)
# cbar.set_label('LOS change (m)')
# Get averaged timeseries for given region
for i in range(len(data_all)):
nanCount=0
sum = []
numPix = 0
for x in range(region[0], region[1]):
for y in range(region[2], region[3]):
if math.isnan(data_all[i][y][x]):
nanCount+=1
else:
sum.append(data_all[i][y][x])
numPix += 1
# print('nanCount = ' + str(nanCount))
# print('Count = ' + str(len(sum)))
# print('Mean value = ' + str(np.mean(sum)))
dates_insar.append(dt.datetime.strptime(titles[i], "%Y%m%d"))
range_change.append(np.mean(sum))
# Plot InSAR time-series for selected pixels
ax2 = plt.subplot(122)
ax2.grid()
ax2.scatter(dates_insar, range_change, zorder=100)
ax2.set_aspect(2.15 * 10**4)
# Load GPS data
gps_data = utilities_GPS.readUNR(gps_filename, data_format)
# Get displacements
plot_displacements = []
# First find start date
search_date = dates_insar[0]
z_init = 666
while z_init == 666:
print('Looking for ' + search_date.strftime('%Y-%m-%d'))
for i in range(len(gps_data.dates)):
print(search_date.strftime('%Y%m%d') + ' = ' + gps_data.dates[i].strftime('%Y%m%d') + '?')
print(search_date.strftime('%Y%m%d') == gps_data.dates[i].strftime('%Y%m%d'))
if search_date.strftime('%Y%m%d') == gps_data.dates[i].strftime('%Y%m%d'):
plot_dates = gps_data.dates[i:]
print('GPS time series start: ' + str(plot_dates[0]))
plot_data = gps_data.up[i:]
z_init = gps_data.up[i]
break
# Try next day
search_date += dt.timedelta(days=1)
search_date -= dt.timedelta(days=1)
print('Using ' + search_date.strftime('%Y-%m-%d'))
print("Initial value: " + str(z_init))
for value in plot_data:
plot_displacements.append(value - z_init)
print(plot_displacements[-1])
plt.grid()
plt.scatter(plot_dates, plot_displacements, marker='.', zorder=101)
# ax2.set_xlim(min(dates_insar) - dt.timedelta(days=30), max(dates_insar) + dt.timedelta(days=30))
ax2.set_xlim(min(dates_insar), max(dates_insar))
# ax2.set_ylim(-0.01, 0.05)
plt.xlabel('Date')
plt.ylabel('LOS range change (m)', rotation=0, labelpad=58)
if save == 'yes':
print('Saving time series to ' + fileDir + output_name_compare)
plt.savefig(fileDir + output_name_compare)
plt.close()
subprocess.call("open " + fileDir + output_name_compare, shell=True)
else:
plt.show()
def format_numeric_result(analysis, result, decimalmark='.', sciformat=1):
"""
Returns the formatted number part of a results value. This is
responsible for deciding the precision, and notation of numeric
values in accordance to the uncertainty. If a non-numeric
result value is given, the value will be returned unchanged.
The following rules apply:
If the "Calculate precision from uncertainties" is enabled in
the Analysis service, and
a) If the non-decimal number of digits of the result is above
the service's ExponentialFormatPrecision, the result will
be formatted in scientific notation.
Example:
Given an Analysis with an uncertainty of 37 for a range of
results between 30000 and 40000, with an
ExponentialFormatPrecision equal to 4 and a result of 32092,
this method will return 3.2092E+04
b) If the number of digits of the integer part of the result is
below the ExponentialFormatPrecision, the result will be
formatted as decimal notation and the resulta will be rounded
in accordance to the precision (calculated from the uncertainty)
Example:
Given an Analysis with an uncertainty of 0.22 for a range of
results between 1 and 10 with an ExponentialFormatPrecision
equal to 4 and a result of 5.234, this method will return 5.2
If the "Calculate precision from Uncertainties" is disabled in the
analysis service, the same rules described above applies, but the
precision used for rounding the result is not calculated from
the uncertainty. The fixed length precision is used instead.
For further details, visit
https://jira.bikalabs.com/browse/LIMS-1334
The default decimal mark '.' will be replaced by the decimalmark
specified.
:param analysis: the analysis from which the uncertainty, precision
and other additional info have to be retrieved
:param result: result to be formatted.
:param decimalmark: decimal mark to use. By default '.'
:param sciformat: 1. The sci notation has to be formatted as aE^+b
2. The sci notation has to be formatted as ax10^b
3. As 2, but with super html entity for exp
4. The sci notation has to be formatted as a·10^b
5. As 4, but with super html entity for exp
By default 1
:result: should be a string to preserve the decimal precision.
:returns: the formatted result as string
"""
try:
result = float(result)
except ValueError:
return result
# continuing with 'nan' result will cause formatting to fail.
if math.isnan(result):
return result
# Scientific notation?
# Get the default precision for scientific notation
threshold = analysis.getExponentialFormatPrecision()
precision = analysis.getPrecision(result)
formatted = _format_decimal_or_sci(result, precision, threshold, sciformat)
return formatDecimalMark(formatted, decimalmark)
def mavlink_packet(self, msg):
'''handle an incoming mavlink packet'''
if not isinstance(self.console, wxconsole.MessageConsole):
return
if not self.console.is_alive():
self.mpstate.console = textconsole.SimpleConsole()
return
type = msg.get_type()
if type == 'HEARTBEAT':
sysid = msg.get_srcSystem()
if not sysid in self.vehicle_list:
self.add_new_vehicle(msg)
if sysid not in self.component_name:
self.component_name[sysid] = {}
compid = msg.get_srcComponent()
if compid not in self.component_name[sysid]:
self.component_name[sysid][compid] = self.component_type_string(msg)
self.update_vehicle_menu()
if self.last_param_sysid_timestamp != self.module('param').new_sysid_timestamp:
'''a new component ID has appeared for parameters'''
self.last_param_sysid_timestamp = self.module('param').new_sysid_timestamp
self.update_vehicle_menu()
if type in ['RADIO', 'RADIO_STATUS']:
# handle RADIO msgs from all vehicles
if msg.rssi < msg.noise+10 or msg.remrssi < msg.remnoise+10:
fg = 'red'
else:
fg = 'black'
self.console.set_status('Radio', 'Radio %u/%u %u/%u' % (msg.rssi, msg.noise, msg.remrssi, msg.remnoise), fg=fg)
if not self.is_primary_vehicle(msg):
# don't process msgs from other than primary vehicle, other than
# updating vehicle list
return
master = self.master
# add some status fields
if type in [ 'GPS_RAW', 'GPS_RAW_INT' ]:
if type == "GPS_RAW":
num_sats1 = master.field('GPS_STATUS', 'satellites_visible', 0)
else:
num_sats1 = msg.satellites_visible
num_sats2 = master.field('GPS2_RAW', 'satellites_visible', -1)
if num_sats2 == -1:
sats_string = "%u" % num_sats1
else:
sats_string = "%u/%u" % (num_sats1, num_sats2)
if ((msg.fix_type >= 3 and master.mavlink10()) or
(msg.fix_type == 2 and not master.mavlink10())):
if (msg.fix_type >= 4):
fix_type = "%u" % msg.fix_type
else:
fix_type = ""
self.console.set_status('GPS', 'GPS: OK%s (%s)' % (fix_type, sats_string), fg='green')
else:
self.console.set_status('GPS', 'GPS: %u (%s)' % (msg.fix_type, sats_string), fg='red')
if master.mavlink10():
gps_heading = int(self.mpstate.status.msgs['GPS_RAW_INT'].cog * 0.01)
else:
gps_heading = self.mpstate.status.msgs['GPS_RAW'].hdg
self.console.set_status('Heading', 'Hdg %s/%u' % (master.field('VFR_HUD', 'heading', '-'), gps_heading))
elif type == 'VFR_HUD':
if master.mavlink10():
alt = master.field('GPS_RAW_INT', 'alt', 0) / 1.0e3
else:
alt = master.field('GPS_RAW', 'alt', 0)
home = self.module('wp').get_home()
if home is not None:
home_lat = home.x
home_lng = home.y
else:
home_lat = None
home_lng = None
lat = master.field('GLOBAL_POSITION_INT', 'lat', 0) * 1.0e-7
lng = master.field('GLOBAL_POSITION_INT', 'lon', 0) * 1.0e-7
rel_alt = master.field('GLOBAL_POSITION_INT', 'relative_alt', 0) * 1.0e-3
agl_alt = None
if self.settings.basealt != 0:
agl_alt = self.console.ElevationMap.GetElevation(lat, lng)
if agl_alt is not None:
agl_alt = self.settings.basealt - agl_alt
else:
try:
agl_alt_home = self.console.ElevationMap.GetElevation(home_lat, home_lng)
except Exception as ex:
print(ex)
agl_alt_home = None
if agl_alt_home is not None:
agl_alt = self.console.ElevationMap.GetElevation(lat, lng)
if agl_alt is not None:
agl_alt = agl_alt_home - agl_alt
if agl_alt is not None:
agl_alt += rel_alt
vehicle_agl = master.field('TERRAIN_REPORT', 'current_height', None)
if vehicle_agl is None:
vehicle_agl = '---'
else:
vehicle_agl = self.height_string(vehicle_agl)
self.console.set_status('AGL', 'AGL %s/%s' % (self.height_string(agl_alt), vehicle_agl))
self.console.set_status('Alt', 'Alt %s' % self.height_string(rel_alt))
self.console.set_status('AirSpeed', 'AirSpeed %s' % self.speed_string(msg.airspeed))
self.console.set_status('GPSSpeed', 'GPSSpeed %s' % self.speed_string(msg.groundspeed))
self.console.set_status('Thr', 'Thr %u' % msg.throttle)
t = time.localtime(msg._timestamp)
flying = False
if self.mpstate.vehicle_type == 'copter':
flying = self.master.motors_armed()
else:
flying = msg.groundspeed > 3
if flying and not self.in_air:
self.in_air = True
self.start_time = time.mktime(t)
elif flying and self.in_air:
self.total_time = time.mktime(t) - self.start_time
self.console.set_status('FlightTime', 'FlightTime %u:%02u' % (int(self.total_time)/60, int(self.total_time)%60))
elif not flying and self.in_air:
self.in_air = False
self.total_time = time.mktime(t) - self.start_time
self.console.set_status('FlightTime', 'FlightTime %u:%02u' % (int(self.total_time)/60, int(self.total_time)%60))
elif type == 'ATTITUDE':
self.console.set_status('Roll', 'Roll %u' % math.degrees(msg.roll))
self.console.set_status('Pitch', 'Pitch %u' % math.degrees(msg.pitch))
elif type in ['SYS_STATUS']:
sensors = { 'AS' : mavutil.mavlink.MAV_SYS_STATUS_SENSOR_DIFFERENTIAL_PRESSURE,
'MAG' : mavutil.mavlink.MAV_SYS_STATUS_SENSOR_3D_MAG,
'INS' : mavutil.mavlink.MAV_SYS_STATUS_SENSOR_3D_ACCEL | mavutil.mavlink.MAV_SYS_STATUS_SENSOR_3D_GYRO,
'AHRS' : mavutil.mavlink.MAV_SYS_STATUS_AHRS,
'RC' : mavutil.mavlink.MAV_SYS_STATUS_SENSOR_RC_RECEIVER,
'TERR' : mavutil.mavlink.MAV_SYS_STATUS_TERRAIN,
'RNG' : mavutil.mavlink.MAV_SYS_STATUS_SENSOR_LASER_POSITION,
'LOG' : mavutil.mavlink.MAV_SYS_STATUS_LOGGING,
}
announce = [ 'RC' ]
for s in sensors.keys():
bits = sensors[s]
present = ((msg.onboard_control_sensors_present & bits) == bits)
enabled = ((msg.onboard_control_sensors_enabled & bits) == bits)
healthy = ((msg.onboard_control_sensors_health & bits) == bits)
if not present:
fg = 'black'
elif not enabled:
fg = 'grey'
elif not healthy:
fg = 'red'
else:
fg = 'green'
# for terrain show yellow if still loading
if s == 'TERR' and fg == 'green' and master.field('TERRAIN_REPORT', 'pending', 0) != 0:
fg = 'yellow'
self.console.set_status(s, s, fg=fg)
for s in announce:
bits = sensors[s]
enabled = ((msg.onboard_control_sensors_enabled & bits) == bits)
healthy = ((msg.onboard_control_sensors_health & bits) == bits)
was_healthy = ((self.last_sys_status_health & bits) == bits)
if enabled and not healthy and was_healthy:
self.say("%s fail" % s)
self.last_sys_status_health = msg.onboard_control_sensors_health
elif type == 'WIND':
self.console.set_status('Wind', 'Wind %u/%.2f' % (msg.direction, msg.speed))
elif type == 'EKF_STATUS_REPORT':
highest = 0.0
vars = ['velocity_variance',
'pos_horiz_variance',
'pos_vert_variance',
'compass_variance',
'terrain_alt_variance']
for var in vars:
v = getattr(msg, var, 0)
highest = max(v, highest)
if highest >= 1.0:
fg = 'red'
elif highest >= 0.5:
fg = 'orange'
else:
fg = 'green'
self.console.set_status('EKF', 'EKF', fg=fg)
elif type == 'HWSTATUS':
if msg.Vcc >= 4600 and msg.Vcc <= 5300:
fg = 'green'
else:
fg = 'red'
self.console.set_status('Vcc', 'Vcc %.2f' % (msg.Vcc * 0.001), fg=fg)
elif type == 'POWER_STATUS':
if msg.flags & mavutil.mavlink.MAV_POWER_STATUS_CHANGED:
fg = 'red'
else:
fg = 'green'
status = 'PWR:'
if msg.flags & mavutil.mavlink.MAV_POWER_STATUS_USB_CONNECTED:
status += 'U'
if msg.flags & mavutil.mavlink.MAV_POWER_STATUS_BRICK_VALID:
status += 'B'
if msg.flags & mavutil.mavlink.MAV_POWER_STATUS_SERVO_VALID:
status += 'S'
if msg.flags & mavutil.mavlink.MAV_POWER_STATUS_PERIPH_OVERCURRENT:
status += 'O1'
if msg.flags & mavutil.mavlink.MAV_POWER_STATUS_PERIPH_HIPOWER_OVERCURRENT:
status += 'O2'
self.console.set_status('PWR', status, fg=fg)
self.console.set_status('Srv', 'Srv %.2f' % (msg.Vservo*0.001), fg='green')
elif type == 'HEARTBEAT':
fmode = master.flightmode
if self.settings.vehicle_name:
fmode = self.settings.vehicle_name + ':' + fmode
self.console.set_status('Mode', '%s' % fmode, fg='blue')
if len(self.vehicle_list) > 1:
self.console.set_status('SysID', 'Sys:%u' % msg.get_srcSystem(), fg='blue')
if self.master.motors_armed():
arm_colour = 'green'
else:
arm_colour = 'red'
armstring = 'ARM'
# add safety switch state
if 'SYS_STATUS' in self.mpstate.status.msgs:
if (self.mpstate.status.msgs['SYS_STATUS'].onboard_control_sensors_enabled & mavutil.mavlink.MAV_SYS_STATUS_SENSOR_MOTOR_OUTPUTS) == 0:
armstring += '(SAFE)'
self.console.set_status('ARM', armstring, fg=arm_colour)
if self.max_link_num != len(self.mpstate.mav_master):
for i in range(self.max_link_num):
self.console.set_status('Link%u'%(i+1), '', row=1)
self.max_link_num = len(self.mpstate.mav_master)
for m in self.mpstate.mav_master:
if self.mpstate.settings.checkdelay:
linkdelay = (self.mpstate.status.highest_msec - m.highest_msec)*1.0e-3
else:
linkdelay = 0
linkline = "Link %s " % (self.link_label(m))
fg = 'dark green'
if m.linkerror:
linkline += "down"
fg = 'red'
else:
packets_rcvd_percentage = 100
if (m.mav_count+m.mav_loss) != 0: #avoid divide-by-zero
packets_rcvd_percentage = (100.0 * m.mav_count) / (m.mav_count + m.mav_loss)
linkbits = ["%u pkts" % m.mav_count,
"%u lost" % m.mav_loss,
"%.2fs delay" % linkdelay,
]
try:
if m.mav.signing.sig_count:
# other end is sending us signed packets
if not m.mav.signing.secret_key:
# we've received signed packets but
# can't verify them
fg = 'orange'
linkbits.append("!KEY")
elif not m.mav.signing.sign_outgoing:
# we've received signed packets but aren't
# signing outselves; this can lead to hairloss
fg = 'orange'
linkbits.append("!SIGNING")
if m.mav.signing.badsig_count:
fg = 'orange'
linkbits.append("%u badsigs" % m.mav.signing.badsig_count)
except AttributeError as e:
# mav.signing.sig_count probably doesn't exist
pass
linkline += "OK {rcv_pct:.1f}% ({bits})".format(
rcv_pct=packets_rcvd_percentage,
bits=", ".join(linkbits))
if linkdelay > 1 and fg == 'dark green':
fg = 'orange'
self.console.set_status('Link%u'%m.linknum, linkline, row=1, fg=fg)
elif type in ['WAYPOINT_CURRENT', 'MISSION_CURRENT']:
wpmax = self.module('wp').wploader.count()
if wpmax > 0:
wpmax = "/%u" % wpmax
else:
wpmax = ""
self.console.set_status('WP', 'WP %u%s' % (msg.seq, wpmax))
lat = master.field('GLOBAL_POSITION_INT', 'lat', 0) * 1.0e-7
lng = master.field('GLOBAL_POSITION_INT', 'lon', 0) * 1.0e-7
if lat != 0 and lng != 0:
airspeed = master.field('VFR_HUD', 'airspeed', 30)
if abs(airspeed - self.speed) > 5:
self.speed = airspeed
else:
self.speed = 0.98*self.speed + 0.02*airspeed
self.speed = max(1, self.speed)
time_remaining = int(self.estimated_time_remaining(lat, lng, msg.seq, self.speed))
self.console.set_status('ETR', 'ETR %u:%02u' % (time_remaining/60, time_remaining%60))
elif type == 'NAV_CONTROLLER_OUTPUT':
self.console.set_status('WPDist', 'Distance %s' % self.dist_string(msg.wp_dist))
self.console.set_status('WPBearing', 'Bearing %u' % msg.target_bearing)
if msg.alt_error > 0:
alt_error_sign = "L"
else:
alt_error_sign = "H"
if msg.aspd_error > 0:
aspd_error_sign = "L"
else:
aspd_error_sign = "H"
if math.isnan(msg.alt_error):
alt_error = "NaN"
else:
alt_error = "%d%s" % (msg.alt_error, alt_error_sign)
self.console.set_status('AltError', 'AltError %s' % alt_error)
self.console.set_status('AspdError', 'AspdError %.1f%s' % (msg.aspd_error*0.01, aspd_error_sign))
def compare_fields(serializers, field, savepoint, dim_bounds):
""" If field is not None compare the field at each savepoint if field is None compare every
field at every savepoint (full archive comparison).
"""
serializer_1, serializer_2 = serializers
field_1 = serializer_1.read(field, savepoint)
field_2 = serializer_2.read(field, savepoint)
dims = serializer_1.get_field_metainfo(field).dims
if len(dims) > len(dim_bounds):
print(" Field dimension '{}' exceeds maximum of 4 dimension")
return False
slices = []
for i in range(0, len(dims)):
slices += [dim_bounds[i]]
# Get a view of a field incorporating the user defined slices
field_view_1 = field_1[slices]
field_view_2 = field_2[slices]
assert field_view_1.size == field_view_2.size
errors = []
num_nans = 0
max_abs_error = 0
max_rel_error = 0
tol = get_config().tol(field)
it_1 = np.nditer(field_view_1, flags=['multi_index'])
it_2 = np.nditer(field_view_2, flags=['multi_index'])
# Iterate the fields
while not it_1.finished and not it_2.finished:
value_1, value_2 = it_1[0], it_2[0]
value_1_isnan, value_2_isnan = isnan(value_1), isnan(value_2)
# Check for NaN values
num_nans += value_1_isnan + value_2_isnan
if value_1_isnan != value_2_isnan:
errors += [
{"index": it_1.multi_index, "value_1": value_1, "value_2": value_2,
"error": float('nan')}]
# Compute error
else:
abs_error = abs(value_2 - value_1)
rel_error = abs((value_2 - value_1) / value_2) if abs(value_2) > 1.0 else 0
err = rel_error if abs(value_2) > 1.0 else abs_error
# Check error
if err > tol:
errors += [
{"index": it_1.multi_index, "value_1": value_1, "value_2": value_2,
"error": err}]
max_abs_error = max(max_abs_error, abs_error)
max_rel_error = max(max_rel_error, rel_error)
it_1.iternext()
it_2.iternext()
# All good!
if len(errors) == 0:
return True
# Report the errors
num_errors = len(errors)
num_errors_displayed = min(get_config().MAX_ERRORS, num_errors)
if num_errors_displayed > 0:
print(" Failed values (displayed {} of {}):".format(num_errors_displayed, num_errors))
for idx in range(0, num_errors_displayed):
print(" {}: value_1 = {:.10f}, value_2 = {:.10f}, error = {:.10e}".format(
errors[idx]["index"], float(errors[idx]["value_1"]), float(errors[idx]["value_2"]),
float(errors[idx]["error"])))
print(" Number of errors: {}".format(num_errors))
print(" Number of NaN: {}".format(num_nans))
print(" Percentage of errors: {:.2f} %".format(100 * num_errors / field_view_1.size))
print(" Maximum absolute error: {:.10e}".format(max_abs_error))
print(" Maximum relative error: {:.10e}".format(max_rel_error))
return False
def test_convert_nan():
got = strong_json.from_json_dict({'__type__': 'float', '__data__': 'nan'})
assert math.isnan(got)
def overlay_ts(master_data, master_titles, region, colors, fileDir, output_name_ts2, labels, save):
# master_data = list of data_all# lists
dates = []
range_change = []
fig = plt.figure(figsize=(25, 10))
# Plot deformation map with selected region/pixels overlain
ax1 = plt.subplot(121)
# ax1 = plt.subplot(111)
# grid[0].set_axis_off()
im = ax1.imshow(master_data[0][-1], cmap=colors, vmin=-0.05, vmax=0.05, aspect=0.75)
ax1.plot([region[0], region[0], region[1], region[1], region[0]], [region[2], region[3], region[3], region[2], region[2]], color='blue', zorder=10000)
ax1.set_title(master_titles[0][-1], fontsize=12, color='black')
ax1.invert_yaxis()
ax1.invert_xaxis()
cbar = fig.colorbar(im)
# cbar.set_label('LOS change (m)')
# Plot time-series for selected pixels
ax2 = plt.subplot(122)
ax2.grid()
# Get averaged timeseries for given region
count = 0
for i in range(len(master_data)):
for j in range(len(master_data[i])):
nanCount = 0
total = []
numPix = 0
for x in range(region[0], region[1]):
for y in range(region[2], region[3]):
if math.isnan(master_data[i][j][y][x]):
nanCount+=1
else:
total.append(master_data[i][j][y][x])
numPix += 1
# print('nanCount = ' + str(nanCount))
# print('Count = ' + str(len(sum)))
# print('Mean value = ' + str(np.mean(sum)))
dates.append(dt.datetime.strptime(master_titles[i][j], "%Y%m%d"))
range_change.append(np.mean(total))
ax2.plot(dates, range_change, linestyle='--', marker='.', zorder=10)
count += 1
# Reset if not last iteration
if count < len(master_data):
dates = []
range_change = []
ax2.legend(labels, loc='lower right')
ax2.set_aspect(2.15 * 10**4)
ax2.set_xlim(min(dates) - dt.timedelta(days=30), max(dates) + dt.timedelta(days=30))
ax2.set_ylim(-0.01, 0.05)
plt.xlabel('Date')
plt.ylabel('LOS range change (m)', rotation=0, labelpad=58)
if save == 'yes':
print('Saving time series to ' + fileDir + output_name_ts2)
plt.savefig(fileDir + output_name_ts2)
plt.close()
subprocess.call("open " + fileDir + output_name_ts2, shell=True)
else:
plt.show()
def simulation_step(self):
"""execute the python simulation by one step"""
l = self.get_line()
f = self.get_file()
if f in self.breakpoints:
if l in self.breakpoints[f]:
raise BreakSim
instruction = self.instructions[self.program_counter]
current_stack = self.registers.get(register_map.tos, 0)
self.max_stack = max([current_stack, self.max_stack])
if self.profile:
trace = instruction.get("trace", "-")
lines = self.files.get(trace.filename, {})
lines[trace.lineno] = lines.get(trace.lineno, 0) + 1
self.files[trace.filename] = lines
literal = 0
if "literal" in instruction:
literal = instruction["literal"]
if "label" in instruction:
literal = instruction["label"]
# read operands
#
a = instruction.get("a", 0)
b = instruction.get("b", 0)
z = instruction.get("z", 0)
operand_b = self.registers.get(b, 0)
operand_a = self.registers.get(a, 0)
operand_z = self.registers.get(z, 0)
this_instruction = self.program_counter
self.program_counter += 1
wait = False
result = None
unsigned_literal = literal
#literal = interpret_twos_complement(unsigned_literal, 16)
signed_literal = interpret_twos_complement(unsigned_literal, 16)
if instruction["op"] == "addl":
literal = literal & 0xffff
# if literal < 0:
# print(instruction["op"])
if instruction["op"] == "stop":
self.program_counter = this_instruction
wait = True
for file_ in self.input_files.values():
file_.close()
for file_ in self.output_files.values():
file_.close()
raise StopSim
elif instruction["op"] == "literal":
if literal & 0x8000:
result = -65536 | literal
else:
result = literal
result &= 0xffffffff
elif instruction["op"] == "addl":
if literal & 0x8000:
sext = -65536 | literal
else:
sext = literal
result = sext + operand_a
result &= 0xffffffff
elif instruction["op"] == "literal_hi":
if literal & 0x8000:
sext = -65536 | literal
else:
sext = literal
result = (sext << 16) | (operand_a & 0x0000ffff)
result &= 0xffffffff
elif instruction["op"] == "store":
offset = interpret_twos_complement(literal, 14)
self.memory.store(operand_z + offset, operand_a)
elif instruction["op"] == "store8":
offset = interpret_twos_complement(literal, 14)
self.memory.store_8(operand_z + offset, operand_a)
elif instruction["op"] == "store16":
offset = interpret_twos_complement(literal, 14)
self.memory.store_16(operand_z + offset, operand_a)
elif instruction["op"] == "load":
result = self.memory.load(operand_a)
elif instruction["op"] == "load_disp":
offset = interpret_twos_complement(literal, 13)
result = self.memory.load(operand_a + offset)
elif instruction["op"] == "load8":
offset = interpret_twos_complement(literal, 13)
result = self.memory.load_8(operand_a + offset,
literal & 0x8000 != 0)
elif instruction["op"] == "load16":
offset = interpret_twos_complement(literal, 13)
result = self.memory.load_16(operand_a + offset,
literal & 0x8000 != 0)
elif instruction["op"] == "call":
result = this_instruction + 1
self.program_counter = literal
elif instruction["op"] == "return":
self.program_counter = operand_a
elif instruction["op"] == "a_lo":
result = self.a_lo
self.a_lo = operand_a
elif instruction["op"] == "b_lo":
result = self.b_lo
self.b_lo = operand_a
elif instruction["op"] == "a_hi":
result = self.a_hi
self.a_hi = operand_a
elif instruction["op"] == "b_hi":
result = self.b_hi
self.b_hi = operand_a
elif instruction["op"] == "a_lo_in":
self.a_lo = operand_a
elif instruction["op"] == "b_lo_in":
self.b_lo = operand_a
elif instruction["op"] == "a_hi_in":
self.a_hi = operand_a
elif instruction["op"] == "b_hi_in":
self.b_hi = operand_a
elif instruction["op"] == "a_lo_out":
result = self.a_lo
elif instruction["op"] == "b_lo_out":
result = self.b_lo
elif instruction["op"] == "a_hi_out":
result = self.a_hi
elif instruction["op"] == "b_hi_out":
result = self.b_hi
elif instruction["op"] == "not":
result = (~operand_a) & 0xffffffff
elif instruction["op"] == "int_to_long":
if operand_a & 0x80000000:
result = 0xffffffff
else:
result = 0
elif instruction["op"] == "int_to_float":
f = float(to_32_signed(self.a_lo))
self.a_lo = float_to_bits(f)
elif instruction["op"] == "float_to_int":
i = bits_to_float(self.a_lo)
if math.isnan(i):
self.a_lo = 0
else:
self.a_lo = int(i) & 0xffffffff
elif instruction["op"] == "long_to_double":
double = float(to_64_signed(chips_c.join_words(self.a_hi, self.a_lo)))
if math.isnan(double):
self.a_hi = 0
self.a_lo = 0
else:
self.a_hi = chips_c.high_word(double_to_bits(double))
self.a_lo = chips_c.low_word(double_to_bits(double))
elif instruction["op"] == "double_to_long":
bits = int(bits_to_double(chips_c.join_words(self.a_hi, self.a_lo)))
bits &= 0xffffffffffffffff
self.a_hi = chips_c.high_word(bits)
self.a_lo = chips_c.low_word(bits)
elif instruction["op"] == "float_to_double":
f = bits_to_float(self.a_lo)
bits = double_to_bits(f)
self.a_hi = chips_c.high_word(bits)
self.a_lo = chips_c.low_word(bits)
elif instruction["op"] == "double_to_float":
f = bits_to_double(chips_c.join_words(self.a_hi, self.a_lo))
self.a_lo = float_to_bits(f)
elif instruction["op"] == "add":
total = add(operand_a, operand_b, 0);
result = total.lo
self.carry = total.hi
elif instruction["op"] == "add_with_carry":
total = add(operand_a, operand_b, self.carry);
result = total.lo
self.carry = total.hi
elif instruction["op"] == "subtract":
total = subtract(operand_a, operand_b, 1);
result = total.lo
self.carry = total.hi
elif instruction["op"] == "subtract_with_carry":
total = subtract(operand_a, operand_b, self.carry);
result = total.lo
self.carry = total.hi
elif instruction["op"] == "multiply":
lw = operand_a * operand_b
self.carry = chips_c.high_word(lw)
result = chips_c.low_word(lw)
elif instruction["op"] == "divide":
a = operand_a
b = operand_b
result = chips_c.divide(a, b)
elif instruction["op"] == "unsigned_divide":
a = operand_a
b = operand_b
result = chips_c.unsigned_divide(a, b)
elif instruction["op"] == "modulo":
a = operand_a
b = operand_b
result = chips_c.modulo(a, b)
elif instruction["op"] == "unsigned_modulo":
a = operand_a
b = operand_b
result = chips_c.unsigned_modulo(a, b)
elif instruction["op"] == "long_divide":
a = chips_c.join_words(self.a_hi, self.a_lo)
b = chips_c.join_words(self.b_hi, self.b_lo)
quotient = chips_c.long_divide(a, b)
self.a_hi = chips_c.high_word(quotient)
self.a_lo = chips_c.low_word(quotient)
elif instruction["op"] == "long_modulo":
a = chips_c.join_words(self.a_hi, self.a_lo)
b = chips_c.join_words(self.b_hi, self.b_lo)
remainder = chips_c.long_modulo(a, b)
self.a_hi = chips_c.high_word(remainder)
self.a_lo = chips_c.low_word(remainder)
elif instruction["op"] == "unsigned_long_divide":
a = chips_c.join_words(self.a_hi, self.a_lo)
b = chips_c.join_words(self.b_hi, self.b_lo)
quotient = chips_c.unsigned_long_divide(a, b)
self.a_hi = chips_c.high_word(quotient)
self.a_lo = chips_c.low_word(quotient)
elif instruction["op"] == "unsigned_long_modulo":
a = chips_c.join_words(self.a_hi, self.a_lo)
b = chips_c.join_words(self.b_hi, self.b_lo)
remainder = chips_c.unsigned_long_modulo(a, b)
self.a_hi = chips_c.high_word(remainder)
self.a_lo = chips_c.low_word(remainder)
elif instruction["op"] == "carry":
result = self.carry
elif instruction["op"] == "or":
result = operand_a | operand_b
elif instruction["op"] == "and":
result = operand_a & operand_b
elif instruction["op"] == "xor":
result = operand_a ^ operand_b
elif instruction["op"] == "shift_left":
total = shift_left(operand_a, operand_b, 0)
result = total.lo
self.carry = total.hi
elif instruction["op"] == "shift_left_with_carry":
total = shift_left(operand_a, operand_b, self.carry)
result = total.lo
self.carry = total.hi
elif instruction["op"] == "shift_right":
total = shift_right(operand_a, operand_b)
result = total.lo
self.carry = total.hi
elif instruction["op"] == "unsigned_shift_right":
total = unsigned_shift_right(operand_a, operand_b, 0)
result = total.lo
self.carry = total.hi
elif instruction["op"] == "shift_right_with_carry":
total = unsigned_shift_right(operand_a, operand_b, self.carry)
result = total.lo
self.carry = total.hi
elif instruction["op"] == "greater":
result = greater(operand_a, operand_b)
elif instruction["op"] == "greater_equal":
result = greater_equal(operand_a, operand_b)
elif instruction["op"] == "unsigned_greater":
result = unsigned_greater(operand_a, operand_b)
elif instruction["op"] == "unsigned_greater_equal":
result = unsigned_greater_equal(operand_a, operand_b)
elif instruction["op"] == "equal":
result = operand_a == operand_b
elif instruction["op"] == "not_equal":
result = operand_a != operand_b
elif instruction["op"] == "jmp_if_false":
if operand_a == 0:
self.program_counter = literal
elif instruction["op"] == "jmp_if_true":
if operand_a != 0:
self.program_counter = literal
elif instruction["op"] == "goto":
self.program_counter = literal
elif instruction["op"] == "timer_low":
result = self.clock&0xffffffff
elif instruction["op"] == "timer_high":
result = self.clock>>32
elif instruction["op"] == "file_read":
value = self.input_files[instruction["filename"]].getline()
result = value
elif instruction["op"] == "float_file_write":
self.output_files[instruction["file_name"]].write(
"%.7f\n" %
bits_to_float(operand_a))
elif instruction["op"] == "unsigned_file_write":
self.output_files[instruction["file_name"]].write(
"%i\n" %
operand_a)
elif instruction["op"] == "file_write":
self.output_files[instruction["file_name"]].write(
"%i\n" %
to_32_signed(operand_a))
elif instruction["op"] == "read":
if operand_a not in self.inputs:
result = 0
else:
input_ = self.inputs[operand_a]
if input_.src_rdy and input_.dst_rdy:
result = input_.q
input_.next_dst_rdy = False
else:
input_.next_dst_rdy = True
wait = True
elif instruction["op"] == "ready":
if operand_a not in self.inputs:
operand_a = 0
else:
input_ = self.inputs[operand_a]
if input_.src_rdy:
result = 1
else:
result = 0
elif instruction["op"] == "output_ready":
if operand_a not in self.outputs:
operand_a = 0
else:
output_ = self.outputs[operand_a]
if output_.dst_rdy:
result = 1
else:
result = 0
elif instruction["op"] == "write":
if operand_a not in self.outputs:
pass
else:
output_ = self.outputs[operand_a]
if output_.src_rdy and output_.dst_rdy:
output_.next_src_rdy = False
else:
output_.q = operand_b
output_.next_src_rdy = True
wait = True
elif instruction["op"] == "float_add":
a = operand_a
b = operand_b
float_ = bits_to_float(a)
floatb = bits_to_float(b)
result = float_to_bits(float_ + floatb)
elif instruction["op"] == "float_subtract":
a = operand_a
b = operand_b
float_ = bits_to_float(a)
floatb = bits_to_float(b)
result = float_to_bits(float_ - floatb)
elif instruction["op"] == "float_multiply":
a = operand_a
b = operand_b
float_ = bits_to_float(a)
floatb = bits_to_float(b)
result = float_to_bits(float_ * floatb)
elif instruction["op"] == "float_divide":
a = operand_a
b = operand_b
float_ = bits_to_float(a)
floatb = bits_to_float(b)
try:
result = float_to_bits(float_ / floatb)
except ZeroDivisionError:
result = float_to_bits(float("nan"))
elif instruction["op"] == "long_float_add":
double = bits_to_double(chips_c.join_words(self.a_hi, self.a_lo))
doubleb = bits_to_double(chips_c.join_words(self.b_hi, self.b_lo))
self.a_hi = chips_c.high_word(double_to_bits(double + doubleb))
self.a_lo = chips_c.low_word(double_to_bits(double + doubleb))
elif instruction["op"] == "long_float_subtract":
double = bits_to_double(chips_c.join_words(self.a_hi, self.a_lo))
doubleb = bits_to_double(chips_c.join_words(self.b_hi, self.b_lo))
self.a_hi = chips_c.high_word(double_to_bits(double - doubleb))
self.a_lo = chips_c.low_word(double_to_bits(double - doubleb))
elif instruction["op"] == "long_float_multiply":
double = bits_to_double(chips_c.join_words(self.a_hi, self.a_lo))
doubleb = bits_to_double(chips_c.join_words(self.b_hi, self.b_lo))
self.a_hi = chips_c.high_word(double_to_bits(double * doubleb))
self.a_lo = chips_c.low_word(double_to_bits(double * doubleb))
elif instruction["op"] == "long_float_divide":
double = bits_to_double(chips_c.join_words(self.a_hi, self.a_lo))
doubleb = bits_to_double(chips_c.join_words(self.b_hi, self.b_lo))
try:
self.a_hi = chips_c.high_word(double_to_bits(double / doubleb))
self.a_lo = chips_c.low_word(double_to_bits(double / doubleb))
except ZeroDivisionError:
self.a_hi = chips_c.high_word(double_to_bits(float("nan")))
self.a_lo = chips_c.low_word(double_to_bits(float("nan")))
elif instruction["op"] == "long_float_file_write":
long_word = chips_c.join_words(self.a_hi, self.a_lo)
self.output_files[instruction["file_name"]].write(
"%.16f\n" %
bits_to_double(long_word))
elif instruction["op"] == "long_file_write":
long_word = chips_c.join_words(self.a_hi, self.a_lo)
self.output_files[instruction["file_name"]].write(
"%f\n" %
long_word)
elif instruction["op"] == "assert":
if operand_a == 0:
raise ChipsAssertionFail(
instruction["file"],
instruction["line"])
elif instruction["op"] == "report":
print "%d (report (int) at line: %s in file: %s)" % (
to_32_signed(self.a_lo),
instruction["line"],
instruction["file"],
)
elif instruction["op"] == "long_report":
print "%d (report (long) at line: %s in file: %s)" % (
to_64_signed(chips_c.join_words(self.a_hi, self.a_lo)),
instruction["line"],
instruction["file"],
)
elif instruction["op"] == "float_report":
print "%f (report (float) at line: %s in file: %s)" % (
bits_to_float(self.a_lo),
instruction["line"],
instruction["file"],
)
elif instruction["op"] == "long_float_report":
print "%s (report (double) at line: %s in file: %s)" % (
bits_to_double(chips_c.join_words(self.a_hi, self.a_lo)),
instruction["line"],
instruction["file"],
)
elif instruction["op"] == "unsigned_report":
print "%d (report (unsigned) at line: %s in file: %s)" % (
self.a_lo,
instruction["line"],
instruction["file"],
)
elif instruction["op"] == "long_unsigned_report":
print "%d (report (unsigned long) at line: %s in file: %s)" % (
chips_c.join_words(self.a_hi, self.a_lo),
instruction["line"],
instruction["file"],
)
elif instruction["op"] == "wait_clocks":
if self.timer == operand_a:
wait = False
self.timer = 0
else:
wait = True
self.timer += 1
else:
print "Unknown machine instruction", instruction["op"]
sys.exit(-1)
# Write data back
if result is not None:
self.registers[z] = result
# manipulate stack pointer
if wait:
self.program_counter = this_instruction
self.clock += 1
read other books, number of books read, distance, computer use, library use,
like the school or not, give math HW, help in studies,
private tuition, Private tuition, Maths is difficult, Solve Maths, Solve Maths in groups,
Draw geometry, Explain answers, Watch TV, Play games, Help in household
'''
count_1 = 0
marks_1 = 0.00
count_2 = 0
marks_2 = 0.00
count_3 = 0
marks_3 = 0.00
count_4 = 0
marks_4 = 0.00
for i in range(0, len(dataframe)):
if (math.isnan(dataframe.iloc[i]['Maths %'])) == False:
if dataframe.iloc[i]['# Books'] == 1:
count_1 = count_1 + 1
marks_1 = marks_1 + dataframe.iloc[i]['Maths %']
if dataframe.iloc[i]['# Books'] == 2:
count_2 = count_2 + 1
marks_2 = marks_2 + dataframe.iloc[i]['Maths %']
if dataframe.iloc[i]['# Books'] == 3:
count_3 = count_3 + 1
marks_3 = marks_3 + dataframe.iloc[i]['Maths %']
if dataframe.iloc[i]['# Books'] == 4:
count_4 = count_4 + 1
marks_4 = marks_4 + dataframe.iloc[i]['Maths %']
print 'Average Percentage of Maths Marks of # Books 1 (No books)- ', ((
marks_1 / count_1))
def is_var_nan(row, col):
return isinstance(sheet.iloc[row, col], float) and math.isnan(
sheet.iloc[row, col])
def text_visual(texts,
scores,
img_h=400,
img_w=600,
threshold=0.,
font_path="./doc/simfang.ttf"):
"""
create new blank img and draw txt on it
args:
texts(list): the text will be draw
scores(list|None): corresponding score of each txt
img_h(int): the height of blank img
img_w(int): the width of blank img
font_path: the path of font which is used to draw text
return(array):
"""
if scores is not None:
assert len(texts) == len(
scores), "The number of txts and corresponding scores must match"
def create_blank_img():
blank_img = np.ones(shape=[img_h, img_w], dtype=np.int8) * 255
blank_img[:, img_w - 1:] = 0
blank_img = Image.fromarray(blank_img).convert("RGB")
draw_txt = ImageDraw.Draw(blank_img)
return blank_img, draw_txt
blank_img, draw_txt = create_blank_img()
font_size = 20
txt_color = (0, 0, 0)
font = ImageFont.truetype(font_path, font_size, encoding="utf-8")
gap = font_size + 5
txt_img_list = []
count, index = 1, 0
for idx, txt in enumerate(texts):
index += 1
if scores[idx] < threshold or math.isnan(scores[idx]):
index -= 1
continue
first_line = True
while str_count(txt) >= img_w // font_size - 4:
tmp = txt
txt = tmp[:img_w // font_size - 4]
if first_line:
new_txt = str(index) + ': ' + txt
first_line = False
else:
new_txt = ' ' + txt
draw_txt.text((0, gap * count), new_txt, txt_color, font=font)
txt = tmp[img_w // font_size - 4:]
if count >= img_h // gap - 1:
txt_img_list.append(np.array(blank_img))
blank_img, draw_txt = create_blank_img()
count = 0
count += 1
if first_line:
new_txt = str(index) + ': ' + txt + ' ' + '%.3f' % (scores[idx])
else:
new_txt = " " + txt + " " + '%.3f' % (scores[idx])
draw_txt.text((0, gap * count), new_txt, txt_color, font=font)
# whether add new blank img or not
if count >= img_h // gap - 1 and idx + 1 < len(texts):
txt_img_list.append(np.array(blank_img))
blank_img, draw_txt = create_blank_img()
count = 0
count += 1
txt_img_list.append(np.array(blank_img))
if len(txt_img_list) == 1:
blank_img = np.array(txt_img_list[0])
else:
blank_img = np.concatenate(txt_img_list, axis=1)
return np.array(blank_img)
def train_one_epoch(self, epoch, train_loader):
self.model.train()
batch_steps = len(train_loader)
step_loss = AverageMeter()
for step, (_, batch) in enumerate(train_loader):
if self.ngpu > 0:
batch = map_to_cuda(batch)
start = time.process_time()
loss = self.model(**batch)
loss = torch.mean(loss) / self.accum_steps
if self.mixed_precision:
import apex.amp as amp
with amp.scale_loss(loss,
self.optimizer.optimizer) as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
if self.grad_noise:
raise NotImplementedError
if self.get_rank() == 0:
step_loss.update(loss.item() * self.accum_steps,
batch['inputs'].size(0))
if step % self.accum_steps == 0:
if self.local_rank == 0:
self.mean_loss.update(step_loss.avg)
grad_norm = torch.nn.utils.clip_grad_norm_(
self.model.parameters(), self.grad_clip)
if math.isnan(grad_norm):
self.logger.warning(
'Grad norm is NAN. DO NOT UPDATE MODEL!')
else:
self.optimizer.step()
self.optimizer.zero_grad()
if self.is_visual and self.local_rank == 0:
self.visulizer.add_scalar('train_loss', loss.item(),
self.global_step)
self.visulizer.add_scalar('lr', self.optimizer.lr,
self.global_step)
if self.global_step % self.log_interval == 0 and self.local_rank == 0:
end = time.process_time()
# process = step * self.world_size / batch_steps * 100
process = step / batch_steps * 100
self.logger.info(
'-Training-Epoch-%d(%.5f%%), Global Step:%d, lr:%.8f, Loss:%.5f, AvgLoss: %.5f, '
'Run Time:%.3f' %
(epoch, process, self.global_step, self.optimizer.lr,
step_loss.avg, self.mean_loss.mean(), end - start))
self.global_step += 1
step_loss.reset()
return self.mean_loss.mean()
for col in df.columns:
for offset, regex in ((0, r'\d+(\.\d+)?'), (1, r'\d+\.\d+\.\d+')):
# find label and ensure all labels are strings
label = df[col][offset]
if isinstance(label, float):
label = str(label)
# pick out those lines that have data for table
if isinstance(label, str) and re.fullmatch(regex, label):
name = df[col][offset + 1]
data = toYear(df[col][4:4+ndec])
# fix instances of line wrap in 4.3 5.1 5.6 9.3 9.4.1 9.4.3 9.4.5 10.2
if math.isnan(data[0]):
name += ' ' + df[col + 1][offset + 1]
data = toYear(df[col + 1][4:4+ndec])
name = re.sub(r' +', r' ', name) # one item has two spaces
name = re.sub(r'\d+$', r'', name) # two items have footnotes
if label in exclItem or (re.match('1[34]', label) and label not in inclItem):
notable[label] = name
continue
parent = re.sub(r'\.\d+$', r'', label)
if parent != label:
if parent in table:
table[parent].isLeaf = False
table[label] = TabItem(name, data, parent)
else:
})
df_counts_2019["year"] = "2019"
df_counts = df_counts_2018.append(df_counts_2019)
df_counts["percent"] = df_counts["percent"] * 100.0
df_counts["answer"] = df_counts["answer"].replace({"yes": "Yes", "no": "No"})
# Create the plot
fig, ax = plt.subplots(figsize=(10, 8))
ax = sns.barplot(x="answer", y="percent", hue="year", data=df_counts)
plt.title(
"Percentage of households in the host community\nwith agricultural land in 2018 and 2019",
fontsize=18)
plt.legend(fontsize=16, loc="upper right")
plt.xlabel(None)
plt.ylabel("Percent of households (%)", fontsize=16)
plt.ylim([0, 100])
plt.xticks(rotation=0, fontsize=14)
# Add percentages to the bars
for p in ax.patches:
width = p.get_width()
height = p.get_height() if not math.isnan(p.get_height()) else 0.0
x, y = p.get_xy()
ax.annotate('{:.0%}'.format(round(height) / 100.0),
(x + width / 2, y + height + 2),
ha='center',
fontsize=14)
plt.tight_layout()
plt.savefig("agricultural_land.png")
plt.close()
for ifeature in drop_feature:
data = data.drop(labels=ifeature, axis=1)
stdscale = StandardScaler()
elements = [
'Iron, Fe', 'Carbon, C', 'Sulfur, S', 'Silicon, Si', 'Phosphorous, P',
'Manganese, Mn', 'Chromium, Cr', 'Nickel, Ni', 'Molybdenum, Mo',
'Copper, Cu'
]
target = 'Thermal Conductivity'
# drop instances with NaN
drop_instance = []
for idx in data.index:
if math.isnan(data.loc[idx, target]):
drop_instance.append(idx)
data_loc = data.drop(drop_instance)
X = data_loc[elements].values
y = data_loc[target].values
print('Shape of dataset:')
print(X.shape)
# standardize features
X = stdscale.fit_transform(X)
y = stdscale.fit_transform(y.reshape(-1, 1))
#X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.75, test_size=0.25, random_state=0)
train_sizes, train_scores, valid_scores = learning_curve(
