def render(self, scene):
""" Render image from stuff inside scene """
width = scene.width
height = scene.height
ar = width / height
camera = scene.camera
im = Image(width, height)
xmin = -1
xmax = 1
ymax = xmax / ar
ymin = -ymax
dx = (xmax - xmin) / (width - 1)
dy = (ymax - ymin) / (height - 1)
pb = ProgressBar(height * width)
for j in range(height):
y = ymin + dy * j
for i in range(width):
x = xmin + dx * i
ray = Ray(camera, Vector(x, y, 0) - camera)
c = self.ray_trace(ray, scene)
im.set_pixel(j, i, c)
pb.update(1)
return im
def __init__(self, parent=None):
super().__init__(parent=parent)
self.nextEnabled = False
title = QLabel("Programming Bootloader")
switchesLabel = QLabel(
"Set the MX2+ DIP switches for bootloader programming as shown below.\nThen power-cycle the SmartDrive."
)
switchesLabel.setWordWrap(True)
self.pixMap = QtGui.QPixmap(
resource.path('images/bootloaderProgramming.jpg'))
self.progressBar = ProgressBar()
self.startButton = QPushButton("Start")
self.startButton.clicked.connect(self.onStart)
self.startButton.show()
self.stopButton = QPushButton("Stop")
self.stopButton.clicked.connect(self.onStop)
self.stopButton.hide()
self.labels = [
title, switchesLabel, self.progressBar, self.startButton,
self.stopButton
]
self.picture = QLabel(self)
self.picture.setPixmap(
self.pixMap.scaled(self.getPictureSize(), Qt.KeepAspectRatio))
self.layout.addWidget(title)
self.layout.addWidget(switchesLabel)
self.layout.addWidget(self.picture)
self.layout.addWidget(self.progressBar)
self.layout.addWidget(self.startButton)
self.layout.addWidget(self.stopButton)
def lightmap(nm, cm, light, ambient):
width, height = nm.size
cwidth, cheight = cm.size
if cwidth != width or cheight != height:
print("Resolutions do not match.")
exit()
shaded = Image.new('RGBA', (width, height), color=(0, 0, 0, 0))
prog = ProgressBar(height)
print("\nLight Map:")
for y in range(0, height):
prog.show(y)
for x in range(0, width):
(cr, cg, cb, ca) = cm.getpixel((x, y))
(r, g, b, a) = nm.getpixel((x, y))
if a != 0:
normal = vmath.Vector3(\
(r/255 - 0.5) * 2,
(g/255 - 0.5) * 2,
(b/255 - 0.5) * 2)
angle = normal.angle(light, 'deg')
illumination = 1 - (angle / 180)
illumination = ambient + (1 - ambient) * illumination
cr = int(illumination * cr)
cg = int(illumination * cg)
cb = int(illumination * cb)
shaded.putpixel((x, y), (cr, cg, cb, ca))
return shaded
def trim_samples(samples,nsamp,pbar=False):
weights = np.array([s.w for s in samples])
weights /= max(weights)
neff = np.sum(weights)
n = weights.size
print "effective number of samples: " , neff, "/", n
# Now trim off the ones that are too small
ntarget = sum([ w if w<1.0/n else 1 for w in weights]) + 0.0
if nsamp>0 and nsamp<ntarget:
weights *= nsamp/neff
else:
weights *= n
if pbar: progress_bar = ProgressBar(samples.size,message="trimming samples ")
else: print "trimming samples"
trimmed_samples = []
for w,s in zip(weights,samples):
if rand() < w:
s.w = max(1.0,w)
trimmed_samples.append(s)
if pbar: progress_bar()
trimmed_samples = np.array(trimmed_samples)
print "Samples trimmed from " , n, " to ", trimmed_samples.size
return trimmed_samples
def gen_var_result_scores(clf, X_train, X_test, y_train=None,
n_rounds=100, verbose=False, fit=True):
"""
Given classifier and training data, return variable importances on the
test data on a per-sample basis. UPDATED.
Result scores represent the difference between the observed score and
the mean score obtained if the specific variable is resampled from the
training data randomly.
"""
if fit:
if verbose:
print 'Training model...'
sys.stdout.flush()
t0 = time.time()
clf.fit(X_train, y_train)
if verbose:
t1 = time.time()
print 'Training took %.2f seconds' % (t1 - t0)
real_scores = clf.predict_proba(X_test)[:, 1]
result_scores = np.zeros(X_test.shape)
if verbose:
pb = ProgressBar()
progress = 0
for var in range(X_train.shape[1]):
single_var_scores = np.zeros([X_test.shape[0], n_rounds])
X_test_mod = np.copy(X_test)
for j in range(n_rounds):
if verbose:
progress += 1
pb.update_progress(progress / float(n_rounds * X_train.shape[1]))
X_test_mod[:, var] = np.random.choice(X_train[:, var], X_test.shape[0], replace=True)
single_var_scores[:, j] = clf.predict_proba(X_test_mod)[:, 1]
result_scores[:, var] = np.abs(real_scores - np.mean(single_var_scores, axis=1))
return result_scores
def cmd_dump(logger, args):
"""
Dump recorded values into a file or to stdout.
"""
if logger.is_busy():
print("Logger is currently recording.")
return 1
config = logger.configuration()
if config["count"] == 0:
print("No records available (nothing has been logged).")
return 0
output = _open_output(args, config)
progress = None if args.no_progress else ProgressBar(config['count'])
if args.data_format:
data_format = args.data_format
else:
data_format = "%c;%d;%t"
if config["humidity"]:
data_format += ";%h"
counter = 0
for values in logger:
for value in values:
stamp, temp, hum = value
record = _format_record(data_format, counter,
stamp.strftime(args.time_format), temp,
hum if config["humidity"] else None)
print(record, file=output)
counter += 1
if progress is not None:
progress += len(values)
if progress is not None:
print()
return 0
def run_ptmcmc_save(sampler, startPos, nSteps, file, **kwargs):
'''runs PT MCMC and saves zero temperature chain to file'''
if not os.path.exists(file):
f = open(file, "w")
f.close()
iStep = 0
bar = ProgressBar()
for pos, prob, like in sampler.sample(startPos,
iterations=nSteps,
storechain=True,
**kwargs):
bar.render(int(100 * iStep / nSteps), 'running MCMC')
iStep += 1
f = open(file, "a")
# pos is shape (ntemps, nwalkers, npars)
# prob is shape (ntemps, nwalkers)
# loop over all walkers for zero temp and append to file
zpos = pos[0, ...]
zprob = prob[0, ...]
for k in range(zpos.shape[0]):
thisPos = zpos[k]
thisProb = zprob[k]
f.write("{0:4d} {1:s} {2:f}\n".format(k,
" ".join(map(str, thisPos)),
thisProb))
f.close()
return sampler
def __init__(self, parent=None):
super().__init__(parent=parent)
self.nextEnabled = False
title = QLabel("Programming SmartDrive Bluetooth")
self.progressBar = ProgressBar()
self.startButton = QPushButton("Start")
self.startButton.clicked.connect(self.onStart)
self.startButton.show()
self.stopButton = QPushButton("Stop")
self.stopButton.clicked.connect(self.onStop)
self.stopButton.hide()
label = QLabel("Will now automatically program SmartDrive Bluetooth.")
label.setWordWrap(True)
self.sn = QLabel("S/N: ")
self.lk = QLabel("License: ")
self.addr = QLabel("Address: ")
self.labels = [
title, label, self.sn, self.lk, self.addr, self.progressBar,
self.startButton, self.stopButton
]
self.layout.addWidget(title)
self.layout.addWidget(label)
self.layout.addWidget(self.sn)
self.layout.addWidget(self.lk)
self.layout.addWidget(self.addr)
self.layout.addWidget(self.progressBar)
self.layout.addWidget(self.startButton)
self.layout.addWidget(self.stopButton)
def run_mcmc_save(sampler, startPos, nSteps, rState, file, **kwargs):
'''runs and MCMC chain with emcee, and saves steps to a file'''
#open chain save file
if file:
f = open(file, "w")
f.close()
iStep = 0
bar = ProgressBar()
for pos, prob, state in sampler.sample(startPos,
iterations=nSteps,
rstate0=rState,
storechain=True,
**kwargs):
if file:
f = open(file, "a")
bar.render(int(100 * iStep / nSteps), 'running MCMC')
iStep += 1
for k in range(pos.shape[0]):
# loop over all walkers and append to file
thisPos = pos[k]
thisProb = prob[k]
if file:
f.write("{0:4d} {1:s} {2:f}\n".format(
k, " ".join(map(str, thisPos)), thisProb))
if file:
f.close()
return sampler
def colormap(hm, gradient, smoothness):
width, height = hm.size
img = Image.new('RGBA', (width, height), color=(0, 0, 0, 0))
colorlist = []
for g in range(len(gradient) - 1):
(start, color) = gradient[g]
(nextStart, nextColor) = gradient[g + 1]
begin = start * smoothness
end = (nextStart - start) * smoothness
sublist = list(Color(color).range_to(Color(nextColor), end))
colorlist += sublist
prog = ProgressBar(height)
print("\nColor Map:")
for y in range(0, height):
prog.show(y)
for x in range(0, width):
value = hm.getpixel((x, y))
if value != -2147483648:
setpixel(x, y, value, colorlist, img)
return img
def partial_plot(clf,
X_,
x_name,
labels,
n_points=100,
lims=None,
n_samp=1000,
categorical=False):
X = X_.copy()
N = len(X)
if lims == None:
x_min = X[x_name].min()
x_max = X[x_name].max()
else:
x_min = lims[0]
x_max = lims[1]
if categorical:
x = np.array([x_min, x_max] * int(n_points / 2.))
else:
x = np.linspace(x_min, x_max, n_points)
p = []
pb = ProgressBar()
for i, x_i in enumerate(x):
X[x_name] = [x_i] * N
_idx = np.random.randint(
N, size=n_samp) #sub sample to reduce time to evaluate
p.append(
clf.predict_proba(X.values[_idx], labels=labels[_idx])[1].mean(0))
pb.update_progress(i / n_points)
return x, np.array(p)
def cal_rows(self, css, scene, h, dh):
width = scene.width
height = scene.height
ar = width / height
camera = scene.camera
xmin = -1
xmax = 1
ymax = xmax / ar
ymin = -ymax
dx = (xmax - xmin) / (width - 1)
dy = (ymax - ymin) / (height - 1)
cs = [] # 3 * width * (dh)
pb = ProgressBar((dh) * width)
for j in range(h, h + dh):
y = ymin + dy * j
for i in range(width):
x = xmin + dx * i
ray = Ray(camera, Vector(x, y, 0) - camera)
c = self.ray_trace(ray, scene)
cs.append(c.x)
cs.append(c.y)
cs.append(c.z)
pb.update(1)
a = h * width * 3
b = (h + dh) * width * 3
css[a:b] = cs
def run_burnin(sampler, startPos, nSteps, storechain=False):
iStep = 0
bar = ProgressBar()
for pos, prob, state in sampler.sample(startPos,
iterations=nSteps,
storechain=storechain):
bar.render(int(100 * iStep / nSteps), 'running Burn In')
iStep += 1
return pos, prob, state
def compute_kernels(slices,weights,pbar=False):
if pbar: progress_bar = ProgressBar(slices.size,message="computing kernels")
else: print "computing kernels"
kernels = []
for s in slices:
kernels.append(gaussian_kde(s,weights=weights))
if pbar: progress_bar()
return np.array(kernels)
def compute_slices(fsamples,xs,pbar=False):
if pbar: progress_bar = ProgressBar(fsamples.size,message="computing slices ")
else: print "computing slices"
slices = []
for f in fsamples:
slices.append(f(xs))
if pbar: progress_bar()
return np.array(slices).T # return transpose
def rebuild_random(magic, data):
"""Create a random.Random() object from the data and the magic vectors
:param list[str] magic: magic vector data
:param str data: observed output from Mersenne Twister
:rtype: random.Random
"""
progress = ProgressBar()
data_vals = [ord(d) for d in data]
state = [0L for _ in xrange(N)]
def dropsubj_run():
pb = ProgressBar(len(PERCENT_REMOVE))
for p in PERCENT_REMOVE:
this_rand = lambda d: replace_percent_subjects(d, p)
cleaner = lambda d: remove_percent_deviant_subjects(d, p)
params = {'p': p, 'cleaner': 'devsubj'}
for row in run_experiment(this_rand, [cleaner], [params]):
yield row
pb.incr_and_errput()
def __init__(self, filename):
dfile = filename.split('.')[0] + '.pkl'
try:
dump = self.loadfile(dfile)
self.bag_of_word = dump['bag_of_word']
self.bag_of_index = dump['bag_of_index']
self.markov_matrix = np.array(dump['markov_matrix'])
except OSError as e:
self.bag_of_word = {}
self.bag_of_index = {}
self.markov_matrix = np.array([[0]])
index = 0
try:
with open(filename, 'r') as file:
print(
"log: this would show up if its a new file. A markov chain will be created and saved"
)
low = (' '.join(file.read().splitlines())).split(' ')
progress = ProgressBar(len(low) - 1, fmt=ProgressBar.FULL)
for i in range(progress.total):
progress.current += 1
progress()
if self.bag_of_word.setdefault(low[i], index) == index:
self.bag_of_index[index] = low[i]
self.markov_matrix = np.pad(self.markov_matrix,
[(0, self.max(index)),
(0, self.max(index))],
mode='constant')
index += 1
if self.bag_of_word.setdefault(low[i + 1],
index) == index:
self.bag_of_index[index] = low[i + 1]
self.markov_matrix = np.pad(self.markov_matrix,
[(0, self.max(index)),
(0, self.max(index))],
mode='constant')
index += 1
self.markov_matrix[self.bag_of_word[low[i]]][
self.bag_of_word[low[i + 1]]] += 1
progress.done()
s = np.sum(self.markov_matrix, axis=1)[:, np.newaxis]
s[s == 0] = 1
self.markov_matrix = self.markov_matrix / s
self.markov_matrix[self.markov_matrix.shape[0] - 1][0] = 1
dump = {}
dump['bag_of_word'] = self.bag_of_word
dump['bag_of_index'] = self.bag_of_index
dump['markov_matrix'] = self.markov_matrix
self.savefile(dump, dfile)
del (dump)
print("log:chain for ", filename, " is created")
except OSError as e:
print("file not found !")
exit()
def __init__(self, url, gitlab, includes=[], excludes=[], concurrency=1, in_file=None, method="http"):
self.in_file = in_file
self.method = method
self.concurrency = concurrency
self.excludes = excludes
self.includes = includes
self.url = url
self.gitlab = gitlab
self.root = Node("", root_path="", url=url)
self.disable_progress = False
self.progress = ProgressBar('* loading tree', self.disable_progress)
def compute_masses(kernels,y,pbar=False):
if pbar: progress_bar = ProgressBar(kernels.size,message="computing masses ")
else: print "computing masses"
masses = []
for k in kernels:
masses.append( compute_pmf(y,k) ) # compute M(x,y) for each value
if pbar: progress_bar()
return np.array(masses).T # return the transpose
def learn(self, samples, epochs=25000, noise=0, test_samples=None, show_progress=True):
''' Learn given distribution using n data
:Parameters:
`samples` : [numpy array, ...]
List of sample sets
`epochs` : [int, ...]
Number of epochs to be ran for each sample set
'''
# Check if samples is a list
if type(samples) not in [tuple,list]:
samples = (samples,)
epochs = (epochs,)
n = 0 # total number of epochs to be ran
for j in range(len(samples)):
n += epochs[j]
self.entropy = []
self.distortion = []
if show_progress:
bar = ProgressBar(widgets=[Percentage(), Bar()], maxval=n).start()
index = 0
for j in range(len(samples)):
self.samples = samples[j]
I = np.random.randint(0,self.samples.shape[0],n)
for i in range(epochs[j]):
# Set sigma and learning rate according to current time
t = index/float(n)
lrate = self.lrate_i*(self.lrate_f/self.lrate_i)**t
sigma = self.sigma_i*(self.sigma_f/self.sigma_i)**t
C = self.codebook.copy()
# Learn data
S = self.samples[I[i]] + noise*(2*np.random.random(len(self.samples[I[i]]))-1)
S = np.minimum(np.maximum(S,0),1)
self.learn_data(S,lrate,sigma)
#self.learn_data(self.samples[I[i]],lrate,sigma)
if i%100 == 0:
self.entropy.append(((self.codebook-C)**2).sum())
if test_samples is not None:
distortion = self.compute_distortion(test_samples)
else:
distortion = self.compute_distortion(self.samples)
self.distortion.append(distortion)
if show_progress:
bar.update(index+1)
index = index+1
if show_progress:
bar.finish()
def minrho_run():
pb = ProgressBar(len(NUM_DROP))
sys.stderr.write("Beginning minrho eval.\n")
pb.errput()
for n in NUM_DROP:
this_rand = lambda d: replace_subjects(d, n)
cleaner = lambda d: remove_most_deviant_subjects(d, n)
params = {'n': n, 'cleaner': 'minrho'}
for row in run_experiment(this_rand, [cleaner], [params]):
yield row
pb.incr_and_errput()
def zscore_run(randomizer, randomizer_name):
pb = ProgressBar(len(ZSCORES) * len(NOISES))
sys.stderr.write("Beginning zscore eval with %s randomization.\n" % randomizer_name)
pb.errput()
for percent_noise in NOISES:
this_rand = lambda d: randomizer(d, percent_noise)
cleaners = [zscore and RemoveDeviantRatings(zscore).scores or BaselineCleaner().scores
for zscore in ZSCORES]
parameters = [dict(cleaner='zscore', p=percent_noise, randomizer=randomizer_name, zscore=str(zscore))
for zscore in ZSCORES]
for row in run_experiment(this_rand, cleaners, parameters):
yield row
pb.incr_and_errput()
async def do_audit(mb_data_file_path: Path, output_path: Path,
concurrency: int, spam: bool) -> None:
configure_logging(spam)
ia_creds = get_ia_credentials()
if ia_creds is None:
loguru.logger.error('IA credentials not found in ~/.ia')
return
s3_access, s3_secret = ia_creds
with mb_data_file_path.open('r') as f:
num_items = sum(1 for l in f)
task_q: asyncio.Queue[tuple[AuditTask,
LogQueue]] = asyncio.Queue(concurrency * 2)
session = ClientSession(
connector=TCPConnector(limit=concurrency),
headers={'Authorization': f'LOW {s3_access}:{s3_secret}'})
async with session:
audit_meta, task_stream = await create_tasks(
AsyncPath(mb_data_file_path), output_path, session, loguru.logger)
num_items = audit_meta['count']
with ProgressBar(num_items) as progress:
queuer = asyncio.create_task(
queue_tasks(task_stream, task_q, progress))
aggregator = ResultAggregator(output_path, progress)
runners = [
asyncio.create_task(task_runner(task_q, aggregator, progress))
for _ in range(concurrency)
]
# Wait until all tasks are queued
await queuer
# Wait until all tasks are done
await task_q.join()
# Terminate the runners, all jobs done
for runner in runners:
runner.cancel()
aggregator.finish()
write_logs(output_path, aggregator)
write_failed_items(output_path, aggregator)
write_tables(output_path, aggregator)
def to_png(tif):
width, height = tif.size
img = Image.new('RGBA', (width, height), color=(0, 0, 0, 0))
prog = ProgressBar(height - 1)
print("\nTo PNG:")
for y in range(0, height - 1):
prog.show(y)
for x in range(0, width - 1):
value = tif.getpixel((x, y))
setpixel(x, y, value, img)
print("\n")
return img
def _align(self, Ys, X, sel=None, record=True):
pb = ProgressBar(final=len(Ys), label='align', tail_label='{i:}')
if record:
info = np.zeros((len(Ys), 11))
# set defaults
sel = sel or slice(None)
X_sel = X[:, sel]
t0 = self._calc_centroid(X_sel)
X0 = X - t0 # centre reference on origin
X0_sel = X0[:, sel]
for i, Y_ in enumerate(Ys):
pb.update()
Y_sel = Y_[:, sel]
t = self._calc_centroid(Y_sel)
Y0_ = Y_ - t # centre frame on origin
Y0_sel = Y0_[:, sel]
R = self._calc_optrot(X0_sel, Y0_sel) # get rotation matrix
Y0 = np.dot(R, Y0_.T) # rotate
Y = Y0 + t0 # centre frame on t0
Ys[i, :, :] = Y
if record:
# ref/frame comparison before alignment
info[i, 0] = self._calc_rmsd(X, Y_) # all
info[i, 2] = self._calc_rmsd(X0, Y0_) # rotational
info[i, 3] = self._calc_rmsd(X0_sel,
Y0_sel) # rotational selection
# ref/frame comparison after alignment
info[i, 4] = self._calc_rmsd(X, Y) # all
info[i, 5] = self._calc_rmsd(X0, Y0) # rotational
# aligned/unaligned frame comparisons
info[i, 6] = self._calc_rmsd(Y_, Y) # all
info[i, 7] = self._calc_rmsd(Y0, Y0_) # rotational
# ref/frame and aligned/unaligned
info[i, 8] = self._calc_rmsd(t, t0) # centre difference
# euler angles
info[i, 8:11] = self._calc_euler(R)
return (Ys, info) if record else Ys
def __init__(self, pdb, dmin, dmax, expand=True):
# inherit from Mixin
self.enumerate_hkl = self._enumerate_hkl_numpy
# enable access to self.F[h, k, l] with slicing support
self.F = self._make_FhklArray()
# coordinates
if expand:
pdb = pdb.get_unitcell()
uc_xyzc = np.dot(pdb.S, [pdb.x, pdb.y, pdb.z])
self.n = len(pdb)
# reflections
hkls, d = self._enumerate_hkl_numpy(dmin, dmax, S=pdb.S)
stol2 = 1.0 / (4.0 * d * d)
# Cromer-Mann scattering factors
assert os.path.isfile('cm.pkl')
with open('cm.pkl', 'rb') as _f:
_A, _B, _C = pickle.load(_f)
f0 = {
e:
_C[e] + sum(_A[e][i] * np.exp(-_B[e][i] * stol2) for i in range(4))
for e in set(pdb.e)
}
# direct summation for each summand
self.Fatoms = np.zeros((len(pdb), len(hkls)), dtype=complex)
p = ProgressBar(final=len(pdb),
label='directsum',
tail_label='{f:>10}')
for i, (xyzc, e, n, b) in enumerate(zip(uc_xyzc.T, pdb.e, pdb.n,
pdb.B)):
p.update()
hx = np.sum(hkls * xyzc, axis=1)
self.Fatoms[i, :] = n * f0[e] * np.exp(-b * stol2) * np.exp(
np.pi * 2j * hx)
# save drange
self._hkls = hkls
self._d = d
self.selection()
def svd_run(randomizer, randomizer_name):
pb = ProgressBar((K + 1) * len(NOISES))
sys.stderr.write("Beginning SVD eval with %s randomization.\n" % randomizer_name)
pb.errput()
for percent_noise in NOISES:
this_rand = lambda d: randomizer(d, percent_noise)
parameters = [{
'cleaner': 'svd',
'p_noise': percent_noise,
'randomizer': randomizer_name,
'k': str(k)
} for k in [None] + range(1, K + 1)]
cleaners = [BaselineCleaner().scores] + [c.scores for c in create_svd_cleaners(K)]
for row in run_experiment(this_rand, cleaners, parameters):
yield row
pb.incr_and_errput()
def download_images(uri_list, download_location, retain_original_naming=True):
num = len(uri_list)
progress = ProgressBar(num, fmt=ProgressBar.FULL)
for i, src in enumerate(uri_list):
############################### DO WORK HERE ###########################
try:
img_data = urlopen(src).read()
if len(img_data) > 0: #Read Success
filename = basename(urlsplit(src)[2]).strip()
if not retain_original_naming:
filetype = filename.split('.')[-1]
filename = str(i + 1) + '.' + filetype
output = open(os.path.join(download_location, filename), 'wb')
output.write(img_data)
output.close()
except Exception as e:
log_error(e)
############################### END OF WORK #$##########################
progress.current += 1
progress()
sleep(0.001)
progress.done()
def normalmap(hm, sobelscale):
width, height = hm.size
nm = Image.new('RGBA', (width, height), color=(0, 0, 0, 0))
prog = ProgressBar(height)
print("\nNormal Map:")
for y in range(0, height):
prog.show(y)
for x in range(0, width):
value = hm.getpixel((x, y))
if value != -2147483648:
normal = sobel(x, y, hm, sobelscale)
color = (\
int(normal.x * 255),
int(normal.y * 255),
int(normal.z * 255), 255)
nm.putpixel((x, y), color)
return nm
