def run(self):
self.progress = progress.Progress(self, self.title)
self.progress.update_progress(0, f'Verifying {self.region} tables.')
self._set_business_tables()
self._set_value_tables()
self._set_lookup_tables()
self._set_query_condition()
self._set_query()
self._set_con()
self._attach_values_db()
self._query_tables()
self.con.close()
if not self.abort:
if self.results:
self._set_output_path()
utilities.Output.output(self.results, self.output_path,
self.columns)
self.progress.update_progress(100, 'Finished.')
utilities.open_file(self.output_path)
self.progress.destroy()
def start_conversion(self):
"""
Extract the appropriate information from GUI and call the
Progress dialog with the suitable argumens.
"""
if not self.ok_to_continue():
return
ext_to = self.output_ext()
_list = self.create_paths_list(self.fnames, ext_to, self.prefix,
self.suffix,
unicode(self.toLineEdit.text()),
self.origCheckBox.isChecked(),
self.overwrite_existing)
tab = self.current_tab()
cmd = ''
size = str('')
mntaspect = False
if tab.name == 'AudioVideo':
cmd = tab.commandLineEdit.text()
elif tab.name == 'Images':
width = tab.widthLineEdit.text()
if width:
height = tab.heightLineEdit.text()
size = str('{0}x{1}'.format(width, height))
mntaspect = tab.aspectCheckBox.isChecked()
else:
self.docconv = True
dialog = progress.Progress(_list, tab.name, cmd,
not self.avconv_prefered, size, mntaspect,
self.deleteCheckBox.isChecked(), self)
dialog.show()
def loadfile(dfile, linect, dformat):
'''Load linect lines from labeled data in dfile according to dformat.
Return a list of DataPoint instances.
dfile -- file name
-- one data point per line
-- values are separated by commas
-- final value is the label
dformat -- data format list
-- tuples with a function to coerce the vaulue and a function
which returns false if the value is invalid
eg:
>>> fmt = 48 * [(float, lambda x: 0 <= x <= 100 )] + \
6 * [(int, lambda x: 0 < x )] + \
1 * [(int, lambda x: x == 1 or x == 0)]
'''
print 'Loading "{}"'.format(dfile)
data = []
with open(dfile, mode='rb') as fd:
with pg.Progress(linect, 2, pg.bar('Lines', 32)) as pr:
for line in fd:
# clean the data & create the datapoint
cleandata = [loadfeature(raw.strip(), fmt) \
for raw, fmt in zip(line.split(','), dformat)]
data.append(DataPoint(cleandata[:-1], cleandata[-1]))
# indicate progress
try:
pr.next()
except TypeError:
break
return data
def get(self, remoteFile='', localFile='', *args):
'''receive file'''
localFile = localFile or remoteFile
localFile = os.path.expanduser(localFile)
resp = self.cd(remoteFile)
if not resp.startswith('5'):
self.cd('..')
return self._get_dir(remoteFile, localFile)
if os.path.isdir(localFile):
raise exc.FailedOperationException(
'Second argument must be file, not directory')
self.send_cmd('TYPE I')
size = self.size(remoteFile)
sock = self._prepare_before_get_data('RETR', remoteFile)
try:
with sock, open(localFile, 'wb') as f:
bar = progress.Progress(size)
if DEBUG:
print(self._lastResp)
for line in sock.makefile('rb'):
bar.update(len(line))
f.write(line)
except Exception as e:
with contextlib.suppress(FileNotFoundError):
os.remove(localFile)
if not isinstance(e, exc.FailedOperationException):
get_resp(self.sock)
raise
else:
return get_resp(self.sock)
def init(self):
self.progress = progress.Progress(max=self.number_of_samples)
self.lockin = np.zeros((1024, self.number_of_samples), dtype=np.float)
self.i = 0
self.stage.query_pos()
self.startpos = self.stage.last_pos()
self.abort = False
def generate_submit_constrained_parameters_random(
self,
num_samples,
dict_parameters_range,
filtering_function=None,
filtering_function_parameters=None,
pbs_submission_infos=None,
submit_jobs=True):
'''
Takes a dictionary of parameters (which should contain low/high values for each or generator function), and
generates num_samples possible parameters randomly.
Checks them with some provided function before accepting them if provided.
If pbs_submission_infos is provided, will additionally automatically create scripts and submit them to PBS.
'''
# Convert specific parameter sampling methods into generators
self.create_generator_random_parameter(dict_parameters_range)
# We will keep all constrained parameters for further use
constrained_parameters = []
fill_parameters_progress = progress.Progress(num_samples)
tested_parameters = 0
# Provide as many experimentally constrained parameters as desired
while len(constrained_parameters) < num_samples:
print "Parameters tested %d, found %d. %.2f%%, %s left - %s" % (
tested_parameters, len(constrained_parameters),
fill_parameters_progress.percentage(),
fill_parameters_progress.time_remaining_str(),
fill_parameters_progress.eta_str())
# Sample new parameter values
new_parameters = {}
for curr_param, param_dict in dict_parameters_range.items():
# Use the provided sampling function (some are predefined)
new_parameters[curr_param] = param_dict['sampling_fct']()
# Check if the new parameters are within the constraints
if (filtering_function and filtering_function(
new_parameters, dict_parameters_range,
filtering_function_parameters)) or (filtering_function is
None):
# Yes, all good
# Append to our parameters
constrained_parameters.append(new_parameters)
# If desired, generate a script and submits it to PBS
if pbs_submission_infos:
self.create_submit_job_parameters(pbs_submission_infos,
new_parameters,
submit=submit_jobs)
fill_parameters_progress.increment()
tested_parameters += 1
return constrained_parameters
def install(self):
self.installbar = progress.Progress(
msg='Installing \'' + self.name + '\'',
success='\'' + self.name + '\' installed')
self.installbar.start()
self.process = subprocess.Popen(
[self.tempfile.name] + cmd_args,
creationflag=subprocess.CREATE_NO_WINDOW)
self.process.wait()
self.installbar.stop()
def loadMHM(version, lines, default_load_callback, strict=False):
version_ = _parse_version(version)
if version_ is None:
raise RuntimeError("Failed to parse version %s" % version)
fprog = progress.Progress(len(lines))
loader = getMHMLoader(version_)
for lineData in lines:
lineData = lineData.strip().split()
loader.loadProperty(lineData, default_load_callback, strict)
fprog.step()
def install(self):
self.installbar = progress.Progress(
msg='Enabling \'' + self.featurename + '\'',
success='\'' + self.featurename + '\' enabled')
self.installbar.start()
self.process = subprocess.Popen([
'runas', '/user:Administrator',
'dism /online /enable-feature /featurename:' + self.featurename
])
self.process.wait()
self.installbar.stop()
def from_dataset(cls, dataset):
stumps = []
print 'Making distinct stumps and caching mistakes.'
maxstumps = len(dataset[0].features) * (len(dataset) - 1 + 2)
with pg.Progress(maxstumps, 2, pg.bar('Stumps', 32)) as p:
# generate pairs of (index, feature vector)
for i, fv in enumerate(it.izip(*(dp.features for dp in dataset))):
for t in cls.thresholds(fv):
s = i, t, Stump.mistakes((i, t, []), dataset)
stumps.append(s)
p.next()
return stumps
def download(self):
self.downloadbar = progress.Progress(
msg='Downloading \'' + self.name + '\'',
success='\'' + self.name + '\' downloaded')
self.downloadbar.start()
r = requests.get(self.url, stream=True)
with tempfile.TemporaryFile() as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
self.downloadbar.stop()
self.tempfile = f
def find_best_classifier_score(classifier, base_settings={}) -> (float, dict):
variations_count = featureEngineering.get_settings_variations_count()
attempt_count = 10
progress_log = progress.Progress(variations_count * attempt_count)
results = list()
for settings_seed in range(0, variations_count):
settings = base_settings | featureEngineering.get_settings_variation(
settings_seed)
avg_test_score = get_avg_test_score(classifier, settings,
attempt_count, progress_log)
results.append((avg_test_score, settings))
best_settings = get_best_settings(results)
return results[0][0], best_settings
def create_user_progress(id, semestercount):
matnr = "student" + str(id)
year = user_start_year
month = 10
day = 1
start = datetime.date(year, month, day)
if semestercount % 2 == 1:
month = 2
year += 1 + (semestercount - 1) / 2
elif semestercount == 0:
day = 15
else:
month = 7
year += semestercount / 2
end = datetime.date(int(year), month, day)
return progress.Progress(id, matnr, user_study, start, end, user_reason,
user_state)
def __init__(self,
spectrometer,
settings,
scanning_points,
stage,
parent=None):
try:
self.spectrometer = spectrometer
self.scanning_points = scanning_points
self.settings = settings
self.stage = stage
self.i = 0
self.n = scanning_points.shape[0]
self.positions = np.zeros((self.n, 2))
self.progress = progress.Progress(max=self.n)
super(ScanThread, self).__init__(spectrometer)
except:
(type, value, traceback) = sys.exc_info()
sys.excepthook(type, value, traceback)
def check():
# march through candidates in an orderly fashion
candidates = words4.keys()
candidates.sort()
ticker = progress.Progress(majormark='.')
write = sys.stdout.write
flush = sys.stdout.flush
is3 = words3.has_key
for wi in candidates:
write(' ' + wi + ' ')
flush()
# examine retricted set of candidates - those with no letters in
# common with wi
jlist = [
w for w in candidates if (wi[0] not in w and wi[1] not in w
and wi[2] not in w and wi[3] not in w)
]
for wj in jlist:
ticker.tick()
# examine a further restricted set of candidates - those with
# no letters in common with wi or wj
klist = [
w for w in jlist if (wj[0] not in w and wj[1] not in w
and wj[2] not in w and wj[3] not in w)
]
for wk in klist:
w = wi + wj + wk
# do we have one of each vowel?
# does each three-letter string form a word?
if (w.count("a") == w.count("e") == w.count("i") ==
w.count("o") == w.count("u") == w.count("y") == 1
and is3(wi[0] + wj[0] + wk[0])
and is3(wi[1] + wj[1] + wk[1])
and is3(wi[2] + wj[2] + wk[2])
and is3(wi[3] + wj[3] + wk[3])):
write(' ' + ` (wi, wj, wk) ` + ' ')
flush()
def init(self):
self.progress = progress.Progress(max=self.number_of_samples)
self.mean = np.zeros(1024, dtype=np.float)
self.i = 0
self.abort = False
def __init__(self, working_dir):
Gtk.Window.__init__(self)
self.connect('destroy', self.destroy_cb)
self.set_position(Gtk.WindowPosition.CENTER_ALWAYS)
self.set_resizable(False)
self.set_title(working_dir)
os.chdir(working_dir)
self.stage = next(discover_stages(), None)
if self.stage is None:
sys.exit("unable to locate THORLABS stage")
self.focus_window = None
self.config_window = None
self.capture_selection = None
self.capture_position = None
self.capture_location = None
# used to normalise start and end times
self.capture_start_pos = None
self.capture_start_time = None
# queue for image saving to not compromise fps
self.capture_image_queue = queue.Queue()
self.vbox = Gtk.VBox(False, 0)
self.add(self.vbox)
self.vbox.show()
fixed = Gtk.Fixed()
self.vbox.pack_start(fixed, False, True, 0)
fixed.show()
self.camera = camera.Camera()
self.preview = preview.Preview(self.camera)
fixed.put(self.preview, 0, 0)
self.preview.show()
eb = Gtk.EventBox()
fixed.put(eb, 0, 0)
eb.show()
self.progress = progress.Progress()
self.progress.set_size_request(preview_width, -1)
eb.add(self.progress)
eb = Gtk.EventBox()
fixed.put(eb, 0, 0)
eb.show()
self.info = info.Info()
self.info.set_size_request(preview_width, -1)
eb.add(self.info)
self.toolbar = Gtk.HBox(False, 5)
self.toolbar.set_border_width(3)
self.vbox.pack_end(self.toolbar, False, False, 0)
self.toolbar.show()
button = Gtk.Button()
quit_image = Gtk.Image.new_from_stock(Gtk.STOCK_QUIT,
Gtk.IconSize.SMALL_TOOLBAR)
quit_image.show()
button.set_tooltip_text("Quit RTIAcquire")
button.connect('clicked', self.destroy_cb, None)
button.add(quit_image)
self.toolbar.pack_end(button, False, False, 0)
button.show()
button = Gtk.Button()
menu_image = Gtk.Image.new_from_stock(Gtk.STOCK_PREFERENCES,
Gtk.IconSize.SMALL_TOOLBAR)
menu_image.show()
button.set_tooltip_text("Camera settings")
button.connect('clicked', self.config_cb, None)
button.add(menu_image)
self.toolbar.pack_end(button, False, False, 0)
button.show()
label = Gtk.Label()
self.toolbar.pack_end(label, False, False, 0)
self.preview.score_label = label
label.show()
spinner = Gtk.SpinButton.new_with_range(1, 4, 0.1)
spinner.set_tooltip_text("Maximum velocity of turntable")
spinner.set_value(self.stage.max_velocity)
self.toolbar.pack_start(spinner, False, False, 0)
self.max_velocity = spinner
spinner.show()
entry = Gtk.Entry.new()
entry.set_tooltip_text("Seal name")
entry.set_placeholder_text("Name of Seal")
self.toolbar.pack_start(entry, False, False, 0)
self.seal_name = entry
entry.show()
radio = Gtk.RadioButton.new_with_label(None, "Direct LED")
self.direct_led = radio
self.toolbar.pack_start(radio, False, False, 0)
radio.show()
radio = Gtk.RadioButton.new_with_label_from_widget(
radio, "Structured Light")
self.toolbar.pack_start(radio, False, False, 0)
radio.show()
button = Gtk.Button("Start Capture")
button.set_tooltip_text("Start seal capture")
button.connect('clicked', self.capture_cb, None)
self.toolbar.pack_start(button, False, False, 0)
self.capture = button
button.show()
self.info.msg('Something Something Something', 'v0.1, July 2017')
self.progress.progress(0.2)
self.preview.set_live(True)
self.show()
def main():
parser = argparse.ArgumentParser()
parser.add_argument("data_store_path")
parser.add_argument("-t", "--template", help="template file name")
parser.add_argument("-e", "--fileext", default="md", help="file extension")
args = parser.parse_args()
app = Flask(__name__)
p = progress.Progress(args.data_store_path, args.fileext)
def get_edit_link(date=None):
week_string = get_week_string(date=date)
return "/%s/edit" % week_string
def get_week_string(date=None):
week_start = progress.get_start_of_week(date_=date)
return week_start.strftime("%Y-%m-%d")
# get the week number in the year
def get_week_number(datestring):
return datetime.strptime(datestring, "%Y-%m-%d").isocalendar()[1] % 53
def get_week_as_html(date=None):
content = p.get_week(date_=date)
if content:
doc = pandoc.Document()
doc.markdown = content
content = str(doc.html)
else:
content = ""
return content, get_edit_link(date)
@app.route('/')
def current_week():
content, edit_link = get_week_as_html()
return render_template('default.html',
content=content,
edit_link=edit_link)
# you might not want this
@app.route('/goals')
def goals_page():
doc = pandoc.Document()
doc.markdown = p.get_file('_goals.md')
content = unicode(str(doc.html), "utf-8")
return render_template('default.html',
content=content,
edit_link=get_edit_link())
@app.route('/archive')
def archive_page():
return render_template('archive.html', edit_link=get_edit_link())
# /YYYY-MM-DD
@app.route('/<datestring>')
def specific_week(datestring):
date = datetime.strptime(datestring, "%Y-%m-%d")
content, edit_link = get_week_as_html(date=date)
return render_template('default.html',
content=content,
edit_link=edit_link)
# /YYYY-MM-DD/edit
@app.route('/<datestring>/edit', methods=['POST', 'GET'])
def edit_specific_week(datestring):
date = datetime.strptime(datestring, "%Y-%m-%d")
if request.method == 'POST':
if request.form['content']:
p.set_week(request.form['content'], date_=date)
return redirect("/%s" % datestring)
content = p.get_week(date_=date, template=args.template)
if not content:
content = "# Week %s: %s" % (get_week_number(datestring),
datestring)
content = '''
<form action="" method="post" class="edit-form">
<textarea name="content" class="edit-textarea">%s</textarea>
<br><br>
<input type="submit">
</form>
''' % content
return render_template('default.html', content=content)
# so it doesn't complain
@app.route('/favicon.ico')
def favicon():
return send_from_directory(app.root_path,
'favicon.ico',
mimetype='image/vnd.microsoft.icon')
app.run(debug=True)
plt.arrow(onlense2_back.r[i, 0], onlense2_back.r[i, 1],
onlense2_back.k[i, 0] / divider,
onlense2_back.k[i, 1] / divider)
plt.plot(screen.points[:, 0], screen.points[:, 1] / (wl * 50))
# for i in range(onscreen.r.shape[0]):
# plt.plot(onscreen.r[i, 0], onscreen.r[i, 1], "bo")
# plt.arrow(onscreen.r[i, 0], onscreen.r[i, 1], onscreen.k[i, 0] / divider, onscreen.k[i, 1] / divider)
plt.savefig("dipole_" + str(int(np.round(theta * 180 / np.pi))) +
"_theta_setup.svg",
dpi=600)
plt.show()
plt.close()
screen.clear()
prog = progress.Progress(max=iterations)
num = 10000 #2000000
for i in range(iterations):
dipole = make_dipole(wl, theta, alpha_max, num, mode='ray')
onlense1_front = lense1.front.interact_with_all_wavelets(dipole)
#onlense1_front.mode = modes['gaussian']
# for j in range(int(iterations/100)):
# dipole = make_dipole(theta, alpha_max, num)
# onlense1_front.append_wavelets(lense1.front.interact_with_all_wavelets(dipole))
onlense1_back = lense1.back.interact_with_all_wavelets(onlense1_front)
onlense2_front = lense2.front.interact_with_all_wavelets(onlense1_back)
onlense2_back = lense2.back.interact_with_all_wavelets(onlense2_front)
def iterate(self, graph, tissue):
eready = 0
prev_ready = -1
prg = progress.Progress(
graph.ecount(),
'Loading tissue specificity data for %s' % self.tissnm[tissue], 1)
inum = 0
while eready > prev_ready:
inum += 1
prg.step(eready - prev_ready, status='iteration %u' % inum)
for e in random.sample(graph.es, graph.ecount()):
if e['giant'][self.tissnm[tissue]] is None:
# print 'for edge #%u between proteins %s (%s) and %s (%s) value for %s is None' % \
# (e.index, graph.vs[e.source]['name'], str(self.ventrez[e.source]), graph.vs[e.target]['name'],
# str(self.ventrez[e.target]), self.tissnm[tissue])
value = self.from_cache(graph.vs[e.source]['name'],
graph.vs[e.target]['name'], tissue)
if value is None:
# print '\tvalue could not be looked up from cache'
entrez = [
self.ventrez[e.source], self.ventrez[e.target]
]
if all(entrez):
# print '\tUniProts successfully translated to
# Entrez: %s. starting query' % str(entrez)
prg.step(
eready - prev_ready,
status=
'downloading (waiting for remote), iter: %u' %
inum)
# print '\tquery params: %s, %s' % (str(tissue),
# str(entrez))
data = self.query(tissue, entrez)
genes = self.genes_dict(data)
# print '\tquery finished and returned data of type %s and length %u' % (str(type(data)),
# 0 if type(data) is not dict else
# len(data['edges']))
prg.step(eready - prev_ready,
status='processing data, iter: %u' % inum)
# print '\titerating edges in data'
for gedge in data['edges']:
# here we get the Gene IDs from the list
# indexes:
epair = (str(genes[gedge['source']]),
str(genes[gedge['target']]))
repair = tuple(reversed(list(epair)))
# print 'looking up edges %s and %s' %
# (str(epair), str(repair))
eid = self.eentrez[epair] if epair in self.eentrez \
else self.eentrez[repair] if repair in self.eentrez else None
if eid is not None:
# print '\tedge %s---%s [%s] in the network with id = %u; setting value to %s' % (
# graph.vs[self.ventrez.index(epair[0])]['name'],
# graph.vs[self.ventrez.index(epair[1])]['name'],
# str(epair), eid, str(gedge['weight']))
graph.es[eid]['giant'][
self.tissnm[tissue]] = gedge['weight']
self.to_cache([
graph.vs[graph.es[eid].source]['name'],
graph.vs[graph.es[eid].target]['name'],
gedge['weight']
], tissue)
else:
pass
# print '\tsome of the UniProts could not be
# translated to Entrez'
else:
# print '\tsetting value from cache: for edge #%u, for tissue %s, value = %s' % \
# (e.index, self.tissnm[tissue], str(value))
e['giant'][self.tissnm[tissue]] = value
prev_ready = eready
eready = len([
e for e in graph.es
if e['giant'][self.tissnm[tissue]] is not None
])
prg.terminate()
def search(self):
# self.mutex.lock()
self.spectrometer.integration_time_micros(
self.settings.search_integration_time * 1000)
# self.mutex.unlock()
spec = self.spectrometer.intensities()
spec = spec[0:1024]
spec = smooth(self.wl, spec)
self.stage.query_pos()
startpos = self.stage.last_pos()
minval = np.min(spec)
maxval = np.max(spec)
d = np.linspace(-self.settings.rasterwidth, self.settings.rasterwidth,
self.settings.rasterdim)
repetitions = 4
self.progress = progress.Progress(max=repetitions)
for j in range(repetitions):
self.stage.query_pos()
origin = self.stage.last_pos()
measured = np.zeros(self.settings.rasterdim)
if j is 4:
d /= 2
if j % 2:
pos = d + origin[0]
else:
pos = d + origin[1]
for k in range(len(pos)):
if j % 2:
self.stage.moveabs(x=pos[k])
else:
self.stage.moveabs(y=pos[k])
if self.abort:
self.stage.moveabs(x=startpos[0], y=startpos[1])
return False
spec = self.spectrometer.intensities()
spec = spec[0:1024]
spec = smooth(self.wl, spec)
self.specSignal.emit(spec)
#initial_guess = (np.max(spec), 600, 200, 0)
#try:
# #def gauss(x, amplitude, xo, fwhm, offset):
# popt, pcov = opt.curve_fit(gauss, np.linspace(0,1023,1024), spec, p0=initial_guess)
# #print(popt)
# measured[k] = popt[0]
#except RuntimeError as e:
# print(e)
measured[k] = np.max(spec[400:800])
maxind = np.argmax(measured[2:(len(pos))])
initial_guess = (maxval - minval, pos[maxind], self.settings.sigma,
minval)
dx = origin[0]
dy = origin[1]
popt = None
fitted = False
try:
popt, pcov = opt.curve_fit(gauss,
pos[2:(len(pos))],
measured[2:(len(pos))],
p0=initial_guess)
#popt, pcov = opt.curve_fit(gauss, pos, measured, p0=initial_guess)
perr = np.diag(pcov)
#print(perr)
if perr[0] > 10000 or perr[1] > 1 or perr[2] > 1:
print(
"Could not determine particle position: Variance too big"
)
elif popt[0] < 1e-1:
print(
"Could not determine particle position: Peak too small"
)
elif popt[1] < (min(pos) - 0.5) or popt[1] > (max(pos) + 0.5):
print(
"Could not determine particle position: Peak outside bounds"
)
else:
fitted = True
except RuntimeError as e:
print(e)
print("Could not determine particle position: Fit error")
if fitted:
if j % 2:
dx = float(popt[1])
else:
dy = float(popt[1])
self.stage.moveabs(x=dx, y=dy)
fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(pos, measured, 'bo')
x = np.linspace(min(pos), max(pos))
if not popt is None:
ax.text(0.1,
0.9,
str(popt[1]) + ' +- ' + str(perr[1]),
ha='left',
va='center',
transform=ax.transAxes)
ax.plot(x, gauss(x, popt[0], popt[1], popt[2], popt[3]), 'g-')
plt.savefig("search_max/search" + str(j) + ".png")
plt.close()
self.progress.next()
self.progressSignal.emit(self.progress.percent,
str(self.progress.eta_td))
self.spectrometer.integration_time_micros(
self.settings.integration_time * 1000)
self.spectrometer.intensities()
return studies
exams = []
with open('../data/allpruefungsleistungen.csv', newline='') as plcsv:
plreader = csv.DictReader(plcsv, delimiter=';')
for r in plreader:
exams += [exam.Exam(r['matnr'], r['studyid'], r['courseid'], r['coursename'], \
toDateTime(r['date']), r['semester'], r['ects'], r['grade'])]
progresses = []
with open('../data/allstudienstatus.csv', newline='') as svcsv:
svreader = csv.DictReader(svcsv, delimiter=';')
for i, r in enumerate(svreader):
progresses += [progress.Progress(i, r['matnr'], r['studyid'], \
toDateTime(r['start']), toDateTime(r['end']), \
r['reason'], r['state'])]
matnrs = set(map(lambda x: x.matnr, progresses))
iter = []
for i, m in enumerate(matnrs):
iter.append((m, i, exams, progresses))
if __name__ == '__main__':
with Pool(5) as pool:
studentdata = pool.starmap(aggregateStudentData, iter)
students = pool.starmap(createStudent, studentdata)
#students = datawrangler.aggregateStudents(exams, progresses)
#print(students[0].studies[0].path.semester)
ids = ["2965-521", "4104-521", "4821-521", "4644-521"]
def get_progress(self, basename): results_dir = self._results_dir(basename) return [ progress.Progress(results_dir) ]
def view_progress(self):
session = create_session()
labels = self._get_labels(session)
progress_window = progress.Progress(labels, parent=self)
progress_window.show()
os.mkdir(savefolder)
print('save folder:', savefolder)
###--save configuration--###
f = open(savefolder + 'setting.txt', 'w')
for arg in vars(args):
f.write('%s:%s\n' % (arg, getattr(args, arg)))
f.write('loaded HPE#1 net:%s\n' % trained_modelFile_hpe1_orig)
f.write('traindataNum_uvr:%s\n' % traindataNum_uvr)
f.write('traindataNum_blur_uvr:%s\n' % traindataNum_blur_uvr)
f.write('traindataNum_VR20:%s\n' % traindataNum_vr20)
f.write('validateNum_uvr:%s\n' % validateNum)
f.close()
#--start
progress_train = progress.Progress(loss_names, pretrain=False)
progress_validate = progress.Progress(loss_names, pretrain=False)
iternum_train = traindataNum_uvr // args.train_batch
iternum_train_blur = traindataNum_blur_uvr // args.train_batch
iternum_vr20 = traindataNum_vr20 // args.train_batch
print('start..')
for epoch in range(args.epochs):
#--train
fusionnet.set_mode('train')
datasetloader_uvr['train'].shuffle()
generator_train_icvl = datasetloader_icvl.generator_learningData(
args.train_batch, 'train', False, 0)
def __init__(self):
gtk.Window.__init__(self)
self.connect('destroy', self.destroy_cb)
self.config_window = None
self.live_hide_timeout = 0
self.light_hop_timeout = 0
self.busy = False
self.leds = ledmap.Ledmap(os.path.join(source_dir, 'data',
'led-maps.txt'))
logging.debug('loaded %d maps', len(self.leds.get_names()))
for name in self.leds.get_names():
bytes = self.leds.get_bytes(name)
logging.debug('%s: %d lights', name, len(bytes))
# where project directories get written, see RTI cap above
self.outdir = options.outdir
self.lights = lights.Lights()
# try to reset the lights ... if this fails, disable dome controls
try:
self.dome_controls = True
name = self.leds.get_names()[0]
self.lights.set_triple(self.leds.get_bytes(name)[0])
except lights.Error as e:
logging.debug('no lights found, disabling dome controls')
self.dome_controls = False
self.vbox = gtk.VBox(False, 0)
self.add(self.vbox)
self.vbox.show()
fixed = gtk.Fixed()
self.vbox.pack_start(fixed, False)
fixed.show()
self.camera = camera.Camera()
self.preview = preview.Preview(self.camera)
fixed.put(self.preview, 0, 0)
self.preview.show()
self.preview.connect('motion_notify_event', self.preview_motion_cb)
if options.verbose:
try:
config = camera.Config(self.camera)
config.prettyprint(sys.stdout, config.get_root_widget())
except:
logging.debug("No Camera detected: unable to print config")
eb = gtk.EventBox()
fixed.put(eb, 0, 0)
eb.show()
self.progress = progress.Progress()
self.progress.set_size_request(preview_width, -1)
eb.add(self.progress)
eb = gtk.EventBox()
fixed.put(eb, 0, 0)
eb.show()
self.info = info.Info()
self.info.set_size_request(preview_width, -1)
eb.add(self.info)
eb = gtk.EventBox()
fixed.put(eb, 20, 380)
eb.show()
self.play_image = gtk.image_new_from_stock(gtk.STOCK_MEDIA_PLAY,
gtk.ICON_SIZE_SMALL_TOOLBAR)
self.pause_image = gtk.image_new_from_stock(gtk.STOCK_MEDIA_PAUSE,
gtk.ICON_SIZE_SMALL_TOOLBAR)
self.live = gtk.Button()
self.live.set_image(self.play_image)
self.live.set_tooltip_text("Start/stop live preview")
self.live.connect('clicked', self.live_cb, None)
eb.add(self.live)
self.live.show()
self.toolbar = gtk.HBox(False, 5)
self.toolbar.set_border_width(3)
self.vbox.pack_end(self.toolbar)
self.toolbar.show()
button = gtk.Button()
quit_image = gtk.image_new_from_stock(gtk.STOCK_QUIT,
gtk.ICON_SIZE_SMALL_TOOLBAR)
quit_image.show()
button.set_tooltip_text("Quit RTIAcquire")
button.connect('clicked', self.destroy_cb, None)
button.add(quit_image)
self.toolbar.pack_end(button, False, False)
button.show()
if self.dome_controls:
self.dome_picker = gtk.combo_box_new_text()
for name in self.leds.get_names():
self.dome_picker.append_text(name)
self.dome_picker.set_active(0)
self.dome_picker.set_tooltip_text("Select lighting system")
self.dome_picker.connect('changed', self.dome_picker_cb, None)
self.toolbar.pack_start(self.dome_picker, False, False)
self.dome_picker.show()
self.light_picker = gtk.SpinButton(climb_rate = 1)
self.light_picker.set_numeric(True)
self.light_picker.set_wrap(True)
self.light_picker.set_increments(1, 1)
self.light_picker.set_tooltip_text("Pick light")
self.light_picker_refresh()
self.light_picker.connect('value_changed',
self.light_picker_cb, None)
self.toolbar.pack_start(self.light_picker, False, False)
self.light_picker.show()
button = gtk.Button()
menu_image = gtk.image_new_from_stock(gtk.STOCK_PREFERENCES,
gtk.ICON_SIZE_SMALL_TOOLBAR)
menu_image.show()
button.set_tooltip_text("Camera settings")
button.connect('clicked', self.config_cb, None)
button.add(menu_image)
self.toolbar.pack_start(button, False, False)
button.show()
button = gtk.Button('Focus')
button.set_tooltip_text("Focus camera automatically")
button.connect('clicked', self.focus_cb, None)
self.toolbar.pack_start(button, False, False)
button.show()
photo_image = gtk.image_new_from_file(
os.path.join(source_dir, 'data', 'camera_24.png'))
photo = gtk.Button()
photo.set_image(photo_image)
photo.set_tooltip_text("Take single photo")
photo.connect('clicked', self.photo_cb, None)
self.toolbar.pack_start(photo, False, False)
photo.show()
if self.dome_controls:
photo = gtk.Button('RTI Preview')
photo.set_tooltip_text("Take preview RTI image")
photo.connect('clicked', self.rti_preview_cb, None)
self.toolbar.pack_start(photo, False, False)
photo.show()
photo = gtk.Button('RTI Capture ...')
photo.set_tooltip_text("Start full RTI acquisition")
photo.connect('clicked', self.rti_capture_cb, None)
self.toolbar.pack_start(photo, False, False)
photo.show()
self.info.msg('Welcome to RTI Acquire', 'v1.3, March 2014')
self.show()
def iterate(self, graph, tissue, size=100):
if self.debug:
self.trace['edges_from_cache'] = []
eready = 0
prev_ready = -1
prg = progress.Progress(
graph.ecount(),
'Loading tissue specificity data for %s' % self.tissnm[tissue], 1)
inum = 0
while eready > prev_ready:
inum += 1
prg.step(eready - prev_ready, status='iteration %u' % inum)
entlst = []
if self.debug:
self.trace['input_edges'] = []
for e in graph.es:
value = None
if e['giant'][self.tissnm[tissue]] is None:
value = self.from_cache(graph.vs[e.source]['name'],
graph.vs[e.target]['name'], tissue)
if value is None:
if self.ventrez[e.source] is not None and self.ventrez[
e.source] not in entlst:
entlst.append(self.ventrez[e.source])
if self.ventrez[e.target] is not None and self.ventrez[
e.target] not in entlst:
entlst.append(self.ventrez[e.target])
#print 'length of query list: %u' % len(entlst)
if self.debug:
self.trace['input_edges'].append(e.index)
if len(entlst) > size or (e.index == graph.ecount() - 1
and len(entlst) > 0):
#print '\t:: starting query'
if self.debug:
self.trace['entrez_list'] = entlst
prg.step(
eready - prev_ready,
status='downloading (waiting for remote), iter: %u'
% inum)
data = self.query(tissue, entlst)
genes = self.genes_dict(data)
prg.step(eready - prev_ready,
status='processing data, iter: %u' % inum)
if self.debug:
self.trace['edges_set'] = []
for gedge in data['edges']:
# here we get the Gene IDs from the list indexes:
epair = (str(genes[gedge['source']]),
str(genes[gedge['target']]))
repair = tuple(reversed(list(epair)))
eid = self.eentrez[epair] if epair in self.eentrez \
else self.eentrez[repair] if repair in self.eentrez else None
if eid is not None:
# print '\tedge %s---%s [%s] in the network with id = %u; setting value to %s' % (
# graph.vs[self.ventrez.index(epair[0])]['name'],
# graph.vs[self.ventrez.index(epair[1])]['name'],
# str(epair), eid, str(gedge['weight']))
graph.es[eid]['giant'][
self.tissnm[tissue]] = gedge['weight']
self.to_cache([
graph.vs[graph.es[eid].source]['name'],
graph.vs[graph.es[eid].target]['name'],
gedge['weight']
], tissue)
if self.debug:
self.trace['edges_set'].append(eid)
prev_ready = eready
entlst = []
else:
e['giant'][self.tissnm[tissue]] = value
if self.debug:
self.trace['edges_from_cache'].append(e.index)
eready = len([
e for e in graph.es
if e['giant'][self.tissnm[tissue]] is not None
])
prg.terminate()
def small_totally_T_adic_polys(q, n, trials=None, term_bound=None, **kwargs):
r"""
Find one or all totally T-adic polynomial over GF(q)[T] with small height
INPUT:
- ``q`` -- a rational prime power
- ``n`` -- a positive integer (the gonality)
- ``trials`` -- optional positive integer
- ``term_bound`` -- an integer bound on the number of terms in the polynomial over GF(q)[T,x]
- Additional keyword arguments are passed to progress.Progress
OUTPUT:
- If trials is None, return all irreducible totally T-adic polynomial over
GF(q)[T] of x-degree n(q+1) and height 1/(q+1).
- If trials is not None, return one such if it can be located in `trials`
random tries, else None.
NOTE:
- This function is an implementation of Algorithm 1 in [FP].
"""
FF = GF(q)
K = FunctionField(FF, names=('T', ))
R = PolynomialRing(K, names=('x', ))
T = K.gen()
x = R.gen()
xdegree = n * (q + 1)
v0 = maclane.function_field_inductive_valuation(T, key_polynomial=x)
w0 = maclane.function_field_decomposition(T, x).extvals()[0]
def GF2_quick_test(f, r):
if f(1) == 0: return False
if w0(f(1 / T)) < -r: return False
if w0(f(T)) < 2 * r: return False
if w0(f(T + 1)) < 2 * r: return False
return True
tests = []
if term_bound is not None:
num_terms = lambda f: sum(
len(val.numerator().dict()) for val in f.dict().values())
tests.append(lambda f: num_terms(f) <= term_bound)
if q == 2:
tests.append(lambda f: GF2_quick_test(f, n))
else:
tests.append(lambda f: quick_tests(f))
tests.append(lambda f: f.is_irreducible())
tests.append(lambda f: special_is_totally_T_adic(n, v0, f))
if trials is not None:
prog = progress.Progress(trials, **kwargs)
for i in range(trials):
f = R(1)
for i in range(n):
f *= T * x - random_poly(K, n, True)
for u in FF:
for i in range(n):
f *= x - u - T * random_poly(K, n - 1, True)
f = reduce_T_power(f, n + 1)
if all(test(f) for test in tests):
prog.finalize()
return f
prog()
prog.finalize()
return None
coefs = []
for i in range(n):
coefs.append(poly_generator(K, n, True))
for u in FF:
for i in range(n):
coefs.append(poly_generator(K, n - 1, True))
num_pols = (q - 1)**(n + q * n) * q**(n * n + q * n * (n - 1))
poly_bits = log(num_pols, 2)
print('Checking 2^({:.1f}) polynomials\n'.format(float(poly_bits)))
prog = progress.Progress(num_pols, **kwargs)
S = set()
for aa in itertools.product(*coefs):
f = R(1)
f *= prod(T * x - aa[i] for i in range(n))
for i, u in enumerate(FF, 1):
f *= prod(x - u - T * aa[i * n + j] for j in range(n))
f = reduce_T_power(f, n + 1)
if all(test(f) for test in tests):
S.add(f)
prog()
prog.finalize()
return S
def main(
): ####################################################################
try:
opts, args = getopt.getopt(sys.argv[1:], "hH:P:p:n:N:d:m:", [
"help", "host=", "profile=", "port=", "nr=", "name=", "delay=",
"sync", "async", "msg="
])
except getopt.GetoptError as err:
print >> sys.stderr, str(err)
usage()
sys.exit(2)
opt_profile = ''
opt_port = '3001'
opt_host = '127.0.0.1'
opt_nr = 1000000
opt_delay = .5
opt_name = "%s [%d]" % (PROG, os.getpid())
opt_msgs = []
opt_async = False
## parse arguments ##
for o, a in opts:
if o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("-P", "--profile"):
opt_profile = a
elif o in ("-p", "--port"):
opt_port = a
elif o in ("-H", "--host"):
opt_host = a
elif o in ("-n", "--name"):
opt_name = a
elif o in ("-N", "--nr"):
opt_nr = int(a)
elif o in ("-d", "--delay"):
opt_delay = float(a)
elif o in ("-m", "--msg"):
opt_msgs.append({'async': opt_async, 'name': a})
elif o in ("-s", "--sync"):
opt_async = False
elif o in ("-a", "--async"):
opt_async = True
if not opt_msgs:
print 'Nothing to do...'
sys.exit(0)
have_sync = not reduce(lambda v, msg: v and msg['async'], opt_msgs, True)
global client
client = mb.Client(opt_name)
client.reg(opt_profile)
# open/subscribe
for msg in opt_msgs:
msg['mid'] = client.open(msg['name'], 'rw')
client.sub(msg['mid'], cb, adaptor=int, async=msg['async'])
# seed traffic...
for msg in opt_msgs:
client.post(msg['mid'], client.id(), 0)
N = len(opt_msgs) * opt_nr
p = progress.Progress(N)
# working cycle/indication
while n < N:
print p.indicator(n),
if have_sync:
time.sleep(opt_delay)
client.dispatch(True, 1)
# close
for msg in opt_msgs:
client.close(msg['mid'])
print p.indicator(n)
client.unreg()
def small_totally_T_adic_polys_GF2(n, num_jobs=1, job=0, **kwargs):
r"""
Search for all totally T-adic polynomials over GF(2)[T] with small height
INPUT:
- ``n`` -- a positive integer (the gonality)
- ``num_jobs`` -- number of jobs into which this computation will be broken (default: 1)
- ``job`` -- index of this job in the full computation (default: 0)
- Additional keyword arguments are passed to progress.Progress
OUTPUT:
- set of all irreducible totally T-adic polynomial over GF(2)[T] of x-degree
`3n` and height `1/3`
NOTES:
- The naive search space has size `f(n)` bits, where
..math:: f(n) = 2n^2 + 3n - 3
This uses a linear algebra approach to cut the naive search space down by
about `4n` bits.
- This function is an implementation of Algorithm 2 in [FP], and is used
for the cases n = 1, 2, 3
"""
xdegree = 3 * n
poly_bits = 2 * n**2 + 3 * n - 3
def make_poly(r, Tring, xring, coefs):
r"""
Construct a polynomial with special Newton poly from a list of coefficients
"""
aa = []
xx = coefs
xx_ptr = 0
T = Tring.gen()
x = xring.gen()
aa.append(Tring(1)) # constant coefficient
for i in range(1, r):
# negative slope terms
aa.append(sum(T**k * xx[xx_ptr + k] for k in range(i + 1)))
xx_ptr += i + 1
aa.append(1 + sum(T**k * xx[xx_ptr + k - 1]
for k in range(1, r + 1))) # vertex term
xx_ptr += r
for i in range(r + 1, 2 * r):
aa.append(sum(T**k * xx[xx_ptr + k]
for k in range(r + 1))) # slope-0 terms
xx_ptr += r + 1
aa.append(1 + sum(T**k * xx[xx_ptr + k - 1]
for k in range(1, r + 1))) # vertex term
xx_ptr += r
for i in range(1, r):
aa.append(sum(T**k * xx[xx_ptr + k] for k in range(r - i + 1)))
xx_ptr += r - i + 1
aa.append(Tring(1)) # leading coefficient
f = sum(T**(r - i) * aa[i] * x**i for i in range(r + 1))
f += sum(aa[i] * x**i for i in range(r + 1, 2 * r + 1))
f += sum(T**i * aa[2 * r + i] * x**(2 * r + i)
for i in range(1, r + 1))
return f
# Construct linear equations using a polynomial in several variables
FF = GF(2)
xx = ['x{}'.format(i) for i in range(poly_bits)]
A = PolynomialRing(FF, names=xx)
xx = A.gens()
AT = PolynomialRing(A, names=('T', ))
RT = PolynomialRing(AT, names=('x', ))
T = AT.gen()
x = RT.gen()
f = make_poly(n, AT, RT, xx)
eqns = []
# Get info from f(T); note first r coefficients vanish
coefs = f(T).dict()
for i in range(n, 2 * n):
if i not in coefs: continue
eqns.append(coefs[i])
# get info from f(T+1)
coefs = f(T + 1).dict()
for i in range(2 * n):
if i not in coefs: continue
eqns.append(coefs[i])
# get info from f(1/T); use f^rev; note first n coefficients vanish
frev = RT(x**(3 * n) * f(1 / x))
coefs = frev(T).dict()
for i in range(n, 2 * n):
if i not in coefs: continue
eqns.append(coefs[i])
# Turn equations into a linear system and solve
zero_vec = tuple([0] * (len(xx)))
Arows = []
b = []
for eqn in eqns:
vecs = set(tuple(term) for term in eqn.dict())
if zero_vec in vecs:
b.append(FF(1))
else:
b.append(FF(0))
w = sum(vector(FF, v) for v in vecs)
Arows.append(w)
A = matrix(FF, Arows)
sol = A.solve_right(vector(FF, b))
ker_matrix = A.right_kernel().matrix()
ker_matrix = ker_matrix.echelon_form(
) # For safety; solve_right should already do this.
ker = ker_matrix.rows()
# Construct solution polynomials and perform
# irreducibility check and totally T-adic check
K = FunctionField(FF, names=('T', ))
R = PolynomialRing(K, names=('x', ))
T = K.gen()
x = R.gen()
v0 = maclane.function_field_inductive_valuation(T, key_polynomial=x)
start, stop = start_and_stop_work(2**len(ker), num_jobs, job)
work = stop - start
print('Checking 2^{:.1f} polynomials\n'.format(math.log(work, 2)))
if num_jobs != 1:
print(' start = {}, stop = {}'.format(start, stop))
sys.stdout.flush()
prog = progress.Progress(work, **kwargs)
irreducible_test = 0
S = set()
for coefs in affine_space_point_iterator_GF2(len(ker),
start=start,
stop=stop):
prog()
cc = sol + sum(c * v for c, v in zip(coefs, ker))
cc = tuple(K(c) for c in cc)
f = make_poly(n, K, R, cc)
prog.profile('make_poly')
if f(1) == 0: continue
# print(f)
f_is_irreducible = f.is_irreducible()
prog.profile('irreducible')
if not f_is_irreducible: continue
irreducible_test += 1
f_is_totally_T_adic = special_is_totally_T_adic(n, v0, f)
prog.profile('totally_T_adic')
if not f_is_totally_T_adic: continue
S.add(f)
prog.finalize()
print('{:5d} are irreducible'.format(irreducible_test))
print('{:5d} are totally T-adic'.format(len(S)))
return S