def _distance_to_W(self, ids=None):
allneighbors = {}
weights = {}
if ids:
ids = np.array(ids)
else:
ids = np.arange(len(self._nmat))
if self.binary:
for i, neighbors in enumerate(self._nmat):
ns = [ni for ni in neighbors if ni != i]
neigh = list(ids[ns])
if len(neigh) == 0:
allneighbors[ids[i]] = []
weights[ids[i]] = []
else:
allneighbors[ids[i]] = neigh
weights[ids[i]] = [1] * len(ns)
else:
self.dmat = self.kd.sparse_distance_matrix(
self.kd, max_distance=self.threshold)
for i, neighbors in enumerate(self._nmat):
ns = [ni for ni in neighbors if ni != i]
neigh = list(ids[ns])
if len(neigh) == 0:
allneighbors[ids[i]] = []
weights[ids[i]] = []
else:
try:
allneighbors[ids[i]] = neigh
weights[ids[i]] = [self.dmat[(
i, j)] ** self.alpha for j in ns]
except ZeroDivisionError:
raise Exception, "Cannot compute inverse distance for elements at same location (distance=0)."
return allneighbors, weights
def generate_pdb(self,pdb_name,shift=0,traduction={"1":"bead","2":"telo","3":"ribo","4":"cent","5":"spbb","6":"rcut","7":"scut"}):
"""
Generate a pdb file according to the molecules given to the LSimu object
traduction must contain as key the type of bead and as item the pdb name
(should be 4 letters name)
(Very ugly code)
"""
#Initial stuff for pdb:
towrite = []
atomid = 0
for n,molecule in enumerate(self.molecules):
for rn,((x,y,z),type_bead) in enumerate(zip(molecule.coords,molecule.types_beads)):
atomid+= 1
name="bead"
residue_name="bea"
chain_id=string.ascii_letters[(n + shift) % len(string.ascii_letters)]
residue_number=rn
name = traduction["%i" % type_bead]
towrite.append(self.one_pdb_atom(x,y,z,atomid,name,residue_name,residue_number,chain_id))
with open(pdb_name,"w") as f:
f.write("".join(towrite))
def initializeGUI(self):
self.d = {"Switches": {}, "Triggers": {}}
# set layout
layout = QtGui.QGridLayout()
self.setLayout(layout)
# get switch names and add them to the layout, and connect their function
layout.addWidget(QtGui.QLabel("Switches"), 0, 0)
switchNames = yield self.server.get_switching_channels()
for order, name in enumerate(switchNames):
button = QtGui.QPushButton(name)
self.d["Switches"][name] = button
button.setCheckable(True)
initstate = yield self.server.get_state(name)
button.setChecked(initstate)
self.setButtonText(button, name)
button.clicked.connect(self.buttonConnection(name, button))
layout.addWidget(button, 0, 1 + order)
# do same for trigger channels
layout.addWidget(QtGui.QLabel("Triggers"), 1, 0)
triggerNames = yield self.server.get_trigger_channels()
for order, name in enumerate(triggerNames):
button = QtGui.QPushButton(name)
button.clicked.connect(self.triggerConnection(name))
self.d["Triggers"][name] = button
layout.addWidget(button, 1, 1 + order)
def restore_sources(self, pubkey, tx):
"""
Restore the sources of a cancelled tx
:param sakia.entities.Transaction tx:
"""
txdoc = TransactionDoc.from_signed_raw(tx.raw)
for offset, output in enumerate(txdoc.outputs):
if output.conditions.left.pubkey == pubkey:
source = Source(currency=self.currency,
pubkey=pubkey,
identifier=txdoc.sha_hash,
type='T',
noffset=offset,
amount=output.amount,
base=output.base)
self._sources_processor.drop(source)
for index, input in enumerate(txdoc.inputs):
source = Source(currency=self.currency,
pubkey=txdoc.issuers[0],
identifier=input.origin_id,
type=input.source,
noffset=input.index,
amount=input.amount,
base=input.base)
if source.pubkey == pubkey:
self._sources_processor.insert(source)
def calculate_zernikes(self, workspace):
zernike_indexes = cpmz.get_zernike_indexes(self.zernike_degree.value + 1)
meas = workspace.measurements
for o in self.objects:
object_name = o.object_name.value
objects = workspace.object_set.get_objects(object_name)
#
# First, get a table of centers and radii of minimum enclosing
# circles per object
#
ij = np.zeros((objects.count + 1, 2))
r = np.zeros(objects.count + 1)
for labels, indexes in objects.get_labels():
ij_, r_ = minimum_enclosing_circle(labels, indexes)
ij[indexes] = ij_
r[indexes] = r_
#
# Then compute x and y, the position of each labeled pixel
# within a unit circle around the object
#
ijv = objects.ijv
l = ijv[:, 2]
yx = (ijv[:, :2] - ij[l, :]) / r[l, np.newaxis]
z = cpmz.construct_zernike_polynomials(
yx[:, 1], yx[:, 0], zernike_indexes)
for image_group in self.images:
image_name = image_group.image_name.value
image = workspace.image_set.get_image(
image_name, must_be_grayscale=True)
pixels = image.pixel_data
mask = (ijv[:, 0] < pixels.shape[0]) & \
(ijv[:, 1] < pixels.shape[1])
mask[mask] = image.mask[ijv[mask, 0], ijv[mask, 1]]
yx_ = yx[mask, :]
l_ = l[mask]
z_ = z[mask, :]
if len(l_) == 0:
for i, (n, m) in enumerate(zernike_indexes):
ftr = self.get_zernike_magnitude_name(image_name, n, m)
meas[object_name, ftr] = np.zeros(0)
if self.wants_zernikes == Z_MAGNITUDES_AND_PHASE:
ftr = self.get_zernike_phase_name(image_name, n, m)
meas[object_name, ftr] = np.zeros(0)
continue
areas = scind.sum(
np.ones(l_.shape, int), labels=l_, index=objects.indices)
for i, (n, m) in enumerate(zernike_indexes):
vr = scind.sum(
pixels[ijv[mask, 0], ijv[mask, 1]] * z_[:, i].real,
labels=l_, index=objects.indices)
vi = scind.sum(
pixels[ijv[mask, 0], ijv[mask, 1]] * z_[:, i].imag,
labels=l_, index=objects.indices)
magnitude = np.sqrt(vr * vr + vi * vi) / areas
ftr = self.get_zernike_magnitude_name(image_name, n, m)
meas[object_name, ftr] = magnitude
if self.wants_zernikes == Z_MAGNITUDES_AND_PHASE:
phase = np.arctan2(vr, vi)
ftr = self.get_zernike_phase_name(image_name, n, m)
meas[object_name, ftr] = phase
def load_text(self):
'''
The text of instances are not stored in the prediction result file,
so you need to call this function to load texts from testing data.
>>> from libshorttext.analyzer import *
>>> insts = InstanceSet('prediction_result_path')
>>> insts.load_text()
This method also load the extra svm features if extra svm files
are used when training.
'''
EMPTY_MESSAGE = '**None**'
sorted_insts = sorted(self.insts, key = lambda inst: inst.idx)
i = 0
for idx, lines in enumerate(izip(*([open(self.filepath, 'r')] + [open(f, 'r') for f in self.extra_svm_files]))):
line = lines[0]
extra_svm_feats = lines[1:]
nr_extra_svm_feats = len(extra_svm_feats)
if idx > sorted_insts[-1].idx:
break
if idx == sorted_insts[i].idx:
try:
sorted_insts[i].text = line.split('\t',1)[1].strip()
except:
sorted_insts[i].text = EMPTY_MESSAGE
sorted_insts[i].extra_svm_feats = [None] * nr_extra_svm_feats
for j, extra_svm_feat in enumerate(extra_svm_feats):
try:
sorted_insts[i].extra_svm_feats[j] = dict(map(lambda t: (int(t[0]), float(t[1])), [feat.split(':') for feat in extra_svm_feat.split(None, 1)[1].split()]))
except:
sorted_insts[i].extra_svm_feats[j] = EMPTY_MESSAGE
i += 1
def _eval_kernel(self):
# get points within bandwidth distance of each point
if not hasattr(self, 'neigh'):
kdtq = self.kdt.query_ball_point
neighbors = [kdtq(self.data[i], r=bwi[0]) for i,
bwi in enumerate(self.bandwidth)]
self.neigh = neighbors
# get distances for neighbors
data = np.array(self.data)
bw = self.bandwidth
kdtq = self.kdt.query
z = []
for i, nids in enumerate(self.neigh):
di, ni = kdtq(self.data[i], k=len(nids))
zi = np.array([dict(zip(ni, di))[nid] for nid in nids]) / bw[i]
z.append(zi)
zs = z
# functions follow Anselin and Rey (2010) table 5.4
if self.function == 'triangular':
self.kernel = [1 - z for z in zs]
elif self.function == 'uniform':
self.kernel = [np.ones(z.shape) * 0.5 for z in zs]
elif self.function == 'quadratic':
self.kernel = [(3. / 4) * (1 - z ** 2) for z in zs]
elif self.function == 'quartic':
self.kernel = [(15. / 16) * (1 - z ** 2) ** 2 for z in zs]
elif self.function == 'gaussian':
c = np.pi * 2
c = c ** (-0.5)
self.kernel = [c * np.exp(-(z ** 2) / 2.) for z in zs]
else:
print 'Unsupported kernel function', self.function
def load_records(self, records):
"""Load in a database and interpret it as a network
First column must be unique keys which define the instance units.
Each column is a pool (names, gangs, ages, etc).
Every row is mutually excitory.
"""
self.units[:] = []
self.pools[:] = []
for counter,record in enumerate(records):
relatedunits = ['%s:%s' % (k,v) for (k,v) in record.items()]
relatedunits.insert(0,'_:%s'% counter)
if not len(relatedunits): continue
key = len(self.units)
for poolnum, name in enumerate(relatedunits):
if poolnum >= len(self.pools):
self.pools.append(Pool(self.unitbyname))
pool = self.pools[poolnum]
if name in self.unitbyname:
unit = self.unitbyname[name]
else:
unit = Unit(name, pool, self.unitbyname)
self.units.append(unit)
pool.addmember(unit)
if poolnum > 0:
self.units[key].addexciter(unit)
unit.addexciter(self.units[key])
def plot_overtime(data_file):
data = performance.load_score_dict(data_file)
avg_sim = []
std_sim = []
# Lets compute the average fraction of matching paths for each case
for index, time_step in enumerate(data):
if index == 0:
continue
prev_step = data[index - 1]
sim_list = []
for pair_index, pair in enumerate(time_step):
curr_chain = set([x[0] for x in pair])
print curr_chain
prev_chain = set([x[0] for x in prev_step[pair_index]])
if len(curr_chain) == 0 or len(prev_chain) == 0:
continue
sim = float(len(curr_chain & prev_chain)) / len(curr_chain)
sim_list.append(sim)
avg_sim.append(np.mean(sim_list))
std_sim.append(np.std(sim_list))
print "Next Time Step!"
plotting.overtime_plot(avg_sim, std_sim)
def recompute_unread(min_date = None):
from r2.models import Inbox, Account, Comment, Message
from r2.lib.db import queries
def load_accounts(inbox_rel):
accounts = set()
q = inbox_rel._query(eager_load = False, data = False,
sort = desc("_date"))
if min_date:
q._filter(inbox_rel.c._date > min_date)
for i in fetch_things2(q):
accounts.add(i._thing1_id)
return accounts
accounts_m = load_accounts(Inbox.rel(Account, Message))
for i, a in enumerate(accounts_m):
a = Account._byID(a)
print "%s / %s : %s" % (i, len(accounts_m), a)
queries.get_unread_messages(a).update()
queries.get_unread_comments(a).update()
queries.get_unread_selfreply(a).update()
accounts = load_accounts(Inbox.rel(Account, Comment)) - accounts_m
for i, a in enumerate(accounts):
a = Account._byID(a)
print "%s / %s : %s" % (i, len(accounts), a)
queries.get_unread_comments(a).update()
queries.get_unread_selfreply(a).update()
def load_level(filename):
""" Loads a level from a given text file.
Args:
filename: The name of the txt file containing the maze.
Returns:
The loaded level (dict) containing the locations of walls (set), the locations of spaces (dict), and
a mapping of locations to waypoints (dict).
"""
walls = set()
spaces = {}
waypoints = {}
with open(filename, "r") as f:
for j, line in enumerate(f.readlines()):
for i, char in enumerate(line):
if char == '\n':
continue
elif char == WALL:
walls.add((i, j))
elif char.isnumeric():
spaces[(i, j)] = float(char)
elif char.islower():
spaces[(i, j)] = 1.
waypoints[char] = (i, j)
level = {'walls': walls,
'spaces': spaces,
'waypoints': waypoints}
return level
def links(self, data_matrix):
data_size = data_matrix.shape[0]
kernel_matrix = pairwise_kernels(data_matrix, metric=self.metric, **self.kwds)
# compute instance density as average pairwise similarity
density = np.sum(kernel_matrix, 0) / data_size
# compute list of nearest neighbors
kernel_matrix_sorted = np.argsort(-kernel_matrix)
# make matrix of densities ordered by nearest neighbor
density_matrix = density[kernel_matrix_sorted]
# if a denser neighbor cannot be found then assign link to the instance itself
link_ids = list(range(density_matrix.shape[0]))
# for all instances determine link link
for i, row in enumerate(density_matrix):
i_density = row[0]
# for all neighbors from the closest to the furthest
for jj, d in enumerate(row):
# proceed until n_nearest_neighbors have been explored
if self.n_nearest_neighbors is not None and jj > self.n_nearest_neighbors:
break
j = kernel_matrix_sorted[i, jj]
if jj > 0:
j_density = d
# if the density of the neighbor is higher than the density of the instance assign link
if j_density > i_density:
link_ids[i] = j
break
return link_ids
def flux_matrix(self, geo):
"""Returns a sparse matrix which can be used to multiply a vector of connection table values for underground
blocks, to give approximate average fluxes of those values at the block centres."""
natm = geo.num_atmosphere_blocks
nele = geo.num_underground_blocks
conindex = dict([((c.block[0].name, c.block[1].name), i) for i, c in enumerate(self.connectionlist)])
from scipy import sparse
A = sparse.lil_matrix((3 * nele, self.num_connections))
if not self.block_centres_defined:
self.calculate_block_centres(geo)
for iblk, blk in enumerate(self.blocklist[natm:]):
ncons = blk.num_connections
for conname in blk.connection_name:
otherindex, sgn = [(0, -1), (1, 1)][conname[0] == blk.name]
blk2name = conname[otherindex]
icon = conindex[conname]
centre2 = self.block[blk2name].centre
if centre2 <> None:
n = centre2 - blk.centre
n /= np.linalg.norm(n)
else:
n = np.array([0, 0, 1]) # assumed connection to atmosphere
for i, ni in enumerate(n):
A[3 * iblk + i, icon] = -sgn * ni / (ncons * self.connection[conname].area)
return A
def rx_oversampled(frames, ref_frame, modulated_frame, x_preamble, data, rx_kernel, demapper, timeslots, fft_len, cp_len, cs_len):
ref_frame_os = signal.resample(ref_frame, 2 * len(ref_frame))
x_preamble_os = signal.resample(x_preamble, 2 * len(x_preamble))
nyquist_frame_len = cp_len + 2 * fft_len + cs_len + cp_len + timeslots * fft_len + cs_len
n_frames = np.shape(frames)[0]
sync_frames = np.zeros((n_frames, nyquist_frame_len), dtype=np.complex)
print('nyquist sampled frame len', nyquist_frame_len, 'with n_frames', n_frames)
f_start = cp_len + 2 * fft_len + cs_len
d_start = f_start + cp_len
print('data start: ', d_start)
for i, f in enumerate(frames[0:2]):
tf = np.roll(f, 1)
tf[0] = 0
ff = signal.resample(tf, len(f) // 2)
sframe = synchronize_time(ff, ref_frame_os, x_preamble_os, 2 * fft_len, 2 * cp_len)
sframe = signal.resample(sframe, len(sframe) // 2)
sframe = synchronize_freq_offsets(sframe, modulated_frame, x_preamble, fft_len, cp_len, samp_rate=3.125e6)
print(len(sframe), len(ref_frame))
rx_preamble = sframe[cp_len:cp_len + 2 * fft_len]
avg_phase = calculate_avg_phase(rx_preamble, x_preamble)
# m, c = calculate_avg_phase(rx_preamble, x_preamble)
# avg_phase = calculate_avg_phase(sframe, ref_frame)
# phase_eqs = m * np.arange(-cp_len, len(sframe) - cp_len) + c
# sframe *= np.exp(-1j * phase_eqs)
# sframe *= np.exp(-1j * avg_phase)
sync_frames[i] = sframe
rx_data_frame = sframe[d_start:d_start + fft_len * timeslots]
# # rx_data_frame *= np.exp(-1j * avg_phase)
#
demodulate_frame(rx_data_frame, modulated_frame, rx_kernel, demapper, data, timeslots, fft_len)
for i, f in enumerate(sync_frames[0:3]):
rx_data_frame = f[d_start:d_start + fft_len * timeslots]
demodulate_frame(rx_data_frame, modulated_frame, rx_kernel, demapper, data, timeslots, fft_len)
def model(self):
model = {
"hostname": "",
"dns[0]": "",
"dns[1]": "",
"ntp[0]": "",
"ntp[1]": "",
"bond.name": "",
"bond.slaves.selected": "",
"bond.options": defaults.NicBonding.default_options
}
model["hostname"] = defaults.Hostname().retrieve()["hostname"] or \
network.hostname()
# Pull name-/timeservers from config files (not defaults)
nameservers = config.network.nameservers()
if nameservers:
for idx, nameserver in enumerate(nameservers):
model["dns[%d]" % idx] = nameserver
timeservers = config.network.timeservers()
if timeservers:
for idx, timeserver in enumerate(timeservers):
model["ntp[%d]" % idx] = timeserver
model.update(self._model_extra)
return model
def _make_scalar_compound_controller(self, fcurves, keyframes, bez_chans, default_xform):
ctrl = plCompoundController()
subctrls = ("X", "Y", "Z")
for i in subctrls:
setattr(ctrl, i, plLeafController())
exported_frames = ([], [], [])
ctrl_fcurves = { i.array_index: i for i in fcurves }
for keyframe in keyframes:
for i, subctrl in enumerate(subctrls):
fval = keyframe.values.get(i, None)
if fval is not None:
keyframe_type = hsKeyFrame.kBezScalarKeyFrame if i in bez_chans else hsKeyFrame.kScalarKeyFrame
exported = hsScalarKey()
exported.frame = keyframe.frame_num
exported.frameTime = keyframe.frame_time
exported.inTan = keyframe.in_tans[i]
exported.outTan = keyframe.out_tans[i]
exported.type = keyframe_type
exported.value = fval
exported_frames[i].append(exported)
for i, subctrl in enumerate(subctrls):
my_keyframes = exported_frames[i]
# ensure this controller has at least ONE keyframe
if not my_keyframes:
hack_frame = hsScalarKey()
hack_frame.frame = 0
hack_frame.frameTime = 0.0
hack_frame.type = hsKeyFrame.kScalarKeyFrame
hack_frame.value = default_xform[i]
my_keyframes.append(hack_frame)
getattr(ctrl, subctrl).keys = (my_keyframes, my_keyframes[0].type)
return ctrl
def extractNames(li):
finList = []
## Loop through the list that has the HTML page content
for a in li:
## Tokenize the HTML text into smaller blocks of text
for send in nltk.sent_tokenize(str(a)):
smLi = []
## Tokenize the smaller blocks of text in individual words and then add a Part-of-Speech(POS) tag
for index, chunk in enumerate(nltk.pos_tag(nltk.word_tokenize(send))):
## If the POS tag is NNP (noun)
if 'NNP' in chunk[1]:
## If the each character in the word is an alphanumeric character and there are more than 2 characters in the word
if(len(' '.join(e for e in chunk[0] if e.isalnum())) > 2):
## Append the list with the index of the word, chunk that has the POS tag and the link
smLi.append([index, chunk, a[1]])
finList.append(smLi)
nameLi = []
for f in finList:
if len(f) > 0:
strName = ''
for index, i in enumerate(f):
## If strName is blank, declare it with the current word in the list
if strName == '':
strName = i[1][0]
## If index+1 is not at the end of the list, continue
if (index + 1) < len(f):
## If the index is a consecutive index, add to the strName
if i[0] + 1 == f[index + 1][0]:
strName = strName + ' ' + f[index + 1][1][0]
## If the index is not a consecutive index, append strName to the nameLi list with the article link and make the strName blank
else:
if ' ' in strName:
nameLi.append([strName, i[2]])
strName = ''
return nameLi
def loadTrajectoryData(inFile = UJILocDataFile):
with open(UJILocDataFile, 'r') as dataFile:
data = dataFile.read()
# 9-axis IMU data
# trajectory: dictionary with three elements
# N is number of samples in the trajectory (data taken at 10Hz)
# mag: Nx3 numpy array where each line has XYZ mag data
# gyro: Nx3 numpy array where each line has XYZ gyro vel data
# accel: Nx3 numpy array where each line has XYZ lin accelerometer data
segments = data.split("<", 2)
IMUDataStr = segments[0].split('\n')[:-1]
magArr = []
oriArr = []
accelArr = []
for i, lineStr in enumerate(IMUDataStr):
lineStr = lineStr.split(' ', 10)[:-1]
lineStr = [float(x) for x in lineStr]
magArr.append(lineStr[1:4]) # xyz mag data for sample
accelArr.append(lineStr[4:7]) # xyz accelerometer data for single samp
oriArr.append(lineStr[7:10]) # xyz gyro data for sample
# values initially are given as euler angles which are not good for imu-type calculations.
# so we fix em!
gyroArr = rawSensorStateProc.orientationToGyro(oriArr)
initOrientationMatrix = rawSensorStateProc.calcInitialOrientation(oriArr[0])
# IMUData = [{'mag': magArr, 'gyro': gyroArr, 'accel': accelArr}]
# process waypoint data
# each waypoint consists of a latitude coordinate, longitude coordinate,
# and index (what IMU dataopoint it represents)
waypoints = []
waypointStr = segments[1].split(">", 2)
numWaypoints = int(waypointStr[0])
waypointLns = waypointStr[1].lstrip().split('\n')
for i, lineStr in enumerate(waypointLns):
line = lineStr.split(' ', WAYPOINTS_ELEMS_PER_LINE)
line = [float(x) for x in line]
if i == 0:
waypoints.append({'lat': line[0], 'long': line[1], 'index': line[4]})
waypoints.append({'lat': line[2], 'long': line[3], 'index': line[5]})
seqLen = line[5]
traj = ({'waypoints': np.array(waypoints), 'mag': np.array(magArr), 'gyro': np.array(gyroArr),
'accel': np.array(accelArr), 'orientSensed': np.array(oriArr),
'initOrient': initOrientationMatrix, 'seqLen': seqLen})
return traj
# loadTrajectoryData()
def run_image_find(self):
if self.ss_labels is None or self.s_labels is None:
return
featured_labels = self.tree.get_featured( VARIABLE['edge_mode'] )
# print( featured_labels )
object_color = { }
color = 1
label_img = numpy.zeros( (len( self.ss_labels), len( self.s_labels[0])) )
for object_name, object_featured in featured_labels.items():
object_color[ object_name ] = color
for i, ss_label in enumerate( self.ss_labels ):
print(i, ss_label, object_featured.keys())
if ss_label in object_featured.keys():
for j, label in enumerate( self.s_labels[i] ):
if label in object_featured[ ss_label ]:
label_img[i,j] = color
color += 1
for row in label_img:
print(row)
print( self.ss_labels )
print( object_color )
self.piximage.color_objects(label_img)
def get_unigram_probs(vocab, counts, smooth_constant):
special_symbol_ids = [vocab[x] for x in SPECIAL_SYMBOLS]
vocab_size = len(vocab) - len(SPECIAL_SYMBOLS)
num_words_with_non_zero_counts = 0
for word_id, count in enumerate(counts):
if word_id in special_symbol_ids:
continue
if counts[word_id] > 0:
num_words_with_non_zero_counts += 1
if num_words_with_non_zero_counts < vocab_size and smooth_constant == 0.0:
sys.exit(sys.argv[0] + ": --smooth-unigram-counts should not be zero, "
"since there are words with zero-counts")
smooth_count = smooth_constant * num_words_with_non_zero_counts / vocab_size
total_counts = 0.0
for word_id, count in enumerate(counts):
if word_id in special_symbol_ids:
continue
counts[word_id] += smooth_count
total_counts += counts[word_id]
probs = []
for count in counts:
probs.append(count / total_counts)
return probs
def rectangle_plot(self, xparams, yparams, yroots, filled=True, ymarkers=None, xmarkers=None, **kwargs):
self.make_figure(nx=len(xparams), ny=len(yparams))
# f, plots = subplots(len(yparams), len(xparams), sharex='col', sharey='row')
sharey = None
yshares = []
for x, xparam in enumerate(xparams):
sharex = None
for y, (yparam, roots) in enumerate(zip(yparams, yroots)):
# f.sca(plots[y, x])
if x > 0: sharey = yshares[y]
ax = self.subplot(x + 1, y + 1, sharex=sharex, sharey=sharey)
if y == 0: sharex = ax
self.plot_2d(roots, param_pair=[xparam, yparam], filled=filled, do_xlabel=y == len(yparams) - 1,
do_ylabel=x == 0, add_legend_proxy=x == 0 and y == 0)
if ymarkers is not None and ymarkers[y] is not None: self.add_y_marker(ymarkers[y], **kwargs)
if xmarkers is not None and xmarkers[x] is not None: self.add_x_marker(xmarkers[x], **kwargs)
if y == 0: lims = xlim()
else: lims = (min(xlim()[0], lims[0]), max(xlim()[1], lims[1]))
if y != len(yparams) - 1: setp(ax.get_xticklabels(), visible=False)
if x != 0: setp(ax.get_yticklabels(), visible=False)
if x == 0: yshares.append(ax)
sharex.set_xlim(lims)
self.spaceTicks(sharex.xaxis)
sharex.set_xlim(sharex.xaxis.get_view_interval())
for ax in yshares:
self.spaceTicks(ax.yaxis)
ax.set_ylim(ax.yaxis.get_view_interval())
subplots_adjust(wspace=0, hspace=0)
self.finish_plot(no_gap=True)
def classListBox2(self, id): pageNumber = self.noteBook.GetCurrent() cname = self.classBox[pageNumber].GetEntry(id).GetText().Data() self.methodBox[pageNumber].RemoveAll() self.labelBox[pageNumber].RemoveAll() methods = self.cmap[cname]['methods'] labels= self.cmap[cname]['labels'] self.cmap[cname]['selected'] = True # List methods names = self.cmap[cname]['sortedmethods'] for index, name in enumerate(names): if not methods[name]: continue self.methodBox[pageNumber].AddEntry(name, index) self.methodBox[pageNumber].Layout() # List getByLabels names = labels.keys() names.sort() for index, name in enumerate(names): if not labels[name]: continue self.labelBox[pageNumber].AddEntry(name, index) self.labelBox[pageNumber].Layout()
def WorkBook_writeSheet(self, filename):
columns = ['Date', 'Month', 'ID #', 'Contact ID #', \
'Talked to Person X?', 'Closeness/Trust with X', \
'Connecting ID', 'Connector ID']
# Writes to csv file
with open(filename, 'w') as f:
writer = csv.writer(f)
writer.writerow(columns)
for row in self.sheet:
date = row[DATE_COLUMN]
month = row[MONTH_COLUMN]
connecting = row[CONNECTING_COLUMN]
connector = row[CONNECTOR_COLUMN]
talkVal = row[TALKED_WEIGHT]
closeVal = row[CLOSENESS_WEIGHT]
connectingID = row[CONNECTING_ID_COLUMN]
connectorID = row[CONNECTOR_ID_COLUMN]
row = [date, month, connecting, connector, talkVal, \
closeVal, connectingID, connectorID]
writer.writerow(row)
# Converts from the written csv file to xlsx
for csvfile in glob.glob(os.path.join('.', '*.csv')):
workbook = Workbook(csvfile[0:-4] + '.xlsx')
worksheet = workbook.add_worksheet()
with open(csvfile, 'rb') as f:
reader = csv.reader(f)
for r, row in enumerate(reader):
for c, col in enumerate(row):
worksheet.write(r, c, col)
workbook.close()
sys.exit()
def validate_label_generation():
mals1_df = pd.read_csv('data/sorted-train-labels-vs251-252.csv')
mals2_df = pd.read_csv('data/sorted-train-labels-vs263-264-apt.csv')
counter = 0
m1_x = np.array(mals1_df['malware_type_x'])
m1_f = np.array(mals1_df['family_name'])
m1_sl = np.array(mals1_df['sample_label'])
m1_fl = np.array(mals1_df['family_label'])
m2_x = np.array(mals2_df['malware_type_x'])
m21_f = np.array(mals2_df['family_name'])
m2_sl = np.array(mals2_df['sample_label'])
m2_fl = np.array(mals2_df['family_label'])
for idx1, mname1 in enumerate(m1_x):
for idx2, mname2 in enumerate(m2_x):
if mname1 == mname2:
if m1_sl[idx1] != m2_sl[idx2]:
print("Sample label incongruence: {:d} {:d}".format(m1_sl[idx1], m2_sl[idx2]))
counter += 1
if (m1_fl[idx1] != m2_fl[idx2]):
print("Family label incongruence: {:d} {:d}".format(m1_fl[idx1], m2_fl[idx2]))
counter += 1
if (idx1 % 1000) == 0:
print("Processed {:d} malware names.".format(idx1))
print("Total Incongruence Errors: {:d}".format(counter))
return
def knapsack_unbounded_dp(items, C):
# order by max value per item size
items = sorted(items, key=lambda item: item[VALUE]/float(item[SIZE]), reverse=True)
# Sack keeps track of max value so far as well as the count of each item in the sack
print('!')
sack = [(0, [0 for i in items]) for i in range(0, C+1)] # value, [item counts]
print('!')
for i,item in enumerate(items):
name, size, value = item
for c in range(size, C+1):
print(sack)
sackwithout = sack[c-size] # previous max sack to try adding this item to
trial = sackwithout[0] + value
used = sackwithout[1][i]
if sack[c][0] < trial:
# old max sack with this added item is better
sack[c] = (trial, sackwithout[1][:])
sack[c][1][i] +=1 # use one more
value, bagged = sack[C]
numbagged = sum(bagged)
size = sum(items[i][1]*n for i,n in enumerate(bagged))
# convert to (iten, count) pairs) in name order
bagged = sorted((items[i][NAME], n) for i,n in enumerate(bagged) if n)
return value, size, numbagged, bagged
def testPeriodsMonths(self):
""" Test iteration over periods (months) """
dt = datetime.datetime
ef = S3TimePlotEventFrame(dt(2011, 1, 5),
dt(2011, 4, 28),
slots="months")
expected = [(dt(2011, 1, 5), dt(2011, 2, 5)),
(dt(2011, 2, 5), dt(2011, 3, 5)),
(dt(2011, 3, 5), dt(2011, 4, 5)),
(dt(2011, 4, 5), dt(2011, 4, 28))]
for i, period in enumerate(ef):
self.assertEqual(period.start, expected[i][0])
self.assertEqual(period.end, expected[i][1])
ef = S3TimePlotEventFrame(dt(2011, 1, 5),
dt(2011, 8, 16),
slots="3 months")
expected = [(dt(2011, 1, 5), dt(2011, 4, 5)),
(dt(2011, 4, 5), dt(2011, 7, 5)),
(dt(2011, 7, 5), dt(2011, 8, 16))]
for i, period in enumerate(ef):
self.assertEqual(period.start, expected[i][0])
self.assertEqual(period.end, expected[i][1])
def categorize(data, colnum, missingvals, ranges=[]):
categories = set()
for row in data:
if row[colnum] not in missingvals:
categories.add(row[colnum])
catlist = list(categories)
catlist.sort()
# print(', '.join(['%i: %s' % (n, catlist[n]) for n in xrange(len(catlist))]), "(with missing vals:", missingvals, ")")
missing_indices = []
for index, row in enumerate(data):
if row[colnum] in missingvals: # missing data
row[colnum] = 0
missing_indices.append(index)
else: # this row doesn't have missing data.
if len(ranges) > 0: # find val in ranges and use that index.
found = False
for i, r in enumerate(ranges):
if isinstance(r, basestring): # compare strings.
if r in row[colnum]:
row[colnum] = i
found = True
break
elif isinstance(r, ( int, long )) and not re.search('[a-zA-Z]', row[colnum]):
# ref : http://stackoverflow.com/questions/3501382/checking-whether-a-variable-is-an-integer-or-not
if float(row[colnum]) >= r and len(ranges) > i+1 and isinstance(ranges[i+1], ( int, long )) and float(row[colnum]) < ranges[i+1]:
row[colnum] = i
found = True
break
if not found:
print(row[colnum]) # error here
else: # no ranges given, so just set category of appearance.
row[colnum] = catlist.index(row[colnum])+1
return missing_indices
def lcs_dy_prog(s1, s2):
table = np.zeros((len(s1), len(s2)), dtype=np.int)
def lookup(i, j):
if i < 0 or j < 0:
return 0
else:
return table[i, j]
# find length of the lcs
for i, c1 in enumerate(s1):
for j, c2 in enumerate(s2):
if c1 == c2:
table[i, j] = lookup(i - 1, j - 1) + 1
else:
table[i, j] = max(lookup(i - 1, j), lookup(i, j - 1))
# backtrac to find lcs (not unique)
i = len(s1) - 1
j = len(s2) - 1
res = ""
while i >= 0 and j >= 0:
if s1[i] == s2[j]:
res += s1[i]
i -= 1
j -= 1
else:
if lookup(i - 1,j) > lookup(i, j - 1):
i -= 1
else:
j -= 1
res = res[::-1]
return res
def testPeriodsWeeks(self):
""" Test iteration over periods (weeks) """
dt = datetime.datetime
ef = S3TimePlotEventFrame(dt(2011, 1, 5),
dt(2011, 1, 28),
slots="weeks")
expected = [(dt(2011, 1, 5), dt(2011, 1, 12)),
(dt(2011, 1, 12), dt(2011, 1, 19)),
(dt(2011, 1, 19), dt(2011, 1, 26)),
(dt(2011, 1, 26), dt(2011, 1, 28))]
for i, period in enumerate(ef):
self.assertEqual(period.start, expected[i][0])
self.assertEqual(period.end, expected[i][1])
ef = S3TimePlotEventFrame(dt(2011, 1, 5),
dt(2011, 2, 16),
slots="2 weeks")
expected = [(dt(2011, 1, 5), dt(2011, 1, 19)),
(dt(2011, 1, 19), dt(2011, 2, 2)),
(dt(2011, 2, 2), dt(2011, 2, 16))]
for i, period in enumerate(ef):
self.assertEqual(period.start, expected[i][0])
self.assertEqual(period.end, expected[i][1])
def plots_1d(self, roots, params=None, legend_labels=None, legend_ncol=None, nx=None,
paramList=None, roots_per_param=False, share_y=None, markers=None, xlims=None):
if roots_per_param:
params = [self.check_param(roots[i][0], param) for i, param in enumerate(params)]
else: params = self.get_param_array(roots[0], params)
if paramList is not None:
wantedParams = self.paramNameListFromFile(paramList)
params = [param for param in params if param.name in wantedParams]
nparam = len(params)
if share_y is None: share_y = self.settings.prob_label is not None and nparam > 1
plot_col, plot_row = self.make_figure(nparam, nx=nx)
plot_roots = roots
for i, param in enumerate(params):
subplot(plot_row, plot_col, i + 1)
if roots_per_param: plot_roots = roots[i]
if markers is not None and i < len(markers): marker = markers[i]
else: marker = None
# self.plot_1d(plot_roots, param, no_ylabel=share_y and i % self.plot_col > 0, marker=marker, prune=(None, 'both')[share_y])
self.plot_1d(plot_roots, param, no_ylabel=share_y and i % self.plot_col > 0, marker=marker)
if xlims is not None: xlim(xlims[i][0], xlims[i][1])
if share_y: self.spaceTicks(gca().xaxis, expand=True)
self.finish_plot([legend_labels, roots][legend_labels is None], legend_ncol=legend_ncol)
if share_y: subplots_adjust(wspace=0)
return plot_col, plot_row
def dt_freq(dt, ignore_M=False):
'''
Detect the (highest) frequency in the passed date time index, and return it
in a format the pandas can interpret.
Also tries to detect whether the found freq is monthly or higher (slow)
Parameters:
-------
dt : pd.DatetimeIndex
The index that is explored
Returns:
--------
unit : str
Unit string of the used resolution
ignore_M : bool
Do not attempt to find out if the daily freq is monthly or lower (slow)
'''
if dt.freq is not None: # if we have the info, it's trivial
return dt.freq.n, dt.freq.name
months, years = dt.month.values, dt.year.values
first_month, first_year = months[0], years[0]
last_month, last_year = months[-1], years[-1]
dm = days_in_month(months, years, astype=float) # days in month
sm = np.array([int(timedelta(days=d).total_seconds()) for d in dm]) # seconds in month
seconds = []
for ns, sec_in_m in zip(np.diff(dt.values), sm):
# month, day, hour, minute
i = ns.astype('timedelta64[s]').astype(int)
seconds.append(i)
sampled = np.array(seconds)
# now check if we can resample to months:
resample = [('m', 60.), ('H', 60.), ('D', 24.)]
highest_freq = 's'
for freq, m in resample:
if all(sampled % m == 0.): # can be resampled, up to daily from seconds
sampled = sampled / m
highest_freq = freq
if highest_freq == 'D' and not any(sampled < 28) and not ignore_M: # it could be monthly, check
df_month = pd.DataFrame(index=pd.period_range('{}-{}'.format(first_year, first_month),
'{}-{}'.format(last_year, last_month), freq='M'))
df_month['passed'] = np.nan
df_month['days'] = np.nan
for i, (y, m) in enumerate(zip(years, months)):
df_month.loc['{}-{:02d}'.format(y,m),'passed'] = i
df_month['passed'] = df_month['passed'].bfill()
for y, m in zip(df_month.index.year, df_month.index.month):
df_month.loc['{}-{:02d}'.format(y,m),'days'] = days_in_month(m, y, float)
df_month = df_month[1:]
m_should_days = df_month.groupby(['passed']).sum()['days'].values
if len(m_should_days) == len(sampled) and all(m_should_days == sampled):
# it is monthly or lower
m_freq = min(df_month.groupby(['passed']).count()['days'].values)
return m_freq, 'M'
else:
return min(sampled), highest_freq
else:
return min(sampled), highest_freq
albu.HorizontalFlip(p=.5),
# Rotate(5, p=.5)
])
# image_augmenter = albu.Compose([albu.GaussNoise(p=.5),
# albu.RandomBrightnessContrast(p=.5)])
image_augmenter = None
dataset = CityscapesDataset(split='train',
net_type='deeplab',
ignore_index=19,
debug=True,
affine_augmenter=affine_augmenter,
image_augmenter=image_augmenter)
dataloader = DataLoader(dataset, batch_size=8, shuffle=True)
print(len(dataset))
for i, batched in enumerate(dataloader):
images, labels = batched
if i == 0:
fig, axes = plt.subplots(8, 2, figsize=(20, 48))
plt.tight_layout()
for j in range(8):
axes[j][0].imshow(
minmax_normalize(images[j],
norm_range=(0, 1),
orig_range=(-1, 1)))
axes[j][1].imshow(labels[j])
axes[j][0].set_xticks([])
axes[j][0].set_yticks([])
axes[j][1].set_xticks([])
axes[j][1].set_yticks([])
plt.savefig('dataset/cityscapes.png')
################ LASSO ################
# Parameter grid to go through.
param_grid_lasso = {'alpha': [0.001, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08,
0.09, 0.1, 0.2, 0.3, 0.4, 0.5, 1]}
# Construct (random) combinations of the parameters (in this case just all of them)
N_ITER = len(param_grid_lasso['alpha'])
param_sampler = ParameterSampler(param_grid_lasso, n_iter=N_ITER)
# Preallocate space for the results
lasso_validation_results = pd.DataFrame(np.zeros((N_ITER, len(param_grid_lasso)+2)),
columns=['alpha', 'coeff', 'score'])
# Loop over the different parameters.
for idx, param in enumerate(param_sampler):
lasso_obj = LassoEstimator(**param)
lasso_obj.fit(features_train, target_train)
predict = lasso_obj.predict(features_validation)
score = lasso_obj.score(target_validation, predict)
lasso_validation_results.iloc[idx, :] = [param['alpha'],
lasso_obj.coef_.astype('object'),
score]
print(lasso_validation_results)
# Get parameters of the best performing model
idx_lasso_model = np.argmin(lasso_validation_results['score'])
lasso_model = lasso_validation_results['alpha'][idx_lasso_model]
################ NN ################
x_test, x_validation, y_test, y_validation = train_test_split(x_test,
y_test,
random_state=0,
train_size=0.3)
#数据归一化
scaler = preprocessing.StandardScaler().fit(x_train) #保存训练集的标准差和均值
x_train = scaler.transform(x_train) #训练集数据归一化
x_test = scaler.transform(x_test) #测试集使用训练集数据归一化
x_validation = scaler.transform(x_validation) #验证集使用训练集数据归一化
#从2^-5-2^15区间中寻找最优C值，从2^-15-2^3区间寻找最优γ值
CScale = [-5, -3, -1, 1, 3, 5, 7, 9, 11, 13, 15]
#gammaScale = [-15, -13, -11, -9, -7, -5,-3,-1,1,3]
score = {'c': 0.0, 'g': 0.0, 'score': 0.0}
for Cindex, c in enumerate(CScale):
# for Gindex,g in enumerate(gammaScale):
#clf = svm.SVC(C=pow(2,c), kernel='rbf', gamma=pow(2,g), decision_function_shape='ovo')
clf = svm.SVC(C=pow(2, c),
kernel='rbf',
gamma=7.464263932294464,
decision_function_shape='ovo')
clf.fit(x_train, y_train.ravel())
#result=clf.score(x_train, y_train)
result = clf.score(x_test, y_test)
if result > score['score']:
score['c'] = Cindex
#score['g']=Gindex
score['score'] = result
n = 10
def detectCharsInPlates(listOfPossiblePlates, filePath):
intPlateCounter = 0
imgContours = None
contours = []
if len(listOfPossiblePlates) == 0: # if list of possible plates is empty
return listOfPossiblePlates # return
# end if
# at this point we can be sure the list of possible plates has at least one plate
for index, possiblePlate in enumerate(
listOfPossiblePlates
): # for each possible plate, this is a big for loop that takes up most of the function
possiblePlate.imgGrayscale, possiblePlate.imgThresh = Preprocess.preprocess(
possiblePlate.imgPlate
) # preprocess to get grayscale and threshold images
adaptivePlate = cv2.adaptiveThreshold(possiblePlate.imgGrayscale, 255,
cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
cv2.THRESH_BINARY, 11, 2)
blurPlate = cv2.GaussianBlur(adaptivePlate, (5, 5), 0)
ret, processedPlate = cv2.threshold(
blurPlate, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
if Main.showSteps == True: # show steps ###################################################
cv2.imshow("5a", possiblePlate.imgPlate)
cv2.imshow("5b", possiblePlate.imgGrayscale)
cv2.imshow("5c.adaptive", adaptivePlate)
cv2.imshow("5d.blur", blurPlate)
cv2.imshow("5e.otsu", processedPlate)
# end if # show steps #####################################################################
# increase size of plate image for easier viewing and char detection
possiblePlate.imgThresh = cv2.resize(possiblePlate.imgThresh, (0, 0),
fx=1.6,
fy=1.6)
# find all possible chars in the plate,
# this function first finds all contours, then only includes contours that could be chars (without comparison to other chars yet)
listOfPossibleCharsInPlate = findPossibleCharsInPlate(adaptivePlate)
listOfPossibleCharsInPlate.sort(key=lambda Char: Char.intCenterX)
if Main.showSteps == True: # show steps ###################################################
showContours(possiblePlate, listOfPossibleCharsInPlate)
listOfListsOfChars = findListOfListsOfMatchingChars(
listOfPossibleCharsInPlate, minChars=3, maxAngle=10)
if len(listOfListsOfChars) == 0:
continue
# find chars that have same heights
listOfListsOfChars1 = [
getEqualHeightList(x) for x in listOfListsOfChars
]
listOfListsOfChars2 = getEqualHeightList(listOfListsOfChars1, mode=1)
# remove Distance Char
listOfListsOfChars3 = [
removeDistanceChar(x) for x in listOfListsOfChars2
]
# flatten list
listOfCharsInPlate = [
char for listChars in listOfListsOfChars3 for char in listChars
]
# remove inner Chars
listOfCharsInPlate = removeInnerChars(listOfCharsInPlate)
# number of plate elements must be > 6
if len(listOfCharsInPlate) >= 6:
possiblePlate.isPlate = True
if Main.showSteps == True: # show steps #######################################################
showListOfLists(possiblePlate, listOfCharsInPlate)
# end of big for loop that takes up most of the function
possiblePlate.strChars = recognizeCharsInPlate(
possiblePlate.imgGrayscale, listOfCharsInPlate, index,
filePath)
print("predict: ", possiblePlate.strChars)
else:
continue
listOfPlates = [plate for plate in listOfPossiblePlates if plate.isPlate]
if Main.showSteps == True:
print(
"\nchar detection complete, click on any image and press a key to continue . . .\n"
)
cv2.waitKey(0)
# end if
return listOfPlates
def recognizeCharsInPlate(imgThresh, listOfMatchingChars, order, filePath):
height, width = imgThresh.shape
imgThreshColor = np.zeros((height, width, 3), np.uint8)
listOfMatchingChars.sort(key=charPlace) # sort chars from left to right
cv2.cvtColor(
imgThresh, cv2.COLOR_GRAY2BGR, imgThreshColor
) # make color version of threshold image so we can draw contours in color on it
charImages = []
for i, currentChar in enumerate(
listOfMatchingChars): # for each char in plate
pt1 = (currentChar.intBoundingRectX, currentChar.intBoundingRectY)
pt2 = ((currentChar.intBoundingRectX +
currentChar.intBoundingRectWidth),
(currentChar.intBoundingRectY +
currentChar.intBoundingRectHeight))
cv2.rectangle(imgThreshColor, pt1, pt2, Main.SCALAR_GREEN,
2) # draw green box around the char
# crop char out of threshold image
imgROI = imgThresh[pt1[1]:pt2[1], pt1[0]:pt2[0]]
imgROIResized = cv2.resize(
imgROI, RESIZED_CHAR
) # resize image, this is necessary for char recognition
# retreive binary image from the char images
adaptivePlate = cv2.adaptiveThreshold(imgROIResized, 255,
cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
cv2.THRESH_BINARY_INV, 11, 2)
blurPlate = cv2.GaussianBlur(adaptivePlate, (5, 5), 0)
ret, im = cv2.threshold(blurPlate, 0, 255,
cv2.THRESH_BINARY + cv2.THRESH_OTSU)
if Main.showSteps == True:
cv2.imshow("resize_" + str(i), im)
cv2.imshow("img_thresh", imgThresh)
charImages.append(im)
# end for
if Main.showSteps == True: # show steps #######################################################
cv2.imshow("8", imgThreshColor)
# end if # show steps #########################################################################
cv2.waitKey(0)
cv2.destroyAllWindows()
chars = recognizeLetter(charImages)
strChars = "".join(chars)
# Gen Plate Letter
# you can write all chars found in a folder by changing the following code
if Main.sourceFolder:
for i, im in enumerate(charImages):
fileName = filePath.split("/")[-1].split(".")
plateFolder = join(Main.targetFolder, fileName[0])
extractedPlateName = "_" + str(order) + "_" + str(
i) + "_" + chars[i] + "." + fileName[1]
resizedChar = cv2.resize(im, RESIZED_CHAR)
borderChar = cv2.copyMakeBorder(resizedChar,
5,
5,
5,
5,
cv2.BORDER_CONSTANT,
value=0)
extractedChar = cv2.resize(borderChar, RESIZED_CHAR)
cv2.imwrite(plateFolder + extractedPlateName, extractedChar)
return strChars
def get_stats(username):
ghuser = username.lower()
firstname = flask.request.args.get("firstname", username)
eventscores = flask.g.redis.zrevrangebyscore("gh:user:{0}:event"
.format(ghuser), "+inf", 5,
0, 10, withscores=True)
events = [e[0] for e in eventscores]
evtcounts = [int(e[1]) for e in eventscores]
# Get the user histogram.
pipe = flask.g.redis.pipeline()
# Get the time zone.
pipe.get("gh:user:{0}:tz".format(ghuser))
# Get the total number of events.
pipe.zscore("gh:user", ghuser)
# Get full commit schedule.
pipe.hgetall("gh:user:{0}:date".format(ghuser))
# Get the daily schedule for each type of event.
[pipe.hgetall("gh:user:{0}:event:{1}:day".format(ghuser, e))
for e in events]
# Get the hourly schedule for each type of event.
[pipe.hgetall("gh:user:{0}:event:{1}:hour".format(ghuser, e))
for e in events]
# Get the distribution of languages contributed to.
pipe.zrevrange("gh:user:{0}:lang".format(ghuser), 0, -1, withscores=True)
# Get the vulgarity (and vulgar rank) of the user.
pipe.zrevrange("gh:user:{0}:curse".format(ghuser), 0, -1, withscores=True)
pipe.zcount("gh:curse:user", 4, "+inf")
pipe.zrevrank("gh:curse:user", ghuser)
# Get connected users.
pipe.zrevrangebyscore("gh:user:{0}:connection".format(ghuser), "+inf", 5,
0, 5)
# Fetch the data from the database.
raw = pipe.execute()
# Get the general stats.
tz = int(raw[0]) if raw[0] is not None and raw[0] != "None" else None
total = int(raw[1]) if raw[1] is not None else 0
if total == 0:
return json.dumps({"message":
"Couldn't find any stats for this user."}), 404
# Get the schedule histograms.
n, m = 3, len(events)
week = zip(*[make_hist(d.iteritems(), 7)
for k, d in zip(events, raw[n:n + m])])
offset = tz + 8 if tz is not None else 0
day = zip(*[make_hist(d.iteritems(), 24, offset=offset)
for k, d in zip(events, raw[n + m:n + 2 * m])])
# If there's no weekly schedule, we don't have enough info.
if not len(week):
return json.dumps({"message":
"Couldn't find any stats for this user."}), 404
# Get the language proportions.
n = n + 2 * m
langs = raw[n]
curses = raw[n + 1]
# Parse the vulgarity factor.
vulgarity = None
try:
vulgarity = int(100 * float(raw[n + 3]) / float(raw[n + 2])) + 1
except:
pass
# Get the connected users.
connections = [c for c in raw[n + 4] if c.lower() != ghuser]
# Get language rank.
langrank = None
langname = None
if len(langs):
lang = langs[0][0]
langname = language_users.get(lang, "{0} expert".format(lang))
# Made up number. How many contributions count as enough? 20? Sure.
pipe.zcount("gh:lang:{0}:user".format(lang), 50, "+inf")
pipe.zrevrank("gh:lang:{0}:user".format(lang), ghuser)
ltot, lrank = pipe.execute()
# This user is in the top N percent of users of language "lang".
try:
langrank = (lang, int(100 * float(lrank) / float(ltot)) + 1)
except:
pass
# Get neighbors.
neighbors = get_neighbors(ghuser)
# Figure out the representative weekly schedule.
hacker_type = "a pretty inconsistent hacker"
if len(week):
mu = np.sum(week, axis=1)
mu /= np.sum(mu)
hacker_type = mean_desc[np.argmin(np.sum(np.abs(means - mu[None, :]),
axis=1))]
# Build a human readable summary.
summary = "<p>"
if langname:
adj = np.random.choice(["a high caliber", "a heavy hitting",
"a serious", "an awesome",
"a top notch", "a trend setting",
"a champion"])
summary += ("{0} is {2} <a href=\"#languages\">{1}</a>"
.format(firstname, langname, adj))
if langrank and langrank[1] < 50:
summary += (" (one of the top {0}% most active {1} users)"
.format(langrank[1], langrank[0]))
if len(events):
if events[0] in evtactions:
summary += (" who <a href=\"#events\">would rather be {0} "
"instead of pushing code</a>").format(
evtactions[events[0]])
elif events[0] == "PushEvent":
summary += " who <a href=\"#events\">loves pushing code</a>"
summary += ". "
summary += "{0} is <a href=\"#week\">{1}</a>".format(firstname,
hacker_type)
if len(day):
best_hour = np.argmax(np.sum(day, axis=1))
if 0 <= best_hour < 7:
tod = "wee hours"
elif 7 <= best_hour < 12:
tod = "morning"
elif 12 <= best_hour < 18:
tod = "mid-afternoon"
elif 18 <= best_hour < 21:
tod = "evening"
else:
tod = "late evening"
summary += " who seems to <a href=\"#day\">work best in the {0}</a>" \
.format(tod)
summary += ". "
if vulgarity:
if vulgarity < 50:
summary += ("I hate to say it but {0} does seem—as one of "
"the top {1}% most vulgar users on GitHub—to "
"be <a href=\"#swearing\">a tad foul-mouthed</a> "
"(with a particular affinity "
"for filthy words like '{2}').").format(firstname,
vulgarity,
curses[0][0])
elif vulgarity < 100:
summary += ("I hate to say it but {0} is becoming—as one of "
"the top {1}% most vulgar users on GitHub—"
"<a href=\"#swearing\">a tad foul-mouthed</a> "
"(with a particular affinity "
"for filthy words like '{2}').").format(firstname,
vulgarity,
curses[0][0])
summary += "</p>"
# Add similar and connected users to summary.
if len(week) and (len(neighbors) or len(connections)):
summary += "<p>"
if len(neighbors):
ind = np.random.randint(len(neighbors))
summary += ("{0}'s behavior is quite similar to <a "
"href=\"{2}\">{1}</a>'s but <span "
"class=\"comparison\" data-url=\"{3}\"></span>. ") \
.format(firstname, neighbors[ind],
flask.url_for(".user", username=neighbors[ind]),
flask.url_for(".compare", username=ghuser,
other=neighbors[ind]))
if len(neighbors) == 2:
ind = (ind + 1) % 2
summary += ("<a href=\"{1}\">{0}</a>'s activity stream also "
"shows remarkable similarities to {2}'s "
"behavior. ").format(neighbors[ind],
flask.url_for(".user",
username=
neighbors[ind]),
firstname)
elif len(neighbors) > 2:
summary += ("It would also be impossible to look at {0}'s "
"activity stream and not compare it to those "
"of ").format(firstname)
cus = []
for i in range(len(neighbors)):
if i != ind:
cus.append("<a href=\"{1}\">{0}</a>"
.format(neighbors[i],
flask.url_for(".user",
username=
neighbors[i])))
summary += ", ".join(cus[:-1])
summary += " and " + cus[-1] + ". "
if len(connections):
ind = 0
summary += ("It seems—from their activity streams—"
"that {0} and <a href=\"{2}\">{1}</a> are probably "
"friends or at least virtual friends. With this in "
"mind, it's worth noting that <span "
"class=\"comparison\" data-url=\"{3}\"></span>. ") \
.format(firstname, connections[ind],
flask.url_for(".user", username=connections[ind]),
flask.url_for(".compare", username=ghuser,
other=connections[ind]))
if len(connections) > 2:
summary += ("There is also an obvious connection between "
"{0} and ").format(firstname)
cus = []
for i in range(len(connections)):
if i != ind:
cus.append("<a href=\"{1}\">{0}</a>"
.format(connections[i],
flask.url_for(".user",
username=
connections[i])))
summary += ", ".join(cus[:-1])
summary += " and " + cus[-1] + ". "
summary += "</p>"
# Summary text for schedule graphs.
sctxt = ""
if len(events):
sctxt = ("<p>The two following graphs show {0}'s average weekly and "
"daily schedules. These charts give significant insight "
"into {0}'s character as a developer. ").format(firstname)
if len(events) == 1:
sctxt += "All of the events in {0}'s activity stream are {1}. " \
.format(firstname, evtverbs.get(events[0]))
else:
sctxt += ("The colors in the charts indicate the fraction of "
"events that are ")
for i, e in enumerate(events):
if i == len(events) - 1:
sctxt += "and "
sctxt += ("<span class=\"evttype\" data-ind=\"{1}\">{0}"
"</span>").format(evtverbs.get(e), i)
if i < len(events) - 1:
sctxt += ", "
sctxt += ". "
sctxt += """</p>
<div class="hist-block">
<div id="week" class="hist"></div>
<div id="day" class="hist"></div>
</div>
<p>"""
sctxt += ("Based on this average weekly schedule, we can "
"describe {0} as "
"<strong>{1}</strong>. ").format(firstname, hacker_type)
if len(day):
if best_hour == 0:
tm = "midnight"
elif best_hour == 12:
tm = "noon"
else:
tm = "{0}{1}".format(best_hour % 12,
"am" if best_hour < 12 else "pm")
sctxt += ("Since {0}'s most active time is around {1}, I would "
"conclude that {0} works best in the "
"<strong>{2}</strong>. ").format(firstname, tm, tod)
sctxt += ("It is important to note that an attempt has been made "
"to show the daily schedule in the correct time zone "
"but this procedure is imperfect at best. ")
sctxt += "</p>"
if len(events) > 1:
sctxt += ("<p>The following chart shows number of events of "
"different types in {0}'s activity stream. In the "
"time frame included in this analysis, {0}'s event "
"stream included "
"a total of {1} events and they are all ") \
.format(firstname, sum(evtcounts))
for i, e in enumerate(events):
if i == len(events) - 1:
sctxt += "or "
sctxt += ("<span class=\"evttype\" data-ind=\"{1}\">{0}"
"</span>").format(evtverbs.get(e), i)
if i < len(events) - 1:
sctxt += ", "
sctxt += ". "
sctxt += """</p><div class="hist-block">
<div id="events"></div>
</div>"""
if langs and len(langs) > 1:
sctxt += ("<p>{0} has contributed to repositories in {1} "
"different languages. ").format(firstname, len(langs))
sctxt += ("In particular, {0} is a serious <strong>{1}</strong> "
"expert").format(firstname, langs[0][0])
ls = [float(l[1]) for l in langs]
if (ls[0] - ls[1]) / sum(ls) < 0.25:
sctxt += (" with a surprisingly broad knowledge of {0} "
"as well").format(langs[1][0])
sctxt += ". "
sctxt += ("The following chart shows the number of contributions "
"made by {0} to repositories where the main "
"language is listed as ").format(firstname)
for i, l in enumerate(langs):
if i == len(langs) - 1:
sctxt += "or "
sctxt += ("<span class=\"evttype\" data-ind=\"{1}\">{0}"
"</span>").format(l[0], i)
if i < len(langs) - 1:
sctxt += ", "
sctxt += "."
sctxt += """</p><div class="hist-block">
<div id="languages"></div>
</div>"""
if langs and len(langs) == 1:
sctxt += ("<p>{0} seems to speak only one programming language: "
"<strong>{1}</strong>. Maybe it's about time to branch "
"out a bit.</p>").format(firstname, langs[0][0])
# Format the results.
results = {"summary": summary}
results["events"] = [" ".join(re.findall("([A-Z][a-z]+)", e))
for e in events]
results["event_counts"] = evtcounts
results["tz"] = tz
results["total"] = total
results["week"] = week
results["hacker_type"] = hacker_type
results["day"] = day
results["schedule_text"] = sctxt
# results["activity"] = raw[2].items()
results["languages"] = langs
results["lang_user"] = langname
results["language_rank"] = langrank
results["curses"] = curses
results["vulgarity"] = vulgarity
results["similar_users"] = neighbors
results["connected_users"] = connections
return json.dumps(results)
exit(1)
if opts.store_file:
vcd_store = VCDStore(fs_db_path=opts.store_file)
else:
vcd_store = VCDStore()
overwrite = opts.overwrite
### Loading up clip IDs and numpy file, pairing parallel rows
print "Extracting clip IDs and features..."
clip_ids = map(int, (l.strip(' \t\n') for l in open(opts.clip_id_file).readlines()))
npy_matrix = np.load(opts.npy_file)
assert len(clip_ids) == npy_matrix.shape[0], \
"ERROR: Number of clip ids found did not match the numpy matrix size! " \
"(clip ids: %d) (np arrays: %d)" \
% (len(clip_ids), npy_matrix.shape[0])
print "-->", len(clip_ids)
### Creating VCDStoreElements
elems = [None] * len(clip_ids)
for i, (cid, np_array) in enumerate(zip(clip_ids, npy_matrix)):
elems[i] = VCDStoreElement(descr_name, cid, np_array)
print "VCD Elements generated:", len(elems)
### Storage into VCDStore
vcd_store.store_feature(elems, overwrite=overwrite)
def _compare_document(self, system, reference):
"""Compute the F1 score for a single document, given a system output
and a reference. This is done by computing a precision according to the
best possible matching of annotations from the system's perspective,
and a recall according to the best possible matching of annotations
from the reference perspective. Gives some partial credit to
annotations that match with the wrong severity.
Args:
system: dictionary mapping doc id's to lists of annotations
reference: dictionary mapping doc id's to lists of annotations
Returns:
The F1 score of a single document.
"""
num_f1_scores = 8
all_matched = np.zeros((num_f1_scores, len(system), len(reference)))
annotation_labels = defaultdict(list)
for i, system_annotation in enumerate(system):
for j, reference_annotation in enumerate(reference):
(
matched,
severity_exact_matched,
category_exact_matched,
both_exact_matched,
category_exact_top_matched,
severity_discount_matched,
category_discount_matched,
both_discount_matched,
) = reference_annotation.count_overlap(
system_annotation,
severity_match=self.severity_match,
category_match=self.category_match
)
all_matched[:, i, j] = [
matched,
severity_exact_matched,
category_exact_matched,
both_exact_matched,
category_exact_top_matched,
severity_discount_matched,
category_discount_matched,
both_discount_matched,
]
# only consider a label as a 'prediction' when there is _some_ overlap
if matched > 0:
annotation_labels['system_severities'].append(
system_annotation.severity)
annotation_labels['system_categories'].append(
system_annotation.category)
annotation_labels['reference_severities'].append(
reference_annotation.severity)
annotation_labels['reference_categories'].append(
reference_annotation.category)
lengths_sys = np.array([len(annotation) for annotation in system])
lengths_ref = np.array([len(annotation) for annotation in reference])
if lengths_sys.sum() == 0:
# no system annotations
precision = np.ones(num_f1_scores)
elif lengths_ref.sum() == 0:
# there were no references
precision = np.zeros(num_f1_scores)
else:
# normalize by annotation length
precision_by_annotation = all_matched.max(2) / lengths_sys
precision = precision_by_annotation.mean(1)
# same as above, for recall now
if lengths_ref.sum() == 0:
recall = np.ones(num_f1_scores)
elif lengths_sys.sum() == 0:
recall = np.zeros(num_f1_scores)
else:
recall_by_annotation = all_matched.max(1) / lengths_ref
recall = recall_by_annotation.mean(1)
# simultaneous division of all the values, with division by zero defaulting to 0
f1 = np.divide(
2*precision*recall,
precision + recall,
out=np.zeros_like(precision),
where=precision + recall != 0
)
assert 0. <= f1.min() and f1.max() <= 1.
return tuple(f1), dict(annotation_labels)
def main():
global tmpSettingsDir, availableBuildSystems
qtVersionsForQuick = ["5.3"]
availableBuildSystems = ["qmake", "Qbs"]
if platform.system() != 'Darwin':
qtVersionsForQuick.append("5.4")
if which("cmake"):
availableBuildSystems.append("CMake")
else:
test.warning("Could not find cmake in PATH - several tests won't run without.")
startApplication("qtcreator" + SettingsPath)
if not startedWithoutPluginError():
return
kits = getConfiguredKits()
test.log("Collecting potential project types...")
availableProjectTypes = []
invokeMenuItem("File", "New File or Project...")
categoriesView = waitForObject(":New.templateCategoryView_QTreeView")
catModel = categoriesView.model()
projects = catModel.index(0, 0)
test.compare("Projects", str(projects.data()))
comboBox = findObject(":New.comboBox_QComboBox")
targets = zip(*kits.values())[0]
test.verify(comboBox.enabled, "Verifying whether combobox is enabled.")
test.compare(comboBox.currentText, "All Templates")
try:
selectFromCombo(comboBox, "All Templates")
except:
test.warning("Could not explicitly select 'All Templates' from combobox.")
for category in [item.replace(".", "\\.") for item in dumpItems(catModel, projects)]:
# skip non-configurable
if "Import" in category:
continue
clickItem(categoriesView, "Projects." + category, 5, 5, 0, Qt.LeftButton)
templatesView = waitForObject("{name='templatesView' type='QListView' visible='1'}")
# needed because categoriesView and templatesView using same model
for template in dumpItems(templatesView.model(), templatesView.rootIndex()):
template = template.replace(".", "\\.")
# skip non-configurable
if not template in ["Qt Quick UI Prototype", "Qt Canvas 3D Application",
"Auto Test Project"]: # FIXME
availableProjectTypes.append({category:template})
safeClickButton("Cancel")
for current in availableProjectTypes:
category = current.keys()[0]
template = current.values()[0]
displayedPlatforms = __createProject__(category, template)
if template == "Qt Quick Application" or template == "Qt Quick Controls Application":
for counter, qtVersion in enumerate(qtVersionsForQuick):
def additionalFunc(displayedPlatforms, qtVersion):
requiredQtVersion = __createProjectHandleQtQuickSelection__(qtVersion)
__modifyAvailableTargets__(displayedPlatforms, requiredQtVersion, True)
handleBuildSystemVerifyKits(category, template, kits, displayedPlatforms,
additionalFunc, qtVersion)
# are there more Quick combinations - then recreate this project
if counter < len(qtVersionsForQuick) - 1:
displayedPlatforms = __createProject__(category, template)
continue
elif template == "Qt Quick Controls 2 Application": # needs a Qt5.7
def additionalFunc(displayedPlatforms):
clickButton(waitForObject(":Next_QPushButton")) # ignore this details page for now
handleBuildSystemVerifyKits(category, template, kits, displayedPlatforms, additionalFunc)
continue
elif template.startswith("Plain C"):
handleBuildSystemVerifyKits(category, template, kits, displayedPlatforms)
continue
handleBuildSystemVerifyKits(category, template, kits, displayedPlatforms)
invokeMenuItem("File", "Exit")
import calendar
import sys, traceback
import logging
import os
logger = logging.getLogger('PricesPaidTrans')
hdlr = logging.FileHandler('../logs/PricesPaidTrans.log')
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.setLevel(logging.ERROR)
# This works in 2.7, which I don't have on the AWS instance yet!
# monthLookup = {v.upper(): k for k,v in enumerate(calendar.month_abbr)}
monthLookup = dict((v.upper(), k) for k, v in enumerate(calendar.month_abbr))
# This are magic numbers that are simply based on the RevAuc files
# as Laura presented them to me.
def findMonthFromAbbrev(a):
global monthLookup
try:
m = monthLookup[a]
return m
except KeyError as e:
logger.error('Caught error ' + repr(e) + a)
return 0
# Need to test this month lookup stuff, right now
def draw_trefoil3d(x=None, y=None, z=None, c=None, s='z', ax=None, fig=None, view=(90, -90), **kwargs):
"""Plot trefoil knot.
"""
if c is None:
c = np.copy(z)
cmap = plt.get_cmap(kwargs.get('cmap', "brg"), 3)
norm = mpl.colors.Normalize(c.min(), c.max())
# extract x, y, z
if s == 'z':
zbins = np.linspace(z.min(), z.max(), num=10)
zbins = np.digitize(z, zbins)
s = zbins**2
# combine features
X = np.c_[x, y, z]
# plot data
fig, axes = plt.subplots(1, 3, figsize=(15,5),subplot_kw=dict(projection='3d'))
# 3 views
for ax_i, ax in enumerate(axes):
if ax_i == 0:
xcol, ycol, zcol = 0, 1, 2
elif ax_i == 1:
xcol, ycol, zcol = 0, 2, 1
elif ax_i == 2:
xcol, ycol, zcol = 1, 2, 0
zbins = np.linspace(z.min(), z.max(), num=10)
zbins = np.digitize(z, zbins)
ax.scatter(X[:,xcol], X[:,ycol], X[:,zcol], c=c, s=s, alpha=0.8, cmap=cmap, norm=norm)
ax.set_xlabel(list('xyz')[xcol], fontweight='bold')
ax.set_ylabel(list('xyz')[ycol], fontweight='bold')
ax.set_zlabel(list('xyz')[zcol], fontweight='bold')
ax.set_xticks([])
ax.set_yticks([])
ax.set_zticks([])
# Get rid of colored axes planes
# First remove fill
ax.xaxis.pane.fill = False
ax.yaxis.pane.fill = False
ax.zaxis.pane.fill = False
# Now set color to white (or whatever is "invisible")
ax.xaxis.pane.set_edgecolor('w')
ax.yaxis.pane.set_edgecolor('w')
ax.zaxis.pane.set_edgecolor('w')
# Bonus: To get rid of the grid as well:
ax.grid(False)
#if ax_i == 1:
# view = (view[0]+0, view[1]+90)
#if ax_i == 2:
# view = (view[0]+90, view[1]+0)
ax.set_title("view = {}".format(view))
ax.view_init(*view)
return axes
plt.ioff()
currFolder = os.path.dirname(os.path.realpath(__file__))
os.chdir(currFolder)
patchesToShow = 'finalPatches'
#patchesToShow = 'finalPatchesMarked'
columnNum = 4
trialList = [
'160208_M193206_Trial1.pkl',
]
for k, trialName in enumerate(trialList):
trialPath = os.path.join(currFolder,trialName)
trial, _ = rm.loadTrial(trialPath)
finalPatches = getattr(trial,patchesToShow)
numOfPatches = len(finalPatches.keys())
rowNum = numOfPatches // columnNum + 1
f = plt.figure(figsize=(10,10))
f.suptitle(trialName)
ax = f.add_subplot(111)
rm.plotPatches(finalPatches,plotaxis=ax,markersize=0)
for key,patch in finalPatches.iteritems():
center = patch.getCenter()
db = config_dict['数据库名称'].strip()
connect = pymysql.connect(host='localhost',
user=user,
passwd=passwd,
db=db,
charset='utf8')
# 捕获异常信息
try:
cursor = connect.cursor()
cursor.execute("truncate table jd_help")
connect.commit()
coupon = []
with open('coupon.txt', 'r') as f:
coupons = f.readlines()
for k, cou in enumerate(coupons):
coupon = 'http://coupon.m.jd.com/coupons/show.action?{}&to=m.jd.com'.format(
cou.strip())
res = requests.get(
url=coupon,
headers={
'User-Agent':
'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1'
})
ur_yuan = res.content.decode('utf-8')
limitStr = re.search('limitStr":"(.*?)"', ur_yuan, re.S).group(1)
discount = re.search('discount":"(.*?)"', ur_yuan, re.S).group(1)
quota = re.search('quota":"(.*?)"', ur_yuan, re.S).group(1)
batchid = re.search('batchId":"(.*?)"', ur_yuan, re.S).group(1)
batchcount = re.search('batchCount":(\d+),', ur_yuan, re.S).group(1)
des = limitStr + quota + '-' + discount
def to_one_hot(labels, dimension=3):
results = np.zeros((len(labels), dimension))
for i, label in enumerate(labels):
results[i, label] = 1.
return results
def flush(self):
if not len(self.incoming_stats):
logger.info("Nothing to report")
return
logger.info("Loss on {} batches".format(len(self.incoming_stats)))
batch_stats = merge_tensor_namedtuple_list(self.incoming_stats, BatchStats)
batch_stats.write_summary(self.action_names)
print_details = "Loss:\n"
td_loss_mean = float(batch_stats.td_loss.mean())
self.td_loss.append(td_loss_mean)
print_details = print_details + "TD LOSS: {0:.3f}\n".format(td_loss_mean)
if batch_stats.logged_rewards is not None:
flattened_rewards = torch.flatten(batch_stats.logged_rewards).tolist()
self.running_reward.extend(flattened_rewards)
if batch_stats.reward_loss is not None:
reward_loss_mean = float(batch_stats.reward_loss.mean())
self.reward_loss.append(reward_loss_mean)
print_details = print_details + "REWARD LOSS: {0:.3f}\n".format(
reward_loss_mean
)
if batch_stats.imitator_loss is not None:
imitator_loss_mean = float(batch_stats.imitator_loss.mean())
self.imitator_loss.append(imitator_loss_mean)
print_details = print_details + "IMITATOR LOSS: {0:.3f}\n".format(
imitator_loss_mean
)
if batch_stats.model_values is not None and self.action_names:
self.model_values.append(
dict(zip(self.action_names, batch_stats.model_values.mean(dim=0)))
)
self.model_value_stds.append(
dict(zip(self.action_names, batch_stats.model_values.std(dim=0)))
)
if batch_stats.model_values_on_logged_actions is not None:
self.logged_action_q_value.append(
batch_stats.model_values_on_logged_actions.mean().item()
)
if (
batch_stats.logged_actions is not None
and batch_stats.model_action_idxs is not None
):
logged_action_counts = {
action: (batch_stats.logged_actions == i).sum().item()
for i, action in enumerate(self.action_names)
}
model_action_counts = {
action: (batch_stats.model_action_idxs == i).sum().item()
for i, action in enumerate(self.action_names)
}
print_details += "The distribution of logged actions : {}\n".format(
logged_action_counts
)
print_details += "The distribution of model actions : {}\n".format(
model_action_counts
)
for action, count in logged_action_counts.items():
self.logged_action_counts[action] += count
self.model_action_counts.append(model_action_counts)
for action, count in model_action_counts.items():
self.model_action_counts_cumulative[action] += count
total = float(sum(model_action_counts.values()))
self.model_action_distr.append(
{action: count / total for action, count in model_action_counts.items()}
)
print_details += "Batch Evaluator Finished"
for print_detail in print_details.split("\n"):
logger.info(print_detail)
self.incoming_stats.clear()
def _base_iterator(self,
circuit: circuits.Circuit,
qubit_order: ops.QubitOrderOrList,
initial_state: Union[int, np.ndarray],
all_measurements_are_terminal=False) -> Iterator:
qubits = ops.QubitOrder.as_qubit_order(qubit_order).order_for(
circuit.all_qubits())
qid_shape = protocols.qid_shape(qubits)
qubit_map = {q: i for i, q in enumerate(qubits)}
initial_matrix = qis.to_valid_density_matrix(initial_state,
len(qid_shape),
qid_shape=qid_shape,
dtype=self._dtype)
measured = collections.defaultdict(
bool) # type: Dict[Tuple[cirq.Qid, ...], bool]
if len(circuit) == 0:
yield DensityMatrixStepResult(initial_matrix, {}, qubit_map,
self._dtype)
return
state = _StateAndBuffers(len(qid_shape),
initial_matrix.reshape(qid_shape * 2))
def on_stuck(bad_op: ops.Operation):
return TypeError(
"Can't simulate operations that don't implement "
"SupportsUnitary, SupportsConsistentApplyUnitary, "
"SupportsMixture, SupportsChannel or is a measurement: {!r}".
format(bad_op))
def keep(potential_op: ops.Operation) -> bool:
return (protocols.has_channel(potential_op)
or isinstance(potential_op.gate, ops.MeasurementGate))
noisy_moments = self.noise.noisy_moments(circuit,
sorted(circuit.all_qubits()))
for moment in noisy_moments:
measurements = collections.defaultdict(
list) # type: Dict[str, List[int]]
channel_ops_and_measurements = protocols.decompose(
moment, keep=keep, on_stuck_raise=on_stuck)
for op in channel_ops_and_measurements:
indices = [qubit_map[qubit] for qubit in op.qubits]
# TODO: support more general measurements.
# Github issue: https://github.com/quantumlib/Cirq/issues/1357
if all_measurements_are_terminal and measured[op.qubits]:
continue
if isinstance(op.gate, ops.MeasurementGate):
measured[op.qubits] = True
meas = op.gate
if all_measurements_are_terminal:
continue
if self._ignore_measurement_results:
for i, q in enumerate(op.qubits):
self._apply_op_channel(
ops.phase_damp(1).on(q), state, [indices[i]])
else:
invert_mask = meas.full_invert_mask()
# Measure updates inline.
bits, _ = (density_matrix_utils.measure_density_matrix(
state.tensor,
indices,
qid_shape=qid_shape,
out=state.tensor,
seed=self._prng))
corrected = [
bit ^ (bit < 2 and mask)
for bit, mask in zip(bits, invert_mask)
]
key = protocols.measurement_key(meas)
measurements[key].extend(corrected)
else:
self._apply_op_channel(op, state, indices)
yield DensityMatrixStepResult(density_matrix=state.tensor,
measurements=measurements,
qubit_map=qubit_map,
dtype=self._dtype)
cfg = Config.Config(options.config)
box = options.box
inputData = cfg.getVariables(box, "inputDataNtu")
treeName = cfg.getVariables(box, "treeName")
outFile = cfg.getVariables(box, "outFile")
loose = cfg.getVariables(box, "loose")
medium = cfg.getVariables(box, "medium")
dR = cfg.getVariables(box, "deltaR")
###################################################################
print treeName
tchain = rt.TChain(treeName)
for i, line in enumerate(inputData):
tchain.Add(line)
nEntries = tchain.GetEntries()
print 'Number of entries: ', nEntries
# Book histos
#bins,min,max
ptRange = [100, 0, 1000]
mjjRange = [150, 500, 2000]
etaRange = [100, -3.5, 3.5]
csvRange = [20, 0, 1]
allHistos = []
##calo
# set path to ground truth
str_gt = os.path.join(str_gt_path, 'derivatives', str_subj, 'labeled',
str_subj + '_labels_v01.nii.gz')
# %%
# Prepare lists for different metrics
resultsAcc = []
resultsF1 = []
# Load ground truth image
tns_gt = func_nii_to_tf_tensor(str_gt, dtype=np.int32)
# loop over models
for indModel, sg_model in enumerate(lst_sg_models):
model_name = lst_model_headers[indModel]
print('---Working on model: ' + model_name)
# Append model name to lists for row header
resultsAcc.append([model_name])
resultsF1.append([model_name])
# loop over augmentations
for aug_stride in lst_aug_strides:
print('---Working on augmentation: ' + aug_stride)
# derive path to predicted segmentations
str_ps = os.path.join(str_sg_path, sg_model, aug_stride, 'pred.nii.gz')
# derive path to predicted probabilities
def grader(e, ans):
state = json.loads(ans) #['answer']
response = state['response']
colors = state['colors']
#return {'ok': False, 'msg': '%s' % str(response)}
Numeric = 'NUMERIC_RESPONSE'
questionTable = [
[
'Mass of Falling Object<br>M(kg)', 'Distance Object Fall<br>D(m)',
'Times for Fall<br>t(s)',
'Calculated Average Acceleration of Falling Object<br>a(m/s^2)',
'Calculated Tension in String<br>T(N)',
'Calculated Torque Exerted on Post of Apparatus<br>T*D(m*N)'
],
['0.100', '0.500', '9.7, 10.0, 10.3', '0.010', Numeric, Numeric],
['0.150', '0.500', '6.9, 7.1, 7.3', '0.020', Numeric, Numeric],
['0.200', '0.500', '5.8, 5.7, 5.6', '0.031', Numeric, Numeric],
['0.250', '0.500', '4.9, 4.9, 4.8', '0.042', Numeric, Numeric],
]
answerTable = [
[
'Mass of Falling Object<br>M(kg)', 'Distance Object Fall<br>D(m)',
'Times for Fall<br>t(s)',
'Calculated Average Acceleration of Falling Object<br>a(m/s^2)',
'Calculated Tension in String<br>T(N)',
'Calculated Torque Exerted on Post of Apparatus<br>T*D(m*N)'
],
['0.100', '0.500', '9.7, 10.0, 10.3', '0.010', 0.96, 0.0011],
['0.150', '0.500', '6.9, 7.1, 7.3', '0.020', 1.47, 0.0016],
['0.200', '0.500', '5.8, 5.7, 5.6', '0.031', 1.95, 0.0021],
['0.250', '0.500', '4.9, 4.9, 4.8', '0.042', 2.44, 0.0027],
]
### Must remove header for comparison in loop below
### JQuery takes cells without a header, so i,j values must only be fore table body.
answerTable = answerTable[1::]
def stringComparison(R, K):
'''
R: Response from student
K: Answer from (K)ey
'''
if str(R) == str(K):
return True
else:
return False
def testNumeric(X):
'''
X: entry taken from Active Table
'''
try:
float(X)
return True
except:
return False
def numericalComparison(R, K, **kwargs):
'''
R: Response from student
K: Answer from (K)ey
tolerance: percentage tolerance allowed in numerical response
*** Implement edX tolerance: https://github.com/edx/edx-platform/blob/dbc465a51871bd685dd925c23bf73b981e07abe6/common/lib/capa/capa/util.py#L14
Currently Calculated: return 100.*abs(student - instructor)/instructor <= tolerance
'''
percent_tolerance = kwargs.get('percent_tolerance', 1.0)
if testNumeric(R) and testNumeric(K):
return 100. * abs(float(R) -
float(K)) / float(K) <= percent_tolerance
else:
#print "Issue with grading rubric. Could not convert Response to float."
return False
def returnID(row, column):
return str(row) + '___' + str(column).replace(' ', '')
def feedback(cellType, R, K):
'''
cellType: type of cell input, options are [NUMERIC_RESPONSE,STRING_RESPONSE]
R: short-hand for student Response Value
K: short-hand for answer Key Value
'''
if cellType == 'NUMERIC_RESPONSE':
return numericalComparison(R, K, percent_tolerance=5.0)
#return stringComparison(R,K)
elif cellType == 'STRING_RESPONSE':
return stringComparison(R, K)
### Loop through question Table and use indices to check answer table
header = questionTable[0]
numCorrects = 0
for i, row in enumerate(questionTable[1::]):
for j, cellType in enumerate(row):
ID = returnID(i, header[j])
#print i,j,header[j],ID,answerTable[i][j]
### Check for Existent Active Cell
if ID in response:
### Compare answer key and contents of cell
if feedback(cellType, response[ID], answerTable[i][j]):
numCorrects = numCorrects + 1
else:
state['colors'][ID] = 'LightPink'
# ### Check for Active Cell
# if cellType == 'NUMERIC_RESPONSE':
# ID = returnID(i,header[j])
# ### Test that ID is in passed answer from edX
# if ID in response:
# #print i,j,header[j],ID,answerTable[i][j]
# ### Compare Responses to Key
# if not stringComparison(response[ID],answerTable[i][j]):
# #print "Incorrect"
# state['colors'][ID] = 'LightPink'
# else:
# numCorrects = numCorrects + 1
if numCorrects == len(response.keys()):
return {'ok': True, 'msg': 'Great job!'}
else:
return {
'ok':
False,
'msg':
'You have %s cells out of %s correct.' %
(str(numCorrects), str(len(response.keys())))
}
# coding: utf-8
import string
import numpy as np
task_time = {letter: 61 + i for i, letter in enumerate(string.ascii_uppercase)}
def next_task(tasklist):
for task in string.ascii_uppercase:
try:
if len(tasklist[task]) == 0:
return task
except KeyError:
pass
return None
def set_task_done(task, tasklist):
for dependencies in tasklist.values():
try:
dependencies.remove(task)
except ValueError:
pass
todo = set(string.ascii_uppercase)
tasks = {letter: [] for letter in string.ascii_uppercase}
with open('input7.txt') as f:
for line in f:
_, pre, _, _, _, _, _, post, _, _ = line.split()
tasks[post].append(pre)
def search(query, docTermIndex):
print('\nRetrieving documents for query \'{}\'\n'.format(query))
qlist = query.strip().split()
#Remove stop words
modified = [
term for term in qlist if term not in stopwords.words('english')
]
EXPAND = False
'''Expansion
EXPAND = True
for term in modified:
syns = wordnet.synsets(term)
for i in range(len(syns)):
#print(syns[i].lemmas()[0].name())
new.append(syns[i].lemmas()[0].name())
''' ''''''
if EXPAND == False:
new = modified
before_stem = np.unique(new)
ps = stem.PorterStemmer()
after_stem = [ps.stem(word) for word in before_stem]
mQuery = np.unique(after_stem)
mQuery
#read potentially relevant docs into matrix using tf*idf weights
qMatrix = pd.DataFrame(np.zeros((0, len(mQuery))), columns=mQuery)
for term in mQuery:
if term in docTermIndex.keys():
termInfo = docTermIndex.get(term)
for occurence in termInfo.occList:
if occurence.docID not in qMatrix.index:
toAppend = pd.Series(np.zeros(len(qMatrix.columns)),
index=qMatrix.columns,
name=occurence.docID)
toAppend[term] = occurence.count * termInfo.idf
qMatrix = qMatrix.append(toAppend)
else:
qMatrix.loc[occurence.docID,
term] = occurence.count * termInfo.idf
#compute tfxidf vector of query
#print(qMatrix.columns)
q_vect = [docTermIndex.get(term).idf for term in qMatrix.columns]
#Get cosine similarities for query
matrix_norm = np.array(
[np.linalg.norm(qMatrix.iloc[i]) for i in range(len(qMatrix))])
q_norm = np.linalg.norm(q_vect)
sims = np.dot(qMatrix, q_vect) / (matrix_norm * q_norm)
dists = 1 - sims
idx = np.argsort(dists)
user_docs = qMatrix.iloc[idx[:10]].index
classes = pd.read_csv('classes.csv', index_col=1)
for i, path in enumerate(user_docs):
parts = path.split('/')
group = parts[1]
file = parts[2]
print('----{}: File {} in folder {}-----\n'.format(i + 1, file, group))
with open(path, 'r', errors='ignore') as myfile:
data = myfile.read()
art = data
ind = art.find('\n\n')
art = art[ind + 2:]
#If article(and not post), re-index again to get rid of tags
if art[0:10].find('archive') != -1:
ind = art.find('\n\n')
art = art[ind + 2:]
mid = len(art) // 2
midmid = mid // 2
print('---------------------------------------------\n')
print(art[mid:mid + 200])
print('---------------------------------------------\n')
return user_docs
data = [111, 444, 555, 888, 412, 647, 951,
789, 654, 258, 5, 4, 234,
543, 345, 123235245, 213,
435, 2334, 345
]
min_valid_amount = 200
max_valid_amount = 400
for index in range(len(data) - 1, -1, -1):
if data[index] > max_valid_amount or data[index] < min_valid_amount:
print(index, data)
del data[index]
print(data)
# another backwards/reverse way
top_index = len(data) - 1
for index, number in enumerate(reversed(data)):
if number < min_valid_amount or number > min_valid_amount:
print(top_index - index, number)
del data[top_index - index]
print(data)
def tempo_polyco_table_writer(polycoTable, filename='polyco.dat'):
"""
Write tempo style polyco file from an astropy table
Tempo style polyco file:
The polynomial ephemerides are written to file 'polyco.dat'. Entries
are listed sequentially within the file. The file format is::
Line Columns Item
---- ------- -----------------------------------
1 1-10 Pulsar Name
11-19 Date (dd-mmm-yy)
20-31 UTC (hhmmss.ss)
32-51 TMID (MJD)
52-72 DM
74-79 Doppler shift due to earth motion (10^-4)
80-86 Log_10 of fit rms residual in periods
2 1-20 Reference Phase (RPHASE)
21-38 Reference rotation frequency (F0)
39-43 Observatory number
44-49 Data span (minutes)
50-54 Number of coefficients
55-75 Observing frequency (MHz)
76-80 Binary phase
3* 1-25 Coefficient 1 (COEFF(1))
26-50 Coefficient 2 (COEFF(2))
51-75 Coefficient 3 (COEFF(3))
* Subsequent lines have three coefficients each, up to NCOEFF
One polyco file could include more then one entrie
The pulse phase and frequency at time T are then calculated as::
DT = (T-TMID)*1440
PHASE = RPHASE + DT*60*F0 + COEFF(1) + DT*COEFF(2) + DT^2*COEFF(3) + ....
FREQ(Hz) = F0 + (1/60)*(COEFF(2) + 2*DT*COEFF(3) + 3*DT^2*COEFF(4) + ....)
Parameters
---------
polycoTable: astropy table
Polycos style table
filename : str
Name of the output poloco file.
References
----------
http://tempo.sourceforge.net/ref_man_sections/tz-polyco.txt
"""
f = open(filename, 'w')
lenTable = len(polycoTable)
if lenTable == 0:
errorMssg = ("Insufficent polyco data." +
" Please make sure polycoTable has data.")
raise AttributeError(errorMssg)
for i in range(lenTable):
entry = polycoTable['entry'][i]
psrname = polycoTable['psr'][i].ljust(10)
dateDMY = polycoTable['date'][i].ljust(10)
utcHMS = polycoTable['utc'][i][0:9].ljust(10)
tmid_mjd = utils.longdouble2str(entry.tmid.value) + ' '
dm = str(polycoTable['dm'][i]).ljust(72 - 52 + 1)
dshift = str(polycoTable['doppler'][i]).ljust(79 - 74 + 1)
logrms = str(polycoTable['logrms'][i]).ljust(80 - 86 + 1)
line1 = psrname + dateDMY + utcHMS + tmid_mjd + dm + dshift + logrms + '\n'
# Get the reference phase
rph = (entry.rphase.int + entry.rphase.frac).value[0]
# FIXME: sometimes raises error, sometimes loses precision!
rphase = utils.longdouble2str(rph)[0:19].ljust(20)
f0 = ('%.12lf' % entry.f0).ljust(38 - 21 + 1)
obs = entry.obs.ljust(43 - 39 + 1)
tspan = str(round(entry.mjdspan.to('min').value,
4))[0].ljust(49 - 44 + 1)
if len(tspan) >= (49 - 44 + 1): # Hack to fix read errors in python
tspan = tspan + ' '
ncoeff = str(entry.ncoeff).ljust(54 - 50 + 1)
obsfreq = str(polycoTable['obsfreq'][i]).ljust(75 - 55 + 1)
binPhase = str(polycoTable['binary_phase'][i]).ljust(80 - 76 + 1)
line2 = rphase + f0 + obs + tspan + ncoeff + obsfreq + binPhase + '\n'
coeffBlock = ""
for j, coeff in enumerate(entry.coeffs):
coeffBlock += ('%.17e' % coeff).ljust(25)
if (j + 1) % 3 == 0:
coeffBlock += '\n'
f.write(line1 + line2 + coeffBlock)
f.close()
raw_imgs = crop_imgs
raw_masks = crop_masks
height, width = raw_imgs.shape[1:3]
zoomed_imgs = zoom(raw_imgs, (slice_resolution, new_height/height, new_width/width))
np.save(os.path.join(des_home, x+".npy"), zoomed_imgs)
zoomed_masks = zoom(raw_masks, (slice_resolution, new_height/height, new_width/width))
np.save(os.path.join(des_home, x+"-dlmask.npy"), zoomed_masks)
immasks = np.concatenate([zoomed_imgs, zoomed_masks*255], axis=2)[length//2]
cv2.imwrite(f"debug/{x}.png", immasks)
#imgs2vid(immasks, "debug/{}.avi".format(x))
#for x in pe_list:
# resize_cta_images(x)
#
#exit()
#for x in orcale_normal_list:
# resize_cta_images(x, "orcale_normal")
from concurrent import futures
num_threads=10
with futures.ProcessPoolExecutor(max_workers=num_threads) as executor:
fs = [executor.submit(resize_cta_images, x, ) for x in pe_list[::-1]]
for i, f in enumerate(futures.as_completed(fs)):
print ("{}/{} done...".format(i, len(fs)))
def expand(self, epsF, wfs):
if wfs.nspins == 1:
epsF = [epsF]
elif not self.fixmom:
epsF = [epsF, epsF]
if self.nos == None:
self.nos = wfs.nbands
# Check dimension of lists
if len(self.wf_u) == len(wfs.kpt_u):
wf_u = self.wf_u
p_uai = self.p_uai
else:
raise RuntimeError('List of wavefunctions has wrong size')
c_un = []
p_u = []
for u, kpt in enumerate(wfs.kpt_u):
# Inner product of pseudowavefunctions
wf = np.reshape(wf_u[u], -1)
Wf_n = kpt.psit_nG
Wf_n = np.reshape(Wf_n, (len(kpt.f_n), -1))
Porb_n = np.dot(Wf_n.conj(), wf) * wfs.gd.dv
# Correction to obtain inner product of AE wavefunctions
for a, p_i in p_uai[u].items():
for n in range(wfs.nbands):
for i in range(len(p_i)):
for j in range(len(p_i)):
Porb_n[n] += (kpt.P_ani[a][n][i].conj() *
wfs.setups[a].dO_ii[i][j] * p_i[j])
wfs.gd.comm.sum(Porb_n)
#print 'Kpt:', kpt.k, ' Spin:', kpt.s, \
# ' Sum_n|<orb|nks>|^2:', sum(abs(Porb_n)**2)
p_u.append(np.array([sum(abs(Porb_n)**2)], dtype=float))
# Starting from KS orbitals with largest overlap,
# fill in the expansion coeffients
c_n = np.zeros(wfs.nbands, dtype=complex)
nos = 0
bandpriority = np.argsort(abs(Porb_n)**2)[::-1]
for n in bandpriority:
if (kpt.eps_n[n] > epsF[kpt.s] + self.Estart
and kpt.eps_n[n] < epsF[kpt.s] + self.Eend):
c_n[n] = Porb_n[n]
nos += 1
if nos == self.nos:
break
# Normalize expansion coefficients
c_n /= np.sqrt(sum(abs(c_n)**2))
c_un.append(c_n)
for s in range(wfs.nspins):
for k in range(wfs.nibzkpts):
p = wfs.collect_auxiliary(p_u, k, s)
if wfs.world.rank == 0:
self.txt.write('Kpt: %d, Spin: %d, ' \
'Sum_n|<orb|nks>|^2: %f\n' % (k, s, p))
return c_un
def _train(path_to_train_lmdb_dir, path_to_val_lmdb_dir, path_to_log_dir,
path_to_restore_checkpoint_file, training_options, max_steps):
batch_size = training_options['batch_size']
initial_learning_rate = training_options['learning_rate']
initial_patience = training_options['patience']
num_steps_to_show_loss = 100
num_steps_to_check = training_options["validation_interval"]
step = 0
patience = initial_patience
best_accuracy = 0.0
duration = 0.0
model = Model()
model.cuda()
transform = transforms.Compose([
transforms.RandomCrop([54, 54]),
transforms.ToTensor(),
transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
])
train_loader = torch.utils.data.DataLoader(Dataset(path_to_train_lmdb_dir, transform),
batch_size=batch_size, shuffle=True,
num_workers=0, pin_memory=True)
evaluator = Evaluator(path_to_val_lmdb_dir)
optimizer = optim.SGD(model.parameters(), lr=initial_learning_rate, momentum=0.9, weight_decay=0.0005)
scheduler = StepLR(optimizer, step_size=training_options['decay_steps'], gamma=training_options['decay_rate'])
if path_to_restore_checkpoint_file is not None:
assert os.path.isfile(path_to_restore_checkpoint_file), '%s not found' % path_to_restore_checkpoint_file
step = model.restore(path_to_restore_checkpoint_file)
scheduler.last_epoch = step
print('Model restored from file: %s' % path_to_restore_checkpoint_file)
path_to_losses_npy_file = os.path.join(path_to_log_dir, 'losses.npy')
if os.path.isfile(path_to_losses_npy_file):
losses = np.load(path_to_losses_npy_file)
else:
losses = np.empty([0], dtype=np.float32)
path_to_test_losses_npy_file = os.path.join(path_to_log_dir, 'test_losses.npy')
if os.path.isfile(path_to_test_losses_npy_file):
test_losses = np.load(path_to_test_losses_npy_file)
else:
test_losses = np.empty([0], dtype=np.float32)
train_loss_array = []
val_loss_array = []
model_checkpoints = []
model_saved = False
# Used to save model (checkpoint) every 2 epochs
model_save_counter = 0
while True:
for batch_idx, (images, length_labels, digits_labels, _) in enumerate(train_loader):
start_time = time.time()
images, length_labels, digits_labels = images.cuda(), length_labels.cuda(), [digit_labels.cuda() for digit_labels in digits_labels]
length_logits, digit1_logits, digit2_logits, digit3_logits, digit4_logits, digit5_logits = model.train()(images)
loss = _loss(length_logits, digit1_logits, digit2_logits, digit3_logits, digit4_logits, digit5_logits, length_labels, digits_labels)
optimizer.zero_grad()
loss.backward()
optimizer.step()
scheduler.step()
step += 1
duration += time.time() - start_time
if step % num_steps_to_show_loss == 0:
examples_per_sec = batch_size * num_steps_to_show_loss / duration
duration = 0.0
print('=> %s: step %d, loss = %f, learning_rate = %f (%.1f examples/sec)' % (
datetime.now(), step, loss.item(), scheduler.get_lr()[0], examples_per_sec))
if step % num_steps_to_check != 0:
continue
model_save_counter += 1
losses = np.append(losses, loss.item())
np.save(path_to_losses_npy_file, losses)
train_loss_array.append((step, loss.item()))
print('=> Evaluating on validation dataset...')
accuracy, test_loss_args = evaluator.evaluate(model)
test_loss = _loss(*test_loss_args)
val_loss_array.append((step, test_loss.item()))
print('==> accuracy = %f, best accuracy %f' % (accuracy, best_accuracy))
# print(f'==> loss = {test_loss}')
# Save model every 2 epochs
if model_save_counter >= 2 or step in [1000, 2000, 3000, 4000, 5000]:
path_to_checkpoint_file = model.store(path_to_log_dir, step=step)
print('=> Model saved to file: %s' % path_to_checkpoint_file)
model_save_counter = 0
model_saved = True
model_checkpoints.append((step, f"model-{step}.pth"))
if accuracy > best_accuracy:
patience = initial_patience
best_accuracy = accuracy
else:
patience -= 1
print("Train losses: ", train_loss_array)
print("Saved Model Checkpoints: ", model_checkpoints)
print('=> patience = %d' % patience)
if patience == 0 or step >= max_steps:
if not model_saved:
path_to_checkpoint_file = model.store(path_to_log_dir, step=step)
print('=> Model MANUALLY saved to file: %s' % path_to_checkpoint_file)
model_checkpoints.append((step, f"model-{step}.pth"))
training_output = {
"model_checkpoints": model_checkpoints,
"train_loss": train_loss_array,
"val_loss": val_loss_array,
}
print("TRAINING OUTPUT -----------------------------")
print(training_output)
return training_output
# Сами минимальный и максимальный элементы в сумму не включать.
# Примечание: попытайтесь решить задания без использования функций max, min, sum, sorted и их аналогов, в том числе
# написанных самостоятельно.
from random import randint
SIZE = 20
MIN_ITEM = 0
MAX_ITEM = 100
summ = 0
arr = [randint(MIN_ITEM, MAX_ITEM) for _ in range(SIZE)]
print('Исходный массив')
print(arr)
min_el = max_el = arr[0]
min_i = max_i = 0
for i, el in enumerate(arr):
if min_el > el:
min_el = el
min_i = i
if max_el < el:
max_el = el
max_i = i
if min_i > max_i:
min_i, max_i = max_i, min_i
print('Полученный массив')
print(arr[min_i+1:max_i])
for i in range(min_i+1, max_i):
summ += arr[i]
print(f'Сумма элементов {summ}')
import csv
import ast
import pickle
import utilFunc
import pprint
with open('C:/Users/이재원/Documents/code/Sodict.txt', 'r',
encoding="UTF-8") as inf:
aa = inf.read()
dicList = []
tokenNum = 0
newData = ""
for i in aa:
if i == '{':
newData = ""
tokenNum = tokenNum + 1
newData = newData + i
elif i == '}':
tokenNum = tokenNum - 1
newData = newData + i
dicList.append(newData)
else:
newData = newData + i
for num, value in enumerate(dicList):
dicData = ast.literal_eval(value)
dicList[num] = dicData
print(dicData['DEPTH 0'])
# init graph and train
som.init_graph()
som.train(colors)
# get output grid
image_grid = som.get_centroids()
# map colours to their closest neurons
mapped = som.map_vects(colors)
# plot
plt.imshow(image_grid)
plt.title('Colour SOM (' + str(som._m) + ' x ' + str(som._n) + '), ' +
str(som._n_iterations) + ' iterations')
for i, m in enumerate(mapped):
plt.text(m[1],
m[0],
color_names[i],
ha='center',
va='center',
bbox=dict(facecolor='white', alpha=0.5, lw=0))
plt.show()
# By running the session created in the method [init_graph()](#init_graph), we can evaluate the tensors and display the SOM data, for example the **weight vectors**:
# In[10]:
#np.set_printoptions(threshold=np.inf) # <--- with this option, the entire array is printed (we do not wish it here)
print(som._sess.run(som._weightage_vects))
