def plot_energy(S, filename):
cen = np_array(S**2).cumsum() / np_array(S**2).sum() * 100
DPI = 100
fig = plt.figure()
# Energy subplot
ax1 = fig.add_subplot(2, 1, 1)
line = ax1.plot(S**2, "o-", linewidth=1)
ax1.set_yscale("log")
plt.title("Basis vector vs Energy")
plt.xlabel("Basis vector number")
plt.ylabel("Energy")
plt.axis([0, None, None, None])
plt.grid(True)
# Cumulative energy subplot
ax2 = fig.add_subplot(2, 1, 2)
line = ax2.plot(cen, "o-", linewidth=1)
ax2.axis([0, None, 90, 101])
plt.title("Cumulative Energy")
plt.xlabel("Basis vector number")
plt.ylabel("Cumulative Energy")
plt.grid(True)
plt.tight_layout()
plt.savefig('{}.png'.format(filename), bbox_inches='tight', dpi=DPI)
return
def compare_csv_decimal_files(file1, file2, header=True, timeseries=False):
"""
This function compares two csv files
"""
#CHECK NUM LINES
with open_csv(file1) as fh1, \
open_csv(file2) as fh2:
assert sum(1 for line1 in fh1) == sum(1 for line2 in fh2)
with open_csv(file1) as fh1, \
open_csv(file2) as fh2:
csv1 = csv.reader(fh1)
csv2 = csv.reader(fh2)
if header:
assert next(csv1) == next(csv2) #header
while True:
try:
row1 = next(csv1)
row2 = next(csv2)
compare_start_index = 0
if timeseries:
assert row1[0] == row2[0] #check dates
compare_start_index=1
assert_almost_equal(np_array(row1[compare_start_index:], dtype=np_float32),
np_array(row2[compare_start_index:], dtype=np_float32),
decimal=2)
except StopIteration:
break
pass
return True
def __init__(self, dbFileName, force=False, scaleFactor=1000):
# data
self.dataManager = GMDataManager() # most data is saved to hdf
self.dbFileName = dbFileName # db containing all the data we'd like to use
self.condition = "" # condition will be supplied at loading time
# --> NOTE: ALL of the arrays in this section are in sync
# --> each one holds information for an individual contig
self.indices = np_array([]) # indices into the data structure based on condition
self.covProfiles = np_array([]) # coverage based coordinates
self.transformedCP = np_array([]) # the munged data points
self.averageCoverages = np_array([]) # average coverage across all stoits
self.kmerSigs = np_array([]) # raw kmer signatures
self.kmerVals = np_array([]) # PCA'd kmer sigs
self.contigNames = np_array([])
self.contigLengths = np_array([])
self.contigColours = np_array([]) # calculated from kmerVals
self.binIds = np_array([]) # list of bin IDs
# --> end section
# meta
self.validBinIds = {} # valid bin ids -> numMembers
self.binnedRowIndicies = {} # dictionary of those indices which belong to some bin
self.restrictedRowIndicies = {} # dictionary of those indices which can not be binned yet
self.numContigs = 0 # this depends on the condition given
self.numStoits = 0 # this depends on the data which was parsed
# contig links
self.links = {}
# misc
self.forceWriting = force # overwrite existng values silently?
self.scaleFactor = scaleFactor # scale every thing in the transformed data to this dimension
def real_imaginary_freq_domain(samples): """ Apply fft on the samples and return the real and imaginary parts in separate """ freq_domain = fft(samples) freq_domain_real = np_array([abs(x.real) for x in freq_domain]) freq_domain_imag = np_array([abs(x.imag) for x in freq_domain]) return freq_domain_real, freq_domain_imag
def __init_statistics(self):
stats = self.raw_stats
if stats is not None:
combined = np_array([[int(team), stats['oprs'][team], stats['dprs'][team],
stats['ccwms'][team]] for team in stats['oprs'].keys()], np_object)
else:
teams = self.get_team()[:, 0]
num_teams = len(teams)
combined = np_rot90(
np_array([teams, np_zeros(num_teams), np_zeros(num_teams), np_zeros(num_teams)], np_object))[::-1]
self.stats = combined
def centroidfinder(cvimage, color, threshold):
lo = [ c - t for c, t in zip(color, threshold) ]
hi = [ c + t for c, t in zip(color, threshold) ]
mat = cv.CreateMat(cvimage.height, cvimage.width, cv.CV_8U)
cv.InRangeS(cvimage, lo, hi, mat)
data = [ [x, y] for x, y in product(range(mat.height), range(mat.width))
if int(mat[x, y]) ]
np_data = np_array(data)
np_centroids = np_array( [ [0, 0], [0, mat.width],
[mat.height, 0], [mat.height, mat.width] ])
centroids, labels = kmeans2(np_data, np_centroids)
return [ (x, y) for x, y in centroids.tolist() ]
def _interp_line(self, line):
# handles = line.getViewHandlePositions()
# x = []
# y = []
# for h in handles:
# x.append(h[1].x())
# y.append(h[1].y())
(x,y) = self._handle2points(line)
xi = range(int(x[0]),int(x[-1])+1)
yi = np_interp(np_array(xi), np_array(x), np_array(y))
return (xi,yi)
def __init_matches(self):
for match_type, var in [['qm', 'qualification_matches'], ['qf', 'quarter_final_matches'],
['sf', 'semi_final_matches'], ['f', 'final_matches']]:
num_matches = self.__count_matches(self.raw_matches, match_type)
if num_matches is not 0:
# zero = range(num_matches)
red_teams = np_zeros((num_matches,), np_object)
blue_teams = np_zeros((num_matches,), np_object)
blue_scores = np_zeros((num_matches,), np_object)
red_scores = np_zeros((num_matches,), np_object)
match_code = np_zeros((num_matches,), np_object)
match_numbers = np_arange(1, num_matches + 1, 1)
for match in self.raw_matches:
if match['comp_level'] == match_type:
match_num = match['match_number'] - 1
red_teams[match_num] = [np_int(match['alliances']['red']['teams'][0][3:]),
np_int(match['alliances']['red']['teams'][1][3:]),
np_int(match['alliances']['red']['teams'][2][3:])]
red_scores[match_num] = [-1 if match['alliances']['red']['score'] is None
else match['alliances']['red']['score'],
-1 if match['score_breakdown']['red']['auto'] is None
else match['score_breakdown']['red']['auto'],
-1 if match['score_breakdown']['red']['foul'] is None
else match['score_breakdown']['red']['foul']]
blue_teams[match_num] = [np_int(match['alliances']['blue']['teams'][0][3:]),
np_int(match['alliances']['blue']['teams'][1][3:]),
np_int(match['alliances']['blue']['teams'][2][3:])]
blue_scores[match_num] = [-1 if match['alliances']['blue']['score'] is None
else match['alliances']['blue']['score'],
-1 if match['score_breakdown']['blue']['auto'] is None
else match['score_breakdown']['blue']['auto'],
-1 if match['score_breakdown']['blue']['foul'] is None
else match['score_breakdown']['blue']['foul']]
match_code[match_num] = match['key']
red_win = np_array(red_scores.tolist())[:, 0] > np_array(blue_scores.tolist())[:, 0]
winner = np_array(['blue'] * len(red_win))
winner[red_win] = 'red'
self.__setattr__(var,
np_rot90(np_array([[match_type] * num_matches, match_numbers, red_teams, blue_teams,
red_scores, blue_scores, winner, match_code], np_object))[::-1])
def updateAlgoData():
"""
Update from raw data into FPs directly used by location.fixPosWLAN() from WppDB(wpp_clusterid, wpp_cfps).
1) Retrieve latest incremental rawdata(csv) from remote FTP server(hosted by FPP).
2) Decompress bzip2, import CSV into wpp_uprecsinfo with its ver_uprecs, Update ver_uprecs in wpp_uprecsver.
3) Incr clustering inserted rawdata for direct algo use.
"""
dbips = DB_OFFLINE
for dbip in dbips:
dbsvr = dbsvrs[dbip]
wppdb = WppDB(dsn=dbsvr['dsn'], dbtype=dbsvr['dbtype'])
ver_wpp = wppdb.getRawdataVersion()
# Sync rawdata into wpp_uprecsinfo from remote FTP server.
print 'Probing rawdata version > [%s]' % ver_wpp
vers_fpp,localbzs = syncFtpUprecs(FTPCFG, ver_wpp)
if not vers_fpp: print 'Not found!'; continue
else: print 'Found new vers: %s' % vers_fpp
# Handle each bzip2 file.
alerts = {'vers':[], 'details':''}
tab_rd = 'wpp_uprecsinfo'
for bzfile in localbzs:
# Filter out the ver_uprecs info from the name of each bzip file.
ver_bzfile = bzfile.split('_')[-1].split('.')[0]
# Update ver_uprecs in wpp_uprecsver to ver_bzfile.
wppdb.setRawdataVersion(ver_bzfile)
print '%s\nUpdate ver_uprecs -> [%s]' % ('-'*40, ver_bzfile)
# Decompress bzip2.
sys.stdout.write('Decompress & append rawdata ... ')
csvdat = csv.reader( BZ2File(bzfile) )
try:
indat = np_array([ line for line in csvdat ])
except csv.Error, e:
sys.exit('\n\nERROR: %s, line %d: %s!\n' % (bzfile, csvdat.line_num, e))
# Append ver_uprecs(auto-incr),area_ok(0),area_try(0) to raw 16-col fp.
append_info = np_array([ [ver_bzfile,0,0] for i in xrange(len(indat)) ])
indat_withvers = np_append(indat, append_info, axis=1).tolist(); print 'Done'
# Import csv into wpp_uprecsinfo.
try:
sys.stdout.write('Import rawdata: ')
wppdb.insertMany(table_name=tab_rd, indat=indat_withvers, verb=True)
except Exception, e:
_lineno = sys._getframe().f_lineno
_file = sys._getframe().f_code.co_filename
alerts['details'] += '\n[ver:%s][%s:%s]: %s' % \
(ver_bzfile, _file, _lineno, str(e).replace('\n', ' '))
alerts['vers'].append(ver_bzfile)
print 'ERROR: Insert Rawdata Failed!'
continue
def iterboxed(self, rows):
"""Iterator that yields each scanline in boxed row flat pixel
format. `rows` should be an iterator that yields the bytes of
each row in turn.
"""
def asvalues(raw):
"""Convert a row of raw bytes into a flat row. Result will
be a freshly allocated object, not shared with
argument.
"""
if self.bitdepth == 8:
return np_array(raw,'uint8')
else:
raw = tostring(raw)
return np_array(struct.unpack('!%dH' % (len(raw)//2), raw),'uint%d' % self.bitdepth)
assert self.bitdepth < 8
width = self.width
# Samples per byte
spb = 8//self.bitdepth
out = array('B')
mask = 2**self.bitdepth - 1
shifts = [self.bitdepth * i
for i in reversed(list(range(spb)))]
for o in raw:
out.extend([mask&(o>>i) for i in shifts])
return out[:width]
return np_array(map(asvalues, rows))
def get_zone_count_estimates(location_id, door_count_placement_view_pair, start_date, end_date, adjusted=False):
"""Iterates through .csv files to return a list of (datetime, zone_count)
ARGS
location_id: location_id of installation, eg '55'
door_count_placement_view_pair: placement and view id pair, e.g. ('3333230','0')
start_date: in format YYYY-MM-DD, <datetime>
end_date: in format YYYY-MM-DD. range is exclusive '<'. <datetime>
adjusted: to select between raw data or adusted <bool>. if adjusted is chosen but not available, returns raw.
RETURN
array with (datetime, zone_count) tuples
"""
datetime_zone_count_pairs = []
day = timedelta(days = 1)
curr_day = start_date
while curr_day < end_date:
date_str = date2str(curr_day, "%Y-%m-%d")
fullpath = ANALYSIS_FOLDER_GLOBAL+str(location_id)+'/'+gtfilename(location_id,door_count_placement_view_pair,curr_day)
if DEBUG:
print 'get_zone_count_estimates: reading file:', fullpath
data = read_csv(fullpath)
for idx in range(len(data)):
ts = utc.localize(get_datetime_from_csv_row(data[idx]), is_dst=None).astimezone(utc)
if ts >= start_date and ts < end_date:
datetime_zone_count_pairs.append(get_zone_count(data[idx], adjusted))
curr_day += day
datetime_zone_count_pairs = np_array(datetime_zone_count_pairs)
return datetime_zone_count_pairs
def amplitude_regularization(signal, bits=16, factor=0.7):
"""
ARGS:
signal: signal amplitudes, should be in the range [-1.0, 1.0], numpy array of numbers
bits: bit-depth value, <int>
factor: 0.7 by default, as suggested by Gerald Friedland @ ICSI
RETURN:
regularized: amplitude regularized signal, <number> or numpy array of numbers
"""
if isinstance(signal, list):
signal = np_array(signal)
elif isinstance(signal, (int, long, float, complex)):
raise Exception("Invalid arg")
# convert amplitude [-1.0, 1.0] to N-bit samples
half_n_bits = 2**(bits-1)
signal_scaled_to_n_bits = (signal + 1) * half_n_bits
# regularize
regularized = signal_scaled_to_n_bits ** factor
# scale back to [-1.0,1.0]
regularized -= half_n_bits
regularized /= half_n_bits
return regularized
def pca(self, data_matrix):
"""Perform PCA.
Principal components are given in self.pca,
and the variance in self.variance.
Parameters
----------
data_matrix : list of lists
List of tetranucleotide signatures
"""
cols = len(data_matrix[0])
data_matrix = np_reshape(np_array(data_matrix), (len(data_matrix), cols))
pca = PCA()
pc, variance = pca.pca_matrix(data_matrix, 3, bCenter=True, bScale=False)
# ensure pc matrix has at least 3 dimensions
if pc.shape[1] == 1:
pc = np_append(pc, np_zeros((pc.shape[0], 2)), 1)
variance = np_append(variance[0], np_ones(2))
elif pc.shape[1] == 2:
pc = np_append(pc, np_zeros((pc.shape[0], 1)), 1)
variance = np_append(variance[0:2], np_ones(1))
return pc, variance
def sim(self, src, tar):
"""Return the Steffensen similarity of two strings.
Parameters
----------
src : str
Source string (or QGrams/Counter objects) for comparison
tar : str
Target string (or QGrams/Counter objects) for comparison
Returns
-------
float
Steffensen similarity
Examples
--------
>>> cmp = Steffensen()
>>> cmp.sim('cat', 'hat')
0.24744247205786737
>>> cmp.sim('Niall', 'Neil')
0.1300991207720166
>>> cmp.sim('aluminum', 'Catalan')
0.011710186806836031
>>> cmp.sim('ATCG', 'TAGC')
4.1196952743871653e-05
.. versionadded:: 0.4.0
"""
if src == tar:
return 1.0
if not src or not tar:
return 0.0
self._tokenize(src, tar)
a = self._intersection_card()
b = self._src_only_card()
c = self._tar_only_card()
d = self._total_complement_card()
n = a + b + c + d
p = np_array([[a, b], [c, d]]) / n
psisq = 0.0
for i in range(len(p)):
pi_star = p[i, :].sum()
for j in range(len(p[i])):
pj_star = p[:, j].sum()
num = p[i, j] * (p[i, j] - pi_star * pj_star) ** 2
if num:
psisq += num / (
pi_star * (1 - pi_star) * pj_star * (1 - pj_star)
)
return psisq
def hz2mel(f):
""" Convert a number or numpy numerical array of frequencies in Hz into mel
ARGS: Frequency or array of frequencies, <number> or numpy array of numbers
RETURN: Mel frequency(ies), <number> or numpy array of numbers
"""
if isinstance(f, list):
f = np_array(f)
return 1127.01048 * log(f/700.0 +1)
def get_sample_energy(samples):
"""
ARGS:
samples: samples of a signal
"""
if isinstance(samples, list) or isinstance(samples, tuple):
samples = np_array(samples)
return sum(samples**2)
def mel2hz(m):
""" Convert a number or numpy numerical array of frequency in mel into Hz
ARGS: Mel Frequency or array of mel frequencies, <number> or numpy array of numbers
RETURN: frequency(ies) in Hz, <number> or numpy array of numbers
"""
if isinstance(m, list):
m = np_array(m)
return (exp(m / 1127.01048) - 1) * 700
def __init__(self, wheels):
"""
Create a new chassis, specifying a set of wheels.
:param wheels:
A sequence of :class:`triangula.chassis.HoloChassis.OmniWheel` objects defining the wheels for this chassis.
"""
self.wheels = wheels
self._matrix_coefficients = np_array([[wheel.co_x, wheel.co_y, wheel.co_theta] for wheel in self.wheels])
def makeColourProfile(self):
"""Make a colour profile based on ksig information"""
working_data = np_array(self.kmerSigs, copy=True)
Center(working_data,verbose=0)
p = PCA(working_data)
components = p.pc()
# now make the colour profile based on PC1
self.kmerVals = np_array([float(i) for i in components[:,0]])
# normalise to fit between 0 and 1
self.kmerVals -= np_min(self.kmerVals)
self.kmerVals /= np_max(self.kmerVals)
if(False):
plt.figure(1)
plt.subplot(111)
plt.plot(components[:,0], components[:,1], 'r.')
plt.show()
def init_classifier_fn(self, **kwargs):
cs_df = self._academic_clusterer.courses_features
AcademicFailureEstimator.COURSES = cs_df['course'].values
se_df = self._academic_clusterer.semesters_features
sf_df = self._academic_clusterer.students_features
gpa_df = self._academic_clusterer.ha_df.drop_duplicates(['student','GPA'])
ss_df = pd_merge( se_df, sf_df, on='student' )
ss_df = pd_merge( ss_df, gpa_df, on='student' )
ss_df = pd_merge( ss_df, cs_df, on='course' )
data = ss_df.apply( self.get_ss_features, axis=1 )
data = np_array( data.tolist() )
X = data
y = ss_df['ha_reprobado'].apply(lambda x: 0 if x else 1).values
# H = np_unique( X[:,0] )
# H = np_array( [ H, np_zeros( len(H) ) ] ).T
# l = np_ones( len( H ) )
# X = np_append( X, H, axis=0)
# y = np_append( y, l )
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.30,
random_state=7)
# logreg = LogisticRegression(random_state=7)
logreg = AdaBoostClassifier(random_state=10)
logreg = CalibratedClassifierCV( logreg, cv=2, method='sigmoid')
# logreg = GaussianNB()
logreg.fit(X, y)
logreg_prob = logreg.predict_proba
logreg_predict = logreg.predict
y_pred = logreg.predict(X_test)
recall = recall_score(y_test, y_pred)
def quality(data):
_z_ = logreg_predict(data)[0]
sample = X[ y==_z_ ]
sample_ = X[ y==(1-_z_) ]
d = np_linalg_norm( [data] - sample )
d_ = np_linalg_norm( [data] - sample_ )
r = np_max( d_ )/np_max( d )
# r = np_mean( d )/np_mean( d_ )
# r = np_min( d )/np_min( d_ )
# r = 0.5 * ( r + recall )
if r > 1:
r = abs( 1-r )
r = 0.5 * ( r + recall )
return str( r )
clf = lambda data: [ logreg_prob( data ), quality(data) ]
self._clf = clf
"""
def get_team_rankings(self, team='all'):
# ['Rank' 'Team' 'Qual Avg' 'Auto' 'Container' 'Coopertition' 'Litter' 'Tote' 'Played']
request = np.array(self.raw_rankings[1:]).astype(np.int)
# print(np.array(self.raw_rankings[0]))
if team is 'all':
return request
getter = request[request[:, 1] == team]
if len(getter) is not 0:
return request[request[:, 1] == team][0]
return np_array([0, team, 0, 0, 0, 0, 0, 0, 0])
def run(self):
dump_logger = getLogger('dumpscraper')
# Let's invoke the getscore runner and tell him to work on training data
dump_logger.info("Calculating dump score...")
running = getscore.DumpScraperGetscore(self.settings, self.parentArgs)
running.run()
# First of all let's feed the classifier with the training data
training = scipy_genfromtxt(self.settings['data_dir'] + "/" + "training/features.csv", delimiter=",", skip_header=1, usecols=(0, 1, 2))
target = scipy_genfromtxt(self.settings['data_dir'] + "/" + "training/features.csv", delimiter=",", skip_header=1, usecols=(-2))
clf = sklearn.neighbors.KNeighborsClassifier(10, weights='uniform')
clf.fit(training, target)
trash_count = hash_count = plain_count = 0
cleared = []
with open(self.settings['data_dir'] + "/" + 'features.csv', 'rb') as csvfile:
reader = csv_reader(csvfile)
for line in reader:
if line[0] == 'Trash score':
continue
features = np_array(line[0:3])
features = features.reshape(1, -1)
label = clf.predict(features)
if label == 0:
folder = 'trash'
trash_count += 1
elif label == 1:
folder = 'hash'
hash_count += 1
elif label == 2:
folder = 'plain'
plain_count += 1
target_file = self.settings['data_dir'] + "/" + 'organized/' + folder + "/" + line[-1]
target_dir = path.dirname(target_file)
# If asked for a clean run, let's delete the entire folder before copying any file
if self.parentArgs.clean and target_dir not in cleared and path.exists(target_dir):
cleared.append(target_dir)
shutil_rmtree(target_dir)
if not path.exists(target_dir):
makedirs(target_dir)
shutil_copyfile(self.settings['data_dir'] + "/" + 'raw/' + line[-1], target_file)
dump_logger.info("Trash files: " + str(trash_count))
dump_logger.info("Hash files: " + str(hash_count))
dump_logger.info("Plain files: " + str(plain_count))
dump_logger.info("Operation completed")
def ftom(f):
""" convert frequency to midi note
ARGS
f: frequency(ies) in Hz > 0, <number> or numpy number array
RETURN
m: MIDI note with A 440 as reference, <float>
"""
if isinstance(f, list):
f = np_array(f)
m = 17.3123405046 * log(.12231220585 * f)
return m
def _get_numeric_node_col_type_numeric(self, array):
"""array: sorted array by the numeric column"""
split = self.get_numeric_col_split(array)
if split.split_index is None:
return None, None
thr = (array[split.split_index - 1, 0] + array[split.split_index, 0]) / 2
left_indices = array[:split.split_index, 3]
right_indices = array[split.split_index:, 3]
indices = {'left': left_indices, 'right': right_indices}
indices = {k: np_array(v).astype(int) for k, v in indices.items()}
return NumericBinaryNode(array.shape[0], split.impurity, self.col_name, thr), indices
def extractDAISY(images): featureVecs = [] for image in images: featureVecs.append(feature.daisy(image, step=8, radius=8, rings=3).flatten()) featureVecs = np_array(featureVecs) return featureVecs
def equilibrium_pts(self):
#Finds equlibrium points as [M,N,Z]
from numpy import array as np_array
from numpy.linalg import solve as np_solve
g = self.k2 * (self.r2 - self.d2) / self.r2
A = np_array([[self.r1/self.k1,self.a1,0], [-self.a2,0,self.b],\
[0,self.b,self.r2/self.k2]])
c = np_array([self.r1, self.d1, self.r2 - self.d2])
self.E0,self.E1,self.E2,self.E3 = [3*[0],[self.k1,0,0],[0,0,g],\
[self.k1,0,g]]
self.E4 = [0,(self.b*self.r2*g-self.r2*self.d1)/(self.k2*self.b**2),\
self.d1/self.b]
self.E5 = np_solve(A, c).tolist()
def mungeCbar(cbar):
'''Set dims of the colorbar by hacking the mainframe and coping with
different matplotlib versions'''
try:
# older matplotlib
cbar.outline.set_ydata([0.15] * 2 + [0.85] * 4 + [0.15] * 3)
except AttributeError:
# matplotlib 1.4.x
tmp_xy = cbar.outline.get_xy()
tmp_xy[:, 1] = np_array(([0.15] * 2 + [0.85] * 4 + [0.15] * 3))
cbar.outline.set_xy(tmp_xy)
def getBreakpointsByCardinality(self, cardinality):
if cardinality not in self.breakpointsByCardinality:
frac = 1.0 / cardinality
list_percent = []
for i_fl in np_arange(frac, 1.0, frac):
list_percent.append(i_fl)
self.breakpointsByCardinality[cardinality] = (
np_array(norm.ppf(list_percent)) * self.std + self.mean)
return self.breakpointsByCardinality[cardinality]
def numpy_full_list(array, desired_length):
'''retuns array with desired length by repeating last item'''
if not isinstance(array, ndarray):
array = np_array(array)
length_diff = desired_length - array.shape[0]
if length_diff > 0:
new_part = np_repeat(array[np_newaxis, -1], length_diff, axis=0)
return np_concatenate((array, new_part))[:desired_length]
return array[:desired_length]
def mungeCbar(cbar):
'''Set dims of the colorbar by hacking the mainframe and coping with
different matplotlib versions'''
try:
# older matplotlib
cbar.outline.set_ydata([0.15] * 2 + [0.85] * 4 + [0.15] * 3)
except AttributeError:
# matplotlib 1.4.x
tmp_xy = cbar.outline.get_xy()
tmp_xy[:,1] = np_array(([0.15] * 2 + [0.85] * 4 + [0.15] * 3))
cbar.outline.set_xy(tmp_xy)
def read_input(self, sample_name: str, start_idx: int, end_idx: int):
self.__check_indexes(start_idx, end_idx)
full_transform = self.read_full_transform(sample_name)
if full_transform.shape[0] < self.max_time_steps * self.time_size:
return np_array([])
x = self.normalize_input(full_transform[start_idx:end_idx, :])
return np_reshape(
x, (1, self.max_time_steps, self.time_size, self.freq_size))
def get_hist_sub_image(self, offset, dimensions):
x, y = offset
width, height = dimensions
sub = self.source[y:y + height, x:x + width]
hist_values = histogram1d(sub,
bins=self.no_of_colors,
range=[0, self.no_of_colors])
def to_hist_value(color):
return hist_values[color]
return np_array(to_hist_value(sub))
def pre_emphasis(signal, p):
""" Apply pre-emphasis filter
ARGS
signal: signal amplitudes, should be in the range [-1.0,1.0], array of numbers
RETURN
Filtered signal, array of numbers
"""
if isinstance(signal, list):
signal = np_array(signal)
elif isinstance(signal, (int, long, float, complex)):
raise Exception("Invalid arg")
return scipy.signal.lfilter([1., -p], 1, signal)
def log_transitions(self):
if len(self.memory) > 0:
basename = self.logdir + "/{}.{}".format(
self.environment_name,
datetime.now().strftime("%Y-%m-%d-%H-%M-%s"))
print("Base Filename: ", basename, flush=True)
state, action, reward, next_state, done = zip(*self.memory)
np_save(basename + "-state.npy",
np_array(state),
allow_pickle=False)
np_save(basename + "-action.npy",
np_array(action),
allow_pickle=False)
np_save(basename + "-reward.npy",
np_array(reward),
allow_pickle=False)
np_save(basename + "-nextstate.npy",
np_array(next_state),
allow_pickle=False)
np_save(basename + "-done.npy", np_array(done), allow_pickle=False)
self.memory.clear()
def read_txt(self, txt_name: str, txt_dir='') -> np_ndarray:
txt_path = self._standard_check(txt_name, txt_dir)
with open(txt_path, 'r') as txt:
lines = txt.readlines()
lines_list = []
for line in lines:
data = [txt_name.replace('.gt.txt', '')] + line.split()
lines_list.append(data)
return np_array(lines_list)
def set_data(self):
data_len = self.col_mat.shape[0]*self.col_mat.shape[1]
self.pos = np_empty((data_len, 3))
self.size = np_ones((data_len))*.1
self.color = np_empty((data_len, 4))
jj = 0
for ii in range(len(self.y)):
for zz in range(len(self.z)):
self.pos[jj,:] = np_array((self.x[ii]/self.kx,self.y[ii]/self.ky,zz/self.kz))
self.color[jj,:] = self.col_mat[ii,zz]/255.
jj = jj + 1
def _toSLH(self):
# These numerically optimal solutions were obtained as outlined in
# my blog post on the Mabuchi-Lab internal blog
# email me ([email protected])for details.
if self.N == 1:
kappa0 = 9.28874141848 / self.tau
kappas = np_array([7.35562929]) / self.tau
Deltas = np_array([3.50876192]) / self.tau
elif self.N == 3:
kappa0 = 14.5869543803 / self.tau
kappas = np_array([ 13.40782559, 9.29869721]) / self.tau
Deltas = np_array([3.48532283, 7.14204585]) / self.tau
elif self.N == 5:
kappa0 = 19.8871474779 / self.tau
kappas = np_array([19.03316217, 10.74270752, 16.28055664]) / self.tau
Deltas = np_array([3.47857213, 10.84138821, 7.03434809]) / self.tau
else:
raise NotImplementedError("The number of cavities to realize the delay must be one of 1,3 or 5.")
h0 = make_namespace_string(self.name, 'C0')
hp = [make_namespace_string(self.name, "C{:d}p".format(n+1)) for n in range((self.N-1)/2)]
hm = [make_namespace_string(self.name, "C{:d}m".format(n+1)) for n in range((self.N-1)/2)]
S = Matrix([1.])
slh0 = SLH(S, Matrix([[sqrt(kappa0) * Destroy(h0)]]), ZeroOperator)
slhp = [SLH(S, Matrix([[sqrt(kj) * Destroy(hj)]]), Dj * Create(hj) * Destroy(hj)) for (kj, Dj, hj) in zip(kappas, Deltas, hp)]
slhm = [SLH(S, Matrix([[sqrt(kj) * Destroy(hj)]]), -Dj * Create(hj) * Destroy(hj)) for (kj, Dj, hj) in zip(kappas, Deltas, hm)]
return freduce(lambda a, b: a << b, slhp + slhm, slh0)
def __init__(self, tree, parent, sax, cardinality):
"""
Initialization function of the rootnode class
:returns: a root node
:rtype: RootNode
"""
self.iSAX_word = np_array([sax, cardinality]).T
Node.__init__(self, parent=parent, name=str(self.iSAX_word))
self.tree = tree
self.sax = sax
self.cardinality = cardinality
self.cardinality_next = np_copy(self.cardinality)
self.cardinality_next = np_array([x*2 for x in self.cardinality_next])
# Number of sequences contained in the node (or by its sons)
self.nb_sequences = 0
""" The incremental computing part for CFOF """
self.mean = np_empty(shape=self.tree.size_word)
# Allows the incremental calculation of self.mean
self.sum = np_empty(shape=self.tree.size_word)
self.std = np_empty(shape=self.tree.size_word)
# Allows the incremental calculation of self.std
self.sn = np_empty(shape=self.tree.size_word)
# Specific to internal nodes
self.nodes = []
self.key_nodes = {}
self.terminal = False
self.level = 0
self.id = RootNode.id_global
RootNode.id_global += 1
def asvalues(raw):
"""Convert a row of raw bytes into a flat row. Result will
be a freshly allocated object, not shared with
argument.
"""
if self.bitdepth == 8:
return np_array(raw, 'uint8')
else:
raw = tostring(raw)
return np_array(struct.unpack('!%dH' % (len(raw) // 2), raw),
'uint%d' % self.bitdepth)
assert self.bitdepth < 8
width = self.width
# Samples per byte
spb = 8 // self.bitdepth
out = array('B')
mask = 2**self.bitdepth - 1
shifts = [self.bitdepth * i for i in reversed(list(range(spb)))]
for o in raw:
out.extend([mask & (o >> i) for i in shifts])
return out[:width]
def asvalues(raw):
"""Convert a row of raw bytes into a flat row. Result will
be a freshly allocated object, not shared with
argument.
"""
if self.bitdepth == 8:
return np_array(raw,'uint8')
else:
raw = tostring(raw)
return np_array(struct.unpack('!%dH' % (len(raw)//2), raw),'uint%d' % self.bitdepth)
assert self.bitdepth < 8
width = self.width
# Samples per byte
spb = 8//self.bitdepth
out = array('B')
mask = 2**self.bitdepth - 1
shifts = [self.bitdepth * i
for i in reversed(list(range(spb)))]
for o in raw:
out.extend([mask&(o>>i) for i in shifts])
return out[:width]
def set_data(self):
data_len = self.col_mat.shape[0] * self.col_mat.shape[1]
self.pos = np_empty((data_len, 3))
self.size = np_ones((data_len)) * .1
self.color = np_empty((data_len, 4))
jj = 0
for ii in range(len(self.y)):
for zz in range(len(self.z)):
self.pos[jj, :] = np_array(
(self.x[ii] / self.kx, self.y[ii] / self.ky, zz / self.kz))
self.color[jj, :] = self.col_mat[ii, zz] / 255.
jj = jj + 1
def getCentroidStats(self, profile):
"""Calculate the centroids of the profile"""
working_list = profile[self.rowIndices]
# return the mean and stdev
# we divide by std so we need to make sure it's never 0
tmp_stds = np_std(working_list, axis=0)
mean_std = np_mean(tmp_stds)
try:
std = np_array([x if x != 0 else mean_std for x in tmp_stds])
except:
std = mean_std
return (np_median(working_list,axis=0), std)
def getCentroidStats(self, profile):
"""Calculate the centroids of the profile"""
working_list = profile[self.rowIndices]
# return the mean and stdev
# we divide by std so we need to make sure it's never 0
tmp_stds = np_std(working_list, axis=0)
mean_std = np_mean(tmp_stds)
try:
std = np_array([x if x != 0 else mean_std for x in tmp_stds])
except:
std = mean_std
return (np_median(working_list, axis=0), std)
def de_mean(signal, axis=-1):
""" De-mean signal(zero center)
ARGS:
signal: signal alitudeii, shouldin the range [-1.0,1.0], <number> or array of numbers
axis: axis to apply mean, <int>
RETURN:
Zero centered signal, <number> or numpy array of numbers
"""
if isinstance(signal, list):
signal = np_array(signal)
elif isinstance(signal, (int, long, float, complex)):
raise Exception("Invalid arg")
return signal - mean(signal,axis)
def getCNNSamples(dir, mult=5): allVectors = getPreparedImagesCNN(dir) N = allVectors.shape[1] posPairs = [] for i in range(N): posPairs.append(np.concatenate((allVectors[0][i],allVectors[1][i]),axis=2)) OGRange = np_array(range(N),dtype=int) negPairs = [] for i in range(mult): negIndex = np_array(range(N),dtype=int) np.random.shuffle(negIndex) while 0 in (negIndex-OGRange): np.random.shuffle(negIndex) for i in range(N): negPairs.append(np.concatenate((allVectors[0][i],allVectors[1][negIndex[i]]),axis=2)) return np_array(posPairs),np_array(negPairs)
def toTensor(X):
if type(X) is typeTensor:
return X
if isDict(X):
X = list(X)
if isList(X):
X = np_array(X)
try:
if X.dtype == np_bool:
X = X.astype(np_uint8)
return _toTensor(X)
except:
return X
def read_image_sequence(self, video_name: str):
self.set_image_sequence_name(video_name)
# Set video directory
video_dir = os_path_join(self.im_seqs_dir, video_name)
# List all the images included inside the video directory
images_names = os_listdir(video_dir)
# Sort the images with respect to their name
sorted_images_names = sorted(images_names)
# Read every image and store it in a list.
# Turn this list to a numpy array.
self.left_sequence = np_array([
cv2_imread(os_path_join(video_dir, img_name))
for img_name in sorted_images_names if img_name.endswith('L.jpg')
])
self.right_sequence = np_array([
cv2_imread(os_path_join(video_dir, img_name))
for img_name in sorted_images_names if img_name.endswith('R.jpg')
])
return self.left_sequence, self.right_sequence
def getHeaderMid(self, listP, offset, size):
list = [0] * size
for i in range(0, size):
list[i] = listP[i + offset][2][0][0]
midy = (listP[size - 1 + offset][1] + listP[offset][1]) >> 1
src = np_array(list)
left = StripRegion._filteringAnomaly(src, StripRegion._modeCheck)
src = StripRegion._filteringAnomaly(left, StripRegion._two_sigma)
midx = np_average(src)
self.points = [()] * 2
self.points[0] = (midx, midy)
def get_cct(self, methods="Hernandez 1999"):
'''
approximate CCT using CIE 1931 xy values
'''
x, y, z = self.get_xyz()
if 0 in [x, y, z]:
return 0.0
logs.logger.debug(f"x = {x}, y = {y}, z = {z}")
if isinstance(methods, str):
methods = [methods]
ccts = list()
for curr_method in methods:
if curr_method == 'me_mccamy':
# McCamy's Approx
small_x = x / (x + y + z)
small_y = y / (x + y + z)
n = (small_x - 0.3320) / (0.1858 - small_y)
cct = 437 * (n**3) + 3601 * (n**2) + 6861 * n + 5517
if DEBUG:
logs.logger.debug(
f"[me_mccamy] calc x = {small_x}, calc y = {small_y} | Calc CCT = {cct} K"
)
elif curr_method in XY_TO_CCT_METHODS:
xyz_arr = np_array([x, y, z])
xy_arr = XYZ_to_xy(xyz_arr)
cct = xy_to_CCT(xy_arr, curr_method)
if DEBUG:
logs.logger.debug(
f"[{curr_method}] calc x,y = {xy_arr} | CCT = {cct}")
else:
options = ["me_mccamy"] + list(XY_TO_CCT_METHODS)
logs.logger.error(
f"{curr_method} Not found!\nCCT calculation methods: \n {options}"
)
return
ccts.append(int(cct))
if len(ccts) == 1:
return ccts[0]
else:
return ccts
def getLastOrder(self):
# if not live
if not self.app.isLive():
self.last_action = 'SELL'
return
orders = self.account.getOrders(self.app.getMarket(), '', 'done')
if len(orders) > 0:
last_order = orders[-1:]
# if orders exist and last order is a buy
if str(last_order.action.values[0]) == 'buy':
self.last_buy_size = float(last_order[last_order.action == 'buy']['size'])
self.last_buy_filled = float(last_order[last_order.action == 'buy']['filled'])
self.last_buy_price = float(last_order[last_order.action == 'buy']['price'])
# binance orders do not show fees
if self.app.getExchange() == 'coinbasepro':
self.last_buy_fee = float(last_order[last_order.action == 'buy']['fees'])
self.last_action = 'BUY'
return
else:
self.minimumOrderQuote()
self.last_action = 'SELL'
self.last_buy_price = 0.0
return
else:
base = float(self.account.getBalance(self.app.getBaseCurrency()))
quote = float(self.account.getBalance(self.app.getQuoteCurrency()))
# nil base or quote funds
if base == 0.0 and quote == 0.0:
sys.tracebacklimit = 0
raise Exception(f'Insufficient Funds! ({self.app.getBaseCurrency()}={str(base)}, {self.app.getQuoteCurrency()}={str(base)})')
# determine last action by comparing normalised [0,1] base and quote balances
order_pairs = np_array([ base, quote ])
order_pairs_normalised = (order_pairs - np_min(order_pairs)) / np_ptp(order_pairs)
if order_pairs_normalised[0] < order_pairs_normalised[1]:
self.minimumOrderQuote()
self.last_action = 'SELL'
elif order_pairs_normalised[0] > order_pairs_normalised[1]:
self.minimumOrderBase()
self.last_action = 'BUY'
else:
self.last_action = 'WAIT'
return
def plot_generated_nodes(partitioner):
import matplotlib.cm as cm
import matplotlib.pyplot as plt
from matplotlib.collections import PatchCollection
from matplotlib.patches import Rectangle
from numpy import array as np_array
areas = list()
fig = plt.figure()
fig.set_size_inches(15, 20)
ax1 = fig.add_subplot(211)
xs = [(s.llx, s.urx) for s in partitioner.sinks + [partitioner.source]]
ys = [(s.lly, s.ury) for s in partitioner.sinks + [partitioner.source]]
x_max, y_max = max([x[1] for x in xs]), max([y[1] for y in ys])
x_min, y_min = min([x[0] for x in xs]), min([y[0] for y in ys])
for s in partitioner.sinks:
x, y = s.llx, s.lly
w, h = s.width, s.height
rectangle = Rectangle((x, y), w, h, alpha=0.3)
ax1.add_patch(rectangle)
x, y = partitioner.source.llx, partitioner.source.lly
w, h = partitioner.source.width, partitioner.source.height
rectangle = Rectangle((x, y), w, h, facecolor='red', alpha=0.3)
ax1.add_patch(rectangle)
ax1.set_xlim([x_min, x_max])
ax1.set_ylim([x_min, y_max])
patches = list()
for n in partitioner.nodes:
x, y = n.llx, n.lly
w, h = n.width, n.height
rectangle = Rectangle((x, y), w, h)
patches.append(rectangle)
areas.append(n.area)
ax2 = fig.add_subplot(212)
col = PatchCollection(patches, alpha=0.4)
col.set(array=np_array(areas), cmap='jet')
ax2.add_collection(col)
ax2.set_xlim([x_min, x_max])
ax2.set_ylim([x_min, y_max])
# plt.colorbar(col)
fig.savefig('test.png', dpi=200)
def predict(self, data: list):
"""Загрузка модели из файла весов и совершение прогноза
:param data: данные для совершения прогноза
:return: список с результатом прогноза модели
"""
try:
prediction_data = np_array(data).reshape(-1, len(data))
prediction_data = self.xscaler.transform(prediction_data)
predicted = self.model.predict(prediction_data)
return self.yscaler.inverse_transform(predicted)
except TypeError as e:
# TODO: Logger
return None
def get_booked_bases(
base_class, booked_bases_list
): # runs on class init, saves a list of booked bases at the time of init to self.booked
index_start = index_end = None
gc = service_account(filename=_secret_file)
sh = gc.open_by_key(cfg.database["jaeger_cal"])
ws = sh.worksheet("Current")
cal_export = np_array(ws.get_all_values())
date_col = cal_export[:, 0]
for index, value in enumerate(date_col):
if not index_start and value == dt.now(tz.utc).strftime('%b-%d'):
# gets us the header for the current date section in the google sheet
index_start = index + 1
continue
if value == (dt.now(tz.utc) + td(days=1)).strftime('%b-%d'):
# gets us the header for tomorrow's date in the sheet
index_end = index # now we know the range on the google sheet to look for base availability
break
if index_start is None or index_end is None:
log.warning(
f"Unable to find date range in Jaeger calendar for today's date. Returned: '{index_start}' "
f"to '{index_end}'")
return
today_bookings = cal_export[index_start:index_end]
for booking in today_bookings:
try:
start_time = date_parser(
booking[10]) # 45 mins before start of reservation
if booking[11] != "":
end_time = date_parser(booking[11])
else:
end_time = date_parser(booking[9])
if start_time <= dt.now(tz.utc) <= end_time:
splitting_chars = ['/', ',', '&', '(', ')']
booked_bases = booking[3]
for sc in splitting_chars:
booked_bases = booked_bases.replace(sc, ';')
booked_bases = [
_identify_base_from_name(base, base_class)
for base in booked_bases.split(";")
]
for booked in booked_bases:
if booked is not None and booked not in booked_bases_list:
booked_bases_list.append(booked)
except (ValueError, TypeError) as e:
log.warning(
f"Skipping invalid line in Jaeger Calendar:\n{booking}\nError: {e}"
)
def generate_points(_vars, const=None):
logging.debug(f"Running generate_points")
def lamb_const_linspace(var, const):
logging.debug(f"Running lamb_const_linspace")
if var.is_constant: return const
else: return var.linspace
if const is not None:
logging.debug(f"const is not None")
args = tuple(
lamb_const_linspace(v, const) for v in self.Vars.values())
else:
logging.debug(f"const is not not None")
args = tuple(v.linspace for v in self.Vars.values())
logging.debug(f"using args {args}")
gpts = self.func(*args)
# use [x][x] to exclude datatype element
logging.debug(f"ASSIGNING POINTS with function {self.func}")
# logging.debug(f"{ gpts[0][0] }")
self.x.append(np_array(gpts[0][0]))
self.y.append(np_array(gpts[1][0]))
def naive_search_with_np(n):
prime_numbers = np_array([2])
i = prime_numbers[0]
while len(prime_numbers) != n:
i += 1
for d in np_arange(2, i):
i_is_prime = True
if i%d == 0:
i_is_prime = False
break
if i_is_prime:
prime_numbers = np_append(prime_numbers, i)
print('Байт:', getsizeof(prime_numbers))
def get_pola_data(dataset, on_off, clean=True):
for key in on_off.keys():
t0 = on_off[key].t_start.unix
tf = on_off[key].t_stop.unix
for dkey in DATA_KEYS:
on_off[key].data[dkey] = np_array([
x[dkey] for x in dataset.iterrows()
if ((x['time'] + x['subtime'] * CLOCK) > t0) and (
(x['time'] + x['subtime'] * CLOCK) < tf)
])
if clean and on_off[key].data[DATA_KEYS[0]].shape == (0, ):
on_off.pop(key)