def call_feature_demon(imageurl):
result_dic = {}
#import pdb; pdb.set_trace()
try:
fe_starttime = time.time()
url = url_prefix % imageurl
response = urllib2.urlopen(url)
logging.info('extract_feature done, %.4f', time.time() - fe_starttime)
if response <> None:
retrieved_items = json.loads(response.read())
if retrieved_items['result']:
result_dic['url'] = retrieved_items['url']
result_dic['predicted_category'] = retrieved_items['category']
result_dic['scores'] = \
np.trim_zeros((np.asarray(retrieved_items['score']) * 100).astype(np.uint8))
#print(result_dic['scores'].shape)
result_dic['predicted_category'] = result_dic['predicted_category'][0:result_dic['scores'].size]
result_dic['feature'] = np.asarray(retrieved_items['feature'])
result_dic['predicted_category_gc'] = retrieved_items['category_gc']
result_dic['scores_gc'] = \
np.trim_zeros((np.asarray(retrieved_items['score_gc']) * 100).astype(np.uint8))
#print(result_dic['scores'].shape)
result_dic['predicted_category_gc'] = result_dic['predicted_category_gc'][0:result_dic['scores_gc'].size]
result_dic['sentence'] = retrieved_items['sentence']
except Exception as err:
logging.info('call_feature_demon error: %s', err)
return {'result': False, 'feature': None}
return result_dic
def computePrecision(R_cv, B_cv, predictions, threshold, n): ''' Computes Precision@N metric on the cross-validation set. Precision@N is the percentage of movies the user rated above threshold in the recommendation list of size n ''' cv_predictions = np.multiply(predictions, B_cv) sorted_predictions = np.fliplr(np.sort(cv_predictions))[:,:n] top_indices = np.fliplr(np.argsort(cv_predictions))[:,:n] num_users = R_cv.shape[0] precision = np.zeros(num_users, dtype=float) for user_id in range(num_users): user_liked = np.ceil(threshold)<= np.trim_zeros(R_cv[user_id,top_indices[user_id]]) user_disliked = np.trim_zeros(R_cv[user_id,top_indices[user_id]]) <= np.floor(threshold) # we think that a recommendation is good if the predicted rating # is grater than threshold not_recommended = np.trim_zeros(sorted_predictions[user_id]) < threshold recommended = np.trim_zeros(sorted_predictions[user_id]) >= threshold tp = np.count_nonzero(np.logical_or(not_recommended, user_liked)) fp = np.count_nonzero(np.logical_and(recommended, user_disliked)) precision[user_id] = float(tp) / (tp+fp) mean_precision = np.mean(precision) return mean_precision
def trim(self, min_qual, clone = True):
"""return a new object with DNA trimmed according to some min_qual
call using my_object.trim(clone=False) if you wish to
mask, trim, and return the current object.
"""
self._qual_check()
if clone: # pragma: no cover
rval = self.clone()
else: # pragma: no cover
rval = self
rval.min_qual = min_qual
trim, comparison = rval._check()
if trim:
rval.snapshot()
# use temp array so we don't change inner quality values (we are *not* masking)
temp = deepcopy(rval.quality)
temp[numpy.where(comparison)[0]] = 0
l = len(rval.quality) - len(numpy.trim_zeros(temp,'f'))
r = len(rval.quality) - len(numpy.trim_zeros(temp,'b'))
rval.quality = rval.quality[l:len(rval.quality) - r]
rval.sequence = rval.sequence[l:len(rval.sequence) - r]
rval.trimming = 't'
return rval
def redundancyAnalysis(m):
CHECK = False
dims = m.shape
vals = np.zeros((dims[1]*dims[2]*dims[3], (1<<PRECISION+1)))
#if CHECK:
aux = np.zeros(257)
auxVal = np.zeros(257)
index = 0
for c in range(dims[1]):
for x in range(dims[2]):
for y in range(dims[3]):
for i in range(dims[0]):
if CHECK:
if aux[ toInteger(m[i][c][x][y]) ] > 0:
if m[i][c][x][y] != auxVal[ toInteger(m[i][c][x][y]) ]:
print "ALARM"
auxVal[ toInteger(m[i][c][x][y]) ] = m[i][c][x][y]
aux[ toInteger(m[i][c][x][y]) ] += 1
vals[index][ toInteger(m[i][c][x][y]) ] += 1;
index += 1 # same value for all the filters
vals = vals.flatten()
vals = np.sort(vals)
#print vals
vals = np.trim_zeros(vals)
vals -= 1
vals = np.trim_zeros(vals)
vals = vals[::-1]
#print vals
vals = np.cumsum(vals)
vals /= dims[1]*dims[2]*dims[3]*dims[0]
vals = sample(vals,SAMPLES)
return vals
def axis_data(axis):
"""Gets the bounds of a masked area along a certain axis"""
x = mask.sum(axis)
trimmed_front = N.trim_zeros(x,"f")
offset = len(x)-len(trimmed_front)
size = len(N.trim_zeros(trimmed_front,"b"))
return offset,size
def plot_abilities_one_components(self, team_ids, **kwargs):
import matplotlib.pyplot as plt
import seaborn as sns
figsize = kwargs.get('figsize',(15,5))
if self.latent_variables.estimated is False:
raise Exception("No latent variables estimated!")
else:
plt.figure(figsize=figsize)
if type(team_ids) == type([]):
if type(team_ids[0]) == str:
for team_id in team_ids:
plt.plot(np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values()).T[self.team_dict[team_id]],
trim='b'), label=self.team_strings[self.team_dict[team_id]])
else:
for team_id in team_ids:
plt.plot(np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values()).T[team_id],
trim='b'), label=self.team_strings[team_id])
else:
if type(team_ids) == str:
plt.plot(np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values()).T[self.team_dict[team_ids]],
trim='b'), label=self.team_strings[self.team_dict[team_ids]])
else:
plt.plot(np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values()).T[team_ids],
trim='b'), label=self.team_strings[team_ids])
plt.legend()
plt.ylabel("Power")
plt.xlabel("Games")
plt.show()
def calc_power(N, field, dk, Nshot, Lx, Ly, norm): dkx = (2.*np.pi)/Lx dky = (2.*np.pi)/Ly V= Lx*Ly power = np.zeros(N, dtype=float) Nmodes = np.zeros(N, dtype=int) for ix in range (0,N): if ix <=N/2.: kx = ix*dkx else: kx = (ix-N)*dkx for iy in range (0,N): if iy <=N/2.: ky = iy*dky else: ky = (iy-N)*dky kval = (kx*kx + ky*ky)**0.5 if kval >0: power[int(kval/dk)] = power[int(kval/dk)] + field[ix][iy].real**2 + field[ix][iy].imag**2 - Nshot Nmodes[int(kval/dk)] = Nmodes[int(kval/dk)]+1 iNonZeros = np.where(Nmodes != 0) iZeros = np.where(Nmodes == 0) power[iNonZeros] = power[iNonZeros]/Nmodes[iNonZeros] k=np.linspace(dkx/2., ((N-1)*dkx)+dkx/2. ,num=N ) k[iZeros]= 0. return V*np.trim_zeros(power)/norm, np.trim_zeros(k)
def trim_zeros(self):
"""Remove the leading and trailing zeros.
"""
tmp = self.numpy()
f = len(self)-len(_numpy.trim_zeros(tmp, trim='f'))
b = len(self)-len(_numpy.trim_zeros(tmp, trim='b'))
return self[f:len(self)-b]
def parse_coverage_bam(fbam, genomecoveragebed="genomeCoverageBed"):
"""
Arguments:
- `fbam`: file to read coverage from. Needs samtools and bedtools (genomeCoverageBed)
- `genomecoveragebed`: path to genomeCoverageBed binary
"""
# forward and reverse coverage
# int is essential; see note below
fw_cov = numpy.zeros((MAX_LEN), dtype=numpy.int32)
rv_cov = numpy.zeros((MAX_LEN), dtype=numpy.int32)
file_genome = tempfile.NamedTemporaryFile(delete=False)
genome_from_bam(fbam, file_genome)
file_genome.close()
basic_cmd = "%s -ibam %s -g %s -d" % (
genomecoveragebed, fbam, file_genome.name)
for strand in ["+", "-"]:
cmd = "%s -strand %s" % (basic_cmd, strand)
try:
process = subprocess.Popen(cmd.split(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except OSError:
raise OSError, ("It seems %s is not installed" % cmd.split()[0])
(stdoutdata, stderrdata) = process.communicate()
retcode = process.returncode
if retcode != 0:
raise OSError("Called command exited with error code '%d'." \
" Command was '%s'. stderr was: '%s'" % (
retcode, cmd, stderrdata))
for line in str.splitlines(stderrdata):
if len(line.strip()) == 0:
continue
LOG.warn("Got following stderr message from genomeCoverageBed: %s" % line)
for line in str.splitlines(stdoutdata):
if len(line) == 0:
continue
(chr, pos, cov) = line.split('\t')
pos = int(pos)-1 # we use zero offset
cov = int(cov)
assert pos < MAX_LEN
if strand == '+':
fw_cov[pos] = cov
elif strand == '-':
rv_cov[pos] = cov
os.unlink(file_genome.name)
return (numpy.trim_zeros(fw_cov, trim='b'),
numpy.trim_zeros(rv_cov, trim='b'))
def fit(self, data):
magnitude = data[0]
time = data[1]
global m_21
global m_31
global m_32
Nsf = 100
Np = 100
sf1 = np.zeros(Nsf)
sf2 = np.zeros(Nsf)
sf3 = np.zeros(Nsf)
f = interp1d(time, magnitude)
time_int = np.linspace(np.min(time), np.max(time), Np)
mag_int = f(time_int)
for tau in np.arange(1, Nsf):
sf1[tau-1] = np.mean(np.power(np.abs(mag_int[0:Np-tau] - mag_int[tau:Np]) , 1.0))
sf2[tau-1] = np.mean(np.abs(np.power(np.abs(mag_int[0:Np-tau] - mag_int[tau:Np]) , 2.0)))
sf3[tau-1] = np.mean(np.abs(np.power(np.abs(mag_int[0:Np-tau] - mag_int[tau:Np]) , 3.0)))
sf1_log = np.log10(np.trim_zeros(sf1))
sf2_log = np.log10(np.trim_zeros(sf2))
sf3_log = np.log10(np.trim_zeros(sf3))
m_21, b_21 = np.polyfit(sf1_log, sf2_log, 1)
m_31, b_31 = np.polyfit(sf1_log, sf3_log, 1)
m_32, b_32 = np.polyfit(sf2_log, sf3_log, 1)
return m_21
def avgEpisodeVValue(self):
""" Returns the average V value on the episode (on time steps where a non-random action has been taken)
"""
if (len(self._Vs_on_last_episode) == 0):
return -1
if(np.trim_zeros(self._Vs_on_last_episode)!=[]):
return np.average(np.trim_zeros(self._Vs_on_last_episode))
else:
return 0
def assert_numden_almost_equal(self, n1, n2, d1, d2):
n1[np.abs(n1) < 1e-10] = 0.
n1 = np.trim_zeros(n1)
d1[np.abs(d1) < 1e-10] = 0.
d1 = np.trim_zeros(d1)
n2[np.abs(n2) < 1e-10] = 0.
n2 = np.trim_zeros(n2)
d2[np.abs(d2) < 1e-10] = 0.
d2 = np.trim_zeros(d2)
np.testing.assert_array_almost_equal(n1, n2)
np.testing.assert_array_almost_equal(d2, d2)
def printSummaryOutput(maincounts):
print OUTPUT + "\\summaryoutput.txt"
with open(OUTPUT + "\\summaryoutput.txt", 'wb') as f:
for index in maincounts.keys():
f.write(index + " ***************\n")
f.write("Total Count: " + str(maincounts[index]['totnum']) + '\n')
f.write("Total Females: " + str(maincounts[index]['numfemales']) + '\n')
f.write("Total Males: " + str(maincounts[index]['nummales']) + '\n')
f.write("Mean Length of Stay: " + str(np.mean(np.trim_zeros(np.nan_to_num(maincounts[index]['meanlengthofstay'])))) + '\n')
f.write("Mean Age: " + str(np.mean(np.trim_zeros(maincounts[index]['meanage']))) + '\n')
f.write("Mean Travel Events: " + str(np.mean(np.trim_zeros(maincounts[index]['meantravelevents']))) + '\n')
def get_history(self, state, action):
"""
:param state:
:param action:
:return: h(s,a), i.e. the number of times given action was selected in given state in the current episode
"""
state = tuple(np.trim_zeros(state, 'f'))
action = tuple(np.trim_zeros(action, 'f'))
if (state, action) in self.state_action_history:
return self.state_action_history[(state, action)]
return 0
def predict_two_components(self, team_1, team_2, team_1b, team_2b, neutral=False):
"""
Returns team 1's probability of winning
"""
if self.latent_variables.estimated is False:
raise Exception("No latent variables estimated!")
else:
if type(team_1) == str:
team_1_ability = np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values())[0].T[self.team_dict[team_1]], trim='b')[-1]
team_2_ability = np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values())[0].T[self.team_dict[team_2]], trim='b')[-1]
team_1_b_ability = np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values())[1].T[self.team_dict[team_1]], trim='b')[-1]
team_2_b_ability = np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values())[1].T[self.team_dict[team_2]], trim='b')[-1]
else:
team_1_ability = np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values())[0].T[team_1], trim='b')[-1]
team_2_ability = np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values())[0].T[team_2], trim='b')[-1]
team_1_b_ability = np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values())[1].T[team_1_b], trim='b')[-1]
team_2_b_ability = np.trim_zeros(self._model_abilities(self.latent_variables.get_z_values())[1].T[team_2_b], trim='b')[-1]
t_z = self.transform_z()
if neutral is False:
return self.link(t_z[0] + team_1_ability - team_2_ability + team_1_b_ability - team_2_b_ability)
else:
return self.link(team_1_ability - team_2_ability + team_1_b_ability - team_2_b_ability)
def largestDistrict(data):
largestDis=0
data_new = data.dropna(subset=['Location','PoliceDistrict'])
policeDis=np.array(data_new['PoliceDistrict'])
Location=np.array(data_new['Location'])
dic= collections.defaultdict(list)
for i in range(policeDis.shape[0]):
dic[policeDis[i]]+=[Location[i]]
for key,value in dic.items():
x=np.zeros(len(value))
y=np.zeros(len(value))
xi=0
for i in range(len(value)):
l=value[i][1:-2]
loc=l.split(",")
if float(loc[0])>26 and float(loc[0])<30 and float(loc[1])>-93 and float(loc[1])<-87:
x[xi]=float(loc[0])
y[xi]=float(loc[1])
xi+=1
x=np.trim_zeros(x)
y=np.trim_zeros(y)
stdX=np.std(x)
stdY=np.std(y)
meanX=np.mean(x)
meanY=np.mean(y)
bootom_la=math.radians(meanX-stdX)
top_la=math.radians(meanX+stdX)
delta_la=math.radians(2*stdX)
delta_long=0
a=math.sin(delta_la)*math.sin(delta_la)+math.cos(bootom_la)*math.cos(top_la)*math.sin(delta_long/2)*math.sin(delta_long/2)
c=2* math.atan2(math.sqrt(a),math.sqrt(1-a))
d_x=6371*c/2
delta_la=0
delta_long=math.radians(2*stdY)
la=math.radians(meanX)
a=math.sin(delta_la)*math.sin(delta_la)+math.cos(la)*math.cos(la)*math.sin(delta_long/2)*math.sin(delta_long/2)
c=2*math.atan2(math.sqrt(a),math.sqrt(1-a))
d_y=6371*c/2
area=math.pi*d_x*d_y
print area, key, meanX,meanY,stdX,stdY
largestDis=max(area,largestDis)
return largestDis
def clean_flux(self,flux, xDef = 1, lambdas = np.array([])):
'''clean a flux array to cross corralate to determine RV shift
eliminates NaNs
moving median to reduce peaks
optional: increase resolution by xDef times
'''
#Copy to output in case of no resampling
fluxHD = flux
newLambdas = lambdas
#clean NaNs and median outliers
fluxHD[np.isnan(fluxHD)] = 0
fluxNeat = fluxHD
fluxMed = signal.medfilt(fluxHD,5)
w = np.where(abs((fluxHD-fluxMed)/np.maximum(fluxMed,50)) > 0.4)
for ix in w[0]:
fluxNeat[ix] = fluxMed[ix]
#if enough data -> resample
if ((xDef>1) and (len(lambdas)>0)):
fFluxHD = interpolate.interp1d(lambdas,fluxNeat)
lambdas = np.arange(min(lambdas), max(lambdas),(max(lambdas)-min(lambdas))/len(lambdas)/xDef)
fluxNeat = fFluxHD(lambdas)
#remove trailing zeros, devide by fitted curve (flatten) and apply tukey window
fluxNeat = np.trim_zeros(fluxNeat,'f')
lambdas = lambdas[-len(fluxNeat):]
fluxNeat = np.trim_zeros(fluxNeat,'b')
lambdas = lambdas[:len(fluxNeat)]
if ((lambdas.shape[0]>0) & (fluxNeat.shape[0]>0)):
fFluxNeat = optimize.curve_fit(self.cubic, lambdas, fluxNeat, p0 = [1,1,1,1])
fittedCurve = self.cubic(lambdas, fFluxNeat[0][0], fFluxNeat[0][1], fFluxNeat[0][2], fFluxNeat[0][3])
# plt.plot(fittedCurve)
# plt.plot(fluxNeat)
# plt.show()
# plt.plot(fluxNeat/fittedCurve-1)
# plt.show()
fluxFlat = fluxNeat/fittedCurve-1
fluxWindow = fluxFlat * self.tukey(0.1, len(fluxFlat))
else:
fluxWindow = np.array([])
print 'empty after removing zeros'
return lambdas, fluxWindow
def trim_zeros(track, front=True, back=True):
if front:
values = np.trim_zeros(track.values, trim="f")
times = track.times[len(track) - len(values):]
else:
values = track.values
times = track.times
if back:
vallen = len(values)
values = np.trim_zeros(values, trim="b")
if vallen != len(values):
times = times[:len(values) - vallen]
track.values = values
track.times = times
def add_to_history(self, state, action):
"""
Adds a state, action pair to the history of the current episode, or increases its counter if already present
:param state:
:param action:
:return:
"""
state = tuple(np.trim_zeros(state, 'f'))
action = tuple(np.trim_zeros(action, 'f'))
if (state, action) in self.state_action_history:
self.state_action_history[(state, action)] += 1
else:
self.state_action_history[(state, action)] = 1
def slotted_autocorrelation(self, data, time, T, K,
second_round=False, K1=100):
slots = np.zeros((K, 1))
i = 1
# make time start from 0
time = time - np.min(time)
# subtract mean from mag values
m = np.mean(data)
data = data - m
prod = np.zeros((K, 1))
pairs = np.subtract.outer(time, time)
pairs[np.tril_indices_from(pairs)] = 10000000
ks = np.int64(np.floor(np.abs(pairs) / T + 0.5))
# We calculate the slotted autocorrelation for k=0 separately
idx = np.where(ks == 0)
prod[0] = ((sum(data ** 2) + sum(data[idx[0]] *
data[idx[1]])) / (len(idx[0]) + len(data)))
slots[0] = 0
# We calculate it for the rest of the ks
if second_round is False:
for k in np.arange(1, K):
idx = np.where(ks == k)
if len(idx[0]) != 0:
prod[k] = sum(data[idx[0]] * data[idx[1]]) / (len(idx[0]))
slots[i] = k
i = i + 1
else:
prod[k] = np.infty
else:
for k in np.arange(K1, K):
idx = np.where(ks == k)
if len(idx[0]) != 0:
prod[k] = sum(data[idx[0]] * data[idx[1]]) / (len(idx[0]))
slots[i - 1] = k
i = i + 1
else:
prod[k] = np.infty
np.trim_zeros(prod, trim='b')
slots = np.trim_zeros(slots, trim='b')
return prod / prod[0], np.int64(slots).flatten()
def sample(session, model, x, x_mask, graph=None):
"""Randomly samples translations from a RNNModel.
Args:
session: TensorFlow session.
model: a RNNModel object.
x: Numpy array with shape (factors, max_seq_len, batch_size).
x_mask: Numpy array with shape (max_seq_len, batch_size).
graph: a SampleGraph object (to allow reuse if sampling repeatedly).
Returns:
A list of NumPy arrays (one for each input sentence in x).
"""
feed_dict = {model.inputs.x: x, model.inputs.x_mask: x_mask}
if graph is None:
graph = SampleGraph(model)
sampled_ys = session.run(graph.outputs, feed_dict=feed_dict)
sampled_ys = sampled_ys.T
samples = []
for sample in sampled_ys:
sample = numpy.trim_zeros(list(sample), trim='b')
sample.append(0)
samples.append(sample)
assert len(samples) == x.shape[-1]
return samples
def mask_and_trim(self, min_qual, clone = True):
"""return a new object with DNA masked and trimmed according to some min_qual
call using my_object.mask_and_trim(clone=False) if you wish to
mask, trim, and return the current object.
"""
self._qual_check()
if clone: # pragma: no cover
rval = self.clone()
else: # pragma: no cover
rval = self
rval.min_qual = min_qual
trim, comparison = rval._check()
if trim:
rval.snapshot()
sequence_array = numpy.array(list(rval.sequence))
# get values where the qual is low and replace them w/ N
sequence_array[numpy.where(comparison)[0]] = 'N'
# trim and adjust rval.sequence
rval.sequence = sequence_array.tostring().lstrip('N').rstrip('N')
# trim and adjust rval.quality
temp = deepcopy(rval.quality)
temp[numpy.where(comparison)[0]] = 0
rval.quality[numpy.where(comparison)[0]] = 0
rval.quality = numpy.trim_zeros(rval.quality)
# ensure the trimming didn't remove something inside the read
assert len(rval.sequence) == len(rval.quality)
rval.trimming = 'mt'
return rval
def get_max_width(char):
max_width = 0
for byte in char:
trimmed = np.trim_zeros(byte, 'b')
max_width = max(len(trimmed), max_width)
return max_width
def get_session_data(session):
'''
Gather data from session: MDP, Agent, Env data, hashed by aeb; then aggregate.
@returns {dict, dict} session_mdp_data, session_data
'''
data_names = AGENT_DATA_NAMES + ENV_DATA_NAMES
mdp_data_names = ['t', 'epi'] + data_names
agg_data_names = ['epi'] + list(DATA_AGG_FNS.keys())
data_h_v_dict = {data_name: session.aeb_space.get_history_v(data_name) for data_name in data_names}
session_mdp_data, session_data = {}, {}
for aeb in session.aeb_space.aeb_list:
data_h_dict = {data_name: data_h_v[aeb] for data_name, data_h_v in data_h_v_dict.items()}
# trim back to remove any incomplete sessions due to multienv termination
complete_done_h = np.trim_zeros(data_h_dict['done'], 'b')
# offset properly to bin separate episodes
reset_bin = np.concatenate([[0.], complete_done_h[:-1]])
data_len = len(reset_bin)
reset_idx = reset_bin.astype('bool')
nonreset_idx = ~reset_idx
data_h_dict['t'] = np.ones(reset_idx.shape)
data_h_dict['epi'] = reset_idx.astype(int).cumsum()
mdp_df = pd.DataFrame({
data_name: data_h_dict[data_name][:data_len]
for data_name in mdp_data_names})
mdp_df = mdp_df.reindex(mdp_data_names, axis=1)
aeb_df = mdp_df[agg_data_names].groupby('epi').agg(DATA_AGG_FNS)
aeb_df.reset_index(drop=False, inplace=True)
session_mdp_data[aeb], session_data[aeb] = mdp_df, aeb_df
logger.debug(f'{session_data}')
data_size_in_bytes = util.memory_size(session_mdp_data)
logger.debug(f'Size of session data: {data_size_in_bytes} MB')
if data_size_in_bytes > 25:
logger.warn(f'Session data > 25 MB')
return session_mdp_data, session_data
def extract_area(dataset, geometry, **kwargs):
"""
Compute a spectrum using the basic GDAL driver to read a single band of the spectrum.
This is faster than the rasterio version and is the default.
"""
im = dataset.__gdal__
bands = kwargs.pop("bands", list(range(im.RasterCount)))
mask = create_mask(dataset,geometry)
maskt, offset = offset_mask(mask)
yo,xo = offset
ys,xs = maskt.shape
for i in (0,1):
assert N.allclose(maskt.sum(axis=i), N.trim_zeros(mask.sum(axis=i)))
maskt = maskt.transpose() # Conform to GDAL's fascist X-first expectations
maska = N.repeat(
N.expand_dims(maskt,0),
len(bands),
axis=0)
buffer=im.ReadRaster(xo,yo,xs,ys,
band_list=[b+1 for b in bands])
arr = N.fromstring(buffer, dtype=dataset.dtype)
arr = arr.reshape((len(bands), xs, ys))
arr = N.ma.masked_array(arr, arr==dataset.nodata)
arr[maska==False] = N.ma.masked
xarr = N.array([xo+0.5 for i in range(xs)]).reshape((xs,0))
yarr = N.array([yo+0.5 for i in range(ys)]).reshape((ys,1))
import IPython; IPython.embed()
return arr
def _parse_coeffs(self, coeff_dict):
"""Convert dict of 'c#' coefficients into a list
of coefficients in decreasing order, i.e. ['c2','c1','c0']
Parameters
----------
coeff_dict: dict
The dictionary of coefficients
Returns
-------
np.ndarray
The sequence of coefficient values
"""
params = {cN: coeff for cN, coeff in coeff_dict.items()
if cN.startswith('c') and cN[1:].isdigit()}
self.parameters = Parameters(**params)
# Parse 'c#' keyword arguments as coefficients
coeffs = np.zeros(100)
for k, v in self.parameters.dict.items():
if k.lower().startswith('c') and k[1:].isdigit():
coeffs[int(k[1:])] = v[0]
# Trim zeros and reverse
self.coeffs = np.trim_zeros(coeffs)[::-1]
def write_string(string, offset_x=0, offset_y=0, kerning=True):
str_buf = []
space = [0x00] * 5
gap = [0x00] * 3
if kerning:
space = [0x00] * 2
gap = [0x00]
for char in string:
if char == ' ':
str_buf += space
else:
char_data = numpy.array(_get_char(char))
if kerning:
char_data = numpy.trim_zeros(char_data)
str_buf += list(char_data)
str_buf += gap # Gap between chars
for x in range(len(str_buf)):
for y in range(7):
p = (str_buf[x] & (1 << y)) > 0
set_pixel(offset_x + x, offset_y + y, p)
l = len(str_buf)
del str_buf
return l
def process(self, samples, unit, target):
if "f" in self.trim:
temporary = numpy.empty_like(samples)
temporary[:] = samples
temporary[numpy.abs(samples) <= self.cutoff] = 0
temporary = numpy.trim_zeros(temporary, trim="f")
samples = samples[samples.shape[0] - temporary.shape[0]:]
if "b" in self.trim:
temporary = numpy.empty_like(samples)
temporary[:] = samples
temporary[numpy.abs(temporary) <= self.cutoff] = 0
temporary = numpy.trim_zeros(temporary, trim="b")
samples = samples[0:temporary.shape[0]]
return samples, unit
def _reconstruct_hypotheses(ys, parents, cost, beam_size):
"""Converts raw beam search outputs into a more usable form.
Args:
ys: NumPy array with shape (max_seq_len, beam_size*batch_size).
parents: NumPy array with same shape as ys.
cost: NumPy array with same shape as ys.
beam_size: integer.
Returns:
A list of lists of (translation, score) pairs. The outer list contains
one list for each input sentence in the batch. The inner lists contain
k elements (where k is the beam size).
"""
def reconstruct_single(ys, parents, hypoId, hypo, pos):
if pos < 0:
hypo.reverse()
return hypo
else:
hypo.append(ys[pos, hypoId])
hypoId = parents[pos, hypoId]
return reconstruct_single(ys, parents, hypoId, hypo, pos - 1)
hypotheses = []
batch_size = ys.shape[1] // beam_size
pos = ys.shape[0] - 1
for batch in range(batch_size):
hypotheses.append([])
for beam in range(beam_size):
i = batch*beam_size + beam
hypo = reconstruct_single(ys, parents, i, [], pos)
hypo = numpy.trim_zeros(hypo, trim='b') # b for back
hypo.append(0)
hypotheses[batch].append((hypo, cost[i]))
return hypotheses
def enet_path(X, y, alpha=0.5, **kwargs):
""" Convenience wrapper for running elastic_net that transforms coefs
Args:
X (np.ndarray): 2D (n_obs x n_features) design matrix
y (np.ndarray): 1D independent variable
alpha (float): ElasticNet mixing parameter (0 <= alpha <= 1.0)
specifying the mix of Ridge L2 (``alpha=0``) to Lasso L1
(``alpha=1``) regularization (default: 0.5).
kwargs: additional arguments provided to GLMNET
Returns:
tuple: intercepts, coefficients, rsquareds, and lambdas as np.ndarrays
"""
n_lambdas, intercepts_, coefs_, ia, nin, rsquareds_, lambdas, _, jerr \
= elastic_net(X, y, alpha, **kwargs)
# ia is 1 indexed
ia = np.trim_zeros(ia, 'b') - 1
# glmnet.f returns coefficients as 'compressed' array that
# requires re-indexing using ia and nin
coefs = np.zeros_like(coefs_)
coefs[ia, :] = coefs_[:np.max(nin), :n_lambdas]
return intercepts_, coefs, rsquareds_, lambdas
def guess(self, sequence, algorithm='sage'):
"""
Return the minimal CFiniteSequence that generates the sequence.
Assume the first value has index 0.
INPUT:
- ``sequence`` -- list of integers
- ``algorithm`` -- string
- 'sage' - the default is to use Sage's matrix kernel function
- 'pari' - use Pari's implementation of LLL
- 'bm' - use Sage's Berlekamp-Massey algorithm
OUTPUT:
- a CFiniteSequence, or 0 if none could be found
With the default kernel method, trailing zeroes are chopped
off before a guessing attempt. This may reduce the data
below the accepted length of six values.
EXAMPLES::
sage: C.<x> = CFiniteSequences(QQ)
sage: C.guess([1,2,4,8,16,32])
C-finite sequence, generated by 1/(-2*x + 1)
sage: r = C.guess([1,2,3,4,5])
Traceback (most recent call last):
...
ValueError: Sequence too short for guessing.
With Berlekamp-Massey, if an odd number of values is given, the last one is dropped.
So with an odd number of values the result may not generate the last value::
sage: r = C.guess([1,2,4,8,9], algorithm='bm'); r
C-finite sequence, generated by 1/(-2*x + 1)
sage: r[0:5]
[1, 2, 4, 8, 16]
"""
S = self.polynomial_ring()
if algorithm == 'bm':
from sage.matrix.berlekamp_massey import berlekamp_massey
if len(sequence) < 2:
raise ValueError('Sequence too short for guessing.')
R = PowerSeriesRing(QQ, 'x')
if len(sequence) % 2 == 1:
sequence = sequence[:-1]
l = len(sequence) - 1
denominator = S(berlekamp_massey(sequence).list()[::-1])
numerator = R(S(sequence) * denominator, prec=l).truncate()
return CFiniteSequence(numerator / denominator)
elif algorithm == 'pari':
global _gp
if len(sequence) < 6:
raise ValueError('Sequence too short for guessing.')
if _gp is None:
_gp = Gp()
_gp("ggf(v)=local(l,m,p,q,B);l=length(v);B=floor(l/2);\
if(B<3,return(0));m=matrix(B,B,x,y,v[x-y+B+1]);\
q=qflll(m,4)[1];if(length(q)==0,return(0));\
p=sum(k=1,B,x^(k-1)*q[k,1]);\
q=Pol(Pol(vector(l,n,v[l-n+1]))*p+O(x^(B+1)));\
if(polcoeff(p,0)<0,q=-q;p=-p);q=q/p;p=Ser(q+O(x^(l+1)));\
for(m=1,l,if(polcoeff(p,m-1)!=v[m],return(0)));q")
_gp.set('gf', sequence)
_gp("gf=ggf(gf)")
num = S(sage_eval(_gp.eval("Vec(numerator(gf))"))[::-1])
den = S(sage_eval(_gp.eval("Vec(denominator(gf))"))[::-1])
if num == 0:
return 0
else:
return CFiniteSequence(num / den)
else:
from sage.matrix.constructor import matrix
from sage.functions.other import floor, ceil
from numpy import trim_zeros
l = len(sequence)
while l > 0 and sequence[l - 1] == 0:
l -= 1
sequence = sequence[:l]
if l == 0:
return 0
if l < 6:
raise ValueError('Sequence too short for guessing.')
hl = ceil(ZZ(l) / 2)
A = matrix([sequence[k:k + hl] for k in range(hl)])
K = A.kernel()
if K.dimension() == 0:
return 0
R = PolynomialRing(QQ, 'x')
den = R(trim_zeros(K.basis()[-1].list()[::-1]))
if den == 1:
return 0
offset = next((i for i, x in enumerate(sequence) if x != 0), None)
S = PowerSeriesRing(QQ, 'x', default_prec=l - offset)
num = S(R(sequence) * den).add_bigoh(floor(ZZ(l) / 2 +
1)).truncate()
if num == 0 or sequence != S(num / den).list():
return 0
else:
return CFiniteSequence(num / den)
def nodeGraphGeneration():
nodeMatrix = []
adjacentNodeDict = {}
nodeElevationDict, numRows, numCols = generateNodeElevationDict()
#Creates 2D Matrix of node ids, the shape of which is determined by the number size of the user selected region
for i in range(numRows):
nodeRow = []
for j in range(numCols):
nodeRow.append(i * numCols + (j + 1))
nodeMatrix.append(nodeRow)
#Adds zero array first row
boundedNodeMatrix = [np.zeros(numCols + 2, dtype=int)]
#Adds zeros on either end of each row, and appends them to boundedNodeMatrix
for i in nodeMatrix:
i = np.insert(i, 0, 0)
i = np.insert(i, numCols + 1, 0)
boundedNodeMatrix.append(i)
#Adds zero array for last row
boundedNodeMatrix.append(np.zeros(numCols + 2, dtype=int))
#Iterates through each cell in the table
for row in range(1, numRows + 1):
for column in range(1, numCols + 1):
#Adjacent nodes to a current node are discovered (eight nodes surrounding the current node, similar to the 5 on a standard keyboard numPad)
adjacentNodeMatrix = [
boundedNodeMatrix[row - 1][column],
boundedNodeMatrix[row - 1][column + 1],
boundedNodeMatrix[row][column + 1],
boundedNodeMatrix[row + 1][column + 1],
boundedNodeMatrix[row + 1][column],
boundedNodeMatrix[row + 1][column - 1],
boundedNodeMatrix[row][column - 1],
boundedNodeMatrix[row - 1][column - 1]
]
adjacentNodeMatrix = np.sort(adjacentNodeMatrix, kind='mergesort')
adjacentNodeMatrix = np.trim_zeros(adjacentNodeMatrix)
#Dictionary containing respective distances
adjacentNodeDistanceDict = {}
#Stores distances of each node to their eight neighbors in a dictionary
for i in adjacentNodeMatrix:
adjacentNodeElevation = nodeElevationDict[i]
currentNodeElevation = nodeElevationDict[boundedNodeMatrix[row]
[column]]
peakXY = XYCoordDict[regionHighPt[0]]
nodeXY = XYCoordDict[i]
distanceHeuristic = np.sqrt((peakXY[0] - nodeXY[0])**2 +
(peakXY[1] - nodeXY[1])**2)
#Distance modified to account for human energy conserving behavior
adjacentNodeDistanceDict.update({
i: ((adjacentNodeElevation - currentNodeElevation)**2 *
distanceHeuristic)
})
adjacentNodeDict.update(
{boundedNodeMatrix[row][column]: adjacentNodeDistanceDict})
nodeMap = adjacentNodeDict
return nodeMap, numRows, numCols
def createPickleFromRawData(params, writeToFile=False):
data = pd.read_csv('electricity/LD2011_2014.txt',
sep=";",
index_col=0,
parse_dates=True)
data.index = data.index - pd.DateOffset(minutes=5)
num_timeseries = data.shape[1]
data = data.resample('H').mean()
# xF = ['Hour', 'DayOfWeek']
clients = data.columns[0:370].values.tolist()
data['Hour'] = data.index.hour # - 1
data['Day'] = data.index.day - 1
data['Month'] = data.index.month - 1
data['DayOfWeek'] = data.index.dayofweek
data['Weekend'] = ((data['DayOfWeek'] == 5) | (data['DayOfWeek'] == 6)) * 1
xF = ['Hour', 'Day', 'Month', 'DayOfWeek', 'Weekend']
if params.isTsId: xF += ['tsId']
XX = list()
lagXX = list()
YY = list()
for i, client in enumerate(clients):
print(i)
ts = np.trim_zeros(data.loc[:, client], trim='f')
ts = ts.to_frame()
lags = [{'hours': 1}]
if params.isLagFeat:
lags += [{'weeks': 1}, {'weeks': 2}, {'years': 1}]
lagX = list()
for lag in lags:
lagX.append(ts.iloc[:, 0].get(
pd.DatetimeIndex(ts.index) -
pd.DateOffset(**lag)).fillna(0).reset_index(drop=True))
lagX = pd.concat(lagX, axis=1, ignore_index=True)
lagX = lagX.values.tolist()
lagXX.append(lagX)
Y = ts.values.tolist()
YY.append(Y)
# ts['Hour'] = pd.DatetimeIndex(ts.index).hour
# ts['Day'] = pd.DatetimeIndex(ts.index).day - 1
# ts['Month'] = pd.DatetimeIndex(ts.index).month - 1
# ts['Year'] = pd.DatetimeIndex(ts.index).year - 2011
# ts['DayOfWeek'] = pd.DatetimeIndex(ts.index).dayofweek
# ts['Weekend'] = ((ts['DayOfWeek'] == 5) | (ts['DayOfWeek'] == 6)) * 1
if params.isTsId:
ts['tsId'] = i
X = data.loc[ts.index, xF].values.tolist()
XX.append(X)
# print(X[:10])
# print('---------')
# print(lagX[:10])
# print('---------')
# print(Y[:10])
# print('------------------------')
# Create feature_dict and emb_dict
features = xF
feature_dict = OrderedDict([(x, i) for i, x in enumerate(features)])
emb_list = OrderedDict([('Hour', (24, 9)), ('Day', (31, 10)),
('month', (12, 6)), ('DayOfWeek', (7, 4))])
if params.isTsId:
emb_list.append(('tsId', (370, 20)))
#('minute', (4, 2)),
#('hour', (24, 9)),
#('day', (31, 10)),
#('month', (12, 6)),
#('year', (5, 3)),
#('dayofweek', (7, 4)),
#('tsId', (370, 20)),
emb_dict = OrderedDict(emb_list)
if writeToFile:
with open('datasets/electricity.pkl', 'wb') as f:
pickle.dump(XX, f)
pickle.dump(lagXX, f)
pickle.dump(YY, f)
pickle.dump(feature_dict, f)
pickle.dump(emb_dict, f)
return XX, lagXX, YY, feature_dict, emb_dict
args.directories.sort(
) #sort directories once, so they will be in the same order when we read images and commands
time_format = '%Y-%m-%d_%H-%M-%S'
trainstart = datetime.datetime.now()
time_string = trainstart.strftime(time_format)
steer = np.array([]) #this is where we store the steering training data
data_lengths = [
] #this will hold the lengths of each file of steering data after zeros are trimmed.
#this loops through the input directories, and then through the files in the directories to load in the steering data:
for directory in args.directories:
ctlfiles = glob.glob(os.path.join(directory, 'commands*.npz'))
for ctlfile in sorted(ctlfiles):
ctldata = np.load(ctlfile)['arr_0']
data_to_append = np.trim_zeros(ctldata[:, 0], trim='b')
data_lengths.append(len(data_to_append))
steer = np.concatenate(
(steer, data_to_append[args.delay:len(data_to_append)]),
axis=0) #note that we compensate for delay here
#use these values to normalize target data before training
steerSampleMean = steer.mean()
steerSampleSTD = steer.std()
#these values get saved to un-normalize network output during testing
np.savez("steerstats.npz", [steerSampleMean, steerSampleSTD])
#defines image size:
nrows = 78
ncols = 128
def construct(self):
LineAnim, LineObjects = linInterp(0,N)
triList = []
labList = []
triLabTups = np.zeros(N, dtype=object)
triLabGroups = np.zeros(N, dtype=object)
for n in range(1,N):
triLabTups[n-1] = [mktri(n),mkLab(n)]
triLabGroups[n-1] = Group(mktri(n), mkLab(n))
triList.append( ShowCreation( triLabTups[n-1][0] ) )
labList.append( ShowCreation( triLabTups[n-1][1] ) )
label = TexMobject('f(x) = \\sqrt{x}', color=color).move_to([4.5,3,0])
label.scale(1.25)
LineAnim.append(FadeIn(label))
self.play(*triList)
self.wait(1)
self.play(*labList)
self.wait(1)
self.play(*LineAnim)
self.wait(1)
flipList = []
rotList = []
centList = []
finRotList = []
for n in range(1,len(triLabGroups)-1):
obj2 = triLabGroups[n]
tri = triLabTups[n]
flipList.append(flipAnim(obj2[0]))
flipList.append( ApplyMethod(obj2[1].shift, .5*LEFT) )
rotList.append(rotAnim(obj2,n))
centList.append(centerAtOrigin(obj2, n))
tupArray = np.array(triLabTups).flatten()
tupArray = np.trim_zeros(tupArray)
i = 0
for ele in tupArray:
if i==0:
i+=1
continue
for el in ele:
self.remove(el)
i+=1
self.play(*flipList, run_time=.8)
self.wait(1)
self.play(*rotList, run_time=.8)
self.wait(1)
self.play(*centList, run_time=.8)
self.wait(1)
fade = [FadeOut(label)]
for li in LineObjects:
fade.append(FadeOut(li))
self.play(*fade)
self.wait(.2)
self.remove(triLabTups[0][0] )
self.remove(triLabTups[0][1] )
toGroup = np.array(triLabGroups).flatten()
toGroup = np.trim_zeros(toGroup)
initGroup = Group(*toGroup)
self.play(ApplyMethod(initGroup.shift, [7,1.5,0]) , run_time=.75)
self.wait(.5)
for n in range(1,len(triLabGroups)-1):
obj2 = triLabGroups[n]
tri = triLabTups[n]
finRotList.append(rotToPlace(obj2,n, about=toGroup[0].get_corner(DOWN+LEFT)))
self.play(*finRotList, run_time=.7)
self.wait(4)
self.play(FadeOut(initGroup))
def prio_and_senio(parent_theory, f, ndist, do_architecture):
"""Get the arm length prob. distr. and priority vs seniority form C and save it in the
theory DataTable"""
tt = parent_theory.tables[f.file_name_short]
# arm length
lgmax = np.log10(react_dist[ndist].contents.arch_maxwt * 1.01)
lgmin = np.log10(react_dist[ndist].contents.monmass / 1.01)
num_armwt_bin = react_dist[ndist].contents.num_armwt_bin
lgstep = (lgmax - lgmin) / (1.0 * num_armwt_bin)
tmp_x = np.power(
10,
[lgmin + ibin * lgstep - 0.5 * lgstep for ibin in range(1, num_armwt_bin + 1)],
)
tmp_y = [
react_dist[ndist].contents.numin_armwt_bin[ibin]
for ibin in range(1, num_armwt_bin + 1)
]
# trim right zeros
tmp_y = np.trim_zeros(tmp_y, "b")
new_len = len(tmp_y)
tmp_x = tmp_x[:new_len]
# trim left zeros
tmp_y = np.trim_zeros(tmp_y, "f")
new_len = len(tmp_y)
tmp_x = tmp_x[-new_len:]
tt.extra_tables["proba_arm_wt"] = np.zeros((new_len, 2))
tt.extra_tables["proba_arm_wt"][:, 0] = tmp_x
tt.extra_tables["proba_arm_wt"][:, 1] = tmp_y
# normalize
try:
tt.extra_tables["proba_arm_wt"][:, 1] /= tt.extra_tables["proba_arm_wt"][
:, 1
].sum()
except ZeroDivisionError:
pass
# number of branch points branch point
max_num_br = react_dist[ndist].contents.max_num_br
# if max_num_br < 100:
rmax = min(max_num_br + 1, pb_global_const.MAX_NBR)
tt.extra_tables["proba_br_pt"] = np.zeros((max_num_br + 1, 2))
tt.extra_tables["proba_br_pt"][:, 0] = np.arange(max_num_br + 1)
tt.extra_tables["proba_br_pt"][:, 1] = [
react_dist[ndist].contents.numin_num_br_bin[i] for i in range(max_num_br + 1)
]
try:
tt.extra_tables["proba_br_pt"][:, 1] /= tt.extra_tables["proba_br_pt"][
:, 1
].sum()
except ZeroDivisionError:
pass
# else:
# # bin the data
# tmp_x = list(np.arange(max_num_br + 1))
# tmp_y = [react_dist[ndist].contents.numin_num_br_bin[i] for i in range(max_num_br + 1)]
# hist, bin_edge = np.histogram(tmp_x, bins=20, weights=tmp_y, density=True)
# tt.extra_tables['proba_br_pt'] = np.zeros((len(hist), 2))
# tt.extra_tables['proba_br_pt'][:, 0] = np.diff(bin_edge) / 2 + bin_edge[:-1]
# tt.extra_tables['proba_br_pt'][:, 1] = hist
if not do_architecture:
return
# P&S
max_prio = return_max_prio()
max_senio = return_max_senio()
avarmlen_v_senio = [
return_avarmlen_v_senio(ct.c_int(s), ct.c_int(ndist))
for s in range(1, max_senio + 1)
]
avarmlen_v_prio = [
return_avarmlen_v_prio(ct.c_int(p), ct.c_int(ndist))
for p in range(1, max_prio + 1)
]
avprio_v_senio = [
return_avprio_v_senio(ct.c_int(s)) for s in range(1, max_senio + 1)
]
avsenio_v_prio = [
return_avsenio_v_prio(ct.c_int(p)) for p in range(1, max_prio + 1)
]
proba_senio = [return_proba_senio(ct.c_int(s)) for s in range(1, max_senio + 1)]
proba_prio = [return_proba_prio(ct.c_int(p)) for p in range(1, max_prio + 1)]
tt.extra_tables["avarmlen_v_senio"] = np.zeros((max_senio, 2))
tt.extra_tables["avarmlen_v_senio"][:, 0] = np.arange(1, max_senio + 1)
tt.extra_tables["avarmlen_v_senio"][:, 1] = avarmlen_v_senio[:]
tt.extra_tables["avarmlen_v_prio"] = np.zeros((max_prio, 2))
tt.extra_tables["avarmlen_v_prio"][:, 0] = np.arange(1, max_prio + 1)
tt.extra_tables["avarmlen_v_prio"][:, 1] = avarmlen_v_prio[:]
tt.extra_tables["avprio_v_senio"] = np.zeros((max_senio, 2))
tt.extra_tables["avprio_v_senio"][:, 0] = np.arange(1, max_senio + 1)
tt.extra_tables["avprio_v_senio"][:, 1] = avprio_v_senio[:]
tt.extra_tables["avsenio_v_prio"] = np.zeros((max_prio, 2))
tt.extra_tables["avsenio_v_prio"][:, 0] = np.arange(1, max_prio + 1)
tt.extra_tables["avsenio_v_prio"][:, 1] = avsenio_v_prio[:]
tt.extra_tables["proba_senio"] = np.zeros((max_senio, 2))
tt.extra_tables["proba_senio"][:, 0] = np.arange(1, max_senio + 1)
tt.extra_tables["proba_senio"][:, 1] = proba_senio[:]
tt.extra_tables["proba_prio"] = np.zeros((max_prio, 2))
tt.extra_tables["proba_prio"][:, 0] = np.arange(1, max_prio + 1)
tt.extra_tables["proba_prio"][:, 1] = proba_prio[:]
def plot_meanCF(data,
clusters,
ylim=[0.0, 90],
centroids=None,
order=None,
ylab='',
xlab='',
title=None):
'''
plot the mean per cluster or the centroid and the percentiles
data: the dataset N*M as a numpy array
centroids: centroids as a 2D numpy array N*M*K
clusters: list of index of assignment (assigned)
date: the date to ve written on the file
ylim: limits of the y axis
name: the name of the output file
'''
maxClusters = max(clusters) + 1
#days=['Monday','Tuesday','Wednesday','Thursday','Friday','Saturday','Sunday']
nonZeroMax = np.max(np.count_nonzero(data, axis=1))
x = np.arange(0, nonZeroMax + 2, 1)
if divmod(maxClusters, 5)[1] != 0:
line = int(math.ceil(max(clusters) / 5) + 1)
else:
line = int(math.ceil(int((maxClusters) / 5)))
#print('line'+str(line))
f, axs = plt.subplots(line,
5,
figsize=(5 * 5, 5 * line),
sharex='col',
sharey='row')
f.subplots_adjust(hspace=.2, wspace=.05)
axs = axs.ravel()
for i in range(maxClusters):
#print(i)
#axs[i].axis('off')
if order is not None:
i2 = order[i]
else:
i2 = i
try:
Y = np.array([
np.concatenate([[0],
np.pad(np.trim_zeros(data[int(j)]),
(0,
int(nonZeroMax + 1 -
len(np.trim_zeros(data[int(j)])))),
'constant')])
for j in np.where(clusters == i2)[0]
],
dtype=float)
if centroids is None:
centroid = np.nanmedian(Y, axis=0)
else:
centroid = centroids[i2]
if i >= (line - 1) * 5.:
# print(i)
axs[i].set_xlabel(xlab, size=20)
for tick in axs[i].xaxis.get_major_ticks():
tick.label.set_fontsize(14)
if i % 5 == 0:
axs[i].set_ylabel(ylab, size=20)
for tick in axs[i].yaxis.get_major_ticks():
tick.label.set_fontsize(14)
axs[i].set_ylim(ylim)
axs[i].set_title('Cluster: ' + str(i2) + '; Size: ' + str(len(Y)),
size=15)
#print(np.shape(Y))
if len(Y) > 1:
for per in zip([2.5, 5, 10, 25], [97.5, 95, 90, 75]):
axs[i].fill_between(x,
np.nanpercentile(Y, per[0], axis=0),
np.nanpercentile(Y, per[1], axis=0),
color="#3F5D7D",
alpha=0.3)
axs[i].plot(x, centroid, lw=1, color="white")
else:
axs[i].plot(x, centroid, lw=1, color="#3F5D7D")
except ValueError:
print('skip: ' + str(i))
#day= datetime.strptime(date,'%Y-%m-%d').weekday()
#f.suptitle('date = '+date+' ('+days[day]+')',size=15)
if title == None:
f.savefig('./gmm-K' + str(maxClusters) + '.png', bbox_inches='tight')
else:
f.savefig(title + '.png', bbox_inches='tight')
plt.clf()
plt.close()
def clean_target_from_padding(target: np.ndarray):
"""Helper function to remove the padding and put the target array in easy to use format"""
return [np.trim_zeros(x.flatten(), 'b') for x in target]
def visualize(self, s, v, a, nt, dt):
"""
Parameters
----------
s : list
the position of the rocket
v : list
the velocity of the rocket
a : list
the acceleration of the rocket
nt : int
the number of time steps
dt : int
the time between each time step
"""
# Set up subplots
f, axs = plt.subplots(nrows=3, ncols=1, sharex=True, figsize=(14,9))
f.tight_layout(pad=3.0)
f.set_facecolor('w')
# Set up common axis and title names
f.text(0.5, 0.02, 'Time During Docking [sec.]', ha='center', fontsize=14)
f.text(0.5, 0.96, 'Rocket Docking Model', ha='center', fontsize=16)
# Set up individual axis titles
axs[0].set_ylabel('Altitude [km]')
axs[0].set_title('Position vs. Time')
axs[1].set_ylabel('Velocity [km/s]')
axs[1].set_title('Velocity vs. Time')
axs[2].set_ylabel('G Force')
axs[2].set_title('Acceleration vs. Time')
# Remove trailing 0s if rocket reached terminal velocity by tmax
if s[-1] <= 0:
s, v, a = map(lambda x: np.trim_zeros(x, 'b'), [s, v, a])
# Calculate common x values and g forces throughout launch
x = np.arange(0, int(len(s)*dt), dt)
gs = self.calculate_g(a)
# Combine data from launch into only a few variables
axes = [axs[0], axs[1], axs[2]]
data = [s, v, gs]
scale_factors = [1000, 1000, 1]
colors = ['b', 'g', 'r']
# Set the x and y axis values for each subplot
for ax, d, sf in zip(axes, data, scale_factors):
if axes == axs[2]:
break
ax.set_ylim(0, 1.3 * np.amax(d) / sf)
ax.set_xlim(0, np.amax(x))
axs[1].set_ylim(-15 , 0)
axs[2].set_ylim(-5, 10)
axs[2].set_xlim(0, np.amax(x))
# Pseudo-animate the launch
for i in range(0, len(x)):
for ax, d, sf, color in zip(axes, data, [1000, 1000, 1], colors):
ax.plot(x[:i], d[:i] / sf, color)
plt.pause(0.0001)
plt.show()
def main():
# print command line arguments
if len(sys.argv) <= 2:
print('usage:\n./plotHM.py heapProfileHM(R/W).out heatmap_folder')
exit()
filename = sys.argv[1]
outFolder = sys.argv[2]
try:
os.mkdir(outFolder)
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
pass
ometaF = open(outFolder+"/meta.txt", "a")
mat = None
mat_name = None
with open(filename) as fp:
line = fp.readline()
cnt = 1
while line:
if line.strip().find("res") != -1 or line.strip().find("size-limit") != -1:
print(line)
ometaF.write(line)
elif line.strip(' ') == '\n':
# do nothing, filter out
pass
elif line.strip().find("fs") != -1:
if mat is not None:
plt.xlabel('mem addr')
plt.ylabel('time')
plt.imshow(mat, cmap='hot', interpolation='nearest')
# estimate locality, is this formula right?
avg_byts_per_bucket = np.sum(mat)/np.size(mat, 1)
total_0_axis = np.sum(mat, axis=0)
locality_0_aixs = np.absolute(total_0_axis - avg_byts_per_bucket)
# Or ?
# locality_0_aixs = total_0_axis - avg_byts_per_bucket * 0.95
# locality_0_aixs[locality_0_aixs < 0] = 0
est_locality = np.sum(locality_0_aixs)/np.sum(mat)
plt.savefig(outFolder+"/"+mat_name+"_"+format(est_locality, '.3f')+".png", dpi=1000)
mat_name = line.strip()
# create new array
mat = None
else:
col = (line.strip('\t\n').replace("\t"," "))
col = list(map(int, col.split()))
col = np.trim_zeros(col, 'b')
col = np.ndarray((1,len(col)), buffer=np.array(col))
if mat is None:
mat = col
else:
if col.shape[1]> mat.shape[1]:
zero_mat = np.zeros((mat.shape[0], col.shape[1]-mat.shape[1]))
mat = np.concatenate([mat, zero_mat], axis=1)
elif col.shape[1] < mat.shape[1]:
pad_zeros = np.zeros(mat.shape[1] - col.shape[1])
pad_zeros = np.ndarray((1,len(pad_zeros)), buffer=pad_zeros)
col = np.concatenate([col, pad_zeros], axis=1)
else:
pass
mat = np.vstack([mat, col])
line = fp.readline()
cnt += 1
if cnt % 1e3 == 0:
print('[%d] 1k line pass' % (cnt/1e3))
# handle the end of file
if mat is not None:
plt.xlabel('mem addr')
plt.ylabel('time')
plt.imshow(mat, cmap='hot', interpolation='nearest')
# estimate locality, is this formula right?
avg_byts_per_bucket = np.sum(mat)/np.size(mat, 1)
total_0_axis = np.sum(mat, axis=0)
locality_0_aixs = np.absolute(total_0_axis - avg_byts_per_bucket)
# Or ?
# locality_0_aixs = total_0_axis - avg_byts_per_bucket * 0.95
# locality_0_aixs[locality_0_aixs < 0] = 0
est_locality = np.sum(locality_0_aixs)/np.sum(mat)
plt.savefig(outFolder+"/"+mat_name+"_"+format(est_locality, '.3f')+".png", dpi=1000)
ometaF.close()
find_rid_targets(local_data_dir_path)
timeseries = []
i = 0
for n in rid_targets:
j = 0
for p in rid_targets[n]:
data = pd.read_csv(p, index_col=0, parse_dates=['Timestamp'])
data.columns = [n]
data_kw = data.resample(freq).sum()
if j == 0:
timeseries.append(np.trim_zeros(data_kw.iloc[:, 0], trim='f'))
else:
timeseries[i] = timeseries[i].append(np.trim_zeros(data_kw.iloc[:, 0], trim='f'))
j += 1
i += 1
# Let us plot the resulting time series for the first ten customers for the time period spanning the first two weeks of 2014.
fig, axs = plt.subplots(1, 2, figsize=(30, 20), sharex=True)
axx = axs.ravel()
for i in range(0, 1):
timeseries[i].loc["2019-11-01":"2019-11-14"].plot(ax=axx[i])
axx[i].set_xlabel("Timestamp")
axx[i].set_ylabel("kW consumption")
plt.show()
def calc_flux_variance(data, model, err_binsize):
kernel = Box1DKernel(err_binsize)
errors = convolve((data - model)**2, kernel, boundary=None)
errors = errors
errors = np.trim_zeros(errors)
return errors
def adaptive_peak_finder(
signal,
vert_offset='unspecified',
window_length='unspecified',
min_peak_distance='unspecified',
min_amount_from_threshold='unspecified'
): # if threshold isnt specified it defaults it is replaced with the mean of the signal
# ------------------------------------------- IMPORTED MODULES -----------------------------------------------
from numpy import array, int_, append, zeros, ceil, trim_zeros
# -------------------------------------------------------------------------------------------------------------
# Checking inputs
if vert_offset == 'unspecified':
print(
'No vert_offset was entered, therefore default of 0 will be used making the threshold = moving mean'
)
vert_offset = 0
if window_length == 'unspecified':
print(
'ERROR, no window_length was provided, if you dont want to update the threshold use the basic_peak_finder function'
)
window_length = int(input('Please re-enter the window length: '))
return (adaptive_peak_finder(signal, vert_offset, window_length))
if window_length % 2 == 0:
print(
'ERROR, an even number was entered for the window length, the window length should be odd'
)
window_length = int(input('Please re-enter the window length: '))
return (adaptive_peak_finder(signal, vert_offset, window_length))
# Initializing varibles (NB when initializing, the most peaks possible would come from a sawtooth wave
# where every point was either a peak or trough, therefore arrays are initialized to half the signal
# input length)
peak_index = zeros(
int(ceil(len(signal) / 2)), dtype=int_
) # initializing a peak_index array where the index of the peak in the signal array will be stored as integers
peak_value = zeros(
int(ceil(len(signal) / 2))
) # initializing a peak_value array where the peak values will be stored
m = 0 # m is the iterator that will step through peak_ arrays indices filling the array with the peak locations and values
ignored_peak_index = zeros(
int(ceil(len(signal) / 2)), dtype=int_
) # initializing an ignored_peak_index array where the index of the peak that didnt meet the min distance requirement between it and the previous peak are stored.
ignored_peak_value = zeros(
int(ceil(len(signal) / 2))
) # initializing a ignored_peak_value array where the ignored peak values that didnt meet the min distance requirement between it and the previous peak will be stored
k = 0 # k is the iterator that will step through ignored_peak_ arrays indices filling the array with the ignored peak locations and values
thresholds = zeros(len(signal) - 1)
thresholds[0] = (
signal[0:int((window_length - 1) / 2) + 1].mean()
) + vert_offset # creating a list to store the thresholds as these can be graphed or used for debugging
# NB the first threshold point is also initilized within.
for i in range(
1, signal.size - 1
): # NB the reason we start at i=1 is because the threshold list is initilized with the threshold for i=0 in it, see above.
# Also the reason we stop at signal.size-1 is we need to stop and not go through the loop for the final element in signal
# because inside the loop we look at signal[i+1] which doesnt exist if i = final element in signal.
# Calculating threshold at each point
if i <= (window_length - 1) / 2:
window_mean = (
signal[0:i + (int((window_length - 1) / 2)) + 1].mean()
) # NB we need to add one to the stop point as we want to include index i+(window-1)/2 in the window mean calc
thresholds[i] = window_mean + vert_offset
elif i >= signal.size - (window_length - 1) / 2:
window_mean = (
signal[i - int((window_length - 1) / 2):signal.size].mean()
) # NB we dont need to add one to the stop point as a stop point of signal.size = len(signal) defines the stop point as the end of signal
thresholds[
i] = window_mean + vert_offset # Incase the threshold is -ve, we need to work out the threshold this way rather than just window_mean*multiple of mean
else:
window_mean = (
signal[i - int((window_length - 1) / 2):i +
(int((window_length - 1) / 2)) + 1].mean()
) # NB we need to add one to the stop point as we want to include index i+(window-1)/2 in the threshold calc
thresholds[
i] = window_mean + vert_offset # Incase the threshold is -ve, we need to work out the threshold this way rather than just window_mean*multiple of mean
# With the threshold calculated we now check to see if the current point meets the criteria to be a peak
if signal[i] >= thresholds[i] and signal[i] > signal[
i -
1]: # checking that the current point of interest is above the threshold and above its previous neighbour
j = 1
while signal[i] == signal[
i +
j]: # If you have a point in the signal consisting of multiple signals, to see if you have a peak you must see if the point AFTER the flat part of the signal decreases!
j = j + 1 # to do this we must step along the flat part of the signal until the gradient of the signal changes. then as seen below we check to see if the point at the change
# in gradient is lower than our flat section. If it is, then the flat section is infact a peak. The index of this peak is considered to be the left most point in the flat section.
# Second criteria to be a peak is that the neighbouring point must be less than signal[i]
if signal[i] > signal[
i +
j]: # signal[i+j] will be the first point after signal[i] that isnt equal to signal[i]. see earlier notes for while statement
if min_peak_distance != 'unspecified' and peak_index[
0] != 0 and i - peak_index[
m -
1] < min_peak_distance: # this condition is only true if min_peak_distance has been specified and and if it is far enough from the previous positively identified peak we have found at least one previous peak needed to compare the min_peak_distance to.
ignored_peak_index[
k] = i # we store the ignored peak data incase the user wants to check what was ignored in the signal
ignored_peak_value[k] = signal[i]
k = k + 1 # k is used to step through ignored_peak arrays as an index is filled
elif min_amount_from_threshold != 'unspecified' and abs(
signal[i] - thresholds[i]
) < min_amount_from_threshold: # we can set an amount a peak must be above the threshold for it to be a true peak, this is handy for when a signal may jump around the threshold a tiny amount and may not be an actual peak.
ignored_peak_index[
k] = i # we store the ignored peak data incase the user wants to check what was ignored in the signal
ignored_peak_value[k] = signal[i]
k = k + 1 # k is used to step through ignored_peak arrays as an index is filled
else: # You need the peak_index and peak_value twice, once for if min_peak_distance is specified and once for if it isnt
peak_index[m] = i
peak_value[m] = signal[i]
m = m + 1 # m is used to step through peak_ arrays as an index is filled
# trimming all unfilled indices of peak_index, peak_value, ignored_peak_index and ignored_peak_values
peak_index = trim_zeros(
peak_index
) # as we cannot possible have a peak at an index location 0, we can trim peak_index and use its length to define the peak_value array. As a peaks value could be 0, so we cant use the trim_zeros function of the peak_value array.
peak_value = peak_value[:len(
peak_index
)] # trimming the peak_value array to the length of peak_index, as they must be the same length, each index is information on the same peak.
ignored_peak_index = trim_zeros(
ignored_peak_index
) # the code is such that we cannot possible have an ignored_peak at an index location 0, so we can trim peak_index and use its length to define the ignored_peak_value array. As an ignored_peak_value could be 0, so we cant use the trim_zeros function of that array.
ignored_peak_value = ignored_peak_value[:len(
ignored_peak_index
)] # trimming the ignored_peak_value array to the length of ignored_peak_index, as they must be the same length, each index is information on the same peak.
return (
peak_index, peak_value, ignored_peak_index, ignored_peak_value,
thresholds
) # a list of the thresholds is stored so it can be plotted or used for debugging purposes
import numpy
import matplotlib.pyplot as pyplot
pyplot.figure()
V_all = []
I_all = []
for name in ['semi-rev-v-ramp-1-11.txt', 'semi-rev-v-ramp-11-12.txt', 'semi-rev-v-ramp-11.8-11.9.txt', 'semi-rev-v-ramp-11.89-11.93.txt' ,'semi-rev-v-ramp-11.93-12.txt', 'semi-rev-v-ramp-11.8-13.txt']:
(T, V, I, ti, ti_V, voltages) = numpy.loadtxt(name)
V = -numpy.absolute(V)
I = -numpy.absolute(I)
ti_V = numpy.trim_zeros(ti_V, 'b')
voltages = numpy.trim_zeros(voltages, 'b')
ramp_index = []
for ti_ramp_pt in ti_V:
for i, time in enumerate(ti):
if time >= ti_ramp_pt or time == ti[-1]:
ramp_index.append(i)
break
av_V = []
av_I = []
for i in ramp_index[1:]:
av_V.append(numpy.average(V[i - 10:i]))
av_I.append(numpy.average(I[i - 10:i]))
if name == 'semi-rev-v-ramp-11.8-13.txt':
av_V = av_V[3:]
av_I = av_I[3:]
elif name == 'semi-rev-v-ramp-11-12.txt':
def read_clevels(fileName):
with open(fileName) as o:
s = o.read().replace('\n', ' ')
lvls = re.search(r'\#\#\$LEVELS=\s?\(.*\)(.*?)\#\#\$', s).group(1)
return np.trim_zeros(np.array(lvls.split(), dtype=float))
import numpy as np a = np.array((0, 0, 0, 1, 2, 3, 0, 2, 1, 0)) np.trim_zeros(a) np.trim_zeros(a, 'b') np.trim_zeros([0, 1, 2, 0])
def forward(self,
x,
im_sizes,
image_offset,
gt_boxes=None,
gt_classes=None,
gt_rels=None,
proposals=None,
train_anchor_inds=None,
return_fmap=False):
"""
Forward pass for Relation detection
Args:
x: Images@[batch_size, 3, IM_SIZE, IM_SIZE]
im_sizes: A numpy array of (h, w, scale) for each image.
image_offset: Offset onto what image we're on for MGPU training (if single GPU this is 0)
parameters for training:
gt_boxes: [num_gt, 4] GT boxes over the batch.
gt_classes: [num_gt, 2] gt boxes where each one is (img_id, class)
gt_rels:
proposals:
train_anchor_inds: a [num_train, 2] array of indices for the anchors that will
be used to compute the training loss. Each (img_ind, fpn_idx)
return_fmap:
Returns:
If train:
scores, boxdeltas, labels, boxes, boxtargets, rpnscores, rpnboxes, rellabels
If test:
prob dists, boxes, img inds, maxscores, classes
"""
s_t = time.time()
verbose = False
def check(sl, een, sst=s_t):
if verbose:
print('{}{}'.format(sl, een - sst))
result = self.detector(x,
im_sizes,
image_offset,
gt_boxes,
gt_classes,
gt_rels,
proposals,
train_anchor_inds,
return_fmap=True)
check('detector', tt())
assert not result.is_none(), 'Empty detection result'
# image_offset refer to Blob
# self.batch_size_per_gpu * index
im_inds = result.im_inds - image_offset
boxes = result.rm_box_priors
obj_scores, box_classes = F.softmax(
result.rm_obj_dists[:, 1:].contiguous(), dim=1).max(1)
box_classes += 1
# TODO: predcls implementation obj_scores and box_classes
num_img = im_inds[-1] + 1
# embed(header='rel_model.py before rel_assignments')
if self.training and result.rel_labels is None:
assert self.mode == 'sgdet'
# only in sgdet mode
# shapes:
# im_inds: (box_num,)
# boxes: (box_num, 4)
# rm_obj_labels: (box_num,)
# gt_boxes: (box_num, 4)
# gt_classes: (box_num, 2) maybe[im_ind, class_ind]
# gt_rels: (rel_num, 4)
# image_offset: integer
result.rel_labels = rel_assignments(im_inds.data,
boxes.data,
result.rm_obj_labels.data,
gt_boxes.data,
gt_classes.data,
gt_rels.data,
image_offset,
filter_non_overlap=True,
num_sample_per_gt=1)
rel_inds = self.get_rel_inds(result.rel_labels, im_inds, boxes)
rois = torch.cat((im_inds[:, None].float(), boxes), 1)
# union boxes feats (NumOfRels, obj_dim)
union_box_feats = self.visual_rep(result.fmap.detach(), rois,
rel_inds[:, 1:].contiguous())
# single box feats (NumOfBoxes, feats)
box_feats = self.obj_feature_map(result.fmap.detach(), rois)
# box spatial feats (NumOfBox, 4)
box_pair_feats = self.fuse_message(union_box_feats, boxes, box_classes,
rel_inds)
box_pair_score = self.relpn_fc(box_pair_feats)
if self.training:
# sampling pos and neg relations here for training
rel_sample_pos, rel_sample_neg = 0, 0
pn_rel_label, pn_pair_score = list(), list()
for i, s, e in enumerate_by_image(
result.rel_labels[:, 0].data.contiguous()):
im_i_rel_label = result.rel_labels[s:e].contiguous()
im_i_box_pair_score = box_pair_score[s:e].contiguous()
im_i_rel_fg_inds = torch.nonzero(
im_i_rel_label[:, -1].contiguous()).squeeze()
im_i_rel_fg_inds = im_i_rel_fg_inds.data.cpu().numpy()
im_i_fg_sample_num = min(RELEVANT_PER_IM,
im_i_rel_fg_inds.shape[0])
if im_i_rel_fg_inds.size > 0:
im_i_rel_fg_inds = np.random.choice(
im_i_rel_fg_inds,
size=im_i_fg_sample_num,
replace=False)
im_i_rel_bg_inds = torch.nonzero(
im_i_rel_label[:, -1].contiguous() == 0).squeeze()
im_i_rel_bg_inds = im_i_rel_bg_inds.data.cpu().numpy()
im_i_bg_sample_num = min(EDGES_PER_IM - im_i_fg_sample_num,
im_i_rel_bg_inds.shape[0])
if im_i_rel_bg_inds.size > 0:
im_i_rel_bg_inds = np.random.choice(
im_i_rel_bg_inds,
size=im_i_bg_sample_num,
replace=False)
#print('{}/{} fg/bg in image {}'.format(im_i_fg_sample_num, im_i_bg_sample_num, i))
rel_sample_pos += im_i_fg_sample_num
rel_sample_neg += im_i_bg_sample_num
im_i_keep_inds = np.append(im_i_rel_fg_inds, im_i_rel_bg_inds)
im_i_pair_score = im_i_box_pair_score[
im_i_keep_inds.tolist()].contiguous()
im_i_rel_pn_labels = Variable(
torch.zeros(im_i_fg_sample_num + im_i_bg_sample_num).type(
torch.LongTensor).cuda(x.get_device()))
im_i_rel_pn_labels[:im_i_fg_sample_num] = 1
pn_rel_label.append(im_i_rel_pn_labels)
pn_pair_score.append(im_i_pair_score)
result.rel_pn_dists = torch.cat(pn_pair_score, 0)
result.rel_pn_labels = torch.cat(pn_rel_label, 0)
result.rel_sample_pos = torch.Tensor([rel_sample_pos]).cuda(
im_i_rel_label.get_device())
result.rel_sample_neg = torch.Tensor([rel_sample_neg]).cuda(
im_i_rel_label.get_device())
box_pair_relevant = F.softmax(box_pair_score, dim=1)
box_pos_pair_ind = torch.nonzero(box_pair_relevant[:, 1].contiguous(
) > box_pair_relevant[:, 0].contiguous()).squeeze()
if box_pos_pair_ind.data.shape == torch.Size([]):
return None
#print('{}/{} trim edges'.format(box_pos_pair_ind.size(0), rel_inds.size(0)))
result.rel_trim_pos = torch.Tensor([box_pos_pair_ind.size(0)]).cuda(
box_pos_pair_ind.get_device())
result.rel_trim_total = torch.Tensor([rel_inds.size(0)
]).cuda(rel_inds.get_device())
if self.trim_graph:
# filtering relations
filter_rel_inds = rel_inds[box_pos_pair_ind.data]
filter_box_pair_feats = box_pair_feats[box_pos_pair_ind.data]
else:
filter_rel_inds = rel_inds
filter_box_pair_feats = box_pair_feats
if self.training:
if self.trim_graph:
filter_rel_labels = result.rel_labels[box_pos_pair_ind.data]
else:
filter_rel_labels = result.rel_labels
num_gt_filtered = torch.nonzero(filter_rel_labels[:, -1])
if num_gt_filtered.shape == torch.Size([]):
num_gt_filtered = 0
else:
num_gt_filtered = num_gt_filtered.size(0)
num_gt_orignial = torch.nonzero(result.rel_labels[:, -1]).size(0)
result.rel_pn_recall = torch.Tensor(
[num_gt_filtered / num_gt_orignial]).cuda(x.get_device())
result.rel_labels = filter_rel_labels
check('trim', tt())
# message passing between boxes and relations
if self.mode in ('sgcls', 'sgdet'):
for _ in range(self.mp_iter_num):
box_feats = self.message_passing(box_feats,
filter_box_pair_feats,
filter_rel_inds)
box_cls_scores = self.cls_fc(box_feats)
result.rm_obj_dists = box_cls_scores
obj_scores, box_classes = F.softmax(
box_cls_scores[:, 1:].contiguous(), dim=1).max(1)
box_classes += 1 # skip background
check('mp', tt())
# RelationCNN
filter_box_pair_feats_fc1 = self.relcnn_fc1(filter_box_pair_feats)
filter_box_pair_score = self.relcnn_fc2(filter_box_pair_feats_fc1)
result.rel_dists = filter_box_pair_score
pred_scores_stage_one = F.softmax(result.rel_dists, dim=1).data
# filter_box_pair_feats is to be added to memory
if self.training:
padded_filter_feats, pack_lengths, re_filter_rel_inds, padded_rel_labels = \
self.pad_sequence(
filter_rel_inds,
filter_box_pair_feats_fc1,
rel_labels=result.rel_labels
)
else:
padded_filter_feats, pack_lengths, re_filter_rel_inds, padded_rel_inds = \
self.pad_sequence(
filter_rel_inds,
filter_box_pair_feats_fc1
)
# trimming zeros to avoid no rel in image
trim_pack_lengths = np.trim_zeros(pack_lengths)
trim_padded_filter_feats = padded_filter_feats[:trim_pack_lengths.
shape[0]]
packed_filter_feats = pack_padded_sequence(trim_padded_filter_feats,
trim_pack_lengths,
batch_first=True)
if self.training:
trim_padded_rel_labels = padded_rel_labels[:trim_pack_lengths.
shape[0]]
packed_rel_labels = pack_padded_sequence(trim_padded_rel_labels,
trim_pack_lengths,
batch_first=True)
rel_mem_dists = self.mem_module(inputs=packed_filter_feats,
rel_labels=packed_rel_labels)
rel_mem_dists = self.re_order_packed_seq(rel_mem_dists,
filter_rel_inds,
re_filter_rel_inds)
result.rel_mem_dists = rel_mem_dists
else:
trim_padded_rel_inds = padded_rel_inds[:trim_pack_lengths.shape[0]]
packed_rel_inds = pack_padded_sequence(trim_padded_rel_inds,
trim_pack_lengths,
batch_first=True)
rel_mem_dists = self.mem_module(inputs=packed_filter_feats,
rel_inds=packed_rel_inds,
obj_classes=box_classes)
rel_mem_probs = self.re_order_packed_seq(rel_mem_dists,
filter_rel_inds,
re_filter_rel_inds)
rel_mem_probs = rel_mem_probs.data
check('mem', tt())
if self.training:
return result
# pad stage one output in rel_mem_probs if it sums zero
for rel_i in range(rel_mem_probs.size(0)):
rel_i_probs = rel_mem_probs[rel_i]
if rel_i_probs.sum() == 0:
rel_mem_probs[rel_i] = pred_scores_stage_one[rel_i]
"""
filter_dets
boxes: bbox regression else [num_box, 4]
obj_scores: [num_box] probabilities for the scores
obj_classes: [num_box] class labels integer
rel_inds: [num_rel, 2] TENSOR consisting of (im_ind0, im_ind1)
pred_scores: [num_rel, num_predicates] including irrelevant class(#relclass + 1)
"""
check('mem processing', tt())
return filter_dets(boxes, obj_scores, box_classes,
filter_rel_inds[:, 1:].contiguous(), rel_mem_probs)
def _trim(data, val):
data_out = np.copy(data)
return np.trim_zeros(data_out - val) + val
def compute(self):
for i in range(self.partition):
if i >= 2:
continue
temptrain_X, temptrain_Y, tempval_X, tempval_Y = self.create_data(
i)
#print((temptrain_X[0]), (temptrain_Y[0]))
self.train_X, self.train_Y = training_set_factorize(
temptrain_X, temptrain_Y, augment_data=self.augment_data_flag)
self.val_X, self.val_Y = validation_set_factorize(
tempval_X, tempval_Y)
self.val_Y = self.val_Y.long()
print(i, "phase 1 completed")
#create dataset
self.train_X = torch.tensor(self.train_X)
self.train_Y = torch.tensor(self.train_Y)
train_set = data_utils.TensorDataset(self.train_X, self.train_Y)
train_loader = data_utils.DataLoader(dataset=train_set,
batch_size=BATCH_SIZE,
drop_last=True,
shuffle=True)
cur_model = deepcopy(self.model).to(device)
ij_recall_history = []
ij_precision_history = []
for j in range(self.epochs):
if j >= 5:
continue
print("epoch %i" % j)
path = 'cnncrf_train_epoch' + str(j) + '_part' + str(i) + '.pt'
cur_model.load_state_dict(torch.load(path))
#validation
p0t1 = 0
p0t0 = 0
p1t1 = 0
p1t0 = 0
for j in range(self.val_X.size(0)):
print(j, self.val_X.size())
val_X = torch.tensor(self.val_X[j])
val_Y = torch.tensor(self.val_Y[j])
val_X = val_X.reshape(VAL_SEQ_LEN, VAL_BATCH_SIZE,
-1).float().contiguous().to(device)
val_Y = val_Y.reshape(
VAL_SEQ_LEN, ).long().contiguous().to(device)
scores, path = cur_model(val_X)
prediction = torch.from_numpy(np.array(path)).reshape(
VAL_SEQ_LEN, )
prediction = prediction.numpy()
val_X = val_X.cpu().numpy()
val_Y = val_Y.cpu().numpy()
last_index = np.trim_zeros(prediction, 'b').shape[
0] #This line might be buggy!!! might want to use the true size
output_Y = prediction[0:last_index]
target_Y = val_Y[0:last_index]
output_Y[output_Y > 1] = 0
target_Y[target_Y > 1] = 0
countp0t1, countp0t0, countp1t1, countp1t0 = self.compute_acc(
output_Y, target_Y)
p0t1 += countp0t1
p0t0 += countp0t0
p1t1 += countp1t1
p1t0 += countp1t0
if ((p1t1 + p1t0) == 0):
ij_precision_history.append(-1)
else:
ij_precision_history.append(p1t1 * 1.0 / (p1t1 + p1t0))
if ((p1t1 + p0t1) == 0):
ij_recall_history.append(-1)
else:
ij_recall_history.append(p1t1 * 1.0 / (p1t1 + p0t1))
self.precision_history.append(ij_precision_history)
self.recall_history.append(ij_recall_history)
print(self.precision_history)
print(self.recall_history)
return self.precision_history, self.recall_history, self.loss_history
def np_trim_zeros(x):
return np.trim_zeros(x)
def inspect(time_leng, pattern_leng, top_print):
#ファイル1のデータカウント
pattern_dict1 = {}
sumpsth = 0
for i, name in enumerate(sheet_names1):
sheet_df1[i] = file1.parse(name)
if(sheet_df1[i] == NaN) :
break
#print(sheet_df1[i][1][1])
start_number = (np.where(sheet_df1[i]['INFORMATION']=="CHANNEL")[0][0]) +cannel_start
end_number = (np.where(sheet_df1[i]['INFORMATION']=="CHANNEL")[0][1]) - cannel_end
sig1 = (sheet_df1[i]['Unnamed: 3'][start_number : end_number]).values
sig1 = sig1.astype("int")
#print(len(sig1))
sig1 = np.trim_zeros(sig1)
# print()
leng = len(sig1)
psth = np.zeros(int((leng/time_leng)+1), dtype=np.int)
l = 0
for k in range(0, leng, time_leng) :
psth[l] = sig1[k : k+time_leng].sum()
l += 1
# print(sig1[len(psth)-1])
if(len(psth) > pattern_leng) :
for k in range(len(psth) - pattern_leng +1) : # PSTHデータからはパターンを重ねて検索している
if (str(psth[k : k + pattern_leng]) in pattern_dict1) :
pattern_dict1[str(psth[k : k + pattern_leng])] += 1
else :
pattern_dict1[str(psth[k : k + pattern_leng])] = 1
sumpsth += (len(psth) -pattern_leng+1)
# print()
# print(sumpsth)
#ファイル2のデータカウント
pattern_dict2 = {}
sum_dict = pattern_dict1.copy()
for i, name in enumerate(sheet_names2):
sheet_df2[i] = file2.parse(name)
start_number = (np.where(sheet_df2[i]['INFORMATION']=="CHANNEL")[0][0]) + cannel_start
end_number = (np.where(sheet_df2[i]['INFORMATION']=="CHANNEL")[0][1]) - cannel_end
sig1 = (sheet_df2[i]['Unnamed: 3'][start_number : end_number]).values
sig1 = sig1.astype("int")
sig1 = np.trim_zeros(sig1)
leng = len(sig1)
# print(sig1[0])
psth = np.zeros(int((leng/time_leng)+1), dtype=np.int)
l = 0
for k in range(j, leng, time_leng) :
psth[l] = sig1[k+j : k+j+time_leng].sum()
l += 1
if(len(psth) > pattern_leng) :
for k in range(len(psth) - pattern_leng +1) :
if (str(psth[k : k+ pattern_leng]) in pattern_dict2) :
pattern_dict2[str(psth[k : k+ pattern_leng])] += 1
sum_dict[str(psth[k : k+ pattern_leng])] += 1
else :
pattern_dict2[str(psth[k : k+ pattern_leng])] = 1
if (str(psth[k : k+ pattern_leng]) in sum_dict) :
sum_dict[str(psth[k : k+ pattern_leng])] += 1
else :
sum_dict[str(psth[k : k+ pattern_leng])] = 1
if(len(pattern_dict1.keys()) == 0 ):
return 0
#情報量の計算準備
pattern_information = 0.0
# print()
# print()
# print(sum(pattern_dict1.values()))
# print(sum(sum_dict.values()))
# print()
sum_pattern1 = sum(pattern_dict1.values()) + len(sum_dict.keys())
sum_pattern2 = sum(pattern_dict2.values()) + len(sum_dict.keys())
# print(sum_pattern1)
# print(sum_pattern2)
# sum_pattern = sum(sum_dict.values())
pattern1 = len(pattern_dict1.keys())
probability1 = np.zeros(pattern1, float)
probability2 = np.zeros(pattern1, float)
top_dict = {}
k = 0
for i in (pattern_dict1.keys()) :
probability1[k] = ((pattern_dict1[i]+1) / sum_pattern1)
if(i in pattern_dict2) :
probability2[k] = ((pattern_dict2[i]+1) / sum_pattern2)
else :
probability2[k] = (1 / sum_pattern2)
info = probability1[k] * math.log2(probability1[k]/probability2[k])
pattern_information += info
top_dict[i] = info
# print(i)
k += 1
if (len(pattern_dict1.keys()) < top_print) :
max_print = len(pattern_dict1.keys())
else :
max_print =top_print
#グラフ出力の準備
print_probability1 = np.zeros(max_print, float)
print_probability2 = np.zeros(max_print, float)
print_kullback = np.zeros(max_print, float)
print_pattern = []
k = 0
for i, v in sorted(top_dict.items(), key=lambda x:-x[1])[0:max_print] :
print_probability1[k] = ((pattern_dict1[i]+1) / sum_pattern1)
if(i in pattern_dict2) :
print_probability2[k] = ((pattern_dict2[i]+1) / sum_pattern2)
else :
print_probability2[k] = (1 / sum_pattern2)
print_kullback[k] = print_probability1[k] * math.log2(print_probability1[k]/print_probability2[k]) #kullback項の保存
print_pattern.append(i)
k += 1
# print(i)
# print_probability2[k] = (pattern_dict2[i]+1 / sum_pattern2)
# print(print_pattern)
#グラフ出力
#パターンの確率比較
fig = plt.figure(dpi=600)
ax = fig.gca()
x = np.arange(len(print_pattern))
w = 0.4
plt.bar(x+w, print_probability1, width=w, color="red", label="after")
plt.bar(x, print_probability2, width=w, color="blue", label="before")
plt.xlim(-(w/2), max_print-(w/2))
plt.xlabel("pattern")
plt.ylabel("probability")
#plt.xticks(x + w/2, print_pattern)
plt.xticks(color="None")
#ax.set_xticklabels(print_pattern, rotation=90)
plt.legend()
plt.savefig(str(time_leng) + "-" + str(pattern_leng) + ".png", bbox_inches='tight')
#パターンのカルバックライブラー項比較
fig = plt.figure(dpi=600)
ax = fig.gca()
plt.bar(print_pattern, print_kullback, color="red", label="after")
plt.xlim(-0.5, max_print-0.5)
plt.xlabel("pattern")
plt.ylabel("probability")
plt.xticks(color="None")
#ax.set_xticklabels(print_pattern, rotation=90)
plt.legend()
#print(print_kullback)
#plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left', borderaxespad=0, ncol=2)
#fig.subplots_adjust(bottom=10)
plt.savefig(str(time_leng) + "-" + str(pattern_leng) + "-kullback.png", bbox_inches='tight')
plt.close()
del pattern_dict1, pattern_dict2, sum_dict, probability1, probability2, top_dict, print_pattern
return pattern_information, sum_pattern1, sum_pattern2
(savings, alpha, gamma, agents))
plt.figure()
plt.plot(cycle, sigma2)
plt.ylabel(r'$\sigma^2_m$')
plt.xlabel('Monte Carlo Cycle')
plt.margins(x=0.1, y=0.1)
plt.tight_layout()
plt.savefig('../fig/sigma_savings_%s_alpha_%s_gamma_%s_agents_%s.pdf' %
(savings, alpha, gamma, agents))
filename = '../output/wealth_pdf_savings_%s_alpha_%s_gamma_%s_agents_%s.txt' % (
savings, alpha, gamma, agents)
m, counts = np.genfromtxt(filename, unpack=True)
counts = np.trim_zeros(counts, 'b')
m = m[:len(counts)]
n = 1 + 3 * float(savings) / (1 - float(savings))
pdf = n**n / scipy.special.gamma(n) * m**(n - 1) * np.exp(-n * m)
plt.figure()
plt.plot(m, pdf)
plt.hist(m, weights=counts, bins=m, normed=True)
plt.ylabel('Normalized distribution of agents')
plt.xlabel('Money')
plt.tight_layout()
plt.savefig('../fig/histogram_savings_%s_alpha_%s_gamma_%s_agents_%s.pdf' %
(savings, alpha, gamma, agents))
m = m + 0.025
def kernelDetection(fluxProfile, kernel, kernels, num):
""" Detects dimming using Mexican Hat kernel for dip detection and set of Fresnel kernels for kernel matching """
""" Prunes profiles"""
light_curve = np.trim_zeros(fluxProfile[1:])
if len(light_curve) == 0:
return -2 # reject empty profiles
med = np.median(light_curve)
indices = np.where(
light_curve > min(med * 2, med + 5 * np.std(light_curve)))
light_curve = np.delete(light_curve, indices)
trunc_profile = np.where(light_curve < 0, 0, light_curve)
if len(trunc_profile) < 1100:
return -2 # reject objects that go out of frame rapidly, ensuring adequate evaluation of median flux
if abs(np.mean(trunc_profile[:1000]) -
np.mean(trunc_profile[-1000:])) > np.std(trunc_profile[:1000]):
return -2 # reject tracking failures
if np.median(trunc_profile) < 5000:
return -2 # reject stars that are very dim, as SNR is too poor
m = np.std(trunc_profile[900:1100]) / np.median(trunc_profile[900:1100])
""" Dip detection"""
# astropy v2.0+ changed convolve_fft quite a bit... see documentation, for now normalize_kernel=False
conv = convolve_fft(trunc_profile, kernel, normalize_kernel=False)
minPeak = np.argmin(conv)
minVal = min(conv)
# if geometric dip (greater than 40%), flag as candidate without template matching
norm_trunc_profile = trunc_profile / np.median(trunc_profile)
if norm_trunc_profile[minPeak] < 0.6:
critTime = np.where(fluxProfile == light_curve[minPeak])[0]
print datetime.datetime.now(
), "Detected >40% dip: frame", str(critTime) + ", star", num
return critTime[0]
if 30 <= minPeak < len(trunc_profile) - 30:
med = np.median(
np.concatenate((trunc_profile[minPeak - 100:minPeak - 30],
trunc_profile[minPeak + 30:minPeak + 100])))
trunc_profile = trunc_profile[(minPeak - 30):(minPeak + 30)]
unc = (
(np.sqrt(abs(trunc_profile) / 100.) / np.median(trunc_profile)) *
100)
unc = np.where(unc == 0, 0.01, unc)
trunc_profile /= med
else:
return -2 # reject events that are cut off at the start/end of time series
bkgZone = conv[10:-10]
if minVal < np.mean(
bkgZone) - 3.75 * np.std(bkgZone): # dip detection threshold
""" Kernel matching"""
old_min = np.inf
for ind in range(0, len(kernels)):
if min(kernels[ind]) > 0.8:
continue
new_min, loc = diffMatch(kernels[ind], trunc_profile, unc)
if new_min < old_min:
active_kernel = ind
min_loc = loc
old_min = new_min
unc_l = unc[min_loc:min_loc + len(kernels[active_kernel])]
thresh = 0
for u in unc_l:
thresh += (abs(m)**1) / (abs(u)**1) # kernel match threshold
if old_min < thresh:
critTime = np.where(fluxProfile == light_curve[minPeak])[0]
print datetime.datetime.now(
), "Detected candidate: frame", str(critTime) + ", star", num
if len(critTime) > 1:
raise ValueError
return critTime[
0] # returns location in original time series where dip occurs
else:
return -1 # reject events that do not pass kernel matching
else:
return -1 # reject events that do not pass dip detection
#the sample rate is the number of bits of infomration recorded per second
print(kernelRate)
print(kernelAudData)
#wav bit type the amount of information recorded in each bit often 8, 16 or 32 bit
kernelAudData.dtype
print("Data Shape:")
print(np.shape(kernelAudData))
#wav length
kernel_len_PRI = kernelAudData.shape[0] / rate
#Final kernel
kernel = np.trim_zeros(kernelAudData, 'fb')
plt.plot(kernel)
# In[7]:
# Padding
print('Channel 1 Length:'), print(len(channel1))
print('NSAMPS Length:'), print(NSAMPS)
while np.mod(NSAMPS, RECORD_SECONDS) != 0:
NSAMPS += 1
print(NSAMPS)
if len(channel1) > NSAMPS:
mod_channel1 = channel1[NSAMPS - len(channel1)]
print("* Shortened *")
def inspect(time_leng, pattern_leng, top_print, count_data):
#ファイル1のデータカウント
pattern_dict1 = {}
sumpsth = 0
for i, name in enumerate(sheet_names1):
sheet_df1[i] = file1.parse(name)
try:
#print(sheet_df1[i][1][1])
start_number = (np.where(sheet_df1[i]['INFORMATION']=="CHANNEL")[0][0]) +cannel_start
end_number = (np.where(sheet_df1[i]['INFORMATION']=="CHANNEL")[0][1]) - cannel_end
sig1 = (sheet_df1[i]['Unnamed: 3'][start_number : end_number]).values
except KeyError :
print("not max data number")
break
sig1 = sig1.astype("int")
#print(len(sig1))
sig1 = np.trim_zeros(sig1)
# print()
leng = len(sig1)
psth = np.zeros(int((leng/time_leng)+1), dtype=np.int)
l = 0
for k in range(0, leng, time_leng) :
psth[l] = sig1[k : k+time_leng].sum()
l += 1
# print(sig1[len(psth)-1])
if(len(psth) > pattern_leng) :
for k in range(len(psth) - pattern_leng +1) : # PSTHデータからはパターンを重ねて検索している
if (str(psth[k : k + pattern_leng]) in pattern_dict1) :
pattern_dict1[str(psth[k : k + pattern_leng])] += 1
else :
pattern_dict1[str(psth[k : k + pattern_leng])] = 1
sumpsth += (len(psth) -pattern_leng+1)
# print()
# print(sumpsth)
#ファイル2のデータカウント
pattern_dict2 = {}
sum_dict = pattern_dict1.copy()
for i, name in enumerate(sheet_names2):
sheet_df2[i] = file2.parse(name)
try :
start_number = (np.where(sheet_df2[i]['INFORMATION']=="CHANNEL")[0][0]) + cannel_start
end_number = (np.where(sheet_df2[i]['INFORMATION']=="CHANNEL")[0][1]) - cannel_end
sig1 = (sheet_df2[i]['Unnamed: 3'][start_number : end_number]).values
except KeyError :
print("not max data number")
break
sig1 = sig1.astype("int")
sig1 = np.trim_zeros(sig1)
leng = len(sig1)
# print(sig1[0])
psth = np.zeros(int((leng/time_leng)+1), dtype=np.int)
l = 0
for k in range(j, leng, time_leng) :
psth[l] = sig1[k+j : k+j+time_leng].sum()
l += 1
if(len(psth) > pattern_leng) :
for k in range(len(psth) - pattern_leng +1) :
if (str(psth[k : k+ pattern_leng]) in pattern_dict2) :
pattern_dict2[str(psth[k : k+ pattern_leng])] += 1
sum_dict[str(psth[k : k+ pattern_leng])] += 1
else :
pattern_dict2[str(psth[k : k+ pattern_leng])] = 1
if (str(psth[k : k+ pattern_leng]) in sum_dict) :
sum_dict[str(psth[k : k+ pattern_leng])] += 1
else :
sum_dict[str(psth[k : k+ pattern_leng])] = 1
sum_pattern1 = sum(pattern_dict1.values()) #+ len(sum_dict.keys())
sum_pattern2 = sum(pattern_dict2.values()) #+ len(sum_dict.keys())
if(len(pattern_dict1.keys()) == 0) :
del pattern_dict1, pattern_dict2, sum_dict, leng, psth, start_number, end_number, sig1, i, l, k, sum_pattern1
if(len(pattern_dict2.keys()) == 0 ):
gc.collect()
return 0, 0, 0
else :
gc.collect()
return 0, 0, sum_pattern2
elif(len(pattern_dict2.keys()) == 0 ):
del pattern_dict1, pattern_dict2, sum_dict, leng, psth, start_number, end_number, sig1, i, l, k, sum_pattern2
gc.collect()
return 0, sum_pattern1, 0
#情報量の計算準備
pattern_information = 0.0
# print()
# print()
# print(sum(pattern_dict1.values()))
# print(sum(sum_dict.values()))
# print()
sum_pattern1 = sum(pattern_dict1.values()) + len(sum_dict.keys())
sum_pattern2 = sum(pattern_dict2.values()) + len(sum_dict.keys())
# print(sum_pattern1)
# print(sum_pattern2)
# sum_pattern = sum(sum_dict.values())
pattern1 = len(pattern_dict1.keys())
probability1 = np.zeros(pattern1, float)
probability2 = np.zeros(pattern1, float)
top_dict = {}
k = 0
for i in (pattern_dict1.keys()) :
probability1[k] = ((pattern_dict1[i]) / sum_pattern1)
if(i in pattern_dict2) :
probability2[k] = ((pattern_dict2[i]+1) / sum_pattern2)
else :
probability2[k] = (1 / sum_pattern2)
info = probability1[k] * math.log2(probability1[k]/probability2[k])
pattern_information += info
top_dict[i] = info
# print(i)
k += 1
if (len(pattern_dict1.keys()) < top_print) :
max_print = len(pattern_dict1.keys())
else :
max_print =top_print
#グラフ出力の準備
# print_probability1 = np.zeros(max_print, float)
# print_probability2 = np.zeros(max_print, float)
# print_count1 = np.zeros(max_print, float)
# print_count2 = np.zeros(max_print, float)
# print_kullback = np.zeros(max_print, float)
# print_pattern = []
k = 0
#print(time_leng, pattern_leng, sum_pattern1, sum_pattern2, file=count_data)
for i, v in sorted(top_dict.items(), key=lambda x:-x[1]) :[0:max_print] :
def sample2str(sample):
trimmed = np.trim_zeros(sample)
if len(trimmed) and trimmed[-1] == EOS_ID:
trimmed = trimmed[:-1]
return " ".join(map(str, trimmed))
def jointSolve(d, orders):
import pylab
path = __path__[0]
lines = {}
lines['cuar'] = numpy.loadtxt(path + "/data/cuar.lines")
lines['hgne'] = numpy.loadtxt(path + "/data/hgne.lines")
lines['xe'] = numpy.loadtxt(path + "/data/xe.lines")
startsoln = numpy.load(path + "/data/esi_wavesolution.dat")
startsoln = numpy.load(path + '/data/shao_wave.dat')
startsoln = [i for i, j in startsoln]
alldata = d['arc']
arclist = d.keys()
arclist.remove('arc')
soln = []
if alldata.shape[1] > 3000:
xvals = numpy.arange(4096.)
resolve = False
cuslice = slice(3860, 3940)
fw1 = 75.
fw2 = 9
WIDTH = 4
else:
xvals = numpy.arange(2048.)
resolve = True
cuslice = slice(1930, 1970)
fw1 = 37.
fw2 = 7
WIDTH = 3
for i in range(10):
solution = startsoln[i]
start, end = orders[i]
if resolve == True:
tmp = numpy.arange(0.5, 4096., 2.)
w = sf.genfunc(tmp, 0., solution)
solution = sf.lsqfit(numpy.array([xvals, w]).T, 'chebyshev', 3)
data = numpy.nanmedian(alldata[start:end], axis=0)
data[numpy.isnan(data)] = 0.
if i == 0:
data[cuslice] = numpy.median(data)
bak = ndimage.percentile_filter(data, 50., int(fw1))
data -= bak
peaks = []
p = ndimage.maximum_filter(data, fw2)
std = clip(numpy.trim_zeros(data), 3.)[1]
peak = numpy.where((p > 30. * std) & (p == data))[0]
for p in peak:
if p - WIDTH < 0 or p + WIDTH + 1 > xvals.size:
continue
x = xvals[p - WIDTH:p + WIDTH + 1].copy()
f = data[p - WIDTH:p + WIDTH + 1].copy()
fitdata = numpy.array([x, f]).T
fit = numpy.array([0., f.max(), xvals[p], 1.])
fit, chi = sf.ngaussfit(fitdata, fit, weight=1)
peaks.append(fit[2])
for converge in range(10):
wave = 10**sf.genfunc(xvals, 0., solution)
refit = []
corr = []
err = wave[wave.size / 2] - wave[wave.size / 2 - 1]
p = 10.**sf.genfunc(peaks, 0., solution)
for arc in arclist:
for k in range(p.size):
if i == 0 and p[k] > 4344.:
continue
cent = p[k]
diff = cent - lines[arc]
corr.append(diff[abs(diff).argmin()])
corr = numpy.array(corr)
corr = corr[abs(corr) < 5 * err]
m, s = clip(corr)
corr = numpy.median(corr[abs(corr - m) < 5. * s])
for arc in arclist:
for k in range(p.size):
if i == 0 and p[k] > 4344.:
continue
pos = peaks[k]
cent = p[k]
diff = abs(cent - lines[arc] - corr)
if diff.min() < 2. * err:
refit.append([pos, lines[arc][diff.argmin()]])
refit = numpy.asarray(refit)
solution = sf.lsqfit(refit, 'polynomial', 3)
refit = []
err = solution['coeff'][1]
p = sf.genfunc(peaks, 0., solution)
for k in range(p.size):
delta = 1e9
match = None
for arc in arclist:
for j in lines[arc]:
if i == 0 and j > 4344.:
continue
diff = abs(p[k] - j)
if diff < delta and diff < 1. * err:
delta = diff
match = j
if match is not None:
refit.append([peaks[k], match])
refit = numpy.asarray(refit)
refit[:, 1] = numpy.log10(refit[:, 1])
solution = sf.lsqfit(refit, 'chebyshev', 3)
solution2 = sf.lsqfit(refit[:, ::-1], 'chebyshev', 3)
g = 10**sf.genfunc(peaks, 0., solution)
g2 = 10**sf.genfunc(peak, 0., solution)
w = 10**sf.genfunc(xvals, 0., solution)
if (w == wave).all():
print("Order %d converged in %d iterations" % (i, converge))
soln.append([solution, solution2])
break
pylab.plot(w, data)
for arc in arclist:
for j in lines[arc]:
if j > w[0] and j < w[-1]:
pylab.axvline(j, c='b')
for j in 10**refit[:, 1]:
if j > w[0] and j < w[-1]:
pylab.axvline(j, c='r')
for j in g:
pylab.axvline(j, c='g')
for j in g2:
pylab.axvline(j, c='c')
pylab.show()
break
return soln
def max_peak_finder(
signal,
threshold='unspecified',
initial_thres='unspecified',
vert_offset='unspecified',
window_length='unspecified',
min_peak_distance='unspecified',
min_amount_from_threshold='unspecified',
peak_value_consistancy_fraction='unspecified'
): # if thershold isnt specified it defaults it is replaced with the mean of the signal
# ------------------------------------------- IMPORTED MODULES -----------------------------------------------
from numpy import array, int_, append, argmax, argsort, zeros, ceil, trim_zeros
# -------------------------------------------------------------------------------------------------------------
# Checking inputs
if threshold == 'unspecified':
if vert_offset == 'unspecified':
# No vert_offset was entered, therefore default of 0 will be used making the threshold = moving mean
vert_offset = 0
if window_length == 'unspecified':
print(
'ERROR, no window_length was provided, if you dont want to update the threshold use the basic_peak_finder function'
)
window_length = int(input('Please re-enter the window length: '))
return (max_peak_finder(signal, threshold, vert_offset,
window_length, min_peak_distance,
min_amount_from_threshold,
peak_value_consistancy_fraction))
if window_length % 2 == 0:
print(
'ERROR, an even number was entered for the window length, the window length should be odd'
)
window_length = int(input('Please re-enter the window length: '))
return (max_peak_finder(signal, threshold, vert_offset,
window_length, min_peak_distance,
min_amount_from_threshold,
peak_value_consistancy_fraction))
if peak_value_consistancy_fraction != 'unspecified' and peak_value_consistancy_fraction <= 0 and peak_value_consistancy_fraction >= 1:
print(
'ERROR, peak_value_consistancy_persentage received an input that was not a fraction between 0 and 1'
)
peak_value_consistancy_fraction = int(
input('Please re-enter the fraction: '))
return (max_peak_finder(signal, threshold, vert_offset,
window_length, min_peak_distance,
min_amount_from_threshold,
peak_value_consistancy_fraction))
elif threshold != 'unspecified' and vert_offset != 'unspecified' or window_length != 'unspecified':
print(
'ERROR! Threshold has been defined as a constant, but so too has one or more of the adaptive arguments: vert_offset or window_length'
)
print(
'If the threshold is fixed as a constant the adaptive arguments are not needed and should be left undefined'
)
return ()
# Initilzing varibles
ignored_peak_index = zeros(
int(ceil(len(signal) / 2)), dtype=int_
) # initializing an ignored_peak_index array where the index of the peak that didnt meet the min distance requirement between it and the previous peak are stored
ignored_peak_value = zeros(
int(ceil(len(signal) / 2))
) # initializing a ignored_peak_value array where the ignored peak values that didnt meet the min distance requirement between it and the previous peak will be stored
m = 0 # m is an iterator that ensures once an index is filled in the ignored_peak_ arrays, the next ignored_peak_ is placed in the next empty index location.
ignored_max_index = zeros(
int(ceil(len(signal) / 2)), dtype=int_
) # initializing an ignored_max_index array where indexs of max values that werent actually local peaks are stored for debugging
k = 0 # k is an iterator that moves through ignored_max_index array. see later
max_peak_index = zeros(
int(ceil(len(signal) / 2)), dtype=int_
) # initializing a max_peak_index array where the index of the peak in the signal array will be stored as integers
max_peak_value = zeros(
int(ceil(len(signal) / 2))
) # initializing a max_peak_value array where the peak values will be stored
n = 0 # n is an iterator that moves through max_peak_ arrays. see later
crossing_index = zeros(
3, dtype=int_
) # creating a list to store the indices of the crossing points
j = 0 # j is the iterate with a range of 0-2 (ie the 3 indices of crossing_index) after j = 2 it will be reset to 0. See later.
outer_crossings = zeros(
len(signal),
dtype=int_) # Storing the outer_crossings for debugging purposes
p = 0 # p is an iterator that moves through outer_crossings arrays. see later
inner_crossings = zeros(
len(signal),
dtype=int_) # Storing the inner_crossings for debugging purposes
q = 0 # q is an iterator that moves through inner_crossings arrays. see later
thresholds = zeros(
len(signal)
) # creating an array to store the thresholds as these are used to check when crossings have occured and can be graphed
# Determining how the first point in the signal is treated w.r.t a threshold
if initial_thres != 'unspecified':
thresholds[0] = initial_thres
start = 1 # with the 0th thershold determine, we want our for loop to start at the second point (1th iteration)
else: # a seperate threshold for the first point in signal was not entered, therefore it is treated like every other point and the function starts at the zeroth point in signal
start = 0
end_check = False # setting an end_check variable which is set to true if the signal ends with an inner crossing and was above the threshold before it.
# Peforming function
for i in range(
start, len(signal)
): # NB the reason we start at i=1 is because the threshold list is initilized with the threshold for i=0 in it, see above.
if threshold == 'unspecified': # this insures if threshold = 'unspecified' then an adaptive threshold is calculated at each iteration
# Calculating threshold at each point using a central smooth algorithm like matlabs smooth function
# if confused by this expression see the example in my Smooth Functions description AND remember that the stop condition value isnt used!
if i <= (window_length - 1) / 2:
window_mean = (
signal[0:window_length].mean()
) # using the mean of a full window length to approximate the mean of the first few points until i > window length
thresholds[i] = window_mean + vert_offset
elif i >= len(signal) - (window_length - 1) / 2:
window_mean = (
signal[-window_length:].mean()
) # using the mean of a full window length to approximate the mean of the last few points where a moving mean would need fewer forward points than avalible
thresholds[
i] = window_mean + vert_offset # Incase the threshold is -ve, we need to work out the threshold this way rather than just window_mean*multiple of mean
else:
window_mean = (
signal[i - int((window_length - 1) / 2):i +
(int((window_length - 1) / 2)) + 1].mean()
) # NB we need to add one to the stop point as we want to include index i+(window-1)/2 in the threshold calc
thresholds[
i] = window_mean + vert_offset # Incase the threshold is -ve, we need to work out the threshold this way rather than just window_mean*multiple of mean
else:
thresholds[
i] = threshold # if thershold was specified, the thresholds array, for every point is just the specified threshold.
# With the threshold calculated we now check to see if the current point meets the criteria to be a peak or trough
# checking to see if point is a positive gradient crossing of the threshold
if signal[i] >= thresholds[i] and signal[i - 1] <= thresholds[
i -
1] and i != 0: # NB if i = 0, signal[i-1]>=thresholds[i-1] --> signal[-1]>=thresholds[-1] which is looking at the end of the signal
crossing_index[j] = i
j = j + 1 # j steps forward one to fill the next index of crossing_index next time a crossing is found.
# checking to see if point is a negetive gradient crossing of the threshold
elif signal[i] <= thresholds[i] and signal[i - 1] >= thresholds[
i -
1] and i != 0: # NB if i = 0, signal[i-1]>=thresholds[i-1] --> signal[-1]>=thresholds[-1] which is looking at the end of the signal
crossing_index[j] = i
j = j + 1 # j steps forward one to fill the next index of crossing_index next time a crossing is found.
# Once 3 crossing have occured we can know there is a peak and trough between the two outer crossings and we can
# go about finding them.
# END PEAK CHECKER (I have the end_check variable just for improved readability)
if i == len(signal) - 1 and j == 2 and signal[
crossing_index[1] - 1] > thresholds[crossing_index[1] - 1]:
end_check = True # the signal was above the threshold between the last outer crossing and final inner crossing which means we need to check for a end peak between these crossings
if j == 3 or end_check == True:
# Argsort returns the INDICES of the sorted VALUES of an array
# I.E. sorted data is the indices of the maximum to minimum values of the data between the crossings, the [::-1] ensures it is max to min values
sorted_data_indices = (argsort(
signal[crossing_index[0]:crossing_index[j - 1]])[::-1]
) + crossing_index[0]
# Checking if the max value between two crossings is infact a local peak or if it is a false positive.
for index in sorted_data_indices:
# the condition below is: if the suspected_index_of_max_peak is not the first element or last element in the total signal and it is a local peak and its above its threshold then it is our max local peak.
if index != 0 and index != len(signal) - 1 and signal[
index -
1] < signal[index] and signal[index + 1] <= signal[
index] and signal[index] >= thresholds[index]:
index_of_max_peak = index # if the above condition is true, the suspected_index_of_max_peak has been shown to be a local peak! And hence it is
# the true index of the max peak between the current outer crossings.
break # once the max peak has been found we want to break.
else: # there was no max peak (which would be weird if we have crossings!) or the index was the first or last point in the signal
ignored_max_index[k] = i
k = k + 1 # k is an iterator that moves through ignored_max_index array.
index_of_max_peak = 'none found'
if index_of_max_peak != 'none found':
if min_peak_distance != 'unspecified' and n > 0 and index_of_max_peak - max_peak_index[
n -
1] < min_peak_distance: # Checking if the max peak in possible peaks is far enough from the previous positively identified max peak
ignored_peak_index[
m] = index_of_max_peak # we store the ignored peak data incase the user wants to check what was ignored in the signal
ignored_peak_value[m] = signal[index_of_max_peak]
m = m + 1 # m is an iterator that ensures once an index is filled in the ignored_peak_ arrays, the next ignored_peak_ is placed in the next empty index location.
elif min_amount_from_threshold != 'unspecified' and abs(
signal[index_of_max_peak] -
thresholds[index_of_max_peak]
) < min_amount_from_threshold: # we can set an amount a peak must be above the threshold for it to be a true peak, this is handy for when a signal may jump around the threshold a tiny amount and may not be an actual peak.
ignored_peak_index[
m] = index_of_max_peak # we store the ignored peak data incase the user wants to check what was ignored in the signal
ignored_peak_value[m] = signal[index_of_max_peak]
m = m + 1 # m is an iterator that ensures once an index is filled in the ignored_peak_ arrays, the next ignored_peak_ is placed in the next empty index location.
elif peak_value_consistancy_fraction != 'unspecified' and n > 0 and signal[
index_of_max_peak] <= (
1 + peak_value_consistancy_fraction
) * signal[max_peak_index[
n - 1]] and signal[index_of_max_peak] >= (
1 - peak_value_consistancy_fraction
) * signal[max_peak_index[
n -
1]]: # checking to see that the value of the peak found is within + or -5% of the previous peaks value, if it is, it is deemed too different to be a correct peak identification. Hence some foward knowledge of the signal is required before using this test.
ignored_peak_index[
m] = index_of_max_peak # we store the ignored peak data incase the user wants to check what was ignored in the signal
ignored_peak_value[m] = signal[index_of_max_peak]
m = m + 1 # m is an iterator that ensures once an index is filled in the ignored_peak_ arrays, the next ignored_peak_ is placed in the next empty index location.
else:
max_peak_index[
n] = index_of_max_peak # adding the index of the current maximum peak to a list of all the peaks
max_peak_value[n] = signal[
index_of_max_peak] # adding the value of the current max peak to a list of all the peaks
n = n + 1 # n is an iterator that ensures once an index is filled in the max_peak_ arrays, the next max_peak_ is placed in the next empty index location.
# STORING AND RESETTING THE CROSSING ARRAY DATA
# Also NB that when finding the argmin or argmax of signal[crossing_index[0]: crossing_index[2]+1], this is just a view of the array signal meaning it will treat this view first
# as its own array with the first element of the array indexed back at 0. To work out the peaks index in the whole signal array we need to add crossing_index[0].
if p == 0:
outer_crossings[p] = crossing_index[0]
outer_crossings[p + 1] = crossing_index[2]
p = p + 2 # p is an iterator that ensures once an index is filled in the outer_crossings, the outer crossing is placed in the next empty index location.
elif j == 3: # if end_check is true, the signal ends on an inner crossing and the for loop ends with j == 2, so we dont want to add crossing_index[2] if j == 2
outer_crossings[p] = crossing_index[
2] # since crossing_index[2] becomes crossing_index[0] (see below) for successive peak finding, if we store crossing_index[0], it will just be the crossing_index[2] of the last peak
p = p + 1 # p is an iterator that ensures once an index is filled in the outer_crossings, the outer crossing is placed in the next empty index location.
inner_crossings[q] = crossing_index[1]
q = q + 1 # q is an iterator that ensures once an index is filled in the inner_crossings arrays, the next inner crossing is placed in the next empty index location.
crossing_index[0] = crossing_index[
2] # Setting the last crossing crossing to be the first crossings in the crossing_index, so we start looking for another two crossings from our current last crossing.
# I.E. our final crossing now becomes our first crossing to continue searching from
j = 1 # resetting j to one so we find our next 2 crossing. NB our first crossing will be the last crossing from the prevous set of 3 crossings as outlined above.
# trimming all unfilled indices of max_peak_ arrays, ignored_peak_ arrays, and crossing arrays
max_peak_index = trim_zeros(
max_peak_index
) # as we cannot possible have a peak at an index location 0, we can trim peak_index and use its length to define the peak_value array. As a peaks value could be 0, so we cant use the trim_zeros function of the peak_value array.
max_peak_value = max_peak_value[:len(
max_peak_index
)] # trimming the peak_value array to the length of peak_index, as they must be the same length, each index is information on the same peak.
ignored_peak_index = trim_zeros(
ignored_peak_index
) # the code is such that we cannot possible have an ignored_peak at an index location 0, so we can trim peak_index and use its length to define the ignored_peak_value array. As an ignored_peak_value could be 0, so we cant use the trim_zeros function of that array.
ignored_peak_value = ignored_peak_value[:len(
ignored_peak_index
)] # trimming the ignored_peak_value array to the length of ignored_peak_index, as they must be the same length, each index is information on the same peak.
ignored_max_index = trim_zeros(ignored_max_index, 'b')
outer_crossings = trim_zeros(outer_crossings, 'b')
inner_crossings = trim_zeros(inner_crossings)
return (max_peak_index, max_peak_value, thresholds, outer_crossings,
inner_crossings, ignored_peak_index)
def encoderfunc(textin, EC):
# Version Selector(partial) #
mode = '0010'
l = len(textin)
version = 0
datacap = 0
terminator = 0
groups = 0
gpoly = 0
g1 = 0
g2 = 0
charcount = '{0:09b}'.format(l)
if EC == 'Low':
bits = [0, 1]
if l < 25:
version = 1
datacap = 152
groups = 1
cwcount = 19
eccw = 7
gpoly = [0, 87, 229, 146, 149, 238, 102, 21]
elif 25 <= l < 47:
version = 2
datacap = 272
groups = 1
cwcount = 34
eccw = 10
gpoly = [0, 251, 67, 46, 61, 118, 70, 64, 94, 32, 45]
elif 47 <= l < 77:
version = 3
datacap = 440
groups = 1
cwcount = 55
eccw = 15
gpoly = [
0, 8, 183, 61, 91, 202, 37, 51, 58, 58, 237, 140, 124, 5, 99,
105
]
elif 77 <= l < 114:
version = 4
datacap = 640
groups = 1
cwcount = 80
eccw = 20
gpoly = [
0, 17, 60, 79, 50, 61, 163, 26, 187, 202, 180, 221, 225, 83,
239, 156, 164, 212, 212, 188, 190
]
elif 114 <= l < 154:
version = 5
datacap = 864
groups = 1
cwcount = 108
eccw = 26
gpoly = [
0, 173, 125, 158, 2, 103, 182, 118, 17, 145, 201, 111, 28, 165,
53, 161, 21, 245, 142, 13, 102, 48, 227, 153, 145, 218, 70
]
elif EC == 'Medium':
bits = [0, 0]
if l < 20:
version = 1
datacap = 128
groups = 1
cwcount = 16
eccw = 10
gpoly = [0, 251, 67, 46, 61, 118, 70, 64, 94, 32, 45]
elif 20 <= l < 38:
version = 2
datacap = 224
groups = 1
cwcount = 28
eccw = 16
gpoly = [
0, 120, 104, 107, 109, 102, 161, 76, 3, 91, 191, 147, 169, 182,
194, 225, 120
]
elif 38 <= l < 61:
version = 3
datacap = 352
groups = 1
cwcount = 44
eccw = 26
gpoly = [
0, 173, 125, 158, 2, 103, 182, 118, 17, 145, 201, 111, 28, 165,
53, 161, 21, 245, 142, 13, 102, 48, 227, 153, 145, 218, 70
]
elif 61 <= l < 90:
version = 4
datacap = 512
groups = 1
cwcount = 64
eccw = 18
gpoly = [
0, 215, 234, 158, 94, 184, 97, 118, 170, 79, 187, 152, 148,
252, 179, 5, 98, 96, 153
]
elif 90 <= l < 122:
version = 5
datacap = 688
groups = 1
cwcount = 86
eccw = 24
gpoly = [
24, 229, 121, 135, 48, 211, 117, 251, 126, 159, 180, 169, 152,
192, 226, 228, 218, 111, 0, 117, 232, 87, 96, 227, 21
]
elif EC == 'Quartile':
bits = [1, 1]
if l < 16:
version = 1
datacap = 104
groups = 1
cwcount = 13
eccw = 13
gpoly = [
13, 74, 152, 176, 100, 86, 100, 106, 104, 130, 218, 206, 140,
78
]
elif 16 <= l < 29:
version = 2
datacap = 176
groups = 1
cwcount = 22
eccw = 22
gpoly = [
0, 210, 171, 247, 242, 93, 230, 14, 109, 221, 53, 200, 74, 8,
172, 98, 80, 219, 134, 160, 105, 165, 231
]
elif 29 <= l < 47:
version = 3
datacap = 272
groups = 1
cwcount = 34
eccw = 18
gpoly = [
0, 215, 234, 158, 94, 184, 97, 118, 170, 79, 187, 152, 148,
252, 179, 5, 98, 96, 153
]
elif 47 <= l < 67:
version = 4
datacap = 384
groups = 1
cwcount = 48
eccw = 26
gpoly = [
0, 173, 125, 158, 2, 103, 182, 118, 17, 145, 201, 111, 28, 165,
53, 161, 21, 245, 142, 13, 102, 48, 227, 153, 145, 218, 70
]
elif 67 <= l < 87:
version = 5
datacap = 496
groups = 2
g1 = 15
g2 = 16
cwcount = 62
eccw = 18
gpoly = [
0, 215, 234, 158, 94, 184, 97, 118, 170, 79, 187, 152, 148,
252, 179, 5, 98, 96, 153
]
elif EC == 'High':
bits = [1, 0]
if l < 10:
version = 1
datacap = 72
groups = 1
cwcount = 9
eccw = 17
elif 10 <= l < 20:
version = 2
datacap = 128
groups = 1
cwcount = 16
eccw = 28
elif 20 <= l < 35:
version = 3
datacap = 208
groups = 1
cwcount = 26
eccw = 22
gpoly = [
0, 210, 171, 247, 242, 93, 230, 14, 109, 221, 53, 200, 74, 8,
172, 98, 80, 219, 134, 160, 105, 165, 231
]
elif 35 <= l < 50:
version = 4
datacap = 288
groups = 1
cwcount = 36
eccw = 16
gpoly = [
0, 120, 104, 107, 109, 102, 161, 76, 3, 91, 191, 147, 169, 182,
194, 225, 120
]
elif 50 <= l < 64:
version = 5
datacap = 368
groups = 2
g1 = 11
g2 = 12
cwcount = 46
eccw = 22
gpoly = [
0, 210, 171, 247, 242, 93, 230, 14, 109, 221, 53, 200, 74, 8,
172, 98, 80, 219, 134, 160, 105, 165, 231
]
if version == 1:
remainder = 0
else:
remainder = 7
print("version=" + str(version))
print("datacap=" + str(datacap))
print("groups=", groups)
# String Split Function #
def split(fn):
while fn:
yield fn[:2]
fn = fn[2:]
# Loop to convert characters to binary encoded data #
a = list(split(textin))
length = len(a)
encoded = ''
for i in range(0, length):
a[i] = a[i].replace('0', '00').replace('1', '01').replace(
'2', '02').replace('3', '03').replace('4', '04').replace(
'5', '05').replace('6', '06').replace('7', '07').replace(
'8', '08').replace('9', '09').replace('A', '10')
a[i] = a[i].replace('B', '11').replace('C', '12').replace(
'D', '13').replace('E', '14').replace('F', '15').replace(
'G', '16').replace('H', '17').replace('I', '18').replace(
'J', '19').replace('K', '20').replace('L', '21')
a[i] = a[i].replace('M', '22').replace('N', '23').replace(
'O', '24').replace('P', '25').replace('Q', '26').replace(
'R', '27').replace('S', '28').replace('T', '29').replace(
'U', '30').replace('V', '31').replace('W', '32')
a[i] = a[i].replace('X', '33').replace('Y', '34').replace(
'Z', '35').replace(' ', '36').replace('$', '37').replace(
'%', '38').replace('*', '39').replace('+', '40').replace(
'-', '41').replace('.',
'42').replace('/',
'43').replace(':', '44')
a[i] = a[i].replace('b', '11').replace('c', '12').replace(
'd', '13').replace('e', '14').replace('f', '15').replace(
'g', '16').replace('h', '17').replace('i', '18').replace(
'j', '19').replace('k', '20').replace('l', '21')
a[i] = a[i].replace('m', '22').replace('n', '23').replace(
'o', '24').replace('p', '25').replace('q', '26').replace(
'r', '27').replace('s', '28').replace('t', '29').replace(
'u', '30').replace('v', '31').replace('w', '32')
a[i] = a[i].replace('x', '33').replace('y', '34').replace(
'z', '35').replace('a', '10')
if len(a[i]) == 4:
a[i] = (int((a[i])[:2]) * 45) + (int((a[i])[2:5]))
a[i] = '{0:011b}'.format(a[i])
elif len(a[i]) == 2:
a[i] = '{0:06b}'.format(int(a[i]))
encoded = encoded + a[i]
print("encoded=", encoded)
datastring2 = (mode + charcount + encoded)
dist = (datacap - len(datastring2))
# Pad bits for short string #
if dist >= 4:
terminator = "0000"
elif dist == 3:
terminator = "000"
elif dist == 2:
terminator = "00"
elif dist == 1:
terminator = "0"
elif dist == 0:
terminator = ""
datastring = (mode + charcount + encoded + terminator)
dist2 = (datacap - len(datastring))
l2 = len(datastring)
pad = (8 - (l2 % 8))
pad = "0" * pad
datastring = datastring + pad
extrapad = int((datacap - len(datastring)) / 8)
# POTENTIAL PROBLEM HERE ##### LOOK AT INT ABOVE #############
for i in range(1, extrapad + 1):
if i % 2 == 1:
datastring = datastring + "11101100"
elif i % 2 == 0:
datastring = datastring + "00010001"
print("datastring=", datastring)
# Creating a log antilog table #
exp2int = [1]
int2exp = [i for i in range(256)]
for i in range(1, 256):
b = exp2int[i - 1] * 2
if b >= 256: # XOR if result is larger than or equal to 256. Use ^^ to XOR two integers
b = b ^ 285 # USED TO BE b^^285, bitwise xor#
exp2int.append(b)
int2exp[b] = i
int2exp[0] = [] # setting zero blank
int2exp[1] = 0 # 1 comes up at 255 again
# Message Polynomial #
line = datastring
mpoly = [int(line[i:i + 8], 2) for i in range(0, len(line), 8)]
print("mpoly=", mpoly)
# Long Division #
print("generator polynomial=", gpoly)
print("message polynomial=", mpoly)
def longdivision(steps, mpoly, gpoly):
for i in range(0, steps):
gap = len(mpoly) - len(gpoly)
m = mpoly[0]
m = int2exp[m]
if gap > 0:
newgpoly = [exp2int[(g + m) % 255] for g in gpoly] + [0] * gap
else:
newgpoly = [exp2int[(g + m) % 255] for g in gpoly]
blank = []
if gap < 0:
mpoly = mpoly + [0] * abs(gap)
for i in range(0, len(newgpoly)):
b = [mpoly[i] ^ newgpoly[i]] # More xor problems?
blank = blank + b
mpoly = numpy.trim_zeros(blank, trim='f')
return mpoly
# interleaving #
if groups == 1:
steps = len(mpoly)
ecwords = longdivision(steps, mpoly, gpoly)
print("ecwords=", ecwords)
message = mpoly + ecwords
message = ['{0:08b}'.format(i) for i in message] + [0] * remainder
blank = ''
for i in message:
blank = blank + str(i)
message = blank
print("full message=", message)
elif groups == 2:
b1 = mpoly[0:g1]
b2 = mpoly[g1:(2 * g1)]
b3 = mpoly[(2 * g1):(2 * g1 + g2)]
b4 = mpoly[(2 * g1 + g2):(2 * g1 + 2 * g2)]
ec1 = longdivision(g1, b1, gpoly)
ec2 = longdivision(g1, b2, gpoly)
ec3 = longdivision(g2, b3, gpoly)
ec4 = longdivision(g2, b4, gpoly)
print("b1=", b1)
print("b2=", b2)
print("b3=", b3)
print("b4=", b4)
print("ec1=", ec1)
print("ec2=", ec2)
print("ec3=", ec3)
print("ec4=", ec4)
blank = []
for i in range(g1):
blank = blank + [b1[i]] + [b2[i]] + [b3[i]] + [b4[i]]
itldata = blank + [b3[-1]] + [b4[-1]]
blank = []
for i in range(len(ec1)):
blank = blank + [ec1[i]] + [ec2[i]] + [ec3[i]] + [ec4[i]]
itlecw = blank
message = itldata + itlecw
message = ['{0:08b}'.format(i) for i in message] + [0] * remainder
blank = ''
for i in message:
blank = blank + str(i)
message = blank
print("full message=", message)
# Graphical Array #
dimension = ((version - 1) * 4) + 21
print("dimension = ", dimension, "x", dimension)
m = numpy.zeros([dimension, dimension])
# Finder & Timing Pattern #
for i in range(7):
m[i][0:7] = 3
m[i][-7:dimension] = 3
for i in range(-1, -8, -1):
m[i][0:7] = 3
for i in range(1, 6):
m[i][1:6] = 0
m[i][-6:dimension - 1] = 0
for i in range(-2, -7, -1):
m[i][1:6] = 0
for i in range(2, 5):
m[i][2:5] = 3
m[i][-5:dimension - 2] = 3
for i in range(-3, -6, -1):
m[i][2:5] = 3
for i in range(8, dimension - 8, 2):
m[i][6] = 3
m[6][i] = 3
# Data graphic #
shift = 1
direction = 1
l = 0
u = 0
g = 1
cap1 = (dimension - 9) * 2
cap2 = dimension * 2
count = 0
counter = 0
count3 = 0
maxcount = ((dimension - 17) / 2) + 4
maxcount3 = (dimension - 17) * 2
maxcol = ((dimension - 7) / 2) - 1
for pos, i in enumerate(message):
if pos > ((dimension - 9) * 8):
counter = counter + 1
if pos > 0 and pos % cap1 == 0 and count < 4:
count = count + 1
shift = shift + 2
direction = -direction
g = -g
u = u + g
elif 4 <= count < maxcount and counter % cap2 == 0:
count = count + 1
shift = shift + 2
direction = -direction
g = -g
u = u + g
if (dimension - 1 - u) == 6:
u = u + (1 * direction)
counter = counter + 2
if count == maxcol and counter % cap2 == 0:
u = u + 8
counter = counter + 14
count = count + 1
if count3 == maxcount3:
shift = shift + 3
direction = -direction
g = -g
u = u + g
elif count3 > maxcount3 and count3 % maxcount3 == 0:
shift = shift + 2
direction = -direction
g = -g
u = u + g
if count >= maxcol:
count3 = count3 + 1
m[dimension - 1 - u][dimension - shift - l] = int(i) + 1
l = (l + 1) % 2
u = u + (pos % 2) * direction
# Masking #
masknum = 0
maskbin = [0, 0, 0]
# maskbin=[0,0,1]
def mask0(m):
for r in range(dimension):
for c in range(dimension):
if m[r][c] == 1 or 2:
if (r + c) % 2 == 0:
if m[r][c] == 2:
m[r][c] = 1
elif m[r][c] == 1:
m[r][c] = 2
def mask1(m):
for r in range(dimension):
for c in range(dimension):
if m[r][c] == 1 or 2:
if r % 2 == 0:
if m[r][c] == 2:
m[r][c] = 1
elif m[r][c] == 1:
m[r][c] = 2
mask0(m)
# mask1(m)
# Information Bits #
gpoly = [1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1]
bitstring = bits + maskbin
infostring = bitstring + [0] * 10
infostring = numpy.trim_zeros(infostring[:5], trim='f') + infostring[5:]
print("infostring=", infostring)
if len(infostring) == 10:
ecstring = infostring + [0]
while len(infostring) >= 11:
gap = len(infostring) - len(gpoly)
if gap != 0:
gpoly = gpoly + [0] * gap
pos = 0
for i, j in zip(infostring, gpoly):
infostring[pos] = bool(i) ^ bool(j)
pos = pos + 1
infostring = numpy.trim_zeros(infostring, trim='f')
ecstring = infostring
print("ecstring=", ecstring)
if len(ecstring) < 10:
ecstring = (10 - len(ecstring)) * [0] + ecstring
print("final ecstring=", ecstring)
ecstring = bitstring + ecstring
qrspec = [1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0]
pos = 0
for i, j in zip(ecstring, qrspec):
qrspec[pos] = (bool(i) ^ bool(j)) * 3
pos = pos + 1
formatstring = qrspec
print("format string=", formatstring)
if EC == 'L':
if masknum == 0:
formatstring = [3, 3, 3, 0, 3, 3, 3, 3, 3, 0, 0, 0, 3, 0, 0]
elif masknum == 1:
formatstring = [3, 3, 3, 0, 0, 3, 0, 3, 3, 3, 3, 0, 0, 3, 3]
elif EC == 'M':
if masknum == 0:
formatstring = [3, 0, 3, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0]
elif masknum == 1:
formatstring = [3, 0, 3, 0, 0, 0, 3, 0, 0, 3, 0, 0, 3, 0, 3]
elif EC == 'Q':
if masknum == 0:
formatstring = [0, 3, 3, 0, 3, 0, 3, 0, 3, 0, 3, 3, 3, 3, 3]
elif masknum == 1:
formatstring = [0, 3, 3, 0, 0, 0, 0, 0, 3, 3, 0, 3, 0, 0, 0]
elif EC == 'H':
if masknum == 0:
formatstring = [0, 0, 3, 0, 3, 3, 0, 3, 0, 0, 0, 3, 0, 0, 3]
elif masknum == 1:
formatstring = [0, 0, 3, 0, 0, 3, 3, 3, 0, 3, 3, 3, 3, 3, 0]
print("formatstring2=", formatstring)
for pos, i in enumerate(range(-8, 0, 1)):
m[8][i] = formatstring[(pos + 7)]
for pos, i in enumerate(range(-1, -8, -1)):
m[i][8] = formatstring[pos]
for pos, i in enumerate([0, 1, 2, 3, 4, 5, 7, 8]):
m[8][i] = formatstring[pos]
for pos, i in enumerate([8, 7, 5, 4, 3, 2, 1, 0]):
m[i][8] = formatstring[(pos + 7)]
m[-8][8] = 3
for i in range(dimension):
for j in range(dimension):
if m[i][j] == 1:
m[i][j] = 0
if m[i][j] == 2:
m[i][j] = 3
plt.matshow(m, fignum=None, cmap='Greys')
plt.savefig("qrcode.png", format='png')
plt.show()
delimiter=',',
usecols=(6, ),
unpack=True)
vale_returns = np.diff(vale) / vale[:-1]
smooth_vale = np.convolve(weights / weights.sum(), vale_returns)[N - 1:-N + 1]
K = 3
t = np.arange(N - 1, len(bhp_returns))
poly_bhp = np.polyfit(t, smooth_bhp, K)
poly_vale = np.polyfit(t, smooth_vale, K)
poly_sub = np.polysub(poly_bhp, poly_vale)
xpoints = np.roots(poly_sub)
print "Intersection points", xpoints
reals = np.isreal(xpoints)
print "Real number?", reals
xpoints = np.select([reals], [xpoints])
xpoints = xpoints.real
print "Real intersection points", xpoints
print "Sans 0s", np.trim_zeros(xpoints)
plot(t, bhp_returns[N - 1:], lw=1.0)
plot(t, smooth_bhp, lw=2.0)
plot(t, vale_returns[N - 1:], lw=1.0)
plot(t, smooth_vale, lw=2.0)
show()