def read(self):
""" loads a raster image and splits it into roughly equal width vertical slices"""
print("Loading input raster {0} and splitting into {1} chunks!".format(
os.path.basename(self.rasterpath), self.num_chunks))
if self.num_chunks < 1:
raise Exception("Cannot split into any fewer than 1 chunk!")
# loads entire raster as numpy array with metadata object
if not self.force_scv:
from dnppy import raster
data, self.metadata = raster.to_numpy(self.rasterpath)
# uses the simpleCV module to import raster without metadata
else:
pass
# split the data and add new chunks to this raster
ys, xs = data.shape
width = xs / float(self.num_chunks)
for c in range(self.num_chunks):
chunk_data = data[:, int(c * width):int((c + 1) * width)]
new_chunk = chunk(c, chunk_data)
self.chunk_list.append(new_chunk)
del data
return
def test_no_input_no_output():
source = BytesIO(b'')
generator = chunk(source)
with raises(StopIteration):
next(generator)
def test_medium_input_no_delimiter():
source = BytesIO(b'0123456789')
generator = chunk(source, limit=6, delimiter=b'9')
with raises(ValueError):
next(generator)
def push_right(self, item):
if self.right.is_full():
if self.middle == None:
self.middle = deque()
self.middle.push_right(self.right)
self.right = chunk.chunk()
self.right.push_back(item)
def push_left(self, item):
if self.left.is_full():
if self.middle == None:
self.middle = deque()
self.middle.push_left(self.left)
self.left = chunk.chunk()
self.left.push_front(item)
def copy(self,rb,chunks,found):
chnk = chunk()
chnk.addTag(found)
chunks.append(chnk)
for rbel in rb.get():
if(rbel!=None):
chnk.append(rbel)
def read(self):
""" loads a raster image and splits it into roughly equal width vertical slices"""
print("Loading input raster {0} and splitting into {1} chunks!".format(
os.path.basename(self.rasterpath), self.num_chunks))
if self.num_chunks <1:
raise Exception("Cannot split into any fewer than 1 chunk!")
# loads entire raster as numpy array with metadata object
if not self.force_scv:
from dnppy import raster
data, self.metadata = raster.to_numpy(self.rasterpath)
# uses the simpleCV module to import raster without metadata
else: pass
# split the data and add new chunks to this raster
ys, xs = data.shape
width = xs / float(self.num_chunks)
for c in range(self.num_chunks):
chunk_data = data[:, int(c * width):int((c+1) * width)]
new_chunk = chunk(c, chunk_data)
self.chunk_list.append(new_chunk)
del data
return
def test_large_reads():
source = BytesIO(b'0123456789')
generator = chunk(source, limit=100)
handle = next(generator)
assert handle.read(100) == b'0123456789'
assert handle.read(100) == b''
def test_medium_input_two_output():
source = BytesIO(b'0123456789')
generator = chunk(source, limit=6, delimiter=b'5')
assert next(generator).read() == b'012345'
assert next(generator).read() == b'6789'
with raises(StopIteration):
next(generator)
def test_no_input_no_output():
source = BytesIO(b'')
generator = chunk(source)
handle = next(generator)
assert handle.read() == b''
with raises(StopIteration):
next(generator)
def test_medium_input_no_delimiter():
source = BytesIO(b'0123456789')
generator = chunk(source, limit=6, delimiter=b'9')
handle = next(generator)
assert handle.read() == b'0123456789'
with raises(StopIteration):
next(generator)
def test_small_input_one_output():
source = BytesIO(b'0123456789')
generator = chunk(source)
handle = next(generator)
assert handle.read() == b'0123456789'
with raises(StopIteration):
next(generator)
def push(self, item):
if self.head.is_full():
if self.tail == None:
self.tail = stack()
self.tail.push(self.head)
self.nb_tail += 1
if self.spare != None:
self.head = self.spare
self.spare = None
else:
self.head = chunk.chunk()
self.head.push(item)
def test_limited_reads():
source = BytesIO(b'0123456789')
generator = chunk(source, limit=2)
handle = next(generator)
assert handle.read(2) == b'01'
assert handle.read(2) == b'23'
assert handle.read(2) == b'45'
assert handle.read(2) == b'67'
assert handle.read(2) == b'89'
assert handle.read(2) == b''
def test_fuzzy():
bytes_to_read = int(2**(random() * 32767 / 1500))
desired_size = int(2**(random() * 32767 / 1500))
with open('/dev/urandom', 'rb') as handle:
content = handle.read(bytes_to_read)
source = BytesIO(content)
output = BytesIO()
for handle in chunk(source, limit=desired_size):
data = _read_one_chunk(handle)
output.write(data)
result = output.getvalue()
assert content == result, \
'source has {source_length} bytes, output has {output_length} bytes'.format(
source_length=len(content),
output_length=len(result)
)
def test():
random.seed()
bytes_to_read = int(2 ** (random.random() * 32767 / 1500))
desired_size = int(2 ** (random.random() * 32767 / 1500))
print("Testing {} bytes with a chunk size of {}".format(bytes_to_read, desired_size))
with open('/dev/urandom', 'rb') as handle:
content = handle.read(bytes_to_read)
input_handle = io.BytesIO(content)
output = io.BytesIO()
block = b''
for chunk_handle in chunk(input_handle, limit=desired_size, delimiter=b'\n'):
# check previous block ends with nl (it's mandatory for every one except the last)
if block:
assert block[-1:] == b'\n'
assert len(block) >= desired_size
returned_less = False
read_blocks = []
while True:
read_size = int(2**(random.random()*22))
current_block = chunk_handle.read(read_size)
read_blocks.append(current_block)
assert len(current_block) <= read_size
if current_block:
# if something was returned, the previous one should have been exact
assert not returned_less
else:
break
returned_less = len(current_block) < read_size
block = b''.join(read_blocks)
assert block.rfind(b'\n', 0, -1) <= desired_size
output.write(block)
out = output.getvalue()
assert content == out, "Compared {} processed bytes to {} original bytes".format(
len(out), len(content)
)
def main():
script, filename, lookaside, lookahead = sys.argv
lookaside = int(lookaside)
lookahead = int(lookahead)
# print "read filter file"
# w is an Nn x Ni ndarray of weights
w = read_filters.read_filters(filename)
# print "break into chunks"
# chunks is a list of Nrows * Tn * Ti weights
(chunks, chunk_idxs) = chunk.chunk(w)
# print "processing each chunk"
for c in chunks:
process_weights(c, lookaside, lookahead)
# print "cycles = ", float(total_reduced_rows)/total_rows
cols = (filename, lookaside, lookahead, total_reduced_rows, total_rows)
for c in cols:
print str(c) + ",",
def main():
script, filename, lookaside, lookahead = sys.argv
lookaside = int(lookaside)
lookahead = int(lookahead)
# print "read filter file"
# w is an Nn x Ni ndarray of weights
w = read_filters.read_filters(filename)
# print "break into chunks"
# chunks is a list of Nrows * Tn * Ti weights
(chunks, chunk_idxs) = chunk.chunk(w)
# print "processing each chunk"
for c in chunks:
process_weights(c, lookaside, lookahead)
# print "cycles = ", float(total_reduced_rows)/total_rows
cols = (filename, lookaside, lookahead, total_reduced_rows, total_rows)
for c in cols:
print str(c) +",",
def getChunks(self, window, overlap, event_label):
try:
if (self.events_table == []):
print("[*]-NO EVENTS IN THIS SESSION")
return []
else:
size_signals = np.shape(self.data)
start = 0
cont = 0
end = start + window
array_chunks = chunk_list.chunk_list()
while end < size_signals[0]:
chunk_data = self.data.ix[start:end - 1, :]
for event in self.events_table:
if (start >= (event.getTimeStart_in_values(self.fm))
and end <
(event.getTimeStart_in_values(self.fm) +
event.getDuration_in_values(self.fm))):
if (event.getLabel() == event_label):
#Es un chunk del tipo event que buscamos
object_chunk = chunk.chunk(
chunk_data, self.file_name, event, self.fm,
self.channels_selected)
array_chunks.Add(
object_chunk
) #Aqui no esta bien <= PROBLEMA !!!!!
cont = cont + 1
else:
pass
else:
pass
start = end - overlap
end = start + window
return array_chunks
except Exception as e:
print("[*]-ERROR TO CREATE CHUNKS => " + str(e))
def __init__(self):
self.head = chunk.chunk()
self.tail = None
self.nb_tail = 0
self.spare = None
codon changes as dict: {'ACT': 'CTG'}, argument "shift" to allow
dinucleotide an bicodon bias investigation?
build on this function: is change synonymous? codon bias change
note that we have to deal w/ case where neiboring positions in same codon
best use some sliding window approach, asking every time is this position
> theshold
'''
pass
'''
Write out feature importance.
'''
a = list(chunk(4, clf.feature_importances_))
df = pd.DataFrame.from_records(a)
df.columns = 'A C T G'.split(' ')
df.unstack().reset_index().to_csv('.../fi_NA.csv', header=None, index=False)
'''
Write out entropy.
'''
s = pd.Series(entropy)
s.to_csv('.../entropy_NA.csv', index=True, header=None)
# for superheat, see below
# pd.DataFrame.from_records(a).to_csv(
# '/Users/pi/tmp/fi_squares.csv', header=None, index=False)
''' R
library(ggplot2)
# library(superheat) # https://github.com/rlbarter/superheat
if reuse_cycle[rc] != reuse_cycle[rc+1]:
diff = reuse_cycle[rc+1] - reuse_cycle[rc]
diff_list.append(diff)
# print ki, n_list
# print reuse_cycle
# glob_dups[k] = [n for (c,n) in sorted(zip(reuse_cycle,n_list), key=lambda pair: pair[0])]
#print "mean buffer time", sum(diff_list)/float(len(diff_list))
for key in glob_dups:
# print key, len(glob_dups[key]), glob_dups[key]
total_dups += len(glob_dups[key])-1
# print key, glob_dups[key]
#print "break into chunks"
# chunks is a list of Nrows * Tn * Ti weights
(chunks, chunk_idxs) = chunk.chunk(w,Nn,Ni,Tnn,Tii,Tn,Ti)
#print "processing each chunk"
np.set_printoptions(threshold=np.inf)
for (c, c_idx) in zip(chunks, chunk_idxs):
process_weights(c, c_idx, lookaside, lookahead, out_limit, in_limit)
left=0
for tsets in buffer:
for tways in tsets:
for key in tways:
for n in tways[key]:
#print "left ", i, n
left += 1
# print "leftover duplicates =", left
'''
8. Assess.
'''
# http://machinelearningmastery.com/feature-importance-and-feature-selection-with-xgboost-in-python/
# clf.fit(X_train, y_train)
# clf.feature_importances_
# np.argsort(clf.feature_importances_)[::-1]
# y_true = y_test
# y_pred = clf.predict(X_test)
clf.feature_importances_ # overall
chunks = chunk(4, clf.feature_importances_)
feat_imp = ['%.3f' % round(sum(i), 3) for i in chunks] # stackoverflow, 56820
feat_ent = entropy[selection]
fp_out = '.../feat_importance_entropy.tsv'
with open(fp_out, 'w+') as outfile:
outfile.write('{}\t{}\n'.format('entropy', 'importance'))
for i in zip(feat_ent, feat_imp):
outfile.write('{}\t{}\n'.format(i[0], i[1]))
def generateChunks(self):
for x in range(0, 15):
for y in range(0, 15):
self.chunk_array[x][y] = chunk(x, y, self.spnoise)
return self.chunk_array
# This program was written in comic sans
import os, sys, pygame
from pygame.locals import *
from path import path
from screen import clock, screen
from pics import rock, timgs
from units import massx, massy
# getting the world
from chunk import chunk
from fake_chunk import fake_chunk
import time
real = chunk()
screen_rect = screen.get_rect()
camera = screen_rect.copy()
dx = dy = 0
black = Color("black")
xback = 0
yback = 0
run = False
dash = False
# starting the loop
while True:
clock.tick(30)
for event in pygame.event.get():
def __init__(self):
self.right = chunk.chunk()
self.left = chunk.chunk()
self.middle = None
def __init__(self):
self.real = chunk(world1)
self.img = pygame.Surface((massx, massy - scale * 2))
self.pos = (0, scale * 2)
self.img.blit(self.real.img, self.real.p_pos)
def __init__(self):
self.real = chunk(world1)
self.img = pygame.Surface((massx, massy - scale * 2))
self.pos = (0, scale * 2)
self.img.blit(self.real.img, self.real.p_pos)
#This program was written in comic sans
import os, sys, pygame
from pygame.locals import *
from path import path
from screen import clock, screen
from pics import rock, timgs
from units import massx, massy
#getting the world
from chunk import chunk
from fake_chunk import fake_chunk
import time
real = chunk()
screen_rect = screen.get_rect()
camera = screen_rect.copy()
dx = dy = 0
black = Color('black')
xback = 0
yback = 0
run = False
dash = False
#starting the loop
while True:
clock.tick(30)
for event in pygame.event.get():
