def anadir(self, widget):
Selector = gtk.FileChooserDialog(
_("Seleccione un archivo"), None, gtk.FILE_CHOOSER_ACTION_OPEN,
(gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL, gtk.STOCK_OK,
gtk.RESPONSE_OK))
Selector.set_default_response(gtk.RESPONSE_CANCEL)
if utils.Archivo_d != None:
Echo = Selector.run()
if Echo == gtk.RESPONSE_OK:
File = Selector.get_filename()
print 'ARCHIVO'
print File
utils.compress(File, utils.Archivo_d)
print 'AÑADIDO'
utils.decompress(utils.Archivo_d, Hboxx)
print 'DESCOMPRIMIDO'
Selector.destroy()
elif Echo == gtk.RESPONSE_CANCEL:
Selector.destroy()
if utils.Archivo_d == None:
self.errores('Copiar_a_nada')
os.chdir('/tmp/Compress/Work')
self.set_model(self.construir_lista())
Uri.set_text(os.getcwd())
Uri.show()
def descomprimir_final(self, widget, dire):
utils.decompress(utils.Archivos, Hboxx, dire, 'Descomprimir')
Hboxx.hide()
os.chdir(dire)
Uri.set_text(os.getcwd())
Uri.show()
self.set_model(self.construir_lista())
def download_file(url, save_path, override, chunksize=65535):
downloaded = 0
if os.path.exists(save_path) and override:
logger.info(save_path + ' will be overridden.')
elif os.path.exists(save_path):
# Resume download
downloaded = os.stat(save_path).st_size
local_timestamp = os.path.getctime(save_path)
logger.info("{} already exists. Send resume request after {} bytes".format(
save_path, downloaded))
# raise FileExistsError()
old_downloaded = downloaded
headers = {}
if downloaded:
headers['Range'] = 'bytes={}-'.format(downloaded)
headers['If-Unmodified-Since'] = timestamp2time(local_timestamp)
logger.info(url + ' is saved as ' + save_path)
res = requests.get(url, headers=headers, stream=True, timeout=15)
mode = 'wb+'
content_len = int(res.headers.get('content-length'))
# Check if server supports range feature, and works as expected.
if res.status_code == 206:
# Content range is in format `bytes 327675-43968289/43968290`, check
# if it starts from where we requested.
content_range = res.headers.get('content-range')
# If file is already downloaded, it will reutrn `bytes */43968290`.
if content_range and \
int(content_range.split(' ')[-1].split('-')[0]) == downloaded:
mode = 'ab+'
elif res.status_code == 416:
# 416 means Range field not support
# TODO:需要重新下载吗
logger.warning("Range not support. Redownloading...")
urlretrieve(url, save_path, reporthook=reporthook4urlretrieve)
return
elif res.status_code == 412:
# 如果所请求的资源在指定的时间之后发生了修改，那么会返回 412 (Precondition Failed) 错误。
logger.warning("Resource Changed, should override. Redownloading...")
urlretrieve(url, save_path, reporthook=reporthook4urlretrieve)
return
file_origin_size = content_len + old_downloaded
with open(save_path, mode) as f:
for chunk in res.iter_content(chunksize):
f.write(chunk)
downloaded += len(chunk)
# TODO:如何显示下载进度
flush_print('Downloading %s %.2f%%: %d | %d' % (save_path, downloaded / file_origin_size * 100,
downloaded, file_origin_size))
# urlretrieve(url, save_path, reporthook=reporthook4urlretrieve)
print()
decompress(save_path)
def download_file(url, save_path, override):
if os.path.exists(save_path) and override: # pragma: no cover
os.remove(save_path)
logger.info(save_path + ' will be overridden.')
logger.info(url + ' is saved as ' + save_path)
urlretrieve(url, save_path, reporthook=reporthook4urlretrieve)
print()
decompress(save_path)
def test_compress_decompress(teardown_env, filesize):
generate_file(filesize=filesize)
compress('_test')
decompress('_test_decomp')
source_file_data = get_content('_test')
dest_file_data = get_content('_test_decomp')
assert source_file_data == dest_file_data
def download_file(url, save_path, override):
if os.path.exists(save_path) and override:
logger.info(save_path + ' will be overridden.')
elif os.path.exists(save_path):
raise FileExistsError()
logger.info(url + ' is saved as ' + save_path)
urlretrieve(url, save_path, reporthook=reporthook4urlretrieve)
print()
decompress(save_path)
def open(self, widget, clickeado=False): # Abrir
if not clickeado:
try:
utils.decompress(utils.Archivos, Hboxx, '/tmp/Compress/Work',
'Abrir')
utils.Archivo_d = Uri.get_text() + "/" + utils.Archivos
os.chdir('/tmp/Compress/Work')
except:
utils.Abrir(Hboxx, entrada)
if clickeado:
utils.Abrir(Hboxx, entrada)
Uri.set_text(os.getcwd())
Uri.show()
self.set_model(self.construir_lista())
def search(cls, words):
doc_array = None
words = [to_ascii(w).lower() for w in words]
for word in words:
kw = Keyword.get(word, readonly=True)
if not kw:
return
if doc_array is None:
doc_array = kw.documents
else:
doc_array &= kw.documents
limit, offset = limit_offset()
ids = list(ranks(doc_array))
ctx.cursor.execute(
'SELECT id, uri, content from document ' \
'WHERE content is not null and id in (%s) ' \
'ORDER BY score desc limit %s offset %s' % (
','.join(str(i) for i in ids), limit, offset)
)
for doc_id, uri, content in ctx.cursor:
match = None
for line in decompress(content).splitlines():
idx = to_ascii(line).lower().find(words[0])
if idx < 0:
continue
match = get_match_context(idx, line)
break
yield doc_id, uri, match, content
def copy(self, widget, W=False):
try:
self.Copiado = utils.Archivos
if not W:
try:
shutil.copy(os.getcwd() + "/" + utils.Archivos,
'/tmp/Compress/')
except:
shutil.copytree(os.getcwd() + "/" + utils.Archivos,
'/tmp/Compress/' + utils.Archivos)
if W:
try:
utils.compress("./" + utils.Archivos, utils.Archivo_d)
utils.decompress(utils.Archivo_d, Hboxx)
except:
self.errores('Copiar_a_nada')
except:
self.errores('Copiar')
def by_id(cls, doc_id):
ctx.cursor.execute('SELECT content '
'FROM document WHERE id = ?', (doc_id, ))
res = next(ctx.cursor, None)
if res is None:
return None
content, = res
if content is not None:
content = decompress(content)
return content
def open_diario(self, widget): # Abrir un zip desde el diario.
chooser = ObjectChooser()
resultado = chooser.run()
if resultado == gtk.RESPONSE_ACCEPT:
Archivo_s = chooser.get_selected_object()
Archivo = Archivo_s.get_file_path()
try:
print Archivo
utils.Archivo_d = Archivo
utils.decompress(Archivo, Hboxx, '/tmp/Compress/Work', 'Abrir',
True)
except IOError:
Mensaje = _("""Oh..Parece que no selecciono un archivo zip""")
info = gtk.MessageDialog(type=gtk.MESSAGE_WARNING,
buttons=gtk.BUTTONS_OK,
message_format=Mensaje)
a = info.run()
if a == gtk.RESPONSE_OK:
info.destroy()
self.set_model(self.construir_lista())
Uri.set_text(os.getcwd())
def create_word_embedding(infile, outfile, min_count, size, downsample, estimator, negative):
def run_on(actual_infile):
assert estimator in ['skipgram', 'cbow']
use_cbow = estimator == 'cbow'
sample = 10e-3
if not downsample:
sample = 1
subprocess.run([ 'tools/word2vec/word2vec' \
, '-train', actual_infile \
, '-output', outfile \
, '-min-count', str(min_count) \
, '-size', str(size) \
, '-sample', str(sample) \
, '-negative', str(negative) \
, '-cbow', str(int(use_cbow)) ])
compress_flag = False
outfilep = PurePath(outfile)
if outfilep.suffix == '.gz':
outfile = str(outfilep.parent / outfilep.stem)
compress_flag = True
infilep = PurePath(infile)
if infilep.suffix == '.gz':
rndstr = ''.join(random.choice(list(string.ascii_uppercase + string.digits)) for _ in range(10))
tmp_filename = '/tmp/' + infilep.stem + rndstr
try:
decompress(infile, keep=True, outfile=tmp_filename)
run_on(tmp_filename)
finally:
if os.path.exists(tmp_filename):
os.remove(tmp_filename)
else:
run_on(infile)
if compress_flag:
compress(outfile)
def read(cls, uri):
ctx.cursor.execute(
'SELECT id, score, content, referer, distance '
'FROM document WHERE uri = ?', (uri, ))
res = next(ctx.cursor, None)
if res is None:
return None
id, score, content, referer, distance = res
if content is not None:
content = decompress(content)
return Document(id,
uri,
score=score,
content=content,
referer=referer,
distance=distance)
import os
# load files
ROOT = os.path.dirname(os.path.realpath(__file__))
clmap = {
0: 'unknown',
1: 'mini',
2: 'punk',
3: 'latin',
4: 'rock',
5: 'techno',
6: 'jazz',
7: 'electro'
}
# database
client = MongoClient('localhost', 27017)
db = client.bheat
collection = db.origset
# generate compressed numpy array for each class of beats
for k, v in clmap.iteritems():
db_beats = collection.find({'class': k})
class_seq = []
for i, b in enumerate(db_beats):
# get np_seq
np_seq = utils.decompress(b['zip'], b['bar'])
class_seq.append(np_seq)
class_seq = np.array(class_seq)
np.savez_compressed(ROOT + '/cache/'+v+'.npz', class_seq)
print('saved', v)
collection.find({
'bar': 128,
'class': 0
}, {
'_id': 1,
'theme_index': 1,
'zip': 1,
'bar': 1
}))
ids = [t['_id'] for t in c0_b]
c0_themes = []
for t in c0_b:
# get np_seq
np_seq = utils.decompress(t['zip'], t['bar'])
tt = np.copy(np_seq[t['theme_index']])
c0_themes.append(tt)
c0_themes = np.array(c0_themes)
identified = {
'unknown': [20, 21, 1, 3, 19, 22],
'mini': [5, 8, 31, 30, 33, 36],
'punk': [0, 2],
'latin': [4],
'rock': [6, 7, 10, 11, 12, 14, 15, 16, 17, 29, 39, 23, 32, 37, 38, 13],
'techno': [9, 40, 41, 42, 43, 44],
'jazz': [24, 25, 26, 27, 28, 18, 34, 35],
'electro': [45, 46]
}
labels = labels['arr_0']
# get ids for mongo
# database
client = MongoClient('localhost', 27017)
db = client.bheat
collection = db.origset
# get class 0 themes
c0_b = list(collection.find({'bar': 128, 'class': 0}, {'_id': 1, 'theme_index': 1, 'zip': 1, 'bar': 1}))
ids = [t['_id'] for t in c0_b]
c0_themes = []
for t in c0_b:
# get np_seq
np_seq = utils.decompress(t['zip'], t['bar'])
tt = np.copy(np_seq[t['theme_index']])
c0_themes.append(tt)
c0_themes = np.array(c0_themes)
identified = {
'unknown': [20, 21, 1, 3, 19, 22],
'mini': [5, 8, 31, 30, 33, 36],
'punk': [0, 2],
'latin': [4],
'rock': [6, 7, 10, 11, 12, 14, 15, 16, 17, 29, 39, 23, 32, 37, 38, 13],
'techno': [9, 40, 41, 42, 43, 44],
'jazz': [24, 25, 26, 27, 28, 18, 34, 35],
'electro': [45, 46]
}
imgBlueComponent = utils.getImageComponent('blue', image)
utils.imgDisplay("blue", imgBlueComponent)
cv2.imwrite("blue-component.jpg", imgBlueComponent)
imgGreenComponent = utils.getImageComponent('green', image)
utils.imgDisplay("green", imgGreenComponent)
cv2.imwrite("green-component.jpg", imgGreenComponent)
for m in [1, 2, 3, 4]:
# Compress the image
compressedImage = utils.compress(image, m)
np.save('image1.npy', image)
np.save('decompressedm{}.npy'.format(m), compressedImage)
decompressedImage = utils.decompress(compressedImage, m)
cv2.imwrite('decompressedm{}.bmp'.format(m), decompressedImage)
utils.imgDisplay('OriginalImage', decompressedImage)
imgRedComponent = utils.getImageComponent('red', decompressedImage)
utils.imgDisplay("red", imgRedComponent)
imgBlueComponent = utils.getImageComponent('blue', decompressedImage)
utils.imgDisplay("blue", imgBlueComponent)
imgGreenComponent = utils.getImageComponent('green', decompressedImage)
utils.imgDisplay("green", imgGreenComponent)
#find and show the psnr
psnr = 255**2
psnr = psnr / (np.square(np.subtract(image, decompressedImage)).mean())
psnr = 10 * np.log10(psnr)
print("at m=" + str(m) + " PSNR=" + str(psnr))
def test_decompress_zip(self):
test_dir = '/tmp/test_zip'
files = utils.decompress('tests/files/dummy.zip', test_dir)
self.assertEquals(files, [os.path.join(test_dir, 'dummy.csv')])
shutil.rmtree(test_dir)
def test_decompress_invalid_type(self):
utils.decompress('tests/files/dummy.csv', '/tmp/test_invalid_type')
'bar': 128,
'diversity': {
'$gt': 0.08
},
'gridicity': {
'$lt': 0.75
},
}).limit(100))
# select random
shuffle(beats)
# decompress in numpy
alll = []
for i, beat in enumerate(beats):
np_beat = utils.decompress(beat['zip'], beat['bar'])
for j, np_bar in enumerate(np_beat):
alll.append(np_bar)
alll = np.array(alll)
print "before:", alll.shape
# downsample
alll = utils.map2twelve(alll)
print "down-sampled:", alll.shape
alll = utils.clean_and_unique_beats(alll)
print "after:", alll.shape
alll_f = alll.reshape((
alll.shape[0],
alll.shape[1] * 12,
def compress_decompress():
compress('_test')
decompress('_test_decomp')
from pymongo import MongoClient
import numpy as np
import utils
import os
# load files
ROOT = os.path.dirname(os.path.realpath(__file__))
# database
client = MongoClient('localhost', 27017)
db = client.bheat
collection = db.origset
beats = collection.find({'bar': 128, 'class': {'$exists': True}})
for i, b in enumerate(beats):
# get np_seq
np_seq = utils.decompress(b['zip'], b['bar'])
theme = np_seq[b['theme_index']]
# compute drum mean
dm = theme.mean(axis=0)
dm = np.delete(dm, np.s_[15::], 0)
ntheme = None
# repare shifted beats
if b['class'] == 0 and dm[0] > 0.0:
# potentiatlly shifted beat
if theme[0][0] > 0.0:
rotate = 0
while theme[0][0] == 0.0:
rotate -= 1
theme = np.roll(theme, rotate, axis=0)
# rotated, we fix all the sequence
np_seq = np.roll(np_seq, rotate, axis=1)
def test_decompress_invalid_type(self):
utils.decompress('tests/files/dummy.csv', '/tmp/test_invalid_type')
def _get_data(self, key):
data = self._reader[key]
data = decompress(key, data)
return data.decode(self.codec, "ignore")
def test_decompress_zip(self):
test_dir = '/tmp/test_zip'
files = utils.decompress('tests/files/dummy.zip', test_dir)
self.assertEquals(files, [os.path.join(test_dir, 'dummy.csv')])
shutil.rmtree(test_dir)
def _get_data(self, key):
data = self._reader[key]
data = decompress(key, data)
return data.decode(self.codec, 'ignore')