def run_compressor (filename):
with open(filename, 'rb') as uncompressed:
freqs = huffman.make_freq_table(uncompressed)
tree = huffman.make_tree(freqs)
uncompressed.seek(0)
with open(filename+'.huf', 'wb') as compressed:
util.compress(tree, uncompressed, compressed)
def use_wav(ycbcr, a_option):
Y = ycbcr[:,:,0]
Cb = ycbcr[:,:,1]
Cr = ycbcr[:,:,2]
# print 'Downsampling Cb and Cr...'
Cb = inter_resample(Cb, 2)
Cr = inter_resample(Cr, 2)
print 'Encoding Y...'
waves = get_wavelets(Y, thres_scale=0.65, no_a=a_option)
eY = encode_wavelets(waves)
print 'Encoding Cb...'
waves = get_wavelets(Cb, thres_scale=4.0)
eCb = encode_wavelets(waves)
print 'Encoding Cr...'
waves = get_wavelets(Cr, thres_scale=4.0)
eCr = encode_wavelets(waves)
pre = binary_int_byte(len(eY)) + binary_int_byte(len(eCb))
a = bitarray.bitarray('',endian='big')
a.frombytes(pre)
uncompressed = a + eY + eCb + eCr
print 'Compressing...'
once = util.compress(uncompressed);
twice = util.compress(once);
return twice
def huffman(infile):
with open(infile, 'rb') as uncompressed:
freqs = make_freq_table(uncompressed)
tree = make_tree(freqs)
uncompressed.seek(0)
with open(infile + '.huf', 'wb') as compressed:
util.compress(tree, uncompressed, compressed)
def run_compressor (filename):
with open(filename, 'rb') as uncompressed:
freqs = huffman.make_freq_table(uncompressed)
tree = huffman.make_tree(freqs)
uncompressed.seek(0)
with open(filename+'.huf', 'wb') as compressed:
util.compress(tree, uncompressed, compressed)
def run_compressor(filename):
# opening file to read
with open(filename, 'rb') as uncompressed:
# makefreqtable reads the whole file
freqs = huffman.make_freq_table(uncompressed)
tree = huffman.make_tree(freqs)
# reinitializing the file 'editing cursor' back to
# the start of the file
uncompressed.seek(0)
with open(filename + '.huf', 'wb') as compressed:
# adding tree_stream as a parameter...????????
util.compress(tree, uncompressed, compressed)
def mse_compr(y, wav='db3',lev=4):
wp = pywt.WaveletPacket2D(data=x, wavelet=wav, maxlevel=lev, mode='sym')
wps = wp.get_level(lev)
dec = map(lambda x: x.data, wps)
paths = map(lambda x: x.path, wps)
data = np.vstack(dec)
s = np.std(data)
wp2 = pywt.WaveletPacket2D(data=None, wavelet=wav, maxlevel=lev, mode='sym')
thres = np.sqrt(squared_mean(y))
res = 0
uncompressed = '';
for p, d in zip(paths, dec):
dd = np.copy(d)
#if p.count("a") == 0:
# rms = np.sqrt(squared_mean(d))
# dd[abs(d - rms) < thres] = rms
# dd[abs(d + rms) < thres] = -rms
dd[abs(d) < thres] = 0
wp2[p] = dd
res += np.sum(dd != 0)
flattened = np.ndarray.flatten(dd);
for coeff in flattened:
uncompressed += binary(coeff);
# drows, dcols = np.shape(dec[0])
# uncompressed = binary(drows) + binary(dcols) + uncompressed;
compressed = util.compress(bitarray.bitarray(uncompressed));
ty = wp2.reconstruct()
return mse(y, ty), len(compressed)
def setHtml(self, html: str) -> None:
if self.fic is None:
self.fic = Fic.lookup((self.ficId, ))
html = getAdapter(FicType(self.fic.sourceId)).extractContent(
self.fic, html)
self.content = util.compress(bytes(html, 'utf-8'))
self.upsert()
def tryCompress(self, data, request=None):
if request is None:
request = self.request
if self.permVars["g"] and "gzip" in (request.getHeader("Accept-Encoding") or ""):
cdata = compress(data)
if request is self.request or len(cdata) < len(data):
request.setHeader("Content-Encoding", "gzip")
return cdata
return data
def encode(self, sound):
H_pitch, H_pitch_mag = lr.piptrack(audio,
sr=self.sr,
n_fft=self.win_len,
hop_length=self.hop_length)
features = compress(H_pitch, H_pitch_mag, n_peaks=16)
return features
def sendBroadcast(self):
try:
d = marshal.dumps(self.value)
except Exception:
d = cPickle.dumps(self.value, -1)
d = compress(d)
self.bytes = len(d)
f = open(self.path, 'wb')
f.write(d)
f.close()
logger.debug("dump to %s", self.path)
def exec_chain_function(c, p, ret, obj, pp):
exec("import " + obj.split('.')[0]) # import the object
exec('cc=' + obj + '(' + pp + ')'
) # create the object with given parameters
h = get_host(cc)
print "connecting: ", h[0], h[1]
con = rpyc.classic.connect(h[0], int(h[1]))
r = send_function(con, cc)
if ret != "cla": # cannot compress the classifier
p[ret] = r
else:
p[ret] = compress(r)
def blockifyObject(self, obj):
try:
buf = marshal.dumps(obj)
except Exception:
buf = cPickle.dumps(obj, -1)
N = self.BlockSize
blockNum = len(buf) / N + 1
val = [BroadcastBlock(i, compress(buf[i*N:i*N+N]))
for i in range(blockNum)]
vi = VariableInfo(val, blockNum, len(buf))
vi.has_blocks = blockNum
return vi
def addFile(parent, name, sourcePath):
# print("'{}' -> '{}'".format(name, sourcePath))
fileObj = parent.findChild(name)
if fileObj is not None:
raise KeyError("Directory '%s' already contains file '%s'" % (parent.path(), name))
fileObj = FWFS.File(parent, name)
cfg.applyRules(fileObj)
fileObj.mtime = os.path.getmtime(sourcePath)
with open(sourcePath, "rb") as fin:
din = fin.read()
ext = os.path.splitext(name)[1]
if ext in ['.json', '.jsonc']:
dout = json.dumps(json.loads(jsmin(din).decode()), separators=(',', ':')).encode()
elif ext in ['.js']:
dout = jsmin(din)
else:
dout = din
fin.close()
# If rules say we should compress this file, give it a go
cmp = fileObj.findObject(FwObt.Compression)
if not cmp is None:
if cmp.compressionType() == CompressionType.gzip:
# print("compressing '" + fileObj.path() + '"')
dcmp = util.compress(dout)
if len(dcmp) < len(dout):
cmp.setOriginalSize(len(dout))
dout = dcmp
else:
# File is bigger, leave uncompressed
fileObj.removeObject(cmp)
else:
print("Unsupported compression type: " + cmp.toString())
sys.exit(1)
fileObj.appendData(dout)
# If required, write copy of generated file
if outFilePath is not None:
path = outFilePath + fileObj.path()
if args.verbose and logfile:
logfile.write("Writing '" + path + "'\n")
util.mkdir(os.path.dirname(path))
with open(path, "wb") as tmp:
tmp.write(dout)
tmp.close()
return fileObj
def write_index_file(self):
with open(conf.word_set_path + ".decompress", 'wb') as out_file:
out_file.write(pickle.dumps(self.word_set))
with open(conf.word2id_map_path + ".decompress", 'wb') as out_file:
out_file.write(pickle.dumps(self.word2id_map))
with open(conf.index_path + ".decompress", 'wb') as out_file:
out_file.write(pickle.dumps(self.index))
with open(conf.doc_length_path + ".decompress", 'wb') as out_file:
out_file.write(pickle.dumps(self.doc_length))
with open(conf.D_path + ".decompress", 'wb') as out_file:
out_file.write(pickle.dumps(self.D))
util.compress(conf.word_set_path + ".decompress", conf.word_set_path)
util.compress(conf.word2id_map_path + ".decompress",
conf.word2id_map_path)
util.compress(conf.index_path + ".decompress", conf.index_path)
util.compress(conf.doc_length_path + ".decompress",
conf.doc_length_path)
util.compress(conf.D_path + ".decompress", conf.D_path)
os.remove(conf.word_set_path + ".decompress")
os.remove(conf.word2id_map_path + ".decompress")
os.remove(conf.index_path + ".decompress")
os.remove(conf.doc_length_path + ".decompress")
os.remove(conf.D_path + ".decompress")
print("Write index to file successfully.")
def _memory_file_insert(instance_uuid: str, header: dict, data: bytes,
collection: str):
"""
Inserts a new document as a compressed file into the database. Header will be saved as metadata.
:param data: The data (bytes) to be compressed and inserted.
:param instance_uuid: The instance_id to which this record belongs to.
:param header: Header is data to identify the file. It will be saved as metadata.
:return: Tuple with (File Object ID, File name).
"""
assert (type(data) is
bytes), "For file compression and saving, data should be bytes."
compressed_data = util.compress(data)
fs = get_grid_fs()
file_name = collection + "_" + str(instance_uuid) + ".snappy"
file_id = fs.put(compressed_data, filename=file_name, metadata=header)
return file_id
def encode_cartoon(rgb):
height, width = rgb[:, :, 0].shape
ycrcb = rgb_to_ycbcr(rgb)
trimmed, low, high = quantize(ycrcb.flatten(), 32)
bytea = bytearray(np.int8(trimmed))
bitar = bitarray.bitarray('')
bb = binary_int_byte(height) + binary_int_byte(width) + \
binary_float_byte(low) + binary_float_byte(high) + \
str(bytea)
bitar.frombytes(bb)
return util.compress(bitar)
def blockifyObject(self, obj):
try:
buf = marshal.dumps(obj)
except Exception:
buf = cPickle.dumps(obj, -1)
buf = compress(buf)
N = self.BlockSize
blockNum = len(buf) / N
if len(buf) % N != 0:
blockNum += 1
val = [BroadcastBlock(i/N, buf[i:i+N])
for i in range(0, len(buf), N)]
vi = VariableInfo(val, blockNum, len(buf))
vi.has_blocks = blockNum
return vi
def download_chapters(novel_path, session, chapters):
for i, (title, url) in enumerate(chapters):
print(f"downloading chapter {i}")
soup = get_soup(session, url)
title = title if title else f"Chapter {i}"
body = get_body(soup) + get_pages(session, soup, url)
filename = f"{i}".zfill(5) + ".json"
volume = max([1, 1 + (i - 1) // 100])
filepath = novel_path.joinpath(str(volume), filename)
filepath.parent.mkdir(parents=True, exist_ok=True)
jdata = {
"body": compress(body),
'chapter_no': i,
'chapter_title': title
}
with filepath.open('w+', encoding='utf-8') as f:
json.dump(jdata, f, separators=(',', ':'))
def createTask(self, o, job, t):
task = mesos_pb2.TaskInfo()
tid = "%s:%s:%s" % (job.id, t.id, t.tried)
task.name = "task %s" % tid
task.task_id.value = tid
task.slave_id.value = o.slave_id.value
task.data = compress(cPickle.dumps((t, t.tried), -1))
task.executor.MergeFrom(self.executor)
if len(task.data) > 1000 * 1024:
logger.warning("task too large: %s %d", t, len(task.data))
cpu = task.resources.add()
cpu.name = "cpus"
cpu.type = 0 # mesos_pb2.Value.SCALAR
cpu.scalar.value = t.cpus
mem = task.resources.add()
mem.name = "mem"
mem.type = 0 # mesos_pb2.Value.SCALAR
mem.scalar.value = t.mem
return task
def createTask(self, o, job, t):
task = mesos_pb2.TaskInfo()
tid = "%s:%s:%s" % (job.id, t.id, t.tried)
task.name = "task %s" % tid
task.task_id.value = tid
task.slave_id.value = o.slave_id.value
task.data = compress(cPickle.dumps((t, t.tried), -1))
task.executor.MergeFrom(self.executor)
if len(task.data) > 1000 * 1024:
logger.warning("task too large: %s %d", t, len(task.data))
cpu = task.resources.add()
cpu.name = 'cpus'
cpu.type = 0 #mesos_pb2.Value.SCALAR
cpu.scalar.value = t.cpus
mem = task.resources.add()
mem.name = 'mem'
mem.type = 0 #mesos_pb2.Value.SCALAR
mem.scalar.value = t.mem
return task
def compress_image(image, window, cutoff, output=OUTPUT_COMPRESSED_IMAGE):
gray = util.im2gray(image)
cropped = util.crop(gray, window)
shaped = util.blockshaped(cropped, window)
dct = fft.dctn(shaped, axes=[1, 2], type=2, norm='ortho')
compressed, mask = util.compress(dct, cutoff)
idct = fft.idctn(compressed, axes=[1, 2], type=2, norm='ortho')
info = np.iinfo(image.dtype)
to_range = (info.min, info.max)
if output == OUTPUT_DCT:
out = dct
from_range = (out.min(), out.max())
elif output == OUTPUT_MASK:
out = mask
from_range = (out.min(), out.max())
elif output == OUTPUT_COMPRESSED_DCT:
out = compressed
from_range = (out.min(), out.max())
elif output == OUTPUT_COMPRESSED_IMAGE:
out = idct
from_range = to_range # No interpolation needed, just clipping
else:
raise ValueError("Invalid value for 'output' parameter.")
# If min == max, min should map to 0
if from_range[0] == from_range[1]:
from_range = (from_range[0], from_range[1] + 1)
normalized = np.interp(out, from_range, to_range)
converted = np.round(normalized).astype(image.dtype)
if output == OUTPUT_MASK:
result = converted
else:
result = util.unblockshaped(converted, cropped.shape)
return result
def saveWebRequest(created: int,
url: str,
status: int,
response: Optional[str],
source: str = None) -> None:
#import psycopg2
responseBytes: Optional[bytes] = None
if response is not None:
responseBytes = util.compress(response.encode('utf-8'))
global __scrapeSource
if source is None:
source = __scrapeSource
conn = openMinerva()
curs = conn.cursor()
curs.execute(('insert into web(created, url, status, response, source)' +
'values(%s, %s, %s, %s, %s)'),
(created, url, status, responseBytes, source))
curs.close()
closeMinerva()
def run(self, attempId):
logger.debug("shuffling %d of %s", self.partition, self.rdd)
numOutputSplits = self.partitioner.numPartitions
getPartition = self.partitioner.getPartition
mergeValue = self.aggregator.mergeValue
createCombiner = self.aggregator.createCombiner
buckets = [{} for i in range(numOutputSplits)]
for k, v in self.rdd.iterator(self.split):
bucketId = getPartition(k)
bucket = buckets[bucketId]
r = bucket.get(k, None)
if r is not None:
bucket[k] = mergeValue(r, v)
else:
bucket[k] = createCombiner(v)
for i in range(numOutputSplits):
path = LocalFileShuffle.getOutputFile(self.shuffleId,
self.partition, i)
if os.path.exists(path):
continue
tpath = path + ".%s.%s" % (socket.gethostname(), os.getpid())
if marshalable(buckets[i]):
flag, d = 'm', marshal.dumps(buckets[i])
else:
flag, d = 'p', cPickle.dumps(buckets[i], -1)
cd = compress(d)
f = open(tpath, 'wb', 1024 * 4096)
f.write(flag + struct.pack("I", 5 + len(cd)))
f.write(cd)
f.close()
if not os.path.exists(path):
os.rename(tpath, path)
else:
os.unlink(tpath)
return LocalFileShuffle.getServerUri()
def run(self, attempId):
logger.debug("shuffling %d of %s", self.partition, self.rdd)
numOutputSplits = self.partitioner.numPartitions
getPartition = self.partitioner.getPartition
mergeValue = self.aggregator.mergeValue
createCombiner = self.aggregator.createCombiner
buckets = [{} for i in range(numOutputSplits)]
for k,v in self.rdd.iterator(self.split):
bucketId = getPartition(k)
bucket = buckets[bucketId]
r = bucket.get(k, None)
if r is not None:
bucket[k] = mergeValue(r, v)
else:
bucket[k] = createCombiner(v)
for i in range(numOutputSplits):
path = LocalFileShuffle.getOutputFile(self.shuffleId, self.partition, i)
if os.path.exists(path):
continue
tpath = path + ".%s.%s" % (socket.gethostname(), os.getpid())
if marshalable(buckets[i]):
flag, d = 'm', marshal.dumps(buckets[i])
else:
flag, d = 'p', cPickle.dumps(buckets[i], -1)
cd = compress(d)
f = open(tpath, 'wb', 1024*4096)
f.write(flag + struct.pack("I", 5 + len(cd)))
f.write(cd)
f.close()
if not os.path.exists(path):
os.rename(tpath, path)
else:
os.unlink(tpath)
return LocalFileShuffle.getServerUri()
def save(self, filename):
self.archive_memory()
with open(filename, 'wb') as f:
f.write(compress(self))
def save_data(data):
print('start save_data')
with open(config.FILE, 'w') as fd: json.dump(list(data), fd, ensure_ascii=False, separators=(',', ':'))
print('end save_data')
data = load_data()
while True:
t0 = time.time()
videos = util.trending_videos(api_key, regionCode='RU')
published_at = [v.pop('publishedAt') for v in videos]
videos = util.compress(videos, config.COMPRESS_SCHEMA)
now = datetime.datetime.now(tz=config.TIMEZONE)
timestamp = int(now.timestamp())
data.append([timestamp, videos])
data = util.drop_old(data)
save_data(data)
util.plot(data, published_at)
gc.collect()
t_elapsed = time.time() - t0
print(now, f'elapsed in {t_elapsed}')
sleep_time = config.TIME_LAG - t_elapsed
if sleep_time > 0:
print(f'sleep for {sleep_time}')
time.sleep(sleep_time)
from pathlib import Path
name = '303_bach'
here = Path(__file__).resolve().parent.parent
samples_dir = here.joinpath('samples')
output_dir = here.joinpath('output')
audio, sr = sfio.load(str(samples_dir.joinpath('%s.mp3' % name)))
print(audio.shape)
n_fft = 2048
hop_length = n_fft / 4
H_pitch, H_pitch_mag = lr.piptrack(audio,
sr=sr,
n_fft=n_fft,
hop_length=hop_length)
features = compress(H_pitch, H_pitch_mag, n_peaks=16)
print("features.shape=", features.shape)
recon = reconstruct(features, n_fft=n_fft, sr=sr, hop_length=hop_length)
audio = sfio.save(str(output_dir.joinpath('%s_reconstructed.mp3' % name)),
recon,
sr=sr)
def contour(self, solution, params):
"""
how well does the solo use contour? This includes an evaluation
of intervallic diversity: in general, we want a good mix of half
steps and whole steps, along with larger leaps of 3rds-octaves. When
considering just a stream of eighth notes (a bebop line), we penalize
intervalls larger than octaves, and "too many" consecutive large
(third or larger) intervalls
"""
if len(solution) < 2:
return 0
score = 0
sol = util.make_pitches(solution)
up = "up"
down = "down"
same = "same"
intervals = [] # intervals
directions = [] # recent interval direction (up or down)
# build intervals and directions from solution
for i in [n + 1 for n in range(len(solution) - 1)]:
intervals.append(sol[i] - sol[i - 1])
if sol[i] < sol[i - 1]:
directions.append(down)
elif sol[i] > sol[i - 1]:
directions.append(up)
else:
directions.append(same)
abs_intervals = [abs(n) for n in intervals]
# incentivize varied contour and intervals
if len(solution) >= 5:
score += params["interval_variety"][len(
util.compress(abs_intervals))]
score += params["direction_variety"][len(
util.compress(directions))]
# slightly incentivize small intervals over large ones
score += np.dot([abs_intervals.count(i + 1) for i in range(12)],
params["interval_weights"])
# slight penalization for repeating notes
score += params["same"][directions.count(same) - 1]
# disincentivize leaps larger than an octave
for interval in abs_intervals:
if interval in theory.large_leap:
score += params["large_leap"]
# incentivize a downward or upward line
pitch_diff = sum(intervals)
if pitch_diff > 12:
score += params["line"][0]
elif pitch_diff > 7:
score += params["line"][1]
elif pitch_diff > 3:
score += params["line"][2]
return score
final = main.compress_image(data, windowsize, threshold)
plt.subplot(1, 2, 1)
plt.imshow(img, cmap='gray', vmin=0, vmax=255)
plt.subplot(1, 2, 2)
plt.imshow(final, cmap='gray', vmin=0, vmax=255)
plt.show()
################## TEST ##################
prova_blocco = np.array([[[231, 32, 233, 161, 24, 71, 140, 245],
[247, 40, 248, 245, 124, 204, 36, 107],
[234, 202, 245, 167, 9, 217, 239, 173],
[193, 190, 100, 167, 43, 180, 8, 70],
[ 11, 24, 210, 177, 81, 243, 8, 112],
[ 97, 195, 203, 47, 125, 114, 165, 181],
[193, 70, 174, 167, 41, 30, 127, 245],
[ 87, 149, 57, 192, 65, 129, 178, 228]]])
prova_c = dctn(prova_blocco, type=2, norm='ortho')
prova_compressed = util.compress(prova_c, threshold=5)
prova_ff = idctn(prova_compressed, type=2, norm='ortho')
prova_normalized_ff = np.clip(np.round(prova_ff), 0, 255)
def archive(self):
self._finalize()
self.archived = True
return compress(self)