def anadir(self, widget):
Selector = gtk.FileChooserDialog(
_("Seleccione un archivo"), None, gtk.FILE_CHOOSER_ACTION_OPEN,
(gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL, gtk.STOCK_OK,
gtk.RESPONSE_OK))
Selector.set_default_response(gtk.RESPONSE_CANCEL)
if utils.Archivo_d != None:
Echo = Selector.run()
if Echo == gtk.RESPONSE_OK:
File = Selector.get_filename()
print 'ARCHIVO'
print File
utils.compress(File, utils.Archivo_d)
print 'AÑADIDO'
utils.decompress(utils.Archivo_d, Hboxx)
print 'DESCOMPRIMIDO'
Selector.destroy()
elif Echo == gtk.RESPONSE_CANCEL:
Selector.destroy()
if utils.Archivo_d == None:
self.errores('Copiar_a_nada')
os.chdir('/tmp/Compress/Work')
self.set_model(self.construir_lista())
Uri.set_text(os.getcwd())
Uri.show()
def test_compress_decompress(teardown_env, filesize):
generate_file(filesize=filesize)
compress('_test')
decompress('_test_decomp')
source_file_data = get_content('_test')
dest_file_data = get_content('_test_decomp')
assert source_file_data == dest_file_data
def create_grams(gramconfig):
makedirs('data/grams', exist_ok=True)
corpuspath = config.corpuspath(gramconfig.corpus)
grampath = config.grampath(gramconfig)
if path.isfile(grampath):
return
uncompressed_grampath = str(Path(grampath).parent / Path(grampath).stem)
with gzip.open(corpuspath, mode='rt') as inf:
with open(uncompressed_grampath, mode='wt') as outf: # may overwrite
for line in inf:
c = gramconfig
for gram, skip in iter_grams(line, c.gram_size, c.skipwords, c.skippos, c.filter_skips):
completegram = ' '.join(chain(gram[ : c.skippos], skip, gram[c.skippos : ]))
print(completegram, file=outf)
subprocess.run(['/usr/bin/shuf', '-o', uncompressed_grampath, uncompressed_grampath])
compress(uncompressed_grampath)
def generate_trajectory(env, model, get_video=False):
"""
Generates lists of states, actions, and rewards for one complete episode.
:param env: The openai gym environment
:param model: The model used to generate the actions
:returns: A tuple of lists (states, actions, rewards), where each list has length equal to the number of timesteps in the episode
"""
# Initialize variables and states.
states = []
frames = []
actions = []
rewards = []
current_episode_reward = 0
state = env.reset()
done = False
# NOOP until green light
for _ in range(88):
env.step([0, 0, 0, 0, 0])
# Set up the break condition - runs for maximum of EPISODE_LENGTH seconds.
start_time = time.time()
while time.time() < start_time + EPISODE_LENGTH:
# Break out of loop if episode ended.
if done:
break
# Get probabilities.
state, original = compress(state, COMPRESS_FACTOR)
probabilities = model.call(
tf.expand_dims(tf.cast(state, tf.float32), axis=0))
probabilities = tf.squeeze(probabilities)
probabilities = np.reshape(probabilities.numpy(), [model.num_actions])
# Select action.
possible_actions = np.arange(model.num_actions)
action = np.random.choice(possible_actions, 1, p=probabilities)[0]
discrete_actions = DiscreteActions()
actual_action = discrete_actions.ACTION_MAP[action][1]
# Save state and apply action, take a step.
states.append(state)
if get_video:
frames.append(original)
actions.append(action)
state, rwd, done, _ = env.step(actual_action)
rewards.append(rwd)
current_episode_reward += rwd
# Output video if specified.
if get_video:
if "-lo" in sys.argv:
observe(np.array(frames), sys.argv[3])
if "-o" in sys.argv:
observe(np.array(frames), sys.argv[2])
# Return.
return states, actions, rewards
def start_encrypting(self):
"""Check if there are directory and password selected, then clear the password field,
then compress,shred and encrypt the compressed file"""
if (self.directory) and (self.password.text()):
self.captured_encrypt_pass = self.password.text()
self.password.clear()
QtTest.QTest.qWait(5 * 1000)
compress(self.directory)
shred(self.directory)
encrypt(self.captured_encrypt_pass)
alert = QMessageBox()
alert.setText('Done')
alert.exec_()
else:
alert = QMessageBox()
alert.setText('Choose a directory and password first')
alert.exec_()
def allc_count_context_worker_wrap(
input_allc_file,
output_file,
compress=True,
overwrite=False,
dirname=DIRNAME, # package directory
):
"""
"""
logging.info("processing: {}".format(input_allc_file))
if not overwrite:
if os.path.isfile(output_file) or os.path.isfile(output_file+'.gz') or os.path.isfile(output_file+'.bgz'):
logging.info("File exists "+output_file+", skipping...")
return 0
sp.run([os.path.join(dirname, "_allc_count_contexts_worker.sh"), input_allc_file, output_file])
if compress:
utils.compress(output_file)
logging.info("Done. Results saved to {}".format(output_file))
return
def gzip(response, level=1, mime_types=['text/html', 'text/plain']):
"""Try to gzip the response body if Content-Type in mime_types.
response.headers['Content-Type'] must be set to one of the
values in the mime_types arg before calling this function.
No compression is performed if any of the following hold:
* The client sends no Accept-Encoding request header
* No 'gzip' or 'x-gzip' is present in the Accept-Encoding header
* No 'gzip' or 'x-gzip' with a qvalue > 0 is present
* The 'identity' value is given with a qvalue > 0.
"""
if not response.body:
# Response body is empty (might be a 304 for instance)
return response
# If returning cached content (which should already have been gzipped),
# don't re-zip.
if getattr(response.request, "cached", False):
return response
acceptable = response.request.headers.elements('Accept-Encoding')
if not acceptable:
# If no Accept-Encoding field is present in a request,
# the server MAY assume that the client will accept any
# content coding. In this case, if "identity" is one of
# the available content-codings, then the server SHOULD use
# the "identity" content-coding, unless it has additional
# information that a different content-coding is meaningful
# to the client.
return response
ct = response.headers.get('Content-Type', 'text/html').split(';')[0]
for coding in acceptable:
if coding.value == 'identity' and coding.qvalue != 0:
return response
if coding.value in ('gzip', 'x-gzip'):
if coding.qvalue == 0:
return response
if ct in mime_types:
# Return a generator that compresses the page
varies = response.headers.get("Vary", "")
varies = [x.strip() for x in varies.split(",") if x.strip()]
if "Accept-Encoding" not in varies:
varies.append("Accept-Encoding")
response.headers['Vary'] = ", ".join(varies)
response.headers['Content-Encoding'] = 'gzip'
response.body = compress(response.body, level)
if response.headers.has_key("Content-Length"):
# Delete Content-Length header so finalize() recalcs it.
del response.headers["Content-Length"]
return response
return HTTPError(response.request, response, 406, "identity, gzip")
def gzip(response, level=1, mime_types=["text/html", "text/plain"]):
"""Try to gzip the response body if Content-Type in mime_types.
response.headers['Content-Type'] must be set to one of the
values in the mime_types arg before calling this function.
No compression is performed if any of the following hold:
* The client sends no Accept-Encoding request header
* No 'gzip' or 'x-gzip' is present in the Accept-Encoding header
* No 'gzip' or 'x-gzip' with a qvalue > 0 is present
* The 'identity' value is given with a qvalue > 0.
"""
if not response.body:
# Response body is empty (might be a 304 for instance)
return response
# If returning cached content (which should already have been gzipped),
# don't re-zip.
if getattr(response.request, "cached", False):
return response
acceptable = response.request.headers.elements("Accept-Encoding")
if not acceptable:
# If no Accept-Encoding field is present in a request,
# the server MAY assume that the client will accept any
# content coding. In this case, if "identity" is one of
# the available content-codings, then the server SHOULD use
# the "identity" content-coding, unless it has additional
# information that a different content-coding is meaningful
# to the client.
return response
ct = response.headers.get("Content-Type", "text/html").split(";")[0]
for coding in acceptable:
if coding.value == "identity" and coding.qvalue != 0:
return response
if coding.value in ("gzip", "x-gzip"):
if coding.qvalue == 0:
return response
if ct in mime_types:
# Return a generator that compresses the page
varies = response.headers.get("Vary", "")
varies = [x.strip() for x in varies.split(",") if x.strip()]
if "Accept-Encoding" not in varies:
varies.append("Accept-Encoding")
response.headers["Vary"] = ", ".join(varies)
response.headers["Content-Encoding"] = "gzip"
response.body = compress(response.body, level)
if response.headers.has_key("Content-Length"):
# Delete Content-Length header so finalize() recalcs it.
del response.headers["Content-Length"]
return response
return HTTPError(response.request, response, 406, "identity, gzip")
def save(self, *args, **kwargs):
if self.image:
# call the compress function
new_image = utils.compress(self.image)
# set self.image to new_image
self.image = new_image
# save
super().save(*args, **kwargs)
if not self.slug:
self.slug = slugify(f"{self.title}")
super().save(*args, **kwargs)
def dashboard():
page_title = "Reminder - Dashboard"
name = "Hassan"
reminders = get_all_reminders()
template = render_template('dashboard.html',
page_title=page_title,
name=name,
reminders=reminders)
return compress(template)
def write(cls, uri, document):
if not document.dirty:
return
content = compress(document.content)
ctx.cursor.execute(
'UPDATE document SET '
'content = ?, '
'score = ?, '
'referer = ?, '
'distance = ? '
'WHERE id = ?', (content, document.score, document.referer,
document.distance, document.id))
def copy(self, widget, W=False):
try:
self.Copiado = utils.Archivos
if not W:
try:
shutil.copy(os.getcwd() + "/" + utils.Archivos,
'/tmp/Compress/')
except:
shutil.copytree(os.getcwd() + "/" + utils.Archivos,
'/tmp/Compress/' + utils.Archivos)
if W:
try:
utils.compress("./" + utils.Archivos, utils.Archivo_d)
utils.decompress(utils.Archivo_d, Hboxx)
except:
self.errores('Copiar_a_nada')
except:
self.errores('Copiar')
def save(self, *args, **kwargs):
if self.image:
# call the compress function
new_image = utils.compress(self.image)
# set self.image to new_image
self.image = new_image
# save
super().save(*args, **kwargs)
if not self.slug:
self.slug = slugify({self.name})
if self.video is not None:
self.video = utils.get_video(self.video)
super().save(*args, **kwargs)
def encrypt(self):
"""Check if there are folder and password selected, clear the password field,
then compress,shred and encrypt the compressed file"""
if (self.lineEdit_folder) and (self.lineEdit_pass_encrypt.text()):
self.captured_encrypt_pass = self.lineEdit_pass_encrypt.text()
self.lineEdit_pass_encrypt.clear()
delay = int(float(self.delay.text()))
QtTest.QTest.qWait(
delay *
1000) # Wait until the delay is over before start encrypting
compress(self.directory)
shred(self.directory)
encrypt(self.captured_encrypt_pass)
alert = QMessageBox()
alert.setText('Done')
alert.exec_()
else:
alert = QMessageBox()
alert.setText('Choose a directory and password first')
alert.exec_()
def reminder(id):
page_title = ''
reminder = get_one_reminder(id)
if reminder['deleted'] == 1:
page_title = "Deleted"
else:
page_title = "Reminder - {0}".format(reminder['title'])
template = render_template('reminder.html',
reminder=reminder,
page_title=page_title)
return compress(template)
def create_word_embedding(infile, outfile, min_count, size, downsample, estimator, negative):
def run_on(actual_infile):
assert estimator in ['skipgram', 'cbow']
use_cbow = estimator == 'cbow'
sample = 10e-3
if not downsample:
sample = 1
subprocess.run([ 'tools/word2vec/word2vec' \
, '-train', actual_infile \
, '-output', outfile \
, '-min-count', str(min_count) \
, '-size', str(size) \
, '-sample', str(sample) \
, '-negative', str(negative) \
, '-cbow', str(int(use_cbow)) ])
compress_flag = False
outfilep = PurePath(outfile)
if outfilep.suffix == '.gz':
outfile = str(outfilep.parent / outfilep.stem)
compress_flag = True
infilep = PurePath(infile)
if infilep.suffix == '.gz':
rndstr = ''.join(random.choice(list(string.ascii_uppercase + string.digits)) for _ in range(10))
tmp_filename = '/tmp/' + infilep.stem + rndstr
try:
decompress(infile, keep=True, outfile=tmp_filename)
run_on(tmp_filename)
finally:
if os.path.exists(tmp_filename):
os.remove(tmp_filename)
else:
run_on(infile)
if compress_flag:
compress(outfile)
def reminders():
count = 0
reminders = get_all_reminders()
# reminder counter
for reminder in reminders:
if reminder['deleted'] == 0:
count += 1
page_title = "Reminders ({}) - All Reminders".format(count)
template = render_template('reminders.html',
reminders=reminders,
page_title=page_title)
return compress(template)
def get_payload_and_headers(size, dump_names=None, compressed=False):
"""Returns a payload and headers ready for posting
:arg size: the size in bytes the payload should be
:arg dump_names: list of dumps to include; default is
``['upload_file_minidump']``
:arg compressed: whether or not to compress the payload
:returns: ``(payload, headers)``
"""
if not dump_names:
dump_names = ['upload_file_minidump']
# Generate the payload and headers for a crash
raw_crash, dumps = utils.generate_sized_crashes(size,
dump_names=dump_names,
compressed=compressed)
# Make sure we have the specified dumps
assert list(sorted(dumps.keys())) == list(sorted(dump_names)), (
'Dumps are missing. There is a bug in the test code. %s vs. %s' %
(dumps.keys(), dump_names))
# Convert the raw crash metadata and dumps into a single dict
crash_payload = utils.assemble_crash_payload(raw_crash, dumps)
# Convert the crash payload dict into a multipart/form-encode byte stream
payload, headers = utils.multipart_encode(crash_payload)
# If we should compress it, compress the byte stream and add relevant
# header
if compressed:
payload = utils.compress(payload)
headers['Content-Encoding'] = 'gzip'
headers['Content-Length'] = str(len(payload))
# Make sure the final payload size is correct
assert len(payload) == size, ('Payload is not the right size! %s vs. %s' %
(len(payload), size))
# Return payload and headers as a tuple
return payload, headers
def eval(self):
self.logger.info('--------------------Evaluation: mAP@50-------------------')
self.ImgNet.eval().cuda()
self.TxtNet.eval().cuda()
re_BI, re_BT, re_L, qu_BI, qu_BT, qu_L = compress(self.database_loader, self.test_loader, self.ImgNet,
self.TxtNet, self.database_dataset, self.test_dataset)
MAP_I2T = calculate_top_map(qu_B=qu_BI, re_B=re_BT, qu_L=qu_L, re_L=re_L, topk=50)
MAP_T2I = calculate_top_map(qu_B=qu_BT, re_B=re_BI, qu_L=qu_L, re_L=re_L, topk=50)
if (self.best_it + self.best_ti) < (MAP_I2T + MAP_T2I):
self.best_it = MAP_I2T
self.best_ti = MAP_T2I
self.logger.info('mAP@50 I->T: %.3f, mAP@50 T->I: %.3f' % (MAP_I2T, MAP_T2I))
self.logger.info('Best MAP of I->T: %.3f, Best mAP of T->I: %.3f' % (self.best_it, self.best_ti))
self.logger.info('--------------------------------------------------------------------')
async def get_media(f: feed,
url: str,
compression: bool = True,
size: int = 320,
filename: str = None) -> IO[bytes]:
async with httpx.AsyncClient(headers=headers,
http2=True,
timeout=None,
verify=False) as client:
r = await client.get(url, headers={"Referer": f.url})
media = BytesIO(r.read())
mediatype = r.headers.get("content-type")
if compression:
if mediatype in ["image/jpeg", "image/png"]:
logger.info(f"压缩: {url} {mediatype}")
media = compress(media, size)
if filename:
media.name = filename
media.seek(0)
return media
async def relink(img):
src = img.attrs.pop("data-src")
img.attrs = {}
logger.info(f"下载图片: {src}")
async with httpx.AsyncClient(headers=headers,
http2=True,
timeout=None,
verify=False) as client:
r = await client.get(f"https:{src}")
media = BytesIO(r.read())
content_length = int(r.headers.get("content-length"))
if content_length > 1024 * 1024 * 5:
mediatype = r.headers.get("content-type")
if mediatype in ["image/jpeg", "image/png"]:
logger.info(
f"图片大小: {content_length} 压缩: {src} {mediatype}")
media = compress(media)
r = await client.post("https://telegra.ph/upload",
files={"upload-file": media.getvalue()})
resp = r.json()
if isinstance(resp, list):
img.attrs["src"] = f"https://telegra.ph{resp[0].get('src')}"
else:
logger.warning(f"{src} -> {resp}")
STYLE = 'jazz'
FNULL = open(os.devnull, 'w')
ROOT = os.path.dirname(os.path.realpath(__file__))
# for playback and tests
beats = np.load(ROOT + '/cache/'+STYLE+'.npz')
beats = beats['arr_0']
# transform bars and keep them unique
print('clean beats ...')
nbeats = []
compress = []
bar = progressbar.ProgressBar()
for beat in bar(beats):
for bar in beat:
cp = utils.compress(bar)
if cp not in compress and 0.01 <= bar.mean() < 0.028:
nbeats.append(bar)
compress.append(cp)
beats = np.array(nbeats)
# two measures is more fun
#beats = beats.reshape((-1,beats.shape[1]*2,20))
print('two measures', beats.shape)
x_train, x_test, _, _ = train_test_split(beats, beats, test_size=0.25, random_state=0)
print('size', x_train.shape, x_test.shape)
# encoder
input_dim = 20
inputs = Input(shape=(x_train.shape[1], input_dim))
encoded = Reshape((x_train.shape[1]*x_train.shape[2],))(inputs)
encoded = Dropout(0.3)(encoded)
utils.imgDisplay('OriginalImage', image)
imgRedComponent = utils.getImageComponent('red', image)
utils.imgDisplay("red", imgRedComponent)
cv2.imwrite("red-component.jpg", imgRedComponent)
imgBlueComponent = utils.getImageComponent('blue', image)
utils.imgDisplay("blue", imgBlueComponent)
cv2.imwrite("blue-component.jpg", imgBlueComponent)
imgGreenComponent = utils.getImageComponent('green', image)
utils.imgDisplay("green", imgGreenComponent)
cv2.imwrite("green-component.jpg", imgGreenComponent)
for m in [1, 2, 3, 4]:
# Compress the image
compressedImage = utils.compress(image, m)
np.save('image1.npy', image)
np.save('decompressedm{}.npy'.format(m), compressedImage)
decompressedImage = utils.decompress(compressedImage, m)
cv2.imwrite('decompressedm{}.bmp'.format(m), decompressedImage)
utils.imgDisplay('OriginalImage', decompressedImage)
imgRedComponent = utils.getImageComponent('red', decompressedImage)
utils.imgDisplay("red", imgRedComponent)
imgBlueComponent = utils.getImageComponent('blue', decompressedImage)
utils.imgDisplay("blue", imgBlueComponent)
imgGreenComponent = utils.getImageComponent('green', decompressedImage)
utils.imgDisplay("green", imgGreenComponent)
def compress_file(self, widget, archivo): # Comprime un archivo
utils.compress(utils.Archivos,
os.getcwd() + "/" + utils.Archivos + ".zip")
self.set_model(self.construir_lista())
def test_compress(teardown_env, content, result):
generate_file(content=content, random=False)
compress('_test')
input_content: bytes = get_content('test.huf')
assert input_content == result
STYLE = 'jazz'
FNULL = open(os.devnull, 'w')
ROOT = os.path.dirname(os.path.realpath(__file__))
# for playback and tests
beats = np.load(ROOT + '/cache/' + STYLE + '.npz')
beats = beats['arr_0']
# transform bars and keep them unique
print('clean beats ...')
nbeats = []
compress = []
bar = progressbar.ProgressBar()
for beat in bar(beats):
for bar in beat:
cp = utils.compress(bar)
if cp not in compress and 0.01 <= bar.mean() < 0.028:
nbeats.append(bar)
compress.append(cp)
beats = np.array(nbeats)
# two measures is more fun
#beats = beats.reshape((-1,beats.shape[1]*2,20))
print('two measures', beats.shape)
x_train, x_test, _, _ = train_test_split(beats,
beats,
test_size=0.25,
random_state=0)
print('size', x_train.shape, x_test.shape)
# encoder
input_dim = 20
beats = collection.find({'bar': 128, 'class': {'$exists': True}})
for i, b in enumerate(beats):
# get np_seq
np_seq = utils.decompress(b['zip'], b['bar'])
theme = np_seq[b['theme_index']]
# compute drum mean
dm = theme.mean(axis=0)
dm = np.delete(dm, np.s_[15::], 0)
ntheme = None
# repare shifted beats
if b['class'] == 0 and dm[0] > 0.0:
# potentiatlly shifted beat
if theme[0][0] > 0.0:
rotate = 0
while theme[0][0] == 0.0:
rotate -= 1
theme = np.roll(theme, rotate, axis=0)
# rotated, we fix all the sequence
np_seq = np.roll(np_seq, rotate, axis=1)
print(utils.draw(np_seq[b['theme_index']]))
ntheme = np_seq
if ntheme is not None:
collection.update_one( {'_id': b['_id']},
{'$set': {'dm': list(dm), 'zip': utils.compress(ntheme)}}
)
else:
collection.update_one({'_id': b['_id']},
{'$set': {'dm': list(dm)}}
)
np_seqs = []
for beat_track in beat_tracks:
np_seqs.append(numpify_beat(beat_track, mf.ticksPerQuarterNote))
flat_seq = flatten_beats(np_seqs)
if flat_seq is not None:
# if everything is okay
# time sig to numbr of steps per bar
ts = time_sigs[0][0]
bar_ticks = int(ts.barDuration.quarterLength*BEAT_DIV)
# clean a bit the sequence
np_seq = pad_reshape_trim(flat_seq, bar_ticks)
if np_seq.shape[0] == 0:
junk(f)
continue
# compress it for mongo
zp = utils.compress(np_seq)
# check for duplicate
if collection.count({'zip': zp}) > 0:
junk(f)
print('found a dup, skipping ...')
else:
# get heuristics
entry = get_stats(np_seq)
entry['zip'] = zp
entry['fname'] = f
collection.insert_one(entry)
else:
# trash
junk(f)
else:
# trash
def mc_region_level_worker(
allc_file, output_file, bed_file,
bed_file_name_column=False,
contexts=CONTEXTS,
compress=True,
cap=2,
overwrite=False,
):
"""
allc_file
bed file:
"""
logging.info("Begin mc_region_level processing: {}".format(allc_file))
# check overlap
if not overwrite:
if os.path.isfile(output_file) or os.path.isfile(output_file+'.gz') or os.path.isfile(output_file+'.bgz'):
logging.info("File exists "+output_file+", skipping...")
return 0
# read in bed file and remove chr prefix
if bed_file_name_column:
columns = ['chr', 'start', 'end', 'name']
df_gtf = pd.read_csv(bed_file, sep="\t", header=None,
names=columns, usecols=[0, 1, 2, 3], dtype={'chr': object, 'name': object})
else:
columns = ['chr', 'start', 'end']
df_gtf = pd.read_csv(bed_file, sep="\t", header=None,
names=columns, usecols=[0, 1, 2], dtype={'chr': object})
df_gtf.chr = df_gtf.chr.apply(lambda x: x[len('chr'):] if x.startswith('chr') else x)
# auto detect whether the allc table has chr prefix or not
allc_peek = utils.read_allc(allc_file,
pindex=False,
remove_chr=False,
nrows=1,
)
chrom_format = allc_peek.iloc[0]['chr']
if chrom_format.startswith('chr'):
allc_chr_prefix = True
else:
allc_chr_prefix = False
# output header
outfile = open(output_file, "w")
for context in contexts:
columns += ['m{}'.format(context), context]
outfile.write('\t'.join(columns)+'\n')
# write results region by region
allc = tabix.open(allc_file)
for i, row in df_gtf.iterrows():
row_out = [str(row.chr), str(row.start), str(row.end)]
if bed_file_name_column:
row_out += [str(row['name'])]
if not row_out[0].startswith('chr'): # enforce output has chr as prefix
row_out[0] = 'chr'+row_out[0]
if allc_chr_prefix:
try:
toquery = 'chr'+str(row['chr']), row['start'], row['end']
records = list(allc.query(*toquery))
except:
raise ValueError("tabix query failed! {}".format(toquery))
else:
try:
toquery = row['chr'], row['start'], row['end']
records = list(allc.query(*toquery))
except:
raise ValueError("tabix query failed for query {}".format(toquery))
for context in contexts:
mc, c = utils.tabix_summary(records, context=context, cap=cap) # remove sites with total_c > 2
row_out += [str(mc), str(c)]
outfile.write('\t'.join(row_out)+'\n')
outfile.close()
if compress:
utils.compress(output_file, suffix='gz')
logging.info("Done. Results saved to {}".format(output_file))
return 0
add_step(sequence, tck_ct/tick, event)
except:
continue;
else:
drumtrack = False
elif type(event) == type(midi.events.EndOfTrackEvent()):
if drumtrack:
drtrack.append(event)
if drumtrack:
# finish this sequnce
complete_sequence(sequence)
if len(sequence)/BEAT_DIV/4 >= 16:
#drpattern.append(drtrack)
#midi.write_midifile("out/"+`ct`+".mid", drpattern)
#midi.write_midifile("out/"+`ct`+"_16.mid", seq2midi(sequence))
try:
np_seq = np.array(sequence)
entry = {
"beat_num": ct,
"beat_zip": compress(np_seq),
"beat_bars": len(np_seq)/BEAT_DIV/4
}
collection.insert_one(entry)
except:
e = sys.exc_info()[0]
print e
continue
print ct
ct+=1
backUpQuery %
("OMG_GameWEB", os.path.join(newDbBackUpPath, "OMG_GameWeb.bak")))
executeSqlQuery(
backUpQuery %
("PS_Billing", os.path.join(newDbBackUpPath, "PS_Billing.bak")))
executeSqlQuery(
backUpQuery %
("PS_ChatLog", os.path.join(newDbBackUpPath, "PS_ChatLog.bak")))
executeSqlQuery(
backUpQuery %
("PS_GameData", os.path.join(newDbBackUpPath, "PS_GameData.bak")))
executeSqlQuery(
backUpQuery %
("PS_GameDefs", os.path.join(newDbBackUpPath, "PS_GameDefs.bak")))
executeSqlQuery(
backUpQuery %
("PS_GameLog", os.path.join(newDbBackUpPath, "PS_GameLog.bak")))
executeSqlQuery(
backUpQuery %
("PS_GMTool", os.path.join(newDbBackUpPath, "PS_GMTool.bak")))
executeSqlQuery(
backUpQuery %
("PS_Statics", os.path.join(newDbBackUpPath, "PS_Statics.bak")))
executeSqlQuery(
backUpQuery %
("PS_UserData", os.path.join(newDbBackUpPath, "PS_UserData.bak")))
utils.compress(dbBackUpPath, currentTime)
else:
utils.show("Operation failed! We do not have %s operation" % operation)
def index():
page_title = "Reminder - Home"
template = render_template('index.html', page_title=page_title)
return compress(template)
def _add_data(self, key, value):
data = value.encode(self.codec, "ignore")
data = compress(key, data)
self._maker.add(key, data)
return
def bin_allc_worker(
allc_file,
genome_size_file,
bin_size,
output_file,
chromosomes=None,
contexts=CONTEXTS,
compress=True,
overwrite=False,
):
logging.info("binc processing: {} {}".format(allc_file, contexts))
if not overwrite:
if os.path.isfile(output_file) or os.path.isfile(
output_file + '.gz') or os.path.isfile(output_file + '.bgz'):
logging.info("File exists " + output_file + ", skipping...")
return 0
chrom_sizes = utils.get_chrom_lengths(genome_size_file)
if chromosomes == None:
chromosomes = chrom_sizes.index.values
# read allc
df_allc = utils.read_allc(allc_file, pindex=False, compression='gzip')
df_allc = df_allc.loc[df_allc.chr.isin(chromosomes)]
# initiate
chr_allc = np.array([])
bins_allc = np.array([])
mc_c_allc = OrderedDict()
for context in contexts:
mc_c_allc['m' + context] = np.array([])
mc_c_allc[context] = np.array([])
for chromosome, df in df_allc.groupby('chr'):
# last bin (incomplete) is discarded
bins = np.arange(0, chrom_sizes[chromosome], bin_size)
# number of intervals is number of bin points -1
chrs = np.asarray([chromosome] * (len(bins) - 1))
bins_allc = np.concatenate([bins_allc, bins[:-1]])
chr_allc = np.concatenate([chr_allc, chrs])
for context in contexts:
df_context = df.loc[df.context.isin(
utils.get_expanded_context(context))]
df_mc_c = df_context.groupby(pd.cut(
df_context.pos, bins)).sum().fillna(0)[['mc', 'c']]
mc_c_allc['m' + context] = np.concatenate(
[mc_c_allc['m' + context], df_mc_c.mc])
mc_c_allc[context] = np.concatenate(
[mc_c_allc[context], df_mc_c.c])
columns = ['chr', 'bin'] + [key for key in mc_c_allc]
binc = pd.DataFrame(columns=columns)
binc['chr'] = chr_allc.astype(object)
binc['bin'] = bins_allc.astype(int)
for key, value in mc_c_allc.items():
binc[key] = value.astype(int)
binc.to_csv(output_file, na_rep='NA', sep="\t", header=True, index=False)
# bgzip
if compress:
utils.compress(output_file, suffix='gz')
logging.info("Done. Results saved to {}".format(output_file))
return
import sys
import cPickle
import utils
for line in sys.stdin:
line = eval(line)
print utils.compress(line)
def compress_decompress():
compress('_test')
decompress('_test_decomp')
