def process_tile(file_path):
# Parse zoom, x, y, out of file name
groups = re.search("(\d+)-(\d+)-(\d+).events.pbf", file_path).groups()
zoom = int(groups[0])
x = int(groups[1])
y = int(groups[2])
print("Processing tile %d-%d-%d" % (zoom, x, y))
# Write into the adjacent events_agg folder
file_path_out = join(output_path, basename(file_path))
with open(file_path_out, 'wb') as file_out:
with open(file_path, 'rb') as file:
for o in read_objects(0, file.read(), SharedStreetsWeeklyBinnedLinearReferences):
# Create a new linear reference with the same settings
out = SharedStreetsBinnedLinearReferences()
out.referenceId = o.referenceId
out.scaledCounts = o.scaledCounts
out.referenceLength = o.referenceLength
out.numberOfBins = o.numberOfBins
out.binPosition.extend(o.binPosition)
# But aggregate the event data
for b in o.binnedPeriodicData:
out.bins.extend([agg_bin(b)])
# Write the data
pbf_data = out.SerializeToString()
size_of_data = len(pbf_data)
_VarintEncoder()(file_out.write, size_of_data, True)
file_out.write(pbf_data)
def _encode_varint(cls, value):
data = []
def write(x):
data.append(x)
protobuf_encoder._VarintEncoder()(write, value)
return b''.join(data)
def _encode_varint(cls, value):
data = []
def write(x):
data.append(x)
protobuf_encoder._VarintEncoder()(write, value)
return b''.join(data)
def generate_pbf(output_file, binned_observations):
for binned_reference in binned_observations:
pbf_data = binned_reference.toPbf()
size_of_data = len(pbf_data)
_VarintEncoder()(output_file.write, size_of_data, True)
output_file.write(pbf_data)
def encode_string(cls, value):
data = []
def write(x):
data.append(x)
encoded = value.encode('utf-8')
protobuf_encoder._VarintEncoder()(write, len(encoded))
write(encoded)
return b''.join(data)
def send_msg(msg, socket):
print("sending")
print(msg)
print("sending complete")
string = msg.SerializeToString()
data = []
_VarintEncoder()(data.append, len(string), None)
size = b''.join(data)
socket.sendall(size + string)
def __init__(self, fname, fdescr, compresslevel=9, autoflush=False):
self._ve = _VarintEncoder()
self._last_descriptor = None
self.autoflush = autoflush
self._fobj = gzip.open(fname, "wb", compresslevel=compresslevel)
self._write_header(fdescr)
def __init__(self, fh, version=1, is_empty=False, read_mode=True):
self.fh = fh
self.is_empty = is_empty
if read_mode == True:
version = ord(self.fh.read(1))
self.varint_encoder = _VarintEncoder()
self.varint_decoder = _DecodeVarint
def __init__(self):
super(Connection,self).__init__()
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.connect(("localhost",8081))
self.varint_encoder = _VarintEncoder()
self.varint_decoder = _DecodeVarint
pass
def __init__(self, factory):
'''
Init the protocol
'''
super(ProtobufDelimitedProtocol, self).__init__()
self._factory = factory
self.varint_encoder = _VarintEncoder()
self.varint_decoder = _DecodeVarint
def __init__(self):
super(Connection, self).__init__()
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.connect(("localhost", 8081))
self.varint_encoder = _VarintEncoder()
self.varint_decoder = _DecodeVarint
pass
def __init__(self, factory):
'''
Init the protocol
'''
super(ProtobufDelimitedProtocol,self).__init__()
self._factory = factory
self.varint_encoder = _VarintEncoder()
self.varint_decoder = _DecodeVarint
def process_tile(file_path):
# Parse zoom, x, y, out of file name
groups = re.search("(\d+)-(\d+)-(\d+).events.pbf", file_path).groups()
zoom = int(groups[0])
x = int(groups[1])
y = int(groups[2])
print("Processing tile %d-%d-%d" % (zoom, x, y))
file_path_out = join(output_path, basename(file_path))
with open(file_path_out, 'wb') as file_out:
with open(file_path, 'rb') as file:
for o in read_objects(0, file.read(), SharedStreetsWeeklyBinnedLinearReferences):
filter_bins(o)
if len(o.binPosition) < 1:
continue
pbf_data = o.SerializeToString()
size_of_data = len(pbf_data)
_VarintEncoder()(file_out.write, size_of_data, True)
file_out.write(pbf_data)
def __init__(self, fh, version=1, is_empty=False, read_mode=None):
if version != 1:
raise PBException('only version 1 streams are supported')
self.fh = fh
self.is_empty = is_empty
if read_mode == 'beginning':
version = ord(self.fh.read(1))
if version != 1:
raise PBException('stream version %d not supported' % version)
self.varint_encoder = _VarintEncoder()
self.varint_decoder = _DecodeVarint
def write_sized_message(message, crc = True):
""" serializes a protobuf message and prefixes it with the length of the message and
suffixes it with an optional crc checksum over the length and the messager
It is a python port of dedupv1::base::WriteSizedMessage
"""
output = []
variant_encoder = encoder._VarintEncoder()
# there we need a method that when it is called writes the bytes. using the append method of a list results in appending
# the bytes to the list
variant_encoder(output.append, message.ByteSize())
output.append(message.SerializeToString())
if (crc):
crc_value = binascii.crc32(output[0]) # crc the size
crc_value = binascii.crc32(output[1], crc_value) # crc the contents based on the size crc. This is equal to crc both at one step
crc_value = crc_value & 0xffffffff # make it unsigned
variant_encoder(output.append, crc_value)
else:
variant_encoder(output.append, 0)
return "".join(output)
# pylint: disable=import-error,no-name-in-module
import google.protobuf
from google.protobuf.internal import encoder as protobuf_encoder
# pylint: disable=import-error,no-name-in-module
from google.protobuf.internal import wire_format as protobuf_wire_format
from krpc.error import EncodingError
from krpc.platform import bytelength
import krpc.schema.KRPC_pb2 as KRPC
from krpc.types import \
Types, ValueType, ClassType, EnumerationType, MessageType, TupleType, \
ListType, SetType, DictionaryType
# The following unpacks the internal protobuf decoders, whose signature
# depends on the version of protobuf installed
# pylint: disable=invalid-name
_pb_VarintEncoder = protobuf_encoder._VarintEncoder()
_pb_SignedVarintEncoder = protobuf_encoder._SignedVarintEncoder()
_pb_DoubleEncoder = protobuf_encoder.DoubleEncoder(1, False, False)
_pb_FloatEncoder = protobuf_encoder.FloatEncoder(1, False, False)
_pb_version = google.protobuf.__version__.split('.')
if int(_pb_version[0]) >= 3 and int(_pb_version[1]) >= 4:
# protobuf v3.4.0 and above
def _VarintEncoder(write, value):
return _pb_VarintEncoder(write, value, True)
def _SignedVarintEncoder(write, value):
return _pb_SignedVarintEncoder(write, value, True)
def _DoubleEncoder(write, value):
return _pb_DoubleEncoder(write, value, True)
def encodeVarint(data):
packedData = []
_VarintEncoder()(packedData.append, data, False)
return b''.join(packedData)
def encode_varint(value):
""" Encode an int as a protobuf varint """
data = []
_VarintEncoder()(data.append, value, False)
return b''.join(data)
import os
import unittest
import krpc.schema.KRPC_pb2 as KRPC
import google.protobuf
from google.protobuf.internal.decoder import _DecodeVarint, _DecodeSignedVarint
from google.protobuf.internal import encoder as protobuf_encoder
from google.protobuf.internal.wire_format import ZigZagEncode, ZigZagDecode
import serial
# The following unpacks the internal protobuf decoders, whose signature
# depends on the version of protobuf installed
# pylint: disable=invalid-name,protected-access
_pb_VarintEncoder = protobuf_encoder._VarintEncoder()
_pb_SignedVarintEncoder = protobuf_encoder._SignedVarintEncoder()
_pb_version = google.protobuf.__version__.split('.')
if int(_pb_version[0]) >= 3 and int(_pb_version[1]) >= 4:
# protobuf v3.4.0 and above
def _VarintEncoder(write, value):
return _pb_VarintEncoder(write, value, True)
def _SignedVarintEncoder(write, value):
return _pb_SignedVarintEncoder(write, value, True)
else:
# protobuf v3.3.0 and below
_VarintEncoder = _pb_VarintEncoder
_SignedVarintEncoder = _pb_SignedVarintEncoder
# pylint: enable=invalid-name,protected-access
def encode_varint(value):
def proto_dump(o, f):
for x in o:
thing = fpb.Thing(**x)
encoder = _VarintEncoder()
encoder(f.write, thing.ByteSize())
f.write(thing.SerializeToString())
def encode(cellid):
output = []
encoder._VarintEncoder()(output.append, cellid)
return "".join(output)
def __init__(self, filename):
self.filename = filename
self.f = open(self.filename, "w")
self.encoder = encoder._VarintEncoder()
def encode_varint(value):
data = []
_VarintEncoder()(data.append, value, None)
return b''.join(data)
def __init__(self, filename): self.filename = filename self.f = open(self.filename, "w") self.encoder = encoder._VarintEncoder()
def encode(cellid):
output = []
encoder._VarintEncoder()(output.append, cellid)
return ''.join(output)
from google.protobuf.internal import encoder
_EncodeVarint = encoder._VarintEncoder()
def write_delimited_to(out_file, message):
msg_size = message.ByteSize()
pieces = []
_EncodeVarint(pieces.append, msg_size)
out_file.write(b"".join(pieces))
out_file.write(message.SerializeToString())
def read_gold_props(gold_props_file):
""" Read gold predicates from CoNLL-formatted file.
"""
gold_props = []
props = []
with open(gold_props_file, 'r') as f:
for line in f:
line = line.strip()
if line == '':
gold_props.append(props)
props = []
else:
props.append(line.split()[0])
f.close()
if len(props) > 0:
gold_props.append(props)
return gold_props
def _encode(self, cellid):
output = []
encoder._VarintEncoder()(output.append, cellid)
return ''.join(output)
def proto_dump(o, f):
for x in o:
pbs = fpb.Thing(**x).SerializeToString()
encoder = _VarintEncoder()
encoder(f.write, len(pbs))
f.write(pbs)