Beispiel #1
0
def process_tile(file_path):
    # Parse zoom, x, y, out of file name
    groups = re.search("(\d+)-(\d+)-(\d+).events.pbf", file_path).groups()
    zoom = int(groups[0])
    x = int(groups[1])
    y = int(groups[2])

    print("Processing tile %d-%d-%d" % (zoom, x, y))

    # Write into the adjacent events_agg folder
    file_path_out = join(output_path, basename(file_path))

    with open(file_path_out, 'wb') as file_out:
        with open(file_path, 'rb') as file:
            for o in read_objects(0, file.read(), SharedStreetsWeeklyBinnedLinearReferences):
                # Create a new linear reference with the same settings
                out = SharedStreetsBinnedLinearReferences()
                out.referenceId = o.referenceId
                out.scaledCounts = o.scaledCounts
                out.referenceLength = o.referenceLength
                out.numberOfBins = o.numberOfBins
                out.binPosition.extend(o.binPosition)

                # But aggregate the event data
                for b in o.binnedPeriodicData:
                    out.bins.extend([agg_bin(b)])

                # Write the data
                pbf_data = out.SerializeToString()
                size_of_data = len(pbf_data)
                _VarintEncoder()(file_out.write, size_of_data, True)
                file_out.write(pbf_data)
Beispiel #2
0
    def _encode_varint(cls, value):
        data = []

        def write(x):
            data.append(x)

        protobuf_encoder._VarintEncoder()(write, value)
        return b''.join(data)
Beispiel #3
0
    def _encode_varint(cls, value):
        data = []

        def write(x):
            data.append(x)

        protobuf_encoder._VarintEncoder()(write, value)
        return b''.join(data)
Beispiel #4
0
def generate_pbf(output_file, binned_observations):

    for binned_reference in binned_observations:

        pbf_data = binned_reference.toPbf()
        size_of_data = len(pbf_data)
        _VarintEncoder()(output_file.write, size_of_data, True)
        output_file.write(pbf_data)
Beispiel #5
0
 def encode_string(cls, value):
     data = []
     def write(x):
         data.append(x)
     encoded = value.encode('utf-8')
     protobuf_encoder._VarintEncoder()(write, len(encoded))
     write(encoded)
     return b''.join(data)
Beispiel #6
0
def send_msg(msg, socket):
    print("sending")
    print(msg)
    print("sending complete")
    string = msg.SerializeToString()
    data = []
    _VarintEncoder()(data.append, len(string), None)
    size = b''.join(data)
    socket.sendall(size + string)
Beispiel #7
0
    def __init__(self, fname, fdescr, compresslevel=9, autoflush=False):
        self._ve = _VarintEncoder()
        self._last_descriptor = None
        self.autoflush = autoflush

        self._fobj = gzip.open(fname, "wb", compresslevel=compresslevel)
        self._write_header(fdescr)
Beispiel #8
0
 def __init__(self, fh, version=1, is_empty=False, read_mode=True):
     self.fh = fh
     self.is_empty = is_empty
     if read_mode == True:
         version = ord(self.fh.read(1))
     self.varint_encoder = _VarintEncoder()
     self.varint_decoder = _DecodeVarint
Beispiel #9
0
 def __init__(self):
     super(Connection,self).__init__()
     
     self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     self.socket.connect(("localhost",8081))
     
     self.varint_encoder = _VarintEncoder()
     self.varint_decoder = _DecodeVarint
     
     pass
    def __init__(self, factory):
        '''
        Init the protocol
        '''
        super(ProtobufDelimitedProtocol, self).__init__()

        self._factory = factory

        self.varint_encoder = _VarintEncoder()
        self.varint_decoder = _DecodeVarint
Beispiel #11
0
    def __init__(self):
        super(Connection, self).__init__()

        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.socket.connect(("localhost", 8081))

        self.varint_encoder = _VarintEncoder()
        self.varint_decoder = _DecodeVarint

        pass
 def __init__(self, factory):
     '''
     Init the protocol
     '''
     super(ProtobufDelimitedProtocol,self).__init__()
     
     self._factory = factory
     
     self.varint_encoder = _VarintEncoder()
     self.varint_decoder = _DecodeVarint
def process_tile(file_path):
    # Parse zoom, x, y, out of file name
    groups = re.search("(\d+)-(\d+)-(\d+).events.pbf", file_path).groups()
    zoom = int(groups[0])
    x = int(groups[1])
    y = int(groups[2])

    print("Processing tile %d-%d-%d" % (zoom, x, y))
    file_path_out = join(output_path, basename(file_path))

    with open(file_path_out, 'wb') as file_out:
        with open(file_path, 'rb') as file:
            for o in read_objects(0, file.read(), SharedStreetsWeeklyBinnedLinearReferences):
                filter_bins(o)
                if len(o.binPosition) < 1:
                    continue
                pbf_data = o.SerializeToString()
                size_of_data = len(pbf_data)
                _VarintEncoder()(file_out.write, size_of_data, True)
                file_out.write(pbf_data)
Beispiel #14
0
    def __init__(self, fh, version=1, is_empty=False, read_mode=None):
        if version != 1:
            raise PBException('only version 1 streams are supported')

        self.fh = fh
        self.is_empty = is_empty

        if read_mode == 'beginning':
            version = ord(self.fh.read(1))
            if version != 1:
                raise PBException('stream version %d not supported' % version)

        self.varint_encoder = _VarintEncoder()
        self.varint_decoder = _DecodeVarint
Beispiel #15
0
def write_sized_message(message, crc = True):
    """ serializes a protobuf message and prefixes it with the length of the message and
        suffixes it with an optional crc checksum over the length and the messager

        It is a python port of dedupv1::base::WriteSizedMessage
    """

    output = []
    variant_encoder = encoder._VarintEncoder()

    # there we need a method that when it is called writes the bytes. using the append method of a list results in appending
    # the bytes to the list
    variant_encoder(output.append, message.ByteSize())
    output.append(message.SerializeToString())

    if (crc):
        crc_value = binascii.crc32(output[0]) # crc the size
        crc_value = binascii.crc32(output[1], crc_value) # crc the contents based on the size crc. This is equal to crc both at one step
        crc_value = crc_value & 0xffffffff # make it unsigned
        variant_encoder(output.append, crc_value)
    else:
        variant_encoder(output.append, 0)

    return "".join(output)
Beispiel #16
0
# pylint: disable=import-error,no-name-in-module
import google.protobuf
from google.protobuf.internal import encoder as protobuf_encoder
# pylint: disable=import-error,no-name-in-module
from google.protobuf.internal import wire_format as protobuf_wire_format
from krpc.error import EncodingError
from krpc.platform import bytelength
import krpc.schema.KRPC_pb2 as KRPC
from krpc.types import \
    Types, ValueType, ClassType, EnumerationType, MessageType, TupleType, \
    ListType, SetType, DictionaryType

# The following unpacks the internal protobuf decoders, whose signature
# depends on the version of protobuf installed
# pylint: disable=invalid-name
_pb_VarintEncoder = protobuf_encoder._VarintEncoder()
_pb_SignedVarintEncoder = protobuf_encoder._SignedVarintEncoder()
_pb_DoubleEncoder = protobuf_encoder.DoubleEncoder(1, False, False)
_pb_FloatEncoder = protobuf_encoder.FloatEncoder(1, False, False)
_pb_version = google.protobuf.__version__.split('.')
if int(_pb_version[0]) >= 3 and int(_pb_version[1]) >= 4:
    # protobuf v3.4.0 and above
    def _VarintEncoder(write, value):
        return _pb_VarintEncoder(write, value, True)

    def _SignedVarintEncoder(write, value):
        return _pb_SignedVarintEncoder(write, value, True)

    def _DoubleEncoder(write, value):
        return _pb_DoubleEncoder(write, value, True)
def encodeVarint(data):
    packedData = []
    _VarintEncoder()(packedData.append, data, False)
    return b''.join(packedData)
def encode_varint(value):
    """ Encode an int as a protobuf varint """
    data = []
    _VarintEncoder()(data.append, value, False)
    return b''.join(data)
Beispiel #19
0
import os
import unittest
import krpc.schema.KRPC_pb2 as KRPC
import google.protobuf
from google.protobuf.internal.decoder import _DecodeVarint, _DecodeSignedVarint
from google.protobuf.internal import encoder as protobuf_encoder
from google.protobuf.internal.wire_format import ZigZagEncode, ZigZagDecode
import serial


# The following unpacks the internal protobuf decoders, whose signature
# depends on the version of protobuf installed
# pylint: disable=invalid-name,protected-access
_pb_VarintEncoder = protobuf_encoder._VarintEncoder()
_pb_SignedVarintEncoder = protobuf_encoder._SignedVarintEncoder()
_pb_version = google.protobuf.__version__.split('.')
if int(_pb_version[0]) >= 3 and int(_pb_version[1]) >= 4:
    # protobuf v3.4.0 and above
    def _VarintEncoder(write, value):
        return _pb_VarintEncoder(write, value, True)

    def _SignedVarintEncoder(write, value):
        return _pb_SignedVarintEncoder(write, value, True)
else:
    # protobuf v3.3.0 and below
    _VarintEncoder = _pb_VarintEncoder
    _SignedVarintEncoder = _pb_SignedVarintEncoder
# pylint: enable=invalid-name,protected-access


def encode_varint(value):
Beispiel #20
0
def proto_dump(o, f):
    for x in o:
        thing = fpb.Thing(**x)
        encoder = _VarintEncoder()
        encoder(f.write, thing.ByteSize())
        f.write(thing.SerializeToString())
Beispiel #21
0
def encode(cellid):
    output = []
    encoder._VarintEncoder()(output.append, cellid)
    return "".join(output)
Beispiel #22
0
 def __init__(self, filename):
     self.filename = filename
     self.f = open(self.filename, "w")
     self.encoder = encoder._VarintEncoder()
Beispiel #23
0
def encode_varint(value):
    data = []
    _VarintEncoder()(data.append, value, None)
    return b''.join(data)
Beispiel #24
0
 def __init__(self, filename):
   self.filename = filename
   self.f = open(self.filename, "w")
   self.encoder = encoder._VarintEncoder()
Beispiel #25
0
def encode(cellid):
    output = []
    encoder._VarintEncoder()(output.append, cellid)
    return ''.join(output)
Beispiel #26
0
from google.protobuf.internal import encoder

_EncodeVarint = encoder._VarintEncoder()


def write_delimited_to(out_file, message):
    msg_size = message.ByteSize()
    pieces = []
    _EncodeVarint(pieces.append, msg_size)
    out_file.write(b"".join(pieces))
    out_file.write(message.SerializeToString())


def read_gold_props(gold_props_file):
    """ Read gold predicates from CoNLL-formatted file.
  """
    gold_props = []
    props = []
    with open(gold_props_file, 'r') as f:
        for line in f:
            line = line.strip()
            if line == '':
                gold_props.append(props)
                props = []
            else:
                props.append(line.split()[0])
        f.close()
    if len(props) > 0:
        gold_props.append(props)
    return gold_props
Beispiel #27
0
 def _encode(self, cellid):
     output = []
     encoder._VarintEncoder()(output.append, cellid)
     return ''.join(output)
Beispiel #28
0
def proto_dump(o, f):
    for x in o:
        pbs = fpb.Thing(**x).SerializeToString()
        encoder = _VarintEncoder()
        encoder(f.write, len(pbs))
        f.write(pbs)