def addTable(self,schema):
descriptorProto = DescriptorProto()
tableIDVersion = str(schema['table_id']) + '_' + str(schema['version'])
if tableIDVersion in self.tableSchemaVersionSet:
return
self.tableSchemaVersionSet.add(tableIDVersion)
tmpName = schema['namespace_name'] + '_' + schema['database'] + '_' + schema['table_name'] + '_' + str(schema['version'])
descriptorProto.name = str(tmpName)
self.tableMaxFieldId[schema['table_id']] = 0
fieldTypeDict = {}
for field in schema['fields']:
if 'deleted' in field and field['deleted'] == True:
continue
fieldProto = descriptorProto.field.add()
fieldProto.number = field['field_id']
if fieldProto.number > self.tableMaxFieldId[schema['table_id']]:
self.tableMaxFieldId[schema['table_id']] = fieldProto.number
fieldProto.name = field['field_name']
fieldProto.type = DynamicMessageParser.mysqlType2ProtoType[field['mysql_type']]
fieldTypeDict[fieldProto.number] = field['mysql_type']
if 'default_value' in field:
if field['mysql_type'] in ('DATETIME','DATE','TIME','TIMESTAMP'):
continue
fieldProto.default_value = base64.b64decode(field['default_value'])
descriptor = MakeDescriptor(descriptorProto)
self.tableDescriptor[schema['table_id']] = descriptor
self.tableFieldTypeDict[schema['table_id']] = fieldTypeDict
self.factory.RegisterMessageDescriptor(descriptor)
self.factory.pool.AddDescriptor(descriptor)
def reconstitute_from_bytes(descriptor_proto_bytes):
"""
Reconstitutes a Python protobuf class from the serialized form of
a descriptor proto.
"""
descriptor_proto = DescriptorProto()
descriptor_proto.ParseFromString(descriptor_proto_bytes)
return reconstitute(descriptor_proto)
def __init__(self, java_class, pool=_default_descriptor_pool):
self._descriptor_pool = pool
self._java_class = java_class
jdesc_bytes = java_class.getMethod("getDescriptor", None).invoke(
None, None).toProto().toByteArray()
if PY2:
jdesc_bytes = str(jdesc_bytes)
pdesc_proto = DescriptorProto()
pdesc_proto.MergeFromString(jdesc_bytes)
try:
pdesc_proto_desc = self._descriptor_pool.FindFileContainingSymbol(
pdesc_proto.name).message_types_by_name[pdesc_proto.name]
except KeyError:
pdesc_proto_desc = MakeClassDescriptor(self._descriptor_pool,
pdesc_proto)
self._python_class = _default_symbol_db.GetPrototype(pdesc_proto_desc)
def create_map(cls, jar, enums, pkg, var, number, ftype1, fmsg1, ftype2, fmsg2, \
msg_to_referrers, msg_path_to_obj):
map_obj = DescriptorProto()
map_obj.options.map_entry = True
map_obj.name = '%s$map%d' % (cls.split('.')[-1], number)
map_full_name = map_obj.name
if pkg:
map_full_name = pkg + '.' + map_full_name
if map_full_name not in msg_path_to_obj:
for fnumber, fname, ftype, fmsg in ((1, 'key', ftype1, fmsg1), \
(2, 'value', ftype2, fmsg2)):
field = map_obj.field.add()
if type(ftype) is str:
if ftype.isnumeric():
ftype = int(ftype)
else:
enum_name, enum_var = ftype.rsplit('.', 1)
enum_code = jar.decomp(enum_name, True).raw
ftype = type_consts[enum_code.split(enum_var + ' = new ')
[1].split('"')[1].lower()]
if fmsg and ftype in (field.TYPE_GROUP, field.TYPE_ENUM,
field.TYPE_MESSAGE):
if '.' not in fmsg and pkg:
fmsg = pkg + '.' + fmsg
msg_to_referrers[fmsg].append(
(fname, map_full_name,
ftype == FieldDescriptorProto.TYPE_GROUP))
field.type_name = '.' + fmsg
if ftype == field.TYPE_ENUM:
if fmsg:
create_enum(jar, enums, fmsg, msg_path_to_obj)
else:
ftype = field.TYPE_INT32
field.type = ftype
field.name = fname
field.number = fnumber
msg_path_to_obj[map_full_name] = map_obj
return ('repeated', 'message', map_full_name, None, var)
def reconstitute_from_bytes(self, descriptor_proto_bytes):
"""
Reconstitutes a Python protobuf class from a byte stream. The
intended purpose of this function is to create a Protobuf
Python class from a byte stream sent from another service. This
way, services can define arbitrary data types and send schemas
for those types to other services.
Args:
descriptor_proto_bytes: Serialized protocol buffer describing
a single class
Returns:
A Python class for the message encoded in
descriptor_proto_bytes.
"""
descriptor_proto = DescriptorProto()
descriptor_proto.ParseFromString(descriptor_proto_bytes)
return self.reconstitute(descriptor_proto)
def create_collection(self, database, name, descriptor):
descriptor_proto = DescriptorProto()
descriptor.CopyToProto(descriptor_proto)
request = CreateCollectionRequest(database=database, name=name)
request.schema.MergeFrom(descriptor_proto)
return self.stub.CreateCollection(request)
def extract_j2me(jar, cls, enums, gen_classes_j2me, protobuftype_cls, consts,
msg_path_to_obj, msg_to_referrers):
code = jar.decomp(cls, True)
cls = cls.replace('$', '.')
if not code.raw:
print('(Jad failed)')
return
"""
Unique step: look for calls to ProtoBufType.addElement(int, int, Object)
"""
code = sub('(?:java\.lang\.Long.valueOf|new Long)\((.+?)L?\)', r'\1',
code.raw)
for var in findall('(\w+) = new ', code):
print('\nIn %s.%s:' % (cls, var))
message = DescriptorProto()
my_namer = namer()
summary = {}
fields = search('%s((?:\.\w+\(\d+, \d+, .+?\))+)' % var, code)
public_var = search(' (\w+) = %s;' % var, code)
if public_var:
var = public_var.group(1)
message.name = var
if fields:
for ftypeandlabel, fnumber, fdefaultormsg in findall(
'\.\w+\((\d+), (\d+), (.+?)\)', fields.group(1)):
field = message.field.add()
# Use int32 instead of enum (we don't have enum contents),
# Use bytes and string instead of data and text (Protobuf 1 types).
types = {
17: 'double',
18: 'float',
19: 'int64',
20: 'uint64',
21: 'int32',
22: 'fixed64',
23: 'fixed32',
24: 'bool',
25: 'bytes',
26: 'group',
27: 'message',
28: 'string',
29: 'uint32',
30: 'int32',
31: 'sfixed32',
32: 'sfixed64',
33: 'sint32',
34: 'sint64',
35: 'bytes',
36: 'string'
}
ftype = types[int(ftypeandlabel) & 0xff]
labels = {1: 'required', 2: 'optional', 4: 'repeated'}
flabel = labels[int(ftypeandlabel) >> 8]
field.name = next(my_namer)
field.number = int(fnumber)
field.label = label_consts[flabel]
if fdefaultormsg != 'null':
if ftype in ('group', 'message'):
if '.' in fdefaultormsg:
field.type_name = '.' + fdefaultormsg
msg_to_referrers[fdefaultormsg].append(
(field.name, cls + '.' + var,
ftype == 'group'))
if fdefaultormsg not in msg_path_to_obj: # Classes empty or to be created
msg_path_to_obj[
fdefaultormsg] = DescriptorProto()
msg_path_to_obj[
fdefaultormsg].name = fdefaultormsg.split(
'.')[-1]
else:
field.type_name = '.' + cls + '.' + fdefaultormsg
msg_to_referrers[cls + '.' + fdefaultormsg].append(
(field.name, cls + '.' + var,
ftype == 'group'))
else:
if fdefaultormsg in consts:
fdefaultormsg = consts[fdefaultormsg]
parse_default(field, ftype, fdefaultormsg)
else:
fdefaultormsg = None
if ftype in ('group', 'message'):
ftype = 'bytes'
field.type = type_consts[ftype]
summary[int(fnumber)] = (flabel, ftype, fdefaultormsg)
msg_path_to_obj[cls + '.' + var] = message
print(summary)
body = body.split(cond2)[0]
if '.DEFAULT_INSTANCE)' in body:
fenumormsg = body.split('.DEFAULT_INSTANCE)')[0].split(
'(')[-1].split(', ')[-1]
if ftype == 'bytes':
ftype = 'message'
fields[fnumber] = (flabel, ftype, fenumormsg, fdefault, var)
print(fields)
"""
Final step: Build the DescriptorProto object
"""
message = DescriptorProto()
message.name = cls.rsplit('.', 1)[-1]
seen_vars = {}
my_namer = namer()
oneofs = {}
# Are these variables proguarded and should we rename these?
use_namer = all(
len(i[4]) < 3 or i[4].endswith('_fld') for i in fields.values())
all_vars = [field[4] for field in fields.values()]
for number, (flabel, ftype, fenumormsg, fdefault,
var) in sorted(fields.items()):
field = message.field.add()
def extract_j2me(jar, cls, enums, gen_classes_j2me, protobuftype_cls, consts,
msg_path_to_obj, msg_to_referrers):
code = jar.decomp(cls, True)
cls = cls.replace('$', '.')
if not code.raw:
print('(Jad failed)')
return
"""
First step: look for calls to ProtoBufType.addElement(int, int, Object)
"""
code = sub('(?:new )?[\w$.]+\((-?\d+)[LDF]?\)', r'\1', code.raw)
fields_for_msg = defaultdict(str)
for var in findall('(\w+) = new ', code):
fields_for_msg[var]
while True:
# Case 1: handle embedded groups
decl = list(finditer('(\(new \w+\(("\w+")\)\)(?=((?:\.\w+\(\d+, \d+, .+?\))+)\)))', code))[::-1]
if not decl:
# Case 2: general case, handle messages
decl = list(finditer('( (\w+)(?=((?:\.\w+\(\d+, \d+, .+?\))+);))', code))
if not decl:
break
prefix, var, fields = decl[0].groups()
if var[0] == '"':
var = var.strip('"')
# If group, avoid name conflicts
while var in fields_for_msg:
var += '_'
else:
# If message, handle object variable reassignements
public_var = findall(' %s = ([a-zA-Z_][\w$]*);' % var, code[:decl[0].start()])
if public_var and public_var[-1] != 'null':
var = public_var[-1]
code = code.replace(prefix + fields, var, 1)
# Store var, fields
fields_for_msg[var] += fields
"""
Final step: Build the DescriptorProto object
"""
for var, fields in fields_for_msg.items():
message = DescriptorProto()
my_namer = namer()
summary = {}
print('\nIn %s.%s:' % (cls, var))
message.name = var
if fields:
for ftypeandlabel, fnumber, fdefaultormsg in findall('\.\w+\((\d+), (\d+), (.+?)\)', fields):
field = message.field.add()
# Use int32 instead of enum (we don't have enum contents),
# Use bytes and string instead of data and text (Protobuf 1 types).
types = {17: 'double', 18: 'float', 19: 'int64', 20: 'uint64', 21: 'int32', 22: 'fixed64', 23: 'fixed32', 24: 'bool', 25: 'bytes', 26: 'group', 27: 'message', 28: 'string', 29: 'uint32', 30: 'int32', 31: 'sfixed32', 32: 'sfixed64', 33: 'sint32', 34: 'sint64', 35: 'bytes', 36: 'string'}
ftype = types[int(ftypeandlabel) & 0xff]
labels = {1: 'required', 2: 'optional', 4: 'repeated'}
flabel = labels[int(ftypeandlabel) >> 8]
field.name = next(my_namer)
field.number = int(fnumber)
field.label = label_consts[flabel]
if fdefaultormsg != 'null':
if ftype in ('group', 'message'):
if '.' in fdefaultormsg:
field.type_name = '.' + fdefaultormsg
msg_to_referrers[fdefaultormsg].append((field.name, cls + '.' + var, ftype == 'group'))
if fdefaultormsg not in msg_path_to_obj: # Classes empty or to be created
msg_path_to_obj[fdefaultormsg] = DescriptorProto()
msg_path_to_obj[fdefaultormsg].name = fdefaultormsg.split('.')[-1]
else:
field.type_name = '.' + cls + '.' + fdefaultormsg
msg_to_referrers[cls + '.' + fdefaultormsg].append((field.name, cls + '.' + var, ftype == 'group'))
else:
if fdefaultormsg in consts:
fdefaultormsg = consts[fdefaultormsg]
parse_default(field, ftype, fdefaultormsg)
else:
fdefaultormsg = None
if ftype in ('group', 'message'):
ftype = 'bytes'
field.type = type_consts[ftype]
summary[int(fnumber)] = (flabel, ftype, fdefaultormsg)
msg_path_to_obj[cls + '.' + var] = message
print(summary)
def translate_descriptor(config_descriptor: Descriptor) -> ConfigSpec:
proto = DescriptorProto()
config_descriptor.CopyToProto(proto) # type: ignore
return translate_descriptor_proto(proto)