Python encodeEntry示例

示例#1

def consumer_metadata(val):
    global buff_meta, size_meta
    global current_file_buf, curr_file_index
    item_ = val[0]
    file_name = val[1]

    if size_meta > args.space * 1000000 or (
            "/".join(curr_file_index.split('/')[0:-1]) != "/".join(
                file_name.split('/')[0:-1]) and curr_file_index != ""):
        m = hashlib.md5()
        m.update(buff_meta[0].SerializeToString())
        curr_hash = m.hexdigest()
        print('writing meta', os.path.join(args.output, curr_hash))
        writer = open(os.path.join(args.output, curr_hash), 'bw')
        decoder.encodeEntry(writer.write, buff_meta)
        writer.close()
        if args.mzip:
            subprocess.call([
                "7z", 'a', '-mx9',
                os.path.join(args.output, curr_hash) + ".7z",
                os.path.join(args.output, curr_hash)
            ])
            os.remove(os.path.join(args.output, curr_hash))
        buff_meta = []
        size_meta = 0
    if curr_file_index != file_name:
        curr_file_index = file_name
    print("%d elements in buffer" % len(buff_meta))
    size_meta += len(item_.SerializeToString())
    buff_meta.append(item_)

示例#2

文件： list.py 项目： Networks-Learning/wikipedia-protobuf

def store(buff):
    m = hashlib.md5()
    m.update(buff[0].SerializeToString())
    curr_hash = m.hexdigest()
    writer = open(os.path.join(args.output, curr_hash), 'wb')
    decoder.encodeEntry(writer.write, buff)
    writer.close()

示例#3

文件： extract_db.py 项目： Networks-Learning/wikipedia-protobuf

def finalize():
  global buff_meta
  if len(buff_meta) > 0:
    print('writing last set of meta')

    m = hashlib.md5()
    m.update(buff_meta[0].SerializeToString())
    curr_hash = m.hexdigest()
    writer = open(os.path.join(args.output,curr_hash),'bw')
    decoder.encodeEntry(writer.write,buff_meta)
    writer.close()
    if args.mzip:
      subprocess.call(["7z",'a','-mx9',os.path.join(args.output,curr_hash)+".7z",os.path.join(args.output,curr_hash)])
      os.remove(os.path.join(args.output,curr_hash))
    buff_meta = []
    size_meta = 0

示例#4

def finalize():
    global buff_meta
    if len(buff_meta) > 0:
        print('writing last set of meta')

        m = hashlib.md5()
        m.update(buff_meta[0].SerializeToString())
        curr_hash = m.hexdigest()
        writer = open(os.path.join(args.output, curr_hash), 'bw')
        decoder.encodeEntry(writer.write, buff_meta)
        writer.close()
        if args.mzip:
            subprocess.call([
                "7z", 'a', '-mx9',
                os.path.join(args.output, curr_hash) + ".7z",
                os.path.join(args.output, curr_hash)
            ])
            os.remove(os.path.join(args.output, curr_hash))
        buff_meta = []
        size_meta = 0

示例#5

文件： extract_db.py 项目： Networks-Learning/wikipedia-protobuf

def consumer_metadata(val):
  global buff_meta,size_meta
  global current_file_buf, curr_file_index
  item_ = val[0]
  file_name = val[1]

  if size_meta > args.space*1000000 or ("/".join(curr_file_index.split('/')[0:-1]) != "/".join(file_name.split('/')[0:-1]) and curr_file_index != ""):
    m = hashlib.md5()
    m.update(buff_meta[0].SerializeToString())
    curr_hash = m.hexdigest()
    print('writing meta', os.path.join(args.output,curr_hash))
    writer = open(os.path.join(args.output,curr_hash),'bw')
    decoder.encodeEntry(writer.write,buff_meta)
    writer.close()
    if args.mzip:
      subprocess.call(["7z",'a','-mx9',os.path.join(args.output,curr_hash)+".7z",os.path.join(args.output,curr_hash)])
      os.remove(os.path.join(args.output,curr_hash))
    buff_meta = []
    size_meta = 0
  if curr_file_index != file_name:
    curr_file_index = file_name
  print("%d elements in buffer" % len(buff_meta))
  size_meta += len(item_.SerializeToString())
  buff_meta.append(item_)