예제 #1
0
def shogunAllProto(argv):
    '''
    Generate XXX.pb.{h,cc} files from all available XXX.proto
    files in the source directory.

    Depends: protoc (protobuf-compiler)
    Input: .proto
    Output: .pb.cc, .pb.h
    '''
    ag = argparse.ArgumentParser()
    ag.add_argument('-o', help='write ninja file', type=str, default='all_proto.ninja')
    ag = ag.parse_args(argv)
    print(red('Argument Dump:'))
    pprint(vars(ag))

    # (1) initialize ninja file
    cursor = Writer(open(ag.o, 'w'))
    ninjaCommonHeader(cursor, ag)

    # (2) glob all proto
    protos = glob.glob(f'**/*.proto', recursive=True)
    print(cyan('AllProto:'), f'globbed {len(protos)} .proto files')

    # (3) generate .pb.cc, .pb.h
    for proto in protos:
        cursor.build([ proto.replace('.proto', '.pb.cc'),
            proto.replace('.proto', '.pb.h')], 'rule_PROTOC', proto)

    # done
    cursor.close()
예제 #2
0
def shogunProtoText(argv):
    '''
    Build a binary ELF executable named proto_text, which generates
    XXX.pb_text{.cc,.h,-impl.h} files from a given XXX.proto file.
    This binary file is for one-time use.

    Depends: shogunAllProto
    Input: bazelDump, cxx source
    Output: proto_text
    '''
    ag = argparse.ArgumentParser()
    ag.add_argument('-i', help='list of source files', type=str, required=True)
    ag.add_argument('-g', help='list of generated files', type=str, required=True)
    ag.add_argument('-o', help='where to write the ninja file', type=str, default='proto_text.ninja')
    ag = ag.parse_args(argv)
    print(red('Argument Dump:'))
    pprint(vars(ag))

    # (0) read bazel dump and apply hardcoded filters
    srclist = bazelPreprocess([l.strip() for l in open(ag.i, 'r').readlines()])
    genlist = bazelPreprocess([l.strip() for l in open(ag.g, 'r').readlines()])
    _, srclist = eGrep('.*.h$', srclist) # we don't need to deal with header here
    _, srclist = eGrep('^third_party', srclist) # no third_party stuff
    _, srclist = eGrep('.*windows/.*', srclist) # no windoge source
    _, srclist = eGrep('.*.proto$', srclist) # already dealt with in (2)

    # (1) Instantiate ninja writer
    cursor = Writer(open(ag.o, 'w'))
    ninjaCommonHeader(cursor, ag)

    # (2) deal with generated files
    # (2.1) .pb.cc and .pb.h files are generated in shogunAllProto
    _, genlist = eGrep('.*.pb.h$', genlist)
    pbcclist, genlist = eGrep('.*.pb.cc$', genlist)
    if len(genlist) > 0:
        print(yellow('Remainders:'), genlist)

    # (3) deal with source files
    cclist, srclist = eGrep('.*.cc', srclist)
    objlist = []
    for cc in cclist + pbcclist:
        obj = cc.replace('.cc', '.o')
        objlist.append(cursor.build(obj, 'rule_CXX_OBJ', cc)[0])
    if len(srclist) > 0:
        print(yellow('Remainders:'), srclist)

    # (4) link objects into the final ELF
    cursor.build(f'proto_text', 'rule_CXX_EXEC', inputs=objlist,
            variables={'LIBS': '-lpthread -lprotobuf -ldouble-conversion'})

    # done
    cursor.close()
예제 #3
0
def shogunCCOP(argv):
    '''
    Generate tensorflow cc ops : tensorflow/cc/ops/*.cc and *.h

    Depends: AllProto, proto_text, libtensorflow_framework
    Input: cc source, bazel dump
    Output: one-time-use binary "XXX_gen_cc" and generated .cc .h files.
    '''
    ag = argparse.ArgumentParser()
    ag.add_argument('-i', help='list of source files', type=str, required=True)
    ag.add_argument('-g', help='list of generated files', type=str, required=True)
    ag.add_argument('-o', help='where to write the ninja file', type=str, default='ccop.ninja')
    ag = ag.parse_args(argv)
    print(red('Argument Dump:'))
    pprint(vars(ag))

    # (0) read bazel dump and apply hardcoded filters
    genlist = bazelPreprocess([l.strip() for l in open(ag.g, 'r').readlines()])

    # (1) Instantiate ninja writer
    cursor = Writer(open(ag.o, 'w'))
    ninjaCommonHeader(cursor, ag)

    # (2) filter unrelated files, we only want cc_op related files.
    _, genlist = eGrep('.*.pb.h', genlist)
    _, genlist = eGrep('.*.pb.cc', genlist)
    _, genlist = eGrep('.*.pb_text.h', genlist)
    _, genlist = eGrep('.*.pb_text-impl.h', genlist)
    _, genlist = eGrep('.*.pb_text.cc', genlist)

    # (3) XXX_gen_cc
    # (3.1) deal with a missing source
    cursor.build('tensorflow/core/ops/user_ops.cc', 'COPY', inputs='tensorflow/core/user_ops/fact.cc')

    # (3.2) build several common objects
    main_cc = ['tensorflow/core/framework/op_gen_lib.cc',
        'tensorflow/cc/framework/cc_op_gen.cc',
        'tensorflow/cc/framework/cc_op_gen_main.cc',
        ]
    main_obj = [x.replace('.cc', '.o') for x in main_cc]
    for cc in main_cc:
        cursor.build(cc.replace('.cc', '.o'), 'rule_CXX_OBJ', inputs=cc)

    # (3.2) build executables and generate file with executable
    gen_ccopcc, genlist = eGrep('.*/cc/ops/.*.cc', genlist)
    gen_ccoph, genlist = eGrep('.*/cc/ops/.*.h', genlist)
    opnamelist = list(set(os.path.basename(x.replace('.cc', '').replace('.h', ''))
        for x in (gen_ccopcc + gen_ccoph) if 'internal' not in x ))

    for opname in opnamelist:
        coreopcc = 'tensorflow/core/ops/' + opname + '.cc'
        ccopcc   = 'tensorflow/cc/ops/'   + opname + '.cc'

        # build corresponding elf executable
        cursor.build(f'{opname}_gen_cc', 'rule_CXX_EXEC', inputs=[coreopcc] + main_obj,
            variables={'SHOGUN_EXTRA': '-I. -L. -ltf_ccop'})

        # generate file
        cursor.build([ccopcc.replace('.cc', '.h'), ccopcc], 'rule_CC_OP_GEN', inputs=f'{opname}_gen_cc',
                variables={'cc_op_gen_internal': '0' if opname != 'sendrecv_ops' else '1'},
                implicit_outputs=[ccopcc.replace('.cc', '_internal.h'), ccopcc.replace('.cc', '_internal.cc')])

    ## done
    cursor.close()
예제 #4
0
def shogunTFLib_framework(argv):
    '''
    Build libtensorflow_framework.so. With slight modification, this
    function should be able to build libtensorflow_android.so too.

    Depends: AllProto, proto_text
    Input: bazelDump, cxx source
    Output: libtensorflow_framework.so
    '''
    ag = argparse.ArgumentParser()
    ag.add_argument('-i', help='list of source files', type=str, required=True)
    ag.add_argument('-g', help='list of generated files', type=str, required=True)
    ag.add_argument('-o', help='where to write the ninja file', type=str, default='libtensorflow_framework.ninja')
    ag = ag.parse_args(argv)
    print(red('Argument Dump:'))
    pprint(vars(ag))

    # (0) read bazel dump and apply hardcoded filters
    srclist = bazelPreprocess([l.strip() for l in open(ag.i, 'r').readlines()])
    genlist = bazelPreprocess([l.strip() for l in open(ag.g, 'r').readlines()])
    _, srclist = eGrep('.*proto_text.gen_proto_text_functions.cc', srclist)
    _, srclist = eGrep('^third_party', srclist)
    _, srclist = eGrep('.*/windows/.*', srclist) # no windoge source.

    # (1) Initialize ninja file
    cursor = Writer(open(ag.o, 'w'))
    ninjaCommonHeader(cursor, ag)

    # (2) deal with generated files
    # (2.1) .pb.h and .pb.cc are already generated by shogunAllProto
    gen_pbh, genlist = eGrep('.*.pb.h', genlist)
    gen_pbcc, genlist = eGrep('.*.pb.cc', genlist)

    # (2.2) .pb_text.*
    pbtlist = [x for x in genlist if any(x.endswith(y) for y in ('.pb_text.h', '.pb_text.cc', '.pb_text-impl.h'))]
    pbtlist = [x.replace('.pb_text.h', '.proto').replace('.pb_text.cc', '.proto').replace('.pb_text-impl.h', '.proto') for x in pbtlist]
    gen_pbth, genlist = eGrep('.*.pb_text.h', genlist)
    gen_pbtih, genlist = eGrep('.*.pb_text-impl.h', genlist)
    gen_pbtcc, genlist = eGrep('.*.pb_text.cc', genlist)
    for pbt in list(set(pbtlist)):
        cursor.build([
            pbt.replace('.proto', '.pb_text.h'),
            pbt.replace('.proto', '.pb_text.cc'),
            pbt.replace('.proto', '.pb_text-impl.h')
            ], 'rule_PROTO_TEXT', pbt)

    # (2.3) finish dealing with generated files
    if genlist:
        print(yellow('Remainders:'), genlist)
        assert(len(genlist) == 1)

    # (3) deal with source files
    # (3.1) filter-out headers
    _, srclist = eGrep('.*.proto$', srclist) # done in (2)
    src_hdrs, srclist = eGrep('.*.h$', srclist)

    # (3.2) compile .cc source
    src_cc, srclist = eGrep('.*.cc', srclist)
    objlist = []
    for cc in src_cc + gen_pbcc + gen_pbtcc + genlist:
        variables = {}
        obj = cursor.build(cc.replace('.cc', '.o'), 'rule_CXX_OBJ', inputs=cc, variables=variables)[0]
        objlist.append(obj)

    # (4) link the final executable
    cursor.build('libtensorflow_framework.so', 'rule_CXX_SHLIB', inputs=objlist,
            variables={'LIBS': '-lfarmhash -lhighwayhash -lsnappy -lgif'
            + ' -ldouble-conversion -lz -lprotobuf -ljpeg -lnsync -lnsync_cpp'
            + ' -lpthread',
            'SHOGUN_EXTRA': f'-Wl,--soname=libtensorflow_framework.so.{tf_soversion}'
            + f' -Wl,--version-script tensorflow/tf_framework_version_script.lds'
            + f'  -fvisibility=hidden'})

    # (5) a temporary shared object used by shogunCCOP
    libccop = [x for x in objlist if all(y not in x for y in ('core/kernels', 'core/ops'))]
    cursor.build('libtf_ccop.so', 'rule_CXX_SHLIB', inputs=libccop,
            variables={'LIBS': '-lfarmhash -lhighwayhash -lsnappy -lgif'
            + ' -ldouble-conversion -lz -lprotobuf -ljpeg -lnsync -lnsync_cpp'
            + ' -lpthread'})

    # done
    cursor.close()