예제 #1
0
class GitVCSTests(unittest2.TestCase):
    def setUp(self):
        self.config = Config()
        init_data = {}
        self.gitcvs = GitVCS(**init_data)

    def tearDown(self):
        self.config.reset_configuration()

    def test_find_commit(self):
        commits = self.gitcvs.get_commits()
        self.failIf(len(commits) == 0)
        expected = commits[0]
        result = self.gitcvs.find_commit(str(expected))[0]
        self.assertEquals(result.hexsha, expected.hexsha)
        self.assertEquals(type(result), type(expected))

    def test_git_stub(self):
        commit_hash = "76191292fe70015eeee241fb9b663af6953ffb5e"
        commits = self.gitcvs.get_commits()
        data = self.gitcvs.find_commit(commit_hash)

        if data is not None and len(data) > 2:
            commitA = data[0]
            commitB = commits[data[1] + 1]
            files_list = self.gitcvs.get_modified_files(commitB, commitA, None)
            expected_files = set()
            expected_files.add(".travis.yaml")
            expected_files.add(".travis.yml")
            expected_files.add("README.md")
            expected_files.add("build_thrift.sh")
            expected_files.add("setup.py")
            expected_files.add("thrift_medusa/config/travis-ci.yaml")
            for item in files_list:
                self.assertTrue(item in expected_files)
예제 #2
0
 def __get_client__(self):
     self.config = Config()
     self.config.reset_configuration()
     compiler = None
     for item in self.config.get_thrift_option("compilers"):
         compiler = ThriftCompiler(item)
     return Documentation([], compiler)
예제 #3
0
파일: client.py 프로젝트: safaci2000/medusa
 def __init__(self, services, thrift_compiler):
     self.compiler = thrift_compiler
     self.config = Config()
     self.sandbox_work = self.config.work_dir
     self.status = Status()
     self.thrift_helper = Thrift(self.compiler)
     self.log = Log(log_file="status.log", logger_name="status").log
     self.services = services
예제 #4
0
    def __get_client__(self):
        self.config = Config()
        compiler = None
        for item in self.config.get_thrift_option("compilers"):
            compiler = ThriftCompiler(item)
            break

        return RubyClient([], compiler)
예제 #5
0
 def __init__(self):
     self.client_list = []
     self.remove_structure("logs")
     wize_mkdir("logs")
     self.business_objects = []
     self.service_objects = []
     self.config = Config()
     self.log = Log(log_file=os.path.join(self.config.repo_dir, "logs/status.log"),
                    logger_name="definitions").get_logger()
예제 #6
0
 def test_bad_init_data(self):
     dict = {}
     conf = Config()
     bad_compiler = ThriftCompiler(dict)
     self.assertEquals(bad_compiler.version, "0.6.1")
     self.assertEquals(bad_compiler.postfix, "")
     language = "java"
     self.assertEquals(bad_compiler.is_language_supported("java"), False)
     self.assertEquals(bad_compiler.language_options(), conf.get_thrift_option("global_compiler_options")[language])
예제 #7
0
def display_compilers():
    """
    Will display the list of all current supported compilers defined in configuration.
    """
    config = Config()
    compilers = config.get_thrift_option("compilers")
    for item in compilers:
        print("found compiler %s with binary at: %s which supports: %s languages" % (item.get('name'), item.get('bin'),
                                                                                     ', '.join(map(str, item.get(
                                                                                         'supported_languages')))))
    sys.exit(0)
예제 #8
0
 def test_bad_init_data(self):
     dict = {}
     conf = Config()
     bad_compiler = ThriftCompiler(dict)
     self.assertEquals(bad_compiler.version, "0.6.1")
     self.assertEquals(bad_compiler.postfix, "")
     language = "java"
     self.assertEquals(bad_compiler.is_language_supported("java"), False)
     self.assertEquals(
         bad_compiler.language_options(),
         conf.get_thrift_option("global_compiler_options")[language])
예제 #9
0
 def __get_client__(self):
     self.config = Config()
     self.config.reset_configuration()
     compiler = None
     for item in self.config.get_thrift_option("compilers"):
         compiler = ThriftCompiler(item)
     return Client([], compiler)
예제 #10
0
    def setUp(self):
        self.config = Config()
        self.config.reset_configuration()
        compilers = self.config.get_thrift_option("compilers")
        if compilers is None or len(compilers) == 0:
            self.helper = Thrift(None)
        else:
            thrift_compiler = ThriftCompiler(compilers[0])
            self.helper = Thrift(thrift_compiler)

        project_path = os.path.join(
            os.path.dirname(
                os.path.abspath(inspect.getfile(inspect.currentframe()))),
            "../")
        project_path = os.path.realpath(project_path)
        self.config.repo_dir = project_path
예제 #11
0
    def __get_client__(self):
        self.config = Config()
        compiler = None
        for item in self.config.get_thrift_option("compilers"):
            compiler = ThriftCompiler(item)
            break

        return RubyClient([], compiler)
예제 #12
0
class RubyClientTests(unittest.TestCase):
    def setUp(self):
        self.client = self.__get_client__()
        self.client.initialize()
        self.service_name = os.path.join(os.getcwd(), "../thrift/services/", "wizecommerce.services.example.thrift")

    def __get_client__(self):
        self.config = Config()
        compiler = None
        for item in self.config.get_thrift_option("compilers"):
            compiler = ThriftCompiler(item)
            break

        return RubyClient([], compiler)

    def tearDown(self):
        self.config.reset_configuration()
예제 #13
0
 def __init__(self, services, thrift_compiler):
     self.compiler = thrift_compiler
     self.config = Config()
     self.sandbox_work = self.config.work_dir
     self.status = Status()
     self.thrift_helper = Thrift(self.compiler)
     self.log = Log(log_file="status.log", logger_name="status").log
     self.services = services
class DocumentationClientTests(unittest2.TestCase):
    def setUp(self):
        self.dict = {}
        self.client = self.__get_client__()
        # self.client.initialize()
        self.service_name = os.path.join(os.getcwd(), "../thrift/services/", "wizecommerce.services.example.thrift")

    def __get_client__(self):
        self.config = Config()
        self.config.reset_configuration()
        compiler = None
        for item in self.config.get_thrift_option("compilers"):
            compiler = ThriftCompiler(item)
        return Documentation([], compiler)

    def test_dummy(self):
        self.assertEquals(1,1)
예제 #15
0
def set_compiler(override_compiler):
    """
    Allows user to explicitly use a particular compiler when building thrift artifacts.
    """
    config = Config()
    compilers = config.get_thrift_option("compilers")
    found = False
    compiler = None
    for item in compilers:
        if item['name'] == override_compiler:
            found = True
            compiler = item

    if not found:
        print("compiler {compiler} was not found in yaml configuration".format(compiler=override_compiler))
        sys.exit(1)

    config.set_thrift_option("compilers", [compiler])
예제 #16
0
class RubyClientTests(unittest.TestCase):
    def setUp(self):
        self.client = self.__get_client__()
        self.client.initialize()
        self.service_name = os.path.join(
            os.getcwd(), "../thrift/services/",
            "wizecommerce.services.example.thrift")

    def __get_client__(self):
        self.config = Config()
        compiler = None
        for item in self.config.get_thrift_option("compilers"):
            compiler = ThriftCompiler(item)
            break

        return RubyClient([], compiler)

    def tearDown(self):
        self.config.reset_configuration()
예제 #17
0
class DocumentationClientTests(unittest2.TestCase):
    def setUp(self):
        self.dict = {}
        self.client = self.__get_client__()
        # self.client.initialize()
        self.service_name = os.path.join(
            os.getcwd(), "../thrift/services/",
            "wizecommerce.services.example.thrift")

    def __get_client__(self):
        self.config = Config()
        self.config.reset_configuration()
        compiler = None
        for item in self.config.get_thrift_option("compilers"):
            compiler = ThriftCompiler(item)
        return Documentation([], compiler)

    def test_dummy(self):
        self.assertEquals(1, 1)
예제 #18
0
    def setUp(self):
        self.config = Config()
        self.config.reset_configuration()
        compilers = self.config.get_thrift_option("compilers")
        if compilers is None or len(compilers) == 0:
            self.helper = Thrift(None)
        else:
            thrift_compiler = ThriftCompiler(compilers[0])
            self.helper = Thrift(thrift_compiler)

        project_path = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), "../")
        project_path = os.path.realpath(project_path)
        self.config.repo_dir = project_path
예제 #19
0
    def __init__(self, compiler):
        self.compiler=compiler
        self.version_pattern = re.compile("const\s+string\s+VERSION\s*=\s*['\"]+(.*)['\"]+")
        self.java_namespace_pattern = re.compile("namespace\s+java\s+(.*)")
        self.ruby_namespace_pattern = re.compile("namespace\s+rb\s+(.*)")
        self.data_type_pattern = re.compile(
            "\s+(\d+):\s+([required,optional]+)\s+([bool,byte,i16,i32,i64,string,double,string]+)\s+(\w+)")

        self.config = Config()
        self.log = Log(log_file="status.log", logger_name="status").log
        # defines the language compiler to spawn.
        self.build = {"ruby": self.__thrift_ruby_build__, "java": self.__thrift_java_build__,
                      "doc": self.__thrift_doc_build__}
예제 #20
0
class JavaClientTests(unittest.TestCase):
    def setUp(self):
        self.client = self.__get_client__()
        self.client.initialize()
        self.service_name = os.path.join(os.getcwd(), "../thrift/services/", "wizecommerce.services.example.thrift")


    def __get_client__(self):
        self.config = Config()
        compiler = None
        for item in self.config.get_thrift_option("compilers"):
            compiler = ThriftCompiler(item)
            compiler.postfix = ""
            break

        return JavaClient([], compiler)

    @httprettified
    def test_check_version_pretty(self):
        self.config.set_local(False)
        expected = False
        expected_url = 'http://nexus.corp.nextag.com:8081/content/repositories/releases/com/wizecommerce/data/tag-client/0.0.13/tag-client-0.0.13.jar'
        HTTPretty.register_uri(HTTPretty.GET, expected_url, body="Hello")
        check_value = self.client.check_version(groupId='com.wizecommerce.data', artifactId='tag-client',
                                                version='0.0.13')
        self.failUnlessEqual(check_value, expected)
        check_value = self.client.check_version(groupId='com.wizecommerce.data', artifactId='tag-client',
                                                version='X.Y.ZZ')
        expected = True
        self.failUnlessEqual(check_value, expected)


    @httprettified
    def test_check_shaded_version_pretty(self):
        self.config.set_local(False)
        expected = False
        expected_url = 'http://crepo.corp.nextag.com/repo/components/com/wizecommerce/data/tag-client-shaded/0.0.13/tag-client-shaded-0.0.13.jar'
        HTTPretty.register_uri(HTTPretty.GET, expected_url, body="Hello")
        check_value_shaded = self.client.check_shaded_version(groupId='com.wizecommerce.data', artifactId='tag-client',
                                                              version='0.0.13')
        print check_value_shaded
        print expected

        assert check_value_shaded == expected
        self.failUnlessEqual(check_value_shaded, expected)

        check_value_shaded = self.client.check_shaded_version(groupId='com.wizecommerce.data', artifactId='tag-client',
                                                              version='X.Y.ZZ')
        expected = True
        assert check_value_shaded == expected

    def tearDown(self):
        self.config.reset_configuration()
예제 #21
0
class BaseClientTests(unittest2.TestCase):
    def setUp(self):
        self.dict = {}
        self.client = self.__get_client__()
        # self.client.initialize()
        self.service_name = os.path.join(
            os.getcwd(), "../thrift/services/",
            "wizecommerce.services.example.thrift")

    def __get_client__(self):
        self.config = Config()
        self.config.reset_configuration()
        compiler = None
        for item in self.config.get_thrift_option("compilers"):
            compiler = ThriftCompiler(item)
        return Client([], compiler)

    def test_not_implemented_methods(self):
        client = self.__get_client__()
        self.assertRaises(NotImplementedError, client.__build_dependency__,
                          "DummyFile")
        self.assertRaises(NotImplementedError, client.__build_client__,
                          "service")
        self.assertRaises(NotImplementedError, client.check_version,
                          **self.dict)
        self.assertRaises(NotImplementedError,
                          client.__deploy_production_artifact__, self.dict,
                          "boo")
        self.assertRaises(NotImplementedError,
                          client.__deploy_local_artifact__, self.dict, "boo")
        self.assertRaises(NotImplementedError, client.finalize)
        self.assertRaises(NotImplementedError, client.initialize)

    def test_deploy_object(self):
        client = self.__get_client__()
        self.config.set_local(True)
        self.assertRaisesRegexp(NotImplementedError,
                                ".*Deploy Local Artifact.*",
                                client.deploy_object, self.dict, "dummy")
        client = self.__get_client__()
        self.config.set_local(False)
        self.assertRaisesRegexp(NotImplementedError,
                                ".*Deploy Production Artifact.*",
                                client.deploy_object, self.dict, "dummy")

    def test_sandbox(self):
        client = self.__get_client__()
        client.set_sandbox("DummySandbox")
        self.assertEquals("DummySandbox", client.get_sandbox())
예제 #22
0
class BaseClientTests(unittest2.TestCase):
    def setUp(self):
        self.dict = {}
        self.client = self.__get_client__()
        # self.client.initialize()
        self.service_name = os.path.join(os.getcwd(), "../thrift/services/", "wizecommerce.services.example.thrift")

    def __get_client__(self):
        self.config = Config()
        self.config.reset_configuration()
        compiler = None
        for item in self.config.get_thrift_option("compilers"):
            compiler = ThriftCompiler(item)
        return Client([], compiler)

    def test_not_implemented_methods(self):
        client = self.__get_client__()
        self.assertRaises(NotImplementedError, client.__build_dependency__, "DummyFile")
        self.assertRaises(NotImplementedError, client.__build_client__, "service")
        self.assertRaises(NotImplementedError, client.check_version, **self.dict)
        self.assertRaises(NotImplementedError, client.__deploy_production_artifact__, self.dict, "boo")
        self.assertRaises(NotImplementedError, client.__deploy_local_artifact__, self.dict, "boo")
        self.assertRaises(NotImplementedError, client.finalize)
        self.assertRaises(NotImplementedError, client.initialize)

    def test_deploy_object(self):
        client = self.__get_client__()
        self.config.set_local(True)
        self.assertRaisesRegexp(
            NotImplementedError, ".*Deploy Local Artifact.*", client.deploy_object, self.dict, "dummy"
        )
        client = self.__get_client__()
        self.config.set_local(False)
        self.assertRaisesRegexp(
            NotImplementedError, ".*Deploy Production Artifact.*", client.deploy_object, self.dict, "dummy"
        )

    def test_sandbox(self):
        client = self.__get_client__()
        client.set_sandbox("DummySandbox")
        self.assertEquals("DummySandbox", client.get_sandbox())
예제 #23
0
class PublishClient():
    """
      The purpose of this class is to setup the environment for processing various service objects
    """

    def __init__(self):
        self.client_list = []
        self.remove_structure("logs")
        wize_mkdir("logs")
        self.business_objects = []
        self.service_objects = []
        self.config = Config()
        self.log = Log(log_file=os.path.join(self.config.repo_dir, "logs/status.log"),
                       logger_name="definitions").get_logger()

    def remove_structure(self, dir):
        """
            Simple method that deletes a directory
        """
        cmd = ['rm', '-fr', dir]
        self.local_assert(subprocess.call(cmd), "failed to run command: {cmd}".format(cmd=str(cmd)))
        return 0

    def local_assert(self, exit_code, message):
        """
        Defines a cleaner version of an assert that is probably more helpful.
        """
        if exit_code != 0:
            self.log.error(message)
            sys.exit(exit_code)

    def create_structure(self):
        """
            Remove old directory structure and re-copy all the files and dependencies
            from the appropriate repos.
        """
        self.remove_structure(self.config.work_dir)
        os.mkdir(self.config.work_dir)

        self.business_objects = build_file_list(self.config.get_path(type="business_object"), ".thrift")
        self.service_objects = build_file_list(self.config.get_path(type="service_object"), ".thrift")
        self.enum_objects = build_file_list(self.config.get_path(type="enum_object"), ".thrift")
        self.exception_objects = build_file_list(self.config.get_path(type="exception_object"), ".thrift")


    def update_client_list(self, thrift_objects, compilers):
        """
          Build a list of all clients for each language and compiler type.

          Note: Multiple thrift compilers not currently supported.
        """
        self.client_list = []
        for item in compilers:
            if self.config.is_java and item.is_language_supported("java"):
                self.client_list.append(JavaClient(thrift_objects, item))
            if self.config.is_ruby and item.is_language_supported("ruby"):
                self.client_list.append(RubyClient(thrift_objects, item))
            if self.config.is_doc_enabled and item.is_language_supported("doc"):
                self.client_list.append(Documentation(thrift_objects, item))

    def process_thrift_services(self):
        """
            This method will iterate through all the service and business object thrift files, and
            deploy the maven artifacts and its dependencies
        """
        compiler_list = []
        for item in self.config.get_thrift_option("compilers"):
            t = ThriftCompiler(item)
            compiler_list.append(t)

        #ensure that vcs is enabled, and thrift-file override hasn't been passed in.
        thrift_objects = []
        if self.config.is_local() and self.config.get_service_override() is not None:
            pass
        elif not self.config.is_vcs or self.config.is_local():
            #flatten list
            thrift_objects = self.service_objects + self.business_objects + self.enum_objects + self.exception_objects
        else:
            vcs = self.config.get_vcs_instance()
            file_objects = vcs.get_modified_files()
            if file_objects is None or len(file_objects) == 0:
                self.config.is_vcs = False
                thrift_objects = self.service_objects + self.business_objects + self.enum_objects + self.exception_objects
            else:
                self.log.info("Using list of objects from VCS")
                thrift_objects = map(lambda current_file: os.path.basename(current_file), file_objects)
                self.log.info("VCS object list is: " + str(thrift_objects))

        if self.config.is_local() and self.config.get_service_override() is not None:
            self.service_objects = []
            thrift_objects = [self.config.get_service_override()]

        self.update_client_list(thrift_objects, compiler_list)

        process_list = []

        for client in self.client_list:
            p = Process(target=client.run)
            p.start()
            process_list.append(p)

        #wait for all threads that have been started to terminate.
        map(lambda proc: proc.join(), process_list)

        # #Check exit codes
        for proc in process_list:
            self.local_assert(proc.exitcode, str(proc))
예제 #24
0
 def setUp(self):
     self.config = Config()
     init_data = {}
     self.gitcvs = GitVCS(**init_data)
예제 #25
0
class Thrift():
    def __init__(self, compiler):
        self.compiler=compiler
        self.version_pattern = re.compile("const\s+string\s+VERSION\s*=\s*['\"]+(.*)['\"]+")
        self.java_namespace_pattern = re.compile("namespace\s+java\s+(.*)")
        self.ruby_namespace_pattern = re.compile("namespace\s+rb\s+(.*)")
        self.data_type_pattern = re.compile(
            "\s+(\d+):\s+([required,optional]+)\s+([bool,byte,i16,i32,i64,string,double,string]+)\s+(\w+)")

        self.config = Config()
        self.log = Log(log_file="status.log", logger_name="status").log
        # defines the language compiler to spawn.
        self.build = {"ruby": self.__thrift_ruby_build__, "java": self.__thrift_java_build__,
                      "doc": self.__thrift_doc_build__}

    def __setup_maven__(self, sandbox="default"):
        """
            removes the old maven structure, and recreates it
        """
        if sandbox == "default":
            sanbox = self.config.work_dir

        os.system("rm -fr %s" % (os.path.join(sandbox, "src")))
        os.system("rm -f %s" % os.path.join(sandbox, "pom.xml"))
        wize_mkdir(self.config.maven_dir)
        wize_mkdir(self.config.get_thrift_option("thrift"))

    def check_file(self, thrift_file):
        """
            checks whether a file exists or not in the workdir
        """
        thrift_file = self.get_thrift_full_path(thrift_file)


        if not os.path.exists(thrift_file):
            raise IOError("file {prettyfile} does not exist".format(prettyfile=thrift_file))


    def __thrift_java_build__(self, thrift_file, sandbox="default"):
        """
        compile the thrift file passed in to java format.
        :param thrift_file:
        """
        if sandbox == "default":
            sandbox = self.config.work_dir
        thrift_file = self.get_thrift_full_path(thrift_file)
        #removes previously generated code
        os.system("rm -fr %s" % os.path.join(sandbox, self.config.java_sandbox))
        os.chdir(sandbox)
        self.log("Building java class for %s" % thrift_file)
        cmd = "{thrift_binary} -I {business_objects} -I {exceptions} -I {enums} -I {services} {options} {language_options} {file}".format(
            business_objects=self.config.get_path(type="business_object"),
            services=self.config.get_path(type="service_object"), exceptions=self.config.get_path(type="exception_object"),
            enums=self.config.get_path(type="enum_object"), thrift_binary=self.compiler.bin,
            file=thrift_file, options=self.compiler.options, language_options=self.compiler.language_options("java"))

        exit_code = subprocess.call(shlex.split(cmd))
        if not exit_code == 0:
            self.log("failed to compile thrift file {file}".format(file=thrift_file))
            sys.exit(1)

    def __thrift_doc_build__(self, thrift_file, sandbox="default"):
        if sandbox == "default":
            sandbox = self.config.work_dir
        thrift_file = self.get_thrift_full_path(thrift_file)
        os.system("rm -fr %s" % os.path.join(sandbox, self.config.doc_sandbox))
        wize_mkdir(self.config.doc_sandbox)

        full_path = self.get_thrift_full_path(thrift_file)
        dependencies = self.read_thrift_dependencies_recursively(full_path)
        for item in dependencies:
                dependency_file = self.get_thrift_full_path(item)
                cmd = "{thrift_binary} -I {business_objects} -I {exceptions} -I {enums} -I {services} {options} " \
                      "{language_options} -o {destination} {file} ".format(
                    business_objects=self.config.get_path(type="business_object"),
                    exceptions=self.config.get_path(type="exception_object"),
                    enums=self.config.get_path(type="enum_object"),
                    services=self.config.get_path(type="service_object"),
                    thrift_binary=self.compiler.bin, file=dependency_file, options=self.compiler.options,
                    language_options=self.compiler.language_options("doc"),
                    destination=sandbox)
                exit_code = subprocess.call(shlex.split(cmd))
                if not exit_code == 0:
                    self.log("failed to compile thrift dependency {dependency} for file {file}".format(
                        dependency=dependency_file, file=dependency_file))
                    sys.exit(1)


        self.log("Generating documentation for %s" % thrift_file)

        cmd = "{thrift_binary} -I {business_objects} -I {exceptions} -I {enums} -I {services} {options} " \
              "{language_options} -o {destination} {file}".format(
            business_objects=self.config.get_path(type="business_object"),
            exceptions=self.config.get_path(type="exception_object"),
            enums=self.config.get_path(type="enum_object"), services=self.config.get_path(type="service_object"),
            thrift_binary=self.compiler.bin,
            file=thrift_file, options=self.compiler.options, language_options=self.compiler.language_options("doc"),
            destination=sandbox)

        exit_code = subprocess.call(shlex.split(cmd))
        if not exit_code == 0:
            self.log("failed to compile thrift file {file}".format(file=thrift_file))
            sys.exit(1)


    def __thrift_ruby_build__(self, thrift_file, sandbox="default"):
        """
        compile the thrift file passed in to ruby format.
        :param thrift_file:
        """
        if sandbox == "default":
            sandbox = self.config.work_dir

        os.chdir(sandbox)
        thrift_file = self.get_thrift_full_path(thrift_file)
        os.system("rm -fr %s" % (self.config.get_ruby_option("sandbox", True)))
        os.system("rm -fr %s" % (os.path.join(self.config.work_dir, "ruby")))
        cmd = "{thrift_binary} -I {business_objects} -I {exceptions} -I {enums} -I {services} {options} " \
              "{language_options}  {file}".format( business_objects=self.config.get_path(type="business_object"),
            exceptions=self.config.get_path(type="exception_object"),
            enums=self.config.get_path(type="enum_object"), services=self.config.get_path(type="service_object"),
            thrift_binary=self.compiler.bin,
            file=thrift_file, options=self.compiler.options, language_options=self.compiler.language_options("ruby"))
        exit_code = subprocess.call(shlex.split(cmd))
        if exit_code != 0:
            self.log("failed to compile thrift file %s" % thrift_file)
            sys.exit(exit_code)

        self.log("Building ruby class for %s" % thrift_file)

    def is_service(self, thrift_file):
        """
         boolean check on thrift file whether true if it contains the service_prefix in the file name.

         Usually it's 'service' or whatever you overrode the value with in the config file.
        """
        if thrift_file.find(self.config.get_global_option("service_prefix")) != -1:
            return True
        else:
            return False


    def is_business_object(self, thrift_file):
        """
         boolean check on thrift file whether true if it contains the object_prefix in the file name.

         Usually it's 'bizobj' or whatever you overrode the value with in the config file.
        """
        if thrift_file.find(self.config.get_global_option("object_prefix")) != -1:
            return True
        else:
            return False

    def thrift_build(self, thrift_file, language="java", sandbox="default"):
        """
            compiles a single thrift file and copies content to
            maven structure for consumption.
        """
        if sandbox == "default":
            sandbox = self.config.work_dir
        if self.build.has_key(language):
            self.build.get(language)(thrift_file, sandbox)

    def get_thrift_full_path(self, thrift_file):
        if thrift_file is None:
            return None

        if thrift_file.find("/") >= 0:
            return thrift_file

        if thrift_file.find(self.config.get_global_option("service_prefix")) != -1:
            return os.path.join(self.config.get_path(type="service_object"), thrift_file)
        elif thrift_file.find(self.config.get_global_option("enum_prefix")) != -1:
            return os.path.join(self.config.get_path(type="enum_object"), thrift_file)
        elif thrift_file.find(self.config.get_global_option("exception_prefix")) != -1:
            return os.path.join(self.config.get_path(type="exception_object"), thrift_file)
        else:
            return os.path.join(self.config.get_path(type="business_object"), thrift_file)

    def read_thrift_dependencies_recursively(self, thrift_file):
        dependencies = set(self.read_thrift_dependencies(thrift_file))
        if len(dependencies) is 0:
            return set()
        for item in dependencies:
            dependencies = dependencies.union(set(self.read_thrift_dependencies_recursively(item)))

        return dependencies

    def check_itemized_contraints(self, line, thrift_file):
        """
        If enabled via the yaml configuration.  This method will ensure that certain contraints are enforced.

        These constraints are applicable on individual lines items and do not need full context of the file.

        If any of them fail, it will throw a ConstraintException
        """
        if not self.config.constraints_enabled:
            return

        if self.config.visibility_check_enabled:
            if line.find("bool") >= 0 or line.find("i64") >= 0 or line.find("i32") >= 0 or \
                line.find("byte") >= 0 or line.find("double") >= 0 or line.find("string") >= 0:
                    if line.find("optional") == -1 and line.find("required") == -1 and line.find("VERSION") == -1:
                        self.log("Error: package visibility flag omitted for line: {line} in file: {file}."
                                 .format(line=line.replace("\n", ""), file=thrift_file))
                        raise ConstraintException("visibility flag is missing.  Please use 'optional' or 'required' "
                                                  "to set visibility.  Please check log file for more details.")

    def check_constraints(self, thrift_file, diff, raw_data):
        """
        If enabled via the yaml configuration.  This method will ensure that certain contraints are enforced.

        If any of them fail, it will throw a ConstraintException
        """
        if not self.config.constraints_enabled:
            return

        if self.config.empty_constraint_enabled and self.is_business_object(thrift_file):
            match = re.search("bool.*empty", raw_data)
            if match is not None and raw_data.find("required") == -1:
                self.log("Error: bool empty field must be a required field. Missing in file {file}".format(
                    file=thrift_file))
                raise ConstraintException("empty flag is required.  Please ensure to include a boolean empty field "
                                              "in your struct")

        #Everything below this line in under Development.
        ## Needs further development and interaction with VCS module
        if self.config.check_field_ordering_enabled:
            ### pseudologic:  get current thrift file, call a helper method which will extract the field Ids, names,
            ### and other metrics.  retrieve previous version of file and do the same operation, compare resulting
            ### constructs.  If thrift patterns are broke.  ie.  field ordering is changed, required is removed and such.
            ### throw an exception.
            pass
        if self.config.check_version_increment_enabled:
            ## pseudologic:  check thrift_file via VCS.  If file is changed, ensure that VERSION number has changed.
            if diff is not None and diff != "" and diff.find("VERSION") == -1:
                self.log("Error, You forgot to increment the version for {file}".format(file=thrift_file))
                raise ConstraintException("a file change requires a version increment.")


    ## Under development, not integrated into any clients yet.
    def decipher_thrift_file(self, thrift_file):
        """
        This method will read a thrift file and pull out any data that we understand
        for further logic processing.

        ie.  numeric index, type, field name, VERSION constant etc.

        """
        if self.is_service(thrift_file):
            return {}

        thrift_file = self.get_thrift_full_path(thrift_file)
        self.check_file(thrift_file)
        fp = open(thrift_file, 'r')
        data = fp.readlines()
        fp.close()
        thrift_meta = {}
        for line in data:
            match = self.version_pattern.match(line)
            if match is not None:
                thrift_meta['version'] = match.group(1)
            match = self.java_namespace_pattern.match(line)
            if match is not None:
                thrift_meta['java_namespace'] = match.group(1)
            match = self.ruby_namespace_pattern.match(line)
            if match is not None:
                thrift_meta['ruby_namespace'] = match.group(1)
            match = self.data_type_pattern.match(line)
            if match is not None:
                if match.lastindex != 4:
                    self.log("Error: found a discrepancy when parsing {file}".format(thrift_file=file))
                    sys.exit(1)
                constraint_type = {}
                constraint_type['visibility'] = match.group(2)
                constraint_type['type'] = match.group(3)
                constraint_type['name'] = match.group(4)
                thrift_meta[match.group(1)] = constraint_type

        return thrift_meta

    def read_thrift_dependencies(self, thrift_file):
        """
            Parse a thrift file and extract the includes returning it as a list of
            it's dependencies.
        """
        self.log("Processing %s" % thrift_file)
        is_enum = False
        is_exception = False
        thrift_file = self.get_thrift_full_path(thrift_file)
        self.check_file(thrift_file)
        fp = open(thrift_file, 'r')
        data = fp.readlines()
        raw_data = "".join(data)
        self.check_constraints(thrift_file, None, raw_data)

        fp.close()
        includes = []
        object_count = 0
        for line in data:
            #skip comments
            if line.startswith("#"):
                continue
            if not self.is_service(thrift_file):
                self.check_itemized_contraints(line, thrift_file)
            if line.startswith("enum"):
                is_enum = True
            if line.startswith("exception"):
                is_exception = True
            if line.startswith("include"):
                line = line.strip()
                begin = line.find("\"")
                end = line.find("\"", begin + 1)
                includes.append(line[begin + 1:end])
            if line.startswith("struct") or line.startswith("enum") or line.startswith("exception"):
                object_count += 1

        if object_count > 1:
            self.log(
                "tsk..tsk..tsk, you tried to build an object {file} that contained multiple structs/enums.".format(
                    file=thrift_file))
            sys.exit(1)

        if is_enum:
            ndx = thrift_file.find(self.config.get_global_option("enum_prefix"))
            if ndx < 0:
                self.log("error, any enum should be defined inside a .enum file")
                sys.exit(1)

        if is_exception:
            ndx = thrift_file.find(self.config.get_global_option("exception_prefix"))
            if ndx < 0:
                self.log("error, any exception should be defined inside a .exception file")
                sys.exit(1)

        return includes

    def read_thrift_properties(self, thrift_file):
        """
            Given a thrift file it will return the groupid, version and artifact name
            as hashmap/dictionary.
        """
        thrift_file = self.get_thrift_full_path(thrift_file)
        if not os.path.exists(thrift_file):
            return {}
        meta_data = {}
        f = open(thrift_file)
        raw = f.read()
        f.close()

        meta_data["VERSION"] = self.__extract_string__(pattern="VERSION", raw=raw, default="0.0.0")
        meta_data["GROUPID"] = self.__extract_string__(pattern="GROUPID", raw=raw, default=self.config.get_global_option("group_id"))
        meta_data["ARTIFACTID"] = self.get_object_name(thrift_file)

        return meta_data

    @staticmethod
    def __extract_string__(pattern, raw, default=None):
        """
            Given a text line it will try to extract the value from a thrift constant matching the pattern passed in.
        """
        match = re.search(".*{search_pattern}.*=".format(search_pattern=pattern), raw)
        if match is None:
            return default
        start = match.start() + len(match.group(0))
        end = raw.find("\n", start)
        if start == -1 or end == -1:
            return default
        result = re.sub("[',\"]", "", raw[start:end]) # remove quotes
        return re.sub("#.*", "", result).strip()

    def get_object_name(self, raw, delimter="-"):
        """
        Build an artifact name from a filename.

        ie.  given wizecommerce.bizobj.foobar.example.thrift it should return  foobar-example-bizobj
             given wizecommerce.services.foobar.example.thrift it should return  foobar-example-client
        """
        raw = os.path.basename(raw).replace(".thrift", "")
        pattern = "{company_name}.".format(company_name=self.config.get_global_option("company_name"))
        ndx = raw.find(pattern)
        if ndx < 0:
	    self.log("could not find company prefix pattern: {pattern}".format(pattern=pattern))
            sys.exit(ndx)
        raw = raw[ndx + len(pattern):]
        ndx = raw.find(".")
        if ndx < 0:
            self.log("could not find delimiter in file name")
            sys.exit(ndx)

        postfix = raw[:ndx]
        if postfix == self.config.get_global_option("service_prefix"):
            postfix = "client"
            raw = raw.replace(self.config.get_global_option("service_prefix") + ".", "")
        else:
            raw = raw.replace(postfix + ".", "")

        raw = raw.replace(".", delimter) + delimter + postfix
        return raw
예제 #26
0
def main():
    parser = argparse.ArgumentParser(description='Client Generation Script')
    parser.add_argument('--local', action="store_true", dest="local", default=False, help="Enables Local Mode")
    parser.add_argument('--profile', action="store_true", dest="profile", default=False, help="Profiles App")
    parser.add_argument("--docOnly", action="store_true", dest="doc_only", default=False)
    parser.add_argument('--ruby', action="store_true", dest="ruby", default=False,
                        help="Enables RubyMode, default is Ruby + Java (Local Mode Only)")
    parser.add_argument('--java', action="store_true", dest="java", default=False,
                        help="Enables JavaMode, default is Ruby + Java  (Local Mode Only) ")
    parser.add_argument('--thrift-file', action="store", dest="thrift_file", type=str,
                        help="Override list of services, and use the one specified (Local Mode Only)\nThis overrides vcs intelligence")
    parser.add_argument('--config', action="store", dest="config", type=str,
                        help="Override default config file and specify your own yaml config")
    parser.add_argument('--compilers', action="store_true", dest="compilers", default=False,
                        help="will list all supported compilers. (Not fully supported)")
    parser.add_argument('--set-compiler', action="store", dest="compiler", type=str,
                        help="accepts a valid compiler name defined in the yaml config.")



    args = parser.parse_args()
    if args.config is None:
        config = Config()
    else:
        config = Config(args.config)

    config.set_local(args.local)

    if args.thrift_file is not None:
        config.set_service_override(os.path.basename(args.thrift_file))

    if args.thrift_file is not None:
        config.set_local(True)

    if args.compilers:
        display_compilers()

    if args.compiler is not None:
        set_compiler(args.compiler)

    publish_client = PublishClient()

    ## these options can only be used in conjunction with local mode
    if config.is_local():
        if args.ruby and args.java:
            print "WARNING: you really should use rubyOverride or JavaOverride, " \
                  "if you pass both it can will fall back on default behavior.  (ie. omit both of them)"
        elif args.ruby:
            config.set_languages({"ruby": True})
        elif args.java:
            config.set_languages({"java": True})

    if args.doc_only:
        config.set_languages({})
        config.is_doc_enabled = True

    if args.profile:
        import cProfile

        cProfile.run('profileProject()')
    else:
        # Create Repo Structure
        publish_client.create_structure()
        # Loop through all of our services check for updates
        publish_client.process_thrift_services()
예제 #27
0
파일: client.py 프로젝트: safaci2000/medusa
class Client():
    """
        For the most part, the purpose fo this class is to define an API that you can extend
        to implement your own Client for any language that isn't already supported.
    """
    def get_sandbox(self):
        return self.sandbox

    def set_sandbox(self, value):
        self.sandbox = value

    def __init__(self, services, thrift_compiler):
        self.compiler = thrift_compiler
        self.config = Config()
        self.sandbox_work = self.config.work_dir
        self.status = Status()
        self.thrift_helper = Thrift(self.compiler)
        self.log = Log(log_file="status.log", logger_name="status").log
        self.services = services

    def run(self):
        self.initialize()

        for service in self.services:
            self.process_service(service)

        self.finalize()

    def __build_dependency__(self, business_object):
        """
            Recursively build the dependency and return a list of all dependencies found and successfully built.
        """
        raise NotImplementedError("Build Dependency needs to be overridden")
        ##if previous error code has been found, aborting.

    def __build_client__(self, service):
        """
            This method is called to build the actual client, in our case that includes
            all of our services.
        """
        raise NotImplementedError("Build Client needs to be overridden")

    def deploy_object(self, properties, business_object, postfix=""):
        """
            The purpose of this method is to handle the deployment / install.
            If set to localMode it will call the appropriate method for handling local installs,
            if doing a production deployment it will call the appropriate method

            postfix str:  a string appended to the artifact name used for releasing snapshot for example.
            properties dict:  list of properties used for doing deployment.
            businessObject str: string containing the name of the thrift file.

        """
        ##Local mode will only build snapshots, for now.
        if self.config.is_local():
            return self.__deploy_local_artifact__(
                properties=properties,
                thrift_object=business_object,
                postfix=postfix)
        else:
            return self.__deploy_production_artifact__(
                properties=properties,
                thrift_object=business_object,
                postfix=postfix)
        return 0

    def check_version(self, **kwargs):
        """
            Returns a boolean checking if the artifact exists on the deployment server.
        """
        raise NotImplementedError("Check Version needs to be overridden")

    def __deploy_production_artifact__(self,
                                       properties,
                                       thrift_object,
                                       postfix=""):
        """
            Implement a method responsible for deploying your artifact to your production server.
        """
        raise NotImplementedError(
            "Deploy Production Artifact needs to be overridden")

    def __deploy_local_artifact__(self, properties, thrift_object, postfix=""):
        """
            Implement a method responsible for performing a local install.
        """
        raise NotImplementedError(
            "Deploy Local Artifact needs to be overridden")

    def finalize(self):
        """
            This method is called as a last step to either clean , publish or whatever needs to be done.
        """
        raise NotImplementedError("Finalize needs to be overridden")

    def initialize(self):
        """
            Optional, used to initialize/construct any environment or file system settings that need
            to be setup that are specific to the client.
        """
        raise NotImplementedError("Initialize method has not been overridden")

    def process_service(self, service):
        """
            This method builds the client and all dependencies assuming appropriate
            metadata is contained in the thrift file.
        """
        os.chdir(self.sandbox_work)

        dependencies = self.thrift_helper.read_thrift_dependencies(service)

        #Adding the service file as well to the list.
        if len(dependencies) == 0:
            print "No dependencies for %s" % service
        else:
            for dependency in dependencies:
                self.local_assert(
                    self.__build_dependency__(dependency),
                    "Failed to process dependencies for {service}".format(
                        service=dependency))

        self.local_assert(
            self.__build_client__(service),
            "Failed to build Client for {service}".format(
                service=str(service)))
        return 0

    def local_assert(self, exit_code, message, prefix="ERROR: "):
        if exit_code != 0:
            self.log(prefix + message)
            sys.exit(exit_code)
예제 #28
0
class JavaClientTests(unittest.TestCase):
    def setUp(self):
        self.client = self.__get_client__()
        self.client.initialize()
        self.service_name = os.path.join(
            os.getcwd(), "../thrift/services/",
            "wizecommerce.services.example.thrift")

    def __get_client__(self):
        self.config = Config()
        compiler = None
        for item in self.config.get_thrift_option("compilers"):
            compiler = ThriftCompiler(item)
            compiler.postfix = ""
            break

        return JavaClient([], compiler)

    @httprettified
    def test_check_version_pretty(self):
        self.config.set_local(False)
        expected = False
        expected_url = 'http://nexus.corp.nextag.com:8081/content/repositories/releases/com/wizecommerce/data/tag-client/0.0.13/tag-client-0.0.13.jar'
        HTTPretty.register_uri(HTTPretty.GET, expected_url, body="Hello")
        check_value = self.client.check_version(
            groupId='com.wizecommerce.data',
            artifactId='tag-client',
            version='0.0.13')
        self.failUnlessEqual(check_value, expected)
        check_value = self.client.check_version(
            groupId='com.wizecommerce.data',
            artifactId='tag-client',
            version='X.Y.ZZ')
        expected = True
        self.failUnlessEqual(check_value, expected)

    @httprettified
    def test_check_shaded_version_pretty(self):
        self.config.set_local(False)
        expected = False
        expected_url = 'http://crepo.corp.nextag.com/repo/components/com/wizecommerce/data/tag-client-shaded/0.0.13/tag-client-shaded-0.0.13.jar'
        HTTPretty.register_uri(HTTPretty.GET, expected_url, body="Hello")
        check_value_shaded = self.client.check_shaded_version(
            groupId='com.wizecommerce.data',
            artifactId='tag-client',
            version='0.0.13')
        print check_value_shaded
        print expected

        assert check_value_shaded == expected
        self.failUnlessEqual(check_value_shaded, expected)

        check_value_shaded = self.client.check_shaded_version(
            groupId='com.wizecommerce.data',
            artifactId='tag-client',
            version='X.Y.ZZ')
        expected = True
        assert check_value_shaded == expected

    def tearDown(self):
        self.config.reset_configuration()
예제 #29
0
 def __init__(self, properties):
     self.config = Config()
     self.meta_data = properties.copy()
예제 #30
0
class ThriftCompiler(object):
    """
    This should allow support for multiple thrift compilers.  Passing in specialized options etc.  This part of the code
    isn't really used yet, and is mainly a placeholder for now.
    """

    def __init__(self, properties):
        self.config = Config()
        self.meta_data = properties.copy()

    @property
    def name(self):
        return self.meta_data.get("name")

    @name.setter
    def name(self, value):
        self.meta_data['name'] = value

    @property
    def bin(self):
        return self.meta_data.get("bin")

    @bin.setter
    def bin(self, value):
        self.meta_data['bin'] = value

    @property
    def options(self):
        return self.meta_data.get("options")

    @options.setter
    def options(self, value):
        self.meta_data['options'] = value

    @property
    def version(self):
        try:
            return self.meta_data["version"]
        except:
            return "0.6.1"  ##our default version

    @version.setter
    def version(self, value):
        self.meta_data["version"] = value

    def language_options(self, language="java"):
        try:
            return self.meta_data.get("language_options").get(language)
        except:
            #fallback on global options.
            try:
                data = self.config.get_thrift_option("global_compiler_options")
                return data[language]
            except:
                return None


    @property
    def languages(self):
        return self.meta_data.get("supported_languages")

    @languages.setter
    def languages(self, value):
        self.meta_data['supported_languages'] = value

    def is_language_supported(self, value):
        if self.languages is None:
            return False

        for lang in self.languages:
            if lang == value:
                return True
        return False

    @property
    def postfix(self):
        try:
            return self.meta_data["compiler_postfix"]
        except:
            return ""

    @postfix.setter
    def postfix(self, value):
        self.meta_data["compiler_postfix"] = value
예제 #31
0
 def __init__(self, properties):
     self.config = Config()
     self.meta_data = properties.copy()
예제 #32
0
class ThriftTests(unittest.TestCase):

    def setUp(self):
        self.config = Config()
        self.config.reset_configuration()
        compilers = self.config.get_thrift_option("compilers")
        if compilers is None or len(compilers) == 0:
            self.helper = Thrift(None)
        else:
            thrift_compiler = ThriftCompiler(compilers[0])
            self.helper = Thrift(thrift_compiler)

        project_path = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), "../")
        project_path = os.path.realpath(project_path)
        self.config.repo_dir = project_path

    def test_extract_string(self):
        raw = \
            """
namespace java com.wizecommerce.service.common

const string VERSION = "0.0.6"

# Type Definitions
typedef i32 Integer

        """
        expected = "0.0.6"

        result = self.helper.__extract_string__("VERSION", raw, "0.0.0")
        self.failUnlessEqual(result, expected)

    # def reset_config(self):
    #     self.config.reset_configuration()
    #     project_path = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), "../")
    #     project_path = os.path.realpath(project_path)
    #     self.config.repo_dir = project_path


    def test_check_file(self):
        thrift_file = "wizecommerce.services.example.thrift"
        self.helper.check_file(thrift_file)
        #we should reach this point, without any exception thrown.
        thrift_file = "wizecommerce.service.dummyfile.thrift"
        self.assertRaises(IOError,  self.helper.check_file,thrift_file)

    def test_thrift_java_build(self):
        real_methods = {'subprocess': subprocess.call, 'system': os.system, 'chdir': os.chdir,
                        'get_thrift_full_path': self.helper.get_thrift_full_path}
        subprocess.call = MagicMock()
        os.system = MagicMock()
        os.chdir = MagicMock()
        self.helper.get_thrift_full_path = MagicMock()
        thrift_file = "dummy.thrift"
        self.helper.get_thrift_full_path.return_value = thrift_file
        subprocess.call.return_value = 0
        self.helper.thrift_build(thrift_file)
        #########################
        # verify subprocess.call
        #########################
        expected_values = {'compiler': "/usr/local/bin/thrift",
                           'options': "java:private-members,hashcode", 'size': 12,
                           'file': thrift_file}
        call_list = subprocess.call.call_args_list
        self.assertEquals(len(call_list), 1)
        self.verify_subprocess_call(subprocess.call.call_args, expected_values)
        #########################
        # verify chdir calls
        #########################
        call_list = os.chdir.call_args_list
        self.assertEquals(len(call_list), 1)
        call_args = os.chdir.call_args[0][0]
        self.assertEquals(call_args, self.config.work_dir)
        #########################
        # verify system calls
        #########################
        call_list = os.system.call_args_list
        call = call_list[0]
        self.assertEquals(len(call_list), 1)
        parameter = call[0][0]
        self.assertNotEqual(parameter.find("rm -fr"), -1)
        self.assertNotEqual(parameter.find("gen-java"), -1)
        #########################
        # reset all system calls
        #########################
        subprocess.call = real_methods['subprocess']
        os.system = real_methods['system']
        os.chdir = real_methods['chdir']
        self.helper.get_thrift_full_path = real_methods['get_thrift_full_path']


    def test_thrift_ruby_build(self):
        thrift_file = "dummy.thrift"
        real_methods = {'subprocess' : subprocess.call, 'system' : os.system, 'chdir' : os.chdir,
                        'get_thrift_full_path' : self.helper.get_thrift_full_path}
        subprocess.call = MagicMock()
        os.system = MagicMock()
        os.chdir = MagicMock()
        self.helper.get_thrift_full_path = MagicMock()
        self.helper.get_thrift_full_path.return_value = thrift_file
        subprocess.call.return_value = 0
        self.helper.thrift_build(thrift_file=thrift_file, language="ruby")
        #########################
        # verify subprocess.call
        #########################
        expected_values = {'compiler': "/usr/local/bin/thrift",
                           'options': "--gen rb", 'size': 12,
                           'file': thrift_file}
        call_list = subprocess.call.call_args_list
        self.assertEquals(len(call_list), 1)
        self.verify_subprocess_call(subprocess.call.call_args, expected_values)
        #########################
        # verify chdir calls
        #########################
        call_list = os.chdir.call_args_list
        self.assertEquals(len(call_list), 1)
        call_args = os.chdir.call_args[0][0]
        self.assertEquals(call_args, self.config.work_dir)
        #########################
        # verify system calls
        #########################
        call_list = os.system.call_args_list
        call = call_list[0]
        self.assertEquals(len(call_list), 2)
        parameter = call[0][0]
        self.assertNotEqual(parameter.find("rm -fr"), -1)
        self.assertNotEqual(parameter.find("gen-rb"), -1)
        #next call
        call = call_list[1]
        parameter = call[0][0]
        self.assertNotEqual(parameter.find("rm -fr"), -1)
        print parameter
        self.assertNotEqual(parameter.find("work/ruby"), -1)
        #########################
        # reset all system calls
        #########################
        subprocess.call = real_methods['subprocess']
        os.system = real_methods['system']
        os.chdir = real_methods['chdir']
        self.helper.get_thrift_full_path = real_methods['get_thrift_full_path']



    def intercept_thrift_doc_full_path(self, request):
        """
        This method intercepts the call for self.helper.read_full_path and simply returns the
        value passed in.
        """
        return request

    def verify_subprocess_call(self, method_call_args, expected_values):
        call_args = method_call_args[0][0]
        self.assertEquals(len(call_args), expected_values['size'])
        parameter = " ".join(call_args)
        #verify compiler
        self.assertNotEqual(parameter.find(expected_values['compiler']), -1)
        #verify file name
        self.assertNotEqual(parameter.find(expected_values['file']), -1)
        #verify language/options
        self.assertNotEqual(parameter.find(expected_values['options']), -1)
        #verify includes
        self.assertNotEqual(parameter.format("thrift/services"), -1)
        self.assertNotEqual(parameter.format("business-objects"), -1)


    def test_thrift_doc_build(self):
        thrift_file = "dummy.thrift"
        real_methods = {'subprocess' : subprocess.call, 'system' : os.system,
                        'get_thrift_full_path' : self.helper.get_thrift_full_path,
                        'read_thrift_dependencies_recursively': self.helper.read_thrift_dependencies_recursively }
        dependencies = ['dependency1.thrift', 'dependency2.thrift']
        subprocess.call = MagicMock()
        os.system = MagicMock()
        self.helper.get_thrift_full_path = MagicMock()
        self.helper.get_thrift_full_path.return_value = thrift_file
        self.helper.get_thrift_full_path = self.intercept_thrift_doc_full_path
        subprocess.call.return_value = 0
        self.helper.read_thrift_dependencies_recursively = MagicMock()
        self.helper.read_thrift_dependencies_recursively.return_value = dependencies
        self.helper.thrift_build(thrift_file=thrift_file, language="doc")
         #########################
        # verify subprocess.call
        #########################
        expected_values = {'compiler': "/usr/local/bin/thrift",
                           'options': "--gen html", 'size': 14,
                           'file': thrift_file}
        call_list = subprocess.call.call_args_list
        self.assertEquals(len(call_list), 3)
        count = 0
        for dependency_file in dependencies:
            call = call_list[count]
            expected_values['file'] = dependency_file
            call_list = subprocess.call.call_args_list
            self.verify_subprocess_call(call, expected_values)
            count += 1
        #verify client
        expected_values['file'] = thrift_file
        call = call_list[count]
        self.verify_subprocess_call(call, expected_values)
        #########################
        # verify system calls
        #########################
        call_list = os.system.call_args_list
        call = call_list[0]
        self.assertEquals(len(call_list), 1)
        parameter = call[0][0]
        self.assertNotEqual(parameter.find("rm -fr"), -1)
        self.assertNotEqual(parameter.find("gen-html"), -1)
        #########################
        # reset all system calls
        #########################
        subprocess.call = real_methods['subprocess']
        os.system = real_methods['system']
        self.helper.get_thrift_full_path = real_methods['get_thrift_full_path']
        self.helper.read_thrift_dependencies_recursively = real_methods['read_thrift_dependencies_recursively']

    def test_is_service(self):
        dummy = "foobar.services.thrift"
        result = self.helper.is_service(dummy)
        self.assertTrue(result)
        dummy = "foobar.dummy.thrift"
        result = self.helper.is_service(dummy)
        self.assertFalse(result)

    def test_read_thrift_dependencies(self):
        real_file = "wizecommerce.services.example.thrift"
        dependencies = self.helper.read_thrift_dependencies_recursively(real_file)
        self.assertEquals(len(dependencies), 3)
        all_deps = " ".join(dependencies)
        self.assertNotEqual(all_deps.find("exception.invalid"), -1)
        self.assertNotEqual(all_deps.find("enum.example_change"), -1)
        self.assertNotEqual(all_deps.find("bizobj.example"), -1)



    ##TODO: possible enhancement, mock out the file we're reading.
    def test_read_thrift_properties(self):
        thrift_file = "wizecommerce.services.example.thrift"
        properties = self.helper.read_thrift_properties(thrift_file)
        self.assertEquals('example-client', properties['ARTIFACTID'])
        self.assertEquals('0.0.5', properties['VERSION'])
        self.assertEquals('com.wizecommerce.data', properties['GROUPID'])
        print properties


    def test_thrift_compiler(self):
        init = {}
        init['name'] = 'testThrift'
        init['bin'] = '/dev/null'
        init['options'] = 'testoptions'
        init['supported_languages'] = ['java', 'ruby', 'python']
        t = ThriftCompiler(init)
        self.failUnless('testThrift' == t.name)
        self.failUnless('/dev/null' == t.bin)
        self.failUnless('testoptions' == t.options)
        self.failUnless(len(t.languages) == 3)
        self.failUnless(t.is_language_supported('ruby'))
        self.failUnless(not t.is_language_supported('erlang') )

    def test_object_name(self):
        th = Thrift(None)
        src = "/home/user/foobar/blah/wizecommerce.bizobj.sellerprogramrecord.thrift"
        result = th.get_object_name(src)
        expected = "sellerprogramrecord-bizobj"
        self.failUnlessEqual(expected, result)

    def test_complex_object_name(self):
        th = Thrift(None)
        src = "/home/user/foobar/blah/wizecommerce.bizobj.search.foobar.sellerprogramrecord.thrift"
        result = th.get_object_name(src)
        expected = "search-foobar-sellerprogramrecord-bizobj"
        self.failUnlessEqual(expected, result)

    def test_special_case_object_name(self):
        th = Thrift(None)
        src = "/home/user/foobar/blah/wizecommerce.services.sellerprogramrecord.thrift"
        result = th.get_object_name(src)
        expected = "sellerprogramrecord-client"
        self.failUnlessEqual(expected, result)
예제 #33
0
class ThriftCompiler(object):
    """
    This should allow support for multiple thrift compilers.  Passing in specialized options etc.  This part of the code
    isn't really used yet, and is mainly a placeholder for now.
    """
    def __init__(self, properties):
        self.config = Config()
        self.meta_data = properties.copy()

    @property
    def name(self):
        return self.meta_data.get("name")

    @name.setter
    def name(self, value):
        self.meta_data['name'] = value

    @property
    def bin(self):
        return self.meta_data.get("bin")

    @bin.setter
    def bin(self, value):
        self.meta_data['bin'] = value

    @property
    def options(self):
        return self.meta_data.get("options")

    @options.setter
    def options(self, value):
        self.meta_data['options'] = value

    @property
    def version(self):
        try:
            return self.meta_data["version"]
        except:
            return "0.6.1"  ##our default version

    @version.setter
    def version(self, value):
        self.meta_data["version"] = value

    def language_options(self, language="java"):
        try:
            return self.meta_data.get("language_options").get(language)
        except:
            #fallback on global options.
            try:
                data = self.config.get_thrift_option("global_compiler_options")
                return data[language]
            except:
                return None

    @property
    def languages(self):
        return self.meta_data.get("supported_languages")

    @languages.setter
    def languages(self, value):
        self.meta_data['supported_languages'] = value

    def is_language_supported(self, value):
        if self.languages is None:
            return False

        for lang in self.languages:
            if lang == value:
                return True
        return False

    @property
    def postfix(self):
        try:
            return self.meta_data["compiler_postfix"]
        except:
            return ""

    @postfix.setter
    def postfix(self, value):
        self.meta_data["compiler_postfix"] = value
예제 #34
0
class ThriftTests(unittest.TestCase):
    def setUp(self):
        self.config = Config()
        self.config.reset_configuration()
        compilers = self.config.get_thrift_option("compilers")
        if compilers is None or len(compilers) == 0:
            self.helper = Thrift(None)
        else:
            thrift_compiler = ThriftCompiler(compilers[0])
            self.helper = Thrift(thrift_compiler)

        project_path = os.path.join(
            os.path.dirname(
                os.path.abspath(inspect.getfile(inspect.currentframe()))),
            "../")
        project_path = os.path.realpath(project_path)
        self.config.repo_dir = project_path

    def test_extract_string(self):
        raw = \
            """
namespace java com.wizecommerce.service.common

const string VERSION = "0.0.6"

# Type Definitions
typedef i32 Integer

        """
        expected = "0.0.6"

        result = self.helper.__extract_string__("VERSION", raw, "0.0.0")
        self.failUnlessEqual(result, expected)

    # def reset_config(self):
    #     self.config.reset_configuration()
    #     project_path = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), "../")
    #     project_path = os.path.realpath(project_path)
    #     self.config.repo_dir = project_path

    def test_check_file(self):
        thrift_file = "wizecommerce.services.example.thrift"
        self.helper.check_file(thrift_file)
        #we should reach this point, without any exception thrown.
        thrift_file = "wizecommerce.service.dummyfile.thrift"
        self.assertRaises(IOError, self.helper.check_file, thrift_file)

    def test_thrift_java_build(self):
        real_methods = {
            'subprocess': subprocess.call,
            'system': os.system,
            'chdir': os.chdir,
            'get_thrift_full_path': self.helper.get_thrift_full_path
        }
        subprocess.call = MagicMock()
        os.system = MagicMock()
        os.chdir = MagicMock()
        self.helper.get_thrift_full_path = MagicMock()
        thrift_file = "dummy.thrift"
        self.helper.get_thrift_full_path.return_value = thrift_file
        subprocess.call.return_value = 0
        self.helper.thrift_build(thrift_file)
        #########################
        # verify subprocess.call
        #########################
        expected_values = {
            'compiler': "/usr/local/bin/thrift",
            'options': "java:private-members,hashcode",
            'size': 12,
            'file': thrift_file
        }
        call_list = subprocess.call.call_args_list
        self.assertEquals(len(call_list), 1)
        self.verify_subprocess_call(subprocess.call.call_args, expected_values)
        #########################
        # verify chdir calls
        #########################
        call_list = os.chdir.call_args_list
        self.assertEquals(len(call_list), 1)
        call_args = os.chdir.call_args[0][0]
        self.assertEquals(call_args, self.config.work_dir)
        #########################
        # verify system calls
        #########################
        call_list = os.system.call_args_list
        call = call_list[0]
        self.assertEquals(len(call_list), 1)
        parameter = call[0][0]
        self.assertNotEqual(parameter.find("rm -fr"), -1)
        self.assertNotEqual(parameter.find("gen-java"), -1)
        #########################
        # reset all system calls
        #########################
        subprocess.call = real_methods['subprocess']
        os.system = real_methods['system']
        os.chdir = real_methods['chdir']
        self.helper.get_thrift_full_path = real_methods['get_thrift_full_path']

    def test_thrift_ruby_build(self):
        thrift_file = "dummy.thrift"
        real_methods = {
            'subprocess': subprocess.call,
            'system': os.system,
            'chdir': os.chdir,
            'get_thrift_full_path': self.helper.get_thrift_full_path
        }
        subprocess.call = MagicMock()
        os.system = MagicMock()
        os.chdir = MagicMock()
        self.helper.get_thrift_full_path = MagicMock()
        self.helper.get_thrift_full_path.return_value = thrift_file
        subprocess.call.return_value = 0
        self.helper.thrift_build(thrift_file=thrift_file, language="ruby")
        #########################
        # verify subprocess.call
        #########################
        expected_values = {
            'compiler': "/usr/local/bin/thrift",
            'options': "--gen rb",
            'size': 12,
            'file': thrift_file
        }
        call_list = subprocess.call.call_args_list
        self.assertEquals(len(call_list), 1)
        self.verify_subprocess_call(subprocess.call.call_args, expected_values)
        #########################
        # verify chdir calls
        #########################
        call_list = os.chdir.call_args_list
        self.assertEquals(len(call_list), 1)
        call_args = os.chdir.call_args[0][0]
        self.assertEquals(call_args, self.config.work_dir)
        #########################
        # verify system calls
        #########################
        call_list = os.system.call_args_list
        call = call_list[0]
        self.assertEquals(len(call_list), 2)
        parameter = call[0][0]
        self.assertNotEqual(parameter.find("rm -fr"), -1)
        self.assertNotEqual(parameter.find("gen-rb"), -1)
        #next call
        call = call_list[1]
        parameter = call[0][0]
        self.assertNotEqual(parameter.find("rm -fr"), -1)
        print parameter
        self.assertNotEqual(parameter.find("work/ruby"), -1)
        #########################
        # reset all system calls
        #########################
        subprocess.call = real_methods['subprocess']
        os.system = real_methods['system']
        os.chdir = real_methods['chdir']
        self.helper.get_thrift_full_path = real_methods['get_thrift_full_path']

    def intercept_thrift_doc_full_path(self, request):
        """
        This method intercepts the call for self.helper.read_full_path and simply returns the
        value passed in.
        """
        return request

    def verify_subprocess_call(self, method_call_args, expected_values):
        call_args = method_call_args[0][0]
        self.assertEquals(len(call_args), expected_values['size'])
        parameter = " ".join(call_args)
        #verify compiler
        self.assertNotEqual(parameter.find(expected_values['compiler']), -1)
        #verify file name
        self.assertNotEqual(parameter.find(expected_values['file']), -1)
        #verify language/options
        self.assertNotEqual(parameter.find(expected_values['options']), -1)
        #verify includes
        self.assertNotEqual(parameter.format("thrift/services"), -1)
        self.assertNotEqual(parameter.format("business-objects"), -1)

    def test_thrift_doc_build(self):
        thrift_file = "dummy.thrift"
        real_methods = {
            'subprocess':
            subprocess.call,
            'system':
            os.system,
            'get_thrift_full_path':
            self.helper.get_thrift_full_path,
            'read_thrift_dependencies_recursively':
            self.helper.read_thrift_dependencies_recursively
        }
        dependencies = ['dependency1.thrift', 'dependency2.thrift']
        subprocess.call = MagicMock()
        os.system = MagicMock()
        self.helper.get_thrift_full_path = MagicMock()
        self.helper.get_thrift_full_path.return_value = thrift_file
        self.helper.get_thrift_full_path = self.intercept_thrift_doc_full_path
        subprocess.call.return_value = 0
        self.helper.read_thrift_dependencies_recursively = MagicMock()
        self.helper.read_thrift_dependencies_recursively.return_value = dependencies
        self.helper.thrift_build(thrift_file=thrift_file, language="doc")
        #########################
        # verify subprocess.call
        #########################
        expected_values = {
            'compiler': "/usr/local/bin/thrift",
            'options': "--gen html",
            'size': 14,
            'file': thrift_file
        }
        call_list = subprocess.call.call_args_list
        self.assertEquals(len(call_list), 3)
        count = 0
        for dependency_file in dependencies:
            call = call_list[count]
            expected_values['file'] = dependency_file
            call_list = subprocess.call.call_args_list
            self.verify_subprocess_call(call, expected_values)
            count += 1
        #verify client
        expected_values['file'] = thrift_file
        call = call_list[count]
        self.verify_subprocess_call(call, expected_values)
        #########################
        # verify system calls
        #########################
        call_list = os.system.call_args_list
        call = call_list[0]
        self.assertEquals(len(call_list), 1)
        parameter = call[0][0]
        self.assertNotEqual(parameter.find("rm -fr"), -1)
        self.assertNotEqual(parameter.find("gen-html"), -1)
        #########################
        # reset all system calls
        #########################
        subprocess.call = real_methods['subprocess']
        os.system = real_methods['system']
        self.helper.get_thrift_full_path = real_methods['get_thrift_full_path']
        self.helper.read_thrift_dependencies_recursively = real_methods[
            'read_thrift_dependencies_recursively']

    def test_is_service(self):
        dummy = "foobar.services.thrift"
        result = self.helper.is_service(dummy)
        self.assertTrue(result)
        dummy = "foobar.dummy.thrift"
        result = self.helper.is_service(dummy)
        self.assertFalse(result)

    def test_read_thrift_dependencies(self):
        real_file = "wizecommerce.services.example.thrift"
        dependencies = self.helper.read_thrift_dependencies_recursively(
            real_file)
        self.assertEquals(len(dependencies), 3)
        all_deps = " ".join(dependencies)
        self.assertNotEqual(all_deps.find("exception.invalid"), -1)
        self.assertNotEqual(all_deps.find("enum.example_change"), -1)
        self.assertNotEqual(all_deps.find("bizobj.example"), -1)

    ##TODO: possible enhancement, mock out the file we're reading.
    def test_read_thrift_properties(self):
        thrift_file = "wizecommerce.services.example.thrift"
        properties = self.helper.read_thrift_properties(thrift_file)
        self.assertEquals('example-client', properties['ARTIFACTID'])
        self.assertEquals('0.0.5', properties['VERSION'])
        self.assertEquals('com.wizecommerce.data', properties['GROUPID'])
        print properties

    def test_thrift_compiler(self):
        init = {}
        init['name'] = 'testThrift'
        init['bin'] = '/dev/null'
        init['options'] = 'testoptions'
        init['supported_languages'] = ['java', 'ruby', 'python']
        t = ThriftCompiler(init)
        self.failUnless('testThrift' == t.name)
        self.failUnless('/dev/null' == t.bin)
        self.failUnless('testoptions' == t.options)
        self.failUnless(len(t.languages) == 3)
        self.failUnless(t.is_language_supported('ruby'))
        self.failUnless(not t.is_language_supported('erlang'))

    def test_object_name(self):
        th = Thrift(None)
        src = "/home/user/foobar/blah/wizecommerce.bizobj.sellerprogramrecord.thrift"
        result = th.get_object_name(src)
        expected = "sellerprogramrecord-bizobj"
        self.failUnlessEqual(expected, result)

    def test_complex_object_name(self):
        th = Thrift(None)
        src = "/home/user/foobar/blah/wizecommerce.bizobj.search.foobar.sellerprogramrecord.thrift"
        result = th.get_object_name(src)
        expected = "search-foobar-sellerprogramrecord-bizobj"
        self.failUnlessEqual(expected, result)

    def test_special_case_object_name(self):
        th = Thrift(None)
        src = "/home/user/foobar/blah/wizecommerce.services.sellerprogramrecord.thrift"
        result = th.get_object_name(src)
        expected = "sellerprogramrecord-client"
        self.failUnlessEqual(expected, result)
예제 #35
0
class Client():
    """
        For the most part, the purpose fo this class is to define an API that you can extend
        to implement your own Client for any language that isn't already supported.
    """
    def get_sandbox(self):
        return self.sandbox

    def set_sandbox(self, value):
        self.sandbox = value


    def __init__(self, services, thrift_compiler):
        self.compiler = thrift_compiler
        self.config = Config()
        self.sandbox_work = self.config.work_dir
        self.status = Status()
        self.thrift_helper = Thrift(self.compiler)
        self.log = Log(log_file="status.log", logger_name="status").log
        self.services = services

    def run(self):
        self.initialize()

        for service in self.services:
            self.process_service(service)

        self.finalize()

    def __build_dependency__(self, business_object):
        """
            Recursively build the dependency and return a list of all dependencies found and successfully built.
        """
        raise NotImplementedError("Build Dependency needs to be overridden")
        ##if previous error code has been found, aborting.

    def __build_client__(self, service):
        """
            This method is called to build the actual client, in our case that includes
            all of our services.
        """
        raise NotImplementedError("Build Client needs to be overridden")

    def deploy_object(self, properties, business_object, postfix=""):
        """
            The purpose of this method is to handle the deployment / install.
            If set to localMode it will call the appropriate method for handling local installs,
            if doing a production deployment it will call the appropriate method

            postfix str:  a string appended to the artifact name used for releasing snapshot for example.
            properties dict:  list of properties used for doing deployment.
            businessObject str: string containing the name of the thrift file.

        """
        ##Local mode will only build snapshots, for now.
        if self.config.is_local():
            return self.__deploy_local_artifact__(properties=properties, thrift_object=business_object, postfix=postfix)
        else:
            return self.__deploy_production_artifact__(properties=properties, thrift_object=business_object, postfix=postfix)
        return 0

    def check_version(self, **kwargs):
        """
            Returns a boolean checking if the artifact exists on the deployment server.
        """
        raise NotImplementedError("Check Version needs to be overridden")

    def __deploy_production_artifact__(self, properties, thrift_object, postfix=""):
        """
            Implement a method responsible for deploying your artifact to your production server.
        """
        raise NotImplementedError("Deploy Production Artifact needs to be overridden")

    def __deploy_local_artifact__(self, properties, thrift_object, postfix=""):
        """
            Implement a method responsible for performing a local install.
        """
        raise NotImplementedError("Deploy Local Artifact needs to be overridden")

    def finalize(self):
        """
            This method is called as a last step to either clean , publish or whatever needs to be done.
        """
        raise NotImplementedError("Finalize needs to be overridden")

    def initialize(self):
        """
            Optional, used to initialize/construct any environment or file system settings that need
            to be setup that are specific to the client.
        """
        raise NotImplementedError("Initialize method has not been overridden")

    def process_service(self, service):
        """
            This method builds the client and all dependencies assuming appropriate
            metadata is contained in the thrift file.
        """
        os.chdir(self.sandbox_work)

        dependencies = self.thrift_helper.read_thrift_dependencies(service)

        #Adding the service file as well to the list.
        if len(dependencies) == 0:
            print "No dependencies for %s" % service
        else:
            for dependency in dependencies:
                self.local_assert(self.__build_dependency__(dependency), "Failed to process dependencies for {service}".format(service=dependency))

        self.local_assert(self.__build_client__(service),  "Failed to build Client for {service}".format(service=str(service)))
        return 0

    def local_assert(self, exit_code, message, prefix="ERROR: "):
        if exit_code != 0:
            self.log(prefix + message)
            sys.exit(exit_code)