class GitVCSTests(unittest2.TestCase):
def setUp(self):
self.config = Config()
init_data = {}
self.gitcvs = GitVCS(**init_data)
def tearDown(self):
self.config.reset_configuration()
def test_find_commit(self):
commits = self.gitcvs.get_commits()
self.failIf(len(commits) == 0)
expected = commits[0]
result = self.gitcvs.find_commit(str(expected))[0]
self.assertEquals(result.hexsha, expected.hexsha)
self.assertEquals(type(result), type(expected))
def test_git_stub(self):
commit_hash = "76191292fe70015eeee241fb9b663af6953ffb5e"
commits = self.gitcvs.get_commits()
data = self.gitcvs.find_commit(commit_hash)
if data is not None and len(data) > 2:
commitA = data[0]
commitB = commits[data[1] + 1]
files_list = self.gitcvs.get_modified_files(commitB, commitA, None)
expected_files = set()
expected_files.add(".travis.yaml")
expected_files.add(".travis.yml")
expected_files.add("README.md")
expected_files.add("build_thrift.sh")
expected_files.add("setup.py")
expected_files.add("thrift_medusa/config/travis-ci.yaml")
for item in files_list:
self.assertTrue(item in expected_files)
def __get_client__(self):
self.config = Config()
self.config.reset_configuration()
compiler = None
for item in self.config.get_thrift_option("compilers"):
compiler = ThriftCompiler(item)
return Documentation([], compiler)
def __init__(self, services, thrift_compiler):
self.compiler = thrift_compiler
self.config = Config()
self.sandbox_work = self.config.work_dir
self.status = Status()
self.thrift_helper = Thrift(self.compiler)
self.log = Log(log_file="status.log", logger_name="status").log
self.services = services
def __get_client__(self):
self.config = Config()
compiler = None
for item in self.config.get_thrift_option("compilers"):
compiler = ThriftCompiler(item)
break
return RubyClient([], compiler)
def __init__(self):
self.client_list = []
self.remove_structure("logs")
wize_mkdir("logs")
self.business_objects = []
self.service_objects = []
self.config = Config()
self.log = Log(log_file=os.path.join(self.config.repo_dir, "logs/status.log"),
logger_name="definitions").get_logger()
def test_bad_init_data(self):
dict = {}
conf = Config()
bad_compiler = ThriftCompiler(dict)
self.assertEquals(bad_compiler.version, "0.6.1")
self.assertEquals(bad_compiler.postfix, "")
language = "java"
self.assertEquals(bad_compiler.is_language_supported("java"), False)
self.assertEquals(bad_compiler.language_options(), conf.get_thrift_option("global_compiler_options")[language])
def display_compilers():
"""
Will display the list of all current supported compilers defined in configuration.
"""
config = Config()
compilers = config.get_thrift_option("compilers")
for item in compilers:
print("found compiler %s with binary at: %s which supports: %s languages" % (item.get('name'), item.get('bin'),
', '.join(map(str, item.get(
'supported_languages')))))
sys.exit(0)
def test_bad_init_data(self):
dict = {}
conf = Config()
bad_compiler = ThriftCompiler(dict)
self.assertEquals(bad_compiler.version, "0.6.1")
self.assertEquals(bad_compiler.postfix, "")
language = "java"
self.assertEquals(bad_compiler.is_language_supported("java"), False)
self.assertEquals(
bad_compiler.language_options(),
conf.get_thrift_option("global_compiler_options")[language])
def __get_client__(self):
self.config = Config()
self.config.reset_configuration()
compiler = None
for item in self.config.get_thrift_option("compilers"):
compiler = ThriftCompiler(item)
return Client([], compiler)
def setUp(self):
self.config = Config()
self.config.reset_configuration()
compilers = self.config.get_thrift_option("compilers")
if compilers is None or len(compilers) == 0:
self.helper = Thrift(None)
else:
thrift_compiler = ThriftCompiler(compilers[0])
self.helper = Thrift(thrift_compiler)
project_path = os.path.join(
os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe()))),
"../")
project_path = os.path.realpath(project_path)
self.config.repo_dir = project_path
def __get_client__(self):
self.config = Config()
compiler = None
for item in self.config.get_thrift_option("compilers"):
compiler = ThriftCompiler(item)
break
return RubyClient([], compiler)
class RubyClientTests(unittest.TestCase):
def setUp(self):
self.client = self.__get_client__()
self.client.initialize()
self.service_name = os.path.join(os.getcwd(), "../thrift/services/", "wizecommerce.services.example.thrift")
def __get_client__(self):
self.config = Config()
compiler = None
for item in self.config.get_thrift_option("compilers"):
compiler = ThriftCompiler(item)
break
return RubyClient([], compiler)
def tearDown(self):
self.config.reset_configuration()
def __init__(self, services, thrift_compiler):
self.compiler = thrift_compiler
self.config = Config()
self.sandbox_work = self.config.work_dir
self.status = Status()
self.thrift_helper = Thrift(self.compiler)
self.log = Log(log_file="status.log", logger_name="status").log
self.services = services
class DocumentationClientTests(unittest2.TestCase):
def setUp(self):
self.dict = {}
self.client = self.__get_client__()
# self.client.initialize()
self.service_name = os.path.join(os.getcwd(), "../thrift/services/", "wizecommerce.services.example.thrift")
def __get_client__(self):
self.config = Config()
self.config.reset_configuration()
compiler = None
for item in self.config.get_thrift_option("compilers"):
compiler = ThriftCompiler(item)
return Documentation([], compiler)
def test_dummy(self):
self.assertEquals(1,1)
def set_compiler(override_compiler):
"""
Allows user to explicitly use a particular compiler when building thrift artifacts.
"""
config = Config()
compilers = config.get_thrift_option("compilers")
found = False
compiler = None
for item in compilers:
if item['name'] == override_compiler:
found = True
compiler = item
if not found:
print("compiler {compiler} was not found in yaml configuration".format(compiler=override_compiler))
sys.exit(1)
config.set_thrift_option("compilers", [compiler])
class RubyClientTests(unittest.TestCase):
def setUp(self):
self.client = self.__get_client__()
self.client.initialize()
self.service_name = os.path.join(
os.getcwd(), "../thrift/services/",
"wizecommerce.services.example.thrift")
def __get_client__(self):
self.config = Config()
compiler = None
for item in self.config.get_thrift_option("compilers"):
compiler = ThriftCompiler(item)
break
return RubyClient([], compiler)
def tearDown(self):
self.config.reset_configuration()
class DocumentationClientTests(unittest2.TestCase):
def setUp(self):
self.dict = {}
self.client = self.__get_client__()
# self.client.initialize()
self.service_name = os.path.join(
os.getcwd(), "../thrift/services/",
"wizecommerce.services.example.thrift")
def __get_client__(self):
self.config = Config()
self.config.reset_configuration()
compiler = None
for item in self.config.get_thrift_option("compilers"):
compiler = ThriftCompiler(item)
return Documentation([], compiler)
def test_dummy(self):
self.assertEquals(1, 1)
def setUp(self):
self.config = Config()
self.config.reset_configuration()
compilers = self.config.get_thrift_option("compilers")
if compilers is None or len(compilers) == 0:
self.helper = Thrift(None)
else:
thrift_compiler = ThriftCompiler(compilers[0])
self.helper = Thrift(thrift_compiler)
project_path = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), "../")
project_path = os.path.realpath(project_path)
self.config.repo_dir = project_path
def __init__(self, compiler):
self.compiler=compiler
self.version_pattern = re.compile("const\s+string\s+VERSION\s*=\s*['\"]+(.*)['\"]+")
self.java_namespace_pattern = re.compile("namespace\s+java\s+(.*)")
self.ruby_namespace_pattern = re.compile("namespace\s+rb\s+(.*)")
self.data_type_pattern = re.compile(
"\s+(\d+):\s+([required,optional]+)\s+([bool,byte,i16,i32,i64,string,double,string]+)\s+(\w+)")
self.config = Config()
self.log = Log(log_file="status.log", logger_name="status").log
# defines the language compiler to spawn.
self.build = {"ruby": self.__thrift_ruby_build__, "java": self.__thrift_java_build__,
"doc": self.__thrift_doc_build__}
class JavaClientTests(unittest.TestCase):
def setUp(self):
self.client = self.__get_client__()
self.client.initialize()
self.service_name = os.path.join(os.getcwd(), "../thrift/services/", "wizecommerce.services.example.thrift")
def __get_client__(self):
self.config = Config()
compiler = None
for item in self.config.get_thrift_option("compilers"):
compiler = ThriftCompiler(item)
compiler.postfix = ""
break
return JavaClient([], compiler)
@httprettified
def test_check_version_pretty(self):
self.config.set_local(False)
expected = False
expected_url = 'http://nexus.corp.nextag.com:8081/content/repositories/releases/com/wizecommerce/data/tag-client/0.0.13/tag-client-0.0.13.jar'
HTTPretty.register_uri(HTTPretty.GET, expected_url, body="Hello")
check_value = self.client.check_version(groupId='com.wizecommerce.data', artifactId='tag-client',
version='0.0.13')
self.failUnlessEqual(check_value, expected)
check_value = self.client.check_version(groupId='com.wizecommerce.data', artifactId='tag-client',
version='X.Y.ZZ')
expected = True
self.failUnlessEqual(check_value, expected)
@httprettified
def test_check_shaded_version_pretty(self):
self.config.set_local(False)
expected = False
expected_url = 'http://crepo.corp.nextag.com/repo/components/com/wizecommerce/data/tag-client-shaded/0.0.13/tag-client-shaded-0.0.13.jar'
HTTPretty.register_uri(HTTPretty.GET, expected_url, body="Hello")
check_value_shaded = self.client.check_shaded_version(groupId='com.wizecommerce.data', artifactId='tag-client',
version='0.0.13')
print check_value_shaded
print expected
assert check_value_shaded == expected
self.failUnlessEqual(check_value_shaded, expected)
check_value_shaded = self.client.check_shaded_version(groupId='com.wizecommerce.data', artifactId='tag-client',
version='X.Y.ZZ')
expected = True
assert check_value_shaded == expected
def tearDown(self):
self.config.reset_configuration()
class BaseClientTests(unittest2.TestCase):
def setUp(self):
self.dict = {}
self.client = self.__get_client__()
# self.client.initialize()
self.service_name = os.path.join(
os.getcwd(), "../thrift/services/",
"wizecommerce.services.example.thrift")
def __get_client__(self):
self.config = Config()
self.config.reset_configuration()
compiler = None
for item in self.config.get_thrift_option("compilers"):
compiler = ThriftCompiler(item)
return Client([], compiler)
def test_not_implemented_methods(self):
client = self.__get_client__()
self.assertRaises(NotImplementedError, client.__build_dependency__,
"DummyFile")
self.assertRaises(NotImplementedError, client.__build_client__,
"service")
self.assertRaises(NotImplementedError, client.check_version,
**self.dict)
self.assertRaises(NotImplementedError,
client.__deploy_production_artifact__, self.dict,
"boo")
self.assertRaises(NotImplementedError,
client.__deploy_local_artifact__, self.dict, "boo")
self.assertRaises(NotImplementedError, client.finalize)
self.assertRaises(NotImplementedError, client.initialize)
def test_deploy_object(self):
client = self.__get_client__()
self.config.set_local(True)
self.assertRaisesRegexp(NotImplementedError,
".*Deploy Local Artifact.*",
client.deploy_object, self.dict, "dummy")
client = self.__get_client__()
self.config.set_local(False)
self.assertRaisesRegexp(NotImplementedError,
".*Deploy Production Artifact.*",
client.deploy_object, self.dict, "dummy")
def test_sandbox(self):
client = self.__get_client__()
client.set_sandbox("DummySandbox")
self.assertEquals("DummySandbox", client.get_sandbox())
class BaseClientTests(unittest2.TestCase):
def setUp(self):
self.dict = {}
self.client = self.__get_client__()
# self.client.initialize()
self.service_name = os.path.join(os.getcwd(), "../thrift/services/", "wizecommerce.services.example.thrift")
def __get_client__(self):
self.config = Config()
self.config.reset_configuration()
compiler = None
for item in self.config.get_thrift_option("compilers"):
compiler = ThriftCompiler(item)
return Client([], compiler)
def test_not_implemented_methods(self):
client = self.__get_client__()
self.assertRaises(NotImplementedError, client.__build_dependency__, "DummyFile")
self.assertRaises(NotImplementedError, client.__build_client__, "service")
self.assertRaises(NotImplementedError, client.check_version, **self.dict)
self.assertRaises(NotImplementedError, client.__deploy_production_artifact__, self.dict, "boo")
self.assertRaises(NotImplementedError, client.__deploy_local_artifact__, self.dict, "boo")
self.assertRaises(NotImplementedError, client.finalize)
self.assertRaises(NotImplementedError, client.initialize)
def test_deploy_object(self):
client = self.__get_client__()
self.config.set_local(True)
self.assertRaisesRegexp(
NotImplementedError, ".*Deploy Local Artifact.*", client.deploy_object, self.dict, "dummy"
)
client = self.__get_client__()
self.config.set_local(False)
self.assertRaisesRegexp(
NotImplementedError, ".*Deploy Production Artifact.*", client.deploy_object, self.dict, "dummy"
)
def test_sandbox(self):
client = self.__get_client__()
client.set_sandbox("DummySandbox")
self.assertEquals("DummySandbox", client.get_sandbox())
class PublishClient():
"""
The purpose of this class is to setup the environment for processing various service objects
"""
def __init__(self):
self.client_list = []
self.remove_structure("logs")
wize_mkdir("logs")
self.business_objects = []
self.service_objects = []
self.config = Config()
self.log = Log(log_file=os.path.join(self.config.repo_dir, "logs/status.log"),
logger_name="definitions").get_logger()
def remove_structure(self, dir):
"""
Simple method that deletes a directory
"""
cmd = ['rm', '-fr', dir]
self.local_assert(subprocess.call(cmd), "failed to run command: {cmd}".format(cmd=str(cmd)))
return 0
def local_assert(self, exit_code, message):
"""
Defines a cleaner version of an assert that is probably more helpful.
"""
if exit_code != 0:
self.log.error(message)
sys.exit(exit_code)
def create_structure(self):
"""
Remove old directory structure and re-copy all the files and dependencies
from the appropriate repos.
"""
self.remove_structure(self.config.work_dir)
os.mkdir(self.config.work_dir)
self.business_objects = build_file_list(self.config.get_path(type="business_object"), ".thrift")
self.service_objects = build_file_list(self.config.get_path(type="service_object"), ".thrift")
self.enum_objects = build_file_list(self.config.get_path(type="enum_object"), ".thrift")
self.exception_objects = build_file_list(self.config.get_path(type="exception_object"), ".thrift")
def update_client_list(self, thrift_objects, compilers):
"""
Build a list of all clients for each language and compiler type.
Note: Multiple thrift compilers not currently supported.
"""
self.client_list = []
for item in compilers:
if self.config.is_java and item.is_language_supported("java"):
self.client_list.append(JavaClient(thrift_objects, item))
if self.config.is_ruby and item.is_language_supported("ruby"):
self.client_list.append(RubyClient(thrift_objects, item))
if self.config.is_doc_enabled and item.is_language_supported("doc"):
self.client_list.append(Documentation(thrift_objects, item))
def process_thrift_services(self):
"""
This method will iterate through all the service and business object thrift files, and
deploy the maven artifacts and its dependencies
"""
compiler_list = []
for item in self.config.get_thrift_option("compilers"):
t = ThriftCompiler(item)
compiler_list.append(t)
#ensure that vcs is enabled, and thrift-file override hasn't been passed in.
thrift_objects = []
if self.config.is_local() and self.config.get_service_override() is not None:
pass
elif not self.config.is_vcs or self.config.is_local():
#flatten list
thrift_objects = self.service_objects + self.business_objects + self.enum_objects + self.exception_objects
else:
vcs = self.config.get_vcs_instance()
file_objects = vcs.get_modified_files()
if file_objects is None or len(file_objects) == 0:
self.config.is_vcs = False
thrift_objects = self.service_objects + self.business_objects + self.enum_objects + self.exception_objects
else:
self.log.info("Using list of objects from VCS")
thrift_objects = map(lambda current_file: os.path.basename(current_file), file_objects)
self.log.info("VCS object list is: " + str(thrift_objects))
if self.config.is_local() and self.config.get_service_override() is not None:
self.service_objects = []
thrift_objects = [self.config.get_service_override()]
self.update_client_list(thrift_objects, compiler_list)
process_list = []
for client in self.client_list:
p = Process(target=client.run)
p.start()
process_list.append(p)
#wait for all threads that have been started to terminate.
map(lambda proc: proc.join(), process_list)
# #Check exit codes
for proc in process_list:
self.local_assert(proc.exitcode, str(proc))
def setUp(self):
self.config = Config()
init_data = {}
self.gitcvs = GitVCS(**init_data)
class Thrift():
def __init__(self, compiler):
self.compiler=compiler
self.version_pattern = re.compile("const\s+string\s+VERSION\s*=\s*['\"]+(.*)['\"]+")
self.java_namespace_pattern = re.compile("namespace\s+java\s+(.*)")
self.ruby_namespace_pattern = re.compile("namespace\s+rb\s+(.*)")
self.data_type_pattern = re.compile(
"\s+(\d+):\s+([required,optional]+)\s+([bool,byte,i16,i32,i64,string,double,string]+)\s+(\w+)")
self.config = Config()
self.log = Log(log_file="status.log", logger_name="status").log
# defines the language compiler to spawn.
self.build = {"ruby": self.__thrift_ruby_build__, "java": self.__thrift_java_build__,
"doc": self.__thrift_doc_build__}
def __setup_maven__(self, sandbox="default"):
"""
removes the old maven structure, and recreates it
"""
if sandbox == "default":
sanbox = self.config.work_dir
os.system("rm -fr %s" % (os.path.join(sandbox, "src")))
os.system("rm -f %s" % os.path.join(sandbox, "pom.xml"))
wize_mkdir(self.config.maven_dir)
wize_mkdir(self.config.get_thrift_option("thrift"))
def check_file(self, thrift_file):
"""
checks whether a file exists or not in the workdir
"""
thrift_file = self.get_thrift_full_path(thrift_file)
if not os.path.exists(thrift_file):
raise IOError("file {prettyfile} does not exist".format(prettyfile=thrift_file))
def __thrift_java_build__(self, thrift_file, sandbox="default"):
"""
compile the thrift file passed in to java format.
:param thrift_file:
"""
if sandbox == "default":
sandbox = self.config.work_dir
thrift_file = self.get_thrift_full_path(thrift_file)
#removes previously generated code
os.system("rm -fr %s" % os.path.join(sandbox, self.config.java_sandbox))
os.chdir(sandbox)
self.log("Building java class for %s" % thrift_file)
cmd = "{thrift_binary} -I {business_objects} -I {exceptions} -I {enums} -I {services} {options} {language_options} {file}".format(
business_objects=self.config.get_path(type="business_object"),
services=self.config.get_path(type="service_object"), exceptions=self.config.get_path(type="exception_object"),
enums=self.config.get_path(type="enum_object"), thrift_binary=self.compiler.bin,
file=thrift_file, options=self.compiler.options, language_options=self.compiler.language_options("java"))
exit_code = subprocess.call(shlex.split(cmd))
if not exit_code == 0:
self.log("failed to compile thrift file {file}".format(file=thrift_file))
sys.exit(1)
def __thrift_doc_build__(self, thrift_file, sandbox="default"):
if sandbox == "default":
sandbox = self.config.work_dir
thrift_file = self.get_thrift_full_path(thrift_file)
os.system("rm -fr %s" % os.path.join(sandbox, self.config.doc_sandbox))
wize_mkdir(self.config.doc_sandbox)
full_path = self.get_thrift_full_path(thrift_file)
dependencies = self.read_thrift_dependencies_recursively(full_path)
for item in dependencies:
dependency_file = self.get_thrift_full_path(item)
cmd = "{thrift_binary} -I {business_objects} -I {exceptions} -I {enums} -I {services} {options} " \
"{language_options} -o {destination} {file} ".format(
business_objects=self.config.get_path(type="business_object"),
exceptions=self.config.get_path(type="exception_object"),
enums=self.config.get_path(type="enum_object"),
services=self.config.get_path(type="service_object"),
thrift_binary=self.compiler.bin, file=dependency_file, options=self.compiler.options,
language_options=self.compiler.language_options("doc"),
destination=sandbox)
exit_code = subprocess.call(shlex.split(cmd))
if not exit_code == 0:
self.log("failed to compile thrift dependency {dependency} for file {file}".format(
dependency=dependency_file, file=dependency_file))
sys.exit(1)
self.log("Generating documentation for %s" % thrift_file)
cmd = "{thrift_binary} -I {business_objects} -I {exceptions} -I {enums} -I {services} {options} " \
"{language_options} -o {destination} {file}".format(
business_objects=self.config.get_path(type="business_object"),
exceptions=self.config.get_path(type="exception_object"),
enums=self.config.get_path(type="enum_object"), services=self.config.get_path(type="service_object"),
thrift_binary=self.compiler.bin,
file=thrift_file, options=self.compiler.options, language_options=self.compiler.language_options("doc"),
destination=sandbox)
exit_code = subprocess.call(shlex.split(cmd))
if not exit_code == 0:
self.log("failed to compile thrift file {file}".format(file=thrift_file))
sys.exit(1)
def __thrift_ruby_build__(self, thrift_file, sandbox="default"):
"""
compile the thrift file passed in to ruby format.
:param thrift_file:
"""
if sandbox == "default":
sandbox = self.config.work_dir
os.chdir(sandbox)
thrift_file = self.get_thrift_full_path(thrift_file)
os.system("rm -fr %s" % (self.config.get_ruby_option("sandbox", True)))
os.system("rm -fr %s" % (os.path.join(self.config.work_dir, "ruby")))
cmd = "{thrift_binary} -I {business_objects} -I {exceptions} -I {enums} -I {services} {options} " \
"{language_options} {file}".format( business_objects=self.config.get_path(type="business_object"),
exceptions=self.config.get_path(type="exception_object"),
enums=self.config.get_path(type="enum_object"), services=self.config.get_path(type="service_object"),
thrift_binary=self.compiler.bin,
file=thrift_file, options=self.compiler.options, language_options=self.compiler.language_options("ruby"))
exit_code = subprocess.call(shlex.split(cmd))
if exit_code != 0:
self.log("failed to compile thrift file %s" % thrift_file)
sys.exit(exit_code)
self.log("Building ruby class for %s" % thrift_file)
def is_service(self, thrift_file):
"""
boolean check on thrift file whether true if it contains the service_prefix in the file name.
Usually it's 'service' or whatever you overrode the value with in the config file.
"""
if thrift_file.find(self.config.get_global_option("service_prefix")) != -1:
return True
else:
return False
def is_business_object(self, thrift_file):
"""
boolean check on thrift file whether true if it contains the object_prefix in the file name.
Usually it's 'bizobj' or whatever you overrode the value with in the config file.
"""
if thrift_file.find(self.config.get_global_option("object_prefix")) != -1:
return True
else:
return False
def thrift_build(self, thrift_file, language="java", sandbox="default"):
"""
compiles a single thrift file and copies content to
maven structure for consumption.
"""
if sandbox == "default":
sandbox = self.config.work_dir
if self.build.has_key(language):
self.build.get(language)(thrift_file, sandbox)
def get_thrift_full_path(self, thrift_file):
if thrift_file is None:
return None
if thrift_file.find("/") >= 0:
return thrift_file
if thrift_file.find(self.config.get_global_option("service_prefix")) != -1:
return os.path.join(self.config.get_path(type="service_object"), thrift_file)
elif thrift_file.find(self.config.get_global_option("enum_prefix")) != -1:
return os.path.join(self.config.get_path(type="enum_object"), thrift_file)
elif thrift_file.find(self.config.get_global_option("exception_prefix")) != -1:
return os.path.join(self.config.get_path(type="exception_object"), thrift_file)
else:
return os.path.join(self.config.get_path(type="business_object"), thrift_file)
def read_thrift_dependencies_recursively(self, thrift_file):
dependencies = set(self.read_thrift_dependencies(thrift_file))
if len(dependencies) is 0:
return set()
for item in dependencies:
dependencies = dependencies.union(set(self.read_thrift_dependencies_recursively(item)))
return dependencies
def check_itemized_contraints(self, line, thrift_file):
"""
If enabled via the yaml configuration. This method will ensure that certain contraints are enforced.
These constraints are applicable on individual lines items and do not need full context of the file.
If any of them fail, it will throw a ConstraintException
"""
if not self.config.constraints_enabled:
return
if self.config.visibility_check_enabled:
if line.find("bool") >= 0 or line.find("i64") >= 0 or line.find("i32") >= 0 or \
line.find("byte") >= 0 or line.find("double") >= 0 or line.find("string") >= 0:
if line.find("optional") == -1 and line.find("required") == -1 and line.find("VERSION") == -1:
self.log("Error: package visibility flag omitted for line: {line} in file: {file}."
.format(line=line.replace("\n", ""), file=thrift_file))
raise ConstraintException("visibility flag is missing. Please use 'optional' or 'required' "
"to set visibility. Please check log file for more details.")
def check_constraints(self, thrift_file, diff, raw_data):
"""
If enabled via the yaml configuration. This method will ensure that certain contraints are enforced.
If any of them fail, it will throw a ConstraintException
"""
if not self.config.constraints_enabled:
return
if self.config.empty_constraint_enabled and self.is_business_object(thrift_file):
match = re.search("bool.*empty", raw_data)
if match is not None and raw_data.find("required") == -1:
self.log("Error: bool empty field must be a required field. Missing in file {file}".format(
file=thrift_file))
raise ConstraintException("empty flag is required. Please ensure to include a boolean empty field "
"in your struct")
#Everything below this line in under Development.
## Needs further development and interaction with VCS module
if self.config.check_field_ordering_enabled:
### pseudologic: get current thrift file, call a helper method which will extract the field Ids, names,
### and other metrics. retrieve previous version of file and do the same operation, compare resulting
### constructs. If thrift patterns are broke. ie. field ordering is changed, required is removed and such.
### throw an exception.
pass
if self.config.check_version_increment_enabled:
## pseudologic: check thrift_file via VCS. If file is changed, ensure that VERSION number has changed.
if diff is not None and diff != "" and diff.find("VERSION") == -1:
self.log("Error, You forgot to increment the version for {file}".format(file=thrift_file))
raise ConstraintException("a file change requires a version increment.")
## Under development, not integrated into any clients yet.
def decipher_thrift_file(self, thrift_file):
"""
This method will read a thrift file and pull out any data that we understand
for further logic processing.
ie. numeric index, type, field name, VERSION constant etc.
"""
if self.is_service(thrift_file):
return {}
thrift_file = self.get_thrift_full_path(thrift_file)
self.check_file(thrift_file)
fp = open(thrift_file, 'r')
data = fp.readlines()
fp.close()
thrift_meta = {}
for line in data:
match = self.version_pattern.match(line)
if match is not None:
thrift_meta['version'] = match.group(1)
match = self.java_namespace_pattern.match(line)
if match is not None:
thrift_meta['java_namespace'] = match.group(1)
match = self.ruby_namespace_pattern.match(line)
if match is not None:
thrift_meta['ruby_namespace'] = match.group(1)
match = self.data_type_pattern.match(line)
if match is not None:
if match.lastindex != 4:
self.log("Error: found a discrepancy when parsing {file}".format(thrift_file=file))
sys.exit(1)
constraint_type = {}
constraint_type['visibility'] = match.group(2)
constraint_type['type'] = match.group(3)
constraint_type['name'] = match.group(4)
thrift_meta[match.group(1)] = constraint_type
return thrift_meta
def read_thrift_dependencies(self, thrift_file):
"""
Parse a thrift file and extract the includes returning it as a list of
it's dependencies.
"""
self.log("Processing %s" % thrift_file)
is_enum = False
is_exception = False
thrift_file = self.get_thrift_full_path(thrift_file)
self.check_file(thrift_file)
fp = open(thrift_file, 'r')
data = fp.readlines()
raw_data = "".join(data)
self.check_constraints(thrift_file, None, raw_data)
fp.close()
includes = []
object_count = 0
for line in data:
#skip comments
if line.startswith("#"):
continue
if not self.is_service(thrift_file):
self.check_itemized_contraints(line, thrift_file)
if line.startswith("enum"):
is_enum = True
if line.startswith("exception"):
is_exception = True
if line.startswith("include"):
line = line.strip()
begin = line.find("\"")
end = line.find("\"", begin + 1)
includes.append(line[begin + 1:end])
if line.startswith("struct") or line.startswith("enum") or line.startswith("exception"):
object_count += 1
if object_count > 1:
self.log(
"tsk..tsk..tsk, you tried to build an object {file} that contained multiple structs/enums.".format(
file=thrift_file))
sys.exit(1)
if is_enum:
ndx = thrift_file.find(self.config.get_global_option("enum_prefix"))
if ndx < 0:
self.log("error, any enum should be defined inside a .enum file")
sys.exit(1)
if is_exception:
ndx = thrift_file.find(self.config.get_global_option("exception_prefix"))
if ndx < 0:
self.log("error, any exception should be defined inside a .exception file")
sys.exit(1)
return includes
def read_thrift_properties(self, thrift_file):
"""
Given a thrift file it will return the groupid, version and artifact name
as hashmap/dictionary.
"""
thrift_file = self.get_thrift_full_path(thrift_file)
if not os.path.exists(thrift_file):
return {}
meta_data = {}
f = open(thrift_file)
raw = f.read()
f.close()
meta_data["VERSION"] = self.__extract_string__(pattern="VERSION", raw=raw, default="0.0.0")
meta_data["GROUPID"] = self.__extract_string__(pattern="GROUPID", raw=raw, default=self.config.get_global_option("group_id"))
meta_data["ARTIFACTID"] = self.get_object_name(thrift_file)
return meta_data
@staticmethod
def __extract_string__(pattern, raw, default=None):
"""
Given a text line it will try to extract the value from a thrift constant matching the pattern passed in.
"""
match = re.search(".*{search_pattern}.*=".format(search_pattern=pattern), raw)
if match is None:
return default
start = match.start() + len(match.group(0))
end = raw.find("\n", start)
if start == -1 or end == -1:
return default
result = re.sub("[',\"]", "", raw[start:end]) # remove quotes
return re.sub("#.*", "", result).strip()
def get_object_name(self, raw, delimter="-"):
"""
Build an artifact name from a filename.
ie. given wizecommerce.bizobj.foobar.example.thrift it should return foobar-example-bizobj
given wizecommerce.services.foobar.example.thrift it should return foobar-example-client
"""
raw = os.path.basename(raw).replace(".thrift", "")
pattern = "{company_name}.".format(company_name=self.config.get_global_option("company_name"))
ndx = raw.find(pattern)
if ndx < 0:
self.log("could not find company prefix pattern: {pattern}".format(pattern=pattern))
sys.exit(ndx)
raw = raw[ndx + len(pattern):]
ndx = raw.find(".")
if ndx < 0:
self.log("could not find delimiter in file name")
sys.exit(ndx)
postfix = raw[:ndx]
if postfix == self.config.get_global_option("service_prefix"):
postfix = "client"
raw = raw.replace(self.config.get_global_option("service_prefix") + ".", "")
else:
raw = raw.replace(postfix + ".", "")
raw = raw.replace(".", delimter) + delimter + postfix
return raw
def main():
parser = argparse.ArgumentParser(description='Client Generation Script')
parser.add_argument('--local', action="store_true", dest="local", default=False, help="Enables Local Mode")
parser.add_argument('--profile', action="store_true", dest="profile", default=False, help="Profiles App")
parser.add_argument("--docOnly", action="store_true", dest="doc_only", default=False)
parser.add_argument('--ruby', action="store_true", dest="ruby", default=False,
help="Enables RubyMode, default is Ruby + Java (Local Mode Only)")
parser.add_argument('--java', action="store_true", dest="java", default=False,
help="Enables JavaMode, default is Ruby + Java (Local Mode Only) ")
parser.add_argument('--thrift-file', action="store", dest="thrift_file", type=str,
help="Override list of services, and use the one specified (Local Mode Only)\nThis overrides vcs intelligence")
parser.add_argument('--config', action="store", dest="config", type=str,
help="Override default config file and specify your own yaml config")
parser.add_argument('--compilers', action="store_true", dest="compilers", default=False,
help="will list all supported compilers. (Not fully supported)")
parser.add_argument('--set-compiler', action="store", dest="compiler", type=str,
help="accepts a valid compiler name defined in the yaml config.")
args = parser.parse_args()
if args.config is None:
config = Config()
else:
config = Config(args.config)
config.set_local(args.local)
if args.thrift_file is not None:
config.set_service_override(os.path.basename(args.thrift_file))
if args.thrift_file is not None:
config.set_local(True)
if args.compilers:
display_compilers()
if args.compiler is not None:
set_compiler(args.compiler)
publish_client = PublishClient()
## these options can only be used in conjunction with local mode
if config.is_local():
if args.ruby and args.java:
print "WARNING: you really should use rubyOverride or JavaOverride, " \
"if you pass both it can will fall back on default behavior. (ie. omit both of them)"
elif args.ruby:
config.set_languages({"ruby": True})
elif args.java:
config.set_languages({"java": True})
if args.doc_only:
config.set_languages({})
config.is_doc_enabled = True
if args.profile:
import cProfile
cProfile.run('profileProject()')
else:
# Create Repo Structure
publish_client.create_structure()
# Loop through all of our services check for updates
publish_client.process_thrift_services()
class Client():
"""
For the most part, the purpose fo this class is to define an API that you can extend
to implement your own Client for any language that isn't already supported.
"""
def get_sandbox(self):
return self.sandbox
def set_sandbox(self, value):
self.sandbox = value
def __init__(self, services, thrift_compiler):
self.compiler = thrift_compiler
self.config = Config()
self.sandbox_work = self.config.work_dir
self.status = Status()
self.thrift_helper = Thrift(self.compiler)
self.log = Log(log_file="status.log", logger_name="status").log
self.services = services
def run(self):
self.initialize()
for service in self.services:
self.process_service(service)
self.finalize()
def __build_dependency__(self, business_object):
"""
Recursively build the dependency and return a list of all dependencies found and successfully built.
"""
raise NotImplementedError("Build Dependency needs to be overridden")
##if previous error code has been found, aborting.
def __build_client__(self, service):
"""
This method is called to build the actual client, in our case that includes
all of our services.
"""
raise NotImplementedError("Build Client needs to be overridden")
def deploy_object(self, properties, business_object, postfix=""):
"""
The purpose of this method is to handle the deployment / install.
If set to localMode it will call the appropriate method for handling local installs,
if doing a production deployment it will call the appropriate method
postfix str: a string appended to the artifact name used for releasing snapshot for example.
properties dict: list of properties used for doing deployment.
businessObject str: string containing the name of the thrift file.
"""
##Local mode will only build snapshots, for now.
if self.config.is_local():
return self.__deploy_local_artifact__(
properties=properties,
thrift_object=business_object,
postfix=postfix)
else:
return self.__deploy_production_artifact__(
properties=properties,
thrift_object=business_object,
postfix=postfix)
return 0
def check_version(self, **kwargs):
"""
Returns a boolean checking if the artifact exists on the deployment server.
"""
raise NotImplementedError("Check Version needs to be overridden")
def __deploy_production_artifact__(self,
properties,
thrift_object,
postfix=""):
"""
Implement a method responsible for deploying your artifact to your production server.
"""
raise NotImplementedError(
"Deploy Production Artifact needs to be overridden")
def __deploy_local_artifact__(self, properties, thrift_object, postfix=""):
"""
Implement a method responsible for performing a local install.
"""
raise NotImplementedError(
"Deploy Local Artifact needs to be overridden")
def finalize(self):
"""
This method is called as a last step to either clean , publish or whatever needs to be done.
"""
raise NotImplementedError("Finalize needs to be overridden")
def initialize(self):
"""
Optional, used to initialize/construct any environment or file system settings that need
to be setup that are specific to the client.
"""
raise NotImplementedError("Initialize method has not been overridden")
def process_service(self, service):
"""
This method builds the client and all dependencies assuming appropriate
metadata is contained in the thrift file.
"""
os.chdir(self.sandbox_work)
dependencies = self.thrift_helper.read_thrift_dependencies(service)
#Adding the service file as well to the list.
if len(dependencies) == 0:
print "No dependencies for %s" % service
else:
for dependency in dependencies:
self.local_assert(
self.__build_dependency__(dependency),
"Failed to process dependencies for {service}".format(
service=dependency))
self.local_assert(
self.__build_client__(service),
"Failed to build Client for {service}".format(
service=str(service)))
return 0
def local_assert(self, exit_code, message, prefix="ERROR: "):
if exit_code != 0:
self.log(prefix + message)
sys.exit(exit_code)
class JavaClientTests(unittest.TestCase):
def setUp(self):
self.client = self.__get_client__()
self.client.initialize()
self.service_name = os.path.join(
os.getcwd(), "../thrift/services/",
"wizecommerce.services.example.thrift")
def __get_client__(self):
self.config = Config()
compiler = None
for item in self.config.get_thrift_option("compilers"):
compiler = ThriftCompiler(item)
compiler.postfix = ""
break
return JavaClient([], compiler)
@httprettified
def test_check_version_pretty(self):
self.config.set_local(False)
expected = False
expected_url = 'http://nexus.corp.nextag.com:8081/content/repositories/releases/com/wizecommerce/data/tag-client/0.0.13/tag-client-0.0.13.jar'
HTTPretty.register_uri(HTTPretty.GET, expected_url, body="Hello")
check_value = self.client.check_version(
groupId='com.wizecommerce.data',
artifactId='tag-client',
version='0.0.13')
self.failUnlessEqual(check_value, expected)
check_value = self.client.check_version(
groupId='com.wizecommerce.data',
artifactId='tag-client',
version='X.Y.ZZ')
expected = True
self.failUnlessEqual(check_value, expected)
@httprettified
def test_check_shaded_version_pretty(self):
self.config.set_local(False)
expected = False
expected_url = 'http://crepo.corp.nextag.com/repo/components/com/wizecommerce/data/tag-client-shaded/0.0.13/tag-client-shaded-0.0.13.jar'
HTTPretty.register_uri(HTTPretty.GET, expected_url, body="Hello")
check_value_shaded = self.client.check_shaded_version(
groupId='com.wizecommerce.data',
artifactId='tag-client',
version='0.0.13')
print check_value_shaded
print expected
assert check_value_shaded == expected
self.failUnlessEqual(check_value_shaded, expected)
check_value_shaded = self.client.check_shaded_version(
groupId='com.wizecommerce.data',
artifactId='tag-client',
version='X.Y.ZZ')
expected = True
assert check_value_shaded == expected
def tearDown(self):
self.config.reset_configuration()
def __init__(self, properties):
self.config = Config()
self.meta_data = properties.copy()
class ThriftCompiler(object):
"""
This should allow support for multiple thrift compilers. Passing in specialized options etc. This part of the code
isn't really used yet, and is mainly a placeholder for now.
"""
def __init__(self, properties):
self.config = Config()
self.meta_data = properties.copy()
@property
def name(self):
return self.meta_data.get("name")
@name.setter
def name(self, value):
self.meta_data['name'] = value
@property
def bin(self):
return self.meta_data.get("bin")
@bin.setter
def bin(self, value):
self.meta_data['bin'] = value
@property
def options(self):
return self.meta_data.get("options")
@options.setter
def options(self, value):
self.meta_data['options'] = value
@property
def version(self):
try:
return self.meta_data["version"]
except:
return "0.6.1" ##our default version
@version.setter
def version(self, value):
self.meta_data["version"] = value
def language_options(self, language="java"):
try:
return self.meta_data.get("language_options").get(language)
except:
#fallback on global options.
try:
data = self.config.get_thrift_option("global_compiler_options")
return data[language]
except:
return None
@property
def languages(self):
return self.meta_data.get("supported_languages")
@languages.setter
def languages(self, value):
self.meta_data['supported_languages'] = value
def is_language_supported(self, value):
if self.languages is None:
return False
for lang in self.languages:
if lang == value:
return True
return False
@property
def postfix(self):
try:
return self.meta_data["compiler_postfix"]
except:
return ""
@postfix.setter
def postfix(self, value):
self.meta_data["compiler_postfix"] = value
def __init__(self, properties):
self.config = Config()
self.meta_data = properties.copy()
class ThriftTests(unittest.TestCase):
def setUp(self):
self.config = Config()
self.config.reset_configuration()
compilers = self.config.get_thrift_option("compilers")
if compilers is None or len(compilers) == 0:
self.helper = Thrift(None)
else:
thrift_compiler = ThriftCompiler(compilers[0])
self.helper = Thrift(thrift_compiler)
project_path = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), "../")
project_path = os.path.realpath(project_path)
self.config.repo_dir = project_path
def test_extract_string(self):
raw = \
"""
namespace java com.wizecommerce.service.common
const string VERSION = "0.0.6"
# Type Definitions
typedef i32 Integer
"""
expected = "0.0.6"
result = self.helper.__extract_string__("VERSION", raw, "0.0.0")
self.failUnlessEqual(result, expected)
# def reset_config(self):
# self.config.reset_configuration()
# project_path = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), "../")
# project_path = os.path.realpath(project_path)
# self.config.repo_dir = project_path
def test_check_file(self):
thrift_file = "wizecommerce.services.example.thrift"
self.helper.check_file(thrift_file)
#we should reach this point, without any exception thrown.
thrift_file = "wizecommerce.service.dummyfile.thrift"
self.assertRaises(IOError, self.helper.check_file,thrift_file)
def test_thrift_java_build(self):
real_methods = {'subprocess': subprocess.call, 'system': os.system, 'chdir': os.chdir,
'get_thrift_full_path': self.helper.get_thrift_full_path}
subprocess.call = MagicMock()
os.system = MagicMock()
os.chdir = MagicMock()
self.helper.get_thrift_full_path = MagicMock()
thrift_file = "dummy.thrift"
self.helper.get_thrift_full_path.return_value = thrift_file
subprocess.call.return_value = 0
self.helper.thrift_build(thrift_file)
#########################
# verify subprocess.call
#########################
expected_values = {'compiler': "/usr/local/bin/thrift",
'options': "java:private-members,hashcode", 'size': 12,
'file': thrift_file}
call_list = subprocess.call.call_args_list
self.assertEquals(len(call_list), 1)
self.verify_subprocess_call(subprocess.call.call_args, expected_values)
#########################
# verify chdir calls
#########################
call_list = os.chdir.call_args_list
self.assertEquals(len(call_list), 1)
call_args = os.chdir.call_args[0][0]
self.assertEquals(call_args, self.config.work_dir)
#########################
# verify system calls
#########################
call_list = os.system.call_args_list
call = call_list[0]
self.assertEquals(len(call_list), 1)
parameter = call[0][0]
self.assertNotEqual(parameter.find("rm -fr"), -1)
self.assertNotEqual(parameter.find("gen-java"), -1)
#########################
# reset all system calls
#########################
subprocess.call = real_methods['subprocess']
os.system = real_methods['system']
os.chdir = real_methods['chdir']
self.helper.get_thrift_full_path = real_methods['get_thrift_full_path']
def test_thrift_ruby_build(self):
thrift_file = "dummy.thrift"
real_methods = {'subprocess' : subprocess.call, 'system' : os.system, 'chdir' : os.chdir,
'get_thrift_full_path' : self.helper.get_thrift_full_path}
subprocess.call = MagicMock()
os.system = MagicMock()
os.chdir = MagicMock()
self.helper.get_thrift_full_path = MagicMock()
self.helper.get_thrift_full_path.return_value = thrift_file
subprocess.call.return_value = 0
self.helper.thrift_build(thrift_file=thrift_file, language="ruby")
#########################
# verify subprocess.call
#########################
expected_values = {'compiler': "/usr/local/bin/thrift",
'options': "--gen rb", 'size': 12,
'file': thrift_file}
call_list = subprocess.call.call_args_list
self.assertEquals(len(call_list), 1)
self.verify_subprocess_call(subprocess.call.call_args, expected_values)
#########################
# verify chdir calls
#########################
call_list = os.chdir.call_args_list
self.assertEquals(len(call_list), 1)
call_args = os.chdir.call_args[0][0]
self.assertEquals(call_args, self.config.work_dir)
#########################
# verify system calls
#########################
call_list = os.system.call_args_list
call = call_list[0]
self.assertEquals(len(call_list), 2)
parameter = call[0][0]
self.assertNotEqual(parameter.find("rm -fr"), -1)
self.assertNotEqual(parameter.find("gen-rb"), -1)
#next call
call = call_list[1]
parameter = call[0][0]
self.assertNotEqual(parameter.find("rm -fr"), -1)
print parameter
self.assertNotEqual(parameter.find("work/ruby"), -1)
#########################
# reset all system calls
#########################
subprocess.call = real_methods['subprocess']
os.system = real_methods['system']
os.chdir = real_methods['chdir']
self.helper.get_thrift_full_path = real_methods['get_thrift_full_path']
def intercept_thrift_doc_full_path(self, request):
"""
This method intercepts the call for self.helper.read_full_path and simply returns the
value passed in.
"""
return request
def verify_subprocess_call(self, method_call_args, expected_values):
call_args = method_call_args[0][0]
self.assertEquals(len(call_args), expected_values['size'])
parameter = " ".join(call_args)
#verify compiler
self.assertNotEqual(parameter.find(expected_values['compiler']), -1)
#verify file name
self.assertNotEqual(parameter.find(expected_values['file']), -1)
#verify language/options
self.assertNotEqual(parameter.find(expected_values['options']), -1)
#verify includes
self.assertNotEqual(parameter.format("thrift/services"), -1)
self.assertNotEqual(parameter.format("business-objects"), -1)
def test_thrift_doc_build(self):
thrift_file = "dummy.thrift"
real_methods = {'subprocess' : subprocess.call, 'system' : os.system,
'get_thrift_full_path' : self.helper.get_thrift_full_path,
'read_thrift_dependencies_recursively': self.helper.read_thrift_dependencies_recursively }
dependencies = ['dependency1.thrift', 'dependency2.thrift']
subprocess.call = MagicMock()
os.system = MagicMock()
self.helper.get_thrift_full_path = MagicMock()
self.helper.get_thrift_full_path.return_value = thrift_file
self.helper.get_thrift_full_path = self.intercept_thrift_doc_full_path
subprocess.call.return_value = 0
self.helper.read_thrift_dependencies_recursively = MagicMock()
self.helper.read_thrift_dependencies_recursively.return_value = dependencies
self.helper.thrift_build(thrift_file=thrift_file, language="doc")
#########################
# verify subprocess.call
#########################
expected_values = {'compiler': "/usr/local/bin/thrift",
'options': "--gen html", 'size': 14,
'file': thrift_file}
call_list = subprocess.call.call_args_list
self.assertEquals(len(call_list), 3)
count = 0
for dependency_file in dependencies:
call = call_list[count]
expected_values['file'] = dependency_file
call_list = subprocess.call.call_args_list
self.verify_subprocess_call(call, expected_values)
count += 1
#verify client
expected_values['file'] = thrift_file
call = call_list[count]
self.verify_subprocess_call(call, expected_values)
#########################
# verify system calls
#########################
call_list = os.system.call_args_list
call = call_list[0]
self.assertEquals(len(call_list), 1)
parameter = call[0][0]
self.assertNotEqual(parameter.find("rm -fr"), -1)
self.assertNotEqual(parameter.find("gen-html"), -1)
#########################
# reset all system calls
#########################
subprocess.call = real_methods['subprocess']
os.system = real_methods['system']
self.helper.get_thrift_full_path = real_methods['get_thrift_full_path']
self.helper.read_thrift_dependencies_recursively = real_methods['read_thrift_dependencies_recursively']
def test_is_service(self):
dummy = "foobar.services.thrift"
result = self.helper.is_service(dummy)
self.assertTrue(result)
dummy = "foobar.dummy.thrift"
result = self.helper.is_service(dummy)
self.assertFalse(result)
def test_read_thrift_dependencies(self):
real_file = "wizecommerce.services.example.thrift"
dependencies = self.helper.read_thrift_dependencies_recursively(real_file)
self.assertEquals(len(dependencies), 3)
all_deps = " ".join(dependencies)
self.assertNotEqual(all_deps.find("exception.invalid"), -1)
self.assertNotEqual(all_deps.find("enum.example_change"), -1)
self.assertNotEqual(all_deps.find("bizobj.example"), -1)
##TODO: possible enhancement, mock out the file we're reading.
def test_read_thrift_properties(self):
thrift_file = "wizecommerce.services.example.thrift"
properties = self.helper.read_thrift_properties(thrift_file)
self.assertEquals('example-client', properties['ARTIFACTID'])
self.assertEquals('0.0.5', properties['VERSION'])
self.assertEquals('com.wizecommerce.data', properties['GROUPID'])
print properties
def test_thrift_compiler(self):
init = {}
init['name'] = 'testThrift'
init['bin'] = '/dev/null'
init['options'] = 'testoptions'
init['supported_languages'] = ['java', 'ruby', 'python']
t = ThriftCompiler(init)
self.failUnless('testThrift' == t.name)
self.failUnless('/dev/null' == t.bin)
self.failUnless('testoptions' == t.options)
self.failUnless(len(t.languages) == 3)
self.failUnless(t.is_language_supported('ruby'))
self.failUnless(not t.is_language_supported('erlang') )
def test_object_name(self):
th = Thrift(None)
src = "/home/user/foobar/blah/wizecommerce.bizobj.sellerprogramrecord.thrift"
result = th.get_object_name(src)
expected = "sellerprogramrecord-bizobj"
self.failUnlessEqual(expected, result)
def test_complex_object_name(self):
th = Thrift(None)
src = "/home/user/foobar/blah/wizecommerce.bizobj.search.foobar.sellerprogramrecord.thrift"
result = th.get_object_name(src)
expected = "search-foobar-sellerprogramrecord-bizobj"
self.failUnlessEqual(expected, result)
def test_special_case_object_name(self):
th = Thrift(None)
src = "/home/user/foobar/blah/wizecommerce.services.sellerprogramrecord.thrift"
result = th.get_object_name(src)
expected = "sellerprogramrecord-client"
self.failUnlessEqual(expected, result)
class ThriftCompiler(object):
"""
This should allow support for multiple thrift compilers. Passing in specialized options etc. This part of the code
isn't really used yet, and is mainly a placeholder for now.
"""
def __init__(self, properties):
self.config = Config()
self.meta_data = properties.copy()
@property
def name(self):
return self.meta_data.get("name")
@name.setter
def name(self, value):
self.meta_data['name'] = value
@property
def bin(self):
return self.meta_data.get("bin")
@bin.setter
def bin(self, value):
self.meta_data['bin'] = value
@property
def options(self):
return self.meta_data.get("options")
@options.setter
def options(self, value):
self.meta_data['options'] = value
@property
def version(self):
try:
return self.meta_data["version"]
except:
return "0.6.1" ##our default version
@version.setter
def version(self, value):
self.meta_data["version"] = value
def language_options(self, language="java"):
try:
return self.meta_data.get("language_options").get(language)
except:
#fallback on global options.
try:
data = self.config.get_thrift_option("global_compiler_options")
return data[language]
except:
return None
@property
def languages(self):
return self.meta_data.get("supported_languages")
@languages.setter
def languages(self, value):
self.meta_data['supported_languages'] = value
def is_language_supported(self, value):
if self.languages is None:
return False
for lang in self.languages:
if lang == value:
return True
return False
@property
def postfix(self):
try:
return self.meta_data["compiler_postfix"]
except:
return ""
@postfix.setter
def postfix(self, value):
self.meta_data["compiler_postfix"] = value
class ThriftTests(unittest.TestCase):
def setUp(self):
self.config = Config()
self.config.reset_configuration()
compilers = self.config.get_thrift_option("compilers")
if compilers is None or len(compilers) == 0:
self.helper = Thrift(None)
else:
thrift_compiler = ThriftCompiler(compilers[0])
self.helper = Thrift(thrift_compiler)
project_path = os.path.join(
os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe()))),
"../")
project_path = os.path.realpath(project_path)
self.config.repo_dir = project_path
def test_extract_string(self):
raw = \
"""
namespace java com.wizecommerce.service.common
const string VERSION = "0.0.6"
# Type Definitions
typedef i32 Integer
"""
expected = "0.0.6"
result = self.helper.__extract_string__("VERSION", raw, "0.0.0")
self.failUnlessEqual(result, expected)
# def reset_config(self):
# self.config.reset_configuration()
# project_path = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), "../")
# project_path = os.path.realpath(project_path)
# self.config.repo_dir = project_path
def test_check_file(self):
thrift_file = "wizecommerce.services.example.thrift"
self.helper.check_file(thrift_file)
#we should reach this point, without any exception thrown.
thrift_file = "wizecommerce.service.dummyfile.thrift"
self.assertRaises(IOError, self.helper.check_file, thrift_file)
def test_thrift_java_build(self):
real_methods = {
'subprocess': subprocess.call,
'system': os.system,
'chdir': os.chdir,
'get_thrift_full_path': self.helper.get_thrift_full_path
}
subprocess.call = MagicMock()
os.system = MagicMock()
os.chdir = MagicMock()
self.helper.get_thrift_full_path = MagicMock()
thrift_file = "dummy.thrift"
self.helper.get_thrift_full_path.return_value = thrift_file
subprocess.call.return_value = 0
self.helper.thrift_build(thrift_file)
#########################
# verify subprocess.call
#########################
expected_values = {
'compiler': "/usr/local/bin/thrift",
'options': "java:private-members,hashcode",
'size': 12,
'file': thrift_file
}
call_list = subprocess.call.call_args_list
self.assertEquals(len(call_list), 1)
self.verify_subprocess_call(subprocess.call.call_args, expected_values)
#########################
# verify chdir calls
#########################
call_list = os.chdir.call_args_list
self.assertEquals(len(call_list), 1)
call_args = os.chdir.call_args[0][0]
self.assertEquals(call_args, self.config.work_dir)
#########################
# verify system calls
#########################
call_list = os.system.call_args_list
call = call_list[0]
self.assertEquals(len(call_list), 1)
parameter = call[0][0]
self.assertNotEqual(parameter.find("rm -fr"), -1)
self.assertNotEqual(parameter.find("gen-java"), -1)
#########################
# reset all system calls
#########################
subprocess.call = real_methods['subprocess']
os.system = real_methods['system']
os.chdir = real_methods['chdir']
self.helper.get_thrift_full_path = real_methods['get_thrift_full_path']
def test_thrift_ruby_build(self):
thrift_file = "dummy.thrift"
real_methods = {
'subprocess': subprocess.call,
'system': os.system,
'chdir': os.chdir,
'get_thrift_full_path': self.helper.get_thrift_full_path
}
subprocess.call = MagicMock()
os.system = MagicMock()
os.chdir = MagicMock()
self.helper.get_thrift_full_path = MagicMock()
self.helper.get_thrift_full_path.return_value = thrift_file
subprocess.call.return_value = 0
self.helper.thrift_build(thrift_file=thrift_file, language="ruby")
#########################
# verify subprocess.call
#########################
expected_values = {
'compiler': "/usr/local/bin/thrift",
'options': "--gen rb",
'size': 12,
'file': thrift_file
}
call_list = subprocess.call.call_args_list
self.assertEquals(len(call_list), 1)
self.verify_subprocess_call(subprocess.call.call_args, expected_values)
#########################
# verify chdir calls
#########################
call_list = os.chdir.call_args_list
self.assertEquals(len(call_list), 1)
call_args = os.chdir.call_args[0][0]
self.assertEquals(call_args, self.config.work_dir)
#########################
# verify system calls
#########################
call_list = os.system.call_args_list
call = call_list[0]
self.assertEquals(len(call_list), 2)
parameter = call[0][0]
self.assertNotEqual(parameter.find("rm -fr"), -1)
self.assertNotEqual(parameter.find("gen-rb"), -1)
#next call
call = call_list[1]
parameter = call[0][0]
self.assertNotEqual(parameter.find("rm -fr"), -1)
print parameter
self.assertNotEqual(parameter.find("work/ruby"), -1)
#########################
# reset all system calls
#########################
subprocess.call = real_methods['subprocess']
os.system = real_methods['system']
os.chdir = real_methods['chdir']
self.helper.get_thrift_full_path = real_methods['get_thrift_full_path']
def intercept_thrift_doc_full_path(self, request):
"""
This method intercepts the call for self.helper.read_full_path and simply returns the
value passed in.
"""
return request
def verify_subprocess_call(self, method_call_args, expected_values):
call_args = method_call_args[0][0]
self.assertEquals(len(call_args), expected_values['size'])
parameter = " ".join(call_args)
#verify compiler
self.assertNotEqual(parameter.find(expected_values['compiler']), -1)
#verify file name
self.assertNotEqual(parameter.find(expected_values['file']), -1)
#verify language/options
self.assertNotEqual(parameter.find(expected_values['options']), -1)
#verify includes
self.assertNotEqual(parameter.format("thrift/services"), -1)
self.assertNotEqual(parameter.format("business-objects"), -1)
def test_thrift_doc_build(self):
thrift_file = "dummy.thrift"
real_methods = {
'subprocess':
subprocess.call,
'system':
os.system,
'get_thrift_full_path':
self.helper.get_thrift_full_path,
'read_thrift_dependencies_recursively':
self.helper.read_thrift_dependencies_recursively
}
dependencies = ['dependency1.thrift', 'dependency2.thrift']
subprocess.call = MagicMock()
os.system = MagicMock()
self.helper.get_thrift_full_path = MagicMock()
self.helper.get_thrift_full_path.return_value = thrift_file
self.helper.get_thrift_full_path = self.intercept_thrift_doc_full_path
subprocess.call.return_value = 0
self.helper.read_thrift_dependencies_recursively = MagicMock()
self.helper.read_thrift_dependencies_recursively.return_value = dependencies
self.helper.thrift_build(thrift_file=thrift_file, language="doc")
#########################
# verify subprocess.call
#########################
expected_values = {
'compiler': "/usr/local/bin/thrift",
'options': "--gen html",
'size': 14,
'file': thrift_file
}
call_list = subprocess.call.call_args_list
self.assertEquals(len(call_list), 3)
count = 0
for dependency_file in dependencies:
call = call_list[count]
expected_values['file'] = dependency_file
call_list = subprocess.call.call_args_list
self.verify_subprocess_call(call, expected_values)
count += 1
#verify client
expected_values['file'] = thrift_file
call = call_list[count]
self.verify_subprocess_call(call, expected_values)
#########################
# verify system calls
#########################
call_list = os.system.call_args_list
call = call_list[0]
self.assertEquals(len(call_list), 1)
parameter = call[0][0]
self.assertNotEqual(parameter.find("rm -fr"), -1)
self.assertNotEqual(parameter.find("gen-html"), -1)
#########################
# reset all system calls
#########################
subprocess.call = real_methods['subprocess']
os.system = real_methods['system']
self.helper.get_thrift_full_path = real_methods['get_thrift_full_path']
self.helper.read_thrift_dependencies_recursively = real_methods[
'read_thrift_dependencies_recursively']
def test_is_service(self):
dummy = "foobar.services.thrift"
result = self.helper.is_service(dummy)
self.assertTrue(result)
dummy = "foobar.dummy.thrift"
result = self.helper.is_service(dummy)
self.assertFalse(result)
def test_read_thrift_dependencies(self):
real_file = "wizecommerce.services.example.thrift"
dependencies = self.helper.read_thrift_dependencies_recursively(
real_file)
self.assertEquals(len(dependencies), 3)
all_deps = " ".join(dependencies)
self.assertNotEqual(all_deps.find("exception.invalid"), -1)
self.assertNotEqual(all_deps.find("enum.example_change"), -1)
self.assertNotEqual(all_deps.find("bizobj.example"), -1)
##TODO: possible enhancement, mock out the file we're reading.
def test_read_thrift_properties(self):
thrift_file = "wizecommerce.services.example.thrift"
properties = self.helper.read_thrift_properties(thrift_file)
self.assertEquals('example-client', properties['ARTIFACTID'])
self.assertEquals('0.0.5', properties['VERSION'])
self.assertEquals('com.wizecommerce.data', properties['GROUPID'])
print properties
def test_thrift_compiler(self):
init = {}
init['name'] = 'testThrift'
init['bin'] = '/dev/null'
init['options'] = 'testoptions'
init['supported_languages'] = ['java', 'ruby', 'python']
t = ThriftCompiler(init)
self.failUnless('testThrift' == t.name)
self.failUnless('/dev/null' == t.bin)
self.failUnless('testoptions' == t.options)
self.failUnless(len(t.languages) == 3)
self.failUnless(t.is_language_supported('ruby'))
self.failUnless(not t.is_language_supported('erlang'))
def test_object_name(self):
th = Thrift(None)
src = "/home/user/foobar/blah/wizecommerce.bizobj.sellerprogramrecord.thrift"
result = th.get_object_name(src)
expected = "sellerprogramrecord-bizobj"
self.failUnlessEqual(expected, result)
def test_complex_object_name(self):
th = Thrift(None)
src = "/home/user/foobar/blah/wizecommerce.bizobj.search.foobar.sellerprogramrecord.thrift"
result = th.get_object_name(src)
expected = "search-foobar-sellerprogramrecord-bizobj"
self.failUnlessEqual(expected, result)
def test_special_case_object_name(self):
th = Thrift(None)
src = "/home/user/foobar/blah/wizecommerce.services.sellerprogramrecord.thrift"
result = th.get_object_name(src)
expected = "sellerprogramrecord-client"
self.failUnlessEqual(expected, result)
class Client():
"""
For the most part, the purpose fo this class is to define an API that you can extend
to implement your own Client for any language that isn't already supported.
"""
def get_sandbox(self):
return self.sandbox
def set_sandbox(self, value):
self.sandbox = value
def __init__(self, services, thrift_compiler):
self.compiler = thrift_compiler
self.config = Config()
self.sandbox_work = self.config.work_dir
self.status = Status()
self.thrift_helper = Thrift(self.compiler)
self.log = Log(log_file="status.log", logger_name="status").log
self.services = services
def run(self):
self.initialize()
for service in self.services:
self.process_service(service)
self.finalize()
def __build_dependency__(self, business_object):
"""
Recursively build the dependency and return a list of all dependencies found and successfully built.
"""
raise NotImplementedError("Build Dependency needs to be overridden")
##if previous error code has been found, aborting.
def __build_client__(self, service):
"""
This method is called to build the actual client, in our case that includes
all of our services.
"""
raise NotImplementedError("Build Client needs to be overridden")
def deploy_object(self, properties, business_object, postfix=""):
"""
The purpose of this method is to handle the deployment / install.
If set to localMode it will call the appropriate method for handling local installs,
if doing a production deployment it will call the appropriate method
postfix str: a string appended to the artifact name used for releasing snapshot for example.
properties dict: list of properties used for doing deployment.
businessObject str: string containing the name of the thrift file.
"""
##Local mode will only build snapshots, for now.
if self.config.is_local():
return self.__deploy_local_artifact__(properties=properties, thrift_object=business_object, postfix=postfix)
else:
return self.__deploy_production_artifact__(properties=properties, thrift_object=business_object, postfix=postfix)
return 0
def check_version(self, **kwargs):
"""
Returns a boolean checking if the artifact exists on the deployment server.
"""
raise NotImplementedError("Check Version needs to be overridden")
def __deploy_production_artifact__(self, properties, thrift_object, postfix=""):
"""
Implement a method responsible for deploying your artifact to your production server.
"""
raise NotImplementedError("Deploy Production Artifact needs to be overridden")
def __deploy_local_artifact__(self, properties, thrift_object, postfix=""):
"""
Implement a method responsible for performing a local install.
"""
raise NotImplementedError("Deploy Local Artifact needs to be overridden")
def finalize(self):
"""
This method is called as a last step to either clean , publish or whatever needs to be done.
"""
raise NotImplementedError("Finalize needs to be overridden")
def initialize(self):
"""
Optional, used to initialize/construct any environment or file system settings that need
to be setup that are specific to the client.
"""
raise NotImplementedError("Initialize method has not been overridden")
def process_service(self, service):
"""
This method builds the client and all dependencies assuming appropriate
metadata is contained in the thrift file.
"""
os.chdir(self.sandbox_work)
dependencies = self.thrift_helper.read_thrift_dependencies(service)
#Adding the service file as well to the list.
if len(dependencies) == 0:
print "No dependencies for %s" % service
else:
for dependency in dependencies:
self.local_assert(self.__build_dependency__(dependency), "Failed to process dependencies for {service}".format(service=dependency))
self.local_assert(self.__build_client__(service), "Failed to build Client for {service}".format(service=str(service)))
return 0
def local_assert(self, exit_code, message, prefix="ERROR: "):
if exit_code != 0:
self.log(prefix + message)
sys.exit(exit_code)