def SubRespireExternal(self, build_filepath, build_function_name,
additional_deps=[], **kwargs):
if not os.path.isabs(build_filepath):
# Make any relative paths relative to the source respire module that is
# declaring them.
build_filepath = os.path.join(os.path.dirname(self.build_filepath),
build_filepath)
(params_content, futures) = json_utils.EncodeToJSON(kwargs)
# Check to see if the parameters contain any futures. If they do, then
# we insert a layer of indirection to resolve the futures first, so that
# we can take a hash of |kwargs| using the resolved futures, allowing us
# to coalesce registry computations for the same parameters regardless of
# whether those parameters are specified explicitly or via futures.
if futures:
return self._SubRespireExternalPrivate(
inspect.getsourcefile(sys.modules[__name__]), '_ResolveFutures',
{'forward_filepath': build_filepath,
'forward_function': build_function_name,
'forward_params': kwargs,
'forward_additional_deps': additional_deps})
else:
return self._SubRespireExternalPrivate(
build_filepath, build_function_name, kwargs,
additional_deps=additional_deps)
def _MakeSystemCommandForPythonFunctionCall(self, function, params):
AssertIsValidRespireFunction(function)
module_filepath = inspect.getsourcefile(sys.modules[function.__module__])
(params_content, futures) = json_utils.EncodeToJSON(params)
if futures:
raise Exception('Passing futures as parameters to PythonFunctions is '
'not yet supported.')
filepaths = _GetPythonFunctionCallFilePaths(
self.out_dir, module_filepath, function.__name__,
params_content)
registry_helpers._EnsureUniqueFileWithContentsExists(
filepaths.params, params_content)
# Setup a function call that calls our "python function runner" Python
# script, which would in turn deserialize the parameters and call the
# desired function with them.
# The '-B' option is used to avoid making ".pyc" files, which were found
# to result in timing issues resulting in occasionally failing tests.
command = [sys.executable, '-B',
inspect.getsourcefile(python_function_main), module_filepath,
function.__name__, filepaths.params, filepaths.deps]
return (command, filepaths.deps)
def test_EncodeDecodeTupleToJSON(self):
a = Bar(do=1, re='foo', mi=[1, 2, 3], fa={'hello': 'there'})
b = json_utils.EncodeToJSON(a)[0]
c = json_utils.DecodeFromJSON(b)
self.assertTrue(isinstance(c, Bar))
self.assertEqual(c.do, 1)
self.assertEqual(c.re, 'foo')
self.assertEqual(c.mi, [1, 2, 3])
self.assertEqual(c.fa, {'hello': 'there'})
def test_EncodeDecodeObjectToJSON(self):
a = Foo(1)
b = json_utils.EncodeToJSON(a)[0]
c = json_utils.DecodeFromJSON(b)
self.assertTrue(isinstance(c, Foo))
self.assertEqual(c.a, 1)
self.assertEqual(c.b, 'foo')
self.assertEqual(c.c, [1, 2, 3])
self.assertEqual(c.d, {'blah': 'boo'})
def test_EncodeDecodeNestedObjectToJSON(self):
a = Foo(1)
b = Foo(a)
c = json_utils.EncodeToJSON(b)[0]
d = json_utils.DecodeFromJSON(c)
self.assertTrue(isinstance(d, Foo))
self.assertTrue(isinstance(d.a, Foo))
self.assertEqual(d.a.a, 1)
self.assertEqual(d.a.b, 'foo')
self.assertEqual(d.a.c, [1, 2, 3])
self.assertEqual(d.a.d, {'blah': 'boo'})
self.assertEqual(d.b, 'foo')
self.assertEqual(d.c, [1, 2, 3])
self.assertEqual(d.d, {'blah': 'boo'})
def DoFlatten(output_filepath, flattened_output_filepath):
with open(output_filepath, 'r') as f:
output_content = f.read()
# Decode the data, and let the decode function handle the Future
# flattening.
output = json_utils.DecodeFromJSONWithFlattening(output_content,
expand_objects=False)
# Re-encode the flattened data for writing.
(flattened_output_content, futures) = json_utils.EncodeToJSON(output)
if futures:
raise Exception('Expected there to be no futures after flattening.')
with open(flattened_output_filepath, 'w') as f:
f.write(flattened_output_content)
def EnsureGenRegistryInputFilesExist(out_dir,
build_filepath,
build_function_name,
params,
additional_deps=[]):
(params_content, futures) = json_utils.EncodeToJSON(params)
sub_respire_filepaths = GetSubRespireFilepaths(
out_dir,
GetHashedBaseFilename(build_filepath, build_function_name,
params_content))
_EnsureUniqueFileWithContentsExists(sub_respire_filepaths.params_filepath,
params_content)
if not os.path.exists(sub_respire_filepaths.gen_registry_filepath):
gen_registry_contents = _MakeGenRegistryContents(
build_filepath, build_function_name, out_dir,
sub_respire_filepaths, futures, additional_deps)
_EnsureUniqueFileWithContentsExists(
sub_respire_filepaths.gen_registry_filepath, gen_registry_contents)
return sub_respire_filepaths
def SubRespire(build_filepath,
params_file,
out_dir,
function_name,
timestamp_file=None):
with open(params_file, 'r') as f:
params_content = f.read()
registry_files_basename = registry_helpers.GetHashedBaseFilename(
build_filepath, function_name, params_content)
respire_filepaths = registry_helpers.GetSubRespireFilepaths(
out_dir, registry_files_basename)
build_dir = os.path.dirname(build_filepath)
# Make sure that the path has the current respire module's directory in it
# first thing.
sys.path.insert(0, build_dir)
# Change the system current directory to the build file's directory to make
# it so that build files can be more hermetic in assuming the cwd is the
# directory they reside in.
original_cwd = os.getcwd()
os.chdir(build_dir)
# Make sure that all modules have access to the respire standard build
# library.
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'buildlib'))
build_module = respire_python_wrapper_helpers.LoadSourceModule(
'respire_build', build_filepath)
registry = Registry(out_dir, build_filepath, build_module,
respire_filepaths)
params = json_utils.DecodeFromJSONWithFlattening(params_content)
if not hasattr(build_module, function_name):
raise Exception(
'Requested respire function, "%s", not found in build file: %s.' %
(function_name, build_filepath))
function = getattr(build_module, function_name)
if not _IsFunctionParameter('registry', function):
raise Exception(
'You may only pass functions with a "registry" parameter to '
'SubRespire().')
out_params = function(registry=registry, **params)
(out_params_content, out_futures) = json_utils.EncodeToJSON(out_params)
_GenerateSystemCommandForFlattenedOutput(
registry, out_futures, respire_filepaths.output_filepath,
respire_filepaths.flattened_output_filepath)
if timestamp_file:
# Timestamp the timestamp file!
open(timestamp_file, 'w').close()
registry_contents = registry.CompileToString()
respire_python_wrapper_helpers.WriteToFileIfContentsDiffer(
respire_filepaths.registry_filepath, registry_contents)
respire_python_wrapper_helpers.WriteToFileIfContentsDiffer(
respire_filepaths.output_filepath, out_params_content)
deps = (respire_python_wrapper_helpers.
GetCurrentNonBuiltInImportDependencySources() + registry.deps)
deps_file_content = '\n'.join(deps)
respire_python_wrapper_helpers.WriteToFileIfContentsDiffer(
respire_filepaths.deps_filepath, deps_file_content)
os.chdir(original_cwd)
return True