def test_running_job_with_different_args_runs_it_again(self, method):
"""Job can be run twice with different args."""
job = MockJob()
jobstamp.run(job, 1, jobstamps_cache_output_directory=os.getcwd(), jobstamps_method=method)
jobstamp.run(job, 2, jobstamps_cache_output_directory=os.getcwd(), jobstamps_method=method)
job.assert_has_calls([call(1), call(2)])
def test_dependency_out_of_date_when_dependency_updated(self, method):
"""Dependency is marked out of date when it is updated."""
job = MockJob()
dependency = os.path.join(os.getcwd(), "dependency")
cwd = os.getcwd()
with open(dependency, "w") as dependency_file:
dependency_file.write("Contents")
jobstamp.run(
job, 1, jobstamps_dependencies=[dependency], jobstamps_cache_output_directory=cwd, jobstamps_method=method
)
time.sleep(1)
with open(dependency, "w") as dependency_file:
dependency_file.write("Updated")
ret = jobstamp.out_of_date(
MockJob(),
1,
jobstamps_dependencies=[dependency],
jobstamps_cache_output_directory=cwd,
jobstamps_method=method,
)
self.assertEqual(dependency, ret)
def test_running_job_twice_only_runs_underlying_job_once(self, method):
"""Job is not run twice when there are no deps and already stamped."""
job = MockJob()
jobstamp.run(job, 1, jobstamps_cache_output_directory=os.getcwd(), jobstamps_method=method)
jobstamp.run(job, 1, jobstamps_cache_output_directory=os.getcwd(), jobstamps_method=method)
job.assert_called_once_with(1)
def test_nothing_out_of_date_when_stampfile_does_exist(self, method):
"""Nothing marked out of date when stamp file doesn't exist."""
job = MockJob()
cwd = os.getcwd()
jobstamp.run(job, 1, jobstamps_cache_output_directory=cwd, jobstamps_method=method)
ret = jobstamp.out_of_date(job, 1, jobstamps_cache_output_directory=cwd, jobstamps_method=method)
self.assertEqual(None, ret)
def test_running_job_twice_returns_expected_value(self, method):
"""Job is run again when there is no stamp."""
job = MockJob()
job.return_value = "expected"
jobstamp.run(job, 1, jobstamps_cache_output_directory=os.getcwd(), jobstamps_method=method)
value = jobstamp.run(job, 1, jobstamps_cache_output_directory=os.getcwd())
self.assertEqual(value, job.return_value)
def test_job_runs_again_when_dependency_added(self, method):
"""Job runs again when a new dependency is added."""
job = MockJob()
dependency = os.path.join(os.getcwd(), "dependency")
with open(dependency, "w") as dependency_file:
dependency_file.write("Contents")
jobstamp.run(
job,
1,
jobstamps_dependencies=[dependency],
jobstamps_cache_output_directory=os.getcwd(),
jobstamps_method=method,
)
time.sleep(1)
dependency_new = os.path.join(os.getcwd(), "dependency_new")
with open(dependency_new, "w") as dependency_file:
dependency_file.write("Contents")
jobstamp.run(
job, 1, jobstamps_dependencies=[dependency, dependency_new], jobstamps_cache_output_directory=os.getcwd()
)
job.assert_has_calls([call(1), call(1)])
def test_raise_if_cachedir_exists_as_file(self): # suppress(no-self-use)
"""Raise IOError if specified cache dir exists and is a file."""
cache_entry = os.path.join(os.getcwd(), "cache")
with open(cache_entry, "w") as cache_entry_file:
cache_entry_file.write("Regular file")
with ExpectedException(IOError):
jobstamp.run(lambda: None, jobstamps_cache_output_directory=cache_entry)
def test_job_runs_again_when_output_file_doesnt_exist(self):
"""Job can be run twice when output file doesn't exist."""
job = MockJob()
expected_outputs = os.path.join(os.getcwd(), "expected_output")
jobstamp.run(job, 1, jobstamps_output_files=[expected_outputs], jobstamps_cache_output_directory=os.getcwd())
jobstamp.run(job, 1, jobstamps_output_files=[expected_outputs], jobstamps_cache_output_directory=os.getcwd())
job.assert_has_calls([call(1), call(1)])
def test_running_twice_with_jobstamps_disabled_runs_twice(self, method):
"""Job is run twice when JOBSTAMPS_DISABLED is set."""
os.environ["JOBSTAMPS_DISABLED"] = "1"
self.addCleanup(lambda: os.environ.pop("JOBSTAMPS_DISABLED", None))
job = MockJob()
jobstamp.run(job, 1, jobstamps_cache_output_directory=os.getcwd(), jobstamps_method=method)
jobstamp.run(job, 1, jobstamps_cache_output_directory=os.getcwd(), jobstamps_method=method)
job.assert_has_calls([call(1), call(1)])
def test_running_job_creates_cache_directory(self):
"""Running job creates stamp file directory."""
os.chdir("..")
cache_directory = self._temporary_directory
shutil.rmtree(cache_directory)
job = MockJob()
jobstamp.run(job, 1, jobstamps_cache_output_directory=cache_directory)
self.assertThat(cache_directory, DirExists())
def test_can_create_cache_directory_in_nested_directories(self):
"""Running job creates stamp file directory, even if nested."""
os.chdir("..")
cache_directory = os.path.join(self._temporary_directory, "nested")
shutil.rmtree(self._temporary_directory)
job = MockJob()
jobstamp.run(job, 1, jobstamps_cache_output_directory=cache_directory)
self.assertThat(cache_directory, DirExists())
def test_output_file_out_of_date_when_output_file_doesnt_exist(self):
"""Output file is out of date when output file doesn't exist."""
expected_outputs = os.path.join(os.getcwd(), "expected_output")
cwd = os.getcwd()
job = MockJob()
jobstamp.run(job, 1, jobstamps_output_files=[expected_outputs], jobstamps_cache_output_directory=cwd)
ret = jobstamp.out_of_date(
job, 1, jobstamps_output_files=[expected_outputs], jobstamps_cache_output_directory=cwd
)
self.assertEqual(expected_outputs, ret)
def test_job_runs_once_when_output_file_exists(self):
"""Job runs only once when output file exists."""
job = MockJob()
expected_outputs = os.path.join(os.getcwd(), "expected_output")
jobstamp.run(job, 1, jobstamps_output_files=[expected_outputs], jobstamps_cache_output_directory=os.getcwd())
with open(expected_outputs, "w") as expected_outputs_file:
expected_outputs_file.write("Expected output")
jobstamp.run(job, 1, jobstamps_output_files=[expected_outputs], jobstamps_cache_output_directory=os.getcwd())
job.assert_called_once_with(1)
def test_dependency_out_of_date_when_dependency_doesnt_exist(self, method):
"""Dependency is marked out of date when it does not exist."""
job = MockJob()
dependency = os.path.join(os.getcwd(), "dependency")
cwd = os.getcwd()
jobstamp.run(
job, 1, jobstamps_dependencies=[dependency], jobstamps_cache_output_directory=cwd, jobstamps_method=method
)
ret = jobstamp.out_of_date(
job, 1, jobstamps_dependencies=[dependency], jobstamps_cache_output_directory=cwd, jobstamps_method=method
)
self.assertEqual(dependency, ret)
def main(arguments=None): # suppress(unused-function)
"""Entry point for the spellcheck linter."""
dictionary_path = os.path.abspath("DICTIONARY")
result = _parse_arguments(arguments)
num_errors = 0
for found_filename in result.files:
file_path = os.path.abspath(found_filename)
with io.open(file_path, "r+", encoding="utf-8") as found_file:
jobstamps_dependencies = [file_path]
if os.path.exists(dictionary_path):
jobstamps_dependencies.append(dictionary_path)
if (result.technical_terms and
os.path.exists(result.technical_terms)):
jobstamps_dependencies.append(result.technical_terms)
kwargs = {
"jobstamps_dependencies": jobstamps_dependencies,
"jobstamps_cache_output_directory": result.stamp_file_path,
}
errors = jobstamp.run(spellcheck,
found_file.read(),
result.technical_terms,
result.spellcheck_cache,
**kwargs)
for error in errors:
_report_spelling_error(error, file_path)
num_errors += len(errors)
return num_errors
def test_job_runs_again_when_dependency_deleted(self, method):
"""Job runs again when dependency is later deleted."""
job = MockJob()
dependency = os.path.join(os.getcwd(), "dependency")
with open(dependency, "w") as dependency_file:
dependency_file.write("Contents")
jobstamp.run(
job,
1,
jobstamps_dependencies=[dependency],
jobstamps_cache_output_directory=os.getcwd(),
jobstamps_method=method,
)
os.remove(dependency)
jobstamp.run(job, 1, jobstamps_dependencies=[dependency], jobstamps_cache_output_directory=os.getcwd())
job.assert_has_calls([call(1), call(1)])
def _stamped_deps(stamp_directory, func, dependencies, *args, **kwargs):
"""Run func, assumed to have dependencies as its first argument."""
if not isinstance(dependencies, list):
jobstamps_dependencies = [dependencies]
else:
jobstamps_dependencies = dependencies
kwargs.update(
{"jobstamps_cache_output_directory": stamp_directory, "jobstamps_dependencies": jobstamps_dependencies}
)
return jobstamp.run(func, dependencies, *args, **kwargs)
def test_job_runs_again_when_dependency_doesnt_exist(self, method):
"""Job can be run twice with different args when dep doesn't exist."""
job = MockJob()
dependency = os.path.join(os.getcwd(), "dependency")
jobstamp.run(
job,
1,
jobstamps_dependencies=[dependency],
jobstamps_cache_output_directory=os.getcwd(),
jobstamps_method=method,
)
jobstamp.run(
job,
1,
jobstamps_dependencies=[dependency],
jobstamps_cache_output_directory=os.getcwd(),
jobstamps_method=method,
)
job.assert_has_calls([call(1), call(1)])
def _stamped_deps(stamp_directory, func, dependencies, *args, **kwargs):
"""Run func, assumed to have dependencies as its first argument."""
if not isinstance(dependencies, list):
jobstamps_dependencies = [dependencies]
else:
jobstamps_dependencies = dependencies
kwargs.update({
"jobstamps_cache_output_directory": stamp_directory,
"jobstamps_dependencies": jobstamps_dependencies
})
return jobstamp.run(func, dependencies, *args, **kwargs)
def _run_lint_on_file_stamped(*args):
"""Run linter functions on file_path, stamping in stamp_file_path."""
# We pass an empty dictionary as keyword arguments here to work
# around a bug in frosted, which crashes when no keyword arguments
# are passed
#
# suppress(E204)
stamp_args, stamp_kwargs = _run_lint_on_file_stamped_args(*args,
**{})
return jobstamp.run(_run_lint_on_file_exceptions,
*stamp_args,
**stamp_kwargs)
def test_job_runs_once_only_once_when_dependency_up_to_date(self, method):
"""Job runs only once when stamp is more recent than dependency."""
job = MockJob()
dependency = os.path.join(os.getcwd(), "dependency")
with open(dependency, "w") as dependency_file:
dependency_file.write("Contents")
jobstamp.run(
job,
1,
jobstamps_dependencies=[dependency],
jobstamps_cache_output_directory=os.getcwd(),
jobstamps_method=method,
)
jobstamp.run(
job,
1,
jobstamps_dependencies=[dependency],
jobstamps_cache_output_directory=os.getcwd(),
jobstamps_method=method,
)
job.assert_called_once_with(1)
def test_job_runs_again_when_dependency_not_up_to_date(self, method):
"""Job runs again when dependency is more recent than stamp."""
job = MockJob()
dependency = os.path.join(os.getcwd(), "dependency")
with open(dependency, "w") as dependency_file:
dependency_file.write("Contents")
jobstamp.run(
job,
1,
jobstamps_dependencies=[dependency],
jobstamps_cache_output_directory=os.getcwd(),
jobstamps_method=method,
)
time.sleep(1)
with open(dependency, "w") as dependency_file:
dependency_file.write("Updated")
jobstamp.run(job, 1, jobstamps_dependencies=[dependency], jobstamps_cache_output_directory=os.getcwd())
job.assert_has_calls([call(1), call(1)])
def main(arguments=None):
"""Entry point for the linter."""
result = _parse_arguments(arguments)
num_errors = 0
for found_file_name in result.files:
with open(found_file_name, "r+") as found_file:
file_path = os.path.abspath(found_file_name)
file_contents = found_file.read()
file_lines = file_contents.splitlines(True)
try:
kwargs = OrderedDict()
if result.namespace is not None:
kwargs["namespace"] = result.namespace[0]
if result.indent is not None:
kwargs["indent"] = result.indent[0]
kwargs.update(
_jobstamps_kwargs(file_path, result.stamp_directory))
errors = jobstamp.run(
lint,
file_contents, # suppress(star-args)
_sorted_if_exists(result.whitelist),
_sorted_if_exists(result.blacklist),
**kwargs)
except RuntimeError as err:
msg = "RuntimeError in processing {0} - {1}".format(
file_path, str(err))
raise RuntimeError(msg)
for error in errors:
if not should_ignore(file_lines[error[1].line - 1], error[0]):
_report_lint_error(error, file_path)
if (result.fix_what_you_can
and error[1].replacement is not None):
_apply_replacement(error, found_file, file_lines)
sys.stderr.write(" ... FIXED\n")
break
sys.stderr.write("\n")
num_errors += 1
return num_errors
def main(arguments=None):
"""Entry point for the linter."""
result = _parse_arguments(arguments)
num_errors = 0
for found_file_name in result.files:
with open(found_file_name, "r+") as found_file:
file_path = os.path.abspath(found_file_name)
file_contents = found_file.read()
file_lines = file_contents.splitlines(True)
try:
kwargs = OrderedDict()
if result.namespace is not None:
kwargs["namespace"] = result.namespace[0]
if result.indent is not None:
kwargs["indent"] = result.indent[0]
kwargs.update(_jobstamps_kwargs(file_path,
result.stamp_directory))
errors = jobstamp.run(lint,
file_contents, # suppress(star-args)
_sorted_if_exists(result.whitelist),
_sorted_if_exists(result.blacklist),
**kwargs)
except RuntimeError as err:
msg = "RuntimeError in processing {0} - {1}".format(file_path,
str(err))
raise RuntimeError(msg)
for error in errors:
if not should_ignore(file_lines[error[1].line - 1], error[0]):
_report_lint_error(error, file_path)
if (result.fix_what_you_can and
error[1].replacement is not None):
_apply_replacement(error, found_file, file_lines)
sys.stderr.write(" ... FIXED\n")
break
sys.stderr.write("\n")
num_errors += 1
return num_errors
def main(argv=None): # suppress(unused-function)
"""Entry point for jobstamp command.
This will parse arguments and run the specified command in a subprocess. If
the command has already been run, then the last captured stdout and
stderr of the command will be printed on the command line.
"""
argv = argv or sys.argv
if "--" not in argv:
sys.stderr.write("""Must specify command after '--'.\n""")
return 1
cmd_index = argv.index("--")
args, cmd = (argv[1:cmd_index], argv[cmd_index + 1:])
parser = argparse.ArgumentParser(description="""Cache results from jobs""")
parser.add_argument("--dependencies",
metavar="PATH",
nargs="*",
help="""A list of paths which, if more recent than """
"""the last time this job was invoked, will """
"""cause the job to be re-invoked.""")
parser.add_argument("--output-files",
metavar="PATH",
nargs="*",
help="""A list of expected output paths form this """
"""command, which, if they do not exist, will """
"""cause the job to be re-invoked.""")
parser.add_argument("--stamp-directory",
metavar="DIRECTORY",
type=str,
help="""A directory to store cached results from """
"""this command. If a matching invocation is """
"""used and the files specified in """
"""--dependencies and --output-files are """
"""up-to-date, then the cached stdout, stderr """
"""and return code is used and the command is """
"""not run again.""")
parser.add_argument("--use-hashes",
action="store_true",
help="""Use hash comparison in order to determine """
"""if dependencies have changed since the last """
"""invocation of the job. This method is """
"""slower, but can withstand files being """
"""copied or moved.""")
namespace = parser.parse_args(args)
stamp_directory = namespace.stamp_directory
if namespace.use_hashes:
method = jobstamp.HashMethod
else:
method = jobstamp.MTimeMethod
result = jobstamp.run(_run_cmd,
cmd,
jobstamps_dependencies=namespace.dependencies,
jobstamps_output_files=namespace.output_files,
jobstamps_cache_output_directory=stamp_directory,
jobstamps_method=method)
sys.stdout.write(result["stdout"].decode())
sys.stderr.write(result["stderr"].decode())
return result["code"]
def test_running_job_once_runs_job(self): # suppress(no-self-use)
"""Job is run initially when there is no stamp."""
job = MockJob()
jobstamp.run(job, 1, jobstamps_cache_output_directory=os.getcwd())
job.assert_called_with(1)