def MergeSeriesCollections(
series: typing.Iterator[
me_pb2.SeriesCollection]) -> me_pb2.SeriesCollection:
"""Merge the given series collections into a single SeriesCollection.
Args:
series: The SeriesCollection messages to merge.
Returns:
A SeriesCollection message.
Raises:
ValueError: If there are Series with duplicate names.
"""
series = list(labtypes.flatten(list(f.series) for f in series))
# Create a map from series name to a list of series protos.
names_to_series = collections.defaultdict(list)
[names_to_series[s.name].append(s) for s in series]
# Concatenate each list of series with the same name.
concatenated_series = [
ConcatenateSeries(s) for s in names_to_series.values()
]
return me_pb2.SeriesCollection(
series=sorted(concatenated_series, key=lambda s: s.name))
def Rsync(src: str, dst: str, host_port: int, excludes: typing.List[str],
dry_run: bool, verbose: bool, delete: bool, progress: bool):
"""Private helper method to invoke rsync with appropriate arguments."""
cmd = [
'rsync',
'-ah',
str(src),
str(dst),
'-e',
f'ssh -p {host_port}',
] + labtypes.flatten([['--exclude', p] for p in excludes])
if dry_run:
cmd.append('--dry-run')
if verbose:
cmd.append('--verbose')
if delete:
cmd.append('--delete')
if progress:
cmd.append('--progress')
logging.info(' '.join(cmd))
p = subprocess.Popen(cmd)
p.communicate()
if p.returncode:
raise subprocess.SubprocessError(
f'rsync failed with returncode {p.returncode}')
def ls(root: typing.Union[str, pathlib.Path] = ".",
abspaths=False,
recursive=False):
"""
Return a list of files in directory.
Directory listings are sorted alphabetically. If the named
directory is a file, return it's path.
Examples:
>>> ls("foo")
["a", "b", "c"]
>>> ls("foo/a")
["foo/a"]
>>> ls("foo", abspaths=True)
["/home/test/foo/a", "/home/test/foo/b", "/home/test/foo/c"]
>>> ls("foo", recursive=True)
["a", "b", "b/d", "b/d/e", "c"]
Arguments:
root (str): Path to directory. Can be relative or absolute.
abspaths (bool, optional): Return absolute paths if true.
recursive (bool, optional): Recursively list subdirectories if
true.
Returns:
list of str: A list of paths.
Raises:
OSError: If root directory does not exist.
"""
def _expand_subdirs(file):
if isdir(path(root, file)):
return [file] + [
path(file, x) for x in ls(path(root, file), recursive=True)
]
else:
return [file]
if isfile(root):
# If argument is a file, return path.
return [abspath(root)] if abspaths else [basename(root)]
elif abspaths:
# Get relative names.
relpaths = ls(root, recursive=recursive, abspaths=False)
# Prepend the absolute path to each relative name.
base = abspath(root)
return [path(base, relpath) for relpath in relpaths]
elif recursive:
# Recursively expand subdirectories.
paths = ls(root, abspaths=abspaths, recursive=False)
return labtypes.flatten([_expand_subdirs(file) for file in paths])
else:
# List directory contents.
return list(sorted(os.listdir(root)))
def GetProtos(export_path: pathlib.Path, outcomes: typing.List[str],
max_src_len: int) -> typing.List[TrainingProto]:
paths = sorted(
labtypes.flatten(
[list((export_path / outcome).iterdir()) for outcome in outcomes]))
protos = []
for path in paths:
proto = pbutil.FromFile(path, TrainingProto())
if len(proto.src) > max_src_len:
continue
protos.append(proto)
return protos
def main(argv):
"""Main entry point."""
if len(argv) > 1:
unknown_args = ', '.join(argv[1:])
raise app.UsageError(f'Unknown arguments "{unknown_args}"')
logging.info('Initializing datastore.')
config = pathlib.Path(FLAGS.datastore)
ds = datastore.DataStore.FromFile(config)
output_dir = pathlib.Path(FLAGS.output_directory)
# Make directories to write the classifications to. We use the same shorthand
# classification names as in Table 2 of the paper:
#
# http://chriscummins.cc/pub/2018-issta.pdf
(output_dir / 'bc').mkdir(parents=True, exist_ok=True)
(output_dir / 'bto').mkdir(exist_ok=True)
(output_dir / 'abf').mkdir(exist_ok=True)
(output_dir / 'arc').mkdir(exist_ok=True)
(output_dir / 'awo').mkdir(exist_ok=True)
(output_dir / 'pass').mkdir(exist_ok=True)
result_dirs = [
pathlib.Path(x) for x in FLAGS.input_directories
if pathlib.Path(x).is_dir()
]
results_paths = labtypes.flatten([
pathlib.Path(x) for x in fs.lsfiles(x, recursive=True, abspaths=True)
] for x in result_dirs)
logging.info('Importing %d results into datastore ...', len(results_paths))
with ds.Session(commit=True) as s:
for path in progressbar.ProgressBar()(results_paths):
# Instantiating a result from file has the side effect of adding the
# result object to the datastore's session.
result.Result.FromFile(s, path)
with ds.Session() as s:
testcases = s.query(testcase.Testcase)
logging.info('Difftesting the results of %d testcases ...',
testcases.count())
for t in progressbar.ProgressBar(
max_value=testcases.count())(testcases):
DifftestTestcase(s, t, output_dir)
df = ReadClassificationsToTable(output_dir)
print()
print(
'Table of results. For each testbed, this shows the number of results')
print('of each class, using the same shortand as in Table 2 of the paper:')
print('http://chriscummins.cc/pub/2018-issta.pdf')
print()
print(df.to_string(index=False))
print()
print('Individual classified programs are written to: '
f"'{output_dir}/<class>/<device>/'")
def _CommandLineInvocation(self, args: typing.List[str],
flags: typing.Dict[str, str],
volumes: typing.Dict[typing.Union[str,
pathlib.Path],
str], timeout: int,
entrypoint: str) -> typing.List[str]:
entrypoint_args = ['--entrypoint', entrypoint] if entrypoint else []
volume_args = [
f'-v{src}:{dst}' for src, dst in (volumes or {}).items()
]
flags_args = labtypes.flatten([[f'--{k}', str(v)]
for k, v in (flags or {}).items()])
return _Docker(['run'] + entrypoint_args + volume_args +
[self.image_name] + args + flags_args, timeout)
def test_flatten(self):
self._test([1, 2, 3], labtypes.flatten([[1], [2, 3]]))
'LLVM-bzip2-512K-v0',
'LLVM-bzip2-1M-v0',
'LLVM-queens-8x8-v0',
'LLVM-queens-14x14-v0',
'LLVM-delayed-reward-bzip2-512K-v0',
'LLVM-delayed-reward-bzip2-1M-v0',
'LLVM-delayed-reward-queens-8x8-v0',
'LLVM-delayed-reward-queens-14x14-v0',
]
# A default environment name, registered below.
DEFAULT_ENV_ID = 'LLVM-delayed-reward-queens-14x14-v0'
# The list of opt passes which defines the action space.
DEFAULT_PASS_LIST = list(
sorted(set(labtypes.flatten(llvm_util.GetOptArgs(['-O3'])))))
# Environment generator functions.
def _GetEntryPoint(delayed_reward: bool) -> str:
if delayed_reward:
return ('phd.experimental.compilers.random_opt.implementation:'
'LlvmOptDelayedRewardEnv')
else:
return 'phd.experimental.compilers.random_opt.implementation:LlvmOptEnv'
def _GetBzip2EnvironmentArgs(dataset_size: str, delayed_reward: bool):
return {
'entry_point': _GetEntryPoint(delayed_reward),
def test_flatten():
assert labtypes.flatten([[1], [2, 3]]) == [1, 2, 3]