def get_all_sampler_datasets():
datasets = []
sampledirs = []
for versioncache in fs.ls(fs.path("~/.cache/clgen"), abspaths=True):
samplerdir = fs.path(versioncache, "sampler")
if fs.isdir(samplerdir):
sampledirs += fs.ls(samplerdir, abspaths=True)
for samplerdir in sampledirs:
inpath = fs.path(samplerdir, "kernels.db")
if fs.isfile(inpath):
datasets.append(inpath)
return datasets
def import_from_dir(self, indir: Path) -> None:
""" import program sources from a directory """
with Session() as s:
num_progs = self.num_programs(s)
# Print a preamble message:
paths = fs.ls(indir, abspaths=True)
num_to_import = len(paths)
print(f"{Colors.BOLD}{num_to_import}{Colors.END} programs are "
"to be imported.")
bar_max = num_progs + num_to_import
bar = progressbar.ProgressBar(initial_value=num_progs,
max_value=bar_max,
redirect_stdout=True)
# The actual import loop:
buf = []
for i, path in enumerate(paths):
buf.append(self.import_from_file(s, path))
# Update progress bar
num_progs += 1
bar.update(num_progs)
if len(buf) >= dsmith.DB_BUF_SIZE:
save_proxies(s, buf)
num_progs = self.num_programs(s)
buf = []
save_proxies(s, buf)
print(f"All done! Imported {Colors.BOLD}{num_to_import}{Colors.END} "
f"programs. You now have {Colors.BOLD}{num_progs}{Colors.END} "
"{self} programs in the database")
def __len__(self):
"""
Get the number of entries in the cache.
Returns:
int: Number of entries in the cache.
"""
return len(list(fs.ls(self.path)))
def __iter__(self):
"""
Iterate over all cached files.
Returns:
iterable: Paths in cache.
"""
for path in fs.ls(self.path, abspaths=True):
yield path
def import_from_dir(self, indir: Path) -> None:
""" import program sources from a directory """
with Session() as s:
start_num_progs = self.num_programs(s)
def _save(proxies):
# Create records from proxies:
programs = [proxy.to_record(s) for proxy in proxies]
logging.warning(getattr(type(programs[0]), "sha1"))
import sys
sys.exit(0)
# Filter duplicates in the set of new records:
programs = dict((program.sha1, program) for program in programs).values()
# Fetch a list of dupe keys already in the database:
sha1s = [program.sha1 for program in programs]
dupes = set(x[0] for x in s.query(Program.sha1).filter(Program.sha1.in_(sha1s)))
# Filter the list of records to import, excluding dupes:
uniq = [program for program in programs if program.sha1 not in dupes]
# Import those suckas:
s.add_all(uniq)
s.commit()
nprog, nuniq = len(programs), len(uniq)
logging.info(f"imported {nuniq} of {nprog} unique programs")
num_progs = self.num_programs(s)
# Print a preamble message:
paths = fs.ls(indir, abspaths=True)
num_to_import = humanize.intcomma(len(paths))
print(f"{Colors.BOLD}{num_to_import}{Colors.END} files are "
"to be imported.")
bar = progressbar.ProgressBar(redirect_stdout=True)
# The actual import loop:
buf = []
for i, path in enumerate(bar(paths)):
buf.append(self.import_from_file(s, path))
if len(buf) >= dsmith.DB_BUF_SIZE:
save_proxies_uniq_on(s, buf, "sha1")
buf = []
save_proxies_uniq_on(s, buf, "sha1")
num_imported = humanize.intcomma(self.num_programs(s) - start_num_progs)
num_progs = humanize.intcomma(self.num_programs(s))
print(f"All done! Imported {Colors.BOLD}{num_imported}{Colors.END} "
f"new {self} programs. You now have "
f"{Colors.BOLD}{num_progs}{Colors.END} {self} programs in the "
"database")
def ls(self, **kwargs):
"""
List files in cache.
Arguments:
**kwargs: Keyword options to pass to fs.ls().
Returns:
iterable: List of files.
"""
return fs.ls(self.path, **kwargs)
def test_ls_abspaths():
fs.cp("lib/labm8/data/test/testdir", "/tmp/testdir")
assert fs.ls("/tmp/testdir", abspaths=True) == [
"/tmp/testdir/a",
"/tmp/testdir/b",
"/tmp/testdir/c",
"/tmp/testdir/d",
]
assert fs.ls("/tmp/testdir", recursive=True, abspaths=True) == [
"/tmp/testdir/a",
"/tmp/testdir/b",
"/tmp/testdir/c",
"/tmp/testdir/c/e",
"/tmp/testdir/c/f",
"/tmp/testdir/c/f/f",
"/tmp/testdir/c/f/f/i",
"/tmp/testdir/c/f/h",
"/tmp/testdir/c/g",
"/tmp/testdir/d",
]
fs.rm("/tmp/testdir")
def main(argv):
if len(argv) > 1:
unknown_args = ', '.join(argv[1:])
raise app.UsageError(f"Unknown arguments {unknown_args}")
logging.info('Preparing OpenCL testbed.')
config = harness_pb2.CldriveHarness()
config.opencl_env.extend([env.OclgrindOpenCLEnvironment().name])
config.opencl_opt.extend([FLAGS.opencl_opt])
harness = cldrive.CldriveHarness(config)
assert len(harness.testbeds) >= 1
input_directories = FLAGS.input_directories
logging.info('Reading testcases from: %s', ' '.join(input_directories))
output_directory = pathlib.Path(FLAGS.output_directory)
logging.info('Writing results to %s', output_directory)
output_directory.mkdir(parents=True, exist_ok=True)
# Load testcases.
testcase_dirs = [
pathlib.Path(x) for x in input_directories if
pathlib.Path(x).is_dir()]
if not testcase_dirs:
raise app.UsageError('No --input_directories found.')
testcase_paths = labtypes.flatten(
[[pathlib.Path(y) for y in fs.ls(x, abspaths=True)]
for x in testcase_dirs])
testcases = [
pbutil.FromFile(path, deepsmith_pb2.Testcase()) for path in testcase_paths]
logging.info('Read %d testcases.', len(testcases))
if not len(testcases):
raise app.UsageError("No testcases found: '%s'",
' '.join(input_directories))
# Execute testcases.
req = harness_pb2.RunTestcasesRequest()
req.testbed.CopyFrom(harness.testbeds[0])
req.testcases.extend(testcases)
res = harness.RunTestcases(req, None)
# Write results to file.
for testcase, result in zip(testcases, res.results):
result_id = crypto.md5_str(str(testcase))
pbutil.ToFile(result, output_directory / f'{result_id}.pbtxt')
logging.info('Executed %d testcases and wrote results to %s',
len(res.results), output_directory)
execution_times = [
result.profiling_events[0].duration_ms for result in res.results]
logging.info('Average time to evaluate testcase: %.2f ms',
sum(execution_times) / len(execution_times))
def test_ls_recursive():
assert fs.ls("lib/labm8/data/test/testdir", recursive=True) == [
"a",
"b",
"c",
"c/e",
"c/f",
"c/f/f",
"c/f/f/i",
"c/f/h",
"c/g",
"d",
]
def _ExportProtos() -> None:
proto_dir = pathlib.Path(FLAGS.proto_dir)
assert proto_dir
credentials = _GetMySqlCredentials()
cnx = MySQLdb.connect(database='dsmith_04_opencl',
host='cc1',
user=credentials[0],
password=credentials[1])
cursor = cnx.cursor()
(proto_dir / 'testcases').mkdir(parents=True, exist_ok=True)
(proto_dir / 'results').mkdir(parents=True, exist_ok=True)
for program_id in FLAGS.program_ids:
logging.info("Exporting OpenCL program %s", program_id)
_ExportOpenCLResults(cursor, program_id, proto_dir)
cursor.close()
cnx.close()
logging.info('Exported %d testcases and %d results',
len(fs.ls(proto_dir / 'testcases')),
len(fs.ls(proto_dir / 'results')))
def test_ls_single_file():
assert ["a"] == fs.ls("lib/labm8/data/test/testdir/a")
def test_ls_bad_path():
with pytest.raises(OSError):
fs.ls("/not/a/real/path/bro")
def test_ls_empty_dir():
fs.mkdir("/tmp/labm8.empty")
assert not fs.ls("/tmp/labm8.empty")
fs.rm("/tmp/labm8.empty")
def test_ls():
assert ["a", "b", "c", "d"] == fs.ls("lib/labm8/data/test/testdir")