def run(self, ctx):
for env in self.envs0:
ctx.env[env] = '0'
for env in self.envs1:
ctx.env[env] = '1'
file_path = ctx.create_holey_file(16 * t.MiB, 'testfile')
ctx.env['PMEM_LOG_LEVEL'] = '15'
ctx.exec(
'pmem_eADR_functions',
self.test_case,
file_path,
)
regex1 = F"{self.function_name}_mov_{self.isa}_noflush"
regex2 = F"{self.function_name}_movnt_{self.isa}" \
F"_empty{self.perfbarrier}"
regex3 = F"{self.function_name}_mov_{self.isa}_empty"
regex = F"({regex1})|({regex2})|({regex3})"
log_file = self.get_log_file_by_prefix("pmem")
log_content = open(log_file).read()
match = re.search(regex, log_content)
if match is None:
futils.fail(F"Failed to find any occurrence of "
F"{self.function_name}"
F" with eADR in pmem_{self.testnum}.log")
def _run_match(self):
"""Match log files"""
cwd_listdir = [path.join(self.cwd, f) for f in os.listdir(self.cwd)]
suffix = '{}.log.match'.format(self.testnum)
def is_matchfile(f):
"""Match file ends with specific suffix and a char before suffix
is not a digit"""
before_suffix = -len(suffix) - 1
return path.isfile(f) and f.endswith(suffix) and \
not f[before_suffix].isdigit()
match_files = filter(is_matchfile, cwd_listdir)
prefix = 'perl ' if sys.platform == 'win32' else ''
match_cmd = prefix + path.join(futils.ROOTDIR, 'match')
for mf in match_files:
cmd = '{} {}'.format(match_cmd, mf)
proc = sp.run(cmd.split(), stdout=sp.PIPE, cwd=self.cwd,
stderr=sp.STDOUT, universal_newlines=True)
if proc.returncode != 0:
futils.fail(proc.stdout, exit_code=proc.returncode)
else:
self.msg.print_verbose(proc.stdout)
def run(self, ctx):
file_path = ctx.create_holey_file(16 * t.MiB, 'testfile')
ctx.env['PMEM2_LOG_LEVEL'] = '15'
ctx.exec('pmem2_future', self.test_case, file_path, self.size)
log_file = self.get_log_file_by_prefix("pmem2")
log_content = open(log_file).read()
# We do not do any checks if the granularity is byte because
# in the case vdm instances do not have to call
# anything in order to persist the memory.
if isinstance(ctx.granularity, g.CacheLine):
regex = "((pmem2_drain)|(memory_barrier))"
matches = re.findall(regex, log_content)
expected = ["pmem2_drain", "memory_barrier"]
# We expect matches of:
# [pmem2_drain, memory_barrier] because calling pmem2_memcpy_async
# should ensure persistence by calling pmem2_drain and
# memory_barrier after that.
if check_match(matches, expected) is not True:
futils.fail(F"Failed to find exact match to "
F"pmem2_drain+memory_barrier call!"
F"Got {matches} instead.")
elif isinstance(ctx.granularity, g.Page):
regex = "pmem2_log_flush"
matches = re.findall(regex, log_content)
# We expect exactly one call of pmem2_log_flush which should
# be called by pmem2_memcpy_async
if len(matches) != 1:
futils.fail(F"Failed to find exactly one "
F"pmem2_log_flush call! "
F"Got {len(matches)} instead.")
def _run_match(self):
"""Match log files"""
cwd_listdir = [path.join(self.cwd, f) for f in os.listdir(self.cwd)]
suffix = '{}.log.match'.format(self.testnum)
def is_matchfile(f):
"""Match file ends with specific suffix and a char before suffix
is not a digit"""
before_suffix = -len(suffix) - 1
return path.isfile(f) and f.endswith(suffix) and \
not f[before_suffix].isdigit()
match_files = filter(is_matchfile, cwd_listdir)
prefix = 'perl ' if sys.platform == 'win32' else ''
match_cmd = prefix + path.join(futils.ROOTDIR, 'match')
for mf in match_files:
cmd = '{} {}'.format(match_cmd, mf)
proc = sp.run(cmd.split(),
stdout=sp.PIPE,
cwd=self.cwd,
stderr=sp.STDOUT,
universal_newlines=True)
if proc.returncode != 0:
futils.fail(proc.stdout, exit_code=proc.returncode)
else:
self.msg.print_verbose(proc.stdout)
def get_size(self, path):
"""
Returns size of the file or dax device.
Value "2**64 - 1" is checked because pmemdetect in case of error prints it.
"""
proc = tools.pmemdetect(self, '-z', path)
if int(proc.stdout) != 2**64 - 1:
return int(proc.stdout)
futils.fail('Could not get size of the file, it is inaccessible or does not exist')
def get_size(self, path):
"""
Returns size of the file or dax device.
Value "2**64 - 1" is checked because pmemdetect in case of error prints it.
"""
proc = tools.pmemdetect(self, '-z', path)
if int(proc.stdout) != 2**64 - 1:
return int(proc.stdout)
futils.fail('Could not get size of the file, it is inaccessible or does not exist')
def set_parts(self, *parts):
"""
Adds a main pool to the poolset file.
The function accepts a list of Parts of any length.
Should not be called more than once.
"""
if not self.parts:
self.parts = list(parts)
else:
futils.fail('This function should not be called more than once.')
def supports_map_sync(self, path):
"""Checks if MAP_SYNC is supported on a filesystem from given path"""
proc = tools.pmemdetect(self, '-s', path)
if proc.returncode == tools.PMEMDETECT_ERROR:
futils.fail(proc.stdout)
if proc.returncode == tools.PMEMDETECT_TRUE:
return True
if proc.returncode == tools.PMEMDETECT_FALSE:
return False
futils.fail('Unknown value {} returned by pmemdetect'.format(proc.returncode))
def is_devdax(self, path):
"""Checks if given path points to device dax"""
proc = tools.pmemdetect(self, '-d', path)
if proc.returncode == tools.PMEMDETECT_ERROR:
futils.fail(proc.stdout)
if proc.returncode == tools.PMEMDETECT_TRUE:
return True
if proc.returncode == tools.PMEMDETECT_FALSE:
return False
futils.fail('Unknown value {} returned by pmemdetect'.format(proc.returncode))
def supports_map_sync(self, path):
"""Checks if MAP_SYNC is supported on a filesystem from given path"""
proc = tools.pmemdetect(self, '-s', path)
if proc.returncode == tools.PMEMDETECT_ERROR:
futils.fail(proc.stdout)
if proc.returncode == tools.PMEMDETECT_TRUE:
return True
if proc.returncode == tools.PMEMDETECT_FALSE:
return False
futils.fail('Unknown value {} returned by pmemdetect'.format(proc.returncode))
def is_devdax(self, path):
"""Checks if given path points to device dax"""
proc = tools.pmemdetect(self, '-d', path)
if proc.returncode == tools.PMEMDETECT_ERROR:
futils.fail(proc.stdout)
if proc.returncode == tools.PMEMDETECT_TRUE:
return True
if proc.returncode == tools.PMEMDETECT_FALSE:
return False
futils.fail('Unknown value {} returned by pmemdetect'.format(proc.returncode))
def set_parts(self, *parts):
"""
Adds a main pool to the poolset file.
The function accepts a list of Parts of any length.
Should not be called more than once.
"""
if not self.parts:
self.parts = list(parts)
else:
futils.fail('This function should not be called more than once.')
def exec(self, cmd, *args, expected_exitcode=0, stderr_file=None):
"""Execute binary in current test context"""
tmp = self._env.copy()
futils.add_env_common(tmp, os.environ.copy())
# change cmd into list for supbrocess type compliance
cmd = [
cmd,
]
if sys.platform == 'win32':
cmd[0] = os.path.join(self.build.exedir, cmd[0]) + '.exe'
else:
cmd[0] = os.path.join(self.cwd, cmd[0]) + \
self.build.exesuffix
if self.valgrind:
cmd = self.valgrind.cmd + cmd
# cast all provided args to strings (required by subprocess run())
# so that exec() can accept args of any printable type
cmd.extend([str(a) for a in args])
if self.conf.tracer:
cmd = shlex.split(self.conf.tracer) + cmd
# process stdout and stderr are not redirected - this lets running
# tracer command in an interactive session
proc = sp.run(cmd, env=tmp, cwd=self.cwd)
else:
if stderr_file:
f = open(os.path.join(self.cwd, stderr_file), 'w')
# let's create a dictionary of arguments to the run func
run_kwargs = {
'env': tmp,
'cwd': self.cwd,
'timeout': self.conf.timeout,
'stdout': sp.PIPE,
'universal_newlines': True,
'stderr': sp.STDOUT if stderr_file is None else f
}
proc = sp.run(cmd, **run_kwargs)
if stderr_file:
f.close()
if expected_exitcode is not None and \
proc.returncode != expected_exitcode:
futils.fail(proc.stdout, exit_code=proc.returncode)
self.msg.print_verbose(proc.stdout)
def exec(self, cmd, *args, expected_exit=0):
"""Execute binary in current test context"""
env = {**self.env, **os.environ.copy(), **self.test.utenv}
# change cmd into list for supbrocess type compliance
cmd = [
cmd,
]
if sys.platform == 'win32':
env['PATH'] = self.build.libdir + os.pathsep +\
envconfig['GLOBAL_LIB_PATH'] + os.pathsep +\
env.get('PATH', '')
cmd[0] = os.path.join(self.build.exedir, cmd[0]) + '.exe'
else:
if self.test.ld_preload:
env['LD_PRELOAD'] = env.get('LD_PRELOAD', '') + os.pathsep +\
self.test.ld_preload
self.valgrind.handle_ld_preload(self.test.ld_preload)
env['LD_LIBRARY_PATH'] = self.build.libdir + os.pathsep +\
envconfig['GLOBAL_LIB_PATH'] + os.pathsep +\
env.get('LD_LIBRARY_PATH', '')
cmd[0] = os.path.join(self.test.cwd, cmd[0]) + self.build.exesuffix
if self.valgrind:
cmd = self.valgrind.cmd + cmd
cmd = cmd + list(args)
if self.conf.tracer:
cmd = shlex.split(self.conf.tracer) + cmd
# process stdout and stderr are not redirected - this lets running
# tracer command in interactive session
proc = sp.run(cmd, env=env, cwd=self.test.cwd)
else:
proc = sp.run(cmd,
env=env,
cwd=self.test.cwd,
timeout=self.conf.timeout,
stdout=sp.PIPE,
stderr=sp.STDOUT,
universal_newlines=True)
if proc.returncode != expected_exit:
futils.fail(proc.stdout, exit_code=proc.returncode)
if sys.platform != 'win32' and expected_exit == 0 \
and not self.valgrind.validate_log():
futils.fail(proc.stdout)
self.msg.print_verbose(proc.stdout)
def _check_pools_size(self):
""""
Validate if pool and replicas have more than 8MiB (minimum pool size).
This function does not validate remote replicas sizes.
"""
size = 0
for part in self.parts:
size += part.size
if size < POOL_MIN_SIZE:
futils.fail('The pool has to have at least 8 MiB')
for replica in self.replicas:
size = 0
for part in replica:
size += part.size
if size < POOL_MIN_SIZE:
futils.fail('The pool has to have at least 8 MiB')
def _check_pools_size(self):
""""
Validate if pool and replicas have more than 8MiB (minimum pool size).
This function does not validate remote replicas sizes.
"""
size = 0
for part in self.parts:
size += part.size
if size < POOL_MIN_SIZE:
futils.fail('The pool has to have at least 8 MiB')
for replica in self.replicas:
size = 0
for part in replica:
size += part.size
if size < POOL_MIN_SIZE:
futils.fail('The pool has to have at least 8 MiB')
def exec(self, cmd, *args, expected_exitcode=0):
"""Execute binary in current test context"""
tmp = self._env.copy()
futils.add_env_common(tmp, os.environ.copy())
# change cmd into list for supbrocess type compliance
cmd = [
cmd,
]
if sys.platform == 'win32':
cmd[0] = os.path.join(self.build.exedir, cmd[0]) + '.exe'
else:
cmd[0] = os.path.join(self.cwd, cmd[0]) + \
self.build.exesuffix
if self.valgrind:
cmd = self.valgrind.cmd + cmd
# cast all provided args to strings (required by subprocess run())
# so that exec() can accept args of any printable type
cmd.extend([str(a) for a in args])
if self.conf.tracer:
cmd = shlex.split(self.conf.tracer) + cmd
# process stdout and stderr are not redirected - this lets running
# tracer command in an interactive session
proc = sp.run(cmd, env=tmp, cwd=self.cwd)
else:
proc = sp.run(cmd,
env=tmp,
cwd=self.cwd,
timeout=self.conf.timeout,
stdout=sp.PIPE,
stderr=sp.STDOUT,
universal_newlines=True)
if expected_exitcode is not None and \
proc.returncode != expected_exitcode:
futils.fail(proc.stdout, exit_code=proc.returncode)
self.msg.print_verbose(proc.stdout)
def create(self):
"""
Create a poolset file with basic space validation. If CREATE arguments
ware passed to parts, create parts files as well.
"""
self._check_pools_size()
required_size = self._get_required_size()
free_space = ctx.get_free_space()
if required_size > free_space:
futils.fail(
'Not enough space available to create parts files. There is '
'{}, and poolset requires {}'.format(free_space,
required_size))
for part in self.parts:
part._process(self.ctx)
for replica in self.replicas:
for part in replica:
part._process(self.ctx)
self.options = list(set(self.options))
poolsetpath = os.path.join(self.ctx.testdir, self.path)
with open(poolsetpath, 'w') as poolset:
print('PMEMPOOLSET', end='\n\n', file=poolset)
for option in self.options:
print('OPTION', option, end='\n\n', file=poolset)
for part in self.parts:
print(part, file=poolset)
print(file=poolset)
for replica in self.replicas:
print("REPLICA", file=poolset)
for part in replica:
print(part, file=poolset)
print(file=poolset)
for remote in self.remote:
print("REPLICA", remote[1], remote[0], file=poolset)
def exec(self, cmd, *args, expected_exit=0):
"""Execute binary in current test context"""
cmd_args = ' '.join(args) if args else ''
env = {**self.env, **os.environ.copy(), **self.test.utenv}
if sys.platform == 'win32':
env['PATH'] = self.build.libdir + os.pathsep +\
envconfig['GLOBAL_LIB_PATH'] + os.pathsep +\
env.get('PATH', '')
cmd = os.path.join(self.build.exedir, cmd) + '.exe'
else:
if self.test.ld_preload:
env['LD_PRELOAD'] = env.get('LD_PRELOAD', '') + os.pathsep +\
self.test.ld_preload
self.valgrind.handle_ld_preload(self.test.ld_preload)
env['LD_LIBRARY_PATH'] = self.build.libdir + os.pathsep +\
envconfig['GLOBAL_LIB_PATH'] + os.pathsep +\
env.get('LD_LIBRARY_PATH', '')
cmd = os.path.join(self.test.cwd, cmd) + self.build.exesuffix
cmd = '{} {}'.format(self.valgrind.cmd, cmd)
cmd = '{} {}'.format(cmd, cmd_args)
proc = sp.run(cmd,
env=env,
cwd=self.test.cwd,
shell=True,
timeout=self.conf.timeout,
stdout=sp.PIPE,
stderr=sp.STDOUT,
universal_newlines=True)
if sys.platform != 'win32' and expected_exit == 0 \
and not self.valgrind.validate_log():
self.test.fail(proc.stdout)
if proc.returncode != expected_exit:
futils.fail(proc.stdout, exit_code=proc.returncode)
else:
self.msg.print_verbose(proc.stdout)
def create(self):
"""
Create a poolset file with basic space validation. If CREATE arguments
ware passed to parts, create parts files as well.
"""
self._check_pools_size()
required_size = self._get_required_size()
free_space = ctx.get_free_space()
if required_size > free_space:
futils.fail('Not enough space available to create parts files. There is '
'{}, and poolset requires {}'.format(free_space,
required_size))
for part in self.parts:
part._process(self.ctx)
for replica in self.replicas:
for part in replica:
part._process(self.ctx)
self.options = list(set(self.options))
poolsetpath = os.path.join(self.ctx.testdir, self.path)
with open(poolsetpath, 'w') as poolset:
print('PMEMPOOLSET', end='\n\n', file=poolset)
for option in self.options:
print('OPTION', option, end='\n\n', file=poolset)
for part in self.parts:
print(part, file=poolset)
print(file=poolset)
for replica in self.replicas:
print("REPLICA", file=poolset)
for part in replica:
print(part, file=poolset)
print(file=poolset)
for remote in self.remote:
print("REPLICA", remote[1], remote[0], file=poolset)
def exec(self, cmd, *args, expected_exit=0):
"""Execute binary in current test context"""
cmd_args = ' '.join(args) if args else ''
env = {**self.env, **os.environ.copy(), **self.test.utenv}
if sys.platform == 'win32':
env['PATH'] = self.build.libdir + os.pathsep +\
envconfig['GLOBAL_LIB_PATH'] + os.pathsep +\
env.get('PATH', '')
cmd = os.path.join(self.build.exedir, cmd) + '.exe'
else:
if self.test.ld_preload:
env['LD_PRELOAD'] = env.get('LD_PRELOAD', '') + os.pathsep +\
self.test.ld_preload
self.valgrind.handle_ld_preload(self.test.ld_preload)
env['LD_LIBRARY_PATH'] = self.build.libdir + os.pathsep +\
envconfig['GLOBAL_LIB_PATH'] + os.pathsep +\
env.get('LD_LIBRARY_PATH', '')
cmd = os.path.join(self.test.cwd, cmd) + self.build.exesuffix
cmd = '{} {}'.format(self.valgrind.cmd, cmd)
cmd = '{} {}'.format(cmd, cmd_args)
proc = sp.run(cmd, env=env, cwd=self.test.cwd, shell=True,
timeout=self.conf.timeout, stdout=sp.PIPE,
stderr=sp.STDOUT, universal_newlines=True)
if sys.platform != 'win32' and expected_exit == 0 \
and not self.valgrind.validate_log():
self.test.fail(proc.stdout)
if proc.returncode != expected_exit:
futils.fail(proc.stdout, exit_code=proc.returncode)
else:
self.msg.print_verbose(proc.stdout)
def __init__(self):
if sys.platform == 'win32':
futils.fail('ndctl is not available on Windows')
self.version = self._get_ndctl_version()
self.ndctl_list_output = self._cmd_out_to_json('list', '-RBNDMv')
def __init__(self, path, size):
if size < PART_MIN_SIZE:
futils.fail('The part should have at least 2 MiB')
self.size = size
self.path = path
def exec(self, cmd, *args, expected_exitcode=0, stderr_file=None,
stdout_file=None):
"""
Execute binary in the current test context as a separate process.
Execution takes place in test cwd and uses environment variables
stored in Context 'env' attribute. Timeout for the execution
is set based on the execution configuration.
Args:
cmd (str): command to be executed
*args: Variable length command arguments list
expected_exitcode (int): if process exit code differs from
expected, Fail is thrown. Defaults to 0. Ignored
if set to None.
stderr_file (str): path to file in which stderr output is
stored. Stored in a string if None. Defaults to None.
stdout_file (str): path to file in which stdout output is
stored. Stored in a string if None. Defaults to None.
If neither stderr_file nor stdout_file are set, both outputs
are merged into single stdout output and stored in a string.
"""
tmp = self._env.copy()
futils.add_env_common(tmp, os.environ.copy())
# change cmd into list for supbrocess type compliance
cmd = [cmd, ]
if sys.platform == 'win32':
cmd[0] = os.path.join(self.build.exedir, cmd[0]) + '.exe'
else:
cmd[0] = os.path.join(self.cwd, cmd[0]) + \
self.build.exesuffix
if self.valgrind:
cmd = self.valgrind.cmd + cmd
# cast all provided args to strings (required by subprocess run())
# so that exec() can accept args of any printable type
cmd.extend([str(a) for a in args])
if self.conf.tracer:
cmd = shlex.split(self.conf.tracer) + cmd
# process stdout and stderr are not redirected - this lets running
# tracer command in an interactive session
proc = sp.run(cmd, env=tmp, cwd=self.cwd)
else:
if stderr_file:
f = open(os.path.join(self.cwd, stderr_file), 'w')
# let's create a dictionary of arguments to the run func
run_kwargs = {
'env': tmp,
'cwd': self.cwd,
'timeout': self.conf.timeout,
'stdout': sp.PIPE,
'universal_newlines': True,
'stderr': sp.STDOUT if stderr_file is None else f}
proc = sp.run(cmd, **run_kwargs)
if stderr_file:
f.close()
if expected_exitcode is not None and \
proc.returncode != expected_exitcode:
futils.fail(proc.stdout, exit_code=proc.returncode)
if stdout_file is not None:
with open(os.path.join(self.cwd, stdout_file), 'w') as f:
f.write(proc.stdout)
self.msg.print_verbose(proc.stdout)
def __init__(self, ctx, path):
if not ctx.is_devdax(path):
futils.fail('Part with path "{}" does not point to dax device'
''.format(path))
_Part.__init__(self, path, ctx.get_size(path))
def __init__(self):
if sys.platform != 'win32':
futils.fail('Ipmctl tool class is currently implemented only'
' for Windows - for Linux use ndctl instead')
def __init__(self, ctx, path):
if not ctx.is_devdax(path):
futils.fail('Part with path "{}" does not point to dax device'
''.format(path))
_Part.__init__(self, path, ctx.get_size(path))
def __init__(self, path, size):
if size < PART_MIN_SIZE:
futils.fail('The part should have at least 2 MiB')
self.size = size
self.path = path