def load(cls):
path = npf.find_local(sys.modules["npf.npf"].options.experimental_design)
assert path is not None
with open(path) as fd:
csvreader = csv.reader(fd)
data = [i for i in csvreader]
ExperimentalDesign.matrix = np.array([[float(j) for j in i] for i in data])
def __init__(self, name, executor, tags):
self.executor = executor
self.name = name
self._nics = []
self.tags = []
self.nfs = True
self.addr = 'localhost'
self.port = 22
self.arch = ''
self.active_nics = range(32)
self.multi = None
self.mode = "bash"
# Always fill 32 random nics address that will be overwriten by config eventually
self._gen_random_nics()
clusterFileName = 'cluster/' + name + (
'.node' if not name.endswith(".node") else "")
try:
clusterFilePath = npf.find_local(clusterFileName, critical=False)
f = open(clusterFilePath, 'r')
for i, line in enumerate(f):
line = line.strip()
if not line or line.startswith("#") or line.startswith("//"):
continue
match = re.match(
r'((?P<tag>[a-zA-Z]+[a-zA-Z0-9]*):)?(?P<nic_idx>[0-9]+):(?P<type>'
+ NIC.TYPES + ')=(?P<val>[a-z0-9:_.]+)', line,
re.IGNORECASE)
if match:
if match.group('tag') and not match.group('tag') in tags:
continue
self._nics[int(match.group('nic_idx'))][match.group(
'type')] = match.group('val')
continue
match = re.match(
r'(?P<var>' + Variable.ALLOWED_NODE_VARS + ')=(?P<val>.*)',
line, re.IGNORECASE)
if match:
if match.group('var') == 'nfs':
self.nfs = get_bool(match.group('val'))
setattr(executor, match.group('var'), match.group('val'))
continue
raise Exception("%s:%d : Unknown node config line %s" %
(clusterFile, i, line))
except FileNotFoundError as e:
print(
"%s could not be found, we will connect to %s with SSH using default parameters"
% (clusterFilePath, name))
self._find_nics()
def __init__(self, repo, options):
self.name = None
self._current_build = None
version = repo.split('@')
if len(version) > 1:
self.version=version[1]
else:
self.version=None
overwrite_name = version[0].split(':')
add_tags = overwrite_name[0].split('+')
overwrite_branch = add_tags[0].split('/')
self.reponame = overwrite_branch[0]
self.tags = []
self.options = options
self.branch = 'master'
self.make = 'make -j12'
self.clean = 'make clean'
self.bin_folder = 'bin'
self.env = OrderedDict()
self.bin_name = self.reponame # Wild guess that may work some times...
self.build_info = None
self.configure = ''
self._last_100 = None
if self.reponame == 'None':
self.url = None
else:
repo_path = npf.find_local('repo/' + self.reponame + '.repo')
f = open(repo_path, 'r')
for line in f:
line = line.strip()
line = re.sub(r'(^|[ ])//.*$', '', line)
if line.startswith("#"):
continue
if not line:
continue
s = line.split('=', 1)
var = s[0].strip()
var = var.split(':',1)
if len(var) > 1:
have_all=True
for v in var[0].split(','):
if not v in options.tags:
have_all = False
break
if not have_all:
continue
self.reponame += '-' + var[0]
var = var[1]
else:
var = var[0]
val = s[1].strip()
append = False
if var.endswith('+'):
var = var[:-1]
append = True
if is_numeric(val) and var != 'branch':
val = float(val)
if val.is_integer():
val = int(val)
if not var in repo_variables:
raise Exception("Unknown variable %s" % var)
elif var == "parent":
parent = Repository(val, options)
for attr in repo_variables:
if not hasattr(parent, attr):
continue
pval = getattr(parent, attr)
if attr == "method":
for m, c in repo_methods.items():
if c == type(pval):
method = m
break
else:
setattr(self, attr, pval)
elif var == "method":
val = val.lower()
if not val in repo_methods:
raise Exception("Unknown method %s" % val)
val = repo_methods[val]
elif var == "tags":
if append:
self.tags += val.split(',')
else:
self.tags = val.split(',')
continue
elif var == "env":
ed = VariableFactory.build(var, val).vdict
if append:
self.env += ed
else:
self.env = ed
continue
if append:
setattr(self, var, getattr(self, var) + " " + val)
else:
setattr(self, var, val)
self.method = self.method(self) # Instanciate the method
self.overriden_variables = {}
if len(add_tags) > 1:
for spec in add_tags[1].split(','):
sp = spec.split('=')
if len(sp) == 1:
self.tags.append(spec)
else:
self.overriden_variables[sp[0]] = sp[1]
self._id = self.reponame + "-" + add_tags[1]
else:
self._id = self.reponame
if len(overwrite_branch) > 1:
self.branch = overwrite_branch[1]
if len(overwrite_name) > 1:
self.name = overwrite_name[1]
self._build_path = os.path.dirname(
(options.build_folder if not options.build_folder is None else npf.npf_root()+'/build/') + self.reponame + '/')
def __init__(self, testie_path, options, tags=None, role=None):
if os.path.exists(npf.find_local(testie_path)):
testie_path = npf.find_local(testie_path)
else:
if not os.path.exists(npf.find_local(testie_path + '.testie')):
raise Exception("Could not find testie %s" % testie_path)
testie_path = npf.find_local(testie_path + '.testie')
self.sections = []
self.files = []
self.scripts = []
self.imports = []
self.requirements = []
self.filename = os.path.basename(testie_path)
self.options = options
self.tags = tags if tags else []
self.role = role
try:
section = None
f = open(testie_path, 'r')
for i, line in enumerate(f):
line = re.sub(r'(^|[ ])//.*$', '', line)
if line.startswith('#') and section is None:
print(
"Warning : comments now use // instead of #. This will be soon deprecated"
)
continue
if line.strip() == '' and not section:
continue
if line.startswith("%"):
result = line[1:]
section = SectionFactory.build(self, result.strip())
if not section is SectionNull:
self.sections.append(section)
elif section is None:
raise Exception(
"Bad syntax, file must start by a section. Line %d :\n%s"
% (i, line))
else:
section.content += line
f.close()
if not hasattr(self, "info"):
self.info = Section("info")
self.info.content = self.filename
self.sections.append(self.info)
if not hasattr(self, "stdin"):
self.stdin = Section("stdin")
self.sections.append(self.stdin)
if not hasattr(self, "variables"):
self.variables = SectionVariable()
self.sections.append(self.variables)
if not hasattr(self, "config"):
self.config = SectionConfig()
self.sections.append(self.config)
for section in self.sections:
section.finish(self)
except Exception as e:
raise Exception("An exception occured while parsing %s :\n%s" %
(testie_path, e.__str__()))
# Check that all reference roles are defined
known_roles = {'self', 'default'}.union(set(npf.roles.keys()))
for script in self.get_scripts():
known_roles.add(script.get_role())
# for file in self.files:
# for nicref in re.finditer(Node.NICREF_REGEX, file.content, re.IGNORECASE):
# if nicref.group('role') not in known_roles:
# raise Exception("Unknown role %s" % nicref.group('role'))
# Create imports testies
for imp in self.imports:
from npf.module import Module
imp.testie = Module(imp.module, options, tags, imp.get_role())
if len(imp.testie.variables.dynamics()) > 0:
raise Exception(
"Imports cannot have dynamic variables. Their parents decides what's dynamic."
)
if 'delay' in imp.params:
for script in imp.testie.scripts:
delay = script.params.setdefault('delay', 0)
script.params['delay'] = float(delay) + float(
imp.params['delay'])
del imp.params['delay']
overriden_variables = {}
for k, v in imp.params.items():
overriden_variables[k] = VariableFactory.build(k, v)
imp.testie.variables.override_all(overriden_variables)
for script in imp.testie.scripts:
if script.get_role():
raise Exception(
'Modules cannot have roles, their importer defines it')
script._role = imp.get_role()
for excludes in self.config.get_list("role_exclude"):
s = excludes.split('+')
m = set()
for role in s:
node = npf.node(role)
if node in m:
raise Exception(
"Roles %s cannot be on the same node ! Please use --cluster argument to set them accross nodes"
% ' and '.join(s))
m.add(node)