def collect_design_metrics(block: Block) -> Dict:
backend_request = block.build_backend_request()
dimacs_header = __generate_cnf(block).split('\n')[0].split(' ')
return {
'full_factor_count': len(block.design),
'crossing_factor_count': len(block.crossing),
'constraint_count': len(block.constraints),
'block_length': block.trials_per_sample(),
'block_length_factorial': factorial(block.trials_per_sample()),
'low_level_request_count': len(backend_request.ll_requests),
'cnf_total_variables': int(dimacs_header[2]),
'cnf_total_clauses': int(dimacs_header[3])
}
def collect_design_metrics(block: Block) -> Dict:
"""Given a block, this function will collect various metrics pertaining to
the block and return them in a dictionary.
"""
backend_request = block.build_backend_request()
dimacs_header = __generate_cnf(block).split('\n')[0].split(' ')
return {
'full_factor_count': len(block.design),
'crossing_factor_count': len(block.crossing),
'constraint_count': len(block.constraints),
'block_length': block.trials_per_sample(),
'block_length_factorial': factorial(block.trials_per_sample()),
'low_level_request_count': len(backend_request.ll_requests),
'cnf_total_variables': int(dimacs_header[2]),
'cnf_total_clauses': int(dimacs_header[3])
}
def __apply_derivation(self, block: Block, backend_request: BackendRequest) -> None:
trial_size = block.variables_per_trial()
cross_size = block.trials_per_sample()
iffs = []
for n in range(cross_size):
or_clause = Or(list(And(list(map(lambda x: x + (n * trial_size) + 1, l))) for l in self.dependent_idxs))
iffs.append(Iff(self.derived_idx + (n * trial_size) + 1, or_clause))
(cnf, new_fresh) = block.cnf_fn(And(iffs), backend_request.fresh)
backend_request.cnfs.append(cnf)
backend_request.fresh = new_fresh
def decode(block: Block, solution: List[int]) -> dict:
# Sort the list and remove any negative (false) variables
solution.sort()
solution = list(filter(lambda v: v > 0, solution))
# Separate into simple/complex variables.
simple_variables = list(
filter(lambda v: v <= block.grid_variables(), solution))
complex_variables = list(
filter(lambda v: v > block.grid_variables(), solution))
experiment = cast(dict, {})
# Simple factors
tuples = list(map(lambda v: block.decode_variable(v),
simple_variables))
string_tuples = list(
map(lambda t: (t[0].factor_name, t[1].external_name), tuples))
for (factor_name, level_name) in string_tuples:
if factor_name not in experiment:
experiment[factor_name] = []
experiment[factor_name].append(level_name)
# Complex factors - The challenge here is knowing when to insert '', rather than using the variables.
# Start after 'width' trials, and shift 'stride' trials for each variable.
complex_factors = list(
filter(lambda f: f.has_complex_window, block.design))
for f in complex_factors:
# Get variables for this factor
start = block.first_variable_for_level(f, f.levels[0]) + 1
end = start + block.variables_for_factor(f)
variables = list(
filter(lambda n: n in range(start, end), complex_variables))
# Get the level names for the variables in the solution.
level_tuples = list(
map(lambda v: block.decode_variable(v), variables))
level_names = list(
map(lambda t: (t[1].external_name), level_tuples))
# Intersperse empty strings for the trials to which this factor does not apply.
#level_names = list(intersperse('', level_names, f.levels[0].window.stride - 1))
#level_names = list(repeat('', f.levels[0].window.width - 1)) + level_names
level_names_fill = []
for n in range(block.trials_per_sample()):
level_names_fill.append(
level_names.pop(0) if f.applies_to_trial(n + 1) else '')
experiment[f.factor_name] = level_names_fill
return experiment
def apply(block: Block, backend_request: BackendRequest) -> None:
next_var = 1
for _ in range(block.trials_per_sample()):
for f in filter(lambda f: not f.has_complex_window, block.design):
number_of_levels = len(f.levels)
new_request = LowLevelRequest("EQ", 1, list(range(next_var, next_var + number_of_levels)))
backend_request.ll_requests.append(new_request)
next_var += number_of_levels
for f in filter(lambda f: f.has_complex_window, block.design):
variables_for_factor = block.variables_for_factor(f)
var_list = list(map(lambda n: n + next_var, range(variables_for_factor)))
chunks = list(chunk_list(var_list, len(f.levels)))
backend_request.ll_requests += list(map(lambda v: LowLevelRequest("EQ", 1, v), chunks))
next_var += variables_for_factor
def __apply_derivation_with_complex_window(self, block: Block, backend_request: BackendRequest) -> None:
trial_size = block.variables_per_trial()
trial_count = block.trials_per_sample()
iffs = []
f = self.factor
window = f.levels[0].window
t = 0
for n in range(trial_count):
if not f.applies_to_trial(n + 1):
continue
num_levels = len(f.levels)
get_trial_size = lambda x: trial_size if x < block.grid_variables() else len(block.decode_variable(x+1)[0].levels)
or_clause = Or(list(And(list(map(lambda x: x + (t * window.stride * get_trial_size(x) + 1), l))) for l in self.dependent_idxs))
iffs.append(Iff(self.derived_idx + (t * num_levels) + 1, or_clause))
t += 1
(cnf, new_fresh) = block.cnf_fn(And(iffs), backend_request.fresh)
backend_request.cnfs.append(cnf)
backend_request.fresh = new_fresh
def __count_solutions(block: Block) -> int:
fc_size = block.trials_per_sample()
return math.factorial(fc_size)
def sample(block: Block,
sample_count: int,
min_search: bool = False) -> SamplingResult:
backend_request = block.build_backend_request()
if block.errors:
for e in block.errors:
print(e)
if "WARNING" not in e:
return SamplingResult([], {})
solutions = sample_uniform(
sample_count, CNF(backend_request.get_cnfs_as_json()),
backend_request.fresh - 1, block.variables_per_sample(),
backend_request.get_requests_as_generation_requests(), False)
if not solutions:
from sweetpea.constraints import AtLeastKInARow
if min_search:
return SamplingResult([], {})
else:
max_constraints = list(
map(
lambda x: cast(AtLeastKInARow, x).max_trials_required,
filter(lambda c: isinstance(c, AtLeastKInARow),
block.constraints)))
if max_constraints:
print(
"No solution found... We require a minimum trials contraint to find a solution."
)
max_constraint = max(max_constraints)
min_constraint = block.trials_per_sample() + 1
original_min_trials = block.min_trials
last_valid_min_contraint = max_constraint
last_valid = SamplingResult([], {})
progress = tqdm(total=math.ceil(
math.log(max_constraint - min_constraint)) + 1,
file=sys.stdout)
while True:
current_constraint = int(
(max_constraint - min_constraint + 1) /
2) + min_constraint
block.min_trials = original_min_trials
c = minimum_trials(current_constraint)
c.validate(block)
c.apply(block, None)
block.constraints.append(c)
res = UnigenSamplingStrategy.sample(
block, sample_count, True)
progress.update(1)
if res.samples:
if current_constraint <= min_constraint:
print(
"Optimal minimum trials contraint is at ",
current_constraint, ".")
return res
else:
last_valid_min_contraint = current_constraint
last_valid = res
max_constraint = current_constraint - 1
else:
if max_constraint <= current_constraint:
print(
"Optimal minimum trials contraint is at ",
last_valid_min_contraint, ".")
return last_valid
else:
min_constraint = current_constraint + 1
progress.close()
return result
else:
return SamplingResult([], {})
result = list(
map(lambda s: SamplingStrategy.decode(block, s.assignment),
solutions))
return SamplingResult(result, {})
def __generate_sample(block: Block, cnf: CNF,
sample_metrics: dict) -> dict:
sample_metrics['trials'] = []
# Start a 'committed' list of CNFs
committed = cast(List[And], [])
for trial_number in range(block.trials_per_sample()):
trial_start_time = time()
trial_metrics = {'t': trial_number + 1, 'solver_calls': []}
solver_calls = cast(List[dict], trial_metrics['solver_calls'])
# Get the variable list for this trial.
variables = block.variable_list_for_trial(trial_number + 1)
variables = list(filter(lambda i: i != [], variables))
potential_trials = list(map(list, product(*variables)))
trial_metrics['potential_trials'] = len(potential_trials)
# Use env var to switch between filtering and not
if GuidedSamplingStrategy.__prefilter_enabled():
# Flatten the list
flat_vars = list(chain(*variables))
# Check SAT for each one
unsat = []
for v in flat_vars:
t_start = time()
full_cnf = cnf + CNF(cnf_to_json(committed)) + CNF(
cnf_to_json([And([v])]))
allowed = cnf_is_satisfiable(full_cnf)
duration_seconds = time() - t_start
solver_calls.append({
'time': duration_seconds,
'SAT': allowed
})
if not allowed:
unsat.append(v)
# TODO: Count filtering SAT calls separately?
# Filter out any potential trials with those vars set
filtered_pts = []
for pt in potential_trials:
if any(uv in pt for uv in unsat):
continue
else:
filtered_pts.append(pt)
# Record the number filterd out for metrics
trial_metrics['prefiltered_out'] = len(potential_trials) - len(
filtered_pts)
potential_trials = filtered_pts
allowed_trials = []
for potential_trial in potential_trials:
start_time = time()
full_cnf = cnf + CNF(cnf_to_json(committed)) + CNF(
cnf_to_json([And(potential_trial)]))
allowed = cnf_is_satisfiable(full_cnf)
duration_seconds = time() - start_time
solver_calls.append({'time': duration_seconds, 'SAT': allowed})
if allowed:
allowed_trials.append(potential_trial)
trial_metrics['allowed_trials'] = len(allowed_trials)
trial_metrics['solver_call_count'] = len(solver_calls)
sample_metrics['trials'].append(trial_metrics)
# Randomly sample a single trial from the uniform distribution of the allowed trials,
# and commit that trial to the committed sequence.
trial_idx = np.random.randint(0, len(allowed_trials))
committed.append(And(allowed_trials[trial_idx]))
trial_metrics['time'] = time() - trial_start_time
# Aggregate the total solver calls
sample_metrics['solver_call_count'] = 0
for tm in sample_metrics['trials']:
sample_metrics['solver_call_count'] += tm['solver_call_count']
# Flatten the committed trials into a list of integers and decode it.
solution = GuidedSamplingStrategy.__committed_to_solution(committed)
return SamplingStrategy.decode(block, solution)
def __generate_encoding_diagram(blk: Block) -> str:
diagram_str = ""
design_size = blk.variables_per_trial()
num_trials = blk.trials_per_sample()
num_vars = blk.variables_per_sample()
largest_number_len = len(str(num_vars))
header_widths = []
row_format_str = '| {:>7} |'
for f in blk.design:
# length of all levels concatenated for this factor
level_names = list(map(get_external_level_name, f.levels))
level_name_widths = [
max(largest_number_len, l) for l in list(map(len, level_names))
]
level_names_width = sum(level_name_widths) + len(
level_names) - 1 # Extra length for spaces in between names.
factor_header_width = max(len(f.factor_name), level_names_width)
header_widths.append(factor_header_width)
# If the header is longer than the level widths combined, then they need to be lengthened.
diff = factor_header_width - level_names_width
if diff > 0:
idx = 0
while diff > 0:
level_name_widths[idx] += 1
idx += 1
diff -= 1
if idx >= len(level_name_widths):
idx = 0
# While we're here, build up the row format str.
row_format_str = reduce(lambda a, b: a + ' {{:^{}}}'.format(b),
level_name_widths, row_format_str)
row_format_str += ' |'
header_format_str = reduce(lambda a, b: a + ' {{:^{}}} |'.format(b),
header_widths, '| {:>7} |')
factor_names = list(map(lambda f: f.factor_name, blk.design))
header_str = header_format_str.format(*["Trial"] + factor_names)
row_width = len(header_str)
# First line
diagram_str += ('-' * row_width) + '\n'
# Header
diagram_str += header_str + '\n'
# Level names
all_level_names = [
ln for (fn, ln) in get_all_external_level_names(blk.design)
]
diagram_str += row_format_str.format(*['#'] + all_level_names) + '\n'
# Separator
diagram_str += ('-' * row_width) + '\n'
# Variables
for t in range(num_trials):
args = [str(t + 1)]
for f in blk.design:
if f.applies_to_trial(t + 1):
variables = [
blk.first_variable_for_level(f, l) + 1 for l in f.levels
]
if f.has_complex_window():
def acc_width(w) -> int:
return w.width + (
acc_width(w.args[0].levels[0].window) -
1 if w.args[0].has_complex_window() else 0)
width = acc_width(f.levels[0].window)
stride = f.levels[0].window.stride
stride_offset = (stride - 1) * int(t / stride)
offset = t - width + 1 - stride_offset
variables = list(
map(lambda n: n + len(variables) * offset, variables))
else:
variables = list(
map(lambda n: n + design_size * t, variables))
args += list(map(str, variables))
else:
args += list(repeat('', len(f.levels)))
diagram_str += row_format_str.format(*args) + '\n'
# Footer
diagram_str += ('-' * row_width) + '\n'
return diagram_str
def sample(block: Block,
sample_count: int,
min_search: bool = False) -> SamplingResult:
backend_request = block.build_backend_request()
if block.errors:
for e in block.errors:
print(e)
if "WARNING" not in e:
return SamplingResult([], {})
solutions = sample_uniform(
sample_count, CNF(backend_request.get_cnfs_as_json()),
backend_request.fresh - 1, block.variables_per_sample(),
backend_request.get_requests_as_generation_requests(), False)
# This section deals with the problem caused by a corner case created
# by at_least_k_in_a_row_constraint. I.e. in some cases this cotnraint
# requires the support of a minimum_trials contraint to find valid
# solutions. This will find the optimal minimum trials constraint to
# the user using binary search with trial and error.
if not solutions:
from sweetpea.constraints import AtLeastKInARow
if min_search:
return SamplingResult([], {})
else:
atleast_constraints = cast(
List[AtLeastKInARow],
filter(lambda c: isinstance(c, AtLeastKInARow),
block.constraints))
max_constraints = list(
map(lambda x: x.max_trials_required, atleast_constraints))
if max_constraints:
print(
"No solution found... We require a minimum trials contraint to find a solution."
)
max_constraint = max(max_constraints)
min_constraint = block.trials_per_sample() + 1
original_min_trials = block.min_trials
last_valid_min_contraint = max_constraint
last_valid = SamplingResult([], {})
progress = tqdm(
total=ceil(log(max_constraint - min_constraint)) + 1,
file=sys.stdout)
while True:
current_constraint = int(
(max_constraint - min_constraint + 1) /
2) + min_constraint
block.min_trials = original_min_trials
c = minimum_trials(current_constraint)
c.validate(block)
c.apply(block, None)
block.constraints.append(c)
res = UnigenSamplingStrategy.sample(
block, sample_count, True)
progress.update(1)
if res.samples:
if current_constraint <= min_constraint:
print(
"Optimal minimum trials contraint is at ",
current_constraint, ".")
return res
else:
last_valid_min_contraint = current_constraint
last_valid = res
max_constraint = current_constraint - 1
else:
if max_constraint <= current_constraint:
print(
"Optimal minimum trials contraint is at ",
last_valid_min_contraint, ".")
return last_valid
else:
min_constraint = current_constraint + 1
progress.close()
return result
else:
return SamplingResult([], {})
result = list(
map(lambda s: SamplingStrategy.decode(block, s.assignment),
solutions))
return SamplingResult(result, {})