def batch_calculate_state(circuits, param_resolvers, simulator):
"""Compute states using a given simulator using parallel processing.
Returns a NumPy array containing the final circuit state for each
`cirq.Circuit` in `circuits`, given that the corresponding
`cirq.ParamResolver` in `param_resolvers` was used to resolve any symbols
in it. If simulator is a `cirq.DensityMatrixSimulator` this final state will
be a density matrix, if simulator is a `cirq.Simulator` this final state
will be a state vector. More specifically for a given `i`
`batch_calculate_state` will use `param_resolvers[i]` to resolve the symbols
in `circuits[i]` and then place the final state in the return list at index
`i`.
Args:
circuits: Python `list` of `cirq.Circuit`s.
param_resolvers: Python `list` of `cirq.ParamResolver`s, where
`param_resolvers[i]` is the resolver to be used with `circuits[i]`.
simulator: Simulator object. Currently
supported are `cirq.DensityMatrixSimulator` and `cirq.Simulator`.
Returns:
`np.ndarray` containing the resulting state information. The array will
have dimensions: [len(circuits), <size of biggest state>] in the
case of `cirq.Simulator`. In the case of `cirq.DensityMatrixSimulator`
the shape is
[len(circuits), <size of biggest state>, <size of biggest state>]
"""
_validate_inputs(circuits, param_resolvers, simulator, 'analytic')
if _check_empty(circuits):
empty_ret = np.zeros((0, 0), dtype=np.complex64)
if isinstance(simulator, cirq.DensityMatrixSimulator):
empty_ret = np.zeros((0, 0, 0), dtype=np.complex64)
return empty_ret
biggest_circuit = max(len(circuit.all_qubits()) for circuit in circuits)
if isinstance(simulator, cirq.DensityMatrixSimulator):
return_mem_shape = (len(circuits), 1 << biggest_circuit,
1 << biggest_circuit)
post_process = lambda x: x.final_density_matrix
elif isinstance(simulator, cirq.Simulator):
return_mem_shape = (len(circuits), 1 << biggest_circuit)
post_process = lambda x: x.final_state_vector
else:
raise TypeError('Simulator {} is not supported by '
'batch_calculate_state.'.format(type(simulator)))
shared_array = _make_complex_view(return_mem_shape, -2)
input_args = _prep_pool_input_args(range(len(circuits)), circuits,
param_resolvers)
with ProcessPool(processes=None,
initializer=_setup_dict,
initargs=(shared_array, return_mem_shape, simulator,
post_process)) as pool:
pool.starmap(_state_worker_func, list(input_args))
return _convert_complex_view_to_result(shared_array, return_mem_shape)
def start(self):
"""
Create a process pool to execute concurrent requests.
:return:
"""
process_pool = ProcessPool(processes=self.concurrency)
process_pool.map(self.run, range(self.concurrency))
process_pool.close()
process_pool.join()
def persist_corpus(data_path: str, uri: str, processes: int):
ppool = ProcessPool(processes)
file_names = [file_name for file_name in get_files_gen(data_path)]
with Database(uri) as session:
session.drop_db()
session.create_db()
session.start_transaction()
with tqdm(total=len(file_names)) as progress:
for docs in tqdm(ppool.imap_unordered(worker_job, file_names)):
session.insert_many(docs)
progress.update()
def main():
spider_list = []
for spiders in all_spiders:
for name in dir(spiders):
spider_instance = getattr(spiders, name)
if inspect.isclass(spider_instance):
spider_list.append(spider_instance)
# pool = ThreadPool(8)
pool = ProcessPool(4)
pool.map(start, spider_list)
pool.close()
pool.join()
print("ALL TASK DONE")
def run(self):
begin = int(time.time())
pool = ProcessPool(args.n) if args.m == 'proc' else ThreadPool(args.n)
if self.f == 'ping':
result = pool.map(self.run_ping, self.ip)
else:
result = pool.map(self.run_tcp_port, [port for port in range(65535 + 1)])
pool.close()
pool.join()
end = int(time.time())
if self.cost_print:
print('cost time:%s' % (end - begin))
# print('result:', result)
if self.file_name:
with open(self.file_name, 'w') as f:
f.write(json.dumps(result))
def create_frames(self, states):
self.validate_states(states)
states_to_be_created = self.get_nonexistent_states(states)
if len(states_to_be_created) != 0 and self.psd is None:
raise RuntimeError(
"Need to generate states but no psd file supplied.")
#ProcessPool for CPU bound image creation, ThreadPool for IO bound file saving
with ProcessPool(initializer=ignore_signal
) as process_pool, ThreadPool() as thread_pool:
try:
for image, state in process_pool.imap_unordered(
self.generate_image, states_to_be_created):
save_image = partial(image.save,
self.args.directory / state.filename)
thread_pool.submit(save_image)
except KeyboardInterrupt:
process_pool.terminate()
thread_pool.shutdown(wait=False)
raise KeyboardInterrupt from None
def batch_sample(circuits, param_resolvers, n_samples, simulator):
"""Sample from circuits using parallel processing.
Returns a `np.ndarray` containing n_samples samples from all the circuits in
circuits given that the corresponding `cirq.ParamResolver` in
`param_resolvers` was used to resolve any symbols. Specifically the
returned array at index `i,j` will correspond to a `np.ndarray` of
booleans representing bitstring `j` that was sampled from `circuits[i]`.
Samples are drawn using the provided simulator object (Currently supported
are `cirq.DensityMatrixSimulator` and `cirq.Simulator`).
Note: In order to keep numpy shape consistent, smaller circuits will
have sample bitstrings padded with -2 on "qubits that don't exist
in the circuit".
Args:
circuits: Python `list` of `cirq.Circuit`s.
param_resolvers: Python `list` of `cirq.ParamResolver`s, where
`param_resolvers[i]` is the resolver to be used with `circuits[i]`.
n_samples: `int` describing number of samples to draw from each
circuit.
simulator: Simulator object. Currently
supported are `cirq.DensityMatrixSimulator` and `cirq.Simulator`.
Returns:
`np.ndarray` containing the samples with invalid qubits blanked out.
It's shape is
[len(circuits), n_samples, <# qubits in largest circuit>].
circuits that are smaller than #qubits in largest circuit have null
qubits in bitstrings mapped to -2.
"""
_validate_inputs(circuits, param_resolvers, simulator, 'sample')
if _check_empty(circuits):
return np.zeros((0, 0, 0), dtype=np.int32)
if not isinstance(n_samples, int):
raise TypeError('n_samples must be an int.'
'Given: {}'.format(type(n_samples)))
if n_samples <= 0:
raise ValueError('n_samples must be > 0.')
biggest_circuit = max(len(circuit.all_qubits()) for circuit in circuits)
return_mem_shape = (len(circuits), n_samples, biggest_circuit)
shared_array = _make_simple_view(return_mem_shape, -2, np.int32, 'i')
if isinstance(simulator, cirq.DensityMatrixSimulator):
post_process = lambda state, size, n_samples: \
cirq.sample_density_matrix(
state.final_density_matrix, [i for i in range(size)],
repetitions=n_samples)
elif isinstance(simulator, cirq.Simulator):
post_process = lambda state, size, n_samples: cirq.sample_state_vector(
state.final_state_vector, list(range(size)), repetitions=n_samples)
else:
raise TypeError(
'Simulator {} is not supported by batch_sample.'.format(
type(simulator)))
input_args = list(
_prep_pool_input_args(range(len(circuits)), circuits, param_resolvers,
[n_samples] * len(circuits)))
with ProcessPool(processes=None,
initializer=_setup_dict,
initargs=(shared_array, return_mem_shape, simulator,
post_process)) as pool:
pool.starmap(_sample_worker_func, input_args)
return _convert_simple_view_to_result(shared_array, np.int32,
return_mem_shape)
def batch_calculate_sampled_expectation(circuits, param_resolvers, ops,
n_samples, simulator):
"""Compute expectations from sampling circuits using parallel processing.
Returns a `np.ndarray` containing the expectation values of `ops`
applied to a specific circuit in `circuits`, given that the
corresponding `cirq.ParamResolver` in `param_resolvers` was used to resolve
any symbols in the circuit. Specifically the returned array at index `i,j`
will be equal to the expectation value of `ops[i][j]` on `circuits[i]` with
`param_resolvers[i]` used to resolve any symbols in `circuits[i]`.
Expectation estimations will be carried out using the simulator object
(`cirq.DensityMatrixSimulator` and `cirq.Simulator` are currently supported)
. Expectations for ops[i][j] are estimated by drawing n_samples[i][j]
samples.
Args:
circuits: Python `list` of `cirq.Circuit`s.
param_resolvers: Python `list` of `cirq.ParamResolver`s, where
`param_resolvers[i]` is the resolver to be used with `circuits[i]`.
ops: 2d Python `list` of `cirq.PauliSum` objects where `ops[i][j]` will
be used to calculate the expectation on `circuits[i]` for all `j`,
after `param_resolver[i]` is used to resolve any parameters
in the circuit.
n_samples: 2d Python `list` of `int`s where `n_samples[i][j]` is
equal to the number of samples to draw in each term of `ops[i][j]`
when estimating the expectation.
simulator: Simulator object. Currently supported are
`cirq.DensityMatrixSimulator` and `cirq.Simulator`.
Returns:
`np.ndarray` containing the expectation values. Shape is:
[len(circuits), len(ops[0])]
"""
_validate_inputs(circuits, param_resolvers, simulator, 'sample')
if _check_empty(circuits):
return np.zeros((0, 0), dtype=np.float32)
if not isinstance(ops, (list, tuple, np.ndarray)):
raise TypeError('ops must be a list or array.'
' Given: {}'.format(type(ops)))
if len(ops) != len(circuits):
raise ValueError('Shape of ops and circuits do not match.')
if len(n_samples) != len(circuits):
raise ValueError('Shape of n_samples does not match circuits.')
for sub_list in n_samples:
if not isinstance(sub_list, (list, tuple, np.ndarray)):
raise TypeError('Elements of n_elements must be lists of ints.')
for x in sub_list:
if not isinstance(x, int):
raise TypeError('Non-integer value found in n_samples.')
if x <= 0:
raise ValueError('n_samples contains sample value <= 0.')
for sub_list in ops:
if not isinstance(sub_list, (list, tuple, np.ndarray)):
raise TypeError('elements of ops must be type list.')
for x in sub_list:
if not isinstance(x, cirq.PauliSum):
raise TypeError('ops must contain only cirq.PauliSum objects.'
' Given: {}'.format(type(x)))
return_mem_shape = (len(circuits), len(ops[0]))
shared_array = _make_simple_view(return_mem_shape, -2, np.float32, 'f')
# avoid mutating ops array
ops = np.copy(ops)
# TODO (mbbrough): make cirq PauliSums pickable at some point ?
for i in range(len(ops)):
for j in range(len(ops[i])):
ops[i][j] = serializer.serialize_paulisum(ops[i][j])
input_args = list(
_prep_pool_input_args(list(
itertools.product(range(len(circuits)), range(len(ops[0])))),
circuits,
param_resolvers,
ops,
n_samples,
slice_args=False))
with ProcessPool(processes=None,
initializer=_setup_dict,
initargs=(shared_array, return_mem_shape, simulator,
None)) as pool:
pool.starmap(_sample_expectation_worker_func, input_args)
return _convert_simple_view_to_result(shared_array, np.float32,
return_mem_shape)
def batch_calculate_expectation(circuits, param_resolvers, ops, simulator):
"""Compute expectations from circuits using parallel processing.
Returns a `np.ndarray` containing the expectation values of `ops`
applied to a specific circuit in `circuits`, given that the
corresponding `cirq.ParamResolver` in `param_resolvers` was used to resolve
any symbols in the circuit. Specifically the returned array at index `i,j`
will be equal to the expectation value of `ops[i][j]` on `circuits[i]` with
`param_resolvers[i]` used to resolve any symbols in `circuits[i]`.
Expectation calculations will be carried out using the simulator object
(`cirq.DensityMatrixSimulator` and `cirq.Simulator` are currently supported)
Args:
circuits: Python `list` of `cirq.Circuit`s.
param_resolvers: Python `list` of `cirq.ParamResolver`s, where
`param_resolvers[i]` is the resolver to be used with `circuits[i]`.
ops: 2d Python `list` of `cirq.PauliSum` objects where `ops[i][j]` will
be used to calculate the expectation on `circuits[i]` for all `j`,
after `param_resolver[i]` is used to resolve any parameters
in the circuit.
simulator: Simulator object. Currently supported are
`cirq.DensityMatrixSimulator` and `cirq.Simulator`.
Returns:
`np.ndarray` containing the expectation values. Shape is:
[len(circuits), len(ops[0])]
"""
_validate_inputs(circuits, param_resolvers, simulator, 'analytic')
if not isinstance(ops, (list, tuple, np.ndarray)):
raise TypeError('ops must be a list or array.'
' Given: {}'.format(type(ops)))
if len(ops) != len(circuits):
raise ValueError('Shape of ops and circuits do not match.')
for sub_list in ops:
if not isinstance(sub_list, (list, tuple, np.ndarray)):
raise TypeError('elements of ops must be type list.')
for x in sub_list:
if not isinstance(x, cirq.PauliSum):
raise TypeError('ops must contain only cirq.PauliSum objects.'
' Given: {}'.format(type(x)))
return_mem_shape = (len(circuits), len(ops[0]))
if isinstance(simulator, cirq.DensityMatrixSimulator):
post_process = lambda op, state, order: sum(
x._expectation_from_density_matrix_no_validation(
state.final_density_matrix, order) for x in op).real
elif isinstance(simulator, cirq.Simulator):
post_process = \
lambda op, state, order: op.expectation_from_state_vector(
state.final_state_vector, order).real
else:
raise TypeError('Simulator {} is not supported by '
'batch_calculate_expectation.'.format(type(simulator)))
shared_array = _make_simple_view(return_mem_shape, -2, np.float32, 'f')
# avoid mutating ops array
ops = np.copy(ops)
# TODO (mbbrough): make cirq PauliSUms pickable at some point ?
for i in range(len(ops)):
for j in range(len(ops[i])):
ops[i][j] = serializer.serialize_paulisum(ops[i][j])
input_args = list(
_prep_pool_input_args(list(
itertools.product(range(len(circuits)), range(len(ops[0])))),
circuits,
param_resolvers,
ops,
slice_args=False))
with ProcessPool(processes=None,
initializer=_setup_dict,
initargs=(shared_array, return_mem_shape, simulator,
post_process)) as pool:
pool.starmap(_analytical_expectation_worker_func, input_args)
return _convert_simple_view_to_result(shared_array, np.float32,
return_mem_shape)
def star(self):
process_pool = ProcessPool(processes=self.concurrency)
process_pool.map(self.run, range(self.concurrency))
process_pool.close()
process_pool.join()
def get_surrogate_model_data():
"""
Yields structural data for surrogate model.
GET parameters:
- "modeltype": Model type can be specified with GET param (currently only decision tree with "rules" supported).
- "objs": Objectives with objs=alpha,beta,...
- "ids": List of embedding IDs to consider, with ids=1,2,3,... Note: If "ids" is not specified, all embeddings
are used to construct surrogate model.
- "n_bins": Number of bins to use for surrogate model's predictions.
:return: Jsonified structure of surrogate model for DR metadata.
"""
metadata_template = app.config["METADATA_TEMPLATE"]
surrogate_model_type = request.args["modeltype"]
objective_name = request.args["objs"]
number_of_bins = int(
request.args["n_bins"]) if request.args["n_bins"] is not None else 5
ids = request.args.get("ids")
# ------------------------------------------------------
# 1. Check for mistakes in parameters.
# ------------------------------------------------------
if surrogate_model_type not in ["rules"]:
return "Surrogate model " + surrogate_model_type + " is not supported.", 400
if objective_name not in metadata_template["objectives"]:
return "Objective " + objective_name + " is not supported.", 400
# ------------------------------------------------------
# 2. Pre-select embeddings to use for surrogate model.
# ------------------------------------------------------
ids = list(map(int, ids.split(","))) if ids is not None else None
features_df = app.config["EMBEDDING_METADATA"]["features_preprocessed"]
labels_df = app.config["EMBEDDING_METADATA"]["labels"]
# Consider filtered IDs before creating model(s).
if ids is not None:
features_df = features_df.iloc[ids]
labels_df = labels_df.iloc[ids]
class_encodings = pd.DataFrame(
pd.cut(labels_df[objective_name], number_of_bins))
bin_labels = class_encodings[objective_name].unique()
with ProcessPool(math.floor(psutil.cpu_count(logical=True))) as pool:
rule_data = list(
tqdm(pool.imap(
partial(Utils.extract_rules,
features_df=features_df,
class_encodings=class_encodings,
objective_name=objective_name), bin_labels),
total=len(bin_labels)))
rule_data = pd.DataFrame(
[item for sublist in rule_data for item in sublist],
columns=["rule", "precision", "recall", "support", "from", "to"])
# Bin data for frontend.
for attribute in ["precision", "recall", "support"]:
quantiles = pd.cut(rule_data[attribute], number_of_bins)
rule_data[attribute + "#histogram"] = quantiles.apply(lambda x: x.left)
rule_data["from#histogram"] = rule_data["from"]
rule_data["to#histogram"] = rule_data["to"]
rule_data.rule = rule_data.rule.str.replace(" and ", "<br>")
return rule_data.to_json(orient='records')