def plot(self):
counter = Counter()
for combination in itertools_product(c.plot().most_common()
for c in self.components):
counter[self.evaluate(c[0]
for c in combination)] += functools_reduce(
operator_mul,
[c[1] for c in combination])
return counter
def _variants(self):
options = self.tree_html.cssselect('ul.saleAttr')
variants_initial = {}
for option in options:
option_name = option.xpath('../preceding-sibling::'
'div[@class="dt"]/text()')[0]
values = option.xpath('li/a/@title | li[not(a)]/text()')
variants_initial[option_name] = values
variant_names = variants_initial.keys()
variant_values = list(
itertools_product(*variants_initial.itervalues()))
variants = []
for variant_value in variant_values:
properties = {}
for i, v in enumerate(variant_value):
properties[variant_names[i]] = v
data = dict(price=None, selected=None, properties=properties)
variants.append(data)
return variants
def fit_gp(self, num_samples=1, hp_tune_criterion=None):
""" Fits a GP according to the tuning criterion. Returns the best GP along with the
hyper-parameters. """
if hp_tune_criterion is None:
hp_tune_criterion = self.options.hp_tune_criterion
if hp_tune_criterion == 'ml':
if self.ml_hp_tune_opt_method in ['direct', 'rand', 'pdoo']:
best_cts_hps = None
best_dscr_hps = None
best_other_params = None
best_hps_val = -np.inf
for dscr_hps in itertools_product(*self.dscr_hp_vals):
opt_cts_val, opt_cts_hps, opt_other_params = \
self._optimise_cts_hps_for_given_dscr_hps(dscr_hps)
if opt_cts_val > best_hps_val:
best_cts_hps = list(opt_cts_hps)
best_dscr_hps = list(dscr_hps)
best_other_params = opt_other_params
best_hps_val = opt_cts_val
opt_gp = self.build_gp(best_cts_hps,
best_dscr_hps,
other_gp_params=best_other_params)
opt_hps = (best_cts_hps, best_dscr_hps)
return 'fitted_gp', opt_gp, opt_hps
elif self.ml_hp_tune_opt_method == 'rand_exp_sampling':
sample_cts_hps, sample_dscr_hps, sample_other_gp_params, sample_probs = \
self._sample_cts_dscr_hps_for_rand_exp_sampling()
return ('sample_hps_with_probs', sample_cts_hps,
sample_dscr_hps, sample_other_gp_params, sample_probs)
elif hp_tune_criterion == 'post_sampling':
sample_cts_hps, sample_dscr_hps, sample_other_gp_params = \
self._sample_cts_dscr_hps_for_post_sampling(num_samples)
if num_samples == 1:
opt_gp = self.build_gp(
sample_cts_hps[0],
sample_dscr_hps[0],
other_gp_params=sample_other_gp_params[0])
opt_hps = (sample_cts_hps, sample_dscr_hps)
return 'post_fitted_gp', opt_gp, opt_hps
else:
return ('post_sample_hps_with_probs', sample_cts_hps,
sample_dscr_hps, sample_other_gp_params)
def constrained_permutations(init_golf_course,
list_holes,
list_balls,
list_dist_balls,
nb_of_balls=None):
"""
:param init_golf_course: the golf course input
:param list_holes: the list of holes on the golf course
:param list_balls: the list of balls to put in the holes on the golf course
:param list_dist_balls: the list of the distances to respect for each ball
:param nb_of_balls: the number of elements we need to permute in the list (must be equal to the number of balls)
:return: this function yields for each possible permutation, a golf course with the "ball-to-hole" paths set and
the sum of the distances of those paths
permutation example:
holes: [A, B, C]
balls: [1, 2]
possible hole permutations with r = 2: [A, B], [B, A], [A, C], [C, A], [B, C], [C, B]
possible "ball-to-hole" paths: [1-A, 2-B], [1-B, 2-A], [1-A, 2-C], [1-C, 2-A], [1-B, 2-C], [1-C, 2-B]
"""
nb_of_holes = len(list_holes)
nb_of_permuted_elt = nb_of_holes if nb_of_balls is None else nb_of_balls
couples_not_possible = []
# going to iterate the indices (nb_of_holes**nb_of_permuted_elt) times:
for indices in itertools_product(range(nb_of_holes),
repeat=nb_of_permuted_elt):
# we select nb_of_permuted_elt within nb_of_holes (order counts)
# to do that, we select only permutations without duplicate elements
if len(set(indices)) == nb_of_permuted_elt:
# check that this permutation is possible ("ball-to-hole" length VS distance to be respected)
if is_distance_ok(list_balls, list_holes, list_dist_balls,
indices):
# if (indices[i], i) couple has already been passed and is not possible
# then we go to the next possible permutation
for i, indices_i in enumerate(indices):
if (indices_i, i) in couples_not_possible:
break
# for each new possible permutation, we deep-copy the golf course
golf_course = copy_deepcopy(init_golf_course)
list_dist_ball_hole = []
# for each (ball, hole) couple in the permutation, we check if there is a possible path
for i in range(nb_of_permuted_elt):
# because memory address of golf_course changes in the backtracking function
get_path_ball_to_hole, golf_course = get_path(
list_balls[i], list_holes[indices[i]],
list_dist_balls[i], list_dist_ball_hole, golf_course)
# if for one of the (ball, hole) couple, we don't find a "ball-to-hole" path
# then we go to the next possible permutation
if not get_path_ball_to_hole:
couples_not_possible.append((indices[i], i))
break
# if we found a path for each of the (ball, hole) couples of this permutation:
if len(list_dist_ball_hole) == nb_of_permuted_elt:
# we yield the golf course and the sum of the distances of each path
yield golf_course, sum(list_dist_ball_hole)
def product(list_of_lists):
return list(itertools_product(*list_of_lists))
def __build_all_pathways(
pathways: Dict,
transfos: Dict,
sink_molecules: List,
rr_reactions: Dict,
compounds_cache: Dict,
max_subpaths_filter: int,
lower_flux_bound: float,
upper_flux_bound: float,
logger: Logger = getLogger(__name__)
) -> Dict:
"""Builds pathways based on all combinations over
reaction rules and template reactions (see
`build_pathway_combinatorics` documentation).
Parameters
----------
pathways: Dict
Metabolic pathways as list of chemical
reactions where each reaction is defined by:
- transformation ID,
- reaction rule ID, and
- template reaction ID
transfos: Dict
Full chemical transformations
sink_molecules: List
Sink chemical species IDs
rr_reactions: Dict
Reaction rules cache
compounds_cache: Dict
Compounds cache
max_subpaths_filter: int
Number of pathways (best) kept per master pathway
lower_flux_bound: float
Lower flux bound for all new reactions created
upper_flux_bound: float
Upper flux bound for all new reactions created
logger: Logger, optional
Returns
-------
Set of ranked rpPathway objects
"""
res_pathways = {}
nb_pathways = 0
nb_unique_pathways = 0
## PATHWAYS
for path_idx, transfos_lst in pathways.items():
# Combine over multiple template reactions
sub_pathways = list(itertools_product(*transfos_lst))
## SUB-PATHWAYS
# # Keep only topX best sub_pathways
# # within a same master pathway
res_pathways[path_idx] = []
for sub_path_idx in range(len(sub_pathways)):
pathway = rpPathway(
id=str(path_idx).zfill(3)+'_'+str(sub_path_idx+1).zfill(4),
logger=logger
)
logger.debug(pathway.get_id())
## ITERATE OVER REACTIONS
nb_reactions = len(sub_pathways[sub_path_idx])
for rxn_idx in range(nb_reactions):
rxn = sub_pathways[sub_path_idx][rxn_idx]
transfo_id = rxn['rp2_transfo_id']
transfo = transfos[transfo_id]
rule_ids = rxn['rule_ids']
tmpl_rxn_id = rxn['tmpl_rxn_ids']
## COMPOUNDS
# Template reaction compounds
added_cmpds = transfo['complement'][rule_ids][tmpl_rxn_id]['added_cmpds']
# Add missing compounds to the cache
for side in added_cmpds.keys():
for spe_id in added_cmpds[side].keys():
logger.debug(f'Add missing compound {spe_id}')
if spe_id not in Cache.get_objects():
try:
rpCompound(
id=spe_id,
smiles=compounds_cache[spe_id]['smiles'],
inchi=compounds_cache[spe_id]['inchi'],
inchikey=compounds_cache[spe_id]['inchikey'],
formula=compounds_cache[spe_id]['formula'],
name=compounds_cache[spe_id]['name']
)
except KeyError:
rpCompound(
id=spe_id
)
## REACTION
# Compounds from original transformation
core_species = {
'right': deepcopy(transfo['right']),
'left': deepcopy(transfo['left'])
}
compounds = __add_compounds(core_species, added_cmpds)
# revert reaction index (forward)
rxn_idx_forward = nb_reactions - rxn_idx
rxn = rpReaction(
id='rxn_'+str(rxn_idx_forward),
ec_numbers=transfo['ec'],
reactants=dict(compounds['left']),
products=dict(compounds['right']),
lower_flux_bound=lower_flux_bound,
upper_flux_bound=upper_flux_bound
)
# write infos
for info_id, info in sub_pathways[sub_path_idx][rxn_idx].items():
getattr(rxn, 'set_'+info_id)(info)
rxn.set_rule_score(rr_reactions[rule_ids][tmpl_rxn_id]['rule_score'])
rxn.set_idx_in_path(rxn_idx_forward)
# Add at the beginning of the pathway
# to have the pathway in forward direction
# Search for the target in the current reaction
target_id = [spe_id for spe_id in rxn.get_products_ids() if 'TARGET' in spe_id]
if target_id != []:
target_id = target_id[0]
else:
target_id = None
logger.debug(f'rxn: {rxn._to_dict()}')
pathway.add_reaction(
rxn=rxn,
target_id=target_id
)
## TRUNK SPECIES
pathway.add_species_group(
'trunk',
[
spe_id
for value
in core_species.values()
for spe_id in value.keys()
]
)
## COMPLETED SPECIES
pathway.add_species_group(
'completed',
[
spe_id
for value
in added_cmpds.values()
for spe_id in value.keys()
]
)
## SINK
pathway.set_sink_species(
list(
set(pathway.get_species_ids()) & set(sink_molecules)
)
)
nb_pathways += 1
## RANK AMONG ALL SUB-PATHWAYS OF THE CURRENT MASTER PATHWAY
res_pathways[path_idx] = __keep_unique_pathways(
res_pathways[path_idx],
pathway,
logger
)
nb_unique_pathways += len(res_pathways[path_idx])
# Flatten lists of pathways
pathways = sum(
[
pathways
for pathways in res_pathways.values()
], [])
# Globally sort pathways
pathways = sorted(pathways)[-max_subpaths_filter:]
logger.info(f'Pathways statistics')
logger.info(f'-------------------')
logger.info(f' pathways: {nb_pathways}')
logger.info(f' unique pathways: {nb_unique_pathways}')
logger.info(f' selected pathways: {len(pathways)} (topX filter = {max_subpaths_filter})')
# Return topX pathway objects
return [
pathway.object
for pathway in pathways
]
# Transform the list of Item into a list of Pathway
results = {}
nb_sel_pathways = 0
for res_pathway_idx, res_pathway in res_pathways.items():
results[res_pathway_idx] = [pathway.object for pathway in res_pathway]
nb_sel_pathways += len(results[res_pathway_idx])
logger.info(f'Pathways selected: {nb_sel_pathways}/{nb_pathways}')
return results
def expandgrid(*itrs):
product = list(itertools_product(*itrs))
return {i: [x[i] for x in product]
for i in range(len(itrs))}
def main(image_urls,
access_id,
secret_key,
regions,
volume_types=None,
volume_via_s3=True,
ex_virt_types=None,
push_notifications=False,
compose_id=None):
"""
The `ec2.ec2initiate.main` function iterates over the image urls and start
uploading the image to the specified regions. The `image_urls`,
`access_id`, and `regions` are the required params. `volume_types`,
`ex_virt_types` are optional arguments, if not passed the values are picked
up from the fedimg configuration.
`volume_via_s3`, `push_notifications`, and `compose_id` are optional params
with default values.
Args:
image_urls (list): List of the image urls to create AMIs. (reques
access_id (str): AWS EC2 access id
secret_key (str): AWS_EC2 secret key
regions (list): List of AWS regions the AMI to be uploaded.
volume_types (list): List of supported volumes for the AMIs to
be created.
volume_via_s3 (bool): If `True`, the images are uploaded via s3 method
else using creating builder instances.
ex_virt_types (list): List of the supported virts for the AMIs to
be created.
push_notifications (bool): If `True` the messages will be pushed to
fedmsg, else skipped.
compose_id: id of the current compose in process.
"""
root_volume_size = AWS_ROOT_VOLUME_SIZE
published_images = []
if volume_types is None:
volume_types = AWS_VOLUME_TYPES
if regions is None:
regions = [AWS_BASE_REGION]
for image_url in image_urls:
# If the virt types is not provided then select the supported virt
# types from the image.
if ex_virt_types is None:
virt_types = get_virt_types_from_url(image_url)
else:
virt_types = ex_virt_types
try:
source = get_source_from_image(image_url)
if not source:
raise ValueError
image_architecture = get_file_arch(image_url)
uploader = EC2ImageUploader(compose_id=compose_id,
access_key=access_id,
secret_key=secret_key,
root_volume_size=root_volume_size,
image_architecture=image_architecture,
volume_via_s3=volume_via_s3,
push_notifications=push_notifications,
image_url=image_url)
publisher = EC2ImagePublisher(
compose_id=compose_id,
access_key=access_id,
secret_key=secret_key,
push_notifications=push_notifications,
image_url=image_url)
combinations = itertools_product(
*[regions, virt_types, volume_types])
for region, virt_type, volume_type in combinations:
uploader.set_region(region)
_log.debug('(uploader) Region is set to: %r' % region)
uploader.set_image_virt_type(virt_type)
_log.debug('(uploader) Virtualization type '
'is set to: %r' % virt_type)
image_name = get_image_name_from_image(image_url=image_url,
virt_type=virt_type,
region=region,
volume_type=volume_type)
uploader.set_image_name(image_name)
uploader.set_image_volume_type(volume_type)
_log.debug('(uploader) Volume type is set to: %r' %
volume_type)
uploader.set_availability_zone_for_region()
if push_notifications:
fedimg.messenger.notify(
topic='image.upload',
msg=dict(
image_url=image_url,
image_name=get_image_name_from_ami_name_for_fedmsg(
image_name),
destination=region,
service='EC2',
status='started',
compose=compose_id,
extra=dict(virt_type=virt_type,
vol_type=volume_type)))
image = uploader.create_image(source)
published_images.extend(
publisher.publish_images(
region_image_mapping=[(region, image.id)]))
except Exception as e:
_log.debug(e.message)
#TODO: Implement the clean up of the images if failed.
# uploader.clean_up(image_id=image.id, delete_snapshot=True)
shutil.rmtree(os.path.dirname(source))
return published_images