def __init__(self, args, gene_set):
self.args = args
self.gene_set = gene_set
self.biotypes = gene_set.biotypes
self.work_dir = os.path.join(args.workDir, gene_set.sourceGenome, gene_set.geneSet)
self.metrics_dir = os.path.join(self.work_dir, 'transMap_gene_set_metrics')
self.output_dir = os.path.join(args.outputDir, gene_set.sourceGenome, gene_set.geneSet)
self.jobtree_dir = os.path.join(args.jobTreeDir, gene_set.sourceGenome, gene_set.geneSet)
self.db = os.path.join(self.work_dir, 'comparativeAnnotator', 'classification.db')
self.target_genomes = frozenset(set(args.targetGenomes) - set([gene_set.sourceGenome]))
self.ordered_target_genomes = tuple([x for x in gene_set.orderedTargetGenomes if x in self.target_genomes])
self.query_cfg = QueryCfg(self)
self.query_target_cfgs = frozendict((genome, QueryTargetCfg(self, genome)) for genome in self.target_genomes)
self.tm_plots = frozendict((biotype, TransMapPlotCfg(self, self.query_cfg, self.query_target_cfgs, biotype))
for biotype in self.biotypes)
self.gene_set_plots = GeneSetPlotCfg(self, self.query_target_cfgs, self.metrics_dir, mode='transMap')
if args.augustus is True:
mode = 'AugustusTMR' if args.augustusHints is not None else 'AugustusTM'
self.aug_metrics_dir = os.path.join(self.work_dir, '{}_consensus_gene_set_metrics'.format(mode))
self.augustus_genomes = frozenset(set(args.augustusGenomes) - set([gene_set.sourceGenome]))
self.ordered_augustus_genomes = tuple([x for x in gene_set.orderedTargetGenomes
if x in self.augustus_genomes])
self.augustus_cfgs = frozendict((genome, AugustusCfg(self.query_target_cfgs[genome], genome,
mode, self.aug_metrics_dir))
for genome in self.augustus_genomes)
self.augustus_gene_set_plots = GeneSetPlotCfg(self, self.query_target_cfgs, self.aug_metrics_dir, mode)
def __init__(self, cfg, target_genome, mode, metrics_dir):
self.__dict__.update(vars(cfg)) # bring in all configurations from query-target
self.query_target_cfg = cfg
self.query_cfg = cfg.query_cfg
self.query_genome = cfg.query_genome
self.gene_set_name = cfg.gene_set_name
self.mode = mode
self.args = cfg.args
self.target_genome = target_genome
self.fasta = cfg.genome_fasta
self.chrom_sizes = cfg.chrom_sizes
self.transcript_fasta = cfg.transcript_fasta
# files that will be produced
self.augustus_gtf = os.path.join(cfg.work_dir, mode, target_genome + '.gtf')
self.augustus_gp = os.path.join(cfg.work_dir, mode, target_genome + '.gp')
self.augustus_bed = os.path.join(cfg.work_dir, mode, target_genome + '.bed')
self.tmr_transcript_fasta = os.path.join(cfg.work_dir, mode, target_genome + '.transcripts.fa')
self.vector_gp = os.path.join(cfg.work_dir, mode, target_genome + '.vector.gp')
# run augustus
tmr_jobtree = os.path.join(self.jobtree_dir, mode, self.target_genome)
self.tmr = TMRJobTree(self, tmr_jobtree)
# align augustus
align_jobtree = os.path.join(self.jobtree_dir, 'align' + mode, self.target_genome)
self.align = AlignAugustusJobTree(self, align_jobtree)
# comparative Annotator
aug_comp_ann_jobtree = os.path.join(self.jobtree_dir, 'augustusComparativeAnnotator', self.target_genome)
self.comp_ann = CompAnnJobTree(self, aug_comp_ann_jobtree, mode='augustus')
# consensus gene set
self.gene_set_dir = os.path.join(self.output_dir, '{}_consensus_gene_set'.format(mode), target_genome)
self.geneset_gps = frozendict((x, os.path.join(self.gene_set_dir, x + '.gp')) for x in self.biotypes)
self.combined_gp = os.path.join(self.gene_set_dir, 'combined.gp')
self.combined_gtf = os.path.join(self.gene_set_dir, 'combined.gtf')
self.geneset_gtfs = frozendict((x, os.path.join(self.gene_set_dir, x + '.gtf')) for x in self.biotypes)
self.metrics_dir = metrics_dir
self.pickled_metrics = os.path.join(self.metrics_dir, '{}.pickle'.format(target_genome))
def __init__(self, _head, _bindings={}, _source=None, _id=None, **kwargs):
self._head = _head
assert isinstance(_head, str) or isinstance(_head, unicode)
self._bindings = frozendict(_bindings, **kwargs)
for v in self._bindings.values():
assert(isinstance(v, Term))
if _source is None:
self._source = construction(self._head, self._bindings)
else:
self._source = _source
if _id is None:
self._id = hash((
self._head,
frozendict({k:v.id for k, v in self._bindings.iteritems()})
))
else:
self._id = _id
#TODO fix hash and id collisions (right now we just count on not running into any)
#this is non-trivial, since something might collide with a DB entry
#but it's also quite unlikely to cause trouble
self._hash = hash((
self._head,
self._bindings,
self._source
))
def freeze(cls, other):
"""
A frozen copy of a lex class
"""
return LexClass(frozendict(other.word_to_subclass.items()),
frozendict((k, frozenset(v)) for k, v in
other.subclass_to_words.items()))
def _google_maps_leg_costs(mode, places):
"""
Looks up distances and durations on Google Maps.
:param mode: A Google Maps mode, e.g. 'driving'.
:param places: An iterable of Places.
:return: A dict mapping each of 'duration' and 'distance' to
a frozendict mapping Place pairs to relevant values.
"""
for waypoint in places:
assert isinstance(waypoint, Place)
distance = dict()
duration = dict()
# Call Google Maps API
response = _client_().distance_matrix(origins=[p.address for p in places],
destinations=[p.address for p in places],
mode=mode,
units='metric')
# Verify and parse response
assert response['status'] == 'OK'
rows = response['rows']
assert len(rows) == len(places)
# Populate the dicts distance and duration
for row, origin in zip(rows, places):
row_elements = row['elements'] # There's also data about exact addresses used
assert len(row_elements) == len(places)
for element, destination in zip(row_elements, places):
assert element['status'] == 'OK'
duration[(origin, destination)] = element['duration']['value']
distance[(origin, destination)] = element['distance']['value']
# Construct and return the dict
return {'distance': frozendict(distance), 'duration': frozendict(duration)}
def __init__(self, callback,
args=None, kwargs=None, details_filter=None,
weak=False):
"""Initialize members
:param callback: callback function
:param details_filter: a callback that will be called before the
actual callback that can be used to discard
the event (thus avoiding the invocation of
the actual callback)
:param args: non-keyworded arguments
:type args: list/iterable/tuple
:param kwargs: key-value pair arguments
:type kwargs: dictionary
:param weak: whether the callback provided is referenced via a
weak reference or a strong reference
:type weak: bool
"""
self._uuid = uuidutils.generate_uuid()
self._callback = callback
self._details_filter = details_filter
self._weak = weak
if not args:
self._args = ()
else:
if not isinstance(args, tuple):
self._args = tuple(args)
else:
self._args = args
if not kwargs:
self._kwargs = frozendict()
else:
self._kwargs = frozendict(kwargs)
def it_encodes_frozendict(self):
test_dict = frozendict({
'pgdir': 'playground',
'services': ('default',),
'aliases': frozendict({
'default': ('')
}),
})
result = JSONEncoder(sort_keys=True, indent=4).encode(test_dict)
assert norm_trailing_whitespace_json(result) == '''\
def to_cache_value(self):
return {
'schema_id': self.schema_id,
'schema_json': frozendict(self.schema_json),
'topic_name': self.topic.name,
'base_schema_id': self.base_schema_id,
'status': self.status,
'primary_keys': self.primary_keys,
'note': frozendict(self.note.to_cache_value()) if self.note is not None else None,
'created_at': self.created_at,
'updated_at': self.updated_at
}
def __init__(self, cfg, target_genome):
# input arguments
self.query_cfg = cfg.query_cfg
self.mode = 'transMap'
self.args = cfg.args
self.hal = self.args.hal
self.ref_fasta = cfg.query_cfg.ref_fasta
self.ref_two_bit = cfg.query_cfg.ref_two_bit
self.ref_sizes = cfg.query_cfg.ref_sizes
self.ref_psl = cfg.query_cfg.psl
self.annotation_gp = cfg.gene_set.genePred
self.transcript_fasta = cfg.query_cfg.transcript_fasta
self.flat_transcript_fasta = self.transcript_fasta + '.flat'
self.target_genome = target_genome
self.query_genome = cfg.gene_set.sourceGenome
self.attrs_tsv = cfg.gene_set.attributesTsv
self.gene_set_name = cfg.gene_set.geneSet
self.db = cfg.db
self.jobtree_dir = cfg.jobtree_dir
self.output_dir = cfg.output_dir
self.work_dir = cfg.work_dir
self.metrics_dir = cfg.metrics_dir
self.biotypes = cfg.biotypes
# genome files
self.genome_work_dir = os.path.join(cfg.work_dir, 'genome_files')
self.genome_fasta = os.path.join(self.genome_work_dir, target_genome + '.fa')
self.genome_two_bit = os.path.join(self.genome_work_dir, target_genome + '.2bit')
self.chrom_sizes = os.path.join(self.genome_work_dir, target_genome + '.chrom.sizes')
self.flat_fasta = self.genome_fasta + '.flat'
# to make QueryCfg and TargetQueryCfg compatible for production of genome files
self.genome = self.target_genome
# chaining config (passed to jobTree)
self.chaining = ChainingCfg(self)
# transMap
self.tm_work_dir = os.path.join(cfg.work_dir, 'transMap')
self.psl = os.path.join(self.tm_work_dir, target_genome + '.psl')
self.gp = os.path.join(self.tm_work_dir, target_genome + '.gp')
self.bed = os.path.join(self.tm_work_dir, target_genome + '.bed')
# comparativeAnnotator
comp_ann_jobtree = os.path.join(self.jobtree_dir, 'comparativeAnnotator', self.target_genome)
self.comp_ann = CompAnnJobTree(self, comp_ann_jobtree, mode=self.mode)
# final gene set
self.gene_set_dir = os.path.join(self.output_dir, 'transMap_gene_set', target_genome)
self.geneset_gps = frozendict((x, os.path.join(self.gene_set_dir, x + '.gp')) for x in self.biotypes)
self.combined_gp = os.path.join(self.gene_set_dir, 'combined.gp')
self.combined_gtf = os.path.join(self.gene_set_dir, 'combined.gtf')
self.geneset_gtfs = frozendict((x, os.path.join(self.gene_set_dir, x + '.gtf')) for x in self.biotypes)
self.pickled_metrics = os.path.join(self.metrics_dir, '{}.pickle'.format(target_genome))
self.filter_chroms = self.args.filterChroms
def test_get_context_from_cmdln(t_config):
path = os.path.join(os.path.dirname(__file__), "data", "good.json")
c = deepcopy(dict(DEFAULT_CONFIG))
with open(path) as fh:
c.update(json.load(fh))
expected_creds = frozendict(c['credentials'])
del(c['credentials'])
expected_config = frozendict(config.apply_product_config(c))
def noop(*args, **kwargs):
pass
context, credentials = config.get_context_from_cmdln([path])
assert credentials == expected_creds
assert context.config == expected_config
def discrete_finite_max(pmf, leave_unoptimized=None):
neg_log_p = pmf.Param['NegLogP']
if leave_unoptimized:
comparison_bases = {}
conditioned_and_unoptimized_vars = set(pmf.Cond) | set(leave_unoptimized)
for var_names_and_values___frozen_dict in neg_log_p:
comparison_basis = {}
for var in (set(var_names_and_values___frozen_dict) & conditioned_and_unoptimized_vars):
comparison_basis[var] = var_names_and_values___frozen_dict[var]
comparison_bases[var_names_and_values___frozen_dict] = frozendict(comparison_basis)
else:
comparison_bases = pmf.CondInstances
neg_log_mins = {}
for var_names_and_values___frozen_dict, func_value in neg_log_p.items():
comparison_basis = comparison_bases[var_names_and_values___frozen_dict]
if comparison_basis in neg_log_mins:
neg_log_mins[comparison_basis] = min(neg_log_mins[comparison_basis], func_value)
else:
neg_log_mins[comparison_basis] = func_value
optims = {}
for var_names_and_values___frozen_dict, func_value in neg_log_p.items():
if func_value <= neg_log_mins[comparison_bases[var_names_and_values___frozen_dict]]:
optims[var_names_and_values___frozen_dict] = func_value
return DiscreteFinitePMF(var_names_and_syms=pmf.Vars.copy(), p_or_neg_log_p=optims, p=False,
cond=pmf.Cond.copy(), scope=pmf.Scope.copy())
def __init__(self, cfg, jobtree_dir):
args = cfg.args
self.__dict__.update(vars(args.jobTreeOptions))
self.defaultMemory = 8 * 1024 ** 3
self.jobTree = jobtree_dir
tm_2_hints_script = 'submodules/comparativeAnnotator/augustus/transMap2hints.pl' # TODO: don't hardcode
assert os.path.exists(tm_2_hints_script)
tm_2_hints_params = ("--ep_cutoff=0 --ep_margin=12 --min_intron_len=40 --start_stop_radius=5 --tss_tts_radius=5"
" --utrend_cutoff=6 --in=/dev/stdin --out=/dev/stdout")
self.tm_2_hints_cmd = " ".join([tm_2_hints_script, tm_2_hints_params])
# can we run TMR or just TM?
self.cfgs = {1: "submodules/comparativeAnnotator/augustus/extrinsic.ETM1.cfg"}
if args.augustusHints is not None:
self.cfgs[2] = "submodules/comparativeAnnotator/augustus/extrinsic.ETM2.cfg"
self.cfgs = frozendict(self.cfgs)
assert all([os.path.exists(x) for x in self.cfgs.itervalues()])
self.augustus_bin = 'submodules/augustus/bin/augustus'
assert os.path.exists(self.augustus_bin)
self.padding = 20000
self.max_gene_size = 3000000
self.hints_db = args.augustusHints
self.out_gtf = cfg.augustus_gtf
self.genome = cfg.target_genome
self.fasta = cfg.fasta
self.chrom_sizes = cfg.chrom_sizes
self.input_gp = cfg.vector_gp
def _unwrap_object(obj, nested=False):
obj_type = obj['_type']
value = obj.get('value', None)
if obj_type == 'none':
return None
if obj_type in ('bool', 'str', 'int', 'float'):
return locate(obj_type)(value)
if obj_type == 'decimal':
return Decimal(value)
if obj_type == 'datetime':
return datetime.datetime.utcfromtimestamp(value)
if obj_type in ('list', 'dict'):
return locate(obj_type)(unwraps(value)) if nested else value
if obj_type in ('set', 'frozenset', 'tuple'):
if nested:
value = unwraps(value)
return locate(obj_type)(value)
if obj_type == 'frozendict':
if nested:
value = unwraps(value)
return frozendict(value)
raise ValueError(repr(obj) + ' cannot be decoded.')
def _to_hashable_value(value):
"""Return a frozen, hashable copy of "value".
mixed value - The pseudo-hashable value, as defined in the comments
for cached_generator.
return object - The frozen, hashable copy.
"""
if isinstance(value, (list, tuple)):
return tuple(list([_to_hashable_value(element) for element in value]))
elif isinstance(value, (frozenset, set)):
return frozenset(
list([_to_hashable_value(element) for element in value]))
elif (isinstance(
value,
(dict, frozendict.frozendict, frozendict.FrozenOrderedDict))):
if (isinstance(
value,
(collections.OrderedDict, frozendict.FrozenOrderedDict))):
hashable_value = collections.OrderedDict()
else:
hashable_value = {}
for key, sub_value in value.iteritems():
hashable_value[key] = _to_hashable_value(sub_value)
if isinstance(value, collections.OrderedDict):
return frozendict.frozendict(hashable_value)
else:
return frozendict.FrozenOrderedDict(hashable_value)
else:
return value
def op(self, op=mul, other=None, r=False, **kwargs):
math_dict = MathDict()
if hasattr(other, 'keys'):
for item_0, item_1 in product(self.items(), other.items()):
vars_and_values_0___frozen_dict, func_value_0 = item_0
vars_and_values_1___frozen_dict, func_value_1 = item_1
same_vars_same_values = True
for var in (set(vars_and_values_0___frozen_dict) & set(vars_and_values_1___frozen_dict)):
if vars_and_values_0___frozen_dict[var] != vars_and_values_1___frozen_dict[var]:
same_vars_same_values = False
break
if same_vars_same_values:
if r:
value = op(func_value_1, func_value_0, **kwargs)
else:
value = op(func_value_0, func_value_1, **kwargs)
math_dict[frozendict(set(vars_and_values_0___frozen_dict.items()) |
set(vars_and_values_1___frozen_dict.items()))] = value
elif other is None:
for k, v in self.items():
math_dict[k] = op(v, **kwargs)
elif r:
for k, v in self.items():
math_dict[k] = op(other, v, **kwargs)
else:
for k, v in self.items():
math_dict[k] = op(v, other, **kwargs)
return math_dict
def genReport(self, data):
"""deduplicate gathered case data"""
dupecount = 0
output = dict()
for change in data["records"]:
for line in change["FeedTrackedChanges"]["records"]:
if line is not None:
if line["NewValue"] in (
"Ready For Close",
"Closed",
"Cancelled",
"Closed as Duplicate"):
caseid = nestedGet(["Parent", "CaseNumber"], change)
changedate = dateparser.parse(change["CreatedDate"])
# need to account for >1 escalation per case
if caseid in output:
# chronological order - latest gets it
if output[caseid]["Date"] > changedate:
dupecount += 1
continue
if nestedGet(["Parent", "Cancel_Effective_Date__c"],
change) is not None:
teardown = True
else:
teardown = False
output[caseid] = frozendict(
Name=nestedGet(["CreatedBy", "Name"], change),
Case=caseid,
Status=line["NewValue"],
Teardown=teardown,
Date=changedate)
print "Found and removed", dupecount, "cases handled more than once."
print "Credit for duplicates given to latest resolver."
return output
def compile_dags(rules, forest):
trees = split_dags(forest)
prev_schedules = defaultdict(list)
prev_schedules[frozendict()] = []
for component in trees:
schedules = {}
for out_resources in all_out_resources(component, rules):
best = [None, None]
def update_best(sched):
cost = sum(match.rule.cost for match in sched)
if not best[0] or best[1] > cost:
best[0] = (sched, externs)
best[1] = cost
for externs in prev_schedules:
matches = {}
for expr in component:
matches.update(find_matches(rules, expr, externs))
sched = compile_exprs(matches, component, out_resources, frozenset(), set(), {}, externs)
if sched != None:
update_best(sched)
if not best[0]:
return None
sched, externs = best[0]
schedules[out_resources] = prev_schedules[externs] + sched
prev_schedules = schedules
best_externs = min(prev_schedules, key = lambda r: sum(match.rule.cost for match in prev_schedules[r]))
return prev_schedules[best_externs]
def _decode_state_dict(input):
"""Decodes a state dict encoded using `_encode_state_dict` above
"""
if input is None:
return None
return frozendict({(etype, state_key,): v for etype, state_key, v in input})
def optim(self, max_or_min=max, leave_unoptimized=None):
if max_or_min is max:
comp = ge
else:
comp = le
if leave_unoptimized:
comparison_bases = {}
conditioned_and_unoptimized_vars = set(self.Cond) | set(leave_unoptimized)
for vars_and_values___frozen_dict in self.Mapping:
comparison_basis = {}
for var in (set(vars_and_values___frozen_dict) & conditioned_and_unoptimized_vars):
comparison_basis[var] = vars_and_values___frozen_dict[var]
comparison_bases[vars_and_values___frozen_dict] = frozendict(comparison_basis)
else:
comparison_bases = self.CondInstances
optim_values = {}
for vars_and_values___frozen_dict, func_value in self.Mapping.items():
comparison_basis = comparison_bases[vars_and_values___frozen_dict]
if comparison_basis in optim_values:
optim_values[comparison_basis] = max_or_min(optim_values[comparison_basis], func_value)
else:
optim_values[comparison_basis] = func_value
optims = {}
for vars_and_values___frozen_dict, func_value in self.Mapping.items():
if comp(func_value, optim_values[comparison_bases[vars_and_values___frozen_dict]]):
optims[vars_and_values___frozen_dict] = func_value
return MathFunc(self.Vars.copy(), optims, cond=self.Cond.copy(), scope=self.Scope.copy())
def __init__(self, var_names_and_syms={}, mapping={}, param={}, cond={}, scope={}, compile=False):
if not hasattr(self, 'Vars'):
self.Vars = var_names_and_syms # {var_name: var_symbol} dict
if not hasattr(self, 'Param'):
self.Param = param
self.Cond = cond # {var_name: var_value} dict, var_value can be None if conditioning is generic
self.Scope = dict.fromkeys(set(var_names_and_syms) - set(cond))
vars_with_fixed_scope_values = {} # to keep track of scope variables with fixed values (i.e. points in space)
for var, value in scope.items():
if (var in self.Scope) and (value is not None):
self.Scope[var] = value # "points-in-space"
vars_with_fixed_scope_values[var] = value
s0 = set(vars_with_fixed_scope_values.items())
if hasattr(mapping, 'keys'):
self.Mapping = MathDict()
self.CondInstances = {}
for vars_and_values___frozen_dict, func_value in mapping.items():
if set(vars_and_values___frozen_dict.items()) >= s0:
self.Mapping[vars_and_values___frozen_dict] = func_value
condition_instance = {}
for var in (set(vars_and_values___frozen_dict) & set(cond)):
condition_instance[var] = vars_and_values___frozen_dict[var]
self.CondInstances[vars_and_values___frozen_dict] = frozendict(condition_instance)
else:
self.Mapping = mapping
if not hasattr(self, 'CompyledFunc'):
self.CompyledFunc = None
if compile:
self.compile()
def marg(self, *marginalized_vars, **kwargs):
itself = lambda x: x
if 'transf' in kwargs:
transf_func = kwargs['transf']
else:
transf_func = itself
if 'reduce_func' in kwargs:
reduce_func = kwargs['reduce_func']
else:
reduce_func = add
if 'rev_transf' in kwargs:
rev_transf_func = kwargs['rev_transf']
else:
rev_transf_func = itself
var_names_and_symbols___dict = self.Vars.copy() # just to be careful
scope = self.Scope.copy() # just to be careful
mapping = self.Mapping.copy() # just to be careful
for marginalized_var in marginalized_vars:
del var_names_and_symbols___dict[marginalized_var]
del scope[marginalized_var]
d = {}
for vars_and_values___frozen_dict, func_value in mapping.items():
marginalized_var_value = vars_and_values___frozen_dict[marginalized_var]
fd = frozendict(set(vars_and_values___frozen_dict.items()) -
{(marginalized_var, marginalized_var_value)})
if fd in d:
d[fd] = reduce_func(d[fd], transf_func(func_value))
else:
d[fd] = transf_func(func_value)
mapping = {k: rev_transf_func(v) for k, v in d.items()}
return MathFunc(var_names_and_symbols___dict, mapping, cond=self.Cond.conds(), scope=scope)
def shift_time_subscripts(obj, t, *matrix_symbols_to_shift):
if isinstance(obj, MathDict):
return MathDict({shift_time_subscripts(key, t): shift_time_subscripts(value, t)
for key, value in obj.items()})
elif isinstance(obj, frozendict):
return frozendict({shift_time_subscripts(key, t): shift_time_subscripts(value, t)
for key, value in obj.items()})
elif isinstance(obj, tuple):
if len(obj) == 2 and not(isinstance(obj[0], (int, float))) and isinstance(obj[1], int):
return shift_time_subscripts(obj[0], t), obj[1] + t
else:
return tuple(shift_time_subscripts(item, t) for item in obj)
elif isinstance(obj, list):
return [shift_time_subscripts(item, t) for item in obj]
elif isinstance(obj, set):
return {shift_time_subscripts(item, t) for item in obj}
elif isinstance(obj, dict):
return {shift_time_subscripts(key, t): shift_time_subscripts(value, t) for key, value in obj.items()}
elif isinstance(obj, MatrixSymbol):
args = obj.args
if isinstance(args[0], tuple):
return MatrixSymbol(shift_time_subscripts(args[0], t), args[1], args[2])
else:
return deepcopy(obj)
elif is_non_atomic_sympy_expr(obj):
return obj.xreplace({matrix_symbol: shift_time_subscripts(matrix_symbol, t)
for matrix_symbol in matrix_symbols_to_shift})
else:
return deepcopy(obj)
def backward_factor(self, t___list, observations___dict={}, max_t=0):
T = max(max(observations___dict.keys()), max_t)
if isinstance(t___list, int):
t = t___list
if t == T:
state_var_symbol = {(self.state_var, t): self.observation_pdf(t).Vars[(self.state_var, t)]}
if self.observation_pdf_template.is_discrete_finite():
var_values___frozen_dicts = self.observation_pdf(t).Params['NegLogP'].keys()
state_var_domain =\
set(frozendict({(self.state_var, t): var_values___frozen_dict[(self.state_var, t)]})
for var_values___frozen_dict in var_values___frozen_dicts)
return OnePMF(var_names_and_syms=state_var_symbol, var_names_and_values=state_var_domain,
cond={(self.state_var, t): None})
else:
return OnePDF(cond={(self.observation_var, t): None})
else:
b = self.transition_pdf(t + 1) * self.observation_pdf(t + 1)
if (t + 1) in observations___dict:
b = b.at({(self.observation_var, t + 1): observations___dict[t + 1]})
b = (b * self.backward_factor(t + 1, observations___dict))\
.marg((self.state_var, t + 1))
return b
elif isinstance(t___list, (list, range, tuple)):
d = {}
t = T
state_var_symbol = {(self.state_var, t): self.observation_pdf(t).Vars[(self.state_var, t)]}
if self.observation_pdf_template.is_discrete_finite():
var_values___frozen_dicts = self.observation_pdf(t).Param['NegLogP'].keys()
state_var_domain =\
set(frozendict({(self.state_var, t): var_values___frozen_dict[(self.state_var, t)]})
for var_values___frozen_dict in var_values___frozen_dicts)
b = {t: OnePMF(var_names_and_syms=state_var_symbol, var_names_and_values=state_var_domain,
cond={(self.state_var, t): None})}
else:
b = {t: OnePDF(cond={(self.observation_var, t): None})}
if t in t___list:
d[t] = b[t]
for t in reversed(range(min(t___list), T)):
b[t] = self.transition_pdf(t + 1) * self.observation_pdf(t + 1)
if (t + 1) in observations___dict:
b[t] = b[t].at({(self.observation_var, t + 1): observations___dict[t + 1]})
b[t] = (b[t] * b[t + 1])\
.marg((self.state_var, t + 1))
if t in t___list:
d[t] = b[t]
return d
def make_observations_joinable(daily_observations):
return (
frozendict({
"Date": daily_observations["Date"],
"WBAN": daily_observations["WBAN"],
}),
daily_observations
)
def to_hashable(self, x):
if type(x) == dict:
return frozendict(x)
elif type(x) == list:
#return set((self.to_hashable(i) for i in x))
return frozenset(x)
else:
return x
def all_out_resources(exprs, rules):
resources = {rule.resource for rule in rules} - {'m'}
memory_cells = islice(repeat('m'), len(exprs))
for i in xrange(len(exprs) + 1):
for resource_list in permutations(resources, i):
all_resource_lists = permutations(chain(resource_list, repeat('m', len(exprs) - i)))
for all_resource_list in all_resource_lists:
yield frozendict({expr: resource for (expr, resource) in zip(exprs, all_resource_list)})
def make(head, bindings=None, **kwargs):
if bindings is None:
bindings = {}
bindings = frozendict(bindings, **kwargs)
return quoted_term(
head=quote(head),
#can't use quote because the values of bindings are already quotations
bindings=term.Term.from_dict_of_str(bindings)
)
def test_backrelate(self):
class A(DataObject):
pass
@backrelate({'b':(A,True)})
class B(DataObject):
pass
class C(B):
pass
class D(A):
pass
eq_(B.relationships, frozendict({}))
eq_(A.relationships, frozendict({'b':(B,True)}))
eq_(C.relationships, frozendict({}))
eq_(D.relationships, frozendict({'b':(B,True)}))
def bindings(self):
return frozendict(
head=quote(self.represents.head),
bindings=quote(self.represents.bindings),
_modifier=properties.both(
has_id(quote(self.represents.id)),
has_source(quote(self.represents.source))
)
)
def __init__(self, state, state_group, prev_group=None, delta_ids=None):
self.state = frozendict(state)
self.state_group = state_group
self.prev_group = prev_group
self.delta_ids = frozendict(delta_ids) if delta_ids is not None else None
# The `state_id` is a unique ID we generate that can be used as ID for
# this collection of state. Usually this would be the same as the
# state group, but on worker instances we can't generate a new state
# group each time we resolve state, so we generate a separate one that
# isn't persisted and is used solely for caches.
# `state_id` is either a state_group (and so an int) or a string. This
# ensures we don't accidentally persist a state_id as a stateg_group
if state_group:
self.state_id = state_group
else:
self.state_id = _gen_state_id()
par_dict={
tau_leaf: 0.0027,
tau_wood: 5e-05,
tau_root: 0.002,
tau_excessC: 0.05,
tau_litter: 0.029,
tau_cwd: 0.001,
tau_soil: 1e-04,
Q_h: 1.4,
m_resp_frac: 0.5,
DON_leach_prop: 0.0015,
leach_rate: 0.00001 # day^{-1}
,
nitr_rate: 0.0001 # day^{-1}
},
func_dict=frozendict({})
# state_var_units= gC*m^{-2}
# time_unit=day
)
in_fl_c = carbon_in_fluxes_by_symbol_1(c_in_t, xc)
internal_fl_c = carbon_internal_fluxes_by_symbol_1(A_c, xc)
out_fl_c = carbon_out_fluxes_by_symbol_1(A_c, xc)
in_fl_n = {N_labile: U_Nfix, N_NH4: Ndep_NH4, N_NO3: Ndep_NO3}
out_fl_n = {N_soil: L_DON, N_NH4: U_NH4, N_NO3: U_NO3 + L_NO3}
internal_fl_n = {
(N_bud, N_leaf): a_budN2leaf,
(N_bud, N_labile): a_budN2Ramain,
(N_labile, N_bud): a_budN,
(N_leaf, N_litter): t_leafN,
(N_leaf, N_labile): t_retransN,
def freeze(item):
item['fields'] = frozendict(item['fields'])
return frozendict(item)
class SerializerForm(forms.Form):
"""
Sync api serializer form errors to django form errors.
"""
task_id = None
action = None
_serializer = None
_api_method = frozendict({
'create': 'POST',
'update': 'PUT',
'delete': 'DELETE',
'get': 'GET'
})
_api_call = NotImplemented # Set in descendant class
_api_response = None
_exclude_fields = frozenset()
_custom_fields = frozendict()
_custom_field_opts = frozendict()
_custom_widgets = frozendict()
_custom_widget_attrs = frozendict()
_ignore_empty_fields = frozenset()
_field_text_class = 'narrow input-transparent'
_field_checkbox_class = 'normal-check'
_field_select_class = 'narrow input-select2'
_field_readonly_class = 'uneditable-input'
def __init__(self, request, obj, *args, **kwargs):
self._request = request
self._obj = obj
self._read_only_fields = set()
init = kwargs.pop('init', False)
# Initial data are useful only for updates, or enabled manually by param
if (obj and request.POST.get('action', None) == 'update') or init:
kwargs['initial'] = self._initial_data(request, obj)
# Parent constructor
super(SerializerForm, self).__init__(*args, **kwargs)
# Copy serializer fields
if self._serializer:
for name, field in iteritems(self._serializer.base_fields):
field_not_defined = name not in self.fields
field_not_excluded = name not in self._exclude_fields
if field_not_defined and field_not_excluded:
self.fields[name] = self._serializer_field(name, field)
# Set fancy placeholder
for key, field in self.fields.items():
field.widget.attrs['placeholder'] = self._get_placeholder(
field, key)
def _get_placeholder(self, field, field_name, default=''):
# Python circular imports
from gui.utils import tags_to_string
try:
value = self.initial[field_name]
except KeyError:
value = field.widget.attrs.get('placeholder', default)
else:
if isinstance(value, (list, tuple)):
value = tags_to_string(value)
if value is None:
return ''
return text_type(value).replace('\r\n', ' ').replace('\n', ' ')
# noinspection PyMethodMayBeStatic
def _build_field(self, name, serializer_field, form_field_class,
**form_field_options):
"""Process converted field information and returns form field.
Suitable for overriding in descendant classes.
"""
return form_field_class(**form_field_options)
def _serializer_field(self, name, field):
"""
Convert serializer field to django form field.
"""
opts = {
'label': field.label,
'help_text': field.help_text,
'required': field.required,
}
if isinstance(
field,
(fields.ChoiceField, fields.IntegerChoiceField, RelatedField)):
field_class = forms.ChoiceField
widget_class = forms.Select
widget_attrs = {'class': self._field_select_class}
opts['required'] = False
try:
opts['choices'] = field.choices
except AttributeError:
pass
try:
if isinstance(field, fields.IntegerChoiceField) or isinstance(
opts['choices'][0][0], int):
field_class = forms.TypedChoiceField
opts['coerce'] = int
opts['empty_value'] = None
except (KeyError, IndexError, TypeError):
pass
elif isinstance(field, fields.BooleanField):
field_class = forms.BooleanField
widget_class = forms.CheckboxInput
widget_attrs = {'class': self._field_checkbox_class}
opts['required'] = False
else:
widget_class = forms.TextInput
widget_attrs = {'class': self._field_text_class}
field_class = forms.CharField
if isinstance(field, fields.IntegerField):
field_class = forms.IntegerField
widget_class = NumberInput
if field.help_text and '(MB)' in field.help_text:
widget_class = ByteSizeInput
widget_attrs['class'] += ' ' + 'input-mbytes'
elif isinstance(field, fields.BaseArrayField):
if isinstance(field, fields.IntegerArrayField):
field_class = IntegerArrayField
else:
field_class = ArrayField
widget_class = ArrayWidget
elif isinstance(field, fields.BaseDictField):
field_class = DictField
widget_class = DictWidget
if field.read_only:
widget_attrs['class'] += ' ' + self._field_readonly_class
if opts['required']:
widget_attrs['required'] = 'required'
if field.read_only:
widget_attrs['disabled'] = 'disabled'
self._read_only_fields.add(name)
field_class = self._custom_fields.get(name, field_class)
widget_class = self._custom_widgets.get(name, widget_class)
widget_attrs.update(self._custom_widget_attrs.get(name, {}))
opts.update(self._custom_field_opts.get(name, {}))
opts['widget'] = widget_class(attrs=widget_attrs)
return self._build_field(name, field, field_class, **opts)
@staticmethod
def _blank(value):
"""Return empty string instead of None"""
if not value:
return ''
return value
@staticmethod
def _null(value):
"""Return None instead of empty string"""
if not value:
return None
return value
def _initial_data(self, request, obj):
"""Data initialized from DB model object"""
if self._serializer:
if hasattr(self._serializer,
'_model_'): # InstanceSerializer(request, instance)
# noinspection PyCallingNonCallable
return self._serializer(self._request, obj).data
else: # class Serializer(instance)
# noinspection PyCallingNonCallable
return self._serializer(obj).data
return {}
def _input_data(self):
"""Data collected from form"""
return self.cleaned_data
def _final_data(self, data=None):
"""Data that are send to API for validation (in create _input_data, in update _has_changed data)"""
if data is None:
data = self._input_data()
return data
def _has_changed(self):
"""Parse _input_data from form and compare with _initial_data from DB and return data that has changed"""
ret = {}
for key, val in self._input_data().items():
if key in self._read_only_fields:
continue
if key in self._ignore_empty_fields and not val:
logger.debug(
'SerializerForm._has_changed [%s]: %s (%s) is empty and will be ignored',
key, val, type(val))
continue
try:
initial_val = self.initial[key]
except KeyError:
ret[key] = val
logger.debug(
'SerializerForm._has_changed [%s]: %s (%s) is missing in initial data',
key, val, type(val))
else:
if initial_val != val:
ret[key] = val
logger.debug(
'SerializerForm._has_changed [%s]: %s (%s) != %s (%s)',
key, initial_val, type(initial_val), val, type(val))
return ret
def _set_api_task_id(self, data):
"""Set task_id from TaskResponse"""
# noinspection PyBroadException
try:
self.task_id = data['task_id']
except:
pass
def _set_custom_api_errors(self, errors):
pass
def _add_error(self, field_name, error):
if field_name not in self._errors:
self._errors[field_name] = self.error_class()
if isinstance(error, (list, tuple)):
self._errors[field_name].extend(error)
else:
self._errors[field_name].append(error)
def _set_api_errors(self, data):
# errors is a dict error output from API
if not data or not isinstance(data, dict):
return None
errors = data.get('result', data)
if isinstance(errors, dict): # Classic serializer error task output
# Pair API errors to Django form errors
for field in self.fields:
if field in errors:
self._add_error(field, errors[field]) # should be lazy
try:
del self.cleaned_data[field]
except KeyError:
pass
if 'non_field_errors' in errors:
self._add_error(NON_FIELD_ERRORS,
errors['non_field_errors']) # should be lazy
elif 'detail' in errors:
self._add_error(NON_FIELD_ERRORS,
ugettext(errors['detail'])) # should be noop
else: # More serious api error
if isinstance(
errors,
list): # Maybe we have errors from multiple serializers
for err in errors:
self._set_api_errors(err)
elif errors:
self._add_error(NON_FIELD_ERRORS, errors)
self._set_custom_api_errors(errors)
# noinspection PyUnusedLocal
@classmethod
def api_call(cls, action, obj, request, args=(), data=()):
method = cls._api_method[action]
logger.info('Calling API view %s %s(%s, data=%s) by user %s in DC %s',
method, cls._api_call.__name__, args, data, request.user,
request.dc)
return call_api_view(request,
method,
cls._api_call.__func__,
*args,
data=dict(data),
log_response=True)
def save(self, action=None, args=()):
# For security reasons you can limit action from view
if not action:
action = self.data.get('action')
# Save action
self.action = action
# For security reasons data must have only cleaned_data for this form
if action == 'update':
data = self._final_data(self._has_changed())
if not data: # Nothing changed
return 204
else:
data = self._final_data()
# Saving socket.io session ID. Used when sending messages about hostname change, so we don't send useless
# message to this user
self._request.siosid = self.data.get('siosid', None)
# Call API function (also updates tasklog)
res = self.api_call(action,
self._obj,
self._request,
args=args,
data=data)
self._api_response = res.data
if res.status_code in (200, 201):
self._set_api_task_id(res.data)
else:
self._set_api_errors(res.data)
return res.status_code
def set_error(self, key, value):
"""Set custom error"""
if self._errors is None:
self._errors = {}
self._add_error(key, value)
sign_omnija
from signingscript.exceptions import SigningServerError
from signingscript.utils import is_autograph_signing_format
log = logging.getLogger(__name__)
FORMAT_TO_SIGNING_FUNCTION = frozendict({
# TODO: Remove the next item (in favor of the regex one), once Focus is migrated
"autograph_focus": sign_jar,
"autograph_apk_.+": sign_jar,
"autograph_hash_only_mar384(:\\w+)?": sign_mar384_with_autograph_hash,
"autograph_stage_mar384(:\\w+)?": sign_mar384_with_autograph_hash,
"gpg": sign_gpg,
"autograph_gpg": sign_gpg_with_autograph,
"jar": sign_jar,
"focus-jar": sign_jar,
"macapp": sign_macapp,
"osslsigncode": sign_signcode,
"sha2signcode": sign_signcode,
# sha2signcodestub uses a generic sign_file
"signcode": sign_signcode,
"widevine": sign_widevine,
"autograph_widevine": sign_widevine,
"autograph_omnija": sign_omnija,
"default": sign_file,
})
# task_cert_type {{{1
def task_cert_type(context):
"""Extract task certificate type.
def upload_image(
self: api_client.Api,
body: typing.Union[SchemaForRequestBodyMultipartFormData,
Unset] = unset,
path_params: RequestPathParams = frozendict(),
content_type: str = 'multipart/form-data',
accept_content_types: typing.Tuple[str] = _all_accept_content_types,
stream: bool = False,
timeout: typing.Optional[typing.Union[int, typing.Tuple]] = None,
skip_deserialization: bool = False,
) -> typing.Union[ApiResponseFor200,
api_client.ApiResponseWithoutDeserialization]:
"""
uploads an image
:param skip_deserialization: If true then api_response.response will be set but
api_response.body and api_response.headers will not be deserialized into schema
class instances
"""
self._verify_typed_dict_inputs(RequestPathParams, path_params)
_path_params = {}
for parameter in (request_path_pet_id, ):
parameter_data = path_params.get(parameter.name, unset)
if parameter_data is unset:
continue
serialized_data = parameter.serialize(parameter_data)
_path_params.update(serialized_data)
_headers = HTTPHeaderDict()
# TODO add cookie handling
if accept_content_types:
for accept_content_type in accept_content_types:
_headers.add('Accept', accept_content_type)
_fields = None
_body = None
if body is not unset:
serialized_data = request_body_body.serialize(body, content_type)
_headers.add('Content-Type', content_type)
if 'fields' in serialized_data:
_fields = serialized_data['fields']
elif 'body' in serialized_data:
_body = serialized_data['body']
response = self.api_client.call_api(
resource_path=_path,
method=_method,
path_params=_path_params,
headers=_headers,
fields=_fields,
body=_body,
auth_settings=_auth,
stream=stream,
timeout=timeout,
)
if skip_deserialization:
api_response = api_client.ApiResponseWithoutDeserialization(
response=response)
else:
response_for_status = _status_code_to_response.get(
str(response.status))
if response_for_status:
api_response = response_for_status.deserialize(
response, self.api_client.configuration)
else:
api_response = api_client.ApiResponseWithoutDeserialization(
response=response)
if not 200 <= response.status <= 299:
raise exceptions.ApiException(api_response=api_response)
return api_response
DEFAULT_CONFIG = frozendict({
"taskcluster_root_url": "https://taskcluster.net",
# Worker identification
"provisioner_id": "test-dummy-provisioner",
"worker_group": "test-dummy-workers",
"worker_type": "dummy-worker-myname",
"worker_id": os.environ.get("SCRIPTWORKER_WORKER_ID", "dummy-worker-myname1"),
"credentials": frozendict({
"clientId": "...",
"accessToken": "...",
"certificate": "...",
}),
# Worker log settings
"log_datefmt": "%Y-%m-%dT%H:%M:%S",
"log_fmt": "%(asctime)s %(levelname)8s - %(message)s",
"watch_log_file": False,
# intervals are expressed in seconds
"task_max_timeout": 60 * 20,
"reclaim_interval": 300,
"poll_interval": 10,
"sign_key_timeout": 60 * 2,
"reversed_statuses": frozendict({
-11: STATUSES['intermittent-task'],
-15: STATUSES['intermittent-task'],
}),
# Report this status on max_timeout. `intermittent-task` will rerun the
# task automatically. `internal-error` or other will require manual
# intervention.
"task_max_timeout_status": STATUSES['intermittent-task'],
"invalid_reclaim_status": STATUSES['intermittent-task'],
"task_script": ("bash", "-c", "echo foo && sleep 19 && exit 1"),
"verbose": True,
# Task settings
"work_dir": "...",
"log_dir": "...",
"artifact_dir": "...",
"task_log_dir": "...", # set this to ARTIFACT_DIR/public/logs
"artifact_upload_timeout": 60 * 20,
"aiohttp_max_connections": 15,
# chain of trust settings
"sign_chain_of_trust": True,
"verify_chain_of_trust": False, # TODO True
"verify_cot_signature": False,
"cot_job_type": "unknown", # e.g., signing
"cot_product": "firefox",
"cot_version": 3,
"min_cot_version": 2,
"max_chain_length": 20,
# Calls to Github API are limited to 60 an hour. Using an API token allows to raise the limit to
# 5000 per hour. https://developer.github.com/v3/#rate-limiting
"github_oauth_token": "",
# ed25519 settings
"ed25519_private_key_path": "...",
"ed25519_public_keys": frozendict({
"docker-worker": tuple([
'J+PAKmq3jkS2uCpBk5WU2ycrnTFPwZujJT4OHAxm38I=',
]),
"generic-worker": tuple([
'6UPrVTyw0EPQV7bCEMXo+5jNR4clbK55JWG74bBJHZQ=',
]),
"scriptworker": tuple([
'DaEKQ79ZC/X+7O8zwm8iyhwTlgyjRSi/TDd63fh2JG0=',
]),
}),
"project_configuration_url": "https://hg.mozilla.org/ci/ci-configuration/raw-file/default/projects.yml",
"pushlog_url": "{repo}/json-pushes?changeset={revision}&tipsonly=1&version=2&full=1",
"chain_of_trust_hash_algorithm": "sha256",
"cot_schema_path": os.path.join(os.path.dirname(__file__), "data", "cot_v1_schema.json"),
# for download url validation. The regexes need to define a 'filepath'.
'valid_artifact_rules': (frozendict({
"schemes": ("https", ),
"netlocs": ("queue.taskcluster.net", ),
"path_regexes": (r"^/v1/task/(?P<taskId>[^/]+)(/runs/\\d+)?/artifacts/(?P<filepath>.*)$", ),
}), ),
# scriptworker identification
"scriptworker_worker_pools": (
"scriptworker-prov-v1/balrogworker-v1",
"scriptworker-prov-v1/beetmoverworker-v1",
"scriptworker-prov-v1/pushapk-v1",
"scriptworker-prov-v1/signing-linux-v1",
"scriptworker-prov-v1/treescriptworker-v1",
"scriptworker-k8s/gecko-1-balrog",
"scriptworker-k8s/gecko-3-balrog",
"scriptworker-k8s/gecko-1-beetmover",
"scriptworker-k8s/gecko-3-beetmover",
"scriptworker-k8s/gecko-1-pushapk",
"scriptworker-k8s/gecko-3-pushapk",
"scriptworker-k8s/gecko-1-signing",
"scriptworker-k8s/gecko-3-signing",
"scriptworker-k8s/gecko-t-signing",
"scriptworker-k8s/gecko-1-tree",
"scriptworker-k8s/gecko-3-tree",
),
"scriptworker_provisioners": (
"scriptworker-prov-v1",
"scriptworker-k8s",
),
# valid hash algorithms for chain of trust artifacts
"valid_hash_algorithms": (
"sha256",
"sha512",
),
# decision task cot
"valid_decision_worker_pools": frozendict({
"by-cot-product": frozendict({
"firefox": (
"aws-provisioner-v1/gecko-1-decision",
"aws-provisioner-v1/gecko-2-decision",
"aws-provisioner-v1/gecko-3-decision",
),
"thunderbird": (
"aws-provisioner-v1/gecko-1-decision",
"aws-provisioner-v1/gecko-2-decision",
"aws-provisioner-v1/gecko-3-decision",
),
"mobile": (
# gecko-focus was for mozilla-mobile releases (bug 1455290) for more details.
# TODO: Remove it once not used anymore
"aws-provisioner-v1/gecko-focus",
"aws-provisioner-v1/mobile-1-decision",
# We haven't had the need for mobile-2-decision yet
# https://bugzilla.mozilla.org/show_bug.cgi?id=1512631#c6
"aws-provisioner-v1/mobile-3-decision",
),
"application-services": (
"aws-provisioner-v1/app-services-1-decision",
"aws-provisioner-v1/app-services-3-decision",
),
}),
}),
# docker-image cot
"valid_docker_image_worker_pools": frozendict({
"by-cot-product": frozendict({
"firefox": (
"aws-provisioner-v1/gecko-1-images",
"aws-provisioner-v1/gecko-2-images",
"aws-provisioner-v1/gecko-3-images",
),
"thunderbird": (
"aws-provisioner-v1/gecko-1-images",
"aws-provisioner-v1/gecko-2-images",
"aws-provisioner-v1/gecko-3-images",
),
"mobile": (
"aws-provisioner-v1/mobile-1-images", # there is no mobile level 2.
"aws-provisioner-v1/mobile-3-images",
),
"application-services": (
"aws-provisioner-v1/app-services-1-images",
"aws-provisioner-v1/app-services-3-images",
),
}),
}),
# for trace_back_to_*_tree. These repos have access to restricted scopes;
# all other repos are relegated to CI scopes.
'trusted_vcs_rules': {
'by-cot-product': frozendict({
'firefox': (frozendict({
"schemes": ("https", "ssh", ),
"netlocs": ("hg.mozilla.org", ),
"path_regexes": (
r"^(?P<path>/mozilla-(central|unified))(/|$)",
r"^(?P<path>/integration/(autoland|fx-team|mozilla-inbound))(/|$)",
r"^(?P<path>/releases/mozilla-(beta|release|esr\d+))(/|$)",
r"^(?P<path>/projects/(birch|jamun|maple|oak))(/|$)",
),
}),),
# XXX We should also check the mozilla-central tree that is being used.
'thunderbird': (frozendict({
"schemes": ("https", "ssh", ),
"netlocs": ("hg.mozilla.org", ),
"path_regexes": (
r"^(?P<path>/comm-central)(/|$)",
r"^(?P<path>/releases/comm-(beta|esr\d+))(/|$)",
),
}),),
'mobile': (frozendict({
"schemes": ("https", "ssh", ),
"netlocs": ("github.com", ),
"path_regexes": tuple([
r"^(?P<path>/mozilla-mobile/(?:android-components|focus-android|reference-browser|fenix|firefox-tv))(/|.git|$)",
]),
}),),
'application-services': (frozendict({
"schemes": ("https", "ssh", ),
"netlocs": ("github.com", ),
"path_regexes": (
r"^(?P<path>/mozilla/application-services)(/|.git|$)",
),
}),),
}),
},
'valid_tasks_for': {
'by-cot-product': frozendict({
'firefox': ('hg-push', 'cron', 'action',),
'thunderbird': ('hg-push', 'cron', 'action',),
'mobile': (
'cron',
# On staging releases, level 1 docker images may be built in the pull-request graph
'github-pull-request',
# Similarly, docker images can be built on regular push. This is usually the case
# for level 3 images
'github-push',
'github-release',
),
'application-services': (
# On staging releases, level 1 docker images may be built in the pull-request graph
'github-pull-request',
# Similarly, docker images can be built on regular push. This is usually the case
# for level 3 images
'github-release',
),
}),
},
'official_github_repos_owner': {
'by-cot-product': frozendict({
'firefox': '',
'thunderbird': '',
'mobile': 'mozilla-mobile',
'application-services': 'mozilla',
}),
},
# Map scopes to restricted-level
'cot_restricted_scopes': {
'by-cot-product': frozendict({
'firefox': frozendict({
'project:releng:addons.mozilla.org:server:production': 'all-release-branches',
'project:releng:balrog:server:nightly': 'all-nightly-branches',
'project:releng:balrog:server:beta': 'beta',
'project:releng:balrog:server:release': 'release',
'project:releng:balrog:server:esr': 'esr',
'project:releng:beetmover:bucket:nightly': 'all-nightly-branches',
'project:releng:beetmover:bucket:release': 'all-release-branches',
'project:releng:bouncer:server:production': 'all-production-branches',
# Fennec rides the 68 train. This means, Fennec Nightly will first be shipped off
# mozilla-central then mozilla-esr68. Fennec Beta and Release will move to
# mozilla-esr68 too.
'project:releng:googleplay:aurora': 'esr68',
'project:releng:googleplay:beta': 'esr68',
'project:releng:googleplay:release': 'esr68',
'project:releng:signing:cert:nightly-signing': 'all-nightly-branches',
'project:releng:signing:cert:release-signing': 'all-release-branches',
'project:releng:snapcraft:firefox:beta': 'beta-or-release', # Needed on release for RCs
'project:releng:snapcraft:firefox:candidate': 'release',
'project:releng:snapcraft:firefox:esr': 'esr',
'project:releng:ship-it:production': 'all-production-branches',
'project:releng:treescript:action:push': 'all-release-branches',
}),
'thunderbird': frozendict({
'project:comm:thunderbird:releng:balrog:server:nightly': 'all-nightly-branches',
'project:comm:thunderbird:releng:balrog:server:beta': 'beta',
'project:comm:thunderbird:releng:balrog:server:esr': 'esr',
'project:comm:thunderbird:releng:beetmover:bucket:nightly': 'all-nightly-branches',
'project:comm:thunderbird:releng:beetmover:bucket:release': 'all-release-branches',
'project:comm:thunderbird:releng:bouncer:server:production': 'all-release-branches',
'project:comm:thunderbird:releng:signing:cert:nightly-signing': 'all-nightly-branches',
'project:comm:thunderbird:releng:signing:cert:release-signing': 'all-release-branches',
}),
'mobile': frozendict({
'project:mobile:android-components:releng:beetmover:bucket:maven-production': 'android-components-repo',
'project:mobile:android-components:releng:beetmover:bucket:maven-snapshot-production': 'android-components-repo',
'project:mobile:fenix:releng:googleplay:product:fenix': 'fenix-repo',
'project:mobile:fenix:releng:signing:cert:nightly-signing': 'fenix-repo',
'project:mobile:fenix:releng:signing:cert:beta-signing': 'fenix-repo',
'project:mobile:fenix:releng:signing:cert:production-signing': 'fenix-repo',
'project:mobile:focus:googleplay:product:focus': 'focus-repo',
'project:mobile:focus:releng:signing:cert:release-signing': 'focus-repo',
'project:mobile:reference-browser:releng:googleplay:product:reference-browser': 'reference-browser-repo',
'project:mobile:reference-browser:releng:signing:cert:release-signing': 'reference-browser-repo',
'project:mobile:firefox-tv:releng:googleplay:product:firefox-tv': 'firefox-tv-repo',
'project:mobile:firefox-tv:releng:signing:cert:production-signing': 'firefox-tv-repo',
}),
'application-services': frozendict({
'project:mozilla:application-services:releng:beetmover:bucket:maven-production': 'application-services-repo',
}),
}),
},
# Map restricted-level to trees
'cot_restricted_trees': {
'by-cot-product': frozendict({
'firefox': frozendict({
# Which repos can perform release actions?
# XXX remove /projects/maple and birch when taskcluster relpro
# migration is tier1 and landed on mozilla-central
# XXX remove /projects/jamun when we no longer run staging releases
# from it
'all-release-branches': (
"/releases/mozilla-beta",
"/releases/mozilla-release",
"/releases/mozilla-esr60",
"/releases/mozilla-esr68",
"/projects/birch",
"/projects/jamun",
"/projects/maple",
),
# Limit things like pushapk to just these branches
'release': (
"/releases/mozilla-release",
),
'beta': (
"/releases/mozilla-beta",
),
'beta-or-release': (
"/releases/mozilla-beta",
"/releases/mozilla-release",
),
'esr': (
"/releases/mozilla-esr60",
"/releases/mozilla-esr68",
),
'esr68': (
"/releases/mozilla-esr68",
),
'nightly': (
"/mozilla-central",
),
# Which repos can do nightly signing?
# XXX remove /projects/maple and birch when taskcluster relpro
# migration is tier1 and landed on mozilla-central
# XXX remove /projects/jamun when we no longer run staging releases
# from it
# XXX remove /projects/oak when we no longer test updates against it
'all-nightly-branches': (
"/mozilla-central",
"/releases/mozilla-unified",
"/releases/mozilla-beta",
"/releases/mozilla-release",
"/releases/mozilla-esr60",
"/releases/mozilla-esr68",
"/projects/birch",
"/projects/jamun",
"/projects/oak",
"/projects/maple",
),
'all-production-branches': (
"/mozilla-central",
"/releases/mozilla-beta",
"/releases/mozilla-release",
"/releases/mozilla-esr60",
"/releases/mozilla-esr68",
),
'all-staging-branches': (
"/projects/birch",
"/projects/jamun",
"/projects/maple",
),
}),
'thunderbird': frozendict({
'all-release-branches': (
"/releases/comm-beta",
"/releases/comm-esr60",
"/releases/comm-esr68",
),
'beta': (
"/releases/comm-beta",
),
'esr': (
"/releases/comm-esr60",
"/releases/comm-esr68",
),
'all-nightly-branches': (
"/comm-central",
"/releases/comm-beta",
"/releases/comm-esr60",
"/releases/comm-esr68",
),
'nightly': (
"/comm-central",
),
}),
'mobile': frozendict({
'fenix-repo': (
'/mozilla-mobile/fenix',
),
'focus-repo': (
'/mozilla-mobile/focus-android',
),
'android-components-repo': (
'/mozilla-mobile/android-components',
),
'reference-browser-repo': (
'/mozilla-mobile/reference-browser',
),
'firefox-tv-repo': (
'/mozilla-mobile/firefox-tv',
),
}),
'application-services': frozendict({
'application-services-repo': (
'/mozilla/application-services',
)
}),
}),
},
'prebuilt_docker_image_task_types': {
'by-cot-product': frozendict({
'firefox': ('decision', 'action', 'docker-image'),
'thunderbird': ('decision', 'action', 'docker-image'),
'mobile': 'any', # all allowed
'application-services': 'any', # all allowed
}),
},
'source_env_prefix': {
'by-cot-product': frozendict({
'firefox': 'GECKO',
'thunderbird': 'COMM',
'mobile': 'MOBILE',
'application-services': 'APPSERVICES',
})
},
})
def _draw_contour(self, x, y, a=1., b=1., c=0.,
levels=(1,), level_kwargs: EllipseCommand._LevelKwargsType = (frozendict(),),
_options=frozendict(), zorder=0, **options: Optionable._OptionsType):
return EllipseCommand(*map(self._num_fmt, (x, y, a, b, c)),
levels=levels, level_kwargs=level_kwargs, options={**options, **_options}, zorder=zorder)
def freeze_dict(obj):
return frozendict((k, freeze(v)) for k, v in obj.items())
def solve_mouselab(formula, env, to_what=None):
"""
Use the program found by the PLP method to act once in a particular
instatiation of the Mouselab MDP.
Parameters
----------
formula : [ StateActionProgram ]
env : Trial
to_what : [ int ]
To what state reset the environment
Returns
-------
actions : ( int )
A tuple of taken actions
reward : int
state_actions_distribution : dict
frozendict([ int ] : [ int ]): [ int ]
A set of admissable actions for observed states written as [node_id]
: [value]
"""
env.reset_observations(to_what=to_what)
state = env
actions = [ac for ac in range(NUM_ACTIONS)]
clicks = tuple()
state_actions_distribution = {}
while 1:
if (len(clicks) > 0 and clicks[-1] == TERM) or \
len(clicks) > NUM_ACTIONS:
break
all_bad = True
admissable_actions = []
shuffle(actions)
for act in actions:
if formula(state, act):
all_bad = False
admissable_actions.append(act)
if all_bad:
admissable_actions = [0]
choice = randint(0, len(admissable_actions) - 1)
act = admissable_actions[choice]
clicks = clicks + (act, )
observed = [n.label for n in state.observed_nodes if n.label != 0]
observed_vals = [n.value for n in state.observed_nodes if n.label != 0]
froze_state = frozendict(
{k: v
for k, v in zip(observed, observed_vals)})
froze_action = frozenset(admissable_actions)
state_actions_distribution[froze_state] = froze_action
if act not in [n.label for n in state.observed_nodes]:
state.node_map[act].observe()
best_path = [
node for node in env.node_map.values()
if is_on_highest_expected_value_path(state, node.label)
]
depths = {i: [] for i in state.level_map.keys()}
for node in best_path:
depths[node.depth].append(node)
best_path = []
for dep, nodes in depths.items():
best_path.append(nodes[0].value)
reward = sum(best_path) - len(clicks) + 1
if len(set(clicks)) != len(clicks):
reward = 1e-10 ## penalty for clicking observed nodes;
## the formula is no good if it does that
return clicks, reward, state_actions_distribution
class SlowAugs(str, Enum):
rotate = "rotate"
shift_scale_rotate = "shift_scale_rotate"
shift_hsv = "shift_hsv"
equalize = "equalize"
to_gray = "to_gray"
resize512 = "resize512"
resize300 = "resize300"
resize256 = "resize256"
resize224 = "resize224"
contrast = "contrast"
crop = "crop",
bright = "bright"
SLOW_AUGS_DICT = frozendict(
rotate=Rotate(always_apply=True),
shift_scale_rotate=ShiftScaleRotate(always_apply=True),
shift_hsv=HueSaturationValue(always_apply=True),
equalize=Equalize(always_apply=True),
to_gray=ToGray(always_apply=True),
resize512=Resize(512, 512, always_apply=True),
resize300=Resize(300, 300, always_apply=True),
resize256=Resize(256, 256, always_apply=True),
resiz224=Resize(224, 224, always_apply=True),
contrast=RandomContrast(always_apply=True),
crop=RandomCrop(64, 64, always_apply=True),
bright=RandomBrightness(always_apply=True))
def get_permission_groups() -> frozendict:
"""Get all defined permission groups.
"""
return frozendict(_groups)
def parse_input(my_input):
return frozendict({
int(tile_data.split(':\n')[0].split(' ')[1]):
tuple(tile_data.split(':\n')[1].split('\n'))
for tile_data in my_input.strip('\n').split('\n\n')
})
import json
from vms.models import Vm, Image, SnapshotDefine, Snapshot, BackupDefine, Backup, TagVm, Subnet
from gui.utils import get_order_by, get_pager
from gui.exceptions import HttpRedirectException
from gui.dc.views import dc_switch
from api.vm.utils import get_vms as api_get_vms, get_vm as api_get_vm
from api.vm.define.serializers import VmDefineSerializer, VmDefineDiskSerializer, VmDefineNicSerializer
from api.vm.define.views import vm_define, vm_define_nic, vm_define_disk
from api.vm.snapshot.vm_snapshot_list import VmSnapshotList
from api.vm.backup.vm_backup_list import VmBackupList
from api.utils.views import call_api_view
logger = getLogger(__name__)
REVERSE_OSTYPES = frozendict((ostype, key) for key, ostype in Vm.OSTYPE)
def get_vm(request, hostname, exists_ok=True, noexists_fail=True, auto_dc_switch=True, sr=('dc', 'owner')):
"""
Get VM object or raise 404.
"""
if auto_dc_switch:
try:
vm = Vm.objects.filter(slavevm__isnull=True).select_related(*sr).get(hostname=hostname)
except Vm.DoesNotExist:
raise Http404
else:
if vm.dc != request.dc:
# Switch to VM's DC -> will return False if user is not allowed to switch to DC
if dc_switch(request, vm.dc.name):
return self.get_hexes_of_distance(3, exclude_river)
@dataclass
class BridgeSlot(Location):
connected_hexes: Tuple[Hex, Hex]
@dataclass
class Map:
locations: frozendict = frozendict()
hexes: Tuple[Hex, ...] = ()
bridge_slots: Tuple[BridgeSlot, ...] = ()
def __init__(self, hexes: Tuple[Hex, ...], bridge_slots: Tuple[BridgeSlot,
...]):
self.hexes = hexes
self.bridge_slots = bridge_slots
self.locations = {x.location_id: x for x in hexes + bridge_slots}
_adjacent_terrains = frozendict({
Terrain.MOUNTAIN: (Terrain.WASTELAND, Terrain.DESERT),
Terrain.DESERT: (Terrain.MOUNTAIN, Terrain.FIELD),
Terrain.FIELD: (Terrain.DESERT, Terrain.SWAMP),
Terrain.SWAMP: (Terrain.FIELD, Terrain.LAKE),
Terrain.LAKE: (Terrain.SWAMP, Terrain.FOREST),
Terrain.FOREST: (Terrain.LAKE, Terrain.WASTELAND),
Terrain.WASTELAND: (Terrain.FOREST, Terrain.MOUNTAIN)
})
def interpret_config_file(filename: str) -> ConfigInfo:
"""
Returns a ConfigInfo.
"""
try:
basename = os.path.basename(filename)
base = basename.replace(SUFFIX, "")
# now we have something like
# package-node.config_name.date
# or
# package-node.config_name
if not "." in base:
msg = f"Invalid filename {filename!r}."
raise dtu.DTConfigException(msg)
tokens = base.split(".")
if len(tokens) > 3:
msg = f"Too many periods/tokens (tokens={tokens})"
raise dtu.DTConfigException(msg)
if len(tokens) <= 2:
# package-node.config_name
package_node = tokens[0]
if not "-" in package_node:
msg = f'Expected a "-" in "{package_node}".'
raise dtu.DTConfigException(msg)
i = package_node.index("-")
package_name = package_node[:i]
node_name = package_node[i + 1 :]
else:
package_name = node_name = None # FIXME: should we bail?
config_name = tokens[1]
if len(tokens) == 3:
# package-node.config_name.date
date_effective = tokens[2]
else:
date_effective = "20170101"
try:
date_effective = parse(date_effective)
except:
msg = f'Cannot interpret "{date_effective}" as a date.'
raise dtu.DTConfigException(msg)
# now read file
with open(filename) as f:
contents = f.read()
try:
try:
data = yaml.load(contents, Loader=yaml.Loader)
except YAMLError as e:
dtu.raise_wrapped(dtu.DTConfigException, e, "Invalid YAML", compact=True)
raise
if not isinstance(data, dict):
msg = "Expected a dictionary inside."
raise dtu.DTConfigException(msg)
for field in ["description", "values"]:
if not field in data:
msg = f'Missing field "{field}".'
raise dtu.DTConfigException(msg)
description = data.pop("description")
if not isinstance(description, str):
msg = f'I expected that "description" is a string, obtained {description!r}.'
raise dtu.DTConfigException(msg)
extends = data.pop("extends", [])
if not isinstance(extends, list):
msg = f'I expected that "extends" is a list, obtained {extends!r}.'
raise dtu.DTConfigException(msg)
values = data.pop("values")
if not isinstance(values, dict):
msg = f'I expected that "values" is a dictionary, obtained {type(values)}.'
raise dtu.DTConfigException(msg)
# Freeze the data
extends = tuple(extends)
values = frozendict(values)
except dtu.DTConfigException as e:
msg = "Could not interpret the contents of the file\n"
msg += f" {dtu.friendly_path(filename)}\n"
msg += "Contents:\n" + dtu.indent(contents, " > ")
dtu.raise_wrapped(dtu.DTConfigException, e, msg, compact=True)
raise
return ConfigInfo(
filename=filename,
package_name=package_name,
node_name=node_name,
config_name=config_name,
date_effective=date_effective,
extends=extends,
description=description,
values=values,
# not decided
valid=None,
error_if_invalid=None,
)
except dtu.DTConfigException as e:
msg = f"Invalid file {dtu.friendly_path(filename)}"
dtu.raise_wrapped(dtu.DTConfigException, e, msg, compact=True)
car = { 'brand':'Honda', 'model': 'Jazz', 'year' : 2017}
type(car)
car
#access
car['brand']
car['year']
car.get('year')
dir(car)
car['brand'] = 'MARUTI'
car #mutable , value can be changed
#%%frozen Dictionaries
#pip install frozendict #install this library
from frozendict import frozendict
fd = frozendict({ 'brand': 'Honda' })
fd
fd['brand'] = 'HYUNDIA' #immutatble
#%% { curly bracket, comma}
#Set - ordered collection of simple items, immutable
set1 = set(['india', 'pakistan', 'england', 'australia','india'])
set1
type(set1)
set2 = {'INDIA','PAKISTAN', "INDIA"} #better way
set2
set3 = {'Australia', 'South Africa', 'INDIA'}
set3
sorted(set3)
set2.union(set3) #set2 | set3
from frozendict import frozendict
RAW_SENTIMENT_LABELS = ('negative', 'neutral', 'positive')
NN_RSL_TO_INT = frozendict({v: i for i, v in enumerate(RAW_SENTIMENT_LABELS)})
class VocabUtil:
def __init__(self):
self.raw_sentiment_labels = RAW_SENTIMENT_LABELS
self.nn_rsl_to_int = NN_RSL_TO_INT
def get_output_vocab_size(self) -> int:
return len(self.nn_rsl_to_int)
def build(cls, fmapper, honored):
obj = object.__new__(HaloScheme)
obj._mapper = frozendict(fmapper)
obj._honored = frozendict(honored)
return obj
def _draw_hist(self, m, v, _options=frozendict(), **options):
return AddplotExpression(f'exp(-(x-{self._num_fmt(m)})^2 / 2 / {self._num_fmt(v)}) / {self._num_fmt(v**0.5)}',
options={**options, **_options})
def omapper(self):
"""
Logical decomposition of the DOMAIN region into OWNED and CORE sub-regions.
This is "cumulative" over all DiscreteFunctions in the HaloScheme; it also
takes into account IterationSpace offsets induced by SubDomains/SubDimensions.
Examples
--------
Consider a HaloScheme comprising two one-dimensional Functions, ``u``
and ``v``. ``u``'s halo, on the LEFT and RIGHT DataSides respectively,
is (2, 2), while ``v``'s is (4, 4). The situation is depicted below.
^^oo----------------oo^^ u
^^^^oooo------------oooo^^^^ v
Where '^' represents a HALO point, 'o' a OWNED point, and '-' a CORE point.
Together, the 'o' and '-' points constitute the DOMAIN region.
In this example, the "cumulative" OWNED size is (left=4, right=4), that is
the max on each DataSide across all Functions, namely ``u`` and ``v``.
The ``omapper`` will contain the following entries:
[(((d, CORE, CENTER),), {d: (d_m + 4, d_M - 4)}),
(((d, OWNED, LEFT),), {d: (d_m, min(d_m + 3, d_M))}),
(((d, OWNED, RIGHT),), {d: (max(d_M - 3, d_m), d_M)})]
In presence of SubDomains (or, more generally, iteration over SubDimensions),
the "true" DOMAIN is actually smaller. For example, consider again the
example above, but now with a SubDomain that excludes the first ``nl``
and the last ``nr`` DOMAIN points, where ``nl >= 0`` and ``nr >= 0``. Often,
``nl`` and ``nr`` are referred to as the "thickness" of the SubDimension (see
also SubDimension.__doc__). For example, the situation could be as below
^^ooXXX----------XXXoo^^ u
^^^^ooooX----------Xoooo^^^^ v
Where 'X' is a CORE point excluded by the computation due to the SubDomain.
Here, the 'o' points are outside of the SubDomain, but in general they could
also be inside. The ``omapper`` is constructed taking into account that
SubDomains are iterated over with min point ``d_m + nl`` and max point
``d_M - nr``. Here, the ``omapper`` is:
[(((d, CORE, CENTER),), {d: (d_m + 4, d_M - 4),
nl: (max(nl - 4, 0),),
nr: (max(nr - 4, 0),)}),
(((d, OWNED, LEFT),), {d: (d_m, min(d_m + 3, d_M - nr)),
nl: (nl,),
nr: (0,)}),
(((d, OWNED, RIGHT),), {d: (max(d_M - 3, d_m + nl), d_M),
nl: (0,),
nr: (nr,)})]
To convince ourselves that this makes sense, we consider a number of cases.
For now, we assume ``|d_M - d_m| > HALO``, that is the left-HALO and right-HALO
regions do not overlap.
1. The SubDomain thickness is 0, which is like there were no SubDomains.
By instantiating the template above with ``nl = 0`` and ``nr = 0``,
it is trivial to see that we fall back to the non-SubDomain case.
2. The SubDomain thickness is as big as the HALO region size, that is
``nl = 4`` and ``nr = 4``. The ``omapper`` is such that no iterations
will be performed in the OWNED regions (i.e., "everything is CORE").
3. The SubDomain left-thickness is smaller than the left-HALO region size,
while the SubDomain right-thickness is larger than the right-Halo region
size. This means that some left-OWNED points are within the SubDomain,
while the RIGHT-OWNED are outside. For example, take ``nl = 1`` and
``nr = 5``; the iteration regions will then be:
- (CORE, CENTER): {d: (d_m + 4, d_M - 4), nl: (0,), nr: (1,)}, so
the min point is ``d_m + 4``, while the max point is ``d_M - 5``.
- (OWNED, LEFT): {d: (d_m, d_m + 3), nl: (1,), nr: (0,)}, so the
min point is ``d_m + 1``, while the max point is ``dm + 3``.
- (OWNED, RIGHT): {d: (d_M - 3, d_M), nl: (0,), nr: (5,)}, so the
min point is ``d_M - 3``, while the max point is ``d_M - 5``,
which implies zero iterations in this region.
Let's now assume that the left-HALO and right-HALO regions overlap. For example,
``d_m = 0`` and ``d_M = 1`` (i.e., the DOMAIN only has two points), with the HALO
size that is still (4, 4).
4. Let's take ``nl = 1`` and ``nr = 0``. That is, only one point is in
the SubDomain and should be updated. We again instantiate the iteration
regions and obtain:
- (CORE, CENTER): {d: (d_m + 4, d_M - 4), nl: (0,), nr: (0,)}, so
the min point is ``d_m + 4 = 4``, while the max point is
``d_M - 4 = -3``, which implies zero iterations in this region.
- (OWNED, LEFT): {d: (d_m, min(d_m + 3, d_M - nr)), nl: (1,), nr: (0,)},
so the min point is ``d_m + 1 = 1``, while the max point is
``min(d_m + 3, d_M - nr) = min(3, 1) = 1``, which implies that there
is exactly one point in this region.
- (OWNED, RIGHT): {d: (max(d_M - 3, d_m + nl), d_M), nl: (0,), nr: (0,)},
so the min point is ``max(d_M - 3, d_m + nl) = max(-2, 1) = 1``, while
the max point is ``d_M = 1``, which implies that there is exactly one
point in this region, and this point is redundantly computed as it's
logically the same as that in the (OWNED, LEFT) region.
Notes
-----
For each Function, the '^' and 'o' are exactly the same on *all MPI
ranks*, so the output of this method is guaranteed to be consistent
across *all MPI ranks*.
"""
items = [((d, CENTER), (d, LEFT), (d, RIGHT)) for d in self.dimensions]
processed = []
for item in product(*items):
where = []
mapper = {}
for d, s in item:
osl, osr = self.owned_size[d]
# Handle SubDomain/SubDimensions to-honor offsets
nl = Max(0, *[i for i, _ in self.honored.get(d, [])])
nr = Max(0, *[i for _, i in self.honored.get(d, [])])
if s is CENTER:
where.append((d, CORE, s))
mapper[d] = (d.symbolic_min + osl, d.symbolic_max - osr)
if nl != 0:
mapper[nl] = (Max(nl - osl, 0), )
if nr != 0:
mapper[nr] = (Max(nr - osr, 0), )
else:
where.append((d, OWNED, s))
if s is LEFT:
mapper[d] = (d.symbolic_min,
Min(d.symbolic_min + osl - 1,
d.symbolic_max - nr))
if nl != 0:
mapper[nl] = (nl, )
mapper[nr] = (0, )
else:
mapper[d] = (Max(d.symbolic_max - osr + 1,
d.symbolic_min + nl), d.symbolic_max)
if nr != 0:
mapper[nl] = (0, )
mapper[nr] = (nr, )
processed.append((tuple(where), frozendict(mapper)))
_, core = processed.pop(0)
owned = processed
return OMapper(core, owned)
def test_illustration():
"""
A `Illustration` can be cast from a dict.
"""
class _Section(DocElem):
"""Test Section"""
def __init__(self, title, *args, **kwargs):
super().__init__(*args, title=title, **kwargs)
@field.cast(Narrative)
def narrative(self):
raise FieldIsRequired
@field.cast(Data)
def data(self):
return NA
@field.cast(Illustration)
def illustration(self):
return NA
def __call__(self, adapter, model):
return Record(
narrative=self.narrative(adapter, model),
data=self.data(adapter, model),
illustration=self.illustration(adapter, model))
p = Bars(xvar="layer", xlabel="Layer",
yvar="cell_density", ylabel="Cell Density")
d = Document("test")
s = _Section("Illustration", parent=d,
narrative="Test",
data=mock_cell_density(),
illustration=frozendict(
caption="Caption",
figures=p(mock_cell_density())))
assert isinstance(s.narrative, Narrative), type(s.narrative)
assert isinstance(s.data, Data), type(s.data)
try:
illustration = s.illustration
except AttributeError:
assert False
assert isinstance(illustration, Illustration), type(illustration)
v = s(MockAdapter(), MockModel())
assert v.narrative == "Test"
assert isinstance(v.data, pd.DataFrame)
assert "layer" in v.data.columns
assert "cell_density" in v.data.columns
try:
vill = v.illustration
except AttributeError:
assert False
try:
caption = vill.caption
except AttributeError:
assert False
assert caption == "Caption"
try:
figures = vill.figures
except AttributeError:
assert False
def sumweights(weights: Sequence[Tuple[float, frozendict]]):
ts = [item for i in weights for item in i.keys()]
if ts:
return frozendict({min(ts): 0})
else:
return frozendict()
def __torch_function__(self, func, types, args=(), kwargs=frozendict()):
ret = super().__torch_function__(func, types, args, kwargs)
return self._meta_update(ret) if isinstance(ret, type(self)) else ret
def test_get_state_for_event(self):
# this defaults to a linear DAG as each new injection defaults to whatever
# forward extremities are currently in the DB for this room.
e1 = self.inject_state_event(self.room, self.u_alice,
EventTypes.Create, "", {})
e2 = self.inject_state_event(self.room, self.u_alice, EventTypes.Name,
"", {"name": "test room"})
e3 = self.inject_state_event(
self.room,
self.u_alice,
EventTypes.Member,
self.u_alice.to_string(),
{"membership": Membership.JOIN},
)
e4 = self.inject_state_event(
self.room,
self.u_bob,
EventTypes.Member,
self.u_bob.to_string(),
{"membership": Membership.JOIN},
)
e5 = self.inject_state_event(
self.room,
self.u_bob,
EventTypes.Member,
self.u_bob.to_string(),
{"membership": Membership.LEAVE},
)
# check we get the full state as of the final event
state = self.get_success(
self.storage.state.get_state_for_event(e5.event_id))
self.assertIsNotNone(e4)
self.assertStateMapEqual(
{
(e1.type, e1.state_key): e1,
(e2.type, e2.state_key): e2,
(e3.type, e3.state_key): e3,
# e4 is overwritten by e5
(e5.type, e5.state_key): e5,
},
state,
)
# check we can filter to the m.room.name event (with a '' state key)
state = self.get_success(
self.storage.state.get_state_for_event(
e5.event_id, StateFilter.from_types([(EventTypes.Name, "")])))
self.assertStateMapEqual({(e2.type, e2.state_key): e2}, state)
# check we can filter to the m.room.name event (with a wildcard None state key)
state = self.get_success(
self.storage.state.get_state_for_event(
e5.event_id,
StateFilter.from_types([(EventTypes.Name, None)])))
self.assertStateMapEqual({(e2.type, e2.state_key): e2}, state)
# check we can grab the m.room.member events (with a wildcard None state key)
state = self.get_success(
self.storage.state.get_state_for_event(
e5.event_id,
StateFilter.from_types([(EventTypes.Member, None)])))
self.assertStateMapEqual(
{
(e3.type, e3.state_key): e3,
(e5.type, e5.state_key): e5
}, state)
# check we can grab a specific room member without filtering out the
# other event types
state = self.get_success(
self.storage.state.get_state_for_event(
e5.event_id,
state_filter=StateFilter(
types=frozendict({
EventTypes.Member:
frozenset({self.u_alice.to_string()})
}),
include_others=True,
),
))
self.assertStateMapEqual(
{
(e1.type, e1.state_key): e1,
(e2.type, e2.state_key): e2,
(e3.type, e3.state_key): e3,
},
state,
)
# check that we can grab everything except members
state = self.get_success(
self.storage.state.get_state_for_event(
e5.event_id,
state_filter=StateFilter(
types=frozendict({EventTypes.Member: frozenset()}),
include_others=True,
),
))
self.assertStateMapEqual(
{
(e1.type, e1.state_key): e1,
(e2.type, e2.state_key): e2
}, state)
#######################################################
# _get_state_for_group_using_cache tests against a full cache
#######################################################
room_id = self.room.to_string()
group_ids = self.get_success(
self.storage.state.get_state_groups_ids(room_id, [e5.event_id]))
group = list(group_ids.keys())[0]
# test _get_state_for_group_using_cache correctly filters out members
# with types=[]
(
state_dict,
is_all,
) = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_cache,
group,
state_filter=StateFilter(types=frozendict(
{EventTypes.Member: frozenset()}),
include_others=True),
)
self.assertEqual(is_all, True)
self.assertDictEqual(
{
(e1.type, e1.state_key): e1.event_id,
(e2.type, e2.state_key): e2.event_id,
},
state_dict,
)
(
state_dict,
is_all,
) = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_members_cache,
group,
state_filter=StateFilter(types=frozendict(
{EventTypes.Member: frozenset()}),
include_others=True),
)
self.assertEqual(is_all, True)
self.assertDictEqual({}, state_dict)
# test _get_state_for_group_using_cache correctly filters in members
# with wildcard types
(
state_dict,
is_all,
) = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_cache,
group,
state_filter=StateFilter(types=frozendict(
{EventTypes.Member: None}),
include_others=True),
)
self.assertEqual(is_all, True)
self.assertDictEqual(
{
(e1.type, e1.state_key): e1.event_id,
(e2.type, e2.state_key): e2.event_id,
},
state_dict,
)
(
state_dict,
is_all,
) = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_members_cache,
group,
state_filter=StateFilter(types=frozendict(
{EventTypes.Member: None}),
include_others=True),
)
self.assertEqual(is_all, True)
self.assertDictEqual(
{
(e3.type, e3.state_key): e3.event_id,
# e4 is overwritten by e5
(e5.type, e5.state_key): e5.event_id,
},
state_dict,
)
# test _get_state_for_group_using_cache correctly filters in members
# with specific types
(
state_dict,
is_all,
) = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_cache,
group,
state_filter=StateFilter(
types=frozendict(
{EventTypes.Member: frozenset({e5.state_key})}),
include_others=True,
),
)
self.assertEqual(is_all, True)
self.assertDictEqual(
{
(e1.type, e1.state_key): e1.event_id,
(e2.type, e2.state_key): e2.event_id,
},
state_dict,
)
(
state_dict,
is_all,
) = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_members_cache,
group,
state_filter=StateFilter(
types=frozendict(
{EventTypes.Member: frozenset({e5.state_key})}),
include_others=True,
),
)
self.assertEqual(is_all, True)
self.assertDictEqual({(e5.type, e5.state_key): e5.event_id},
state_dict)
# test _get_state_for_group_using_cache correctly filters in members
# with specific types
(
state_dict,
is_all,
) = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_members_cache,
group,
state_filter=StateFilter(
types=frozendict(
{EventTypes.Member: frozenset({e5.state_key})}),
include_others=False,
),
)
self.assertEqual(is_all, True)
self.assertDictEqual({(e5.type, e5.state_key): e5.event_id},
state_dict)
#######################################################
# deliberately remove e2 (room name) from the _state_group_cache
cache_entry = self.state_datastore._state_group_cache.get(group)
state_dict_ids = cache_entry.value
self.assertEqual(cache_entry.full, True)
self.assertEqual(cache_entry.known_absent, set())
self.assertDictEqual(
state_dict_ids,
{
(e1.type, e1.state_key): e1.event_id,
(e2.type, e2.state_key): e2.event_id,
},
)
state_dict_ids.pop((e2.type, e2.state_key))
self.state_datastore._state_group_cache.invalidate(group)
self.state_datastore._state_group_cache.update(
sequence=self.state_datastore._state_group_cache.sequence,
key=group,
value=state_dict_ids,
# list fetched keys so it knows it's partial
fetched_keys=((e1.type, e1.state_key), ),
)
cache_entry = self.state_datastore._state_group_cache.get(group)
state_dict_ids = cache_entry.value
self.assertEqual(cache_entry.full, False)
self.assertEqual(cache_entry.known_absent, {(e1.type, e1.state_key)})
self.assertDictEqual(state_dict_ids,
{(e1.type, e1.state_key): e1.event_id})
############################################
# test that things work with a partial cache
# test _get_state_for_group_using_cache correctly filters out members
# with types=[]
room_id = self.room.to_string()
(
state_dict,
is_all,
) = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_cache,
group,
state_filter=StateFilter(types=frozendict(
{EventTypes.Member: frozenset()}),
include_others=True),
)
self.assertEqual(is_all, False)
self.assertDictEqual({(e1.type, e1.state_key): e1.event_id},
state_dict)
room_id = self.room.to_string()
(
state_dict,
is_all,
) = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_members_cache,
group,
state_filter=StateFilter(types=frozendict(
{EventTypes.Member: frozenset()}),
include_others=True),
)
self.assertEqual(is_all, True)
self.assertDictEqual({}, state_dict)
# test _get_state_for_group_using_cache correctly filters in members
# wildcard types
(
state_dict,
is_all,
) = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_cache,
group,
state_filter=StateFilter(types=frozendict(
{EventTypes.Member: None}),
include_others=True),
)
self.assertEqual(is_all, False)
self.assertDictEqual({(e1.type, e1.state_key): e1.event_id},
state_dict)
(
state_dict,
is_all,
) = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_members_cache,
group,
state_filter=StateFilter(types=frozendict(
{EventTypes.Member: None}),
include_others=True),
)
self.assertEqual(is_all, True)
self.assertDictEqual(
{
(e3.type, e3.state_key): e3.event_id,
(e5.type, e5.state_key): e5.event_id,
},
state_dict,
)
# test _get_state_for_group_using_cache correctly filters in members
# with specific types
(
state_dict,
is_all,
) = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_cache,
group,
state_filter=StateFilter(
types=frozendict(
{EventTypes.Member: frozenset({e5.state_key})}),
include_others=True,
),
)
self.assertEqual(is_all, False)
self.assertDictEqual({(e1.type, e1.state_key): e1.event_id},
state_dict)
(
state_dict,
is_all,
) = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_members_cache,
group,
state_filter=StateFilter(
types=frozendict(
{EventTypes.Member: frozenset({e5.state_key})}),
include_others=True,
),
)
self.assertEqual(is_all, True)
self.assertDictEqual({(e5.type, e5.state_key): e5.event_id},
state_dict)
# test _get_state_for_group_using_cache correctly filters in members
# with specific types
(
state_dict,
is_all,
) = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_cache,
group,
state_filter=StateFilter(
types=frozendict(
{EventTypes.Member: frozenset({e5.state_key})}),
include_others=False,
),
)
self.assertEqual(is_all, False)
self.assertDictEqual({}, state_dict)
(
state_dict,
is_all,
) = self.state_datastore._get_state_for_group_using_cache(
self.state_datastore._state_group_members_cache,
group,
state_filter=StateFilter(
types=frozendict(
{EventTypes.Member: frozenset({e5.state_key})}),
include_others=False,
),
)
self.assertEqual(is_all, True)
self.assertDictEqual({(e5.type, e5.state_key): e5.event_id},
state_dict)
DEFAULT_CONFIG = frozendict({
# Worker identification
"provisioner_id":
"test-dummy-provisioner",
"worker_group":
"test-dummy-workers",
"worker_type":
"dummy-worker-myname",
"worker_id":
os.environ.get("SCRIPTWORKER_WORKER_ID", "dummy-worker-myname1"),
"credentials":
frozendict({
"clientId": "...",
"accessToken": "...",
"certificate": "...",
}),
# Worker log settings
"log_datefmt":
"%Y-%m-%dT%H:%M:%S",
"log_fmt":
"%(asctime)s %(levelname)8s - %(message)s",
"log_max_bytes":
1024 * 1024 * 512,
"log_num_backups":
10,
# intervals are expressed in seconds
"artifact_expiration_hours":
24,
"task_max_timeout":
60 * 20,
"reclaim_interval":
300,
"poll_interval":
5,
"sign_key_timeout":
60 * 2,
"task_script": ("bash", "-c", "echo foo && sleep 19 && exit 1"),
"verbose":
True,
# Task settings
"work_dir":
"...",
"log_dir":
"...",
"artifact_dir":
"...",
"task_log_dir":
"...", # set this to ARTIFACT_DIR/public/logs
"git_commit_signing_pubkey_dir":
"...",
"artifact_upload_timeout":
60 * 20,
"aiohttp_max_connections":
15,
# chain of trust settings
"sign_chain_of_trust":
True,
"verify_chain_of_trust":
False, # TODO True
"verify_cot_signature":
False,
"cot_job_type":
"unknown", # e.g., signing
"cot_product":
"firefox",
# Specify a default gpg home other than ~/.gnupg
"gpg_home":
None,
# A list of additional gpg cmdline options
"gpg_options":
None,
# The path to the gpg executable.
"gpg_path":
None,
# The path to the public/secret keyrings, if we're not using the default
"gpg_public_keyring":
'%(gpg_home)s/pubring.gpg',
"gpg_secret_keyring":
'%(gpg_home)s/secring.gpg',
# Boolean to use the gpg agent
"gpg_use_agent":
False,
"gpg_encoding":
'utf-8',
"base_gpg_home_dir":
"...",
"gpg_lockfile":
os.path.join(os.getcwd(), "gpg_homedir.lock"),
"git_key_repo_dir":
"...",
"git_key_repo_url":
"https://github.com/mozilla-releng/cot-gpg-keys.git",
"last_good_git_revision_file":
os.path.join(os.getcwd(), "git_revision"),
"pubkey_path":
"...",
"privkey_path":
"...",
"my_email":
"*****@*****.**",
"chain_of_trust_hash_algorithm":
"sha256",
"cot_schema_path":
os.path.join(os.path.dirname(__file__), "data", "cot_v1_schema.json"),
# for download url validation. The regexes need to define a 'filepath'.
'valid_artifact_rules': (frozendict({
"schemes": ("https", ),
"netlocs": ("queue.taskcluster.net", ),
"path_regexes":
("^/v1/task/(?P<taskId>[^/]+)(/runs/\\d+)?/artifacts/(?P<filepath>.*)$",
),
}), ),
# docker image shas
"docker_image_allowlists":
frozendict({
"decision": (
"sha256:31035ed23eba3ede02b988be39027668d965b9fc45b74b932b2338a4e7936cf9",
"sha256:7320c720c770e9f93df26f7da742db72b334b7ded77539fb240fc4a28363de5a",
"sha256:9db282317340838f0015335d74ed56c4ee0dbad588be33e6999928a181548587",
"sha256:a22b90c7e16191a701760ef4f9159e86289ba598bf8ff5b22b7b94867530460d",
),
"docker-image": (
"sha256:74c5a18ce1768605ce9b1b5f009abac1ff11b55a007e2d03733cd6e95847c747",
"sha256:d438d7818b6a47a0b1d49943ab12b5c504b65161806658e4c28f5f2aac821b9e",
"sha256:13b80a7a6b8e10c6096aba5a435529fbc99b405f56012e57cc6835facf4b40fb",
)
}),
# git gpg homedir layout
"gpg_homedirs":
frozendict({
"docker-worker":
frozendict({
"type": "flat",
"ignore_suffixes": (".md", )
}),
"generic-worker":
frozendict({
"type": "flat",
"ignore_suffixes": (".md", )
}),
"scriptworker":
frozendict({
"type": "signed",
"ignore_suffixes": (".md", )
}),
}),
# scriptworker identification
"scriptworker_worker_types": (
"balrogworker-v1",
"beetmoverworker-v1",
"pushapk-v1",
"signing-linux-v1",
),
"scriptworker_provisioners": ("scriptworker-prov-v1", ),
# valid hash algorithms for chain of trust artifacts
"valid_hash_algorithms": (
"sha256",
"sha512",
),
# decision task cot
"valid_decision_worker_types": ("gecko-decision", ),
# docker-image cot
"valid_docker_image_worker_types": (
"taskcluster-images", # TODO: Remove this image once docker-images is the only valid worker type
"gecko-images",
),
# for trace_back_to_*_tree. These repos have access to restricted scopes;
# all other repos are relegated to CI scopes.
'valid_vcs_rules': (
frozendict({
# TODO index by cot_product
"schemes": (
"https",
"ssh",
),
"netlocs": ("hg.mozilla.org", ),
"path_regexes": (
"^(?P<path>/mozilla-(central|unified))(/|$)",
"^(?P<path>/integration/(autoland|fx-team|mozilla-inbound))(/|$)",
"^(?P<path>/releases/mozilla-(aurora|beta|release|esr45|esr52))(/|$)",
# XXX remove /projects/date when taskcluster nightly migration is
# tier1 and landed on mozilla-central
# XXX remove /projects/jamun when we no longer release firefox
# from it
"^(?P<path>/projects/(date|jamun))(/|$)",
),
}), ),
# Map scopes to restricted-level
'cot_restricted_scopes':
frozendict({
'firefox':
frozendict({
'project:releng:balrog:server:nightly':
'nightly',
'project:releng:balrog:server:aurora':
'aurora',
'project:releng:balrog:server:beta':
'beta',
'project:releng:balrog:server:release':
'release',
'project:releng:balrog:server:esr':
'esr',
'project:releng:balrog:nightly':
'all-nightly-branches',
'project:releng:beetmover:bucket:release':
'all-release-branches',
'project:releng:googleplay:release':
'release',
'project:releng:signing:cert:release-signing':
'all-release-branches',
'project:releng:googleplay:beta':
'betatest',
'project:releng:googleplay:aurora':
'auroratest',
'project:releng:beetmover:bucket:nightly':
'all-nightly-branches',
'project:releng:signing:cert:nightly-signing':
'all-nightly-branches',
})
}),
# Map restricted-level to trees
'cot_restricted_trees':
frozendict({
'firefox':
frozendict({
# Which repos can perform release actions?
# Allow aurora for staging betas.
# XXX remove /projects/jamun when we no longer release firefox
# from it
'all-release-branches': (
"/releases/mozilla-aurora",
"/releases/mozilla-beta",
"/releases/mozilla-release",
"/releases/mozilla-esr45",
"/releases/mozilla-esr52",
"/projects/jamun",
),
# Limit things like pushapk to just these branches
'release': ("/releases/mozilla-release", ),
'beta': ("/releases/mozilla-beta", ),
# TODO remove it once pushapk is landed on beta
'betatest': (
"/releases/mozilla-beta",
"/projects/jamun",
),
'aurora': ("/releases/mozilla-aurora", ),
# TODO remove it once pushapk is landed on aurora
'auroratest': (
"/releases/mozilla-aurora",
"/projects/date",
),
'esr': (
"/releases/mozilla-esr45",
"/releases/mozilla-esr52",
),
'nightly': ("/mozilla-central", ),
# Which repos can do nightly signing?
# XXX remove /projects/date when taskcluster nightly migration is
# tier1 and landed on mozilla-central
# XXX remove /projects/jamun when we no longer release firefox
# from it
'all-nightly-branches': (
"/mozilla-central",
"/releases/mozilla-unified",
"/releases/mozilla-aurora",
"/releases/mozilla-beta",
"/releases/mozilla-release",
"/releases/mozilla-esr45",
"/releases/mozilla-esr52",
"/projects/jamun",
"/projects/date",
),
}),
}),
})
def test_state_filter_difference_no_include_other_minus_include_other(
self):
"""
Tests the StateFilter.approx_difference method
where, in a.approx_difference(b), only b has the include_others flag set.
"""
# (wildcard on state keys) - (wildcard on state keys):
self.assert_difference(
StateFilter.freeze(
{
EventTypes.Member: None,
EventTypes.Create: None
},
include_others=False,
),
StateFilter.freeze(
{
EventTypes.Member: None,
EventTypes.CanonicalAlias: None
},
include_others=True,
),
StateFilter(types=frozendict(), include_others=False),
)
# (wildcard on state keys) - (specific state keys)
# This one is an over-approximation because we can't represent
# 'all state keys except a few named examples'
self.assert_difference(
StateFilter.freeze({EventTypes.Member: None},
include_others=False),
StateFilter.freeze(
{EventTypes.Member: {"@wombat:spqr"}},
include_others=True,
),
StateFilter.freeze({EventTypes.Member: None},
include_others=False),
)
# (wildcard on state keys) - (no state keys)
self.assert_difference(
StateFilter.freeze(
{EventTypes.Member: None},
include_others=False,
),
StateFilter.freeze(
{
EventTypes.Member: set(),
},
include_others=True,
),
StateFilter.freeze(
{EventTypes.Member: None},
include_others=False,
),
)
# (specific state keys) - (wildcard on state keys):
self.assert_difference(
StateFilter.freeze(
{
EventTypes.Member: {"@wombat:spqr", "@spqr:spqr"},
EventTypes.CanonicalAlias: {""},
},
include_others=False,
),
StateFilter.freeze(
{EventTypes.Member: None},
include_others=True,
),
StateFilter(
types=frozendict(),
include_others=False,
),
)
# (specific state keys) - (specific state keys)
# This one is an over-approximation because we can't represent
# 'all state keys except a few named examples'
self.assert_difference(
StateFilter.freeze(
{
EventTypes.Member: {"@wombat:spqr", "@spqr:spqr"},
EventTypes.CanonicalAlias: {""},
},
include_others=False,
),
StateFilter.freeze(
{
EventTypes.Member: {"@wombat:spqr"},
},
include_others=True,
),
StateFilter.freeze(
{
EventTypes.Member: {"@spqr:spqr"},
},
include_others=False,
),
)
# (specific state keys) - (no state keys)
self.assert_difference(
StateFilter.freeze(
{
EventTypes.Member: {"@wombat:spqr", "@spqr:spqr"},
EventTypes.CanonicalAlias: {""},
},
include_others=False,
),
StateFilter.freeze(
{
EventTypes.Member: set(),
},
include_others=True,
),
StateFilter.freeze(
{
EventTypes.Member: {"@wombat:spqr", "@spqr:spqr"},
},
include_others=False,
),
)
def _isotype_annotation_dagfuncs(
isotypemap: Mapping[str, str] = None,
isotypemap_file: str = None,
idmap: Mapping[str, Set[str]] = None,
idmap_file: str = None,
isotype_names: Sequence[str] = None,
) -> hdag.utils.AddFuncDict:
"""Return functions for annotating history DAG nodes with inferred
isotypes.
Args:
isotypemap: A dictionary mapping original IDs to observed isotype names
isotypemap_file: A csv file providing an `isotypemap`
idmap: A dictionary mapping unique sequence IDs to sets of original IDs of observed sequences
idmap_file: A csv file providing an `idmap`
isotype_names: A sequence of isotype names containing values in `isotypemap`, in the correct switching order
Returns:
A :meth:`historydag.utils.AddFuncDict` which may be passed as keyword arguments
to :meth:`historydag.HistoryDag.weight_count`, :meth:`historydag.HistoryDag.trim_optimal_weight`,
or :meth:`historydag.HistoryDag.optimal_weight_annotate`
methods to trim or annotate a :meth:`historydag.HistoryDag` according to isotype parsimony.
Weight format is ``frozendict[Isotype, int]``, where the value is the count of observed isotypes, and 0
for unobserved (internal) nodes.
"""
if isotype_names is None:
isotype_names = default_isotype_order
if isotypemap_file and isotypemap is None:
with open(isotypemap_file, "r") as fh:
isotypemap = dict(
map(lambda x: x.strip(), line.split(",")) for line in fh)
elif isotypemap_file is None:
raise ValueError("Either isotypemap or isotypemap_file is required")
if idmap is None:
if idmap_file is None:
raise TypeError(
"either idmap or idmap_file is required for isotyping")
else:
with open(idmap_file, "r") as fh:
idmap = {}
for line in fh:
seqid, cell_ids = line.rstrip().split(",")
cell_idset = {
cell_id
for cell_id in cell_ids.split(":") if cell_id
}
if len(cell_idset) > 0:
idmap[seqid] = cell_idset
newidmap = explode_idmap(idmap, isotypemap)
newisotype = IsotypeTemplate(isotype_names, weight_matrix=None).new
def start_func(n: hdag.HistoryDagNode):
seqid = n.attr["name"]
if seqid in newidmap:
return frozendict({
newisotype(name): len(oldidset)
for name, oldidset in newidmap[seqid].items()
})
else:
return frozendict()
def sumweights(weights: Sequence[Tuple[float, frozendict]]):
ts = [item for i in weights for item in i.keys()]
if ts:
return frozendict({min(ts): 0})
else:
return frozendict()
return hdag.utils.AddFuncDict(
{
"start_func": start_func,
"edge_weight_func": lambda n1, n2: frozendict(),
"accum_func": sumweights,
},
name="Inferred Isotype",
)
def __init__(self, intervals, parts):
super(DataSpace, self).__init__(intervals)
self._parts = frozendict(parts)
def __init__(self, attr_map=None):
if attr_map is None:
attr_map = {}
self._attr_map = frozendict(attr_map)
def __init__(self, intervals, sub_iterators=None, directions=None):
super(IterationSpace, self).__init__(intervals)
self._sub_iterators = frozendict(sub_iterators or {})
self._directions = frozendict(directions or {})
