def _build_custom_structs(): from collections import namedtuple as _nt rest = 'diff_lines emit_edited'.split() ur = _nt('UpdateResult', ('before_entity', 'after_entity', *rest)) cr = _nt('CreateResult', ('created_entity', *rest)) dr = _nt('DeleteResult', ('deleted_entity', *rest)) attrs = 'for_update for_create for_delete'.split() return _nt('These', attrs)(for_update=ur, for_create=cr, for_delete=dr)
def _define_higher_level_functions(): """We are adamant (for now) that the result of indexing a single node tree be a *stream* (iterator) of name-value pairs that might go in to making an index; it's up to the client to consume this stream and cherry-pick (or otherwise process) the elements in a manner appropriate to the use case. Having said that, here's an example all-purpose ting """ def export(): yield 'public', 'tree_index_via', tree_index_via yield 'public', 'pretend_big_index', pretend_big_index yield 'protected', '_named_tuple_for_big_index', _BigIndex def to_node_tree_index_items(self): return ((k, tree_index_via(k, tup)) for k, tup in self.built.items()) def pretend_big_index(root_EID, items): ti = tree_index_via(root_EID, items) return _NOT_SURE(root_EID, ti) def tree_index_via(root_EID, items): slots, cx_of = {k: None for k in simple_fields}, {} for k, val in items: if 'expanded_children' == k: parent_eid, cx_eids = val assert parent_eid not in cx_of cx_of[parent_eid] = cx_eids continue if slots[k] is not None: xx(f"wasn't expecting there to already be a {k!r}") slots[k] = val return _TreeIndex(root_EID, children_of=cx_of, **slots) from collections import namedtuple as _nt _BigIndex = _nt('_BigIndex', ('built', 'cache')) _BigIndex.to_node_tree_index_items = to_node_tree_index_items simple_fields = """ document_depth_minmax overall_depth business_entity_cache """.split() pub_dct, prot_dct = {}, [] for visi, k, v in export(): if 'public' == visi: pub_dct[k] = v continue assert 'protected' == visi prot_dct.append((k, v)) cls = _nt('_FX', pub_dct.keys()) for k, v in prot_dct: setattr(cls, k, v) # BE CAREFUL return cls(** pub_dct)
def _build_soeifjslefj(): from collections import namedtuple as _nt return _nt( 'selfkse', """ component_path emacs_field_height emacs_field_width emacs_field_y emacs_field_x""".split())
class BM(_nt('_Bm', 'tag, status, payload, seeds')): __slots__ = () def identify(self, uid): """ Returns a new message with updated seeds. Previous seeds (if any) are removed and the provided unique id is stored in the appropriate structure (i.e. seeds dict). """ seeds = pm(**{self.tag: se(uid)}) return self._replace(seeds=seeds)
def _read_manifest_plist(manifestplist): ''' read Manifest.plist into a simple object. Here, the toplevel fields of the plist are converted into namedtuple fields. Subfields are left as dictionaries. ''' m = _plistlib.readPlist(manifestplist) # here too, only convert toplevel fields into namedtuple fields keys = sorted(m.keys()) vals = [m[k] for k in keys] manifest = _nt('manifest', ' '.join(keys))(*vals) return manifest
class API(object): Var = _nt("Var", ("name", "type")) Arg = _nt("Arg", ("name", "type", "kind")) Func = _nt("Func", ("name", "args", "returns")) Class = _nt("Class", ("name", "path", "body")) Module = _nt("Module", ("name", "path", "body")) Unknown = _nt("Unknown", ("name", "type", "info"))
def _read_status_plist(statusfile): ''' read Status.plist into a simple object. Here, the toplevel fields of the plist are converted into namedtuple fields. Subfields are left as dictionaries. ''' s = _plistlib.readPlist(statusfile) # only convert toplevel fields into namedtuple fields, since we have # encountered names that are not proper Python identifiers in some of the # subfields (i.e. start with number or underscore) and I do not want to # mangle those names. keys = sorted(s.keys()) vals = [s[k] for k in keys] status = _nt('status', ' '.join(keys))(*vals) return status
def _db_file_records(db): ''' yield all records from the Files table as namedtuples ''' # perform the query q = '''SELECT * FROM Files''' c = db.cursor().execute(q) # check if columns match what we expect colnames = list(zip(*c.description))[0] # we expect the following columns in the Files table _expected_filerec = ('fileID', 'domain', 'relativePath', 'flags', 'file') if colnames != _expected_filerec: raise ValueError('Unexpected table layout for Files table') # create a namedtuple for the unprocessed file records for convenience record = _nt('file', ' '.join(colnames)) for r in c: r = record(*r) # the file column contains a plist that needs additional parsing p = _db_parse_file_column(r.file) # relativePath should match if p.relpath != r.relativePath: raise ValueError('relativePath mismatch!') # the value in the flags field always seems to correspond to the # filetype when derived from the mode field (1 = RegularFile, # 2=Directory, 3=Symlink). Test this here and abort if this assumption # is broken. if (r.flags == 1 and p.filetype != FileType.RegularFile): raise ValueError('assumption broken on flags field') elif (r.flags == 2 and p.filetype != FileType.Directory): raise ValueError('assumption broken on flags field') elif (r.flags == 4 and p.filetype != FileType.Symlink): raise ValueError('assumption broken on flags field') yield _file_entry(r.fileID, r.domain, r.relativePath, p.uid, p.gid, p.mtime, p.ctime, p.btime, p.inode, p.mode, p.filetype, p.permissions, p.size, p.protection, p.extended_attributes, p.linktarget, p.digest)
def load_all_data(self, tm, calib=None, mapping=None): """ Loads all rows of data for a particular target moduel into memory (in the future a selection of modules) Args: tm (int): The slot number of the target module Kwargs: calib (arraylike): an array with calibration coefficient that should be applied to the data mapping (str or arraylike): a string to select a mapping or an array with the mapping ['ssl2colrow','ssl2asic_ch','raw'] """ if calib is None: calib = 1.0 if mapping is None: mapping = self.map elif isinstance(mapping, str): if mapping == "raw": mapping = np.arange(N_TM_PIX) else: try: mapping = ss_mappings.__getattribute__(mapping) except: raise ValueError("No mapping found with name %s" % mapping) amps = np.zeros((self.n_readouts, N_TM_PIX)) time = np.zeros(self.n_readouts, dtype=np.uint64) cpu_t = np.zeros(self.n_readouts) iro = np.zeros(self.n_readouts, dtype=np.uint64) for i, r in enumerate(self.read()): amps[i, :] = self.raw_data[tm, :] * calib time[i] = self.time cpu_t[i] = self.cpu_t iro[i] = self.iro amps = amps[:, mapping] ssdata = _nt("ssdata", "iro amps time cpu_t tm") return ssdata(iro, amps, time, cpu_t, tm)
def _build_custom_structs(): from collections import namedtuple as _nt these = ('emit_edited', ) fd = _nt('DeleteResult', ('deleted_entity', *these)) return _nt('These', ('for_delete', ))(fd)
if '\n' == char: scn.advance_by_one() assert scn.empty break if '"' == char: xx("have fun parsing escaped quotes") xx(f"unexpected character {char!r}") return spans_via_line # == Models _MinimalSchema = _nt('MiniamSchema', ('field_name_keys', )) _MinimalEntity = _nt('MinimalEntity', ('core_attributes', )) def _AST_definitions(): yield 'line', 'sexps', 'as', 'cells', 's', 'as', 'line_tail' yield 'cell', 's', 's', 'as', 'cell_content' # == Low-level might abstract def _build_stream_join(sep): def stream_join(itr): # :[#612.4] near [#611] things we do with scanners scn = func(itr) yield scn.next() # ..
os.close(fd) except: pass if not cr: cr = (env.get('LINES', 25), env.get('COLUMNS', 80)) return int(cr[1]), int(cr[0]) # ===================================================================== class ShebangNotFoundError(Exception): pass Extension = _nt("Extension", ["name", "interpreters"]) Extension.__doc__ = "The extension that will have all the associated interpreters." class Interpreter(object): """The Interpreter""" def __init__(self, name, version='', extension='', default=False, paths=None): # type: (str, str, str, bool, List[InterpreterPath]) -> None """Constructor. :param name: the name of the interpreter
yield 'natural_key', None yield 'hierarchical_container_type', use_HCT yield 'heading', use_heading yield 'document_datetime', None yield 'body', use_body yield 'children', use_children yield 'next', use_next yield 'annotated_entity_revisions', None def _index_the_graph(graph): kw = {k: v for k, v in _do_index_the_graph(graph)} return _GraphIndex(**kw) _GraphIndex = _nt('_GraphIndex', ('children_of', 'parent_of')) def _do_index_the_graph(graph): children_of, parent_of = {}, {} def _add_parent_child(parent_EID, child_EID): if (arr := children_of.get(parent_EID)) is None: children_of[parent_EID] = (arr := []) assert child_EID not in arr arr.append(child_EID) assert child_EID not in parent_of parent_of[child_EID] = parent_EID
#!/usr/bin/env python3 #coding=utf8 from . import * from collections import namedtuple as _nt Vector3 = _nt('Vector3', 'x y z') def vector3(factory): return Vector3(single(factory), single(factory), single(factory)) add_reader(vector3, 'Microsoft.Xna.Framework.Content.Vector3Reader', 'Microsoft.Xna.Framework.Vector3', True) Rectangle = _nt('Rectangle', 'x y w h') def rectangle(factory): return Rectangle(i32(factory), i32(factory), i32(factory), i32(factory)) add_reader(rectangle, 'Microsoft.Xna.Framework.Content.RectangleReader', 'Microsoft.Xna.Framework.Rectangle', True)
def statistitican_via_collection_path(coll_path): def document_commits_via_title(vendor_document_title): """(before #history-B.4 we could get the history in one commit with a JOIN. but now (to accomodate rigged documents) we do it in two which is fine.) """ c = execute( 'SELECT notecard_based_document_ID, just_kidding_document_type ' 'FROM notecard_based_document ' 'WHERE document_title_from_vendor=?', (vendor_document_title, )) # Maybe we have no record of this document at all (strange) first_row = c.fetchone() if first_row is None: return assert c.fetchone() is None docu_ID, typ, = first_row if 'docu_type_common' == typ: return for_notecard_based_document(docu_ID) assert 'docu_type_rigged' == typ return for_rigged_document(vendor_document_title) # ick/meh def for_rigged_document(vendor_document_title): c = execute( 'SELECT RDC.* ' 'FROM rigged_document_commit AS RDC ' 'JOIN rigged_document AS RD USING (rigged_document_ID) ' 'WHERE RD.document_title_from_vendor=? ' 'ORDER BY datetime(RDC.normal_datetime) ', (vendor_document_title, )) # (we want to make it be commit-graph order not chrono order, # but not badly enough to do it knowing that it's not covered) def mutable_threes(): while True: row = c.fetchone() if not row: break rec = RD_commit_record(*row) dt = datetime_via_record(rec) yield [dt, 'edit', rec] scn = _scanner_via_iterator(mutable_threes()) # If there are no commits in the database for this docu, strange if scn.empty: return return docu_CIs_via_threes_scanner(scn, 'docu_type_rigged') def for_notecard_based_document(docu_ID): c = execute( 'SELECT NBDC.* FROM notecard_based_document_commit as NBDC ' 'WHERE NBDC.notecard_based_document_ID=? ' 'ORDER BY datetime(NBDC.normal_datetime)', (docu_ID, )) def mutable_threes(): while True: row = c.fetchone() if not row: break rec = NB_commit_rec_via_row(row) dt = datetime_via_record(rec) yield [dt, 'edit', rec] scn = _scanner_via_iterator(mutable_threes()) # If there are no commits in the database for this docu, strange if scn.empty: return return docu_CIs_via_threes_scanner(scn, 'docu_type_common') def docu_CIs_via_threes_scanner(scn, typ): scn.peek[1] = 'create' # meh while True: three = scn.next() yield _DocumentCommit(*three, typ) if scn.empty: break # Datetime via record def datetime_via_record(rec): rec.tzinfo # hi return strptime(rec.normal_datetime, '%Y-%m-%d %H:%M:%S') from datetime import datetime as _ strptime = _.strptime # Connect to database from pho.document_history_._model import \ database_via_collection_path_ as func # (it's a sibling file to us but we are an entrypoint file) db = func(coll_path) assert db # Prepare statistics sing = db.singleton_text k = 'mean_and_std' two_as_string = sing.get(k) if two_as_string is None: xx(f"Did you generate the statistics? Not found: {k!r}") mean_s, std_s = two_as_string.split(' ') mean, std = float(mean_s), float(std_s) k = 'mean_and_std_for_rigged' two_as_string = sing.get(k) if two_as_string is None: xx(f"Did you generate the statistics? Not found: {k!r}") mean_s, std_s = two_as_string.split(' ') mean_for_rigged, std_for_rigged = float(mean_s), float(std_s) from pho.document_history_._model import \ RiggedDocumentCommitRecord_ as RD_commit_record NB_commit_rec_via_row = \ db.notecard_based_document_commit_table.NBD_CI_via_row_ execute = db.conn.execute # == BEGIN meh from dataclasses import dataclass as _dataclass from collections import namedtuple as _nt @_dataclass class _Statistician: mean: float std: float mean_for_rigged: float std_for_rigged: float document_commits_via_title: callable db: object _DocumentCommit = _nt('_DocumentCommit', ('datetime', 'verb', 'record', 'document_type')) # == END return _Statistician(mean=mean, std=std, mean_for_rigged=mean_for_rigged, std_for_rigged=std_for_rigged, document_commits_via_title=document_commits_via_title, db=db)
class AbstractSTScan(): """For testing and verification. Coordinates are as usual, but timestamps are just float values, with 0 being the end time, and e.g. 10 being 10 units into the past. """ def __init__(self, coords, timestamps): self.coords = _np.asarray(coords) self.timestamps = _np.asarray(timestamps) if len(self.timestamps) != self.coords.shape[1]: raise ValueError( "Timestamps and Coordinates must be of same length.") self._sort_times_increasing() self._unique_points = self._make_unique_points() self.geographic_radius_limit = 100 self.geographic_population_limit = 0.5 self.time_max_interval = 28 self.time_population_limit = 0.5 self.only_full_disks = False def _sort_times_increasing(self): self.arg_sort = _np.argsort(self.timestamps) self.timestamps = self.timestamps[self.arg_sort] self.coords = self.coords[:, self.arg_sort] def allowed_times_into_past(self): """Find the times into the past which satisfy the constraints of maximum time interval, and maximum time population. :return: Array of times into the past, in increasing order. """ mask = self.timestamps <= self.time_max_interval if not _np.any(mask): return [] times = _np.unique(self.timestamps[mask]) times.sort() index = len(times) - 1 cutoff = times[index] maxsize = int(self.time_population_limit * self.timestamps.shape[0]) while _np.sum(self.timestamps <= cutoff) > maxsize: index -= 1 if index == -1: return [] cutoff = times[index] return times[:index + 1] Disc = _nt("Disc", ["centre", "radius_sq", "mask"]) def _make_unique_points(self): """Return an array of the unique coordinates.""" return _np.array(list(set((x, y) for x, y in self.coords.T))).T @staticmethod def _product(s): if len(s) == 1: yield s else: for i in range(1, len(s) + 1): yield from _itertools.combinations(s, i) def all_discs_around(self, centre): """Find all discs around the centre. Applies the rules above: no disc contains a single point, and the rule about boundary cases. Is a generator, yields pairs (radius**2, mask) """ centre = _np.asarray(centre) limit = self.timestamps.shape[0] * self.geographic_population_limit if self.only_full_disks: distsqun = _np.sum((self.coords - centre[:, None])**2, axis=0) uniques = _np.unique(distsqun) uniques = uniques[uniques <= self.geographic_radius_limit**2] uniques.sort() for d in uniques: mask = distsqun <= d if _np.sum(mask) > limit: return yield (d, mask) return distsqun = _np.sum((self._unique_points - centre[:, None])**2, axis=0) index_array = _np.arange(len(distsqun)) uniques = _np.unique(distsqun) uniques = uniques[uniques <= self.geographic_radius_limit**2] uniques.sort() # Zero case mask = (self.coords[0] == centre[0]) & (self.coords[1] == centre[1]) count = _np.sum(mask) if count > 1: if count > limit: return yield (0, mask) current_mask = mask for d in uniques[1:]: new_indices = index_array[distsqun == d] seen_too_large = False new_mask = current_mask.copy() for to_add in self._product(new_indices): mask = current_mask.copy() for i in to_add: mask |= ((self.coords[0] == self._unique_points[0][i]) & (self.coords[1] == self._unique_points[1][i])) new_mask |= mask if _np.sum(mask) > limit: seen_too_large = True else: yield (d, mask) if seen_too_large: return current_mask = new_mask def all_discs(self): """Generate all discs according to the rules. Is a generator, yielding Disc objects. """ all_masks = set() for centre in self._unique_points.T: for rr, mask in self.all_discs_around(centre): m = tuple(mask) if m not in all_masks: yield self.Disc(centre, rr, mask) all_masks.add(m) Result = _nt("Result", ["centre", "radius", "mask", "time", "statistic"]) def build_times_cutoff(self): """Returns pair (times, cutoff) where `times` is an array of all valid times into the past to test, in increasing order, and `cutoff[i]` is the greatest index, plus one, into `self.timestamps` whose value is `<= times[i]`. """ times = self.allowed_times_into_past() cutoff = [] i = 0 for t in times: while i < self.timestamps.shape[0] and self.timestamps[i] <= t: i += 1 if i == self.timestamps.shape[0]: cutoff.append(self.timestamps.shape[0]) else: cutoff.append(i) return times, cutoff def score_clusters(self): """A generator returning triples `(disc, time, statistic)` describing each cluster of relevance. """ N = self.timestamps.shape[0] times, cutoff = self.build_times_cutoff() for disc in self.all_discs(): space_count = _np.sum(disc.mask) / N for c in cutoff: actual = _np.sum(disc.mask[:c]) expected = space_count * c if actual > 1 and actual > expected: yield (disc, self.timestamps[c - 1], self._statistic(actual, expected, N)) @staticmethod def _not_intersecting(all_clusters, cluster): return [ cc for cc in all_clusters if _np.sum((cc.centre - cluster.centre)**2) > (cluster.radius + cc.radius)**2 ] def find_all_clusters(self): """Find all the disjoint clusters from most to least significant.""" all_clusters = [ self.Result(centre=c[0].centre, radius=_np.sqrt(c[0].radius_sq), mask=c[0].mask, time=c[1], statistic=c[2]) for c in self.score_clusters() ] all_clusters.sort(key=lambda r: -r.statistic) while len(all_clusters) > 0: c = all_clusters[0] yield c all_clusters = self._not_intersecting(all_clusters, c) @staticmethod def _statistic(actual, expected, total): """Calculate the log likelihood""" stat = actual * (_np.log(actual) - _np.log(expected)) stat += (total - actual) * (_np.log(total - actual) - _np.log(total - expected)) return stat def to_satscan(self, filename, offset): """Writes the training data to two SaTScan compatible files. Does *not* currently write settings, so these will need to be entered manually. The timestamps are rounded down to an integer. :param filename: Saves files "filename.geo" and "filename.cas" containing the geometry and "cases" repsectively. :param offset: The "end time" in generic units, from which the `timestamps` are subtracted. """ self.write_to_satscan(filename, offset, self.coords, self.timestamps) @staticmethod def write_to_satscan(filename, offset, coords, timestamps): unique_coords = list(set((x, y) for x, y in coords.T)) with open(filename + ".geo", "w") as geofile: for i, (x, y) in enumerate(unique_coords): print("{}\t{}\t{}".format(i + 1, x, y), file=geofile) unique_times = list(set(t for t in timestamps)) with open(filename + ".cas", "w") as casefile: for i, (t) in enumerate(unique_times): pts = coords.T[timestamps == t] pts = [(x, y) for x, y in pts] import collections c = collections.Counter(pts) for pt in c: index = unique_coords.index(pt) print("{}\t{}\t{}".format(index + 1, c[pt], int(offset - t)), file=casefile)
return _TempCommitRecord(*row) def _create_table_(self): self._execute("""CREATE TEMPORARY TABLE commit_queue ( SHA TEXT PRIMARY KEY, parent_SHA TEXT, child_SHA TEXT, datetime TEXT NOT NULL, message_indented TEXT NOT NULL, file_paths TEXT NOT NULL)""") self._commit() # ?? return _TempCommitRecord = _nt( '_TempCommitRecord', 'SHA parent_SHA child_SHA datetime message_indented file_paths'.split()) class _CommitTable(_Table): def accept_commit(self, temp_rec, parent_ID): SHA, parent_SHA, child_SHA, datetime, message_indented, file_paths = \ temp_rec return self.insert_commit(SHA, datetime, message_indented, parent_ID=parent_ID) def insert_commit(self, SHA, datetime,
_my_assert_sequence_equal(self, act_scn, exp_scn) do_debug = False def _schema_via_sexp(sx): stack = list(reversed(sx)) assert 'schema' == stack.pop() kwargs = {} while stack: k = stack.pop() kwargs[k] = stack.pop() return _Schema(**kwargs) _EndStateSchemaAndEntitySTREAM = _nt('ES01', ('schema', 'entity_STREAM')) _Schema = _nt('MinimalSchema', ('field_name_keys', )) class _JustEnoughEntity: def __init__(self, dct): self.core_attributes = dct def __eq__(self, otr): cad = otr.core_attributes return self.core_attributes == cad # == Low-level and likely to abstract
# Last item only: smunge that ']' on to there line = ''.join((last_line_of_previous, ']\n')) yield line def _build_identifier_builder(_listener, _cstacker=None): def iden_via_primitive(x): # #[#877.4] this might become default assert isinstance(x, str) assert len(x) return x return iden_via_primitive _JustEnoughEntity = _nt('JustEnoughEntity', ('core_attributes', )) _MinimalSchema = _nt('MinimalSchema', ('field_name_keys', )) def _scnlib(): import text_lib.magnetics.scanner_via as module return module def xx(msg=None): raise RuntimeError(msg or "wee") # #history-B.1 initial spike # #born as nonworking stub
def __sub__(self, a): if isinstance(a, IP4Addr): raise TypeError("Can't subtract IP4Addr from IP6Addr") return IP6Addr(long.__sub__(self, a)) def __and__(self, a): if isinstance(a, IP4Addr): raise TypeError("Can't do logical 'and' of IP6Addr and IP4Addr") return IP6Addr(long.__and__(self, a)) def __or__(self, a): if isinstance(a, IP4Addr): raise TypeError("Can't do logical 'or' of IP6Addr and IP4Addr") return IP6Addr(long.__or__(self, a)) IPRangeParentType = _nt("IPRange", ["start", "end"]) class IPRange(IPRangeParentType): __slots__ = () def __new__(cls, param1, param2=None): """ Params may be: start, end - first and last address of a range as IPAddr, string or number string - string in "prefix" format, e.g. 1.2.3.0/24 """ if param2 is None: return cls.fromString(param1) else: return cls.fromPair(param1, param2)
''' raised when extraction fails for some reason ''' pass class MbdbParseError(PearBackError): ''' raised when parsing of older Manifest.mbdb fails ''' pass # instead of defining full classes, I use namedtuples for simplicity # a backup object consists of rootdir, connection to Manifest.db (None when # dealing with backups from devices with iOS9 or below), parsed # status.plist and manifest.plist and a partial generator function for # iterating over parsed file-records from Manifest.db _backup = _nt('iOSbackup', 'backuptype rootdir db status manifest filerecords') # a simple object for representing a single entry in the Files table _file_entry = _nt( 'file_entry', 'fileID domain relativePath uid gid ' 'mtime ctime btime inode mode filetype ' 'permissions size protection ' 'extended_attributes linktarget digest') # namedtuple for representing the values in the plist in the file field in iOS10 # Manifest.db databases _plistvals = _nt( 'plist_values', 'uid gid mtime ctime btime inode mode ' 'filetype permissions size protection ' 'relpath extended_attributes linktarget ' 'digest')
yield value def resolve_variable_value_once(var_name): return _resolve_variable_value_once(var_name, memo, bcoll, listener) def resolve_function_name_and_args(): return _parse_function_call(expression_string, listener) stop = _Stop try: return main() except stop: pass _Funco = _nt('_Funco', ('num_args', 'func', 'locator', 'memo_yikes')) def _load_function_once(func_name, memo, bcoll, listener): dct = _produce_raw_dict(memo, 'value_function', 'funcs_as_unparsed_dict', 'VALUE_FUNCTION_RIGHT_HAND_SIDES', bcoll, listener) # Get func locator expression right hand side as it appeared in schema.rec rhs = dct.get(func_name) if rhs is None: from pho.magnetics_.text_via import oxford_join as func or_list = func(dct.keys(), ' or ') # does repr xx(f"No function {func_name!r}. Did you mean {or_list}?") # Split a function locator like this "foo.bar.baz" into "foo.bar", "baz" md = _function_locator_rx.match(rhs)
lexer = lex.lex() #if sourceCode is not None: # lexer.input(sourceCode) # if __debug__: # for tok in lexer: # #print(tok.value, end=" ") # print("{0:12} {1}".format(tok.value, tok.type)) # #print(tok.value) # print() # TODO nested scopes, symbol tables # TODO simplify resulting structure to eliminate artifacts of handwiring? from collections import namedtuple as _nt ASTLiteralInteger = _nt("ASTLiteral", ["value"]) ASTIdentifier = _nt("ASTIdentifier", ["name"]) ASTApplication = _nt("ASTApplication", ["function", "argument"]) ASTAbstraction = _nt("ASTAbstraction", ["parameter", "body"]) ASTLet = _nt("ASTLet", ["variable", "value", "expression"]) def p_literalinteger(p): "expression : LITERALINTEGER" p[0] = ASTLiteralInteger(int(p[1])) def p_identifier(p): "expression : IDENTIFIER" p[0] = ASTIdentifier(p[1]) def p_application(p): "expression : APPLICATOR expression expression"
slot_w = w - _hand_w - _lb_w - _rb_w # #here1 assert 0 < slot_w # assume concretization negotiated fairly return ItemContructors() def _FFSA(fsa_def): # copy-paste-modify. memoize the FFSA into its own func if not hasattr(fsa_def, '_FFSA_'): from modality_agnostic.magnetics.formal_state_machine_via_definition \ import build_formal_FSA_via_definition_function as func wti = fsa_def.where_to_insert() fsa_def._FFSA_ = func(__name__, fsa_def, where_to_insert=wti) return fsa_def._FFSA_ _min_width_and_value = _nt('_min_w_and_val', ('minimum_width', 'value')) _hand_w = _piece_via_has_focus.width _this_ratio = 2.75 # keymashing this looked fine: 8 chars : 22 chars # Imagining: ' 👉 [ chamoochie]:[fachoochie ]' ' 👉 chamoochie : fachoochie ' # such that: [hand][left bracket][name slot][spine][value slot][right bracket] _lb_w = 1 # left bracket width _rb_w = _lb_w _spine_w = _rb_w + len(':') + _lb_w def xx(msg=None): raise RuntimeError(''.join(('cover me', *((': ', msg) if msg else ()))))
class STScanNumpy(): """For testing and verification; numpy accelerated. Coordinates are as usual, but timestamps are just float values, with 0 being the end time, and e.g. 10 being 10 units into the past. """ def __init__(self, coords, timestamps): self.coords = _np.asarray(coords) self.timestamps = _np.asarray(timestamps) if len(self.timestamps) != self.coords.shape[1]: raise ValueError( "Timestamps and Coordinates must be of same length.") self._sort_times_increasing() self.geographic_radius_limit = 100 self.geographic_population_limit = 0.5 self.time_max_interval = 28 self.time_population_limit = 0.5 self._cache_N = 0 def _sort_times_increasing(self): arg_sort = _np.argsort(self.timestamps) self.timestamps = self.timestamps[arg_sort] self.coords = self.coords[:, arg_sort] def make_time_ranges(self): """Compute the posssible time intervals. :return: Tuple of masks (of shape (N,k) where N is the number of data points), counts (of length k) and the cutoff used for each count (of length k). Hence `masks[:,i]` corresponds to `count[i]` is given by looking at event `<= cutoff[i]` before the end of time. """ unique_times = _np.unique(self.timestamps) unique_times = unique_times[unique_times <= self.time_max_interval] unique_times.sort() time_masks = self.timestamps[:, None] <= unique_times[None, :] limit = self.timestamps.shape[0] * self.time_population_limit time_counts = _np.sum(time_masks, axis=0) m = time_counts <= limit return time_masks[:, m], time_counts[m], unique_times[m] def find_discs(self, centre): """Compute the possible disks. :return: Tuple of masks (of shape (N,k) where N is the number of data points), counts (of length k) and the distances squared from the centre point (of length k). Hence `masks[:,i]` corresponds to `count[i]` is given by looking at event `<= cutoff[i]` before the end of time. """ centre = _np.asarray(centre) distsq = _np.sum((self.coords - centre[:, None])**2, axis=0) unique_dists = _np.unique(distsq) unique_dists = unique_dists[ unique_dists <= self.geographic_radius_limit**2] mask = distsq[:, None] <= unique_dists[None, :] limit = self.timestamps.shape[0] * self.geographic_population_limit space_counts = _np.sum(mask, axis=0) m = (space_counts > 1) & (space_counts <= limit) return mask[:, m], space_counts[m], unique_dists[m] @staticmethod def _calc_actual(space_masks, time_masks, time_counts): # Does this, but >9 times quicker: # uber_mask = space_masks[:,:,None] & time_masks[:,None,:] # actual = _np.sum(uber_mask, axis=0) x = _np.empty((space_masks.shape[1], time_masks.shape[1])) # This is better, but still >20 times slower... #for i, c in enumerate(time_counts): # x[:,i] = _np.sum(space_masks[:c,:], axis=0) current_sum = _np.zeros(space_masks.shape[1]) current_column = 0 for i, c in enumerate(time_counts): while current_column < c: current_sum += space_masks[current_column, :] current_column += 1 x[:, i] = current_sum return x def faster_score_all_new(self): """As :method:`score_all` but yields tuples (centre, distance_array, time_array, statistic_array).""" time_masks, time_counts, times = self.make_time_ranges() N = self.timestamps.shape[0] for centre in self.coords.T: space_masks, space_counts, dists = self.find_discs(centre) actual = self._calc_actual(space_masks, time_masks, time_counts) expected = space_counts[:, None] * time_counts[None, :] / N _mask = (actual > 1) & (actual > expected) actual = _np.ma.array(actual, mask=~_mask) expected = _np.ma.array(expected, mask=~_mask) stats = self._ma_statistic(actual, expected, N) _mask1 = _np.any(_mask, axis=1) if not _np.any(_mask1): continue m = _np.ma.argmax(stats, axis=1)[_mask1] stats = stats[_mask1, :] stats = stats[range(stats.shape[0]), m].data used_dists = dists[_mask1] used_times = times[m] yield centre, used_dists, used_times, stats @staticmethod def _ma_statistic(actual, expected, total): """Calculate the log likelihood""" stat = actual * (_np.ma.log(actual) - _np.ma.log(expected)) stat += (total - actual) * (_np.ma.log(total - actual) - _np.ma.log(total - expected)) return stat @staticmethod def _calc_actual1(space_masks, time_masks, time_counts): x = _np.empty((space_masks.shape[1], time_masks.shape[1]), dtype=_np.int) current_sum = _np.zeros(space_masks.shape[1], dtype=_np.int) current_column = 0 for i, c in enumerate(time_counts): while current_column < c: current_sum += space_masks[current_column, :] current_column += 1 x[:, i] = current_sum return x def faster_score_all(self): """As :method:`score_all` but yields tuples (centre, distance_array, time_array, statistic_array).""" time_masks, time_counts, times = self.make_time_ranges() N = self.timestamps.shape[0] for centre in self.coords.T: space_masks, space_counts, dists = self.find_discs(centre) actual = self._calc_actual1(space_masks, time_masks, time_counts) stcounts = space_counts[:, None] * time_counts[None, :] _mask = (actual > 1) & (N * actual > stcounts) stats = self._ma_statistics_lookup(space_counts, time_counts, stcounts, actual, _mask, N) _mask1 = _np.any(_mask, axis=1) if not _np.any(_mask1): continue m = _np.ma.argmax(stats, axis=1)[_mask1] stats = stats[_mask1, :] stats = stats[range(stats.shape[0]), m].data used_dists = dists[_mask1] used_times = times[m] yield centre, used_dists, used_times, stats @staticmethod def _build_log_lookup(N): lookup = _np.empty(N + 1, dtype=_np.float64) lookup[0] = 1 for i in range(1, N + 1): lookup[i] = i return _np.log(lookup) def _ma_statistics_lookup(self, space_counts, time_counts, stcounts, actual, _mask, N): # Faster version which uses lookup tables if self._cache_N != N: self._cache_N = N self._log_lookup = self._build_log_lookup(N) if N > 2000: self._log_lookup2 = None else: self._log_lookup2 = self._build_log_lookup(N * N) sl = self._log_lookup[space_counts] tl = self._log_lookup[time_counts] y = actual * (self._log_lookup[actual] - sl[:, None] - tl[None, :]) if self._log_lookup2 is None: yy = (N - actual) * (self._log_lookup[N - actual] - _np.log(N * N - stcounts)) else: yy = (N - actual) * (self._log_lookup[N - actual] - self._log_lookup2[N * N - stcounts]) return _np.ma.array(y + yy + N * _np.log(N), mask=~_mask) def faster_score_all_old(self): """As :method:`score_all` but yields tuples (centre, distance_array, time_array, statistic_array).""" time_masks, time_counts, times = self.make_time_ranges() N = self.timestamps.shape[0] for centre in self.coords.T: space_masks, space_counts, dists = self.find_discs(centre) uber_mask = space_masks[:, :, None] & time_masks[:, None, :] actual = _np.sum(uber_mask, axis=0) expected = space_counts[:, None] * time_counts[None, :] / N _mask = (actual > 1) & (actual > expected) used_dists = _np.broadcast_to(dists[:, None], _mask.shape)[_mask] used_times = _np.broadcast_to(times[None, :], _mask.shape)[_mask] actual = actual[_mask] expected = expected[_mask] stats = AbstractSTScan._statistic(actual, expected, N) if len(stats) > 0: yield centre, used_dists, used_times, stats def score_all(self): """Consider all possible space and time regions (which may include many essentially repeated disks) and yield tuples of the centre of disk, the radius squared of the disk, the time span of the region, and the statistic. """ for centre, dists, times, stats in self.faster_score_all(): for d, t, s in zip(dists, times, stats): yield centre, d, t, s @staticmethod def _not_intersecting(scores, centre, radius): return [ cc for cc in scores if (cc[0] - centre[0])**2 + (cc[1] - centre[1])**2 > (radius + cc[2])**2 ] Result = _nt("Result", ["centre", "radius", "time", "statistic"]) def find_all_clusters(self): scores = [] count = 0 for centre, dists, times, stats in self.faster_score_all(): dists = _np.sqrt(dists) scores.extend( zip(_itertools.repeat(centre[0]), _itertools.repeat(centre[1]), dists, times, stats)) count += 1 if len(scores) == 0: return scores = _np.asarray(scores) if len(scores.shape) == 1: scores = scores[None, :] scores = scores[_np.argsort(-scores[:, 4]), :] while scores.shape[0] > 0: best = scores[0] centre = _np.asarray([best[0], best[1]]) radius = best[2] yield self.Result(centre=centre, radius=radius, time=best[3], statistic=best[4]) distances = (scores[:, 0] - best[0])**2 + (scores[:, 1] - best[1])**2 mask = distances > (radius + scores[:, 2])**2 scores = scores[mask, :]
if has: intersections = _intersections(use_insides, use_overhangs, use_kissing) left_thing = left_scn.next() yield left_thing, intersections _when_overhang = { # key, do consume, do stay (False, False): None, (False, True): ('at_stop', False, False), (True, False): ('at_start', True, True), (True, True): ('full', False, False), } _intersections = _nt('_Intersections', ('inside_or_flush', 'overhangs', 'kissing')) def _relationship(one, other): if one < other: return 'overhang' if other < one: return 'inside' assert one == other return 'equal' def _check_ranges(itr): tup = next(itr) start, prev_stop = tup[:2] assert start <= prev_stop
#!/usr/bin/env python3 -tt """ File: point.h ------------- Exports a type representing 2- and 3-dimensional points. These points inherit from tuples, so you can do anything you could otherwise do with tuples (notably, tuple unpacking) Usage: origin = Point() pt = Point(4, 1) x = origin.x # => x = 0 y = pt.y # => y = 1 origin3 = Point3() pt = Point3(3, 4, 5) x, y, z = pt """ from collections import namedtuple as _nt Point = _nt('Point', ['x', 'y']) Point3 = _nt('Point', ['x', 'y', 'z'])
def condition_two(): pass def condition_three(): pass def no_see(): self.fail("no see") listener, emis = em.listener_and_emissions_for(self) case = subject_fellow()(listener) val = case(when_chim_churry_is) assert val is None emi, = emis return emi.to_messages() EndState = _nt('EndState', ('emissions', 'recordings', 'result_value')) def subject_fellow(): from kiss_rdb.magnetics_.collection_via_path import _build_case_function return _build_case_function if __name__ == '__main__': unittest.main() # #born
yield ('--from-arg=<arg>*', 'EXPERIMENTAL pass thru to producer script (yikes)') # (using '-t' is sad, we would rather have again '-f' like ffmpeg # but that would require custom arg parsing that is way out of scope. # it's basically `[opts] arg` twice. #wish [#459.T]) yield '-t', f'{_to_monikers.format_flag}=<fmt>', _same_help from data_pipes.cli import this_screen_ as this_screen yield '-h', '--help', this_screen yield _from_monikers.arg, 'the collection the data comes from' yield _to_monikers.arg, 'the collection the data goes to' _monikers = _nt('Monikers', ('arg', 'format_flag', 'STDIN_STDOUT')) _from_monikers = _monikers('FROM_COLLECTION', '--from-format', 'STDIN') _to_monikers = _monikers('TO_COLLECTION', '--to-format', 'STDOUT') _same_help = "(or will try to infer from file extension if present)" def BUILD_COLLECTION_MAPPER(stderr, vals, foz, rscser): """Harness the power of pipes…""" # .. """EXPERIMENTAL. Created as a cheap-and-easy way to create and populate a collection with a producer script or similar. With FROM_COLLECTION of "-", lines of the indicated format are read from STDIN. With TO_COLLECTION of "-", lines of the specified format are
# Type used when actual type cannot be determined. # While typing.Any could be used here, and would be valid # we need a distinction between explicitly added typing.Any # so typing additions can be treated differently to typing changes. UNKNOWN = "~unknown" # fmt: off class Kind(object): POSITIONAL = 0b0001 KEYWORD = 0b0010 VARIADIC = 0b0100 DEFAULT = 0b1000 class API(object): Var = _nt("Var", ("name", "type")) Arg = _nt("Arg", ("name", "type", "kind")) Func = _nt("Func", ("name", "args", "returns")) Class = _nt("Class", ("name", "path", "body")) Module = _nt("Module", ("name", "path", "body")) Unknown = _nt("Unknown", ("name", "type", "info")) # fmt: on Change = _nt("Change", ("level", "type", "info")) TYPE_CHARS = r"\w[\w\.]*(?:\[[\w\.,\[\] ]*\])?"
from datetime import datetime import numpy as np import struct from collections import namedtuple as _nt import sys N_TM = 32 # Number of target modules in camera N_TM_PIX = 64 # Number of pixels on a Target Module N_BYTES_NUM = 8 # Number of bytes to encode numbers (uint and float) in the SSReadout N_CAM_PIX = N_TM * N_TM_PIX # Number of pixels in the camera # fmt: off _SSMappings = _nt("SSMappings", "ssl2colrow ssl2asic_ch") ss_mappings = _SSMappings( np.array([ 8, 1, 5, 6, 41, 43, 51, 47, 11, 14, 13, 7, 44, 36, 42, 46,
""" Created on 31 Jul 2011 @author: D.W. """ import wx from collections import namedtuple as _nt _id_text= _nt("id_text", "menu toolbar status menu_kind") #=============================================================================== # File ID #=============================================================================== ID_NEW= wx.NewId() id_text_new= _id_text("New\tCtrl+N", "New", "Create a new file", wx.ITEM_NORMAL) ID_OPEN= wx.NewId() id_text_open= _id_text("Open\tCtrl+O", "Open", "Open an existing file", wx.ITEM_NORMAL) ID_SAVE= wx.NewId() id_text_save= _id_text("Save\tCtrl+S", "Save", "Save the current file", wx.ITEM_NORMAL) ID_SAVEAS= wx.NewId() id_text_saveas= _id_text("Save as", "Save as", "Save as a new filename", wx.ITEM_NORMAL) ID_CLOSETAB= wx.NewId()