def ngram_loader(n, lang='en', query=['counts', 'order', 'frequencies', 'sum'], sep=' '): ngrams = {} ret = {} file = pkg_resources.resource_filename( __name__, 'data/' + lang + '/' + ngramfiles[n]) for line in open(file): key, count = line.split(sep) key = key.lower() ngrams[key] = int(count) L = len(key) N = sum([ngrams[k] for k in ngrams]) #print (file) if 'counts' in query: ret['counts'] = shallowcopy(ngrams) if 'order' in query: ret['order'] = [ k[0] for k in sorted([[k, ngrams[k]] for k in ngrams], key=lambda x: x[1]) ] ret['order'].reverse() if 'frequencies' in query: for ngram in ngrams: ngrams[ngram] = ngrams[ngram] / N ret['frequencies'] = shallowcopy(ngrams) if 'sum' in query: ret['sum'] = N return ret
def copy(self): ''' Create a copy of this object and return it. ''' new_me = self.__class__() for name in self._valid_attrs.keys(): if name in self._alias_attrs: continue new_me._attributes[name] = shallowcopy(self._attributes[name]) new_me._attr_defaults[name] = shallowcopy( self._attr_defaults[name]) new_me._loader = self._loader new_me._variable_manager = self._variable_manager new_me._validated = self._validated new_me._finalized = self._finalized new_me._uuid = self._uuid # if the ds value was set on the object, copy it to the new copy too if hasattr(self, '_ds'): new_me._ds = self._ds return new_me
def copy(x): if not x._son: return shallowcopy(x) else: x_ = shallowcopy(x) x_._son = [Ego.copy(t) if isinstance(t, Ego) else t for t in x._son] return x_
def copy(self): ''' Create a copy of this object and return it. ''' try: new_me = self.__class__() except RuntimeError as e: raise AnsibleError( "Exceeded maximum object depth. This may have been caused by excessive role recursion", orig_exc=e) for name in self._valid_attrs.keys(): if name in self._alias_attrs: continue new_me._attributes[name] = shallowcopy(self._attributes[name]) new_me._attr_defaults[name] = shallowcopy( self._attr_defaults[name]) new_me._loader = self._loader new_me._variable_manager = self._variable_manager new_me._validated = self._validated new_me._finalized = self._finalized new_me._uuid = self._uuid # if the ds value was set on the object, copy it to the new copy too if hasattr(self, '_ds'): new_me._ds = self._ds return new_me
def signal(self, frame: Union[Beads, Cycles], key=None) -> np.ndarray: "returns the signal to subtract from beads" task = self.task next(iter(frame.keys())) # unlazyfy # type: ignore # Discard the FixedBeadDetectionProcessor from the # frame as it will created an exception which is # meaningless in this context frame = shallowcopy(frame) frame.data = shallowcopy(frame.data) frame.data.actions = [ i for i in frame.data.actions if 'FixedBeadDetectionProcessor' not in str(i) ] data = cast(Dict, frame.data) itr = (cast(Iterable[int], task.beads) if key is None else cast( Iterable[int], list(zip(task.beads, repeat(key))))) if len(task.beads) == 0: sub = 0. elif len(task.beads) == 1: sub = np.copy(data[task.beads[0]]) else: sub = task.agg.process(itr, frame) return task.filter(sub) if task.filter else sub
def _iter(self, sel=None) -> Iterator[Tuple[int, np.ndarray]]: if isinstance(self.data, Beads) and self.cycles is None: beads = cast(Beads, self.data) if sel is None: sel = self.selected if sel is None: yield from beads.__iter__() # pylint: disable=no-member elif beads.selected: parent = frozenset(beads.keys()) sel = [ i for i in shallowcopy(beads).selecting(sel, True).keys() if i in parent ] yield from shallowcopy(beads).selecting(sel, clear=True).__iter__() return itr = ((bead, self.data[bead]) for bead in self.keys(sel)) if self.cycles is not None: cyc = self.cycles def _get(arr): ind1 = None if cyc.start is None else self.track.phases[ cyc.start, 0] ind2 = None if cyc.stop is None else self.track.phases[ cyc.stop, 0] return arr[ind1:ind2] itr = ((bead, _get(arr)) for bead, arr in itr) yield from itr
def __init__( self, # pylint: disable=too-many-arguments data: DataType, task: Task = None, pool: ProcessPoolExecutor = None, gen: Iterator[TrackView] = None, start: Iterator[TrackView] = None, level: Level = Level(0), **_) -> None: if isinstance(task, ProcessPoolExecutor): assert pool is None task, pool = None, task data = (Cache(list(pickle.loads(data))) if isinstance(data, bytes) else data if isinstance( data, Cache) else Cache(list(data))).keepupto(task) if pool is not None and not hasattr(pool, 'nworkers'): nproc = getattr(pool, '_max_workers', None) if nproc is None: nproc = cpu_count() setattr(pool, 'nworkers', nproc) gen = (iter(shallowcopy(i) for i in start) if start is not None else iter( shallowcopy(i) for i in gen) if gen is not None else None) level = toenum(Level, level) self.data: Cache = data self.pool: Optional[ProcessPoolExecutor] = pool self.gen: Optional[Iterator[TrackView]] = gen self.level: Level = toenum(Level, level)
def renamebeads(trk: TRACKS, *beads: Tuple[int, int]) -> TRACKS: "returns a track without the given beads" if isinstance(trk, TracksDict): return _applytodict(renamebeads, trk, beads, {}) trk.load() cpy = shallowcopy(trk) rep = dict(beads) cpy.data = {rep.get(i, i): j for i, j in trk.data.items()} cpy.fov = shallowcopy(trk.fov) cpy.fov.beads = {rep.get(i, i): j for i, j in trk.fov.beads.items()} return cpy
def __getitem__(self:TSelf, keys) -> Union[TSelf, np.ndarray]: if (isellipsis(keys) or (isinstance(keys, tuple) and all(isellipsis(i) for i in keys))): return shallowcopy(self) if (isinstance(keys, _m_KEYS) or (isinstance(keys, tuple) and all(isinstance(i, _m_KEYS) for i in keys))): # this is NOT a slice return self.get(keys) # consider this a slice cpy = shallowcopy(self) return (cpy.new() if cpy.selected else cpy).selecting(keys)
def copy(self): new = type(self)(path=self._db_path, active_table=self._active_table, columns=shallowcopy(self.columns), verbose=False) new.columns = shallowcopy(self.columns) new.has_internal_ids = self.has_internal_ids new._n_rows = self._n_rows new._state_query = shallowcopy(self._state_query) new._col_types = shallowcopy(self._col_types) new._queries_wo_optimize = self._queries_wo_optimize new._n_rows_before_opt = self._n_rows_before_opt new._user_vec_cache_limit = self._default_user_vec_cache_limit new._item_vec_cache_limit = self._default_item_vec_cache_limit return new
def smart_copy(obj, memo=None, shared_attrs=(), shallowcopy_attrs=(), deepcopy_attrs=(), check_intersections=True, **options): """This function can be used as a direct replacement of __deepcopy__ in classes wishing to provide their own implementation. It uses 'deepcopy_attrs' and 'shallowcopy_attrs' to determine which object attributes should be deep-copied and those which should be shallow-copied.""" # determine which attributes are shared, and which to make shallow or deep copies of shared_attrs = _determine_attr_set(shared_attrs, obj) shallowcopy_attrs = _determine_attr_set(shallowcopy_attrs, obj) deepcopy_attrs = _determine_attr_set(deepcopy_attrs, obj) # check if all attribute sets are empty if len(shared_attrs) == 0 and len(shallowcopy_attrs) == 0 and len(deepcopy_attrs) == 0: raise Exception("all copy sets are empty - nothing to copy") # check for intersections between attribute sets if check_intersections: attr_sets = [set(shared_attrs), set(shallowcopy_attrs), set(deepcopy_attrs)] for attr_set1, attr_set2 in itertools.combinations(attr_sets, 2): conflicting_attrs = attr_set1.intersection(attr_set2) if len(conflicting_attrs) > 0: raise Exception("intersection between attribute sets - %s" % (conflicting_attrs,)) # initialize memo if necessary, and update it with the provided options if memo is None: memo = {} if len(options) > 0: memo.update(options) # fetch or create a clone object and fill it clone = fetch_copy(obj, memo) for attr in shared_attrs: setattr(clone, attr, getattr(obj, attr)) for attr in shallowcopy_attrs: setattr(clone, attr, shallowcopy(getattr(obj, attr))) for attr in deepcopy_attrs: setattr(clone, attr, deepcopy(getattr(obj, attr), memo)) return clone
def __ior__(self, other): for key, value in other.items(): if key in self: self[key] |= value else: self[key] = shallowcopy(value) return self
def copy(self): ''' Create a copy of this object and return it. ''' try: new_me = self.__class__() except RuntimeError as e: raise AnsibleError( "Exceeded maximum object depth. This may have been caused by excessive role recursion", orig_exc=e) for name in self.fattributes: setattr(new_me, name, shallowcopy(getattr(self, f'_{name}', Sentinel))) new_me._loader = self._loader new_me._variable_manager = self._variable_manager new_me._validated = self._validated new_me._finalized = self._finalized new_me._uuid = self._uuid # if the ds value was set on the object, copy it to the new copy too if hasattr(self, '_ds'): new_me._ds = self._ds return new_me
def process_segment(chunk, delimiter): c = self._open_cursor() projected_cols = shallowcopy(self.columns) # rid is never read from file - remove it to not cause issues with zip(row, projected_cols) projected_cols.remove('rid') try: csv_reader = csv.reader(chunk, delimiter=delimiter) records = [] for row in csv_reader: record = [] for value, col in zip(row, projected_cols): if col is None: # accept None cols when reading as skip cols continue elif col == 'interaction': # todo: do we really want this? record.append(float(value)) else: record.append(value) records.append(tuple(record) + (self._n_rows,)) self._n_rows += 1 except ValueError as e: raise Exception('Failed to process dataset chunk: Make sure that the data you\'re loading does not ' 'have a header - if it does, set the argument "has_header" to true.\n' 'More details: {e}'.format(e=str(e))) projected_cols.append('rid') # since rid is automatically added (explicitly) add to projected_cols projected_cols = [col for col in projected_cols if col is not None] # remove any skip cols cols = ','.join(projected_cols) vars = ','.join('?' * len(projected_cols)) c.executemany(f'INSERT INTO interactions ({cols}) VALUES ({vars})', records) c.close()
def __ixor__(self, other): for key, value in other.iteritems(): if key in self: self[key] ^= value else: self[key] = shallowcopy(value) return self
def join_pipe(self, pipes): assert len(pipes) > 0, 'No sense making a null pipe' assert isinstance(pipes[0],Pipe), '%s is not a Pipe object' % pipes[0] jp = shallowcopy(pipes[0]) for p in pipes[1:]: assert isinstance(p,Pipe), '%s is not a Pipe object' % p jp = jp.curry(p) return jp
def dropbeads(trk: TRACKS, *beads: int) -> TRACKS: "returns a track without the given beads" if isinstance(trk, TracksDict): return _applytodict(dropbeads, trk, beads, {}) trk.load() if len(beads) == 1 and isinstance(beads[0], (tuple, list, set, frozenset)): beads = tuple(beads[0]) cpy = shallowcopy(trk) good = frozenset(trk.data.keys()) - frozenset(beads) cpy.data = {i: trk.data[i] for i in good} cpy.fov = shallowcopy(trk.fov) good = good & frozenset(trk.fov.beads) cpy.fov.beads = {i: trk.fov.beads[i] for i in good} setattr(cpy, '_secondaries', dict(getattr(trk, '_secondaries'))) return cpy
def __getitem__(self: TDictType, key: Union[List, Any]) -> Union[TDictType, TrackType]: if isellipsis(key): return shallowcopy(self) if isinstance(key, str) and len(key) and key[0] == '~' and key not in self: key = ['~', key[1:]] if isinstance(key, list): other = shallowcopy(self) bad = set(key) - {'~'} if '~' in key else set(other) - set(key) for i in bad: other.pop(i) return other return super().__getitem__(key)
def clone(self,envother): """Clones the interface into another environment """ clone = shallowcopy(self) clone.prob = RaveCreateModule(envother,'TaskManipulation') clone.robot = envother.GetRobot(self.robot.GetName()) envother.Add(clone.prob,True,clone.args) return clone
def __init__(self, event, subscribers=[]): if inspect.isclass(event): self.event = event() else: self.event = shallowcopy(event) self.subscribers = subscribers pass
def _newRound(self, round_id): self.round = self.RoundClass( round_id, shallowcopy(self.players), self.button, self.small_blind, self.big_blind )
def clone(self,envother): """Clones the interface into another environment """ clone = shallowcopy(self) clone.prob = RaveCreateModule(envother,'VisualFeedback') clone.robot = envother.GetRobot(self.robot.GetName()) if envother.AddModule(clone.prob,clone.args) != 0: raise ValueError('module failed to initialize') return clone
def copy(self, deep=False): if deep: math_func = deepcopy(self) else: math_func = shallowcopy(self) math_func.Cond = deepcopy(math_func.Cond) # need to be careful with Cond and Scope math_func.Scope = deepcopy(math_func.Scope) # need to be careful with Cond and Scope math_func.CompyledFunc = None # remove compiled version because changes are likely to be made on the copy return math_func
def plugins(command, env): return [ # PluginPrint(debug=False), # PluginWrite(stdout='foo.log'.format(**env), stderr='bar.log'), # PluginProgress(name=str(env) if env else command), # PluginProgress(name=command.format(**env)), PluginJson(details={'bin': "{bin}".format(**env)}), PluginEnv(env=shallowcopy(env)) ]
def clone(self,envother): """Clones the interface into another environment """ clone = shallowcopy(self) clone.prob = RaveCreateModule(envother,'Grasper') clone.robot = envother.GetRobot(self.robot.GetName()) clone.avoidlinks = [clone.robot.GetLink(link.GetName()) for link in self.avoidlinks] envother.Add(clone.prob,True,clone.args) return clone
def infer_col_types(first_line): projected_cols = shallowcopy(self.columns) # rid is never read from file - remove it to not cause issues with zip(row, projected_cols) projected_cols.remove('rid') col_types = {'rid': int} for value, col in zip(first_line, projected_cols): col_types[col] = self._infer_value_type(value) return col_types
def clone(self,envother): clone = shallowcopy(self) clone.env = envother clone.robot = clone.env.GetRobot(self.robot.GetName()) clone.irmodels = [] for irmodel in self.irmodels: try: clone.irmodels.append(irmodel.clone(envother)) except openrave_exception,e: print e
def clone(self,envother): """Clones the interface into another environment """ clone = shallowcopy(self) clone.prob = RaveCreateModule(envother,'Grasper') clone.robot = envother.GetRobot(self.robot.GetName()) clone.avoidlinks = [clone.robot.GetLink(link.GetName()) for link in self.avoidlinks] if envother.AddModule(clone.prob,clone.args) != 0: raise ValueError('module failed to initialize') return clone
def clone(self,envother): clone = shallowcopy(self) clone.env = envother clone.envreal = self.env clone.robot = clone.env.GetRobot(self.robot.GetName()) clone.grmodel = self.grmodel.clone(envother) if self.grmodel is not None else None clone.basemanip = self.basemanip.clone(envother) clone.taskmanip = self.taskmanip.clone(envother) if clone.switchpatterns is not None: clone.taskmanip.SwitchModels(switchpatterns=clone.switchpatterns) return clone
def subtractbeads(self, **kwa) -> TracksDict: "creates a new TracksDict with subtracted beads" alg = FixedBeadDetection(**kwa) itr = ((i, alg, j) for i, j in self.tracks.items()) with ProcessPoolExecutor() as pool: info = dict(pool.map(self._subtractbeads, itr)) itms = cast(TracksDict, type(self.tracks)()) for i, beads in info.items(): itms[i] = shallowcopy(self.tracks[i]) itms[i].tasks.subtraction = [j[-1] for j in beads] return itms
def path(cls, pathtype, path, **kwa): "creates a path using provided arguments" if isinstance(path, dict): return pathtype(**path, **kwa) if isinstance(path, pathtype): return update(shallowcopy(path), **kwa) if isinstance(path, (tuple, str, Path)): return pathtype(track=path, **kwa) raise TypeError('Could not create {} using {}'.format(pathtype, path))
def __enter__(self): if self.ctrl is None: return None data = super().__enter__() if data is not self.ctrl.data: ctrl = shallowcopy(self.ctrl) ctrl.data = data else: ctrl = self.ctrl return ctrl if self.copy is None else next( iter(ctrl.run(copy=self.copy)))
def _cfg_subslvr_dims(self, subslvr_cfg, P): for dim in self._dims.itervalues(): name = dim.name if name in P: # Copy dimension data for reconfiguration sub_dim = shallowcopy(dim) sub_dim.update(local_size=P[name], lower_extent=0, upper_extent=P[name]) subslvr_cfg[name] = sub_dim return subslvr_cfg
def _savetrack(args): if not isinstance(args[2], dict): try: PickleIO.save(args[1], args[2]) except Exception as exc: raise TrackIOError( f"Could not save {args[2].path} [{args[2].key}]") from exc else: PickleIO.save(args[1], args[2]) new = type(args[2]).__new__(type(args[2])) # type: ignore new.__dict__.update(shallowcopy(args[2].__dict__)) setattr(new, '_path', args[1]) return args[0], new
def __getitem__(self, beads) -> Track: if isinstance(beads, list): return selectbeads(self._trk, *beads) if np.isscalar(beads): return selectbeads(self._trk, beads) if isinstance(beads, tuple): if len(beads) != 2: raise KeyError("Key should be a (beads, cycles) tuple") trk = self._trk if isellipsis(beads[0]) else self.__getitem__( beads[0]) trk = trk if isellipsis(beads[1]) else selectcycles(trk, beads[1]) return shallowcopy(self._trk) if trk is self._trk else trk raise NotImplementedError()
def _iter(self, sel=None) -> Iterator[Tuple[CYCLEKEY, np.ndarray]]: if isinstance(self.data, Cycles): cycles = cast(Cycles, self.data) if sel is None: sel = self.selected if sel is None: yield from cycles.__iter__() # pylint: disable=no-member return if cycles.selected: parent = frozenset(cycles.keys()) sel = [ i for i in shallowcopy(cycles).selecting(sel, True).keys() if i in parent ] yield from shallowcopy(cycles).selecting(sel, clear=True).__iter__() elif self.direct: tmp = cast(dict, self.data) yield from ((key, tmp[key]) for key in self.keys(sel)) else: yield from self.__iterfrombeads(sel)
def clone(trk: TRACKS) -> TRACKS: """ Deeper shallow copy of the track. The track containers are copied but the numpy arrays are the same. """ if isinstance(trk, Track): track = cast(Track, trk) state = track.__getstate__() state.pop('fov', None) state = deepcopy(state) for i in ('data', 'secondaries', 'fov'): state[i] = shallowcopy(getattr(track, f'_{i}')) return type(track)(**state) return type(trk)({i: clone(j) for i, j in cast(TracksDict, trk).items()})
def plugins(command, env): return [ PluginPrint(), PluginPrintCommand(), # PluginWrite(stdout='foo.log'.format(**env), stderr='bar.log'), # PluginProgress(name=str(env) if env else command), PluginProgress( name="x{nproc} {flow123d} -s {problem_config}".format( nproc=env['nproc'], flow123d=io.end_path(env['flow123d'], 2), problem_config=io.end_path(env['problem_config'], 2), )), # PluginJson(details={ 'problem_config': "{problem_config}".format(**env) }), PluginEnv(env=shallowcopy(env)), PluginTimer() ]
def get_binary_info(): def plugins(command, env): return [ # PluginPrint(debug=False), # PluginWrite(stdout='foo.log'.format(**env), stderr='bar.log'), # PluginProgress(name=str(env) if env else command), # PluginProgress(name=command.format(**env)), PluginJson(details={'bin': "{bin}".format(**env)}), PluginEnv(env=shallowcopy(env)) ] # # exec_util.exec_all(command, options.variables, plugins) info = dict() variables = ['bin:git g++ gcc python java mono perl fortran gfortran make cmake'] which_cmd = 'where "{bin}"' if platform.system() == "Windows" else 'which "{bin}"' for executor in exec_util.exec_all(which_cmd, variables, plugins): # grab environment env = executor.plugins.get('PluginEnv').env bin_name = env['bin'] # clean up json json = shallowcopy(executor.plugins.get('PluginJson').json) json['path'] = ''.join(json['stdout']).strip() del json['stdout'] del json['stderr'] del json['command'] del json['exit_code'] # try to get more info about binary env['version'] = bin_version_flag.get(bin_name, '--version') result = exec_util.exec_all('{bin} {version}'.format(**env))[0] output = ' '.join(result.stderr + result.stdout).replace('\n', ' ') results = re.findall(bin_version_regexp.get(bin_name, bin_version_regexp['']), output) # some --version return non zero exit code if not results: if not json['path']: json['missing'] = True del json['path'] else: json['version'] = results[0] info[bin_name] = json return info
def encodew( self, w, d_encode): d_encode_local = shallowcopy(d_encode) l_res = [] casestr = isinstance(w, basestring) for p in w: if p in d_encode_local: l_res.append( d_encode_local[p] ) continue if isinstance (p , basestring ) and casestr == True: #sinon produira un join d_encode_local[p] = p l_res.append( p ) continue else : d_encode_local[p] = unichr(len(d_encode_local)+128) l_res.append( d_encode_local[p] ) res = ''.join([ d_encode_local[p] for p in w ]) return res, d_encode_local
def run_benchmarks(tests=None, cores=None, timeout=0.4, tries=2): tests = shallowcopy(all_tests.keys()) if tests is None else set(tests) cores = range(1, 5) if cores is None else cores measurement = BenchmarkMeasurement() measurement.configure(timeout, tries, cores) info = dict() for test in all_tests: if test not in tests: continue try: info[test] = measurement.measure(all_tests.get(test), test) except Exception as e: print e return info
def copy(self): ''' Create a copy of this object and return it. ''' new_me = self.__class__() for name in self._valid_attrs.keys(): new_me._attributes[name] = shallowcopy(self._attributes[name]) new_me._loader = self._loader new_me._variable_manager = self._variable_manager new_me._validated = self._validated new_me._finalized = self._finalized new_me._uuid = self._uuid # if the ds value was set on the object, copy it to the new copy too if hasattr(self, '_ds'): new_me._ds = self._ds return new_me
def _resolve_function(self, name, args): assert name in self.pipes, 'No such pipe: %s' % name pipe = shallowcopy(self.pipes[name]) pipe.fill_args(map(self.resolve, args)) return pipe
def deck_generator(this): deck = shallowcopy(this.__deck) shuffle(deck) return (card for card in deck) # returns a generator