def ngram_loader(n,
lang='en',
query=['counts', 'order', 'frequencies', 'sum'],
sep=' '):
ngrams = {}
ret = {}
file = pkg_resources.resource_filename(
__name__, 'data/' + lang + '/' + ngramfiles[n])
for line in open(file):
key, count = line.split(sep)
key = key.lower()
ngrams[key] = int(count)
L = len(key)
N = sum([ngrams[k] for k in ngrams])
#print (file)
if 'counts' in query:
ret['counts'] = shallowcopy(ngrams)
if 'order' in query:
ret['order'] = [
k[0] for k in sorted([[k, ngrams[k]] for k in ngrams],
key=lambda x: x[1])
]
ret['order'].reverse()
if 'frequencies' in query:
for ngram in ngrams:
ngrams[ngram] = ngrams[ngram] / N
ret['frequencies'] = shallowcopy(ngrams)
if 'sum' in query:
ret['sum'] = N
return ret
def copy(self):
'''
Create a copy of this object and return it.
'''
new_me = self.__class__()
for name in self._valid_attrs.keys():
if name in self._alias_attrs:
continue
new_me._attributes[name] = shallowcopy(self._attributes[name])
new_me._attr_defaults[name] = shallowcopy(
self._attr_defaults[name])
new_me._loader = self._loader
new_me._variable_manager = self._variable_manager
new_me._validated = self._validated
new_me._finalized = self._finalized
new_me._uuid = self._uuid
# if the ds value was set on the object, copy it to the new copy too
if hasattr(self, '_ds'):
new_me._ds = self._ds
return new_me
def copy(x):
if not x._son:
return shallowcopy(x)
else:
x_ = shallowcopy(x)
x_._son = [Ego.copy(t) if isinstance(t, Ego) else t for t in x._son]
return x_
def copy(self):
'''
Create a copy of this object and return it.
'''
try:
new_me = self.__class__()
except RuntimeError as e:
raise AnsibleError(
"Exceeded maximum object depth. This may have been caused by excessive role recursion",
orig_exc=e)
for name in self._valid_attrs.keys():
if name in self._alias_attrs:
continue
new_me._attributes[name] = shallowcopy(self._attributes[name])
new_me._attr_defaults[name] = shallowcopy(
self._attr_defaults[name])
new_me._loader = self._loader
new_me._variable_manager = self._variable_manager
new_me._validated = self._validated
new_me._finalized = self._finalized
new_me._uuid = self._uuid
# if the ds value was set on the object, copy it to the new copy too
if hasattr(self, '_ds'):
new_me._ds = self._ds
return new_me
def signal(self, frame: Union[Beads, Cycles], key=None) -> np.ndarray:
"returns the signal to subtract from beads"
task = self.task
next(iter(frame.keys())) # unlazyfy # type: ignore
# Discard the FixedBeadDetectionProcessor from the
# frame as it will created an exception which is
# meaningless in this context
frame = shallowcopy(frame)
frame.data = shallowcopy(frame.data)
frame.data.actions = [
i for i in frame.data.actions
if 'FixedBeadDetectionProcessor' not in str(i)
]
data = cast(Dict, frame.data)
itr = (cast(Iterable[int], task.beads) if key is None else cast(
Iterable[int], list(zip(task.beads, repeat(key)))))
if len(task.beads) == 0:
sub = 0.
elif len(task.beads) == 1:
sub = np.copy(data[task.beads[0]])
else:
sub = task.agg.process(itr, frame)
return task.filter(sub) if task.filter else sub
def _iter(self, sel=None) -> Iterator[Tuple[int, np.ndarray]]:
if isinstance(self.data, Beads) and self.cycles is None:
beads = cast(Beads, self.data)
if sel is None:
sel = self.selected
if sel is None:
yield from beads.__iter__() # pylint: disable=no-member
elif beads.selected:
parent = frozenset(beads.keys())
sel = [
i for i in shallowcopy(beads).selecting(sel, True).keys()
if i in parent
]
yield from shallowcopy(beads).selecting(sel, clear=True).__iter__()
return
itr = ((bead, self.data[bead]) for bead in self.keys(sel))
if self.cycles is not None:
cyc = self.cycles
def _get(arr):
ind1 = None if cyc.start is None else self.track.phases[
cyc.start, 0]
ind2 = None if cyc.stop is None else self.track.phases[
cyc.stop, 0]
return arr[ind1:ind2]
itr = ((bead, _get(arr)) for bead, arr in itr)
yield from itr
def __init__(
self, # pylint: disable=too-many-arguments
data: DataType,
task: Task = None,
pool: ProcessPoolExecutor = None,
gen: Iterator[TrackView] = None,
start: Iterator[TrackView] = None,
level: Level = Level(0),
**_) -> None:
if isinstance(task, ProcessPoolExecutor):
assert pool is None
task, pool = None, task
data = (Cache(list(pickle.loads(data)))
if isinstance(data, bytes) else data if isinstance(
data, Cache) else Cache(list(data))).keepupto(task)
if pool is not None and not hasattr(pool, 'nworkers'):
nproc = getattr(pool, '_max_workers', None)
if nproc is None:
nproc = cpu_count()
setattr(pool, 'nworkers', nproc)
gen = (iter(shallowcopy(i)
for i in start) if start is not None else iter(
shallowcopy(i)
for i in gen) if gen is not None else None)
level = toenum(Level, level)
self.data: Cache = data
self.pool: Optional[ProcessPoolExecutor] = pool
self.gen: Optional[Iterator[TrackView]] = gen
self.level: Level = toenum(Level, level)
def renamebeads(trk: TRACKS, *beads: Tuple[int, int]) -> TRACKS:
"returns a track without the given beads"
if isinstance(trk, TracksDict):
return _applytodict(renamebeads, trk, beads, {})
trk.load()
cpy = shallowcopy(trk)
rep = dict(beads)
cpy.data = {rep.get(i, i): j for i, j in trk.data.items()}
cpy.fov = shallowcopy(trk.fov)
cpy.fov.beads = {rep.get(i, i): j for i, j in trk.fov.beads.items()}
return cpy
def __getitem__(self:TSelf, keys) -> Union[TSelf, np.ndarray]:
if (isellipsis(keys)
or (isinstance(keys, tuple) and all(isellipsis(i) for i in keys))):
return shallowcopy(self)
if (isinstance(keys, _m_KEYS)
or (isinstance(keys, tuple)
and all(isinstance(i, _m_KEYS) for i in keys))):
# this is NOT a slice
return self.get(keys)
# consider this a slice
cpy = shallowcopy(self)
return (cpy.new() if cpy.selected else cpy).selecting(keys)
def copy(self):
new = type(self)(path=self._db_path, active_table=self._active_table, columns=shallowcopy(self.columns),
verbose=False)
new.columns = shallowcopy(self.columns)
new.has_internal_ids = self.has_internal_ids
new._n_rows = self._n_rows
new._state_query = shallowcopy(self._state_query)
new._col_types = shallowcopy(self._col_types)
new._queries_wo_optimize = self._queries_wo_optimize
new._n_rows_before_opt = self._n_rows_before_opt
new._user_vec_cache_limit = self._default_user_vec_cache_limit
new._item_vec_cache_limit = self._default_item_vec_cache_limit
return new
def smart_copy(obj, memo=None, shared_attrs=(), shallowcopy_attrs=(), deepcopy_attrs=(),
check_intersections=True, **options):
"""This function can be used as a direct replacement of __deepcopy__ in classes wishing to
provide their own implementation. It uses 'deepcopy_attrs' and 'shallowcopy_attrs' to determine
which object attributes should be deep-copied and those which should be shallow-copied."""
# determine which attributes are shared, and which to make shallow or deep copies of
shared_attrs = _determine_attr_set(shared_attrs, obj)
shallowcopy_attrs = _determine_attr_set(shallowcopy_attrs, obj)
deepcopy_attrs = _determine_attr_set(deepcopy_attrs, obj)
# check if all attribute sets are empty
if len(shared_attrs) == 0 and len(shallowcopy_attrs) == 0 and len(deepcopy_attrs) == 0:
raise Exception("all copy sets are empty - nothing to copy")
# check for intersections between attribute sets
if check_intersections:
attr_sets = [set(shared_attrs), set(shallowcopy_attrs), set(deepcopy_attrs)]
for attr_set1, attr_set2 in itertools.combinations(attr_sets, 2):
conflicting_attrs = attr_set1.intersection(attr_set2)
if len(conflicting_attrs) > 0:
raise Exception("intersection between attribute sets - %s" % (conflicting_attrs,))
# initialize memo if necessary, and update it with the provided options
if memo is None:
memo = {}
if len(options) > 0:
memo.update(options)
# fetch or create a clone object and fill it
clone = fetch_copy(obj, memo)
for attr in shared_attrs:
setattr(clone, attr, getattr(obj, attr))
for attr in shallowcopy_attrs:
setattr(clone, attr, shallowcopy(getattr(obj, attr)))
for attr in deepcopy_attrs:
setattr(clone, attr, deepcopy(getattr(obj, attr), memo))
return clone
def __ior__(self, other):
for key, value in other.items():
if key in self:
self[key] |= value
else:
self[key] = shallowcopy(value)
return self
def copy(self):
'''
Create a copy of this object and return it.
'''
try:
new_me = self.__class__()
except RuntimeError as e:
raise AnsibleError(
"Exceeded maximum object depth. This may have been caused by excessive role recursion",
orig_exc=e)
for name in self.fattributes:
setattr(new_me, name,
shallowcopy(getattr(self, f'_{name}', Sentinel)))
new_me._loader = self._loader
new_me._variable_manager = self._variable_manager
new_me._validated = self._validated
new_me._finalized = self._finalized
new_me._uuid = self._uuid
# if the ds value was set on the object, copy it to the new copy too
if hasattr(self, '_ds'):
new_me._ds = self._ds
return new_me
def process_segment(chunk, delimiter):
c = self._open_cursor()
projected_cols = shallowcopy(self.columns)
# rid is never read from file - remove it to not cause issues with zip(row, projected_cols)
projected_cols.remove('rid')
try:
csv_reader = csv.reader(chunk, delimiter=delimiter)
records = []
for row in csv_reader:
record = []
for value, col in zip(row, projected_cols):
if col is None: # accept None cols when reading as skip cols
continue
elif col == 'interaction': # todo: do we really want this?
record.append(float(value))
else:
record.append(value)
records.append(tuple(record) + (self._n_rows,))
self._n_rows += 1
except ValueError as e:
raise Exception('Failed to process dataset chunk: Make sure that the data you\'re loading does not '
'have a header - if it does, set the argument "has_header" to true.\n'
'More details: {e}'.format(e=str(e)))
projected_cols.append('rid') # since rid is automatically added (explicitly) add to projected_cols
projected_cols = [col for col in projected_cols if col is not None] # remove any skip cols
cols = ','.join(projected_cols)
vars = ','.join('?' * len(projected_cols))
c.executemany(f'INSERT INTO interactions ({cols}) VALUES ({vars})', records)
c.close()
def __ixor__(self, other):
for key, value in other.iteritems():
if key in self:
self[key] ^= value
else:
self[key] = shallowcopy(value)
return self
def join_pipe(self, pipes):
assert len(pipes) > 0, 'No sense making a null pipe'
assert isinstance(pipes[0],Pipe), '%s is not a Pipe object' % pipes[0]
jp = shallowcopy(pipes[0])
for p in pipes[1:]:
assert isinstance(p,Pipe), '%s is not a Pipe object' % p
jp = jp.curry(p)
return jp
def dropbeads(trk: TRACKS, *beads: int) -> TRACKS:
"returns a track without the given beads"
if isinstance(trk, TracksDict):
return _applytodict(dropbeads, trk, beads, {})
trk.load()
if len(beads) == 1 and isinstance(beads[0], (tuple, list, set, frozenset)):
beads = tuple(beads[0])
cpy = shallowcopy(trk)
good = frozenset(trk.data.keys()) - frozenset(beads)
cpy.data = {i: trk.data[i] for i in good}
cpy.fov = shallowcopy(trk.fov)
good = good & frozenset(trk.fov.beads)
cpy.fov.beads = {i: trk.fov.beads[i] for i in good}
setattr(cpy, '_secondaries', dict(getattr(trk, '_secondaries')))
return cpy
def __getitem__(self: TDictType,
key: Union[List, Any]) -> Union[TDictType, TrackType]:
if isellipsis(key):
return shallowcopy(self)
if isinstance(key,
str) and len(key) and key[0] == '~' and key not in self:
key = ['~', key[1:]]
if isinstance(key, list):
other = shallowcopy(self)
bad = set(key) - {'~'} if '~' in key else set(other) - set(key)
for i in bad:
other.pop(i)
return other
return super().__getitem__(key)
def clone(self,envother):
"""Clones the interface into another environment
"""
clone = shallowcopy(self)
clone.prob = RaveCreateModule(envother,'TaskManipulation')
clone.robot = envother.GetRobot(self.robot.GetName())
envother.Add(clone.prob,True,clone.args)
return clone
def __init__(self, event, subscribers=[]):
if inspect.isclass(event):
self.event = event()
else:
self.event = shallowcopy(event)
self.subscribers = subscribers
pass
def _newRound(self, round_id):
self.round = self.RoundClass(
round_id,
shallowcopy(self.players),
self.button,
self.small_blind,
self.big_blind
)
def clone(self,envother):
"""Clones the interface into another environment
"""
clone = shallowcopy(self)
clone.prob = RaveCreateModule(envother,'VisualFeedback')
clone.robot = envother.GetRobot(self.robot.GetName())
if envother.AddModule(clone.prob,clone.args) != 0:
raise ValueError('module failed to initialize')
return clone
def copy(self, deep=False):
if deep:
math_func = deepcopy(self)
else:
math_func = shallowcopy(self)
math_func.Cond = deepcopy(math_func.Cond) # need to be careful with Cond and Scope
math_func.Scope = deepcopy(math_func.Scope) # need to be careful with Cond and Scope
math_func.CompyledFunc = None # remove compiled version because changes are likely to be made on the copy
return math_func
def plugins(command, env):
return [
# PluginPrint(debug=False),
# PluginWrite(stdout='foo.log'.format(**env), stderr='bar.log'),
# PluginProgress(name=str(env) if env else command),
# PluginProgress(name=command.format(**env)),
PluginJson(details={'bin': "{bin}".format(**env)}),
PluginEnv(env=shallowcopy(env))
]
def clone(self,envother):
"""Clones the interface into another environment
"""
clone = shallowcopy(self)
clone.prob = RaveCreateModule(envother,'Grasper')
clone.robot = envother.GetRobot(self.robot.GetName())
clone.avoidlinks = [clone.robot.GetLink(link.GetName()) for link in self.avoidlinks]
envother.Add(clone.prob,True,clone.args)
return clone
def clone(self,envother):
"""Clones the interface into another environment
"""
clone = shallowcopy(self)
clone.prob = RaveCreateModule(envother,'Grasper')
clone.robot = envother.GetRobot(self.robot.GetName())
clone.avoidlinks = [clone.robot.GetLink(link.GetName()) for link in self.avoidlinks]
envother.Add(clone.prob,True,clone.args)
return clone
def infer_col_types(first_line):
projected_cols = shallowcopy(self.columns)
# rid is never read from file - remove it to not cause issues with zip(row, projected_cols)
projected_cols.remove('rid')
col_types = {'rid': int}
for value, col in zip(first_line, projected_cols):
col_types[col] = self._infer_value_type(value)
return col_types
def clone(self,envother):
clone = shallowcopy(self)
clone.env = envother
clone.robot = clone.env.GetRobot(self.robot.GetName())
clone.irmodels = []
for irmodel in self.irmodels:
try:
clone.irmodels.append(irmodel.clone(envother))
except openrave_exception,e:
print e
def clone(self,envother):
"""Clones the interface into another environment
"""
clone = shallowcopy(self)
clone.prob = RaveCreateModule(envother,'Grasper')
clone.robot = envother.GetRobot(self.robot.GetName())
clone.avoidlinks = [clone.robot.GetLink(link.GetName()) for link in self.avoidlinks]
if envother.AddModule(clone.prob,clone.args) != 0:
raise ValueError('module failed to initialize')
return clone
def clone(self,envother):
clone = shallowcopy(self)
clone.env = envother
clone.envreal = self.env
clone.robot = clone.env.GetRobot(self.robot.GetName())
clone.grmodel = self.grmodel.clone(envother) if self.grmodel is not None else None
clone.basemanip = self.basemanip.clone(envother)
clone.taskmanip = self.taskmanip.clone(envother)
if clone.switchpatterns is not None:
clone.taskmanip.SwitchModels(switchpatterns=clone.switchpatterns)
return clone
def subtractbeads(self, **kwa) -> TracksDict:
"creates a new TracksDict with subtracted beads"
alg = FixedBeadDetection(**kwa)
itr = ((i, alg, j) for i, j in self.tracks.items())
with ProcessPoolExecutor() as pool:
info = dict(pool.map(self._subtractbeads, itr))
itms = cast(TracksDict, type(self.tracks)())
for i, beads in info.items():
itms[i] = shallowcopy(self.tracks[i])
itms[i].tasks.subtraction = [j[-1] for j in beads]
return itms
def path(cls, pathtype, path, **kwa):
"creates a path using provided arguments"
if isinstance(path, dict):
return pathtype(**path, **kwa)
if isinstance(path, pathtype):
return update(shallowcopy(path), **kwa)
if isinstance(path, (tuple, str, Path)):
return pathtype(track=path, **kwa)
raise TypeError('Could not create {} using {}'.format(pathtype, path))
def __enter__(self):
if self.ctrl is None:
return None
data = super().__enter__()
if data is not self.ctrl.data:
ctrl = shallowcopy(self.ctrl)
ctrl.data = data
else:
ctrl = self.ctrl
return ctrl if self.copy is None else next(
iter(ctrl.run(copy=self.copy)))
def _cfg_subslvr_dims(self, subslvr_cfg, P):
for dim in self._dims.itervalues():
name = dim.name
if name in P:
# Copy dimension data for reconfiguration
sub_dim = shallowcopy(dim)
sub_dim.update(local_size=P[name],
lower_extent=0, upper_extent=P[name])
subslvr_cfg[name] = sub_dim
return subslvr_cfg
def _savetrack(args):
if not isinstance(args[2], dict):
try:
PickleIO.save(args[1], args[2])
except Exception as exc:
raise TrackIOError(
f"Could not save {args[2].path} [{args[2].key}]") from exc
else:
PickleIO.save(args[1], args[2])
new = type(args[2]).__new__(type(args[2])) # type: ignore
new.__dict__.update(shallowcopy(args[2].__dict__))
setattr(new, '_path', args[1])
return args[0], new
def _cfg_subslvr_dims(self, subslvr_cfg, P):
for dim in self._dims.itervalues():
name = dim.name
if name in P:
# Copy dimension data for reconfiguration
sub_dim = shallowcopy(dim)
sub_dim.update(local_size=P[name],
lower_extent=0,
upper_extent=P[name])
subslvr_cfg[name] = sub_dim
return subslvr_cfg
def __getitem__(self, beads) -> Track:
if isinstance(beads, list):
return selectbeads(self._trk, *beads)
if np.isscalar(beads):
return selectbeads(self._trk, beads)
if isinstance(beads, tuple):
if len(beads) != 2:
raise KeyError("Key should be a (beads, cycles) tuple")
trk = self._trk if isellipsis(beads[0]) else self.__getitem__(
beads[0])
trk = trk if isellipsis(beads[1]) else selectcycles(trk, beads[1])
return shallowcopy(self._trk) if trk is self._trk else trk
raise NotImplementedError()
def _iter(self, sel=None) -> Iterator[Tuple[CYCLEKEY, np.ndarray]]:
if isinstance(self.data, Cycles):
cycles = cast(Cycles, self.data)
if sel is None:
sel = self.selected
if sel is None:
yield from cycles.__iter__() # pylint: disable=no-member
return
if cycles.selected:
parent = frozenset(cycles.keys())
sel = [
i for i in shallowcopy(cycles).selecting(sel, True).keys()
if i in parent
]
yield from shallowcopy(cycles).selecting(sel,
clear=True).__iter__()
elif self.direct:
tmp = cast(dict, self.data)
yield from ((key, tmp[key]) for key in self.keys(sel))
else:
yield from self.__iterfrombeads(sel)
def clone(trk: TRACKS) -> TRACKS:
"""
Deeper shallow copy of the track.
The track containers are copied but the numpy arrays are the same.
"""
if isinstance(trk, Track):
track = cast(Track, trk)
state = track.__getstate__()
state.pop('fov', None)
state = deepcopy(state)
for i in ('data', 'secondaries', 'fov'):
state[i] = shallowcopy(getattr(track, f'_{i}'))
return type(track)(**state)
return type(trk)({i: clone(j) for i, j in cast(TracksDict, trk).items()})
def plugins(command, env):
return [
PluginPrint(),
PluginPrintCommand(),
# PluginWrite(stdout='foo.log'.format(**env), stderr='bar.log'),
# PluginProgress(name=str(env) if env else command),
PluginProgress(
name="x{nproc} {flow123d} -s {problem_config}".format(
nproc=env['nproc'],
flow123d=io.end_path(env['flow123d'], 2),
problem_config=io.end_path(env['problem_config'], 2),
)),
# PluginJson(details={ 'problem_config': "{problem_config}".format(**env) }),
PluginEnv(env=shallowcopy(env)),
PluginTimer()
]
def get_binary_info():
def plugins(command, env):
return [
# PluginPrint(debug=False),
# PluginWrite(stdout='foo.log'.format(**env), stderr='bar.log'),
# PluginProgress(name=str(env) if env else command),
# PluginProgress(name=command.format(**env)),
PluginJson(details={'bin': "{bin}".format(**env)}),
PluginEnv(env=shallowcopy(env))
]
#
# exec_util.exec_all(command, options.variables, plugins)
info = dict()
variables = ['bin:git g++ gcc python java mono perl fortran gfortran make cmake']
which_cmd = 'where "{bin}"' if platform.system() == "Windows" else 'which "{bin}"'
for executor in exec_util.exec_all(which_cmd, variables, plugins):
# grab environment
env = executor.plugins.get('PluginEnv').env
bin_name = env['bin']
# clean up json
json = shallowcopy(executor.plugins.get('PluginJson').json)
json['path'] = ''.join(json['stdout']).strip()
del json['stdout']
del json['stderr']
del json['command']
del json['exit_code']
# try to get more info about binary
env['version'] = bin_version_flag.get(bin_name, '--version')
result = exec_util.exec_all('{bin} {version}'.format(**env))[0]
output = ' '.join(result.stderr + result.stdout).replace('\n', ' ')
results = re.findall(bin_version_regexp.get(bin_name, bin_version_regexp['']), output)
# some --version return non zero exit code
if not results:
if not json['path']:
json['missing'] = True
del json['path']
else:
json['version'] = results[0]
info[bin_name] = json
return info
def encodew( self, w, d_encode): d_encode_local = shallowcopy(d_encode) l_res = [] casestr = isinstance(w, basestring) for p in w: if p in d_encode_local: l_res.append( d_encode_local[p] ) continue if isinstance (p , basestring ) and casestr == True: #sinon produira un join d_encode_local[p] = p l_res.append( p ) continue else : d_encode_local[p] = unichr(len(d_encode_local)+128) l_res.append( d_encode_local[p] ) res = ''.join([ d_encode_local[p] for p in w ]) return res, d_encode_local
def run_benchmarks(tests=None, cores=None, timeout=0.4, tries=2):
tests = shallowcopy(all_tests.keys()) if tests is None else set(tests)
cores = range(1, 5) if cores is None else cores
measurement = BenchmarkMeasurement()
measurement.configure(timeout, tries, cores)
info = dict()
for test in all_tests:
if test not in tests:
continue
try:
info[test] = measurement.measure(all_tests.get(test), test)
except Exception as e:
print e
return info
def copy(self):
'''
Create a copy of this object and return it.
'''
new_me = self.__class__()
for name in self._valid_attrs.keys():
new_me._attributes[name] = shallowcopy(self._attributes[name])
new_me._loader = self._loader
new_me._variable_manager = self._variable_manager
new_me._validated = self._validated
new_me._finalized = self._finalized
new_me._uuid = self._uuid
# if the ds value was set on the object, copy it to the new copy too
if hasattr(self, '_ds'):
new_me._ds = self._ds
return new_me
def _resolve_function(self, name, args): assert name in self.pipes, 'No such pipe: %s' % name pipe = shallowcopy(self.pipes[name]) pipe.fill_args(map(self.resolve, args)) return pipe
def deck_generator(this):
deck = shallowcopy(this.__deck)
shuffle(deck)
return (card for card in deck) # returns a generator
