def __init__(self, states, symbols, transitions, initial, final) :
self.states = frozenset(states)
self.symbols = frozenset(symbols)
self.transitions = transitions.copy()
self.initial = initial
self.final = frozenset(final)
self.check_integrity()
def get_types_and_names(cls):
types, names = [], []
for stype, name, _ in cls.service_info_iterator(cls.ALL_SERVICES):
types.append(stype)
names.append(name)
return frozenset(types), frozenset(names)
def parse_junction(self, jc, tokenizer):
jc.cursors = []
# Take a peek at the tokenizer's stream to see if we
# have a conjunction or disjunction
token = tokenizer.peek()
s = tokenizer.s[token.pos:]
(andpos, orpos) = self.find_nonparen(s, ['&', '|'])
if orpos >= 0 and (andpos < 0 or orpos < andpos):
conjunction = False
mbr = frozenset()
else:
conjunction = True
mbr = frozenset(self.allN)
while tokenizer.available():
mbr = self.parse_join(jc, tokenizer, conjunction, mbr)
token = tokenizer.token()
if token != None:
if token.kind == Token.OR:
self.assertTrue(not conjunction)
elif token.kind == Token.AND:
self.assertTrue(conjunction)
elif token.kind == Token.RPAREN:
break
else:
self.err(token, 'unexpected token')
return mbr
def editBonding(self, bond, _netinfo):
"""
Modifies the bond so that the bond in the system ends up with the
same slave and options configuration that are requested. Makes a
best effort not to interrupt connectivity.
"""
nicsToSet = frozenset(nic.name for nic in bond.slaves)
currentNics = frozenset(_netinfo.getNicsForBonding(bond.name))
nicsToAdd = nicsToSet
nicsToRemove = currentNics
if bond.areOptionsApplied():
nicsToAdd -= currentNics
nicsToRemove -= nicsToSet
for nic in nicsToRemove:
slave = Nic(nic, self, _netinfo=_netinfo)
self.configApplier.removeBondSlave(bond, slave)
slave.remove()
if not bond.areOptionsApplied():
self.configApplier.ifdown(bond)
self.configApplier.addBondOptions(bond)
for slave in bond.slaves:
if slave.name in nicsToAdd:
self.configApplier.addBondSlave(bond, slave)
self.configApplier.ifup(bond)
self.runningConfig.setBonding(
bond.name, {'options': bond.options,
'nics': [slave.name for slave in bond.slaves],
'switch': 'legacy'})
def test_exclude_related(self):
"""Test for specifying excluded columns on related models."""
date = datetime.date(1999, 12, 31)
person = self.Person(name='Test', age=10, other=20, birth_date=date)
computer = self.Computer(name='foo', vendor='bar', buy_date=date)
self.session.add(person)
person.computers.append(computer)
self.session.commit()
exclude = frozenset(['name', 'age', 'computers', 'computers.id',
'computers.name'])
self.manager.create_api(self.Person, exclude_columns=exclude)
exclude = frozenset(['name', 'age', 'computers.id', 'computers.name'])
self.manager.create_api(self.Person, url_prefix='/api2',
exclude_columns=exclude)
response = self.app.get('/api/person/%s' % person.id)
person_dict = loads(response.data)
for column in 'name', 'age', 'computers':
assert column not in person_dict
for column in 'id', 'other', 'birth_date':
assert column in person_dict
response = self.app.get('/api2/person/%s' % person.id)
person_dict = loads(response.data)
assert 'computers' in person_dict
for column in 'id', 'name':
assert column not in person_dict['computers'][0]
for column in 'vendor', 'owner_id', 'buy_date':
assert column in person_dict['computers'][0]
def testGetLists(self):
"""Should be able to get lists from a service"""
self.assertEqual(self.service.get_list_count(), 3)
list_a = self.service.get_list("test-list-1")
self.assertTrue(list_a.description, "An example test list")
self.assertEqual(list_a.size, 42)
self.assertEqual(list_a.count, 42)
self.assertEqual(len(list_a), 42)
self.assertEqual(list_a.title, "test1")
self.assertTrue(list_a.is_authorized)
self.assertEqual(list_a.list_type, "Employee")
self.assertEqual(list_a.tags, frozenset(["tag1", "tag2", "tag3"]))
list_a = self.service.get_list("test-list-2")
self.assertTrue(list_a.description, "Another example test list")
self.assertEqual(list_a.size, 7)
self.assertEqual(len(list_a), 7)
self.assertEqual(list_a.count, 7)
self.assertTrue(not list_a.is_authorized)
self.assertEqual(list_a.tags, frozenset([]))
list_c = self.service.get_list("test-list-3")
self.assertTrue(list_c.description, "Yet Another example test list")
self.assertEqual(list_c.size, 8)
self.assertTrue(list_c.is_authorized)
def alter_size():
list_a.size = 10
def alter_type():
list_a.list_type = "foo"
self.assertRaises(AttributeError, alter_size)
self.assertRaises(AttributeError, alter_type)
def izipstrict(*args):
"""
XXX broken! this function does not always raise the error it claims to
implement; it needs a layer of counters to see how much was used from each
iterable....
Returns a generator very like the itertools izip function. The arguments
are treated like arguments to izip. Unlike izip this function ensures that
all the iterators finished after the same number of items. It raises a
StrictError if not all the ierators have finished.
If StrictError is raised, in addition to a human-readable message, the
exception will have attributes 'finished', 'unfinished' and 'values', all
tuples corresponding respectively to the indices in the argument list of the
iterators that had finished, the indices of the iterators that had not
finished, and the initial yields from those unfinished iterators. Note that
process of getting each initial yield can itself finish iterators in the
unfinished set.
>>> tuple(izipstrict(xrange(5), xrange(5)))
((0, 0), (1, 1), (2, 2), (3, 3), (4, 4))
Uncaught error
>>> tuple(izipstrict(xrange(5), xrange(4), xrange(1,6), xrange(4)))
Traceback (most recent call last):
...
StrictError: out of 4 iterators, 2 were unfinished: at argument indices (0, 2)
Caught error and attributes
>>> try: tuple(izipstrict(xrange(5), xrange(4), xrange(1,6), xrange(4)))
... except StrictError, e: print e.finished, e.unfinished, e.values
(1, 3) (0, 2) (4, 5)
"""
nexts = tuple(iter(arg).next for arg in args)
finished = list()
build = list()
build_append = build.append
while True:
del build[:]
for index, next in enumerate(nexts):
try:
build_append(next())
except StopIteration:
finished.append(index)
if finished and build:
unfinished = tuple(sorted(frozenset(xrange(len(nexts))) - frozenset(finished)))
assert len(unfinished) == len(build)
err = StrictError("out of %d iterators, %d were unfinished: at argument indices %s"
% (len(nexts), len(unfinished), tuple(unfinished)))
err.finished = tuple(finished)
err.unfinished = unfinished
err.values = tuple(build)
raise err
if build:
yield tuple(build)
else:
assert len(finished) == len(nexts)
raise StopIteration
def _remove_allpaths(hps, conditions):
"""Hacky way to recognize some kinds of false dependencies
Better would be logic programming.
"""
potential_conds = {}
for k, v in hps.items():
if v["node"].name in ("randint", "categorical"):
upper = v["node"].arg["upper"].obj
potential_conds[k] = frozenset([EQ(k, ii) for ii in range(upper)])
for k, v in hps.items():
if len(v["conditions"]) > 1:
all_conds = [[c for c in cond if c is not True] for cond in v["conditions"]]
all_conds = [cond for cond in all_conds if len(cond) >= 1]
if len(all_conds) == 0:
v["conditions"] = set([conditions])
continue
depvar = all_conds[0][0].name
all_one_var = all(len(cond) == 1 and cond[0].name == depvar for cond in all_conds)
if all_one_var:
conds = [cond[0] for cond in all_conds]
if frozenset(conds) == potential_conds[depvar]:
v["conditions"] = set([conditions])
continue
def force_trace_widths(board):
microstrip_layers = frozenset(('1_top', '6_bot'))
stripline_layers = frozenset(('3_inner', '4_inner'))
se_50_microstrip_width = '0.1778'
se_50_stripline_width = '0.1651'
diff_90_microstrip_width = '0.127'
diff_90_stripline_width = '0.127'
for element in board:
if element[0] == 'segment':
segment = OrderedDict([(v[0], v[1:]) for v in element[1:]])
assert len(segment['net']) == 1
net_name = net_by_number[int(segment['net'][0])]
assert len(segment['layer']) == 1
layer = segment['layer'][0]
new_width = None
if net_name in nets_by_net_class['50_se']:
if layer in microstrip_layers:
new_width = [se_50_microstrip_width]
if layer in stripline_layers:
new_width = [se_50_stripline_width]
elif net_name in nets_by_net_class['90_diff']:
if layer in microstrip_layers:
new_width = [diff_90_microstrip_width]
if layer in stripline_layers:
new_width = [diff_90_stripline_width]
if new_width:
segment['width'] = new_width
new_elements = [[a] + b for a, b in segment.items()]
element[1:] = new_elements
def count_by_annotator(corpus):
"""
Return variety of by-annotator counts
"""
annotators = frozenset(k.annotator for k in corpus
if k.annotator is not None)
acounts = PerAnno(struct=defaultdict(empty_counts),
acts=defaultdict(empty_counts),
rlabels=defaultdict(empty_counts),
links=defaultdict(empty_counts))
for annotator in annotators:
units, discourse = anno_subcorpus(corpus, annotator)
for kdoc in frozenset(k.doc for k in discourse):
ksubdocs = frozenset(k.subdoc for k in discourse
if k.doc == kdoc)
acounts.struct[annotator]["doc"] += 1
acounts.struct[annotator]["subdoc"] += len(ksubdocs)
for k in units:
count(corpus[k], {},
counts=acounts.acts[annotator],
pred_extract=(educe.stac.is_edu, hinted_type))
for k in discourse:
count(corpus[k], dict(SEGMENT_CATEGORIES),
counts=acounts.struct[annotator])
count(corpus[k], dict(LINK_CATEGORIES),
counts=acounts.links[annotator])
count(corpus[k], {},
counts=acounts.rlabels[annotator],
pred_extract=(educe.stac.is_relation_instance,
lambda x: x.type))
return acounts
def CheckPrereq(self):
owned_groups = frozenset(self.owned_locks(locking.LEVEL_NODEGROUP))
assert self.group_uuid in owned_groups
# Check if locked instances are still correct
owned_instance_names = frozenset(self.owned_locks(locking.LEVEL_INSTANCE))
if self.op.conflicts_check:
CheckNodeGroupInstances(self.cfg, self.group_uuid, owned_instance_names)
self.netparams = {
constants.NIC_MODE: self.network_mode,
constants.NIC_LINK: self.network_link,
}
objects.NIC.CheckParameterSyntax(self.netparams)
self.group = self.cfg.GetNodeGroup(self.group_uuid)
#if self.network_mode == constants.NIC_MODE_BRIDGED:
# _CheckNodeGroupBridgesExist(self, self.network_link, self.group_uuid)
self.connected = False
if self.network_uuid in self.group.networks:
self.LogWarning("Network '%s' is already mapped to group '%s'" %
(self.network_name, self.group.name))
self.connected = True
# check only if not already connected
elif self.op.conflicts_check:
pool = network.AddressPool(self.cfg.GetNetwork(self.network_uuid))
_NetworkConflictCheck(
self, lambda nic: pool.Contains(nic.ip), "connect to",
[instance_info for (_, instance_info) in
self.cfg.GetMultiInstanceInfoByName(owned_instance_names)])
def _update_beliefs(self, sending_clique, recieving_clique, operation):
"""
This is belief-update method.
Parameters
----------
sending_clique: node (as the operation is on junction tree, node should be a tuple)
Node sending the message
recieving_clique: node (as the operation is on junction tree, node should be a tuple)
Node recieving the message
operation: str ('marginalize' | 'maximize')
The operation to do for passing messages between nodes.
Takes belief of one clique and uses it to update the belief of the
neighboring ones.
"""
sepset = frozenset(sending_clique).intersection(frozenset(recieving_clique))
sepset_key = frozenset((sending_clique, recieving_clique))
# \sigma_{i \rightarrow j} = \sum_{C_i - S_{i, j}} \beta_i
# marginalize the clique over the sepset
sigma = getattr(self.clique_beliefs[sending_clique], operation)(list(frozenset(sending_clique) - sepset),
inplace=False)
# \beta_j = \beta_j * \frac{\sigma_{i \rightarrow j}}{\mu_{i, j}}
self.clique_beliefs[recieving_clique] *= (sigma / self.sepset_beliefs[sepset_key]
if self.sepset_beliefs[sepset_key] else sigma)
# \mu_{i, j} = \sigma_{i \rightarrow j}
self.sepset_beliefs[sepset_key] = sigma
def __init__(self, prefix, channels, subdirs=(), specs_to_add=(), specs_to_remove=()):
"""
Args:
prefix (str):
The conda prefix / environment location for which the :class:`Solver`
is being instantiated.
channels (Sequence[:class:`Channel`]):
A prioritized list of channels to use for the solution.
subdirs (Sequence[str]):
A prioritized list of subdirs to use for the solution.
specs_to_add (Set[:class:`MatchSpec`]):
The set of package specs to add to the prefix.
specs_to_remove (Set[:class:`MatchSpec`]):
The set of package specs to remove from the prefix.
"""
self.prefix = prefix
self.channels = IndexedSet(Channel(c) for c in channels or context.channels)
self.subdirs = tuple(s for s in subdirs or context.subdirs)
self.specs_to_add = frozenset(MatchSpec.merge(s for s in specs_to_add))
self.specs_to_remove = frozenset(MatchSpec.merge(s for s in specs_to_remove))
assert all(s in context.known_subdirs for s in self.subdirs)
self._index = None
self._r = None
self._prepared = False
def fromdict(cls, name, density):
symbol = Symbol(name)
domain = SingleFiniteDomain(symbol, frozenset(density.keys()))
density = dict((frozenset(((symbol, val),)), prob)
for val, prob in density.items())
density = Dict(density)
return FinitePSpace.__new__(cls, domain, density)
def mincoverages_upto_maxk__defi(m, q, dontcares, maxk, counters, mincov):
# In deficiency counter mode we act on all deficient ktuples
# and reduce their deficiency, if appropriate.
endk = maxk + 1
minc = [0] * endk
_append_new_defi_position(counters, m, q, mincov)
shift = m - q # put the q-gram into the window at the rightmost shift
sdc = list(range(shift)) + [d+shift for d in dontcares]
sdcset = frozenset(sdc)
D = len(sdc)
for k in range(1, endk):
# iterate over deficient tuples or over k-tuples with all dontcares?
ck = counters[k]
if k*len(ck) <= binom(D, k):
# iterate over keys
if k == 1:
to_reduce = [i for i in ck if i in sdcset]
else:
to_reduce = [dt for dt in ck if frozenset(dt) <= sdcset]
else:
# iterate over k-tuples from sdc
combos = combinations(sdc, k) if k > 1 else sdc
to_reduce = [dt for dt in combos if dt in ck]
# decrement all items of counters[k] in to_reduce
for key in to_reduce:
if ck[key] > 1:
ck[key] -= 1
else:
del ck[key]
# compute minimum coverage for each k = 1..maxk
minc[k] = mincov - (max(ck.values()) if len(ck) > 0 else 0)
return minc, counters
def __call__(self, oeb, context):
oeb.logger.info('Flattening CSS and remapping font sizes...')
self.context = self.opts =context
self.oeb = oeb
self.filter_css = frozenset()
if self.opts.filter_css:
try:
self.filter_css = {x.strip().lower() for x in
self.opts.filter_css.split(',')}
except:
self.oeb.log.warning('Failed to parse filter_css, ignoring')
else:
from calibre.ebooks.oeb.normalize_css import normalize_filter_css
self.filter_css = frozenset(normalize_filter_css(self.filter_css))
self.oeb.log.debug('Filtering CSS properties: %s'%
', '.join(self.filter_css))
for item in oeb.manifest.values():
# Make all links to resources absolute, as these sheets will be
# consolidated into a single stylesheet at the root of the document
if item.media_type in OEB_STYLES:
cssutils.replaceUrls(item.data, item.abshref,
ignoreImportRules=True)
self.body_font_family, self.embed_font_rules = self.get_embed_font_info(
self.opts.embed_font_family)
# Store for use in output plugins/transforms that generate content,
# like the AZW3 output inline ToC.
self.oeb.store_embed_font_rules = EmbedFontsCSSRules(self.body_font_family,
self.embed_font_rules)
self.stylize_spine()
self.sbase = self.baseline_spine() if self.fbase else None
self.fmap = FontMapper(self.sbase, self.fbase, self.fkey)
self.flatten_spine()
def test_creation():
metaedge_tuples = [
('compound', 'disease', 'indication', 'both'),
('disease', 'gene', 'association', 'both'),
('compound', 'gene', 'target', 'both'),
]
metanode_ids = 'compound', 'disease', 'gene'
metagraph = graph.MetaGraph.from_edge_tuples(metaedge_tuples)
# check that nodes got added to metagraph_node_dict
assert frozenset(metagraph.node_dict) == frozenset(metanode_ids)
for metanode in metagraph.node_dict.values():
assert isinstance(metanode, graph.MetaNode)
# check that metanode.get_id() and hash(metanode) are working as expected
for metanode_id in metanode_ids:
metanode = metagraph.node_dict[metanode_id]
assert metanode.identifier == metanode_id
assert metanode.get_id() == metanode_id
assert hash(metanode) == hash(metanode_id)
g = graph.Graph(metagraph)
ms = g.add_node('disease', 'DOID:2377', 'multiple sclerosis')
assert ms.metanode.identifier == 'disease'
assert ms.identifier == 'DOID:2377'
assert ms.name == 'multiple sclerosis'
with pytest.raises(KeyError):
# misordered args
g.add_node('DOID:2377', 'multiple sclerosis', 'disease')
def test_merge_entries(fx_stages, fx_test_feeds, fx_test_entries):
stage1, stage2 = fx_stages
feed1, feed2 = fx_test_feeds
entry0 = feed2.entries[0]
entry1, entry2 = fx_test_entries
assert feed1.id == feed2.id
feed1.entries.append(entry1)
feed2.entries.append(entry2)
print(feed1.entries)
print(feed2.entries)
assert entry1 in feed1.entries and entry2 in feed2.entries
assert entry2 not in feed1.entries and entry1 not in feed2.entries
with stage1:
stage1.feeds[get_hash(feed1.id)] = feed1
with stage2:
stage2.feeds[get_hash(feed2.id)] = feed2
with stage1:
feed1 = stage1.feeds[get_hash(feed1.id)]
with stage2:
feed2 = stage2.feeds[get_hash(feed2.id)]
print(repr(entry1))
print(repr(entry2))
print(feed1.entries)
print(feed2.entries)
assert (frozenset(entry.id for entry in feed1.entries) ==
frozenset(entry.id for entry in feed2.entries) ==
frozenset([entry0.id, entry2.id, entry1.id]))
def resolve_user_identifier(lookup_fields, required, args, kwargs):
"""
Resolves a user identifier from the given args
and kwargs.
If a user identifier is given, it should either be
keyword arguments, or positional arguments that match the fields in
settings.LDAP_AUTH_USER_LOOKUP_FIELDS.
"""
# Raises a type error if the args are incorrect.
def raise_error():
raise TypeError("Expected arguments: {lookup_fields}".format(
lookup_fields = ", ".join(map(force_text, lookup_fields)),
))
# Cannot use both args and kwargs.
if args and kwargs:
raise TypeError("Cannot use both args and kwargs to identify a user")
# Parse args.
if args:
if len(lookup_fields) != len(args):
raise_error()
return dict(zip(lookup_fields, args))
# Parse kwargs.
if kwargs:
if frozenset(lookup_fields) != frozenset(kwargs.keys()):
raise_error()
return kwargs.copy()
# No user identifier.
if required:
raise_error()
# All done!
return {}
def generate(self, nfa_graph):
states = []
visited = {}
nodes = []
node = frozenset(self.default_closure([nfa_graph.states[0]]))
nodes.append(node)
visited[node] = self.create_state(states)
visited[node].rules = [r.rule for r in node if r.rule != None]
index = 0
while index < len(nodes):
for i in self.get_inputs(nodes[index]):
node = frozenset(self.input_closure(nodes[index], i))
if node not in visited:
nodes.append(node)
visited[node] = self.create_state(states)
visited[node].rules = [r.rule for r in node if r.rule != None]
visited[nodes[index]].consume(i, visited[node])
index = index + 1
dfa_graph = DFAGraph()
dfa_graph.states = states
return dfa_graph
def from_board(board):
board = np.copy(board)
curr_group_id = 0
lib_tracker = LibertyTracker()
for color in (WHITE, BLACK):
while color in board:
curr_group_id += 1
found_color = np.where(board == color)
coord = found_color[0][0], found_color[1][0]
chain, reached = find_reached(board, coord)
liberties = frozenset(r for r in reached if board[r] == EMPTY)
new_group = Group(curr_group_id, frozenset(
chain), liberties, color)
lib_tracker.groups[curr_group_id] = new_group
for s in chain:
lib_tracker.group_index[s] = curr_group_id
place_stones(board, FILL, chain)
lib_tracker.max_group_id = curr_group_id
liberty_counts = np.zeros([N, N], dtype=np.uint8)
for group in lib_tracker.groups.values():
num_libs = len(group.liberties)
for s in group.stones:
liberty_counts[s] = num_libs
lib_tracker.liberty_cache = liberty_counts
return lib_tracker
def _ite(f, g, h):
"""Return node that results from recursively applying ITE(f, g, h)."""
# ITE(f, 1, 0) = f
if g is BDDNODEONE and h is BDDNODEZERO:
return f
# ITE(f, 0, 1) = -f
elif g is BDDNODEZERO and h is BDDNODEONE:
return _neg(f)
# ITE(1, g, h) = g
elif f is BDDNODEONE:
return g
# ITE(0, g, h) = h
elif f is BDDNODEZERO:
return h
# ITE(f, g, g) = g
elif g is h:
return g
else:
# ITE(f, g, h) = ITE(x, ITE(fx', gx', hx'), ITE(fx, gx, hx))
root = min(node.root for node in (f, g, h) if node.root > 0)
upoint0 = frozenset([root]), frozenset()
upoint1 = frozenset(), frozenset([root])
fv0, gv0, hv0 = [_urestrict(node, upoint0) for node in (f, g, h)]
fv1, gv1, hv1 = [_urestrict(node, upoint1) for node in (f, g, h)]
return bddnode(root, _ite(fv0, gv0, hv0), _ite(fv1, gv1, hv1))
def test_blankpattern(self):
# Make sure when tuple or something has no values no regex is generated.
# Fixes bug #661354
test_locale = _strptime.LocaleTime()
test_locale.timezone = (frozenset(), frozenset())
self.assertEqual(_strptime.TimeRE(test_locale).pattern("%Z"), '',
"with timezone == ('',''), TimeRE().pattern('%Z') != ''")
def test_metanode__properties():
node = MetaNode()
assert_equal(node.input_files, frozenset())
assert_equal(node.output_files, frozenset())
assert_equal(node.executables, frozenset())
assert_equal(node.auxiliary_files, frozenset())
assert_equal(node.requirements, frozenset())
def _get_autounmask_changes(self, pkg): needed_use_config_change = self.depgraph._dynamic_config._needed_use_config_changes.get(pkg) if needed_use_config_change is None: return frozenset() use, changes = needed_use_config_change return frozenset(changes.keys())
def test_constructor__requirements():
node = Node(requirements = id)
assert_equal(node.requirements, frozenset([id]))
node = Node(requirements = [id])
assert_equal(node.requirements, frozenset([id]))
node = Node(requirements = [id, str])
assert_equal(node.requirements, frozenset([id, str]))
def test_metanode__nodes():
subnodes = [Node(), Node()]
dependencies = [Node(), Node()]
node = MetaNode(subnodes=iter(subnodes),
dependencies=iter(dependencies))
assert_equal(node.subnodes, frozenset(subnodes))
assert_equal(node.dependencies, frozenset(dependencies))
def get_partitions(tree):
"""
Get all of the partitions implied by a tree.
Each positive branch implies a partition.
Each partition is a frozenset of two frozensets of leaf names.
The return value is the set of these partitions.
Note that the word 'partition' is a python keyword,
so the word 'part' will be used here instead.
@param tree: a tree object
@return: the set of partitions implied by the tree.
"""
# map a directed branch id to the set of leaf names in its subtree
d = _get_branch_id_to_leaf_name_set(tree)
# for each branch in the tree get the frozenset of leaf names on each end of the branch
parts = set()
for node in tree.gen_non_root_nodes():
parent = node.get_parent()
directed_branches = (node.get_directed_branch_to(parent), parent.get_directed_branch_to(node))
branch_length = directed_branches[0].get_branch_length()
for dbranch in directed_branches:
assert dbranch.get_branch_length() == branch_length
if branch_length > 0:
leaf_sets = [d[id(dbranch)] for dbranch in directed_branches]
part = frozenset(frozenset(leaf_set) for leaf_set in leaf_sets)
parts.add(part)
# return the set of partitions
return parts
def test_change_mapping(self):
""" Test using multiple input datasets, like if you were calculating a change. """
with mock.patch('cwsl.core.pattern_dataset.PatternDataSet.glob_fs') as mock_glob:
fake_file_1 = '/a/fake/file_1956_red.nc'
fake_file_2 = '/a/fake/file_1981_red.nc'
mock_glob.return_value = [fake_file_1, fake_file_2]
first_pattern_ds = PatternDataSet("/a/fake/file_%date%_%colour%.nc",
set([Constraint('date', ['1956'])]))
second_pattern_ds = PatternDataSet("/a/fake/file_%date%_%colour%.nc",
set([Constraint('date', ['1981'])]))
# Overwrite the valid combinations for these mock datasets.
first_pattern_ds.valid_combinations = set([frozenset([Constraint('colour', ['red']),
Constraint('date', ['1956'])])])
second_pattern_ds.valid_combinations = set([frozenset([Constraint('colour', ['red']),
Constraint('date', ['1981'])])])
the_process_unit = ProcessUnit([first_pattern_ds, second_pattern_ds],
"/a/final/output/file_%start_date%_%end_date%_%colour%.txt",
'echo', map_dict={'start_date': ('date', 0),
'end_date': ('date', 1)})
ds_result = the_process_unit.execute(simulate=True)
outfiles = [file_thing for file_thing in ds_result.files]
self.assertEqual(len(outfiles), 1)
expected_string = self.script_header + "mkdir -p /a/final/output\necho /a/fake/file_1956_red.nc /a/fake/file_1981_red.nc /a/final/output/file_1956_1981_red.txt\n"
self.assertEqual(expected_string, the_process_unit.scheduler.job.to_str())
def create_initial_revisions(self, app, model_class, verbosity=2, **kwargs):
"""
all stolen from management/__init__.py :)
"""
# Import the relevant admin module.
try:
import_module("%s.admin" % app.__name__.rsplit(".", 1)[0])
except ImportError:
pass
# Check all models for empty revisions.
if revision.is_registered(model_class):
content_type = ContentType.objects.get_for_model(model_class)
# Get the id for all models that have not got at least one revision.
# HACK: This join can't be done in the database, for potential incompatibilities
# between unicode object_ids and integer pks on strict backends like postgres.
versioned_ids = frozenset(Version.objects.filter(content_type=content_type).values_list("object_id", flat=True).distinct().iterator())
all_ids = frozenset(unicode(id) for id in model_class._default_manager.values_list("pk", flat=True).iterator())
unversioned_ids = all_ids - versioned_ids
# Create the initial revision for all unversioned models.
created_count = 0
for unversioned_obj in model_class._default_manager.filter(pk__in=unversioned_ids).iterator():
self.version_save(unversioned_obj)
created_count += 1
# Print out a message, if feeling verbose.
if created_count > 0 and verbosity >= 2:
self.stdout.write(u"Created %s initial revisions for model %s.\n" % (created_count, model_class._meta.verbose_name))
else:
if verbosity >= 2:
self.stdout.write(u"Model %s is not registered.\n" % (model_class._meta.verbose_name))
def load(self, field_xso):
self._value = frozenset(
self.field.type_.parse(value)
for value in field_xso.values
)
super().load(field_xso)
if encoding:
try:
return str(r.content, encoding)
except UnicodeError:
tried_encodings.append(encoding)
# Fall back:
try:
return str(r.content, encoding, errors='replace')
except TypeError:
return r.content
# The unreserved URI characters (RFC 3986)
UNRESERVED_SET = frozenset(
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + "0123456789-._~")
def unquote_unreserved(uri):
"""Un-escape any percent-escape sequences in a URI that are unreserved
characters. This leaves all reserved, illegal and non-ASCII bytes encoded.
"""
parts = uri.split('%')
for i in range(1, len(parts)):
h = parts[i][0:2]
if len(h) == 2 and h.isalnum():
try:
c = chr(int(h, 16))
except ValueError:
raise InvalidURL("Invalid percent-escape sequence: '%s'" % h)
from pathlib import Path
with (Path(__file__).parent / "input.txt").open() as puzzle_input_file:
puzzle_input_raw = puzzle_input_file.read()
import itertools
import functools
flights = {frozenset([x.split()[0], x.split()[2]]): int(x.split()[-1]) for x in puzzle_input_raw.splitlines()}
destinations = set(itertools.chain(*{x for x in flights}))
route_distances = {}
for route in itertools.permutations(destinations, len(destinations)):
distance = functools.reduce(lambda d, f: d + flights[frozenset(f)], zip(route, route[1:]), 0)
route_distances[route] = distance
shortest_route = max(route_distances.items(), key=lambda x: x[1])
print(shortest_route)
def _convert_indexed_to_array(expr):
if isinstance(expr, Sum):
function = expr.function
summation_indices = expr.variables
subexpr, subindices = _convert_indexed_to_array(function)
subindicessets = {j: i for i in subindices if isinstance(i, frozenset) for j in i}
summation_indices = sorted(set([subindicessets.get(i, i) for i in summation_indices]), key=default_sort_key)
# TODO: check that Kronecker delta is only contracted to one other element:
kronecker_indices = set([])
if isinstance(function, Mul):
for arg in function.args:
if not isinstance(arg, KroneckerDelta):
continue
arg_indices = sorted(set(arg.indices), key=default_sort_key)
if len(arg_indices) == 2:
kronecker_indices.update(arg_indices)
kronecker_indices = sorted(kronecker_indices, key=default_sort_key)
# Check dimensional consistency:
shape = get_shape(subexpr)
if shape:
for ind, istart, iend in expr.limits:
i = _get_argindex(subindices, ind)
if istart != 0 or iend+1 != shape[i]:
raise ValueError("summation index and array dimension mismatch: %s" % ind)
contraction_indices = []
subindices = list(subindices)
if isinstance(subexpr, ArrayDiagonal):
diagonal_indices = list(subexpr.diagonal_indices)
dindices = subindices[-len(diagonal_indices):]
subindices = subindices[:-len(diagonal_indices)]
for index in summation_indices:
if index in dindices:
position = dindices.index(index)
contraction_indices.append(diagonal_indices[position])
diagonal_indices[position] = None
diagonal_indices = [i for i in diagonal_indices if i is not None]
for i, ind in enumerate(subindices):
if ind in summation_indices:
pass
if diagonal_indices:
subexpr = _array_diagonal(subexpr.expr, *diagonal_indices)
else:
subexpr = subexpr.expr
axes_contraction = defaultdict(list)
for i, ind in enumerate(subindices):
include = all(j not in kronecker_indices for j in ind) if isinstance(ind, frozenset) else ind not in kronecker_indices
if ind in summation_indices and include:
axes_contraction[ind].append(i)
subindices[i] = None
for k, v in axes_contraction.items():
if any(i in kronecker_indices for i in k) if isinstance(k, frozenset) else k in kronecker_indices:
continue
contraction_indices.append(tuple(v))
free_indices = [i for i in subindices if i is not None]
indices_ret = list(free_indices)
indices_ret.sort(key=lambda x: free_indices.index(x))
return _array_contraction(
subexpr,
*contraction_indices,
free_indices=free_indices
), tuple(indices_ret)
if isinstance(expr, Mul):
args, indices = zip(*[_convert_indexed_to_array(arg) for arg in expr.args])
# Check if there are KroneckerDelta objects:
kronecker_delta_repl = {}
for arg in args:
if not isinstance(arg, KroneckerDelta):
continue
# Diagonalize two indices:
i, j = arg.indices
kindices = set(arg.indices)
if i in kronecker_delta_repl:
kindices.update(kronecker_delta_repl[i])
if j in kronecker_delta_repl:
kindices.update(kronecker_delta_repl[j])
kindices = frozenset(kindices)
for index in kindices:
kronecker_delta_repl[index] = kindices
# Remove KroneckerDelta objects, their relations should be handled by
# ArrayDiagonal:
newargs = []
newindices = []
for arg, loc_indices in zip(args, indices):
if isinstance(arg, KroneckerDelta):
continue
newargs.append(arg)
newindices.append(loc_indices)
flattened_indices = [kronecker_delta_repl.get(j, j) for i in newindices for j in i]
diagonal_indices, ret_indices = _get_diagonal_indices(flattened_indices)
tp = _array_tensor_product(*newargs)
if diagonal_indices:
return _array_diagonal(tp, *diagonal_indices), ret_indices
else:
return tp, ret_indices
if isinstance(expr, MatrixElement):
indices = expr.args[1:]
diagonal_indices, ret_indices = _get_diagonal_indices(indices)
if diagonal_indices:
return _array_diagonal(expr.args[0], *diagonal_indices), ret_indices
else:
return expr.args[0], ret_indices
if isinstance(expr, ArrayElement):
indices = expr.indices
diagonal_indices, ret_indices = _get_diagonal_indices(indices)
if diagonal_indices:
return _array_diagonal(expr.name, *diagonal_indices), ret_indices
else:
return expr.name, ret_indices
if isinstance(expr, Indexed):
indices = expr.indices
diagonal_indices, ret_indices = _get_diagonal_indices(indices)
if diagonal_indices:
return _array_diagonal(expr.base, *diagonal_indices), ret_indices
else:
return expr.args[0], ret_indices
if isinstance(expr, IndexedBase):
raise NotImplementedError
if isinstance(expr, KroneckerDelta):
return expr, expr.indices
if isinstance(expr, Add):
args, indices = zip(*[_convert_indexed_to_array(arg) for arg in expr.args])
args = list(args)
# Check if all indices are compatible. Otherwise expand the dimensions:
index0 = []
shape0 = []
for arg, arg_indices in zip(args, indices):
arg_indices_set = set(arg_indices)
arg_indices_missing = arg_indices_set.difference(index0)
index0.extend([i for i in arg_indices if i in arg_indices_missing])
arg_shape = get_shape(arg)
shape0.extend([arg_shape[i] for i, e in enumerate(arg_indices) if e in arg_indices_missing])
for i, (arg, arg_indices) in enumerate(zip(args, indices)):
if len(arg_indices) < len(index0):
missing_indices_pos = [i for i, e in enumerate(index0) if e not in arg_indices]
missing_shape = [shape0[i] for i in missing_indices_pos]
arg_indices = tuple(index0[j] for j in missing_indices_pos) + arg_indices
args[i] = _array_tensor_product(OneArray(*missing_shape), args[i])
permutation = Permutation([arg_indices.index(j) for j in index0])
# Perform index permutations:
args[i] = _permute_dims(args[i], permutation)
return _array_add(*args), tuple(index0)
if isinstance(expr, Pow):
subexpr, subindices = _convert_indexed_to_array(expr.base)
if isinstance(expr.exp, (int, Integer)):
diags = zip(*[(2*i, 2*i + 1) for i in range(expr.exp)])
arr = _array_diagonal(_array_tensor_product(*[subexpr for i in range(expr.exp)]), *diags)
return arr, subindices
return expr, ()
def __init__(self, dct):
self._symbols = frozenset(dct.items())
def merge_metadata_results(self, merge_on_identifiers=False):
'''
Merge results with identical title and authors or an identical
identifier
'''
# First title/author
groups = {}
for result in self.results:
title = lower(result.title if result.title else '')
key = (title, tuple(lower(x) for x in result.authors))
if key not in groups:
groups[key] = []
groups[key].append(result)
if len(groups) != len(self.results):
self.results = []
for rgroup in groups.itervalues():
rel = [r.average_source_relevance for r in rgroup]
if len(rgroup) > 1:
result = self.merge(rgroup, None, do_asr=False)
result.average_source_relevance = sum(rel) / len(rel)
else:
result = rgroup[0]
self.results.append(result)
if merge_on_identifiers:
# Now identifiers
groups, empty = {}, []
for result in self.results:
key = set()
for typ, val in result.identifiers.iteritems():
if typ and val:
key.add((typ, val))
if key:
key = frozenset(key)
match = None
for candidate in list(groups):
if candidate.intersection(key):
# We have at least one identifier in common
match = candidate.union(key)
results = groups.pop(candidate)
results.append(result)
groups[match] = results
break
if match is None:
groups[key] = [result]
else:
empty.append(result)
if len(groups) != len(self.results):
self.results = []
for rgroup in groups.itervalues():
rel = [r.average_source_relevance for r in rgroup]
if len(rgroup) > 1:
result = self.merge(rgroup, None, do_asr=False)
result.average_source_relevance = sum(rel) / len(rel)
elif rgroup:
result = rgroup[0]
self.results.append(result)
if empty:
self.results.extend(empty)
self.results.sort(key=attrgetter('average_source_relevance'))
IdlTypes are picklable because we store them in interfaces_info.
"""
from collections import defaultdict
################################################################################
# IDL types
################################################################################
INTEGER_TYPES = frozenset([
# http://www.w3.org/TR/WebIDL/#dfn-integer-type
'byte',
'octet',
'short',
'unsigned short',
# int and unsigned are not IDL types
'long',
'unsigned long',
'long long',
'unsigned long long',
])
NUMERIC_TYPES = (INTEGER_TYPES | frozenset([
# http://www.w3.org/TR/WebIDL/#dfn-numeric-type
'float',
'unrestricted float',
'double',
'unrestricted double',
]))
# http://www.w3.org/TR/WebIDL/#dfn-primitive-type
PRIMITIVE_TYPES = (frozenset(['boolean']) | NUMERIC_TYPES)
BASIC_TYPES = (PRIMITIVE_TYPES | frozenset([
from calibre.ebooks.metadata.opf2 import metadata_to_opf
from calibre.library.save_to_disk import find_plugboard
from calibre.srv.errors import HTTPNotFound, BookNotFound
from calibre.srv.routes import endpoint, json
from calibre.srv.utils import http_date, get_db, get_use_roman
from calibre.utils.config_base import tweaks
from calibre.utils.date import timestampfromdt
from calibre.utils.img import scale_image, image_from_data
from calibre.utils.filenames import ascii_filename, atomic_rename
from calibre.utils.shared_file import share_open
from polyglot.urllib import quote
from polyglot.binary import as_hex_unicode
plugboard_content_server_value = 'content_server'
plugboard_content_server_formats = ['epub', 'mobi', 'azw3']
update_metadata_in_fmts = frozenset(plugboard_content_server_formats)
lock = Lock()
# Get book formats/cover as a cached filesystem file {{{
# We cannot store mtimes in the filesystem since some operating systems (OS X)
# have only one second precision for mtimes
mtimes = {}
rename_counter = 0
def reset_caches():
mtimes.clear()
def open_for_write(fname):
FILE_CACHE_MAXSIZE = 'file_cache_maxsize'
CMAP_SEQUENTIAL = 'cmap_sequential'
CMAP_DIVERGENT = 'cmap_divergent'
KEEP_ATTRS = 'keep_attrs'
OPTIONS = {
DISPLAY_WIDTH: 80,
ARITHMETIC_JOIN: 'inner',
ENABLE_CFTIMEINDEX: True,
FILE_CACHE_MAXSIZE: 128,
CMAP_SEQUENTIAL: 'viridis',
CMAP_DIVERGENT: 'RdBu_r',
KEEP_ATTRS: 'default'
}
_JOIN_OPTIONS = frozenset(['inner', 'outer', 'left', 'right', 'exact'])
def _positive_integer(value):
return isinstance(value, int) and value > 0
_VALIDATORS = {
DISPLAY_WIDTH: _positive_integer,
ARITHMETIC_JOIN: _JOIN_OPTIONS.__contains__,
ENABLE_CFTIMEINDEX: lambda value: isinstance(value, bool),
FILE_CACHE_MAXSIZE: _positive_integer,
KEEP_ATTRS: lambda choice: choice in [True, False, 'default']
}
def test_frozenset(self):
f = frozenset(range(10))
self.assertEqual(self.dump_and_load(f), f)
def __init__(self, blocks, partitions):
self.blocks = frozenset(blocks)
self.partitions = partitions
self._config = None
# Minimum value for timeouts.
_MIN_TIMEOUT_SECS = 1 if utils.is_local_dev_server() else 30
# The world started on 2010-01-01 at 00:00:00 UTC. The rationale is that using
# EPOCH (1970) means that 40 years worth of keys are wasted.
#
# Note: This creates a 'naive' object instead of a formal UTC object. Note that
# datetime.datetime.utcnow() also return naive objects. That's python.
_BEGINING_OF_THE_WORLD = datetime.datetime(2010, 1, 1, 0, 0, 0, 0)
# Used for isolated files.
_HASH_CHARS = frozenset('0123456789abcdef')
_NAMESPACE_RE = re.compile(r'[a-z0-9A-Z\-._]+')
### Properties validators must come before the models.
def _validate_isolated(prop, value):
if value:
if not _HASH_CHARS.issuperset(value) or len(value) != 40:
raise datastore_errors.BadValueError(
'%s must be lowercase hex of length 40, not %s' % (prop._name, value))
def _validate_hostname(prop, value):
# pylint: disable=unused-argument
kar_types = frozenset([ "char", "signed char", "unsigned char", "int", "signed", "signed int", "unsigned", "unsigned int", "short", "short int", "short signed", "short signed int", "short unsigned", "short unsigned int", "signed short", "signed short int", "unsigned short", "unsigned short int", "long", "long int", "long signed", "long signed int", "long unsigned", "long unsigned int", "signed long", "signed long int", "unsigned long", "unsigned long int", "float", "double", ])
def complement(self):
blocks = frozenset(range(self.partitions)) - self.blocks
return Region(blocks, self.partitions)
e_set=set()
num_set = {1,2,3}
#hashed lists, stored unmutable objects unsorted, unique objects to store , but set is mutable
print(2 in num_set)
#odd or even set
odd_set=set()
even_set=set()
for i in range(1,11):
if i%2:
odd_set.add(i)
else:
even_set.add(i)
print(even_set)
print(odd_set)
#operations
union_set=odd_set|even_set
intersect_set=union_set & odd_set
difference_set = union_set - odd_set
print("union: " , union_set)
print("intersect: " , intersect_set)
print("difference: " , difference_set)
#unmutable set
frozen = frozenset()
ENGLISH_STOP_WORDS = frozenset([
"a", "about", "above", "across", "after", "afterwards", "again", "against",
"all", "almost", "alone", "along", "already", "also", "although", "always",
"am", "among", "amongst", "amoungst", "amount", "an", "and", "another",
"any", "anyhow", "anyone", "anything", "anyway", "anywhere", "are",
"around", "as", "at", "back", "be", "became", "because", "become",
"becomes", "becoming", "been", "before", "beforehand", "behind", "being",
"below", "beside", "besides", "between", "beyond", "bill", "both",
"bottom", "but", "by", "call", "can", "cannot", "cant", "co", "con",
"could", "couldnt", "cry", "de", "describe", "detail", "do", "done",
"down", "due", "during", "each", "eg", "eight", "either", "eleven", "else",
"elsewhere", "empty", "enough", "etc", "even", "ever", "every", "everyone",
"everything", "everywhere", "except", "few", "fifteen", "fifty", "fill",
"find", "fire", "first", "five", "for", "former", "formerly", "forty",
"found", "four", "from", "front", "full", "further", "get", "give", "go",
"had", "has", "hasnt", "have", "he", "hence", "her", "here", "hereafter",
"hereby", "herein", "hereupon", "hers", "herself", "him", "himself", "his",
"how", "however", "hundred", "i", "ie", "if", "in", "inc", "indeed",
"interest", "into", "is", "it", "its", "itself", "keep", "last", "latter",
"latterly", "least", "less", "ltd", "made", "many", "may", "me",
"meanwhile", "might", "mill", "mine", "more", "moreover", "most", "mostly",
"move", "much", "must", "my", "myself", "name", "namely", "neither",
"never", "nevertheless", "next", "nine", "no", "nobody", "none", "noone",
"nor", "not", "nothing", "now", "nowhere", "of", "off", "often", "on",
"once", "one", "only", "onto", "or", "other", "others", "otherwise", "our",
"ours", "ourselves", "out", "over", "own", "part", "per", "perhaps",
"please", "put", "rather", "re", "same", "see", "seem", "seemed",
"seeming", "seems", "serious", "several", "she", "should", "show", "side",
"since", "sincere", "six", "sixty", "so", "some", "somehow", "someone",
"something", "sometime", "sometimes", "somewhere", "still", "such",
"system", "take", "ten", "than", "that", "the", "their", "them",
"themselves", "then", "thence", "there", "thereafter", "thereby",
"therefore", "therein", "thereupon", "these", "they", "thick", "thin",
"third", "this", "those", "though", "three", "through", "throughout",
"thru", "thus", "to", "together", "too", "top", "toward", "towards",
"twelve", "twenty", "two", "un", "under", "until", "up", "upon", "us",
"very", "via", "was", "we", "well", "were", "what", "whatever", "when",
"whence", "whenever", "where", "whereafter", "whereas", "whereby",
"wherein", "whereupon", "wherever", "whether", "which", "while", "whither",
"who", "whoever", "whole", "whom", "whose", "why", "will", "with",
"within", "without", "would", "yet", "you", "your", "yours", "yourself",
"yourselves"])
def freeze(d):
if isinstance(d, dict):
return frozenset((key, freeze(value)) for key, value in d.items())
elif isinstance(d, list):
return tuple(freeze(value) for value in d)
return d
from .debug_info import log_messages
from .mixins import MQTT_ENTITY_COMMON_SCHEMA, MqttEntity, async_setup_entry_helper
_LOGGER = logging.getLogger(__name__)
CONF_MIN = "min"
CONF_MAX = "max"
CONF_STEP = "step"
DEFAULT_NAME = "MQTT Number"
DEFAULT_OPTIMISTIC = False
MQTT_NUMBER_ATTRIBUTES_BLOCKED = frozenset(
{
number.ATTR_MAX,
number.ATTR_MIN,
number.ATTR_STEP,
}
)
def validate_config(config):
"""Validate that the configuration is valid, throws if it isn't."""
if config.get(CONF_MIN) >= config.get(CONF_MAX):
raise vol.Invalid(f"'{CONF_MAX}' must be > '{CONF_MIN}'")
return config
PLATFORM_SCHEMA = vol.All(
mqtt.MQTT_RW_PLATFORM_SCHEMA.extend(
def validate_flavor(self, flavor_metadata):
LOG.debug('Provider %s no-op, validate_flavor metadata: %s',
self.__class__.__name__, flavor_metadata)
flavor_hash = hash(frozenset(flavor_metadata.items()))
self.driverconfig[flavor_hash] = (flavor_metadata, 'validate_flavor')
elif count or winType == "win:GUID" or count == "win:count":
#special case for GUIDS, consider them as structs
self.count = "win:count"
else:
self.count = "win:null"
def getTopLevelElementsByTagName(node,tag):
dataNodes = []
for element in node.getElementsByTagName(tag):
if element.parentNode == node:
dataNodes.append(element)
return dataNodes
ignoredXmlTemplateAttribes = frozenset(["map","outType"])
usedXmlTemplateAttribes = frozenset(["name","inType","count", "length"])
def parseTemplateNodes(templateNodes):
#return values
allTemplates = {}
for templateNode in templateNodes:
structCounts = {}
arrays = {}
templateName = templateNode.getAttribute('tid')
var_Dependecies = {}
fnPrototypes = FunctionSignature()
dataNodes = getTopLevelElementsByTagName(templateNode,'data')
class UtmpParser(dtfabric_parser.DtFabricBaseParser):
"""Parser for Linux libc6 utmp files."""
NAME = 'utmp'
DATA_FORMAT = 'Linux libc6 utmp file'
_DEFINITION_FILE = 'utmp.yaml'
_EMPTY_IP_ADDRESS = (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
_SUPPORTED_TYPES = frozenset(range(0, 10))
_DEAD_PROCESS_TYPE = 8
def _ReadEntry(self, parser_mediator, file_object, file_offset):
"""Reads an utmp entry.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): a file-like object.
file_offset (int): offset of the data relative to the start of
the file-like object.
Returns:
tuple: containing:
int: timestamp, which contains the number of microseconds
since January 1, 1970, 00:00:00 UTC.
UtmpEventData: event data of the utmp entry read.
list[str]: warning messages emitted by the parser.
Raises:
ParseError: if the entry cannot be parsed.
"""
entry_map = self._GetDataTypeMap('linux_libc6_utmp_entry')
warning_strings = []
try:
entry, _ = self._ReadStructureFromFileObject(
file_object, file_offset, entry_map)
except (ValueError, errors.ParseError) as exception:
raise errors.ParseError((
'Unable to parse utmp entry at offset: 0x{0:08x} with error: '
'{1!s}.').format(file_offset, exception))
if entry.type not in self._SUPPORTED_TYPES:
raise errors.ParseError('Unsupported type: {0:d}'.format(entry.type))
encoding = parser_mediator.codepage or 'utf-8'
try:
username = entry.username.split(b'\x00')[0]
username = username.decode(encoding)
except UnicodeDecodeError:
warning_strings.append('unable to decode username string')
username = None
try:
terminal = entry.terminal.split(b'\x00')[0]
terminal = terminal.decode(encoding)
except UnicodeDecodeError:
warning_strings.append('unable to decode terminal string')
terminal = None
if terminal == '~':
terminal = 'system boot'
try:
hostname = entry.hostname.split(b'\x00')[0]
hostname = hostname.decode(encoding)
except UnicodeDecodeError:
warning_strings.append('unable to decode hostname string')
hostname = None
if not hostname or hostname == ':0':
hostname = 'localhost'
if entry.ip_address[4:] == self._EMPTY_IP_ADDRESS[4:]:
ip_address = self._FormatPackedIPv4Address(entry.ip_address[:4])
else:
ip_address = self._FormatPackedIPv6Address(entry.ip_address)
# TODO: add termination status.
event_data = UtmpEventData()
event_data.hostname = hostname
event_data.exit_status = entry.exit_status
event_data.ip_address = ip_address
event_data.offset = file_offset
event_data.pid = entry.pid
event_data.terminal = terminal
event_data.terminal_identifier = entry.terminal_identifier
event_data.type = entry.type
event_data.username = username
timestamp = entry.microseconds + (
entry.timestamp * definitions.MICROSECONDS_PER_SECOND)
return timestamp, event_data, warning_strings
def ParseFileObject(self, parser_mediator, file_object):
"""Parses an utmp file-like object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): a file-like object.
Raises:
UnableToParseFile: when the file cannot be parsed.
"""
file_offset = 0
try:
timestamp, event_data, warning_strings = self._ReadEntry(
parser_mediator, file_object, file_offset)
except errors.ParseError as exception:
raise errors.UnableToParseFile(
'Unable to parse first utmp entry with error: {0!s}'.format(
exception))
if not timestamp:
raise errors.UnableToParseFile(
'Unable to parse first utmp entry with error: missing timestamp')
if not event_data.username and event_data.type != self._DEAD_PROCESS_TYPE:
raise errors.UnableToParseFile(
'Unable to parse first utmp entry with error: missing username')
if warning_strings:
all_warnings = ', '.join(warning_strings)
raise errors.UnableToParseFile(
'Unable to parse first utmp entry with error: {0:s}'.format(
all_warnings))
date_time = dfdatetime_posix_time.PosixTimeInMicroseconds(
timestamp=timestamp)
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_START)
parser_mediator.ProduceEventWithEventData(event, event_data)
file_offset = file_object.tell()
file_size = file_object.get_size()
while file_offset < file_size:
if parser_mediator.abort:
break
try:
timestamp, event_data, warning_strings = self._ReadEntry(
parser_mediator, file_object, file_offset)
except errors.ParseError:
# Note that the utmp file can contain trailing data.
break
date_time = dfdatetime_posix_time.PosixTimeInMicroseconds(
timestamp=timestamp)
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_START)
parser_mediator.ProduceEventWithEventData(event, event_data)
for warning_string in warning_strings:
parser_mediator.ProduceExtractionWarning(warning_string)
file_offset = file_object.tell()
def treewidth_decomp(G, heuristic=min_fill_in_heuristic):
graph = {n: set(G[n]) - set([n]) for n in G}
# stack containing nodes and neighbors in the order from the heuristic
node_stack = []
# get first node from heuristic
elim_node = heuristic(graph)
while elim_node is not None:
# connect all neighbours with each other
nbrs = graph[elim_node]
for u, v in itertools.permutations(nbrs, 2):
if v not in graph[u]:
graph[u].add(v)
# push node and its current neighbors on stack
node_stack.append((elim_node, nbrs))
# remove node from graph
for u in graph[elim_node]:
graph[u].remove(elim_node)
del graph[elim_node]
elim_node = heuristic(graph)
# the abort condition is met; put all remaining nodes into one bag
decomp = nx.Graph()
first_bag = frozenset(graph.keys())
decomp.add_node(first_bag)
treewidth = len(first_bag) - 1
while node_stack:
# get node and its neighbors from the stack
(curr_node, nbrs) = node_stack.pop()
# find a bag all neighbors are in
old_bag = None
for bag in decomp.nodes:
if nbrs <= bag:
old_bag = bag
break
if old_bag is None:
# no old_bag was found: just connect to the first_bag
old_bag = first_bag
# create new node for decomposition
nbrs.add(curr_node)
new_bag = frozenset(nbrs)
# update treewidth
treewidth = max(treewidth, len(new_bag) - 1)
# add edge to decomposition (implicitly also adds the new node)
decomp.add_edge(old_bag, new_bag)
decomp = nx.convert_node_labels_to_integers(decomp, label_attribute='bag')
# nice_tree_decomp(out)
# nx.draw(decomp, with_labels=True)
# plt.show()
return treewidth, decomp
#grouping by business: every pair needs to group
inputGroup = dict(inputIndex.groupByKey().mapValues(lambda row: set(row)).collect())
#number of minhash
minhash_num = 51
hash_para = HashParaGenerator(minhash_num, userBin)
# r=2/b=12/recall=0.99 r=3/b=16/recall=0.89
bands = 18
#minhash
signs = inputIndex.flatMap(lambda row: hashValue(row, hash_para)).reduceByKey(lambda x, y: y if x >= y else x).persist(StorageLevel.DISK_ONLY)
# output > ((business_id, hashindex), minhashvalue)
signs_bands = signs.map(lambda row: ((row[0][1]%bands, row[0][0]), (row[0][1], row[1]))).groupByKey()\
.map(lambda row: (frozenset(row[1]), row[0][1])).groupByKey().map(lambda business: sorted(list(business[1])))\
.filter(lambda bucket: len(bucket)>1).persist(StorageLevel.DISK_ONLY)
# output > (bandIndex, business_id), (hashIndex, hashValue)
# output > (frozenset(hashIndex, hashValue)), business_id
# output > business_ids
candidate_pairs = signs_bands.flatMap(lambda bucket: [pair for pair in itertools.combinations(bucket, 2)]).distinct().sortBy(lambda pair: (pair[0], pair[1])).persist(StorageLevel.DISK_ONLY)
# Jaccard similarity for every candidate pairs from LSH
PairsSim = candidate_pairs.map(lambda pair: Jarccard(pair, inputGroup)).filter(lambda sim: sim[1]>=0.5).sortBy(lambda pair: (pair[0][0], pair[0][1])).persist(StorageLevel.DISK_ONLY)
#write txt file
header = sc.parallelize(["business_id_1, business_id_2, similarity"],1)
PairsSimOutput = PairsSim.map(lambda sim: sim[0][0]+','+sim[0][1]+','+str(sim[1]))
output = '\n'.join(header.union(PairsSimOutput).collect())
with open(outputFilePath, 'w') as fd:
class BuildFSM(TreeParser):
grammarFileName = "src\\SavedFSM\\BuildFSM.g"
antlr_version = version_str_to_tuple("3.1.3 Mar 18, 2009 10:09:25")
antlr_version_str = "3.1.3 Mar 18, 2009 10:09:25"
tokenNames = tokenNames
def __init__(self, input, state=None, *args, **kwargs):
if state is None:
state = RecognizerSharedState()
super(BuildFSM, self).__init__(input, state, *args, **kwargs)
# memory here is used only for logging and debugging purposes.
# We append bebugging information to memory so we can print it later.
self.memory = []
self.main_fsm = FSMBuilderState()
self.current_fsm = self.main_fsm
# $ANTLR start "description"
# src\\SavedFSM\\BuildFSM.g:107:1: description : ^( PROTOCOL ( activityDef )+ ) ;
def description(self, ):
try:
try:
# src\\SavedFSM\\BuildFSM.g:107:12: ( ^( PROTOCOL ( activityDef )+ ) )
# src\\SavedFSM\\BuildFSM.g:107:14: ^( PROTOCOL ( activityDef )+ )
pass
self.match(self.input, PROTOCOL, self.FOLLOW_PROTOCOL_in_description52)
self.match(self.input, DOWN, None)
# src\\SavedFSM\\BuildFSM.g:107:25: ( activityDef )+
cnt1 = 0
while True: #loop1
alt1 = 2
LA1_0 = self.input.LA(1)
if ((RESV <= LA1_0 <= SEND) or (RECLABEL <= LA1_0 <= PARALLEL) or LA1_0 == GLOBAL_ESCAPE or LA1_0 == 47 or (49 <= LA1_0 <= 50)) :
alt1 = 1
if alt1 == 1:
# src\\SavedFSM\\BuildFSM.g:107:25: activityDef
pass
self._state.following.append(self.FOLLOW_activityDef_in_description54)
self.activityDef()
self._state.following.pop()
else:
if cnt1 >= 1:
break #loop1
eee = EarlyExitException(1, self.input)
raise eee
cnt1 += 1
self.match(self.input, UP, None)
#action start
print "ProtocolDefinition"
#action end
except RecognitionException, re:
self.reportError(re)
self.recover(self.input, re)
finally:
pass
return
# $ANTLR end "description"
# $ANTLR start "activityDef"
# src\\SavedFSM\\BuildFSM.g:108:1: activityDef : ( ^( RESV ( ^( VALUE ( (val= ID vtype= ( INT | STRING ) ) )* ) ) rlabel= ID (rtype= ID )* role= ID ( ^( ASSERT (assertion= ASSERTION )? ) ) ) | ^( SEND ( ^( VALUE ( (val= ID vtype= ( INT | STRING ) ) )* ) ) slabel= ID (stype= ID )* role= ID ( ^( ASSERT (assertion= ASSERTION )? ) ) ) | ^( 'choice' ( ^( BRANCH ( activityDef )+ ) )+ ) | ^( PARALLEL ( ^( BRANCH ( activityDef )+ ) )+ ) | ^( 'repeat' ( ^( BRANCH ( activityDef )+ ) ) ) | ^( 'rec' label= ID ( ^( BRANCH ( activityDef )+ ) ) ) | ^( 'RECLABEL' labelID= ID ) | ^( GLOBAL_ESCAPE ( ^( 'do' ( ( activityDef )+ ) ) ) ( ^( 'interrupt' roleName ( ( activityDef )+ ) ) ) ) );
def activityDef(self, ):
val = None
vtype = None
rlabel = None
rtype = None
role = None
assertion = None
slabel = None
stype = None
label = None
labelID = None
try:
try:
# src\\SavedFSM\\BuildFSM.g:108:12: ( ^( RESV ( ^( VALUE ( (val= ID vtype= ( INT | STRING ) ) )* ) ) rlabel= ID (rtype= ID )* role= ID ( ^( ASSERT (assertion= ASSERTION )? ) ) ) | ^( SEND ( ^( VALUE ( (val= ID vtype= ( INT | STRING ) ) )* ) ) slabel= ID (stype= ID )* role= ID ( ^( ASSERT (assertion= ASSERTION )? ) ) ) | ^( 'choice' ( ^( BRANCH ( activityDef )+ ) )+ ) | ^( PARALLEL ( ^( BRANCH ( activityDef )+ ) )+ ) | ^( 'repeat' ( ^( BRANCH ( activityDef )+ ) ) ) | ^( 'rec' label= ID ( ^( BRANCH ( activityDef )+ ) ) ) | ^( 'RECLABEL' labelID= ID ) | ^( GLOBAL_ESCAPE ( ^( 'do' ( ( activityDef )+ ) ) ) ( ^( 'interrupt' roleName ( ( activityDef )+ ) ) ) ) )
alt16 = 8
LA16 = self.input.LA(1)
if LA16 == RESV:
alt16 = 1
elif LA16 == SEND:
alt16 = 2
elif LA16 == 47:
alt16 = 3
elif LA16 == PARALLEL:
alt16 = 4
elif LA16 == 49:
alt16 = 5
elif LA16 == 50:
alt16 = 6
elif LA16 == RECLABEL:
alt16 = 7
elif LA16 == GLOBAL_ESCAPE:
alt16 = 8
else:
nvae = NoViableAltException("", 16, 0, self.input)
raise nvae
if alt16 == 1:
# src\\SavedFSM\\BuildFSM.g:109:2: ^( RESV ( ^( VALUE ( (val= ID vtype= ( INT | STRING ) ) )* ) ) rlabel= ID (rtype= ID )* role= ID ( ^( ASSERT (assertion= ASSERTION )? ) ) )
pass
self.match(self.input, RESV, self.FOLLOW_RESV_in_activityDef66)
#action start
local_context = []
#action end
self.match(self.input, DOWN, None)
# src\\SavedFSM\\BuildFSM.g:110:5: ( ^( VALUE ( (val= ID vtype= ( INT | STRING ) ) )* ) )
# src\\SavedFSM\\BuildFSM.g:110:6: ^( VALUE ( (val= ID vtype= ( INT | STRING ) ) )* )
pass
self.match(self.input, VALUE, self.FOLLOW_VALUE_in_activityDef76)
if self.input.LA(1) == DOWN:
self.match(self.input, DOWN, None)
# src\\SavedFSM\\BuildFSM.g:110:14: ( (val= ID vtype= ( INT | STRING ) ) )*
while True: #loop2
alt2 = 2
LA2_0 = self.input.LA(1)
if (LA2_0 == ID) :
alt2 = 1
if alt2 == 1:
# src\\SavedFSM\\BuildFSM.g:110:15: (val= ID vtype= ( INT | STRING ) )
pass
# src\\SavedFSM\\BuildFSM.g:110:15: (val= ID vtype= ( INT | STRING ) )
# src\\SavedFSM\\BuildFSM.g:110:16: val= ID vtype= ( INT | STRING )
pass
val=self.match(self.input, ID, self.FOLLOW_ID_in_activityDef82)
vtype = self.input.LT(1)
if (INT <= self.input.LA(1) <= STRING):
self.input.consume()
self._state.errorRecovery = False
else:
mse = MismatchedSetException(None, self.input)
raise mse
#action start
if ((val is not None) and (vtype is not None)): local_context.append((val.text, vtype.text))
#action end
else:
break #loop2
self.match(self.input, UP, None)
rlabel=self.match(self.input, ID, self.FOLLOW_ID_in_activityDef107)
#action start
#INFO: This is the way to write comments in actions self.memory.append('resv' + rlabel.text)
#action end
# src\\SavedFSM\\BuildFSM.g:112:5: (rtype= ID )*
while True: #loop3
alt3 = 2
LA3_0 = self.input.LA(1)
if (LA3_0 == ID) :
LA3_1 = self.input.LA(2)
if (LA3_1 == ID) :
alt3 = 1
if alt3 == 1:
# src\\SavedFSM\\BuildFSM.g:112:6: rtype= ID
pass
rtype=self.match(self.input, ID, self.FOLLOW_ID_in_activityDef121)
#action start
self.memory.append(rtype.text)
#action end
else:
break #loop3
role=self.match(self.input, ID, self.FOLLOW_ID_in_activityDef131)
# src\\SavedFSM\\BuildFSM.g:113:5: ( ^( ASSERT (assertion= ASSERTION )? ) )
# src\\SavedFSM\\BuildFSM.g:113:6: ^( ASSERT (assertion= ASSERTION )? )
pass
self.match(self.input, ASSERT, self.FOLLOW_ASSERT_in_activityDef139)
if self.input.LA(1) == DOWN:
self.match(self.input, DOWN, None)
# src\\SavedFSM\\BuildFSM.g:113:15: (assertion= ASSERTION )?
alt4 = 2
LA4_0 = self.input.LA(1)
if (LA4_0 == ASSERTION) :
alt4 = 1
if alt4 == 1:
# src\\SavedFSM\\BuildFSM.g:113:16: assertion= ASSERTION
pass
assertion=self.match(self.input, ASSERTION, self.FOLLOW_ASSERTION_in_activityDef144)
self.match(self.input, UP, None)
self.match(self.input, UP, None)
#action start
self.current_fsm.add_transition(TransitionFactory.create(LocalType.RESV,rlabel, role), assertion, local_context)
#action end
elif alt16 == 2:
# src\\SavedFSM\\BuildFSM.g:119:3: ^( SEND ( ^( VALUE ( (val= ID vtype= ( INT | STRING ) ) )* ) ) slabel= ID (stype= ID )* role= ID ( ^( ASSERT (assertion= ASSERTION )? ) ) )
pass
self.match(self.input, SEND, self.FOLLOW_SEND_in_activityDef163)
#action start
local_context = []
#action end
self.match(self.input, DOWN, None)
# src\\SavedFSM\\BuildFSM.g:120:12: ( ^( VALUE ( (val= ID vtype= ( INT | STRING ) ) )* ) )
# src\\SavedFSM\\BuildFSM.g:120:13: ^( VALUE ( (val= ID vtype= ( INT | STRING ) ) )* )
pass
self.match(self.input, VALUE, self.FOLLOW_VALUE_in_activityDef180)
if self.input.LA(1) == DOWN:
self.match(self.input, DOWN, None)
# src\\SavedFSM\\BuildFSM.g:120:21: ( (val= ID vtype= ( INT | STRING ) ) )*
while True: #loop5
alt5 = 2
LA5_0 = self.input.LA(1)
if (LA5_0 == ID) :
alt5 = 1
if alt5 == 1:
# src\\SavedFSM\\BuildFSM.g:120:22: (val= ID vtype= ( INT | STRING ) )
pass
# src\\SavedFSM\\BuildFSM.g:120:22: (val= ID vtype= ( INT | STRING ) )
# src\\SavedFSM\\BuildFSM.g:120:23: val= ID vtype= ( INT | STRING )
pass
val=self.match(self.input, ID, self.FOLLOW_ID_in_activityDef186)
vtype = self.input.LT(1)
if (INT <= self.input.LA(1) <= STRING):
self.input.consume()
self._state.errorRecovery = False
else:
mse = MismatchedSetException(None, self.input)
raise mse
#action start
if ((val is not None) and (vtype is not None)): local_context.append((val.text, vtype.text))
#action end
else:
break #loop5
self.match(self.input, UP, None)
slabel=self.match(self.input, ID, self.FOLLOW_ID_in_activityDef212)
#action start
self.memory.append('send' + slabel.text)
#action end
# src\\SavedFSM\\BuildFSM.g:121:61: (stype= ID )*
while True: #loop6
alt6 = 2
LA6_0 = self.input.LA(1)
if (LA6_0 == ID) :
LA6_1 = self.input.LA(2)
if (LA6_1 == ID) :
alt6 = 1
if alt6 == 1:
# src\\SavedFSM\\BuildFSM.g:121:63: stype= ID
pass
stype=self.match(self.input, ID, self.FOLLOW_ID_in_activityDef222)
#action start
self.memory.append(stype.text)
#action end
else:
break #loop6
role=self.match(self.input, ID, self.FOLLOW_ID_in_activityDef232)
# src\\SavedFSM\\BuildFSM.g:122:5: ( ^( ASSERT (assertion= ASSERTION )? ) )
# src\\SavedFSM\\BuildFSM.g:122:6: ^( ASSERT (assertion= ASSERTION )? )
pass
self.match(self.input, ASSERT, self.FOLLOW_ASSERT_in_activityDef240)
if self.input.LA(1) == DOWN:
self.match(self.input, DOWN, None)
# src\\SavedFSM\\BuildFSM.g:122:15: (assertion= ASSERTION )?
alt7 = 2
LA7_0 = self.input.LA(1)
if (LA7_0 == ASSERTION) :
alt7 = 1
if alt7 == 1:
# src\\SavedFSM\\BuildFSM.g:122:16: assertion= ASSERTION
pass
assertion=self.match(self.input, ASSERTION, self.FOLLOW_ASSERTION_in_activityDef245)
self.match(self.input, UP, None)
self.match(self.input, UP, None)
#action start
self.memory.append('In SEND assertion')
#action end
#action start
self.current_fsm.add_transition(TransitionFactory.create(LocalType.SEND,slabel, role), assertion, local_context)
#action end
elif alt16 == 3:
# src\\SavedFSM\\BuildFSM.g:127:3: ^( 'choice' ( ^( BRANCH ( activityDef )+ ) )+ )
pass
self.match(self.input, 47, self.FOLLOW_47_in_activityDef264)
#action start
self.memory.append('enter choice state')
self.current_fsm.choice_start_state.append(self.current_fsm.get_current_state())
self.current_fsm.choice_end_state.append(self.current_fsm.state_gen.next())
#action end
self.match(self.input, DOWN, None)
# src\\SavedFSM\\BuildFSM.g:132:2: ( ^( BRANCH ( activityDef )+ ) )+
cnt9 = 0
while True: #loop9
alt9 = 2
LA9_0 = self.input.LA(1)
if (LA9_0 == BRANCH) :
alt9 = 1
if alt9 == 1:
# src\\SavedFSM\\BuildFSM.g:132:3: ^( BRANCH ( activityDef )+ )
pass
self.match(self.input, BRANCH, self.FOLLOW_BRANCH_in_activityDef274)
#action start
self.memory.append('enter choice branch and save the current state')
self.current_fsm.move_current_state(self.current_fsm.choice_start_state[-1])
#action end
self.match(self.input, DOWN, None)
# src\\SavedFSM\\BuildFSM.g:137:4: ( activityDef )+
cnt8 = 0
while True: #loop8
alt8 = 2
LA8_0 = self.input.LA(1)
if ((RESV <= LA8_0 <= SEND) or (RECLABEL <= LA8_0 <= PARALLEL) or LA8_0 == GLOBAL_ESCAPE or LA8_0 == 47 or (49 <= LA8_0 <= 50)) :
alt8 = 1
if alt8 == 1:
# src\\SavedFSM\\BuildFSM.g:137:4: activityDef
pass
self._state.following.append(self.FOLLOW_activityDef_in_activityDef280)
self.activityDef()
self._state.following.pop()
else:
if cnt8 >= 1:
break #loop8
eee = EarlyExitException(8, self.input)
raise eee
cnt8 += 1
self.match(self.input, UP, None)
#action start
self.memory.append('exit choice branch and set the current state to the end state for the choice')
self.current_fsm.fsm.add_transition(self.current_fsm.fsm.EMPTY_TRANSITION, self.current_fsm.get_current_state(), self.current_fsm.choice_end_state[-1])
#action end
else:
if cnt9 >= 1:
break #loop9
eee = EarlyExitException(9, self.input)
raise eee
cnt9 += 1
self.match(self.input, UP, None)
#action start
self.memory.append('set the current state to be equal to the end state for the choice')
self.current_fsm.move_current_state(self.current_fsm.choice_end_state[-1])
self.current_fsm.choice_end_state.pop()
self.current_fsm.choice_start_state.pop()
#action end
elif alt16 == 4:
# src\\SavedFSM\\BuildFSM.g:149:4: ^( PARALLEL ( ^( BRANCH ( activityDef )+ ) )+ )
pass
self.match(self.input, PARALLEL, self.FOLLOW_PARALLEL_in_activityDef299)
#action start
self.memory.append('enter parallel state')
self.parallel_root = self.current_fsm
#action end
self.match(self.input, DOWN, None)
# src\\SavedFSM\\BuildFSM.g:154:2: ( ^( BRANCH ( activityDef )+ ) )+
cnt11 = 0
while True: #loop11
alt11 = 2
LA11_0 = self.input.LA(1)
if (LA11_0 == BRANCH) :
alt11 = 1
if alt11 == 1:
# src\\SavedFSM\\BuildFSM.g:154:3: ^( BRANCH ( activityDef )+ )
pass
self.match(self.input, BRANCH, self.FOLLOW_BRANCH_in_activityDef316)
#action start
self.memory.append('enter parallel branch')
nested_fsm = FSMBuilderState(self.parallel_root)
self.parallel_root.fsm.add_fsm_to_memory(self.parallel_root.get_current_state(), nested_fsm.fsm)
self.current_fsm = nested_fsm
#action end
self.match(self.input, DOWN, None)
# src\\SavedFSM\\BuildFSM.g:161:2: ( activityDef )+
cnt10 = 0
while True: #loop10
alt10 = 2
LA10_0 = self.input.LA(1)
if ((RESV <= LA10_0 <= SEND) or (RECLABEL <= LA10_0 <= PARALLEL) or LA10_0 == GLOBAL_ESCAPE or LA10_0 == 47 or (49 <= LA10_0 <= 50)) :
alt10 = 1
if alt10 == 1:
# src\\SavedFSM\\BuildFSM.g:161:3: activityDef
pass
self._state.following.append(self.FOLLOW_activityDef_in_activityDef325)
self.activityDef()
self._state.following.pop()
else:
if cnt10 >= 1:
break #loop10
eee = EarlyExitException(10, self.input)
raise eee
cnt10 += 1
self.match(self.input, UP, None)
#action start
self.memory.append('exit parallel branch')
self.current_fsm.add_transition(self.current_fsm.fsm.END_PAR_TRANSITION)
#action end
else:
if cnt11 >= 1:
break #loop11
eee = EarlyExitException(11, self.input)
raise eee
cnt11 += 1
self.match(self.input, UP, None)
#action start
self.memory.append('exit parallel state')
self.current_fsm = self.current_fsm.parent
self.current_fsm.fsm.add_transition(self.current_fsm.fsm.EMPTY_TRANSITION, self.current_fsm.get_current_state(), self.current_fsm.move_current_state())
#action end
elif alt16 == 5:
# src\\SavedFSM\\BuildFSM.g:171:3: ^( 'repeat' ( ^( BRANCH ( activityDef )+ ) ) )
pass
self.match(self.input, 49, self.FOLLOW_49_in_activityDef346)
#action start
self.memory.append('enter repeat state')
#action end
self.match(self.input, DOWN, None)
# src\\SavedFSM\\BuildFSM.g:173:2: ( ^( BRANCH ( activityDef )+ ) )
# src\\SavedFSM\\BuildFSM.g:173:3: ^( BRANCH ( activityDef )+ )
pass
self.match(self.input, BRANCH, self.FOLLOW_BRANCH_in_activityDef355)
self.match(self.input, DOWN, None)
# src\\SavedFSM\\BuildFSM.g:173:12: ( activityDef )+
cnt12 = 0
while True: #loop12
alt12 = 2
LA12_0 = self.input.LA(1)
if ((RESV <= LA12_0 <= SEND) or (RECLABEL <= LA12_0 <= PARALLEL) or LA12_0 == GLOBAL_ESCAPE or LA12_0 == 47 or (49 <= LA12_0 <= 50)) :
alt12 = 1
if alt12 == 1:
# src\\SavedFSM\\BuildFSM.g:173:13: activityDef
pass
self._state.following.append(self.FOLLOW_activityDef_in_activityDef358)
self.activityDef()
self._state.following.pop()
#action start
self.memory.append('repeat statement')
#action end
else:
if cnt12 >= 1:
break #loop12
eee = EarlyExitException(12, self.input)
raise eee
cnt12 += 1
self.match(self.input, UP, None)
self.match(self.input, UP, None)
#action start
self.memory.append('exit repeat state')
#action end
elif alt16 == 6:
# src\\SavedFSM\\BuildFSM.g:176:10: ^( 'rec' label= ID ( ^( BRANCH ( activityDef )+ ) ) )
pass
self.match(self.input, 50, self.FOLLOW_50_in_activityDef382)
self.match(self.input, DOWN, None)
label=self.match(self.input, ID, self.FOLLOW_ID_in_activityDef388)
#action start
self.memory.append('enter rec state ' + label.text + str(self.current_fsm.get_current_state()))
self.current_fsm.recursions_states.setdefault(label.text, (self.current_fsm.format_state_name(self.current_fsm.get_current_state()), True))
#action end
# src\\SavedFSM\\BuildFSM.g:180:2: ( ^( BRANCH ( activityDef )+ ) )
# src\\SavedFSM\\BuildFSM.g:180:3: ^( BRANCH ( activityDef )+ )
pass
self.match(self.input, BRANCH, self.FOLLOW_BRANCH_in_activityDef404)
self.match(self.input, DOWN, None)
# src\\SavedFSM\\BuildFSM.g:180:12: ( activityDef )+
cnt13 = 0
while True: #loop13
alt13 = 2
LA13_0 = self.input.LA(1)
if ((RESV <= LA13_0 <= SEND) or (RECLABEL <= LA13_0 <= PARALLEL) or LA13_0 == GLOBAL_ESCAPE or LA13_0 == 47 or (49 <= LA13_0 <= 50)) :
alt13 = 1
if alt13 == 1:
# src\\SavedFSM\\BuildFSM.g:180:13: activityDef
pass
self._state.following.append(self.FOLLOW_activityDef_in_activityDef407)
self.activityDef()
self._state.following.pop()
#action start
self.memory.append('rec statement')
#action end
else:
if cnt13 >= 1:
break #loop13
eee = EarlyExitException(13, self.input)
raise eee
cnt13 += 1
self.match(self.input, UP, None)
self.match(self.input, UP, None)
#action start
(start_state, isActive) = self.current_fsm.recursions_states.get(label.text)
self.memory.append('exit rec state ' + label.text + 'start_state' + str(start_state))
self.current_fsm.recursions_states[label.text] = (start_state, False)
#action end
elif alt16 == 7:
# src\\SavedFSM\\BuildFSM.g:187:3: ^( 'RECLABEL' labelID= ID )
pass
self.match(self.input, RECLABEL, self.FOLLOW_RECLABEL_in_activityDef425)
self.match(self.input, DOWN, None)
labelID=self.match(self.input, ID, self.FOLLOW_ID_in_activityDef432)
#action start
(start_rec_state, isActive) = self.current_fsm.recursions_states.get(labelID.text)
self.memory.append('rec label:' + labelID.text + 'starts from state:' + str(start_rec_state))
if isActive:
self.current_fsm.fsm.add_transition(self.current_fsm.fsm.EMPTY_TRANSITION,
self.current_fsm.format_state_name(self.current_fsm.get_current_state()),
start_rec_state)
# Generate unreachable state for the choice construct
self.current_fsm.move_current_state()
else: raise ExceptionFSM('Calling a recusrion label from a recursion that is not valid')
#action end
self.match(self.input, UP, None)
#action start
# Do not need it for no
#self.current_fsm.fsm.copy_transitions(self.current_fsm.recursions_states[labelID.text], self.current_fsm.get_current_state())
#action end
elif alt16 == 8:
# src\\SavedFSM\\BuildFSM.g:204:3: ^( GLOBAL_ESCAPE ( ^( 'do' ( ( activityDef )+ ) ) ) ( ^( 'interrupt' roleName ( ( activityDef )+ ) ) ) )
pass
self.match(self.input, GLOBAL_ESCAPE, self.FOLLOW_GLOBAL_ESCAPE_in_activityDef446)
self.match(self.input, DOWN, None)
# src\\SavedFSM\\BuildFSM.g:205:5: ( ^( 'do' ( ( activityDef )+ ) ) )
# src\\SavedFSM\\BuildFSM.g:205:6: ^( 'do' ( ( activityDef )+ ) )
pass
self.match(self.input, 56, self.FOLLOW_56_in_activityDef455)
self.match(self.input, DOWN, None)
# src\\SavedFSM\\BuildFSM.g:205:13: ( ( activityDef )+ )
# src\\SavedFSM\\BuildFSM.g:205:14: ( activityDef )+
pass
# src\\SavedFSM\\BuildFSM.g:205:14: ( activityDef )+
cnt14 = 0
while True: #loop14
alt14 = 2
LA14_0 = self.input.LA(1)
if ((RESV <= LA14_0 <= SEND) or (RECLABEL <= LA14_0 <= PARALLEL) or LA14_0 == GLOBAL_ESCAPE or LA14_0 == 47 or (49 <= LA14_0 <= 50)) :
alt14 = 1
if alt14 == 1:
# src\\SavedFSM\\BuildFSM.g:205:14: activityDef
pass
self._state.following.append(self.FOLLOW_activityDef_in_activityDef458)
self.activityDef()
self._state.following.pop()
else:
if cnt14 >= 1:
break #loop14
eee = EarlyExitException(14, self.input)
raise eee
cnt14 += 1
#action start
self.current_fsm.fsm.final_state = self.current_fsm.get_current_state()
#action end
self.match(self.input, UP, None)
# src\\SavedFSM\\BuildFSM.g:206:5: ( ^( 'interrupt' roleName ( ( activityDef )+ ) ) )
# src\\SavedFSM\\BuildFSM.g:206:6: ^( 'interrupt' roleName ( ( activityDef )+ ) )
pass
self.match(self.input, 57, self.FOLLOW_57_in_activityDef471)
self.match(self.input, DOWN, None)
self._state.following.append(self.FOLLOW_roleName_in_activityDef473)
self.roleName()
self._state.following.pop()
#action start
self.memory.append('before setting interrupt_transition to True')
self.current_fsm.interrupt_start_state = self.current_fsm.move_current_state()
self.current_fsm.set_interrupt_transition = True
#action end
# src\\SavedFSM\\BuildFSM.g:209:56: ( ( activityDef )+ )
# src\\SavedFSM\\BuildFSM.g:209:57: ( activityDef )+
pass
# src\\SavedFSM\\BuildFSM.g:209:57: ( activityDef )+
cnt15 = 0
while True: #loop15
alt15 = 2
LA15_0 = self.input.LA(1)
if ((RESV <= LA15_0 <= SEND) or (RECLABEL <= LA15_0 <= PARALLEL) or LA15_0 == GLOBAL_ESCAPE or LA15_0 == 47 or (49 <= LA15_0 <= 50)) :
alt15 = 1
if alt15 == 1:
# src\\SavedFSM\\BuildFSM.g:209:57: activityDef
pass
self._state.following.append(self.FOLLOW_activityDef_in_activityDef482)
self.activityDef()
self._state.following.pop()
else:
if cnt15 >= 1:
break #loop15
eee = EarlyExitException(15, self.input)
raise eee
cnt15 += 1
self.match(self.input, UP, None)
self.match(self.input, UP, None)
except RecognitionException, re:
self.reportError(re)
self.recover(self.input, re)
finally:
pass
return
# $ANTLR end "activityDef"
# $ANTLR start "roleName"
# src\\SavedFSM\\BuildFSM.g:211:1: roleName : ID ;
def roleName(self, ):
try:
try:
# src\\SavedFSM\\BuildFSM.g:211:9: ( ID )
# src\\SavedFSM\\BuildFSM.g:211:11: ID
pass
self.match(self.input, ID, self.FOLLOW_ID_in_roleName495)
except RecognitionException, re:
self.reportError(re)
self.recover(self.input, re)
finally:
pass
return
# $ANTLR end "roleName"
# $ANTLR start "labelName"
# src\\SavedFSM\\BuildFSM.g:212:1: labelName : ID ;
def labelName(self, ):
try:
try:
# src\\SavedFSM\\BuildFSM.g:212:10: ( ID )
# src\\SavedFSM\\BuildFSM.g:212:12: ID
pass
self.match(self.input, ID, self.FOLLOW_ID_in_labelName501)
except RecognitionException, re:
self.reportError(re)
self.recover(self.input, re)
finally:
pass
return
# $ANTLR end "labelName"
# $ANTLR start "roleDef"
# src\\SavedFSM\\BuildFSM.g:213:1: roleDef : ID ;
def roleDef(self, ):
try:
try:
# src\\SavedFSM\\BuildFSM.g:213:8: ( ID )
# src\\SavedFSM\\BuildFSM.g:213:10: ID
pass
self.match(self.input, ID, self.FOLLOW_ID_in_roleDef507)
except RecognitionException, re:
self.reportError(re)
self.recover(self.input, re)
finally:
pass
return
# $ANTLR end "roleDef"
# $ANTLR start "primitivetype"
# src\\SavedFSM\\BuildFSM.g:214:1: primitivetype : INT ;
def primitivetype(self, ):
try:
try:
# src\\SavedFSM\\BuildFSM.g:214:15: ( INT )
# src\\SavedFSM\\BuildFSM.g:214:16: INT
pass
self.match(self.input, INT, self.FOLLOW_INT_in_primitivetype513)
except RecognitionException, re:
self.reportError(re)
self.recover(self.input, re)
finally:
pass
return
# $ANTLR end "primitivetype"
# Delegated rules
FOLLOW_PROTOCOL_in_description52 = frozenset([2])
FOLLOW_activityDef_in_description54 = frozenset([3, 12, 13, 18, 19, 22, 47, 49, 50])
FOLLOW_RESV_in_activityDef66 = frozenset([2])
FOLLOW_VALUE_in_activityDef76 = frozenset([2])
FOLLOW_ID_in_activityDef82 = frozenset([5, 6])
FOLLOW_set_in_activityDef86 = frozenset([3, 24])
FOLLOW_ID_in_activityDef107 = frozenset([24])
FOLLOW_ID_in_activityDef121 = frozenset([24])
FOLLOW_ID_in_activityDef131 = frozenset([21])
FOLLOW_ASSERT_in_activityDef139 = frozenset([2])
FOLLOW_ASSERTION_in_activityDef144 = frozenset([3])
FOLLOW_SEND_in_activityDef163 = frozenset([2])
FOLLOW_VALUE_in_activityDef180 = frozenset([2])
FOLLOW_ID_in_activityDef186 = frozenset([5, 6])
FOLLOW_set_in_activityDef191 = frozenset([3, 24])
FOLLOW_ID_in_activityDef212 = frozenset([24])
FOLLOW_ID_in_activityDef222 = frozenset([24])
FOLLOW_ID_in_activityDef232 = frozenset([21])
FOLLOW_ASSERT_in_activityDef240 = frozenset([2])
FOLLOW_ASSERTION_in_activityDef245 = frozenset([3])
FOLLOW_47_in_activityDef264 = frozenset([2])
FOLLOW_BRANCH_in_activityDef274 = frozenset([2])
FOLLOW_activityDef_in_activityDef280 = frozenset([3, 12, 13, 18, 19, 22, 47, 49, 50])
FOLLOW_PARALLEL_in_activityDef299 = frozenset([2])
FOLLOW_BRANCH_in_activityDef316 = frozenset([2])
FOLLOW_activityDef_in_activityDef325 = frozenset([3, 12, 13, 18, 19, 22, 47, 49, 50])
FOLLOW_49_in_activityDef346 = frozenset([2])
FOLLOW_BRANCH_in_activityDef355 = frozenset([2])
FOLLOW_activityDef_in_activityDef358 = frozenset([3, 12, 13, 18, 19, 22, 47, 49, 50])
FOLLOW_50_in_activityDef382 = frozenset([2])
FOLLOW_ID_in_activityDef388 = frozenset([16])
FOLLOW_BRANCH_in_activityDef404 = frozenset([2])
FOLLOW_activityDef_in_activityDef407 = frozenset([3, 12, 13, 18, 19, 22, 47, 49, 50])
FOLLOW_RECLABEL_in_activityDef425 = frozenset([2])
FOLLOW_ID_in_activityDef432 = frozenset([3])
FOLLOW_GLOBAL_ESCAPE_in_activityDef446 = frozenset([2])
FOLLOW_56_in_activityDef455 = frozenset([2])
FOLLOW_activityDef_in_activityDef458 = frozenset([3, 12, 13, 18, 19, 22, 47, 49, 50])
FOLLOW_57_in_activityDef471 = frozenset([2])
FOLLOW_roleName_in_activityDef473 = frozenset([3, 12, 13, 18, 19, 22, 47, 49, 50])
FOLLOW_activityDef_in_activityDef482 = frozenset([3, 12, 13, 18, 19, 22, 47, 49, 50])
FOLLOW_ID_in_roleName495 = frozenset([1])
FOLLOW_ID_in_labelName501 = frozenset([1])
FOLLOW_ID_in_roleDef507 = frozenset([1])
FOLLOW_INT_in_primitivetype513 = frozenset([1])
def flatten_options(options):
return reduce(lambda x,y: x.union(y),
options,
frozenset())
cl.__original_init__ = cl.__init__
# In order of debuggers like PDB being able to step through the code,
# we add a fake linecache entry.
linecache.cache[unique_filename] = (
len(script),
None,
script.splitlines(True),
unique_filename
)
cl.__init__ = init
return cl
return wrap
_VALID_INITS = frozenset(["characteristic_init", "__init__"])
def immutable(attrs):
"""
Class decorator that makes *attrs* of a class immutable.
That means that *attrs* can only be set from an initializer. If anyone
else tries to set one of them, an :exc:`AttributeError` is raised.
.. versionadded:: 14.0
"""
# In this case, we just want to compare (native) strings.
attrs = frozenset(attr.name if isinstance(attr, Attribute) else attr
for attr in _ensure_attributes(attrs, NOTHING)
if attr.exclude_from_immutable is False)
class Connection(ConnectionBase):
''' Local chroot based connections '''
transport = 'chroot'
has_pipelining = True
# su currently has an undiagnosed issue with calculating the file
# checksums (so copy, for instance, doesn't work right)
# Have to look into that before re-enabling this
become_methods = frozenset(C.BECOME_METHODS).difference(('su',))
def __init__(self, play_context, new_stdin, *args, **kwargs):
super(Connection, self).__init__(play_context, new_stdin, *args, **kwargs)
self.chroot = self._play_context.remote_addr
if os.geteuid() != 0:
raise AnsibleError("chroot connection requires running as root")
# we're running as root on the local system so do some
# trivial checks for ensuring 'host' is actually a chroot'able dir
if not os.path.isdir(self.chroot):
raise AnsibleError("%s is not a directory" % self.chroot)
chrootsh = os.path.join(self.chroot, 'bin/sh')
if not is_executable(chrootsh):
raise AnsibleError("%s does not look like a chrootable dir (/bin/sh missing)" % self.chroot)
self.chroot_cmd = distutils.spawn.find_executable('chroot')
if not self.chroot_cmd:
raise AnsibleError("chroot command not found in PATH")
def _connect(self):
''' connect to the chroot; nothing to do here '''
super(Connection, self)._connect()
if not self._connected:
self._display.vvv("THIS IS A LOCAL CHROOT DIR", host=self.chroot)
self._connected = True
def _buffered_exec_command(self, cmd, stdin=subprocess.PIPE):
''' run a command on the chroot. This is only needed for implementing
put_file() get_file() so that we don't have to read the whole file
into memory.
compared to exec_command() it looses some niceties like being able to
return the process's exit code immediately.
'''
executable = C.DEFAULT_EXECUTABLE.split()[0] if C.DEFAULT_EXECUTABLE else '/bin/sh'
local_cmd = [self.chroot_cmd, self.chroot, executable, '-c', cmd]
self._display.vvv("EXEC %s" % (local_cmd), host=self.chroot)
p = subprocess.Popen(local_cmd, shell=False, stdin=stdin,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
return p
def exec_command(self, cmd, in_data=None, sudoable=False):
''' run a command on the chroot '''
super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable)
p = self._buffered_exec_command(cmd)
stdout, stderr = p.communicate(in_data)
return (p.returncode, stdout, stderr)
def _prefix_login_path(self, remote_path):
''' Make sure that we put files into a standard path
If a path is relative, then we need to choose where to put it.
ssh chooses $HOME but we aren't guaranteed that a home dir will
exist in any given chroot. So for now we're choosing "/" instead.
This also happens to be the former default.
Can revisit using $HOME instead if it's a problem
'''
if not remote_path.startswith(os.path.sep):
remote_path = os.path.join(os.path.sep, remote_path)
return os.path.normpath(remote_path)
def put_file(self, in_path, out_path):
''' transfer a file from local to chroot '''
super(Connection, self).put_file(in_path, out_path)
self._display.vvv("PUT %s TO %s" % (in_path, out_path), host=self.chroot)
out_path = pipes.quote(self._prefix_login_path(out_path))
try:
with open(in_path, 'rb') as in_file:
try:
p = self._buffered_exec_command('dd of=%s bs=%s' % (out_path, BUFSIZE), stdin=in_file)
except OSError:
raise AnsibleError("chroot connection requires dd command in the chroot")
try:
stdout, stderr = p.communicate()
except:
traceback.print_exc()
raise AnsibleError("failed to transfer file %s to %s" % (in_path, out_path))
if p.returncode != 0:
raise AnsibleError("failed to transfer file %s to %s:\n%s\n%s" % (in_path, out_path, stdout, stderr))
except IOError:
raise AnsibleError("file or module does not exist at: %s" % in_path)
def fetch_file(self, in_path, out_path):
''' fetch a file from chroot to local '''
super(Connection, self).fetch_file(in_path, out_path)
self._display.vvv("FETCH %s TO %s" % (in_path, out_path), host=self.chroot)
in_path = pipes.quote(self._prefix_login_path(in_path))
try:
p = self._buffered_exec_command('dd if=%s bs=%s' % (in_path, BUFSIZE))
except OSError:
raise AnsibleError("chroot connection requires dd command in the chroot")
with open(out_path, 'wb+') as out_file:
try:
chunk = p.stdout.read(BUFSIZE)
while chunk:
out_file.write(chunk)
chunk = p.stdout.read(BUFSIZE)
except:
traceback.print_exc()
raise AnsibleError("failed to transfer file %s to %s" % (in_path, out_path))
stdout, stderr = p.communicate()
if p.returncode != 0:
raise AnsibleError("failed to transfer file %s to %s:\n%s\n%s" % (in_path, out_path, stdout, stderr))
def close(self):
''' terminate the connection; nothing to do here '''
super(Connection, self).close()
self._connected = False
class PolishAction(InterfaceAction):
name = 'Polish Books'
action_spec = (_('Polish books'), 'polish.png',
_('Apply the shine of perfection to your books'), _('P'))
dont_add_to = frozenset(['context-menu-device'])
action_type = 'current'
accepts_drops = True
def accept_enter_event(self, event, mime_data):
if mime_data.hasFormat("application/calibre+from_library"):
return True
return False
def accept_drag_move_event(self, event, mime_data):
if mime_data.hasFormat("application/calibre+from_library"):
return True
return False
def drop_event(self, event, mime_data):
mime = 'application/calibre+from_library'
if mime_data.hasFormat(mime):
self.dropped_ids = tuple(map(int, str(mime_data.data(mime)).split()))
QTimer.singleShot(1, self.do_drop)
return True
return False
def do_drop(self):
book_id_map = self.get_supported_books(self.dropped_ids)
del self.dropped_ids
if book_id_map:
self.do_polish(book_id_map)
def genesis(self):
self.qaction.triggered.connect(self.polish_books)
self.report = Report(self.gui)
def location_selected(self, loc):
enabled = loc == 'library'
self.qaction.setEnabled(enabled)
def get_books_for_polishing(self):
rows = [r.row() for r in
self.gui.library_view.selectionModel().selectedRows()]
if not rows or len(rows) == 0:
d = error_dialog(self.gui, _('Cannot polish'),
_('No books selected'))
d.exec_()
return None
db = self.gui.library_view.model().db
ans = (db.id(r) for r in rows)
ans = self.get_supported_books(ans)
for fmts in ans.itervalues():
for x in fmts:
if x.startswith('ORIGINAL_'):
from calibre.gui2.dialogs.confirm_delete import confirm
if not confirm(_(
'One of the books you are polishing has an {0} format.'
' Polishing will use this as the source and overwrite'
' any existing {1} format. Are you sure you want to proceed?').format(
x, x[len('ORIGINAL_'):]), 'confirm_original_polish', title=_('Are you sure?'),
confirm_msg=_('Ask for this confirmation again')):
return {}
break
return ans
def get_supported_books(self, book_ids):
from calibre.ebooks.oeb.polish.main import SUPPORTED
db = self.gui.library_view.model().db
supported = set(SUPPORTED)
for x in SUPPORTED:
supported.add('ORIGINAL_'+x)
ans = [(x, set((db.formats(x, index_is_id=True) or '').split(','))
.intersection(supported)) for x in book_ids]
ans = [x for x in ans if x[1]]
if not ans:
error_dialog(self.gui, _('Cannot polish'),
_('Polishing is only supported for books in the %s'
' formats. Convert to one of those formats before polishing.')
%_(' or ').join(sorted(SUPPORTED)), show=True)
ans = OrderedDict(ans)
for fmts in ans.itervalues():
for x in SUPPORTED:
if ('ORIGINAL_'+x) in fmts:
fmts.discard(x)
return ans
def polish_books(self):
book_id_map = self.get_books_for_polishing()
if not book_id_map:
return
self.do_polish(book_id_map)
def do_polish(self, book_id_map):
d = Polish(self.gui.library_view.model().db, book_id_map, parent=self.gui)
if d.exec_() == d.Accepted and d.jobs:
show_reports = bool(d.show_reports.isChecked())
for desc, data, book_id, base, is_orig in reversed(d.jobs):
job = self.gui.job_manager.run_job(
Dispatcher(self.book_polished), 'gui_polish', args=(data,),
description=desc)
job.polish_args = (book_id, base, data['files'], show_reports, is_orig)
if d.jobs:
self.gui.jobs_pointer.start()
self.gui.status_bar.show_message(
ngettext('Start polishing the book', 'Start polishing of {} books',
len(d.jobs)).format(len(d.jobs)), 2000)
def book_polished(self, job):
if job.failed:
self.gui.job_exception(job)
return
db = self.gui.current_db
book_id, base, files, show_reports, is_orig = job.polish_args
fmts = set()
for path in files:
fmt = path.rpartition('.')[-1].upper()
if tweaks['save_original_format_when_polishing'] and not is_orig[fmt]:
fmts.add(fmt)
db.save_original_format(book_id, fmt, notify=False)
with open(path, 'rb') as f:
db.add_format(book_id, fmt, f, index_is_id=True)
self.gui.status_bar.show_message(job.description + _(' completed'), 2000)
try:
shutil.rmtree(base)
parent = os.path.dirname(base)
os.rmdir(parent)
except:
pass
self.gui.tags_view.recount()
if self.gui.current_view() is self.gui.library_view:
current = self.gui.library_view.currentIndex()
if current.isValid():
self.gui.library_view.model().current_changed(current, QModelIndex())
if show_reports:
self.report(db.title(book_id, index_is_id=True), book_id, fmts, job, job.result)
class JsonFile(object):
"""A streaming file-ish interface for JSON content.
Usage:
writer = JsonFile('path')
writer.open('w')
writer.write(json_serializable_python_object) # We serialize for you.
writer.write(another_json_serializable_python_object)
writer.close() # Must close before read.
reader = JsonFile('path')
reader.open('r') # Only 'r' and 'w' are supported.
for entity in reader:
do_something_with(entity) # We deserialize back to Python for you.
self.reader.reset() # Reset read pointer to head.
contents = self.reader.read() # Returns {'rows': [...]}.
for entity in contents['rows']:
do_something_with(entity) # Again, deserialized back to Python.
reader.close()
with syntax is not supported. Cannot be used inside the App Engine
container where the filesystem is read-only.
Internally, each call to write will take a Python object, serialize it, and
write the contents as one line to the json file. On __iter__ we deserialize
one line at a time, generator-style, to avoid OOM unless serialization/de-
serialization of one object exhausts memory.
"""
# When writing to files use \n instead of os.linesep; see
# http://docs.python.org/2/library/os.html.
_LINE_TEMPLATE = ',\n %s'
_MODE_READ = 'r'
_MODE_WRITE = 'w'
_MODES = frozenset([_MODE_READ, _MODE_WRITE])
_PREFIX = '{"rows": ['
_SUFFIX = ']}'
def __init__(self, path):
self._first = True
self._file = None
self._path = path
def __iter__(self):
assert self._file
return self
def close(self):
"""Closes the file; must close before read."""
assert self._file
if not self._file.closed: # Like file, allow multiple close calls.
if self.mode == self._MODE_WRITE:
self._file.write('\n' + self._SUFFIX)
self._file.close()
@property
def mode(self):
"""Returns the mode the file was opened in."""
assert self._file
return self._file.mode
@property
def name(self):
"""Returns string name of the file."""
assert self._file
return self._file.name
def next(self):
"""Retrieves the next line and deserializes it into a Python object."""
assert self._file
line = self._file.readline()
if line.startswith(self._PREFIX):
line = self._file.readline()
if line.endswith(self._SUFFIX):
raise StopIteration()
line = line.strip()
if line.endswith(','):
line = line[:-1]
return loads(line)
def open(self, mode):
"""Opens the file in the given mode string ('r, 'w' only)."""
assert not self._file
assert mode in self._MODES
self._file = open(self._path, mode)
if self.mode == self._MODE_WRITE:
self._file.write(self._PREFIX)
def read(self):
"""Reads the file into a single Python object; may exhaust memory.
Returns:
dict. Format: {'rows': [...]} where the value is a list of de-
serialized objects passed to write.
"""
assert self._file
return loads(self._file.read())
def reset(self):
"""Resets file's position to head."""
assert self._file
self._file.seek(0)
def write(self, python_object):
"""Writes serialized JSON representation of python_object to file.
Args:
python_object: object. Contents to write. Must be JSON-serializable.
Raises:
ValueError: if python_object cannot be JSON-serialized.
"""
assert self._file
template = self._LINE_TEMPLATE
if self._first:
template = template[1:]
self._first = False
self._file.write(template % dumps(python_object))
def __init__(self, raw_hypergraph):
self.data = {k: frozenset(map(frozenset,v))
for (k,v) in raw_hypergraph.items()}
