def save_csv(result, option):
def max_ccn(f):
cyclomatic_complexities = list(func.cyclomatic_complexity for func in f.function_list)
return 0 if (cyclomatic_complexities == []) else max(cyclomatic_complexities)
result = filter(lambda f: f.function_list != [], result)
total_files_count = len(result)
nloc_unhealthy_files = filter(lambda f: f.nloc > option.length, result)
fanout_unhealthy_files = filter(lambda f: len(f.dependency_list) > option.DEP, result)
ccn_unhealthy_files = [ccn for ccn in map(max_ccn, result) if ccn > option.CCN]
cpd_unhealthy_files = set([file['file'] for info in option.cpd_infos for file in info['files']])
avg_nloc_serverity = sum(map(lambda f: f.nloc, nloc_unhealthy_files)) / (option.length * len(nloc_unhealthy_files)) if len(nloc_unhealthy_files) > 0 else 1
avg_fanout_serverity = sum(map(lambda f: len(f.dependency_list), fanout_unhealthy_files)) / (option.DEP * len(fanout_unhealthy_files)) if len(fanout_unhealthy_files) > 0 else 1
avg_ccn_serverity = sum(ccn_unhealthy_files) / (option.CCN * len(ccn_unhealthy_files)) if len(ccn_unhealthy_files) > 0 else 1
avg_cpd_serverity = sum(map(lambda info: info['tokens'], option.cpd_infos)) / (option.cpd_tokens * len(option.cpd_infos)) if len(option.cpd_infos) > 0 else 1
percent_nloc_warning = len(nloc_unhealthy_files)/ total_files_count
percent_fanout_warning = len(fanout_unhealthy_files) / total_files_count
percent_ccn_warning = len(ccn_unhealthy_files) / total_files_count
percent_cpd_warning = len(cpd_unhealthy_files) / total_files_count
# today = os.popen("git log -1 --pretty=format:'%ad' --date=short").read()
today = datetime.date.today().__str__()
try:
with open(option.csv_file, 'r') as f:
f_csv = csv.DictReader(f)
rows = filter(lambda r: r['date'] != today, [row for row in f_csv])
except IOError, e:
rows = []
def convert_node(fields, x, names={}, import_data=None):
name = x.__class__.__name__
conv = lambda x:convert_node(fields, x, names=names, import_data=import_data)
if name == 'Str':
return x.s.decode('utf-8') if isinstance(x.s, bytes) else x.s
elif name == 'Num':
return x.n
elif name in {'Set', 'List', 'Tuple'}:
func = {'Set':set, 'List':list, 'Tuple':tuple}[name]
return func(map(conv, x.elts))
elif name == 'Dict':
keys, values = map(conv, x.keys), map(conv, x.values)
return dict(zip(keys, values))
elif name == 'Call':
if len(x.args) != 1 and len(x.keywords) != 0:
raise TypeError('Unsupported function call for fields: %s' % (fields,))
return tuple(map(conv, x.args))[0]
elif name == 'Name':
if x.id not in names:
if import_data is not None and x.id in import_data[0]:
return get_import_data(x.id, import_data[0][x.id], *import_data[1:])
raise ValueError('Could not find name %s for fields: %s' % (x.id, fields))
return names[x.id]
elif name == 'BinOp':
if x.right.__class__.__name__ == 'Str':
return x.right.s.decode('utf-8') if isinstance(x.right.s, bytes) else x.right.s
raise TypeError('Unknown datatype %s for fields: %s' % (x, fields))
def pad_larger(*arrays):
"""Pad the smallest of `n` arrays.
Parameters
----------
arrays : tuple of array_like
Raises
------
AssertionError
Returns
-------
ret : tuple
Tuple of zero padded arrays.
"""
assert all(map(isinstance, arrays, itools.repeat(np.ndarray))), \
("all arguments must be instances of ndarray or implement the ndarray"
" interface")
if len(arrays) == 2:
return pad_larger2(*arrays)
sizes = np.fromiter(map(operator.attrgetter('size'), arrays), int)
lsize = sizes.max()
ret = []
for array, size in zip(arrays, sizes):
size_diff = lsize - size
ret.append(np.pad(array, (0, size_diff), 'constant',
constant_values=(0,)))
ret.append(lsize)
return ret
def initialize_constants():
src = read_cal_file('constants.py')
nv = re.search(r'numeric_version\s+=\s+\((\d+), (\d+), (\d+)\)', src)
calibre_constants['version'] = '%s.%s.%s' % (nv.group(1), nv.group(2), nv.group(3))
calibre_constants['appname'] = re.search(r'__appname__\s+=\s+(u{0,1})[\'"]([^\'"]+)[\'"]', src).group(2)
epsrc = re.compile(r'entry_points = (\{.*?\})', re.DOTALL).search(read_cal_file('linux.py')).group(1)
entry_points = eval(epsrc, {'__appname__': calibre_constants['appname']})
def e2b(ep):
return re.search(r'\s*(.*?)\s*=', ep).group(1).strip()
def e2s(ep, base='src'):
return (base + os.path.sep + re.search(r'.*=\s*(.*?):', ep).group(1).replace('.', '/') + '.py').strip()
def e2m(ep):
return re.search(r'.*=\s*(.*?)\s*:', ep).group(1).strip()
def e2f(ep):
return ep[ep.rindex(':') + 1:].strip()
calibre_constants['basenames'] = basenames = {}
calibre_constants['functions'] = functions = {}
calibre_constants['modules'] = modules = {}
calibre_constants['scripts'] = scripts = {}
for x in ('console', 'gui'):
y = x + '_scripts'
basenames[x] = list(map(e2b, entry_points[y]))
functions[x] = list(map(e2f, entry_points[y]))
modules[x] = list(map(e2m, entry_points[y]))
scripts[x] = list(map(e2s, entry_points[y]))
src = read_cal_file('ebooks/__init__.py')
be = re.search(r'^BOOK_EXTENSIONS\s*=\s*(\[.+?\])', src, flags=re.DOTALL | re.MULTILINE).group(1)
calibre_constants['book_extensions'] = json.loads(be.replace("'", '"'))
def __init__(self, sfnt):
self.sfnt = sfnt
self.head = self.sfnt[b"head"]
hhea = self.sfnt[b"hhea"]
hhea.read_data(self.sfnt[b"hmtx"])
self.ascent = hhea.ascender
self.descent = hhea.descender
self.bbox = (self.head.x_min, self.head.y_min, self.head.x_max, self.head.y_max)
self._advance_widths = hhea.advance_widths
self.cmap = self.sfnt[b"cmap"]
self.units_per_em = self.head.units_per_em
self.os2 = self.sfnt[b"OS/2"]
self.os2.read_data()
self.post = self.sfnt[b"post"]
self.post.read_data()
self.names = get_all_font_names(self.sfnt[b"name"].raw, raw_is_table=True)
self.is_otf = "CFF " in self.sfnt.tables
self._sig = hash(self.sfnt[b"name"].raw)
# Metrics for embedding in PDF
pdf_scale = self.pdf_scale = lambda x: int(round(x * 1000.0 / self.units_per_em))
self.pdf_ascent, self.pdf_descent = map(pdf_scale, (self.os2.typo_ascender, self.os2.typo_descender))
self.pdf_bbox = tuple(map(pdf_scale, self.bbox))
self.pdf_capheight = pdf_scale(getattr(self.os2, "cap_height", self.os2.typo_ascender))
self.pdf_avg_width = pdf_scale(self.os2.average_char_width)
self.pdf_stemv = 50 + int((self.os2.weight_class / 65.0) ** 2)
def convert_node(fields, x, names={}, import_data=None):
name = x.__class__.__name__
conv = lambda x: convert_node(fields, x, names=names, import_data=import_data)
if name == "Str":
return x.s.decode("utf-8") if isinstance(x.s, bytes) else x.s
elif name == "Num":
return x.n
elif name in {"Set", "List", "Tuple"}:
func = {"Set": set, "List": list, "Tuple": tuple}[name]
return func(map(conv, x.elts))
elif name == "Dict":
keys, values = map(conv, x.keys), map(conv, x.values)
return dict(zip(keys, values))
elif name == "Call":
if len(x.args) != 1 and len(x.keywords) != 0:
raise TypeError("Unsupported function call for fields: %s" % (fields,))
return tuple(map(conv, x.args))[0]
elif name == "Name":
if x.id not in names:
if import_data is not None and x.id in import_data[0]:
return get_import_data(x.id, import_data[0][x.id], *import_data[1:])
raise ValueError("Could not find name %s for fields: %s" % (x.id, fields))
return names[x.id]
elif name == "BinOp":
if x.right.__class__.__name__ == "Str":
return x.right.s.decode("utf-8") if isinstance(x.right.s, bytes) else x.right.s
raise TypeError("Unknown datatype %s for fields: %s" % (x, fields))
def __init__(self, sfnt):
for table in (b'head', b'hhea', b'hmtx', b'cmap', b'OS/2', b'post',
b'name'):
if table not in sfnt:
raise UnsupportedFont('This font has no %s table'%table)
self.sfnt = sfnt
self.head = self.sfnt[b'head']
hhea = self.sfnt[b'hhea']
hhea.read_data(self.sfnt[b'hmtx'])
self.ascent = hhea.ascender
self.descent = hhea.descender
self.bbox = (self.head.x_min, self.head.y_min, self.head.x_max,
self.head.y_max)
self._advance_widths = hhea.advance_widths
self.cmap = self.sfnt[b'cmap']
self.units_per_em = self.head.units_per_em
self.os2 = self.sfnt[b'OS/2']
self.os2.read_data()
self.post = self.sfnt[b'post']
self.post.read_data()
self.names = get_all_font_names(self.sfnt[b'name'].raw, raw_is_table=True)
self.is_otf = 'CFF ' in self.sfnt.tables
self._sig = hash(self.sfnt[b'name'].raw)
# Metrics for embedding in PDF
pdf_scale = self.pdf_scale = lambda x:int(round(x*1000./self.units_per_em))
self.pdf_ascent, self.pdf_descent = map(pdf_scale,
(self.os2.typo_ascender, self.os2.typo_descender))
self.pdf_bbox = tuple(map(pdf_scale, self.bbox))
self.pdf_capheight = pdf_scale(getattr(self.os2, 'cap_height',
self.os2.typo_ascender))
self.pdf_avg_width = pdf_scale(self.os2.average_char_width)
self.pdf_stemv = 50 + int((self.os2.weight_class / 65.0) ** 2)
def plugin_to_index(plugin, count):
title = '<h3><img src="/plugin-icon.png"><a href=%s title="Plugin forum thread">%s</a></h3>' % ( # noqa
quoteattr(plugin["thread_url"]),
escape(plugin["name"]),
)
released = datetime(*tuple(map(int, re.split(r"\D", plugin["last_modified"])))[:6]).strftime("%e %b, %Y").lstrip()
details = [
"Version: <b>%s</b>" % escape(".".join(map(str, plugin["version"]))),
"Released: <b>%s</b>" % escape(released),
"Author: %s" % escape(plugin["author"]),
"History: %s" % escape("Yes" if plugin["history"] else "No"),
"calibre: %s" % escape(".".join(map(str, plugin["minimum_calibre_version"]))),
"Platforms: %s" % escape(", ".join(sorted(plugin["supported_platforms"]) or ["all"])),
]
if plugin["uninstall"]:
details.append("Uninstall: %s" % escape(", ".join(plugin["uninstall"])))
if plugin["donate"]:
details.append('<a href=%s title="Donate">Donate</a>' % quoteattr(plugin["donate"]))
block = []
for li in details:
if li.startswith("calibre:"):
block.append("<br>")
block.append("<li>%s</li>" % li)
block = "<ul>%s</ul>" % ("\n".join(block))
downloads = ('\xa0<span class="download-count">[%d total downloads]</span>' % count) if count else ""
zipfile = '<div class="end"><a href=%s title="Download plugin" download=%s>Download plugin \u2193</a>%s</div>' % (
quoteattr(plugin["file"]),
quoteattr(plugin["name"] + ".zip"),
downloads,
)
desc = plugin["description"] or ""
if desc:
desc = "<p>%s</p>" % desc
return "%s\n%s\n%s\n%s\n\n" % (title, desc, block, zipfile)
def plugin_to_index(plugin):
title = '<h3><img src="http://icons.iconarchive.com/icons/oxygen-icons.org/oxygen/32/Apps-preferences-plugin-icon.png"><a href=%s title="Plugin forum thread">%s</a></h3>' % ( # noqa
quoteattr(plugin['thread_url']), escape(plugin['name']))
released = datetime(*tuple(map(int, re.split(r'\D', plugin['last_modified'])))[:6]).strftime('%e %b, %Y').lstrip()
details = [
'Version: <b>%s</b>' % escape('.'.join(map(str, plugin['version']))),
'Released: <b>%s</b>' % escape(released),
'Author: %s' % escape(plugin['author']),
'History: %s' % escape('Yes' if plugin['history'] else 'No'),
'calibre: %s' % escape('.'.join(map(str, plugin['minimum_calibre_version']))),
'Platforms: %s' % escape(', '.join(sorted(plugin['supported_platforms']) or ['all'])),
]
if plugin['uninstall']:
details.append('Uninstall: %s' % escape(', '.join(plugin['uninstall'])))
if plugin['donate']:
details.append('<a href=%s title="Donate">Donate</a>' % quoteattr(plugin['donate']))
block = []
for li in details:
if li.startswith('calibre:'):
block.append('<br>')
block.append('<li>%s</li>' % li)
block = '<ul>%s</ul>' % ('\n'.join(block))
zipfile = '<div class="end"><a href=%s title="Download plugin" download=%s>Download plugin \u2193</a></div>' % (
quoteattr(plugin['file']), quoteattr(plugin['name'] + '.zip'))
desc = plugin['description'] or ''
if desc:
desc = '<p>%s</p>' % desc
return '%s\n%s\n%s\n%s\n\n' % (title, desc, block, zipfile)
def test_collate(self):
"""Test collate_iters function"""
indicies = [index(i) for i in [0, 1, 2, 3]]
helper = lambda i: indicies[i]
makeiter1 = lambda: iter(indicies)
makeiter2 = lambda: map(helper, [0, 1, 3])
makeiter3 = lambda: map(helper, [1, 2])
outiter = patchdir.collate_iters([makeiter1(), makeiter2()])
assert Iter.equal(outiter,
iter([(indicies[0], indicies[0]),
(indicies[1], indicies[1]),
(indicies[2], None),
(indicies[3], indicies[3])]))
assert Iter.equal(patchdir.collate_iters([makeiter1(),
makeiter2(),
makeiter3()]),
iter([(indicies[0], indicies[0], None),
(indicies[1], indicies[1], indicies[1]),
(indicies[2], None, indicies[2]),
(indicies[3], indicies[3], None)]), 1)
assert Iter.equal(patchdir.collate_iters([makeiter1(), iter([])]),
map(lambda i: (i, None), indicies))
assert Iter.equal(map(lambda i: (i, None), indicies),
patchdir.collate_iters([makeiter1(), iter([])]))
def unique_justseen(iterable, key=None):
"""List unique elements, preserving order. Remember only the element just \
seen.
Parameters
----------
iterable : `collections.Iterable`
Iterables to check.
key : `collections.Callable`, `None`, optional
If ``None`` the values are taken as they are. If it's a callable the
callable is applied to the value before comparing it.
Default is ``None``.
Returns
-------
iterable : generator
An iterable containing all unique values just seen in the iterable.
Examples
--------
>>> from nddata.utils.itertools_recipes import unique_justseen
>>> list(unique_justseen('AAAABBBCCDAABBB'))
['A', 'B', 'C', 'D', 'A', 'B']
>>> list(unique_justseen('ABBCcAD', str.lower))
['A', 'B', 'C', 'A', 'D']
"""
return map(next, map(itemgetter(1), groupby(iterable, key)))
def createMorphology(self, nmlcell, moosecell, symmetric=False):
"""Create the MOOSE compartmental morphology in `moosecell` using the
segments in NeuroML2 cell `nmlcell`. Create symmetric
compartments if `symmetric` is True.
"""
morphology = nmlcell.morphology
segments = morphology.segment
id_to_segment = dict([(seg.id, seg) for seg in segments])
if symmetric:
compclass = moose.SymCompartment
else:
compclass = moose.Compartment
# segment names are used as compartment names - assuming
# naming convention does not clash with that in MOOSE
cellpath = moosecell.path
id_to_comp = {}
for seg in segments:
if seg.name is not None:
id_to_comp[seg.id] = compclass('%s/%s' % (cellpath, seg.name))
else:
id_to_comp[seg.id] = compclass('%s/comp_%s' % (cellpath, seg.id))
# Now assign the positions and connect up axial resistance
if not symmetric:
src, dst = 'axial', 'raxial'
else:
src, dst = 'proximal', 'distal'
for segid, comp in id_to_comp.items():
segment = id_to_segment[segid]
try:
parent = id_to_segment[str(segment.parent.segment)]
except AttributeError:
parent = None
self.moose_to_nml[comp] = segment
self.nml_to_moose[segment] = comp
p0 = segment.proximal
if p0 is None:
if parent:
p0 = parent.distal
else:
raise Exception('No proximal point and no parent segment for segment: name=%s, id=%s' % (segment.name, segment.id))
comp.x0, comp.y0, comp.z0 = (x * self.lunit for x in map(float, (p0.x, p0.y, p0.z)))
p1 = segment.distal
comp.x, comp.y, comp.z = (x * self.lunit for x in map(float, (p1.x, p1.y, p1.z)))
comp.length = np.sqrt((comp.x - comp.x0)**2
+ (comp.y - comp.y0)**2
+ (comp.z - comp.z0)**2)
# This can pose problem with moose where both ends of
# compartment have same diameter. We are averaging the two
# - may be splitting the compartment into two is better?
comp.diameter = (float(p0.diameter)+float(p1.diameter)) * self.lunit / 2
if parent:
pcomp = id_to_comp[parent.id]
moose.connect(comp, src, pcomp, dst)
sg_to_segments = {}
for sg in morphology.segmentGroup:
sg_to_segments[sg.id] = [id_to_segment[str(m.segment)] for m in sg.member]
self._cell_to_sg[nmlcell] = sg_to_segments
return id_to_comp, id_to_segment, sg_to_segments
def _generate_constant_defs(macro_defs, macro_type_map):
constants = '\n'.join(map(
lambda (name, macro): ' constexpr {type} const {name} = {value};'.format(
type=macro_type_map[name], name=name, value=list(macro.get_tokens())[1].spelling),
filter(lambda (name, _): name in macro_type_map, macro_defs.items())))
if 'OFP_NO_BUFFER' not in macro_defs:
return '\n'.join([constants, ' constexpr std::uint32_t const OFP_NO_BUFFER = 0xffffffff;'])
return constants
def __init__(self, items, level1=DEFAULT_LEVEL1, level2=DEFAULT_LEVEL2, level3=DEFAULT_LEVEL3):
items = map(lambda x: normalize('NFC', unicode(x)), filter(None, items))
items = tuple(map(lambda x: x.encode('utf-8'), items))
sort_keys = tuple(map(primary_sort_key, items))
speedup, err = plugins['matcher']
if speedup is None:
raise RuntimeError('Failed to load the matcher plugin with error: %s' % err)
self.m = speedup.Matcher(items, sort_keys, level1.encode('utf-8'), level2.encode('utf-8'), level3.encode('utf-8'))
def _generate_enum_decls(enum_decls):
return '\n\n'.join(map(
lambda (name, enum): (
"""\
enum {name}
{{
{members}
}};\
""".format(name=name, members=_generate_enum_members(enum.get_children()))),
enum_decls.items()))
def __call__(self, data):
self.beginResetModel()
self.files = data['files']
self.total_size = sum(map(itemgetter(3), self.files))
self.images_size = sum(map(itemgetter(3), (f for f in self.files if f.category == 'image')))
self.fonts_size = sum(map(itemgetter(3), (f for f in self.files if f.category == 'font')))
psk = numeric_sort_key
self.sort_keys = tuple((psk(entry.dir), psk(entry.basename), entry.size, psk(self.CATEGORY_NAMES.get(entry.category, '')))
for entry in self.files)
self.endResetModel()
def stringtotime(timestring):
"""Return time in seconds from w3 or duplicity timestring
If there is an error parsing the string, or it doesn't look
like a valid datetime string, return None.
"""
try:
date, daytime = timestring[:19].split("T")
if len(timestring) == 16:
# new format for filename time
year, month, day = map(int,
[date[0:4], date[4:6], date[6:8]])
hour, minute, second = map(int,
[daytime[0:2], daytime[2:4], daytime[4:6]])
else:
# old format for filename time
year, month, day = map(int, date.split("-"))
hour, minute, second = map(int,
daytime.split(globals.time_separator))
assert 1900 < year < 2100, year
assert 1 <= month <= 12
assert 1 <= day <= 31
assert 0 <= hour <= 23
assert 0 <= minute <= 59
assert 0 <= second <= 61 # leap seconds
# We want to return the time in units of seconds since the
# epoch. Unfortunately the only functin that does this
# works in terms of the current timezone and we have a
# timezone offset in the string.
timetuple = (year, month, day, hour, minute, second, -1, -1, 0)
if len(timestring) == 16:
# as said in documentation, time.gmtime() and timegm() are each others' inverse.
# As far as UTC format is used in new file format,
# do not rely on system's python DST and tzdata settings
# and use functions that working with UTC
utc_in_secs = calendar.timegm(timetuple)
else:
# mktime assumed that the tuple was a local time. Compensate
# by subtracting the value for the current timezone.
# We don't need to worry about DST here because we turned it
# off in the tuple
local_in_secs = time.mktime(timetuple)
utc_in_secs = local_in_secs - time.timezone
# Now apply the offset that we were given in the time string
# This gives the correct number of seconds from the epoch
# even when we're not in the same timezone that wrote the
# string
if len(timestring) == 16:
return int(utc_in_secs)
else:
return int(utc_in_secs + tzdtoseconds(timestring[19:]))
except (TypeError, ValueError, AssertionError):
return None
def apply_color_space(self, color, pattern, stroke=False):
wl = self.current_page.write_line
if color is not None and pattern is None:
wl(' '.join(map(fmtnum, color)) + (' RG' if stroke else ' rg'))
elif color is None and pattern is not None:
wl('/Pattern %s /%s %s'%('CS' if stroke else 'cs', pattern,
'SCN' if stroke else 'scn'))
elif color is not None and pattern is not None:
col = ' '.join(map(fmtnum, color))
wl('/PCSp %s %s /%s %s'%('CS' if stroke else 'cs', col, pattern,
'SCN' if stroke else 'scn'))
def __init__(self, qwe):
self.qwe = qwe
self.attributes = {unicode(x):unicode(qwe.attribute(x)) for x in
qwe.attributeNames()}
self.input_controls = list(map(Control, qwe.findAll('input')))
rc = [y for y in self.input_controls if y.type == 'radio']
self.input_controls = [ic for ic in self.input_controls if ic.type != 'radio']
rc_names = {x.name for x in rc}
self.radio_controls = {name:RadioControl(name, [z.qwe for z in rc if z.name == name]) for name in rc_names}
selects = list(map(SelectControl, qwe.findAll('select')))
self.select_controls = {x.name:x for x in selects}
def _generate_adapt_structs(collector):
return '\n\n'.join(map(
lambda (name, struct): (
"""\
BOOST_FUSION_ADAPT_STRUCT(
canard::network::openflow::v{version}::v{version}_detail::{name},
{member_ppseq}
)\
""".format(version=collector.version, name=name, member_ppseq=_generate_member_ppseq(
collector.version, struct.members, is_skip_ofp_match=(collector.version == 13)))),
collector.struct_decls.items()))
def pyimplementation():
if hasattr(_platform, 'python_implementation'):
return _platform.python_implementation()
elif sys.platform.startswith('java'):
return 'Jython ' + sys.platform
elif hasattr(sys, 'pypy_version_info'):
v = '.'.join(map(str, sys.pypy_version_info[:3]))
if sys.pypy_version_info[3:]:
v += '-' + ''.join(map(str, sys.pypy_version_info[3:]))
return 'PyPy ' + v
else:
return 'CPython'
def pyimplementation():
if hasattr(_platform, "python_implementation"):
return _platform.python_implementation()
elif sys.platform.startswith("java"):
return "Jython " + sys.platform
elif hasattr(sys, "pypy_version_info"):
v = ".".join(map(str, sys.pypy_version_info[:3]))
if sys.pypy_version_info[3:]:
v += "-" + "".join(map(str, sys.pypy_version_info[3:]))
return "PyPy " + v
else:
return "CPython"
def get(ctx, rd, what, book_id, library_id):
book_id, rest = book_id.partition('_')[::2]
try:
book_id = int(book_id)
except Exception:
raise HTTPNotFound('Book with id %r does not exist' % book_id)
db = get_db(ctx, rd, library_id)
if db is None:
raise HTTPNotFound('Library %r not found' % library_id)
with db.safe_read_lock:
if not ctx.has_id(rd, db, book_id):
raise BookNotFound(book_id, db)
library_id = db.server_library_id # in case library_id was None
if what == 'thumb':
sz = rd.query.get('sz')
w, h = 60, 80
if sz is None:
if rest:
try:
w, h = map(int, rest.split('_'))
except Exception:
pass
elif sz == 'full':
w = h = None
elif 'x' in sz:
try:
w, h = map(int, sz.partition('x')[::2])
except Exception:
pass
else:
try:
w = h = int(sz)
except Exception:
pass
return cover(ctx, rd, library_id, db, book_id, width=w, height=h)
elif what == 'cover':
return cover(ctx, rd, library_id, db, book_id)
elif what == 'opf':
mi = db.get_metadata(book_id, get_cover=False)
rd.outheaders['Content-Type'] = 'application/oebps-package+xml; charset=UTF-8'
rd.outheaders['Last-Modified'] = http_date(timestampfromdt(mi.last_modified))
return metadata_to_opf(mi)
elif what == 'json':
from calibre.srv.ajax import book_to_json
data, last_modified = book_to_json(ctx, rd, db, book_id)
rd.outheaders['Last-Modified'] = http_date(timestampfromdt(last_modified))
return json(ctx, rd, get, data)
else:
try:
return book_fmt(ctx, rd, library_id, db, book_id, what.lower())
except NoSuchFormat:
raise HTTPNotFound('No %s format for the book %r' % (what.lower(), book_id))
def numbers(self, name, step=0, type=int, counts=False):
"""Generate decoded numeric term values, optionally with frequency counts.
:param name: field name
:param step: precision step to select terms
:param type: int or float
:param counts: include frequency counts
"""
convert = util.NumericUtils.sortableLongToDouble if issubclass(type, float) else int
termsenum = index.MultiFields.getTerms(self.indexReader, name).iterator(None)
termsenum = search.PrefixTermsEnum(termsenum, util.BytesRef(chr(ord(" ") + step)))
values = map(convert, map(util.NumericUtils.prefixCodedToLong, termsenum))
return ((value, termsenum.docFreq()) for value in values) if counts else values
def _generate_struct_decls(struct_decls, is_skip_ofp_match=False):
return '\n\n'.join(map(
lambda (name, struct): (
"""\
struct {name}
{{
{members}
}};
static_assert(sizeof({name}) == {size}, "");\
""".format(
name=name,
members=_generate_struct_members(struct.members, is_skip_ofp_match),
size=_struct_size(struct, is_skip_ofp_match))),
struct_decls.items()))
def testInterface(self):
"Distributed reading and writing."
resources = client.Resources(self.hosts, limit=1)
assert resources.unicast('GET', '/')
assert not resources.unicast('POST', '/terms')
responses = resources.broadcast('GET', '/')
assert len(responses) == len(resources)
for response in responses:
(directory, count), = response().items()
assert count == 0 and directory.startswith('org.apache.lucene.store.RAMDirectory@')
responses = resources.broadcast('PUT', '/fields/text')
assert all(response() == {'index': 'ANALYZED', 'store': 'NO', 'termvector': 'NO'} for response in responses)
responses = resources.broadcast('PUT', '/fields/name', {'store': 'yes', 'index': 'not_analyzed'})
assert all(response() == {'index': 'NOT_ANALYZED', 'store': 'YES', 'termvector': 'NO'} for response in responses)
doc = {'name': 'sample', 'text': 'hello world'}
responses = resources.broadcast('POST', '/docs', [doc])
assert all(response() is None for response in responses)
response = resources.unicast('POST', '/docs', [doc])
assert response() is None
responses = resources.broadcast('POST', '/update')
assert all(response() >= 1 for response in responses)
responses = resources.broadcast('GET', '/search?q=text:hello')
docs = []
for response in responses:
result = response()
assert result['count'] >= 1
docs += result['docs']
assert len(docs) == len(resources) + 1
assert len(set(doc['__id__'] for doc in docs)) == 2
self.stop(self.servers.pop(0))
self.assertRaises(socket.error, resources.broadcast, 'GET', '/')
assert resources.unicast('GET', '/')()
del resources[self.hosts[0]]
assert all(resources.broadcast('GET', '/'))
assert list(map(len, resources.values())) == [1, 1]
time.sleep(self.config['server.socket_timeout'] + 1)
assert resources.unicast('GET', '/')
counts = list(map(len, resources.values()))
assert set(counts) == set([0, 1])
assert resources.broadcast('GET', '/')
assert list(map(len, resources.values())) == counts[::-1]
host = self.hosts[1]
resource = resources.request(host, 'GET', '/')
resource.getresponse = lambda: getresponse(socket.error)
self.assertRaises(socket.error, resources.getresponse, host, resource)
resource = resources.request(host, 'GET', '/')
resource.getresponse = lambda: getresponse(httplib.BadStatusLine)
assert resources.getresponse(host, resource) is None
resources.clear()
self.assertRaises(ValueError, resources.unicast, 'GET', '/')
def __init__(self, items, level1=DEFAULT_LEVEL1, level2=DEFAULT_LEVEL2, level3=DEFAULT_LEVEL3, scorer=None):
with wlock:
if not workers:
requests, results = Queue(), Queue()
w = [Worker(requests, results) for i in range(max(1, cpu_count()))]
[x.start() for x in w]
workers.extend(w)
items = map(lambda x: normalize('NFC', unicode(x)), filter(None, items))
self.items = items = tuple(items)
tasks = split(items, len(workers))
self.task_maps = [{j:i for j, (i, _) in enumerate(task)} for task in tasks]
scorer = scorer or default_scorer
self.scorers = [scorer(tuple(map(itemgetter(1), task_items))) for task_items in tasks]
self.sort_keys = None
def dump(self, items, out_stream, pdf_metadata):
opts = self.opts
self.outline = Outline(self.toc, items)
page_size = get_page_size(self.opts)
xdpi, ydpi = self.view.logicalDpiX(), self.view.logicalDpiY()
ml, mr = opts.margin_left, opts.margin_right
margin_side = min(ml, mr)
ml, mr = ml - margin_side, mr - margin_side
self.doc = PdfDevice(out_stream, page_size=page_size, left_margin=ml,
top_margin=0, right_margin=mr, bottom_margin=0,
xdpi=xdpi, ydpi=ydpi, errors=self.log.error,
debug=self.log.debug, compress=not
opts.uncompressed_pdf)
self.page.setViewportSize(QSize(self.doc.width(), self.doc.height()))
self.render_queue = items
self.total_items = len(items)
# TODO: Test margins
mt, mb = map(self.doc.to_px, (opts.margin_top, opts.margin_bottom))
ms = self.doc.to_px(margin_side, vertical=False)
self.margin_top, self.margin_size, self.margin_bottom = map(
lambda x:int(floor(x)), (mt, ms, mb))
self.painter = QPainter(self.doc)
self.doc.set_metadata(title=pdf_metadata.title,
author=pdf_metadata.author,
tags=pdf_metadata.tags)
self.painter.save()
try:
if self.cover_data is not None:
p = QPixmap()
p.loadFromData(self.cover_data)
if not p.isNull():
draw_image_page(QRect(0, 0, self.doc.width(), self.doc.height()),
self.painter, p,
preserve_aspect_ratio=self.opts.preserve_cover_aspect_ratio)
self.doc.end_page()
finally:
self.painter.restore()
QTimer.singleShot(0, self.render_book)
self.loop.exec_()
# TODO: Outline and links
self.painter.end()
if self.doc.errors_occurred:
raise Exception('PDF Output failed, see log for details')
def sort_q_values(header_val):
'Get sorted items from an HTTP header of type: a;q=0.5, b;q=0.7...'
if not header_val:
return []
def item(x):
e, r = x.partition(';')[::2]
p, v = r.partition('=')[::2]
q = 1.0
if p == 'q' and v:
try:
q = max(0.0, min(1.0, float(v.strip())))
except Exception:
pass
return e.strip(), q
return tuple(map(itemgetter(0), sorted(map(item, parse_http_list(header_val)), key=itemgetter(1), reverse=True)))
def refresh_ids(self, ids):
self.cache.clear_caches(book_ids=ids)
try:
return list(map(self.id_to_index, ids))
except ValueError:
pass
return None
def ddg_search(terms,
site=None,
br=None,
log=prints,
safe_search=False,
dump_raw=None,
timeout=60):
# https://duck.co/help/results/syntax
terms = map(ddg_term, terms)
terms = [quote_term(t) for t in terms]
if site is not None:
terms.append(quote_term(('site:' + site)))
q = '+'.join(terms)
url = 'https://duckduckgo.com/html/?q={q}&kp={kp}'.format(
q=q, kp=1 if safe_search else -1)
log('Making ddg query: ' + url)
br = br or browser()
root = query(br, url, 'ddg', dump_raw, timeout=timeout)
ans = []
for a in root.xpath(
'//*[@class="results"]//*[@class="result__title"]/a[@href and @class="result__a"]'
):
ans.append(Result(ddg_href(a.get('href')), tostring(a), None))
return ans, url
def find_identical_books(mi, data):
author_map, aid_map, title_map, lang_map = data
found_books = None
for a in mi.authors:
author_ids = author_map.get(icu_lower(a))
if author_ids is None:
return set()
books_by_author = {
book_id
for aid in author_ids for book_id in aid_map.get(aid, ())
}
if found_books is None:
found_books = books_by_author
else:
found_books &= books_by_author
if not found_books:
return set()
ans = set()
titleq = fuzzy_title(mi.title)
for book_id in found_books:
title = title_map.get(book_id, '')
if fuzzy_title(title) == titleq:
ans.add(book_id)
langq = tuple(
filter(lambda x: x and x != 'und',
map(canonicalize_lang, mi.languages or ())))
if not langq:
return ans
def lang_matches(book_id):
book_langq = lang_map.get(book_id)
return not book_langq or langq == book_langq
return {book_id for book_id in ans if lang_matches(book_id)}
def _in(d):
ret = []
neg = False
for i in d:
if i[0] == 'range':
subs = map(unichr, range(i[1][0], i[1][1] + 1))
if neg:
for char in subs:
try:
ret.remove(char)
except:
pass
else:
ret.extend(subs)
elif i[0] == 'literal':
if neg:
try:
ret.remove(unichr(i[1]))
except:
pass
else:
ret.append(unichr(i[1]))
elif i[0] == 'category':
subs = CATEGORIES.get(i[1], [''])
if neg:
for char in subs:
try:
ret.remove(char)
except:
pass
else:
ret.extend(subs)
elif i[0] == 'negate':
ret = list(CATEGORIES['category_any'])
neg = True
return ret
def _in(d):
ret = []
neg = False
for i in d:
if i[0] == sre_parse.RANGE:
subs = map(unichr, range(i[1][0], i[1][1] + 1))
if neg:
for char in subs:
try:
ret.remove(char)
except:
pass
else:
ret.extend(subs)
elif i[0] == sre_parse.LITERAL:
if neg:
try:
ret.remove(unichr(i[1]))
except:
pass
else:
ret.append(unichr(i[1]))
elif i[0] == sre_parse.CATEGORY:
subs = CATEGORIES.get(i[1], [''])
if neg:
for char in subs:
try:
ret.remove(char)
except:
pass
else:
ret.extend(subs)
elif i[0] == sre_parse.NEGATE:
ret = list(CATEGORIES['category_any'])
neg = True
return ret
def write_widths(self, objects):
glyphs = sorted(self.used_glyphs | {0})
widths = {
g: self.metrics.pdf_scale(w)
for g, w in zip(glyphs, self.metrics.glyph_widths(glyphs))
}
counter = Counter()
for g, w in widths.items():
counter[w] += 1
most_common = counter.most_common(1)[0][0]
self.descendant_font['DW'] = most_common
widths = {g: w for g, w in widths.items() if w != most_common}
groups = Array()
for k, g in groupby(enumerate(widths.keys()),
lambda i_x: i_x[0] - i_x[1]):
group = list(map(itemgetter(1), g))
gwidths = [widths[g] for g in group]
if len(set(gwidths)) == 1 and len(group) > 1:
w = (min(group), max(group), gwidths[0])
else:
w = (min(group), Array(gwidths))
groups.extend(w)
self.descendant_font['W'] = objects.add(groups)
def main():
t = int(input())
for _ in range(t):
n = int(input())
a = list(map(int, input().split()))
l = []
for i in range(n):
if (a[i] == 1 and (len(l) == 0 or l[-1] != a[i])):
l.append(a[i])
else:
l.append(a[i])
for i in range(len(l)):
if (l[i] == 1):
oi = i
break
xi = oi
ans = 0
for i in range(oi + 1, len(l)):
if (a[i] == 1):
ans += (i - xi - 1)
xi = i
print(ans)
def select_rows(self,
identifiers,
using_ids=True,
change_current=True,
scroll=True):
'''
Select rows identified by identifiers. identifiers can be a set of ids,
row numbers or QModelIndexes.
'''
rows = set([x.row() if hasattr(x, 'row') else x for x in identifiers])
if using_ids:
rows = set([])
identifiers = set(identifiers)
m = self.model()
for row in xrange(m.rowCount(QModelIndex())):
if m.id(row) in identifiers:
rows.add(row)
rows = list(sorted(rows))
if rows:
row = rows[0]
if change_current:
self.set_current_row(row, select=False)
if scroll:
self.scroll_to_row(row)
sm = self.selectionModel()
sel = QItemSelection()
m = self.model()
max_col = m.columnCount(QModelIndex()) - 1
# Create a range based selector for each set of contiguous rows
# as supplying selectors for each individual row causes very poor
# performance if a large number of rows has to be selected.
for k, g in itertools.groupby(enumerate(rows), lambda (i, x): i - x):
group = list(map(operator.itemgetter(1), g))
sel.merge(
QItemSelection(m.index(min(group), 0),
m.index(max(group), max_col)), sm.Select)
def parse_part():
line = buf.readline()
if not line:
raise ValueError('Premature end of message')
if not line.startswith(b'--' + sep):
raise ValueError('Malformed start of multipart message: %s' % reprlib.repr(line))
if line.endswith(b'--'):
return None
headers = read_headers(buf.readline)
cr = headers.get('Content-Range')
if not cr:
raise ValueError('Missing Content-Range header in sub-part')
if not cr.startswith('bytes '):
raise ValueError('Malformed Content-Range header in sub-part, no prefix')
try:
start, stop = map(lambda x: int(x.strip()), cr.partition(' ')[-1].partition('/')[0].partition('-')[::2])
except Exception:
raise ValueError('Malformed Content-Range header in sub-part, failed to parse byte range')
content_length = stop - start + 1
ret = buf.read(content_length)
if len(ret) != content_length:
raise ValueError('Malformed sub-part, length of body not equal to length specified in Content-Range')
buf.readline()
return (start, ret)
def extract_Pframe_all(tI, hIlm, listmodes):
# Build precessing-frame waveform - uses GWFrames
T_data = tI
LM_data = np.array(list(map(list, listmodes)), dtype=np.int32)
mode_data = np.array([hIlm[lm] for lm in listmodes])
W_v3 = GWFrames.Waveform(T_data, LM_data, mode_data)
W_v3.SetFrameType(1);
W_v3.SetDataType(1);
W_v3.TransformToCoprecessingFrame();
# Time series for the dominant radiation vector
quat = W_v3.Frame()
logquat = np.array(list(map(lambda q: q.log(), quat)))
quatseries = np.array(list(map(lambda q: np.array([q[0], q[1], q[2], q[3]]), quat)))
logquatseries = np.array(list(map(lambda q: np.array([q[0], q[1], q[2], q[3]]), logquat)))
Vfseries = np.array(list(map(Vf_from_quat, quat)))
eulerVfseries = np.array(list(map(euler_from_quat, quat)))
alphaVfseries = np.unwrap(eulerVfseries[:,0])
betaVfseries = eulerVfseries[:,1]
gammaVfseries = np.unwrap(eulerVfseries[:,2])
eulerVfseries = np.array([alphaVfseries, betaVfseries, gammaVfseries]).T
return [eulerVfseries, quatseries, logquatseries, Vfseries]
def hist2d(x,
y,
bins=20,
range=None,
weights=None,
levels=None,
smooth=None,
ax=None,
color=None,
plot_datapoints=True,
plot_density=True,
plot_contours=True,
no_fill_contours=False,
fill_contours=False,
contour_kwargs=None,
contourf_kwargs=None,
data_kwargs=None,
**kwargs):
"""
Plot a 2-D histogram of samples.
Parameters
----------
x : array_like[nsamples,]
The samples.
y : array_like[nsamples,]
The samples.
levels : array_like
The contour levels to draw.
ax : matplotlib.Axes
A axes instance on which to add the 2-D histogram.
plot_datapoints : bool
Draw the individual data points.
plot_density : bool
Draw the density colormap.
plot_contours : bool
Draw the contours.
no_fill_contours : bool
Add no filling at all to the contours (unlike setting
``fill_contours=False``, which still adds a white fill at the densest
points).
fill_contours : bool
Fill the contours.
contour_kwargs : dict
Any additional keyword arguments to pass to the `contour` method.
contourf_kwargs : dict
Any additional keyword arguments to pass to the `contourf` method.
data_kwargs : dict
Any additional keyword arguments to pass to the `plot` method when
adding the individual data points.
"""
if ax is None:
ax = plt.gca()
# Set the default range based on the data range if not provided.
if range is None:
if "extent" in kwargs:
logging.warn("Deprecated keyword argument 'extent'. "
"Use 'range' instead.")
range = kwargs["extent"]
else:
range = [[x.min(), x.max()], [y.min(), y.max()]]
# Set up the default plotting arguments.
if color is None:
color = "k"
# Choose the default "sigma" contour levels.
if levels is None:
levels = 1.0 - np.exp(-0.5 * np.arange(0.5, 2.1, 0.5)**2)
# This is the color map for the density plot, over-plotted to indicate the
# density of the points near the center.
density_cmap = LinearSegmentedColormap.from_list("density_cmap",
[color, (1, 1, 1, 0)])
# This color map is used to hide the points at the high density areas.
white_cmap = LinearSegmentedColormap.from_list("white_cmap", [(1, 1, 1),
(1, 1, 1)],
N=2)
# This "color map" is the list of colors for the contour levels if the
# contours are filled.
rgba_color = colorConverter.to_rgba(color)
contour_cmap = [list(rgba_color) for l in levels] + [rgba_color]
for i, l in enumerate(levels):
contour_cmap[i][-1] *= float(i) / (len(levels) + 1)
# We'll make the 2D histogram to directly estimate the density.
try:
H, X, Y = np.histogram2d(x.flatten(),
y.flatten(),
bins=bins,
range=list(map(np.sort, range)),
weights=weights)
except ValueError:
raise ValueError("It looks like at least one of your sample columns "
"have no dynamic range. You could try using the "
"'range' argument.")
if smooth is not None:
if gaussian_filter is None:
raise ImportError("Please install scipy for smoothing")
H = gaussian_filter(H, smooth)
# Compute the density levels.
Hflat = H.flatten()
inds = np.argsort(Hflat)[::-1]
Hflat = Hflat[inds]
sm = np.cumsum(Hflat)
sm /= sm[-1]
V = np.empty(len(levels))
for i, v0 in enumerate(levels):
try:
V[i] = Hflat[sm <= v0][-1]
except:
V[i] = Hflat[0]
V.sort()
m = np.diff(V) == 0
if np.any(m):
logging.warning("Too few points to create valid contours")
while np.any(m):
V[np.where(m)[0][0]] *= 1.0 - 1e-4
m = np.diff(V) == 0
V.sort()
# Compute the bin centers.
X1, Y1 = 0.5 * (X[1:] + X[:-1]), 0.5 * (Y[1:] + Y[:-1])
# Extend the array for the sake of the contours at the plot edges.
H2 = H.min() + np.zeros((H.shape[0] + 4, H.shape[1] + 4))
H2[2:-2, 2:-2] = H
H2[2:-2, 1] = H[:, 0]
H2[2:-2, -2] = H[:, -1]
H2[1, 2:-2] = H[0]
H2[-2, 2:-2] = H[-1]
H2[1, 1] = H[0, 0]
H2[1, -2] = H[0, -1]
H2[-2, 1] = H[-1, 0]
H2[-2, -2] = H[-1, -1]
X2 = np.concatenate([
X1[0] + np.array([-2, -1]) * np.diff(X1[:2]),
X1,
X1[-1] + np.array([1, 2]) * np.diff(X1[-2:]),
])
Y2 = np.concatenate([
Y1[0] + np.array([-2, -1]) * np.diff(Y1[:2]),
Y1,
Y1[-1] + np.array([1, 2]) * np.diff(Y1[-2:]),
])
if plot_datapoints:
if data_kwargs is None:
data_kwargs = dict()
data_kwargs["color"] = data_kwargs.get("color", color)
data_kwargs["ms"] = data_kwargs.get("ms", 2.0)
data_kwargs["mec"] = data_kwargs.get("mec", "none")
data_kwargs["alpha"] = data_kwargs.get("alpha", 0.1)
ax.plot(x, y, "o", zorder=-1, rasterized=True, **data_kwargs)
# Plot the base fill to hide the densest data points.
if (plot_contours or plot_density) and not no_fill_contours:
ax.contourf(X2,
Y2,
H2.T, [V.min(), H.max()],
cmap=white_cmap,
antialiased=False)
if plot_contours and fill_contours:
if contourf_kwargs is None:
contourf_kwargs = dict()
contourf_kwargs["colors"] = contourf_kwargs.get("colors", contour_cmap)
contourf_kwargs["antialiased"] = contourf_kwargs.get(
"antialiased", False)
ax.contourf(X2, Y2, H2.T,
np.concatenate([[0], V, [H.max() * (1 + 1e-4)]]),
**contourf_kwargs)
# Plot the density map. This can't be plotted at the same time as the
# contour fills.
elif plot_density:
ax.pcolor(X, Y, H.max() - H.T, cmap=density_cmap)
# Plot the contour edge colors.
if plot_contours:
if contour_kwargs is None:
contour_kwargs = dict()
contour_kwargs["colors"] = contour_kwargs.get("colors", color)
ax.contour(X2, Y2, H2.T, V, **contour_kwargs)
ax.set_xlim(range[0])
ax.set_ylim(range[1])
def process_node(node):
name = node['dest']
if name and node['id'] not in seen_map[name]:
ans[name].append({'id': node['id'], 'frag': node['frag']})
seen_map[name].add(node['id'])
tuple(map(process_node, node['children']))
def display(self):
"""Returns a :func:`list` of screen lines as unicode strings."""
return [
"".join(map(operator.attrgetter("data"), line))
for line in self.buffer
]
def corner(xs,
bins=20,
params_range=None,
weights=None,
cov=None,
color="k",
smooth=None,
smooth1d=None,
labels=None,
label_kwargs=None,
show_histograms=True,
show_titles=False,
title_fmt=".2f",
title_kwargs=None,
truths=None,
add_truths=None,
truth_color="#4682b4",
add_truth_colors=None,
cov_color=None,
scale_hist=False,
quantiles=None,
show_quantiles=True,
verbose=False,
figaxes=None,
max_n_ticks=5,
top_ticks=False,
use_math_text=False,
hist_kwargs=None,
**hist2d_kwargs):
"""
Make a *sick* corner plot showing the projections of a data set in a
multi-dimensional space. kwargs are passed to hist2d() or used for
`matplotlib` styling.
Parameters
----------
xs : array_like[nsamples, ndim]
The samples. This should be a 1- or 2-dimensional array. For a 1-D
array this results in a simple histogram. For a 2-D array, the zeroth
axis is the list of samples and the next axis are the dimensions of
the space.
bins : int or array_like[ndim,]
The number of bins to use in histograms, either as a fixed value for
all dimensions, or as a list of integers for each dimension.
weights : array_like[nsamples,]
The weight of each sample. If `None` (default), samples are given
equal weight.
color : str
A ``matplotlib`` style color for all histograms.
smooth, smooth1d : float
The standard deviation for Gaussian kernel passed to
`scipy.ndimage.gaussian_filter` to smooth the 2-D and 1-D histograms
respectively. If `None` (default), no smoothing is applied.
labels : iterable (ndim,)
A list of names for the dimensions. If a ``xs`` is a
``pandas.DataFrame``, labels will default to column names.
label_kwargs : dict
Any extra keyword arguments to send to the `set_xlabel` and
`set_ylabel` methods.
show_histograms : bool
Displays 1-D histogram showing marginalized distributions and quantiles.
show_titles : bool
Displays a title above each 1-D histogram showing the 0.5 quantile
with the upper and lower errors supplied by the quantiles argument.
title_fmt : string
The format string for the quantiles given in titles. If you explicitly
set ``show_titles=True`` and ``title_fmt=None``, the labels will be
shown as the titles. (default: ``.2f``)
title_kwargs : dict
Any extra keyword arguments to send to the `set_title` command.
params_range : iterable (ndim,)
A list where each element is either a length 2 tuple containing
lower and upper bounds or a float in range (0., 1.)
giving the fraction of samples to include in bounds, e.g.,
[(0.,10.), (1.,5), 0.999, etc.].
If a fraction, the bounds are chosen to be equal-tailed.
truths : iterable (ndim,)
A list of reference values to indicate on the plots. Individual
values can be omitted by using ``None``.
truth_color : str
A ``matplotlib`` style color for the ``truths`` makers.
scale_hist : bool
Should the 1-D histograms be scaled in such a way that the zero line
is visible?
quantiles : iterable
A list of fractional quantiles to show on the 1-D histograms as
vertical dashed lines.
verbose : bool
If true, print the values of the computed quantiles.
plot_contours : bool
Draw contours for dense regions of the plot.
use_math_text : bool
If true, then axis tick labels for very large or small exponents will
be displayed as powers of 10 rather than using `e`.
max_n_ticks: int
Maximum number of ticks to try to use
top_ticks : bool
If true, label the top ticks of each axis
fig : matplotlib.Figure
Overplot onto the provided figure object.
hist_kwargs : dict
Any extra keyword arguments to send to the 1-D histogram plots.
**hist2d_kwargs
Any remaining keyword arguments are sent to `corner.hist2d` to generate
the 2-D histogram plots.
"""
if quantiles is None:
quantiles = []
if title_kwargs is None:
title_kwargs = dict()
if label_kwargs is None:
label_kwargs = dict()
# Try filling in labels from pandas.DataFrame columns.
if labels is None:
try:
labels = xs.columns
except AttributeError:
pass
# Deal with 1D sample lists.
xs = np.atleast_1d(xs)
if len(xs.shape) == 1:
xs = np.atleast_2d(xs)
else:
assert len(xs.shape) == 2, "The input sample array must be 1- or 2-D."
xs = xs.T
assert xs.shape[0] <= xs.shape[1], "I don't believe that you want more " \
"dimensions than samples!"
# Parse the weight array.
if weights is not None:
weights = np.asarray(weights)
if weights.ndim != 1:
raise ValueError("Weights must be 1-D")
if xs.shape[1] != weights.shape[0]:
raise ValueError("Lengths of weights must match number of samples")
# Parse the parameter ranges.
if params_range is None:
if "extents" in hist2d_kwargs:
logging.warn("Deprecated keyword argument 'extents'. "
"Use 'params_range' instead.")
params_range = hist2d_kwargs.pop("extents")
else:
params_range = [[x.min(), x.max()] for x in xs]
# Check for parameters that never change.
m = np.array([e[0] == e[1] for e in params_range], dtype=bool)
if np.any(m):
raise ValueError(
("It looks like the parameter(s) in "
"column(s) {0} have no dynamic range. "
"Please provide a `params_range` argument.").format(
", ".join(map("{0}".format,
np.arange(len(m))[m]))))
else:
# If any of the extents are percentiles, convert them to ranges.
# Also make sure it's a normal list.
params_range = list(params_range)
for i, pr in enumerate(params_range):
if pr is None:
params_range[i] = [xs[i].min(), xs[i].max()]
else:
try:
emin, emax = params_range[i]
except TypeError:
q = [
0.5 - 0.5 * params_range[i],
0.5 + 0.5 * params_range[i]
]
params_range[i] = quantile(xs[i], q, weights=weights)
if len(params_range) != xs.shape[0]:
raise ValueError("Dimension mismatch between samples and params_range")
# Parse the bin specifications.
try:
bins = [int(bins) for _ in params_range]
except TypeError:
if len(bins) != len(params_range):
raise ValueError(
"Dimension mismatch between bins and params_range")
# Some magic numbers for pretty axis layout.
K = len(xs)
if show_histograms: # total number of plots: grid nplot*nplot
nplot = K
else:
nplot = K - 1
factor = 2.0 # size of one side of one panel
lbdim = 0.5 * factor # size of left/bottom margin
trdim = 0.2 * factor # size of top/right margin
whspace = 0.05 # w/hspace size
plotdim = factor * nplot + factor * (nplot - 1.) * whspace
dim = lbdim + plotdim + trdim
# Create a new figure if one wasn't provided.
# if fig is None:
# fig, axes = plt.subplots(nplot, nplot, figsize=(dim, dim))
# else:
# print(type(fig.axes))
# try:
# axes = np.array(fig.axes).reshape((nplot, nplot))
# except:
# raise ValueError("Provided figure has {0} axes, but data has "
# "dimensions K={1}".format(len(fig.axes), nplot))
if figaxes is None:
fig, axes = plt.subplots(nplot, nplot, figsize=(dim, dim))
else:
fig, axes = figaxes
#idea is to pass in covariance, otherwise concoct something from the 1-sigma range.
# if(cov==None):
# print("concocting covar elements from 1-sigma ranges")
# cov=np.zeros((K,K))
# for k in np.arange(K):
# q_16, q_50, q_84 = quantile(xs[k], [0.16, 0.5, 0.84],weights=weights)
# deltax=(q_84-q_16)/2.0
# cov[k,k]=deltax**2
# print("cov=",cov)
# Format the figure.
lb = lbdim / dim
tr = (lbdim + plotdim) / dim
fig.subplots_adjust(left=lb,
bottom=lb,
right=tr,
top=tr,
wspace=whspace,
hspace=whspace)
# If asked to show the 1D histograms
if show_histograms:
# Set up the default histogram keywords.
if hist_kwargs is None:
hist_kwargs = dict()
hist_kwargs["color"] = hist_kwargs.get("color", color)
if smooth1d is None:
hist_kwargs["histtype"] = hist_kwargs.get("histtype", "step")
for i, x in enumerate(xs):
# Deal with masked arrays.
if hasattr(x, "compressed"):
x = x.compressed()
if np.shape(xs)[0] == 1:
ax = axes
else:
ax = axes[i, i]
# Plot the histograms.
if smooth1d is None:
n, _, _ = ax.hist(x,
bins=bins[i],
weights=weights,
density=True,
range=np.sort(params_range[i]),
**hist_kwargs)
else:
if gaussian_filter is None:
raise ImportError("Please install scipy for smoothing")
n, b = np.histogram(x,
bins=bins[i],
weights=weights,
density=True,
range=np.sort(params_range[i]))
n = gaussian_filter(n, smooth1d)
x0 = np.array(list(zip(b[:-1], b[1:]))).flatten()
y0 = np.array(list(zip(n, n))).flatten()
ax.plot(x0, y0, **hist_kwargs)
if truths is not None and truths[i] is not None:
ax.axvline(truths[i], color=truth_color)
if add_truths is not None:
for add_truth, add_truth_color in zip(add_truths,
add_truth_colors):
if add_truth[i] is not None:
ax.axvline(add_truth[i], color=add_truth_color)
# Plot quantiles if wanted.
if show_quantiles:
if len(quantiles) > 0:
qvalues = quantile(x, quantiles, weights=weights)
for q in qvalues:
ax.axvline(q, ls="dashed", color=color)
if verbose:
print("Quantiles:")
for item in zip(quantiles, qvalues):
print(item)
if show_titles:
title = None
if title_fmt is not None:
# Compute the quantiles for the title. This might redo
# unneeded computation but who cares.
q_16, q_50, q_84 = quantile(x, [0.16, 0.5, 0.84],
weights=weights)
q_m, q_p = q_50 - q_16, q_84 - q_50
# Format the quantile display.
fmt = "{{0:{0}}}".format(title_fmt).format
title = r"${{{0}}}_{{-{1}}}^{{+{2}}}$"
title = title.format(fmt(q_50), fmt(q_m), fmt(q_p))
# Add in the column name if it's given.
if labels is not None:
title = "{0} = {1}".format(labels[i], title)
elif labels is not None:
title = "{0}".format(labels[i])
if title is not None:
ax.set_title(title, **title_kwargs)
# Set up the axes.
ax.set_xlim(params_range[i])
if scale_hist:
maxn = np.max(n)
ax.set_ylim(-0.1 * maxn, 1.1 * maxn)
else:
ax.set_ylim(0, 1.1 * np.max(n))
ax.set_yticklabels([])
ax.xaxis.set_major_locator(MaxNLocator(max_n_ticks, prune="lower"))
if i < K - 1:
if top_ticks:
ax.xaxis.set_ticks_position("top")
[l.set_rotation(45) for l in ax.get_xticklabels()]
else:
ax.set_xticklabels([])
else:
[l.set_rotation(45) for l in ax.get_xticklabels()]
if labels is not None:
ax.set_xlabel(labels[i], **label_kwargs)
ax.xaxis.set_label_coords(0.5, -0.3)
# use MathText for axes ticks
ax.xaxis.set_major_formatter(
ScalarFormatter(useMathText=use_math_text))
for j, y in enumerate(xs):
if np.shape(xs)[0] == 1:
ax = axes
else:
ax = axes[i, j]
if j > i:
ax.set_frame_on(False)
ax.set_xticks([])
ax.set_yticks([])
continue
elif j == i:
continue
# Deal with masked arrays.
if hasattr(y, "compressed"):
y = y.compressed()
hist2d(y,
x,
ax=ax,
range=[params_range[j], params_range[i]],
weights=weights,
color=color,
smooth=smooth,
bins=[bins[j], bins[i]],
**hist2d_kwargs)
# Add an error ellipse based on the provided covariance matrix
if cov is not None:
# #center
# cx=truths[j]#need to add checking for availability of truths?
# cy=truths[i]
# #ang=math.acos(cov[0,1]/math.sqrt(cov[0,0]*cov[1,1]))*180/math.pi
# print (j,i,labels[j],labels[i],"center=",cx,cy)
# N_thetas=60
# dtheta=2.0*math.pi/(N_thetas-1)
# thetas=np.arange(0,(2.0*math.pi+dtheta),dtheta)
# Cplus=(cov[i,i]+cov[j,j])/2.0
# Cminus=(-cov[i,i]+cov[j,j])/2.0
# print("cov[ii],cov[ij],cov[jj],Cplus,Cminus:",cov[i,i],cov[i,j],cov[j,j],Cplus,Cminus)
# ang=-math.pi/4.
# root=cov[i,j]/math.sqrt(cov[i,i]*cov[j,j])
# acoeff=math.sqrt(1-root)
# bcoeff=math.sqrt(1+root)
# xcoeff=math.sqrt(cov[j,j])
# ycoeff=math.sqrt(cov[i,i])
# print("a2,b2",acoeff*acoeff,bcoeff*bcoeff)
# print("a,b,ang, xcoeff,ycoeff, root=",acoeff,bcoeff,ang,xcoeff,ycoeff,root)
# elxs=[cx+xcoeff*(acoeff*math.cos(th)*math.cos(ang)-bcoeff*math.sin(th)*math.sin(ang)) for th in thetas]
# elys=[cy+ycoeff*(acoeff*math.cos(th)*math.sin(ang)+bcoeff*math.sin(th)*math.cos(ang)) for th in thetas]
# #print (thetas)
# #print (elxs)
# #print (elys)
# ax.plot(elxs,elys,color='r')
#
#center
if truths is None:
raise ValueError(
'Error: truths is None but is required to plot Fisher ellipses.'
)
cx = truths[
j] # need to add checking for availability of truths?
cy = truths[i]
N_thetas = 60
dtheta = 2.0 * math.pi / (N_thetas - 1)
thetas = np.arange(0, (2.0 * math.pi + dtheta), dtheta)
ang = -math.pi / 4.
root = cov[i, j] / math.sqrt(cov[i, i] * cov[j, j])
if (root > 1): root = 1
if (root < -1): root = -1
acoeff = math.sqrt(1 - root)
bcoeff = math.sqrt(1 + root)
xcoeff = math.sqrt(cov[j, j])
ycoeff = math.sqrt(cov[i, i])
if "levels" in hist2d_kwargs:
levels = hist2d_kwargs["levels"]
else:
levels == 1.0 - np.exp(
-0.5 * np.arange(0.5, 2.1, 0.5)**2)
for xlev in levels:
# in the next line we convert the credibility limit
# to a "sigma" limit for a 2-d normal
# this becomes a scale-factor for the error ellipse
# 1-exp(x^2/(-2)=y
# -2*log(1-y)=x^2
lev_fac = math.sqrt(-2 * math.log(1 - xlev))
elxs = [
cx + lev_fac * xcoeff *
(acoeff * math.cos(th) * math.cos(ang) -
bcoeff * math.sin(th) * math.sin(ang))
for th in thetas
]
elys = [
cy + lev_fac * ycoeff *
(acoeff * math.cos(th) * math.sin(ang) +
bcoeff * math.sin(th) * math.cos(ang))
for th in thetas
]
if cov_color is None:
cov_color = 'k'
ax.plot(elxs, elys, color=cov_color)
if truths is not None:
if truths[i] is not None and truths[j] is not None:
ax.plot(truths[j], truths[i], "s", color=truth_color)
if truths[j] is not None:
ax.axvline(truths[j], color=truth_color)
if truths[i] is not None:
ax.axhline(truths[i], color=truth_color)
if add_truths is not None:
for add_truth, add_truth_color in zip(
add_truths, add_truth_colors):
print(add_truth)
if add_truth[i] is not None and add_truth[
j] is not None:
ax.plot(add_truth[j],
add_truth[i],
"s",
color=add_truth_color)
if add_truth[j] is not None:
ax.axvline(add_truth[j], color=add_truth_color)
if add_truth[i] is not None:
ax.axhline(add_truth[i], color=add_truth_color)
ax.xaxis.set_major_locator(
MaxNLocator(max_n_ticks, prune="lower"))
ax.yaxis.set_major_locator(
MaxNLocator(max_n_ticks, prune="lower"))
if i < K - 1:
ax.set_xticklabels([])
else:
[l.set_rotation(45) for l in ax.get_xticklabels()]
if labels is not None:
ax.set_xlabel(labels[j], **label_kwargs)
ax.xaxis.set_label_coords(0.5, -0.3)
# use MathText for axes ticks
ax.xaxis.set_major_formatter(
ScalarFormatter(useMathText=use_math_text))
if j > 0:
ax.set_yticklabels([])
else:
[l.set_rotation(45) for l in ax.get_yticklabels()]
if labels is not None:
ax.set_ylabel(labels[i], **label_kwargs)
ax.yaxis.set_label_coords(-0.3, 0.5)
# use MathText for axes ticks
ax.yaxis.set_major_formatter(
ScalarFormatter(useMathText=use_math_text))
# If not asking for the histograms to be plotted
else:
for i, x in enumerate(xs):
if i == 0:
continue
# Deal with masked arrays.
if hasattr(x, "compressed"):
x = x.compressed()
for j, y in enumerate(xs):
if j == K - 1:
continue
if np.shape(xs)[0] == 2:
ax = axes
else:
ax = axes[i - 1, j]
if j > i - 1:
ax.set_frame_on(False)
ax.set_xticks([])
ax.set_yticks([])
continue
elif j == i:
continue
# Deal with masked arrays.
if hasattr(y, "compressed"):
y = y.compressed()
hist2d(y,
x,
ax=ax,
range=[params_range[j], params_range[i]],
weights=weights,
color=color,
smooth=smooth,
bins=[bins[j], bins[i]],
**hist2d_kwargs)
# Add an error ellipse based on the provided covariance matrix
if cov is not None:
# #center
# cx=truths[j]#need to add checking for availability of truths?
# cy=truths[i]
# #ang=math.acos(cov[0,1]/math.sqrt(cov[0,0]*cov[1,1]))*180/math.pi
# print (j,i,labels[j],labels[i],"center=",cx,cy)
# N_thetas=60
# dtheta=2.0*math.pi/(N_thetas-1)
# thetas=np.arange(0,(2.0*math.pi+dtheta),dtheta)
# Cplus=(cov[i,i]+cov[j,j])/2.0
# Cminus=(-cov[i,i]+cov[j,j])/2.0
# print("cov[ii],cov[ij],cov[jj],Cplus,Cminus:",cov[i,i],cov[i,j],cov[j,j],Cplus,Cminus)
# ang=-math.pi/4.
# root=cov[i,j]/math.sqrt(cov[i,i]*cov[j,j])
# acoeff=math.sqrt(1-root)
# bcoeff=math.sqrt(1+root)
# xcoeff=math.sqrt(cov[j,j])
# ycoeff=math.sqrt(cov[i,i])
# print("a2,b2",acoeff*acoeff,bcoeff*bcoeff)
# print("a,b,ang, xcoeff,ycoeff, root=",acoeff,bcoeff,ang,xcoeff,ycoeff,root)
# elxs=[cx+xcoeff*(acoeff*math.cos(th)*math.cos(ang)-bcoeff*math.sin(th)*math.sin(ang)) for th in thetas]
# elys=[cy+ycoeff*(acoeff*math.cos(th)*math.sin(ang)+bcoeff*math.sin(th)*math.cos(ang)) for th in thetas]
# #print (thetas)
# #print (elxs)
# #print (elys)
# ax.plot(elxs,elys,color='r')
#
#center
cx = truths[
j] # need to add checking for availability of truths?
cy = truths[i]
N_thetas = 60
dtheta = 2.0 * math.pi / (N_thetas - 1)
thetas = np.arange(0, (2.0 * math.pi + dtheta), dtheta)
ang = -math.pi / 4.
root = cov[i, j] / math.sqrt(cov[i, i] * cov[j, j])
if (root > 1): root = 1
if (root < -1): root = -1
acoeff = math.sqrt(1 - root)
bcoeff = math.sqrt(1 + root)
xcoeff = math.sqrt(cov[j, j])
ycoeff = math.sqrt(cov[i, i])
if "levels" in hist2d_kwargs:
levels = hist2d_kwargs["levels"]
else:
levels == 1.0 - np.exp(
-0.5 * np.arange(0.5, 2.1, 0.5)**2)
for xlev in levels:
# in the next line we convert the credibility limit
# to a "sigma" limit for a 2-d normal
# this becomes a scale-factor for the error ellipse
# 1-exp(x^2/(-2)=y
# -2*log(1-y)=x^2
lev_fac = math.sqrt(-2 * math.log(1 - xlev))
elxs = [
cx + lev_fac * xcoeff *
(acoeff * math.cos(th) * math.cos(ang) -
bcoeff * math.sin(th) * math.sin(ang))
for th in thetas
]
elys = [
cy + lev_fac * ycoeff *
(acoeff * math.cos(th) * math.sin(ang) +
bcoeff * math.sin(th) * math.cos(ang))
for th in thetas
]
if cov_color is None:
cov_color = 'k'
ax.plot(elxs, elys, color=cov_color)
if truths is not None:
if truths[i] is not None and truths[j] is not None:
ax.plot(truths[j], truths[i], "s", color=truth_color)
if truths[j] is not None:
ax.axvline(truths[j], color=truth_color)
if truths[i] is not None:
ax.axhline(truths[i], color=truth_color)
if add_truths is not None:
for add_truth, add_truth_color in zip(
add_truths, add_truth_colors):
print(add_truth)
if add_truth[i] is not None and add_truth[
j] is not None:
ax.plot(add_truth[j],
add_truth[i],
"s",
color=add_truth_color)
if add_truth[j] is not None:
ax.axvline(add_truth[j], color=add_truth_color)
if add_truth[i] is not None:
ax.axhline(add_truth[i], color=add_truth_color)
ax.xaxis.set_major_locator(
MaxNLocator(max_n_ticks, prune="lower"))
ax.yaxis.set_major_locator(
MaxNLocator(max_n_ticks, prune="lower"))
if i < K - 1:
ax.set_xticklabels([])
else:
[l.set_rotation(45) for l in ax.get_xticklabels()]
if labels is not None:
ax.set_xlabel(labels[j], **label_kwargs)
ax.xaxis.set_label_coords(0.5, -0.3)
# use MathText for axes ticks
ax.xaxis.set_major_formatter(
ScalarFormatter(useMathText=use_math_text))
if j > 0:
ax.set_yticklabels([])
else:
[l.set_rotation(45) for l in ax.get_yticklabels()]
if labels is not None:
ax.set_ylabel(labels[i], **label_kwargs)
ax.yaxis.set_label_coords(-0.3, 0.5)
# use MathText for axes ticks
ax.yaxis.set_major_formatter(
ScalarFormatter(useMathText=use_math_text))
return fig, axes
def uindex(index):
"Convert an index (a tuple of path parts) to unicode for printing"
if index:
return os.path.join(*map(ufn, index))
else:
return u'.'
def is_namespace(name):
"Is the name either a valid campaign name or core?"
return name in map(os.path.basename, scopelist())
def get_categories(dbcache,
sort='name',
book_ids=None,
first_letter_sort=False):
if sort not in CATEGORY_SORTS:
raise ValueError('sort ' + sort + ' not a valid value')
fm = dbcache.field_metadata
book_rating_map = dbcache.fields['rating'].book_value_map
lang_map = dbcache.fields['languages'].book_value_map
categories = {}
book_ids = frozenset(book_ids) if book_ids else book_ids
pm_cache = {}
def get_metadata(book_id):
ans = pm_cache.get(book_id)
if ans is None:
ans = pm_cache[book_id] = dbcache._get_proxy_metadata(book_id)
return ans
bids = None
first_letter_sort = bool(first_letter_sort)
for category, is_multiple, is_composite in find_categories(fm):
tag_class = create_tag_class(category, fm)
sort_on, reverse = sort, False
if is_composite:
if bids is None:
bids = dbcache._all_book_ids(
) if book_ids is None else book_ids
cats = dbcache.fields[category].get_composite_categories(
tag_class, book_rating_map, bids, is_multiple, get_metadata)
elif category == 'news':
cats = dbcache.fields['tags'].get_news_category(
tag_class, book_ids)
else:
cat = fm[category]
brm = book_rating_map
dt = cat['datatype']
if dt == 'rating':
if category != 'rating':
brm = dbcache.fields[category].book_value_map
if sort_on == 'name':
sort_on, reverse = 'rating', True
cats = dbcache.fields[category].get_categories(
tag_class, brm, lang_map, book_ids)
if (category != 'authors' and dt == 'text' and cat['is_multiple']
and cat['display'].get('is_names', False)):
for item in cats:
item.sort = author_to_author_sort(item.sort)
cats.sort(key=category_sort_keys[first_letter_sort][sort_on],
reverse=reverse)
categories[category] = cats
# Needed for legacy databases that have multiple ratings that
# map to n stars
for r in categories['rating']:
for x in tuple(categories['rating']):
if r.name == x.name and r.id != x.id:
r.id_set |= x.id_set
r.count = len(r.id_set)
categories['rating'].remove(x)
break
# User categories
user_categories = clean_user_categories(dbcache).copy()
# First add any grouped search terms to the user categories
muc = dbcache.pref('grouped_search_make_user_categories', [])
gst = dbcache.pref('grouped_search_terms', {})
for c in gst:
if c not in muc:
continue
user_categories[c] = []
for sc in gst[c]:
for t in categories.get(sc, ()):
user_categories[c].append([t.name, sc, 0])
if user_categories:
# We want to use same node in the user category as in the source
# category. To do that, we need to find the original Tag node. There is
# a time/space tradeoff here. By converting the tags into a map, we can
# do the verification in the category loop much faster, at the cost of
# temporarily duplicating the categories lists.
taglist = {}
for c, items in categories.iteritems():
taglist[c] = dict(map(lambda t: (icu_lower(t.name), t), items))
# Add the category values to the user categories
for user_cat in sorted(user_categories.iterkeys(), key=sort_key):
items = []
names_seen = {}
user_cat_is_gst = user_cat in gst
for name, label, ign in user_categories[user_cat]:
n = icu_lower(name)
if label in taglist and n in taglist[label]:
if user_cat_is_gst:
# for gst items, make copy and consolidate the tags by name.
if n in names_seen:
# We must combine this node into a previous one with
# the same name ignoring case. As part of the process,
# remember the source categories and correct the
# average rating
t = names_seen[n]
other_tag = taglist[label][n]
t.id_set |= other_tag.id_set
t.count = len(t.id_set)
t.original_categories.add(other_tag.category)
total_rating = 0
count = 0
for id_ in t.id_set:
rating = book_rating_map.get(id_, 0)
if rating:
total_rating += rating / 2
count += 1
if total_rating and count:
t.avg_rating = total_rating / count
else:
# Must deepcopy so we don't share the id_set between nodes
t = copy.deepcopy(taglist[label][n])
t.original_categories = {t.category}
names_seen[n] = t
items.append(t)
else:
items.append(taglist[label][n])
# else: do nothing, to not include nodes w zero counts
cat_name = '@' + user_cat # add the '@' to avoid name collision
items.sort(key=category_sort_keys[False][sort])
categories[cat_name] = items
# ### Finally, the saved searches category ####
items = []
queries = dbcache._search_api.saved_searches.queries
for srch in sorted(queries, key=sort_key):
items.append(
Tag(srch,
sort=srch,
search_expression=queries[srch],
category='search',
is_editable=False))
if len(items):
categories['search'] = items
return categories
def ajax_books_in(self,
category,
item,
sort='title',
num=25,
offset=0,
sort_order='asc',
get_additional_fields=''):
'''
Return the books (as list of ids) present in the specified category.
'''
try:
dname, ditem = map(decode_name, (category, item))
except:
raise cherrypy.HTTPError(404,
'Invalid encoded param: %r' % category)
try:
num = int(num)
except:
raise cherrypy.HTTPError(404, "Invalid num: %r" % num)
try:
offset = int(offset)
except:
raise cherrypy.HTTPError(404, "Invalid offset: %r" % offset)
if sort_order not in ('asc', 'desc'):
sort_order = 'asc'
sfield = self.db.data.sanitize_sort_field_name(sort)
if sfield not in self.db.field_metadata.sortable_field_keys():
raise cherrypy.HTTPError(404,
'%s is not a valid sort field' % sort)
if dname in ('allbooks', 'newest'):
ids = self.search_cache('')
elif dname == 'search':
try:
ids = self.search_cache('search:"%s"' % ditem)
except:
raise cherrypy.HTTPError(404,
'Search: %r not understood' % ditem)
else:
try:
cid = int(ditem)
except:
raise cherrypy.HTTPError(
404, 'Category id %r not an integer' % ditem)
if dname == 'news':
dname = 'tags'
ids = self.db.get_books_for_category(dname, cid)
all_ids = set(self.search_cache(''))
# Implement restriction
ids = ids.intersection(all_ids)
ids = list(ids)
self.db.data.multisort(fields=[(sfield, sort_order == 'asc')],
subsort=True,
only_ids=ids)
total_num = len(ids)
ids = ids[offset:offset + num]
result = {
'total_num':
total_num,
'sort_order':
sort_order,
'offset':
offset,
'num':
len(ids),
'sort':
sort,
'base_url':
absurl(self.opts.url_prefix,
'/ajax/books_in/%s/%s' % (category, item)),
'book_ids':
ids
}
if get_additional_fields:
additional_fields = {}
for field in get_additional_fields.split(','):
field = field.strip()
if field:
flist = additional_fields[field] = []
for id_ in ids:
flist.append(
self.db.new_api.field_for(field,
id_,
default_value=None))
if additional_fields:
result['additional_fields'] = additional_fields
return result
from future_builtins import map __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal [email protected]' __docformat__ = 'restructuredtext en' __appname__ = u'calibre' numeric_version = (2, 58, 0) __version__ = u'.'.join(map(unicode, numeric_version)) __author__ = u"Kovid Goyal <*****@*****.**>" ''' Various run time constants. ''' import sys, locale, codecs, os, importlib, collections _plat = sys.platform.lower() iswindows = 'win32' in _plat or 'win64' in _plat isosx = 'darwin' in _plat isnewosx = isosx and getattr(sys, 'new_app_bundle', False) isfreebsd = 'freebsd' in _plat isnetbsd = 'netbsd' in _plat isdragonflybsd = 'dragonfly' in _plat isbsd = isfreebsd or isnetbsd or isdragonflybsd islinux = not(iswindows or isosx or isbsd) isfrozen = hasattr(sys, 'frozen') isunix = isosx or islinux isportable = os.environ.get('CALIBRE_PORTABLE_BUILD', None) is not None ispy3 = sys.version_info.major > 2 isxp = iswindows and sys.getwindowsversion().major < 6 is64bit = sys.maxsize > (1 << 32)
def mi():
return map(int, input().strip().split(" "))
def do_split(split_point, log, before=True):
'''
Split tree into a *before* and an *after* tree at ``split_point``.
:param split_point: The Element at which to split
:param before: If True tree is split before split_point, otherwise after split_point
:return: before_tree, after_tree
'''
if before:
# We cannot adjust for after since moving an after split point to a
# parent will cause breakage if the parent contains any content
# after the original split point
split_point = adjust_split_point(split_point, log)
tree = split_point.getroottree()
path = tree.getpath(split_point)
tree, tree2 = copy.deepcopy(tree), copy.deepcopy(tree)
root, root2 = tree.getroot(), tree2.getroot()
body, body2 = map(get_body, (root, root2))
split_point = root.xpath(path)[0]
split_point2 = root2.xpath(path)[0]
def nix_element(elem, top=True):
# Remove elem unless top is False in which case replace elem by its
# children
parent = elem.getparent()
if top:
parent.remove(elem)
else:
index = parent.index(elem)
parent[index:index + 1] = list(elem.iterchildren())
# Tree 1
hit_split_point = False
keep_descendants = False
split_point_descendants = frozenset(split_point.iterdescendants())
for elem in tuple(body.iterdescendants()):
if elem is split_point:
hit_split_point = True
if before:
nix_element(elem)
else:
# We want to keep the descendants of the split point in
# Tree 1
keep_descendants = True
# We want the split point element, but not its tail
elem.tail = '\n'
continue
if hit_split_point:
if keep_descendants:
if elem in split_point_descendants:
# elem is a descendant keep it
continue
else:
# We are out of split_point, so prevent further set
# lookups of split_point_descendants
keep_descendants = False
nix_element(elem)
# Tree 2
ancestors = frozenset(XPath('ancestor::*')(split_point2))
for elem in tuple(body2.iterdescendants()):
if elem is split_point2:
if not before:
# Keep the split point element's tail, if it contains non-whitespace
# text
tail = elem.tail
if tail and not tail.isspace():
parent = elem.getparent()
idx = parent.index(elem)
if idx == 0:
parent.text = (parent.text or '') + tail
else:
sib = parent[idx - 1]
sib.tail = (sib.tail or '') + tail
# Remove the element itself
nix_element(elem)
break
if elem in ancestors:
# We have to preserve the ancestors as they could have CSS
# styles that are inherited/applicable, like font or
# width. So we only remove the text, if any.
elem.text = '\n'
else:
nix_element(elem, top=False)
body2.text = '\n'
return tree, tree2
def chars(lines):
return ["".join(map(operator.attrgetter("data"), line)) for line in lines]
def msi():
return map(str, input().strip().split(" "))
def virtualize_resources(self):
changed = set()
link_uid = self.book_render_data['link_uid']
resource_template = link_uid + '|{}|'
xlink_xpath = XPath('//*[@xl:href]')
link_xpath = XPath('//h:a[@href]')
res_link_xpath = XPath('//h:link[@href]')
def link_replacer(base, url):
if url.startswith('#'):
frag = urlunquote(url[1:])
if not frag:
return url
changed.add(base)
return resource_template.format(encode_url(base, frag))
purl = urlparse(url)
if purl.netloc or purl.query:
return url
if purl.scheme and purl.scheme != 'file':
return url
if not purl.path or purl.path.startswith('/'):
return url
url, frag = purl.path, purl.fragment
name = self.href_to_name(url, base)
if name:
if self.has_name(name):
frag = urlunquote(frag)
url = resource_template.format(encode_url(name, frag))
else:
if isinstance(name, unicode):
name = name.encode('utf-8')
url = 'missing:' + force_unicode(quote(name), 'utf-8')
changed.add(base)
return url
ltm = self.book_render_data['link_to_map']
for name, mt in self.mime_map.iteritems():
mt = mt.lower()
if mt in OEB_STYLES:
replaceUrls(self.parsed(name), partial(link_replacer, name))
self.virtualized_names.add(name)
elif mt in OEB_DOCS:
self.virtualized_names.add(name)
root = self.parsed(name)
for link in res_link_xpath(root):
ltype = (link.get('type') or 'text/css').lower()
rel = (link.get('rel') or 'stylesheet').lower()
if ltype != 'text/css' or rel != 'stylesheet':
# This link will not be loaded by the browser anyway
# and will causes the resource load check to hang
link.attrib.clear()
changed.add(name)
rewrite_links(root, partial(link_replacer, name))
for a in link_xpath(root):
href = a.get('href')
if href.startswith(link_uid):
a.set('href', 'javascript:void(0)')
parts = decode_url(href.split('|')[1])
lname, lfrag = parts[0], parts[1]
ltm.setdefault(lname,
{}).setdefault(lfrag or '',
set()).add(name)
a.set(
'data-' + link_uid,
json.dumps({
'name': lname,
'frag': lfrag
},
ensure_ascii=False))
else:
a.set('target', '_blank')
a.set('rel', 'noopener noreferrer')
changed.add(name)
elif mt == 'image/svg+xml':
self.virtualized_names.add(name)
changed.add(name)
xlink = XLINK('href')
for elem in xlink_xpath(self.parsed(name)):
elem.set(xlink, link_replacer(name, elem.get(xlink)))
for name, amap in ltm.iteritems():
for k, v in tuple(amap.iteritems()):
amap[k] = tuple(v) # needed for JSON serialization
tuple(map(self.dirty, changed))
def utc_trns( loctime ):
terms = loctime.translate( trtab ).split()
if len( terms ) == 7:
# convert .123 into 123000 microseconds
terms[6] += '0' * ( 6 - len( terms[6] ))
return datetime.datetime( *map( int, terms ), tzinfo=pytz.utc )
def _load_index(self):
'Load the index, automatically removing incorrectly sized thumbnails and pruning to fit max_size'
try:
os.makedirs(self.location)
except OSError as err:
if err.errno != errno.EEXIST:
self.log('Failed to make thumbnail cache dir:',
as_unicode(err))
self.total_size = 0
self.items = OrderedDict()
order = self._read_order()
def listdir(*args):
try:
return os.listdir(os.path.join(*args))
except EnvironmentError:
return () # not a directory or no permission or whatever
entries = ('/'.join((parent, subdir, entry))
for parent in listdir(self.location)
for subdir in listdir(self.location, parent)
for entry in listdir(self.location, parent, subdir))
invalidate = set()
try:
with open(os.path.join(self.location, 'invalidate'), 'rb') as f:
raw = f.read()
except EnvironmentError as err:
if getattr(err, 'errno', None) != errno.ENOENT:
self.log('Failed to read thumbnail invalidate data:',
as_unicode(err))
else:
try:
os.remove(os.path.join(self.location, 'invalidate'))
except EnvironmentError as err:
self.log('Failed to remove thumbnail invalidate data:',
as_unicode(err))
else:
def record(line):
try:
uuid, book_id = line.partition(' ')[0::2]
book_id = int(book_id)
return (uuid, book_id)
except Exception:
return None
invalidate = {record(x) for x in raw.splitlines()}
items = []
try:
for entry in entries:
try:
uuid, name = entry.split('/')[0::2]
book_id, timestamp, size, thumbnail_size = name.split('-')
book_id, timestamp, size = int(book_id), float(
timestamp), int(size)
thumbnail_size = tuple(
map(int,
thumbnail_size.partition('x')[0::2]))
except (ValueError, TypeError, IndexError, KeyError,
AttributeError):
continue
key = (uuid, book_id)
path = os.path.join(self.location, entry)
if self.thumbnail_size == thumbnail_size and key not in invalidate:
items.append(
(key, Entry(path, size, timestamp, thumbnail_size)))
self.total_size += size
else:
self._do_delete(path)
except EnvironmentError as err:
self.log('Failed to read thumbnail cache dir:', as_unicode(err))
self.items = OrderedDict(
sorted(items, key=lambda x: order.get(hash(x[0]), 0)))
self._apply_size()
def test_history_timestamp_abbreviations():
"""Test timezone abbreviation support. """
abbrev = timestamp.support_abbreviations( 'CA', reset=True )
assert sorted( abbrev ) == ['ADT', 'AST', 'CDT', 'CST', 'EDT', 'EST', 'MDT', 'MST', 'NDT', 'NST', 'PDT', 'PST']
# Perform all the remaining timezone abbreviation tests relative to a known range of times, to
# avoid differences in the future due to timezone changes.
ts = timestamp( "2014-04-24 08:00:00 MDT" )
assert near( ts.value, 1398348000.0 )
# Try to add all of the Americas to the CA abbreviations already supported; can't be done (too
# many inconsistencies)
try:
abbrev = timestamp.support_abbreviations( 'America' )
assert False, "Many zones should have been ambiguously abbreviated"
except AmbiguousTimeZoneError as exc:
assert "America/Mazatlan" in str( exc )
exclude = [
'America/Mazatlan', 'America/Merida', 'America/Mexico_City', 'America/Monterrey',
'America/Bahia_Banderas', 'America/Cancun', 'America/Chihuahua', 'America/Havana',
'America/Santa_Isabel', 'America/Grand_Turk', 'America/Cayman', 'America/Port-au-Prince',
'America/Metlakatla',
]
#print()
#print( "America, w/o %r" % ( exclude ))
abbrev = timestamp.support_abbreviations( 'America', exclude=exclude )
#print( sorted( abbrev ))
#print( reprlib.repr( timestamp._tzabbrev ))
pytz_version = tuple( map( int, pytz.__version__.split( '.' )))
if pytz_version < (2015,4):
logging.warning( "pytz < 2015.4; HADT/HAST vs. HDT/HST" )
assert sorted( abbrev ) == ['ACT', 'AKDT', 'AKST', 'AMST', 'AMT', 'ART', 'BOT', 'BRST', 'BRT', 'CLST', 'CLT',
'COT', 'ECT', 'EGST', 'EGT', 'FNT', 'GFT', 'GMT', 'GYT', 'HADT', 'HAST',
'PET', 'PMDT', 'PMST', 'PYST', 'PYT', 'SRT', 'UYST', 'UYT', 'VET', 'WGST', 'WGT']
elif pytz_version < (2015,7):
logging.warning( "pytz < 2015.7; had UYST" )
assert sorted( abbrev ) == ['ACT', 'AKDT', 'AKST', 'AMST', 'AMT', 'ART', 'BOT', 'BRST', 'BRT', 'CLST', 'CLT',
'COT', 'ECT', 'EGST', 'EGT', 'FNT', 'GFT', 'GMT', 'GYT', 'HDT', 'HST',
'PET', 'PMDT', 'PMST', 'PYST', 'PYT', 'SRT', 'UYST', 'UYT', 'VET', 'WGST', 'WGT']
elif pytz_version < (2017,2):
assert sorted( abbrev ) == ['ACT', 'AKDT', 'AKST', 'AMST', 'AMT', 'ART', 'BOT', 'BRST', 'BRT', 'CLST', 'CLT',
'COT', 'ECT', 'EGST', 'EGT', 'FNT', 'GFT', 'GMT', 'GYT', 'HDT', 'HST',
'PET', 'PMDT', 'PMST', 'PYST', 'PYT', 'SRT', 'UYT', 'VET', 'WGST', 'WGT']
else:
# As of pytz 2017.2, alot of these zones are now using time zones consistent with CA; only a few added.
assert sorted( abbrev ) == ['AKDT', 'AKST', 'GMT', 'HDT', 'HST']
# We *can* add Europe/Berlin
abbrev = timestamp.support_abbreviations( 'Europe/Berlin' )
assert sorted( abbrev ) == ['CEST', 'CET']
assert 'CEST' in timestamp._tzabbrev
assert 'EEST' not in timestamp._tzabbrev
# And all of Europe, w/o some troublesome time zones
exclude = [ 'Europe/Simferopol', 'Europe/Istanbul', 'Europe/Minsk', 'Europe/Chisinau', 'Europe/Dublin' ]
#print()
#print( "Europe, w/o %r" % ( exclude ))
abbrev = timestamp.support_abbreviations( 'Europe', exclude=exclude )
#print( sorted( abbrev ))
if pytz_version < (2015,2):
assert sorted( abbrev ) == ['BST', 'EEST', 'EET', 'MSK', 'SAMT', 'WEST', 'WET']
elif pytz_version < (2016,3):
assert sorted( abbrev ) == ['BST', 'EEST', 'EET', 'IST', 'MSK', 'SAMT', 'WEST', 'WET']
elif pytz_version < (2016,7):
assert sorted( abbrev ) == ['BST', 'EEST', 'EET', 'IST', 'MSK', 'SAMT', 'WEST', 'WET']
elif pytz_version < (2018,5):
assert sorted( abbrev ) == ['BST', 'EEST', 'EET', 'IST', 'MSK', 'WEST', 'WET']
else:
assert sorted( abbrev ) == ['BST', 'EEST', 'EET', 'MSK', 'WEST', 'WET']
assert 'EEST' in timestamp._tzabbrev
try:
timestamp.support_abbreviations( 'Asia' )
assert False, "Asia/Jerusalem IST should have mismatched Europe/Dublin IST"
except AmbiguousTimeZoneError as exc:
assert "Asia/Jerusalem" in str( exc )
assert near( parse_offset( '< 1:00:00.001' ), -3600.001 )
assert near( parse_offset( '<:1.001' ), -1.001 )
assert near( parse_offset( '>1:0.001' ), 60.001 )
assert near( parse_offset( '>1' ), 1 )
# While Asia is internally very inconsistent (eg. EEST), countries should be internally consisent
abbrev = timestamp.support_abbreviations( 'JO', reset=True ) # Jordan
#print( sorted( abbrev ))
assert sorted( abbrev ) == [ 'EEST', 'EET']
z,dst,off = timestamp._tzabbrev['EEST']
assert str(z) == 'Asia/Amman' and dst == True and format_offset( timedelta_total_seconds( off ), ms=None ) == "> 3:00:00"
abbrev = timestamp.support_abbreviations( 'IE', reset=True ) # Israel
#print( sorted( abbrev ))
assert sorted( abbrev ) == [ 'GMT', 'IST' ]
# Jordan, Israel and Lebanon only work if we pick a region to exclude, for one EEST definition
abbrev = timestamp.support_abbreviations( ['JO', 'IE', 'LB'],
exclude=[ 'Asia/Amman' ], reset=True )
#print( sorted( abbrev ))
assert sorted( abbrev ) == [ 'EEST', 'EET', 'GMT', 'IST' ]
z,dst,off = timestamp._tzabbrev['EEST']
assert str(z) == 'Asia/Beirut' and dst == True and format_offset( timedelta_total_seconds( off ), ms=None ) == "> 3:00:00"
# Australia zones incompatible with a bunch of other timezone abbreviations, eg. CST; reset
abbrev = timestamp.support_abbreviations( 'Australia', reset=True )
#print( sorted( abbrev ))
#print( repr( timestamp._tzabbrev ))
if pytz_version < (2017,2):
assert sorted( abbrev ) == ['ACDT', 'ACST', 'ACWST', 'AEDT', 'AEST', 'AWST', 'LHDT', 'LHST']
z,dst,off = timestamp._tzabbrev['LHST']
assert str(z) == 'Australia/Lord_Howe' and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == ">10:30:00"
else:
assert sorted( abbrev ) == ['ACDT', 'ACST', 'AEDT', 'AEST', 'AWST']
# Ensure that non-ambiguous (DST-specific) zone abbreviations override ambiguous (no longer
# relevant, as pytz >= 2014.7 no longer contains dst == None for some of the Australian zones
# without DST)
abbrev = timestamp.support_abbreviations( [ 'Australia/Adelaide' ], reset=True )
assert sorted( abbrev ) == [ 'ACDT', 'ACST' ]
z,dst,off = timestamp._tzabbrev['ACST']
assert str(z) == 'Australia/Adelaide' and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == "> 9:30:00"
abbrev = timestamp.support_abbreviations( [ 'Australia/Adelaide', 'Australia/Darwin' ], reset=True )
#print( sorted( abbrev ))
#print( repr( timestamp._tzabbrev ))
z,dst,off = timestamp._tzabbrev['ACST']
assert str(z) in ( 'Australia/Darwin',
'Australia/Adelaide' ) and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == "> 9:30:00"
# Check that zones with complete, permanent offset changes (not just DST) are handled. We know
# that within a year of 2014-04-28, the America/Eirunepe (west Amazonas) zone had such a change
# (pre pytz 2017.2, anyway...)
if pytz_version < (2017,2):
abbrev = timestamp.support_abbreviations( [ 'America/Eirunepe' ], at=datetime.datetime( 2014, 4, 28 ), reset=True)
#print( sorted( abbrev ))
assert sorted( abbrev ) == [ 'ACT', 'AMT' ]
z,dst,off = timestamp._tzabbrev['ACT']
assert str(z) == 'America/Eirunepe' and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == "< 5:00:00"
z,dst,off = timestamp._tzabbrev['AMT']
assert str(z) == 'America/Eirunepe' and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == "< 4:00:00"
def main( argv=None ):
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog = """\
If an address starting with '/' is provided (eg. --address /dev/ttyS1), then
a Modbus/RTU serial framer is assumed. Otherwise, a Modbus/TCP framer is used.
The --evil option takes the following :
truncate -- return only part of the response
delay[:#.#[-#.#]]-- delay response by #.#[-#.#] seconds (default == 5)
corrupt[:<shat>] -- corrupt Modbus/TCP protocol response (default == "transaction")
:transaction -- Transaction ID (only relevant to Modbus/TCP servers)
:protocol -- Protocol ID
:unit -- Unit number
:function -- Function code
:registers -- Amount of response data
Register range(s) and value(s) must be supplied:
<begin>[-<end>][=<val>[,<val>]]
EXAMPLE
modbus_sim.py --address localhost:7502 --evil delay:2.5 40001-40100
Starts a simulated Modbus/TCP PLC serving Holding registers 40001-40100
== 0, on port 7502 on interface 'localhost', which delays all responses
for 2.5 seconds.
modbus_sim.py --address /dev/ttyS0 --evil delay:.01-.1 40001-40100
Starts a simulated Modbus/RTU PLC serving Holding registers 40001-40100
== 0, on serial port /dev/ttyS0, which delays all responses for between
.01-.1 seconds.
""" )
parser.add_argument( '-v', '--verbose',
default=1, action="count", help="Display logging information." )
parser.add_argument('-l', '--log',
type=str, default=None, help="Direct log output to the specified file" )
parser.add_argument( '-a', '--address', default="0.0.0.0:502",
help="Default [interface][:port] to bind to (default: any, port 502)" )
parser.add_argument( '-r', '--range', default=1,
help="Number of ports to try, if busy (default: 1)" )
parser.add_argument( '-e', '--evil', default=None,
help="Evil Modbus/TCP protocol framer (default: None)" )
parser.add_argument( '-c', '--config', default=None,
help="""JSON config data for Modbus framer (eg. {"baudrate":19200}) (default: None)""" )
parser.add_argument( 'registers', nargs="+" )
args = parser.parse_args( argv )
# Deduce logging level and target file (if any)
levelmap = {
0: logging.CRITICAL,
1: logging.ERROR,
2: logging.WARNING,
3: logging.INFO,
4: logging.DEBUG,
}
cpppo.log_cfg['level'] = ( levelmap[args.verbose]
if args.verbose in levelmap
else logging.DEBUG )
if args.log:
cpppo.log_cfg['filename']= args.log # log rotation not supported
logging.basicConfig( **cpppo.log_cfg )
#---------------------------------------------------------------------------#
# run the server you want
#---------------------------------------------------------------------------#
# Deduce interface:port to bind, and correct types. Interface defaults to
# '' (INADDR_ANY) if only :port is supplied. Port defaults to 502 if only
# interface is supplied. After this block, 'address' is always a tuple like
# ("interface",502). If '/', then start a Modbus/RTU serial server,
# otherwise a Modbus/TCP network server. Create an address_sequence
# yielding all the relevant target addresses we might need to try.
# We must initialize 'framer' here (even if its the same as the 'starter'
# default), because we may make an Evil...() derived class below...
starter_kwds = {}
if args.address.startswith( '/' ):
starter = StartRtuServerLogging
framer = modbus_rtu_framer_collecting
try:
import serial
except ImportError:
logging.error( "Modbus/RTU not supported; ensure PySerial is available" )
raise
starter_kwds = {
# Default serial configs; may be overridden, eg:
# --config '{"baudrate":19200, "slaves":[1,2,3]}'
'stopbits': 1,
'bytesize': 8,
'parity': serial.PARITY_NONE,
'baudrate': 4800,
'timeout': 0.5,
'slaves': None,
'ignore_missing_slaves': True,
}
address_sequence = [ args.address ]
assert args.range == 1, \
"A range of serial ports is unsupported"
else:
starter = StartTcpServerLogging
framer = ModbusSocketFramer
address = args.address.split(':')
assert 1 <= len( address ) <= 2
address = (
str( address[0] ),
int( address[1] ) if len( address ) > 1 else Defaults.Port )
log.info( "--server '%s' produces address=%r", args.address, address )
address_sequence = (
(address[0],port)
for port in range( address[1], address[1] + int( args.range ))
)
#---------------------------------------------------------------------------#
# Evil Framers, manipulate packets resulting from underlying Framers
#---------------------------------------------------------------------------#
if args.evil == "truncate":
class EvilFramerTruncateResponse( framer ):
def buildPacket(self, message):
''' Creates a *truncated* ready to send modbus packet. Truncates from 1
to all of the bytes, before returning response.
:param message: The populated request/response to send
'''
packet = super( EvilFramerTruncateResponse, self ).buildPacket( message )
datalen = len( packet )
corrlen = datalen - random.randint( 1, datalen )
log.info( "Corrupting response; truncating from %d to %d bytes", datalen, corrlen )
return packet[:corrlen]
framer = EvilFramerTruncateResponse
log.info( "--evil '%s' uses EvilFramerTruncateResponse", args.evil )
elif args.evil and args.evil.startswith( 'delay' ):
class EvilFramerDelayResponse( framer ):
delay = 5
def buildPacket(self, message):
''' Creates a ready to send modbus packet but delays the return.
:param message: The populated request/response to send
'''
packet = super( EvilFramerDelayResponse, self ).buildPacket( message )
log.info( "Delaying response for %s seconds", self.delay )
delay = self.delay
if isinstance( delay, (list,tuple) ):
delay = random.uniform( *delay )
time.sleep( delay )
return packet
framer = EvilFramerDelayResponse
# If a "--evil delay:1.5" is provided, pull out the number and change
# the ModbusSockerFramerDelayResponse class' .delay value to the specified
# number of seconds
req = args.evil.split( ':', 1 )
assert 1 <= len( req ) <= 2
if len( req ) == 2:
# Specified delay value or range
delay = tuple( map( float, req[1].split( '-' )))
assert 1 <= len( delay ) <= 2
EvilFramerDelayResponse.delay = delay if len( delay ) > 1 else delay[0]
log.info( "--evil '%s' uses EvilFramerDelayResponse, which delays responses for %s seconds",
args.evil, EvilFramerDelayResponse.delay )
elif args.evil and args.evil.startswith( 'corrupt' ):
class EvilFramerCorruptResponse( framer ):
what = "transaction"
def buildPacket(self, message):
''' Creates a *corrupted* ready to send modbus packet. Truncates from 1
to all of the bytes, before returning response.
:param message: The populated request/response to send
WARNING: pymodbus seems to swallow any exceptions thrown by these
methods. This seems like a defect; it should log them, at least.
'''
try:
log.info("Encoding package")
message.encode()
if self.what == "transaction":
message.transaction_id ^= 0xFFFF
packet = super( EvilFramerCorruptResponse, self ).buildPacket( message )
message.transaction_id ^= 0xFFFF
elif self.what == "registers":
if isinstance( message, ReadRegistersResponseBase ):
# These have '.registers' attribute, which is a list.
# Add/remove some
saveregs = message.registers
if len( message.registers ) == 0 or random.randint( 0, 1 ):
message.registers += [999]
else:
message.registers = message.registers[:-1]
packet = super( EvilFramerCorruptResponse, self ).buildPacket( message )
message.registers = saveregs
elif isinstance( message, WriteSingleRegisterResponse ):
# Flip the responses address bits and then flip them back.
message.address ^= 0xFFFF
packet = super( EvilFramerCorruptResponse, self ).buildPacket( message )
message.address ^= 0xFFFF
elif isinstance( message, WriteMultipleRegistersResponse ):
# Flip the responses address bits and then flip them back.
message.address ^= 0xFFFF
packet = super( EvilFramerCorruptResponse, self ).buildPacket( message )
message.address ^= 0xFFFF
else:
raise NotImplementedException(
"Unhandled class for register corruption; not implemented" )
elif self.what == "protocol":
message.protocol_id ^= 0xFFFF
packet = super( EvilFramerCorruptResponse, self ).buildPacket( message )
message.protocol_id ^= 0xFFFF
elif self.what == "unit":
message.unit_id ^= 0xFF
packet = super( EvilFramerCorruptResponse, self ).buildPacket( message )
message.unit_id ^= 0xFF
elif self.what == "function":
message.function_code ^= 0xFF
packet = super( EvilFramerCorruptResponse, self ).buildPacket( message )
message.function_code ^= 0xFF
else:
raise NotImplementedException(
"Unknown corruption specified; not implemented" )
except Exception:
log.info( "Could not build corrupt packet: %s", traceback.format_exc() )
return packet
framer = EvilFramerCorruptResponse
# If a "--evil corrupt:<what>" is provided, corrupt the specified entry.
req = args.evil.split(":", 1)
assert 1 <= len( req ) <= 2
if len( req ) == 2:
EvilFramerCorruptResponse.what = req[1]
log.info( "--evil '%s' uses EvilFramerCorruptResponse, which corrupts the responses %s entry",
args.evil, EvilFramerCorruptResponse.what )
elif args.evil:
log.error( "Unrecognized --evil argument: %s", args.evil )
return 1
if args.config:
try:
starter_kwds.update( **json.loads( args.config ))
except Exception as exc:
log.error( "Failed to parse JSON --config Modbus Framer config: %s; %s", args.config, exc )
raise
#---------------------------------------------------------------------------#
# Start the PLC simulation engine on a port in the range; will serve forever
#---------------------------------------------------------------------------#
for address in address_sequence:
try:
for k in sorted( starter_kwds.keys() ):
log.info( "config: %24s: %s", k, starter_kwds[k] )
starter( registers=args.registers, framer=framer, address=address, **starter_kwds )
except KeyboardInterrupt:
return 1
except Exception:
log.info( "Couldn't start PLC simulator on %s: %s",
address, traceback.format_exc() )
log.error( "Failed to start PLC simulator on %s, over a range of %s ports",
args.address, args.range )
return 1
d['tweak_fmt_order'] = ['EPUB', 'AZW3']
d['update_metadata_from_calibre'] = True
d['nestable_dock_widgets'] = False
d['dock_top_left'] = 'horizontal'
d['dock_top_right'] = 'horizontal'
d['dock_bottom_left'] = 'horizontal'
d['dock_bottom_right'] = 'horizontal'
d['preview_serif_family'] = 'Liberation Serif'
d['preview_sans_family'] = 'Liberation Sans'
d['preview_mono_family'] = 'Liberation Mono'
d['preview_standard_font_family'] = 'serif'
d['preview_base_font_size'] = 18
d['preview_mono_font_size'] = 14
d['preview_minimum_font_size'] = 8
d['remove_existing_links_when_linking_sheets'] = True
d['charmap_favorites'] = list(map(ord, '\xa0\u2002\u2003\u2009\xad' '‘’“”‹›«»‚„' '—–§¶†‡©®™' '→⇒•·°±−×÷¼½½¾' '…µ¢£€¿¡¨´¸ˆ˜' 'ÀÁÂÃÄÅÆÇÈÉÊË' 'ÌÍÎÏÐÑÒÓÔÕÖØ' 'ŒŠÙÚÛÜÝŸÞßàá' 'âãäåæçèéêëìí' 'îïðñòóôõöøœš' 'ùúûüýÿþªºαΩ∞')) # noqa
d['folders_for_types'] = {'style':'styles', 'image':'images', 'font':'fonts', 'audio':'audio', 'video':'video'}
d['pretty_print_on_open'] = False
d['disable_completion_popup_for_search'] = False
d['saved_searches'] = []
d['insert_tag_mru'] = ['p', 'div', 'li', 'h1', 'h2', 'h3', 'h4', 'em', 'strong', 'td', 'tr']
d['spell_check_case_sensitive_sort'] = False
d['inline_spell_check'] = True
d['custom_themes'] = {}
d['remove_unused_classes'] = False
d['global_book_toolbar'] = [
'new-file', 'open-book', 'save-book', None, 'global-undo', 'global-redo', 'create-checkpoint', None, 'donate', 'user-manual']
d['global_tools_toolbar'] = [
'check-book', 'spell-check-book', 'edit-toc', 'insert-character',
'manage-fonts', 'smarten-punctuation', 'remove-unused-css', 'show-reports'
]
def parse_graminit_c(self, filename):
"""Parse the .c file written by pgen. (Internal)
The file looks as follows. The first two lines are always this:
#include "pgenheaders.h"
#include "grammar.h"
After that come four blocks:
1) one or more state definitions
2) a table defining dfas
3) a table defining labels
4) a struct defining the grammar
A state definition has the following form:
- one or more arc arrays, each of the form:
static arc arcs_<n>_<m>[<k>] = {
{<i>, <j>},
...
};
- followed by a state array, of the form:
static state states_<s>[<t>] = {
{<k>, arcs_<n>_<m>},
...
};
"""
try:
f = open(filename)
except IOError as err:
print("Can't open %s: %s" % (filename, err))
return False
# The code below essentially uses f's iterator-ness!
lineno = 0
# Expect the two #include lines
lineno, line = lineno + 1, next(f)
assert line == '#include "pgenheaders.h"\n', (lineno, line)
lineno, line = lineno + 1, next(f)
assert line == '#include "grammar.h"\n', (lineno, line)
# Parse the state definitions
lineno, line = lineno + 1, next(f)
allarcs = {}
states = []
while line.startswith("static arc "):
while line.startswith("static arc "):
mo = re.match(r"static arc arcs_(\d+)_(\d+)\[(\d+)\] = {$",
line)
assert mo, (lineno, line)
n, m, k = map(int, mo.groups())
arcs = []
for _ in range(k):
lineno, line = lineno + 1, next(f)
mo = re.match(r"\s+{(\d+), (\d+)},$", line)
assert mo, (lineno, line)
i, j = map(int, mo.groups())
arcs.append((i, j))
lineno, line = lineno + 1, next(f)
assert line == "};\n", (lineno, line)
allarcs[(n, m)] = arcs
lineno, line = lineno + 1, next(f)
mo = re.match(r"static state states_(\d+)\[(\d+)\] = {$", line)
assert mo, (lineno, line)
s, t = map(int, mo.groups())
assert s == len(states), (lineno, line)
state = []
for _ in range(t):
lineno, line = lineno + 1, next(f)
mo = re.match(r"\s+{(\d+), arcs_(\d+)_(\d+)},$", line)
assert mo, (lineno, line)
k, n, m = map(int, mo.groups())
arcs = allarcs[n, m]
assert k == len(arcs), (lineno, line)
state.append(arcs)
states.append(state)
lineno, line = lineno + 1, next(f)
assert line == "};\n", (lineno, line)
lineno, line = lineno + 1, next(f)
self.states = states
# Parse the dfas
dfas = {}
mo = re.match(r"static dfa dfas\[(\d+)\] = {$", line)
assert mo, (lineno, line)
ndfas = int(mo.group(1))
for i in range(ndfas):
lineno, line = lineno + 1, next(f)
mo = re.match(r'\s+{(\d+), "(\w+)", (\d+), (\d+), states_(\d+),$',
line)
assert mo, (lineno, line)
symbol = mo.group(2)
number, x, y, z = map(int, mo.group(1, 3, 4, 5))
assert self.symbol2number[symbol] == number, (lineno, line)
assert self.number2symbol[number] == symbol, (lineno, line)
assert x == 0, (lineno, line)
state = states[z]
assert y == len(state), (lineno, line)
lineno, line = lineno + 1, next(f)
mo = re.match(r'\s+("(?:\\\d\d\d)*")},$', line)
assert mo, (lineno, line)
first = {}
rawbitset = eval(mo.group(1))
for i, c in enumerate(rawbitset):
byte = ord(c)
for j in range(8):
if byte & (1 << j):
first[i * 8 + j] = 1
dfas[number] = (state, first)
lineno, line = lineno + 1, next(f)
assert line == "};\n", (lineno, line)
self.dfas = dfas
# Parse the labels
labels = []
lineno, line = lineno + 1, next(f)
mo = re.match(r"static label labels\[(\d+)\] = {$", line)
assert mo, (lineno, line)
nlabels = int(mo.group(1))
for i in range(nlabels):
lineno, line = lineno + 1, next(f)
mo = re.match(r'\s+{(\d+), (0|"\w+")},$', line)
assert mo, (lineno, line)
x, y = mo.groups()
x = int(x)
if y == "0":
y = None
else:
y = eval(y)
labels.append((x, y))
lineno, line = lineno + 1, next(f)
assert line == "};\n", (lineno, line)
self.labels = labels
# Parse the grammar struct
lineno, line = lineno + 1, next(f)
assert line == "grammar _PyParser_Grammar = {\n", (lineno, line)
lineno, line = lineno + 1, next(f)
mo = re.match(r"\s+(\d+),$", line)
assert mo, (lineno, line)
ndfas = int(mo.group(1))
assert ndfas == len(self.dfas)
lineno, line = lineno + 1, next(f)
assert line == "\tdfas,\n", (lineno, line)
lineno, line = lineno + 1, next(f)
mo = re.match(r"\s+{(\d+), labels},$", line)
assert mo, (lineno, line)
nlabels = int(mo.group(1))
assert nlabels == len(self.labels), (lineno, line)
lineno, line = lineno + 1, next(f)
mo = re.match(r"\s+(\d+)$", line)
assert mo, (lineno, line)
start = int(mo.group(1))
assert start in self.number2symbol, (lineno, line)
self.start = start
lineno, line = lineno + 1, next(f)
assert line == "};\n", (lineno, line)
try:
lineno, line = lineno + 1, next(f)
except StopIteration:
pass
else:
assert 0, (lineno, line)
def getter(w):
return list(
map(unicode, (w.item(i).text() for i in xrange(w.count()))))
