def get_older_than_required(self, t):
"""
Returns list of old backup sets required by new sets
This function is similar to the previous one, but it only
returns the times of sets which are old but part of the chains
where the newer end of the chain is newer than t.
"""
assert self.values_set
new_chains = filter(lambda c: c.end_time >= t, self.all_backup_chains)
result_sets = []
for chain in new_chains:
old_sets = filter(lambda s: s.get_time() < t, chain.get_all_sets())
result_sets.extend(old_sets)
return self.sort_sets(result_sets)
def get_containing_volumes(self, index_prefix):
"""
Return list of volume numbers that may contain index_prefix
"""
return filter(lambda vol_num:
self.volume_info_dict[vol_num].contains(index_prefix),
self.volume_info_dict.keys())
def first(predicate_or_None, iterable, default=None):
"""
Returns the first item of iterable for which predicate(item) is true.
If predicate is None, matches the first item that is true.
Returns value of default in case of no matching items.
"""
return next(filter(predicate_or_None, iterable), default)
def partition(iterable, pred):
"""Use a predicate to partition entries into false entries and true \
entries.
Parameters
----------
iterable : `collections.Iterable`
Iterable to partition.
pred : `collections.Callable`
The predicate which determines the group in which the value of the
iterable belongs.
Returns
-------
false_values : generator
An iterable containing the values for which the predicate was True.
true_values : generator
An iterable containing the values for which the predicate was True.
Examples
--------
>>> from nddata.utils.itertools_recipes import partition
>>> def is_odd(val): return val % 2
>>> [list(i) for i in partition(range(10), is_odd)]
[[0, 2, 4, 6, 8], [1, 3, 5, 7, 9]]
"""
t1, t2 = tee(iterable)
return filterfalse(pred, t1), filter(pred, t2)
def add(self, code, toplevel_code=None):
if code in self:
return
if toplevel_code is None:
filename = code.co_filename
if filename.endswith((".pyc", ".pyo")):
filename = filename[:-1]
if not os.path.exists(filename):
print('ERROR: Could not find file ' + filename)
if filename.startswith(("ipython-input", "<ipython-input")):
print(
"NOTE: %mprun can only be used on functions defined in"
" physical files, and not in the IPython environment.")
return
toplevel_code = code
(sub_lines, start_line) = inspect.getsourcelines(code)
linenos = range(start_line,
start_line + len(sub_lines))
self._toplevel.append((filename, code, linenos))
self[code] = {}
else:
self[code] = self[toplevel_code]
for subcode in filter(inspect.iscode, code.co_consts):
self.add(subcode, toplevel_code=toplevel_code)
def __init__(self, searcher, query, field, terms=False, fields=False, tag='', formatter=None, encoder=None):
if tag:
formatter = highlight.SimpleHTMLFormatter('<{}>'.format(tag), '</{}>'.format(tag))
scorer = (highlight.QueryTermScorer if terms else highlight.QueryScorer)(query, *(searcher.indexReader, field) * (not fields))
highlight.Highlighter.__init__(self, *filter(None, [formatter, encoder, scorer]))
self.searcher, self.field = searcher, field
self.selector = HashSet(Arrays.asList([field]))
def find_duplicate_filenames(paths):
d = defaultdict(list)
for path in paths:
key = os.path.basename(path).lower()
d[key].append(path)
return filter(lambda x: len(x[1]) > 1, d.items())
def solve(exclusive_limit):
inclusive_limit = exclusive_limit - 1
numbers = islice(count(1), inclusive_limit)
multiples = filter(multiple_of_3_or_5, numbers)
return sum(multiples)
def get_all_source_files(paths, exclude_patterns):
'''
Function counts md5 hash for the given file
and checks if it isn't a duplicate using set
of hashes for previous files
'''
hash_set = set()
def _validate_file(pathname):
return (
pathname in paths or (
CodeReader.get_reader(pathname) and
all(not fnmatch(pathname, p) for p in exclude_patterns) and
_not_duplicate(pathname)))
def _not_duplicate(full_path_name):
fhash = md5_hash_file(full_path_name)
if not fhash or fhash not in hash_set:
hash_set.add(fhash)
return True
def all_listed_files(paths):
for path in paths:
if os.path.isfile(path):
yield path
else:
for root, _, files in os.walk(path, topdown=False):
for filename in files:
yield os.path.join(root, filename)
return filter(_validate_file, all_listed_files(paths))
def __init__(self, searcher, query, field, terms=False, fields=False, tag='', formatter=None, encoder=None):
if tag:
formatter = lucene.SimpleHTMLFormatter('<{0}>'.format(tag), '</{0}>'.format(tag))
scorer = (lucene.QueryTermScorer if terms else lucene.QueryScorer)(query, *(searcher.indexReader, field) * (not fields))
lucene.Highlighter.__init__(self, *filter(None, [formatter, encoder, scorer]))
self.searcher, self.field = searcher, field
self.selector = lucene.MapFieldSelector([field])
def _generate_constant_defs(macro_defs, macro_type_map):
constants = '\n'.join(map(
lambda (name, macro): ' constexpr {type} const {name} = {value};'.format(
type=macro_type_map[name], name=name, value=list(macro.get_tokens())[1].spelling),
filter(lambda (name, _): name in macro_type_map, macro_defs.items())))
if 'OFP_NO_BUFFER' not in macro_defs:
return '\n'.join([constants, ' constexpr std::uint32_t const OFP_NO_BUFFER = 0xffffffff;'])
return constants
def correct(self, word):
"Generate ordered sets of words by increasing edit distance."
previous, edits = set(), {word: 0}
for distance in range(len(word)):
yield sorted(filter(self.__contains__, edits), key=self.__getitem__, reverse=True)
previous.update(edits)
groups = map(self.edits, edits, edits.values())
edits = dict((edit, group[edit]) for group in groups for edit in group if edit not in previous)
def traverse_node(node, filename, func, level=0):
if func(node, level):
return
# indent_str = ''.join(itertools.repeat(' ', level))
# print "{}{} : {}".format(indent_str, node.kind.name, node.displayname)
for child in filter(
lambda node: str(node.location.file) == filename,
node.get_children()):
traverse_node(child, filename, func, level + 1)
def load_csv(profile, csv_file):
"""
Return tuple (column_names, data_rows).
column_names - a list with selected column names
data_rows - an iterator over row tuples (dn, timestamp, values)
"""
csv_reader = create_csv_reader(profile, csv_file)
header = csv_reader.next()
fields = profile.field_selector(header)
values = ValuesExtractor(fields)
check_header(header, fields)
header_checks = [
check for check in
[
profile.timestamp.header_check(),
profile.identifier.header_check()
]
if not check is None
]
for check in header_checks:
check(header)
record_checks = [
check for check in
[
profile.timestamp.record_check(),
profile.identifier.record_check()
]
if not check is None
]
include_record = partial(record_passes_checks, record_checks)
include_row = create_row_check(header)
records = filter(
include_record,
(
dict(zip(header, [item.decode('utf-8') for item in row]))
for line_nr, row in enumerate(csv_reader)
if profile.ignore_field_mismatches or include_row(line_nr, row)
)
)
extract_raw_data_row = compose(tuple, raw_data_row_extractor(
profile.identifier.from_record,
profile.timestamp.from_record,
values.from_record
))
return fields, map(extract_raw_data_row, records)
def tracemalloc_tool():
# .. cross-platform but but requires Python 3.4 or higher ..
stat = next(filter(lambda item: str(item).startswith(filename),
tracemalloc.take_snapshot().statistics('filename')))
mem = stat.size / _TWO_20
if timestamps:
return mem, time.time()
else:
return mem
def components(self, field, reader):
source = tokens = (
self.tokenizer.tokenStream(field, reader)
if isinstance(self.tokenizer, analysis.Analyzer)
else self.tokenizer(reader)
)
for filter in self.filters:
tokens = filter(tokens)
return source, tokens
def test_overall(self):
results = draftcombine.SpotShooting(season=self.season)
assert results
overall = results.overall()
assert overall
stats = next(filter(lambda d: d['PLAYER_NAME'] == self.player_name, overall))
assert stats
assert stats['PLAYER_NAME'] == self.player_name
assert stats['COLLEGE_BREAK_LEFT_MADE'] == None
assert stats['COLLEGE_BREAK_LEFT_PCT'] == None
assert stats['COLLEGE_BREAK_RIGHT_ATTEMPT'] == None
assert stats['COLLEGE_BREAK_RIGHT_MADE'] == None
assert stats['COLLEGE_BREAK_RIGHT_PCT'] == None
assert stats['COLLEGE_CORNER_LEFT_ATTEMPT'] == None
assert stats['COLLEGE_CORNER_LEFT_MADE'] == None
assert stats['COLLEGE_CORNER_LEFT_PCT'] == None
assert stats['COLLEGE_CORNER_RIGHT_ATTEMPT'] == None
assert stats['COLLEGE_CORNER_RIGHT_MADE'] == None
assert stats['COLLEGE_CORNER_RIGHT_PCT'] == None
assert stats['COLLEGE_TOP_KEY_ATTEMPT'] == None
assert stats['COLLEGE_TOP_KEY_MADE'] == None
assert stats['COLLEGE_TOP_KEY_PCT'] == None
assert stats['FIFTEEN_BREAK_LEFT_ATTEMPT'] == None
assert stats['FIFTEEN_BREAK_LEFT_MADE'] == None
assert stats['FIFTEEN_BREAK_LEFT_PCT'] == None
assert stats['FIFTEEN_BREAK_RIGHT_ATTEMPT'] == None
assert stats['FIFTEEN_BREAK_RIGHT_MADE'] == None
assert stats['FIFTEEN_BREAK_RIGHT_PCT'] == None
assert stats['FIFTEEN_CORNER_LEFT_ATTEMPT'] == None
assert stats['FIFTEEN_CORNER_LEFT_MADE'] == None
assert stats['FIFTEEN_CORNER_LEFT_PCT'] == None
assert stats['FIFTEEN_CORNER_RIGHT_ATTEMPT'] == None
assert stats['FIFTEEN_CORNER_RIGHT_MADE'] == None
assert stats['FIFTEEN_CORNER_RIGHT_PCT'] == None
assert stats['FIFTEEN_TOP_KEY_ATTEMPT'] == None
assert stats['FIFTEEN_TOP_KEY_MADE'] == None
assert stats['FIFTEEN_TOP_KEY_PCT'] == None
assert stats['NBA_BREAK_LEFT_ATTEMPT'] == None
assert stats['NBA_BREAK_LEFT_MADE'] == None
assert stats['NBA_BREAK_LEFT_PCT'] == None
assert stats['NBA_BREAK_RIGHT_ATTEMPT'] == None
assert stats['NBA_BREAK_RIGHT_MADE'] == None
assert stats['NBA_BREAK_RIGHT_PCT'] == None
assert stats['NBA_CORNER_LEFT_ATTEMPT'] == None
assert stats['NBA_CORNER_LEFT_MADE'] == None
assert stats['NBA_CORNER_LEFT_PCT'] == None
assert stats['NBA_CORNER_RIGHT_ATTEMPT'] == None
assert stats['NBA_CORNER_RIGHT_MADE'] == None
assert stats['NBA_CORNER_RIGHT_PCT'] == None
assert stats['NBA_TOP_KEY_ATTEMPT'] == None
assert stats['NBA_TOP_KEY_MADE'] == None
assert stats['NBA_TOP_KEY_PCT'] == None
assert stats['PLAYER_ID'] == None
def translate_query_params(cls, query=None, **kwargs):
"""
Translate an arbirtary keyword argument to the expected query.
In the v2 API, many endpoints expect a particular query argument to be
in the form of `query=xxx` where `xxx` would be the name of perhaps
the name, ID or otherwise. This function ought to take a more aptly
named parameter specified in `TRANSLATE_QUERY_PARAM`, and substitute it
into the `query` keyword argument. The purpose is so that some models
(optionally) have nicer named keyword arguments than `query` for easier
to read python.
If a query argument is given then the output should be that value. If a
substitute value is given as a keyword specified in
`TRANSLATE_QUERY_PARAM`(and query is not) then the `query` argument
will be that keyword argument.
Eg. No query param
query = None
TRANSLATE_QUERY_PARAM = ('name',)
kwargs = {'name': 'PagerDuty'}
...
output = {'query': 'PagerDuty'}
or, query param explicitly
query = 'XXXXPlopperDuty'
TRANSLATE_QUERY_PARAM = ('name',)
kwargs = {'name': 'PagerDuty'}
...
output = {'query': 'XXXXPlopperDuty'}
XXX: Clean this up. It's *too* flexible.
"""
values = []
output = kwargs.copy()
# remove any of the TRANSLATE_QUERY_PARAMs in output
for param in cls.TRANSLATE_QUERY_PARAM:
popped = output.pop(param, None)
if popped is not None:
values.append(popped)
# if query is not provided, use the first parameter we removed from
# the kwargs
try:
output['query'] = next(filter(None, values))
except StopIteration:
pass
# if query is provided, just use it
if query is not None:
output['query'] = query
return output
def parse_query(query):
matches = grammar.findall(query)
if not matches:
raise QueryParseException("wrong query")
fields = list(map(parse_field,
[first(filter(None, match)) for match in matches]))
return fields
def _generate_is_error_code_enum_specialization(version, error_type_enum, ignore_types):
return '\n\n'.join(map(
lambda error: (
"""\
template <> struct is_error_code_enum<canard::network::openflow::v{version}::ofp_{name}_code>
{{
static bool const value = true;
}};\
""".format(version=version, name=_strip_OFPET(error.displayname))),
filter(lambda enum: enum.displayname not in ignore_types, error_type_enum.get_children())))
def buck2diac(buckAnalysis, keepSegmentation=True):
'''
Given a segmented and tagged Arabic word from the Buckwalter analyzer, perform
orthographic normalization to produce the surface diacritized form
(also in the Buckwalter encoding) with morphemes separated by '+' (unless
'keepSegmentation' is false, in which case the unsegmented form will be returned).
Normalization involves contracting li+Al+ to li+, adding Shadda (gemination mark) to
sun letters after Al+, changing Alef Wasla to Alef, etc.
'''
assert ' ' not in buckAnalysis,buckAnalysis
# Strip tags from morphemes
vs = '+'.join(filter(lambda m: m!='(null)', (x.split('/')[0] for x in buckAnalysis.split('+') if x!='')))
# Normalization
''' In MADA-3.1/MADA/ALMOR3.pm:
#update Sept 11 2008: to match up behavior of sun/moon letters in BAMA2
# code segemnt from BAMA-2
my $voc_str = $$prefix_value{"diac"}."+".$$stem_value{"diac"}."+".$$suffix_value{"diac"};
$voc_str =~ s/^((wa|fa)?(bi|ka)?Al)\+([tvd\*rzs\$SDTZln])/$1$4~/; # not moon letters
$voc_str =~ s/^((wa|fa)?lil)\+([tvd\*rzs\$SDTZln])/$1$3~/; # not moon letters
$voc_str =~ s/A\+a([pt])/A$1/; # e.g.: Al+HayA+ap
$voc_str =~ s/\{/A/g;
$voc_str =~ s/\+//g;
'''
# TODO: this is ad hoc for now--not sure if it's precisely how SAMA/MADA do it.
vs = re.sub(r'^li[+]Al[+]', 'li+l+', vs) # contraction
vs = re.sub('^([+]?)min[+]m(A|an)$', r'\1mi+m~\2', vs) # contraction, e.g. +min/PREP+mA/REL_PRON+ -> mim~A
vs = re.sub(r'Y[+]', 'y+', vs) # Y -> y / _+
vs = re.sub(r'y[+]F', 'Y+F', vs) # e.g. bi/PREP+maEonY/NOUN+F/CASE_INDEF_GEN -> bimaEonYF
vs = re.sub(r'y[+]([tn])', r'yo+\1', vs) # 'o' epenthesis? e.g. wa/CONJ+>aroDay/PV+tu/PVSUFF_SUBJ:1S+hA/PVSUFF_DO:3FS -> wa>aroDayotuhA
vs = re.sub(r't[+]h', 'to+h', vs) # 'o' deletion? e.g. +jaEal/PV+at/PVSUFF_SUBJ:3FS+hu/PVSUFF_DO:3MS -> jaEalatohu
vs = re.sub(r'^Einod[+]a[+]mA$', 'EindamA', vs) # +Einod/NOUN+a/CASE_DEF_ACC+mA/SUB_CONJ+
vs = re.sub(r'^li[+][*]alika$', 'li*`lika', vs) # +li/PREP+*alika/DEM_PRON_MS+
# end ad hoc
# add Shadda to sun letters after Al+
# NOTE: needed to add [+]? in some places after clitics
vs = re.sub(r'^((wa[+]?|fa[+]?)?(bi[+]?|ka[+]?)?Al)\+([tvd\*rzs\$SDTZln])', r'\1\4~', vs, 1) # not moon letters
vs = re.sub(r'^((wa[+]?|fa[+]?)?li[+]?l)\+([tvd\*rzs\$SDTZln])', r'\1\3~', vs, 1); # not moon letters
# simplify Aa before p or t, e.g.: Al+HayA+ap
vs = re.sub(r'A\+a([pt])', r'A\1', vs, 1)
# convert Alef Wasla to plain Alef (http://en.wikipedia.org/wiki/Arabic_diacritics#.CA.BCAlif_wa.E1.B9.A3la)
vs = vs.replace('{', 'A')
# use the desired segmentation character
if not keepSegmentation:
vs = vs.replace('+', '')
return vs
def _generate_make_error_code_funcs(version, error_type_enum, ignore_types):
return '\n\n'.join(map(
lambda error: (
"""\
inline auto make_error_code(ofp_{name}_code const e)
-> boost::system::error_code
{{
return {{e + 1, v{version}::{name}_category()}};
}}\
""".format(version=version, name=_strip_OFPET(error.displayname))),
filter(lambda enum: enum.displayname not in ignore_types, error_type_enum.get_children())))
def _generate_error_category_funcs(version, error_type_enum, ignore_types):
return '\n\n'.join(map(
lambda error: (
"""\
inline auto {name}_category()
-> boost::system::error_category&
{{
static v{version}_detail::{name}_category instance{{}};
return instance;
}}\
""".format(version=version, name=_strip_OFPET(error.displayname))),
filter(lambda enum: enum.displayname not in ignore_types, error_type_enum.get_children())))
def parse_class(node):
class_assigments = filter(lambda x:x.__class__.__name__ == 'Assign', ast.iter_child_nodes(node))
found = {}
for node in class_assigments:
targets = {getattr(t, 'id', None) for t in node.targets}
targets.discard(None)
fields = field_names.intersection(targets)
if fields:
val = convert_node(fields, node.value, names=names, import_data=import_data)
for field in fields:
found[field] = val
return found
def __call__(self, where_clause=None):
'''
The navigation chain is invoked. Optionally, a conditional
*where-clause* in the form of a function may be provided, e.g
>>> chain(lambda selected: selected.Name == 'test')
'''
handle = self.handle or list()
if where_clause:
handle = filter(where_clause, handle)
return QuerySet(handle)
def select_one(self, where_clause=None):
'''
Select a single instance from the instance pool. Optionally, a
conditional *where-clause* in the form of a function may be provided.
'''
if isinstance(where_clause, dict):
s = self.query(where_clause)
elif where_clause:
s = iter(filter(where_clause, self.storage))
else:
s = iter(self.storage)
return next(s, None)
def select_many(self, where_clause=None):
'''
Select several instances from the instance pool. Optionally,
a conditional *where-clause* in the form of a function may be provided.
'''
if isinstance(where_clause, dict):
s = self.query(where_clause)
elif where_clause:
s = filter(where_clause, self.storage)
else:
s = iter(self.storage)
return QuerySet(s)
def glob_get_filename_sf(self, filename, include):
"""Get a selection function given a normal filename
Some of the parsing is better explained in
filelist_parse_line. The reason this is split from normal
globbing is things are a lot less complicated if no special
globbing characters are used.
"""
if not filename.startswith(self.prefix):
raise FilePrefixError(filename)
index = tuple(filter(lambda x: x,
filename[len(self.prefix):].split("/")))
return self.glob_get_tuple_sf(index, include)
def _generate_member_ppseq(version, members, is_skip_ofp_match):
def to_type_and_name_tuple(mem):
if mem.is_array():
return _enumerate_array(version, mem)
return ' ({type}, {name})'.format(
type=_maybe_qualify(version, mem.type), name=mem.name)
return '\n'.join(map(
to_type_and_name_tuple,
filter(
lambda mem: not (
(is_skip_ofp_match and mem.type == 'ofp_match')
or (mem.is_array() and (mem.element_count == 0 or mem.element_type == 'char'))),
members)))
def sort_reflexive(set_of_instances, rel_id, phrase):
'''
Sort a *set of instances* in the order they appear across a conditional and
reflexive association. The first instance in the resulting ordered set is
**not** associated to an instance across the given *phrase*.
'''
if not isinstance(set_of_instances, QuerySet):
raise MetaException('The collection to sort must be a QuerySet')
if not set_of_instances.first:
return QuerySet()
if isinstance(rel_id, int):
rel_id = 'R%d' % rel_id
# Figure out the phrase in the other direction
metaclass = get_metaclass(set_of_instances.first)
for link in metaclass.links.values():
if link.to_metaclass != metaclass:
continue
if link.rel_id != rel_id:
continue
if link.phrase == phrase:
continue
other_phrase = link.phrase
break
else:
raise UnknownLinkException(metaclass.kind, metaclass.kind, rel_id, phrase)
first_filt = lambda sel: not navigate_one(sel).nav(metaclass.kind, rel_id, phrase)()
first_instances = list(filter(first_filt, set_of_instances))
if not first_instances:
#the instance sequence is recursive, start anywhere
first_instances = [set_of_instances.first]
def sequence_generator():
for first in first_instances:
inst = first
while inst:
if inst in set_of_instances:
yield inst
inst = navigate_one(inst).nav(metaclass.kind, rel_id, other_phrase)()
if inst is first:
break
return QuerySet(sequence_generator())
def _case_for_each_member(members, ignore_to_string_enums):
return '\n'.join(map(
lambda mem: ' case {name}: return "{name}";'.format(name=mem.displayname),
filter(lambda mem: mem.displayname not in ignore_to_string_enums, members)))
def _create_path(self, *args):
"""Create URL path for endpoint and args."""
args = filter(None, args)
path = self.endpoint + '/'.join(args) + '.%s' % (self.format)
return path
def browser(request, path='', template="cloud_browser/browser.html"):
"""View files in a file path.
:param request: The request.
:param path: Path to resource, including container as first part of path.
:param template: Template to render.
"""
from itertools import islice
try:
# pylint: disable=redefined-builtin
from future_builtins import filter
except ImportError:
# pylint: disable=import-error
from builtins import filter
# Inputs.
container_path, object_path = path_parts(path)
incoming = request.POST or request.GET or {}
marker = incoming.get('marker', None)
marker_part = incoming.get('marker_part', None)
if marker_part:
marker = path_join(object_path, marker_part)
# Get and adjust listing limit.
limit_default = settings.CLOUD_BROWSER_DEFAULT_LIST_LIMIT
def limit_test(num):
return num > 0 and (MAX_LIMIT is None or num <= MAX_LIMIT - 1)
limit = get_int(incoming.get('limit', limit_default), limit_default,
limit_test)
# Q1: Get all containers.
# We optimize here by not individually looking up containers later,
# instead going through this in-memory list.
# TODO: Should page listed containers with a ``limit`` and ``marker``.
conn = get_connection()
containers = conn.get_containers()
marker_part = None
container = None
objects = None
if container_path != '':
# Find marked container from list.
def cont_eq(container):
return container.name == container_path
filtered_conts = filter(cont_eq, containers)
cont_list = list(islice(filtered_conts, 1))
if not cont_list:
raise Http404("No container at: %s" % container_path)
# Q2: Get objects for instant list, plus one to check "next".
container = cont_list[0]
objects = container.get_objects(object_path, marker, limit + 1)
marker = None
# If over limit, strip last item and set marker.
if len(objects) == limit + 1:
objects = objects[:limit]
marker = objects[-1].name
marker_part = relpath(marker, object_path)
return render(
request, template, {
'path': path,
'marker': marker,
'marker_part': marker_part,
'limit': limit,
'breadcrumbs': _breadcrumbs(path),
'container_path': container_path,
'containers': containers,
'container': container,
'object_path': object_path,
'objects': objects
})
def parse_cmdline_options(arglist):
"""Parse argument list"""
global select_opts, select_files, full_backup
global list_current, collection_status, cleanup, remove_time, verify
def set_log_fd(fd):
if fd < 1:
raise optparse.OptionValueError(
"log-fd must be greater than zero.")
log.add_fd(fd)
def set_time_sep(sep, opt):
if sep == '-':
raise optparse.OptionValueError(
"Dash ('-') not valid for time-separator.")
globals.time_separator = sep
old_fn_deprecation(opt)
def add_selection(o, s, v, p):
select_opts.append((s, v))
def add_filelist(o, s, v, p):
filename = v
select_opts.append((s, filename))
try:
select_files.append(open(filename, "r"))
except IOError:
log.FatalError(
_("Error opening file %s") % filename,
log.ErrorCode.cant_open_filelist)
def print_ver(o, s, v, p):
print "duplicity %s" % (globals.version)
sys.exit(0)
def add_rename(o, s, v, p):
globals.rename[os.path.normcase(os.path.normpath(v[0]))] = v[1]
parser = OPHelpFix(option_class=DupOption, usage=usage())
# If this is true, only warn and don't raise fatal error when backup
# source directory doesn't match previous backup source directory.
parser.add_option("--allow-source-mismatch", action="store_true")
# Set to the path of the archive directory (the directory which
# contains the signatures and manifests of the relevent backup
# collection), and for checkpoint state between volumes.
# TRANSL: Used in usage help to represent a Unix-style path name. Example:
# --archive-dir <path>
parser.add_option("--archive-dir", type="file", metavar=_("path"))
# Asynchronous put/get concurrency limit
# (default of 0 disables asynchronicity).
parser.add_option("--asynchronous-upload",
action="store_const",
const=1,
dest="async_concurrency")
parser.add_option("--compare-data", action="store_true")
# config dir for future use
parser.add_option("--config-dir",
type="file",
metavar=_("path"),
help=optparse.SUPPRESS_HELP)
# for testing -- set current time
parser.add_option("--current-time",
type="int",
dest="current_time",
help=optparse.SUPPRESS_HELP)
# Don't actually do anything, but still report what would be done
parser.add_option("--dry-run", action="store_true")
# TRANSL: Used in usage help to represent an ID for a GnuPG key. Example:
# --encrypt-key <gpg_key_id>
parser.add_option("--encrypt-key",
type="string",
metavar=_("gpg-key-id"),
dest="",
action="callback",
callback=lambda o, s, v, p: globals.gpg_profile.
recipients.append(v)) # @UndefinedVariable
# secret keyring in which the private encrypt key can be found
parser.add_option("--encrypt-secret-keyring",
type="string",
metavar=_("path"))
parser.add_option(
"--encrypt-sign-key",
type="string",
metavar=_("gpg-key-id"),
dest="",
action="callback",
callback=lambda o, s, v, p:
(globals.gpg_profile.recipients.append(v), set_sign_key(v)))
# TRANSL: Used in usage help to represent a "glob" style pattern for
# matching one or more files, as described in the documentation.
# Example:
# --exclude <shell_pattern>
parser.add_option("--exclude",
action="callback",
metavar=_("shell_pattern"),
dest="",
type="string",
callback=add_selection)
parser.add_option("--exclude-device-files",
action="callback",
dest="",
callback=add_selection)
parser.add_option("--exclude-filelist",
type="file",
metavar=_("filename"),
dest="",
action="callback",
callback=add_filelist)
parser.add_option("--exclude-filelist-stdin",
action="callback",
dest="",
callback=lambda o, s, v, p:
(select_opts.append(
("--exclude-filelist", "standard input")),
select_files.append(sys.stdin), stdin_deprecation(o)),
help=optparse.SUPPRESS_HELP)
parser.add_option(
"--exclude-globbing-filelist",
type="file",
metavar=_("filename"),
dest="",
action="callback",
callback=lambda o, s, v, p:
(add_filelist(o, s, v, p), old_globbing_filelist_deprecation(s)),
help=optparse.SUPPRESS_HELP)
# TRANSL: Used in usage help to represent the name of a file. Example:
# --log-file <filename>
parser.add_option("--exclude-if-present",
metavar=_("filename"),
dest="",
type="file",
action="callback",
callback=add_selection)
parser.add_option("--exclude-other-filesystems",
action="callback",
dest="",
callback=add_selection)
# TRANSL: Used in usage help to represent a regular expression (regexp).
parser.add_option("--exclude-regexp",
metavar=_("regular_expression"),
dest="",
type="string",
action="callback",
callback=add_selection)
# Exclude any files with modification dates older than this from the backup
parser.add_option("--exclude-older-than",
type="time",
metavar=_("time"),
dest="",
action="callback",
callback=add_selection)
# Whether we should be particularly aggressive when cleaning up
parser.add_option("--extra-clean", action="store_true")
# used in testing only - raises exception after volume
parser.add_option("--fail-on-volume",
type="int",
help=optparse.SUPPRESS_HELP)
# used to provide a prefix on top of the defaul tar file name
parser.add_option("--file-prefix",
type="string",
dest="file_prefix",
action="store")
# used to provide a suffix for manifest files only
parser.add_option("--file-prefix-manifest",
type="string",
dest="file_prefix_manifest",
action="store")
# used to provide a suffix for archive files only
parser.add_option("--file-prefix-archive",
type="string",
dest="file_prefix_archive",
action="store")
# used to provide a suffix for sigature files only
parser.add_option("--file-prefix-signature",
type="string",
dest="file_prefix_signature",
action="store")
# used in testing only - skips upload for a given volume
parser.add_option("--skip-volume", type="int", help=optparse.SUPPRESS_HELP)
# If set, restore only the subdirectory or file specified, not the
# whole root.
# TRANSL: Used in usage help to represent a Unix-style path name. Example:
# --archive-dir <path>
parser.add_option("--file-to-restore",
"-r",
action="callback",
type="file",
metavar=_("path"),
dest="restore_dir",
callback=lambda o, s, v, p: setattr(
p.values, "restore_dir", v.rstrip('/')))
# Used to confirm certain destructive operations like deleting old files.
parser.add_option("--force", action="store_true")
# FTP data connection type
parser.add_option("--ftp-passive",
action="store_const",
const="passive",
dest="ftp_connection")
parser.add_option("--ftp-regular",
action="store_const",
const="regular",
dest="ftp_connection")
# If set, forces a full backup if the last full backup is older than
# the time specified
parser.add_option("--full-if-older-than",
type="time",
dest="full_force_time",
metavar=_("time"))
parser.add_option("--gio",
action="callback",
dest="use_gio",
callback=lambda o, s, v, p:
(setattr(p.values, o.dest, True), old_fn_deprecation(s)))
parser.add_option("--gpg-binary", type="file", metavar=_("path"))
parser.add_option("--gpg-options", action="extend", metavar=_("options"))
# TRANSL: Used in usage help to represent an ID for a hidden GnuPG key. Example:
# --hidden-encrypt-key <gpg_key_id>
parser.add_option("--hidden-encrypt-key",
type="string",
metavar=_("gpg-key-id"),
dest="",
action="callback",
callback=lambda o, s, v, p: globals.gpg_profile.
hidden_recipients.append(v)) # @UndefinedVariable
# ignore (some) errors during operations; supposed to make it more
# likely that you are able to restore data under problematic
# circumstances. the default should absolutely always be False unless
# you know what you are doing.
parser.add_option(
"--ignore-errors",
action="callback",
dest="ignore_errors",
callback=lambda o, s, v, p: (log.Warn(
_("Running in 'ignore errors' mode due to %s; please "
"re-consider if this was not intended") % s),
setattr(p.values, "ignore_errors", True)))
# Whether to use the full email address as the user name when
# logging into an imap server. If false just the user name
# part of the email address is used.
parser.add_option("--imap-full-address", action="store_true")
# Name of the imap folder where we want to store backups.
# Can be changed with a command line argument.
# TRANSL: Used in usage help to represent an imap mailbox
parser.add_option("--imap-mailbox", metavar=_("imap_mailbox"))
parser.add_option("--include",
action="callback",
metavar=_("shell_pattern"),
dest="",
type="string",
callback=add_selection)
parser.add_option("--include-filelist",
type="file",
metavar=_("filename"),
dest="",
action="callback",
callback=add_filelist)
parser.add_option("--include-filelist-stdin",
action="callback",
dest="",
callback=lambda o, s, v, p:
(select_opts.append(
("--include-filelist", "standard input")),
select_files.append(sys.stdin), stdin_deprecation(o)),
help=optparse.SUPPRESS_HELP)
parser.add_option(
"--include-globbing-filelist",
type="file",
metavar=_("filename"),
dest="",
action="callback",
callback=lambda o, s, v, p:
(add_filelist(o, s, v, p), old_globbing_filelist_deprecation(s)),
help=optparse.SUPPRESS_HELP)
parser.add_option("--include-regexp",
metavar=_("regular_expression"),
dest="",
type="string",
action="callback",
callback=add_selection)
parser.add_option("--log-fd",
type="int",
metavar=_("file_descriptor"),
dest="",
action="callback",
callback=lambda o, s, v, p: set_log_fd(v))
# TRANSL: Used in usage help to represent the name of a file. Example:
# --log-file <filename>
parser.add_option("--log-file",
type="file",
metavar=_("filename"),
dest="",
action="callback",
callback=lambda o, s, v, p: log.add_file(v))
# Maximum block size for large files
parser.add_option("--max-blocksize", type="int", metavar=_("number"))
# TRANSL: Used in usage help (noun)
parser.add_option("--name", dest="backup_name", metavar=_("backup name"))
# If set to false, then do not encrypt files on remote system
parser.add_option("--no-encryption",
action="store_false",
dest="encryption")
# If set to false, then do not compress files on remote system
parser.add_option("--no-compression",
action="store_false",
dest="compression")
# If set, print the statistics after every backup session
parser.add_option("--no-print-statistics",
action="store_false",
dest="print_statistics")
# If true, filelists and directory statistics will be split on
# nulls instead of newlines.
parser.add_option("--null-separator", action="store_true")
# number of retries on network operations
# TRANSL: Used in usage help to represent a desired number of
# something. Example:
# --num-retries <number>
parser.add_option("--num-retries", type="int", metavar=_("number"))
# File owner uid keeps number from tar file. Like same option in GNU tar.
parser.add_option("--numeric-owner", action="store_true")
# Whether the old filename format is in effect.
parser.add_option("--old-filenames",
action="callback",
dest="old_filenames",
callback=lambda o, s, v, p:
(setattr(p.values, o.dest, True), old_fn_deprecation(s)))
# Level of Redundancy in % for Par2 files
parser.add_option("--par2-redundancy", type="int", metavar=_("number"))
# Verbatim par2 options
parser.add_option("--par2-options", action="extend", metavar=_("options"))
# Used to display the progress for the full and incremental backup operations
parser.add_option("--progress", action="store_true")
# Used to control the progress option update rate in seconds. Default: prompts each 3 seconds
parser.add_option("--progress-rate", type="int", metavar=_("number"))
# option to trigger Pydev debugger
parser.add_option("--pydevd", action="store_true")
# option to rename files during restore
parser.add_option("--rename",
type="file",
action="callback",
nargs=2,
callback=add_rename)
# Restores will try to bring back the state as of the following time.
# If it is None, default to current time.
# TRANSL: Used in usage help to represent a time spec for a previous
# point in time, as described in the documentation. Example:
# duplicity remove-older-than time [options] target_url
parser.add_option("--restore-time",
"--time",
"-t",
type="time",
metavar=_("time"))
# user added rsync options
parser.add_option("--rsync-options", action="extend", metavar=_("options"))
# Whether to create European buckets (sorry, hard-coded to only
# support european for now).
parser.add_option("--s3-european-buckets", action="store_true")
# Whether to use S3 Reduced Redudancy Storage
parser.add_option("--s3-use-rrs", action="store_true")
# Whether to use S3 Infrequent Access Storage
parser.add_option("--s3-use-ia", action="store_true")
# Whether to use "new-style" subdomain addressing for S3 buckets. Such
# use is not backwards-compatible with upper-case buckets, or buckets
# that are otherwise not expressable in a valid hostname.
parser.add_option("--s3-use-new-style", action="store_true")
# Whether to use plain HTTP (without SSL) to send data to S3
# See <https://bugs.launchpad.net/duplicity/+bug/433970>.
parser.add_option("--s3-unencrypted-connection", action="store_true")
# Chunk size used for S3 multipart uploads.The number of parallel uploads to
# S3 be given by chunk size / volume size. Use this to maximize the use of
# your bandwidth. Defaults to 25MB
parser.add_option(
"--s3-multipart-chunk-size",
type="int",
action="callback",
metavar=_("number"),
callback=lambda o, s, v, p: setattr(
p.values, "s3_multipart_chunk_size", v * 1024 * 1024))
# Number of processes to set the Processor Pool to when uploading multipart
# uploads to S3. Use this to control the maximum simultaneous uploads to S3.
parser.add_option("--s3-multipart-max-procs",
type="int",
metavar=_("number"))
# Number of seconds to wait for each part of a multipart upload to S3. Use this
# to prevent hangups when doing a multipart upload to S3.
parser.add_option("--s3-multipart-max-timeout",
type="int",
metavar=_("number"))
# Option to allow the s3/boto backend use the multiprocessing version.
parser.add_option("--s3-use-multiprocessing", action="store_true")
# Option to allow use of server side encryption in s3
parser.add_option("--s3-use-server-side-encryption",
action="store_true",
dest="s3_use_sse")
# scp command to use (ssh pexpect backend)
parser.add_option("--scp-command", metavar=_("command"))
# sftp command to use (ssh pexpect backend)
parser.add_option("--sftp-command", metavar=_("command"))
# allow the user to switch cloudfiles backend
parser.add_option("--cf-backend", metavar=_("pyrax|cloudfiles"))
# If set, use short (< 30 char) filenames for all the remote files.
parser.add_option("--short-filenames",
action="callback",
dest="short_filenames",
callback=lambda o, s, v, p:
(setattr(p.values, o.dest, True), old_fn_deprecation(s)))
# TRANSL: Used in usage help to represent an ID for a GnuPG key. Example:
# --encrypt-key <gpg_key_id>
parser.add_option("--sign-key",
type="string",
metavar=_("gpg-key-id"),
dest="",
action="callback",
callback=lambda o, s, v, p: set_sign_key(v))
# default to batch mode using public-key encryption
parser.add_option("--ssh-askpass", action="store_true")
# user added ssh options
parser.add_option("--ssh-options", action="extend", metavar=_("options"))
# user added ssl options (webdav backend)
parser.add_option(
"--ssl-cacert-file",
metavar=_("pem formatted bundle of certificate authorities"))
parser.add_option("--ssl-no-check-certificate", action="store_true")
# Working directory for the tempfile module. Defaults to /tmp on most systems.
parser.add_option("--tempdir",
dest="temproot",
type="file",
metavar=_("path"))
# network timeout value
# TRANSL: Used in usage help. Example:
# --timeout <seconds>
parser.add_option("--timeout", type="int", metavar=_("seconds"))
# Character used like the ":" in time strings like
# 2002-08-06T04:22:00-07:00. The colon isn't good for filenames on
# windows machines.
# TRANSL: abbreviation for "character" (noun)
parser.add_option("--time-separator",
type="string",
metavar=_("char"),
action="callback",
callback=lambda o, s, v, p: set_time_sep(v, s))
# Whether to specify --use-agent in GnuPG options
parser.add_option("--use-agent", action="store_true")
parser.add_option("--verbosity",
"-v",
type="verbosity",
metavar="[0-9]",
dest="",
action="callback",
callback=lambda o, s, v, p: log.setverbosity(v))
parser.add_option("-V", "--version", action="callback", callback=print_ver)
# volume size
# TRANSL: Used in usage help to represent a desired number of
# something. Example:
# --num-retries <number>
parser.add_option("--volsize",
type="int",
action="callback",
metavar=_("number"),
callback=lambda o, s, v, p: setattr(
p.values, "volsize", v * 1024 * 1024))
# If set, collect only the file status, not the whole root.
parser.add_option("--file-changed",
action="callback",
type="file",
metavar=_("path"),
dest="file_changed",
callback=lambda o, s, v, p: setattr(
p.values, "file_changed", v.rstrip('/')))
# parse the options
(options, args) = parser.parse_args()
# Copy all arguments and their values to the globals module. Don't copy
# attributes that are 'hidden' (start with an underscore) or whose name is
# the empty string (used for arguments that don't directly store a value
# by using dest="")
for f in filter(lambda x: x and not x.startswith("_"), dir(options)):
v = getattr(options, f)
# Only set if v is not None because None is the default for all the
# variables. If user didn't set it, we'll use defaults in globals.py
if v is not None:
setattr(globals, f, v)
socket.setdefaulttimeout(globals.timeout)
# expect no cmd and two positional args
cmd = ""
num_expect = 2
# process first arg as command
if args:
cmd = args.pop(0)
possible = [c for c in commands if c.startswith(cmd)]
# no unique match, that's an error
if len(possible) > 1:
command_line_error("command '%s' not unique, could be %s" %
(cmd, possible))
# only one match, that's a keeper
elif len(possible) == 1:
cmd = possible[0]
# no matches, assume no cmd
elif not possible:
args.insert(0, cmd)
if cmd == "cleanup":
cleanup = True
num_expect = 1
elif cmd == "collection-status":
collection_status = True
num_expect = 1
elif cmd == "full":
full_backup = True
num_expect = 2
elif cmd == "incremental":
globals.incremental = True
num_expect = 2
elif cmd == "list-current-files":
list_current = True
num_expect = 1
elif cmd == "remove-older-than":
try:
arg = args.pop(0)
except Exception:
command_line_error("Missing time string for remove-older-than")
globals.remove_time = dup_time.genstrtotime(arg)
num_expect = 1
elif cmd == "remove-all-but-n-full" or cmd == "remove-all-inc-of-but-n-full":
if cmd == "remove-all-but-n-full":
globals.remove_all_but_n_full_mode = True
if cmd == "remove-all-inc-of-but-n-full":
globals.remove_all_inc_of_but_n_full_mode = True
try:
arg = args.pop(0)
except Exception:
command_line_error("Missing count for " + cmd)
globals.keep_chains = int(arg)
if not globals.keep_chains > 0:
command_line_error(cmd + " count must be > 0")
num_expect = 1
elif cmd == "verify":
verify = True
if len(args) != num_expect:
command_line_error("Expected %d args, got %d" %
(num_expect, len(args)))
# expand pathname args, but not URL
for loc in range(len(args)):
if '://' not in args[loc]:
args[loc] = expand_fn(args[loc])
# Note that ProcessCommandLine depends on us verifying the arg
# count here; do not remove without fixing it. We must make the
# checks here in order to make enough sense of args to identify
# the backend URL/lpath for args_to_path_backend().
if len(args) < 1:
command_line_error("Too few arguments")
elif len(args) == 1:
backend_url = args[0]
elif len(args) == 2:
lpath, backend_url = args_to_path_backend(args[0],
args[1]) # @UnusedVariable
else:
command_line_error("Too many arguments")
if globals.backup_name is None:
globals.backup_name = generate_default_backup_name(backend_url)
# set and expand archive dir
set_archive_dir(
expand_archive_dir(globals.archive_dir, globals.backup_name))
log.Info(
_("Using archive dir: %s") % (util.ufn(globals.archive_dir.name), ))
log.Info(_("Using backup name: %s") % (globals.backup_name, ))
return args
def solve(factors_range):
factors = list(permutations(factors_range, 2))
products = map(lambda pair: mul(*pair), factors)
palindromes = filter(palindrome, products)
return max(palindromes)
def _filter_status(self, status):
return filter(lambda x: x.status == status, self)
def find_pred(self, pred: 'Callable[[Tree], bool]') -> 'Iterator[Tree]':
"""Returns all nodes of the tree that evaluate pred(node) as true."""
return filter(pred, self.iter_subtrees())
def parse_yaml(hosts, config):
"""Parses host information from the output of yaml.load"""
export = {'_meta': {'hostvars': {}}}
for host_types in hosts['hosts']:
for host_type, providers in host_types.items():
export[host_type] = {}
export[host_type]['hosts'] = []
key = '~/.ssh/nodejs_build_%s' % host_type
export[host_type]['vars'] = {'ansible_ssh_private_key_file': key}
for provider in providers:
for provider_name, hosts in provider.items():
for host, metadata in hosts.items():
# some hosts have metadata appended to provider
# which requires underscore
delimiter = "_" if host.count('-') is 3 else "-"
hostname = '{}-{}{}{}'.format(host_type, provider_name,
delimiter, host)
export[host_type]['hosts'].append(hostname)
hostvars = {}
try:
parsed_host = parse_host(hostname)
for k, v in parsed_host.items():
hostvars.update(
{k: v[0] if type(v) is dict else v})
except Exception as e:
raise Exception('Failed to parse host: %s' % e)
if 'ip' in metadata:
hostvars.update({'ansible_host': metadata['ip']})
del metadata['ip']
if 'port' in metadata:
hostvars.update(
{'ansible_port': str(metadata['port'])})
del metadata['port']
if 'user' in metadata:
hostvars.update({'ansible_user': metadata['user']})
hostvars.update({'ansible_become': True})
del metadata['user']
hostvars.update(metadata)
# add specific options from config
for option in filter(lambda s: s.startswith('hosts:'),
config.sections()):
# remove `hosts:`
if option[6:] in hostname:
for o in config.items(option):
# configparser returns tuples of key, value
hostvars.update({o[0]: o[1]})
export['_meta']['hostvars'][hostname] = {}
export['_meta']['hostvars'][hostname].update(hostvars)
export[host_type]['hosts'].sort()
return export
def layer(name, features):
"""Make a vector_tile.Tile.Layer from GeoJSON features."""
pbl = vector_tile_pb2.tile.layer()
pbl.name = name
pbl.version = 1
pb_keys = []
pb_vals = []
pb_features = []
for j, f in enumerate(chain.from_iterable(singles(ob) for ob in features)):
pbf = vector_tile_pb2.tile.feature()
pbf.id = j
# Pack up the feature geometry.
g = f.get('geometry')
if g:
gtype = g['type']
coords = g['coordinates']
if gtype == 'Point':
geometry = [(1 << 3) + 1] + [(n << 1) ^ (n >> 31)
for n in map(int, coords)]
elif gtype == 'LineString':
num = len(coords)
geometry = [0] * (4 + 2 * (num - 1))
geometry[0] = (1 << 3) + 1
geometry[1:3] = ((n << 1) ^ (n >> 31)
for n in map(int, coords[0]))
geometry[3] = ((num - 1) << 3) + 2
for i, (prev, pair) in enumerate(pairwise(coords), 1):
prev = map(int, prev)
pair = map(int, pair)
geometry[2 * i + 2:2 * i +
4] = ((n << 1) ^ (n >> 31)
for n in (pair[0] - prev[0],
pair[1] - prev[1]))
pbf.geometry.extend(geometry)
elif gtype == 'Polygon':
rings = []
for ring in coords:
num = len(ring)
geometry = [0] * (5 + 2 * (num - 1))
geometry[0] = (1 << 3) + 1
geometry[1:3] = ((n << 1) ^ (n >> 31)
for n in map(int, ring[0]))
geometry[3] = ((num - 1) << 3) + 2
for i, (prev, pair) in enumerate(pairwise(ring), 1):
prev = map(int, prev)
pair = map(int, pair)
geometry[2 * i + 2:2 * i +
4] = ((n << 1) ^ (n >> 31)
for n in (pair[0] - prev[0],
pair[1] - prev[1]))
geometry[-1] = (1 << 3) + 7
pbf.geometry.extend(geometry)
pbf.type = geom_type_map[gtype]
# Pack up feature properties.
props = f.get('properties', {})
tags = [0] * (2 * len(props))
for i, (k, v) in enumerate(props.items()):
if k not in pb_keys:
pb_keys.append(k)
if v not in pb_vals:
pb_vals.append(v)
tags[i * 2:i * 2 + 2] = pb_keys.index(k), pb_vals.index(v)
pbf.tags.extend(tags)
pb_features.append(pbf)
# Finish up the layer.
pbl.keys.extend(map(str, pb_keys))
pbl.values.extend(map(value, filter(None, pb_vals)))
return pbl
def output_resources(self):
return filter(lambda a: isinstance(a, pypeliner.resources.Resource),
self.outputs)
def _capability2packages( self, cap ):
""" Find packages that implement the capability 'cap'. If cap is a
list, find the union of packages implementing those capabilities. """
# Convert to list if single capability given
if 'list' == type(cap).__name__:
caps = cap
else:
caps = [ cap ]
# Prime the pump.
(result, todo, pack_lists, uninstalled) = ([], [], [], [])
# Build two lists: 'results' is a list of capabilities found in our cache.
# 'todo' is a list of packages we need to query with rpm.
for one_cap in caps:
if one_cap.startswith('rpmlib'):
continue
# if this capability is cached, add its packages to the results.
if one_cap in self._transmogrify_cache:
new1 = self._transmogrify_cache[one_cap]
result.append(new1)
else:
# if not cached, add to todo list.
todo.append(one_cap)
todo = list(set(todo))
# Ask RPM for capabilities provided by each package in the todo list.
if len(todo) > 0:
# Because RPM can return more than one package that provides a given
# capability, the only way to maintain corraspondance between
# packages and capabilities is to force delimitors into RPM's output.
todo_cooked = reduce(lambda r,v: r + [ self._rpm_delimiter, v],
todo[1:], todo[:1])
todo_cooked.append(self._rpm_delimiter)
# Run the RPM command. Because of the forced delimitors, RPM will
# give "no package provides" errors in its output and return with
# a non-zero # return code.
try:
cmd_list =["rpm", "-q", "--whatprovides"] + todo_cooked
packages = proc.check_output( cmd_list, stderr=proc.STDOUT)
except proc.CalledProcessError as e:
packages = e.output
if "bytes" == type(packages).__name__:
packages = "".join(list(map(chr,packages)))
packages = packages.rstrip().split('\n')
# Use the "no package provides @_@" messages to seperate the output
# from RPM into a list of lists. Each element of the top level list
# corrasponds to a given capability and is a list of packages
# providing that capabilits.
a_pack_list = []
for pack in packages:
if self._rpm_delimiter in pack:
pack_lists.append(list(set(filter(self._good_for_rpm,
a_pack_list))))
a_pack_list = []
else:
a_pack_list.append(pack)
assert len(todo) == len(pack_lists), \
"todo and pack_list have different lengths"
new_stuff = zip(todo,pack_lists)
self._transmogrify_cache.update(new_stuff)
for k,v in new_stuff:
if [] == v:
uninstalled.append(k)
return list(chain.from_iterable(result + pack_lists)) + uninstalled
def parse_metadata(raw, namelist, zf):
module = ast.parse(raw, filename='__init__.py')
top_level_imports = filter(lambda x: x.__class__.__name__ == 'ImportFrom',
ast.iter_child_nodes(module))
top_level_classes = tuple(
filter(lambda x: x.__class__.__name__ == 'ClassDef',
ast.iter_child_nodes(module)))
top_level_assigments = filter(lambda x: x.__class__.__name__ == 'Assign',
ast.iter_child_nodes(module))
defaults = {
'name': '',
'description': '',
'supported_platforms': ['windows', 'osx', 'linux'],
'version': (1, 0, 0),
'author': 'Unknown',
'minimum_calibre_version': (0, 9, 42)
}
field_names = set(defaults)
imported_names = {}
plugin_import_found = set()
all_imports = []
for node in top_level_imports:
names = getattr(node, 'names', [])
mod = getattr(node, 'module', None)
if names and mod:
names = [Alias(n.name, getattr(n, 'asname', None)) for n in names]
if mod in {
'calibre.customize',
'calibre.customize.conversion',
'calibre.ebooks.metadata.sources.base',
'calibre.ebooks.metadata.covers',
'calibre.devices.interface',
'calibre.ebooks.metadata.fetch',
} or re.match(r'calibre\.devices\.[a-z0-9]+\.driver',
mod) is not None:
inames = {n.asname or n.name for n in names}
inames = {x for x in inames if x.lower() != x}
plugin_import_found |= inames
else:
all_imports.append((mod, [n.name for n in names]))
imported_names[n.asname or n.name] = mod
if not plugin_import_found:
return all_imports
import_data = (imported_names, zf, namelist)
names = {}
for node in top_level_assigments:
targets = {getattr(t, 'id', None) for t in node.targets}
targets.discard(None)
for x in targets - field_names:
try:
val = convert_node({x}, node.value, import_data=import_data)
except Exception:
pass
else:
names[x] = val
def parse_class(node):
class_assigments = filter(lambda x: x.__class__.__name__ == 'Assign',
ast.iter_child_nodes(node))
found = {}
for node in class_assigments:
targets = {getattr(t, 'id', None) for t in node.targets}
targets.discard(None)
fields = field_names.intersection(targets)
if fields:
val = convert_node(fields,
node.value,
names=names,
import_data=import_data)
for field in fields:
found[field] = val
return found
if top_level_classes:
for node in top_level_classes:
bases = {getattr(x, 'id', None) for x in node.bases}
if not bases.intersection(plugin_import_found):
continue
found = parse_class(node)
if 'name' in found and 'author' in found:
defaults.update(found)
return defaults
for node in top_level_classes:
found = parse_class(node)
if 'name' in found and 'author' in found and 'version' in found:
defaults.update(found)
return defaults
raise ValueError('Could not find plugin class')
def select_many_in(inst_set, where_cond, order_by):
s = filter(where_cond, inst_set)
if order_by:
s = order_by(s)
return xtuml.QuerySet(s)
def find_pred(self, pred):
"""Returns all nodes of the tree that evaluate pred(node) as true."""
return filter(pred, self.iter_subtrees())
def create_periodical_index(self): # {{{
periodical_node = iter(self.oeb.toc).next()
periodical_node_offset = self.serializer.body_start_offset
periodical_node_size = (self.serializer.body_end_offset -
periodical_node_offset)
normalized_sections = []
id_offsets = self.serializer.id_offsets
periodical = PeriodicalIndexEntry(periodical_node_offset,
self.cncx[periodical_node.title],
self.cncx[periodical_node.klass], 0)
periodical.length = periodical_node_size
periodical.first_child_index = 1
periodical.image_index = self.masthead_offset
seen_sec_offsets = set()
seen_art_offsets = set()
for sec in periodical_node:
normalized_articles = []
try:
offset = id_offsets[sec.href]
label = self.cncx[sec.title]
klass = self.cncx[sec.klass]
except:
continue
if offset in seen_sec_offsets:
continue
seen_sec_offsets.add(offset)
section = PeriodicalIndexEntry(offset, label, klass, 1)
section.parent_index = 0
for art in sec:
try:
offset = id_offsets[art.href]
label = self.cncx[art.title]
klass = self.cncx[art.klass]
except:
continue
if offset in seen_art_offsets:
continue
seen_art_offsets.add(offset)
article = PeriodicalIndexEntry(offset, label, klass, 2)
normalized_articles.append(article)
article.author_offset = self.cncx[art.author]
article.desc_offset = self.cncx[art.description]
if getattr(art, 'toc_thumbnail', None) is not None:
try:
ii = self.serializer.images[art.toc_thumbnail] - 1
if ii > -1:
article.image_index = ii
except KeyError:
pass # Image not found in serializer
if normalized_articles:
normalized_articles.sort(key=lambda x: x.offset)
normalized_sections.append((section, normalized_articles))
normalized_sections.sort(key=lambda x: x[0].offset)
# Set lengths
for s, x in enumerate(normalized_sections):
sec, normalized_articles = x
try:
sec.length = normalized_sections[s + 1][0].offset - sec.offset
except:
sec.length = self.serializer.body_end_offset - sec.offset
for i, art in enumerate(normalized_articles):
try:
art.length = normalized_articles[i + 1].offset - art.offset
except:
art.length = sec.offset + sec.length - art.offset
# Filter
for i, x in list(enumerate(normalized_sections)):
sec, normalized_articles = x
normalized_articles = list(
filter(lambda x: x.length > 0, normalized_articles))
normalized_sections[i] = (sec, normalized_articles)
normalized_sections = list(
filter(lambda x: x[0].length > 0 and x[1], normalized_sections))
# Set indices
i = 0
for sec, articles in normalized_sections:
i += 1
sec.index = i
sec.parent_index = 0
for sec, articles in normalized_sections:
for art in articles:
i += 1
art.index = i
art.parent_index = sec.index
for sec, normalized_articles in normalized_sections:
sec.first_child_index = normalized_articles[0].index
sec.last_child_index = normalized_articles[-1].index
# Set lengths again to close up any gaps left by filtering
for s, x in enumerate(normalized_sections):
sec, articles = x
try:
next_offset = normalized_sections[s + 1][0].offset
except:
next_offset = self.serializer.body_end_offset
sec.length = next_offset - sec.offset
for a, art in enumerate(articles):
try:
next_offset = articles[a + 1].offset
except:
next_offset = sec.next_offset
art.length = next_offset - art.offset
# Sanity check
for s, x in enumerate(normalized_sections):
sec, articles = x
try:
next_sec = normalized_sections[s + 1][0]
except:
if (sec.length == 0
or sec.next_offset != self.serializer.body_end_offset):
raise ValueError('Invalid section layout')
else:
if next_sec.offset != sec.next_offset or sec.length == 0:
raise ValueError('Invalid section layout')
for a, art in enumerate(articles):
try:
next_art = articles[a + 1]
except:
if (art.length == 0 or art.next_offset != sec.next_offset):
raise ValueError('Invalid article layout')
else:
if art.length == 0 or art.next_offset != next_art.offset:
raise ValueError('Invalid article layout')
# Flatten
indices = [periodical]
for sec, articles in normalized_sections:
indices.append(sec)
periodical.last_child_index = sec.index
for sec, articles in normalized_sections:
for a in articles:
indices.append(a)
return indices
def _populate_envvars(self):
"""Return CGI-related env vars
Mostly straight from ``CGIHTTPRequestHandler.run_cgi``.
This takes request header fields and parsed-path info and
sets env vars required by rfc3875_ and some HTTP ones.
.. _rfc3875: https://tools.ietf.org/html/rfc3875#section-4.1
"""
full_env = dict(os.environ)
# As required by git-http-backend(1); These never change.
cgi_env = dict(self.git_env)
cgi_env["GIT_HTTP_EXPORT_ALL"] = ""
cgi_env.update(self.auth_env)
# FIXME previous comment said, "Fallback for when auth isn't used,"
# but that implies auth is mandatory, which it's not
if self.path.endswith("git-receive-pack"):
cgi_env.setdefault("REMOTE_USER", full_env.get("USER", "nobody"))
# Vanilla from here on down
always = {
"SCRIPT_NAME": "git-http-backend",
"SERVER_SOFTWARE": self.version_string(),
"SERVER_NAME": self.server.server_name,
"GATEWAY_INTERFACE": "CGI/1.1",
"SERVER_PROTOCOL": self.protocol_version,
"SERVER_PORT": str(self.server.server_port),
"REQUEST_METHOD": self.command,
"REMOTE_ADDR": self.client_address[0],
}
cgi_env.update(always)
if hasattr(self.headers, "get_content_type"):
cgi_env["CONTENT_TYPE"] = self.headers.get(
"content-type", self.headers.get_content_type())
else: # 27
cgi_env["CONTENT_TYPE"] = (self.headers.typeheader
or self.headers.type)
length = self.headers.get("content-length")
if length:
cgi_env["CONTENT_LENGTH"] = length
referer = self.headers.get("referer")
if referer:
cgi_env["HTTP_REFERER"] = referer
accept = []
# Actual type is X-<custom>
for line in self.headers.getallmatchingheaders("accept"):
if line[:1] in "\t\n\r ":
accept.append(line.strip())
else:
accept += accept + line[7:].split(",")
cgi_env["HTTP_ACCEPT"] = ",".join(accept)
ua = self.headers.get("user-agent")
if ua:
cgi_env["HTTP_USER_AGENT"] = ua
if hasattr(self.headers, "get_all"):
co = filter(None, self.headers.get_all("cookie", []))
else:
co = filter(None, self.headers.getheaders("cookie"))
cookie_str = ", ".join(co)
if cookie_str:
cgi_env["HTTP_COOKIE"] = cookie_str
config["DEBUG"] and self.dlog("envvars", **cgi_env)
full_env.update(cgi_env)
# 4.1 says: "an optional meta-variable may be omitted (left unset) if
# its value is NULL" (a zero-length string).
#
# EDIT: upstream includes CONTENT_LENGTH here but it has been removed
# below. The RFC says it can be NULL but also says it MUST be set IFF
# content exists (meaning unset otherwise). Since we only read what's
# reported by the request (and discard the rest), there's no sense in
# using a fallback, no?
rfcvars = (
"QUERY_STRING",
"REMOTE_HOST", # SHOULD (can also be REMOTE_HOST)
"HTTP_USER_AGENT",
"HTTP_COOKIE",
"HTTP_REFERER",
)
for k in rfcvars:
full_env.setdefault(k, "")
return full_env
def find_pred(self, pred):
"Find all nodes where pred(tree) == True"
return filter(pred, self.iter_subtrees())
def secret_generator(self, string, *args, **kwargs):
return filter(
self.is_formally_valid,
super(JwtTokenDetector,
self).secret_generator(string, *args, **kwargs),
)
def getleastindex(elems):
"""Return the first index in elems, assuming elems isn't empty"""
return min(map(lambda elem: elem.index, filter(lambda x: x, elems)))
# The current directory is where setup.py is.
def package_files(directory):
paths = []
for (path, directories, filenames) in os.walk(directory):
for filename in filenames:
paths.append(os.path.join('..', path, filename))
return paths
extra_files = package_files('survol/www')
# The zip archive contains directories: docs, survol and tests.
with open(os.path.join('survol', '__init__.py')) as f:
__version__ = ast.parse(
next(filter(lambda line: line.startswith('__version__'),
f))).body[0].value.s
with open('README.txt') as readme_file:
README = readme_file.read()
# FIXME: survol.__doc__ = '\nSurvol library\n' ...
setup(
name='survol',
version=__version__,
description='Understanding legacy applications',
long_description=README,
author='Primhill Computers',
author_email='*****@*****.**',
url='http://www.primhillcomputers.com/survol.html',
packages=find_packages(),
def list_categories(self, name=None):
categories = self.settings.global_settings.categories
return list(filter(self.find_lambda(name), categories))
def select_many_in(inst_set, where_cond):
s = filter(where_cond, inst_set)
return xtuml.QuerySet(s)
def multi_finalize(fn, d):
appends = (d.getVar("__BBAPPEND", True) or "").split()
for append in appends:
logger.debug(2, "Appending .bbappend file %s to %s", append, fn)
bb.parse.BBHandler.handle(append, d, True)
onlyfinalise = d.getVar("__ONLYFINALISE", False)
safe_d = d
d = bb.data.createCopy(safe_d)
try:
finalize(fn, d)
except bb.parse.SkipPackage as e:
d.setVar("__SKIPPED", e.args[0])
datastores = {"": safe_d}
versions = (d.getVar("BBVERSIONS", True) or "").split()
if versions:
pv = orig_pv = d.getVar("PV", True)
baseversions = {}
def verfunc(ver, d, pv_d = None):
if pv_d is None:
pv_d = d
overrides = d.getVar("OVERRIDES", True).split(":")
pv_d.setVar("PV", ver)
overrides.append(ver)
bpv = baseversions.get(ver) or orig_pv
pv_d.setVar("BPV", bpv)
overrides.append(bpv)
d.setVar("OVERRIDES", ":".join(overrides))
versions = list(_expand_versions(versions))
for pos, version in enumerate(list(versions)):
try:
pv, bpv = version.split(":", 2)
except ValueError:
pass
else:
versions[pos] = pv
baseversions[pv] = bpv
if pv in versions and not baseversions.get(pv):
versions.remove(pv)
else:
pv = versions.pop()
# This is necessary because our existing main datastore
# has already been finalized with the old PV, we need one
# that's been finalized with the new PV.
d = bb.data.createCopy(safe_d)
verfunc(pv, d, safe_d)
try:
finalize(fn, d)
except bb.parse.SkipPackage as e:
d.setVar("__SKIPPED", e.args[0])
_create_variants(datastores, versions, verfunc)
extended = d.getVar("BBCLASSEXTEND", True) or ""
if extended:
# the following is to support bbextends with arguments, for e.g. multilib
# an example is as follows:
# BBCLASSEXTEND = "multilib:lib32"
# it will create foo-lib32, inheriting multilib.bbclass and set
# BBEXTENDCURR to "multilib" and BBEXTENDVARIANT to "lib32"
extendedmap = {}
variantmap = {}
for ext in extended.split():
eext = ext.split(':', 2)
if len(eext) > 1:
extendedmap[ext] = eext[0]
variantmap[ext] = eext[1]
else:
extendedmap[ext] = ext
pn = d.getVar("PN", True)
def extendfunc(name, d):
if name != extendedmap[name]:
d.setVar("BBEXTENDCURR", extendedmap[name])
d.setVar("BBEXTENDVARIANT", variantmap[name])
else:
d.setVar("PN", "%s-%s" % (pn, name))
bb.parse.BBHandler.inherit(extendedmap[name], fn, 0, d)
safe_d.setVar("BBCLASSEXTEND", extended)
_create_variants(datastores, extendedmap.keys(), extendfunc)
for variant, variant_d in datastores.iteritems():
if variant:
try:
if not onlyfinalise or variant in onlyfinalise:
finalize(fn, variant_d, variant)
except bb.parse.SkipPackage as e:
variant_d.setVar("__SKIPPED", e.args[0])
if len(datastores) > 1:
variants = filter(None, datastores.iterkeys())
safe_d.setVar("__VARIANTS", " ".join(variants))
datastores[""] = d
return datastores
import os
from slack_cleaner2 import SlackCleaner, match
try:
from future_builtins import filter
except ImportError:
pass
s = SlackCleaner(os.environ['TOKEN'], 1)
with s.log.group('clear bot channels'):
for msg in s.msgs(filter(match('.*-bots'), s.conversations)):
msg.delete()
# with s.log.group('delete older than 3 months'):
# for msg in filter(is_not_pinned(), s.msgs(before=a_while_ago(months=3))):
# msg.delete()
# with s.log.group('delete ims older than 1 month'):
# for msg in filter(is_not_pinned(), s.msgs(s.ims, before=a_while_ago(months=1))):
# msg.delete()
s.log.summary()
_, _, children = tree
if len(children):
return max(maxdepth(i, count + 1) for i in children.values())
else:
return count
if __name__ == "__main__":
wordlist = []
defintionlist = []
wordlist, definitionlist = dict.load_dict()
print("Creating BK-tree...\n")
dict_tree = timeof(BKTree, levenshtein, filter(len, wordlist),
filter(len, definitionlist))
print("Size of dictionary: " + str(len(wordlist)) + "\n")
while (1):
print("DICTIONARY USING BK-TREES\n")
print("1: Insert a new word into the dictionary")
print("2: Look up a word in the dictionary")
print("3: Check whether the spelling of a given word is correct")
print(
"4: Compare searches by brute query and Levenshtein distance-based query"
)
print("5: Depth")
print("6: Exit\n")
def map_filter(self, function, test):
"""Filter with test then apply function."""
if function is None:
function = lambda x: x # return results of filter
return list(map(function, filter(test, self.entities)))
def inclusive_alternation_branch(branch_node):
character_ranges = (CharacterRange.from_op_node(node)
for node in branch_node.children)
return any(
cr1.overlap(cr2) for cr1, cr2 in itertools.combinations(
filter(None, character_ranges), 2))
def parse_product(self, response):
product = response.meta['product']
cond_set(
product, 'brand',
response.xpath("//div[contains(@class,'prodTitlePlus')]"
"/span[@itemprop='brand']/text()").extract())
if not product.get('brand', None):
cond_set(
product, 'brand',
response.xpath(
'//*[@itemprop="brand"]//span/text()').extract())
cond_set(
product, 'title',
response.xpath(
"//div[contains(@class,'prodTitle')]/h1/span[@itemprop='name']"
"/text()").extract())
# Title key must be present even if it is blank
cond_set_value(product, 'title', "")
sold_out = response.xpath(
'//*[@itemprop="availability" and @href="http://schema.org/SoldOut"]'
)
cond_set_value(product, 'no_longer_available', 1 if
(not response.body or sold_out) else 0)
cond_set(
product, 'image_url',
response.xpath("//div[@id='plImageHolder']/img/@src").extract())
url = "http://m.samsclub.com/api/sams/samsapi/v2/productInfo?repositoryId={}&class=product&loadType=full&bypassEGiftCardViewOnly=true&clubId={}"
product_id = response.xpath(
'//input[@id="pProductId"]/@value').extract()[0]
product_data = json.loads(
requests.get(url.format(product_id, self.clubno)).text)
try:
price = float(
product_data.get('sa')[0].get('onlinePrice').get(
'listPrice').replace('$', '').replace(',', ''))
cond_set_value(product, 'price',
Price(price=price, priceCurrency='USD'))
except:
pass
try:
price = float(
product_data.get('sa')[0].get('onlinePrice').get(
'finalPrice').replace('$', '').replace(',', ''))
cond_set_value(product, 'price_with_discount',
Price(price=price, priceCurrency='USD'))
except:
pass
try:
price = float(
product_data.get('sa')[0].get('clubPrice').get(
'listPrice').replace('$', '').replace(',', ''))
cond_set_value(product, 'price_club',
Price(price=price, priceCurrency='USD'))
except:
pass
try:
price = float(
product_data.get('sa')[0].get('clubPrice').get(
'finalPrice').replace('$', '').replace(',', ''))
cond_set_value(product, 'price_club_with_discount',
Price(price=price, priceCurrency='USD'))
except:
pass
regex = "(prod\d+)"
reseller_id = re.findall(regex, response.url)
reseller_id = reseller_id[0] if reseller_id else None
cond_set_value(product, "reseller_id", reseller_id)
cond_set(
product,
'description',
response.xpath("//div[@itemprop='description']").extract(),
)
cond_set(product,
'model',
response.xpath("//span[@itemprop='model']/text()").extract(),
conv=string.strip)
if product.get('model', '').strip().lower() == 'null':
product['model'] = ''
product['locale'] = "en-US"
# Categories
categorie_filters = [u'sam\u2019s club']
# Clean and filter categories names from breadcrumb
bc = response.xpath('//*[@id="breadcrumb"]//a/text()').extract()
bc = [b.strip() for b in bc if b.strip()]
if not bc or len(bc) == 1:
bc = response.xpath(".//*[@id='breadcrumb']//text()").extract()
bc = [b.strip() for b in bc if b.strip()]
if not bc:
bc = response.xpath(
'//*[@id="breadcrumb"]//a//*[@itemprop="title"]/text()'
).extract()
bc = [b.strip() for b in bc if b.strip()]
categories = list(
filter((lambda x: x.lower() not in categorie_filters),
map((lambda x: x.strip()), bc)))
category = categories[-1] if categories else None
cond_set_value(product, 'categories', categories)
cond_set_value(product, 'category', category)
# Subscribe and save
subscribe_and_save = response.xpath('//*[@class="subscriptionDiv" and \
not(@style="display: none;")]/input[@id="pdpSubCheckBox"]')
cond_set_value(product, 'subscribe_and_save',
1 if subscribe_and_save else 0)
# Shpping
shipping_included = response.xpath('//*[@class="freeDelvryTxt"]')
cond_set_value(product, 'shipping_included',
1 if shipping_included else 0)
oos_in_both = response.xpath(
'//*[@class="biggraybtn" and'
' text()="Out of stock online and in club"]')
# Available in Store
available_store = response.xpath(
'//*[(@id="addtocartsingleajaxclub" or'
'@id="variantMoneyBoxButtonInitialLoadClub")'
'and contains(text(),"Pick up in Club")]') or \
response.xpath(
'//li[contains(@class,"pickupIcon")]'
'/following-sibling::li[contains'
'(translate(text(), "ABCDEFGHIJKLMNOPQRSTUVWXYZ",'
' "abcdefghijklmnopqrstuvwxyz"),"ready as soon as")]')
cond_set_value(product, 'available_store',
1 if available_store and not oos_in_both else 0)
# Available Online
available_online = response.xpath(
'//*[(@id="addtocartsingleajaxonline" \
or @id="variantMoneyBoxButtonInitialLoadOnline")]')
cond_set_value(product, 'available_online',
1 if available_online and not oos_in_both else 0)
if str(product.get('available_online', None)) == '0' and str(
product.get('available_store', None)) == '0':
product['is_out_of_stock'] = True
if not shipping_included and not product.get('no_longer_available'):
productId = ''.join(
response.xpath('//*[@id="mbxProductId"]/@value').extract())
if not productId:
productId = self._product_id(response)
pSkuId = ''.join(
response.xpath('//*[@id="mbxSkuId"]/@value').extract())
# This is fixing bug with sku and prod_id extraction for bundle products
if not productId or not pSkuId:
js_sku_prodid = response.xpath(
'.//script[contains(text(), "var skuId") and contains(text(), "var productId")]/text()'
).extract()
js_sku_prodid = ''.join(
js_sku_prodid) if js_sku_prodid else None
if js_sku_prodid:
rgx = r'(prod\d+)'
match_list = re.findall(rgx, js_sku_prodid)
productId = match_list[0] if match_list else None
rgx = r'(sku\d+)'
match_list = re.findall(rgx, js_sku_prodid)
pSkuId = match_list[0] if match_list else None
shipping_prices_url = "http://www.samsclub.com/sams/shop/product/moneybox/shippingDeliveryInfo.jsp?zipCode=%s&productId=%s&skuId=%s" % (
self.zip_code, productId, pSkuId)
return Request(shipping_prices_url,
meta={
'product': product,
'prod_id': productId
},
callback=self._parse_shipping_cost)
elif not product.get('buyer_reviews'):
productId = ''.join(
response.xpath('//*[@id="mbxProductId"]/@value').extract())
if not productId:
productId = self._product_id(response)
reviews_url = self._REVIEWS_URL.format(prod_id=productId)
return Request(reviews_url,
meta={
'product': product,
'prod_id': productId
},
callback=self._load_reviews)
return product
def find_pred(self, pred):
return filter(pred, self.iter_subtrees())
def buck2diac(buckAnalysis, keepSegmentation=True):
'''
Given a segmented and tagged Arabic word from the Buckwalter analyzer, perform
orthographic normalization to produce the surface diacritized form
(also in the Buckwalter encoding) with morphemes separated by '+' (unless
'keepSegmentation' is false, in which case the unsegmented form will be returned).
Normalization involves contracting li+Al+ to li+, adding Shadda (gemination mark) to
sun letters after Al+, changing Alef Wasla to Alef, etc.
'''
assert ' ' not in buckAnalysis, buckAnalysis
# Strip tags from morphemes
vs = '+'.join(
filter(lambda m: m != '(null)',
(x.split('/')[0] for x in buckAnalysis.split('+') if x != '')))
# Normalization
''' In MADA-3.1/MADA/ALMOR3.pm:
#update Sept 11 2008: to match up behavior of sun/moon letters in BAMA2
# code segemnt from BAMA-2
my $voc_str = $$prefix_value{"diac"}."+".$$stem_value{"diac"}."+".$$suffix_value{"diac"};
$voc_str =~ s/^((wa|fa)?(bi|ka)?Al)\+([tvd\*rzs\$SDTZln])/$1$4~/; # not moon letters
$voc_str =~ s/^((wa|fa)?lil)\+([tvd\*rzs\$SDTZln])/$1$3~/; # not moon letters
$voc_str =~ s/A\+a([pt])/A$1/; # e.g.: Al+HayA+ap
$voc_str =~ s/\{/A/g;
$voc_str =~ s/\+//g;
'''
# TODO: this is ad hoc for now--not sure if it's precisely how SAMA/MADA do it.
vs = re.sub(r'^li[+]Al[+]', 'li+l+', vs) # contraction
vs = re.sub('^([+]?)min[+]m(A|an)$', r'\1mi+m~\2',
vs) # contraction, e.g. +min/PREP+mA/REL_PRON+ -> mim~A
vs = re.sub(r'Y[+]', 'y+', vs) # Y -> y / _+
vs = re.sub(r'y[+]F', 'Y+F',
vs) # e.g. bi/PREP+maEonY/NOUN+F/CASE_INDEF_GEN -> bimaEonYF
vs = re.sub(
r'y[+]([tn])', r'yo+\1', vs
) # 'o' epenthesis? e.g. wa/CONJ+>aroDay/PV+tu/PVSUFF_SUBJ:1S+hA/PVSUFF_DO:3FS -> wa>aroDayotuhA
vs = re.sub(
r't[+]h', 'to+h', vs
) # 'o' deletion? e.g. +jaEal/PV+at/PVSUFF_SUBJ:3FS+hu/PVSUFF_DO:3MS -> jaEalatohu
vs = re.sub(r'^Einod[+]a[+]mA$', 'EindamA',
vs) # +Einod/NOUN+a/CASE_DEF_ACC+mA/SUB_CONJ+
vs = re.sub(r'^li[+][*]alika$', 'li*`lika',
vs) # +li/PREP+*alika/DEM_PRON_MS+
# end ad hoc
# add Shadda to sun letters after Al+
# NOTE: needed to add [+]? in some places after clitics
vs = re.sub(r'^((wa[+]?|fa[+]?)?(bi[+]?|ka[+]?)?Al)\+([tvd\*rzs\$SDTZln])',
r'\1\4~', vs, 1) # not moon letters
vs = re.sub(r'^((wa[+]?|fa[+]?)?li[+]?l)\+([tvd\*rzs\$SDTZln])', r'\1\3~',
vs, 1)
# not moon letters
# simplify Aa before p or t, e.g.: Al+HayA+ap
vs = re.sub(r'A\+a([pt])', r'A\1', vs, 1)
# convert Alef Wasla to plain Alef (http://en.wikipedia.org/wiki/Arabic_diacritics#.CA.BCAlif_wa.E1.B9.A3la)
vs = vs.replace('{', 'A')
# use the desired segmentation character
if not keepSegmentation:
vs = vs.replace('+', '')
return vs
