def get_older_than_required(self, t): """ Returns list of old backup sets required by new sets This function is similar to the previous one, but it only returns the times of sets which are old but part of the chains where the newer end of the chain is newer than t. """ assert self.values_set new_chains = filter(lambda c: c.end_time >= t, self.all_backup_chains) result_sets = [] for chain in new_chains: old_sets = filter(lambda s: s.get_time() < t, chain.get_all_sets()) result_sets.extend(old_sets) return self.sort_sets(result_sets)
def get_containing_volumes(self, index_prefix): """ Return list of volume numbers that may contain index_prefix """ return filter(lambda vol_num: self.volume_info_dict[vol_num].contains(index_prefix), self.volume_info_dict.keys())
def first(predicate_or_None, iterable, default=None): """ Returns the first item of iterable for which predicate(item) is true. If predicate is None, matches the first item that is true. Returns value of default in case of no matching items. """ return next(filter(predicate_or_None, iterable), default)
def partition(iterable, pred): """Use a predicate to partition entries into false entries and true \ entries. Parameters ---------- iterable : `collections.Iterable` Iterable to partition. pred : `collections.Callable` The predicate which determines the group in which the value of the iterable belongs. Returns ------- false_values : generator An iterable containing the values for which the predicate was True. true_values : generator An iterable containing the values for which the predicate was True. Examples -------- >>> from nddata.utils.itertools_recipes import partition >>> def is_odd(val): return val % 2 >>> [list(i) for i in partition(range(10), is_odd)] [[0, 2, 4, 6, 8], [1, 3, 5, 7, 9]] """ t1, t2 = tee(iterable) return filterfalse(pred, t1), filter(pred, t2)
def add(self, code, toplevel_code=None): if code in self: return if toplevel_code is None: filename = code.co_filename if filename.endswith((".pyc", ".pyo")): filename = filename[:-1] if not os.path.exists(filename): print('ERROR: Could not find file ' + filename) if filename.startswith(("ipython-input", "<ipython-input")): print( "NOTE: %mprun can only be used on functions defined in" " physical files, and not in the IPython environment.") return toplevel_code = code (sub_lines, start_line) = inspect.getsourcelines(code) linenos = range(start_line, start_line + len(sub_lines)) self._toplevel.append((filename, code, linenos)) self[code] = {} else: self[code] = self[toplevel_code] for subcode in filter(inspect.iscode, code.co_consts): self.add(subcode, toplevel_code=toplevel_code)
def __init__(self, searcher, query, field, terms=False, fields=False, tag='', formatter=None, encoder=None): if tag: formatter = highlight.SimpleHTMLFormatter('<{}>'.format(tag), '</{}>'.format(tag)) scorer = (highlight.QueryTermScorer if terms else highlight.QueryScorer)(query, *(searcher.indexReader, field) * (not fields)) highlight.Highlighter.__init__(self, *filter(None, [formatter, encoder, scorer])) self.searcher, self.field = searcher, field self.selector = HashSet(Arrays.asList([field]))
def find_duplicate_filenames(paths): d = defaultdict(list) for path in paths: key = os.path.basename(path).lower() d[key].append(path) return filter(lambda x: len(x[1]) > 1, d.items())
def solve(exclusive_limit): inclusive_limit = exclusive_limit - 1 numbers = islice(count(1), inclusive_limit) multiples = filter(multiple_of_3_or_5, numbers) return sum(multiples)
def get_all_source_files(paths, exclude_patterns): ''' Function counts md5 hash for the given file and checks if it isn't a duplicate using set of hashes for previous files ''' hash_set = set() def _validate_file(pathname): return ( pathname in paths or ( CodeReader.get_reader(pathname) and all(not fnmatch(pathname, p) for p in exclude_patterns) and _not_duplicate(pathname))) def _not_duplicate(full_path_name): fhash = md5_hash_file(full_path_name) if not fhash or fhash not in hash_set: hash_set.add(fhash) return True def all_listed_files(paths): for path in paths: if os.path.isfile(path): yield path else: for root, _, files in os.walk(path, topdown=False): for filename in files: yield os.path.join(root, filename) return filter(_validate_file, all_listed_files(paths))
def __init__(self, searcher, query, field, terms=False, fields=False, tag='', formatter=None, encoder=None): if tag: formatter = lucene.SimpleHTMLFormatter('<{0}>'.format(tag), '</{0}>'.format(tag)) scorer = (lucene.QueryTermScorer if terms else lucene.QueryScorer)(query, *(searcher.indexReader, field) * (not fields)) lucene.Highlighter.__init__(self, *filter(None, [formatter, encoder, scorer])) self.searcher, self.field = searcher, field self.selector = lucene.MapFieldSelector([field])
def _generate_constant_defs(macro_defs, macro_type_map): constants = '\n'.join(map( lambda (name, macro): ' constexpr {type} const {name} = {value};'.format( type=macro_type_map[name], name=name, value=list(macro.get_tokens())[1].spelling), filter(lambda (name, _): name in macro_type_map, macro_defs.items()))) if 'OFP_NO_BUFFER' not in macro_defs: return '\n'.join([constants, ' constexpr std::uint32_t const OFP_NO_BUFFER = 0xffffffff;']) return constants
def correct(self, word): "Generate ordered sets of words by increasing edit distance." previous, edits = set(), {word: 0} for distance in range(len(word)): yield sorted(filter(self.__contains__, edits), key=self.__getitem__, reverse=True) previous.update(edits) groups = map(self.edits, edits, edits.values()) edits = dict((edit, group[edit]) for group in groups for edit in group if edit not in previous)
def traverse_node(node, filename, func, level=0): if func(node, level): return # indent_str = ''.join(itertools.repeat(' ', level)) # print "{}{} : {}".format(indent_str, node.kind.name, node.displayname) for child in filter( lambda node: str(node.location.file) == filename, node.get_children()): traverse_node(child, filename, func, level + 1)
def load_csv(profile, csv_file): """ Return tuple (column_names, data_rows). column_names - a list with selected column names data_rows - an iterator over row tuples (dn, timestamp, values) """ csv_reader = create_csv_reader(profile, csv_file) header = csv_reader.next() fields = profile.field_selector(header) values = ValuesExtractor(fields) check_header(header, fields) header_checks = [ check for check in [ profile.timestamp.header_check(), profile.identifier.header_check() ] if not check is None ] for check in header_checks: check(header) record_checks = [ check for check in [ profile.timestamp.record_check(), profile.identifier.record_check() ] if not check is None ] include_record = partial(record_passes_checks, record_checks) include_row = create_row_check(header) records = filter( include_record, ( dict(zip(header, [item.decode('utf-8') for item in row])) for line_nr, row in enumerate(csv_reader) if profile.ignore_field_mismatches or include_row(line_nr, row) ) ) extract_raw_data_row = compose(tuple, raw_data_row_extractor( profile.identifier.from_record, profile.timestamp.from_record, values.from_record )) return fields, map(extract_raw_data_row, records)
def tracemalloc_tool(): # .. cross-platform but but requires Python 3.4 or higher .. stat = next(filter(lambda item: str(item).startswith(filename), tracemalloc.take_snapshot().statistics('filename'))) mem = stat.size / _TWO_20 if timestamps: return mem, time.time() else: return mem
def components(self, field, reader): source = tokens = ( self.tokenizer.tokenStream(field, reader) if isinstance(self.tokenizer, analysis.Analyzer) else self.tokenizer(reader) ) for filter in self.filters: tokens = filter(tokens) return source, tokens
def test_overall(self): results = draftcombine.SpotShooting(season=self.season) assert results overall = results.overall() assert overall stats = next(filter(lambda d: d['PLAYER_NAME'] == self.player_name, overall)) assert stats assert stats['PLAYER_NAME'] == self.player_name assert stats['COLLEGE_BREAK_LEFT_MADE'] == None assert stats['COLLEGE_BREAK_LEFT_PCT'] == None assert stats['COLLEGE_BREAK_RIGHT_ATTEMPT'] == None assert stats['COLLEGE_BREAK_RIGHT_MADE'] == None assert stats['COLLEGE_BREAK_RIGHT_PCT'] == None assert stats['COLLEGE_CORNER_LEFT_ATTEMPT'] == None assert stats['COLLEGE_CORNER_LEFT_MADE'] == None assert stats['COLLEGE_CORNER_LEFT_PCT'] == None assert stats['COLLEGE_CORNER_RIGHT_ATTEMPT'] == None assert stats['COLLEGE_CORNER_RIGHT_MADE'] == None assert stats['COLLEGE_CORNER_RIGHT_PCT'] == None assert stats['COLLEGE_TOP_KEY_ATTEMPT'] == None assert stats['COLLEGE_TOP_KEY_MADE'] == None assert stats['COLLEGE_TOP_KEY_PCT'] == None assert stats['FIFTEEN_BREAK_LEFT_ATTEMPT'] == None assert stats['FIFTEEN_BREAK_LEFT_MADE'] == None assert stats['FIFTEEN_BREAK_LEFT_PCT'] == None assert stats['FIFTEEN_BREAK_RIGHT_ATTEMPT'] == None assert stats['FIFTEEN_BREAK_RIGHT_MADE'] == None assert stats['FIFTEEN_BREAK_RIGHT_PCT'] == None assert stats['FIFTEEN_CORNER_LEFT_ATTEMPT'] == None assert stats['FIFTEEN_CORNER_LEFT_MADE'] == None assert stats['FIFTEEN_CORNER_LEFT_PCT'] == None assert stats['FIFTEEN_CORNER_RIGHT_ATTEMPT'] == None assert stats['FIFTEEN_CORNER_RIGHT_MADE'] == None assert stats['FIFTEEN_CORNER_RIGHT_PCT'] == None assert stats['FIFTEEN_TOP_KEY_ATTEMPT'] == None assert stats['FIFTEEN_TOP_KEY_MADE'] == None assert stats['FIFTEEN_TOP_KEY_PCT'] == None assert stats['NBA_BREAK_LEFT_ATTEMPT'] == None assert stats['NBA_BREAK_LEFT_MADE'] == None assert stats['NBA_BREAK_LEFT_PCT'] == None assert stats['NBA_BREAK_RIGHT_ATTEMPT'] == None assert stats['NBA_BREAK_RIGHT_MADE'] == None assert stats['NBA_BREAK_RIGHT_PCT'] == None assert stats['NBA_CORNER_LEFT_ATTEMPT'] == None assert stats['NBA_CORNER_LEFT_MADE'] == None assert stats['NBA_CORNER_LEFT_PCT'] == None assert stats['NBA_CORNER_RIGHT_ATTEMPT'] == None assert stats['NBA_CORNER_RIGHT_MADE'] == None assert stats['NBA_CORNER_RIGHT_PCT'] == None assert stats['NBA_TOP_KEY_ATTEMPT'] == None assert stats['NBA_TOP_KEY_MADE'] == None assert stats['NBA_TOP_KEY_PCT'] == None assert stats['PLAYER_ID'] == None
def translate_query_params(cls, query=None, **kwargs): """ Translate an arbirtary keyword argument to the expected query. In the v2 API, many endpoints expect a particular query argument to be in the form of `query=xxx` where `xxx` would be the name of perhaps the name, ID or otherwise. This function ought to take a more aptly named parameter specified in `TRANSLATE_QUERY_PARAM`, and substitute it into the `query` keyword argument. The purpose is so that some models (optionally) have nicer named keyword arguments than `query` for easier to read python. If a query argument is given then the output should be that value. If a substitute value is given as a keyword specified in `TRANSLATE_QUERY_PARAM`(and query is not) then the `query` argument will be that keyword argument. Eg. No query param query = None TRANSLATE_QUERY_PARAM = ('name',) kwargs = {'name': 'PagerDuty'} ... output = {'query': 'PagerDuty'} or, query param explicitly query = 'XXXXPlopperDuty' TRANSLATE_QUERY_PARAM = ('name',) kwargs = {'name': 'PagerDuty'} ... output = {'query': 'XXXXPlopperDuty'} XXX: Clean this up. It's *too* flexible. """ values = [] output = kwargs.copy() # remove any of the TRANSLATE_QUERY_PARAMs in output for param in cls.TRANSLATE_QUERY_PARAM: popped = output.pop(param, None) if popped is not None: values.append(popped) # if query is not provided, use the first parameter we removed from # the kwargs try: output['query'] = next(filter(None, values)) except StopIteration: pass # if query is provided, just use it if query is not None: output['query'] = query return output
def parse_query(query): matches = grammar.findall(query) if not matches: raise QueryParseException("wrong query") fields = list(map(parse_field, [first(filter(None, match)) for match in matches])) return fields
def _generate_is_error_code_enum_specialization(version, error_type_enum, ignore_types): return '\n\n'.join(map( lambda error: ( """\ template <> struct is_error_code_enum<canard::network::openflow::v{version}::ofp_{name}_code> {{ static bool const value = true; }};\ """.format(version=version, name=_strip_OFPET(error.displayname))), filter(lambda enum: enum.displayname not in ignore_types, error_type_enum.get_children())))
def buck2diac(buckAnalysis, keepSegmentation=True): ''' Given a segmented and tagged Arabic word from the Buckwalter analyzer, perform orthographic normalization to produce the surface diacritized form (also in the Buckwalter encoding) with morphemes separated by '+' (unless 'keepSegmentation' is false, in which case the unsegmented form will be returned). Normalization involves contracting li+Al+ to li+, adding Shadda (gemination mark) to sun letters after Al+, changing Alef Wasla to Alef, etc. ''' assert ' ' not in buckAnalysis,buckAnalysis # Strip tags from morphemes vs = '+'.join(filter(lambda m: m!='(null)', (x.split('/')[0] for x in buckAnalysis.split('+') if x!=''))) # Normalization ''' In MADA-3.1/MADA/ALMOR3.pm: #update Sept 11 2008: to match up behavior of sun/moon letters in BAMA2 # code segemnt from BAMA-2 my $voc_str = $$prefix_value{"diac"}."+".$$stem_value{"diac"}."+".$$suffix_value{"diac"}; $voc_str =~ s/^((wa|fa)?(bi|ka)?Al)\+([tvd\*rzs\$SDTZln])/$1$4~/; # not moon letters $voc_str =~ s/^((wa|fa)?lil)\+([tvd\*rzs\$SDTZln])/$1$3~/; # not moon letters $voc_str =~ s/A\+a([pt])/A$1/; # e.g.: Al+HayA+ap $voc_str =~ s/\{/A/g; $voc_str =~ s/\+//g; ''' # TODO: this is ad hoc for now--not sure if it's precisely how SAMA/MADA do it. vs = re.sub(r'^li[+]Al[+]', 'li+l+', vs) # contraction vs = re.sub('^([+]?)min[+]m(A|an)$', r'\1mi+m~\2', vs) # contraction, e.g. +min/PREP+mA/REL_PRON+ -> mim~A vs = re.sub(r'Y[+]', 'y+', vs) # Y -> y / _+ vs = re.sub(r'y[+]F', 'Y+F', vs) # e.g. bi/PREP+maEonY/NOUN+F/CASE_INDEF_GEN -> bimaEonYF vs = re.sub(r'y[+]([tn])', r'yo+\1', vs) # 'o' epenthesis? e.g. wa/CONJ+>aroDay/PV+tu/PVSUFF_SUBJ:1S+hA/PVSUFF_DO:3FS -> wa>aroDayotuhA vs = re.sub(r't[+]h', 'to+h', vs) # 'o' deletion? e.g. +jaEal/PV+at/PVSUFF_SUBJ:3FS+hu/PVSUFF_DO:3MS -> jaEalatohu vs = re.sub(r'^Einod[+]a[+]mA$', 'EindamA', vs) # +Einod/NOUN+a/CASE_DEF_ACC+mA/SUB_CONJ+ vs = re.sub(r'^li[+][*]alika$', 'li*`lika', vs) # +li/PREP+*alika/DEM_PRON_MS+ # end ad hoc # add Shadda to sun letters after Al+ # NOTE: needed to add [+]? in some places after clitics vs = re.sub(r'^((wa[+]?|fa[+]?)?(bi[+]?|ka[+]?)?Al)\+([tvd\*rzs\$SDTZln])', r'\1\4~', vs, 1) # not moon letters vs = re.sub(r'^((wa[+]?|fa[+]?)?li[+]?l)\+([tvd\*rzs\$SDTZln])', r'\1\3~', vs, 1); # not moon letters # simplify Aa before p or t, e.g.: Al+HayA+ap vs = re.sub(r'A\+a([pt])', r'A\1', vs, 1) # convert Alef Wasla to plain Alef (http://en.wikipedia.org/wiki/Arabic_diacritics#.CA.BCAlif_wa.E1.B9.A3la) vs = vs.replace('{', 'A') # use the desired segmentation character if not keepSegmentation: vs = vs.replace('+', '') return vs
def _generate_make_error_code_funcs(version, error_type_enum, ignore_types): return '\n\n'.join(map( lambda error: ( """\ inline auto make_error_code(ofp_{name}_code const e) -> boost::system::error_code {{ return {{e + 1, v{version}::{name}_category()}}; }}\ """.format(version=version, name=_strip_OFPET(error.displayname))), filter(lambda enum: enum.displayname not in ignore_types, error_type_enum.get_children())))
def _generate_error_category_funcs(version, error_type_enum, ignore_types): return '\n\n'.join(map( lambda error: ( """\ inline auto {name}_category() -> boost::system::error_category& {{ static v{version}_detail::{name}_category instance{{}}; return instance; }}\ """.format(version=version, name=_strip_OFPET(error.displayname))), filter(lambda enum: enum.displayname not in ignore_types, error_type_enum.get_children())))
def parse_class(node): class_assigments = filter(lambda x:x.__class__.__name__ == 'Assign', ast.iter_child_nodes(node)) found = {} for node in class_assigments: targets = {getattr(t, 'id', None) for t in node.targets} targets.discard(None) fields = field_names.intersection(targets) if fields: val = convert_node(fields, node.value, names=names, import_data=import_data) for field in fields: found[field] = val return found
def __call__(self, where_clause=None): ''' The navigation chain is invoked. Optionally, a conditional *where-clause* in the form of a function may be provided, e.g >>> chain(lambda selected: selected.Name == 'test') ''' handle = self.handle or list() if where_clause: handle = filter(where_clause, handle) return QuerySet(handle)
def select_one(self, where_clause=None): ''' Select a single instance from the instance pool. Optionally, a conditional *where-clause* in the form of a function may be provided. ''' if isinstance(where_clause, dict): s = self.query(where_clause) elif where_clause: s = iter(filter(where_clause, self.storage)) else: s = iter(self.storage) return next(s, None)
def select_many(self, where_clause=None): ''' Select several instances from the instance pool. Optionally, a conditional *where-clause* in the form of a function may be provided. ''' if isinstance(where_clause, dict): s = self.query(where_clause) elif where_clause: s = filter(where_clause, self.storage) else: s = iter(self.storage) return QuerySet(s)
def glob_get_filename_sf(self, filename, include): """Get a selection function given a normal filename Some of the parsing is better explained in filelist_parse_line. The reason this is split from normal globbing is things are a lot less complicated if no special globbing characters are used. """ if not filename.startswith(self.prefix): raise FilePrefixError(filename) index = tuple(filter(lambda x: x, filename[len(self.prefix):].split("/"))) return self.glob_get_tuple_sf(index, include)
def _generate_member_ppseq(version, members, is_skip_ofp_match): def to_type_and_name_tuple(mem): if mem.is_array(): return _enumerate_array(version, mem) return ' ({type}, {name})'.format( type=_maybe_qualify(version, mem.type), name=mem.name) return '\n'.join(map( to_type_and_name_tuple, filter( lambda mem: not ( (is_skip_ofp_match and mem.type == 'ofp_match') or (mem.is_array() and (mem.element_count == 0 or mem.element_type == 'char'))), members)))
def sort_reflexive(set_of_instances, rel_id, phrase): ''' Sort a *set of instances* in the order they appear across a conditional and reflexive association. The first instance in the resulting ordered set is **not** associated to an instance across the given *phrase*. ''' if not isinstance(set_of_instances, QuerySet): raise MetaException('The collection to sort must be a QuerySet') if not set_of_instances.first: return QuerySet() if isinstance(rel_id, int): rel_id = 'R%d' % rel_id # Figure out the phrase in the other direction metaclass = get_metaclass(set_of_instances.first) for link in metaclass.links.values(): if link.to_metaclass != metaclass: continue if link.rel_id != rel_id: continue if link.phrase == phrase: continue other_phrase = link.phrase break else: raise UnknownLinkException(metaclass.kind, metaclass.kind, rel_id, phrase) first_filt = lambda sel: not navigate_one(sel).nav(metaclass.kind, rel_id, phrase)() first_instances = list(filter(first_filt, set_of_instances)) if not first_instances: #the instance sequence is recursive, start anywhere first_instances = [set_of_instances.first] def sequence_generator(): for first in first_instances: inst = first while inst: if inst in set_of_instances: yield inst inst = navigate_one(inst).nav(metaclass.kind, rel_id, other_phrase)() if inst is first: break return QuerySet(sequence_generator())
def _case_for_each_member(members, ignore_to_string_enums): return '\n'.join(map( lambda mem: ' case {name}: return "{name}";'.format(name=mem.displayname), filter(lambda mem: mem.displayname not in ignore_to_string_enums, members)))
def _create_path(self, *args): """Create URL path for endpoint and args.""" args = filter(None, args) path = self.endpoint + '/'.join(args) + '.%s' % (self.format) return path
def browser(request, path='', template="cloud_browser/browser.html"): """View files in a file path. :param request: The request. :param path: Path to resource, including container as first part of path. :param template: Template to render. """ from itertools import islice try: # pylint: disable=redefined-builtin from future_builtins import filter except ImportError: # pylint: disable=import-error from builtins import filter # Inputs. container_path, object_path = path_parts(path) incoming = request.POST or request.GET or {} marker = incoming.get('marker', None) marker_part = incoming.get('marker_part', None) if marker_part: marker = path_join(object_path, marker_part) # Get and adjust listing limit. limit_default = settings.CLOUD_BROWSER_DEFAULT_LIST_LIMIT def limit_test(num): return num > 0 and (MAX_LIMIT is None or num <= MAX_LIMIT - 1) limit = get_int(incoming.get('limit', limit_default), limit_default, limit_test) # Q1: Get all containers. # We optimize here by not individually looking up containers later, # instead going through this in-memory list. # TODO: Should page listed containers with a ``limit`` and ``marker``. conn = get_connection() containers = conn.get_containers() marker_part = None container = None objects = None if container_path != '': # Find marked container from list. def cont_eq(container): return container.name == container_path filtered_conts = filter(cont_eq, containers) cont_list = list(islice(filtered_conts, 1)) if not cont_list: raise Http404("No container at: %s" % container_path) # Q2: Get objects for instant list, plus one to check "next". container = cont_list[0] objects = container.get_objects(object_path, marker, limit + 1) marker = None # If over limit, strip last item and set marker. if len(objects) == limit + 1: objects = objects[:limit] marker = objects[-1].name marker_part = relpath(marker, object_path) return render( request, template, { 'path': path, 'marker': marker, 'marker_part': marker_part, 'limit': limit, 'breadcrumbs': _breadcrumbs(path), 'container_path': container_path, 'containers': containers, 'container': container, 'object_path': object_path, 'objects': objects })
def parse_cmdline_options(arglist): """Parse argument list""" global select_opts, select_files, full_backup global list_current, collection_status, cleanup, remove_time, verify def set_log_fd(fd): if fd < 1: raise optparse.OptionValueError( "log-fd must be greater than zero.") log.add_fd(fd) def set_time_sep(sep, opt): if sep == '-': raise optparse.OptionValueError( "Dash ('-') not valid for time-separator.") globals.time_separator = sep old_fn_deprecation(opt) def add_selection(o, s, v, p): select_opts.append((s, v)) def add_filelist(o, s, v, p): filename = v select_opts.append((s, filename)) try: select_files.append(open(filename, "r")) except IOError: log.FatalError( _("Error opening file %s") % filename, log.ErrorCode.cant_open_filelist) def print_ver(o, s, v, p): print "duplicity %s" % (globals.version) sys.exit(0) def add_rename(o, s, v, p): globals.rename[os.path.normcase(os.path.normpath(v[0]))] = v[1] parser = OPHelpFix(option_class=DupOption, usage=usage()) # If this is true, only warn and don't raise fatal error when backup # source directory doesn't match previous backup source directory. parser.add_option("--allow-source-mismatch", action="store_true") # Set to the path of the archive directory (the directory which # contains the signatures and manifests of the relevent backup # collection), and for checkpoint state between volumes. # TRANSL: Used in usage help to represent a Unix-style path name. Example: # --archive-dir <path> parser.add_option("--archive-dir", type="file", metavar=_("path")) # Asynchronous put/get concurrency limit # (default of 0 disables asynchronicity). parser.add_option("--asynchronous-upload", action="store_const", const=1, dest="async_concurrency") parser.add_option("--compare-data", action="store_true") # config dir for future use parser.add_option("--config-dir", type="file", metavar=_("path"), help=optparse.SUPPRESS_HELP) # for testing -- set current time parser.add_option("--current-time", type="int", dest="current_time", help=optparse.SUPPRESS_HELP) # Don't actually do anything, but still report what would be done parser.add_option("--dry-run", action="store_true") # TRANSL: Used in usage help to represent an ID for a GnuPG key. Example: # --encrypt-key <gpg_key_id> parser.add_option("--encrypt-key", type="string", metavar=_("gpg-key-id"), dest="", action="callback", callback=lambda o, s, v, p: globals.gpg_profile. recipients.append(v)) # @UndefinedVariable # secret keyring in which the private encrypt key can be found parser.add_option("--encrypt-secret-keyring", type="string", metavar=_("path")) parser.add_option( "--encrypt-sign-key", type="string", metavar=_("gpg-key-id"), dest="", action="callback", callback=lambda o, s, v, p: (globals.gpg_profile.recipients.append(v), set_sign_key(v))) # TRANSL: Used in usage help to represent a "glob" style pattern for # matching one or more files, as described in the documentation. # Example: # --exclude <shell_pattern> parser.add_option("--exclude", action="callback", metavar=_("shell_pattern"), dest="", type="string", callback=add_selection) parser.add_option("--exclude-device-files", action="callback", dest="", callback=add_selection) parser.add_option("--exclude-filelist", type="file", metavar=_("filename"), dest="", action="callback", callback=add_filelist) parser.add_option("--exclude-filelist-stdin", action="callback", dest="", callback=lambda o, s, v, p: (select_opts.append( ("--exclude-filelist", "standard input")), select_files.append(sys.stdin), stdin_deprecation(o)), help=optparse.SUPPRESS_HELP) parser.add_option( "--exclude-globbing-filelist", type="file", metavar=_("filename"), dest="", action="callback", callback=lambda o, s, v, p: (add_filelist(o, s, v, p), old_globbing_filelist_deprecation(s)), help=optparse.SUPPRESS_HELP) # TRANSL: Used in usage help to represent the name of a file. Example: # --log-file <filename> parser.add_option("--exclude-if-present", metavar=_("filename"), dest="", type="file", action="callback", callback=add_selection) parser.add_option("--exclude-other-filesystems", action="callback", dest="", callback=add_selection) # TRANSL: Used in usage help to represent a regular expression (regexp). parser.add_option("--exclude-regexp", metavar=_("regular_expression"), dest="", type="string", action="callback", callback=add_selection) # Exclude any files with modification dates older than this from the backup parser.add_option("--exclude-older-than", type="time", metavar=_("time"), dest="", action="callback", callback=add_selection) # Whether we should be particularly aggressive when cleaning up parser.add_option("--extra-clean", action="store_true") # used in testing only - raises exception after volume parser.add_option("--fail-on-volume", type="int", help=optparse.SUPPRESS_HELP) # used to provide a prefix on top of the defaul tar file name parser.add_option("--file-prefix", type="string", dest="file_prefix", action="store") # used to provide a suffix for manifest files only parser.add_option("--file-prefix-manifest", type="string", dest="file_prefix_manifest", action="store") # used to provide a suffix for archive files only parser.add_option("--file-prefix-archive", type="string", dest="file_prefix_archive", action="store") # used to provide a suffix for sigature files only parser.add_option("--file-prefix-signature", type="string", dest="file_prefix_signature", action="store") # used in testing only - skips upload for a given volume parser.add_option("--skip-volume", type="int", help=optparse.SUPPRESS_HELP) # If set, restore only the subdirectory or file specified, not the # whole root. # TRANSL: Used in usage help to represent a Unix-style path name. Example: # --archive-dir <path> parser.add_option("--file-to-restore", "-r", action="callback", type="file", metavar=_("path"), dest="restore_dir", callback=lambda o, s, v, p: setattr( p.values, "restore_dir", v.rstrip('/'))) # Used to confirm certain destructive operations like deleting old files. parser.add_option("--force", action="store_true") # FTP data connection type parser.add_option("--ftp-passive", action="store_const", const="passive", dest="ftp_connection") parser.add_option("--ftp-regular", action="store_const", const="regular", dest="ftp_connection") # If set, forces a full backup if the last full backup is older than # the time specified parser.add_option("--full-if-older-than", type="time", dest="full_force_time", metavar=_("time")) parser.add_option("--gio", action="callback", dest="use_gio", callback=lambda o, s, v, p: (setattr(p.values, o.dest, True), old_fn_deprecation(s))) parser.add_option("--gpg-binary", type="file", metavar=_("path")) parser.add_option("--gpg-options", action="extend", metavar=_("options")) # TRANSL: Used in usage help to represent an ID for a hidden GnuPG key. Example: # --hidden-encrypt-key <gpg_key_id> parser.add_option("--hidden-encrypt-key", type="string", metavar=_("gpg-key-id"), dest="", action="callback", callback=lambda o, s, v, p: globals.gpg_profile. hidden_recipients.append(v)) # @UndefinedVariable # ignore (some) errors during operations; supposed to make it more # likely that you are able to restore data under problematic # circumstances. the default should absolutely always be False unless # you know what you are doing. parser.add_option( "--ignore-errors", action="callback", dest="ignore_errors", callback=lambda o, s, v, p: (log.Warn( _("Running in 'ignore errors' mode due to %s; please " "re-consider if this was not intended") % s), setattr(p.values, "ignore_errors", True))) # Whether to use the full email address as the user name when # logging into an imap server. If false just the user name # part of the email address is used. parser.add_option("--imap-full-address", action="store_true") # Name of the imap folder where we want to store backups. # Can be changed with a command line argument. # TRANSL: Used in usage help to represent an imap mailbox parser.add_option("--imap-mailbox", metavar=_("imap_mailbox")) parser.add_option("--include", action="callback", metavar=_("shell_pattern"), dest="", type="string", callback=add_selection) parser.add_option("--include-filelist", type="file", metavar=_("filename"), dest="", action="callback", callback=add_filelist) parser.add_option("--include-filelist-stdin", action="callback", dest="", callback=lambda o, s, v, p: (select_opts.append( ("--include-filelist", "standard input")), select_files.append(sys.stdin), stdin_deprecation(o)), help=optparse.SUPPRESS_HELP) parser.add_option( "--include-globbing-filelist", type="file", metavar=_("filename"), dest="", action="callback", callback=lambda o, s, v, p: (add_filelist(o, s, v, p), old_globbing_filelist_deprecation(s)), help=optparse.SUPPRESS_HELP) parser.add_option("--include-regexp", metavar=_("regular_expression"), dest="", type="string", action="callback", callback=add_selection) parser.add_option("--log-fd", type="int", metavar=_("file_descriptor"), dest="", action="callback", callback=lambda o, s, v, p: set_log_fd(v)) # TRANSL: Used in usage help to represent the name of a file. Example: # --log-file <filename> parser.add_option("--log-file", type="file", metavar=_("filename"), dest="", action="callback", callback=lambda o, s, v, p: log.add_file(v)) # Maximum block size for large files parser.add_option("--max-blocksize", type="int", metavar=_("number")) # TRANSL: Used in usage help (noun) parser.add_option("--name", dest="backup_name", metavar=_("backup name")) # If set to false, then do not encrypt files on remote system parser.add_option("--no-encryption", action="store_false", dest="encryption") # If set to false, then do not compress files on remote system parser.add_option("--no-compression", action="store_false", dest="compression") # If set, print the statistics after every backup session parser.add_option("--no-print-statistics", action="store_false", dest="print_statistics") # If true, filelists and directory statistics will be split on # nulls instead of newlines. parser.add_option("--null-separator", action="store_true") # number of retries on network operations # TRANSL: Used in usage help to represent a desired number of # something. Example: # --num-retries <number> parser.add_option("--num-retries", type="int", metavar=_("number")) # File owner uid keeps number from tar file. Like same option in GNU tar. parser.add_option("--numeric-owner", action="store_true") # Whether the old filename format is in effect. parser.add_option("--old-filenames", action="callback", dest="old_filenames", callback=lambda o, s, v, p: (setattr(p.values, o.dest, True), old_fn_deprecation(s))) # Level of Redundancy in % for Par2 files parser.add_option("--par2-redundancy", type="int", metavar=_("number")) # Verbatim par2 options parser.add_option("--par2-options", action="extend", metavar=_("options")) # Used to display the progress for the full and incremental backup operations parser.add_option("--progress", action="store_true") # Used to control the progress option update rate in seconds. Default: prompts each 3 seconds parser.add_option("--progress-rate", type="int", metavar=_("number")) # option to trigger Pydev debugger parser.add_option("--pydevd", action="store_true") # option to rename files during restore parser.add_option("--rename", type="file", action="callback", nargs=2, callback=add_rename) # Restores will try to bring back the state as of the following time. # If it is None, default to current time. # TRANSL: Used in usage help to represent a time spec for a previous # point in time, as described in the documentation. Example: # duplicity remove-older-than time [options] target_url parser.add_option("--restore-time", "--time", "-t", type="time", metavar=_("time")) # user added rsync options parser.add_option("--rsync-options", action="extend", metavar=_("options")) # Whether to create European buckets (sorry, hard-coded to only # support european for now). parser.add_option("--s3-european-buckets", action="store_true") # Whether to use S3 Reduced Redudancy Storage parser.add_option("--s3-use-rrs", action="store_true") # Whether to use S3 Infrequent Access Storage parser.add_option("--s3-use-ia", action="store_true") # Whether to use "new-style" subdomain addressing for S3 buckets. Such # use is not backwards-compatible with upper-case buckets, or buckets # that are otherwise not expressable in a valid hostname. parser.add_option("--s3-use-new-style", action="store_true") # Whether to use plain HTTP (without SSL) to send data to S3 # See <https://bugs.launchpad.net/duplicity/+bug/433970>. parser.add_option("--s3-unencrypted-connection", action="store_true") # Chunk size used for S3 multipart uploads.The number of parallel uploads to # S3 be given by chunk size / volume size. Use this to maximize the use of # your bandwidth. Defaults to 25MB parser.add_option( "--s3-multipart-chunk-size", type="int", action="callback", metavar=_("number"), callback=lambda o, s, v, p: setattr( p.values, "s3_multipart_chunk_size", v * 1024 * 1024)) # Number of processes to set the Processor Pool to when uploading multipart # uploads to S3. Use this to control the maximum simultaneous uploads to S3. parser.add_option("--s3-multipart-max-procs", type="int", metavar=_("number")) # Number of seconds to wait for each part of a multipart upload to S3. Use this # to prevent hangups when doing a multipart upload to S3. parser.add_option("--s3-multipart-max-timeout", type="int", metavar=_("number")) # Option to allow the s3/boto backend use the multiprocessing version. parser.add_option("--s3-use-multiprocessing", action="store_true") # Option to allow use of server side encryption in s3 parser.add_option("--s3-use-server-side-encryption", action="store_true", dest="s3_use_sse") # scp command to use (ssh pexpect backend) parser.add_option("--scp-command", metavar=_("command")) # sftp command to use (ssh pexpect backend) parser.add_option("--sftp-command", metavar=_("command")) # allow the user to switch cloudfiles backend parser.add_option("--cf-backend", metavar=_("pyrax|cloudfiles")) # If set, use short (< 30 char) filenames for all the remote files. parser.add_option("--short-filenames", action="callback", dest="short_filenames", callback=lambda o, s, v, p: (setattr(p.values, o.dest, True), old_fn_deprecation(s))) # TRANSL: Used in usage help to represent an ID for a GnuPG key. Example: # --encrypt-key <gpg_key_id> parser.add_option("--sign-key", type="string", metavar=_("gpg-key-id"), dest="", action="callback", callback=lambda o, s, v, p: set_sign_key(v)) # default to batch mode using public-key encryption parser.add_option("--ssh-askpass", action="store_true") # user added ssh options parser.add_option("--ssh-options", action="extend", metavar=_("options")) # user added ssl options (webdav backend) parser.add_option( "--ssl-cacert-file", metavar=_("pem formatted bundle of certificate authorities")) parser.add_option("--ssl-no-check-certificate", action="store_true") # Working directory for the tempfile module. Defaults to /tmp on most systems. parser.add_option("--tempdir", dest="temproot", type="file", metavar=_("path")) # network timeout value # TRANSL: Used in usage help. Example: # --timeout <seconds> parser.add_option("--timeout", type="int", metavar=_("seconds")) # Character used like the ":" in time strings like # 2002-08-06T04:22:00-07:00. The colon isn't good for filenames on # windows machines. # TRANSL: abbreviation for "character" (noun) parser.add_option("--time-separator", type="string", metavar=_("char"), action="callback", callback=lambda o, s, v, p: set_time_sep(v, s)) # Whether to specify --use-agent in GnuPG options parser.add_option("--use-agent", action="store_true") parser.add_option("--verbosity", "-v", type="verbosity", metavar="[0-9]", dest="", action="callback", callback=lambda o, s, v, p: log.setverbosity(v)) parser.add_option("-V", "--version", action="callback", callback=print_ver) # volume size # TRANSL: Used in usage help to represent a desired number of # something. Example: # --num-retries <number> parser.add_option("--volsize", type="int", action="callback", metavar=_("number"), callback=lambda o, s, v, p: setattr( p.values, "volsize", v * 1024 * 1024)) # If set, collect only the file status, not the whole root. parser.add_option("--file-changed", action="callback", type="file", metavar=_("path"), dest="file_changed", callback=lambda o, s, v, p: setattr( p.values, "file_changed", v.rstrip('/'))) # parse the options (options, args) = parser.parse_args() # Copy all arguments and their values to the globals module. Don't copy # attributes that are 'hidden' (start with an underscore) or whose name is # the empty string (used for arguments that don't directly store a value # by using dest="") for f in filter(lambda x: x and not x.startswith("_"), dir(options)): v = getattr(options, f) # Only set if v is not None because None is the default for all the # variables. If user didn't set it, we'll use defaults in globals.py if v is not None: setattr(globals, f, v) socket.setdefaulttimeout(globals.timeout) # expect no cmd and two positional args cmd = "" num_expect = 2 # process first arg as command if args: cmd = args.pop(0) possible = [c for c in commands if c.startswith(cmd)] # no unique match, that's an error if len(possible) > 1: command_line_error("command '%s' not unique, could be %s" % (cmd, possible)) # only one match, that's a keeper elif len(possible) == 1: cmd = possible[0] # no matches, assume no cmd elif not possible: args.insert(0, cmd) if cmd == "cleanup": cleanup = True num_expect = 1 elif cmd == "collection-status": collection_status = True num_expect = 1 elif cmd == "full": full_backup = True num_expect = 2 elif cmd == "incremental": globals.incremental = True num_expect = 2 elif cmd == "list-current-files": list_current = True num_expect = 1 elif cmd == "remove-older-than": try: arg = args.pop(0) except Exception: command_line_error("Missing time string for remove-older-than") globals.remove_time = dup_time.genstrtotime(arg) num_expect = 1 elif cmd == "remove-all-but-n-full" or cmd == "remove-all-inc-of-but-n-full": if cmd == "remove-all-but-n-full": globals.remove_all_but_n_full_mode = True if cmd == "remove-all-inc-of-but-n-full": globals.remove_all_inc_of_but_n_full_mode = True try: arg = args.pop(0) except Exception: command_line_error("Missing count for " + cmd) globals.keep_chains = int(arg) if not globals.keep_chains > 0: command_line_error(cmd + " count must be > 0") num_expect = 1 elif cmd == "verify": verify = True if len(args) != num_expect: command_line_error("Expected %d args, got %d" % (num_expect, len(args))) # expand pathname args, but not URL for loc in range(len(args)): if '://' not in args[loc]: args[loc] = expand_fn(args[loc]) # Note that ProcessCommandLine depends on us verifying the arg # count here; do not remove without fixing it. We must make the # checks here in order to make enough sense of args to identify # the backend URL/lpath for args_to_path_backend(). if len(args) < 1: command_line_error("Too few arguments") elif len(args) == 1: backend_url = args[0] elif len(args) == 2: lpath, backend_url = args_to_path_backend(args[0], args[1]) # @UnusedVariable else: command_line_error("Too many arguments") if globals.backup_name is None: globals.backup_name = generate_default_backup_name(backend_url) # set and expand archive dir set_archive_dir( expand_archive_dir(globals.archive_dir, globals.backup_name)) log.Info( _("Using archive dir: %s") % (util.ufn(globals.archive_dir.name), )) log.Info(_("Using backup name: %s") % (globals.backup_name, )) return args
def solve(factors_range): factors = list(permutations(factors_range, 2)) products = map(lambda pair: mul(*pair), factors) palindromes = filter(palindrome, products) return max(palindromes)
def _filter_status(self, status): return filter(lambda x: x.status == status, self)
def find_pred(self, pred: 'Callable[[Tree], bool]') -> 'Iterator[Tree]': """Returns all nodes of the tree that evaluate pred(node) as true.""" return filter(pred, self.iter_subtrees())
def parse_yaml(hosts, config): """Parses host information from the output of yaml.load""" export = {'_meta': {'hostvars': {}}} for host_types in hosts['hosts']: for host_type, providers in host_types.items(): export[host_type] = {} export[host_type]['hosts'] = [] key = '~/.ssh/nodejs_build_%s' % host_type export[host_type]['vars'] = {'ansible_ssh_private_key_file': key} for provider in providers: for provider_name, hosts in provider.items(): for host, metadata in hosts.items(): # some hosts have metadata appended to provider # which requires underscore delimiter = "_" if host.count('-') is 3 else "-" hostname = '{}-{}{}{}'.format(host_type, provider_name, delimiter, host) export[host_type]['hosts'].append(hostname) hostvars = {} try: parsed_host = parse_host(hostname) for k, v in parsed_host.items(): hostvars.update( {k: v[0] if type(v) is dict else v}) except Exception as e: raise Exception('Failed to parse host: %s' % e) if 'ip' in metadata: hostvars.update({'ansible_host': metadata['ip']}) del metadata['ip'] if 'port' in metadata: hostvars.update( {'ansible_port': str(metadata['port'])}) del metadata['port'] if 'user' in metadata: hostvars.update({'ansible_user': metadata['user']}) hostvars.update({'ansible_become': True}) del metadata['user'] hostvars.update(metadata) # add specific options from config for option in filter(lambda s: s.startswith('hosts:'), config.sections()): # remove `hosts:` if option[6:] in hostname: for o in config.items(option): # configparser returns tuples of key, value hostvars.update({o[0]: o[1]}) export['_meta']['hostvars'][hostname] = {} export['_meta']['hostvars'][hostname].update(hostvars) export[host_type]['hosts'].sort() return export
def layer(name, features): """Make a vector_tile.Tile.Layer from GeoJSON features.""" pbl = vector_tile_pb2.tile.layer() pbl.name = name pbl.version = 1 pb_keys = [] pb_vals = [] pb_features = [] for j, f in enumerate(chain.from_iterable(singles(ob) for ob in features)): pbf = vector_tile_pb2.tile.feature() pbf.id = j # Pack up the feature geometry. g = f.get('geometry') if g: gtype = g['type'] coords = g['coordinates'] if gtype == 'Point': geometry = [(1 << 3) + 1] + [(n << 1) ^ (n >> 31) for n in map(int, coords)] elif gtype == 'LineString': num = len(coords) geometry = [0] * (4 + 2 * (num - 1)) geometry[0] = (1 << 3) + 1 geometry[1:3] = ((n << 1) ^ (n >> 31) for n in map(int, coords[0])) geometry[3] = ((num - 1) << 3) + 2 for i, (prev, pair) in enumerate(pairwise(coords), 1): prev = map(int, prev) pair = map(int, pair) geometry[2 * i + 2:2 * i + 4] = ((n << 1) ^ (n >> 31) for n in (pair[0] - prev[0], pair[1] - prev[1])) pbf.geometry.extend(geometry) elif gtype == 'Polygon': rings = [] for ring in coords: num = len(ring) geometry = [0] * (5 + 2 * (num - 1)) geometry[0] = (1 << 3) + 1 geometry[1:3] = ((n << 1) ^ (n >> 31) for n in map(int, ring[0])) geometry[3] = ((num - 1) << 3) + 2 for i, (prev, pair) in enumerate(pairwise(ring), 1): prev = map(int, prev) pair = map(int, pair) geometry[2 * i + 2:2 * i + 4] = ((n << 1) ^ (n >> 31) for n in (pair[0] - prev[0], pair[1] - prev[1])) geometry[-1] = (1 << 3) + 7 pbf.geometry.extend(geometry) pbf.type = geom_type_map[gtype] # Pack up feature properties. props = f.get('properties', {}) tags = [0] * (2 * len(props)) for i, (k, v) in enumerate(props.items()): if k not in pb_keys: pb_keys.append(k) if v not in pb_vals: pb_vals.append(v) tags[i * 2:i * 2 + 2] = pb_keys.index(k), pb_vals.index(v) pbf.tags.extend(tags) pb_features.append(pbf) # Finish up the layer. pbl.keys.extend(map(str, pb_keys)) pbl.values.extend(map(value, filter(None, pb_vals))) return pbl
def output_resources(self): return filter(lambda a: isinstance(a, pypeliner.resources.Resource), self.outputs)
def _capability2packages( self, cap ): """ Find packages that implement the capability 'cap'. If cap is a list, find the union of packages implementing those capabilities. """ # Convert to list if single capability given if 'list' == type(cap).__name__: caps = cap else: caps = [ cap ] # Prime the pump. (result, todo, pack_lists, uninstalled) = ([], [], [], []) # Build two lists: 'results' is a list of capabilities found in our cache. # 'todo' is a list of packages we need to query with rpm. for one_cap in caps: if one_cap.startswith('rpmlib'): continue # if this capability is cached, add its packages to the results. if one_cap in self._transmogrify_cache: new1 = self._transmogrify_cache[one_cap] result.append(new1) else: # if not cached, add to todo list. todo.append(one_cap) todo = list(set(todo)) # Ask RPM for capabilities provided by each package in the todo list. if len(todo) > 0: # Because RPM can return more than one package that provides a given # capability, the only way to maintain corraspondance between # packages and capabilities is to force delimitors into RPM's output. todo_cooked = reduce(lambda r,v: r + [ self._rpm_delimiter, v], todo[1:], todo[:1]) todo_cooked.append(self._rpm_delimiter) # Run the RPM command. Because of the forced delimitors, RPM will # give "no package provides" errors in its output and return with # a non-zero # return code. try: cmd_list =["rpm", "-q", "--whatprovides"] + todo_cooked packages = proc.check_output( cmd_list, stderr=proc.STDOUT) except proc.CalledProcessError as e: packages = e.output if "bytes" == type(packages).__name__: packages = "".join(list(map(chr,packages))) packages = packages.rstrip().split('\n') # Use the "no package provides @_@" messages to seperate the output # from RPM into a list of lists. Each element of the top level list # corrasponds to a given capability and is a list of packages # providing that capabilits. a_pack_list = [] for pack in packages: if self._rpm_delimiter in pack: pack_lists.append(list(set(filter(self._good_for_rpm, a_pack_list)))) a_pack_list = [] else: a_pack_list.append(pack) assert len(todo) == len(pack_lists), \ "todo and pack_list have different lengths" new_stuff = zip(todo,pack_lists) self._transmogrify_cache.update(new_stuff) for k,v in new_stuff: if [] == v: uninstalled.append(k) return list(chain.from_iterable(result + pack_lists)) + uninstalled
def parse_metadata(raw, namelist, zf): module = ast.parse(raw, filename='__init__.py') top_level_imports = filter(lambda x: x.__class__.__name__ == 'ImportFrom', ast.iter_child_nodes(module)) top_level_classes = tuple( filter(lambda x: x.__class__.__name__ == 'ClassDef', ast.iter_child_nodes(module))) top_level_assigments = filter(lambda x: x.__class__.__name__ == 'Assign', ast.iter_child_nodes(module)) defaults = { 'name': '', 'description': '', 'supported_platforms': ['windows', 'osx', 'linux'], 'version': (1, 0, 0), 'author': 'Unknown', 'minimum_calibre_version': (0, 9, 42) } field_names = set(defaults) imported_names = {} plugin_import_found = set() all_imports = [] for node in top_level_imports: names = getattr(node, 'names', []) mod = getattr(node, 'module', None) if names and mod: names = [Alias(n.name, getattr(n, 'asname', None)) for n in names] if mod in { 'calibre.customize', 'calibre.customize.conversion', 'calibre.ebooks.metadata.sources.base', 'calibre.ebooks.metadata.covers', 'calibre.devices.interface', 'calibre.ebooks.metadata.fetch', } or re.match(r'calibre\.devices\.[a-z0-9]+\.driver', mod) is not None: inames = {n.asname or n.name for n in names} inames = {x for x in inames if x.lower() != x} plugin_import_found |= inames else: all_imports.append((mod, [n.name for n in names])) imported_names[n.asname or n.name] = mod if not plugin_import_found: return all_imports import_data = (imported_names, zf, namelist) names = {} for node in top_level_assigments: targets = {getattr(t, 'id', None) for t in node.targets} targets.discard(None) for x in targets - field_names: try: val = convert_node({x}, node.value, import_data=import_data) except Exception: pass else: names[x] = val def parse_class(node): class_assigments = filter(lambda x: x.__class__.__name__ == 'Assign', ast.iter_child_nodes(node)) found = {} for node in class_assigments: targets = {getattr(t, 'id', None) for t in node.targets} targets.discard(None) fields = field_names.intersection(targets) if fields: val = convert_node(fields, node.value, names=names, import_data=import_data) for field in fields: found[field] = val return found if top_level_classes: for node in top_level_classes: bases = {getattr(x, 'id', None) for x in node.bases} if not bases.intersection(plugin_import_found): continue found = parse_class(node) if 'name' in found and 'author' in found: defaults.update(found) return defaults for node in top_level_classes: found = parse_class(node) if 'name' in found and 'author' in found and 'version' in found: defaults.update(found) return defaults raise ValueError('Could not find plugin class')
def select_many_in(inst_set, where_cond, order_by): s = filter(where_cond, inst_set) if order_by: s = order_by(s) return xtuml.QuerySet(s)
def find_pred(self, pred): """Returns all nodes of the tree that evaluate pred(node) as true.""" return filter(pred, self.iter_subtrees())
def create_periodical_index(self): # {{{ periodical_node = iter(self.oeb.toc).next() periodical_node_offset = self.serializer.body_start_offset periodical_node_size = (self.serializer.body_end_offset - periodical_node_offset) normalized_sections = [] id_offsets = self.serializer.id_offsets periodical = PeriodicalIndexEntry(periodical_node_offset, self.cncx[periodical_node.title], self.cncx[periodical_node.klass], 0) periodical.length = periodical_node_size periodical.first_child_index = 1 periodical.image_index = self.masthead_offset seen_sec_offsets = set() seen_art_offsets = set() for sec in periodical_node: normalized_articles = [] try: offset = id_offsets[sec.href] label = self.cncx[sec.title] klass = self.cncx[sec.klass] except: continue if offset in seen_sec_offsets: continue seen_sec_offsets.add(offset) section = PeriodicalIndexEntry(offset, label, klass, 1) section.parent_index = 0 for art in sec: try: offset = id_offsets[art.href] label = self.cncx[art.title] klass = self.cncx[art.klass] except: continue if offset in seen_art_offsets: continue seen_art_offsets.add(offset) article = PeriodicalIndexEntry(offset, label, klass, 2) normalized_articles.append(article) article.author_offset = self.cncx[art.author] article.desc_offset = self.cncx[art.description] if getattr(art, 'toc_thumbnail', None) is not None: try: ii = self.serializer.images[art.toc_thumbnail] - 1 if ii > -1: article.image_index = ii except KeyError: pass # Image not found in serializer if normalized_articles: normalized_articles.sort(key=lambda x: x.offset) normalized_sections.append((section, normalized_articles)) normalized_sections.sort(key=lambda x: x[0].offset) # Set lengths for s, x in enumerate(normalized_sections): sec, normalized_articles = x try: sec.length = normalized_sections[s + 1][0].offset - sec.offset except: sec.length = self.serializer.body_end_offset - sec.offset for i, art in enumerate(normalized_articles): try: art.length = normalized_articles[i + 1].offset - art.offset except: art.length = sec.offset + sec.length - art.offset # Filter for i, x in list(enumerate(normalized_sections)): sec, normalized_articles = x normalized_articles = list( filter(lambda x: x.length > 0, normalized_articles)) normalized_sections[i] = (sec, normalized_articles) normalized_sections = list( filter(lambda x: x[0].length > 0 and x[1], normalized_sections)) # Set indices i = 0 for sec, articles in normalized_sections: i += 1 sec.index = i sec.parent_index = 0 for sec, articles in normalized_sections: for art in articles: i += 1 art.index = i art.parent_index = sec.index for sec, normalized_articles in normalized_sections: sec.first_child_index = normalized_articles[0].index sec.last_child_index = normalized_articles[-1].index # Set lengths again to close up any gaps left by filtering for s, x in enumerate(normalized_sections): sec, articles = x try: next_offset = normalized_sections[s + 1][0].offset except: next_offset = self.serializer.body_end_offset sec.length = next_offset - sec.offset for a, art in enumerate(articles): try: next_offset = articles[a + 1].offset except: next_offset = sec.next_offset art.length = next_offset - art.offset # Sanity check for s, x in enumerate(normalized_sections): sec, articles = x try: next_sec = normalized_sections[s + 1][0] except: if (sec.length == 0 or sec.next_offset != self.serializer.body_end_offset): raise ValueError('Invalid section layout') else: if next_sec.offset != sec.next_offset or sec.length == 0: raise ValueError('Invalid section layout') for a, art in enumerate(articles): try: next_art = articles[a + 1] except: if (art.length == 0 or art.next_offset != sec.next_offset): raise ValueError('Invalid article layout') else: if art.length == 0 or art.next_offset != next_art.offset: raise ValueError('Invalid article layout') # Flatten indices = [periodical] for sec, articles in normalized_sections: indices.append(sec) periodical.last_child_index = sec.index for sec, articles in normalized_sections: for a in articles: indices.append(a) return indices
def _populate_envvars(self): """Return CGI-related env vars Mostly straight from ``CGIHTTPRequestHandler.run_cgi``. This takes request header fields and parsed-path info and sets env vars required by rfc3875_ and some HTTP ones. .. _rfc3875: https://tools.ietf.org/html/rfc3875#section-4.1 """ full_env = dict(os.environ) # As required by git-http-backend(1); These never change. cgi_env = dict(self.git_env) cgi_env["GIT_HTTP_EXPORT_ALL"] = "" cgi_env.update(self.auth_env) # FIXME previous comment said, "Fallback for when auth isn't used," # but that implies auth is mandatory, which it's not if self.path.endswith("git-receive-pack"): cgi_env.setdefault("REMOTE_USER", full_env.get("USER", "nobody")) # Vanilla from here on down always = { "SCRIPT_NAME": "git-http-backend", "SERVER_SOFTWARE": self.version_string(), "SERVER_NAME": self.server.server_name, "GATEWAY_INTERFACE": "CGI/1.1", "SERVER_PROTOCOL": self.protocol_version, "SERVER_PORT": str(self.server.server_port), "REQUEST_METHOD": self.command, "REMOTE_ADDR": self.client_address[0], } cgi_env.update(always) if hasattr(self.headers, "get_content_type"): cgi_env["CONTENT_TYPE"] = self.headers.get( "content-type", self.headers.get_content_type()) else: # 27 cgi_env["CONTENT_TYPE"] = (self.headers.typeheader or self.headers.type) length = self.headers.get("content-length") if length: cgi_env["CONTENT_LENGTH"] = length referer = self.headers.get("referer") if referer: cgi_env["HTTP_REFERER"] = referer accept = [] # Actual type is X-<custom> for line in self.headers.getallmatchingheaders("accept"): if line[:1] in "\t\n\r ": accept.append(line.strip()) else: accept += accept + line[7:].split(",") cgi_env["HTTP_ACCEPT"] = ",".join(accept) ua = self.headers.get("user-agent") if ua: cgi_env["HTTP_USER_AGENT"] = ua if hasattr(self.headers, "get_all"): co = filter(None, self.headers.get_all("cookie", [])) else: co = filter(None, self.headers.getheaders("cookie")) cookie_str = ", ".join(co) if cookie_str: cgi_env["HTTP_COOKIE"] = cookie_str config["DEBUG"] and self.dlog("envvars", **cgi_env) full_env.update(cgi_env) # 4.1 says: "an optional meta-variable may be omitted (left unset) if # its value is NULL" (a zero-length string). # # EDIT: upstream includes CONTENT_LENGTH here but it has been removed # below. The RFC says it can be NULL but also says it MUST be set IFF # content exists (meaning unset otherwise). Since we only read what's # reported by the request (and discard the rest), there's no sense in # using a fallback, no? rfcvars = ( "QUERY_STRING", "REMOTE_HOST", # SHOULD (can also be REMOTE_HOST) "HTTP_USER_AGENT", "HTTP_COOKIE", "HTTP_REFERER", ) for k in rfcvars: full_env.setdefault(k, "") return full_env
def find_pred(self, pred): "Find all nodes where pred(tree) == True" return filter(pred, self.iter_subtrees())
def secret_generator(self, string, *args, **kwargs): return filter( self.is_formally_valid, super(JwtTokenDetector, self).secret_generator(string, *args, **kwargs), )
def getleastindex(elems): """Return the first index in elems, assuming elems isn't empty""" return min(map(lambda elem: elem.index, filter(lambda x: x, elems)))
# The current directory is where setup.py is. def package_files(directory): paths = [] for (path, directories, filenames) in os.walk(directory): for filename in filenames: paths.append(os.path.join('..', path, filename)) return paths extra_files = package_files('survol/www') # The zip archive contains directories: docs, survol and tests. with open(os.path.join('survol', '__init__.py')) as f: __version__ = ast.parse( next(filter(lambda line: line.startswith('__version__'), f))).body[0].value.s with open('README.txt') as readme_file: README = readme_file.read() # FIXME: survol.__doc__ = '\nSurvol library\n' ... setup( name='survol', version=__version__, description='Understanding legacy applications', long_description=README, author='Primhill Computers', author_email='*****@*****.**', url='http://www.primhillcomputers.com/survol.html', packages=find_packages(),
def list_categories(self, name=None): categories = self.settings.global_settings.categories return list(filter(self.find_lambda(name), categories))
def select_many_in(inst_set, where_cond): s = filter(where_cond, inst_set) return xtuml.QuerySet(s)
def multi_finalize(fn, d): appends = (d.getVar("__BBAPPEND", True) or "").split() for append in appends: logger.debug(2, "Appending .bbappend file %s to %s", append, fn) bb.parse.BBHandler.handle(append, d, True) onlyfinalise = d.getVar("__ONLYFINALISE", False) safe_d = d d = bb.data.createCopy(safe_d) try: finalize(fn, d) except bb.parse.SkipPackage as e: d.setVar("__SKIPPED", e.args[0]) datastores = {"": safe_d} versions = (d.getVar("BBVERSIONS", True) or "").split() if versions: pv = orig_pv = d.getVar("PV", True) baseversions = {} def verfunc(ver, d, pv_d = None): if pv_d is None: pv_d = d overrides = d.getVar("OVERRIDES", True).split(":") pv_d.setVar("PV", ver) overrides.append(ver) bpv = baseversions.get(ver) or orig_pv pv_d.setVar("BPV", bpv) overrides.append(bpv) d.setVar("OVERRIDES", ":".join(overrides)) versions = list(_expand_versions(versions)) for pos, version in enumerate(list(versions)): try: pv, bpv = version.split(":", 2) except ValueError: pass else: versions[pos] = pv baseversions[pv] = bpv if pv in versions and not baseversions.get(pv): versions.remove(pv) else: pv = versions.pop() # This is necessary because our existing main datastore # has already been finalized with the old PV, we need one # that's been finalized with the new PV. d = bb.data.createCopy(safe_d) verfunc(pv, d, safe_d) try: finalize(fn, d) except bb.parse.SkipPackage as e: d.setVar("__SKIPPED", e.args[0]) _create_variants(datastores, versions, verfunc) extended = d.getVar("BBCLASSEXTEND", True) or "" if extended: # the following is to support bbextends with arguments, for e.g. multilib # an example is as follows: # BBCLASSEXTEND = "multilib:lib32" # it will create foo-lib32, inheriting multilib.bbclass and set # BBEXTENDCURR to "multilib" and BBEXTENDVARIANT to "lib32" extendedmap = {} variantmap = {} for ext in extended.split(): eext = ext.split(':', 2) if len(eext) > 1: extendedmap[ext] = eext[0] variantmap[ext] = eext[1] else: extendedmap[ext] = ext pn = d.getVar("PN", True) def extendfunc(name, d): if name != extendedmap[name]: d.setVar("BBEXTENDCURR", extendedmap[name]) d.setVar("BBEXTENDVARIANT", variantmap[name]) else: d.setVar("PN", "%s-%s" % (pn, name)) bb.parse.BBHandler.inherit(extendedmap[name], fn, 0, d) safe_d.setVar("BBCLASSEXTEND", extended) _create_variants(datastores, extendedmap.keys(), extendfunc) for variant, variant_d in datastores.iteritems(): if variant: try: if not onlyfinalise or variant in onlyfinalise: finalize(fn, variant_d, variant) except bb.parse.SkipPackage as e: variant_d.setVar("__SKIPPED", e.args[0]) if len(datastores) > 1: variants = filter(None, datastores.iterkeys()) safe_d.setVar("__VARIANTS", " ".join(variants)) datastores[""] = d return datastores
import os from slack_cleaner2 import SlackCleaner, match try: from future_builtins import filter except ImportError: pass s = SlackCleaner(os.environ['TOKEN'], 1) with s.log.group('clear bot channels'): for msg in s.msgs(filter(match('.*-bots'), s.conversations)): msg.delete() # with s.log.group('delete older than 3 months'): # for msg in filter(is_not_pinned(), s.msgs(before=a_while_ago(months=3))): # msg.delete() # with s.log.group('delete ims older than 1 month'): # for msg in filter(is_not_pinned(), s.msgs(s.ims, before=a_while_ago(months=1))): # msg.delete() s.log.summary()
_, _, children = tree if len(children): return max(maxdepth(i, count + 1) for i in children.values()) else: return count if __name__ == "__main__": wordlist = [] defintionlist = [] wordlist, definitionlist = dict.load_dict() print("Creating BK-tree...\n") dict_tree = timeof(BKTree, levenshtein, filter(len, wordlist), filter(len, definitionlist)) print("Size of dictionary: " + str(len(wordlist)) + "\n") while (1): print("DICTIONARY USING BK-TREES\n") print("1: Insert a new word into the dictionary") print("2: Look up a word in the dictionary") print("3: Check whether the spelling of a given word is correct") print( "4: Compare searches by brute query and Levenshtein distance-based query" ) print("5: Depth") print("6: Exit\n")
def map_filter(self, function, test): """Filter with test then apply function.""" if function is None: function = lambda x: x # return results of filter return list(map(function, filter(test, self.entities)))
def inclusive_alternation_branch(branch_node): character_ranges = (CharacterRange.from_op_node(node) for node in branch_node.children) return any( cr1.overlap(cr2) for cr1, cr2 in itertools.combinations( filter(None, character_ranges), 2))
def parse_product(self, response): product = response.meta['product'] cond_set( product, 'brand', response.xpath("//div[contains(@class,'prodTitlePlus')]" "/span[@itemprop='brand']/text()").extract()) if not product.get('brand', None): cond_set( product, 'brand', response.xpath( '//*[@itemprop="brand"]//span/text()').extract()) cond_set( product, 'title', response.xpath( "//div[contains(@class,'prodTitle')]/h1/span[@itemprop='name']" "/text()").extract()) # Title key must be present even if it is blank cond_set_value(product, 'title', "") sold_out = response.xpath( '//*[@itemprop="availability" and @href="http://schema.org/SoldOut"]' ) cond_set_value(product, 'no_longer_available', 1 if (not response.body or sold_out) else 0) cond_set( product, 'image_url', response.xpath("//div[@id='plImageHolder']/img/@src").extract()) url = "http://m.samsclub.com/api/sams/samsapi/v2/productInfo?repositoryId={}&class=product&loadType=full&bypassEGiftCardViewOnly=true&clubId={}" product_id = response.xpath( '//input[@id="pProductId"]/@value').extract()[0] product_data = json.loads( requests.get(url.format(product_id, self.clubno)).text) try: price = float( product_data.get('sa')[0].get('onlinePrice').get( 'listPrice').replace('$', '').replace(',', '')) cond_set_value(product, 'price', Price(price=price, priceCurrency='USD')) except: pass try: price = float( product_data.get('sa')[0].get('onlinePrice').get( 'finalPrice').replace('$', '').replace(',', '')) cond_set_value(product, 'price_with_discount', Price(price=price, priceCurrency='USD')) except: pass try: price = float( product_data.get('sa')[0].get('clubPrice').get( 'listPrice').replace('$', '').replace(',', '')) cond_set_value(product, 'price_club', Price(price=price, priceCurrency='USD')) except: pass try: price = float( product_data.get('sa')[0].get('clubPrice').get( 'finalPrice').replace('$', '').replace(',', '')) cond_set_value(product, 'price_club_with_discount', Price(price=price, priceCurrency='USD')) except: pass regex = "(prod\d+)" reseller_id = re.findall(regex, response.url) reseller_id = reseller_id[0] if reseller_id else None cond_set_value(product, "reseller_id", reseller_id) cond_set( product, 'description', response.xpath("//div[@itemprop='description']").extract(), ) cond_set(product, 'model', response.xpath("//span[@itemprop='model']/text()").extract(), conv=string.strip) if product.get('model', '').strip().lower() == 'null': product['model'] = '' product['locale'] = "en-US" # Categories categorie_filters = [u'sam\u2019s club'] # Clean and filter categories names from breadcrumb bc = response.xpath('//*[@id="breadcrumb"]//a/text()').extract() bc = [b.strip() for b in bc if b.strip()] if not bc or len(bc) == 1: bc = response.xpath(".//*[@id='breadcrumb']//text()").extract() bc = [b.strip() for b in bc if b.strip()] if not bc: bc = response.xpath( '//*[@id="breadcrumb"]//a//*[@itemprop="title"]/text()' ).extract() bc = [b.strip() for b in bc if b.strip()] categories = list( filter((lambda x: x.lower() not in categorie_filters), map((lambda x: x.strip()), bc))) category = categories[-1] if categories else None cond_set_value(product, 'categories', categories) cond_set_value(product, 'category', category) # Subscribe and save subscribe_and_save = response.xpath('//*[@class="subscriptionDiv" and \ not(@style="display: none;")]/input[@id="pdpSubCheckBox"]') cond_set_value(product, 'subscribe_and_save', 1 if subscribe_and_save else 0) # Shpping shipping_included = response.xpath('//*[@class="freeDelvryTxt"]') cond_set_value(product, 'shipping_included', 1 if shipping_included else 0) oos_in_both = response.xpath( '//*[@class="biggraybtn" and' ' text()="Out of stock online and in club"]') # Available in Store available_store = response.xpath( '//*[(@id="addtocartsingleajaxclub" or' '@id="variantMoneyBoxButtonInitialLoadClub")' 'and contains(text(),"Pick up in Club")]') or \ response.xpath( '//li[contains(@class,"pickupIcon")]' '/following-sibling::li[contains' '(translate(text(), "ABCDEFGHIJKLMNOPQRSTUVWXYZ",' ' "abcdefghijklmnopqrstuvwxyz"),"ready as soon as")]') cond_set_value(product, 'available_store', 1 if available_store and not oos_in_both else 0) # Available Online available_online = response.xpath( '//*[(@id="addtocartsingleajaxonline" \ or @id="variantMoneyBoxButtonInitialLoadOnline")]') cond_set_value(product, 'available_online', 1 if available_online and not oos_in_both else 0) if str(product.get('available_online', None)) == '0' and str( product.get('available_store', None)) == '0': product['is_out_of_stock'] = True if not shipping_included and not product.get('no_longer_available'): productId = ''.join( response.xpath('//*[@id="mbxProductId"]/@value').extract()) if not productId: productId = self._product_id(response) pSkuId = ''.join( response.xpath('//*[@id="mbxSkuId"]/@value').extract()) # This is fixing bug with sku and prod_id extraction for bundle products if not productId or not pSkuId: js_sku_prodid = response.xpath( './/script[contains(text(), "var skuId") and contains(text(), "var productId")]/text()' ).extract() js_sku_prodid = ''.join( js_sku_prodid) if js_sku_prodid else None if js_sku_prodid: rgx = r'(prod\d+)' match_list = re.findall(rgx, js_sku_prodid) productId = match_list[0] if match_list else None rgx = r'(sku\d+)' match_list = re.findall(rgx, js_sku_prodid) pSkuId = match_list[0] if match_list else None shipping_prices_url = "http://www.samsclub.com/sams/shop/product/moneybox/shippingDeliveryInfo.jsp?zipCode=%s&productId=%s&skuId=%s" % ( self.zip_code, productId, pSkuId) return Request(shipping_prices_url, meta={ 'product': product, 'prod_id': productId }, callback=self._parse_shipping_cost) elif not product.get('buyer_reviews'): productId = ''.join( response.xpath('//*[@id="mbxProductId"]/@value').extract()) if not productId: productId = self._product_id(response) reviews_url = self._REVIEWS_URL.format(prod_id=productId) return Request(reviews_url, meta={ 'product': product, 'prod_id': productId }, callback=self._load_reviews) return product
def find_pred(self, pred): return filter(pred, self.iter_subtrees())
def buck2diac(buckAnalysis, keepSegmentation=True): ''' Given a segmented and tagged Arabic word from the Buckwalter analyzer, perform orthographic normalization to produce the surface diacritized form (also in the Buckwalter encoding) with morphemes separated by '+' (unless 'keepSegmentation' is false, in which case the unsegmented form will be returned). Normalization involves contracting li+Al+ to li+, adding Shadda (gemination mark) to sun letters after Al+, changing Alef Wasla to Alef, etc. ''' assert ' ' not in buckAnalysis, buckAnalysis # Strip tags from morphemes vs = '+'.join( filter(lambda m: m != '(null)', (x.split('/')[0] for x in buckAnalysis.split('+') if x != ''))) # Normalization ''' In MADA-3.1/MADA/ALMOR3.pm: #update Sept 11 2008: to match up behavior of sun/moon letters in BAMA2 # code segemnt from BAMA-2 my $voc_str = $$prefix_value{"diac"}."+".$$stem_value{"diac"}."+".$$suffix_value{"diac"}; $voc_str =~ s/^((wa|fa)?(bi|ka)?Al)\+([tvd\*rzs\$SDTZln])/$1$4~/; # not moon letters $voc_str =~ s/^((wa|fa)?lil)\+([tvd\*rzs\$SDTZln])/$1$3~/; # not moon letters $voc_str =~ s/A\+a([pt])/A$1/; # e.g.: Al+HayA+ap $voc_str =~ s/\{/A/g; $voc_str =~ s/\+//g; ''' # TODO: this is ad hoc for now--not sure if it's precisely how SAMA/MADA do it. vs = re.sub(r'^li[+]Al[+]', 'li+l+', vs) # contraction vs = re.sub('^([+]?)min[+]m(A|an)$', r'\1mi+m~\2', vs) # contraction, e.g. +min/PREP+mA/REL_PRON+ -> mim~A vs = re.sub(r'Y[+]', 'y+', vs) # Y -> y / _+ vs = re.sub(r'y[+]F', 'Y+F', vs) # e.g. bi/PREP+maEonY/NOUN+F/CASE_INDEF_GEN -> bimaEonYF vs = re.sub( r'y[+]([tn])', r'yo+\1', vs ) # 'o' epenthesis? e.g. wa/CONJ+>aroDay/PV+tu/PVSUFF_SUBJ:1S+hA/PVSUFF_DO:3FS -> wa>aroDayotuhA vs = re.sub( r't[+]h', 'to+h', vs ) # 'o' deletion? e.g. +jaEal/PV+at/PVSUFF_SUBJ:3FS+hu/PVSUFF_DO:3MS -> jaEalatohu vs = re.sub(r'^Einod[+]a[+]mA$', 'EindamA', vs) # +Einod/NOUN+a/CASE_DEF_ACC+mA/SUB_CONJ+ vs = re.sub(r'^li[+][*]alika$', 'li*`lika', vs) # +li/PREP+*alika/DEM_PRON_MS+ # end ad hoc # add Shadda to sun letters after Al+ # NOTE: needed to add [+]? in some places after clitics vs = re.sub(r'^((wa[+]?|fa[+]?)?(bi[+]?|ka[+]?)?Al)\+([tvd\*rzs\$SDTZln])', r'\1\4~', vs, 1) # not moon letters vs = re.sub(r'^((wa[+]?|fa[+]?)?li[+]?l)\+([tvd\*rzs\$SDTZln])', r'\1\3~', vs, 1) # not moon letters # simplify Aa before p or t, e.g.: Al+HayA+ap vs = re.sub(r'A\+a([pt])', r'A\1', vs, 1) # convert Alef Wasla to plain Alef (http://en.wikipedia.org/wiki/Arabic_diacritics#.CA.BCAlif_wa.E1.B9.A3la) vs = vs.replace('{', 'A') # use the desired segmentation character if not keepSegmentation: vs = vs.replace('+', '') return vs