def complete_env_var(line): """ Complete names of environment variables This function tokenizes the line and computes completions based on environment variables starting with the last token It returns a pair: - the line expanded up to the longest common sequence among the completions - the list of all possible completions """ tokens = parse_line(line) if tokens == [] or (line[-1] in sep_chars and parse_line(line) == parse_line(line + ' ')): tokens += [''] # This saves some checks later # Account for the VAR=VALUE syntax (token_prefix, equals, token_orig) = tokens[-1].rpartition('=') token_prefix += equals if token_orig.count('%') % 2 == 0 and token_orig.strip('"').endswith('%'): [lead, prefix] = token_orig.strip('"').rstrip('%').rsplit('%', 2) else: [lead, prefix] = token_orig.strip('"').rsplit('%', 1) if token_orig.strip('"').endswith('%') and prefix != '': completions = [prefix] else: completions = [ var for var in os.environ if var.lower().startswith(prefix.lower()) ] completions.sort() if completions != []: # Find longest prefix common_string = find_common_prefix(prefix, completions) quote = '' # Assume no quoting needed by default, then check for spaces for completion in completions: if contains_special_char(os.environ[completion]): quote = '"' break result = line[0:len(line) - len(token_orig)] + quote + lead + '%' + common_string if len(completions) == 1: result += '%' + quote return (result, []) else: return (result, completions) else: # No completion possible, return original line return (line, [])
def complete_env_var(line): """ Complete names of environment variables This function tokenizes the line and computes completions based on environment variables starting with the last token It returns a pair: - the line expanded up to the longest common sequence among the completions - the list of all possible completions """ tokens = parse_line(line) if tokens == [] or (line[-1] in sep_chars and parse_line(line) == parse_line(line + ' ')): tokens += [''] # This saves some checks later # Account for the VAR=VALUE syntax (token_prefix, equals, token_orig) = tokens[-1].rpartition('=') token_prefix += equals if token_orig.count('%') % 2 == 0 and token_orig.strip('"').endswith('%'): [lead, prefix] = token_orig.strip('"').rstrip('%').rsplit('%', 2) else: [lead, prefix] = token_orig.strip('"').rsplit('%', 1) if token_orig.strip('"').endswith('%') and prefix != '': completions = [prefix] else: completions = [var for var in os.environ if var.lower().startswith(prefix.lower())] completions.sort() if completions != []: # Find longest prefix common_string = find_common_prefix(prefix, completions) quote = '' # Assume no quoting needed by default, then check for spaces for completion in completions: if contains_special_char(os.environ[completion]): quote = '"' break result = line[0 : len(line) - len(token_orig)] + quote + lead + '%' + common_string if len(completions) == 1: result += '%' + quote return (result, []) else: return (result, completions) else: # No completion possible, return original line return (line, [])
def write_submission(writer, reader, target_labels, process_row_func, get_profiles_func, personal_recommendations, common_recommendations, product_stats, personal_recommendations_weight): total = 0 writer.write("ncodpers,added_products\n") removed_rows = 0 while True: line = reader.readline()[:-1] total += 1 if line == '': break row = parse_line(line) # Write before row processing user = get_user(row) writer.write(user + ',') prow = process_row_func(row) if prow is None or len(prow) == 0: removed_rows += 1 logging.debug("--- Removed row : {}".format(row)) writer.write("\n") continue predicted = compute_predictions(prow, get_profiles_func, personal_recommendations, common_recommendations, product_stats, personal_recommendations_weight) for p in predicted: writer.write(target_labels[p] + ' ') if total % 1000000 == 0: logging.info('Read {} lines'.format(total)) writer.write("\n") logging.info("-- Removed rows : %s" % removed_rows)
def complete_file_simple(line): """ Complete names of files or directories This function tokenizes the line and computes file and directory completions starting with the last token. It returns a pair: - the line expanded up to the longest common sequence among the completions - the list of all possible completions (first dirs, then files) """ tokens = parse_line(line) if tokens == [] or (line[-1] in sep_chars and parse_line(line) == parse_line(line + ' ')): tokens += [''] # This saves us some checks later token = tokens[-1].replace('"', '') (path_to_complete, _, prefix) = token.rpartition('\\') if path_to_complete == '' and token != '' and token[0] == '\\': path_to_complete = '\\' # print '\n\n', path_to_complete, '---', prefix, '\n\n' if path_to_complete == '': dir_to_complete = os.getcwd() elif path_to_complete == '\\': dir_to_complete = os.getcwd()[0:3] else: dir_to_complete = expand_env_vars(path_to_complete) + '\\' # This is the wildcard matcher used throughout the function matcher = wildcard_to_regex(prefix + '*') completions = [] if os.path.isdir(dir_to_complete): try: completions = [ elem for elem in os.listdir(dir_to_complete) if matcher.match(elem) ] except OSError: # Cannot complete, probably access denied pass # Sort directories first, also append '\'; then, files completions_dirs = [ elem + '\\' for elem in completions if os.path.isdir(dir_to_complete + '\\' + elem) ] completions_files = [ elem for elem in completions if os.path.isfile(dir_to_complete + '\\' + elem) ] completions = completions_dirs + completions_files if (len(tokens) == 1 or tokens[-2] in seq_tokens) and path_to_complete == '': # We are at the beginning of a command ==> also complete from the path completions_path = [] for elem_in_path in os.environ['PATH'].split(';'): dir_to_complete = expand_env_vars(elem_in_path) + '\\' try: completions_path += [ elem for elem in os.listdir(dir_to_complete) if matcher.match(elem) and os.path.isfile(dir_to_complete + '\\' + elem) and has_exec_extension(elem) and not elem in completions and not elem in completions_path ] except OSError: # Cannot complete, probably access denied pass # Add internal commands internal_commands = [ 'assoc', 'call', 'cd', 'chdir', 'cls', 'color', 'copy', 'date', 'del', 'dir', 'echo', 'endlocal', 'erase', 'exit', 'for', 'ftype', 'goto', 'if', 'md', 'mkdir', 'move', 'path', 'pause', 'popd', 'prompt', 'pushd', 'rem', 'ren', 'rename', 'rd', 'rmdir', 'set', 'setlocal', 'shift', 'start', 'time', 'title', 'type', 'ver', 'verify', 'vol' ] if sys.getwindowsversion()[0] >= 6: # Windows 7 or newer internal_commands.append('mklink') completions_path += [ elem for elem in internal_commands if matcher.match(elem) and not elem in completions and not elem in completions_path ] # Sort in lexical order (case ignored) completions_path.sort(key=str.lower) # Remove .com, .exe or .bat extension where possible completions_path_no_ext = [ strip_extension(elem) for elem in completions_path ] completions_path_nice = [] for i in range(0, len(completions_path_no_ext)): similar = [ elem for elem in completions_path_no_ext if elem == completions_path_no_ext[i] ] similar += [ elem for elem in completions if strip_extension(elem) == completions_path_no_ext[i] ] if len(similar) == 1 and has_exec_extension( completions_path[i]) and len(prefix) < len( completions_path[i]) - 3: # No similar executables, don't use extension completions_path_nice.append(completions_path_no_ext[i]) else: # Similar executables found, keep extension completions_path_nice.append(completions_path[i]) completions += completions_path_nice if completions != []: # Find the longest common sequence common_string = find_common_prefix(prefix, completions) if path_to_complete == '': completed_file = common_string elif path_to_complete == '\\': completed_file = '\\' + common_string else: completed_file = path_to_complete + '\\' + common_string if expand_env_vars(completed_file).find(' ') >= 0 or \ (prefix != '' and [elem for elem in completions if contains_special_char(elem)] != []) or \ (prefix == '' and [elem for elem in completions if starts_with_special_char(elem)] != []): # We add quotes if one of the following holds: # * the (env-expanded) completed string contains whitespace # * there is a prefix and at least one of the valid completions contains whitespace # * there is no prefix and at least one completion _starts_ with whitespace start_quote = '"' else: start_quote = '' # Build the result result = line[0:len(line) - len(tokens[-1])] + start_quote + completed_file if len(completions) == 1: # We can close the quotes if we have completed to a unique filename if start_quote == '"': end_quote = '"' else: end_quote = '' if result[-1] == '\\': # Directory -- we want the backslash (if any) AFTER the closing quote result = result[:-1] + end_quote + '\\' else: # File -- add space if the completion is unique result += end_quote result += ' ' return (result, completions) else: # No expansion was made, return original line return (line, [])
def complete_wildcard(line): """ Complete file/dir wildcards This function tokenizes the line and computes file and directory completions starting with the last token which is a wildcard It returns a pair: - the line expanded up to the longest common sequence among the completions - the list of all possible completions (first dirs, then files) """ tokens = parse_line(line) if tokens == [] or (line[-1] in sep_chars and parse_line(line) == parse_line(line + ' ')): tokens += [''] # This saves us some checks later token = tokens[-1].replace('"', '') (path_to_complete, _, prefix) = token.rpartition('\\') if path_to_complete == '' and token != '' and token[0] == '\\': path_to_complete = '\\' # print '\n\n', path_to_complete, '---', prefix, '\n\n' if path_to_complete == '': dir_to_complete = os.getcwd() elif path_to_complete == '\\': dir_to_complete = os.getcwd()[0:3] else: dir_to_complete = expand_env_vars(path_to_complete) + '\\' # This is the wildcard matcher used throughout the function matcher = wildcard_to_regex(prefix + '*') completions = [] if os.path.isdir(dir_to_complete): try: completions = [ elem for elem in os.listdir(dir_to_complete) if matcher.match(elem) ] except OSError: # Cannot complete, probably access denied pass # Sort directories first, also append '\'; then, files completions_dirs = [ elem + '\\' for elem in completions if os.path.isdir(dir_to_complete + '\\' + elem) ] completions_files = [ elem for elem in completions if os.path.isfile(dir_to_complete + '\\' + elem) ] completions = completions_dirs + completions_files if completions != []: completed_suffixes = [] for c in completions: match = matcher.match(c) completed_suffixes.append(match.group(match.lastindex)) if len(completions) == 1: # Only one match, we can inline it and replace the wildcards common_string = completions[0] else: # Multiple matches, find the longest common sequence common_string = prefix + find_common_prefix( prefix, completed_suffixes) if path_to_complete == '': completed_file = common_string elif path_to_complete == '\\': completed_file = '\\' + common_string else: completed_file = path_to_complete + '\\' + common_string if expand_env_vars(completed_file).find(' ') >= 0 or \ (prefix != '' and [elem for elem in completions if contains_special_char(elem)] != []) or \ (prefix == '' and [elem for elem in completions if starts_with_special_char(elem)] != []): # We add quotes if one of the following holds: # * the (env-expanded) completed string contains whitespace # * there is a prefix and at least one of the valid completions contains whitespace # * there is no prefix and at least one completion _starts_ with whitespace start_quote = '"' else: start_quote = '' # Build the result result = line[0:len(line) - len(tokens[-1])] + start_quote + completed_file if len(completions) == 1 or \ not common_string.endswith('*') and \ max([len(c) for c in completed_suffixes]) == len(common_string) - len(prefix): # We can close the quotes if all the completions have the same suffix or # there exists only one matching file if start_quote == '"': end_quote = '"' else: end_quote = '' if result[-1] == '\\': # Directory -- we want the backslash (if any) AFTER the closing quote result = result[:-1] + end_quote + '\\' else: # File -- add space if the completion is unique result += end_quote result += ' ' return (result, completions) else: # No expansion was made, return original line return (line, [])
def complete_file_alternate(line): """ Complete names of files or directories using an alternate tokenization This function tokenizes the line by tring to interpret the last token as a semicolon-separated list of paths, optionally preceded by an equals char. It returns a pair: - the line expanded up to the longest common sequence among the completions - the list of all possible completions (first dirs, then files) """ tokens = parse_line(line) if tokens == [] or (line[-1] in sep_chars and parse_line(line) == parse_line(line + ' ')): tokens += [''] # This saves us some checks later (last_token_prefix, equal_char, last_token) = tokens[-1].replace('"', '').rpartition('=') last_token_prefix += equal_char paths = last_token.split(';') token = paths[-1] # print '\n\nTokens:', tokens, '\n\nCompleting:', token, '\n\n' (path_to_complete, _, prefix) = token.rpartition('\\') if path_to_complete == '' and token != '' and token[0] == '\\': path_to_complete = '\\' # print '\n\n', path_to_complete, '---', prefix, '\n\n' if path_to_complete == '': dir_to_complete = os.getcwd() elif path_to_complete == '\\': dir_to_complete = os.getcwd()[0:3] else: dir_to_complete = expand_env_vars(path_to_complete) + '\\' # This is the wildcard matcher used throughout the function matcher = wildcard_to_regex(prefix + '*') completions = [] if os.path.isdir(dir_to_complete): try: completions = [ elem for elem in os.listdir(dir_to_complete) if matcher.match(elem) ] except OSError: # Cannot complete, probably access denied pass # Sort directories first, also append '\'; then, files completions_dirs = [ elem + '\\' for elem in completions if os.path.isdir(dir_to_complete + '\\' + elem) ] completions_files = [ elem for elem in completions if os.path.isfile(dir_to_complete + '\\' + elem) ] completions = completions_dirs + completions_files if completions != []: # Find the longest common sequence common_string = find_common_prefix(prefix, completions) if path_to_complete == '': completed_file = common_string elif path_to_complete == '\\': completed_file = '\\' + common_string else: completed_file = path_to_complete + '\\' + common_string if expand_env_vars(last_token + completed_file).find(' ') >= 0 or \ (prefix != '' and [elem for elem in completions if contains_special_char(elem)] != []) or \ (prefix == '' and [elem for elem in completions if starts_with_special_char(elem)] != []): # We add quotes if one of the following holds: # * the (env-expanded) completed string contains whitespace # * there is a prefix and at least one of the valid completions contains whitespace # * there is no prefix and at least one completion _starts_ with whitespace start_quote = '"' else: start_quote = '' # Build and return the result result = line[0:len(line) - len(tokens[-1])] result += last_token_prefix + start_quote result += last_token[:len(last_token) - len(token)] result += completed_file return (result, completions) else: # No expansion was made, return original line return (line, [])
def complete_file_simple(line): """ Complete names of files or directories This function tokenizes the line and computes file and directory completions starting with the last token. It returns a pair: - the line expanded up to the longest common sequence among the completions - the list of all possible completions (first dirs, then files) """ tokens = parse_line(line) if tokens == [] or (line[-1] in sep_chars and parse_line(line) == parse_line(line + ' ')): tokens += [''] # This saves us some checks later token = tokens[-1].replace('"', '') (path_to_complete, _, prefix) = token.rpartition('\\') if path_to_complete == '' and token != '' and token[0] == '\\': path_to_complete = '\\' # print '\n\n', path_to_complete, '---', prefix, '\n\n' if path_to_complete == '': dir_to_complete = os.getcwd() elif path_to_complete == '\\': dir_to_complete = os.getcwd()[0:3] else: dir_to_complete = expand_env_vars(path_to_complete) + '\\' # This is the wildcard matcher used throughout the function matcher = wildcard_to_regex(prefix + '*') completions = [] if os.path.isdir(dir_to_complete): try: completions = [elem for elem in os.listdir(dir_to_complete) if matcher.match(elem)] except OSError: # Cannot complete, probably access denied pass # Sort directories first, also append '\'; then, files completions_dirs = [elem + '\\' for elem in completions if os.path.isdir(dir_to_complete + '\\' + elem)] completions_files = [elem for elem in completions if os.path.isfile(dir_to_complete + '\\' + elem)] completions = completions_dirs + completions_files if (len(tokens) == 1 or tokens[-2] in seq_tokens) and path_to_complete == '': # We are at the beginning of a command ==> also complete from the path completions_path = [] for elem_in_path in os.environ['PATH'].split(';'): dir_to_complete = expand_env_vars(elem_in_path) + '\\' try: completions_path += [elem for elem in os.listdir(dir_to_complete) if matcher.match(elem) and os.path.isfile(dir_to_complete + '\\' + elem) and has_exec_extension(elem) and not elem in completions and not elem in completions_path] except OSError: # Cannot complete, probably access denied pass # Add internal commands internal_commands = ['assoc', 'call', 'cd', 'chdir', 'cls', 'color', 'copy', 'date', 'del', 'dir', 'echo', 'endlocal', 'erase', 'exit', 'for', 'ftype', 'goto', 'if', 'md', 'mkdir', 'move', 'path', 'pause', 'popd', 'prompt', 'pushd', 'rem', 'ren', 'rename', 'rd', 'rmdir', 'set', 'setlocal', 'shift', 'start', 'time', 'title', 'type', 'ver', 'verify', 'vol'] if sys.getwindowsversion()[0] >= 6: # Windows 7 or newer internal_commands.append('mklink') completions_path += [elem for elem in internal_commands if matcher.match(elem) and not elem in completions and not elem in completions_path] # Sort in lexical order (case ignored) completions_path.sort(key=str.lower) # Remove .com, .exe or .bat extension where possible completions_path_no_ext = [strip_extension(elem) for elem in completions_path] completions_path_nice = [] for i in range(0, len(completions_path_no_ext)): similar = [elem for elem in completions_path_no_ext if elem == completions_path_no_ext[i]] similar += [elem for elem in completions if strip_extension(elem) == completions_path_no_ext[i]] if len(similar) == 1 and has_exec_extension(completions_path[i]) and len(prefix) < len(completions_path[i]) - 3: # No similar executables, don't use extension completions_path_nice.append(completions_path_no_ext[i]) else: # Similar executables found, keep extension completions_path_nice.append(completions_path[i]) completions += completions_path_nice if completions != []: # Find the longest common sequence common_string = find_common_prefix(prefix, completions) if path_to_complete == '': completed_file = common_string elif path_to_complete == '\\': completed_file = '\\' + common_string else: completed_file = path_to_complete + '\\' + common_string if expand_env_vars(completed_file).find(' ') >= 0 or \ (prefix != '' and [elem for elem in completions if contains_special_char(elem)] != []) or \ (prefix == '' and [elem for elem in completions if starts_with_special_char(elem)] != []): # We add quotes if one of the following holds: # * the (env-expanded) completed string contains whitespace # * there is a prefix and at least one of the valid completions contains whitespace # * there is no prefix and at least one completion _starts_ with whitespace start_quote = '"' else: start_quote = '' # Build the result result = line[0 : len(line) - len(tokens[-1])] + start_quote + completed_file if len(completions) == 1: # We can close the quotes if we have completed to a unique filename if start_quote == '"': end_quote = '"' else: end_quote = '' if result[-1] == '\\': # Directory -- we want the backslash (if any) AFTER the closing quote result = result[ : -1] + end_quote + '\\' else: # File -- add space if the completion is unique result += end_quote result += ' ' return (result, completions) else: # No expansion was made, return original line return (line, [])
def complete_wildcard(line): """ Complete file/dir wildcards This function tokenizes the line and computes file and directory completions starting with the last token which is a wildcard It returns a pair: - the line expanded up to the longest common sequence among the completions - the list of all possible completions (first dirs, then files) """ tokens = parse_line(line) if tokens == [] or (line[-1] in sep_chars and parse_line(line) == parse_line(line + ' ')): tokens += [''] # This saves us some checks later token = tokens[-1].replace('"', '') (path_to_complete, _, prefix) = token.rpartition('\\') if path_to_complete == '' and token != '' and token[0] == '\\': path_to_complete = '\\' # print '\n\n', path_to_complete, '---', prefix, '\n\n' if path_to_complete == '': dir_to_complete = os.getcwd() elif path_to_complete == '\\': dir_to_complete = os.getcwd()[0:3] else: dir_to_complete = expand_env_vars(path_to_complete) + '\\' # This is the wildcard matcher used throughout the function matcher = wildcard_to_regex(prefix + '*') completions = [] if os.path.isdir(dir_to_complete): try: completions = [elem for elem in os.listdir(dir_to_complete) if matcher.match(elem)] except OSError: # Cannot complete, probably access denied pass # Sort directories first, also append '\'; then, files completions_dirs = [elem + '\\' for elem in completions if os.path.isdir(dir_to_complete + '\\' + elem)] completions_files = [elem for elem in completions if os.path.isfile(dir_to_complete + '\\' + elem)] completions = completions_dirs + completions_files if completions != []: completed_suffixes = [] for c in completions: match = matcher.match(c) completed_suffixes.append(match.group(match.lastindex)) if len(completions) == 1: # Only one match, we can inline it and replace the wildcards common_string = completions[0] else: # Multiple matches, find the longest common sequence common_string = prefix + find_common_prefix(prefix, completed_suffixes) if path_to_complete == '': completed_file = common_string elif path_to_complete == '\\': completed_file = '\\' + common_string else: completed_file = path_to_complete + '\\' + common_string if expand_env_vars(completed_file).find(' ') >= 0 or \ (prefix != '' and [elem for elem in completions if contains_special_char(elem)] != []) or \ (prefix == '' and [elem for elem in completions if starts_with_special_char(elem)] != []): # We add quotes if one of the following holds: # * the (env-expanded) completed string contains whitespace # * there is a prefix and at least one of the valid completions contains whitespace # * there is no prefix and at least one completion _starts_ with whitespace start_quote = '"' else: start_quote = '' # Build the result result = line[0 : len(line) - len(tokens[-1])] + start_quote + completed_file if len(completions) == 1 or \ not common_string.endswith('*') and \ max([len(c) for c in completed_suffixes]) == len(common_string) - len(prefix): # We can close the quotes if all the completions have the same suffix or # there exists only one matching file if start_quote == '"': end_quote = '"' else: end_quote = '' if result[-1] == '\\': # Directory -- we want the backslash (if any) AFTER the closing quote result = result[ : -1] + end_quote + '\\' else: # File -- add space if the completion is unique result += end_quote result += ' ' return (result, completions) else: # No expansion was made, return original line return (line, [])
def complete_file_alternate(line): """ Complete names of files or directories using an alternate tokenization This function tokenizes the line by tring to interpret the last token as a semicolon-separated list of paths, optionally preceded by an equals char. It returns a pair: - the line expanded up to the longest common sequence among the completions - the list of all possible completions (first dirs, then files) """ tokens = parse_line(line) if tokens == [] or (line[-1] in sep_chars and parse_line(line) == parse_line(line + ' ')): tokens += [''] # This saves us some checks later (last_token_prefix, equal_char, last_token) = tokens[-1].replace('"', '').rpartition('=') last_token_prefix += equal_char paths = last_token.split(';') token = paths[-1] # print '\n\nTokens:', tokens, '\n\nCompleting:', token, '\n\n' (path_to_complete, _, prefix) = token.rpartition('\\') if path_to_complete == '' and token != '' and token[0] == '\\': path_to_complete = '\\' # print '\n\n', path_to_complete, '---', prefix, '\n\n' if path_to_complete == '': dir_to_complete = os.getcwd() elif path_to_complete == '\\': dir_to_complete = os.getcwd()[0:3] else: dir_to_complete = expand_env_vars(path_to_complete) + '\\' # This is the wildcard matcher used throughout the function matcher = wildcard_to_regex(prefix + '*') completions = [] if os.path.isdir(dir_to_complete): try: completions = [elem for elem in os.listdir(dir_to_complete) if matcher.match(elem)] except OSError: # Cannot complete, probably access denied pass # Sort directories first, also append '\'; then, files completions_dirs = [elem + '\\' for elem in completions if os.path.isdir(dir_to_complete + '\\' + elem)] completions_files = [elem for elem in completions if os.path.isfile(dir_to_complete + '\\' + elem)] completions = completions_dirs + completions_files if completions != []: # Find the longest common sequence common_string = find_common_prefix(prefix, completions) if path_to_complete == '': completed_file = common_string elif path_to_complete == '\\': completed_file = '\\' + common_string else: completed_file = path_to_complete + '\\' + common_string if expand_env_vars(last_token + completed_file).find(' ') >= 0 or \ (prefix != '' and [elem for elem in completions if contains_special_char(elem)] != []) or \ (prefix == '' and [elem for elem in completions if starts_with_special_char(elem)] != []): # We add quotes if one of the following holds: # * the (env-expanded) completed string contains whitespace # * there is a prefix and at least one of the valid completions contains whitespace # * there is no prefix and at least one completion _starts_ with whitespace start_quote = '"' else: start_quote = '' # Build and return the result result = line[0 : len(line) - len(tokens[-1])] result += last_token_prefix + start_quote result += last_token[:len(last_token) - len(token)] result += completed_file return (result, completions) else: # No expansion was made, return original line return (line, [])
if state.search_substr is not None: state.search_substr = None state.reset_selection() else: state.history.reset() break elif rec.Char == chr(27): # Esc if scrolling: scrolling = False else: state.handle(ActionCode.ACTION_ESCAPE) save_history(state.history.list, pycmd_data_dir + '\\history', 1000) auto_select = False elif rec.Char == '\t': # Tab tokens = parse_line(state.before_cursor) if tokens == [] or state.before_cursor[-1] in sep_chars: tokens.append('') # This saves some checks later on if tokens[-1].strip('"').count('%') % 2 == 1: (completed, suggestions) = complete_env_var(state.before_cursor) elif has_wildcards(tokens[-1]): (completed, suggestions) = complete_wildcard(state.before_cursor) else: (completed, suggestions) = complete_file(state.before_cursor) cursor_backward(len(state.before_cursor)) state.handle(ActionCode.ACTION_COMPLETE, completed) stdout.write(state.before_cursor + state.after_cursor)
def testReparseLine(self): """Test that reparse of print of first parse is unchanged.""" for input, expected in self.lines_to_parse: first_parse = parse_line(input) second_parse = parse_line(' '.join(first_parse)) self.assertEqual(first_parse, second_parse)
def testParseLine(self): """Test that result of parse_line equals expected result.""" for input, expected in self.lines_to_parse: self.assertEqual(parse_line(input), expected)
elif rec.Char == chr(13): # Enter state.history.reset() break elif rec.Char == chr(27): # Esc if scrolling: scrolling = False else: state.handle(ActionCode.ACTION_ESCAPE) save_history(state.history.list, pycmd_data_dir + '\\history', 1000) auto_select = False elif rec.Char == '\t': # Tab stdout.write(state.after_cursor) # Move cursor to the end tokens = parse_line(state.before_cursor) if tokens == [] or state.before_cursor[-1] in sep_chars: tokens.append('') # This saves some checks later on # Check ... expansion at first last_token_len = len(tokens[-1]) if last_token_len >= 2 and \ (tokens[-1].count('.') == last_token_len) : suggestions = [] expanded_token = '..\\' for i in range(last_token_len - 2) : expanded_token = expanded_token + '..\\' tokens[-1] = expanded_token completed = ' '.join(tokens) elif tokens[-1].strip('"').count('%') % 2 == 1:
def read_data(reader, yearmonth_begin, nb_months, process_row_func, get_profiles_func, return_raw_data=False, personal_recommendations=None, common_recommendations=None, product_stats=None): """ :param reader: :param yearmonth_begin: e.g. 201501 :param nb_months: e.g. 5 -> 201501, 201502, 201503, 201504, 201505 :return: (personal_recommendations, common_recommendations, product_stats) or if return_raw_data == True: (personal_recommendations, common_recommendations, product_stats, raw_data) raw_data : list of read and parsed lines """ def _to_yearmonth_str(yearmonth): year = int(math.floor(yearmonth * 0.01)) month = yearmonth - year * 100 return "%s-%s" % (str(year), str(month).zfill(2)) year = int(math.floor(yearmonth_begin * 0.01)) * 100 dates = [_to_yearmonth_str(yearmonth_begin), ] current_yearmonth = yearmonth_begin for i in range(1, nb_months): current_yearmonth += 1 if current_yearmonth - year > 12: year += 100 current_yearmonth = year + 1 dates.append(_to_yearmonth_str(current_yearmonth)) logging.info("- READ DATA : months to read {}".format(dates)) raw_data = [] if common_recommendations is None: common_recommendations = defaultdict(lambda: defaultdict(float)) if personal_recommendations is None: personal_recommendations = defaultdict(lambda: defaultdict(float)) if product_stats is None: product_stats = defaultdict(int) # Loop on lines in the file reader: removed_rows = 0 total = 0 start = time.time() mean_row_processing = 0.0 mean_update = 0.0 while True: line = reader.readline()[:-1] total += 1 if line == '': break row = parse_line(line) # data : ['2015-01-28', # user id -> '1375586', # profile -> 'N', 'ES', 'H', '35', '2015-01-12', '0', '6', '1', '', '1.0', 'A', 'S', 'N', '', 'KHL', 'N', '1', '29', 'MALAGA', '1', '87218.1', '02 - PARTICULARES', # choices -> '0', '0', '1', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0'] yearmonth_str = row[0][:7] if yearmonth_str not in dates: yearmonth = to_yearmonth(yearmonth_str) if yearmonth - yearmonth_begin < 0: total = 0 continue else: break if return_raw_data: raw_data.append(row) start_mrp = time.time() processed_row = process_row_func(row) mean_row_processing += time.time() - start_mrp if processed_row is None or len(processed_row) == 0: removed_rows += 1 continue last_choice = get_last_choice(get_user(processed_row), personal_recommendations) start_mu = time.time() update_common_recommendations(common_recommendations, processed_row, get_profiles_func, last_choice) update_product_stats(product_stats, processed_row, last_choice) update_personal_recommendations(personal_recommendations, processed_row) mean_update += time.time() - start_mu if total % 100000 == 0: elapsed = time.time() - start start = time.time() mean_row_processing /= float(total) mean_update /= float(total) logging.debug('--- Time analysis : mean row processing : {}, mean update : {}'.format(mean_row_processing, mean_update)) logging.info('-- Processed {} lines : Elapsed {} s. Current month : {}'.format(total, elapsed, yearmonth_str)) logging.debug("-- Removed rows : %s" % removed_rows) if return_raw_data: return personal_recommendations, common_recommendations, product_stats, raw_data return personal_recommendations, common_recommendations, product_stats
state.search_substr = None state.reset_selection() else: state.history.reset() break elif rec.Char == chr(27): # Esc if scrolling: scrolling = False else: state.handle(ActionCode.ACTION_ESCAPE) update_history(state.history.list[-1], pycmd_data_dir + '\\history', save_history_limit) auto_select = False elif rec.Char == '\t': # Tab tokens = parse_line(state.before_cursor) if tokens == [] or state.before_cursor[-1] in sep_chars: tokens.append('') # This saves some checks later on if tokens[-1].strip('"').count('%') % 2 == 1: (completed, suggestions) = complete_env_var(state.before_cursor) elif has_wildcards(tokens[-1]): (completed, suggestions) = complete_wildcard(state.before_cursor) else: (completed, suggestions) = complete_file(state.before_cursor) cursor_backward(len(state.before_cursor)) state.handle(ActionCode.ACTION_COMPLETE, completed) stdout.write(state.before_cursor + state.after_cursor) # Show multiple completions if available if not suggestions: