def complete_env_var(line):
"""
Complete names of environment variables
This function tokenizes the line and computes completions
based on environment variables starting with the last token
It returns a pair:
- the line expanded up to the longest common sequence among the
completions
- the list of all possible completions
"""
tokens = parse_line(line)
if tokens == [] or (line[-1] in sep_chars
and parse_line(line) == parse_line(line + ' ')):
tokens += [''] # This saves some checks later
# Account for the VAR=VALUE syntax
(token_prefix, equals, token_orig) = tokens[-1].rpartition('=')
token_prefix += equals
if token_orig.count('%') % 2 == 0 and token_orig.strip('"').endswith('%'):
[lead, prefix] = token_orig.strip('"').rstrip('%').rsplit('%', 2)
else:
[lead, prefix] = token_orig.strip('"').rsplit('%', 1)
if token_orig.strip('"').endswith('%') and prefix != '':
completions = [prefix]
else:
completions = [
var for var in os.environ if var.lower().startswith(prefix.lower())
]
completions.sort()
if completions != []:
# Find longest prefix
common_string = find_common_prefix(prefix, completions)
quote = '' # Assume no quoting needed by default, then check for spaces
for completion in completions:
if contains_special_char(os.environ[completion]):
quote = '"'
break
result = line[0:len(line) -
len(token_orig)] + quote + lead + '%' + common_string
if len(completions) == 1:
result += '%' + quote
return (result, [])
else:
return (result, completions)
else:
# No completion possible, return original line
return (line, [])
def complete_env_var(line):
"""
Complete names of environment variables
This function tokenizes the line and computes completions
based on environment variables starting with the last token
It returns a pair:
- the line expanded up to the longest common sequence among the
completions
- the list of all possible completions
"""
tokens = parse_line(line)
if tokens == [] or (line[-1] in sep_chars and parse_line(line) == parse_line(line + ' ')):
tokens += [''] # This saves some checks later
# Account for the VAR=VALUE syntax
(token_prefix, equals, token_orig) = tokens[-1].rpartition('=')
token_prefix += equals
if token_orig.count('%') % 2 == 0 and token_orig.strip('"').endswith('%'):
[lead, prefix] = token_orig.strip('"').rstrip('%').rsplit('%', 2)
else:
[lead, prefix] = token_orig.strip('"').rsplit('%', 1)
if token_orig.strip('"').endswith('%') and prefix != '':
completions = [prefix]
else:
completions = [var for var in os.environ if var.lower().startswith(prefix.lower())]
completions.sort()
if completions != []:
# Find longest prefix
common_string = find_common_prefix(prefix, completions)
quote = '' # Assume no quoting needed by default, then check for spaces
for completion in completions:
if contains_special_char(os.environ[completion]):
quote = '"'
break
result = line[0 : len(line) - len(token_orig)] + quote + lead + '%' + common_string
if len(completions) == 1:
result += '%' + quote
return (result, [])
else:
return (result, completions)
else:
# No completion possible, return original line
return (line, [])
def write_submission(writer, reader, target_labels,
process_row_func,
get_profiles_func,
personal_recommendations,
common_recommendations,
product_stats,
personal_recommendations_weight):
total = 0
writer.write("ncodpers,added_products\n")
removed_rows = 0
while True:
line = reader.readline()[:-1]
total += 1
if line == '':
break
row = parse_line(line)
# Write before row processing
user = get_user(row)
writer.write(user + ',')
prow = process_row_func(row)
if prow is None or len(prow) == 0:
removed_rows += 1
logging.debug("--- Removed row : {}".format(row))
writer.write("\n")
continue
predicted = compute_predictions(prow,
get_profiles_func,
personal_recommendations,
common_recommendations,
product_stats, personal_recommendations_weight)
for p in predicted:
writer.write(target_labels[p] + ' ')
if total % 1000000 == 0:
logging.info('Read {} lines'.format(total))
writer.write("\n")
logging.info("-- Removed rows : %s" % removed_rows)
def complete_file_simple(line):
"""
Complete names of files or directories
This function tokenizes the line and computes file and directory
completions starting with the last token.
It returns a pair:
- the line expanded up to the longest common sequence among the
completions
- the list of all possible completions (first dirs, then files)
"""
tokens = parse_line(line)
if tokens == [] or (line[-1] in sep_chars
and parse_line(line) == parse_line(line + ' ')):
tokens += [''] # This saves us some checks later
token = tokens[-1].replace('"', '')
(path_to_complete, _, prefix) = token.rpartition('\\')
if path_to_complete == '' and token != '' and token[0] == '\\':
path_to_complete = '\\'
# print '\n\n', path_to_complete, '---', prefix, '\n\n'
if path_to_complete == '':
dir_to_complete = os.getcwd()
elif path_to_complete == '\\':
dir_to_complete = os.getcwd()[0:3]
else:
dir_to_complete = expand_env_vars(path_to_complete) + '\\'
# This is the wildcard matcher used throughout the function
matcher = wildcard_to_regex(prefix + '*')
completions = []
if os.path.isdir(dir_to_complete):
try:
completions = [
elem for elem in os.listdir(dir_to_complete)
if matcher.match(elem)
]
except OSError:
# Cannot complete, probably access denied
pass
# Sort directories first, also append '\'; then, files
completions_dirs = [
elem + '\\' for elem in completions
if os.path.isdir(dir_to_complete + '\\' + elem)
]
completions_files = [
elem for elem in completions
if os.path.isfile(dir_to_complete + '\\' + elem)
]
completions = completions_dirs + completions_files
if (len(tokens) == 1
or tokens[-2] in seq_tokens) and path_to_complete == '':
# We are at the beginning of a command ==> also complete from the path
completions_path = []
for elem_in_path in os.environ['PATH'].split(';'):
dir_to_complete = expand_env_vars(elem_in_path) + '\\'
try:
completions_path += [
elem for elem in os.listdir(dir_to_complete)
if matcher.match(elem) and os.path.isfile(dir_to_complete +
'\\' + elem)
and has_exec_extension(elem) and not elem in completions
and not elem in completions_path
]
except OSError:
# Cannot complete, probably access denied
pass
# Add internal commands
internal_commands = [
'assoc', 'call', 'cd', 'chdir', 'cls', 'color', 'copy', 'date',
'del', 'dir', 'echo', 'endlocal', 'erase', 'exit', 'for', 'ftype',
'goto', 'if', 'md', 'mkdir', 'move', 'path', 'pause', 'popd',
'prompt', 'pushd', 'rem', 'ren', 'rename', 'rd', 'rmdir', 'set',
'setlocal', 'shift', 'start', 'time', 'title', 'type', 'ver',
'verify', 'vol'
]
if sys.getwindowsversion()[0] >= 6:
# Windows 7 or newer
internal_commands.append('mklink')
completions_path += [
elem for elem in internal_commands if matcher.match(elem)
and not elem in completions and not elem in completions_path
]
# Sort in lexical order (case ignored)
completions_path.sort(key=str.lower)
# Remove .com, .exe or .bat extension where possible
completions_path_no_ext = [
strip_extension(elem) for elem in completions_path
]
completions_path_nice = []
for i in range(0, len(completions_path_no_ext)):
similar = [
elem for elem in completions_path_no_ext
if elem == completions_path_no_ext[i]
]
similar += [
elem for elem in completions
if strip_extension(elem) == completions_path_no_ext[i]
]
if len(similar) == 1 and has_exec_extension(
completions_path[i]) and len(prefix) < len(
completions_path[i]) - 3:
# No similar executables, don't use extension
completions_path_nice.append(completions_path_no_ext[i])
else:
# Similar executables found, keep extension
completions_path_nice.append(completions_path[i])
completions += completions_path_nice
if completions != []:
# Find the longest common sequence
common_string = find_common_prefix(prefix, completions)
if path_to_complete == '':
completed_file = common_string
elif path_to_complete == '\\':
completed_file = '\\' + common_string
else:
completed_file = path_to_complete + '\\' + common_string
if expand_env_vars(completed_file).find(' ') >= 0 or \
(prefix != '' and [elem for elem in completions if contains_special_char(elem)] != []) or \
(prefix == '' and [elem for elem in completions if starts_with_special_char(elem)] != []):
# We add quotes if one of the following holds:
# * the (env-expanded) completed string contains whitespace
# * there is a prefix and at least one of the valid completions contains whitespace
# * there is no prefix and at least one completion _starts_ with whitespace
start_quote = '"'
else:
start_quote = ''
# Build the result
result = line[0:len(line) -
len(tokens[-1])] + start_quote + completed_file
if len(completions) == 1:
# We can close the quotes if we have completed to a unique filename
if start_quote == '"':
end_quote = '"'
else:
end_quote = ''
if result[-1] == '\\':
# Directory -- we want the backslash (if any) AFTER the closing quote
result = result[:-1] + end_quote + '\\'
else:
# File -- add space if the completion is unique
result += end_quote
result += ' '
return (result, completions)
else:
# No expansion was made, return original line
return (line, [])
def complete_wildcard(line):
"""
Complete file/dir wildcards
This function tokenizes the line and computes file and directory
completions starting with the last token which is a wildcard
It returns a pair:
- the line expanded up to the longest common sequence among the
completions
- the list of all possible completions (first dirs, then files)
"""
tokens = parse_line(line)
if tokens == [] or (line[-1] in sep_chars
and parse_line(line) == parse_line(line + ' ')):
tokens += [''] # This saves us some checks later
token = tokens[-1].replace('"', '')
(path_to_complete, _, prefix) = token.rpartition('\\')
if path_to_complete == '' and token != '' and token[0] == '\\':
path_to_complete = '\\'
# print '\n\n', path_to_complete, '---', prefix, '\n\n'
if path_to_complete == '':
dir_to_complete = os.getcwd()
elif path_to_complete == '\\':
dir_to_complete = os.getcwd()[0:3]
else:
dir_to_complete = expand_env_vars(path_to_complete) + '\\'
# This is the wildcard matcher used throughout the function
matcher = wildcard_to_regex(prefix + '*')
completions = []
if os.path.isdir(dir_to_complete):
try:
completions = [
elem for elem in os.listdir(dir_to_complete)
if matcher.match(elem)
]
except OSError:
# Cannot complete, probably access denied
pass
# Sort directories first, also append '\'; then, files
completions_dirs = [
elem + '\\' for elem in completions
if os.path.isdir(dir_to_complete + '\\' + elem)
]
completions_files = [
elem for elem in completions
if os.path.isfile(dir_to_complete + '\\' + elem)
]
completions = completions_dirs + completions_files
if completions != []:
completed_suffixes = []
for c in completions:
match = matcher.match(c)
completed_suffixes.append(match.group(match.lastindex))
if len(completions) == 1:
# Only one match, we can inline it and replace the wildcards
common_string = completions[0]
else:
# Multiple matches, find the longest common sequence
common_string = prefix + find_common_prefix(
prefix, completed_suffixes)
if path_to_complete == '':
completed_file = common_string
elif path_to_complete == '\\':
completed_file = '\\' + common_string
else:
completed_file = path_to_complete + '\\' + common_string
if expand_env_vars(completed_file).find(' ') >= 0 or \
(prefix != '' and [elem for elem in completions if contains_special_char(elem)] != []) or \
(prefix == '' and [elem for elem in completions if starts_with_special_char(elem)] != []):
# We add quotes if one of the following holds:
# * the (env-expanded) completed string contains whitespace
# * there is a prefix and at least one of the valid completions contains whitespace
# * there is no prefix and at least one completion _starts_ with whitespace
start_quote = '"'
else:
start_quote = ''
# Build the result
result = line[0:len(line) -
len(tokens[-1])] + start_quote + completed_file
if len(completions) == 1 or \
not common_string.endswith('*') and \
max([len(c) for c in completed_suffixes]) == len(common_string) - len(prefix):
# We can close the quotes if all the completions have the same suffix or
# there exists only one matching file
if start_quote == '"':
end_quote = '"'
else:
end_quote = ''
if result[-1] == '\\':
# Directory -- we want the backslash (if any) AFTER the closing quote
result = result[:-1] + end_quote + '\\'
else:
# File -- add space if the completion is unique
result += end_quote
result += ' '
return (result, completions)
else:
# No expansion was made, return original line
return (line, [])
def complete_file_alternate(line):
"""
Complete names of files or directories using an alternate tokenization
This function tokenizes the line by tring to interpret the last token as a
semicolon-separated list of paths, optionally preceded by an equals char.
It returns a pair:
- the line expanded up to the longest common sequence among the
completions
- the list of all possible completions (first dirs, then files)
"""
tokens = parse_line(line)
if tokens == [] or (line[-1] in sep_chars
and parse_line(line) == parse_line(line + ' ')):
tokens += [''] # This saves us some checks later
(last_token_prefix, equal_char,
last_token) = tokens[-1].replace('"', '').rpartition('=')
last_token_prefix += equal_char
paths = last_token.split(';')
token = paths[-1]
# print '\n\nTokens:', tokens, '\n\nCompleting:', token, '\n\n'
(path_to_complete, _, prefix) = token.rpartition('\\')
if path_to_complete == '' and token != '' and token[0] == '\\':
path_to_complete = '\\'
# print '\n\n', path_to_complete, '---', prefix, '\n\n'
if path_to_complete == '':
dir_to_complete = os.getcwd()
elif path_to_complete == '\\':
dir_to_complete = os.getcwd()[0:3]
else:
dir_to_complete = expand_env_vars(path_to_complete) + '\\'
# This is the wildcard matcher used throughout the function
matcher = wildcard_to_regex(prefix + '*')
completions = []
if os.path.isdir(dir_to_complete):
try:
completions = [
elem for elem in os.listdir(dir_to_complete)
if matcher.match(elem)
]
except OSError:
# Cannot complete, probably access denied
pass
# Sort directories first, also append '\'; then, files
completions_dirs = [
elem + '\\' for elem in completions
if os.path.isdir(dir_to_complete + '\\' + elem)
]
completions_files = [
elem for elem in completions
if os.path.isfile(dir_to_complete + '\\' + elem)
]
completions = completions_dirs + completions_files
if completions != []:
# Find the longest common sequence
common_string = find_common_prefix(prefix, completions)
if path_to_complete == '':
completed_file = common_string
elif path_to_complete == '\\':
completed_file = '\\' + common_string
else:
completed_file = path_to_complete + '\\' + common_string
if expand_env_vars(last_token + completed_file).find(' ') >= 0 or \
(prefix != '' and [elem for elem in completions if contains_special_char(elem)] != []) or \
(prefix == '' and [elem for elem in completions if starts_with_special_char(elem)] != []):
# We add quotes if one of the following holds:
# * the (env-expanded) completed string contains whitespace
# * there is a prefix and at least one of the valid completions contains whitespace
# * there is no prefix and at least one completion _starts_ with whitespace
start_quote = '"'
else:
start_quote = ''
# Build and return the result
result = line[0:len(line) - len(tokens[-1])]
result += last_token_prefix + start_quote
result += last_token[:len(last_token) - len(token)]
result += completed_file
return (result, completions)
else:
# No expansion was made, return original line
return (line, [])
def complete_file_simple(line):
"""
Complete names of files or directories
This function tokenizes the line and computes file and directory
completions starting with the last token.
It returns a pair:
- the line expanded up to the longest common sequence among the
completions
- the list of all possible completions (first dirs, then files)
"""
tokens = parse_line(line)
if tokens == [] or (line[-1] in sep_chars and parse_line(line) == parse_line(line + ' ')):
tokens += [''] # This saves us some checks later
token = tokens[-1].replace('"', '')
(path_to_complete, _, prefix) = token.rpartition('\\')
if path_to_complete == '' and token != '' and token[0] == '\\':
path_to_complete = '\\'
# print '\n\n', path_to_complete, '---', prefix, '\n\n'
if path_to_complete == '':
dir_to_complete = os.getcwd()
elif path_to_complete == '\\':
dir_to_complete = os.getcwd()[0:3]
else:
dir_to_complete = expand_env_vars(path_to_complete) + '\\'
# This is the wildcard matcher used throughout the function
matcher = wildcard_to_regex(prefix + '*')
completions = []
if os.path.isdir(dir_to_complete):
try:
completions = [elem for elem in os.listdir(dir_to_complete)
if matcher.match(elem)]
except OSError:
# Cannot complete, probably access denied
pass
# Sort directories first, also append '\'; then, files
completions_dirs = [elem + '\\' for elem in completions if os.path.isdir(dir_to_complete + '\\' + elem)]
completions_files = [elem for elem in completions if os.path.isfile(dir_to_complete + '\\' + elem)]
completions = completions_dirs + completions_files
if (len(tokens) == 1 or tokens[-2] in seq_tokens) and path_to_complete == '':
# We are at the beginning of a command ==> also complete from the path
completions_path = []
for elem_in_path in os.environ['PATH'].split(';'):
dir_to_complete = expand_env_vars(elem_in_path) + '\\'
try:
completions_path += [elem for elem in os.listdir(dir_to_complete)
if matcher.match(elem)
and os.path.isfile(dir_to_complete + '\\' + elem)
and has_exec_extension(elem)
and not elem in completions
and not elem in completions_path]
except OSError:
# Cannot complete, probably access denied
pass
# Add internal commands
internal_commands = ['assoc',
'call', 'cd', 'chdir', 'cls', 'color', 'copy',
'date', 'del', 'dir',
'echo', 'endlocal', 'erase', 'exit',
'for', 'ftype',
'goto',
'if',
'md', 'mkdir', 'move',
'path', 'pause', 'popd', 'prompt', 'pushd',
'rem', 'ren', 'rename', 'rd', 'rmdir',
'set', 'setlocal', 'shift', 'start',
'time', 'title', 'type',
'ver', 'verify', 'vol']
if sys.getwindowsversion()[0] >= 6:
# Windows 7 or newer
internal_commands.append('mklink')
completions_path += [elem for elem in internal_commands
if matcher.match(elem)
and not elem in completions
and not elem in completions_path]
# Sort in lexical order (case ignored)
completions_path.sort(key=str.lower)
# Remove .com, .exe or .bat extension where possible
completions_path_no_ext = [strip_extension(elem) for elem in completions_path]
completions_path_nice = []
for i in range(0, len(completions_path_no_ext)):
similar = [elem for elem in completions_path_no_ext if elem == completions_path_no_ext[i]]
similar += [elem for elem in completions if strip_extension(elem) == completions_path_no_ext[i]]
if len(similar) == 1 and has_exec_extension(completions_path[i]) and len(prefix) < len(completions_path[i]) - 3:
# No similar executables, don't use extension
completions_path_nice.append(completions_path_no_ext[i])
else:
# Similar executables found, keep extension
completions_path_nice.append(completions_path[i])
completions += completions_path_nice
if completions != []:
# Find the longest common sequence
common_string = find_common_prefix(prefix, completions)
if path_to_complete == '':
completed_file = common_string
elif path_to_complete == '\\':
completed_file = '\\' + common_string
else:
completed_file = path_to_complete + '\\' + common_string
if expand_env_vars(completed_file).find(' ') >= 0 or \
(prefix != '' and [elem for elem in completions if contains_special_char(elem)] != []) or \
(prefix == '' and [elem for elem in completions if starts_with_special_char(elem)] != []):
# We add quotes if one of the following holds:
# * the (env-expanded) completed string contains whitespace
# * there is a prefix and at least one of the valid completions contains whitespace
# * there is no prefix and at least one completion _starts_ with whitespace
start_quote = '"'
else:
start_quote = ''
# Build the result
result = line[0 : len(line) - len(tokens[-1])] + start_quote + completed_file
if len(completions) == 1:
# We can close the quotes if we have completed to a unique filename
if start_quote == '"':
end_quote = '"'
else:
end_quote = ''
if result[-1] == '\\':
# Directory -- we want the backslash (if any) AFTER the closing quote
result = result[ : -1] + end_quote + '\\'
else:
# File -- add space if the completion is unique
result += end_quote
result += ' '
return (result, completions)
else:
# No expansion was made, return original line
return (line, [])
def complete_wildcard(line):
"""
Complete file/dir wildcards
This function tokenizes the line and computes file and directory
completions starting with the last token which is a wildcard
It returns a pair:
- the line expanded up to the longest common sequence among the
completions
- the list of all possible completions (first dirs, then files)
"""
tokens = parse_line(line)
if tokens == [] or (line[-1] in sep_chars and parse_line(line) == parse_line(line + ' ')):
tokens += [''] # This saves us some checks later
token = tokens[-1].replace('"', '')
(path_to_complete, _, prefix) = token.rpartition('\\')
if path_to_complete == '' and token != '' and token[0] == '\\':
path_to_complete = '\\'
# print '\n\n', path_to_complete, '---', prefix, '\n\n'
if path_to_complete == '':
dir_to_complete = os.getcwd()
elif path_to_complete == '\\':
dir_to_complete = os.getcwd()[0:3]
else:
dir_to_complete = expand_env_vars(path_to_complete) + '\\'
# This is the wildcard matcher used throughout the function
matcher = wildcard_to_regex(prefix + '*')
completions = []
if os.path.isdir(dir_to_complete):
try:
completions = [elem for elem in os.listdir(dir_to_complete) if matcher.match(elem)]
except OSError:
# Cannot complete, probably access denied
pass
# Sort directories first, also append '\'; then, files
completions_dirs = [elem + '\\' for elem in completions if os.path.isdir(dir_to_complete + '\\' + elem)]
completions_files = [elem for elem in completions if os.path.isfile(dir_to_complete + '\\' + elem)]
completions = completions_dirs + completions_files
if completions != []:
completed_suffixes = []
for c in completions:
match = matcher.match(c)
completed_suffixes.append(match.group(match.lastindex))
if len(completions) == 1:
# Only one match, we can inline it and replace the wildcards
common_string = completions[0]
else:
# Multiple matches, find the longest common sequence
common_string = prefix + find_common_prefix(prefix, completed_suffixes)
if path_to_complete == '':
completed_file = common_string
elif path_to_complete == '\\':
completed_file = '\\' + common_string
else:
completed_file = path_to_complete + '\\' + common_string
if expand_env_vars(completed_file).find(' ') >= 0 or \
(prefix != '' and [elem for elem in completions if contains_special_char(elem)] != []) or \
(prefix == '' and [elem for elem in completions if starts_with_special_char(elem)] != []):
# We add quotes if one of the following holds:
# * the (env-expanded) completed string contains whitespace
# * there is a prefix and at least one of the valid completions contains whitespace
# * there is no prefix and at least one completion _starts_ with whitespace
start_quote = '"'
else:
start_quote = ''
# Build the result
result = line[0 : len(line) - len(tokens[-1])] + start_quote + completed_file
if len(completions) == 1 or \
not common_string.endswith('*') and \
max([len(c) for c in completed_suffixes]) == len(common_string) - len(prefix):
# We can close the quotes if all the completions have the same suffix or
# there exists only one matching file
if start_quote == '"':
end_quote = '"'
else:
end_quote = ''
if result[-1] == '\\':
# Directory -- we want the backslash (if any) AFTER the closing quote
result = result[ : -1] + end_quote + '\\'
else:
# File -- add space if the completion is unique
result += end_quote
result += ' '
return (result, completions)
else:
# No expansion was made, return original line
return (line, [])
def complete_file_alternate(line):
"""
Complete names of files or directories using an alternate tokenization
This function tokenizes the line by tring to interpret the last token as a
semicolon-separated list of paths, optionally preceded by an equals char.
It returns a pair:
- the line expanded up to the longest common sequence among the
completions
- the list of all possible completions (first dirs, then files)
"""
tokens = parse_line(line)
if tokens == [] or (line[-1] in sep_chars and parse_line(line) == parse_line(line + ' ')):
tokens += [''] # This saves us some checks later
(last_token_prefix, equal_char, last_token) = tokens[-1].replace('"', '').rpartition('=')
last_token_prefix += equal_char
paths = last_token.split(';')
token = paths[-1]
# print '\n\nTokens:', tokens, '\n\nCompleting:', token, '\n\n'
(path_to_complete, _, prefix) = token.rpartition('\\')
if path_to_complete == '' and token != '' and token[0] == '\\':
path_to_complete = '\\'
# print '\n\n', path_to_complete, '---', prefix, '\n\n'
if path_to_complete == '':
dir_to_complete = os.getcwd()
elif path_to_complete == '\\':
dir_to_complete = os.getcwd()[0:3]
else:
dir_to_complete = expand_env_vars(path_to_complete) + '\\'
# This is the wildcard matcher used throughout the function
matcher = wildcard_to_regex(prefix + '*')
completions = []
if os.path.isdir(dir_to_complete):
try:
completions = [elem for elem in os.listdir(dir_to_complete)
if matcher.match(elem)]
except OSError:
# Cannot complete, probably access denied
pass
# Sort directories first, also append '\'; then, files
completions_dirs = [elem + '\\' for elem in completions if os.path.isdir(dir_to_complete + '\\' + elem)]
completions_files = [elem for elem in completions if os.path.isfile(dir_to_complete + '\\' + elem)]
completions = completions_dirs + completions_files
if completions != []:
# Find the longest common sequence
common_string = find_common_prefix(prefix, completions)
if path_to_complete == '':
completed_file = common_string
elif path_to_complete == '\\':
completed_file = '\\' + common_string
else:
completed_file = path_to_complete + '\\' + common_string
if expand_env_vars(last_token + completed_file).find(' ') >= 0 or \
(prefix != '' and [elem for elem in completions if contains_special_char(elem)] != []) or \
(prefix == '' and [elem for elem in completions if starts_with_special_char(elem)] != []):
# We add quotes if one of the following holds:
# * the (env-expanded) completed string contains whitespace
# * there is a prefix and at least one of the valid completions contains whitespace
# * there is no prefix and at least one completion _starts_ with whitespace
start_quote = '"'
else:
start_quote = ''
# Build and return the result
result = line[0 : len(line) - len(tokens[-1])]
result += last_token_prefix + start_quote
result += last_token[:len(last_token) - len(token)]
result += completed_file
return (result, completions)
else:
# No expansion was made, return original line
return (line, [])
if state.search_substr is not None:
state.search_substr = None
state.reset_selection()
else:
state.history.reset()
break
elif rec.Char == chr(27): # Esc
if scrolling:
scrolling = False
else:
state.handle(ActionCode.ACTION_ESCAPE)
save_history(state.history.list,
pycmd_data_dir + '\\history', 1000)
auto_select = False
elif rec.Char == '\t': # Tab
tokens = parse_line(state.before_cursor)
if tokens == [] or state.before_cursor[-1] in sep_chars:
tokens.append('') # This saves some checks later on
if tokens[-1].strip('"').count('%') % 2 == 1:
(completed,
suggestions) = complete_env_var(state.before_cursor)
elif has_wildcards(tokens[-1]):
(completed,
suggestions) = complete_wildcard(state.before_cursor)
else:
(completed,
suggestions) = complete_file(state.before_cursor)
cursor_backward(len(state.before_cursor))
state.handle(ActionCode.ACTION_COMPLETE, completed)
stdout.write(state.before_cursor + state.after_cursor)
def testReparseLine(self):
"""Test that reparse of print of first parse is unchanged."""
for input, expected in self.lines_to_parse:
first_parse = parse_line(input)
second_parse = parse_line(' '.join(first_parse))
self.assertEqual(first_parse, second_parse)
def testParseLine(self):
"""Test that result of parse_line equals expected result."""
for input, expected in self.lines_to_parse:
self.assertEqual(parse_line(input), expected)
elif rec.Char == chr(13): # Enter
state.history.reset()
break
elif rec.Char == chr(27): # Esc
if scrolling:
scrolling = False
else:
state.handle(ActionCode.ACTION_ESCAPE)
save_history(state.history.list,
pycmd_data_dir + '\\history',
1000)
auto_select = False
elif rec.Char == '\t': # Tab
stdout.write(state.after_cursor) # Move cursor to the end
tokens = parse_line(state.before_cursor)
if tokens == [] or state.before_cursor[-1] in sep_chars:
tokens.append('') # This saves some checks later on
# Check ... expansion at first
last_token_len = len(tokens[-1])
if last_token_len >= 2 and \
(tokens[-1].count('.') == last_token_len) :
suggestions = []
expanded_token = '..\\'
for i in range(last_token_len - 2) :
expanded_token = expanded_token + '..\\'
tokens[-1] = expanded_token
completed = ' '.join(tokens)
elif tokens[-1].strip('"').count('%') % 2 == 1:
def read_data(reader, yearmonth_begin, nb_months,
process_row_func, get_profiles_func,
return_raw_data=False,
personal_recommendations=None, common_recommendations=None, product_stats=None):
"""
:param reader:
:param yearmonth_begin: e.g. 201501
:param nb_months: e.g. 5 -> 201501, 201502, 201503, 201504, 201505
:return:
(personal_recommendations,
common_recommendations,
product_stats)
or if return_raw_data == True:
(personal_recommendations,
common_recommendations,
product_stats,
raw_data)
raw_data : list of read and parsed lines
"""
def _to_yearmonth_str(yearmonth):
year = int(math.floor(yearmonth * 0.01))
month = yearmonth - year * 100
return "%s-%s" % (str(year), str(month).zfill(2))
year = int(math.floor(yearmonth_begin * 0.01)) * 100
dates = [_to_yearmonth_str(yearmonth_begin), ]
current_yearmonth = yearmonth_begin
for i in range(1, nb_months):
current_yearmonth += 1
if current_yearmonth - year > 12:
year += 100
current_yearmonth = year + 1
dates.append(_to_yearmonth_str(current_yearmonth))
logging.info("- READ DATA : months to read {}".format(dates))
raw_data = []
if common_recommendations is None:
common_recommendations = defaultdict(lambda: defaultdict(float))
if personal_recommendations is None:
personal_recommendations = defaultdict(lambda: defaultdict(float))
if product_stats is None:
product_stats = defaultdict(int)
# Loop on lines in the file reader:
removed_rows = 0
total = 0
start = time.time()
mean_row_processing = 0.0
mean_update = 0.0
while True:
line = reader.readline()[:-1]
total += 1
if line == '':
break
row = parse_line(line)
# data : ['2015-01-28',
# user id -> '1375586',
# profile -> 'N', 'ES', 'H', '35', '2015-01-12', '0', '6', '1', '', '1.0', 'A', 'S', 'N', '', 'KHL', 'N', '1', '29', 'MALAGA', '1', '87218.1', '02 - PARTICULARES',
# choices -> '0', '0', '1', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0']
yearmonth_str = row[0][:7]
if yearmonth_str not in dates:
yearmonth = to_yearmonth(yearmonth_str)
if yearmonth - yearmonth_begin < 0:
total = 0
continue
else:
break
if return_raw_data:
raw_data.append(row)
start_mrp = time.time()
processed_row = process_row_func(row)
mean_row_processing += time.time() - start_mrp
if processed_row is None or len(processed_row) == 0:
removed_rows += 1
continue
last_choice = get_last_choice(get_user(processed_row), personal_recommendations)
start_mu = time.time()
update_common_recommendations(common_recommendations, processed_row, get_profiles_func, last_choice)
update_product_stats(product_stats, processed_row, last_choice)
update_personal_recommendations(personal_recommendations, processed_row)
mean_update += time.time() - start_mu
if total % 100000 == 0:
elapsed = time.time() - start
start = time.time()
mean_row_processing /= float(total)
mean_update /= float(total)
logging.debug('--- Time analysis : mean row processing : {}, mean update : {}'.format(mean_row_processing, mean_update))
logging.info('-- Processed {} lines : Elapsed {} s. Current month : {}'.format(total, elapsed, yearmonth_str))
logging.debug("-- Removed rows : %s" % removed_rows)
if return_raw_data:
return personal_recommendations, common_recommendations, product_stats, raw_data
return personal_recommendations, common_recommendations, product_stats
state.search_substr = None
state.reset_selection()
else:
state.history.reset()
break
elif rec.Char == chr(27): # Esc
if scrolling:
scrolling = False
else:
state.handle(ActionCode.ACTION_ESCAPE)
update_history(state.history.list[-1],
pycmd_data_dir + '\\history',
save_history_limit)
auto_select = False
elif rec.Char == '\t': # Tab
tokens = parse_line(state.before_cursor)
if tokens == [] or state.before_cursor[-1] in sep_chars:
tokens.append('') # This saves some checks later on
if tokens[-1].strip('"').count('%') % 2 == 1:
(completed, suggestions) = complete_env_var(state.before_cursor)
elif has_wildcards(tokens[-1]):
(completed, suggestions) = complete_wildcard(state.before_cursor)
else:
(completed, suggestions) = complete_file(state.before_cursor)
cursor_backward(len(state.before_cursor))
state.handle(ActionCode.ACTION_COMPLETE, completed)
stdout.write(state.before_cursor + state.after_cursor)
# Show multiple completions if available
if not suggestions:
