def test_convert_utilities(tmpdir):
import pydevd_file_utils
import sys
test_dir = str(tmpdir.mkdir("Test_Convert_Utilities"))
if sys.platform == 'win32':
normalized = pydevd_file_utils.normcase(test_dir)
assert isinstance(normalized, str) # bytes on py2, unicode on py3
assert normalized.lower() == normalized
assert '~' not in normalized
assert '~' in pydevd_file_utils.convert_to_short_pathname(normalized)
real_case = pydevd_file_utils.get_path_with_real_case(normalized)
assert isinstance(real_case, str) # bytes on py2, unicode on py3
# Note test_dir itself cannot be compared with because pytest may
# have passed the case normalized.
assert real_case.endswith("Test_Convert_Utilities")
else:
# On other platforms, nothing should change
assert pydevd_file_utils.normcase(test_dir) == test_dir
assert pydevd_file_utils.convert_to_short_pathname(
test_dir) == test_dir
assert pydevd_file_utils.get_path_with_real_case(test_dir) == test_dir
def _check_matches(patterns, paths):
if not patterns and not paths:
# Matched to the end.
return True
if (not patterns and paths) or (patterns and not paths):
return False
pattern = normcase(patterns[0])
path = normcase(paths[0])
if not glob.has_magic(pattern):
if pattern != path:
return False
elif pattern == '**':
if len(patterns) == 1:
return True # if ** is the last one it matches anything to the right.
for i in xrange(len(paths)):
# Recursively check the remaining patterns as the
# current pattern could match any number of paths.
if _check_matches(patterns[1:], paths[i:]):
return True
elif not fnmatch.fnmatch(path, pattern):
# Current part doesn't match.
return False
return _check_matches(patterns[1:], paths[1:])
def set_dont_trace_start_end_patterns(self, py_db, start_patterns, end_patterns):
# Note: start/end patterns normalized internally.
start_patterns = tuple(pydevd_file_utils.normcase(x) for x in start_patterns)
end_patterns = tuple(pydevd_file_utils.normcase(x) for x in end_patterns)
# After it's set the first time, we can still change it, but we need to reset the
# related caches.
reset_caches = False
dont_trace_start_end_patterns_previously_set = \
py_db.dont_trace_external_files.__name__ == 'custom_dont_trace_external_files'
if not dont_trace_start_end_patterns_previously_set and not start_patterns and not end_patterns:
# If it wasn't set previously and start and end patterns are empty we don't need to do anything.
return
if not py_db.is_cache_file_type_empty():
# i.e.: custom function set in set_dont_trace_start_end_patterns.
if dont_trace_start_end_patterns_previously_set:
reset_caches = py_db.dont_trace_external_files.start_patterns != start_patterns or \
py_db.dont_trace_external_files.end_patterns != end_patterns
else:
reset_caches = True
def custom_dont_trace_external_files(abs_path):
normalized_abs_path = pydevd_file_utils.normcase(abs_path)
return normalized_abs_path.startswith(start_patterns) or normalized_abs_path.endswith(end_patterns)
custom_dont_trace_external_files.start_patterns = start_patterns
custom_dont_trace_external_files.end_patterns = end_patterns
py_db.dont_trace_external_files = custom_dont_trace_external_files
if reset_caches:
py_db.clear_dont_trace_start_end_patterns_caches()
def test_convert_utilities(tmpdir):
import pydevd_file_utils
test_dir = str(tmpdir.mkdir("Test_Convert_Utilities"))
if IS_WINDOWS:
normalized = pydevd_file_utils.normcase(test_dir)
assert isinstance(normalized, str) # bytes on py2, unicode on py3
assert normalized.lower() == normalized
upper_version = os.path.join(test_dir, 'ÁÉÍÓÚ')
with open(upper_version, 'w') as stream:
stream.write('test')
with open(upper_version, 'r') as stream:
assert stream.read() == 'test'
with open(pydevd_file_utils.normcase(upper_version), 'r') as stream:
assert stream.read() == 'test'
assert '~' not in normalized
for i in range(3): # Check if cache is ok.
if i == 2:
pydevd_file_utils._listdir_cache.clear()
assert pydevd_file_utils.get_path_with_real_case('<does not EXIST>') == '<does not EXIST>'
real_case = pydevd_file_utils.get_path_with_real_case(normalized)
assert isinstance(real_case, str) # bytes on py2, unicode on py3
# Note test_dir itself cannot be compared with because pytest may
# have passed the case normalized.
assert real_case.endswith("Test_Convert_Utilities")
if i == 2:
# Check that we have the expected paths in the cache.
assert pydevd_file_utils._listdir_cache[os.path.dirname(normalized).lower()] == ['Test_Convert_Utilities']
assert pydevd_file_utils._listdir_cache[(os.path.dirname(normalized).lower(), 'Test_Convert_Utilities'.lower())] == real_case
if IS_PY2:
# Test with unicode in python 2 too.
real_case = pydevd_file_utils.get_path_with_real_case(normalized.decode(
getfilesystemencoding()))
assert isinstance(real_case, str) # bytes on py2, unicode on py3
# Note test_dir itself cannot be compared with because pytest may
# have passed the case normalized.
assert real_case.endswith("Test_Convert_Utilities")
# Check that it works with a shortened path.
shortened = pydevd_file_utils.convert_to_short_pathname(normalized)
assert '~' in shortened
with_real_case = pydevd_file_utils.get_path_with_real_case(shortened)
assert with_real_case.endswith('Test_Convert_Utilities')
assert '~' not in with_real_case
else:
# On other platforms, nothing should change
assert pydevd_file_utils.normcase(test_dir) == test_dir
assert pydevd_file_utils.get_path_with_real_case(test_dir) == test_dir
def _get_template_file_name(frame):
try:
if IS_DJANGO19:
# The Node source was removed since Django 1.9
if 'context' in frame.f_locals:
context = frame.f_locals['context']
if hasattr(context, '_has_included_template'):
# if there was included template we need to inspect the previous frames and find its name
back = frame.f_back
while back is not None and frame.f_code.co_name in (
'render', '_render'):
locals = back.f_locals
if 'self' in locals:
self = locals['self']
if self.__class__.__name__ == 'Template' and hasattr(self, 'origin') and \
hasattr(self.origin, 'name'):
return normcase(
_convert_to_str(self.origin.name))
back = back.f_back
else:
if hasattr(context, 'template') and hasattr(context.template, 'origin') and \
hasattr(context.template.origin, 'name'):
return normcase(
_convert_to_str(context.template.origin.name))
return None
elif IS_DJANGO19_OR_HIGHER:
# For Django 1.10 and later there is much simpler way to get template name
if 'self' in frame.f_locals:
self = frame.f_locals['self']
if hasattr(self, 'origin') and hasattr(self.origin, 'name'):
return normcase(_convert_to_str(self.origin.name))
return None
source = _get_source_django_18_or_lower(frame)
if source is None:
pydev_log.debug("Source is None\n")
return None
fname = _convert_to_str(source[0].name)
if fname == '<unknown source>':
pydev_log.debug("Source name is %s\n" % fname)
return None
else:
abs_path_real_path_and_base = get_abs_path_real_path_and_base_from_file(
fname)
return abs_path_real_path_and_base[1]
except:
pydev_log.debug(traceback.format_exc())
return None
def _get_template_file_name(frame):
try:
if IS_DJANGO19_OR_HIGHER:
# The Node source was removed since Django 1.9
if dict_contains(frame.f_locals, 'context'):
context = frame.f_locals['context']
if hasattr(context, 'template') and hasattr(context.template, 'origin') and \
hasattr(context.template.origin, 'name'):
return normcase(context.template.origin.name)
return None
source = _get_source_django_18_or_lower(frame)
if source is None:
pydev_log.debug("Source is None\n")
return None
fname = source[0].name
if fname == '<unknown source>':
pydev_log.debug("Source name is %s\n" % fname)
return None
else:
abs_path_real_path_and_base = get_abs_path_real_path_and_base_from_file(fname)
return abs_path_real_path_and_base[1]
except:
pydev_log.debug(traceback.format_exc())
return None
def _get_template_file_name(frame):
try:
if IS_DJANGO19:
# The Node source was removed since Django 1.9
if 'context' in frame.f_locals:
context = frame.f_locals['context']
if hasattr(context, '_has_included_template'):
# if there was included template we need to inspect the previous frames and find its name
back = frame.f_back
while back is not None and frame.f_code.co_name in ('render', '_render'):
locals = back.f_locals
if 'self' in locals:
self = locals['self']
if self.__class__.__name__ == 'Template' and hasattr(self, 'origin') and \
hasattr(self.origin, 'name'):
return normcase(self.origin.name)
back = back.f_back
else:
if hasattr(context, 'template') and hasattr(context.template, 'origin') and \
hasattr(context.template.origin, 'name'):
return normcase(context.template.origin.name)
return None
elif IS_DJANGO19_OR_HIGHER:
# For Django 1.10 and later there is much simpler way to get template name
if 'self' in frame.f_locals:
self = frame.f_locals['self']
if hasattr(self, 'origin') and hasattr(self.origin, 'name'):
return normcase(self.origin.name)
return None
source = _get_source_django_18_or_lower(frame)
if source is None:
pydev_log.debug("Source is None\n")
return None
fname = source[0].name
if fname == '<unknown source>':
pydev_log.debug("Source name is %s\n" % fname)
return None
else:
abs_path_real_path_and_base = get_abs_path_real_path_and_base_from_file(fname)
return abs_path_real_path_and_base[1]
except:
pydev_log.debug(traceback.format_exc())
return None
def exception_break(plugin, main_debugger, pydb_frame, frame, args, arg):
main_debugger = args[0]
thread = args[3]
exception, value, trace = arg
if main_debugger.django_exception_break and exception is not None:
if exception.__name__ in ['VariableDoesNotExist', 'TemplateDoesNotExist', 'TemplateSyntaxError'] and \
just_raised(trace) and not ignore_exception_trace(trace):
if exception.__name__ == 'TemplateSyntaxError':
# In this case we don't actually have a regular render frame with the context
# (we didn't really get to that point).
token = getattr(value, 'token', None)
lineno = getattr(token, 'lineno', None)
filename = None
if lineno is not None:
get_template_frame = frame
while get_template_frame.f_code.co_name != 'get_template':
get_template_frame = get_template_frame.f_back
origin = None
if get_template_frame is not None:
origin = get_template_frame.f_locals.get('origin')
if hasattr(origin, 'name') and origin.name is not None:
filename = normcase(_convert_to_str(origin.name))
if filename is not None and lineno is not None:
syntax_error_frame = DjangoTemplateSyntaxErrorFrame(
frame, filename, lineno, {
'token': token,
'exception': exception
})
suspend_frame = suspend_django(main_debugger, thread,
syntax_error_frame,
CMD_ADD_EXCEPTION_BREAK)
return True, suspend_frame
elif exception.__name__ == 'VariableDoesNotExist':
if _is_django_variable_does_not_exist_exception_break_context(
frame):
render_frame = _find_django_render_frame(frame)
if render_frame:
suspend_frame = suspend_django(
main_debugger, thread,
DjangoTemplateFrame(render_frame),
CMD_ADD_EXCEPTION_BREAK)
if suspend_frame:
add_exception_to_frame(suspend_frame,
(exception, value, trace))
thread.additional_info.pydev_message = 'VariableDoesNotExist'
suspend_frame.f_back = frame
frame = suspend_frame
return True, frame
return None
def _get_filename_from_origin_in_parent_frame_locals(frame, parent_frame_name):
filename = None
parent_frame = frame
while parent_frame.f_code.co_name != parent_frame_name:
parent_frame = parent_frame.f_back
origin = None
if parent_frame is not None:
origin = parent_frame.f_locals.get('origin')
if hasattr(origin, 'name') and origin.name is not None:
filename = normcase(_convert_to_str(origin.name))
return filename
def _get_filename_from_origin_in_parent_frame_locals(frame, parent_frame_name):
filename = None
parent_frame = frame
while parent_frame.f_code.co_name != parent_frame_name:
parent_frame = parent_frame.f_back
origin = None
if parent_frame is not None:
origin = parent_frame.f_locals.get('origin')
if hasattr(origin, 'name') and origin.name is not None:
filename = normcase(_convert_to_str(origin.name))
return filename
def test_convert_utilities(tmpdir):
import pydevd_file_utils
test_dir = str(tmpdir.mkdir("Test_Convert_Utilities"))
if IS_WINDOWS:
normalized = pydevd_file_utils.normcase(test_dir)
assert isinstance(normalized, str) # bytes on py2, unicode on py3
assert normalized.lower() == normalized
upper_version = os.path.join(test_dir, 'ÁÉÍÓÚ')
with open(upper_version, 'w') as stream:
stream.write('test')
with open(upper_version, 'r') as stream:
assert stream.read() == 'test'
with open(pydevd_file_utils.normcase(upper_version), 'r') as stream:
assert stream.read() == 'test'
assert '~' not in normalized
if not IS_JYTHON:
assert '~' in pydevd_file_utils.convert_to_short_pathname(
normalized)
real_case = pydevd_file_utils.get_path_with_real_case(normalized)
assert isinstance(real_case, str) # bytes on py2, unicode on py3
# Note test_dir itself cannot be compared with because pytest may
# have passed the case normalized.
assert real_case.endswith("Test_Convert_Utilities")
else:
# On other platforms, nothing should change
assert pydevd_file_utils.normcase(test_dir) == test_dir
assert pydevd_file_utils.convert_to_short_pathname(
test_dir) == test_dir
assert pydevd_file_utils.get_path_with_real_case(test_dir) == test_dir
def set_source_mapping(self, absolute_filename, mapping):
'''
:param str absolute_filename:
The filename for the source mapping (bytes on py2 and str on py3).
:param list(SourceMappingEntry) mapping:
A list with the source mapping entries to be applied to the given filename.
:return str:
An error message if it was not possible to set the mapping or an empty string if
everything is ok.
'''
# Let's first validate if it's ok to apply that mapping.
# File mappings must be 1:N, not M:N (i.e.: if there's a mapping from file1.py to <cell1>,
# there can be no other mapping from any other file to <cell1>).
# This is a limitation to make it easier to remove existing breakpoints when new breakpoints are
# set to a file (so, any file matching that breakpoint can be removed instead of needing to check
# which lines are corresponding to that file).
for map_entry in mapping:
existing_source_filename = self._mappings_to_client.get(
map_entry.runtime_source)
if existing_source_filename and existing_source_filename != absolute_filename:
return 'Cannot apply mapping from %s to %s (it conflicts with mapping: %s to %s)' % (
absolute_filename, map_entry.runtime_source,
existing_source_filename, map_entry.runtime_source)
try:
absolute_normalized_filename = pydevd_file_utils.normcase(
absolute_filename)
current_mapping = self._mappings_to_server.get(
absolute_normalized_filename, [])
for map_entry in current_mapping:
del self._mappings_to_client[map_entry.runtime_source]
self._mappings_to_server[absolute_normalized_filename] = sorted(
mapping, key=lambda entry: entry.line)
for map_entry in mapping:
self._mappings_to_client[
map_entry.runtime_source] = absolute_filename
finally:
self._cache.clear()
self._on_source_mapping_changed()
return ''
def map_to_server(self, absolute_filename, lineno):
'''
Convert something as 'file1.py' at line 10 to '<ipython-cell-xxx>' at line 2.
Note that the name should be already normalized at this point.
'''
absolute_normalized_filename = pydevd_file_utils.normcase(
absolute_filename)
changed = False
mappings = self._mappings_to_server.get(absolute_normalized_filename)
if mappings:
i = bisect.bisect(KeyifyList(mappings, lambda entry: entry.line),
lineno)
if i >= len(mappings):
i -= 1
if i == 0:
entry = mappings[i]
else:
entry = mappings[i - 1]
if not entry.contains_line(lineno):
entry = mappings[i]
if not entry.contains_line(lineno):
entry = None
if entry is not None:
lineno = entry.runtime_line + (lineno - entry.line)
absolute_filename = entry.runtime_source
changed = True
return absolute_filename, lineno, changed
def _absolute_normalized_path(self, filename):
'''
Provides a version of the filename that's absolute and normalized.
'''
return normcase(pydevd_file_utils.absolute_path(filename))
def custom_dont_trace_external_files(abs_path):
normalized_abs_path = pydevd_file_utils.normcase(abs_path)
return normalized_abs_path.startswith(
start_patterns) or normalized_abs_path.endswith(end_patterns)
def add_breakpoint(self,
py_db,
original_filename,
breakpoint_type,
breakpoint_id,
line,
condition,
func_name,
expression,
suspend_policy,
hit_condition,
is_logpoint,
adjust_line=False):
'''
:param str original_filename:
Note: must be sent as it was received in the protocol. It may be translated in this
function and its final value will be available in the returned _AddBreakpointResult.
:param str breakpoint_type:
One of: 'python-line', 'django-line', 'jinja2-line'.
:param int breakpoint_id:
:param int line:
Note: it's possible that a new line was actually used. If that's the case its
final value will be available in the returned _AddBreakpointResult.
:param condition:
Either None or the condition to activate the breakpoint.
:param str func_name:
If "None" (str), may hit in any context.
Empty string will hit only top level.
Any other value must match the scope of the method to be matched.
:param str expression:
None or the expression to be evaluated.
:param suspend_policy:
Either "NONE" (to suspend only the current thread when the breakpoint is hit) or
"ALL" (to suspend all threads when a breakpoint is hit).
:param str hit_condition:
An expression where `@HIT@` will be replaced by the number of hits.
i.e.: `@HIT@ == x` or `@HIT@ >= x`
:param bool is_logpoint:
If True and an expression is passed, pydevd will create an io message command with the
result of the evaluation.
:return _AddBreakpointResult:
'''
assert original_filename.__class__ == str, 'Expected str, found: %s' % (
original_filename.__class__, ) # i.e.: bytes on py2 and str on py3
pydev_log.debug('Request for breakpoint in: %s line: %s',
original_filename, line)
# Parameters to reapply breakpoint.
api_add_breakpoint_params = (original_filename, breakpoint_type,
breakpoint_id, line, condition, func_name,
expression, suspend_policy, hit_condition,
is_logpoint)
translated_filename = self.filename_to_server(
original_filename) # Apply user path mapping.
pydev_log.debug('Breakpoint (after path translation) in: %s line: %s',
translated_filename, line)
func_name = self.to_str(func_name)
assert translated_filename.__class__ == str # i.e.: bytes on py2 and str on py3
assert func_name.__class__ == str # i.e.: bytes on py2 and str on py3
# Apply source mapping (i.e.: ipython).
source_mapped_filename, new_line, multi_mapping_applied = py_db.source_mapping.map_to_server(
translated_filename, line)
if multi_mapping_applied:
pydev_log.debug(
'Breakpoint (after source mapping) in: %s line: %s',
source_mapped_filename, new_line)
# Note that source mapping is internal and does not change the resulting filename nor line
# (we want the outside world to see the line in the original file and not in the ipython
# cell, otherwise the editor wouldn't be correct as the returned line is the line to
# which the breakpoint will be moved in the editor).
result = self._AddBreakpointResult(original_filename, line)
# If a multi-mapping was applied, consider it the canonical / source mapped version (translated to ipython cell).
translated_absolute_filename = source_mapped_filename
canonical_normalized_filename = pydevd_file_utils.normcase(
source_mapped_filename)
line = new_line
else:
translated_absolute_filename = pydevd_file_utils.absolute_path(
translated_filename)
canonical_normalized_filename = pydevd_file_utils.canonical_normalized_path(
translated_filename)
if adjust_line and not translated_absolute_filename.startswith(
'<'):
# Validate breakpoints and adjust their positions.
try:
lines = sorted(
_get_code_lines(translated_absolute_filename))
except Exception:
pass
else:
if line not in lines:
# Adjust to the first preceding valid line.
idx = bisect.bisect_left(lines, line)
if idx > 0:
line = lines[idx - 1]
result = self._AddBreakpointResult(original_filename, line)
py_db.api_received_breakpoints[(original_filename, breakpoint_id)] = (
canonical_normalized_filename, api_add_breakpoint_params)
if not translated_absolute_filename.startswith('<'):
# Note: if a mapping pointed to a file starting with '<', don't validate.
if not pydevd_file_utils.exists(translated_absolute_filename):
result.error_code = self.ADD_BREAKPOINT_FILE_NOT_FOUND
return result
if (py_db.is_files_filter_enabled
and not py_db.get_require_module_for_filters()
and py_db.apply_files_filter(
self._DummyFrame(translated_absolute_filename),
translated_absolute_filename, False)):
# Note that if `get_require_module_for_filters()` returns False, we don't do this check.
# This is because we don't have the module name given a file at this point (in
# runtime it's gotten from the frame.f_globals).
# An option could be calculate it based on the filename and current sys.path,
# but on some occasions that may be wrong (for instance with `__main__` or if
# the user dynamically changes the PYTHONPATH).
# Note: depending on the use-case, filters may be changed, so, keep on going and add the
# breakpoint even with the error code.
result.error_code = self.ADD_BREAKPOINT_FILE_EXCLUDED_BY_FILTERS
if breakpoint_type == 'python-line':
added_breakpoint = LineBreakpoint(line,
condition,
func_name,
expression,
suspend_policy,
hit_condition=hit_condition,
is_logpoint=is_logpoint)
breakpoints = py_db.breakpoints
file_to_id_to_breakpoint = py_db.file_to_id_to_line_breakpoint
supported_type = True
else:
add_plugin_breakpoint_result = None
plugin = py_db.get_plugin_lazy_init()
if plugin is not None:
add_plugin_breakpoint_result = plugin.add_breakpoint(
'add_line_breakpoint',
py_db,
breakpoint_type,
canonical_normalized_filename,
line,
condition,
expression,
func_name,
hit_condition=hit_condition,
is_logpoint=is_logpoint)
if add_plugin_breakpoint_result is not None:
supported_type = True
added_breakpoint, breakpoints = add_plugin_breakpoint_result
file_to_id_to_breakpoint = py_db.file_to_id_to_plugin_breakpoint
else:
supported_type = False
if not supported_type:
raise NameError(breakpoint_type)
if DebugInfoHolder.DEBUG_TRACE_BREAKPOINTS > 0:
pydev_log.debug('Added breakpoint:%s - line:%s - func_name:%s\n',
canonical_normalized_filename, line, func_name)
if canonical_normalized_filename in file_to_id_to_breakpoint:
id_to_pybreakpoint = file_to_id_to_breakpoint[
canonical_normalized_filename]
else:
id_to_pybreakpoint = file_to_id_to_breakpoint[
canonical_normalized_filename] = {}
id_to_pybreakpoint[breakpoint_id] = added_breakpoint
py_db.consolidate_breakpoints(canonical_normalized_filename,
id_to_pybreakpoint, breakpoints)
if py_db.plugin is not None:
py_db.has_plugin_line_breaks = py_db.plugin.has_line_breaks()
py_db.on_breakpoints_changed()
return result
