def test_convert_utilities(tmpdir): import pydevd_file_utils import sys test_dir = str(tmpdir.mkdir("Test_Convert_Utilities")) if sys.platform == 'win32': normalized = pydevd_file_utils.normcase(test_dir) assert isinstance(normalized, str) # bytes on py2, unicode on py3 assert normalized.lower() == normalized assert '~' not in normalized assert '~' in pydevd_file_utils.convert_to_short_pathname(normalized) real_case = pydevd_file_utils.get_path_with_real_case(normalized) assert isinstance(real_case, str) # bytes on py2, unicode on py3 # Note test_dir itself cannot be compared with because pytest may # have passed the case normalized. assert real_case.endswith("Test_Convert_Utilities") else: # On other platforms, nothing should change assert pydevd_file_utils.normcase(test_dir) == test_dir assert pydevd_file_utils.convert_to_short_pathname( test_dir) == test_dir assert pydevd_file_utils.get_path_with_real_case(test_dir) == test_dir
def _check_matches(patterns, paths): if not patterns and not paths: # Matched to the end. return True if (not patterns and paths) or (patterns and not paths): return False pattern = normcase(patterns[0]) path = normcase(paths[0]) if not glob.has_magic(pattern): if pattern != path: return False elif pattern == '**': if len(patterns) == 1: return True # if ** is the last one it matches anything to the right. for i in xrange(len(paths)): # Recursively check the remaining patterns as the # current pattern could match any number of paths. if _check_matches(patterns[1:], paths[i:]): return True elif not fnmatch.fnmatch(path, pattern): # Current part doesn't match. return False return _check_matches(patterns[1:], paths[1:])
def set_dont_trace_start_end_patterns(self, py_db, start_patterns, end_patterns): # Note: start/end patterns normalized internally. start_patterns = tuple(pydevd_file_utils.normcase(x) for x in start_patterns) end_patterns = tuple(pydevd_file_utils.normcase(x) for x in end_patterns) # After it's set the first time, we can still change it, but we need to reset the # related caches. reset_caches = False dont_trace_start_end_patterns_previously_set = \ py_db.dont_trace_external_files.__name__ == 'custom_dont_trace_external_files' if not dont_trace_start_end_patterns_previously_set and not start_patterns and not end_patterns: # If it wasn't set previously and start and end patterns are empty we don't need to do anything. return if not py_db.is_cache_file_type_empty(): # i.e.: custom function set in set_dont_trace_start_end_patterns. if dont_trace_start_end_patterns_previously_set: reset_caches = py_db.dont_trace_external_files.start_patterns != start_patterns or \ py_db.dont_trace_external_files.end_patterns != end_patterns else: reset_caches = True def custom_dont_trace_external_files(abs_path): normalized_abs_path = pydevd_file_utils.normcase(abs_path) return normalized_abs_path.startswith(start_patterns) or normalized_abs_path.endswith(end_patterns) custom_dont_trace_external_files.start_patterns = start_patterns custom_dont_trace_external_files.end_patterns = end_patterns py_db.dont_trace_external_files = custom_dont_trace_external_files if reset_caches: py_db.clear_dont_trace_start_end_patterns_caches()
def test_convert_utilities(tmpdir): import pydevd_file_utils test_dir = str(tmpdir.mkdir("Test_Convert_Utilities")) if IS_WINDOWS: normalized = pydevd_file_utils.normcase(test_dir) assert isinstance(normalized, str) # bytes on py2, unicode on py3 assert normalized.lower() == normalized upper_version = os.path.join(test_dir, 'ÁÉÍÓÚ') with open(upper_version, 'w') as stream: stream.write('test') with open(upper_version, 'r') as stream: assert stream.read() == 'test' with open(pydevd_file_utils.normcase(upper_version), 'r') as stream: assert stream.read() == 'test' assert '~' not in normalized for i in range(3): # Check if cache is ok. if i == 2: pydevd_file_utils._listdir_cache.clear() assert pydevd_file_utils.get_path_with_real_case('<does not EXIST>') == '<does not EXIST>' real_case = pydevd_file_utils.get_path_with_real_case(normalized) assert isinstance(real_case, str) # bytes on py2, unicode on py3 # Note test_dir itself cannot be compared with because pytest may # have passed the case normalized. assert real_case.endswith("Test_Convert_Utilities") if i == 2: # Check that we have the expected paths in the cache. assert pydevd_file_utils._listdir_cache[os.path.dirname(normalized).lower()] == ['Test_Convert_Utilities'] assert pydevd_file_utils._listdir_cache[(os.path.dirname(normalized).lower(), 'Test_Convert_Utilities'.lower())] == real_case if IS_PY2: # Test with unicode in python 2 too. real_case = pydevd_file_utils.get_path_with_real_case(normalized.decode( getfilesystemencoding())) assert isinstance(real_case, str) # bytes on py2, unicode on py3 # Note test_dir itself cannot be compared with because pytest may # have passed the case normalized. assert real_case.endswith("Test_Convert_Utilities") # Check that it works with a shortened path. shortened = pydevd_file_utils.convert_to_short_pathname(normalized) assert '~' in shortened with_real_case = pydevd_file_utils.get_path_with_real_case(shortened) assert with_real_case.endswith('Test_Convert_Utilities') assert '~' not in with_real_case else: # On other platforms, nothing should change assert pydevd_file_utils.normcase(test_dir) == test_dir assert pydevd_file_utils.get_path_with_real_case(test_dir) == test_dir
def _get_template_file_name(frame): try: if IS_DJANGO19: # The Node source was removed since Django 1.9 if 'context' in frame.f_locals: context = frame.f_locals['context'] if hasattr(context, '_has_included_template'): # if there was included template we need to inspect the previous frames and find its name back = frame.f_back while back is not None and frame.f_code.co_name in ( 'render', '_render'): locals = back.f_locals if 'self' in locals: self = locals['self'] if self.__class__.__name__ == 'Template' and hasattr(self, 'origin') and \ hasattr(self.origin, 'name'): return normcase( _convert_to_str(self.origin.name)) back = back.f_back else: if hasattr(context, 'template') and hasattr(context.template, 'origin') and \ hasattr(context.template.origin, 'name'): return normcase( _convert_to_str(context.template.origin.name)) return None elif IS_DJANGO19_OR_HIGHER: # For Django 1.10 and later there is much simpler way to get template name if 'self' in frame.f_locals: self = frame.f_locals['self'] if hasattr(self, 'origin') and hasattr(self.origin, 'name'): return normcase(_convert_to_str(self.origin.name)) return None source = _get_source_django_18_or_lower(frame) if source is None: pydev_log.debug("Source is None\n") return None fname = _convert_to_str(source[0].name) if fname == '<unknown source>': pydev_log.debug("Source name is %s\n" % fname) return None else: abs_path_real_path_and_base = get_abs_path_real_path_and_base_from_file( fname) return abs_path_real_path_and_base[1] except: pydev_log.debug(traceback.format_exc()) return None
def _get_template_file_name(frame): try: if IS_DJANGO19_OR_HIGHER: # The Node source was removed since Django 1.9 if dict_contains(frame.f_locals, 'context'): context = frame.f_locals['context'] if hasattr(context, 'template') and hasattr(context.template, 'origin') and \ hasattr(context.template.origin, 'name'): return normcase(context.template.origin.name) return None source = _get_source_django_18_or_lower(frame) if source is None: pydev_log.debug("Source is None\n") return None fname = source[0].name if fname == '<unknown source>': pydev_log.debug("Source name is %s\n" % fname) return None else: abs_path_real_path_and_base = get_abs_path_real_path_and_base_from_file(fname) return abs_path_real_path_and_base[1] except: pydev_log.debug(traceback.format_exc()) return None
def _get_template_file_name(frame): try: if IS_DJANGO19: # The Node source was removed since Django 1.9 if 'context' in frame.f_locals: context = frame.f_locals['context'] if hasattr(context, '_has_included_template'): # if there was included template we need to inspect the previous frames and find its name back = frame.f_back while back is not None and frame.f_code.co_name in ('render', '_render'): locals = back.f_locals if 'self' in locals: self = locals['self'] if self.__class__.__name__ == 'Template' and hasattr(self, 'origin') and \ hasattr(self.origin, 'name'): return normcase(self.origin.name) back = back.f_back else: if hasattr(context, 'template') and hasattr(context.template, 'origin') and \ hasattr(context.template.origin, 'name'): return normcase(context.template.origin.name) return None elif IS_DJANGO19_OR_HIGHER: # For Django 1.10 and later there is much simpler way to get template name if 'self' in frame.f_locals: self = frame.f_locals['self'] if hasattr(self, 'origin') and hasattr(self.origin, 'name'): return normcase(self.origin.name) return None source = _get_source_django_18_or_lower(frame) if source is None: pydev_log.debug("Source is None\n") return None fname = source[0].name if fname == '<unknown source>': pydev_log.debug("Source name is %s\n" % fname) return None else: abs_path_real_path_and_base = get_abs_path_real_path_and_base_from_file(fname) return abs_path_real_path_and_base[1] except: pydev_log.debug(traceback.format_exc()) return None
def exception_break(plugin, main_debugger, pydb_frame, frame, args, arg): main_debugger = args[0] thread = args[3] exception, value, trace = arg if main_debugger.django_exception_break and exception is not None: if exception.__name__ in ['VariableDoesNotExist', 'TemplateDoesNotExist', 'TemplateSyntaxError'] and \ just_raised(trace) and not ignore_exception_trace(trace): if exception.__name__ == 'TemplateSyntaxError': # In this case we don't actually have a regular render frame with the context # (we didn't really get to that point). token = getattr(value, 'token', None) lineno = getattr(token, 'lineno', None) filename = None if lineno is not None: get_template_frame = frame while get_template_frame.f_code.co_name != 'get_template': get_template_frame = get_template_frame.f_back origin = None if get_template_frame is not None: origin = get_template_frame.f_locals.get('origin') if hasattr(origin, 'name') and origin.name is not None: filename = normcase(_convert_to_str(origin.name)) if filename is not None and lineno is not None: syntax_error_frame = DjangoTemplateSyntaxErrorFrame( frame, filename, lineno, { 'token': token, 'exception': exception }) suspend_frame = suspend_django(main_debugger, thread, syntax_error_frame, CMD_ADD_EXCEPTION_BREAK) return True, suspend_frame elif exception.__name__ == 'VariableDoesNotExist': if _is_django_variable_does_not_exist_exception_break_context( frame): render_frame = _find_django_render_frame(frame) if render_frame: suspend_frame = suspend_django( main_debugger, thread, DjangoTemplateFrame(render_frame), CMD_ADD_EXCEPTION_BREAK) if suspend_frame: add_exception_to_frame(suspend_frame, (exception, value, trace)) thread.additional_info.pydev_message = 'VariableDoesNotExist' suspend_frame.f_back = frame frame = suspend_frame return True, frame return None
def _get_filename_from_origin_in_parent_frame_locals(frame, parent_frame_name): filename = None parent_frame = frame while parent_frame.f_code.co_name != parent_frame_name: parent_frame = parent_frame.f_back origin = None if parent_frame is not None: origin = parent_frame.f_locals.get('origin') if hasattr(origin, 'name') and origin.name is not None: filename = normcase(_convert_to_str(origin.name)) return filename
def test_convert_utilities(tmpdir): import pydevd_file_utils test_dir = str(tmpdir.mkdir("Test_Convert_Utilities")) if IS_WINDOWS: normalized = pydevd_file_utils.normcase(test_dir) assert isinstance(normalized, str) # bytes on py2, unicode on py3 assert normalized.lower() == normalized upper_version = os.path.join(test_dir, 'ÁÉÍÓÚ') with open(upper_version, 'w') as stream: stream.write('test') with open(upper_version, 'r') as stream: assert stream.read() == 'test' with open(pydevd_file_utils.normcase(upper_version), 'r') as stream: assert stream.read() == 'test' assert '~' not in normalized if not IS_JYTHON: assert '~' in pydevd_file_utils.convert_to_short_pathname( normalized) real_case = pydevd_file_utils.get_path_with_real_case(normalized) assert isinstance(real_case, str) # bytes on py2, unicode on py3 # Note test_dir itself cannot be compared with because pytest may # have passed the case normalized. assert real_case.endswith("Test_Convert_Utilities") else: # On other platforms, nothing should change assert pydevd_file_utils.normcase(test_dir) == test_dir assert pydevd_file_utils.convert_to_short_pathname( test_dir) == test_dir assert pydevd_file_utils.get_path_with_real_case(test_dir) == test_dir
def set_source_mapping(self, absolute_filename, mapping): ''' :param str absolute_filename: The filename for the source mapping (bytes on py2 and str on py3). :param list(SourceMappingEntry) mapping: A list with the source mapping entries to be applied to the given filename. :return str: An error message if it was not possible to set the mapping or an empty string if everything is ok. ''' # Let's first validate if it's ok to apply that mapping. # File mappings must be 1:N, not M:N (i.e.: if there's a mapping from file1.py to <cell1>, # there can be no other mapping from any other file to <cell1>). # This is a limitation to make it easier to remove existing breakpoints when new breakpoints are # set to a file (so, any file matching that breakpoint can be removed instead of needing to check # which lines are corresponding to that file). for map_entry in mapping: existing_source_filename = self._mappings_to_client.get( map_entry.runtime_source) if existing_source_filename and existing_source_filename != absolute_filename: return 'Cannot apply mapping from %s to %s (it conflicts with mapping: %s to %s)' % ( absolute_filename, map_entry.runtime_source, existing_source_filename, map_entry.runtime_source) try: absolute_normalized_filename = pydevd_file_utils.normcase( absolute_filename) current_mapping = self._mappings_to_server.get( absolute_normalized_filename, []) for map_entry in current_mapping: del self._mappings_to_client[map_entry.runtime_source] self._mappings_to_server[absolute_normalized_filename] = sorted( mapping, key=lambda entry: entry.line) for map_entry in mapping: self._mappings_to_client[ map_entry.runtime_source] = absolute_filename finally: self._cache.clear() self._on_source_mapping_changed() return ''
def map_to_server(self, absolute_filename, lineno): ''' Convert something as 'file1.py' at line 10 to '<ipython-cell-xxx>' at line 2. Note that the name should be already normalized at this point. ''' absolute_normalized_filename = pydevd_file_utils.normcase( absolute_filename) changed = False mappings = self._mappings_to_server.get(absolute_normalized_filename) if mappings: i = bisect.bisect(KeyifyList(mappings, lambda entry: entry.line), lineno) if i >= len(mappings): i -= 1 if i == 0: entry = mappings[i] else: entry = mappings[i - 1] if not entry.contains_line(lineno): entry = mappings[i] if not entry.contains_line(lineno): entry = None if entry is not None: lineno = entry.runtime_line + (lineno - entry.line) absolute_filename = entry.runtime_source changed = True return absolute_filename, lineno, changed
def _absolute_normalized_path(self, filename): ''' Provides a version of the filename that's absolute and normalized. ''' return normcase(pydevd_file_utils.absolute_path(filename))
def custom_dont_trace_external_files(abs_path): normalized_abs_path = pydevd_file_utils.normcase(abs_path) return normalized_abs_path.startswith( start_patterns) or normalized_abs_path.endswith(end_patterns)
def add_breakpoint(self, py_db, original_filename, breakpoint_type, breakpoint_id, line, condition, func_name, expression, suspend_policy, hit_condition, is_logpoint, adjust_line=False): ''' :param str original_filename: Note: must be sent as it was received in the protocol. It may be translated in this function and its final value will be available in the returned _AddBreakpointResult. :param str breakpoint_type: One of: 'python-line', 'django-line', 'jinja2-line'. :param int breakpoint_id: :param int line: Note: it's possible that a new line was actually used. If that's the case its final value will be available in the returned _AddBreakpointResult. :param condition: Either None or the condition to activate the breakpoint. :param str func_name: If "None" (str), may hit in any context. Empty string will hit only top level. Any other value must match the scope of the method to be matched. :param str expression: None or the expression to be evaluated. :param suspend_policy: Either "NONE" (to suspend only the current thread when the breakpoint is hit) or "ALL" (to suspend all threads when a breakpoint is hit). :param str hit_condition: An expression where `@HIT@` will be replaced by the number of hits. i.e.: `@HIT@ == x` or `@HIT@ >= x` :param bool is_logpoint: If True and an expression is passed, pydevd will create an io message command with the result of the evaluation. :return _AddBreakpointResult: ''' assert original_filename.__class__ == str, 'Expected str, found: %s' % ( original_filename.__class__, ) # i.e.: bytes on py2 and str on py3 pydev_log.debug('Request for breakpoint in: %s line: %s', original_filename, line) # Parameters to reapply breakpoint. api_add_breakpoint_params = (original_filename, breakpoint_type, breakpoint_id, line, condition, func_name, expression, suspend_policy, hit_condition, is_logpoint) translated_filename = self.filename_to_server( original_filename) # Apply user path mapping. pydev_log.debug('Breakpoint (after path translation) in: %s line: %s', translated_filename, line) func_name = self.to_str(func_name) assert translated_filename.__class__ == str # i.e.: bytes on py2 and str on py3 assert func_name.__class__ == str # i.e.: bytes on py2 and str on py3 # Apply source mapping (i.e.: ipython). source_mapped_filename, new_line, multi_mapping_applied = py_db.source_mapping.map_to_server( translated_filename, line) if multi_mapping_applied: pydev_log.debug( 'Breakpoint (after source mapping) in: %s line: %s', source_mapped_filename, new_line) # Note that source mapping is internal and does not change the resulting filename nor line # (we want the outside world to see the line in the original file and not in the ipython # cell, otherwise the editor wouldn't be correct as the returned line is the line to # which the breakpoint will be moved in the editor). result = self._AddBreakpointResult(original_filename, line) # If a multi-mapping was applied, consider it the canonical / source mapped version (translated to ipython cell). translated_absolute_filename = source_mapped_filename canonical_normalized_filename = pydevd_file_utils.normcase( source_mapped_filename) line = new_line else: translated_absolute_filename = pydevd_file_utils.absolute_path( translated_filename) canonical_normalized_filename = pydevd_file_utils.canonical_normalized_path( translated_filename) if adjust_line and not translated_absolute_filename.startswith( '<'): # Validate breakpoints and adjust their positions. try: lines = sorted( _get_code_lines(translated_absolute_filename)) except Exception: pass else: if line not in lines: # Adjust to the first preceding valid line. idx = bisect.bisect_left(lines, line) if idx > 0: line = lines[idx - 1] result = self._AddBreakpointResult(original_filename, line) py_db.api_received_breakpoints[(original_filename, breakpoint_id)] = ( canonical_normalized_filename, api_add_breakpoint_params) if not translated_absolute_filename.startswith('<'): # Note: if a mapping pointed to a file starting with '<', don't validate. if not pydevd_file_utils.exists(translated_absolute_filename): result.error_code = self.ADD_BREAKPOINT_FILE_NOT_FOUND return result if (py_db.is_files_filter_enabled and not py_db.get_require_module_for_filters() and py_db.apply_files_filter( self._DummyFrame(translated_absolute_filename), translated_absolute_filename, False)): # Note that if `get_require_module_for_filters()` returns False, we don't do this check. # This is because we don't have the module name given a file at this point (in # runtime it's gotten from the frame.f_globals). # An option could be calculate it based on the filename and current sys.path, # but on some occasions that may be wrong (for instance with `__main__` or if # the user dynamically changes the PYTHONPATH). # Note: depending on the use-case, filters may be changed, so, keep on going and add the # breakpoint even with the error code. result.error_code = self.ADD_BREAKPOINT_FILE_EXCLUDED_BY_FILTERS if breakpoint_type == 'python-line': added_breakpoint = LineBreakpoint(line, condition, func_name, expression, suspend_policy, hit_condition=hit_condition, is_logpoint=is_logpoint) breakpoints = py_db.breakpoints file_to_id_to_breakpoint = py_db.file_to_id_to_line_breakpoint supported_type = True else: add_plugin_breakpoint_result = None plugin = py_db.get_plugin_lazy_init() if plugin is not None: add_plugin_breakpoint_result = plugin.add_breakpoint( 'add_line_breakpoint', py_db, breakpoint_type, canonical_normalized_filename, line, condition, expression, func_name, hit_condition=hit_condition, is_logpoint=is_logpoint) if add_plugin_breakpoint_result is not None: supported_type = True added_breakpoint, breakpoints = add_plugin_breakpoint_result file_to_id_to_breakpoint = py_db.file_to_id_to_plugin_breakpoint else: supported_type = False if not supported_type: raise NameError(breakpoint_type) if DebugInfoHolder.DEBUG_TRACE_BREAKPOINTS > 0: pydev_log.debug('Added breakpoint:%s - line:%s - func_name:%s\n', canonical_normalized_filename, line, func_name) if canonical_normalized_filename in file_to_id_to_breakpoint: id_to_pybreakpoint = file_to_id_to_breakpoint[ canonical_normalized_filename] else: id_to_pybreakpoint = file_to_id_to_breakpoint[ canonical_normalized_filename] = {} id_to_pybreakpoint[breakpoint_id] = added_breakpoint py_db.consolidate_breakpoints(canonical_normalized_filename, id_to_pybreakpoint, breakpoints) if py_db.plugin is not None: py_db.has_plugin_line_breaks = py_db.plugin.has_line_breaks() py_db.on_breakpoints_changed() return result