def test_index_project_files_no_plugins():
remove_db()
time.sleep(1)
with db.DbHelper():
pass
th = SubprocessTaskHandle()
backend.index_project_files(th, [PATH], [], [])
with db.DbHelper() as dbh:
assert len(list(dbh.get_files())) == 3
assert len(list(dbh.get_symbols())) == 0
def test_index_project_files_with_plugins():
remove_db()
time.sleep(1)
with db.DbHelper():
pass
th = SubprocessTaskHandle()
backend.index_project_files(th, [PATH], [], [FakeParserPlugin()])
with db.DbHelper() as dbh:
for row in dbh.get_files():
print(row[db.COL_FILE_PATH])
assert len(list(dbh.get_files())) == 3
assert len(list(dbh.get_symbols())) == 3
def test_delete_project():
remove_db()
with db.DbHelper() as dbh:
# project 1: 2 files, 2 symbols
pid1 = dbh.create_project('/home/colin')
fid1 = dbh.create_file('/path1', pid1)
dbh.create_symbol('setToolTip', 10, 45, 'code-variable',
'/path/to/icon.png', fid1, pid1)
fid2 = dbh.create_file('/path2', pid1)
dbh.create_symbol('setToolTip', 10, 45, 'code-variable',
'/path/to/icon.png', fid2, pid1)
# project 2: 2 files, 2 symbols
pid2 = dbh.create_project('/home/colin2')
fid3 = dbh.create_file('/path3', pid2)
dbh.create_symbol('setToolTip', 10, 45, 'code-variable',
'/path/to/icon.png', fid3, pid2)
fid4 = dbh.create_file('/path4', pid2)
dbh.create_symbol('setToolTip', 10, 45, 'code-variable',
'/path/to/icon.png', fid4, pid2)
assert len(list(dbh.get_files())) == 4
assert len(list(dbh.get_symbols())) == 4
assert dbh.delete_project('/home/colin') is True
# second attempt to delete project should fail
assert dbh.delete_project('/home/colin') is False
assert len(list(dbh.get_files())) == 2
assert len(list(dbh.get_symbols())) == 2
def _parse_symbols(task_handle, file_id, project_id, path, plugin, root_directory):
"""
Parses the symbols of the specified file using the parser plugin.
The symbols are written to the index database.
:param task_handle: task handle, to report progress to the frontend.
:param file_id: id of the file to parse.
:param project_id: parent project id.
:param path: path of the file to parse.
:param plugin: associated parser plugin.
:param root_directory: project directory.
"""
rel_path = os.path.relpath(path, root_directory)
task_handle.report_progress('Parsing %r' % rel_path, -1)
try:
symbols = plugin.parse(path)
except Exception as e:
print(path, e)
else:
# write results to the database
with db.DbHelper() as dbh:
# delete all associated symbols
dbh.delete_file_symbols(file_id)
# write them all and commit once all inserts have been done to
# improve performances
_write_symbols_to_db(
dbh, symbols, file_id, project_id, parent_id=None)
dbh.conn.commit()
def index_project_files(task_handle, project_directories, ignore_patterns,
parser_plugins):
"""
Perform a full indexation of the project files.
:param project_directories: the directories to scan.
:param task_handle: task handle, to report task progress.
:param ignore_patterns: the ignore patterns to respect.
:param parser_plugins: the list of parser plugins.
"""
mime_types.load()
# adjust ignore patterns to always exclude binary files from indexation
ignore_patterns += ['*.exe', '*.dll', '*.usr', '*.so', '*.dylib', '*.psd',
'*.db', '.hackedit', '.eggs', '.cache', '.git', '.svn',
'.hackedit', 'build', 'dist', '_build']
if os.environ.get('TRAVIS', default=None) is not None:
ignore_patterns.remove('build')
parser_plugins = tuple(parser_plugins)
# create projects
proj_ids = []
task_handle.report_progress(_('Creating projects'), -1)
with db.DbHelper() as dbh:
for project_directory in project_directories:
proj_ids.append((dbh.create_project(project_directory),
project_directory))
# create file index
for project_id, project_directory in proj_ids:
_recursive_index_dirs(task_handle, project_directory, ignore_patterns, project_directory,
project_id, parser_plugins)
_clean_project_files(task_handle, project_id, ignore_patterns)
task_handle.report_progress('Finished', 100)
def test_get_project_symbols():
remove_db()
p1 = '/home/colin'
p2 = '/home/colin2'
path1 = '/home/file.py'
path2 = '/home/file2.py'
with db.DbHelper() as dbh:
pid1 = dbh.create_project(p1)
pid2 = dbh.create_project(p2)
fid1 = dbh.create_file(path1, pid1)
fid2 = dbh.create_file(path2, pid2)
dbh.create_symbol('setToolTip', 10, 45, 'code-variable',
'/path/to/icon.png', fid1, pid1)
dbh.create_symbol('setCallTip', 10, 45, 'code-variable',
'/path/to/icon.png', fid1, pid1)
dbh.create_symbol('setTip', 10, 45, 'code-variable',
'/path/to/icon.png', fid2, pid2)
dbh.create_symbol('word', 10, 45, 'code-variable', '/path/to/icon.png',
fid2, pid2)
dbh.create_symbol('word', 10, 45, 'code-variable', '/path/to/icon.png',
fid2, pid2)
assert len(list(dbh.get_symbols())) == 5
assert len(list(dbh.get_symbols(project_ids=[pid1]))) == 2
assert len(list(dbh.get_symbols(project_ids=[pid2]))) == 3
assert len(list(dbh.get_symbols(project_ids=[pid2],
name_filter='wo'))) == 2
def test_has_file():
remove_db()
with db.DbHelper() as dbh:
assert not dbh.has_file('/home/file.py')
pid = dbh.create_project('/home/colin')
dbh.create_file('/home/file.py', pid)
assert dbh.has_file('/home/file.py')
def test_create_project():
remove_db()
with db.DbHelper() as dbh:
assert dbh.get_project('/home/colin') is None
assert dbh.create_project('/home/colin') == 1
assert dbh.create_project('/home/colin') == 1
assert dbh.create_project('/home/colin2') == 2
assert len(list(dbh.get_projects())) == 2
def remove_project(path):
"""
Removes project from the index database
:param path: path of the project to remove
"""
with db.DbHelper() as dbh:
dbh.delete_project(path)
def test_rename_files():
remove_db()
time.sleep(1)
with db.DbHelper():
pass
# create a new temporary file that will get renamed
with open(NEW_FILE, 'w'):
pass
th = SubprocessTaskHandle()
backend.index_project_files(th, [PATH], [], [FakeParserPlugin()])
with db.DbHelper() as dbh:
assert len(list(dbh.get_files())) == 4
os.rename(NEW_FILE, RENAMED_FILE)
backend.rename_files(th, [(NEW_FILE, RENAMED_FILE)])
with db.DbHelper() as dbh:
assert len(list(dbh.get_files())) == 4
def test_update_file():
remove_db()
time.sleep(1)
with db.DbHelper():
pass
th = SubprocessTaskHandle()
backend.index_project_files(th, [PATH], [], [FakeParserPlugin()])
with db.DbHelper() as dbh:
fid = dbh.get_file_by_path(SETUP_PY)
backend.update_file(th, SETUP_PY, fid, PATH, 1,
[FakeParserPlugin()])
# update file time stamp
with open(SETUP_PY, 'r') as fin:
with open(SETUP_PY, 'w') as fout:
fout.write(fin.read())
backend.update_file(th, SETUP_PY, fid, PATH, 1,
[FakeParserPlugin()])
def _update_mtime(file_path):
"""
Update the modification time (mtime) of the specified file.
:param file_path: Path of the file to update.
:return: the new and the old file modification time.
"""
try:
new_mtime = os.path.getmtime(file_path)
except FileNotFoundError:
with db.DbHelper() as dbh:
dbh.delete_file(file_path)
return None, None
else:
with db.DbHelper() as dbh:
old_mtime = dbh.get_file_mtime(file_path)
dbh.update_file(file_path, new_mtime)
return new_mtime, old_mtime
def test_create_file():
remove_db()
with db.DbHelper() as dbh:
pid = dbh.create_project('/home/colin')
assert dbh.create_file('/home/file.py', pid) == 1
assert dbh.create_file('/home/file.py', pid) == 1 # already added
assert dbh.create_file('/home/file2.py', pid) == 2
assert dbh.create_file('/home/file3.py', pid) == 3
assert dbh.get_file_by_id(3)[db.COL_FILE_PATH] == '/home/file3.py'
def test_cleanup():
remove_db()
time.sleep(1)
with db.DbHelper():
pass
# create a new temporary file that will get renamed
with open(NEW_FILE, 'w'):
pass
th = SubprocessTaskHandle()
backend.index_project_files(th, [PATH], [], [FakeParserPlugin()])
with db.DbHelper() as dbh:
nb_files = len(list(dbh.get_files()))
os.remove(NEW_FILE)
# reindexing should remove the deleted file
backend.index_project_files(th, [PATH], [], [FakeParserPlugin()])
with db.DbHelper() as dbh:
assert len(list(dbh.get_files())) == nb_files - 1
def get_file(file_path):
"""
Gets a file from the database
:param file_path: path of the File entry to retrieve.
:returns: File
"""
with db.DbHelper() as dbh:
row = dbh.get_file_by_path(file_path)
if row:
return File(row)
return None
def get_all_projects():
"""
Gets the list of indexed project paths.
:return: list of paths
"""
projects = []
with db.DbHelper() as dbh:
for item in dbh.get_projects():
p = Project(item)
projects.append(p)
return sorted(projects, key=lambda x: x.name)
def _clean_project_files(task_handle, project_id, ignore_patterns):
"""
Removes project files that do not exist or that have been ignored.
:param task_handle: Task handle, to report progress to the frontend.
:param project_id: Id of the project to clean.
:param ignore_patterns: The list of ignore patterns.
"""
task_handle.report_progress('Cleaning project index', -1)
to_delete = []
with db.DbHelper() as db_helper:
for file_item in db_helper.get_files(project_ids=[project_id]):
path = file_item[db.COL_FILE_PATH]
if not os.path.exists(path) or \
is_ignored_path(path, ignore_patterns):
to_delete.append(path)
task_handle.report_progress('Cleaning project index', -1)
with db.DbHelper() as db_helper:
for path in to_delete:
db_helper.delete_file(path, commit=False)
db_helper.conn.commit()
time.sleep(0.001) # allow other process to perform a query
def test_delete_file_symbols():
remove_db()
path = '/home/file.py'
with db.DbHelper() as dbh:
pid = dbh.create_project('/home/colin')
file_id = dbh.create_file(path, pid)
dbh.create_symbol('spam', 10, 45, 'code-variable', '/path/to/icon.png',
file_id, pid)
dbh.create_symbol('eggs', 22, 45, 'code-variable', '/path/to/icon.png',
file_id, pid)
assert len(list(dbh.get_symbols(file_id=file_id))) == 2
dbh.delete_file_symbols(file_id)
assert len(list(dbh.get_symbols(file_id=file_id))) == 0
def delete_files(task_handle, deleted_files):
"""
Removes the specified deleted files from the index database.
:param task_handle: task handle, to report progress update to the frontend
:param deleted_files: the list of deleted files to remove.
:return:
"""
with db.DbHelper() as dbh:
for path in deleted_files:
task_handle.report_progress(_('Delete file %r from index') % path, -1)
dbh.delete_file(path, commit=False)
dbh.conn.commit()
def test_get_project_files():
remove_db()
with db.DbHelper() as dbh:
pid = dbh.create_project('/home/colin')
pid2 = dbh.create_project('/home/colin2')
dbh.create_file('/home/file.py', pid)
dbh.create_file('/home/file.txt', pid)
dbh.create_file('/home/zut.txt', pid)
dbh.create_file('/home/set_tool_tip.txt', pid)
dbh.create_file('/home/set_call_tip.txt', pid)
dbh.create_file('/home/set_tip.txt', pid)
dbh.create_file('/home/set_tip2.txt', pid2)
dbh.create_file('/home/testMyCodeEditor.txt', pid2)
with db.DbHelper() as dbh:
assert len(list(dbh.get_files(project_ids=[pid],
name_filter='file'))) == 2
assert len(list(dbh.get_files(project_ids=[pid],
name_filter='zu'))) == 1
items = list(dbh.get_files(project_ids=[pid], name_filter='set tip'))
assert len(items) == 3
assert len(list(dbh.get_files(project_ids=[pid], name_filter=''))) == 6
# check that using multiple projects works too
assert len(
list(dbh.get_files(project_ids=[pid, pid2],
name_filter='set Tip'))) == 4
assert len(list(dbh.get_files(name_filter='set tip'))) == 4
assert len(list(dbh.get_files(name_filter='g'))) == 0
assert len(list(dbh.get_files())) == 8
assert len(
list(dbh.get_files(project_ids=[pid, pid2],
name_filter='my'))) == 1
assert len(
list(
dbh.get_files(project_ids=[pid, pid2],
name_filter='my editor'))) == 1
assert len(
list(
dbh.get_files(project_ids=[pid, pid2],
name_filter='code editor'))) == 1
def test_unescaped_symbol():
remove_db()
path = '/home/file.py'
with db.DbHelper() as dbh:
pid = dbh.create_project('/home/colin')
file_id = dbh.create_file(path, pid)
assert len(list(dbh.get_symbols(file_id=file_id))) == 0
symbol_id = dbh.create_symbol("78 SOME-VAR VALUE '459'", 10, 45,
'code-variable', '/path/to/icon.png',
file_id, pid)
assert symbol_id == 1
symbols = list(dbh.get_symbols(file_id=file_id))
assert len(symbols) == 1
assert symbols[0][db.COL_SYMBOL_NAME] == "78 SOME-VAR VALUE '459'"
def create_database():
"""
Creates the index database if does not already exists.
:return: Whether the operation succeeded or not.
"""
try:
with db.DbHelper():
pass
except (OSError, sqlite3.OperationalError): # pragma: no cover
_logger().exception("failed to create index database...")
return False
else:
return True
def test_get_files():
remove_db()
with db.DbHelper() as dbh:
pid = dbh.create_project('/home/colin')
assert len(list(dbh.get_files())) == 0
dbh.create_file('/home/file.py', pid)
assert len(list(dbh.get_files())) == 1
dbh.create_file('/home/file.txt', pid)
assert len(list(dbh.get_files())) == 2
pid2 = dbh.create_project('/home/colin2')
dbh.create_file('/home/other.txt', pid2)
assert len(list(dbh.get_files())) == 3
assert len(list(dbh.get_files(project_ids=[pid]))) == 2
assert len(list(dbh.get_files(project_ids=[pid2]))) == 1
def test_file_timestamp():
remove_db()
path = '/home/file.py'
mtime = 1456088731.7597587
with db.DbHelper() as dbh:
pid = dbh.create_project('/home/colin')
with pytest.raises(ValueError):
dbh.get_file_mtime(path)
with pytest.raises(ValueError):
dbh.update_file(path, mtime)
dbh.create_file(path, pid)
# no time stamp
assert dbh.get_file_mtime(path) is None
dbh.update_file(path, mtime)
assert dbh.get_file_mtime(path) == mtime
def get_project_ids(projects):
"""
Gets the id of the specified projects.
:param projects: list of project paths
:return: list of id
"""
project_ids = []
if indexing_enabled():
for proj in projects:
with db.DbHelper() as dbh:
p = dbh.get_project(proj)
if p:
project_ids.append(p[db.COL_PROJECT_ID])
return project_ids
def test_delete_files():
remove_db()
path = '/home/file.py'
with db.DbHelper() as dbh:
assert dbh.delete_file(path) is False
pid = dbh.create_project('/home/colin')
file_id = dbh.create_file(path, pid)
assert dbh.has_file(path)
dbh.create_symbol('spam', 10, 45, 'code-variable', '/path/to/icon.png',
file_id, pid)
dbh.create_symbol('eggs', 22, 45, 'code-variable', '/path/to/icon.png',
file_id, pid)
assert len(list(dbh.get_files(project_ids=[file_id]))) == 1
dbh.delete_file(path)
assert not dbh.has_file(path)
assert len(list(dbh.get_files(project_ids=[file_id]))) == 0
def rename_files(task_handle, renamed_files):
"""
Update renamed files
:param task_handle: task handle, to report progress update to the frontend
:param renamed_files: list of renamed files, each element in the list is a tuple(old_path, new_path).
"""
with db.DbHelper() as dbh:
for old_path, new_path in renamed_files:
task_handle.report_progress(_('Updating renamed file %r -> %r') % (old_path, new_path), -1)
try:
mtime = dbh.get_file_mtime(old_path)
except ValueError:
continue
else:
dbh.update_file(old_path, mtime, new_path=new_path, commit=False)
dbh.conn.commit()
def get_files(name_filter='', projects=None):
"""
Generator that yields all the File entries found in the index database.
Client can restrict the search to the specified project paths.
An optional name filter can be used to filter files by names.
:param name_filter: Optional file name filter.
:param projects: List of projects to search into. If None, all files from
all indexed projects will be used.
:return: A generator that yields class:`File`.
"""
project_ids = None
if projects:
project_ids = get_project_ids(projects)
with db.DbHelper() as dbh:
for itm in dbh.get_files(project_ids=project_ids,
name_filter=name_filter):
yield File(itm)
def test_add_symbols():
remove_db()
path = '/home/file.py'
with db.DbHelper() as dbh:
pid = dbh.create_project('/home/colin')
file_id = dbh.create_file(path, pid)
assert len(list(dbh.get_symbols(file_id=file_id))) == 0
symbol_id = dbh.create_symbol('spam', 10, 45, 'code-variable',
'/path/to/icon.png', file_id, pid)
assert symbol_id == 1
symbols = list(dbh.get_symbols(file_id=file_id))
assert len(symbols) == 1
assert symbols[0][db.COL_SYMBOL_NAME] == 'spam'
assert symbols[0][db.COL_SYMBOL_LINE] == 10
assert symbols[0][db.COL_SYMBOL_COLUMN] == 45
assert symbols[0][db.COL_SYMBOL_ICON_THEME] == 'code-variable'
assert symbols[0][db.COL_SYMBOL_ICON_PATH] == '/path/to/icon.png'
assert symbols[0][db.COL_SYMBOL_PARENT_SYMBOL_ID] == 'null'
assert symbols[0][db.COL_SYMBOL_FILE_ID] == file_id
assert symbols[0][db.COL_SYMBOL_ID] == symbol_id
def _recursive_index_dirs(task_handle, directory, ignore_patterns, project_dir, project_id, parser_plugins):
"""
Performs a recursive indexation of the specified path.
:param task_handle: task handle, to report progress updates to the frontend.
:param directory: path to analyse.
:param ignore_patterns: The list of ignore patterns to respect.
:param project_dir: The root project directory.
:param project_id: Id of the.
:param parser_plugins: The list of parser plugins.
"""
paths = []
rel_dir = os.path.relpath(directory, project_dir)
task_handle.report_progress('Indexing "%s"' % rel_dir, -1)
join = os.path.join
isfile = os.path.isfile
try:
dir_paths = listdir(directory)
except OSError:
return
for path in dir_paths:
try:
path = path.name
except AttributeError:
_logger().debug('using the old python api for scanning dirs')
full_path = join(directory, path)
ignored = is_ignored_path(full_path, ignore_patterns)
if not ignored:
if isfile(full_path):
paths.append(full_path)
else:
_recursive_index_dirs(task_handle, full_path, ignore_patterns,
project_dir, project_id, parser_plugins)
files = []
with db.DbHelper() as dbh:
for path in paths:
fid = dbh.create_file(path, project_id, commit=False)
files.append((path, fid))
dbh.conn.commit()
_index_documents(task_handle, files, project_id, project_dir, parser_plugins)
