def download_collection(self, id, register_name):
"""Download a collection given its ID.
For remote git repositories, we will clone the repository first (or
pull if the repo is already cloned before) and copy the collection to
collections dir.
:param id: The ID of the collection.
:type id: str
:param register_name: The register name of the collection (the
section name of the collection)
:type register_name: unicode
"""
# Copy the specific downloaded collection to collections dir
src_dir = os.path.join(self._path, 'collections', register_name)
if not os.path.exists(src_dir):
error_message = ('Error: The collection does not exist in the '
'repository.')
return False, error_message
dest_dir = local_collection_path(id)
if os.path.exists(dest_dir):
shutil.rmtree(dest_dir)
shutil.copytree(src_dir, dest_dir)
return True, None
def download_collection(self, id, register_name):
"""Download a collection given its ID.
For zip collection, we will download the zip, and extract the
collection to collections dir.
:param id: The ID of the collection.
:type id: str
:param register_name: The register name of the collection (the
section name of the collection)
:type register_name: unicode
"""
# Download the zip first
collection_path = 'collections/%s.zip' % register_name
network_manager = NetworkManager(self.file_url(collection_path))
status, description = network_manager.fetch()
if not status:
return False, description
# Create the zip file
zip_file = QTemporaryFile()
if zip_file.open():
zip_file.write(network_manager.content)
zip_file.close()
zf = ZipFile(zip_file.fileName())
zf.extractall(path=local_collection_path(id))
return True, None
def download_collection(self, id, register_name):
"""Download a collection given its ID.
For remote git repositories, we will clone the repository first (or
pull if the repo is already cloned before) and copy the collection to
collections dir.
:param id: The ID of the collection.
:type id: str
:param register_name: The register name of the collection (the
section name of the collection)
:type register_name: unicode
"""
# Copy the specific downloaded collection to collections dir
src_dir = os.path.join(self._path, 'collections', register_name)
if not os.path.exists(src_dir):
error_message = ('Error: The collection does not exist in the '
'repository.')
return False, error_message
dest_dir = local_collection_path(id)
if os.path.exists(dest_dir):
shutil.rmtree(dest_dir)
shutil.copytree(src_dir, dest_dir)
return True, None
def uninstall(self):
"""Uninstall the SVGs from QGIS."""
# Remove from the searchPaths if the dir empty of collection
settings = QSettings()
search_paths_str = settings.value('svg/searchPathsForSVG')
if not search_paths_str:
search_paths = []
else:
search_paths = search_paths_str.split('|')
collection_directories = os.listdir(local_collection_path())
if len(collection_directories) == 0:
if local_collection_path() in search_paths:
search_paths.remove(local_collection_path())
settings.setValue('svg/searchPathsForSVG', '|'.join(search_paths))
def uninstall(self):
"""Uninstall the SVGs."""
if not Path(self.resource_dir).exists():
return
# Remove from the SVG search paths if there are no SVGs left
# under local_collection_path.
# Have to remove now, to be able to update the SVG search path
shutil.rmtree(self.resource_dir)
# Check if there are no SVG files in the collections directory
svgCount = 0
for filename in local_collection_path().rglob('*'):
if filename.suffix.lower() == "svg":
svgCount += 1
break
search_paths = self.svg_search_paths()
if svgCount == 0:
if str(local_collection_path()) in search_paths:
search_paths.remove(str(local_collection_path()))
self.set_svg_search_paths(search_paths)
def install(self):
"""Install the SVGs from this collection.
Add the collection root directory path to the SVG search path.
"""
# Check if the dir exists, pass silently if it doesn't
if not Path(self.resource_dir).exists():
return
# Add to the SVG search paths
search_paths = self.svg_search_paths()
if str(local_collection_path()) not in search_paths:
search_paths.append(str(local_collection_path()))
self.set_svg_search_paths(search_paths)
# Count the SVGs
valid = 0
for filename in Path(self.resource_dir).rglob('*'):
if filename.suffix.lower().endswith("svg"):
valid += 1
if valid >= 0:
self.collection[SVG] = valid
def install(self):
"""Install the SVGs from this collection in to QGIS.
We simply just add the path to the collection root directory to search
path in QGIS.
"""
# Check if the dir exists, pass installing silently if it doesn't exist
if not os.path.exists(self.resource_dir):
return
# Add to the search paths for SVG
settings = QSettings()
search_paths_str = settings.value('svg/searchPathsForSVG')
if not search_paths_str:
search_paths = []
else:
search_paths = search_paths_str.split('|')
if local_collection_path() not in search_paths:
search_paths.append(local_collection_path())
settings.setValue('svg/searchPathsForSVG', '|'.join(search_paths))
def uninstall(self, collection_id):
"""Uninstall the collection.
:param collection_id: The id of the collection about to be uninstalled.
:type collection_id: str
"""
# Uninstall all types of resources
for resource_handler in BaseResourceHandler.registry.values():
resource_handler_instance = resource_handler(collection_id)
resource_handler_instance.uninstall()
# Remove the collection directory
collection_dir = local_collection_path(collection_id)
if collection_dir.exists():
shutil.rmtree(str(collection_dir))
config.COLLECTIONS[collection_id]["status"] = COLLECTION_NOT_INSTALLED_STATUS
def rebuild_collections(self):
"""Rebuild collections from repositories."""
config.COLLECTIONS = {}
for repo in self._repositories.keys():
repo_collections = self._repositories[repo]
for collection in repo_collections:
collection_id = self._collections_manager.get_collection_id(
collection['register_name'], collection['repository_url'])
config.COLLECTIONS[collection_id] = collection
# Check in the file system if the collection exists for all
# installed collections. If not, also uninstall resources
current_status = config.COLLECTIONS[collection_id]['status']
if current_status == COLLECTION_INSTALLED_STATUS:
collection_path = local_collection_path(collection_id)
if not os.path.exists(collection_path):
# Uninstall the collection
self._collections_manager.uninstall(collection_id)
def uninstall(self, collection_id):
"""Uninstall the collection from QGIS.
:param collection_id: The id of the collection about to be uninstalled.
:type collection_id: str
"""
# Remove the collection directory
collection_dir = local_collection_path(collection_id)
if os.path.exists(collection_dir):
shutil.rmtree(collection_dir)
# Uninstall all type of resources from QGIS
for resource_handler in BaseResourceHandler.registry.values():
resource_handler_instance = resource_handler(collection_id)
resource_handler_instance.uninstall()
config.COLLECTIONS[collection_id]['status'] = \
COLLECTION_NOT_INSTALLED_STATUS
def rebuild_collections(self):
"""Rebuild collections from repositories."""
config.COLLECTIONS = {}
for repo in self._repositories.keys():
repo_collections = self._repositories[repo]
for collection in repo_collections:
collection_id = self._collections_manager.get_collection_id(
collection['register_name'],
collection['repository_url']
)
config.COLLECTIONS[collection_id] = collection
# Check in the file system if the collection exists for all
# installed collections. If not, also uninstall resources
current_status = config.COLLECTIONS[collection_id]['status']
if current_status == COLLECTION_INSTALLED_STATUS:
collection_path = local_collection_path(collection_id)
if not os.path.exists(collection_path):
# Uninstall the collection
self._collections_manager.uninstall(collection_id)
def rebuild_collections(self):
"""Rebuild the collections for all the repositories."""
config.COLLECTIONS = {}
for repo in self._repositories.keys():
repo_collections = self._repositories[repo]
for collection in repo_collections:
collection_id = self._collections_manager.get_collection_id(
collection["register_name"], collection["repository_url"]
)
collection["repository_name"] = repo
config.COLLECTIONS[collection_id] = collection
# Get the collection path (updating if neccessary)
collection_path = local_collection_path(collection_id)
# Check the file system to see if the collection exists.
# If not, also uninstall its resources
current_status = config.COLLECTIONS[collection_id]["status"]
if current_status == COLLECTION_INSTALLED_STATUS:
if not collection_path.exists():
# Uninstall the collection
self._collections_manager.uninstall(collection_id)
def download_collection(self, id, register_name):
"""Download a collection given its ID.
:param id: The ID of the collection.
:type id: str
:param register_name: The register name of the collection (the
section name of the collection)
:type register_name: unicode
"""
# Copy the specific downloaded collection to collections dir
src_dir = Path(self._path) / "collections" / register_name
if not src_dir.exists():
error_message = "Error: The collection does not exist in the " "repository."
return False, error_message
dest_dir = local_collection_path(id)
if dest_dir.exists():
shutil.rmtree(str(dest_dir))
shutil.copytree(str(src_dir), str(dest_dir))
return True, None
def open_collection(self):
"""Slot for when user clicks 'Open' button."""
collection_path = local_collection_path(self._selected_collection_id)
directory_url = QUrl.fromLocalFile(str(collection_path))
QDesktopServices.openUrl(directory_url)
def collection_path(self):
"""Return the local collection path."""
return local_collection_path(self._collection_id)
def resource_dir(self):
"""The root of the resource dir from this resource type."""
resource_dir = local_collection_path(self.collection_id) / self.dir_name()
return resource_dir
def edit_directory(
self, old_repo_name, new_repo_name, old_url, new_url, new_auth_cfg
):
"""Edit the directory of repositories and update the
collections.
Also used to reload repositories (old == new for url and repo_name)
:param old_repo_name: The old name of the repository
:type old_repo_name: str
:param new_repo_name: The new name of the repository
:type new_repo_name: str
:param old_url: The old URL of the repository
:type old_url: str
:param new_url: The new URL of the repository
:type new_url: str
:param new_auth_cfg: The auth config id.
:type new_auth_cfg: str
:return: (status, error)
:rtype: (boolean, string)
"""
old_collections = self._repositories.get(old_repo_name, [])
if (old_repo_name != new_repo_name) and (old_url == new_url):
# Renaming a repository (same URL)
for old_collection in old_collections:
coll_id = self._collections_manager.get_collection_id(
old_collection["register_name"], old_collection["repository_url"]
)
old_path = local_collection_path(coll_id)
# Update the repository name for this collection
config.COLLECTIONS[coll_id]["repository_name"] = new_repo_name
new_path = local_collection_path(coll_id)
# If the repository is renamed (same URL), the directories
# of its collections should be renamed accordingly (so that
# they remain accessible)
if old_path.exists():
old_path.rename(new_path)
new_collections = old_collections
status = True
fetcherror = ""
else:
# old_repo_name == new_repo_name and old_url == new_url
# or new_url != old_url
# Fetch the metadata (metadata.ini) from the new url
repo_handler = BaseRepositoryHandler.get_handler(new_url)
if repo_handler is None:
repo_warning = "No handler for URL '" + str(new_url) + "'!"
LOGGER.warning(repo_warning)
return (False, repo_warning)
if new_auth_cfg:
repo_handler.auth_cfg = new_auth_cfg
status, fetcherror = repo_handler.fetch_metadata()
if status:
# Parse metadata
try:
new_collections = repo_handler.parse_metadata()
except MetadataError as me:
metadata_warning = (
"Error parsing metadata for "
+ str(new_repo_name)
+ ":\n"
+ str(me)
)
LOGGER.warning(metadata_warning)
return (False, metadata_warning)
# raise MetadataError(metadata_warning)
# Get all the installed collections from the old repository
installed_old_collections = []
for old_collection in old_collections:
if old_collection["status"] == COLLECTION_INSTALLED_STATUS:
installed_old_collections.append(old_collection)
# Handling installed collections
# An old collection that is present in the new location
# (URL) is identified by its register name.
# Cases for installed collections:
# 1. Old collection exists in the new, same URL: use the new
# one, else: update the status to INSTALLED
# 2. Old collection exists in the new, different URL: keep them
# both (add the old one). Because they should be treated as
# different collections
# 3. Old collection doesn't exist in the new, same URL: keep
# the old collection
# 4. Old collection doesn't exist in the new, different URL:
# same as 3
for installed_collection in installed_old_collections:
reg_name = installed_collection["register_name"]
is_present = False
for n_coll in new_collections:
# Look for collections that are already present
if n_coll["register_name"] == reg_name:
# Already present
is_present = True
if old_url == new_url:
# Set the status to installed
n_coll["status"] = COLLECTION_INSTALLED_STATUS
# Keep the collection statistics
for key in installed_collection.keys():
if key in [
"models",
"processing",
"rscripts",
"style",
"svg",
"symbol",
"expressions",
]:
n_coll[key] = installed_collection[key]
else:
# Different repository URLs, so append
new_collections.append(installed_collection)
break
if not is_present:
new_collections.append(installed_collection)
# Remove the old repository and add the new one
self._repositories.pop(old_repo_name, None)
self._repositories[new_repo_name] = new_collections
self.rebuild_collections()
# Update QgsSettings
settings = QgsSettings()
settings.beginGroup(repo_settings_group())
settings.remove(old_repo_name)
settings.setValue(new_repo_name + "/url", new_url)
settings.setValue(new_repo_name + "/auth_cfg", new_auth_cfg)
settings.endGroup()
# Serialize repositories every time we successfully edited repo
self.serialize_repositories()
return status, fetcherror
def resource_dir(self):
"""The root of the resource dir from this type of the resource."""
resource_dir = os.path.join(
local_collection_path(self.collection_id), self.dir_name())
return resource_dir
def collection_path(self):
"""Return the local collection path."""
return local_collection_path(self._collection_id)
def open_collection(self):
"""Slot for when user clicks 'Open' button."""
collection_path = local_collection_path(self._selected_collection_id)
directory_url = QUrl.fromLocalFile(collection_path)
QDesktopServices.openUrl(directory_url)
def download_collection(self, id, register_name):
"""Download a collection given its ID.
For remote git repositories, we will clone the repository (or
pull if the repo is already cloned), and copy the collection to
the collections directory.
:param id: The ID of the collection.
:type id: str
:param register_name: The register name of the collection (the
section name of the collection)
:type register_name: unicode
:return: (success (True or False), error message (None if success))
:rtype: (boolean, string)
"""
# Hack to avoid irritating Dulwich / Porcelain ResourceWarning
warnings.filterwarnings("ignore", category=ResourceWarning)
# Clone or pull the repositories first
local_repo_dir = Path(QgsApplication.qgisSettingsDirPath(),
'resource_sharing', 'repositories',
self.git_host, self.git_owner,
self.git_repository)
# Hack to try to avoid locking errors
if local_repo_dir.exists():
try:
shutil.rmtree(str(local_repo_dir))
except:
pass
if not (local_repo_dir / '.git').exists():
local_repo_dir.mkdir(parents=True)
try:
repo = porcelain.clone(self.url,
str(local_repo_dir),
errstream=writeOut)
repo.close() # Try to avoid WinErr 32
except Exception as e:
# Try to clone with https if it is a ssh url
git_parsed = parse(self.url)
if self.url == git_parsed.url2ssh:
try:
repo = porcelain.clone(git_parsed.url2https,
str(local_repo_dir),
errstream=writeOut)
repo.close() # Try to avoid WinErr 32
except Exception as e:
error_message = 'Error: %s' % str(e)
LOGGER.exception(traceback.format_exc())
return False, error_message
else:
error_message = 'Error: %s' % str(e)
LOGGER.exception(traceback.format_exc())
return False, error_message
if not repo:
error_message = ('Error: Cloning the repository of the '
'collection failed.')
return False, error_message
else:
# Hack until dulwich/porcelain handles file removal ???!!!
if local_repo_dir.exists():
shutil.rmtree(str(local_repo_dir))
try:
porcelain.pull(str(local_repo_dir),
self.url,
b'refs/heads/master',
errstream=writeOut)
except Exception as e:
# Try to pull with https if it's ssh url
git_parsed = parse(self.url)
if self.url == git_parsed.url2ssh:
try:
porcelain.pull(str(local_repo_dir),
git_parsed.url2https,
b'refs/heads/master',
errstream=writeOut)
except Exception as e:
error_message = 'Error: %s' % str(e)
LOGGER.exception(traceback.format_exc())
return False, error_message
else:
error_message = 'Error: %s' % str(e)
LOGGER.exception(traceback.format_exc())
return False, error_message
# Copy the specific downloaded collection to the collections dir
src_dir = local_repo_dir / 'collections' / register_name
if not src_dir.exists():
error_message = ('Error: The collection does not exist in the '
'repository.')
return False, error_message
dest_dir = local_collection_path(id)
if dest_dir.exists():
# Remove the existing collection directory
shutil.rmtree(str(dest_dir))
shutil.copytree(str(src_dir), str(dest_dir))
return True, None
def resource_dir(self):
"""The root of the resource dir from this type of the resource."""
resource_dir = os.path.join(local_collection_path(self.collection_id),
self.dir_name())
return resource_dir
def download_collection(self, id, register_name):
"""Download a collection given its ID.
For remote git repositories, we will clone the repository first (or
pull if the repo is already cloned before) and copy the collection to
collections dir.
:param id: The ID of the collection.
:type id: str
:param register_name: The register name of the collection (the
section name of the collection)
:type register_name: unicode
"""
# Clone or pull the repositories first
local_repo_dir = os.path.join(QgsApplication.qgisSettingsDirPath(),
'resource_sharing', 'repositories',
self.git_host, self.git_owner,
self.git_repository)
if not os.path.exists(local_repo_dir):
os.makedirs(local_repo_dir)
try:
repo = porcelain.clone(self.url.encode('utf-8'),
local_repo_dir)
except Exception as e:
# Try to clone with https if it's ssh url
git_parsed = parse(self.url)
if self.url.encode('utf-8') == git_parsed.url2ssh:
try:
repo = porcelain.clone(git_parsed.url2https,
local_repo_dir)
except Exception as e:
error_message = 'Error: %s' % str(e)
LOGGER.exception(traceback.format_exc())
return False, error_message
else:
error_message = 'Error: %s' % str(e)
LOGGER.exception(traceback.format_exc())
return False, error_message
if not repo:
error_message = ('Error: Cloning the repository of the '
'collection failed.')
return False, error_message
else:
try:
porcelain.pull(local_repo_dir, self.url.encode('utf-8'),
b'refs/heads/master')
except Exception as e:
# Try to pull with https if it's ssh url
git_parsed = parse(self.url)
if self.url.encode('utf-8') == git_parsed.url2ssh:
try:
porcelain.pull(local_repo_dir, git_parsed.url2https,
b'refs/heads/master')
except Exception as e:
error_message = 'Error: %s' % str(e)
LOGGER.exception(traceback.format_exc())
return False, error_message
else:
error_message = 'Error: %s' % str(e)
LOGGER.exception(traceback.format_exc())
return False, error_message
# Copy the specific downloaded collection to collections dir
src_dir = os.path.join(local_repo_dir, 'collections', register_name)
if not os.path.exists(src_dir):
error_message = ('Error: The collection does not exist in the '
'repository.')
return False, error_message
dest_dir = local_collection_path(id)
if os.path.exists(dest_dir):
shutil.rmtree(dest_dir)
shutil.copytree(src_dir, dest_dir)
return True, None
def download_collection(self, id, register_name):
"""Download a collection given its ID.
For remote git repositories, we will clone the repository first (or
pull if the repo is already cloned before) and copy the collection to
collections dir.
:param id: The ID of the collection.
:type id: str
:param register_name: The register name of the collection (the
section name of the collection)
:type register_name: unicode
"""
# Clone or pull the repositories first
local_repo_dir = os.path.join(
QgsApplication.qgisSettingsDirPath(),
'resource_sharing',
'repositories',
self.git_host, self.git_owner, self.git_repository
)
if not os.path.exists(os.path.join(local_repo_dir, '.git')):
os.makedirs(local_repo_dir)
try:
repo = porcelain.clone(
self.url, local_repo_dir,
errstream=writeOut
)
except Exception as e:
# Try to clone with https if it's ssh url
git_parsed = parse(self.url)
if self.url == git_parsed.url2ssh:
try:
repo = porcelain.clone(
git_parsed.url2https, local_repo_dir,
errstream=writeOut)
except Exception as e:
error_message = 'Error: %s' % str(e)
LOGGER.exception(traceback.format_exc())
return False, error_message
else:
error_message = 'Error: %s' % str(e)
LOGGER.exception(traceback.format_exc())
return False, error_message
if not repo:
error_message = ('Error: Cloning the repository of the '
'collection failed.')
return False, error_message
else:
try:
porcelain.pull(
local_repo_dir,
self.url,
b'refs/heads/master',
errstream=writeOut
)
except Exception as e:
# Try to pull with https if it's ssh url
git_parsed = parse(self.url)
if self.url == git_parsed.url2ssh:
try:
porcelain.pull(
local_repo_dir,
git_parsed.url2https,
b'refs/heads/master',
errstream=writeOut
)
except Exception as e:
error_message = 'Error: %s' % str(e)
LOGGER.exception(traceback.format_exc())
return False, error_message
else:
error_message = 'Error: %s' % str(e)
LOGGER.exception(traceback.format_exc())
return False, error_message
# Copy the specific downloaded collection to collections dir
src_dir = os.path.join(local_repo_dir, 'collections', register_name)
if not os.path.exists(src_dir):
error_message = ('Error: The collection does not exist in the '
'repository.')
return False, error_message
dest_dir = local_collection_path(id)
if os.path.exists(dest_dir):
shutil.rmtree(dest_dir)
shutil.copytree(src_dir, dest_dir)
return True, None
