def download_sg_diagrams(
db_manager,
site_layer,
diagram_layer,
sg_code_field,
output_directory,
all_features=False,
callback=None):
"""Downloads all SG Diagrams.
:param db_manager: A database manager
:type db_manager: DatabaseManager
:param site_layer: The target layer.
:type site_layer: QgsVectorLayer
:param diagram_layer: Vector layer that has sg code in its field.
:type diagram_layer: QgsVectorLayer
:param sg_code_field: Name of the field that contains sg code
:type sg_code_field: str
:param output_directory: Directory to put the diagram.
:type output_directory: str
:param all_features: If True select all features, else only the selected
ones.
:type all_features: bool
:param callback: A function to all to indicate progress. The function
should accept params 'current' (int) and 'maximum' (int). Defaults to
None.
:type callback: function
:returns: A report listing which files were downloaded and their
download failure or success.
:rtype: str
"""
if callback is None:
callback = print_progress_callback
sg_codes_and_provinces = map_sg_codes_to_provinces(
db_manager, site_layer, diagram_layer, sg_code_field, all_features)
maximum = len(sg_codes_and_provinces)
current = 0
report = ''
for sg_code, province in sg_codes_and_provinces.iteritems():
current += 1
message = 'Downloading SG Code %s from %s' % (sg_code, province)
callback(current, maximum, message)
try:
report += download_sg_diagram(
db_manager,
sg_code,
province,
output_directory,
callback)
except Exception, e:
report += 'Failed to download %s %s %s\n' % (sg_code, province, e)
LOGGER.exception(e)
def download_sg_diagrams(db_manager,
site_layer,
diagram_layer,
sg_code_field,
output_directory,
all_features=False,
callback=None):
"""Downloads all SG Diagrams.
:param db_manager: A database manager
:type db_manager: DatabaseManager
:param site_layer: The target layer.
:type site_layer: QgsVectorLayer
:param diagram_layer: Vector layer that has sg code in its field.
:type diagram_layer: QgsVectorLayer
:param sg_code_field: Name of the field that contains sg code
:type sg_code_field: str
:param output_directory: Directory to put the diagram.
:type output_directory: str
:param all_features: If True select all features, else only the selected
ones.
:type all_features: bool
:param callback: A function to all to indicate progress. The function
should accept params 'current' (int) and 'maximum' (int). Defaults to
None.
:type callback: function
:returns: A report listing which files were downloaded and their
download failure or success.
:rtype: str
"""
if callback is None:
callback = print_progress_callback
sg_codes_and_provinces = map_sg_codes_to_provinces(db_manager, site_layer,
diagram_layer,
sg_code_field,
all_features)
maximum = len(sg_codes_and_provinces)
current = 0
report = ''
for sg_code, province in sg_codes_and_provinces.iteritems():
current += 1
message = 'Downloading SG Code %s from %s' % (sg_code, province)
callback(current, maximum, message)
try:
report += download_sg_diagram(db_manager, sg_code, province,
output_directory, callback)
except Exception, e:
report += 'Failed to download %s %s %s\n' % (sg_code, province, e)
LOGGER.exception(e)
def download_from_url(url, output_directory, filename=None, use_cache=True):
"""Download file from a url and put it under output_directory.
:param url: Url that gives response.
:type url: str
:param output_directory: Directory to put the diagram.
:type output_directory: str
:param filename: Optional filename for downloaded file.
:type filename: str
:param use_cache: If there is a cached copy of the file already in the
output directory, do not refetch it (True) or force refecth it (False).
:type use_cache: bool
:returns: File path if success to download, else None
:rtype: str
"""
if filename is None:
filename = get_filename(url)
LOGGER.info('Download file %s from %s' % (filename, url))
file_path = os.path.join(output_directory, filename)
if os.path.exists(file_path) and use_cache:
LOGGER.info('File %s exists, not downloading' % file_path)
return file_path
# Set Proxy in webpage
proxy = get_proxy()
network_manager = QNetworkAccessManager()
if not proxy is None:
network_manager.setProxy(proxy)
# Download Process
# noinspection PyTypeChecker
downloader = FileDownloader(network_manager, url, file_path)
try:
result = downloader.download()
except IOError as ex:
raise DownloadException(ex)
if result[0] is not True:
_, error_message = result
raise DownloadException(error_message)
if os.path.exists(file_path):
return file_path
else:
return None
def download_from_url(url, output_directory, filename=None, use_cache=True):
"""Download file from a url and put it under output_directory.
:param url: Url that gives response.
:type url: str
:param output_directory: Directory to put the diagram.
:type output_directory: str
:param filename: Optional filename for downloaded file.
:type filename: str
:param use_cache: If there is a cached copy of the file already in the
output directory, do not refetch it (True) or force refecth it (False).
:type use_cache: bool
:returns: File path if success to download, else None
:rtype: str
"""
if filename is None:
filename = get_filename(url)
LOGGER.info('Download file %s from %s' % (filename, url))
file_path = os.path.join(output_directory, filename)
if os.path.exists(file_path) and use_cache:
LOGGER.info('File %s exists, not downloading' % file_path)
return file_path
# Set Proxy in webpage
proxy = get_proxy()
network_manager = QNetworkAccessManager()
if not proxy is None:
network_manager.setProxy(proxy)
# Download Process
# noinspection PyTypeChecker
downloader = FileDownloader(network_manager, url, file_path)
try:
result = downloader.download()
except IOError as ex:
raise DownloadException(ex)
if result[0] is not True:
_, error_message = result
raise DownloadException(error_message)
if os.path.exists(file_path):
return file_path
else:
return None
def construct_url(db_manager, sg_code=None, province_name=None):
"""Construct url to download sg diagram.
:param db_manager: A database manager
:type db_manager: DatabaseManager
:param sg_code: SG code.
:type sg_code: str
:param province_name: province_name name.
:type province_name: str
:returns: URL to download sg diagram.
:rtype: str
"""
LOGGER.info('Constructing url for %s %s' % (sg_code, province_name))
if not is_valid_sg_code(sg_code):
raise InvalidSGCodeException
if sg_code is None or province_name is None:
raise UrlException()
if province_name not in PROVINCE_NAMES:
raise NotInSouthAfricaException
base_url = 'http://csg.dla.gov.za/esio/listdocument.jsp?'
reg_division = sg_code[:8]
try:
record = get_office(db_manager, reg_division, province_name)
except DatabaseException:
raise DatabaseException
if record is None or bool(record) is None:
raise DatabaseException
office, office_number, typology = record
erf = sg_code[8:16]
portion = sg_code[16:]
url = base_url + 'regDivision=' + reg_division
url += '&office=' + office
url += '&Noffice=' + office_number
url += '&Erf=' + erf
url += '&Portion=' + portion
return url
def construct_url(db_manager, sg_code=None, province_name=None):
"""Construct url to download sg diagram.
:param db_manager: A database manager
:type db_manager: DatabaseManager
:param sg_code: SG code.
:type sg_code: str
:param province_name: province_name name.
:type province_name: str
:returns: URL to download sg diagram.
:rtype: str
"""
LOGGER.info('Constructing url for %s %s' % (sg_code, province_name))
if not is_valid_sg_code(sg_code):
raise InvalidSGCodeException
if sg_code is None or province_name is None:
raise UrlException()
if province_name not in PROVINCE_NAMES:
raise NotInSouthAfricaException
base_url = 'http://csg.dla.gov.za/esio/listdocument.jsp?'
reg_division = sg_code[:8]
try:
record = get_office(db_manager, reg_division, province_name)
except DatabaseException:
raise DatabaseException
if record is None or bool(record) is None:
raise DatabaseException
office, office_number, typology = record
erf = sg_code[8:16]
portion = sg_code[16:]
url = base_url + 'regDivision=' + reg_division
url += '&office=' + office
url += '&Noffice=' + office_number
url += '&Erf=' + erf
url += '&Portion=' + portion
return url
def download_sg_diagram(
db_manager, sg_code, province_name, output_directory, callback=None):
"""Download sg diagram using sg_code and put it under output_directory.
:param db_manager: A database manager
:type db_manager: DatabaseManager
:param sg_code: Surveyor General code.
:type sg_code: str
:param province_name: Province name.
:type province_name: str
:param output_directory: Directory to put the diagram.
:type output_directory: str
:param callback: A function to all to indicate progress. The function
should accept params 'current' (int) and 'maximum' (int). Defaults to
None.
:type callback: function
:returns: A report listing which files were downloaded and their
download failure or success.
:rtype: str
"""
if callback is None:
callback = print_progress_callback
report = 'Downloading documents for %s in %s\n' % (sg_code, province_name)
try:
download_page = construct_url(db_manager, sg_code, province_name)
except (InvalidSGCodeException,
DatabaseException,
UrlException,
NotInSouthAfricaException) as e:
report += (
'Failed: Downloading SG code %s for province %s because of %s\n' %
(sg_code, province_name, e.reason))
return report
try:
download_links = parse_download_page(download_page)
except ParseException as e:
report += (
'Failed: Downloading SG code %s for province %s because of %s\n' %
(sg_code, province_name, e.reason))
return report
output_directory = os.path.join(output_directory, sg_code)
if not os.path.exists(output_directory):
os.makedirs(output_directory)
count = 0
total = len(download_links)
if total == 0:
report += 'No documents found for %s in %s' % (sg_code, province_name)
for download_link in download_links:
count += 1
message = ('[%s - %s] Downloading file %s of %s' % (
sg_code, province_name, count, total))
callback(count, total, message)
try:
file_path = download_from_url(download_link, output_directory)
if file_path is not None:
report += 'Success: File %i of %i : %s saved to %s\n' % (
count, total, download_link, file_path)
else:
report += 'Failed: File %i of %i : %s \n' % (
count, total, download_link)
except DownloadException as e:
message = 'Failed to download %s for %s in %s because %s' % (
download_link, sg_code, province_name, e.reason)
LOGGER.exception(message)
report += 'Failed: File %i of %i : %s \n' % (
count, total, download_link)
message = 'Downloads completed for %s in %s' % (sg_code, province_name)
callback(count, total, message)
return report
def download_sg_diagram(db_manager,
sg_code,
province_name,
output_directory,
callback=None):
"""Download sg diagram using sg_code and put it under output_directory.
:param db_manager: A database manager
:type db_manager: DatabaseManager
:param sg_code: Surveyor General code.
:type sg_code: str
:param province_name: Province name.
:type province_name: str
:param output_directory: Directory to put the diagram.
:type output_directory: str
:param callback: A function to all to indicate progress. The function
should accept params 'current' (int) and 'maximum' (int). Defaults to
None.
:type callback: function
:returns: A report listing which files were downloaded and their
download failure or success.
:rtype: str
"""
if callback is None:
callback = print_progress_callback
report = 'Downloading documents for %s in %s\n' % (sg_code, province_name)
try:
download_page = construct_url(db_manager, sg_code, province_name)
except (InvalidSGCodeException, DatabaseException, UrlException,
NotInSouthAfricaException) as e:
report += (
'Failed: Downloading SG code %s for province %s because of %s\n' %
(sg_code, province_name, e.reason))
return report
try:
download_links = parse_download_page(download_page)
except ParseException as e:
report += (
'Failed: Downloading SG code %s for province %s because of %s\n' %
(sg_code, province_name, e.reason))
return report
output_directory = os.path.join(output_directory, sg_code)
if not os.path.exists(output_directory):
os.makedirs(output_directory)
count = 0
total = len(download_links)
if total == 0:
report += 'No documents found for %s in %s' % (sg_code, province_name)
for download_link in download_links:
count += 1
message = ('[%s - %s] Downloading file %s of %s' %
(sg_code, province_name, count, total))
callback(count, total, message)
try:
file_path = download_from_url(download_link, output_directory)
if file_path is not None:
report += 'Success: File %i of %i : %s saved to %s\n' % (
count, total, download_link, file_path)
else:
report += 'Failed: File %i of %i : %s \n' % (count, total,
download_link)
except DownloadException as e:
message = 'Failed to download %s for %s in %s because %s' % (
download_link, sg_code, province_name, e.reason)
LOGGER.exception(message)
report += 'Failed: File %i of %i : %s \n' % (count, total,
download_link)
message = 'Downloads completed for %s in %s' % (sg_code, province_name)
callback(count, total, message)
return report