def download_sg_diagrams(
db_manager,
site_layer,
diagram_layer,
sg_code_field,
output_directory,
all_features=False,
callback=None):
"""Downloads all SG Diagrams.
:param db_manager: A database manager
:type db_manager: DatabaseManager
:param site_layer: The target layer.
:type site_layer: QgsVectorLayer
:param diagram_layer: Vector layer that has sg code in its field.
:type diagram_layer: QgsVectorLayer
:param sg_code_field: Name of the field that contains sg code
:type sg_code_field: str
:param output_directory: Directory to put the diagram.
:type output_directory: str
:param all_features: If True select all features, else only the selected
ones.
:type all_features: bool
:param callback: A function to all to indicate progress. The function
should accept params 'current' (int) and 'maximum' (int). Defaults to
None.
:type callback: function
:returns: A report listing which files were downloaded and their
download failure or success.
:rtype: str
"""
if callback is None:
callback = print_progress_callback
sg_codes_and_provinces = map_sg_codes_to_provinces(
db_manager, site_layer, diagram_layer, sg_code_field, all_features)
maximum = len(sg_codes_and_provinces)
current = 0
report = ''
for sg_code, province in sg_codes_and_provinces.iteritems():
current += 1
message = 'Downloading SG Code %s from %s' % (sg_code, province)
callback(current, maximum, message)
try:
report += download_sg_diagram(
db_manager,
sg_code,
province,
output_directory,
callback)
except Exception, e:
report += 'Failed to download %s %s %s\n' % (sg_code, province, e)
LOGGER.exception(e)
def download_sg_diagrams(db_manager,
site_layer,
diagram_layer,
sg_code_field,
output_directory,
all_features=False,
callback=None):
"""Downloads all SG Diagrams.
:param db_manager: A database manager
:type db_manager: DatabaseManager
:param site_layer: The target layer.
:type site_layer: QgsVectorLayer
:param diagram_layer: Vector layer that has sg code in its field.
:type diagram_layer: QgsVectorLayer
:param sg_code_field: Name of the field that contains sg code
:type sg_code_field: str
:param output_directory: Directory to put the diagram.
:type output_directory: str
:param all_features: If True select all features, else only the selected
ones.
:type all_features: bool
:param callback: A function to all to indicate progress. The function
should accept params 'current' (int) and 'maximum' (int). Defaults to
None.
:type callback: function
:returns: A report listing which files were downloaded and their
download failure or success.
:rtype: str
"""
if callback is None:
callback = print_progress_callback
sg_codes_and_provinces = map_sg_codes_to_provinces(db_manager, site_layer,
diagram_layer,
sg_code_field,
all_features)
maximum = len(sg_codes_and_provinces)
current = 0
report = ''
for sg_code, province in sg_codes_and_provinces.iteritems():
current += 1
message = 'Downloading SG Code %s from %s' % (sg_code, province)
callback(current, maximum, message)
try:
report += download_sg_diagram(db_manager, sg_code, province,
output_directory, callback)
except Exception, e:
report += 'Failed to download %s %s %s\n' % (sg_code, province, e)
LOGGER.exception(e)
def download_sg_diagram(
db_manager, sg_code, province_name, output_directory, callback=None):
"""Download sg diagram using sg_code and put it under output_directory.
:param db_manager: A database manager
:type db_manager: DatabaseManager
:param sg_code: Surveyor General code.
:type sg_code: str
:param province_name: Province name.
:type province_name: str
:param output_directory: Directory to put the diagram.
:type output_directory: str
:param callback: A function to all to indicate progress. The function
should accept params 'current' (int) and 'maximum' (int). Defaults to
None.
:type callback: function
:returns: A report listing which files were downloaded and their
download failure or success.
:rtype: str
"""
if callback is None:
callback = print_progress_callback
report = 'Downloading documents for %s in %s\n' % (sg_code, province_name)
try:
download_page = construct_url(db_manager, sg_code, province_name)
except (InvalidSGCodeException,
DatabaseException,
UrlException,
NotInSouthAfricaException) as e:
report += (
'Failed: Downloading SG code %s for province %s because of %s\n' %
(sg_code, province_name, e.reason))
return report
try:
download_links = parse_download_page(download_page)
except ParseException as e:
report += (
'Failed: Downloading SG code %s for province %s because of %s\n' %
(sg_code, province_name, e.reason))
return report
output_directory = os.path.join(output_directory, sg_code)
if not os.path.exists(output_directory):
os.makedirs(output_directory)
count = 0
total = len(download_links)
if total == 0:
report += 'No documents found for %s in %s' % (sg_code, province_name)
for download_link in download_links:
count += 1
message = ('[%s - %s] Downloading file %s of %s' % (
sg_code, province_name, count, total))
callback(count, total, message)
try:
file_path = download_from_url(download_link, output_directory)
if file_path is not None:
report += 'Success: File %i of %i : %s saved to %s\n' % (
count, total, download_link, file_path)
else:
report += 'Failed: File %i of %i : %s \n' % (
count, total, download_link)
except DownloadException as e:
message = 'Failed to download %s for %s in %s because %s' % (
download_link, sg_code, province_name, e.reason)
LOGGER.exception(message)
report += 'Failed: File %i of %i : %s \n' % (
count, total, download_link)
message = 'Downloads completed for %s in %s' % (sg_code, province_name)
callback(count, total, message)
return report
def download_sg_diagram(db_manager,
sg_code,
province_name,
output_directory,
callback=None):
"""Download sg diagram using sg_code and put it under output_directory.
:param db_manager: A database manager
:type db_manager: DatabaseManager
:param sg_code: Surveyor General code.
:type sg_code: str
:param province_name: Province name.
:type province_name: str
:param output_directory: Directory to put the diagram.
:type output_directory: str
:param callback: A function to all to indicate progress. The function
should accept params 'current' (int) and 'maximum' (int). Defaults to
None.
:type callback: function
:returns: A report listing which files were downloaded and their
download failure or success.
:rtype: str
"""
if callback is None:
callback = print_progress_callback
report = 'Downloading documents for %s in %s\n' % (sg_code, province_name)
try:
download_page = construct_url(db_manager, sg_code, province_name)
except (InvalidSGCodeException, DatabaseException, UrlException,
NotInSouthAfricaException) as e:
report += (
'Failed: Downloading SG code %s for province %s because of %s\n' %
(sg_code, province_name, e.reason))
return report
try:
download_links = parse_download_page(download_page)
except ParseException as e:
report += (
'Failed: Downloading SG code %s for province %s because of %s\n' %
(sg_code, province_name, e.reason))
return report
output_directory = os.path.join(output_directory, sg_code)
if not os.path.exists(output_directory):
os.makedirs(output_directory)
count = 0
total = len(download_links)
if total == 0:
report += 'No documents found for %s in %s' % (sg_code, province_name)
for download_link in download_links:
count += 1
message = ('[%s - %s] Downloading file %s of %s' %
(sg_code, province_name, count, total))
callback(count, total, message)
try:
file_path = download_from_url(download_link, output_directory)
if file_path is not None:
report += 'Success: File %i of %i : %s saved to %s\n' % (
count, total, download_link, file_path)
else:
report += 'Failed: File %i of %i : %s \n' % (count, total,
download_link)
except DownloadException as e:
message = 'Failed to download %s for %s in %s because %s' % (
download_link, sg_code, province_name, e.reason)
LOGGER.exception(message)
report += 'Failed: File %i of %i : %s \n' % (count, total,
download_link)
message = 'Downloads completed for %s in %s' % (sg_code, province_name)
callback(count, total, message)
return report
