예제 #1
0
def download_sg_diagrams(
        db_manager,
        site_layer,
        diagram_layer,
        sg_code_field,
        output_directory,
        all_features=False,
        callback=None):
    """Downloads all SG Diagrams.

    :param db_manager: A database manager
    :type db_manager: DatabaseManager

    :param site_layer: The target layer.
    :type site_layer: QgsVectorLayer

    :param diagram_layer: Vector layer that has sg code in its field.
    :type diagram_layer: QgsVectorLayer

    :param sg_code_field: Name of the field that contains sg code
    :type sg_code_field: str

    :param output_directory: Directory to put the diagram.
    :type output_directory: str

    :param all_features: If True select all features, else only the selected
        ones.
    :type all_features: bool

    :param callback: A function to all to indicate progress. The function
        should accept params 'current' (int) and 'maximum' (int). Defaults to
        None.
    :type callback: function

    :returns: A report listing which files were downloaded and their
        download failure or success.
    :rtype: str
    """
    if callback is None:
        callback = print_progress_callback

    sg_codes_and_provinces = map_sg_codes_to_provinces(
        db_manager, site_layer, diagram_layer, sg_code_field, all_features)
    maximum = len(sg_codes_and_provinces)
    current = 0
    report = ''
    for sg_code, province in sg_codes_and_provinces.iteritems():
        current += 1
        message = 'Downloading SG Code %s from %s' % (sg_code, province)
        callback(current, maximum, message)
        try:
            report += download_sg_diagram(
                db_manager,
                sg_code,
                province,
                output_directory,
                callback)
        except Exception, e:
            report += 'Failed to download %s %s %s\n' % (sg_code, province, e)
            LOGGER.exception(e)
예제 #2
0
def download_sg_diagrams(db_manager,
                         site_layer,
                         diagram_layer,
                         sg_code_field,
                         output_directory,
                         all_features=False,
                         callback=None):
    """Downloads all SG Diagrams.

    :param db_manager: A database manager
    :type db_manager: DatabaseManager

    :param site_layer: The target layer.
    :type site_layer: QgsVectorLayer

    :param diagram_layer: Vector layer that has sg code in its field.
    :type diagram_layer: QgsVectorLayer

    :param sg_code_field: Name of the field that contains sg code
    :type sg_code_field: str

    :param output_directory: Directory to put the diagram.
    :type output_directory: str

    :param all_features: If True select all features, else only the selected
        ones.
    :type all_features: bool

    :param callback: A function to all to indicate progress. The function
        should accept params 'current' (int) and 'maximum' (int). Defaults to
        None.
    :type callback: function

    :returns: A report listing which files were downloaded and their
        download failure or success.
    :rtype: str
    """
    if callback is None:
        callback = print_progress_callback

    sg_codes_and_provinces = map_sg_codes_to_provinces(db_manager, site_layer,
                                                       diagram_layer,
                                                       sg_code_field,
                                                       all_features)
    maximum = len(sg_codes_and_provinces)
    current = 0
    report = ''
    for sg_code, province in sg_codes_and_provinces.iteritems():
        current += 1
        message = 'Downloading SG Code %s from %s' % (sg_code, province)
        callback(current, maximum, message)
        try:
            report += download_sg_diagram(db_manager, sg_code, province,
                                          output_directory, callback)
        except Exception, e:
            report += 'Failed to download %s %s %s\n' % (sg_code, province, e)
            LOGGER.exception(e)
예제 #3
0
def download_from_url(url, output_directory, filename=None, use_cache=True):
    """Download file from a url and put it under output_directory.

    :param url: Url that gives response.
    :type url: str

    :param output_directory: Directory to put the diagram.
    :type output_directory: str

    :param filename: Optional filename for downloaded file.
    :type filename: str

    :param use_cache: If there is a cached copy of the file already in the
        output directory, do not refetch it (True) or force refecth it (False).
    :type use_cache: bool

    :returns: File path if success to download, else None
    :rtype: str
    """
    if filename is None:
        filename = get_filename(url)
    LOGGER.info('Download file %s from %s' % (filename, url))
    file_path = os.path.join(output_directory, filename)
    if os.path.exists(file_path) and use_cache:
        LOGGER.info('File %s exists, not downloading' % file_path)
        return file_path

    # Set Proxy in webpage
    proxy = get_proxy()
    network_manager = QNetworkAccessManager()
    if not proxy is None:
        network_manager.setProxy(proxy)

    # Download Process
    # noinspection PyTypeChecker
    downloader = FileDownloader(network_manager, url, file_path)
    try:
        result = downloader.download()
    except IOError as ex:
        raise DownloadException(ex)

    if result[0] is not True:
        _, error_message = result
        raise DownloadException(error_message)

    if os.path.exists(file_path):
        return file_path
    else:
        return None
예제 #4
0
def download_from_url(url, output_directory, filename=None, use_cache=True):
    """Download file from a url and put it under output_directory.

    :param url: Url that gives response.
    :type url: str

    :param output_directory: Directory to put the diagram.
    :type output_directory: str

    :param filename: Optional filename for downloaded file.
    :type filename: str

    :param use_cache: If there is a cached copy of the file already in the
        output directory, do not refetch it (True) or force refecth it (False).
    :type use_cache: bool

    :returns: File path if success to download, else None
    :rtype: str
    """
    if filename is None:
        filename = get_filename(url)
    LOGGER.info('Download file %s from %s' % (filename, url))
    file_path = os.path.join(output_directory, filename)
    if os.path.exists(file_path) and use_cache:
        LOGGER.info('File %s exists, not downloading' % file_path)
        return file_path

    # Set Proxy in webpage
    proxy = get_proxy()
    network_manager = QNetworkAccessManager()
    if not proxy is None:
        network_manager.setProxy(proxy)

    # Download Process
    # noinspection PyTypeChecker
    downloader = FileDownloader(network_manager, url, file_path)
    try:
        result = downloader.download()
    except IOError as ex:
        raise DownloadException(ex)

    if result[0] is not True:
        _, error_message = result
        raise DownloadException(error_message)

    if os.path.exists(file_path):
        return file_path
    else:
        return None
예제 #5
0
def construct_url(db_manager, sg_code=None, province_name=None):
    """Construct url to download sg diagram.

    :param db_manager: A database manager
    :type db_manager: DatabaseManager

    :param sg_code: SG code.
    :type sg_code: str

    :param province_name: province_name name.
    :type province_name: str

    :returns: URL to download sg diagram.
    :rtype: str
    """
    LOGGER.info('Constructing url for %s %s' % (sg_code, province_name))
    if not is_valid_sg_code(sg_code):
        raise InvalidSGCodeException

    if sg_code is None or province_name is None:
        raise UrlException()

    if province_name not in PROVINCE_NAMES:
        raise NotInSouthAfricaException

    base_url = 'http://csg.dla.gov.za/esio/listdocument.jsp?'
    reg_division = sg_code[:8]

    try:
        record = get_office(db_manager, reg_division, province_name)
    except DatabaseException:
        raise DatabaseException

    if record is None or bool(record) is None:
        raise DatabaseException

    office, office_number, typology = record

    erf = sg_code[8:16]
    portion = sg_code[16:]
    url = base_url + 'regDivision=' + reg_division
    url += '&office=' + office
    url += '&Noffice=' + office_number
    url += '&Erf=' + erf
    url += '&Portion=' + portion
    return url
예제 #6
0
def construct_url(db_manager, sg_code=None, province_name=None):
    """Construct url to download sg diagram.

    :param db_manager: A database manager
    :type db_manager: DatabaseManager

    :param sg_code: SG code.
    :type sg_code: str

    :param province_name: province_name name.
    :type province_name: str

    :returns: URL to download sg diagram.
    :rtype: str
    """
    LOGGER.info('Constructing url for %s %s' % (sg_code, province_name))
    if not is_valid_sg_code(sg_code):
        raise InvalidSGCodeException

    if sg_code is None or province_name is None:
        raise UrlException()

    if province_name not in PROVINCE_NAMES:
        raise NotInSouthAfricaException

    base_url = 'http://csg.dla.gov.za/esio/listdocument.jsp?'
    reg_division = sg_code[:8]

    try:
        record = get_office(db_manager, reg_division, province_name)
    except DatabaseException:
        raise DatabaseException

    if record is None or bool(record) is None:
        raise DatabaseException

    office, office_number, typology = record

    erf = sg_code[8:16]
    portion = sg_code[16:]
    url = base_url + 'regDivision=' + reg_division
    url += '&office=' + office
    url += '&Noffice=' + office_number
    url += '&Erf=' + erf
    url += '&Portion=' + portion
    return url
예제 #7
0
def download_sg_diagram(
        db_manager, sg_code, province_name, output_directory, callback=None):
    """Download sg diagram using sg_code and put it under output_directory.

    :param db_manager: A database manager
    :type db_manager: DatabaseManager

    :param sg_code: Surveyor General code.
    :type sg_code: str

    :param province_name: Province name.
    :type province_name: str

    :param output_directory: Directory to put the diagram.
    :type output_directory: str

    :param callback: A function to all to indicate progress. The function
        should accept params 'current' (int) and 'maximum' (int). Defaults to
        None.
    :type callback: function

    :returns: A report listing which files were downloaded and their
        download failure or success.
    :rtype: str
    """
    if callback is None:
        callback = print_progress_callback

    report = 'Downloading documents for %s in %s\n' % (sg_code, province_name)

    try:
        download_page = construct_url(db_manager, sg_code, province_name)
    except (InvalidSGCodeException,
            DatabaseException,
            UrlException,
            NotInSouthAfricaException) as e:
        report += (
            'Failed: Downloading SG code %s for province %s because of %s\n' %
            (sg_code, province_name, e.reason))
        return report
    try:
        download_links = parse_download_page(download_page)
    except ParseException as e:
        report += (
            'Failed: Downloading SG code %s for province %s because of %s\n' %
            (sg_code, province_name, e.reason))
        return report

    output_directory = os.path.join(output_directory, sg_code)
    if not os.path.exists(output_directory):
        os.makedirs(output_directory)

    count = 0
    total = len(download_links)
    if total == 0:
        report += 'No documents found for %s in %s' % (sg_code, province_name)

    for download_link in download_links:
        count += 1
        message = ('[%s - %s] Downloading file %s of %s' % (
            sg_code, province_name, count, total))
        callback(count, total, message)
        try:
            file_path = download_from_url(download_link, output_directory)
            if file_path is not None:
                report += 'Success: File %i of %i : %s saved to %s\n' % (
                    count, total, download_link, file_path)
            else:
                report += 'Failed: File %i of %i : %s \n' % (
                    count, total, download_link)
        except DownloadException as e:
            message = 'Failed to download %s for %s in %s because %s' % (
                download_link, sg_code, province_name, e.reason)
            LOGGER.exception(message)
            report += 'Failed: File %i of %i : %s \n' % (
                count, total, download_link)

    message = 'Downloads completed for %s in %s' % (sg_code, province_name)
    callback(count, total, message)
    return report
예제 #8
0
def download_sg_diagram(db_manager,
                        sg_code,
                        province_name,
                        output_directory,
                        callback=None):
    """Download sg diagram using sg_code and put it under output_directory.

    :param db_manager: A database manager
    :type db_manager: DatabaseManager

    :param sg_code: Surveyor General code.
    :type sg_code: str

    :param province_name: Province name.
    :type province_name: str

    :param output_directory: Directory to put the diagram.
    :type output_directory: str

    :param callback: A function to all to indicate progress. The function
        should accept params 'current' (int) and 'maximum' (int). Defaults to
        None.
    :type callback: function

    :returns: A report listing which files were downloaded and their
        download failure or success.
    :rtype: str
    """
    if callback is None:
        callback = print_progress_callback

    report = 'Downloading documents for %s in %s\n' % (sg_code, province_name)

    try:
        download_page = construct_url(db_manager, sg_code, province_name)
    except (InvalidSGCodeException, DatabaseException, UrlException,
            NotInSouthAfricaException) as e:
        report += (
            'Failed: Downloading SG code %s for province %s because of %s\n' %
            (sg_code, province_name, e.reason))
        return report
    try:
        download_links = parse_download_page(download_page)
    except ParseException as e:
        report += (
            'Failed: Downloading SG code %s for province %s because of %s\n' %
            (sg_code, province_name, e.reason))
        return report

    output_directory = os.path.join(output_directory, sg_code)
    if not os.path.exists(output_directory):
        os.makedirs(output_directory)

    count = 0
    total = len(download_links)
    if total == 0:
        report += 'No documents found for %s in %s' % (sg_code, province_name)

    for download_link in download_links:
        count += 1
        message = ('[%s - %s] Downloading file %s of %s' %
                   (sg_code, province_name, count, total))
        callback(count, total, message)
        try:
            file_path = download_from_url(download_link, output_directory)
            if file_path is not None:
                report += 'Success: File %i of %i : %s saved to %s\n' % (
                    count, total, download_link, file_path)
            else:
                report += 'Failed: File %i of %i : %s \n' % (count, total,
                                                             download_link)
        except DownloadException as e:
            message = 'Failed to download %s for %s in %s because %s' % (
                download_link, sg_code, province_name, e.reason)
            LOGGER.exception(message)
            report += 'Failed: File %i of %i : %s \n' % (count, total,
                                                         download_link)

    message = 'Downloads completed for %s in %s' % (sg_code, province_name)
    callback(count, total, message)
    return report