def test_write_external_data():
    expected_dump_id = "foobar"
    expected_value_triple = ("foobar", "P42", ["foobar"])

    gnd_converter = GndDumpConverter(False)
    xml_dump_converter_mock = XmlDumpConverter(None, None, None)
    def process_dump_mock(dump_file):
        yield expected_value_triple
    xml_dump_converter_mock.process_dump = process_dump_mock
    gnd_converter.xml_dump_converter = xml_dump_converter_mock
    result_mock = ResultWriter(StringIO(), StringIO())
    def write_external_value_mock(dump_id, external_id,
                                  property_id, external_value):
        assert expected_dump_id == dump_id
        assert expected_value_triple[0] == external_id
        assert expected_value_triple[1] == property_id
        assert external_value in expected_value_triple[2]
    result_mock.write_external_value = write_external_value_mock

    gnd_converter.write_external_data(expected_dump_id, None, result_mock)
class GndDumpConverter:
    """
    Dump converter for dumps of the Integrated Authority File (GND) of the
    German National Library. Downloads latest dump files, processes entities and
    writes values to tar file.
    """
    DATA_SOURCE_ITEM_ID = "Q36578"
    IDENTIFIER_PROPERTY_ID = "P227"
    LANGUAGE = "de"
    LICENSE_ITEM_ID = "Q6938433"

    FILE_PREFIXES = {
        "GND-Tpgesamt": "Tpgesamt",
        "GND-Tggesamt": "Tggesamt",
        "GND-Tugesamt": "Tugesamt"
    }
    URL_FORMAT = "http://datendienst.dnb.de/cgi-bin/mabit.pl?cmd=fetch&userID=GNDxml&pass=gndmarcxml{0}{1}&mabheft={2}{0}{3}gndmrc.xml.gz"

    XML_ENTITIES_PATH = "ns:collection/ns:record"
    XML_ENTITY_ID_XPATH = "substring-after(ns:datafield[@tag='035']/ns:subfield[@code='a' and starts-with(./text(), '(DE-588)')], '(DE-588)')"
    XML_NAMESPACES = {
        "ns": "http://www.loc.gov/MARC21/slim"
    }

    def __init__(self, is_quiet=False):
        """
        Creates new GndDumpConverter instance.
        :param is_quiet: If set to True, console output will be suppressed.
        """
        self.is_quiet = is_quiet
        self.xml_dump_converter = XmlDumpConverter(self.XML_ENTITIES_PATH,
                                                   self.XML_ENTITY_ID_XPATH,
                                                   propertymappings.mapping,
                                                   self.XML_NAMESPACES,
                                                   is_quiet)

    def execute(self, result_writer):
        """
        Starts whole convert process.
        :param result_writer: Writer for output of result.
        """
        for dump_id, file_prefix in self.FILE_PREFIXES.iteritems():
            if not self.is_quiet:
                print "Start to convert '{0}'".format(file_prefix)

            dump_file, dump_url, dump_size = self.download_dump(file_prefix)

            uncompressed_dump_file = GzipFile(mode="rb", fileobj=dump_file)

            self.write_external_data(
                dump_id,
                uncompressed_dump_file,
                result_writer)
            result_writer.write_dump_information(
                dump_id,
                self.DATA_SOURCE_ITEM_ID,
                [self.IDENTIFIER_PROPERTY_ID],
                self.LANGUAGE,
                dump_url,
                dump_size,
                self.LICENSE_ITEM_ID)

            uncompressed_dump_file.close()
            dump_file.close()

            if not self.is_quiet:
                print

    def get_dump_url(self, file_prefix, fallback=False, date=datetime.date.today()):
        """
        Returns url of the latest dump with specified prefix.
        If dump file does not exist, fallback option can be set True to
        build url of previous dump.
        This will be applicable, if new dump should be already available,
        but was not published yet.
        :param file_prefix: Prefix of the dump file.
        :param fallback: If set to True, url of previous dump will be returned.
        :param date: Datetime, on which the url generation is based on.
        :return: Url of the dump
        """
        if fallback:
            previous_month = date.month - 4
            previous_year = date.year
            if previous_month <= 0:
                previous_month = (date.month - 4) % 12
                if previous_month == 0:
                    previous_month = 12
                previous_year -= 1
            date = datetime.date(previous_year, previous_month, 1)

        year = date.year
        if date.month == 1:
            index = 3
            month = "10"
            year -= 1
        elif date.month < 6:
            index = 1
            month = "02"
        elif date.month < 10:
            index = 2
            month = "06"
        else:
            index = 3
            month = "10"
        year = str(year)[-2:]

        return self.URL_FORMAT.format(year, index, file_prefix, month)

    def download_dump(self, file_prefix):
        """
        Downloads a dump identified by file prefix to destination file.
        :param file_prefix: Prefix of the dump file.
        :return: List of file object, url and size of downloaded file.
        """
        dump_file = TemporaryFile()
        dump_url = self.get_dump_url(file_prefix)
        try:
            dump_size = downloadutils.download_file(dump_url, dump_file,
                                                    is_quiet=self.is_quiet,
                                                    progress_message="Downloading database dump...{0}")
        except DownloadError as e:
            if e.status_code == 400 or e.status_code == 500:
                dump_url = self.get_dump_url(file_prefix, fallback=True)
                dump_size = downloadutils.download_file(dump_url, dump_file,
                                                        is_quiet=self.is_quiet,
                                                        progress_message="Downloading database dump...{0}")
            else:
                raise

        return dump_file, dump_url, dump_size

    def write_external_data(self, dump_id, dump_file, result_writer):
        """
        Processes dump and writes external values to file.
        :param dump_id: Id of the processing dump.
        :param dump_file: File object of the dump.
        :param result_writer: Current result writer.
        """
        external_data = self.xml_dump_converter.process_dump(dump_file)
        for external_id, property_id, external_values in external_data:
            for external_value in external_values:
                result_writer.write_external_value(dump_id, external_id,
                                                   property_id, external_value)