示例#1
0
def test_process_records_simplify(changes):
    """Test if queue entries for the same object are properly simplified"""
    refs = (
        LiveSyncQueueEntry(type=EntryType.event, event_id=1).object_ref,
        LiveSyncQueueEntry(type=EntryType.event, event_id=2).object_ref
    )
    queue = []
    changes = changes[:3], changes[3:]
    expected = [0, 0]
    for i, ref in enumerate(refs):
        if changes[i][0]:
            queue.append(LiveSyncQueueEntry(change=ChangeType.created, **ref))
            expected[i] |= SimpleChange.created
        if changes[i][1]:
            queue.append(LiveSyncQueueEntry(change=ChangeType.data_changed, **ref))
            queue.append(LiveSyncQueueEntry(change=ChangeType.data_changed, **ref))
            expected[i] |= SimpleChange.updated
        if changes[i][2]:
            queue.append(LiveSyncQueueEntry(change=ChangeType.deleted, **ref))
            expected[i] |= SimpleChange.deleted

    result = process_records(queue)
    assert result == process_records(reversed(queue))  # queue order shouldn't matter
    assert len(result) == sum(1 for x in expected if x)
    for i, ref in enumerate(refs):
        assert (ref in result) == bool(expected[i])
        assert result[ref] == expected[i]
def test_process_records_cascade(mocker, change, cascade):
    """Test if certain changes cascade to child elements"""
    cascade_mock = mocker.patch(
        'indico_livesync.simplify._process_cascaded_category_contents')
    records = [LiveSyncQueueEntry(change=change)]
    process_records(records)
    assert cascade_mock.call_args == (({records[0]} if cascade else set(), ), )
def test_process_records_simplify(changes, mocker, db, create_event,
                                  dummy_agent):
    """Test if queue entries for the same object are properly simplified"""
    event1 = create_event(id_=1)
    event2 = create_event(id_=2)

    db.session.add(dummy_agent)
    db.session.add(event1)
    db.session.add(event2)

    refs = ({
        'type': EntryType.event,
        'event_id': event1.id
    }, {
        'type': EntryType.event,
        'event_id': event2.id
    })

    queue = []
    changes = changes[:3], changes[3:]
    expected = [0, 0]
    for i, ref in enumerate(refs):
        if changes[i][0]:
            queue.append(
                LiveSyncQueueEntry(change=ChangeType.created,
                                   agent=dummy_agent,
                                   **ref))
            expected[i] |= SimpleChange.created
        if changes[i][1]:
            queue += [
                LiveSyncQueueEntry(change=ChangeType.data_changed,
                                   agent=dummy_agent,
                                   **ref),
                LiveSyncQueueEntry(change=ChangeType.data_changed,
                                   agent=dummy_agent,
                                   **ref)
            ]
            expected[i] |= SimpleChange.updated
        if changes[i][2]:
            queue.append(
                LiveSyncQueueEntry(change=ChangeType.deleted,
                                   agent=dummy_agent,
                                   **ref))
            expected[i] |= SimpleChange.deleted

    db.session.flush()

    result = process_records(queue)
    assert result == process_records(
        reversed(queue))  # queue order shouldn't matter
    assert len(result) == sum(1 for x in expected if x)

    result_refs = {obj.id: change for obj, change in result.viewitems()}
    for i, ref in enumerate(refs):
        assert (ref['event_id'] in list(result_refs)) == bool(expected[i])
        assert result_refs.get(ref['event_id'], 0) == expected[i]
示例#4
0
def test_process_records_category_ignored(mocker, change, invalid):
    """Test if categories are only kepy for certain changes"""
    cascade = mocker.patch('indico_livesync.simplify._process_cascaded_category_contents')
    cascade.return_value = [object()]
    records = [LiveSyncQueueEntry(change=change, type=EntryType.category)]
    if invalid:
        with pytest.raises(AssertionError):
            process_records(records)
    else:
        result = process_records(records)
        assert len(result) == 1
        assert result.values()[0] == SimpleChange.updated
示例#5
0
    def run(self, records):
        """Runs the batch upload

        :param records: an iterable containing queue entries
        """
        self_name = type(self).__name__
        for i, batch in enumerate(
                grouper(records, self.BATCH_SIZE, skip_missing=True), 1):
            self.logger.info('%s processing batch %d', self_name, i)
            try:
                for j, proc_batch in enumerate(
                        grouper(process_records(batch).iteritems(),
                                self.BATCH_SIZE,
                                skip_missing=True), 1):
                    self.logger.info('%s uploading chunk #%d (batch %d)',
                                     self_name, j, i)
                    self.upload_records({k: v
                                         for k, v in proc_batch},
                                        from_queue=True)
            except Exception:
                self.logger.exception('%s could not upload batch', self_name)
                return
            self.logger.info('%s finished batch %d', self_name, i)
            self.processed_records(batch)
        self.logger.info('%s finished', self_name)
示例#6
0
def test_process_records_simplify(changes, mocker, db, create_event, dummy_agent):
    """Test if queue entries for the same object are properly simplified"""
    event1 = create_event(id_=1)
    event2 = create_event(id_=2)

    db.session.add(dummy_agent)
    db.session.add(event1)
    db.session.add(event2)

    refs = (
        {'type': EntryType.event, 'event_id': event1.id},
        {'type': EntryType.event, 'event_id': event2.id}
    )

    queue = []
    changes = changes[:3], changes[3:]
    expected = [0, 0]
    for i, ref in enumerate(refs):
        if changes[i][0]:
            queue.append(LiveSyncQueueEntry(change=ChangeType.created, agent=dummy_agent, **ref))
            expected[i] |= SimpleChange.created
        if changes[i][1]:
            queue += [LiveSyncQueueEntry(change=ChangeType.data_changed, agent=dummy_agent, **ref),
                      LiveSyncQueueEntry(change=ChangeType.data_changed, agent=dummy_agent, **ref)]
            expected[i] |= SimpleChange.updated
        if changes[i][2]:
            queue.append(LiveSyncQueueEntry(change=ChangeType.deleted, agent=dummy_agent, **ref))
            expected[i] |= SimpleChange.deleted

    db.session.flush()

    result = process_records(queue)
    assert result == process_records(reversed(queue))  # queue order shouldn't matter
    assert len(result) == sum(1 for x in expected if x)

    result_refs = {obj.id: change for obj, change in result.viewitems()}
    for i, ref in enumerate(refs):
        assert (ref['event_id'] in list(result_refs)) == bool(expected[i])
        assert result_refs.get(ref['event_id'], 0) == expected[i]
示例#7
0
    def run(self, records):
        """Runs the batch upload

        :param records: an iterable containing queue entries
        """
        self_name = type(self).__name__
        for i, batch in enumerate(grouper(records, self.BATCH_SIZE, skip_missing=True), 1):
            self.logger.info('%s processing batch %d', self_name, i)
            try:
                for j, proc_batch in enumerate(grouper(
                        process_records(batch).iteritems(), self.BATCH_SIZE, skip_missing=True), 1):
                    self.logger.info('%s uploading chunk #%d (batch %d)', self_name, j, i)
                    self.upload_records({k: v for k, v in proc_batch}, from_queue=True)
            except Exception:
                self.logger.exception('%s could not upload batch', self_name)
                return
            self.logger.info('%s finished batch %d', self_name, i)
            self.processed_records(batch)
        self.logger.info('%s finished', self_name)
示例#8
0
    def run(self):
        records = self.fetch_records()
        if not records:
            self._print(cformat('%{yellow!}No records%{reset}'))
            return

        self._print(cformat('%{white!}Raw changes:%{reset}'))
        for record in records:
            self._print(record)

        self._print()
        self._print(cformat('%{white!}Simplified/cascaded changes:%{reset}'))
        for obj, change in process_records(records).items():
            self._print(cformat('%{white!}{}%{reset}: {}').format(_change_str(change), obj))

        self._print()
        self._print(cformat('%{white!}Resulting MarcXML:%{reset}'))
        uploader = DebugUploader(self)
        uploader.run(records)
        self.update_last_run()
示例#9
0
    def run(self):
        records = self.fetch_records()
        if not records:
            self._print(cformat('%{yellow!}No records%{reset}'))
            return

        self._print(cformat('%{white!}Raw changes:%{reset}'))
        for record in records:
            self._print(record)

        self._print()
        self._print(cformat('%{white!}Simplified/cascaded changes:%{reset}'))
        for obj, change in process_records(records).iteritems():
            self._print(cformat('%{white!}{}%{reset}: {}').format(_change_str(change), obj))

        self._print()
        self._print(cformat('%{white!}Resulting MarcXML:%{reset}'))
        uploader = DebugUploader(self)
        uploader.run(records)
        self.update_last_run()
示例#10
0
    def process_queue(self, uploader, allowed_categories=()):
        records = self.fetch_records(allowed_categories)
        if not records:
            print(cformat('%{yellow!}No records%{reset}'))
            return

        print(cformat('%{white!}Raw changes:%{reset}'))
        for record in records:
            print(record)

        print()
        print(cformat('%{white!}Simplified/cascaded changes:%{reset}'))
        for obj, change in process_records(records).items():
            print(
                cformat('%{white!}{}%{reset}: {}').format(
                    _change_str(change), obj))

        print()
        print(cformat('%{white!}Resulting records:%{reset}'))
        uploader._is_queue_run = True
        uploader.run(records)
        self.update_last_run()
示例#11
0
def test_process_records_cascade(mocker, change, cascade):
    """Test if certain changes cascade to child elements"""
    cascade_mock = mocker.patch('indico_livesync.simplify._process_cascaded_category_contents')
    records = [LiveSyncQueueEntry(change=change)]
    process_records(records)
    assert cascade_mock.call_args == (({records[0]} if cascade else set(),),)
示例#12
0
def test_process_records_cascade(mocker, change, cascade):
    """Test if certain changes cascade to child elements"""
    cascade_mock = mocker.patch('indico_livesync.simplify._cascade')
    records = [LiveSyncQueueEntry(change=change)]
    process_records(records)
    assert cascade_mock.called == cascade