Beispiel #1
0
    def test_connection_error(self, emails):
        sch = RemoteScheduler(host="this_host_doesnt_exist",
                              port=1337,
                              connect_timeout=1)
        worker = Worker(scheduler=sch)

        self.waits = 0

        def dummy_wait():
            self.waits += 1

        sch._wait = dummy_wait

        class A(DummyTask):
            pass

        a = A()
        self.assertEqual(emails, [])
        worker.add(a)
        self.assertEqual(self.waits, 2)  # should attempt to add it 3 times
        self.assertNotEquals(emails, [])
        self.assertTrue(
            emails[0].find("Luigi: Framework error while scheduling %s" %
                           (a, )) != -1)
        worker.stop()
    def test_workflow(self):
        # set up directories:
        src_path = os.path.join(self.temp_rootdir, "src")
        os.mkdir(src_path)
        counts_path = os.path.join(self.temp_rootdir, "counts")
        os.mkdir(counts_path)
        report_path = os.path.join(self.temp_rootdir, "report.csv")
        data_filepath = os.path.join(self.temp_rootdir, "geoloc.dat")
        with open(data_filepath, 'w') as data_file:
            data_file.write("Dummy geolocation data.")

        # create input:
        log_filepath = os.path.join(src_path, "tracking.log")
        with open(log_filepath, 'w') as log_file:
            log_file.write(self._create_event_log_line())
            log_file.write('\n')
            log_file.write(
                self._create_event_log_line(username="******",
                                            ip=FakeGeoLocation.ip_address_2))
            log_file.write('\n')

        end_date = '2014-04-01'
        task = UsersPerCountryReportWorkflow(
            mapreduce_engine='local',
            name='test',
            src=[src_path],
            end_date=datetime.datetime.strptime(end_date, '%Y-%m-%d').date(),
            geolocation_data=data_filepath,
            counts=counts_path,
            report=report_path,
        )
        worker = luigi.worker.Worker()
        worker.add(task)
        with patch(
                'edx.analytics.tasks.user_location.pygeoip') as mock_pygeoip:
            mock_pygeoip.GeoIP = Mock(return_value=FakeGeoLocation())
            worker.run()
        worker.stop()

        output_lines = []
        with open(report_path) as report_file:
            output_lines = report_file.readlines()

        self.assertEquals(len(output_lines), 3)
        self.assertEquals(output_lines[0].strip('\n'),
                          UsersPerCountryReport.create_header(end_date))
        expected = UsersPerCountryReport.create_csv_entry(
            0.5, 1, FakeGeoLocation.country_name_1,
            FakeGeoLocation.country_code_1)
        self.assertEquals(output_lines[1].strip('\n'), expected)
        expected = UsersPerCountryReport.create_csv_entry(
            0.5, 1, FakeGeoLocation.country_name_2,
            FakeGeoLocation.country_code_2)
        self.assertEquals(output_lines[2].strip('\n'), expected)
    def test_workflow(self):
        # set up directories:
        src_path = os.path.join(self.temp_rootdir, "src")
        os.mkdir(src_path)
        counts_path = os.path.join(self.temp_rootdir, "counts")
        os.mkdir(counts_path)
        report_path = os.path.join(self.temp_rootdir, "report.csv")
        data_filepath = os.path.join(self.temp_rootdir, "geoloc.dat")
        with open(data_filepath, 'w') as data_file:
            data_file.write("Dummy geolocation data.")

        # create input:
        log_filepath = os.path.join(src_path, "tracking.log")
        with open(log_filepath, 'w') as log_file:
            log_file.write(self._create_event_log_line())
            log_file.write('\n')
            log_file.write(self._create_event_log_line(username="******", ip=FakeGeoLocation.ip_address_2))
            log_file.write('\n')

        end_date = '2014-04-01'
        task = UsersPerCountryReportWorkflow(
            mapreduce_engine='local',
            name='test',
            src=[src_path],
            end_date=datetime.datetime.strptime(end_date, '%Y-%m-%d').date(),
            geolocation_data=data_filepath,
            counts=counts_path,
            report=report_path,
        )
        worker = luigi.worker.Worker()
        worker.add(task)
        with patch('edx.analytics.tasks.user_location.pygeoip') as mock_pygeoip:
            mock_pygeoip.GeoIP = Mock(return_value=FakeGeoLocation())
            worker.run()
        worker.stop()

        output_lines = []
        with open(report_path) as report_file:
            output_lines = report_file.readlines()

        self.assertEquals(len(output_lines), 3)
        self.assertEquals(output_lines[0].strip('\n'), UsersPerCountryReport.create_header(end_date))
        expected = UsersPerCountryReport.create_csv_entry(
            0.5, 1, FakeGeoLocation.country_name_1, FakeGeoLocation.country_code_1
        )
        self.assertEquals(output_lines[1].strip('\n'), expected)
        expected = UsersPerCountryReport.create_csv_entry(
            0.5, 1, FakeGeoLocation.country_name_2, FakeGeoLocation.country_code_2
        )
        self.assertEquals(output_lines[2].strip('\n'), expected)
Beispiel #4
0
    def test_connection_error(self):
        sch = RemoteScheduler(host="this_host_doesnt_exist", port=1337)
        worker = Worker(scheduler=sch)

        self.waits = 0

        def dummy_wait():
            self.waits += 1

        sch._wait = dummy_wait

        class A(DummyTask):
            pass

        a = A()
        self.assertEquals(self.last_email, None)
        worker.add(a)
        self.assertEquals(self.waits, 2)  # should attempt to add it 3 times
        self.assertNotEquals(self.last_email, None)
        self.assertEquals(self.last_email[0], "Luigi: Framework error while scheduling %s" % (a,))
        worker.stop()
Beispiel #5
0
 def _run(self, worker):
     worker.run()
     worker.stop()
     self.finish()