def test_insert_record(self, uow):
        syrx_file_handler = SyrxFileHandler()
        records = Mock()

        syrx_file_handler.insert_records(records)

        uow.return_value.energy_records.insert_equipment_point_records.assert_called_with(records)
    def test_get_equipment_point_records_successful(self, get_weather_data, populate_weatherstation_id_on_records, uow):
        syrx_file_handler = SyrxFileHandler()
        syrx_file_handler.date_time = Mock()
        syrx_file_handler.date_time_str = Mock()

        weather_data = {"ws1": {pytz.utc.localize(datetime.datetime(2014, 9, 8, 0, 0, 0)): "weather_record_1",
                                pytz.utc.localize(datetime.datetime(2014, 9, 8, 13, 0, 0)): "weather_record_2",
                                pytz.utc.localize(datetime.datetime(2014, 8, 8, 13, 0, 0)): "weather_record_3"}}
        get_weather_data.return_value = weather_data

        records = [{"value": 100, "timestamp": "2014-09-08 00:15:00", "weatherstation_id": "ws1", "syrx_num": "400000-0001-237323-EP-001"},
                   {"value": 100, "timestamp": "2014-09-08 13:00:00", "weatherstation_id": "ws1", "syrx_num": "400000-0001-237323-EP-001"},
                   {"value": 100, "timestamp": "2014-08-08 13:00:00", "weatherstation_id": "ws1", "syrx_num": "400000-0001-237323-EP-001"}]

        rv = syrx_file_handler.get_equipment_point_records(records)

        populate_weatherstation_id_on_records.assert_called_with(records)
        get_weather_data.assert_called_with(records)

        assert len(rv.equipment_point_records) == 3
        assert len(rv.bad_records) == 0

        uow.return_value.energy_records.get_equipment_point_record.assert_has_calls([
            call(pytz.utc.localize(datetime.datetime(2014, 9, 8, 0, 15, 0)), "400000-0001-237323-EP-001", 100,
                 "weather_record_1", syrx_file_handler.date_time),
            call(pytz.utc.localize(datetime.datetime(2014, 9, 8, 13, 0, 0)), "400000-0001-237323-EP-001", 100,
                 "weather_record_2", syrx_file_handler.date_time),
            call(pytz.utc.localize(datetime.datetime(2014, 8, 8, 13, 0, 0)), "400000-0001-237323-EP-001", 100,
                 "weather_record_3", syrx_file_handler.date_time)

        ])
    def test_process_johnson_records(self, get_record_ranges, insert_records, delete_old_records,
                                     get_equipment_point_records, uow):
        syrx_file_handler = SyrxFileHandler()

        equipment_point_records = EquipmentPointRecordsReturn()
        equipment_point_records.equipment_point_records = [
            {"name": "record0"},
            {"name": "record1"}
        ]
        equipment_point_records.bad_records = [
            {"name": "record2"},
            {"name": "record3"}
        ]
        get_equipment_point_records.return_value = equipment_point_records

        records = Mock()
        bad_record_file = io.StringIO()

        rv = syrx_file_handler.process_syrx_records(records, bad_record_file)

        get_equipment_point_records.assert_called_with(records)
        delete_old_records.assert_called_with(equipment_point_records.equipment_point_records)
        insert_records.assert_called_with(equipment_point_records.equipment_point_records)

        bad_record_file.seek(0)
        bad_records_written_data = bad_record_file.read().strip().split("\n")
        assert len(bad_records_written_data) == 2
        assert json.loads(bad_records_written_data[0]) == {"name": "record2"}
        assert json.loads(bad_records_written_data[1]) == {"name": "record3"}

        get_record_ranges.assert_called_with(equipment_point_records.equipment_point_records)

        assert rv.num_good_records == 2
        assert rv.num_bad_records == 2
        assert rv.record_ranges == get_record_ranges.return_value
    def test_process_syrx_file_mass_records(self, uow):
        syrx_file_handler = SyrxFileHandler()
        syrx_file_handler.logger = Mock()
        syrx_file_handler.date_str = Mock()

        bad_record_file_mock = MagicMock()

        summary = ProcessSyrxRecordsReturn()
        summary.num_good_records = 200
        summary.num_bad_records = 0
        syrx_file_handler.process_syrx_records = Mock(return_value=summary)

        read_file_stream = io.StringIO()
        for i in range(400):
            read_file_stream.write(u'{"name": "record"}\n')

        # add extra record to force small batch for good measure
        read_file_stream.write(u'{"name": "record"}')
        read_file_stream.seek(0)

        parsed_list = [{"name": "record"} for i in range(200)]

        rv = syrx_file_handler.process_syrx_file(read_file_stream, bad_record_file_mock)

        syrx_file_handler.process_syrx_records.assert_has_calls([call(parsed_list,  bad_record_file_mock),
                                                                 call(parsed_list, bad_record_file_mock),
                                                                 call([{"name": "record"}], bad_record_file_mock)])
        assert rv.num_good_records == 600
    def test_get_unique_syrx_nums(self, uow):
        syrx_file_handler = SyrxFileHandler()

        records = [{"syrx_num": "syrx_num_1"}, {"syrx_num": "syrx_num_1"}, {"syrx_num": "syrx_num_2"}]

        rv = syrx_file_handler.get_unique_syrx_nums(records)

        assert rv == ["syrx_num_1", "syrx_num_2"]
    def test_get_weather_data(self, get_weather_data_records, get_weatherstation_years_to_retrieve, uow):
        syrx_file_handler = SyrxFileHandler()
        records = Mock()

        rv = syrx_file_handler.get_weather_data(records)

        get_weatherstation_years_to_retrieve.assert_called_with(records)
        get_weather_data_records.assert_called_with(get_weatherstation_years_to_retrieve.return_value)
        assert rv == get_weather_data_records.return_value
    def test_delete_old_record(self, uow):
        syrx_file_handler = SyrxFileHandler()

        records = [{"syrx_num": "syrx_num_1", "date": "date_1"},
                   {"syrx_num": "syrx_num_2", "date": "date_2"},
                   {"syrx_num": "syrx_num_3", "date": "date_3"}]

        record_keys = [[r["syrx_num"], r["date"]] for r in records]

        syrx_file_handler.delete_old_records(records)

        uow.return_value.energy_records.delete_equipment_point_records.assert_called_with(record_keys)
    def test_get_weatherstation_years_to_retrieve(self, uow):
        syrx_file_handler = SyrxFileHandler()

        records = [{"weatherstation_id": "ws1", "timestamp": "2014-09-08 13:15"},
                   {"weatherstation_id": "ws1", "timestamp": "2014-10-08 13:15"},
                   {"weatherstation_id": "ws1", "timestamp": "2015-10-08 13:15"},
                   {"weatherstation_id": "ws2", "timestamp": "2015-10-08 13:15"},
                   {"weatherstation_id": "ws3", "timestamp": None}]

        rv = syrx_file_handler.get_weatherstation_years_to_retrieve(records)

        assert rv["ws1"] == {2014, 2015}
        assert rv["ws2"] == {2015}
        assert rv["ws3"] == set()
    def test_flatten_record_ranges(self, uow):
        syrx_file_handler = SyrxFileHandler()

        records = [RecordRange("syrx_num_1", datetime.datetime(2012, 1, 1), datetime.datetime(2012, 3, 1)),
                   RecordRange("syrx_num_1", datetime.datetime(2012, 4, 1), datetime.datetime(2012, 6, 1)),
                   RecordRange("syrx_num_2", datetime.datetime(2012, 7, 1), datetime.datetime(2012, 9, 1)),
                   RecordRange("syrx_num_2", datetime.datetime(2012, 10, 1), datetime.datetime(2012, 12, 1))]

        rv = syrx_file_handler.flatten_record_ranges(records)

        assert rv[0].syrx_num == "syrx_num_1"
        assert rv[0].start_date == datetime.datetime(2012, 1, 1)
        assert rv[0].end_date == datetime.datetime(2012, 6, 1)

        assert rv[1].syrx_num == "syrx_num_2"
        assert rv[1].start_date == datetime.datetime(2012, 7, 1)
        assert rv[1].end_date == datetime.datetime(2012, 12, 1)
    def test_process_syrx_file_path_no_bad_records(self, move, remove, open_mock, path_join, named_temporary_file, uow):
        syrx_file_handler = SyrxFileHandler()
        syrx_file_handler.logger = Mock()
        syrx_file_handler.syrx_upload_folder = Mock()
        syrx_file_handler.date_str = Mock()

        summary = ProcessSyrxRecordsReturn()
        summary.num_bad_records = 0
        syrx_file_handler.process_syrx_file = Mock(return_value=summary)

        file_path = MagicMock()

        bad_record_file_mock = named_temporary_file.return_value
        read_file_mock = MagicMock()

        def open_mock_side_effect(path, mode):
            if path == file_path and mode == "r":
                return read_file_mock
            else:
                return None
        open_mock.side_effect = open_mock_side_effect

        syrx_file_handler.process_syrx_file_path(file_path)

        named_temporary_file.assert_called_with(delete=False)
        syrx_file_handler.process_syrx_file.assert_called_with(read_file_mock.__enter__.return_value,
                                                               bad_record_file_mock)

        remove.assert_has_calls([call(file_path), call(bad_record_file_mock.name)])
        bad_record_file_mock.close.assert_called_with()
    def test_get_record_ranges(self, uow):
        syrx_file_handler = SyrxFileHandler()

        records = [{"syrx_num": "syrx_num_1", "date": datetime.datetime(2012, 1, 1)},
                   {"syrx_num": "syrx_num_1", "date": datetime.datetime(2012, 2, 1)},
                   {"syrx_num": "syrx_num_1", "date": datetime.datetime(2012, 3, 1)},
                   {"syrx_num": "syrx_num_2", "date": datetime.datetime(2012, 2, 1)},
                   {"syrx_num": "syrx_num_2", "date": datetime.datetime(2012, 3, 1)}]

        rv = syrx_file_handler.get_record_ranges(records)

        assert rv[0].syrx_num == "syrx_num_1"
        assert rv[0].start_date == datetime.datetime(2012, 1, 1)
        assert rv[0].end_date == datetime.datetime(2012, 3, 1)

        assert rv[1].syrx_num == "syrx_num_2"
        assert rv[1].start_date == datetime.datetime(2012, 2, 1)
        assert rv[1].end_date == datetime.datetime(2012, 3, 1)
    def test_get_weather_data_records(self, uow):
        syrx_file_handler = SyrxFileHandler()

        weatherstation_years_to_retrieve = {"ws1": {2014, 2015}, "ws2": {2015}}
        jan_1_2014 = pytz.utc.localize(datetime.datetime(2014, 1, 1))
        jan_1_2015 = pytz.utc.localize(datetime.datetime(2015, 1, 1))

        def get_noaa_records_side_effect(weatherstation_id, years):
            return [{"weatherstation_id": weatherstation_id,
                     "datetimeutc": pytz.utc.localize(datetime.datetime(y, 1, 1))}
                    for y in years]

        uow.return_value.weather_stations.get_noaa_records.side_effect = get_noaa_records_side_effect

        rv = syrx_file_handler.get_weather_data_records(weatherstation_years_to_retrieve)

        assert rv["ws1"] == {jan_1_2014: {"weatherstation_id": "ws1", "datetimeutc": jan_1_2014},
                             jan_1_2015: {"weatherstation_id": "ws1", "datetimeutc": jan_1_2015}}
        assert rv["ws2"] == {jan_1_2015: {"weatherstation_id": "ws2", "datetimeutc": jan_1_2015}}
    def test_process_syrx_file(self, uow):
        syrx_file_handler = SyrxFileHandler()
        syrx_file_handler.logger = Mock()
        syrx_file_handler.syrx_upload_folder = Mock()
        syrx_file_handler.date_str = Mock()

        bad_record_file_mock = MagicMock()

        summary = ProcessSyrxRecordsReturn()
        summary.num_good_records = 3
        summary.num_bad_records = 1
        syrx_file_handler.process_syrx_records = Mock(return_value=summary)

        read_file_stream = io.StringIO()
        read_file_stream.write(u'{"name": "record0"}\n'
                               u'{"name": "record1"}\n'
                               u'{"name": "record2"}\n'
                               u'{"name": "record3"}')
        read_file_stream.seek(0)

        parsed_list = [{"name": "record0"}, {"name": "record1"}, {"name": "record2"}, {"name": "record3"}]

        rv = syrx_file_handler.process_syrx_file(read_file_stream, bad_record_file_mock)

        syrx_file_handler.process_syrx_records.assert_called_with(parsed_list, bad_record_file_mock)
        assert rv.num_good_records == 3 and rv.num_bad_records == 1
    def test_get_equipment_point_records_missing_weather_data(self, get_weather_data, populate_weatherstation_id_on_records, uow):
        syrx_file_handler = SyrxFileHandler()
        syrx_file_handler.date_time = Mock()
        syrx_file_handler.date_time_str = Mock()

        weather_data = {"ws1": {pytz.utc.localize(datetime.datetime(2014, 8, 8, 0, 0, 0)): "weather_record_1"}}
        get_weather_data.return_value = weather_data

        records = [{"value": 100, "timestamp": "2014-09-08 00:15:00", "weatherstation_id": "ws1", "syrx_num": "400000-0001-237323-EP-001"}]

        rv = syrx_file_handler.get_equipment_point_records(records)

        populate_weatherstation_id_on_records.assert_called_with(records)
        get_weather_data.assert_called_with(records)

        assert len(rv.equipment_point_records) == 0
        assert len(rv.bad_records) == 1

        assert rv.bad_records[0] == {"value": 100, "timestamp": "2014-09-08 00:15:00", "weatherstation_id": "ws1",
                                     "syrx_num": "400000-0001-237323-EP-001",
                                     "error": {"date": syrx_file_handler.date_time_str,
                                               "messages": ['Time 2014-09-08 00:00:00 not found in weather data']}}
    def test_process_johnson_filename_exception(self, path_join, uow):
        syrx_file_handler = SyrxFileHandler()
        syrx_file_handler.logger = Mock()
        syrx_file_handler.syrx_upload_folder = Mock()
        syrx_file_handler.date_str = Mock()
        syrx_file_handler.process_syrx_file_path = Mock(side_effect=Exception())

        file_name = Mock()

        syrx_file_handler.process_syrx_filename(0, file_name, 1)
        path_join.assert_called_with(syrx_file_handler.syrx_upload_folder, file_name)
        syrx_file_handler.process_syrx_file_path.assert_called_with(path_join.return_value)
        assert syrx_file_handler.logger.exception.call_count == 1
    def test_compile_equipment_point_records(self, energy_record_compiler, uow):
        syrx_file_handler = SyrxFileHandler()

        range1 = RecordRange()
        range1.syrx_num = "syrx_num_1"
        range1.start_date = datetime.datetime(2012, 1, 1)
        range1.end_date = datetime.datetime(2012, 3, 1)
        range2 = RecordRange()
        range2.syrx_num = "syrx_num_2"
        range2.start_date = datetime.datetime(2012, 2, 1)
        range2.end_date = datetime.datetime(2012, 3, 1)
        ranges = [range1, range2]

        syrx_file_handler.compile_equipment_point_records(ranges)

        compiler = energy_record_compiler.return_value

        uow.return_value.compiled_energy_records.delete_compiled_equipment_point_records\
            .assert_has_calls([call(range1.syrx_num, range1.start_date, range1.end_date),
                               call(range2.syrx_num, range2.start_date, range2.end_date)])

        compiler.compile_component_point_records_by_year_span\
            .assert_has_calls([call(range1.syrx_num, range1.start_date, range1.end_date),
                               call(range2.syrx_num, range2.start_date, range2.end_date)])
    def test_process_syrx_filename(self, path_join, uow):
        syrx_file_handler = SyrxFileHandler()
        syrx_file_handler.logger = Mock()
        syrx_file_handler.syrx_upload_folder = Mock()
        syrx_file_handler.process_syrx_file_path = Mock()

        file_name = Mock()

        syrx_file_handler.process_syrx_filename(0, file_name, 1)
        path_join.assert_called_with(syrx_file_handler.syrx_upload_folder, file_name)
        syrx_file_handler.process_syrx_file_path.assert_called_with(path_join.return_value)
        assert syrx_file_handler.process_syrx_file_path.call_count == 1
    def test_process_all_syrx_files(self, listdir, uow):
        syrx_file_handler = SyrxFileHandler()
        syrx_file_handler.logger = Mock()
        syrx_file_handler.syrx_upload_folder = Mock()
        syrx_file_handler.process_syrx_filename = Mock()
        syrx_file_handler.process_syrx_filename.return_value = []
        syrx_file_handler.compile_equipment_point_records = Mock()
        listdir.return_value = ["file1", "file2", "file3", "file4", "file5", "file6"]

        syrx_file_handler.process_all_syrx_files()
        syrx_file_handler.process_syrx_filename.assert_has_calls([call(0, "file1", 6),
                                                                  call(1, "file2", 6),
                                                                  call(2, "file3", 6),
                                                                  call(3, "file4", 6),
                                                                  call(4, "file5", 6),
                                                                  call(5, "file6", 6)])