Example #1
0
    def setUp(self):
        parms = settings.BITIA_TEST_ENHYDRIS_INSTALLATION
        self.cookies = enhydris_api.login(parms['base_url'],
                                          parms['user'],
                                          parms['password'])

        # Create two time series
        j = {
            'gentity': parms['station_id'],
            'variable': parms['variable_id'],
            'unit_of_measurement': parms['unit_of_measurement_id'],
            'time_zone': parms['time_zone_id'],
        }
        self.ts1_id = enhydris_api.post_model(
            parms['base_url'], self.cookies, 'Timeseries', j)
        self.ts2_id = enhydris_api.post_model(
            parms['base_url'], self.cookies, 'Timeseries', j)
        assert self.ts1_id != self.ts2_id

        # Add some data (all but the last record) to the database
        ts = Timeseries(self.ts1_id)
        ts.read(StringIO(self.timeseries1_top))
        enhydris_api.post_tsdata(parms['base_url'], self.cookies, ts)
        ts = Timeseries(self.ts2_id)
        ts.read(StringIO(self.timeseries2_top))
        enhydris_api.post_tsdata(parms['base_url'], self.cookies, ts)

        # Temporary directory for cache files
        self.tempdir = tempfile.mkdtemp()
Example #2
0
    def setUp(self):
        get_server_from_env(self.__dict__)

        if not self.class_being_tested or not self.base_url:
            return

        # First 60 nonempty lines of test go to self.file1
        (fd, self.file1) = tempfile.mkstemp(text=True)
        fp = os.fdopen(fd, 'w')
        i = 0
        with open(full_testdata_filename(self.datafilename)) as f:
            for line in f:
                if i >= 60:
                    break
                if not line.strip():
                    continue
                i += 1
                fp.write(line)
        fp.close()

        # Login
        self.cookies = enhydris_api.login(self.base_url,
                                          self.user,
                                          self.password)

        # Create two timeseries
        self.timeseries_id1 = create_timeseries(self.cookies, self.__dict__)
        self.timeseries_id2 = create_timeseries(self.cookies, self.__dict__)
        self.datafile_fields = "0,{0},{1}".format(self.timeseries_id1,
                                                  self.timeseries_id2)
        self.ts1 = Timeseries(self.timeseries_id1)
        self.ts2 = Timeseries(self.timeseries_id2)
Example #3
0
    def setUp(self):
        self.parms = json.loads(os.getenv('PTHELMA_TEST_ENHYDRIS_API'))
        self.cookies = enhydris_api.login(self.parms['base_url'],
                                          self.parms['user'],
                                          self.parms['password'])

        # Create two time series
        j = {
            'gentity': self.parms['station_id'],
            'variable': self.parms['variable_id'],
            'unit_of_measurement': self.parms['unit_of_measurement_id'],
            'time_zone': self.parms['time_zone_id'],
            'time_step': 3,
            'timestamp_offset_minutes': 0,
            'timestamp_offset_months': 0,
            'remarks': 'Très importante',
        }
        self.ts1_id = enhydris_api.post_model(
            self.parms['base_url'], self.cookies, 'Timeseries', j)
        self.ts2_id = enhydris_api.post_model(
            self.parms['base_url'], self.cookies, 'Timeseries', j)
        assert self.ts1_id != self.ts2_id

        # Add some data (all but the last record) to the database
        ts = Timeseries(self.ts1_id)
        ts.read(StringIO(self.timeseries1_top))
        enhydris_api.post_tsdata(self.parms['base_url'], self.cookies, ts)
        ts = Timeseries(self.ts2_id)
        ts.read(StringIO(self.timeseries2_top))
        enhydris_api.post_tsdata(self.parms['base_url'], self.cookies, ts)

        # Temporary directory for cache files
        self.tempdir = tempfile.mkdtemp()
        self.savedcwd = os.getcwd()
        os.chdir(self.tempdir)
Example #4
0
    def test_get_ts_end_date(self):
        v = json.loads(os.getenv('PTHELMA_TEST_ENHYDRIS_API'))
        cookies = enhydris_api.login(v['base_url'], v['user'], v['password'])

        # Create a time series in the database
        j = {
            'gentity': v['station_id'],
            'variable': v['variable_id'],
            'unit_of_measurement': v['unit_of_measurement_id'],
            'time_zone': v['time_zone_id'],
        }
        ts_id = enhydris_api.post_model(v['base_url'], cookies, 'Timeseries',
                                        j)

        # Get its last date while it has no data
        date = enhydris_api.get_ts_end_date(v['base_url'], cookies, ts_id)
        self.assertEqual(date.isoformat(), '0001-01-01T00:00:00')

        # Now upload some data
        ts = Timeseries(ts_id)
        ts.read(StringIO(self.test_timeseries))
        enhydris_api.post_tsdata(v['base_url'], cookies, ts)

        # Get its last date
        date = enhydris_api.get_ts_end_date(v['base_url'], cookies, ts_id)
        self.assertEqual(date.isoformat(), '2014-01-05T08:00:00')

        # Get the last date of a nonexistent time series
        self.assertRaises(requests.HTTPError, enhydris_api.get_ts_end_date,
                          v['base_url'], cookies, ts_id + 1)
Example #5
0
 def setUp(self):
     get_server_from_env(self.__dict__)
     self.ref_ts = Timeseries(0)
     if not self.base_url:
         return
     self.cookies = enhydris_api.login(self.base_url, self.user,
                                       self.password)
     self.timeseries_id = create_timeseries(self.cookies, self.__dict__)
     self.ts = Timeseries(self.timeseries_id)
Example #6
0
    def setUp(self):
        get_server_from_env(self.__dict__)

        # Connect to server
        self.cookies = enhydris_api.login(self.base_url, self.user,
                                          self.password)

        # Create as many timeseries as there are not-to-ignore parameters
        for parm in self.parameters:
            parm['ts_id'] = create_timeseries(self.cookies, self.__dict__
                                              ) if parm['expname'] else 0
Example #7
0
 def append_newer_timeseries(self, start_date, ts1):
     self.session_cookies = enhydris_api.login(self.base_url, self.user,
                                               self.password)
     url = self.base_url + 'timeseries/d/{}/download/{}/?version=3'.format(
         self.timeseries_id, start_date.isoformat())
     r = requests.get(url, cookies=self.session_cookies)
     if r.status_code != 200:
         raise HTTPError('Error {} while getting {}'.format(r.status_code,
                                                            url))
     responseio = StringIO(r.text)
     ts2 = Timeseries()
     ts2.read_file(responseio)
     responseio.seek(0)
     ts1.read_meta(responseio)
     ts1.append(ts2)
Example #8
0
    def test_login(self):
        v = json.loads(os.getenv('PTHELMA_TEST_ENHYDRIS_API'))
        base_url, user, password = v['base_url'], v['user'], v['password']

        # Verify we are logged out
        r = requests.get(base_url)
        self.assertEqual(r.status_code, 200)
        self.assertTrue('Login' in r.text)
        self.assertFalse('Logout' in r.text)

        # Now login and verify we're logged on
        cookies = enhydris_api.login(base_url, user, password)
        r = requests.get(base_url, cookies=cookies)
        self.assertEqual(r.status_code, 200)
        self.assertFalse('Login' in r.text)
        self.assertTrue('Logout' in r.text)
Example #9
0
    def setUp(self):
        get_server_from_env(self.__dict__)

        if not self.class_being_tested or not self.base_url:
            return

        # Login
        self.cookies = enhydris_api.login(self.base_url,
                                          self.user,
                                          self.password)

        # Create two timeseries
        self.timeseries_id1 = create_timeseries(self.cookies, self.__dict__)
        self.timeseries_id2 = create_timeseries(self.cookies, self.__dict__)
        self.datafile_fields = "{0},{1}".format(self.timeseries_id1,
                                                self.timeseries_id2)
        self.ts1 = Timeseries(self.timeseries_id1)
        self.ts2 = Timeseries(self.timeseries_id2)
Example #10
0
 def execute(self):
     cookies = enhydris_api.login(self.config['General']['base_url'],
                                  self.config['General']['user'],
                                  self.config['General']['password'])
     for section in self.config:
         if section == 'General':
             continue
         datafileclass = eval('Datafile_{}'.format(
             self.config[section]['datafile_format']))
         adatafile = datafileclass(self.config['General']['base_url'],
                                   cookies,
                                   self.config[section],
                                   self.logger)
         try:
             adatafile.update_database()
         except MeteologgerError as e:
             msg = 'Error while processing item {0}: {1}'.format(section,
                                                                 str(e))
             sys.stderr.write(msg + '\n')
             self.logger.error(msg)
             self.logger.debug(traceback.format_exc())
Example #11
0
    def test_post_tsdata(self):
        v = json.loads(os.getenv('PTHELMA_TEST_ENHYDRIS_API'))
        cookies = enhydris_api.login(v['base_url'], v['user'], v['password'])

        # Create a time series in the database
        j = {
            'gentity': v['station_id'],
            'variable': v['variable_id'],
            'unit_of_measurement': v['unit_of_measurement_id'],
            'time_zone': v['time_zone_id'],
        }
        ts_id = enhydris_api.post_model(v['base_url'], cookies, 'Timeseries',
                                        j)

        # Now upload some data
        ts = Timeseries(ts_id)
        ts.read(StringIO(self.test_timeseries_top))
        enhydris_api.post_tsdata(v['base_url'], cookies, ts)

        # Read and check the time series
        url = enhydris_api.urljoin(v['base_url'],
                                   'timeseries/d/{}/download/'.format(ts.id))
        r = requests.get(url, cookies=cookies)
        r.raise_for_status()
        self.assertEqual(get_after_blank_line(r.text),
                         self.test_timeseries_top)

        # Upload more data
        ts = Timeseries(ts_id)
        ts.read(StringIO(self.test_timeseries_bottom))
        enhydris_api.post_tsdata(v['base_url'], cookies, ts)

        # Read and check the time series
        url = enhydris_api.urljoin(v['base_url'],
                                   'timeseries/d/{}/download/'.format(ts.id))
        r = requests.get(url, cookies=cookies)
        r.raise_for_status()
        self.assertEqual(get_after_blank_line(r.text),
                         self.test_timeseries)
Example #12
0
 def setUp(self):
     # Create two stations, each one with a time series
     parms = settings.BITIA_TEST_ENHYDRIS_INSTALLATION
     self.cookies = enhydris_api.login(parms['base_url'],
                                       parms['user'],
                                       parms['password'])
     self.station1_id = enhydris_api.post_model(
         parms['base_url'], self.cookies, 'Station',
         {'name': 'station1',
          'srid': 4326,
          'point': 'POINT (23.78743 37.97385)',
          'copyright_holder': 'Joe User',
          'copyright_years': '2014',
          'stype': 1,
          'owner': parms['owner_id'],
          })
     self.timeseries1_id = enhydris_api.post_model(
         parms['base_url'], self.cookies, 'Timeseries',
         {'gentity': self.station1_id,
          'variable': parms['variable_id'],
          'unit_of_measurement': parms['unit_of_measurement_id'],
          'time_zone': parms['time_zone_id']})
     self.station2_id = enhydris_api.post_model(
         parms['base_url'], self.cookies, 'Station',
         {'name': 'station1',
          'srid': 4326,
          'point': 'POINT (24.56789 38.76543)',
          'copyright_holder': 'Joe User',
          'copyright_years': '2014',
          'stype': 1,
          'owner': parms['owner_id'],
          })
     self.timeseries2_id = enhydris_api.post_model(
         parms['base_url'], self.cookies, 'Timeseries',
         {'gentity': self.station2_id,
          'variable': parms['variable_id'],
          'unit_of_measurement': parms['unit_of_measurement_id'],
          'time_zone': parms['time_zone_id']})
Example #13
0
    def update_one_timeseries(base_url, id, user=None, password=None):
        if base_url[-1] != '/':
            base_url += '/'

        # Read timeseries from cache file
        cache_filename = os.path.join(
            settings.BITIA_TIMESERIES_CACHE_DIR, '{}.hts'.format(id))
        ts1 = Timeseries()
        if os.path.exists(cache_filename):
            with open(cache_filename) as f:
                try:
                    ts1.read_file(f)
                except ValueError:
                    # File may be corrupted; continue with empty time series
                    ts1 = Timeseries()

        # Get its end date
        try:
            end_date = ts1.bounding_dates()[1]
        except TypeError:
            # Timeseries is totally empty; no start and end date
            end_date = datetime(1, 1, 1, 0, 0)
        start_date = end_date + timedelta(minutes=1)

        # Get newer timeseries and append it
        session_cookies = enhydris_api.login(base_url, user, password)
        url = base_url + 'timeseries/d/{}/download/{}/'.format(
            id, start_date.isoformat())
        r = requests.get(url, cookies=session_cookies)
        r.raise_for_status()
        ts2 = Timeseries()
        ts2.read_file(StringIO(r.text))
        ts1.append(ts2)

        # Save it
        with open(cache_filename, 'w') as f:
            ts1.write(f)
Example #14
0
def create_ogr_layer_from_stations(group, data_source):
    """
    Create and return an ogr.Layer with stations as its points.

    settings.BITIA_TIMESERIES_GROUPS[group] is a list of time series,
    each time series referring to a station. This function retrieves
    the co-ordinates of each station from Enhydris (unless we have
    them cached in Django's cache) and creates a layer in the
    specified ogr data_source whose features are points; as many
    points as there are stations/timeseries; each point is also given
    a timeseries_id attribute.
    """
    layer = data_source.CreateLayer('test')
    layer.CreateField(ogr.FieldDefn('timeseries_id', ogr.OFTInteger))
    for item in settings.BITIA_TIMESERIES_GROUPS[group]:
        # Get the point from the cache, or fetch it anew on cache miss
        cache_key = 'timeseries_{}_point'.format(item['id'])
        pointwkt = cache.get(cache_key)
        if not pointwkt:
            cookies = enhydris_api.login(item['base_url'],
                                         item.get('user', None),
                                         item.get('password', None))
            tsd = enhydris_api.get_model(item['base_url'], cookies,
                                         'Timeseries', item['id'])
            gpoint = enhydris_api.get_model(item['base_url'], cookies,
                                            'Gpoint', tsd['gentity'])
            pointwkt = gpoint['point']
            cache.set(cache_key, pointwkt)

        # Create feature and put it on the layer
        point = ogr.CreateGeometryFromWkt(pointwkt)
        f = ogr.Feature(layer.GetLayerDefn())
        f.SetGeometry(point)
        f.SetField('timeseries_id', item['id'])
        layer.CreateFeature(f)
    return layer
Example #15
0
    def setUp(self):
        self.tempdir = tempfile.mkdtemp()
        self.config_file = os.path.join(self.tempdir, 'enhydris_cache.conf')
        self.saved_argv = sys.argv
        sys.argv = ['enhydris_cache', '--traceback', self.config_file]
        self.savedcwd = os.getcwd()

        # Create two stations, each one with a time series
        self.parms = json.loads(os.getenv('PTHELMA_TEST_ENHYDRIS_API'))
        self.cookies = enhydris_api.login(self.parms['base_url'],
                                          self.parms['user'],
                                          self.parms['password'])
        self.station1_id = enhydris_api.post_model(
            self.parms['base_url'], self.cookies, 'Station',
            {'name': 'station1',
             'srid': 4326,
             'point': 'POINT (23.78743 37.97385)',
             'copyright_holder': 'Joe User',
             'copyright_years': '2014',
             'stype': 1,
             'owner': self.parms['owner_id'],
             })
        self.timeseries1_id = enhydris_api.post_model(
            self.parms['base_url'], self.cookies, 'Timeseries',
            {'gentity': self.station1_id,
             'variable': self.parms['variable_id'],
             'unit_of_measurement': self.parms['unit_of_measurement_id'],
             'time_zone': self.parms['time_zone_id']})
        self.station2_id = enhydris_api.post_model(
            self.parms['base_url'], self.cookies, 'Station',
            {'name': 'station1',
             'srid': 4326,
             'point': 'POINT (24.56789 38.76543)',
             'copyright_holder': 'Joe User',
             'copyright_years': '2014',
             'stype': 1,
             'owner': self.parms['owner_id'],
             })
        self.timeseries2_id = enhydris_api.post_model(
            self.parms['base_url'], self.cookies, 'Timeseries',
            {'gentity': self.station2_id,
             'variable': self.parms['variable_id'],
             'unit_of_measurement': self.parms['unit_of_measurement_id'],
             'time_zone': self.parms['time_zone_id']})

        # Prepare a configuration file (some tests override it)
        with open(self.config_file, 'w') as f:
            f.write(textwrap.dedent('''\
                [General]
                cache_dir = {self.tempdir}

                [timeseries1]
                base_url = {base_url}
                id = {self.timeseries1_id}
                file = file1
                user = {self.parms[user]}
                password = {self.parms[password]}

                [timeseries2]
                base_url = {base_url}
                id = {self.timeseries2_id}
                file = file2
                user = {self.parms[user]}
                password = {self.parms[password]}
                ''').format(self=self, base_url=self.parms['base_url']))