Exemple #1
0
    def setUp(self):
        self.servic_emanager=DataServiceManager()
        self.dss_service=self.servic_emanager.get_service("vtools.datastore.dss.DssService")

        if os.path.exists(self.test_file_path):
            os.remove(self.test_file_path)

        shutil.copy(self.backup_dss_file,self.test_file_path)
Exemple #2
0
    def __init__(self, methodName="runTest"):
        super(TestDssService, self).__init__(methodName)
        import vtools.datastore.dss
        pkgfile = vtools.datastore.dss.__file__
        self.test_file_path = 'test/testfile.dss'
        self.test_file_path = os.path.join(
            os.path.split(os.path.abspath(pkgfile))[0], self.test_file_path)
        self.backup_dss_file = os.path.join(
            os.path.split(os.path.abspath(pkgfile))[0],
            'test/backup_dssfile/testfile.dss')

        self.service_manager = DataServiceManager()
        self.dss_service = self.service_manager.get_service(
            "vtools.datastore.dss.DssService")
Exemple #3
0
class TestTranslate(unittest.TestCase):

    """ test functionality of translate data references """


    def __init__(self,methodName="runTest"):

        super(TestTranslate,self).__init__(methodName)
        import vtools.test
        pkgfile = vtools.test.__file__
        self.test_file_path='testfile.dss'
        self.test_file_path=os.path.join(os.path.split(os.path.abspath(pkgfile))[0],self.test_file_path)
        self.backup_dss_file=os.path.join(os.path.split(os.path.abspath(pkgfile))[0],'backup/testfile.dss') 
        self.test_file2="dest.dss"
        self.test_file2=os.path.join(os.path.split(os.path.abspath(pkgfile))[0],self.test_file2)
        
        
    def setUp(self):
        self.servic_emanager=DataServiceManager()
        self.dss_service=self.servic_emanager.get_service("vtools.datastore.dss.DssService")

        if os.path.exists(self.test_file_path):
            os.remove(self.test_file_path)
        shutil.copy(self.backup_dss_file,self.test_file_path)
        self.dest_dss=open_dss(self.test_file2)        
                                
    def tearDown(self):
        if os.path.exists(self.test_file_path):
            os.remove(self.test_file_path)
        self.dest_dss.close()
        if os.path.exists(self.test_file2):
            os.remove(self.test_file2)
            
    def test_translate_dss(self):
        
        dssfile_path=self.test_file_path
        dssc=self.dss_service.get_catalog(dssfile_path)
        selector="/HIST*/*/*/*/*/*/"
        dl=dssc.data_references(selector)
    
        ## given translation here
        translation="/${A}_AVER/${B}/${C}//1HOUR/${F}/"
        dest="this/is/only/test/location.dss"
        dt=translate_references(self.dss_service,dl,self.dss_service,dest,translation)
        self.assertEqual(len(dt),5)
Exemple #4
0
import sys, os, time  #,pdb

## vtools import
from vtools.datastore.dss.dss_constants import DSS_DATA_SOURCE
from vtools.datastore.data_service_manager import DataServiceManager
from vtools.data.vtime import parse_time, parse_interval
from vtools.data.timeseries import rts, its
from vtools.data.constants import *
from vtools.datastore.data_reference import DataReferenceFactory
import re
from pydss.hecdss import zchkpn

__all__=["dss_retrieve_ts","dss_store_ts","dss_catalog",\
         "visual_dss_format","dss_delete_ts","is_valid_dss"]

dsm = DataServiceManager()

##############################################
## Public function.
##############################################


def dss_retrieve_ts(dss_file,
                    selector,
                    time_window=None,
                    unique=False,
                    overlap=None):
    """Retrieve timeseries from a dss file 
    based on the specified paths within a given time_window.
    usage: rt=retrieve_dss_data(dss_file,selector)
Exemple #5
0
 def setUp(self):
     self.servic_emanager = DataServiceManager()
     self.excel_service = self.servic_emanager.get_service(
         EXCEL_DATA_SOURCE)
Exemple #6
0
class TestExcelService(unittest.TestCase):
    def __init__(self, methodName="runTest"):

        super(TestExcelService, self).__init__(methodName)
        import vtools.datastore.excel
        pkgfile = vtools.datastore.excel.__file__
        self.test_file_path = 'test\\test.xls'
        self.test_file_path = os.path.join(
            os.path.split(os.path.abspath(pkgfile))[0], self.test_file_path)
        self.backup_xls_file = os.path.join(
            os.path.split(os.path.abspath(pkgfile))[0],
            'test/backup_excelfile/test.xls')
        if os.path.exists(self.test_file_path):
            os.remove(self.test_file_path)
        shutil.copy(self.backup_xls_file, self.test_file_path)

    def setUp(self):
        self.servic_emanager = DataServiceManager()
        self.excel_service = self.servic_emanager.get_service(
            EXCEL_DATA_SOURCE)

    def test_get_catalog(self):
        c = self.excel_service.get_catalog(self.test_file_path)
        self.assert_(type(c) == ExcelCatalog)

    def test_get_data(self):

        c = self.excel_service.get_catalog(self.test_file_path)

        ## filter catalog to get some data reference of rts.
        selector = "worksheet=hydro_flow;range=A6:E110;"
        dref = c.filter_catalog(selector)

        for d in dref:
            ts = self.excel_service.get_data(d)
            self.assert_(ts.is_regular())
            self.assertEqual(len(ts), 104)

        ## filter catalog to get some data reference of its.
        selector = "worksheet=its_data;range=B6:F110;"
        dref = c.filter_catalog(selector)

        for d in dref:
            ts = self.excel_service.get_data(d)
            self.assert_(not (ts.is_regular()))
            self.assertEqual(len(ts), 104)

    def test_add_data(self):
        def rand_gen():
            while True:
                yield random()

        tss = []

        ## create several ts
        # 1
        st = parse_time("1/2/1987 10:30")
        dt = parse_interval("1hour")
        prop={"agency":"dwr","interval":"1hour","station":"rsac045",\
              "datum":"NGVD88","var":"flow"}
        n = 13470
        data = list(islice(rand_gen(), n))
        ts = rts(data, st, dt, prop)
        tss.append(ts)
        # 2
        st = parse_time("3/20/1997 10:30")
        dt = parse_interval("1day")
        prop={"bearu":"usgs","interval":"1day","lat":70.90,\
              "long":34.45,"datum":"NGVD88","var":"stage"}
        n = 40960
        data = list(islice(rand_gen(), n))
        ts = rts(data, st, dt, prop)
        tss.append(ts)

        # 3
        st = parse_time("1/2/1967 4:30")
        dt = parse_interval("15min")
        prop={"place":"uml","interval":"15min","station":"rsac045",\
              "datum":"NGVD88","var":"bod"}
        n = 20000
        data = list(islice(rand_gen(), n))
        ts = rts(data, st, dt, prop)
        tss.append(ts)

        #
        ref=DataReferenceFactory(EXCEL_DATA_SOURCE,"store.xls",\
                                 selector="dss2excel$B5")

        self.excel_service.batch_add(ref, tss)
Exemple #7
0
class TestDssCatalog(unittest.TestCase):
    """ test functionality of dss catalog """
    
    def __init__(self,methodName="runTest"):

        super(TestDssCatalog,self).__init__(methodName)     
        import vtools.datastore.dss
        pkgfile = vtools.datastore.dss.__file__
        self.test_file_path='testfile.dss'
        self.test_file_path=os.path.join(os.path.split(os.path.abspath(pkgfile))[0],self.test_file_path)
        self.backup_dss_file=os.path.join(os.path.split(os.path.abspath(pkgfile))[0],'test/backup_dssfile/testfile.dss')    
        
    def setUp(self):
        self.servic_emanager=DataServiceManager()
        self.dss_service=self.servic_emanager.get_service("vtools.datastore.dss.DssService")

        if os.path.exists(self.test_file_path):
            os.remove(self.test_file_path)

        shutil.copy(self.backup_dss_file,self.test_file_path)
        
    def tearDown(self):        
        if os.path.exists(self.test_file_path):
            os.remove(self.test_file_path)


    def test_get_data_reference(self):
        dssfile_path=self.test_file_path
        dssc=self.dss_service.get_catalog(dssfile_path)
        for entry in dssc.entries():           
            data_ref=dssc.get_data_reference(entry)
        
    def test_copy_remove(self):
        dssfile_path=self.test_file_path
        dssc=self.dss_service.get_catalog(dssfile_path)

        len1=len(dssc.entries())
        entry1=dssc.entries()[-1]
        entry2=dssc.copy(entry1)
        len2=len(dssc.entries())
        # Length of calalog entry list should be
        # increased by 1
        self.assertEqual(len1+1,len2)
        
        A1=entry1.item('A')
        A2=entry2.item('A')
        self.assertEqual(A1+'_COPY',A2)
        
        data_ref1=dssc.get_data_reference(entry1)
        data_ref2=dssc.get_data_reference(entry2)
        
        # Those two dataferfence should have same time extent.
        self.assertEqual(data_ref1.extent,data_ref2.extent)

        # Make time window smaller to speed up
        extent="time_window=(12/1/1991 03:45,12/24/1991 01:30)"
        data_ref1.extent= extent
        data_ref2.extent= extent        
        data1=self.dss_service.get_data(data_ref1)
        data2=self.dss_service.get_data(data_ref2)
        
        for (d1,d2) in zip(data1.data,data2.data):
            self.assertEqual(d1,d2)
        for (t1,t2) in zip(data1.ticks,data2.ticks):
            self.assertEqual(t1,t2)

        # Test remove the new entry.
        dssc.remove(entry2)
        self.assert_(entry2 not in dssc.entries())

        # Try to get data from based on new datareference
        # , service should raise exception.        
        self.assertRaises(DssAccessError,\
                          self.dss_service.get_data,data_ref2)
                
    def test_modify(self):
##
##        import pdb
##        pdb.set_trace()
        dssfile_path=self.test_file_path
        dssc=self.dss_service.get_catalog(dssfile_path)        

        for entry in dssc.entries():
            new_entry=copy(entry)
            A=new_entry.item('A')
            A=A+'_MODIFIED'
            new_entry.set_item('A',A)
            dssc.modify(entry,new_entry)
            
        for newentry in dssc.entries():
            dataref=dssc.get_data_reference(newentry)
            self.assert_('_MODIFIED' in dataref.selector)
            
    def test_filter_catalog(self):
        dssfile_path=self.test_file_path
        dssc=self.dss_service.get_catalog(dssfile_path)
        selector="/HIST*/SLTR*/*//15MIN/*/"
        dl=dssc.filter_catalog(selector)
        self.assertEqual(type(dl),DssCatalog)
        self.assertEqual(len(dl),3)
                
        selector="/HIST+CHAN/SLTMP017/STAGE//15MIN/DWR-CD-SURFWATER/"        
        dl=dssc.filter_catalog(selector)
        self.assertEqual(type(dl),DssCatalog)
        self.assertEqual(len(dl),1)
        
        ## this selector has a trailing space intentionally, I made it still correct by stripping the space.
        selector="/HIST+CHAN/SLTMP017/STAGE//15MIN/DWR-CD-SURFWATER/ "       
        dl=dssc.filter_catalog(selector)
        self.assertEqual(type(dl),DssCatalog)
        self.assertEqual(len(dl),1)
        
        selector="nothing here "      
        self.assertRaises(ValueError,dssc.filter_catalog,selector)
        
        ## this selector intentionaly misses last backslash, it should report error
        selector="/HIST+CHAN/SLTMP017/STAGE//15MIN/DWR-CD-SURFWATER"    
        self.assertRaises(ValueError,dssc.filter_catalog,selector)
        
    def test_filter_catalog2(self):
        
        dssfile_path=self.test_file_path
        dssc=self.dss_service.get_catalog(dssfile_path)
        selector="F=DWR*"
        dl=dssc.filter_catalog(selector)
        self.assertEqual(type(dl),DssCatalog)
        self.assertEqual(len(dl),7)
                
    def test_filter_catalog_dot_bug(self):
        dssfile_path=self.test_file_path
        dssc=self.dss_service.get_catalog(dssfile_path)
        selector="B.SLTMP017,F=USGS-RIM"
        dl=dssc.filter_catalog(selector)
        self.assertEqual(len(dl),1)      
        

    def test_data_references(self):
        
        dssfile_path=self.test_file_path
        dssc=self.dss_service.get_catalog(dssfile_path)
        selector="/HIST*/SLTR*/*//15MIN/*/"
        range="(10/2/1997 1200, 7/4/1998 1315)"
        dl=dssc.data_references(selector,range)
        numd=0
        for d in dl:
            self.assertEqual(type(d),DataReference)
            numd=numd+1
        self.assertEqual(numd,3)

        range=("10/2/1997 12:00", "7/4/1998 13:15")
        dl=dssc.data_references(selector,range)
        numd=0
        for d in dl:
            self.assertEqual(type(d),DataReference)
            numd=numd+1
        self.assertEqual(numd,3)

        range=(datetime(1997,10,2,12), datetime(1998,7,4, 13,15))
        dl=dssc.data_references(selector,range)
        numd=0
        for d in dl:
            self.assertEqual(type(d),DataReference)
            numd=numd+1
        self.assertEqual(numd,3)        
        
        selector="/HIST+CHAN/SLTMP017/STAGE//15MIN/DWR-CD-SURFWATER/"        
        dl=dssc.data_references(selector)
        numd=0
        for d in dl:
            self.assertEqual(type(d),DataReference)
            numd=numd+1
        self.assertEqual(numd,1)
Exemple #8
0
class TestDssService(unittest.TestCase):
    """ test functionality of dss service """
    def __init__(self, methodName="runTest"):
        super(TestDssService, self).__init__(methodName)
        import vtools.datastore.dss
        pkgfile = vtools.datastore.dss.__file__
        self.test_file_path = 'test/testfile.dss'
        self.test_file_path = os.path.join(
            os.path.split(os.path.abspath(pkgfile))[0], self.test_file_path)
        self.backup_dss_file = os.path.join(
            os.path.split(os.path.abspath(pkgfile))[0],
            'test/backup_dssfile/testfile.dss')

        self.service_manager = DataServiceManager()
        self.dss_service = self.service_manager.get_service(
            "vtools.datastore.dss.DssService")

    def setUp(self):
        if os.path.exists(self.test_file_path):
            os.remove(self.test_file_path)
        shutil.copy(self.backup_dss_file, self.test_file_path)
        if os.path.exists('newdss.dss'):
            os.remove('newdss.dss')

    def tearDown(self):
        if os.path.exists(self.test_file_path):
            os.remove(self.test_file_path)

    def test_get_catalog(self):
        dssfile_path = self.test_file_path
        dssc = self.dss_service.get_catalog(dssfile_path)
        self.assertTrue(type(dssc) == DssCatalog)
        entries = dssc.entries()
        for entry in entries:
            self.assertTrue(type(entry) == CatalogEntry)

        self.assertEqual(len(entries), 28)

    def test_get_data(self):

        id = "vtools.datastore.dss.DssService"
        view = ""
        source = self.test_file_path

        # Regular time series.

        selector = "/TUTORIAL/DOWNSTREAM/EC//15MIN/REALISTIC/"
        extent = "time_window=(12/1/1991 03:45,12/24/1991 01:30)"
        data_ref = DataReference(id, source, view, selector, extent)
        data = self.dss_service.get_data(data_ref)
        self.assertTrue(type(data) == TimeSeries)
        l = len(data.data)
        self.assertEqual(len(data.data), 2200)
        self.assertEqual(ticks_to_time(data.ticks[0]),
                         parse('12/1/1991 03:45'))
        # Here dss reading func only read up to right end of
        # time window (not include).
        self.assertEqual(ticks_to_time(data.ticks[l - 1]),
                         parse('12/24/1991 01:30'))

        # regular time series didn't accept overlap option
        overlap = (0, 0)
        self.assertRaises(ValueError, self.dss_service.get_data, data_ref,
                          overlap)

        # Irregular time series.
        selector = "/TUTORIAL/GATE/FLAP_OP//IR-YEAR/CONSTANT/"
        extent = "time_window=(12/11/1991 01:00,04/02/1992 21:50)"
        data_ref = DataReference(id, source, view, selector, extent)
        data = self.dss_service.get_data(data_ref)
        self.assertTrue(type(data) == TimeSeries)
        self.assertEqual(len(data.data), 106)

        ## retrieve data of monthly interval
        ## Note this operation will trigger the exception handling line
        ## of funciton _multiple_window of dss_service.py. For dss_service
        ## will try to forward input time window start time by DSS_MAX_RTS_POINTS
        ## months, which is over the maximum year set by Python Datetime.

        selector = "/SHORT/SYNTHETIC/DATA//1MON/TEST-MONTHLY-INTERVAL/"
        extent = "time_window=(12/1/1990,06/02/1991)"
        data_ref = DataReference(id, source, view, selector, extent)
        data = self.dss_service.get_data(data_ref)
        self.assertTrue(type(data) == TimeSeries)
        self.assertEqual(len(data.data), 5)

    def test_get_its_data_overlap(self):

        id = "vtools.datastore.dss.DssService"
        view = ""
        source = self.test_file_path
        # Irregular time series.
        selector = "/TUTORIAL/GATE/FLAP_OP//IR-YEAR/CONSTANT/"
        extent = "time_window=(12/10/1991 00:05,01/24/1992 20:58)"
        data_ref = DataReference(id, source, view, selector, extent)
        ##  retrieve data within
        overlap = (0, 0)
        data = self.dss_service.get_data(data_ref, overlap)
        self.assertTrue(type(data) == TimeSeries)
        self.assertEqual(len(data.data), 46)
        self.assertEqual(ticks_to_time(data.ticks[0]),
                         parse('12/10/1991 00:08'))

        ##  retrieve data within and preceding
        overlap = (1, 0)
        data = self.dss_service.get_data(data_ref, overlap)
        self.assertTrue(type(data) == TimeSeries)
        self.assertEqual(len(data.data), 47)
        self.assertEqual(ticks_to_time(data.ticks[0]),
                         parse('12/10/1991 00:04'))

        ##  retrieve data within  and following
        overlap = (0, 1)
        data = self.dss_service.get_data(data_ref, overlap)
        self.assertTrue(type(data) == TimeSeries)
        self.assertEqual(len(data.data), 47)
        self.assertEqual(ticks_to_time(data.ticks[-1]),
                         parse('1/24/1992 21:00'))

        ##  retrieve data within
        overlap = (1, 1)
        data = self.dss_service.get_data(data_ref, overlap)
        self.assertTrue(type(data) == TimeSeries)
        self.assertEqual(len(data.data), 48)
        self.assertEqual(ticks_to_time(data.ticks[0]),
                         parse('12/10/1991 00:04'))
        self.assertEqual(ticks_to_time(data.ticks[-1]),
                         parse('1/24/1992 21:00'))

        ## test when time window exaclty coincide with record time points
        ## the data preceding and following time window will still be returned
        ## depends the overlap option
        extent = "time_window=(12/10/1991 00:08,01/24/1992 21:00)"
        data_ref = DataReference(id, source, view, selector, extent)
        ##  retrieve data within
        overlap = (0, 0)
        data = self.dss_service.get_data(data_ref, overlap)
        self.assertTrue(type(data) == TimeSeries)
        self.assertEqual(len(data.data), 47)
        self.assertEqual(ticks_to_time(data.ticks[0]),
                         parse('12/10/1991 00:08'))
        self.assertEqual(ticks_to_time(data.ticks[-1]),
                         parse('1/24/1992 21:00'))

        ##  retrieve data within and preceding
        overlap = (1, 0)
        data = self.dss_service.get_data(data_ref, overlap)
        self.assertTrue(type(data) == TimeSeries)
        self.assertEqual(len(data.data), 48)
        self.assertEqual(ticks_to_time(data.ticks[0]),
                         parse('12/10/1991 00:04'))
        self.assertEqual(ticks_to_time(data.ticks[-1]),
                         parse('1/24/1992 21:00'))

        ##  retrieve data within  and following
        overlap = (0, 1)
        data = self.dss_service.get_data(data_ref, overlap)
        self.assertTrue(type(data) == TimeSeries)
        self.assertEqual(len(data.data), 48)
        self.assertEqual(ticks_to_time(data.ticks[0]),
                         parse('12/10/1991 00:08'))
        self.assertEqual(ticks_to_time(data.ticks[-1]),
                         parse('1/26/1992 08:00'))
        ##  retrieve data within
        overlap = (1, 1)
        data = self.dss_service.get_data(data_ref, overlap)
        self.assertTrue(type(data) == TimeSeries)
        self.assertEqual(len(data.data), 49)
        self.assertEqual(ticks_to_time(data.ticks[0]),
                         parse('12/10/1991 00:04'))
        self.assertEqual(ticks_to_time(data.ticks[-1]),
                         parse('1/26/1992 08:00'))

    def test_get_aggregated_data(self):

        # Regular time series.
        id = "vtools.datastore.dss.DssService"
        view = ""
        selector = "/RLTM+CHAN/SLBAR002/FLOW-EXPORT//1DAY/DWR-OM-JOC-DSM2/"
        source = self.test_file_path
        extent = "time_window=(1/2/1997,1/5/1997)"
        data_ref = DataReference(id, source, view, selector, extent)
        data = self.dss_service.get_data(data_ref)
        self.assertTrue(type(data) == TimeSeries)
        l = len(data.data)
        self.assertEqual(len(data.data), 3)
        self.assertEqual(data.data[0], 11.0)
        self.assertEqual(ticks_to_time(data.ticks[0]), parse('1/2/1997'))
        # Here dss reading func only read up to right end of
        # time window (not include).
        self.assertEqual(ticks_to_time(data.ticks[l - 1]), parse('1/4/1997'))

    def test_get_data_all(self):
        ## here to test pull out all the data from a
        ## path given(through data references).

        dssfile_path = self.test_file_path
        dssc = self.dss_service.get_catalog(dssfile_path)
        entries = dssc.entries()

        ## for each entry ,bulit a datareference without
        ## time window given
        for entry in entries:
            data_ref = dssc.get_data_reference(entry)
            #print data_ref.selector
            #print entry.dimension_scales()[0].get_range()
            ts = self.dss_service.get_data(data_ref)
            self.assertTrue(type(ts) == TimeSeries)

    def test_support_unaligned_ts(self):
        ## create some unaligned ts with aggregated attributes, save to test file and read it back

        ## daily averaged ts
        ts_start = parse_time("01/02/2000 23:00")
        ts_data = [1.0, 2.0, 3.0, 4.0, 5.0]
        interval = parse_interval("1day")
        props = {AGGREGATION: MEAN, TIMESTAMP: PERIOD_START}
        ts = rts(ts_data, ts_start, interval, props)
        selector = "/THIS/TS/UNALIGNED//1DAY/TEST/"
        id = "vtools.datastore.dss.DssService"
        source = self.test_file_path
        data_ref = DataReference(id, source=source, selector=selector)
        self.dss_service.add_data(data_ref, ts)

        dssc = self.dss_service.get_catalog(source)
        data_refs = dssc.data_references(selector)
        data_ref = data_refs.next()
        ts_back = self.dss_service.get_data(data_ref)
        self.assertTrue(ts_back.start == ts.start)
        self.assertTrue(len(ts_back) == len(ts))
        self.assertTrue((ts_back.data == ts.data).all())

        ## hourly averaged ts
        ts_start = parse_time("01/02/2000 23:59")
        ts_data = [1.0, 2.0, 3.0, 4.0, 5.0]
        interval = parse_interval("1hour")
        props = {AGGREGATION: MEAN, TIMESTAMP: PERIOD_START}
        ts = rts(ts_data, ts_start, interval, props)
        selector = "/THIS/TS/UNALIGNED//1HOUR/TEST/"
        data_ref = DataReference(id, source=source, selector=selector)
        self.dss_service.add_data(data_ref, ts)

        dssc = self.dss_service.get_catalog(source)
        data_refs = dssc.data_references(selector)
        data_ref = data_refs.next()
        ts_back = self.dss_service.get_data(data_ref)
        self.assertTrue(ts_back.start == ts.start)
        self.assertTrue(len(ts_back) == len(ts))
        self.assertTrue((ts_back.data == ts.data).all())

        ## 15MIN averaged ts
        ts_start = parse_time("01/02/2000 23:59")
        ts_data = [1.0, 2.0, 3.0, 4.0, 5.0]
        interval = parse_interval("15MIN")
        props = {AGGREGATION: MEAN, TIMESTAMP: PERIOD_START}
        ts = rts(ts_data, ts_start, interval, props)
        selector = "/THIS/TS/UNALIGNED//15MIN/TEST/"
        data_ref = DataReference(id, source=source, selector=selector)
        self.dss_service.add_data(data_ref, ts)

        dssc = self.dss_service.get_catalog(source)
        data_refs = dssc.data_references(selector)
        data_ref = data_refs.next()
        ts_back = self.dss_service.get_data(data_ref)
        self.assertTrue(ts_back.start == ts.start)
        self.assertTrue(len(ts_back) == len(ts))
        self.assertTrue((ts_back.data == ts.data).all())

        ## YEAR averaged ts
        ts_start = parse_time("02/02/2000 ")
        ts_data = [1.0, 2.0, 3.0, 4.0, 5.0]
        interval = parse_interval("1year")
        props = {AGGREGATION: MEAN, TIMESTAMP: PERIOD_START}
        ts = rts(ts_data, ts_start, interval, props)
        selector = "/THIS/TS/UNALIGNED//1YEAR/TEST/"
        data_ref = DataReference(id, source=source, selector=selector)
        self.dss_service.add_data(data_ref, ts)

        dssc = self.dss_service.get_catalog(source)
        data_refs = dssc.data_references(selector)
        data_ref = data_refs.next()
        ts_back = self.dss_service.get_data(data_ref)
        self.assertTrue(ts_back.start == ts.start)
        self.assertTrue(len(ts_back) == len(ts))
        self.assertTrue((ts_back.data == ts.data).all())

    def test_get_save_ts(self):
        ## test ts property unchanged after read and save ts into dss

        selector = "/RLTM+CHAN/SLBAR002/FLOW-EXPORT//1DAY/DWR-OM-JOC/"
        source = self.test_file_path
        dssc = self.dss_service.get_catalog(source)
        ##cagatalog function return a itertor over possible data reference
        ##we only get one data ref
        data_refs = dssc.data_references(selector)
        data_ref = data_refs.next()
        ts = self.dss_service.get_data(data_ref)

        ## then save this ts back into dss in a different path with some
        ## extra props to simulate props in pratical cases
        id = "vtools.datastore.dss.DssService"
        path = "/RLTM+CHAN/SLBAR002_COPY/FLOW-EXPORT//1DAY/DWR-OM-JOC/"
        source = self.test_file_path
        data_ref = DataReference(id, source=source, selector=path)
        self.dss_service.add_data(data_ref, ts)

        ## read this ts back it be sa me length as original one
        data_refs = dssc.data_references(selector)
        data_ref = data_refs.next()
        nts = self.dss_service.get_data(data_ref)
        self.assertTrue(len(ts) == len(nts))

        ##clean up this temp data
        cat = self.dss_service.get_catalog(source)
        cat_filtered = cat.filter_catalog(path)
        cat_filtered.set_editable()
        cat_filtered.remove(cat_filtered.entries()[0])

        ##read catalog again, this path shouldn't be there
        cat_new = self.dss_service.get_catalog(source)
        self.assertEqual(len(cat_new), len(cat) - 1)
        self.assertRaises(DssCatalogError, cat_new.filter_catalog, path)

    def test_save_ts_props(self):
        ## test ts property unchanged after read and save ts into dss

        selector = "/RLTM+CHAN/SLBAR002/FLOW-EXPORT//1DAY/DWR-OM-JOC/"
        source = self.test_file_path
        dssc = self.dss_service.get_catalog(source)
        ##cagatalog function return a itertor over possible data reference
        ##we only get one data ref
        data_refs = dssc.data_references(selector)
        data_ref = data_refs.next()
        ts = self.dss_service.get_data(data_ref)

        self.assertTrue(type(ts) == TimeSeries)
        self.assertTrue(ts.props[AGGREGATION] == MEAN)

        ts.props.clear()
        ts.props[AGGREGATION] = MEAN
        ts.props[TIMESTAMP] = 'PERIOD_START'
        ts.props[UNIT] = 'CFS'
        ts.props["VDATUM"] = "NGVD88"
        ts.props["AUTHOR"] = "John Doe"
        ts.props["MODEL"] = "hydro 7.5"

        ## then save this ts back into dss in a different path
        ## to simulate pratical cases
        id = "vtools.datastore.dss.DssService"
        path = "/RLTM+CHAN/SLBAR002_COPY/FLOW-EXPORT//1DAY/DWR-OM-JOC/"
        source = self.test_file_path
        data_ref = DataReference(id, source=source, selector=path)
        self.dss_service.add_data(data_ref, ts)

        ## read this ts back it AGGREGATION should be MEAN
        ## also with all other properties
        dssc = self.dss_service.get_catalog(source)
        data_refs = dssc.data_references(path)
        data_ref = data_refs.next()
        ts = self.dss_service.get_data(data_ref)

        self.assertTrue(type(ts) == TimeSeries)
        self.assertTrue(ts.props[AGGREGATION] == MEAN)
        self.assertTrue(ts.props["VDATUM"] == "NGVD88")
        self.assertTrue(ts.props["AUTHOR"] == "John Doe")
        self.assertTrue(ts.props["MODEL"] == "hydro 7.5")

        ##clean up this temp data
        cat = self.dss_service.get_catalog(source)
        cat_filtered = cat.filter_catalog(path)
        cat_filtered.set_editable()
        cat_filtered.remove(cat_filtered.entries()[0])

    def test_read_aggregated_rts_timewindow(self):
        ## save some ts into dss file, ts is hourly averaged

        ## save rts first.
        data = range(1000)
        start = "12/21/2000 2:00"
        interval = "1hour"
        prop = {}
        prop[TIMESTAMP] = PERIOD_START
        prop[AGGREGATION] = MEAN
        rt1 = rts(data, start, interval, prop)

        id = "vtools.datastore.dss.DssService"
        path = "/TEST/DOWNSTREAM/EC//1HOUR/DWR/"
        source = self.test_file_path
        data_ref = DataReference(id, source=source, selector=path)
        self.dss_service.add_data(data_ref, rt1)

        ## test return part of stored data up to the end
        ## it should get 992 numbers and value is (8,9,...,1000)
        ## it start datetime shoudl be 12/21/2000 10:00
        extent = "time_window=(12/21/2000 10:00,01/31/2001 18:00)"
        data_ref = DataReference(id, source, None, path, extent)
        rtt2 = self.dss_service.get_data(data_ref)
        self.assertTrue(rtt2.start == parse_time("12/21/2000 10:00"))
        self.assertTrue(len(rtt2) == 992)
        correct_data = range(8, len(rtt2) + 8)
        for i in range(len(rtt2)):
            self.assertTrue(rtt2.data[i] == float(correct_data[i]))

        ## test return middle part of stored data
        ## it should get 12 numbers and value is (8,9,...,19)
        ## it start datetime should be 12/21/2000 10:00, end
        ## at 12/21/2000 21:00 (not include the late side)
        extent = "time_window=(12/21/2000 10:00,12/21/2000 22:00)"
        data_ref = DataReference(id, source, None, path, extent)
        rtt2 = self.dss_service.get_data(data_ref)
        self.assertTrue(rtt2.start == parse_time("12/21/2000 10:00"))
        self.assertTrue(rtt2.end == parse_time("12/21/2000 21:00"))
        self.assertTrue(len(rtt2) == 12)
        correct_data = range(8, len(rtt2) + 8)
        for i in range(len(rtt2)):
            self.assertTrue(rtt2.data[i] == float(correct_data[i]))

        ## test return middle part of stored data
        ## it should get 12 numbers and value is (8,9,...,19)
        ## it start datetime should be 12/21/2000 10:00, end
        ## at 12/21/2000 21:00 (not include the late side)
        ## time window is not given at the correct hourly time points.
        extent = "time_window=(12/21/2000 09:45,12/21/2000 22:15)"
        data_ref = DataReference(id, source, None, path, extent)
        rtt2 = self.dss_service.get_data(data_ref)
        self.assertTrue(rtt2.start == parse_time("12/21/2000 10:00"))
        self.assertTrue(rtt2.end == parse_time("12/21/2000 21:00"))
        self.assertTrue(len(rtt2) == 12)
        correct_data = range(8, len(rtt2) + 8)
        for i in range(len(rtt2)):
            self.assertTrue(rtt2.data[i] == float(correct_data[i]))

        ## test valid timewindow overlap exaclty the last data of
        ## the record
        extent = "time_window=(1/31/2001 17:00,1/31/2001 18:00)"
        data_ref = DataReference(id, source, None, path, extent)
        rtt2 = self.dss_service.get_data(data_ref)
        self.assertTrue(len(rtt2) == 1)
        self.assertTrue(rtt2.data[0] == float(999))

        ## test invalid time window with same start and end
        ## excatly at beginig time sequence
        extent = "time_window=(12/21/2000 02:00,12/21/2000 02:00)"
        data_ref = DataReference(id, source, None, path, extent)
        rtt2 = self.dss_service.get_data(data_ref)
        self.assertTrue(len(rtt2) == 0)

        ## test invalid time window with same start and end in the
        ## middle of time sequence
        extent = "time_window=(12/21/2000 05:00,12/21/2000 05:00)"
        data_ref = DataReference(id, source, None, path, extent)
        rtt2 = self.dss_service.get_data(data_ref)
        self.assertTrue(len(rtt2) == 0)

        ## test invalid time window with same start and end at the
        ## end of time sequence
        extent = "time_window=(12/21/2000 17:00,12/21/2000 17:00)"
        data_ref = DataReference(id, source, None, path, extent)
        rtt2 = self.dss_service.get_data(data_ref)
        self.assertTrue(len(rtt2) == 0)

        ## test invalid time window with same start and end not aligined with interval
        extent = "time_window=(12/21/2000 05:15,12/21/2000 05:15)"
        data_ref = DataReference(id, source, None, path, extent)
        rtt2 = self.dss_service.get_data(data_ref)
        self.assertTrue(len(rtt2) == 0)

        ## test invalid time window with different start and end within a hour interval
        extent = "time_window=(12/21/2000 05:15,12/21/2000 05:55)"
        data_ref = DataReference(id, source, None, path, extent)
        rtt2 = self.dss_service.get_data(data_ref)
        self.assertTrue(len(rtt2) == 0)

        ## test invalid time window with different start and end across two hour intervals
        ## but intervals are incomplete, so it should return no value
        extent = "time_window=(12/21/2000 05:15,12/21/2000 06:55)"
        data_ref = DataReference(id, source, None, path, extent)
        rtt2 = self.dss_service.get_data(data_ref)
        self.assertTrue(len(rtt2) == 0)

        ## test invalid time window with same start and end
        ## excatly at the middle time sequence
        extent = "time_window=(12/21/2000 17:15,12/21/2000 17:15)"
        data_ref = DataReference(id, source, None, path, extent)
        rtt2 = self.dss_service.get_data(data_ref)
        self.assertTrue(len(rtt2) == 0)

        ## test valid time window overlap exactly the first data
        ## at the begining
        extent = "time_window=(12/21/2000 02:00,12/21/2000 03:00)"
        data_ref = DataReference(id, source, None, path, extent)
        rtt2 = self.dss_service.get_data(data_ref)
        self.assertTrue(len(rtt2) == 1)
        self.assertTrue(rtt2.data[0] == float(0))

        ## test valid time window overlap exactly a data in the middle
        extent = "time_window=(12/21/2000 05:00,12/21/2000 06:00)"
        data_ref = DataReference(id, source, None, path, extent)
        rtt2 = self.dss_service.get_data(data_ref)
        self.assertTrue(len(rtt2) == 1)
        self.assertTrue(rtt2.data[0] == float(3))

        ## test valid time window overlap exactly a data at the end
        extent = "time_window=(1/31/2001 17:00,1/31/2001 18:00)"
        data_ref = DataReference(id, source, None, path, extent)
        rtt2 = self.dss_service.get_data(data_ref)
        self.assertTrue(len(rtt2) == 1)
        self.assertTrue(rtt2.data[0] == float(999))

        ## test invalid timewindow before the data starting
        ## but still overlap data block window
        extent = "time_window=(12/21/2000 00:00,12/21/2000 1:00)"
        data_ref = DataReference(id, source, None, path, extent)
        self.assertRaises(ValueError, self.dss_service.get_data, data_ref)

        ## test invalid timewindow overlapping data block window,
        extent = "time_window=(1/31/2001 18:00,1/31/2001 22:00)"
        data_ref = DataReference(id, source, None, path, extent)
        self.assertRaises(ValueError, self.dss_service.get_data, data_ref)

        ## test invalid timewindow not overlapping data block window
        extent = "time_window=(11/21/2000 00:00,11/22/2000 1:00)"
        data_ref = DataReference(id, source, None, path, extent)
        self.assertRaises(ValueError, self.dss_service.get_data, data_ref)

    def test_read_instant_rts_timewindow(self):
        ## save some ts into dss file, ts is hourly spaned instanteous

        ## save rts first.
        data = range(1000)
        start = "12/21/2000 2:00"
        interval = "1hour"
        prop = {}

        rt1 = rts(data, start, interval, prop)

        id = "vtools.datastore.dss.DssService"
        path = "/TEST/DOWNSTREAM/EC//1HOUR/STAGE/"
        source = self.test_file_path
        data_ref = DataReference(id, source=source, selector=path)
        self.dss_service.add_data(data_ref, rt1)

        ## test returning part of stored data up to the end
        ## it should get 992 numbers and value is (8,9,...,1000)
        ## it start datetime shoudl be 12/21/2000 10:00
        extent = "time_window=(12/21/2000 10:00,01/31/2001 18:00)"
        data_ref = DataReference(id, source, None, path, extent)
        rtt2 = self.dss_service.get_data(data_ref)
        self.assertTrue(rtt2.start == parse_time("12/21/2000 10:00"))
        self.assertTrue(len(rtt2) == 992)
        correct_data = range(8, len(rtt2) + 8)
        for i in range(len(rtt2)):
            self.assertTrue(rtt2.data[i] == float(correct_data[i]))

        ## test returning middle part of stored data
        ## it should get 13 numbers and value is (8,9,...,19,20)
        ## it start datetime should be 12/21/2000 10:00, end
        ## at 12/21/2000 22:00 (include the later side)
        extent = "time_window=(12/21/2000 10:00,12/21/2000 22:00)"
        data_ref = DataReference(id, source, None, path, extent)
        rtt2 = self.dss_service.get_data(data_ref)
        self.assertTrue(rtt2.start == parse_time("12/21/2000 10:00"))
        self.assertTrue(rtt2.end == parse_time("12/21/2000 22:00"))
        self.assertTrue(len(rtt2) == 13)
        correct_data = range(8, len(rtt2) + 8)
        for i in range(len(rtt2)):
            self.assertTrue(rtt2.data[i] == float(correct_data[i]))

        ## test valid timewindow overlap exaclty the last data of
        ## the record
        extent = "time_window=(1/31/2001 17:00,1/31/2001 17:00)"
        data_ref = DataReference(id, source, None, path, extent)
        rtt2 = self.dss_service.get_data(data_ref)
        self.assertTrue(len(rtt2) == 1)
        self.assertTrue(rtt2.data[0] == float(999))

        ## test valid time window with same start and end
        ## excatly at begining of the time sequence
        extent = "time_window=(12/21/2000 02:00,12/21/2000 02:00)"
        data_ref = DataReference(id, source, None, path, extent)
        rtt2 = self.dss_service.get_data(data_ref)
        self.assertTrue(len(rtt2) == 1)
        self.assertTrue(rtt2.data[0] == float(0))

        ## test valid time window overlap exactly a data in the middle
        extent = "time_window=(12/21/2000 05:00,12/21/2000 05:00)"
        data_ref = DataReference(id, source, None, path, extent)
        rtt2 = self.dss_service.get_data(data_ref)
        self.assertTrue(len(rtt2) == 1)
        self.assertTrue(rtt2.data[0] == float(3))

        ## test invalid time window with same end and start not aligned with
        ## interval
        extent = "time_window=(12/21/2000 05:15,12/21/2000 05:15)"
        data_ref = DataReference(id, source, None, path, extent)
        rtt2 = self.dss_service.get_data(data_ref)
        self.assertTrue(len(rtt2) == 0)

        ## test invalid time window in the middle with the same earlier and later side
        ## not aligns with time sequence
        extent = "time_window=(12/21/2000 05:15,12/21/2000 05:15)"
        data_ref = DataReference(id, source, None, path, extent)
        rtt2 = self.dss_service.get_data(data_ref)
        self.assertTrue(len(rtt2) == 0)

        ## test invalid timewindow before the data starting
        ## but still overlap data block window
        extent = "time_window=(12/21/2000 00:00,12/21/2000 1:00)"
        data_ref = DataReference(id, source, None, path, extent)
        self.assertRaises(ValueError, self.dss_service.get_data, data_ref)

        ## test invalid timewindow overlapping data block window,
        extent = "time_window=(1/31/2001 18:00,1/31/2001 22:00)"
        data_ref = DataReference(id, source, None, path, extent)
        self.assertRaises(ValueError, self.dss_service.get_data, data_ref)

        ## test invalid timewindow not overlapping data block window
        extent = "time_window=(11/21/2000 00:00,11/22/2000 1:00)"
        data_ref = DataReference(id, source, None, path, extent)
        self.assertRaises(ValueError, self.dss_service.get_data, data_ref)

    def test_save_data(self):
        ## save some ts into dss file, ts may contain
        ## header.

        ## save rts first.
        data = range(1000)
        start = "12/21/2000 2:00"
        interval = "1hour"
        prop = {}
        prop[TIMESTAMP] = PERIOD_START
        prop[AGGREGATION] = MEAN

        prop["datum"] = "NGVD88"
        prop["manager"] = "John Doe"
        prop["model"] = "hydro 7.5"

        rt1 = rts(data, start, interval, prop)

        id = "vtools.datastore.dss.DssService"
        path = "/TEST/DOWNSTREAM/EC//1HOUR/STAGE/"
        source = self.test_file_path
        data_ref = DataReference(id, source=source, selector=path)
        self.dss_service.add_data(data_ref, rt1)
        dssc = self.dss_service.get_catalog(source)
        path = "/TEST/DOWNSTREAM/EC//1HOUR/STAGE/"
        data_ref = dssc.data_references(path).next()
        rtt = self.dss_service.get_data(data_ref)
        self.assertTrue(len(rtt) == len(data))
        self.assertTrue(rtt.props[TIMESTAMP] == PERIOD_START)
        self.assertTrue(rtt.props[AGGREGATION] == MEAN)
        self.assertTrue(rtt.times[0], dtm.datetime(2000, 12, 21, 2))
        extent = "time_window=(12/21/2000 02:00,01/31/2001 18:00)"
        data_ref = DataReference(id, source, None, path, extent)
        rtt2 = self.dss_service.get_data(data_ref)
        self.assertTrue(rtt.start == rtt2.start)
        self.assertTrue(rtt.end == rtt2.end)

        ## then its.

        path = "/HERE/IS/ITS//IR-YEAR/TEST/"
        data = range(20)
        data_ref = DataReference(id, source=source, selector=path)
        prop[AGGREGATION] = INDIVIDUAL

        times=["01/15/1997","02/17/1997","03/5/1997",\
               "04/25/1997","05/1/1997","06/15/1997",\
               "07/25/1997","08/14/1997","09/17/1997",\
               "10/15/1997","11/21/1997","12/3/1997",\
               "01/9/1998","02/15/1998","03/19/1998",\
               "04/15/1998","05/19/1998","06/30/1998",\
               "07/15/1998","08/24/1998"]

        times = map(parse_time, times)
        itt = its(times, data, prop)
        self.dss_service.add_data(data_ref, itt)
        extent = "time_window=(1/10/1997 02:00,09/30/1998 18:00)"
        data_ref = DataReference(id, source, None, path, extent)
        rtt3 = self.dss_service.get_data(data_ref)
        self.assertTrue(parse_time("01/15/1997") == rtt3.start)
        self.assertTrue(parse_time("08/24/1998") == rtt3.end)

    def test_retrievesave_longits(self):
        ## save some ts into dss file, ts may contain
        ## header.

        ## create rts first, and use it ticks and
        ## data to create its
        total_len = 700000
        its_len = 40000  ## 80000 will cause zsitx fail,for dss interal
        ## cache size is 10000. data size
        ## close to this number will cause dss lib abort
        data = range(total_len)
        start = "12/21/1940 2:00"
        interval = "1hour"
        prop = {}
        prop[TIMESTAMP] = INST
        prop[AGGREGATION] = INDIVIDUAL

        prop["datum"] = "NGVD88"
        prop["manager"] = "John Doe"
        prop["model"] = "hydro 7.5"

        rt1 = rts(data, start, interval, prop)

        id = "vtools.datastore.dss.DssService"
        path = "/TEST/DOWNSTREAM/EC//1HOUR/SRT/"
        source = self.test_file_path

        data_ref = DataReference(id, source=source, selector=path)
        self.dss_service.add_data(data_ref, rt1)

        path = "/TEST/DOWNSTREAM/EC//IR-DAY/STAGE/"
        data_ref = DataReference(id, source=source, selector=path)
        i = 0
        start = i * its_len
        end = (i + 1) * its_len
        it1 = its(data[start:end], rt1.ticks[start:end], prop)
        self.dss_service.add_data(data_ref, it1)


#        i=1
#        start = i*its_len
#        end   = (i+1)*its_len
#        it1=its(data[start:end],rt1.ticks[start:end],prop)
#        self.dss_service.add_data(data_ref,it1)
#        i=2
#        start = i*its_len
#        end   = (i+1)*its_len
#        it1=its(data[start:end],rt1.ticks[start:end],prop)
#        self.dss_service.add_data(data_ref,it1)
#        i=3
#        start = i*its_len
#        end   = (i+1)*its_len
#        it1=its(data[start:end],rt1.ticks[start:end],prop)
#        self.dss_service.add_data(data_ref,it1)

#        print "all done"
#        residual_len = total_len - round*its_len
#        if (residual_len>0):
#            start = rounds*its_len
#            end   =start+residual_len
#            it1=its(data[start:end],rt1.ticks[start:end],prop)
#            self.dss_service.add_data(data_ref,it1)

    def test_save2newf(self):
        """ try to save ts to a non exist file."""

        ## save rts first.
        data = range(1000)
        start = "12/21/2000 2:00"
        interval = "1hour"
        prop = {}
        prop[TIMESTAMP] = PERIOD_START
        prop[AGGREGATION] = MEAN

        prop["datum"] = "NGVD88"
        prop["manager"] = "John Doe"
        prop["model"] = "hydro 7.5"

        rt1 = rts(data, start, interval, prop)

        id = "vtools.datastore.dss.DssService"
        path = "/TEST/DOWNSTREAM/EC//1HOUR/STAGE/"
        source = 'newdss.dss'

        data_ref = DataReference(id, source=source, selector=path)
        self.dss_service.add_data(data_ref, rt1)

        self.assertTrue(os.path.exists(source))

    def test_get_two_catalog_same_time(self):
        """ test get two catlogs of same file"""
        c1 = self.dss_service.get_catalog(self.test_file_path)
        c2 = self.dss_service.get_catalog(self.test_file_path)
        self.assertEqual(len(c1), len(c2))

    def test_get_save_ts_manytimes(self):
        """ get some ts from test file and save it back, repeat many times"""

        selector = "/RLTM+CHAN/*/*//*/*/"
        loops = 100
        print "this is a time consuming test case ,be patient..."
        for i in range(loops):
            c = self.dss_service.get_catalog(self.test_file_path)
            data_ref = [df for df in c.data_references(selector)]
            tslist = []
            for a_data_ref in data_ref:
                ts = self.dss_service.get_data(a_data_ref)
                tslist.append(ts)

            for a_data_ref, ts in zip(data_ref, tslist):
                self.dss_service.add_data(a_data_ref, ts)
        print "finish repeating get and save ts %i rounds" % loops