Beispiel #1
0
    def test_flushIndex(self):
        # set up
        index1 = XmlIndex(self.rt1, "/station/station_code", label="code")
        index2 = XmlIndex(self.rt1, "/station/XY/paramXY", label="paramXY")
        self.catalog.registerIndex(index1)
        self.catalog.registerIndex(index2)
        res = Resource(self.rt1, document=newXMLDocument(RAW_XML2))
        self.xmldb.addResource(res)
        self.catalog.indexResource(res)

        # index1 and index2 contain data
        el = self.catalog.dumpIndex(index1)
        self.assertEquals(len(el), 1)
        el = self.catalog.dumpIndex(index2)
        self.assertEquals(len(el), 3)
        # flush index1
        self.catalog.flushIndex(index1)
        # index1 data has gone
        el = self.catalog.dumpIndex(index1)
        self.assertEquals(len(el), 0)
        # index2 data still there
        el = self.catalog.dumpIndex(index2)
        self.assertEquals(len(el), 3)
        # flush index2
        self.catalog.flushIndex(index2)
        el = self.catalog.dumpIndex(index2)
        self.assertEquals(len(el), 0)
        # clean up:
        self.catalog.deleteIndex(index1)
        self.catalog.deleteIndex(index2)
        self.xmldb.deleteResource(res)
Beispiel #2
0
    def test_getIndexes(self):
        """
        """
        index_rt1 = XmlIndex(self.rt1, "/station/XY/paramXY", label='id1')
        index2_rt1 = XmlIndex(self.rt1, "/station/station_code", label='id2')
        index_rt2 = XmlIndex(self.rt2, "/station/XY/paramXY", label='id3')
        self.catalog.registerIndex(index_rt1)
        self.catalog.registerIndex(index2_rt1)
        self.catalog.registerIndex(index_rt2)

        # get by resourcetype:
        res = self.catalog.getIndexes(package_id=self.rt1.package.package_id,
                                      resourcetype_id=self.rt1.resourcetype_id)
        self.assertEquals(len(res), 2)
        self.assertEquals(res[0].resourcetype.resourcetype_id,
                          self.rt1.resourcetype_id)
        self.assertEquals(res[0].xpath, index_rt1.xpath)
        self.assertEquals(res[1].resourcetype.resourcetype_id,
                          self.rt1.resourcetype_id)
        self.assertEquals(res[1].xpath, index2_rt1.xpath)
        res = self.catalog.getIndexes(package_id=self.rt2.package.package_id,
                                      resourcetype_id=self.rt2.resourcetype_id)
        self.assertEquals(len(res), 1)
        self.assertEquals(res[0].resourcetype.resourcetype_id,
                          self.rt2.resourcetype_id)
        self.assertEquals(res[0].xpath, index_rt2.xpath)

        # get by xpath
        res = self.catalog.getIndexes(xpath="/station/XY/paramXY")
        self.assertEquals(len(res), 2)
        self.assertEquals(res[0].resourcetype.resourcetype_id,
                          self.rt1.resourcetype_id)
        self.assertEquals(res[0].xpath, index_rt1.xpath)
        self.assertEquals(res[1].resourcetype.resourcetype_id,
                          self.rt2.resourcetype_id)
        self.assertEquals(res[1].xpath, index_rt2.xpath)
        res = self.catalog.getIndexes(xpath="/station/station_code")
        self.assertEquals(len(res), 1)
        self.assertEquals(res[0].resourcetype.resourcetype_id,
                          self.rt1.resourcetype_id)
        self.assertEquals(res[0].xpath, index2_rt1.xpath)

        # get by rt and xpath
        res = self.catalog.getIndexes(package_id=self.rt1.package.package_id,
                                      resourcetype_id=self.rt1.resourcetype_id,
                                      xpath="/station/XY/paramXY")
        self.assertEquals(len(res), 1)
        self.assertEquals(res[0].resourcetype.resourcetype_id,
                          self.rt1.resourcetype_id)
        self.assertEquals(res[0].xpath, "/station/XY/paramXY")

        # remove:
        self.catalog.deleteIndex(index_rt1)
        self.catalog.deleteIndex(index2_rt1)
        self.catalog.deleteIndex(index_rt2)
 def test_TimestampIndex(self):
     """
     Tests indexing of timestamps.
     """
     # w/ microseconds
     dt = UTCDateTime(2008, 10, 23, 11, 53, 12, 54000)
     idx = XmlIndex(self.rt1, "/station/creation_date",
                    index.TIMESTAMP_INDEX)
     timestr = "%f" % dt.timestamp
     doc = newXMLDocument(RAW_XML1 % (timestr, ""))
     res = idx.eval(doc, self.env)[0]
     self.assertEqual(res.key, dt.datetime)
     # w/o microseconds
     dt = UTCDateTime(2008, 10, 23, 11, 53, 12)
     idx = XmlIndex(self.rt1, "/station/creation_date",
                    index.TIMESTAMP_INDEX)
     timestr = "%f" % dt.timestamp
     doc = newXMLDocument(RAW_XML1 % (timestr, ""))
     res = idx.eval(doc, self.env)[0]
     self.assertEqual(res.key, dt.datetime)
     # negative timestamp works too
     dt = UTCDateTime(1969, 12, 31, 23, 36, 39, 500000)
     idx = XmlIndex(self.rt1, "/station/creation_date",
                    index.TIMESTAMP_INDEX)
     timestr = "%f" % dt.timestamp
     doc = newXMLDocument(RAW_XML1 % (timestr, ""))
     res = idx.eval(doc, self.env)[0]
     self.assertEqual(res.key, dt.datetime)
Beispiel #4
0
 def test_indexResource(self):
     # set up
     res = Resource(self.rt1, document=newXMLDocument(RAW_XML2))
     self.xmldb.addResource(res)
     index1 = XmlIndex(self.rt1, "/station/station_code", label='idx1')
     index2 = XmlIndex(self.rt1, "/station/XY/paramXY", label='idx2')
     index3 = XmlIndex(self.rt1, "/station/test_date", FLOAT_INDEX)
     self.catalog.registerIndex(index1)
     self.catalog.registerIndex(index2)
     self.catalog.registerIndex(index3)
     # index resource
     r = self.catalog.indexResource(res)
     self.assertEquals(len(r), 5)
     # dump first index
     el = self.catalog.dumpIndex(index1)
     self.assertEquals(len(el), 1)
     self.assertEquals(el[0].key, "GENF")
     self.assertEquals(el[0].document.data, res.document.data)
     # dump second index
     el = self.catalog.dumpIndex(index2)
     self.assertEqual(len(el), 3)
     keys = ["0", "2.5", "99"]
     for e in el:
         assert e.key in keys
         keys.remove(e.key)
         self.assertEquals(e.document.data, res.document.data)
     # dumpIndexByResource
     el = self.catalog.dumpIndexByResource(res)
     self.assertEqual(len(el), 5)
     self.assertEquals(el[0].key, "GENF")
     self.assertEquals(el[0].document.data, res.document.data)
     self.assertEquals(el[0].index.xpath, "/station/station_code")
     self.assertTrue(el[1].key in ["0", "2.5", "99"])
     self.assertEquals(el[1].document.data, res.document.data)
     self.assertEquals(el[1].index.xpath, "/station/XY/paramXY")
     self.assertTrue(el[2].key in ["0", "2.5", "99"])
     self.assertEquals(el[2].document.data, res.document.data)
     self.assertEquals(el[2].index.xpath, "/station/XY/paramXY")
     self.assertTrue(el[3].key in ["0", "2.5", "99"])
     self.assertEquals(el[3].document.data, res.document.data)
     self.assertEquals(el[3].index.xpath, "/station/XY/paramXY")
     self.assertEquals(el[4].key, 2008.123456789)
     self.assertEquals(el[4].document.data, res.document.data)
     self.assertEquals(el[4].index.xpath, "/station/test_date")
     # clean up
     self.catalog.deleteIndex(index1)
     self.catalog.deleteIndex(index2)
     self.catalog.deleteIndex(index3)
     self.xmldb.deleteResource(res)
Beispiel #5
0
 def test_deleteIndex(self):
     """
     """
     index = XmlIndex(self.rt1, "/station/XY/paramXY", label='test')
     self.catalog.registerIndex(index)
     res = self.catalog.getIndexes(package_id=self.rt1.package.package_id,
                                   resourcetype_id=self.rt1.resourcetype_id,
                                   xpath="/station/XY/paramXY")
     self.assertEquals(len(res), 1)
     self.catalog.deleteIndex(index)
     res = self.catalog.getIndexes(package_id=self.rt1.package.package_id,
                                   resourcetype_id=self.rt1.resourcetype_id,
                                   xpath="/station/XY/paramXY")
     self.assertEquals(len(res), 0)
    def testIndexCommon(self):
        # index with single node key result:
        si = XmlIndex(self.rt1, xpath="/station/station_code")
        # index with multiple nodes key result:
        mi = XmlIndex(self.rt1, xpath="/station/XY/paramXY")
        # index which is empty on test resource
        ei = XmlIndex(self.rt1, xpath="/station/empty")
#        # index which does not exists on test resource
#        ni = XmlIndex(self.rt1, xpath="/station/XXX")

        test_doc = newXMLDocument(RAW_XML1)

        class Foo(object):
            pass

        # pass a Foo: (which does not implement IXmlDoc)
        self.assertRaises(TypeError, si.eval, Foo())
        # pass an empty XmlDoc:
        self.assertRaises(SeisHubError, si.eval, XmlDocument())

        res = si.eval(test_doc, self.env)
        self.assertEquals(len(res), 1)
        self.assertEquals(res[0].index, si)
        self.assertEquals(res[0].document, test_doc)
        self.assertEquals(res[0].key, 'BERN')

        res = mi.eval(test_doc, self.env)
        self.assertEquals(len(res), 3)
        self.assertEquals(res[0].index, mi)
        self.assertEquals(res[0].document, test_doc)
        self.assertEquals(res[0].key, '20.5')
        self.assertEquals(res[1].index, mi)
        self.assertEquals(res[1].document, test_doc)
        self.assertEquals(res[1].key, '11.5')
        self.assertEquals(res[2].index, mi)
        self.assertEquals(res[2].document, test_doc)
        self.assertEquals(res[2].key, 'blah')

        res = ei.eval(test_doc, self.env)
        self.assertEquals(len(res), 1)
 def testNumericIndex(self):
     test_doc = newXMLDocument(RAW_XML1)
     idx = XmlIndex(self.rt1, "/station/lon", index.NUMERIC_INDEX)
     res = idx.eval(test_doc, self.env)[0]
     self.assertEquals(type(res), NumericIndexElement)
     # self.assertEquals(type(res.key), float)
     self.assertEquals(res.key, '12.51200')
     # elements with wrong data type are ignored
     idx = XmlIndex(self.rt1, "/station/XY/paramXY", index.NUMERIC_INDEX)
     res = idx.eval(test_doc, self.env)
     self.assertEquals(len(res), 2)
 def testTextIndex(self):
     test_doc = newXMLDocument(RAW_XML1)
     idx = XmlIndex(self.rt1, "/station/lon", index.TEXT_INDEX)
     res = idx.eval(test_doc, self.env)[0]
     self.assertEquals(type(res), index.TextIndexElement)
     self.assertEquals(type(res.key), unicode)
     self.assertEquals(res.key, '12.51200')
     # index which is empty on test resource
     idx = XmlIndex(self.rt1, "/station/empty", index.TEXT_INDEX)
     res = idx.eval(test_doc, self.env)
     self.assertEquals(len(res), 1)
 def test_DateIndex(self):
     """
     Tests indexing of dates.
     """
     dt = datetime(2008, 10, 10, 11, 53, 0, 54000)
     # ISO 8601 w/o minus
     idx = XmlIndex(self.rt1, "/station/creation_date", index.DATE_INDEX)
     timestr = dt.strftime("%Y%m%d")
     doc = newXMLDocument(RAW_XML1 % (timestr, ""))
     res = idx.eval(doc, self.env)[0]
     self.assertEqual(res.key, dt.date())
     # ISO 8601 w/ minus
     idx = XmlIndex(self.rt1, "/station/creation_date", index.DATE_INDEX)
     timestr = dt.strftime("%Y-%m-%d")
     doc = newXMLDocument(RAW_XML1 % (timestr, ""))
     res = idx.eval(doc, self.env)[0]
     self.assertEqual(res.key, dt.date())
     # custom format
     idx = XmlIndex(self.rt1, "/station/creation_date", index.DATE_INDEX,
                    options="%d.%m.%Y")
     timestr = dt.strftime("%d.%m.%Y")
     doc = newXMLDocument(RAW_XML1 % (timestr, ""))
     res = idx.eval(doc, self.env)[0]
     self.assertEqual(res.key, dt.date())
Beispiel #10
0
 def test_indexResourceWithGrouping(self):
     # set up
     res = Resource(self.rt1, document=newXMLDocument(RAW_XML4))
     self.xmldb.addResource(res)
     index = XmlIndex(self.rt1, "/station/XY/Z/value",
                      group_path="/station/XY")
     self.catalog.registerIndex(index)
     r = self.catalog.indexResource(res)
     self.assertEquals(len(r), 2)
     el = self.catalog.dumpIndex(index)
     self.assertEquals(len(el), 2)
     self.assertEquals(el[0].key, "3")
     self.assertEquals(el[0].group_pos, 0)
     self.assertEquals(el[0].document.data, res.document.data)
     self.assertEquals(el[1].key, "6")
     self.assertEquals(el[1].group_pos, 1)
     self.assertEquals(el[1].document.data, res.document.data)
     # clean up
     self.catalog.deleteIndex(index)
     self.xmldb.deleteResource(res)
Beispiel #11
0
    def test_registerIndex(self):
        """
        """
        index = XmlIndex(self.rt1, "/station/XY/paramXY", DATETIME_INDEX,
                         "%Y/%m", label='test')
        self.catalog.registerIndex(index)

        res = self.catalog.getIndexes(package_id=self.rt1.package.package_id,
                                      resourcetype_id=self.rt1.resourcetype_id,
                                      xpath="/station/XY/paramXY")[0]
        self.assertEquals(res.resourcetype.resourcetype_id,
                          self.rt1.resourcetype_id)
        self.assertEquals(res.xpath, "/station/XY/paramXY")
        self.assertEquals(res.type, DATETIME_INDEX)
        self.assertEquals(res.options, "%Y/%m")
        # try to add a duplicate:
        self.assertRaises(DuplicateObjectError,
                          self.catalog.registerIndex, index)
        # clean up:
        self.catalog.deleteIndex(index)
Beispiel #12
0
    def testBooleanIndex(self):
        idx = XmlIndex(self.rt1, "/station/bool", index.BOOLEAN_INDEX)
        doc = newXMLDocument(RAW_XML1 % ("", "True"))
        res = idx.eval(doc, self.env)[0]
        self.assertEqual(res.key, True)

        doc = newXMLDocument(RAW_XML1 % ("", "False"))
        res = idx.eval(doc, self.env)[0]
        self.assertEqual(res.key, False)

        doc = newXMLDocument(RAW_XML1 % ("", "1"))
        res = idx.eval(doc, self.env)[0]
        self.assertEqual(res.key, True)

        doc = newXMLDocument(RAW_XML1 % ("", "0"))
        res = idx.eval(doc, self.env)[0]
        self.assertEqual(res.key, False)

        doc = newXMLDocument(RAW_XML1 % ("", "something"))
        res = idx.eval(doc, self.env)[0]
        self.assertEqual(res.key, True)
Beispiel #13
0
    def testIndexGrouping(self):
        doc = newXMLDocument(RAW_XML2)
        idx1 = XmlIndex(self.rt1, "/station/XY/X", index.NUMERIC_INDEX,
                        group_path="/station/XY")
        res = idx1.eval(doc, self.env)
        self.assertEqual(len(res), 2)
        self.assertEqual(res[0].key, '1')
        self.assertEqual(res[0].group_pos, 0)
        self.assertEqual(res[1].key, '4')
        self.assertEqual(res[1].group_pos, 1)

        idx2 = XmlIndex(self.rt1, "/station/XY/Y", index.NUMERIC_INDEX,
                        group_path="/station/XY")
        res = idx2.eval(doc, self.env)
        self.assertEqual(len(res), 2)
        self.assertEqual(res[0].key, '2')
        self.assertEqual(res[0].group_pos, 0)
        self.assertEqual(res[1].key, '5')
        self.assertEqual(res[1].group_pos, 1)

        idx3 = XmlIndex(self.rt1, "/station/XY/Y/@id", index.NUMERIC_INDEX,
                        group_path="/station/XY")
        res = idx3.eval(doc, self.env)
        self.assertEqual(len(res), 2)
        self.assertEqual(res[0].key, '1')
        self.assertEqual(res[0].group_pos, 0)
        self.assertEqual(res[1].key, '2')
        self.assertEqual(res[1].group_pos, 1)

        idx4 = XmlIndex(self.rt1, "/station/XY/Z/value", index.NUMERIC_INDEX,
                        group_path="/station/XY")
        res = idx4.eval(doc, self.env)
        self.assertEqual(len(res), 2)
        self.assertEqual(res[0].key, '3')
        self.assertEqual(res[0].group_pos, 0)
        self.assertEqual(res[1].key, '6')
        self.assertEqual(res[1].group_pos, 1)
Beispiel #14
0
 def test_DateTimeIndex(self):
     """
     Tests indexing of datetimes.
     """
     # setup 
     dt = datetime(2008, 10, 23, 11, 53, 12, 54000)
     dt2 = datetime(2008, 10, 23, 11, 53, 12)
     dt3 = datetime(2008, 10, 23)
     dt4 = datetime(2008, 10, 23, 11)
     dt5 = datetime(2008, 10, 23, 11 , 53)
     # ISO 8601
     idx = XmlIndex(self.rt1, "/station/creation_date",
                    index.DATETIME_INDEX)
     timestr = dt.strftime("%Y%m%dT%H:%M:%S") + ".054000"
     doc = newXMLDocument(RAW_XML1 % (timestr, ""))
     res = idx.eval(doc, self.env)[0]
     self.assertEqual(res.key, dt)
     # ISO 8601 w/ minus
     idx = XmlIndex(self.rt1, "/station/creation_date",
                    index.DATETIME_INDEX)
     timestr = dt.strftime("%Y-%m-%dT%H:%M:%S") + ".054000"
     doc = newXMLDocument(RAW_XML1 % (timestr, ""))
     res = idx.eval(doc, self.env)[0]
     self.assertEqual(res.key, dt)
     # ISO 8601 w/ time zone
     idx = XmlIndex(self.rt1, "/station/creation_date",
                    index.DATETIME_INDEX)
     timestr = dt.strftime("%Y%m%dT%H:%M:%S") + ".054000Z"
     doc = newXMLDocument(RAW_XML1 % (timestr, ""))
     res = idx.eval(doc, self.env)[0]
     self.assertEqual(res.key, dt)
     # ISO 8601 w/o T
     idx = XmlIndex(self.rt1, "/station/creation_date",
                    index.DATETIME_INDEX)
     timestr = dt.strftime("%Y%m%d %H:%M:%S") + ".054000"
     doc = newXMLDocument(RAW_XML1 % (timestr, ""))
     res = idx.eval(doc, self.env)[0]
     self.assertEqual(res.key, dt)
     # ISO 8601 w/o milliseconds 
     idx = XmlIndex(self.rt1, "/station/creation_date",
                    index.DATETIME_INDEX)
     timestr = dt2.strftime("%Y%m%dT%H:%M:%S")
     doc = newXMLDocument(RAW_XML1 % (timestr, ""))
     res = idx.eval(doc, self.env)[0]
     self.assertEqual(res.key, dt2)
     # ISO 8601 w/o time - defaults to 00:00:00
     idx = XmlIndex(self.rt1, "/station/creation_date",
                    index.DATETIME_INDEX)
     timestr = dt3.strftime("%Y-%m-%d")
     doc = newXMLDocument(RAW_XML1 % (timestr, ""))
     res = idx.eval(doc, self.env)[0]
     self.assertEqual(res.key, dt3)
     # ISO 8601 w/o minutes - defaults to :00:00
     idx = XmlIndex(self.rt1, "/station/creation_date",
                    index.DATETIME_INDEX)
     timestr = dt4.strftime("%Y-%m-%dT%H")
     doc = newXMLDocument(RAW_XML1 % (timestr, ""))
     res = idx.eval(doc, self.env)[0]
     self.assertEqual(res.key, dt4)
     # ISO 8601 w/o seconds - defaults to :00
     idx = XmlIndex(self.rt1, "/station/creation_date",
                    index.DATETIME_INDEX)
     timestr = dt5.strftime("%Y-%m-%dT%H:%M")
     doc = newXMLDocument(RAW_XML1 % (timestr, ""))
     res = idx.eval(doc, self.env)[0]
     self.assertEqual(res.key, dt5)
     # with custom format
     # microseconds are ignored since not supported by strftime()
     idx = XmlIndex(self.rt1, "/station/creation_date",
                    index.DATETIME_INDEX, "%H:%M:%S - %Y%m%d")
     timestr = dt.strftime("%H:%M:%S - %Y%m%d")
     doc = newXMLDocument(RAW_XML1 % (timestr, ""))
     res = idx.eval(doc, self.env)[0]
     self.assertEqual(res.key, dt.replace(microsecond=0))
Beispiel #15
0
        # check for grouping indexes
        xpath = xpath.strip()
        group_path = None
        if '#' in xpath:
            try:
                group_path, temp = xpath.split('#')
            except ValueError, e:
                msg = "registerIndex: Invalid index expression. %s"
                raise InvalidParameterError(msg % str(e))
            xpath = '/'.join([group_path, temp])
        type = INDEX_TYPES.get(type.lower())
        _, resourcetype = self.env.registry.objects_from_id(package_id,
                                                            resourcetype_id)
        # generate index
        xmlindex = XmlIndex(resourcetype=resourcetype, label=label,
                            xpath=xpath, type=type, options=options,
                            group_path=group_path)
        xmlindex = self.index_catalog.registerIndex(xmlindex)
        return xmlindex

    def deleteIndex(self, xmlindex=None, _id=None):
        """
        Remove an index using either a XMLIndex object or a index id.
        """
        if _id:
            xmlindex = self.getIndexes(_id=_id)[0]
        self.index_catalog.deleteIndex(xmlindex)

    def deleteAllIndexes(self, package_id, resourcetype_id=None):
        """
        Removes all indexes related to a given package_id and resourcetype_id.