def test_flushIndex(self): # set up index1 = XmlIndex(self.rt1, "/station/station_code", label="code") index2 = XmlIndex(self.rt1, "/station/XY/paramXY", label="paramXY") self.catalog.registerIndex(index1) self.catalog.registerIndex(index2) res = Resource(self.rt1, document=newXMLDocument(RAW_XML2)) self.xmldb.addResource(res) self.catalog.indexResource(res) # index1 and index2 contain data el = self.catalog.dumpIndex(index1) self.assertEquals(len(el), 1) el = self.catalog.dumpIndex(index2) self.assertEquals(len(el), 3) # flush index1 self.catalog.flushIndex(index1) # index1 data has gone el = self.catalog.dumpIndex(index1) self.assertEquals(len(el), 0) # index2 data still there el = self.catalog.dumpIndex(index2) self.assertEquals(len(el), 3) # flush index2 self.catalog.flushIndex(index2) el = self.catalog.dumpIndex(index2) self.assertEquals(len(el), 0) # clean up: self.catalog.deleteIndex(index1) self.catalog.deleteIndex(index2) self.xmldb.deleteResource(res)
def test_getIndexes(self): """ """ index_rt1 = XmlIndex(self.rt1, "/station/XY/paramXY", label='id1') index2_rt1 = XmlIndex(self.rt1, "/station/station_code", label='id2') index_rt2 = XmlIndex(self.rt2, "/station/XY/paramXY", label='id3') self.catalog.registerIndex(index_rt1) self.catalog.registerIndex(index2_rt1) self.catalog.registerIndex(index_rt2) # get by resourcetype: res = self.catalog.getIndexes(package_id=self.rt1.package.package_id, resourcetype_id=self.rt1.resourcetype_id) self.assertEquals(len(res), 2) self.assertEquals(res[0].resourcetype.resourcetype_id, self.rt1.resourcetype_id) self.assertEquals(res[0].xpath, index_rt1.xpath) self.assertEquals(res[1].resourcetype.resourcetype_id, self.rt1.resourcetype_id) self.assertEquals(res[1].xpath, index2_rt1.xpath) res = self.catalog.getIndexes(package_id=self.rt2.package.package_id, resourcetype_id=self.rt2.resourcetype_id) self.assertEquals(len(res), 1) self.assertEquals(res[0].resourcetype.resourcetype_id, self.rt2.resourcetype_id) self.assertEquals(res[0].xpath, index_rt2.xpath) # get by xpath res = self.catalog.getIndexes(xpath="/station/XY/paramXY") self.assertEquals(len(res), 2) self.assertEquals(res[0].resourcetype.resourcetype_id, self.rt1.resourcetype_id) self.assertEquals(res[0].xpath, index_rt1.xpath) self.assertEquals(res[1].resourcetype.resourcetype_id, self.rt2.resourcetype_id) self.assertEquals(res[1].xpath, index_rt2.xpath) res = self.catalog.getIndexes(xpath="/station/station_code") self.assertEquals(len(res), 1) self.assertEquals(res[0].resourcetype.resourcetype_id, self.rt1.resourcetype_id) self.assertEquals(res[0].xpath, index2_rt1.xpath) # get by rt and xpath res = self.catalog.getIndexes(package_id=self.rt1.package.package_id, resourcetype_id=self.rt1.resourcetype_id, xpath="/station/XY/paramXY") self.assertEquals(len(res), 1) self.assertEquals(res[0].resourcetype.resourcetype_id, self.rt1.resourcetype_id) self.assertEquals(res[0].xpath, "/station/XY/paramXY") # remove: self.catalog.deleteIndex(index_rt1) self.catalog.deleteIndex(index2_rt1) self.catalog.deleteIndex(index_rt2)
def test_TimestampIndex(self): """ Tests indexing of timestamps. """ # w/ microseconds dt = UTCDateTime(2008, 10, 23, 11, 53, 12, 54000) idx = XmlIndex(self.rt1, "/station/creation_date", index.TIMESTAMP_INDEX) timestr = "%f" % dt.timestamp doc = newXMLDocument(RAW_XML1 % (timestr, "")) res = idx.eval(doc, self.env)[0] self.assertEqual(res.key, dt.datetime) # w/o microseconds dt = UTCDateTime(2008, 10, 23, 11, 53, 12) idx = XmlIndex(self.rt1, "/station/creation_date", index.TIMESTAMP_INDEX) timestr = "%f" % dt.timestamp doc = newXMLDocument(RAW_XML1 % (timestr, "")) res = idx.eval(doc, self.env)[0] self.assertEqual(res.key, dt.datetime) # negative timestamp works too dt = UTCDateTime(1969, 12, 31, 23, 36, 39, 500000) idx = XmlIndex(self.rt1, "/station/creation_date", index.TIMESTAMP_INDEX) timestr = "%f" % dt.timestamp doc = newXMLDocument(RAW_XML1 % (timestr, "")) res = idx.eval(doc, self.env)[0] self.assertEqual(res.key, dt.datetime)
def test_indexResource(self): # set up res = Resource(self.rt1, document=newXMLDocument(RAW_XML2)) self.xmldb.addResource(res) index1 = XmlIndex(self.rt1, "/station/station_code", label='idx1') index2 = XmlIndex(self.rt1, "/station/XY/paramXY", label='idx2') index3 = XmlIndex(self.rt1, "/station/test_date", FLOAT_INDEX) self.catalog.registerIndex(index1) self.catalog.registerIndex(index2) self.catalog.registerIndex(index3) # index resource r = self.catalog.indexResource(res) self.assertEquals(len(r), 5) # dump first index el = self.catalog.dumpIndex(index1) self.assertEquals(len(el), 1) self.assertEquals(el[0].key, "GENF") self.assertEquals(el[0].document.data, res.document.data) # dump second index el = self.catalog.dumpIndex(index2) self.assertEqual(len(el), 3) keys = ["0", "2.5", "99"] for e in el: assert e.key in keys keys.remove(e.key) self.assertEquals(e.document.data, res.document.data) # dumpIndexByResource el = self.catalog.dumpIndexByResource(res) self.assertEqual(len(el), 5) self.assertEquals(el[0].key, "GENF") self.assertEquals(el[0].document.data, res.document.data) self.assertEquals(el[0].index.xpath, "/station/station_code") self.assertTrue(el[1].key in ["0", "2.5", "99"]) self.assertEquals(el[1].document.data, res.document.data) self.assertEquals(el[1].index.xpath, "/station/XY/paramXY") self.assertTrue(el[2].key in ["0", "2.5", "99"]) self.assertEquals(el[2].document.data, res.document.data) self.assertEquals(el[2].index.xpath, "/station/XY/paramXY") self.assertTrue(el[3].key in ["0", "2.5", "99"]) self.assertEquals(el[3].document.data, res.document.data) self.assertEquals(el[3].index.xpath, "/station/XY/paramXY") self.assertEquals(el[4].key, 2008.123456789) self.assertEquals(el[4].document.data, res.document.data) self.assertEquals(el[4].index.xpath, "/station/test_date") # clean up self.catalog.deleteIndex(index1) self.catalog.deleteIndex(index2) self.catalog.deleteIndex(index3) self.xmldb.deleteResource(res)
def test_deleteIndex(self): """ """ index = XmlIndex(self.rt1, "/station/XY/paramXY", label='test') self.catalog.registerIndex(index) res = self.catalog.getIndexes(package_id=self.rt1.package.package_id, resourcetype_id=self.rt1.resourcetype_id, xpath="/station/XY/paramXY") self.assertEquals(len(res), 1) self.catalog.deleteIndex(index) res = self.catalog.getIndexes(package_id=self.rt1.package.package_id, resourcetype_id=self.rt1.resourcetype_id, xpath="/station/XY/paramXY") self.assertEquals(len(res), 0)
def testIndexCommon(self): # index with single node key result: si = XmlIndex(self.rt1, xpath="/station/station_code") # index with multiple nodes key result: mi = XmlIndex(self.rt1, xpath="/station/XY/paramXY") # index which is empty on test resource ei = XmlIndex(self.rt1, xpath="/station/empty") # # index which does not exists on test resource # ni = XmlIndex(self.rt1, xpath="/station/XXX") test_doc = newXMLDocument(RAW_XML1) class Foo(object): pass # pass a Foo: (which does not implement IXmlDoc) self.assertRaises(TypeError, si.eval, Foo()) # pass an empty XmlDoc: self.assertRaises(SeisHubError, si.eval, XmlDocument()) res = si.eval(test_doc, self.env) self.assertEquals(len(res), 1) self.assertEquals(res[0].index, si) self.assertEquals(res[0].document, test_doc) self.assertEquals(res[0].key, 'BERN') res = mi.eval(test_doc, self.env) self.assertEquals(len(res), 3) self.assertEquals(res[0].index, mi) self.assertEquals(res[0].document, test_doc) self.assertEquals(res[0].key, '20.5') self.assertEquals(res[1].index, mi) self.assertEquals(res[1].document, test_doc) self.assertEquals(res[1].key, '11.5') self.assertEquals(res[2].index, mi) self.assertEquals(res[2].document, test_doc) self.assertEquals(res[2].key, 'blah') res = ei.eval(test_doc, self.env) self.assertEquals(len(res), 1)
def testNumericIndex(self): test_doc = newXMLDocument(RAW_XML1) idx = XmlIndex(self.rt1, "/station/lon", index.NUMERIC_INDEX) res = idx.eval(test_doc, self.env)[0] self.assertEquals(type(res), NumericIndexElement) # self.assertEquals(type(res.key), float) self.assertEquals(res.key, '12.51200') # elements with wrong data type are ignored idx = XmlIndex(self.rt1, "/station/XY/paramXY", index.NUMERIC_INDEX) res = idx.eval(test_doc, self.env) self.assertEquals(len(res), 2)
def testTextIndex(self): test_doc = newXMLDocument(RAW_XML1) idx = XmlIndex(self.rt1, "/station/lon", index.TEXT_INDEX) res = idx.eval(test_doc, self.env)[0] self.assertEquals(type(res), index.TextIndexElement) self.assertEquals(type(res.key), unicode) self.assertEquals(res.key, '12.51200') # index which is empty on test resource idx = XmlIndex(self.rt1, "/station/empty", index.TEXT_INDEX) res = idx.eval(test_doc, self.env) self.assertEquals(len(res), 1)
def test_DateIndex(self): """ Tests indexing of dates. """ dt = datetime(2008, 10, 10, 11, 53, 0, 54000) # ISO 8601 w/o minus idx = XmlIndex(self.rt1, "/station/creation_date", index.DATE_INDEX) timestr = dt.strftime("%Y%m%d") doc = newXMLDocument(RAW_XML1 % (timestr, "")) res = idx.eval(doc, self.env)[0] self.assertEqual(res.key, dt.date()) # ISO 8601 w/ minus idx = XmlIndex(self.rt1, "/station/creation_date", index.DATE_INDEX) timestr = dt.strftime("%Y-%m-%d") doc = newXMLDocument(RAW_XML1 % (timestr, "")) res = idx.eval(doc, self.env)[0] self.assertEqual(res.key, dt.date()) # custom format idx = XmlIndex(self.rt1, "/station/creation_date", index.DATE_INDEX, options="%d.%m.%Y") timestr = dt.strftime("%d.%m.%Y") doc = newXMLDocument(RAW_XML1 % (timestr, "")) res = idx.eval(doc, self.env)[0] self.assertEqual(res.key, dt.date())
def test_indexResourceWithGrouping(self): # set up res = Resource(self.rt1, document=newXMLDocument(RAW_XML4)) self.xmldb.addResource(res) index = XmlIndex(self.rt1, "/station/XY/Z/value", group_path="/station/XY") self.catalog.registerIndex(index) r = self.catalog.indexResource(res) self.assertEquals(len(r), 2) el = self.catalog.dumpIndex(index) self.assertEquals(len(el), 2) self.assertEquals(el[0].key, "3") self.assertEquals(el[0].group_pos, 0) self.assertEquals(el[0].document.data, res.document.data) self.assertEquals(el[1].key, "6") self.assertEquals(el[1].group_pos, 1) self.assertEquals(el[1].document.data, res.document.data) # clean up self.catalog.deleteIndex(index) self.xmldb.deleteResource(res)
def test_registerIndex(self): """ """ index = XmlIndex(self.rt1, "/station/XY/paramXY", DATETIME_INDEX, "%Y/%m", label='test') self.catalog.registerIndex(index) res = self.catalog.getIndexes(package_id=self.rt1.package.package_id, resourcetype_id=self.rt1.resourcetype_id, xpath="/station/XY/paramXY")[0] self.assertEquals(res.resourcetype.resourcetype_id, self.rt1.resourcetype_id) self.assertEquals(res.xpath, "/station/XY/paramXY") self.assertEquals(res.type, DATETIME_INDEX) self.assertEquals(res.options, "%Y/%m") # try to add a duplicate: self.assertRaises(DuplicateObjectError, self.catalog.registerIndex, index) # clean up: self.catalog.deleteIndex(index)
def testBooleanIndex(self): idx = XmlIndex(self.rt1, "/station/bool", index.BOOLEAN_INDEX) doc = newXMLDocument(RAW_XML1 % ("", "True")) res = idx.eval(doc, self.env)[0] self.assertEqual(res.key, True) doc = newXMLDocument(RAW_XML1 % ("", "False")) res = idx.eval(doc, self.env)[0] self.assertEqual(res.key, False) doc = newXMLDocument(RAW_XML1 % ("", "1")) res = idx.eval(doc, self.env)[0] self.assertEqual(res.key, True) doc = newXMLDocument(RAW_XML1 % ("", "0")) res = idx.eval(doc, self.env)[0] self.assertEqual(res.key, False) doc = newXMLDocument(RAW_XML1 % ("", "something")) res = idx.eval(doc, self.env)[0] self.assertEqual(res.key, True)
def testIndexGrouping(self): doc = newXMLDocument(RAW_XML2) idx1 = XmlIndex(self.rt1, "/station/XY/X", index.NUMERIC_INDEX, group_path="/station/XY") res = idx1.eval(doc, self.env) self.assertEqual(len(res), 2) self.assertEqual(res[0].key, '1') self.assertEqual(res[0].group_pos, 0) self.assertEqual(res[1].key, '4') self.assertEqual(res[1].group_pos, 1) idx2 = XmlIndex(self.rt1, "/station/XY/Y", index.NUMERIC_INDEX, group_path="/station/XY") res = idx2.eval(doc, self.env) self.assertEqual(len(res), 2) self.assertEqual(res[0].key, '2') self.assertEqual(res[0].group_pos, 0) self.assertEqual(res[1].key, '5') self.assertEqual(res[1].group_pos, 1) idx3 = XmlIndex(self.rt1, "/station/XY/Y/@id", index.NUMERIC_INDEX, group_path="/station/XY") res = idx3.eval(doc, self.env) self.assertEqual(len(res), 2) self.assertEqual(res[0].key, '1') self.assertEqual(res[0].group_pos, 0) self.assertEqual(res[1].key, '2') self.assertEqual(res[1].group_pos, 1) idx4 = XmlIndex(self.rt1, "/station/XY/Z/value", index.NUMERIC_INDEX, group_path="/station/XY") res = idx4.eval(doc, self.env) self.assertEqual(len(res), 2) self.assertEqual(res[0].key, '3') self.assertEqual(res[0].group_pos, 0) self.assertEqual(res[1].key, '6') self.assertEqual(res[1].group_pos, 1)
def test_DateTimeIndex(self): """ Tests indexing of datetimes. """ # setup dt = datetime(2008, 10, 23, 11, 53, 12, 54000) dt2 = datetime(2008, 10, 23, 11, 53, 12) dt3 = datetime(2008, 10, 23) dt4 = datetime(2008, 10, 23, 11) dt5 = datetime(2008, 10, 23, 11 , 53) # ISO 8601 idx = XmlIndex(self.rt1, "/station/creation_date", index.DATETIME_INDEX) timestr = dt.strftime("%Y%m%dT%H:%M:%S") + ".054000" doc = newXMLDocument(RAW_XML1 % (timestr, "")) res = idx.eval(doc, self.env)[0] self.assertEqual(res.key, dt) # ISO 8601 w/ minus idx = XmlIndex(self.rt1, "/station/creation_date", index.DATETIME_INDEX) timestr = dt.strftime("%Y-%m-%dT%H:%M:%S") + ".054000" doc = newXMLDocument(RAW_XML1 % (timestr, "")) res = idx.eval(doc, self.env)[0] self.assertEqual(res.key, dt) # ISO 8601 w/ time zone idx = XmlIndex(self.rt1, "/station/creation_date", index.DATETIME_INDEX) timestr = dt.strftime("%Y%m%dT%H:%M:%S") + ".054000Z" doc = newXMLDocument(RAW_XML1 % (timestr, "")) res = idx.eval(doc, self.env)[0] self.assertEqual(res.key, dt) # ISO 8601 w/o T idx = XmlIndex(self.rt1, "/station/creation_date", index.DATETIME_INDEX) timestr = dt.strftime("%Y%m%d %H:%M:%S") + ".054000" doc = newXMLDocument(RAW_XML1 % (timestr, "")) res = idx.eval(doc, self.env)[0] self.assertEqual(res.key, dt) # ISO 8601 w/o milliseconds idx = XmlIndex(self.rt1, "/station/creation_date", index.DATETIME_INDEX) timestr = dt2.strftime("%Y%m%dT%H:%M:%S") doc = newXMLDocument(RAW_XML1 % (timestr, "")) res = idx.eval(doc, self.env)[0] self.assertEqual(res.key, dt2) # ISO 8601 w/o time - defaults to 00:00:00 idx = XmlIndex(self.rt1, "/station/creation_date", index.DATETIME_INDEX) timestr = dt3.strftime("%Y-%m-%d") doc = newXMLDocument(RAW_XML1 % (timestr, "")) res = idx.eval(doc, self.env)[0] self.assertEqual(res.key, dt3) # ISO 8601 w/o minutes - defaults to :00:00 idx = XmlIndex(self.rt1, "/station/creation_date", index.DATETIME_INDEX) timestr = dt4.strftime("%Y-%m-%dT%H") doc = newXMLDocument(RAW_XML1 % (timestr, "")) res = idx.eval(doc, self.env)[0] self.assertEqual(res.key, dt4) # ISO 8601 w/o seconds - defaults to :00 idx = XmlIndex(self.rt1, "/station/creation_date", index.DATETIME_INDEX) timestr = dt5.strftime("%Y-%m-%dT%H:%M") doc = newXMLDocument(RAW_XML1 % (timestr, "")) res = idx.eval(doc, self.env)[0] self.assertEqual(res.key, dt5) # with custom format # microseconds are ignored since not supported by strftime() idx = XmlIndex(self.rt1, "/station/creation_date", index.DATETIME_INDEX, "%H:%M:%S - %Y%m%d") timestr = dt.strftime("%H:%M:%S - %Y%m%d") doc = newXMLDocument(RAW_XML1 % (timestr, "")) res = idx.eval(doc, self.env)[0] self.assertEqual(res.key, dt.replace(microsecond=0))
# check for grouping indexes xpath = xpath.strip() group_path = None if '#' in xpath: try: group_path, temp = xpath.split('#') except ValueError, e: msg = "registerIndex: Invalid index expression. %s" raise InvalidParameterError(msg % str(e)) xpath = '/'.join([group_path, temp]) type = INDEX_TYPES.get(type.lower()) _, resourcetype = self.env.registry.objects_from_id(package_id, resourcetype_id) # generate index xmlindex = XmlIndex(resourcetype=resourcetype, label=label, xpath=xpath, type=type, options=options, group_path=group_path) xmlindex = self.index_catalog.registerIndex(xmlindex) return xmlindex def deleteIndex(self, xmlindex=None, _id=None): """ Remove an index using either a XMLIndex object or a index id. """ if _id: xmlindex = self.getIndexes(_id=_id)[0] self.index_catalog.deleteIndex(xmlindex) def deleteAllIndexes(self, package_id, resourcetype_id=None): """ Removes all indexes related to a given package_id and resourcetype_id.