def test_flushIndex(self):
# set up
index1 = XmlIndex(self.rt1, "/station/station_code", label="code")
index2 = XmlIndex(self.rt1, "/station/XY/paramXY", label="paramXY")
self.catalog.registerIndex(index1)
self.catalog.registerIndex(index2)
res = Resource(self.rt1, document=newXMLDocument(RAW_XML2))
self.xmldb.addResource(res)
self.catalog.indexResource(res)
# index1 and index2 contain data
el = self.catalog.dumpIndex(index1)
self.assertEquals(len(el), 1)
el = self.catalog.dumpIndex(index2)
self.assertEquals(len(el), 3)
# flush index1
self.catalog.flushIndex(index1)
# index1 data has gone
el = self.catalog.dumpIndex(index1)
self.assertEquals(len(el), 0)
# index2 data still there
el = self.catalog.dumpIndex(index2)
self.assertEquals(len(el), 3)
# flush index2
self.catalog.flushIndex(index2)
el = self.catalog.dumpIndex(index2)
self.assertEquals(len(el), 0)
# clean up:
self.catalog.deleteIndex(index1)
self.catalog.deleteIndex(index2)
self.xmldb.deleteResource(res)
def test_getIndexes(self):
"""
"""
index_rt1 = XmlIndex(self.rt1, "/station/XY/paramXY", label='id1')
index2_rt1 = XmlIndex(self.rt1, "/station/station_code", label='id2')
index_rt2 = XmlIndex(self.rt2, "/station/XY/paramXY", label='id3')
self.catalog.registerIndex(index_rt1)
self.catalog.registerIndex(index2_rt1)
self.catalog.registerIndex(index_rt2)
# get by resourcetype:
res = self.catalog.getIndexes(package_id=self.rt1.package.package_id,
resourcetype_id=self.rt1.resourcetype_id)
self.assertEquals(len(res), 2)
self.assertEquals(res[0].resourcetype.resourcetype_id,
self.rt1.resourcetype_id)
self.assertEquals(res[0].xpath, index_rt1.xpath)
self.assertEquals(res[1].resourcetype.resourcetype_id,
self.rt1.resourcetype_id)
self.assertEquals(res[1].xpath, index2_rt1.xpath)
res = self.catalog.getIndexes(package_id=self.rt2.package.package_id,
resourcetype_id=self.rt2.resourcetype_id)
self.assertEquals(len(res), 1)
self.assertEquals(res[0].resourcetype.resourcetype_id,
self.rt2.resourcetype_id)
self.assertEquals(res[0].xpath, index_rt2.xpath)
# get by xpath
res = self.catalog.getIndexes(xpath="/station/XY/paramXY")
self.assertEquals(len(res), 2)
self.assertEquals(res[0].resourcetype.resourcetype_id,
self.rt1.resourcetype_id)
self.assertEquals(res[0].xpath, index_rt1.xpath)
self.assertEquals(res[1].resourcetype.resourcetype_id,
self.rt2.resourcetype_id)
self.assertEquals(res[1].xpath, index_rt2.xpath)
res = self.catalog.getIndexes(xpath="/station/station_code")
self.assertEquals(len(res), 1)
self.assertEquals(res[0].resourcetype.resourcetype_id,
self.rt1.resourcetype_id)
self.assertEquals(res[0].xpath, index2_rt1.xpath)
# get by rt and xpath
res = self.catalog.getIndexes(package_id=self.rt1.package.package_id,
resourcetype_id=self.rt1.resourcetype_id,
xpath="/station/XY/paramXY")
self.assertEquals(len(res), 1)
self.assertEquals(res[0].resourcetype.resourcetype_id,
self.rt1.resourcetype_id)
self.assertEquals(res[0].xpath, "/station/XY/paramXY")
# remove:
self.catalog.deleteIndex(index_rt1)
self.catalog.deleteIndex(index2_rt1)
self.catalog.deleteIndex(index_rt2)
def test_TimestampIndex(self):
"""
Tests indexing of timestamps.
"""
# w/ microseconds
dt = UTCDateTime(2008, 10, 23, 11, 53, 12, 54000)
idx = XmlIndex(self.rt1, "/station/creation_date",
index.TIMESTAMP_INDEX)
timestr = "%f" % dt.timestamp
doc = newXMLDocument(RAW_XML1 % (timestr, ""))
res = idx.eval(doc, self.env)[0]
self.assertEqual(res.key, dt.datetime)
# w/o microseconds
dt = UTCDateTime(2008, 10, 23, 11, 53, 12)
idx = XmlIndex(self.rt1, "/station/creation_date",
index.TIMESTAMP_INDEX)
timestr = "%f" % dt.timestamp
doc = newXMLDocument(RAW_XML1 % (timestr, ""))
res = idx.eval(doc, self.env)[0]
self.assertEqual(res.key, dt.datetime)
# negative timestamp works too
dt = UTCDateTime(1969, 12, 31, 23, 36, 39, 500000)
idx = XmlIndex(self.rt1, "/station/creation_date",
index.TIMESTAMP_INDEX)
timestr = "%f" % dt.timestamp
doc = newXMLDocument(RAW_XML1 % (timestr, ""))
res = idx.eval(doc, self.env)[0]
self.assertEqual(res.key, dt.datetime)
def test_indexResource(self):
# set up
res = Resource(self.rt1, document=newXMLDocument(RAW_XML2))
self.xmldb.addResource(res)
index1 = XmlIndex(self.rt1, "/station/station_code", label='idx1')
index2 = XmlIndex(self.rt1, "/station/XY/paramXY", label='idx2')
index3 = XmlIndex(self.rt1, "/station/test_date", FLOAT_INDEX)
self.catalog.registerIndex(index1)
self.catalog.registerIndex(index2)
self.catalog.registerIndex(index3)
# index resource
r = self.catalog.indexResource(res)
self.assertEquals(len(r), 5)
# dump first index
el = self.catalog.dumpIndex(index1)
self.assertEquals(len(el), 1)
self.assertEquals(el[0].key, "GENF")
self.assertEquals(el[0].document.data, res.document.data)
# dump second index
el = self.catalog.dumpIndex(index2)
self.assertEqual(len(el), 3)
keys = ["0", "2.5", "99"]
for e in el:
assert e.key in keys
keys.remove(e.key)
self.assertEquals(e.document.data, res.document.data)
# dumpIndexByResource
el = self.catalog.dumpIndexByResource(res)
self.assertEqual(len(el), 5)
self.assertEquals(el[0].key, "GENF")
self.assertEquals(el[0].document.data, res.document.data)
self.assertEquals(el[0].index.xpath, "/station/station_code")
self.assertTrue(el[1].key in ["0", "2.5", "99"])
self.assertEquals(el[1].document.data, res.document.data)
self.assertEquals(el[1].index.xpath, "/station/XY/paramXY")
self.assertTrue(el[2].key in ["0", "2.5", "99"])
self.assertEquals(el[2].document.data, res.document.data)
self.assertEquals(el[2].index.xpath, "/station/XY/paramXY")
self.assertTrue(el[3].key in ["0", "2.5", "99"])
self.assertEquals(el[3].document.data, res.document.data)
self.assertEquals(el[3].index.xpath, "/station/XY/paramXY")
self.assertEquals(el[4].key, 2008.123456789)
self.assertEquals(el[4].document.data, res.document.data)
self.assertEquals(el[4].index.xpath, "/station/test_date")
# clean up
self.catalog.deleteIndex(index1)
self.catalog.deleteIndex(index2)
self.catalog.deleteIndex(index3)
self.xmldb.deleteResource(res)
def test_deleteIndex(self):
"""
"""
index = XmlIndex(self.rt1, "/station/XY/paramXY", label='test')
self.catalog.registerIndex(index)
res = self.catalog.getIndexes(package_id=self.rt1.package.package_id,
resourcetype_id=self.rt1.resourcetype_id,
xpath="/station/XY/paramXY")
self.assertEquals(len(res), 1)
self.catalog.deleteIndex(index)
res = self.catalog.getIndexes(package_id=self.rt1.package.package_id,
resourcetype_id=self.rt1.resourcetype_id,
xpath="/station/XY/paramXY")
self.assertEquals(len(res), 0)
def testIndexCommon(self):
# index with single node key result:
si = XmlIndex(self.rt1, xpath="/station/station_code")
# index with multiple nodes key result:
mi = XmlIndex(self.rt1, xpath="/station/XY/paramXY")
# index which is empty on test resource
ei = XmlIndex(self.rt1, xpath="/station/empty")
# # index which does not exists on test resource
# ni = XmlIndex(self.rt1, xpath="/station/XXX")
test_doc = newXMLDocument(RAW_XML1)
class Foo(object):
pass
# pass a Foo: (which does not implement IXmlDoc)
self.assertRaises(TypeError, si.eval, Foo())
# pass an empty XmlDoc:
self.assertRaises(SeisHubError, si.eval, XmlDocument())
res = si.eval(test_doc, self.env)
self.assertEquals(len(res), 1)
self.assertEquals(res[0].index, si)
self.assertEquals(res[0].document, test_doc)
self.assertEquals(res[0].key, 'BERN')
res = mi.eval(test_doc, self.env)
self.assertEquals(len(res), 3)
self.assertEquals(res[0].index, mi)
self.assertEquals(res[0].document, test_doc)
self.assertEquals(res[0].key, '20.5')
self.assertEquals(res[1].index, mi)
self.assertEquals(res[1].document, test_doc)
self.assertEquals(res[1].key, '11.5')
self.assertEquals(res[2].index, mi)
self.assertEquals(res[2].document, test_doc)
self.assertEquals(res[2].key, 'blah')
res = ei.eval(test_doc, self.env)
self.assertEquals(len(res), 1)
def testNumericIndex(self):
test_doc = newXMLDocument(RAW_XML1)
idx = XmlIndex(self.rt1, "/station/lon", index.NUMERIC_INDEX)
res = idx.eval(test_doc, self.env)[0]
self.assertEquals(type(res), NumericIndexElement)
# self.assertEquals(type(res.key), float)
self.assertEquals(res.key, '12.51200')
# elements with wrong data type are ignored
idx = XmlIndex(self.rt1, "/station/XY/paramXY", index.NUMERIC_INDEX)
res = idx.eval(test_doc, self.env)
self.assertEquals(len(res), 2)
def testTextIndex(self):
test_doc = newXMLDocument(RAW_XML1)
idx = XmlIndex(self.rt1, "/station/lon", index.TEXT_INDEX)
res = idx.eval(test_doc, self.env)[0]
self.assertEquals(type(res), index.TextIndexElement)
self.assertEquals(type(res.key), unicode)
self.assertEquals(res.key, '12.51200')
# index which is empty on test resource
idx = XmlIndex(self.rt1, "/station/empty", index.TEXT_INDEX)
res = idx.eval(test_doc, self.env)
self.assertEquals(len(res), 1)
def test_DateIndex(self):
"""
Tests indexing of dates.
"""
dt = datetime(2008, 10, 10, 11, 53, 0, 54000)
# ISO 8601 w/o minus
idx = XmlIndex(self.rt1, "/station/creation_date", index.DATE_INDEX)
timestr = dt.strftime("%Y%m%d")
doc = newXMLDocument(RAW_XML1 % (timestr, ""))
res = idx.eval(doc, self.env)[0]
self.assertEqual(res.key, dt.date())
# ISO 8601 w/ minus
idx = XmlIndex(self.rt1, "/station/creation_date", index.DATE_INDEX)
timestr = dt.strftime("%Y-%m-%d")
doc = newXMLDocument(RAW_XML1 % (timestr, ""))
res = idx.eval(doc, self.env)[0]
self.assertEqual(res.key, dt.date())
# custom format
idx = XmlIndex(self.rt1, "/station/creation_date", index.DATE_INDEX,
options="%d.%m.%Y")
timestr = dt.strftime("%d.%m.%Y")
doc = newXMLDocument(RAW_XML1 % (timestr, ""))
res = idx.eval(doc, self.env)[0]
self.assertEqual(res.key, dt.date())
def test_indexResourceWithGrouping(self):
# set up
res = Resource(self.rt1, document=newXMLDocument(RAW_XML4))
self.xmldb.addResource(res)
index = XmlIndex(self.rt1, "/station/XY/Z/value",
group_path="/station/XY")
self.catalog.registerIndex(index)
r = self.catalog.indexResource(res)
self.assertEquals(len(r), 2)
el = self.catalog.dumpIndex(index)
self.assertEquals(len(el), 2)
self.assertEquals(el[0].key, "3")
self.assertEquals(el[0].group_pos, 0)
self.assertEquals(el[0].document.data, res.document.data)
self.assertEquals(el[1].key, "6")
self.assertEquals(el[1].group_pos, 1)
self.assertEquals(el[1].document.data, res.document.data)
# clean up
self.catalog.deleteIndex(index)
self.xmldb.deleteResource(res)
def test_registerIndex(self):
"""
"""
index = XmlIndex(self.rt1, "/station/XY/paramXY", DATETIME_INDEX,
"%Y/%m", label='test')
self.catalog.registerIndex(index)
res = self.catalog.getIndexes(package_id=self.rt1.package.package_id,
resourcetype_id=self.rt1.resourcetype_id,
xpath="/station/XY/paramXY")[0]
self.assertEquals(res.resourcetype.resourcetype_id,
self.rt1.resourcetype_id)
self.assertEquals(res.xpath, "/station/XY/paramXY")
self.assertEquals(res.type, DATETIME_INDEX)
self.assertEquals(res.options, "%Y/%m")
# try to add a duplicate:
self.assertRaises(DuplicateObjectError,
self.catalog.registerIndex, index)
# clean up:
self.catalog.deleteIndex(index)
def testBooleanIndex(self):
idx = XmlIndex(self.rt1, "/station/bool", index.BOOLEAN_INDEX)
doc = newXMLDocument(RAW_XML1 % ("", "True"))
res = idx.eval(doc, self.env)[0]
self.assertEqual(res.key, True)
doc = newXMLDocument(RAW_XML1 % ("", "False"))
res = idx.eval(doc, self.env)[0]
self.assertEqual(res.key, False)
doc = newXMLDocument(RAW_XML1 % ("", "1"))
res = idx.eval(doc, self.env)[0]
self.assertEqual(res.key, True)
doc = newXMLDocument(RAW_XML1 % ("", "0"))
res = idx.eval(doc, self.env)[0]
self.assertEqual(res.key, False)
doc = newXMLDocument(RAW_XML1 % ("", "something"))
res = idx.eval(doc, self.env)[0]
self.assertEqual(res.key, True)
def testIndexGrouping(self):
doc = newXMLDocument(RAW_XML2)
idx1 = XmlIndex(self.rt1, "/station/XY/X", index.NUMERIC_INDEX,
group_path="/station/XY")
res = idx1.eval(doc, self.env)
self.assertEqual(len(res), 2)
self.assertEqual(res[0].key, '1')
self.assertEqual(res[0].group_pos, 0)
self.assertEqual(res[1].key, '4')
self.assertEqual(res[1].group_pos, 1)
idx2 = XmlIndex(self.rt1, "/station/XY/Y", index.NUMERIC_INDEX,
group_path="/station/XY")
res = idx2.eval(doc, self.env)
self.assertEqual(len(res), 2)
self.assertEqual(res[0].key, '2')
self.assertEqual(res[0].group_pos, 0)
self.assertEqual(res[1].key, '5')
self.assertEqual(res[1].group_pos, 1)
idx3 = XmlIndex(self.rt1, "/station/XY/Y/@id", index.NUMERIC_INDEX,
group_path="/station/XY")
res = idx3.eval(doc, self.env)
self.assertEqual(len(res), 2)
self.assertEqual(res[0].key, '1')
self.assertEqual(res[0].group_pos, 0)
self.assertEqual(res[1].key, '2')
self.assertEqual(res[1].group_pos, 1)
idx4 = XmlIndex(self.rt1, "/station/XY/Z/value", index.NUMERIC_INDEX,
group_path="/station/XY")
res = idx4.eval(doc, self.env)
self.assertEqual(len(res), 2)
self.assertEqual(res[0].key, '3')
self.assertEqual(res[0].group_pos, 0)
self.assertEqual(res[1].key, '6')
self.assertEqual(res[1].group_pos, 1)
def test_DateTimeIndex(self):
"""
Tests indexing of datetimes.
"""
# setup
dt = datetime(2008, 10, 23, 11, 53, 12, 54000)
dt2 = datetime(2008, 10, 23, 11, 53, 12)
dt3 = datetime(2008, 10, 23)
dt4 = datetime(2008, 10, 23, 11)
dt5 = datetime(2008, 10, 23, 11 , 53)
# ISO 8601
idx = XmlIndex(self.rt1, "/station/creation_date",
index.DATETIME_INDEX)
timestr = dt.strftime("%Y%m%dT%H:%M:%S") + ".054000"
doc = newXMLDocument(RAW_XML1 % (timestr, ""))
res = idx.eval(doc, self.env)[0]
self.assertEqual(res.key, dt)
# ISO 8601 w/ minus
idx = XmlIndex(self.rt1, "/station/creation_date",
index.DATETIME_INDEX)
timestr = dt.strftime("%Y-%m-%dT%H:%M:%S") + ".054000"
doc = newXMLDocument(RAW_XML1 % (timestr, ""))
res = idx.eval(doc, self.env)[0]
self.assertEqual(res.key, dt)
# ISO 8601 w/ time zone
idx = XmlIndex(self.rt1, "/station/creation_date",
index.DATETIME_INDEX)
timestr = dt.strftime("%Y%m%dT%H:%M:%S") + ".054000Z"
doc = newXMLDocument(RAW_XML1 % (timestr, ""))
res = idx.eval(doc, self.env)[0]
self.assertEqual(res.key, dt)
# ISO 8601 w/o T
idx = XmlIndex(self.rt1, "/station/creation_date",
index.DATETIME_INDEX)
timestr = dt.strftime("%Y%m%d %H:%M:%S") + ".054000"
doc = newXMLDocument(RAW_XML1 % (timestr, ""))
res = idx.eval(doc, self.env)[0]
self.assertEqual(res.key, dt)
# ISO 8601 w/o milliseconds
idx = XmlIndex(self.rt1, "/station/creation_date",
index.DATETIME_INDEX)
timestr = dt2.strftime("%Y%m%dT%H:%M:%S")
doc = newXMLDocument(RAW_XML1 % (timestr, ""))
res = idx.eval(doc, self.env)[0]
self.assertEqual(res.key, dt2)
# ISO 8601 w/o time - defaults to 00:00:00
idx = XmlIndex(self.rt1, "/station/creation_date",
index.DATETIME_INDEX)
timestr = dt3.strftime("%Y-%m-%d")
doc = newXMLDocument(RAW_XML1 % (timestr, ""))
res = idx.eval(doc, self.env)[0]
self.assertEqual(res.key, dt3)
# ISO 8601 w/o minutes - defaults to :00:00
idx = XmlIndex(self.rt1, "/station/creation_date",
index.DATETIME_INDEX)
timestr = dt4.strftime("%Y-%m-%dT%H")
doc = newXMLDocument(RAW_XML1 % (timestr, ""))
res = idx.eval(doc, self.env)[0]
self.assertEqual(res.key, dt4)
# ISO 8601 w/o seconds - defaults to :00
idx = XmlIndex(self.rt1, "/station/creation_date",
index.DATETIME_INDEX)
timestr = dt5.strftime("%Y-%m-%dT%H:%M")
doc = newXMLDocument(RAW_XML1 % (timestr, ""))
res = idx.eval(doc, self.env)[0]
self.assertEqual(res.key, dt5)
# with custom format
# microseconds are ignored since not supported by strftime()
idx = XmlIndex(self.rt1, "/station/creation_date",
index.DATETIME_INDEX, "%H:%M:%S - %Y%m%d")
timestr = dt.strftime("%H:%M:%S - %Y%m%d")
doc = newXMLDocument(RAW_XML1 % (timestr, ""))
res = idx.eval(doc, self.env)[0]
self.assertEqual(res.key, dt.replace(microsecond=0))
# check for grouping indexes
xpath = xpath.strip()
group_path = None
if '#' in xpath:
try:
group_path, temp = xpath.split('#')
except ValueError, e:
msg = "registerIndex: Invalid index expression. %s"
raise InvalidParameterError(msg % str(e))
xpath = '/'.join([group_path, temp])
type = INDEX_TYPES.get(type.lower())
_, resourcetype = self.env.registry.objects_from_id(package_id,
resourcetype_id)
# generate index
xmlindex = XmlIndex(resourcetype=resourcetype, label=label,
xpath=xpath, type=type, options=options,
group_path=group_path)
xmlindex = self.index_catalog.registerIndex(xmlindex)
return xmlindex
def deleteIndex(self, xmlindex=None, _id=None):
"""
Remove an index using either a XMLIndex object or a index id.
"""
if _id:
xmlindex = self.getIndexes(_id=_id)[0]
self.index_catalog.deleteIndex(xmlindex)
def deleteAllIndexes(self, package_id, resourcetype_id=None):
"""
Removes all indexes related to a given package_id and resourcetype_id.