def convert_to_document(revision):
temp = {}
rev_key = [ i for i in dir(revision) if not re.match('^_',i) ]
bill_key = [ i for i in dir(revision.bill) if not re.match('^_',i) ]
for key in rev_key:
if key != 'metadata' and key != 'bill':
temp[key] = getattr(revision,key)
for key in bill_key:
if key != 'metadata' and key!='id' and key!='bill_revs':
temp[key] = getattr(revision.bill,key)
full_path = download(temp['url'])
if full_path:
temp['document'] = pyes.file_to_attachment(full_path)
return temp
def setUp(self):
super(QueryAttachmentTestCase, self).setUp()
mapping = {
"attachment": {
"type": "attachment",
"fields": {
"file": {"store": "yes"},
"date": {"store": "yes"},
"author": {"store": "yes"},
"title": {"store": "yes", "term_vector": "with_positions_offsets"},
"attachment": {"store": "yes"},
},
},
"uuid": {"boost": 1.0, "index": "not_analyzed", "store": "yes", "type": u"string"},
}
# mapping = {
# self.document_type: {
# "_index": {"enabled": "yes"},
# "_id": {"store": "yes"},
# "properties": {
# "attachment": {
# "type": "attachment",
# "fields": {
# "title": {"store": "yes", "term_vector" : "with_positions_offsets"},
# "attachment": {"store":"yes", "term_vector" : "with_positions_offsets"}
# },
# "store":"yes"
#
# },
# "uuid": {"type": "string", "store": "yes", "index": "not_analyzed"}
# },
# "_all": {"store": "yes", "term_vector": "with_positions_offsets"}
# }
# }
self.conn.debug_dump = True
self.conn.create_index(self.index_name)
self.conn.put_mapping(self.document_type, {self.document_type: {"properties": mapping}}, self.index_name)
self.conn.refresh(self.index_name)
self.conn.get_mapping(self.document_type, self.index_name)
self.conn.index(
{"attachment": file_to_attachment(os.path.join("data", "testXHTML.html")), "uuid": "1"},
self.index_name,
self.document_type,
1,
)
self.conn.refresh(self.index_name)
def get_indexable_bills():
revision = select([bill_revs, bills], bill_revs.c.bill_id == bills.c.id)
conn = engine.connect()
result = conn.execute(revision)
data = result.fetchall()
for item in data:
temp = {}
for key in item.keys():
if key == 'id':
continue
elif key == 'url':
full_path = download(item[key])
if not full_path:
continue
temp['document'] = pyes.file_to_attachment(full_path)
else:
temp[key] = item[key]
yield temp
def get_indexable_bills():
revision = select([bill_revs,bills],bill_revs.c.bill_id==bills.c.id)
conn = engine.connect()
result = conn.execute(revision)
data = result.fetchall()
for item in data:
temp = {}
for key in item.keys():
if key == 'id':
continue
elif key == 'url':
full_path = download(item[key])
if not full_path:
continue
temp['document'] = pyes.file_to_attachment(full_path)
else:
temp[key] = item[key]
yield temp
def setUp(self):
super(QueryAttachmentTestCase, self).setUp()
mapping = {
"attachment" : { "type" : "attachment",
'fields':{
"file" : {'store' : "yes"},
"date" : {'store' : "yes"},
"author" : {'store': "yes"},
"title" : {'store': "yes", "term_vector" : "with_positions_offsets"},
"attachment" : {'store': "yes"},
}
},
'uuid': {'boost': 1.0,
'index': 'not_analyzed',
'store': 'yes',
'type': u'string'}
}
# mapping = {
# "test-type": {
# "_index": {"enabled": "yes"},
# "_id": {"store": "yes"},
# "properties": {
# "attachment": {
# "type": "attachment",
# "fields": {
# "title": {"store": "yes", "term_vector" : "with_positions_offsets"},
# "attachment": {"store":"yes", "term_vector" : "with_positions_offsets"}
# },
# "store":"yes"
#
# },
# "uuid": {"type": "string", "store": "yes", "index": "not_analyzed"}
# },
# "_all": {"store": "yes", "term_vector": "with_positions_offsets"}
# }
# }
self.conn.debug_dump = True
self.conn.create_index("test-index")
self.conn.put_mapping("test-type", {"test-type":{'properties':mapping}}, ["test-index"])
self.conn.refresh(["test-index"])
self.conn.get_mapping("test-type", ["test-index"])
self.conn.index({"attachment":file_to_attachment(os.path.join("data", "testXHTML.html")), "uuid":"1" }, "test-index", "test-type", 1)
self.conn.refresh(["test-index"])
def setUp(self):
super(QueryAttachmentTestCase, self).setUp()
mapping = {
"attachment": {
"type": "attachment",
'fields': {
"file": {
'store': "yes"
},
"date": {
'store': "yes"
},
"author": {
'store': "yes"
},
"title": {
'store': "yes",
"term_vector": "with_positions_offsets"
},
"attachment": {
'store': "yes"
},
}
},
'uuid': {
'boost': 1.0,
'index': 'not_analyzed',
'store': 'yes',
'type': u'string'
}
}
# mapping = {
# self.document_type: {
# "_index": {"enabled": "yes"},
# "_id": {"store": "yes"},
# "properties": {
# "attachment": {
# "type": "attachment",
# "fields": {
# "title": {"store": "yes", "term_vector" : "with_positions_offsets"},
# "attachment": {"store":"yes", "term_vector" : "with_positions_offsets"}
# },
# "store":"yes"
#
# },
# "uuid": {"type": "string", "store": "yes", "index": "not_analyzed"}
# },
# "_all": {"store": "yes", "term_vector": "with_positions_offsets"}
# }
# }
self.conn.debug_dump = True
self.conn.create_index(self.index_name)
self.conn.put_mapping(self.document_type,
{self.document_type: {
'properties': mapping
}}, self.index_name)
self.conn.refresh(self.index_name)
self.conn.get_mapping(self.document_type, self.index_name)
self.conn.index(
{
"attachment":
file_to_attachment(os.path.join("data", "testXHTML.html")),
"uuid":
"1"
}, self.index_name, self.document_type, 1)
self.conn.refresh(self.index_name)
