def convert_to_document(revision): temp = {} rev_key = [ i for i in dir(revision) if not re.match('^_',i) ] bill_key = [ i for i in dir(revision.bill) if not re.match('^_',i) ] for key in rev_key: if key != 'metadata' and key != 'bill': temp[key] = getattr(revision,key) for key in bill_key: if key != 'metadata' and key!='id' and key!='bill_revs': temp[key] = getattr(revision.bill,key) full_path = download(temp['url']) if full_path: temp['document'] = pyes.file_to_attachment(full_path) return temp
def setUp(self): super(QueryAttachmentTestCase, self).setUp() mapping = { "attachment": { "type": "attachment", "fields": { "file": {"store": "yes"}, "date": {"store": "yes"}, "author": {"store": "yes"}, "title": {"store": "yes", "term_vector": "with_positions_offsets"}, "attachment": {"store": "yes"}, }, }, "uuid": {"boost": 1.0, "index": "not_analyzed", "store": "yes", "type": u"string"}, } # mapping = { # self.document_type: { # "_index": {"enabled": "yes"}, # "_id": {"store": "yes"}, # "properties": { # "attachment": { # "type": "attachment", # "fields": { # "title": {"store": "yes", "term_vector" : "with_positions_offsets"}, # "attachment": {"store":"yes", "term_vector" : "with_positions_offsets"} # }, # "store":"yes" # # }, # "uuid": {"type": "string", "store": "yes", "index": "not_analyzed"} # }, # "_all": {"store": "yes", "term_vector": "with_positions_offsets"} # } # } self.conn.debug_dump = True self.conn.create_index(self.index_name) self.conn.put_mapping(self.document_type, {self.document_type: {"properties": mapping}}, self.index_name) self.conn.refresh(self.index_name) self.conn.get_mapping(self.document_type, self.index_name) self.conn.index( {"attachment": file_to_attachment(os.path.join("data", "testXHTML.html")), "uuid": "1"}, self.index_name, self.document_type, 1, ) self.conn.refresh(self.index_name)
def get_indexable_bills(): revision = select([bill_revs, bills], bill_revs.c.bill_id == bills.c.id) conn = engine.connect() result = conn.execute(revision) data = result.fetchall() for item in data: temp = {} for key in item.keys(): if key == 'id': continue elif key == 'url': full_path = download(item[key]) if not full_path: continue temp['document'] = pyes.file_to_attachment(full_path) else: temp[key] = item[key] yield temp
def get_indexable_bills(): revision = select([bill_revs,bills],bill_revs.c.bill_id==bills.c.id) conn = engine.connect() result = conn.execute(revision) data = result.fetchall() for item in data: temp = {} for key in item.keys(): if key == 'id': continue elif key == 'url': full_path = download(item[key]) if not full_path: continue temp['document'] = pyes.file_to_attachment(full_path) else: temp[key] = item[key] yield temp
def setUp(self): super(QueryAttachmentTestCase, self).setUp() mapping = { "attachment" : { "type" : "attachment", 'fields':{ "file" : {'store' : "yes"}, "date" : {'store' : "yes"}, "author" : {'store': "yes"}, "title" : {'store': "yes", "term_vector" : "with_positions_offsets"}, "attachment" : {'store': "yes"}, } }, 'uuid': {'boost': 1.0, 'index': 'not_analyzed', 'store': 'yes', 'type': u'string'} } # mapping = { # "test-type": { # "_index": {"enabled": "yes"}, # "_id": {"store": "yes"}, # "properties": { # "attachment": { # "type": "attachment", # "fields": { # "title": {"store": "yes", "term_vector" : "with_positions_offsets"}, # "attachment": {"store":"yes", "term_vector" : "with_positions_offsets"} # }, # "store":"yes" # # }, # "uuid": {"type": "string", "store": "yes", "index": "not_analyzed"} # }, # "_all": {"store": "yes", "term_vector": "with_positions_offsets"} # } # } self.conn.debug_dump = True self.conn.create_index("test-index") self.conn.put_mapping("test-type", {"test-type":{'properties':mapping}}, ["test-index"]) self.conn.refresh(["test-index"]) self.conn.get_mapping("test-type", ["test-index"]) self.conn.index({"attachment":file_to_attachment(os.path.join("data", "testXHTML.html")), "uuid":"1" }, "test-index", "test-type", 1) self.conn.refresh(["test-index"])
def setUp(self): super(QueryAttachmentTestCase, self).setUp() mapping = { "attachment": { "type": "attachment", 'fields': { "file": { 'store': "yes" }, "date": { 'store': "yes" }, "author": { 'store': "yes" }, "title": { 'store': "yes", "term_vector": "with_positions_offsets" }, "attachment": { 'store': "yes" }, } }, 'uuid': { 'boost': 1.0, 'index': 'not_analyzed', 'store': 'yes', 'type': u'string' } } # mapping = { # self.document_type: { # "_index": {"enabled": "yes"}, # "_id": {"store": "yes"}, # "properties": { # "attachment": { # "type": "attachment", # "fields": { # "title": {"store": "yes", "term_vector" : "with_positions_offsets"}, # "attachment": {"store":"yes", "term_vector" : "with_positions_offsets"} # }, # "store":"yes" # # }, # "uuid": {"type": "string", "store": "yes", "index": "not_analyzed"} # }, # "_all": {"store": "yes", "term_vector": "with_positions_offsets"} # } # } self.conn.debug_dump = True self.conn.create_index(self.index_name) self.conn.put_mapping(self.document_type, {self.document_type: { 'properties': mapping }}, self.index_name) self.conn.refresh(self.index_name) self.conn.get_mapping(self.document_type, self.index_name) self.conn.index( { "attachment": file_to_attachment(os.path.join("data", "testXHTML.html")), "uuid": "1" }, self.index_name, self.document_type, 1) self.conn.refresh(self.index_name)