def create_document(self, meta, type=None):
if meta.content_hash:
q = Document.all()
if meta.foreign_id:
q = q.filter(Document.foreign_id == meta.foreign_id)
else:
q = q.filter(Document.content_hash == meta.content_hash)
q = q.filter(Document.source_id == self.source_id)
document = q.first()
if document is None:
document = Document()
document.source_id = self.source_id
document.meta = meta
document.type = type or self.DOCUMENT_TYPE
db.session.add(document)
db.session.flush()
return document
def document_by_meta(cls, collection_id, meta):
q = Document.all()
if meta.foreign_id:
q = q.filter(Document.foreign_id == meta.foreign_id)
elif meta.content_hash:
q = q.filter(Document.content_hash == meta.content_hash)
else:
raise ValueError("No unique criterion for document: %s" % meta)
q = q.filter(Document.collection_id == collection_id)
document = q.first()
if document is None:
document = Document()
document.collection_id = collection_id
document.foreign_id = meta.foreign_id
document.content_hash = meta.content_hash
document.meta = meta
return document
def create_document(self, meta, type=None):
if meta.content_hash:
q = Document.all()
if meta.foreign_id:
q = q.filter(Document.foreign_id == meta.foreign_id)
else:
q = q.filter(Document.content_hash == meta.content_hash)
clause = Collection.id == self.collection_id
q = q.filter(Document.collections.any(clause))
document = q.first()
if document is None:
document = Document()
document.collections = [Collection.by_id(self.collection_id)]
document.meta = meta
document.type = type or self.DOCUMENT_TYPE
db.session.add(document)
db.session.flush()
return document
def test_normalize(self):
for number in PHONE_NUMBERS:
document = Document()
document.add_country('de')
analyzer = PhoneNumberAnalyzer()
analyzer.analyze(document)