def add_doc(self, text, meta, labels):
from api.models import Document, SequenceAnnotation
doc = Document(text=text, meta=meta, project=self.project)
doc.save()
for (start, end, label) in labels:
sa = SequenceAnnotation(start_offset=start,
end_offset=end,
document=doc,
user=self.admin)
sa.label = self.label_by_name[label]
sa.save()
def generate_docs(self, request, queryset):
user = request.user
taiwan_year = datetime.date.today().year - 1911
# NOTE: code format YYYXXXX
# YYY is taiwan year, XXXX is serial number
previous_code = Document.objects.aggregate(Max("code"))["code__max"]
if not previous_code or previous_code < taiwan_year * (10**4):
previous_code = taiwan_year * (10**4)
docs = []
factories = []
for code, factory in enumerate(queryset, start=previous_code + 1):
docs.append(
Document(
factory_id=factory.id,
creator_id=user.id,
code=code,
cet_staff=choose_cet_staff(factory.townname),
))
factory.cet_review_status = "X"
factories.append(factory)
Document.objects.bulk_create(docs)
Factory.objects.bulk_update(factories, ["cet_review_status"])
def test_update_status(client):
# Adding a docclass to the database
docclass_id = add_mock_docclass("test_update_status")
temp_document = Document(
{
"fileID": "Navam",
"userID": "Why",
"status": "Pending",
"docClassID": docclass_id,
"fileName": "MyDoc.docx",
}
)
db.session.add(temp_document)
db.session.commit()
rs = client.put(
"/document/" + str(temp_document.id),
content_type="multipart/form-data",
data={"status": "Missing"},
headers=headers,
)
assert rs.status_code == 200
ret_dict = rs.json # gives you a dictionary
assert ret_dict["success"] == True
def create_mock_document(data):
data["folderID"] = create_folder(
DocumentClass.query.get(data["docClassID"]).name,
FieldPartner.query.get(data["userID"]).folder_id,
)
d = Document(data)
db.session.add(d)
def create_document(file_id, user_id, status, docclass):
temp_document = Document({
"fileID": file_id,
"userID": user_id,
"date": date.fromordinal(730920),
"status": status,
"docClassID": docclass.id,
"fileName": "MyDoc.docx",
"latest": True,
"description": "Yeet",
})
db.session.add(temp_document)
return temp_document
def add_doc(self, text, meta, labels, updated=None, priority=None):
from api.models import Document
doc = Document(text=text, meta=meta, project=self.project)
if updated is not None:
doc.updated_at = updated
if priority is not None:
doc.priority = priority
doc.save()
self._set_annotations(doc, labels)
def add_documents(raw_corpus_file, lda_corpus_file, topics):
"""
Ajoute dans la table documents les articles d'un fichier .tsv contenant 8 colonnes :
- id_article
- titre
- chapo
- texte
- langue
- auteur
- mots
- date
Ajoute également les topics liés à chaque document.
:Parameters:
-`raw_corpus_file`: Le fichier .tsv contenant les documents
-`lda_corpus_file`: Le fichier _lda.mm
-`topics` : Une liste l contenant tous les topics telle que l[i] = Topic(id = i+1)
-`docid` : Un fichier de correspondance id diplo <-> id corpus
"""
try:
lda = corpora.mmcorpus.MmCorpus(lda_corpus_file)
except:
raise IOError("""Impossible de charger le fichier _lda.mm""")
with open(raw_corpus_file, 'r') as raw:
raw.readline() # on ignore la première ligne qui contient les noms des colonnes
transaction.set_autocommit(False)
for docno, raw_line in enumerate(raw):
# lda[docno] donne les topics associés au document raw_line sous la forme
# d'une liste de tuples (id_topic, poids du topic id_topic dans le document docno)
print "Document n°" + str(docno)
doc = Document.create_document(raw_line.rstrip().split('\t'))
doc.save()
for id_topic, weight_in_document in lda[docno]:
doc_topic = DocumentTopic(document=doc,
topic=topics[id_topic],
weight_in_document=weight_in_document)
doc_topic.save()
if docno % 1000 == 0:
transaction.commit()
def test_get_document(client):
rs = client.get("/documents")
assert rs.status_code == 200
ret_dict = rs.json # gives you a dictionary
assert ret_dict["success"] == True
# Adding a docclass to the database
docclass_id = add_mock_docclass("test_get_document")
# create Person and test whether it returns a person
temp_document = Document(
{
"fileID": "DunDunDun",
"userID": "WompWomp",
"status": "Pending",
"docClassID": docclass_id,
"fileName": "MyDoc.docx",
}
)
db.session.add(temp_document)
db.session.commit()
rs = client.get("/documents")
assert rs.status_code == 200
ret_dict = rs.json
assert ret_dict["success"] == True
assert len(ret_dict["result"]["documents"]) == 4
assert ret_dict["result"]["documents"]["Pending"][0]["fileID"] == "DunDunDun"
assert ret_dict["result"]["documents"]["Pending"][0]["userID"] == "WompWomp"
assert ret_dict["result"]["documents"]["Pending"][0]["status"] == "Pending"
rs = client.get("/documents?fid=jalkdf")
assert rs.status_code == 200
rs = client.get("/documents?uid=WompWomp")
ret_dict = rs.json
assert len(ret_dict["result"]["documents"]) == 4
assert ret_dict["result"]["documents"]["Pending"][0]["fileID"] == "DunDunDun"
assert ret_dict["result"]["documents"]["Pending"][0]["userID"] == "WompWomp"
assert ret_dict["result"]["documents"]["Pending"][0]["status"] == "Pending"
d1 = Department(name="DOE")
d1.save()
d2 = Department(name="DOD")
d2.save()
u1 = User.objects.create_user(username="******", password="******")
u1.save()
u1type = Usertype(user=u1, usertype="moderator", department=d1)
u1type.save()
u1.groups.add(moderators)
u2 = User.objects.create_user(username="******", password="******")
u2.save()
u2type = Usertype(user=u2, usertype="researcher", department=d2)
u2type.save()
u2.groups.add(researchers)
doc1 = Document(date_published="2016-12-20 00:00", title="DOEdoc", publisher="blah", institution="blah", status="unread", file_link="blah", PI_first_name="first",
PI_last_name="last", PI_email="*****@*****.**", author_list="1, 2, 3", department=d1)
doc1.save()
#Should all print true
u1.has_perm("api.change_document")
u1.has_perm("api.view_grant")
u2.has_perm("api.add_document")
#Should all print false
u1.has_perm("api.change_department")
u2.has_perm("view_document")
u2.has_perm("delete_document")
def add_document(db_session: Session, document: DocumentIn):
document = Document(**document.dict())
db_session.add(document)
db_session.commit()
document.id = document.id # get fields workaround
return document
def create_new_documents():
"""
used upon assignment of documents to field partner
"""
data = request.form
if data is None:
return create_response(status=400,
message="No body provided for new Document")
token = request.headers.get("token")
headers = {"Content-type": "application/json", "token": token}
message, info = verify_token(token)
if message != None:
return create_response(status=400, message=message)
if info == "fp":
return create_response(
status=400,
message="You do not have permission to delete documents!")
if "userID" not in data:
return create_response(status=400,
message="No UserID provided for new Document")
if "docClassIDs" not in data:
return create_response(status=400,
message="No document classes provided")
if "dueDate" not in data:
return create_response(status=400, message="No due date provided")
userID = data.get("userID")
dueDate = data.get("dueDate")
status = "Missing"
document_class_ids = data.get("docClassIDs").split(",")
document_ids = []
if type(document_class_ids) != list:
document_class_ids = document_class_ids.split(",")
fp = FieldPartner.query.get(userID)
fp.app_status = "In Process"
fp.due_date = dueDate
for document_class_id in document_class_ids:
data = {
"userID": userID,
"status": status,
"docClassID": document_class_id
}
new_doc = Document(data)
doc_dict = new_doc.to_dict()
document_ids.append(doc_dict["_id"])
db.session.add(new_doc)
ret = {"docIDs": document_ids}
db.session.commit()
# Make it return the document ids
return create_response(status=200, data=ret)
def setUp(self):
d = Document()
dep = Department()
try:
dep.name = 'NIH'
dep.save()
except:
self.fail('Department could not be saved!')
try:
d.title = 'Correlations within Causations'
d.institution = 'COS'
d.pifirstname = 'Foo'
d.pilastname = 'Bar'
d.piemail = '*****@*****.**'
d.status = 'Pending'
d.datepublished = datetime.datetime.now() # update this to use datetime
d.filelink = '~/doc.txt'
d.department = dep
d.save()
except:
self.fail('Document could not be saved!')
def setUp(self):
d = Document()
dep = Department()
try:
dep.name = 'NIH'
dep.save()
except:
self.fail('Department could not be saved!')
try:
d.title = 'Correlations within Causations'
d.institution = 'COS'
d.pifirstname = 'Foo'
d.pilastname = 'Bar'
d.piemail = '*****@*****.**'
d.status = 'Pending'
d.datepublished = datetime.datetime.now(
) # update this to use datetime
d.filelink = '~/doc.txt'
d.department = dep
d.save()
except:
self.fail('Document could not be saved!')
u1type = Usertype(user=u1, usertype="moderator", department=d1)
u1type.save()
u1.groups.add(moderators)
u2 = User.objects.create_user(username="******", password="******")
u2.save()
u2type = Usertype(user=u2, usertype="researcher", department=d2)
u2type.save()
u2.groups.add(researchers)
doc1 = Document(date_published="2016-12-20 00:00",
title="DOEdoc",
publisher="blah",
institution="blah",
status="unread",
file_link="blah",
PI_first_name="first",
PI_last_name="last",
PI_email="*****@*****.**",
author_list="1, 2, 3",
department=d1)
doc1.save()
#Should all print true
u1.has_perm("api.change_document")
u1.has_perm("api.view_grant")
u2.has_perm("api.add_document")
#Should all print false
u1.has_perm("api.change_department")
u2.has_perm("view_document")