class DocumentSearchTestCase(unittest.TestCase):
def setUp(self):
# Start the OCR queue
self.default_queue = DocumentQueue.objects.get(name='default')
def test_do_document_ocr(self):
self.document_type = DocumentType(name='test doc type')
self.document_type.save()
self.document = Document(
document_type=self.document_type,
description='description',
)
self.document.save()
file_object = open(os.path.join(settings.SITE_ROOT, 'contrib', 'sample_documents', 'title_page.png'))
new_version = self.document.new_version(file=File(file_object, name='title_page.png'))
file_object.close()
self.failUnlessEqual(self.default_queue.queuedocument_set.count(), 1)
do_document_ocr(self.default_queue.queuedocument_set.all()[0])
self.assertTrue(u'Mayan EDMS' in self.document.pages.all()[0].content)
def tearDown(self):
self.document.delete()
self.document_type.delete()
def save_document(provider, title, href, images, embed, desc, duration, views, categories):
from documents.models import Document, Category, Image
try:
doc = Document.objects.get(href=href)
except Document.DoesNotExist:
doc = Document()
doc.provider = provider
doc.title = title
doc.href = href
doc.embed = embed
doc.desc = desc
doc.duration = duration
locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
doc.views = locale.atoi(views)
doc.save()
if doc.images.count() == 0:
for image_url in images:
response = requests.get(image_url)
if response.status_code == 200:
image = Image()
image.document = doc
image.url = image_url
image.save()
for category in categories:
cat = Category.objects.get_or_create(title=category)
doc.categories.add(cat[0])
def split_doc(request):
if request.is_ajax():
results = {}
cut = int(request.GET["modal_split_cut"][0])
doc = Document.objects.get(pk=int(request.GET["modal_split_doc_id"]))
doc.name = request.GET["modal_split_name"]
new_doc = Document(
name=request.GET["modal_split_new_name"],
owner=request.user,
refer_category=doc.refer_category,
complete=True,
)
pages = doc.pages.all()
new_doc.save()
i = 0
for p in pages:
if i >= cut:
new_doc.pages.add(p)
new_doc.size += p.get_size()
p.refer_document = new_doc
p.save()
doc.size -= p.get_size()
doc.pages.remove(p)
i += 1
doc.refer_category.documents.add(new_doc)
doc.save()
new_doc.save()
results["valid"] = True
return HttpResponse(json.dumps(results))
def report(request, lot_number):
report = Report.objects.get(lot_number=lot_number)
audit_log = target_stream(report)
orders = OrderLineItem.objects.filter(report=report)
newdoc = NewDocumentForm(None)
if 'document_button' in request.POST:
newdoc = NewDocumentForm(request.POST, request.FILES)
if newdoc.is_valid():
doc = Document(type=newdoc.cleaned_data['type'],
file=request.FILES['file'], created_by=request.user)
doc.save()
action.send(request.user, verb="created document", action_object=doc)
rd = ReportDocument(report=report, document=doc, primary_document=newdoc.cleaned_data['primary'], created_by=request.user, internal_cert=newdoc.cleaned_data['internal'])
rd.save()
action.send(request.user, verb="attached document", action_object=rd, target=report)
report.save()
messages.success(request, 'Document upload successful.')
return HttpResponseRedirect(reverse('reports.views.report',
args=[report.lot_number]))
return render_to_response('reports/report.html',
{
'report': report,
'new_document_form': newdoc,
'orders': orders,
'audit_log': audit_log,
},
context_instance=RequestContext(request))
def forwards(self, orm):
"Write your forwards methods here."
# This will transfer all the resumes to the documents
# Note: Don't use "from appname.models import ModelName".
# Use orm.ModelName to refer to models in this application,
# and orm['appname.ModelName'] for models in other applications.
# 1. get all 'original' resumes
# 2. for each 'original' resume
# 3. get corresponding 'non-original' resume which is the image
# 4. create Document entry and store it
print "hello"
for resume in orm['resumes.resume'].objects.filter(original=True):
# get non-original-resume and check if the same user and uploaded within 10 mins
print resume
non_org_resume = orm['resumes.resume'].objects.filter(
id=resume.id + 1)
if non_org_resume and non_org_resume[0].original == False:
non_org_resume = non_org_resume[0]
ten_minutes = datetime.timedelta(minutes=10)
if resume.user == non_org_resume.user and non_org_resume.timestamp - resume.timestamp <= ten_minutes:
s = string.lowercase + string.digits
unique_hash = ''.join(random.sample(s, 10))
new_doc = Document(user_id=resume.user.id,
document_type=1,
image_url=non_org_resume.url,
url=resume.url,
timestamp=resume.timestamp,
unique_hash=unique_hash)
new_doc.save()
def update_or_create_document(yaml_obj):
"""
Submit an object read from our YAML files and it will update it in the
database, creating it if it doesn't already exist.
Returns the database object, and a boolean that is true if a new object
was created.
"""
# Check if the table already exists in the datastore
obj = Document.get_by_key_name(yaml_obj.get('slug'))
# Update the obj if it exists
if obj:
# Loop through the keys and update the object one by one.
for key in yaml_obj.keys():
# With some special casing for projects...
if key == 'project_slug':
proj = Project.get_by_key_name(yaml_obj.get('project_slug'))
obj.project = proj
# ...and for tags.
elif key == 'tags':
obj.tags = get_tag_keys(yaml_obj.get("tags"))
else:
setattr(obj, key, yaml_obj.get(key))
# Save it out
obj.put()
created = False
# Create it if it doesn't
else:
# If it has tags....
if yaml_obj.has_key('tags'):
# Convert to database keys
tags = get_tag_keys(yaml_obj.pop("tags"))
# Load the data
obj = Document(key_name=yaml_obj.get('slug'), **yaml_obj)
# Set the tags
obj.tags = tags
# Otherwise....
else:
# Update the basic values
obj = Document(key_name=yaml_obj.get('slug'), **yaml_obj)
# And clear out the tag data
obj.tags = []
obj.similar_documents = []
# Connected it to a project, if it exists
if yaml_obj.has_key('project_slug'):
proj = Project.get_by_key_name(yaml_obj.get('project_slug'))
obj.project = proj
# Save it out
obj.put()
created = True
# Update the similarity lists of documents with the same tags
taskqueue.add(
url='/_/document/update-similar/',
params=dict(key=obj.key()),
method='GET'
)
# Pass it out
return obj, created
def _add_iban_alias_to_order_confirmation(self,
order_confirmation: Document,
order: Order):
wallet = self.get_user_wallet(order.buyer.mangopay_user_id)
wallet_id = wallet.get("Id")
# List that contains only one item maximum
banking_alias = self.get_wallet_banking_alias(wallet_id)
if banking_alias:
iban = banking_alias["iban"]
bic = banking_alias["bic"]
owner_name = banking_alias["owner_name"]
else:
banking_alias = self.mangopay.create_banking_alias_iban(
user_id=order.buyer.mangopay_user_id,
wallet_id=wallet_id,
name=order.buyer.get_full_name(),
)
iban = banking_alias["IBAN"]
bic = banking_alias["BIC"]
owner_name = banking_alias["OwnerName"]
order_confirmation.seller["iban"] = iban
order_confirmation.seller["bic"] = bic
order_confirmation.bank_account_owner = owner_name
order_confirmation.save()
def test_create_document_object(self):
doc = Document(title="foo", document_type=DocumentType.objects.all()[0],
notes="bonk")
doc.file.save(name="whee", content=ContentFile("wee willy wonka"))
doc.programs = Program.objects.all()[:2]
doc.authors = [self.john.id]
self.assertItemsEqual([doc], Document.objects.all())
def upload_single_file(self,
file_object,
filename=None,
use_file_name=False,
document_type=None,
metadata_dict_list=None,
user=None,
document=None,
new_version_data=None):
new_document = not document
if not document:
document = Document()
if document_type:
document.document_type = document_type
document.save()
apply_default_acls(document, user)
if user:
document.add_as_recent_document_for_user(user)
create_history(HISTORY_DOCUMENT_CREATED, document,
{'user': user})
else:
create_history(HISTORY_DOCUMENT_CREATED, document)
else:
if use_file_name:
filename = None
else:
filename = filename if filename else document.latest_version.filename
if not new_version_data:
new_version_data = {}
try:
new_version = document.new_version(file=file_object,
user=user,
**new_version_data)
except Exception:
# Don't leave the database in a broken state
# document.delete()
transaction.rollback()
raise
if filename:
document.rename(filename)
transformations, errors = self.get_transformation_list()
new_version.apply_default_transformations(transformations)
#TODO: new HISTORY for version updates
if metadata_dict_list and new_document:
# Only do for new documents
save_metadata_list(metadata_dict_list, document, create=True)
warnings = update_indexes(document)
return document
def get_context_data(self, *args, **kwargs):
context = super(FolderContextMixin,
self).get_context_data(*args, **kwargs)
folder = self.get_folder()
if not folder:
return context # Exit in case of usage outside of Folders section, like committees, etc.
account = get_current_account(self.request)
membership = self.request.user.get_membership(account)
# Folder forms
context['folder_add_form'] = FolderAddForm()
context['folder_edit_form'] = FolderEditForm()
context['folder_move_form'] = FolderMoveForm()
# Ordered items (files & folders)
search = self.request.GET.get('search', None)
if search:
descendants = folder.get_descendants()
sub_folders = descendants.filter(name__icontains=search)
documents = Document.objects.filter(account=folder.account,
folder__in=descendants,
name__icontains=search)
else:
sub_folders = folder.children.all()
documents = folder.documents.all()
documents = documents.select_related('user')
items = filter_by_permission(sub_folders, membership, PERMISSIONS.view)
filtered_documents = filter_by_permission(documents, membership,
PERMISSIONS.view)
Document.prefetch_revisions(filtered_documents)
items += filtered_documents
# Order items
default_ordering = 'date' if folder.name == 'Meeting Documents' else 'default'
ordering = self.request.GET.get('o', default_ordering).lower()
items = FolderContextMixin.sort_items(items, ordering)
# append members private folder
if folder.is_account_root and not search:
try:
private_folder = membership.private_folder
except Folder.DoesNotExist:
private_folder = Folder.objects.create_membership_folder(
membership)
items.append(private_folder)
context['show_shared_folder'] = True
# Set helper attributes
FolderContextMixin.set_help_attrs(items)
context['items'] = items
context['ordering'] = ordering
context['search'] = search
qs = folder.get_ancestors(include_self=True).filter(
membership_id=membership.id)
membership_ancestor = qs[0] if len(qs) > 0 else None
context['membership_ancestor'] = membership_ancestor
return context
def add_documents(request, category_id):
if request.is_ajax():
files = request.GET.getlist('files', False)
cat = Category.objects.get(id=category_id)
l_doc = []
l_pdf = []
cmds = []
paths = []
for f in list(files):
mime = MimeTypes()
path = os.path.join(settings.MEDIA_ROOT, settings.UPLOAD_DIR, f)
m = mime.guess_type(path)[0]
d = Document(name=f.encode('ascii', 'ignore'), owner=request.user, refer_category=cat)
d.save()
cat.add_doc(d)
if m == 'application/pdf':
l_pdf.append(([cat], path, f, [d]))
elif m in ['image/png', 'image/jpeg', 'image/bmp']:
im = Image.open(path)
w, h = im.size
new_filename = str(d.id) + '_' + f
new_path = os.path.join(cat.get_absolute_path(), new_filename)
shutil.copy2(path, new_path)
d.add_page(d.get_npages() + 1, new_filename, w, h)
for fu in FileUpload.objects.all():
if fu.file.path == path:
fu.delete()
d.complete = True
d.save()
remove_fileupload([path])
elif m in ['application/msword', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document']:
p = re.compile(r'.[Dd][Oo][Cc][xX]?$')
new_f = p.sub('.pdf', f)
new_path = path.replace(f, new_f)
cmd = 'soffice --headless --convert-to pdf %s --outdir %s/upload' % (path, settings.MEDIA_ROOT)
cmds.append(cmd)
paths.append(path)
l_doc.append(([cat], new_path, new_f, [d]))
elif m in ['application/vnd.ms-excel', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet']:
p = re.compile(r'.[Xx][Ll][Ss][xX]?$')
new_f = p.sub('.pdf', f)
new_path = path.replace(f, new_f)
cmd = 'soffice --headless --convert-to pdf %s --outdir %s/upload' % (path, settings.MEDIA_ROOT)
cmds.append(cmd)
paths.append(path)
l_doc.append(([cat], new_path, new_f, [d]))
else:
print 'ERREUR FORMAT FICHIER'
if len(l_doc):
thread1 = Timer(0, manage_convert_doc_to_pdf, (cmds, paths, l_doc,))
thread1.start()
if len(l_pdf):
thread = Timer(0, manage_convert_pdf_to_jpg, (l_pdf,))
thread.start()
results = {'doc_list': [d.as_json() for d in cat.get_docs()], 'n': cat.count_docs()}
return HttpResponse(json.dumps(results))
class DocumentTestCase(TestCase):
def setUp(self):
self.document_type = DocumentType(name='test doc type')
self.document_type.save()
self.document = Document(
document_type=self.document_type,
description='description',
)
self.document.save()
with open(TEST_DOCUMENT_PATH) as file_object:
self.document.new_version(file=File(file_object, name='mayan_11_1.pdf'))
with open(TEST_KEY_FILE) as file_object:
gpg.import_key(file_object.read())
def test_document_no_signature(self):
self.failUnlessEqual(DocumentVersionSignature.objects.has_detached_signature(self.document), False)
def test_new_document_version_signed(self):
with open(TEST_SIGNED_DOCUMENT_PATH) as file_object:
new_version_data = {
'comment': 'test comment 1',
'version_update': VERSION_UPDATE_MAJOR,
'release_level': RELEASE_LEVEL_FINAL,
'serial': 0,
}
self.document.new_version(file=File(file_object, name='mayan_11_1.pdf.gpg'), **new_version_data)
self.failUnlessEqual(DocumentVersionSignature.objects.has_detached_signature(self.document), False)
self.failUnlessEqual(DocumentVersionSignature.objects.verify_signature(self.document).status, SIGNATURE_STATE_VALID)
def test_detached_signatures(self):
new_version_data = {
'comment': 'test comment 2',
'version_update': VERSION_UPDATE_MAJOR,
'release_level': RELEASE_LEVEL_FINAL,
'serial': 0,
}
with open(TEST_DOCUMENT_PATH) as file_object:
self.document.new_version(file=File(file_object), **new_version_data)
# GPGVerificationError
self.failUnlessEqual(DocumentVersionSignature.objects.verify_signature(self.document), None)
with open(TEST_SIGNATURE_FILE_PATH, 'rb') as file_object:
DocumentVersionSignature.objects.add_detached_signature(self.document, File(file_object))
self.failUnlessEqual(DocumentVersionSignature.objects.has_detached_signature(self.document), True)
self.failUnlessEqual(DocumentVersionSignature.objects.verify_signature(self.document).status, SIGNATURE_STATE_VALID)
def tearDown(self):
self.document.delete()
self.document_type.delete()
def test_search_endpoint_returns_400_if_no_search_terms(
api_client, create_content):
content = create_content(5)
document = Document(title='example.txt', content=content)
document.save()
result = api_client.get(f"{SEARCH_ENDPOINT}")
assert result.status_code == 400
assert "Missing search terms" in result.data
result = api_client.get(f"{SEARCH_ENDPOINT}?term=")
assert result.status_code == 400
assert "Missing search terms" in result.data
def test_document_pdf_file_names(order, document_type, file_pattern):
seller = {"company_name": "Seller Ltd"}
buyer = {"company_name": "Buyer Ltd"}
document = Document(
document_type=Document.TYPES.get_value(document_type),
order=order,
seller=seller,
buyer=buyer,
)
document.save()
assert document.pdf_file_name == file_pattern.format(
order_id=order.id, document_id=document.id, seller=seller, buyer=buyer)
def setUp(self):
self.document_type = DocumentType(name='test doc type')
self.document_type.save()
self.document = Document(
document_type=self.document_type,
description='description',
)
self.document.save()
with open(TEST_DOCUMENT_PATH) as file_object:
self.document.new_version(file_object=File(file_object))
def test_total_price():
document = Document()
document.lines = [{
"amount": 3,
"price": 10,
"vat_rate": 20,
"count": 2,
"seller_user_id": 1
}]
assert document.price == 3 * 10 * 2
assert document.price_gross == 3 * 10 * 2 * 1.20
def test_gross_rounding():
document = Document(lines=[
{
"price": 41,
"count": 1,
"amount": 1,
"vat_rate": 7,
"seller_user_id": 1
},
{
"price": 7.9,
"count": 1,
"amount": 1,
"vat_rate": 7,
"seller_user_id": 2
},
{
"price": 4.89,
"count": 1,
"amount": 1,
"vat_rate": 19,
"seller_user_id": 3,
},
{
"price": 4.89,
"count": 1,
"amount": 1,
"vat_rate": 19,
"seller_user_id": 4,
},
])
assert document.price_gross_cents == 6396
document = Document(lines=[
{
"price": 7.80,
"count": 1,
"amount": 1,
"vat_rate": 7,
"seller_user_id": 4,
},
{
"price": 0.97,
"count": 1,
"amount": 1,
"vat_rate": 7,
"seller_user_id": 4,
},
])
assert document.price_gross_cents == 938
def list(self, request, *args, **kwargs):
self.require_folder_permission('view')
queryset = self.get_queryset().filter(folder=self.folder)
documents = filter_by_permission(queryset,
self.get_current_membership(),
PERMISSIONS.view)
Document.prefetch_revisions(documents)
serializer = self.get_serializer(documents, many=True)
return Response(serializer.data)
def __upload_core(request):
form = DocumentForm(request.POST, request.FILES)
if form.is_valid():
u = request.user
doc = Document(owner=u)
file = request.FILES['file']
doc.filename = request.FILES['file'].name
doc.save()
doc.file.save('file_' + str(doc.id), file)
return doc
else:
return Exception()
def test_search_endpoint_sorts_by_hit_count(search, obama_speech: Document):
obama_speech.save()
result = search("begin", "believe")
assert result.results[0]['search_term'] == "believe"
assert result.results[1]['search_term'] == "begin"
result = search("believe", "begin")
assert result.results[0]['search_term'] == "believe"
assert result.results[1]['search_term'] == "begin"
# TODO: Choose some words
# TODO: Find a pretty way to display the report!
# TODO: Finish README.md
def document_clear_image_cache(request):
check_permissions(request.user, [PERMISSION_DOCUMENT_TOOLS])
previous = request.POST.get('previous', request.GET.get('previous', request.META.get('HTTP_REFERER', '/')))
if request.method == 'POST':
try:
Document.clear_image_cache()
messages.success(request, _(u'Document image cache cleared successfully'))
except Exception, msg:
messages.error(request, _(u'Error clearing document image cache; %s') % msg)
return HttpResponseRedirect(previous)
class FolderTestCase(unittest.TestCase):
def setUp(self):
self.document_type = DocumentType(name='test doc type')
self.document_type.save()
self.document = Document(
document_type=self.document_type,
description='description',
)
self.document.save()
file_object = open(os.path.join(settings.SITE_ROOT, 'contrib', 'sample_documents', 'mayan_11_1.pdf'))
new_version = self.document.new_version(file=File(file_object, name='mayan_11_1.pdf'))
file_object.close()
def test_creation_of_folder(self):
user = User.objects.all()[0]
folder = Folder.objects.create(title='test', user=user)
self.assertEqual(Folder.objects.all().count(), 1)
self.assertEqual(list(Folder.objects.all()), [folder])
folder.delete()
def test_addition_of_documents(self):
user = User.objects.all()[0]
folder = Folder.objects.create(title='test', user=user)
folder.add_document(self.document)
self.assertEqual(folder.documents.count(), 1)
self.assertEqual(list(folder.documents), [self.document])
folder.delete()
def test_addition_and_deletion_of_documents(self):
user = User.objects.all()[0]
folder = Folder.objects.create(title='test', user=user)
folder.add_document(self.document)
self.assertEqual(folder.documents.count(), 1)
self.assertEqual(list(folder.documents), [self.document])
folder.remove_document(self.document)
self.assertEqual(folder.documents.count(), 0)
self.assertEqual(list(folder.documents), [])
folder.delete()
def tearDown(self):
self.document.delete()
self.document_type.delete()
def report(request, lot_number):
report = Report.objects.get(lot_number=lot_number)
audit_log = target_stream(report)
orders = OrderLineItem.objects.filter(report=report)
newdoc = NewDocumentForm(None)
try:
next_report = report.get_next_by_created_at()
except:
next_report = None
try:
prev_report = report.get_previous_by_created_at()
except:
prev_report = None
if 'document_button' in request.POST:
newdoc = NewDocumentForm(request.POST, request.FILES)
if newdoc.is_valid():
doc = Document(type=newdoc.cleaned_data['type'],
file=request.FILES['file'],
created_by=request.user)
doc.save()
action.send(request.user,
verb="created document",
action_object=doc)
rd = ReportDocument(
report=report,
document=doc,
primary_document=newdoc.cleaned_data['primary'],
created_by=request.user,
internal_cert=newdoc.cleaned_data['internal'])
rd.save()
action.send(request.user,
verb="attached document",
action_object=rd,
target=report)
report.save()
messages.success(request, 'Document upload successful.')
return HttpResponseRedirect(
reverse('reports.views.report', args=[report.lot_number]))
return render_to_response('reports/report.html', {
'report': report,
'new_document_form': newdoc,
'orders': orders,
'audit_log': audit_log,
'next_report': next_report,
'prev_report': prev_report,
},
context_instance=RequestContext(request))
def upload_single_file(self, file_object, filename=None, use_file_name=False, document_type=None, metadata_dict_list=None, user=None, document=None, new_version_data=None):
if not document:
document = Document()
if document_type:
document.document_type = document_type
document.save()
if metadata_dict_list:
save_metadata_list(metadata_dict_list, document, create=True)
warnings = update_indexes(document)
if user:
document.add_as_recent_document_for_user(user)
create_history(HISTORY_DOCUMENT_CREATED, document, {'user': user})
else:
create_history(HISTORY_DOCUMENT_CREATED, document)
else:
if use_file_name:
filename = None
else:
filename = filename if filename else document.latest_version.filename
if not new_version_data:
new_version_data = {}
new_version = document.new_version(file=file_object, **new_version_data)
if filename:
new_version.filename = filename
new_version.save()
transformations, errors = self.get_transformation_list()
new_version.apply_default_transformations(transformations)
def test_list(self):
self.folder = Folder(**{'name': 'Images'})
self.folder.save()
files = ['2210571.jpg', 'images.docx', 'rgba1px.png']
for file in files:
path = 'tests/files/%s' % file
f = open(path, 'rb')
document = Document()
document.folder = self.folder
document.name = file
document.file.save(name=document.name, content=File(f))
document.save()
document.save_thumbnail()
f.close()
url = reverse('model_view_list',
args=['documents.Document', 'folder', self.folder.pk])
response = self.client.get(url)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(len(response.data), len(files))
url = reverse(
'model_view_detail',
args=['documents.Document', 'folder', self.folder.pk, files[0]])
response = self.client.get(url)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data['name'], files[0])
def upload_document(self, file_object, document_type, description=None, label=None, language=None, metadata_dict_list=None, metadata_dictionary=None, tag_ids=None, user=None):
"""
Upload an individual document
"""
try:
with transaction.atomic():
document = Document(
description=description or '', document_type=document_type,
label=label or file_object.name,
language=language or setting_language.value
)
document.save(_user=user)
except Exception as exception:
logger.critical(
'Unexpected exception while trying to create new document '
'"%s" from source "%s"; %s',
label or file_object.name, self, exception
)
raise
else:
try:
document_version = document.new_version(
file_object=file_object, _user=user,
)
if user:
document.add_as_recent_document_for_user(user)
Transformation.objects.copy(
source=self, targets=document_version.pages.all()
)
if metadata_dict_list:
save_metadata_list(
metadata_dict_list, document, create=True
)
if metadata_dictionary:
set_bulk_metadata(
document=document,
metadata_dictionary=metadata_dictionary
)
if tag_ids:
for tag in Tag.objects.filter(pk__in=tag_ids):
tag.documents.add(document)
except Exception as exception:
logger.critical(
'Unexpected exception while trying to create version for '
'new document "%s" from source "%s"; %s',
label or file_object.name, self, exception
)
document.delete(to_trash=False)
raise
def setUp(self):
self.admin_user = User.objects.create_superuser(
username=TEST_ADMIN_USERNAME,
email=TEST_ADMIN_EMAIL,
password=TEST_ADMIN_PASSWORD)
self.document_type = DocumentType(name=TEST_DOCUMENT_TYPE)
self.document_type.save()
self.document = Document(document_type=self.document_type, )
self.document.save()
with open(TEST_DOCUMENT_PATH) as file_object:
self.document.new_version(
file_object=File(file_object, name='mayan_11_1.pdf'))
def ready(self):
super(FoldersApp, self).ready()
APIEndPoint(app=self, version_string='1')
Document.add_to_class(
'document_folders',
lambda document: DocumentFolder.objects.filter(documents=document))
ModelPermission.register(
model=Document,
permissions=(permission_folder_add_document,
permission_folder_remove_document))
ModelPermission.register(
model=Folder,
permissions=(permission_acl_edit, permission_acl_view,
permission_folder_delete, permission_folder_edit,
permission_folder_view))
SourceColumn(source=Folder,
label=_('Created'),
attribute='datetime_created')
SourceColumn(source=Folder, label=_('User'), attribute='user')
SourceColumn(source=Folder,
label=_('Documents'),
func=lambda context: context['object'].get_document_count(
user=context['request'].user))
menu_facet.bind_links(links=(link_document_folder_list, ),
sources=(Document, ))
menu_main.bind_links(links=(link_folder_list, ))
menu_multi_item.bind_links(
links=(link_folder_add_multiple_documents, ), sources=(Document, ))
menu_multi_item.bind_links(
links=(link_folder_document_multiple_remove, ),
sources=('folders:folder_view', ))
menu_object.bind_links(links=(link_folder_view, ),
sources=(DocumentFolder, ))
menu_object.bind_links(links=(link_folder_view, link_folder_edit,
link_acl_list, link_folder_delete),
sources=(Folder, ))
menu_secondary.bind_links(links=(link_folder_list, link_folder_create),
sources=(Folder, 'folders:folder_list',
'folders:folder_create'))
menu_sidebar.bind_links(links=(link_folder_add_document, ),
sources=('folders:document_folder_list',
'folders:folder_add_document'))
def initialize_document_checkout_extra_methods():
Document.add_to_class(
'is_checked_out', lambda document: DocumentCheckout.objects.
is_document_checked_out(document))
Document.add_to_class('check_in',
lambda document, user=None: DocumentCheckout.objects.
check_in_document(document, user))
Document.add_to_class(
'checkout_info', lambda document: DocumentCheckout.objects.
document_checkout_info(document))
Document.add_to_class(
'checkout_state', lambda document: DocumentCheckout.objects.
document_checkout_state(document))
Document.add_to_class('is_new_versions_allowed',
lambda document, user=None: DocumentCheckout.objects.
is_document_new_versions_allowed(document, user))
def document_from_sql(name, code, host, db_user, db_password, db_name,
table_name):
description = get_table_description(host, db_user, db_password, db_name,
table_name)
try:
document = Document(name=name, code=code)
except:
raise Exception('Error creating new document, check name and code.')
fields_description = dict()
for field in description:
fields_description[field] = ''
document.fields_description = str(fields_description)
document.save()
return document
def setUp(self):
self.document_type = DocumentType(name='test doc type')
self.document_type.save()
self.document = Document(
document_type=self.document_type,
description='description',
)
self.document.save()
with open(TEST_DOCUMENT_PATH) as file_object:
self.document.new_version(
file_object=File(file_object, name='mayan_11_1.pdf'))
with open(TEST_KEY_FILE) as file_object:
gpg.import_key(file_object.read())
def setUp(self):
# Start the OCR queue
self.default_queue = DocumentQueue.objects.get(name='default')
self.document_type = DocumentType.objects.create(name='test doc type')
self.document = Document(
document_type=self.document_type,
description='description',
)
self.document.save()
with open(TEST_DOCUMENT_PATH) as file_object:
self.document.new_version(
file=File(file_object, name='title_page.png'))
# Clear OCR queue
QueueDocument.objects.all().delete()
class DocumentTestCase(TestCase):
def setUp(self):
self.document_type = DocumentType(name="test doc type")
self.document_type.save()
self.document = Document(document_type=self.document_type, description="description")
self.document.save()
with open(TEST_DOCUMENT_PATH) as file_object:
self.document.new_version(file_object=File(file_object, name="mayan_11_1.pdf"))
with open(TEST_KEY_FILE) as file_object:
gpg.import_key(file_object.read())
def test_document_no_signature(self):
self.failUnlessEqual(DocumentVersionSignature.objects.has_detached_signature(self.document), False)
def test_new_document_version_signed(self):
with open(TEST_SIGNED_DOCUMENT_PATH) as file_object:
new_version_data = {"comment": "test comment 1"}
self.document.new_version(file_object=File(file_object, name="mayan_11_1.pdf.gpg"), **new_version_data)
self.failUnlessEqual(DocumentVersionSignature.objects.has_detached_signature(self.document), False)
self.failUnlessEqual(
DocumentVersionSignature.objects.verify_signature(self.document).status, SIGNATURE_STATE_VALID
)
def test_detached_signatures(self):
new_version_data = {"comment": "test comment 2"}
with open(TEST_DOCUMENT_PATH) as file_object:
self.document.new_version(file_object=File(file_object), **new_version_data)
# GPGVerificationError
self.failUnlessEqual(DocumentVersionSignature.objects.verify_signature(self.document), None)
with open(TEST_SIGNATURE_FILE_PATH, "rb") as file_object:
DocumentVersionSignature.objects.add_detached_signature(self.document, File(file_object))
self.failUnlessEqual(DocumentVersionSignature.objects.has_detached_signature(self.document), True)
self.failUnlessEqual(
DocumentVersionSignature.objects.verify_signature(self.document).status, SIGNATURE_STATE_VALID
)
def tearDown(self):
self.document.delete()
self.document_type.delete()
def upload_document(self,
file_object,
document_type,
description=None,
label=None,
language=None,
querystring=None,
user=None):
"""
Upload an individual document
"""
try:
with transaction.atomic():
document = Document(description=description or '',
document_type=document_type,
label=label or file_object.name,
language=language
or setting_language.value)
document.save(_user=user)
except Exception as exception:
logger.critical(
'Unexpected exception while trying to create new document '
'"%s" from source "%s"; %s', label or file_object.name, self,
exception)
raise
else:
try:
document_version = document.new_version(
file_object=file_object,
_user=user,
)
if user:
document.add_as_recent_document_for_user(user)
Transformation.objects.copy(
source=self, targets=document_version.pages.all())
except Exception as exception:
logger.critical(
'Unexpected exception while trying to create version for '
'new document "%s" from source "%s"; %s',
label or file_object.name,
self,
exception,
exc_info=True)
document.delete(to_trash=False)
raise
else:
WizardStep.post_upload_process(document=document,
querystring=querystring)
return document
def create_document(self):
"""Creates new random documents"""
doc = Document()
doc.name_nl = _faker.text(max_nb_chars=30)
doc.name_en = _faker.text(max_nb_chars=30)
doc.category = random.choice(
[c[0] for c in Document.DOCUMENT_CATEGORIES])
doc.members_only = random.random() < 0.75
doc.file_en.save("{}.txt".format(doc.name_en),
ContentFile(_faker.text(max_nb_chars=120)))
doc.file_nl = doc.file_en
doc.save()
def form_valid(self, form):
new_homework=Homework(
entered_by=self.request.user,
entered_on=datetime.today(),
)
new_homework.save()
new_details=form.save(commit=False)
new_details.deleted=False
new_details.hwk=new_homework
if self.request.user.has_perm('classlists.is_kksastaff'):
#creating multiple copies of the same hwk details record, one for each class
for k in form.cleaned_data['klass']:
new_details.pk=None
new_details.klass=k
new_details.save()
if form.cleaned_data['link']:
new_link=Link(
link=form.cleaned_data['link'],
description=form.cleaned_data['link_description'],
homework=new_homework,
subject=new_details.subject,
)
new_link.save()
for k in form.cleaned_data['klass']:
new_link.klass.add(k)
if form.cleaned_data['attached_file']:
new_document=Document(
attached_file=form.cleaned_data['attached_file'],
filename=form.cleaned_data['attached_file'].name,
description=form.cleaned_data['document_description'],
homework=new_homework,
subject=new_details.subject,
)
new_document.save()
for k in form.cleaned_data['klass']:
new_document.klass.add(k)
else:
new_details.klass=Klass.objects.get(klass_name=self.kwargs['class_url'])
new_details.save()
return HttpResponseRedirect(reverse('homework_view', args=(self.kwargs['class_url'],),))
def test_search_endpoint_can_return_multiple_documents_single_word_query(
search, title):
word = "hello"
first_doc = Document(title=title(), content=word)
first_doc.save()
second_doc = Document(title=title(), content=word)
second_doc.save()
result = search(word)
assert first_doc.title in result.result_for('hello')['documents']
assert second_doc.title in result.result_for('hello')['documents']
def create(self, validated_data):
body = validated_data.get('body')
body = base64_decode(body)[0]
document = Document()
document.account = self.context['account']
document.folder = validated_data.get('folder')
document.type = validated_data.get('type') or Document.OTHER
document.file = ContentFile(body, validated_data.get('name'))
document.user = self.context['user']
old_doc = validated_data.get('old_doc')
if old_doc:
document.previous_version = old_doc.id
document.save()
return document
def test_pdf_indexing(self):
doc = Document()
self.assign_fixture_to_filefield('word_pdf.pdf', doc.file)
self.assertEquals(
"\nLorem ipsum dolor sit amet, consectetur " +
"adipiscing elit.\nPraesent pharetra urna eu arcu blandit " +
"nec pretium odio fermentum. Sed in orci quis risus interdum " +
"lacinia ut eu nisl.\n\n\n", self.index.prepare_text(doc))
def setUp(self):
self.document_type = DocumentType(name="test doc type")
self.document_type.save()
self.document = Document(document_type=self.document_type, description="description")
self.document.save()
with open(TEST_DOCUMENT_PATH) as file_object:
self.document.new_version(file_object=File(file_object))
def mtr_generator(request):
lot_number = request.GET.get('lot_number', None)
if lot_number:
report = Report.objects.get(lot_number=lot_number)
if request.method == 'POST':
mtrform = MTRForm(request.POST)
if mtrform.is_valid():
new_mtr = MTR(properties=mtrform.cleaned_data)
pdf = new_mtr.generate_pdf()
fp = StringIO()
pdf.write(fp)
file = InMemoryUploadedFile(fp, None, "%s.pdf" % report.lot_number,
'application/pdf', fp.len, None)
doc_type = DocumentType.objects.get(name='Material Test Report')
doc = Document(type=doc_type, file=file, created_by=request.user)
doc.save()
rd = ReportDocument(report=report,
document=doc,
primary_document=True,
created_by=request.user,
internal_cert=True)
rd.save()
report.save()
messages.success(request, 'Document upload successful.')
action.send(request.user,
verb="generated an MTR ",
action_object=rd,
target=report)
return HttpResponseRedirect(
reverse('reports.views.report', args=[report.lot_number]))
else:
mtrform = MTRForm(
initial={
'part_number': report.part_number,
'heat_number': report.heat_number,
'size': report.part_number.description,
})
return render_to_response('pdfgen/new_mtr.html', {
'form': mtrform,
},
context_instance=RequestContext(request))
def handle(self, *args, **options):
# Login
self.authorize(options)
# Loop through them...
for obj in Document.all():
# ... and do the deed.
print "Deleted %s" % obj
obj.delete()
def pay_for_document(
document: Document,
author_id: str,
source_wallet_id: str,
destination_wallet_id: str,
amount: float,
fees: float = 0,
db="default",
):
"""
Atomic transaction to pay for the document an mark document as paid
:param document:
:param author_id:
:param source_wallet_id:
:param destination_wallet_id:
:param amount:
:param fees:
:return:
"""
mangopay = MangopayMixin()
with transaction.atomic(using=db):
try:
document = (Document.objects.using(db).filter(
id=document.id).select_for_update(nowait=True).get())
except OperationalError:
raise DuplicateTransferError(
f"Document {document.id} is being paid by other process")
if document.paid:
raise DuplicateTransferError(
f"Document {document.id} already paid")
if source_wallet_id != destination_wallet_id and not document.paid:
mangopay.mangopay.transfer(
author_id,
source_wallet_id,
destination_wallet_id,
amount=amount,
fees=fees,
tag=document.mangopay_tag,
)
document.paid = True
document.save(update_fields=("paid", "updated_at"))
def handle(self, *args, **options):
# Login
self.authorize(options)
# Loop through them...
for obj in Document.all():
# ... and do the deed.
print "Deleted %s" % obj
obj.delete()
def test_powerpoint_2007_indexing(self):
doc = Document()
self.assign_fixture_to_filefield('powerpoint_2007.pptx', doc.file)
self.assertEquals(
"Lorem Ipsum\n" +
"Lorem ipsum dolor sit amet, consectetur adipiscing elit.\n" +
"Praesent pharetra urna eu arcu blandit nec pretium odio " +
"fermentum.\n" +
"Sed in orci quis risus interdum lacinia ut eu nisl.\n",
self.index.prepare_text(doc))
def upload_file(request, slug):
form = UploadFileForm(request.POST, request.FILES)
if form.is_valid():
course = get_object_or_404(Course, slug=slug)
d = Document.new(request.user, course, request.FILES['file'].name,
form.cleaned_data['category'])
course.add_document(d)
transaction.commit()
run_process_file(d.id, request.FILES['file'])
# FIXME add an error management
return HttpResponseRedirect(reverse('course_show', args=[slug]))
def setUp(self):
self.document_type = DocumentType(name="test doc type")
self.document_type.save()
self.document = Document(document_type=self.document_type, description="description")
self.document.save()
with open(TEST_DOCUMENT_PATH) as file_object:
self.document.new_version(file_object=File(file_object, name="mayan_11_1.pdf"))
with open(TEST_KEY_FILE) as file_object:
gpg.import_key(file_object.read())
def setUp(self):
self.document_type = DocumentType(name='test doc type')
self.document_type.save()
self.document = Document(
document_type=self.document_type,
description='description',
)
self.document.save()
with open(TEST_DOCUMENT_PATH) as file_object:
self.document.new_version(file=File(file_object, name='nucleos_11_1.pdf'))
def upload_single_file(self, file_object, filename=None, use_file_name=False, document_type=None, metadata_dict_list=None, user=None, document=None, new_version_data=None):
new_document = not document
if not document:
document = Document()
if document_type:
document.document_type = document_type
document.save()
apply_default_acls(document, user)
if user:
document.add_as_recent_document_for_user(user)
create_history(HISTORY_DOCUMENT_CREATED, document, {'user': user})
else:
create_history(HISTORY_DOCUMENT_CREATED, document)
else:
if use_file_name:
filename = None
else:
filename = filename if filename else document.latest_version.filename
if not new_version_data:
new_version_data = {}
try:
new_version = document.new_version(file=file_object, user=user, **new_version_data)
except Exception:
# Don't leave the database in a broken state
# document.delete()
transaction.rollback()
raise
if filename:
document.rename(filename)
transformations, errors = self.get_transformation_list()
new_version.apply_default_transformations(transformations)
#TODO: new HISTORY for version updates
if metadata_dict_list and new_document:
# Only do for new documents
save_metadata_list(metadata_dict_list, document, create=True)
warnings = update_indexes(document)
class DocumentSearchTestCase(unittest.TestCase):
def setUp(self):
# Start the OCR queue
self.default_queue = DocumentQueue.objects.get(name='default')
def test_do_document_ocr(self):
self.document_type = DocumentType(name='test doc type')
self.document_type.save()
self.document = Document(
document_type=self.document_type,
description='description',
)
self.document.save()
with open(TEST_DOCUMENT_PATH) as file_object:
self.document.new_version(file=File(file_object, name='title_page.png'))
self.failUnlessEqual(self.default_queue.queuedocument_set.count(), 1)
do_document_ocr(self.default_queue.queuedocument_set.all()[0])
self.assertTrue(u'Mayan EDMS' in self.document.pages.all()[0].content)
def tearDown(self):
self.document.delete()
self.document_type.delete()
def upload_document(request):
if request.method == 'GET':
return render_to_response('documents/document_upload.html',
RequestContext(request),
context_instance=RequestContext(request)
)
elif request.method == 'POST':
try:
content_type = ContentType.objects.get(name=request.POST['ctype'])
object_id = request.POST['objid']
except:
content_type = None
object_id = None
try:
int(object_id)
except:
if object_id is not None:
object_id = Layer.objects.get(uuid=object_id).id
file = request.FILES['file']
title = request.POST['title']
document = Document(content_type=content_type, object_id=object_id, title=title, file=file)
document.owner = request.user
document.save()
document.set_default_permissions()
permissionsStr = request.POST['permissions']
permissions = json.loads(permissionsStr)
set_document_permissions(document, permissions)
return HttpResponse(json.dumps({'success': True,'redirect_to':'/documents/' + str(document.id)}))
def mtr_generator(request):
lot_number = request.GET.get('lot_number', None)
if lot_number:
report = Report.objects.get(lot_number=lot_number)
if request.method == 'POST':
mtrform = MTRForm(request.POST)
if mtrform.is_valid():
new_mtr = MTR(properties=mtrform.cleaned_data)
pdf = new_mtr.generate_pdf()
fp = StringIO()
pdf.write(fp)
file = InMemoryUploadedFile(fp, None, "%s.pdf" % report.lot_number,
'application/pdf', fp.len, None)
doc_type = DocumentType.objects.get(name='Material Test Report')
doc = Document(type=doc_type, file=file, created_by=request.user)
doc.save()
rd = ReportDocument(report=report, document=doc, primary_document=True, created_by=request.user, internal_cert=True)
rd.save()
report.save()
messages.success(request, 'Document upload successful.')
action.send(request.user, verb="generated an MTR ", action_object=rd, target=report)
return HttpResponseRedirect(reverse('reports.views.report',
args=[report.lot_number]))
else:
mtrform = MTRForm(initial={
'part_number': report.part_number,
'heat_number': report.heat_number,
'size': report.part_number.description,
})
return render_to_response('pdfgen/new_mtr.html',
{
'form': mtrform,
},
context_instance=RequestContext(request))
def handle_noargs(self, **options):
doc_dir = os.path.join(os.path.dirname(__file__), '..', '..',
'fixtures')
doctypes = DocumentType.objects.all()
programs = Program.objects.all()
users = IntranetUser.objects.all()
for i in range(1000):
for f in os.listdir(doc_dir):
doc = Document()
doc.title = random.random()
doc.file = File(open(os.path.join(doc_dir, f)))
doc.notes = random.random()
doc.document_type = random.choice(doctypes)
doc.save()
doc.programs = random.sample(programs, 2)
doc.authors = random.sample(users, 1)
def setUp(self):
self.admin_user = User.objects.create_superuser(username=TEST_ADMIN_USERNAME, email=TEST_ADMIN_EMAIL, password=TEST_ADMIN_PASSWORD)
self.document_type = DocumentType(name=TEST_DOCUMENT_TYPE)
self.document_type.save()
self.document = Document(
document_type=self.document_type,
)
self.document.save()
with open(TEST_DOCUMENT_PATH) as file_object:
self.document.new_version(file_object=File(file_object, name='mayan_11_1.pdf'))
def setUp(self):
self.document_type = DocumentType(name='test doc type')
self.document_type.save()
self.document = Document(
document_type=self.document_type,
description='description',
)
self.document.save()
file_object = open(os.path.join(settings.PROJECT_ROOT, 'contrib', 'mayan_11_1.pdf'))
new_version = self.document.new_version(file=File(file_object, name='mayan_11_1.pdf'))
file_object.close()
def upload_http(request, slug):
form = UploadHttpForm(request.POST)
if form.is_valid():
course = get_object_or_404(Course, slug=slug)
url = escape(form.cleaned_data["url"])
name = match(r".*/([^/]+)$", url).group(1)
if len(name) < 4:
return HttpResponse("name invalid", "text/html")
doc = Document.new(request.user, course, name, escape(form.cleaned_data["category"]))
course.add_document(doc)
PendingDocument.objects.create(doc=doc, state="queued", url=url)
return HttpResponse("ok", "text/html")
return HttpResponse("form invalid", "text/html")
def forwards(self, orm):
"Write your forwards methods here."
# This will transfer all the resumes to the documents
# Note: Don't use "from appname.models import ModelName".
# Use orm.ModelName to refer to models in this application,
# and orm['appname.ModelName'] for models in other applications.
# 1. get all 'original' resumes
# 2. for each 'original' resume
# 3. get corresponding 'non-original' resume which is the image
# 4. create Document entry and store it
print "hello"
for resume in orm['resumes.resume'].objects.filter(original=True):
# get non-original-resume and check if the same user and uploaded within 10 mins
print resume
non_org_resume = orm['resumes.resume'].objects.filter(id=resume.id+1)
if non_org_resume and non_org_resume[0].original == False:
non_org_resume = non_org_resume[0]
ten_minutes = datetime.timedelta(minutes=10)
if resume.user == non_org_resume.user and non_org_resume.timestamp - resume.timestamp <= ten_minutes:
s=string.lowercase+string.digits
unique_hash = ''.join(random.sample(s,10))
new_doc = Document(user_id=resume.user.id, document_type=1, image_url=non_org_resume.url, url=resume.url, timestamp=resume.timestamp, unique_hash=unique_hash)
new_doc.save()
class DocumentTestCase(unittest.TestCase):
def setUp(self):
self.document_type = DocumentType(name='test doc type')
self.document_type.save()
self.document = Document(
document_type=self.document_type,
description='description',
)
self.document.save()
file_object = open(os.path.join(settings.PROJECT_ROOT, 'contrib', 'mayan_11_1.pdf'))
new_version = self.document.new_version(file=File(file_object, name='mayan_11_1.pdf'))
file_object.close()
def test_document_no_signature(self):
self.failUnlessEqual(DocumentVersionSignature.objects.has_detached_signature(self.document), False)
def test_new_document_version_signed(self):
file_object = open(os.path.join(settings.PROJECT_ROOT, 'contrib', 'mayan_11_1.pdf.gpg'))
new_version_data = {
'comment': 'test comment 1',
'version_update': VERSION_UPDATE_MAJOR,
'release_level': RELEASE_LEVEL_FINAL,
'serial': 0,
}
self.failUnlessEqual(DocumentVersionSignature.objects.has_detached_signature(self.document), False)
# self.failUnlessEqual(DocumentVersionSignature.objects.verify_signature(self.document).status, SIGNATURE_STATE_VALID)
# TODO: verify_signature is failing, check
def test_detached_signatures(self):
new_version_data = {
'comment': 'test comment 2',
'version_update': VERSION_UPDATE_MAJOR,
'release_level': RELEASE_LEVEL_FINAL,
'serial': 0,
}
file_object = open(os.path.join(settings.PROJECT_ROOT, 'contrib', 'mayan_11_1.pdf'))
new_version = self.document.new_version(file=File(file_object), **new_version_data)
file_object.close()
# GPGVerificationError
self.failUnlessEqual(DocumentVersionSignature.objects.verify_signature(self.document), None)
file_object = open(os.path.join(settings.PROJECT_ROOT, 'contrib', 'mayan_11_1.pdf.sig'), 'rb')
DocumentVersionSignature.objects.add_detached_signature(self.document, File(file_object))
file_object.close()
self.failUnlessEqual(DocumentVersionSignature.objects.has_detached_signature(self.document), True)
# self.failUnlessEqual(DocumentVersionSignature.objects.verify_signature(self.document).status, SIGNATURE_STATE_VALID)
# TODO: verify_signature is failing, check
def tearDown(self):
self.document.delete()
self.document_type.delete()
