def update_or_create_document(yaml_obj):
"""
Submit an object read from our YAML files and it will update it in the
database, creating it if it doesn't already exist.
Returns the database object, and a boolean that is true if a new object
was created.
"""
# Check if the table already exists in the datastore
obj = Document.get_by_key_name(yaml_obj.get('slug'))
# Update the obj if it exists
if obj:
# Loop through the keys and update the object one by one.
for key in yaml_obj.keys():
# With some special casing for projects...
if key == 'project_slug':
proj = Project.get_by_key_name(yaml_obj.get('project_slug'))
obj.project = proj
# ...and for tags.
elif key == 'tags':
obj.tags = get_tag_keys(yaml_obj.get("tags"))
else:
setattr(obj, key, yaml_obj.get(key))
# Save it out
obj.put()
created = False
# Create it if it doesn't
else:
# If it has tags....
if yaml_obj.has_key('tags'):
# Convert to database keys
tags = get_tag_keys(yaml_obj.pop("tags"))
# Load the data
obj = Document(key_name=yaml_obj.get('slug'), **yaml_obj)
# Set the tags
obj.tags = tags
# Otherwise....
else:
# Update the basic values
obj = Document(key_name=yaml_obj.get('slug'), **yaml_obj)
# And clear out the tag data
obj.tags = []
obj.similar_documents = []
# Connected it to a project, if it exists
if yaml_obj.has_key('project_slug'):
proj = Project.get_by_key_name(yaml_obj.get('project_slug'))
obj.project = proj
# Save it out
obj.put()
created = True
# Update the similarity lists of documents with the same tags
taskqueue.add(
url='/_/document/update-similar/',
params=dict(key=obj.key()),
method='GET'
)
# Pass it out
return obj, created
def test_search_endpoint_can_return_multiple_documents_single_word_query(
search, title):
word = "hello"
first_doc = Document(title=title(), content=word)
first_doc.save()
second_doc = Document(title=title(), content=word)
second_doc.save()
result = search(word)
assert first_doc.title in result.result_for('hello')['documents']
assert second_doc.title in result.result_for('hello')['documents']
def test_gross_rounding():
document = Document(lines=[
{
"price": 41,
"count": 1,
"amount": 1,
"vat_rate": 7,
"seller_user_id": 1
},
{
"price": 7.9,
"count": 1,
"amount": 1,
"vat_rate": 7,
"seller_user_id": 2
},
{
"price": 4.89,
"count": 1,
"amount": 1,
"vat_rate": 19,
"seller_user_id": 3,
},
{
"price": 4.89,
"count": 1,
"amount": 1,
"vat_rate": 19,
"seller_user_id": 4,
},
])
assert document.price_gross_cents == 6396
document = Document(lines=[
{
"price": 7.80,
"count": 1,
"amount": 1,
"vat_rate": 7,
"seller_user_id": 4,
},
{
"price": 0.97,
"count": 1,
"amount": 1,
"vat_rate": 7,
"seller_user_id": 4,
},
])
assert document.price_gross_cents == 938
def forwards(self, orm):
"Write your forwards methods here."
# This will transfer all the resumes to the documents
# Note: Don't use "from appname.models import ModelName".
# Use orm.ModelName to refer to models in this application,
# and orm['appname.ModelName'] for models in other applications.
# 1. get all 'original' resumes
# 2. for each 'original' resume
# 3. get corresponding 'non-original' resume which is the image
# 4. create Document entry and store it
print "hello"
for resume in orm['resumes.resume'].objects.filter(original=True):
# get non-original-resume and check if the same user and uploaded within 10 mins
print resume
non_org_resume = orm['resumes.resume'].objects.filter(
id=resume.id + 1)
if non_org_resume and non_org_resume[0].original == False:
non_org_resume = non_org_resume[0]
ten_minutes = datetime.timedelta(minutes=10)
if resume.user == non_org_resume.user and non_org_resume.timestamp - resume.timestamp <= ten_minutes:
s = string.lowercase + string.digits
unique_hash = ''.join(random.sample(s, 10))
new_doc = Document(user_id=resume.user.id,
document_type=1,
image_url=non_org_resume.url,
url=resume.url,
timestamp=resume.timestamp,
unique_hash=unique_hash)
new_doc.save()
def upload_single_file(self, file_object, filename=None, use_file_name=False, document_type=None, metadata_dict_list=None, user=None, document=None, new_version_data=None):
if not document:
document = Document()
if document_type:
document.document_type = document_type
document.save()
if metadata_dict_list:
save_metadata_list(metadata_dict_list, document, create=True)
warnings = update_indexes(document)
if user:
document.add_as_recent_document_for_user(user)
create_history(HISTORY_DOCUMENT_CREATED, document, {'user': user})
else:
create_history(HISTORY_DOCUMENT_CREATED, document)
else:
if use_file_name:
filename = None
else:
filename = filename if filename else document.latest_version.filename
if not new_version_data:
new_version_data = {}
new_version = document.new_version(file=file_object, **new_version_data)
if filename:
new_version.filename = filename
new_version.save()
transformations, errors = self.get_transformation_list()
new_version.apply_default_transformations(transformations)
def test_list(self):
self.folder = Folder(**{'name': 'Images'})
self.folder.save()
files = ['2210571.jpg', 'images.docx', 'rgba1px.png']
for file in files:
path = 'tests/files/%s' % file
f = open(path, 'rb')
document = Document()
document.folder = self.folder
document.name = file
document.file.save(name=document.name, content=File(f))
document.save()
document.save_thumbnail()
f.close()
url = reverse('model_view_list',
args=['documents.Document', 'folder', self.folder.pk])
response = self.client.get(url)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(len(response.data), len(files))
url = reverse(
'model_view_detail',
args=['documents.Document', 'folder', self.folder.pk, files[0]])
response = self.client.get(url)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data['name'], files[0])
def upload_single_file(self,
file_object,
filename=None,
use_file_name=False,
document_type=None,
metadata_dict_list=None,
user=None,
document=None,
new_version_data=None):
new_document = not document
if not document:
document = Document()
if document_type:
document.document_type = document_type
document.save()
apply_default_acls(document, user)
if user:
document.add_as_recent_document_for_user(user)
create_history(HISTORY_DOCUMENT_CREATED, document,
{'user': user})
else:
create_history(HISTORY_DOCUMENT_CREATED, document)
else:
if use_file_name:
filename = None
else:
filename = filename if filename else document.latest_version.filename
if not new_version_data:
new_version_data = {}
try:
new_version = document.new_version(file=file_object,
user=user,
**new_version_data)
except Exception:
# Don't leave the database in a broken state
# document.delete()
transaction.rollback()
raise
if filename:
document.rename(filename)
transformations, errors = self.get_transformation_list()
new_version.apply_default_transformations(transformations)
#TODO: new HISTORY for version updates
if metadata_dict_list and new_document:
# Only do for new documents
save_metadata_list(metadata_dict_list, document, create=True)
warnings = update_indexes(document)
return document
def test_pdf_indexing(self):
doc = Document()
self.assign_fixture_to_filefield('word_pdf.pdf', doc.file)
self.assertEquals(
"\nLorem ipsum dolor sit amet, consectetur " +
"adipiscing elit.\nPraesent pharetra urna eu arcu blandit " +
"nec pretium odio fermentum. Sed in orci quis risus interdum " +
"lacinia ut eu nisl.\n\n\n", self.index.prepare_text(doc))
def test_search_endpoint_returns_400_if_no_search_terms(
api_client, create_content):
content = create_content(5)
document = Document(title='example.txt', content=content)
document.save()
result = api_client.get(f"{SEARCH_ENDPOINT}")
assert result.status_code == 400
assert "Missing search terms" in result.data
result = api_client.get(f"{SEARCH_ENDPOINT}?term=")
assert result.status_code == 400
assert "Missing search terms" in result.data
def test_powerpoint_2007_indexing(self):
doc = Document()
self.assign_fixture_to_filefield('powerpoint_2007.pptx', doc.file)
self.assertEquals(
"Lorem Ipsum\n" +
"Lorem ipsum dolor sit amet, consectetur adipiscing elit.\n" +
"Praesent pharetra urna eu arcu blandit nec pretium odio " +
"fermentum.\n" +
"Sed in orci quis risus interdum lacinia ut eu nisl.\n",
self.index.prepare_text(doc))
def test_euro_to_cents_conversion():
document = Document(lines=[{
"price": 18.17,
"count": 1,
"amount": 1,
"vat_rate": 10,
"seller_user_id": 1,
}])
assert document.price_gross_cents == 1999
def upload_document(self, file_object, document_type, description=None, label=None, language=None, metadata_dict_list=None, metadata_dictionary=None, tag_ids=None, user=None):
"""
Upload an individual document
"""
try:
with transaction.atomic():
document = Document(
description=description or '', document_type=document_type,
label=label or file_object.name,
language=language or setting_language.value
)
document.save(_user=user)
except Exception as exception:
logger.critical(
'Unexpected exception while trying to create new document '
'"%s" from source "%s"; %s',
label or file_object.name, self, exception
)
raise
else:
try:
document_version = document.new_version(
file_object=file_object, _user=user,
)
if user:
document.add_as_recent_document_for_user(user)
Transformation.objects.copy(
source=self, targets=document_version.pages.all()
)
if metadata_dict_list:
save_metadata_list(
metadata_dict_list, document, create=True
)
if metadata_dictionary:
set_bulk_metadata(
document=document,
metadata_dictionary=metadata_dictionary
)
if tag_ids:
for tag in Tag.objects.filter(pk__in=tag_ids):
tag.documents.add(document)
except Exception as exception:
logger.critical(
'Unexpected exception while trying to create version for '
'new document "%s" from source "%s"; %s',
label or file_object.name, self, exception
)
document.delete(to_trash=False)
raise
def compose(self):
return (
Document(
buyer=self.buyer,
seller=lines["seller_data"],
lines=lines["lines"],
order=self._order,
delivery_address=self.delivery_address,
document_type=self.DOCUMENT_TYPE.value[0],
)
for lines in self.lines.values()
)
def test_excel_2003_indexing(self):
doc = Document()
self.assign_fixture_to_filefield('excel_2003.xls', doc.file)
self.assertEquals(
"Sheet1\n\tLorem ipsum dolor sit amet, " +
"consectetur adipiscing elit.\t\tPraesent pharetra urna eu " +
"arcu blandit nec pretium odio fermentum.\n\tSed in orci " +
"quis risus interdum lacinia ut eu nisl.\n\t\tSed facilisis " +
"nibh eu diam tincidunt pellentesque semper nulla auctor.\n" +
"\n\nSheet2\n\t\n\n\nSheet3\n\t\n\n\n",
self.index.prepare_text(doc))
def test_document_pdf_file_names(order, document_type, file_pattern):
seller = {"company_name": "Seller Ltd"}
buyer = {"company_name": "Buyer Ltd"}
document = Document(
document_type=Document.TYPES.get_value(document_type),
order=order,
seller=seller,
buyer=buyer,
)
document.save()
assert document.pdf_file_name == file_pattern.format(
order_id=order.id, document_id=document.id, seller=seller, buyer=buyer)
def create_document(self):
"""Creates new random documents"""
self.stdout.write("Creating a document")
doc = Document()
doc.name = _faker.text(max_nb_chars=30)
doc.category = random.choice(
[c[0] for c in Document.DOCUMENT_CATEGORIES])
doc.members_only = random.random() < 0.75
doc.file.save("{}.txt".format(doc.name),
ContentFile(_faker.text(max_nb_chars=120)))
doc.save()
def setUp(self):
self.document_type = DocumentType(name='test doc type')
self.document_type.save()
self.document = Document(
document_type=self.document_type,
description='description',
)
self.document.save()
with open(TEST_DOCUMENT_PATH) as file_object:
self.document.new_version(file_object=File(file_object))
def test_total_price():
document = Document()
document.lines = [{
"amount": 3,
"price": 10,
"vat_rate": 20,
"count": 2,
"seller_user_id": 1
}]
assert document.price == 3 * 10 * 2
assert document.price_gross == 3 * 10 * 2 * 1.20
def upload_document(self,
file_object,
document_type,
description=None,
label=None,
language=None,
querystring=None,
user=None):
"""
Upload an individual document
"""
try:
with transaction.atomic():
document = Document(description=description or '',
document_type=document_type,
label=label or file_object.name,
language=language
or setting_language.value)
document.save(_user=user)
except Exception as exception:
logger.critical(
'Unexpected exception while trying to create new document '
'"%s" from source "%s"; %s', label or file_object.name, self,
exception)
raise
else:
try:
document_version = document.new_version(
file_object=file_object,
_user=user,
)
if user:
document.add_as_recent_document_for_user(user)
Transformation.objects.copy(
source=self, targets=document_version.pages.all())
except Exception as exception:
logger.critical(
'Unexpected exception while trying to create version for '
'new document "%s" from source "%s"; %s',
label or file_object.name,
self,
exception,
exc_info=True)
document.delete(to_trash=False)
raise
else:
WizardStep.post_upload_process(document=document,
querystring=querystring)
return document
def report(request, lot_number):
report = Report.objects.get(lot_number=lot_number)
audit_log = target_stream(report)
orders = OrderLineItem.objects.filter(report=report)
newdoc = NewDocumentForm(None)
try:
next_report = report.get_next_by_created_at()
except:
next_report = None
try:
prev_report = report.get_previous_by_created_at()
except:
prev_report = None
if 'document_button' in request.POST:
newdoc = NewDocumentForm(request.POST, request.FILES)
if newdoc.is_valid():
doc = Document(type=newdoc.cleaned_data['type'],
file=request.FILES['file'],
created_by=request.user)
doc.save()
action.send(request.user,
verb="created document",
action_object=doc)
rd = ReportDocument(
report=report,
document=doc,
primary_document=newdoc.cleaned_data['primary'],
created_by=request.user,
internal_cert=newdoc.cleaned_data['internal'])
rd.save()
action.send(request.user,
verb="attached document",
action_object=rd,
target=report)
report.save()
messages.success(request, 'Document upload successful.')
return HttpResponseRedirect(
reverse('reports.views.report', args=[report.lot_number]))
return render_to_response('reports/report.html', {
'report': report,
'new_document_form': newdoc,
'orders': orders,
'audit_log': audit_log,
'next_report': next_report,
'prev_report': prev_report,
},
context_instance=RequestContext(request))
def setUp(self):
self.admin_user = User.objects.create_superuser(
username=TEST_ADMIN_USERNAME,
email=TEST_ADMIN_EMAIL,
password=TEST_ADMIN_PASSWORD)
self.document_type = DocumentType(name=TEST_DOCUMENT_TYPE)
self.document_type.save()
self.document = Document(document_type=self.document_type, )
self.document.save()
with open(TEST_DOCUMENT_PATH) as file_object:
self.document.new_version(
file_object=File(file_object, name='mayan_11_1.pdf'))
def test_delete(self):
name = '2210571.jpg'
url = reverse(
'model_view_detail',
args=['documents.Document', 'folder', self.folder.pk, name])
path = 'tests/files/%s' % name
f = open(path, 'rb')
document = Document()
document.folder = self.folder
document.name = '2210571.jpg'
document.file.save(name=document.name, content=File(f))
document.save()
f.close()
response = self.client.delete(url)
self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
def setUp(self):
# Start the OCR queue
self.default_queue = DocumentQueue.objects.get(name='default')
self.document_type = DocumentType.objects.create(name='test doc type')
self.document = Document(
document_type=self.document_type,
description='description',
)
self.document.save()
with open(TEST_DOCUMENT_PATH) as file_object:
self.document.new_version(
file=File(file_object, name='title_page.png'))
# Clear OCR queue
QueueDocument.objects.all().delete()
def document_from_sql(name, code, host, db_user, db_password, db_name,
table_name):
description = get_table_description(host, db_user, db_password, db_name,
table_name)
try:
document = Document(name=name, code=code)
except:
raise Exception('Error creating new document, check name and code.')
fields_description = dict()
for field in description:
fields_description[field] = ''
document.fields_description = str(fields_description)
document.save()
return document
def setUp(self):
self.document_type = DocumentType(name='test doc type')
self.document_type.save()
self.document = Document(
document_type=self.document_type,
description='description',
)
self.document.save()
with open(TEST_DOCUMENT_PATH) as file_object:
self.document.new_version(
file_object=File(file_object, name='mayan_11_1.pdf'))
with open(TEST_KEY_FILE) as file_object:
gpg.import_key(file_object.read())
def upload_document(request, mapid=None):
if request.method == 'GET':
return render_to_response('documents/document_upload.html',
RequestContext(request, {
'mapid': mapid,
}),
context_instance=RequestContext(request))
elif request.method == 'POST':
try:
resourceId = str(request.POST['map'])
file = None
try:
file = request.FILES['file']
except MultiValueDictKeyError as fe:
file = None
title = request.POST['title']
html_lnk_document = request.POST['html_lnk_document']
htmllink = None
if (html_lnk_document == "Yes"):
htmllink = request.POST['documentHtmlLink']
document = Document(title=title, file=file, htmllink=htmllink)
if request.user.is_authenticated(): document.owner = request.user
document.save()
document.set_default_permissions()
permissionsStr = request.REQUEST['documentPermissions']
permissions = json.loads(permissionsStr)
set_document_permissions(document, permissions)
if (resourceId != "" and float(resourceId)):
docMap = Map.objects.get(id=resourceId)
if docMap is not None:
document.maps.add(docMap)
docLayer = Layer.objects.get(id=resourceId)
if docLayer is not None:
document.layers.add(docLayer)
except Exception as e:
logger.error('Error inside document upload ' + str(e))
raise e
return HttpResponse(
json.dumps({
'success': True,
'redirect_to': '/maps/' + str(mapid)
}))
def create(self, validated_data):
body = validated_data.get('body')
body = base64_decode(body)[0]
document = Document()
document.account = self.context['account']
document.folder = validated_data.get('folder')
document.type = validated_data.get('type') or Document.OTHER
document.file = ContentFile(body, validated_data.get('name'))
document.user = self.context['user']
old_doc = validated_data.get('old_doc')
if old_doc:
document.previous_version = old_doc.id
document.save()
return document
def setUp(self):
self.project = Project(**{'name': 'Customer A'})
self.project.save()
self.folder = Folder(**{'name': 'Images', 'project': self.project})
self.folder.save()
files = ['2210571.jpg', 'images.docx']
for file in files:
path = 'tests/files/%s' % file
f = open(path, 'rb')
document = Document()
document.folder = self.folder
document.name = file
document.file.save(name=document.name, content=File(f))
document.save()
document.save_thumbnail()
f.close()
def mtr_generator(request):
lot_number = request.GET.get('lot_number', None)
if lot_number:
report = Report.objects.get(lot_number=lot_number)
if request.method == 'POST':
mtrform = MTRForm(request.POST)
if mtrform.is_valid():
new_mtr = MTR(properties=mtrform.cleaned_data)
pdf = new_mtr.generate_pdf()
fp = StringIO()
pdf.write(fp)
file = InMemoryUploadedFile(fp, None, "%s.pdf" % report.lot_number,
'application/pdf', fp.len, None)
doc_type = DocumentType.objects.get(name='Material Test Report')
doc = Document(type=doc_type, file=file, created_by=request.user)
doc.save()
rd = ReportDocument(report=report,
document=doc,
primary_document=True,
created_by=request.user,
internal_cert=True)
rd.save()
report.save()
messages.success(request, 'Document upload successful.')
action.send(request.user,
verb="generated an MTR ",
action_object=rd,
target=report)
return HttpResponseRedirect(
reverse('reports.views.report', args=[report.lot_number]))
else:
mtrform = MTRForm(
initial={
'part_number': report.part_number,
'heat_number': report.heat_number,
'size': report.part_number.description,
})
return render_to_response('pdfgen/new_mtr.html', {
'form': mtrform,
},
context_instance=RequestContext(request))
def get_or_add_Sectionreport(self, section_obj, semester, user, grades, analysis):
_doc = None
for _document in Document.objects.filter(semester=semester,
document_class=self.section_document_class):
try:
_section_code = _document.data_dict['section']
if _section_code == section_obj.section_code:
_doc = _document
break
except:
pass
if _doc is None:
_doc = Document()
_doc.document_class = self.section_document_class
_doc.document_owner = user
_doc.semester = semester
_data = {}
_data['grades'] = grades
_data['section'] = int(section_obj.section_code)
_data['teacher'] = user.username
_data['semester'] = semester.semester_name
_data['academic_year'] = semester.semester_academic_year.academic_year_name
_data['campus'] = section_obj.section_course.program.department.college.campus.campus_name_en
_data['college'] = section_obj.section_course.program.department.college.college_name_en
_data['department'] = section_obj.section_course.program.department.department_name_en
_data['course'] = section_obj.section_course.course_name
_data['filename'] = 'section__' + str(section_obj.section_code)
_data['max_grade'] = 100
_stats = {}
_stats = Section_Measurment(grades).compute_statistics()
if analysis == '' or analysis == None:
pass
else:
_stats['analysis'] = analysis
_doc.data_dict = {**_data, **_stats}
_doc.save()
return _doc
