def init_database():
# Create the database and the database table
db.create_all()
# Fake data
documents = [
Document(
text='It was the best of times, it was the worst of times',
title='A Tale of Two Cities',
),
Document(
text='<html><head /><body><article>Hello</article></body></html>',
title='A random HTML page',
),
Document(
text='<p>A fragment of text',
title='A fragment of text',
),
Document(
text='',
title='Doc1',
),
]
map(lambda doc: db.session.add(doc), documents)
db.session.add(documents[0])
# Commit the changes for the users
db.session.commit()
yield db # this is where the testing happens!
db.drop_all()
def index():
form = DocumentsForm()
if request.method == 'POST':
template_id = form.templates.data
fields = Template_field.query.filter_by(template=template_id)
document = Document()
document.template_id = template_id
db.session.add(document)
db.session.commit()
for field in fields:
document_field = Document_field(document=document.id,
name=field.name,
index=field.index,
alias=field.alias,
template_field=field.id,
group_id=field.group_id,
value=field.value,
comment=field.comment)
db.session.add(document_field)
db.session.commit()
doc_fields = Document_field.query.filter_by(document=document.id)
return redirect(url_for('document', document_id=document.id))
else:
form = DocumentsForm()
form.templates.choices = [(s.id, s.name)
for s in Template.query.filter_by(status=1)]
result = Document.query.all()
documents = [document for document in result]
return render_template('index.html',
title='Home',
documents=documents,
form=form)
def transcribe_document(
document: Document,
aws_bucket_name: str,
lang: str,
):
filekey = document.filename
job_raw = transcribe(
filekey=filekey,
aws_bucket_name=aws_bucket_name,
lang=lang,
)
full_text = job_raw["results"]["transcripts"][0]["transcript"]
transcription = AWSTranscription(raw=job_raw, full_text=full_text)
document.words = []
for i, item in enumerate(job_raw["results"]["items"]):
if item["type"] == "punctuation":
continue
alternative = item["alternatives"][0]
assert "start_time" in item, item
word = Word(
word=alternative["content"],
order=i,
start_time=timedelta(seconds=float(item["start_time"])),
end_time=timedelta(seconds=float(item["end_time"])),
confidence=float(alternative["confidence"]),
)
document.words.append(word)
document.transcription = transcription
def insert_sample_data():
import requests
from app.models import Document
url = 'https://api.github.com/users/moby/repos'
sample_data = json.loads(requests.get(url).content.decode('utf-8'))
for x in sample_data:
d = Document(doc=x)
d.create(d)
def add_fake_data(number_users):
"""Add fake data to the database."""
User.generate_fake(count=number_users)
Document.generate_fake(count=number_users)
Suggestion.generate_fake(count=number_users)
Saved.generate_fake()
Tag.generate_fake(count=number_users)
Tagged.generate_fake()
def document(slug):
document = Document.get_by_slug(slug=slug)
phrases = Document.get_phrases(document)
return render_template(
"user-document.html",
title="Document Analysis",
phrases=phrases,
document=document,
)
def add_document(self, title, manager_page_id):
manager_page = self.page(page_id=manager_page_id)
if manager_page is not None and self.user and self.user.has_perm(
APP_NAME + '.add_link'):
profile = ProfileRepo(user=self.user).me
if profile is not None:
document = Document(title=title,
icon_material='get_app',
profile=profile)
document.save()
manager_page.documents.add(document)
return True
return False
def setup_class(self):
self.base_url = 'http://127.0.0.1:5000'
self.headers = {'content-type': 'application/json'}
self.doc_to_delete = []
# we create a test document
doc = Document(
text=
"Look! I'm even testing my code, you should really hire me... ;)")
db.session.add(doc)
db.session.commit()
document = doc.to_dict()
self.doc_to_delete.append(document['document_id'])
self.doc_id = document['document_id']
def post(self):
args = parser.parse_args()
try:
text_arg = args['text']
doc = Document(text=text_arg)
db.session.add(doc)
db.session.commit()
text = doc.to_dict()
app.logger.info('Text added in db: {}'.format(text_arg))
code = 201
except Exception as e:
text = {}
app.logger.error('Text not added in db: {}'.format(text_arg))
code = 500
return {'document_id': text['document_id']}, code
def fcmat(request):
if not request.POST:
from app.models import Document
from app.forms import DocumentForm
form = DocumentForm() # A empty, unbound form
# Render list page with the documents and the form
return render_to_response('app/inp_fcmat.html', {
'title': 'Displaying Functional Connectivity Matrix',
'form': form
},
context_instance=RequestContext(request))
else:
from app.models import Document
from app.forms import DocumentForm
from django.http import HttpResponseRedirect
from django.core.urlresolvers import reverse
from django.conf import settings #or from my_project import settings
import os
import pyFitMR.fcmat_lib as fcmat_lib
from django.conf import settings #or from my_project import settings
uploadedfile = False
form = DocumentForm(request.POST, request.FILES)
if form.is_valid():
newdoc = Document(docfile=request.FILES['docfile'])
newdoc.save()
matfile = os.path.join(settings.MEDIA_ROOT, newdoc.docfile.name)
uploadedfile = True
else:
FILE_ROOT = os.path.abspath(os.path.dirname(__file__))
matfile = os.path.join(FILE_ROOT, 'CC_testfile.mat')
import scipy.io
#import matplotlib.pyplot as plt
mat = scipy.io.loadmat(matfile)
FC = mat['connectome']
#FC=mat.items()[0][1]
#plt.imshow(FC)
#print FC
script, div = fcmat_lib.plot(FC)
if uploadedfile:
newdoc.docfile.delete()
return render(request, 'app/fcmat_result.html', {
"the_script": script,
"the_div": div
})
def post_documents():
###
# Receive a posted document, and return the document id
##
errors = []
request = flask_rebar.get_validated_body()
app.logger.debug(request)
rc = 201
if not request.get('title').strip():
raise err.BadRequest('Empty title is not allowed')
# The API will be responsible for generating the document ID
## Normalize name. Note: this also commits the record, to avoid a race
doc_id = normalize_doc_id(request['title'])
# Create a new doc
doc = Document(
doc_id=doc_id,
title=request['title'],
text=request['text']
)
db.session.add(doc)
db.session.commit()
return {
'document': doc,
'errors': errors,
}, rc
def upload():
if request.method == 'POST':
file = request.files['file']
x = 0
if request.form.get('privatebool'):
x = 1
newFile = Document(name=file.filename,
username=session['username'],
doc=file.read(),
privateval=x)
db.session.add(newFile)
db.session.commit()
files = []
all_files = Document.query.filter_by(privateval=0)
for f in all_files:
files.append({'fname': f.name, 'fuser': f.username})
#print(files)
session['files'] = files
# return render_template('home.html', user=session['curr_user'])
return render_template('home.html',
user=session['curr_user'],
files=session['files'],
found=True,
ufound=True,
ffound=True)
def edit_transaction(trans_id):
form = EditTransactionForm()
transaction = Transaction.query.filter_by(id=trans_id).first_or_404()
if form.validate_on_submit():
transaction.tr_id = form.tr_id.data
transaction.date = form.date.data
transaction.description = form.description.data
transaction.tr_type = form.tr_type.data
transaction.amount = form.amount.data
filename = secure_filename(form.file.data.filename)
db.session.commit()
if filename != '':
form.file.data.save('uploads/' + filename)
document = Document(filename=filename,
transaction_id=transaction.id)
db.session.add(document)
db.session.commit()
flash('You have sucessfully edited the transaction!')
return redirect(url_for('edit_transaction', trans_id=transaction.id))
elif request.method == 'GET':
form.tr_id.data = transaction.tr_id
form.date.data = transaction.date
form.description.data = transaction.description
form.tr_type.data = transaction.tr_type
form.amount.data = transaction.amount
return render_template('edit_transaction.html',
title='Edit Transaction',
form=form)
def create_document():
form = DocumentForm(request.form)
if request.method == 'POST' and form.validate():
# TODO add validate to form
title = form.title.data
created = form.created.data
text = form.text.data
source_name = form.source_name.data
source_url = form.source_url.data
source = Source(name=source_name, url=source_url)
save_db(source)
document = Document(
title=title, text=text, created=created, author=current_user,
source=source
)
saved = save_db(document)
if saved:
return redirect(url_for('documents.index'))
else:
flash('Документ не был создан')
return render_template(
'documents/create.html', document_create_page=True, form=form
)
def fake():
"Populate tables using fake data"
fake = faker.Faker()
upload_dir = os.path.join(manager.app.instance_path, 'uploads')
if not os.path.isdir(upload_dir):
os.mkdir(upload_dir)
users = [User.generate(fake) for _ in range(0, 10)]
user = users[0]
for obj in users:
db.session.add(obj)
db.session.commit()
pdfdata = base64.decodestring(EMPTY_PDF.strip())
docs = [Document.generate(pdfdata) for _ in range(0, 10)]
for doc in docs:
comments = [Comment.generate(fake, doc.id) for _ in range(0, 4)]
annotations = [Annotation.generate(fake, doc.id, user.id)
for _ in range(0, 2)]
for obj in docs + comments + annotations:
db.session.add(obj)
db.session.commit()
def document_phrase_list_api(slug):
document = Document.get_by_slug(slug=slug)
phrases = Document.get_phrases(document)
result = {}
phrase_list = []
for phrase in phrases:
phrase_list.append(phrase.serialize())
result["phrases"] = phrase_list
result["phraseCount"] = len(phrase_list)
return json.dumps(result)
def kickoff():
posters = []
poster_notes = []
documents = []
document_notes = []
form = AddProjectForm()
if request.method == 'POST':
uploadlist_file = request.files.getlist('file')
uploadlist_note = request.form.getlist('note')
# uploadlist_note will be like ['', ''] when inputs are left blank
project_prefix = datetime.now().strftime('%m%d%H%M')
for i in range(len(uploadlist_file)):
if uploadlist_file[i]:
f_name = project_prefix + '_' + random_filename(
uploadlist_file[i].filename
)[-12:] # this function will remain the original extension, don't bother using os.path
if f_name.lower().endswith(('.png', '.jpg', '.jpeg')):
posters.append(f_name)
poster_notes.append(uploadlist_note[i])
else:
documents.append(f_name)
if uploadlist_note[i]:
document_notes.append(uploadlist_note[i])
else:
document_notes.append(
os.path.splitext(uploadlist_file[i].filename)[0])
# set default filenote same as the filename
uploadlist_file[i].save(
os.path.join(app.config['PROJECT_PATH'], f_name))
title_cn = form.title_cn.data
title_en = form.title_en.data
brief_cn = form.brief_cn.data if form.brief_cn.data else '暂无此项目说明'
brief_en = form.brief_en.data if form.brief_en.data else 'No information available'
startdate = form.startdate.data
enddate = form.enddate.data
category = form.category.data
banner = posters[0] if posters else ""
project = Project(title_cn=title_cn,
title_en=title_en,
brief_cn=brief_cn,
brief_en=brief_en,
startdate=startdate,
enddate=enddate,
filename="*".join(posters),
filenote="*".join(poster_notes),
banner=banner,
category=category)
# filename & filenote are misleading but the model of database is set, no way to change it.
db.session.add(project)
db.session.flush()
for i in range(len(documents)):
document = Document(filename=documents[i],
filenote=document_notes[i],
project=project)
db.session.add(document)
db.session.commit()
flash('New project launched!', 'success')
return redirect(url_for('.project'))
def add_document():
if 'file' not in request.files:
return gen_error('No file sent')
file = request.files['file']
# if user does not select file, browser also
# submit a empty part without filename
if file.filename == '' or not file:
return gen_error('No file selected')
if 'name' not in request.form:
return gen_error('No name document could not be created')
if 'vus' not in request.form:
return gen_error('No vus document could not be created')
vus = map(int, request.form['vus'].split())
vus = VUS.query.filter_by(number=vus[0], code=vus[1]).first()
if vus is None:
return gen_error('Such vus not yet exists in this system')
docs = os.listdir(os.path.join(USER_PATH, 'documents'))
filename = file.filename
if filename in docs:
i = 1
while filename[:-5] + '_' + str(i) + filename[-5:] in docs:
i += 1
filename = filename[:-5] + '_' + str(i) + filename[-5:]
file.save(os.path.join(USER_PATH, 'documents', filename))
d = Document(name=request.form['name'], vus_id=vus.id, filename=filename)
db.session.add(d)
db.session.commit()
return gen_success(filename=filename, message='Success!')
def test_model_document(self):
"""Test to make sure that Document is working properly.
"""
d1 = Document(title="test")
d1.save()
assert d1.type == "document"
u1 = Unit()
u1.save()
d1.children.append(u1)
d1.save()
assert d1.children == [u1]
assert u1.parent == d1
def test_model_document_file(self):
"""Test to make sure that DocumentFile is working properly.
"""
documentfile = DocumentFile()
document1 = Document()
document2 = Document()
project1 = Project()
project2 = Project()
documentfile.path = "/foo/bar"
documentfile.documents = [document1, document2]
documentfile.projects = [project1, project2]
documentfile.save()
assert len(documentfile.documents) == 2
assert len(documentfile.projects) == 2
def fcmat(request):
if not request.POST:
from app.models import Document
from app.forms import DocumentForm
form = DocumentForm() # A empty, unbound form
# Render list page with the documents and the form
return render_to_response(
'app/inp_fcmat.html',
{'title':'Displaying Functional Connectivity Matrix','form': form},
context_instance=RequestContext(request)
)
else:
from app.models import Document
from app.forms import DocumentForm
from django.http import HttpResponseRedirect
from django.core.urlresolvers import reverse
from django.conf import settings #or from my_project import settings
import os
import pyFitMR.fcmat_lib as fcmat_lib
from django.conf import settings #or from my_project import settings
uploadedfile = False
form = DocumentForm(request.POST, request.FILES)
if form.is_valid():
newdoc = Document(docfile = request.FILES['docfile'])
newdoc.save()
matfile = os.path.join(settings.MEDIA_ROOT,newdoc.docfile.name)
uploadedfile = True
else:
FILE_ROOT = os.path.abspath(os.path.dirname(__file__))
matfile = os.path.join(FILE_ROOT,'CC_testfile.mat')
import scipy.io
#import matplotlib.pyplot as plt
mat = scipy.io.loadmat(matfile)
FC=mat['connectome']
#FC=mat.items()[0][1]
#plt.imshow(FC)
#print FC
script, div = fcmat_lib.plot(FC)
if uploadedfile:
newdoc.docfile.delete()
return render(request, 'app/fcmat_result.html', {"the_script":script, "the_div":div})
def post():
raw_dict = request.get_json(force=True)
try:
schema.validate(raw_dict)
doc_dict = raw_dict['data']['attributes']
doc = Document(doc=doc_dict['doc'])
doc.create(doc)
query = Document.query.get(doc.id)
result = schema.dump(query).data
return jsonify(result), 201
except ValidationError as ve:
response = jsonify({'code': 403, 'message': ve.messages})
return response, 403
except SQLAlchemyError as e:
db.session.rollback()
resp = jsonify({'code': 500, 'message': str(e)})
return response, 500
def document_list():
form = DocumentForm()
if form.validate_on_submit():
if current_user.is_authenticated:
Document.add_document(title=form.title.data,
body=form.body.data,
user=current_user)
else:
flash("Please register or login to analyze documents.")
documents = Document.get_all()
return render_template("document-list.html",
title="All documents",
documents=documents)
def remove_stock(component_id, comment):
document = Document(datetime.utcnow(), current_user.id, 'Cписание',
comment)
db.session.add(document)
db.session.commit()
stock = Stock(document.id, component_id, 0)
db.session.add(stock)
db.session.commit()
return redirect(url_for('stock'))
def test_document_belongs_to(self):
"""Check if ``belongs_to()`` on ``Document`` is working properly.
"""
user = User()
project = Project()
document_file = DocumentFile()
document = Document()
project.document_files = [document_file]
document_file.documents = [document]
user.projects = [project]
user.save()
project.save()
document.save()
document_file.save()
assert document.belongs_to(user)
def test_document_belongs_to(self):
"""Check if ``belongs_to()`` on ``Document`` is working properly.
"""
user = User()
project = Project()
document_file = DocumentFile()
document = Document()
project.document_files = [document_file]
document_file.documents = [document]
user.projects = [project]
user.save()
project.save()
document.save()
document_file.save()
assert document.belongs_to(user)
def test_get_documents(self):
"""Test the get_documents method.
"""
document_file1 = DocumentFile()
document_file2 = DocumentFile()
document1 = Document()
document2 = Document()
document3 = Document()
project = Project()
document_file1.documents = [document1, document2]
document_file2.documents = [document3]
project.document_files = [document_file1, document_file2]
project.save()
assert project.get_documents() == [document1, document2, document3]
def upload():
"""Upload a pdf."""
form = UploadDocumentForm()
doc = Document.query.get(current_user.id)
if form.validate_on_submit():
if doc is None:
doc = Document()
doc.user_id = current_user.id
doc.document_urls = form.file_urls.data
db.session.add(doc)
db.session.commit()
return redirect(url_for('checklist.index'))
form.file_urls.data = doc.document_urls if doc is not None else None
return render_template('checklist/upload_document.html', form=form)
def upload_document():
form = CreateDocumentForm()
if form.validate_on_submit():
f = form.file.data
fname = secure_filename(f.filename)
fileext = fname.rsplit('.', 1)[1].lower()
filename = "{title}_{time}.{ext}".format(
title=form.title.data,
time=datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S"),
ext=fileext)
f.save(os.path.join(current_app.config['DOCUMENTS_FOLDER'], filename))
params = {'form_data': form.data, 'filename': filename}
Document.create_document(params=params)
return redirect(url_for('main.documents'))
return render_template('main/upload.jinja2', form=form)
def test_html(tmpdir):
p = tmpdir.join("Andy_Warhol.rst")
content = ("***********\n"
"Andy Warhol\n"
"***********\n"
"**Andy Warhol** was an artist.")
p.write(content)
doc = Document(p)
html = doc.html
assert '<h1 class="title">Andy Warhol</h1>' in html
assert '<p><strong>Andy Warhol</strong> was an artist.</p>' in html
def test_contents(tmpdir):
p = tmpdir.join("Andy_Warhol.rst")
content = ("""
***********
Andy Warhol
***********
**Andy Warhol** was an artist.
""")
p.write(content)
assert Document(p).content == content
def new_document():
form = DocumentForm()
# if form.validate_on_submit():
if request.method == 'POST':
doc = Document(doctype=form.doctype.data,
date=form.date.data, national_context=form.context.data,
citation=form.citation.data, zotero_id=form.zotero.data,
comments=form.comments.data)
db.session.add(doc)
db.session.commit()
return redirect(url_for('get_document', docId=doc.id))
return render_template('new_document.html', form=form)
def post_document(title):
req = request.get_json()
document = Document.query.filter_by(title=title).first()
if document is None:
document = Document(title=title)
if req.get('title'):
req.pop('title')
document.update_attributes(req)
document.save()
return make_response(json.dumps(document.to_dict()))
def list(request):
# Handle file upload
if request.method == 'POST':
form = DocumentForm(request.POST, request.FILES)
if form.is_valid():
newdoc = Document(docfile = request.FILES['docfile'])
newdoc.save()
# Redirect to the document list after POST
print '**********************************'
#print newdoc
dir_name= str(newdoc).split('/')
name_file= str(dir_name[1])
#print settings.MEDIA_ROOT
filepath = str(settings.MEDIA_ROOT) +'/' + str(dir_name[0]) + '/'
#functions.xlsx2geojson(filepath,name_file)
request.session['name_file'] = name_file
request.session['filepath'] = filepath
# return HttpResponseRedirect(reverse('app.views.listdetails'))
array_headers= functions.readheader_xlsx(filepath,name_file)
print array_headers
variable = {'array_headers':simplejson.dumps(array_headers)}
return render_to_response('listdetails.html',variable,context_instance=RequestContext(request))
else:
form = DocumentForm() # A empty, unbound form
# Load documents for the list page
documents = Document.objects.all()
# Render list page with the documents and the form
return render_to_response(
'list.html',
{'documents': documents, 'form': form},
context_instance=RequestContext(request)
)
def upload_doc():
with open(blockchain_root ,"r") as block_init:
parsed_json = json.load(block_init)
chain = BlockChain.BlockChain(parsed_json)
block_init.close()
if request.method == 'POST':
# check if the post request has the file part
users = Users(current_user.id, current_user.username)
# join/add to this
target = os.path.join(APP_ROOT, 'Effichaincy/documents/' + str(users.id) + "")
all_userdirs = os.path.join(APP_ROOT, 'Effichaincy/documents/')
# create folder if nonexistent
if not os.path.isdir(target):
os.makedirs(target)
if 'file' not in request.files:
flash('No file part')
return redirect(request.url)
file = request.files['file']
# if user does not select file, browser submits an empty part without filename
if file.filename == '':
flash('No selected file')
return redirect(request.url)
#if 'file' in request.files:
# return render_template('e_file_already_exists.html', title='Oops')
if file:
doc = Document.Document(target + "/" + file.filename,users.name)
filename = file.filename
file_loc = target + "/" + filename
count = 1
to_dot = filename.find('.')
strip_file = filename[0:to_dot]
sec_filename_num = strip_file + '_v' + str(count) + '.txt'
for dirs in os.listdir(all_userdirs):
current_dir = os.path.join(all_userdirs,dirs)
if os.path.isfile(current_dir + "/" + sec_filename_num):
print("True for: " + current_dir)
for files in os.listdir(current_dir):
file = request.files['file']
file_name = file.filename
num_to_dot = file_name.find('.')
striped_file = file_name[0:num_to_dot]
if striped_file in files:
count = count + 1
sec_filename_num = striped_file + '_v' + str(count) + '.txt'
#sec_filename_num = secure_filename(file_with_num)
file.save(target + "/" + sec_filename_num)
doc.name = target + "/" + sec_filename_num
chain.make_block(chain.chain, doc)
return render_template('upload_completion.html', title='Success', file=sec_filename_num)
def post_document(title):
req = request.get_json()
document = Document.query.filter_by(title=title).first()
if document is None:
document = Document(title=title)
if req.get('title'):
req.pop('title')
document.update_attributes(req)
document.save()
return make_response(json.dumps(document.to_dict()))
def save(self):
super(TextFile, self).save()
lexer = None
content = self.f.value.decode('utf-8')
plain_ext = '.txt', '.log',
if self.ext in plain_ext:
lexer = TextLexer
else:
try:
lexer = get_lexer_for_filename(self.filename)
except ClassNotFound:
try:
lexer = get_lexer_for_mimetype(self.mimetype)
except ClassNotFound:
try:
lexer = guess_lexer(content)
except ClassNotFound:
lexer = TextLexer
html = highlight(content, lexer, HtmlFormatter(linenos=True, lineanchors='line', anchorlinenos = True))
txt = Document()
txt.file_id = self.id
txt.html = html
txt.content = content
txt.save()
return self.base
def test_generate_models(self):
fake = faker.Faker()
user = User.generate(fake)
doc = Document.generate('pdfdata')
comm = Comment.generate(fake, doc)
ann = Annotation.generate(fake, doc, user)
