def load_data(self):
X, Y = [], []
for file in os.listdir(self.path):
if file == 'truth.txt' or file == '.DS_Store':
continue
print "loading file -->" + file
tree = ET.parse(os.path.join(self.path, file))
root = tree.getroot()
document = Document(language=root.get('lang'), name=root.get('id'))
for d in root.findall('document'):
document.add_document(d.text)
user, gender, age_group, extroverted, stable, agreeable, conscientious, open = self.truth[
root.get('id')].split(":::")
traits = PersonalityTraits(extroverted=float(extroverted), stable=float(stable), agreeable=float(agreeable),
conscientious=float(conscientious), open=float(open))
usr = Author(gender=gender, age_group=age_group, traits=traits)
document.author = usr
X.append(document)
Y.append(self.truth[root.get('id')])
print "done loading files"
self.X = X
self.Y = Y
return self
def store_digest(self, digest):
docproof = Document.get_doc(digest)
if docproof:
return {"success" : False, "reason": "existing", "digest": digest, "args": [export_timestamp(docproof.timestamp)]}
d = Document.new(digest)
self.doc = d
return {"success": True, "digest": d.digest}
def store_digest(self, digest):
old_doc = Document.get_doc(digest)
if old_doc:
return {"success" : False, "reason": "existing",
"digest": digest, "args": [export_timestamp(old_doc.timestamp)]}
d = Document.new(digest)
self.doc = d # don't remove, needed for API
return {"success": True, "digest": d.digest}
def get(self):
# Provide login/logout URLs.
user_info = get_user_info()
if user_info is None:
login_url = users.create_login_url('/')
else:
login_url = users.create_logout_url('/')
# Collect list of error messages which gets shown to the user.
error_messages = self.request.params.getall('error_message')
view_user = user_info # for now
did_search = False
# Fetch media for view user.
media = MediaObject.all().filter('owner', user_info)
media = media.filter('lacks_document', True)
media = media.order('creation')
limit = 50
if self.request.get("limit"):
limit = long(self.request.get("limit"))
media = media.fetch(limit)
docs = Document.all().filter('owner', user_info)
tags = self.request.get("tags")
if tags:
did_search = True
for tag in re.split('\s*,\s*', tags):
docs = docs.filter("tags", tag)
docs = docs.fetch(limit)
untagged_docs = Document.all().filter('owner', user_info).filter(
"no_tags", True).fetch(limit)
upcoming_due = Document.all().filter('owner', user_info)
upcoming_due = upcoming_due.filter("due_date !=", None)
upcoming_due = upcoming_due.order("due_date")
upcoming_due = upcoming_due.fetch(30)
top_message = ""
if self.request.get("saved_doc"):
docid = long(self.request.get("saved_doc"))
top_message = "Saved <a href='/doc/%d'>doc %d</a>" % (docid, docid)
# Render view.
self.response.out.write(
template.render('main.html', {
"did_search": did_search,
"media": media,
"docs": docs,
"untagged_docs": untagged_docs,
"upcoming_due_docs": upcoming_due,
"view_user": view_user,
"login_url": login_url,
"user_info": user_info,
"top_message": top_message,
},
debug=True))
def makeMockDocument():
try: os.remove(testfile)
except(OSError): pass
document = Document(testfile)
# set up a mock example
trn = document.database.transaction()
with trn:
for spandata in MockData:
span = Document.persistenceSchema.classes.Span(externalID = spandata['id'], info = spandata['info'])
for tok in spandata['tokens']:
span.addToken(Document.persistenceSchema.classes.Token(tok))
document.addSpan(span)
# make some constituents, cause why not
phrase = Document.persistenceSchema.classes.Constituent()
phrase.add(document.spans[0].tokens[3])
phrase.add(document.spans[0].tokens[2])
phrase.add(document.spans[0].tokens[4])
phrase = Document.persistenceSchema.classes.Constituent()
phrase.add(document.spans[0].tokens[0])
phrase.add(document.spans[0].tokens[1])
phrase.add(document.spans[0].tokens[3])
phrase.add(document.spans[0].tokens[4])
phrase = Document.persistenceSchema.classes.Constituent()
print phrase.refmark
phrase.add(document.spans[1].tokens[1])
phrase.add(document.spans[1].tokens[0])
x = Document.persistenceSchema.classes.DependencyRelation(document.spans[0].constituents[0], document.spans[0].tokens[5])
x = Document.persistenceSchema.classes.DependencyRelation(document.spans[0].tokens[1], document.spans[0].tokens[1])
document.spans[0].tokens[0].variables['PoS'] = 'Noun'
document.spans[0].tokens[0].variables['Number'] = 'singular'
document.spans[0].tokens[0].variables['Gender'] = 'masculine'
document.spans[0].tokens[0].variables['Focus'] = 'yes'
mm = Document.persistenceSchema.classes.ReferenceMark()
document.spans[0].tokens[0].refmark = mm
document.spans[1].tokens[0].refmark = mm
phrase.refmark = mm
document.spans[0].tokens[4].refmark = Document.persistenceSchema.classes.ReferenceMark()
trn.commit()
return document
def get(self):
# Provide login/logout URLs.
user_info = get_user_info()
if user_info is None:
login_url = users.create_login_url('/')
else:
login_url = users.create_logout_url('/')
# Collect list of error messages which gets shown to the user.
error_messages = self.request.params.getall('error_message')
view_user = user_info # for now
did_search = False
# Fetch media for view user.
media = MediaObject.all().filter('owner', user_info)
media = media.filter('lacks_document', True)
media = media.order('creation')
limit = 50
if self.request.get("limit"):
limit = long(self.request.get("limit"))
media = media.fetch(limit)
docs = Document.all().filter('owner', user_info)
tags = self.request.get("tags")
if tags:
did_search = True
for tag in re.split('\s*,\s*', tags):
docs = docs.filter("tags", tag)
docs = docs.fetch(limit)
untagged_docs = Document.all().filter('owner', user_info).filter("no_tags", True).fetch(limit)
upcoming_due = Document.all().filter('owner', user_info)
upcoming_due = upcoming_due.filter("due_date !=", None)
upcoming_due = upcoming_due.order("due_date")
upcoming_due = upcoming_due.fetch(30)
top_message = ""
if self.request.get("saved_doc"):
docid = long(self.request.get("saved_doc"))
top_message = "Saved <a href='/doc/%d'>doc %d</a>" % (docid, docid)
# Render view.
self.response.out.write(template.render('main.html', {
"did_search": did_search,
"media": media,
"docs": docs,
"untagged_docs": untagged_docs,
"upcoming_due_docs": upcoming_due,
"view_user": view_user,
"login_url": login_url,
"user_info": user_info,
"top_message": top_message,
}, debug=True))
def handle(self):
n = 0
for old in DocumentProof.all():
doc = Document.get_doc(old.digest)
if not doc:
doc = Document.new(old.digest)
doc.pending = old.tx == None
doc.tx = old.tx
doc.timestamp = old.timestamp
doc.blockstamp = old.blockstamp
n += 1
if n == 10:
break
return {"success" : True, "processed": n}
def get(self):
actionable = Document.get_actionable()
for d in actionable:
ret = d.blockchain_certify()
subject = "Document certified: %s %s" % (ret['success'], d.digest)
body = subject + "\n\nmesage: %s" % (ret['message'])
def get(self, docid):
user_info = get_user_info()
if user_info is None:
self.redirect("/?error_message=%s" % "login required to view docs")
docid = long(docid)
doc = Document.get_by_id(docid, parent=user_info)
if doc is None:
self.response.out.write("Docid %d not found." % (docid))
return
pages = MediaObject.get(doc.pages)
size = self.request.get("size")
if not size:
size = 1200
show_single_list = long(size) > 600
self.response.out.write(
template.render(
"doc.html",
{
"doc": doc,
"pages": pages,
"user_info": user_info,
"size": size,
"show_single_list": show_single_list,
},
debug=True,
)
)
def _ghost(self, oid, state=None):
if oid is Document:
return Document()
kind, tag, content, close = oid
return kind()
def apostile_create_post():
count = session.query(Apostile).filter(Apostile.number == int(request.form['number'])).count()
if count:
return render_template('message.html', title='Помилка',
msg=f'Апостиль з номером {request.form["number"]} уже існує', **user_config())
doc = Document(
country=request.form['country'],
date=request.form['doc_date'],
author_name=request.form['person_name'],
author_info=request.form['person_position'],
stamp_info=request.form['stamp_info'] if request.form['stamp_info'] else '-/-/-'
)
session.add(doc)
session.flush()
tr_id, tr_type = find_trusted_by_str(request.form['ap_author'])
ap = Apostile(
number=int(request.form['number']),
date=request.form['ap_date'],
is_archived=False,
trusted_id=tr_id, trusted_type=tr_type,
document_id=doc.id
)
session.add(ap)
session.flush()
session.commit()
return redirect(f'/apostile/{ap.id}')
class BasePaymentCallback(JsonAPIHandler):
def handle(self):
test = self.request.get("test") == "true"
try:
tx_hash = self.request.get("transaction_hash")
address = self.request.get("address")
satoshis = int(self.request.get("value"))
payment_address = self.request.get("input_address")
except ValueError, e:
return "error: value error"
if not tx_hash:
return "error: no transaction_hash"
if not address:
return "error: no address"
if satoshis <= 0: # outgoing payment
return "*ok*"
if satoshis < MIN_SATOSHIS_PAYMENT: # not enough
return "*ok*"
if not test:
doc = Document.get_by_address(payment_address)
if not doc:
return "error: couldn't find document"
return self.process_payment(satoshis, doc)
return "*ok*"
def run(self):
result = {}
x, y, y_actual = [], [], []
for file in os.listdir(self.path):
if file == 'truth.txt' or file == '.DS_Store':
continue
tree = ET.parse(os.path.join(self.path, file))
root = tree.getroot()
document = Document(language=root.get('lang'), name=root.get('id'))
for d in root.findall('document'):
document.add_document(d.text)
x_test = [document] # vector
temp_result = {}
for predictor in self.model:
# print predictor
if predictor.name == 'age_gender':
prediction = predictor.clf.predict(x_test) # predict
temp_result.update(
predictor.label_extractor(list(predictor.label_encoder.inverse_transform(prediction))[0]))
document.author.gender = temp_result['gender']
document.author.age_group = temp_result['age_group']
if predictor.name == 'personality':
target = predictor.label_encoder.classes_
prediction = list(predictor.clf.predict_proba(x_test))[0]
prediction = [change_range(p, 1.0, 0.0, 0.5, -0.5) for p in prediction]
temp_result.update(predictor.label_extractor(target, prediction))
document.author.personality_traits.extroverted = temp_result['extroverted']
document.author.personality_traits.agreeable = temp_result['agreeable']
document.author.personality_traits.conscientious = temp_result['conscientious']
document.author.personality_traits.stable = temp_result['stable']
document.author.personality_traits.open = temp_result['open']
result[os.path.splitext(file)[0]] = document
# y.extend(prediction)
# print y
x.append(os.path.splitext(file)[0])
# y_actual.append(predictor.label_extractor(self.truth[root.get('id')]))
self.x_test = x_test
# self.y_prediction = y
# self.y_actual = self.label_encoder.transform(y_actual)
self.result = result
def save_document(title, session):
"""Create a new document; return it."""
## XXX(alexr): need to handle source languages...
document = Document(title, "bob", "en")
session.add(document)
session.commit()
print("added document:", document)
return document
def get(self):
actionable = Document.get_actionable()
for d in actionable:
ret = d.blockchain_certify()
sender_address = "*****@*****.**"
subject = "Document certified: %s %s" % (ret['success'], d.digest)
body = subject + "\n\nmesage: %s" % (ret['message'])
mail.send_mail(sender_address, ADMIN_EMAIL, subject, body)
def get(self):
actionable = Document.get_actionable()
for d in actionable:
ret = d.blockchain_certify()
sender_address = "*****@*****.**"
subject = "Document certified: %s %s" % (ret['success'], d.digest)
body = subject + "\n\nmesage: %s" % (ret['message'])
mail.send_mail(sender_address, ADMIN_EMAIL, subject, body)
def store_media():
"""Store media object info in datastore.
Also updates the user-info record to keep count of media objects.
This function is run as a transaction.
"""
user_info = UserInfo.get_by_key_name("user:%s" % user_email)
if user_info is None:
error_messages.append("User record has been deleted. " "Try uploading again")
return
media = MediaObject(
parent=user_info,
owner=user_info,
blob=blob_info.key(),
creation=blob_info.creation,
content_type=blob_info.content_type,
filename=blob_info.filename,
size=int(blob_info.size),
lacks_document=True,
)
user_info.media_objects += 1
db.put(user_info)
db.put(media)
if bool(is_doc) and is_doc != "0":
tag_list = []
if tags is not None:
tag_list = [x for x in re.split("\s*,\s*", tags) if x]
doc = Document(
parent=user_info,
owner=user_info,
pages=[media.key()],
title=title,
description=description,
no_tags=(len(tag_list) == 0),
tags=tag_list,
)
db.put(doc)
media.document = doc.key()
media.lacks_document = False
db.put(media)
def handle(self):
digest = self.request.get("d")
doc = Document.get_doc(digest)
if not doc or not doc.tx:
return {"success" : False, "error": "format"}
# TODO: add check to prevent double timestamping
tx, message = publish_data_old(doc)
do_check_document(digest)
return {"success" : True, "tx" : tx, "message" : message}
def store_media():
"""Store media object info in datastore.
Also updates the user-info record to keep count of media objects.
This function is run as a transaction.
"""
user_info = UserInfo.get_by_key_name('user:%s' % user_email)
if user_info is None:
error_messages.append('User record has been deleted. '
'Try uploading again')
return
media = MediaObject(
parent=user_info,
owner=user_info,
blob=blob_info.key(),
creation=blob_info.creation,
content_type=blob_info.content_type,
filename=blob_info.filename,
size=int(blob_info.size),
lacks_document=True)
user_info.media_objects += 1
db.put(user_info)
db.put(media)
if bool(is_doc) and is_doc != "0":
tag_list = []
if tags is not None:
tag_list = [x for x in re.split('\s*,\s*', tags) if x]
doc = Document(
parent=user_info,
owner=user_info,
pages=[media.key()],
title=title,
description=description,
no_tags=(len(tag_list)==0),
tags=tag_list)
db.put(doc)
media.document = doc.key()
media.lacks_document = False
db.put(media)
def create_doc(self, content, title, description='', author=''):
"""
Creates an document.
:param data: document's properties as json.
:return:
"""
doc = Document(content=content, title=title, description=description, author=author)
self.db.session.add(doc)
self.db.session.commit()
return doc
def handle(self):
digest = self.request.get("d")
doc = Document.get_doc(digest)
if not doc or not doc.payment_address:
return {"success" : False, "error": "format"}
if not doc.tx:
return {"success" : False, "error": "no transaction"}
return {"success" : True, "tx" : doc.tx}
def get_doc_by_pid(cls, pid):
sql = """select * from doc where person_id=?"""
data = (pid, )
rows = cls.db.query(sql, data)
records = []
for row in rows:
print(row)
records.append(
Document(row[0], row[1], row[2], cls.get_by_doc_id(row[0])))
return records
def makeMockDocument1():
try: os.remove(testfile)
except(OSError): pass
document = Document(testfile)
# set up a mock example
trn = document.database.transaction()
with trn:
for spandata in MockData1:
span = Document.persistenceSchema.classes.Span(externalID = spandata['id'], info = spandata['info'])
for tok in spandata['tokens']:
span.addToken(Document.persistenceSchema.classes.Token(tok))
document.addSpan(span)
trn.commit()
return document
def post(self):
user_info = get_user_info()
if user_info is None:
self.redirect('/?error_message=%s' % 'login required to view docs')
docid = long(self.request.get("docid"))
doc = Document.get_by_id(docid, parent=user_info)
if doc is None:
self.response.out.write("Docid %d not found." % (docid))
return
mode = self.request.get("mode")
if mode == "break":
break_and_delete_doc(user_info, doc)
self.response.out.write(
"[<< <a href='/'>Back</a>] Docid %d deleted and images broken out as un-annotated."
% docid)
return
if mode == "delete":
delete_doc_and_images(user_info, doc)
self.response.out.write(
"[<< <a href='/'>Back</a>] Docid %d and its images deleted."
% docid)
return
# Simple properties:
doc.physical_location = self.request.get("physical_location")
doc.title = self.request.get("title")
# Tags
doc.tags = [
x for x in re.split('\s*,\s*', self.request.get("tags")) if x
]
doc.no_tags = (len(doc.tags) == 0)
# Document Date
date = self.request.get("date")
if date:
doc.doc_date = datetime.datetime.strptime(date, "%Y-%m-%d")
doc.no_date = False
else:
doc.doc_date = None
doc.no_date = True
# Due date
due_date_str = self.request.get("due_date")
doc.due_date = None
if due_date_str:
doc.due_date = datetime.datetime.strptime(due_date_str, "%Y-%m-%d")
def store():
db.put(doc)
db.run_in_transaction(store)
self.redirect("/?saved_doc=" + str(docid))
def process_payment(self, satoshis, digest):
secret = self.request.get("secret")
if len(digest) != 64 or secret != CALLBACK_SECRET or satoshis < MIN_SATOSHIS_PAYMENT:
return {"success" : False, "reason" : "format or payment below " + str(MIN_SATOSHIS_PAYMENT)}
doc = Document.get_doc(digest)
if not doc:
return {"success" : False, "reason" : "Couldn't find document"}
doc.pending = False
doc.put()
return {"success" : True}
def get(self):
archiveable = Document.get_archiveable()
processed = False
for d in archiveable:
res = d.archive()
self.response.write("%s %s<br />" % (d.digest, res))
processed = True
if processed:
self.response.write("Running autoconsolidate<br />")
auto_consolidate()
else:
self.response.write("Finished without operation<br />")
def get(self):
archiveable = Document.get_archiveable()
processed = False
for d in archiveable:
res = d.archive()
self.response.write("%s %s<br />" % (d.digest, res))
processed = True
if processed:
self.response.write("Running autoconsolidate<br />")
auto_consolidate()
else:
self.response.write("Finished without operation<br />")
def make_from_json(jsonData, filename, removeOld=False):
if removeOld:
try:
os.remove(filename)
except (OSError):
pass
# create the document
document = Document(filename)
# set up a mock example
trn = document.database.transaction()
with trn:
document.database.root['documentType'] = jsonData['documentType']
document.database.root['variety'] = jsonData['variety']
for i, spanData in enumerate(jsonData['spans']):
span = Document.persistenceSchema.classes.Span(
externalID=i, info=spanData['spanInfo'])
for i, token in enumerate(spanData['tokens']):
token = Document.persistenceSchema.classes.Token(token)
try:
token.gloss = spanData['lemmas'][i]
except:
pass
span.addToken(token)
document.addSpan(span)
trn.commit()
document.close()
def handle(self):
digest = self.request.get("d")
doc = Document.get_doc(digest)
if not doc or doc.tx:
return {"success" : False, "error": "format"}
# TODO: add check to prevent double timestamping
txid, message = publish_data(doc.digest.decode('hex'))
if txid:
doc.tx = txid
doc.txstamp = datetime.datetime.now()
LatestBlockchainDocuments.get_inst().add_document(digest)
doc.put()
return {"success" : txid is not None, "tx" : txid, "message" : message}
def handle(self):
digest = self.request.get("d")
doc = Document.get_doc(digest)
if not doc or doc.tx:
return {"success": False, "error": "format"}
# TODO: add check to prevent double timestamping
txid, message = publish_data(doc.digest.decode('hex'))
if txid:
doc.tx = txid
doc.txstamp = datetime.datetime.now()
LatestBlockchainDocuments.get_inst().add_document(digest)
doc.put()
return {"success": txid is not None, "tx": txid, "message": message}
def handle(self):
digest = self.request.get("d")
doc = Document.get_doc(digest)
if not digest:
return {"success": False, "reason": "format"}
if not doc:
return {"success": False, "reason": "nonexistent"}
if doc.tx:
return {"success": True, "status": "confirmed", "transaction": doc.tx}
if doc.is_actionable():
return {"success": True, "status": "pending"}
return {"success": True, "status": "registered"}
def handle(self):
digest = self.request.get("d")
doc = Document.get_doc(digest)
if not doc or not doc.payment_address:
return {"success" : False, "error": "format"}
txs = get_txs_for_addr(doc.payment_address)
if not txs or len(txs) == 0:
return {"success" : False, "error": "no transactions"}
tx_hash, tx_timestamp = txs[0]
doc.confirmed(tx_hash, tx_timestamp)
LatestBlockchainDocuments.get_inst().add_document(doc)
return {"success" : True, "tx" : doc.tx}
def post(self):
user_info = get_user_info()
if user_info is None:
self.redirect("/?error_message=%s" % "login required to view docs")
docid = long(self.request.get("docid"))
doc = Document.get_by_id(docid, parent=user_info)
if doc is None:
self.response.out.write("Docid %d not found." % (docid))
return
mode = self.request.get("mode")
if mode == "break":
break_and_delete_doc(user_info, doc)
self.response.out.write(
"[<< <a href='/'>Back</a>] Docid %d deleted and images broken out as un-annotated." % docid
)
return
if mode == "delete":
delete_doc_and_images(user_info, doc)
self.response.out.write("[<< <a href='/'>Back</a>] Docid %d and its images deleted." % docid)
return
# Simple properties:
doc.physical_location = self.request.get("physical_location")
doc.title = self.request.get("title")
# Tags
doc.tags = [x for x in re.split("\s*,\s*", self.request.get("tags")) if x]
doc.no_tags = len(doc.tags) == 0
# Document Date
date = self.request.get("date")
if date:
doc.doc_date = datetime.datetime.strptime(date, "%Y-%m-%d")
doc.no_date = False
else:
doc.doc_date = None
doc.no_date = True
# Due date
due_date_str = self.request.get("due_date")
doc.due_date = None
if due_date_str:
doc.due_date = datetime.datetime.strptime(due_date_str, "%Y-%m-%d")
def store():
db.put(doc)
db.run_in_transaction(store)
self.redirect("/?saved_doc=" + str(docid))
def show(self):
window = self.window
while True:
event, values = window.read()
if event == "submit":
doc = Document(None, values['doc'], self.person.id, [])
doc_id = self.service.add_doc(doc)
if doc_id > 0:
self.make_doc_on_disk(self.person, doc)
break
elif event == "reset":
window.Element("doc").Update(value="")
else:
break
window.Close()
def load_text(document, name, passcode, doc_owner):
""" Load the text from the document into database, and create new db object """
text = document.read()
document = Document(text=text,
name=name,
passcode=passcode,
doc_owner=doc_owner)
db.session.add(document)
db.session.commit()
return document
def save_document(title, tags, segments):
"""Given a document title, tags (as a list of strings), all the segments
(also a list of strings), create a new document and tag it."""
session = get_session()
document = Document(title, "bob", "es")
session.add(document)
session.commit()
docid = document.id
sentences = []
for (segmentid, s) in segments:
sent = Sentence(s.strip(), docid)
sentences.append(sent)
session.add_all(sentences)
session.commit()
for tag in tags:
tag_document(document, tag)
def handle(self):
digest = self.request.get("d")
doc = Document.get_doc(digest)
if not digest:
return {"success": False, "reason": "format"}
if not doc:
return {"success": False, "reason": "nonexistent"}
if doc.tx:
return {"success": True, "status": "confirmed", "transaction": doc.tx, "txstamp": doc.txstamp}
if doc.is_actionable():
return {"success": True, "status": "pending"}
return {"success": True, \
"status": "registered", \
"pay_address": doc.payment_address, \
"price": MIN_SATOSHIS_PAYMENT \
}
def get(self):
if not users.is_current_user_admin():
self.redirect('/?error_message=%s' % 'log-in required')
used = set()
for d in Document.all():
used |= set(d.pages)
dead = dict()
for i in MediaObject.all():
if i.key() not in used:
dead[i.key()] = i
for k in dead:
dead[k].delete()
self.redirect('/')
def save_file(fn):
session = Session()
title = os.path.basename(fn)
document = Document(title, "bob", "en")
session.add(document)
session.commit()
docid = document.id
with open(fn) as infile:
sentences = []
for line in infile:
sent = Sentence(line.strip(), docid)
sentences.append(sent)
session.add_all(sentences)
session.commit()
print("added document:", document)
def get(self):
self.response.headers['Cache-Control'] = "private"
self.response.headers['Content-Type'] = "text/plain; charset=utf-8"
user = UserInfo.get_by_key_name('user:[email protected]')
docs = Document.all().filter('owner', user)
docs = docs.fetch(10000)
self.response.out.write("# got %d docs\n" % len(docs))
for doc in docs:
self.response.out.write("%s tags[%s] date[%s] title[%s] \n" % (doc.display_url, doc.tag_comma_separated, doc.date_yyyy_mm_dd, doc.title_or_empty_string))
for page in doc.pages:
self.response.out.write(" has_page: %d\n" % (page.id_or_name()))
meds = MediaObject.all().filter('owner', user)
meds = meds.fetch(10000)
self.response.out.write("# got %d mediaobjects\n" % len(meds))
for mo in meds:
self.response.out.write("%s creation[%s] size[%d]\n" % (mo.url_path, str(mo.creation), mo.size))
def handle(self):
digest = self.request.get("d")
doc = Document.get_doc(digest)
if not digest:
return {"success": False, "reason": "format"}
if not doc:
return {"success": False, "reason": "nonexistent"}
if doc.tx:
return {"success": True, "status": "confirmed", "transaction": doc.tx, "txstamp": doc.txstamp}
if doc.is_actionable():
return {"success": True, "status": "pending"}
return {
"success": True,
"status": "registered",
"pay_address": doc.payment_address,
"price": MIN_SATOSHIS_PAYMENT,
}
def post(self):
user_info = get_user_info()
if user_info is None:
self.redirect('/?error_message=%s' % 'log-in required')
scan_ids = self.request.get_all("media_id")
scans = MediaObject.get(scan_ids)
doc = Document(
parent=user_info,
owner=user_info,
pages=[scan.key() for scan in scans],
title=None,
description=None)
def make_doc():
db.put(doc)
for scan in scans:
scan.lacks_document = False
scan.document = doc.key()
db.put(scan)
db.run_in_transaction(make_doc)
self.redirect(doc.display_url + "?size=1200")
def get(self, docid):
user_info = get_user_info()
if user_info is None:
self.redirect('/?error_message=%s' % 'login required to view docs')
docid = long(docid)
doc = Document.get_by_id(docid, parent=user_info)
if doc is None:
self.response.out.write("Docid %d not found." % (docid))
return
pages = MediaObject.get(doc.pages)
size = self.request.get("size")
if not size:
size = 1200
show_single_list = long(size) > 600
self.response.out.write(template.render('doc.html',
{"doc": doc,
"pages": pages,
"user_info": user_info,
"size": size,
"show_single_list": show_single_list},
debug=True))
def handle(self):
digest = self.request.get("d")
doc = Document.get_doc(digest)
if not doc or doc.tx:
return {"success" : False, "error": "format"}
ladd, radd = doc.get_address_repr()
ltxs = get_txs_for_addr(ladd)
rtxs = get_txs_for_addr(radd)
if not ltxs or not rtxs:
return {"success" : False, "error": "no transactions"}
intersection = [tx for tx in ltxs if tx in rtxs]
if len(intersection) == 0:
return {"success" : False, "error": "no intersecting"}
tx_hash, tx_timestamp = intersection[0]
doc.confirmed(tx_hash, tx_timestamp)
LatestBlockchainDocuments.get_inst().add_document(doc)
return {"success" : True, "tx" : doc.tx}
def get(self):
self.response.headers['Cache-Control'] = "private"
self.response.headers['Content-Type'] = "text/plain; charset=utf-8"
user = UserInfo.get_by_key_name('user:[email protected]')
docs = Document.all().filter('owner', user)
docs = docs.fetch(10000)
self.response.out.write("# got %d docs\n" % len(docs))
for doc in docs:
self.response.out.write(
"%s tags[%s] date[%s] title[%s] \n" %
(doc.display_url, doc.tag_comma_separated, doc.date_yyyy_mm_dd,
doc.title_or_empty_string))
for page in doc.pages:
self.response.out.write(" has_page: %d\n" %
(page.id_or_name()))
meds = MediaObject.all().filter('owner', user)
meds = meds.fetch(10000)
self.response.out.write("# got %d mediaobjects\n" % len(meds))
for mo in meds:
self.response.out.write("%s creation[%s] size[%d]\n" %
(mo.url_path, str(mo.creation), mo.size))
def get(self):
digest = self.request.get('d')
d = Document.get_doc(digest)
if d and d.pending and d.has_balance():
d.received_payment()
self.response.write("%s %s<br />" % (d.digest, d.payment_address))
def handle(self):
digest = self.request.get("d")
doc = Document.get_doc(digest)
if not doc:
return {"success" : False, "error": "format"}
return doc.blockchain_certify()
def get(self):
actionable = Document.get_actionable()
url = SECRET_ADMIN_PATH + '/autopay'
for d in actionable:
self.response.write('<a href="%s?d=%s">%s</a><br /><br />' % (url, d.digest, d.digest))
'bc2ed3d70b2521dc828ebb32db8d8fd954c6b7a9fc63580e601b1863821bc34e'
), # govtech_strong_password
User(name='Admin',
username='******',
password=
'******'
), # super_duper_whitehacks_strong_password
Module(code='IS200', name='Software Foundations'),
Module(code='IS103', name='Computational Thinking'),
Module(code='IS101', name='Seminar on Information Systems'),
Module(code='WRIT001', name='Academic Writing'),
Lesson(module_code='IS200', name='Lesson 01'),
Lesson(module_code='IS103', name='Lesson 01'),
Lesson(module_code='IS101', name='Lesson 01'),
Lesson(module_code='WRIT001', name='Lesson 01'),
Document(
lesson_id=1,
name='Document 01',
is_draft=False,
content=
'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum'
),
Document(lesson_id=4,
name='Document Flag',
is_draft=True,
content='NOTFLAG{youre_almost_there_try_harder}'),
]
db.session.bulk_save_objects(obj)
db.session.commit()
def handle(self):
confirmed = self.request.get("confirmed")
confirmed = confirmed and confirmed == "true"
return [doc.to_dict() for doc in Document.get_latest(confirmed)]
def get(self):
digest = self.request.get('d')
d = Document.get_doc(digest)
if d and d.pending and d.has_balance():
d.received_payment()
self.response.write("%s %s<br />" % (d.digest, d.payment_address))
def setUp(self):
self.doc = Document()
def handle(self):
digest = self.request.get("d")
doc = Document.get_doc(digest)
if not doc:
return {"success": False, "error": "format"}
return doc.blockchain_certify()
def handle(self):
digest = self.request.get("d")
doc = Document.get_doc(digest)
if not doc:
return {"success" : False, "error": "Document not found"}
return {"success": True, "payment": doc.payment_received()}
def handle(self):
digest = self.request.get("d")
doc = Document.get_doc(digest)
if not doc:
return {"success" : False, "error": "Document not found"}
return {"success": True, "doc": doc.to_dict()}
def get(self):
pending = Document.get_pending()
url = SECRET_ADMIN_PATH + '/autopay'
for d in pending:
self.response.write('<a href="%s?d=%s">%s</a><br /><br />' % (url, d.digest, d.digest))
class DocumentTestCase(unittest.TestCase):
def setUp(self):
self.doc = Document()
def test_insert(self):
self.doc.insert(0, "here")
self.assertEqual(self.doc.visible_text, "here")
self.doc.insert(0, "again ")
self.assertEqual(self.doc.visible_text, "again here")
self.doc.insert(8, "1")
self.assertEqual(self.doc.visible_text, "again he1re")
self.doc.insert(8, "23")
self.assertEqual(self.doc.visible_text, "again he231re")
self.doc.insert(len(self.doc.visible_text), "ing")
self.assertEqual(self.doc.visible_text, "again he231reing")
def test_remove(self):
self.doc.insert(0, "here is some stuff\nover a few lines")
self.assertEqual(self.doc.visible_text, "here is some stuff\nover a few lines")
self.doc.remove(0, 5)
self.assertEqual(self.doc.visible_text, "is some stuff\nover a few lines")
self.doc.remove(1, 1)
self.assertEqual(self.doc.visible_text, "i some stuff\nover a few lines")
self.doc.remove(len(self.doc.visible_text) - 4, 4)
self.assertEqual(self.doc.visible_text, "i some stuff\nover a few l")
def test_search(self):
self.doc.insert(0, "hello there\nthis is a test\nof search")
self.doc.search("hello")
self.assertEqual(self.doc.current_search, "hello")
self.assertEqual(self.doc.visible_text, "hello there")
self.doc.search("o")
self.assertEqual(self.doc.visible_text, "hello there\nof search")
self.doc.search("th")
self.assertEqual(self.doc.visible_text, "hello there\nthis is a test")
self.doc.search("")
self.assertEqual(self.doc.visible_text, "hello there\nthis is a test\nof search")
def test_search_multiple_words(self):
self.doc.insert(0, "hello there\nthis is a test\nof search")
self.doc.search("this test")
self.assertEqual(self.doc.current_search, "this test")
self.assertEqual(self.doc.visible_text, "this is a test")
def test_search_and_insert(self):
self.doc.search("test")
self.assertEqual(self.doc.visible_text, "")
self.doc.search("")
self.doc.insert(0, "hello there\nthis is a test\nof search")
self.doc.search("test")
self.assertEqual(self.doc.visible_text, "this is a test")
self.doc.insert(0, "again ")
self.assertEqual(self.doc.visible_text, "again this is a test")
self.assertEqual(self.doc.text, "hello there\nagain this is a test\nof search")
self.doc.search("")
self.assertEqual(self.doc.visible_text, "hello there\nagain this is a test\nof search")
self.doc.search("search")
self.assertEqual(self.doc.visible_text, "of search")
self.doc.insert(0, "hello ")
self.assertEqual(self.doc.visible_text, "hello of search")
self.doc.search("hello")
self.assertEqual(self.doc.visible_text, "hello there\nhello of search")
self.doc.insert(11, "go")
self.assertEqual(self.doc.visible_text, "hello therego\nhello of search")
self.assertEqual(self.doc.text, "hello therego\nagain this is a test\nhello of search")
self.doc.insert(15, "23")
self.assertEqual(self.doc.visible_text, "hello therego\nh23ello of search")
self.assertEqual(self.doc.text, "hello therego\nagain this is a test\nh23ello of search")
def test_search_and_remove(self):
self.doc.search("")
self.doc.insert(0, "hello there\nthis is a test\nof search\nhello there")
self.doc.search("hello")
self.assertEqual(self.doc.visible_text, "hello there\nhello there")
self.doc.remove(0, 3)
self.assertEqual(self.doc.visible_text, "lo there\nhello there")
self.doc.remove(len(self.doc.visible_text) - 3, 3)
self.assertEqual(self.doc.visible_text, "lo there\nhello th")
# now remove accross multiple visible regions
self.doc.remove(7, 3)
self.assertEqual(self.doc.visible_text, "lo therello th")
# finally check underlying text matches what we expect
self.assertEqual(self.doc.text, "lo therello th\nthis is a test\nof search\n")
def test_insert_remove(self):
# in the simple case (without search) inserting them remove should
# have no effect
self.doc.insert(0, "hello there this is a test\n of inserting and removing\nsome text")
self._check_inserting_and_removing()
def test_insert_remove_while_searching(self):
# when inserting and removing (in that order) with search nothing should change either
self.doc.insert(0, "hello there this is a test\n of inserting and removing\nsome text")
self.doc.search("te")
self._check_inserting_and_removing()
self.doc.search("hello")
self._check_inserting_and_removing()
self.doc.search("inserting")
self._check_inserting_and_removing()
self.doc.search("some")
self._check_inserting_and_removing()
# now try a search with no visible lines
self.doc.search("876fdjfdsf87")
self._check_inserting_and_removing()
def _check_inserting_and_removing(self):
for i in range(0, 200):
text_length = len(self.doc.visible_text)
offset = random.choice(range(text_length + 1))
length = random.choice(range(text_length + 1)) + 1
s = "".join(random.choice("gsgs8hkjefw98y") for i in range(length))
visible_text_before = self.doc.visible_text
text_before = self.doc.text
self.doc.insert(offset, s)
self.assert_(visible_text_before != self.doc.visible_text)
self.assert_(text_before != self.doc.text)
self.doc.remove(offset, length)
self.assertEqual(self.doc.visible_text, visible_text_before)
self.assertEqual(self.doc.text, text_before)
