Exemple #1
0
    def load_data(self):
        X, Y = [], []
        for file in os.listdir(self.path):
            if file == 'truth.txt' or file == '.DS_Store':
                continue
            print "loading file -->" + file
            tree = ET.parse(os.path.join(self.path, file))
            root = tree.getroot()
            document = Document(language=root.get('lang'), name=root.get('id'))
            for d in root.findall('document'):
                document.add_document(d.text)
            user, gender, age_group, extroverted, stable, agreeable, conscientious, open = self.truth[
                root.get('id')].split(":::")

            traits = PersonalityTraits(extroverted=float(extroverted), stable=float(stable), agreeable=float(agreeable),
                                       conscientious=float(conscientious), open=float(open))
            usr = Author(gender=gender, age_group=age_group, traits=traits)
            document.author = usr
            X.append(document)
            Y.append(self.truth[root.get('id')])
        print "done loading files"

        self.X = X
        self.Y = Y
        return self
Exemple #2
0
  def store_digest(self, digest):
    docproof = Document.get_doc(digest)
    if docproof:
      return {"success" : False, "reason": "existing", "digest": digest, "args": [export_timestamp(docproof.timestamp)]}

    d = Document.new(digest)
    self.doc = d
    return {"success": True, "digest": d.digest}
Exemple #3
0
  def store_digest(self, digest):
    old_doc = Document.get_doc(digest)
    if old_doc:
      return {"success" : False, "reason": "existing",
       "digest": digest, "args": [export_timestamp(old_doc.timestamp)]}

    d = Document.new(digest)
    self.doc = d # don't remove, needed for API
    return {"success": True, "digest": d.digest}
Exemple #4
0
    def get(self):
        # Provide login/logout URLs.
        user_info = get_user_info()
        if user_info is None:
            login_url = users.create_login_url('/')
        else:
            login_url = users.create_logout_url('/')

        # Collect list of error messages which gets shown to the user.
        error_messages = self.request.params.getall('error_message')
        view_user = user_info  # for now
        did_search = False

        # Fetch media for view user.
        media = MediaObject.all().filter('owner', user_info)
        media = media.filter('lacks_document', True)
        media = media.order('creation')
        limit = 50
        if self.request.get("limit"):
            limit = long(self.request.get("limit"))
        media = media.fetch(limit)

        docs = Document.all().filter('owner', user_info)
        tags = self.request.get("tags")
        if tags:
            did_search = True
            for tag in re.split('\s*,\s*', tags):
                docs = docs.filter("tags", tag)
        docs = docs.fetch(limit)

        untagged_docs = Document.all().filter('owner', user_info).filter(
            "no_tags", True).fetch(limit)

        upcoming_due = Document.all().filter('owner', user_info)
        upcoming_due = upcoming_due.filter("due_date !=", None)
        upcoming_due = upcoming_due.order("due_date")
        upcoming_due = upcoming_due.fetch(30)

        top_message = ""
        if self.request.get("saved_doc"):
            docid = long(self.request.get("saved_doc"))
            top_message = "Saved <a href='/doc/%d'>doc %d</a>" % (docid, docid)

        # Render view.
        self.response.out.write(
            template.render('main.html', {
                "did_search": did_search,
                "media": media,
                "docs": docs,
                "untagged_docs": untagged_docs,
                "upcoming_due_docs": upcoming_due,
                "view_user": view_user,
                "login_url": login_url,
                "user_info": user_info,
                "top_message": top_message,
            },
                            debug=True))
Exemple #5
0
def makeMockDocument():
  try: os.remove(testfile)
  except(OSError): pass
  
  
  document = Document(testfile)  
  
  # set up a mock example
  trn = document.database.transaction()
  with trn:
    for spandata in MockData:
      span = Document.persistenceSchema.classes.Span(externalID = spandata['id'], info = spandata['info'])
      for tok in spandata['tokens']:
        span.addToken(Document.persistenceSchema.classes.Token(tok))
      document.addSpan(span)
    
    # make some constituents, cause why not  
    phrase = Document.persistenceSchema.classes.Constituent()
    phrase.add(document.spans[0].tokens[3])
    phrase.add(document.spans[0].tokens[2])
    phrase.add(document.spans[0].tokens[4])
    
    phrase = Document.persistenceSchema.classes.Constituent()
    phrase.add(document.spans[0].tokens[0])
    phrase.add(document.spans[0].tokens[1])
    phrase.add(document.spans[0].tokens[3])
    phrase.add(document.spans[0].tokens[4])

    phrase = Document.persistenceSchema.classes.Constituent()
    print phrase.refmark
    phrase.add(document.spans[1].tokens[1])
    phrase.add(document.spans[1].tokens[0])
    
    x = Document.persistenceSchema.classes.DependencyRelation(document.spans[0].constituents[0], document.spans[0].tokens[5])
    x = Document.persistenceSchema.classes.DependencyRelation(document.spans[0].tokens[1], document.spans[0].tokens[1])
    
    document.spans[0].tokens[0].variables['PoS'] = 'Noun'
    document.spans[0].tokens[0].variables['Number'] = 'singular'
    document.spans[0].tokens[0].variables['Gender'] = 'masculine'
    document.spans[0].tokens[0].variables['Focus'] = 'yes'
    
    
    mm = Document.persistenceSchema.classes.ReferenceMark()
    document.spans[0].tokens[0].refmark = mm
    document.spans[1].tokens[0].refmark = mm
    phrase.refmark = mm
    
    document.spans[0].tokens[4].refmark = Document.persistenceSchema.classes.ReferenceMark()
    
    
          
  trn.commit()
  
  return document
Exemple #6
0
  def get(self):
    # Provide login/logout URLs.
    user_info = get_user_info()
    if user_info is None:
      login_url = users.create_login_url('/')
    else:
      login_url = users.create_logout_url('/')

    # Collect list of error messages which gets shown to the user.
    error_messages = self.request.params.getall('error_message')
    view_user = user_info  # for now
    did_search = False

    # Fetch media for view user.
    media = MediaObject.all().filter('owner', user_info)
    media = media.filter('lacks_document', True)
    media = media.order('creation')
    limit = 50
    if self.request.get("limit"):
      limit = long(self.request.get("limit"))
    media = media.fetch(limit)

    docs = Document.all().filter('owner', user_info)
    tags = self.request.get("tags")
    if tags:
      did_search = True
      for tag in re.split('\s*,\s*', tags):
        docs = docs.filter("tags", tag)
    docs = docs.fetch(limit)

    untagged_docs = Document.all().filter('owner', user_info).filter("no_tags", True).fetch(limit)

    upcoming_due = Document.all().filter('owner', user_info)
    upcoming_due = upcoming_due.filter("due_date !=", None)
    upcoming_due = upcoming_due.order("due_date")
    upcoming_due = upcoming_due.fetch(30)

    top_message = ""
    if self.request.get("saved_doc"):
      docid = long(self.request.get("saved_doc"))
      top_message = "Saved <a href='/doc/%d'>doc %d</a>" % (docid, docid)

    # Render view.
    self.response.out.write(template.render('main.html', {
        "did_search": did_search,
        "media": media,
        "docs": docs,
        "untagged_docs": untagged_docs,
        "upcoming_due_docs": upcoming_due,
        "view_user": view_user,
        "login_url": login_url,
        "user_info": user_info,
        "top_message": top_message,
        }, debug=True))
 def handle(self):
     n = 0
     for old in DocumentProof.all():
         doc = Document.get_doc(old.digest)
         if not doc:
             doc = Document.new(old.digest)
             doc.pending = old.tx == None
             doc.tx = old.tx
             doc.timestamp = old.timestamp
             doc.blockstamp = old.blockstamp
             n += 1
             if n == 10:
                 break 
             
     return {"success" : True, "processed": n}
Exemple #8
0
  def get(self):
    actionable = Document.get_actionable()
    for d in actionable:
      ret = d.blockchain_certify()

      subject = "Document certified: %s %s" % (ret['success'], d.digest)
      body = subject + "\n\nmesage: %s" % (ret['message'])
Exemple #9
0
 def get(self, docid):
     user_info = get_user_info()
     if user_info is None:
         self.redirect("/?error_message=%s" % "login required to view docs")
     docid = long(docid)
     doc = Document.get_by_id(docid, parent=user_info)
     if doc is None:
         self.response.out.write("Docid %d not found." % (docid))
         return
     pages = MediaObject.get(doc.pages)
     size = self.request.get("size")
     if not size:
         size = 1200
     show_single_list = long(size) > 600
     self.response.out.write(
         template.render(
             "doc.html",
             {
                 "doc": doc,
                 "pages": pages,
                 "user_info": user_info,
                 "size": size,
                 "show_single_list": show_single_list,
             },
             debug=True,
         )
     )
Exemple #10
0
    def _ghost(self, oid, state=None):

        if oid is Document:
            return Document()

        kind, tag, content, close = oid
        return kind()
Exemple #11
0
def apostile_create_post():
    count = session.query(Apostile).filter(Apostile.number == int(request.form['number'])).count()
    if count:
        return render_template('message.html', title='Помилка',
                               msg=f'Апостиль з номером {request.form["number"]} уже існує', **user_config())

    doc = Document(
        country=request.form['country'],
        date=request.form['doc_date'],
        author_name=request.form['person_name'],
        author_info=request.form['person_position'],
        stamp_info=request.form['stamp_info'] if request.form['stamp_info'] else '-/-/-'
    )
    session.add(doc)
    session.flush()

    tr_id, tr_type = find_trusted_by_str(request.form['ap_author'])
    ap = Apostile(
        number=int(request.form['number']),
        date=request.form['ap_date'],
        is_archived=False,
        trusted_id=tr_id, trusted_type=tr_type,
        document_id=doc.id
    )
    session.add(ap)
    session.flush()
    session.commit()

    return redirect(f'/apostile/{ap.id}')
Exemple #12
0
class BasePaymentCallback(JsonAPIHandler):
    def handle(self):
        test = self.request.get("test") == "true"
        try:
            tx_hash = self.request.get("transaction_hash")
            address = self.request.get("address")
            satoshis = int(self.request.get("value"))
            payment_address = self.request.get("input_address")
        except ValueError, e:
            return "error: value error"
        if not tx_hash:
            return "error: no transaction_hash"

        if not address:
            return "error: no address"

        if satoshis <= 0:  # outgoing payment
            return "*ok*"
        if satoshis < MIN_SATOSHIS_PAYMENT:  # not enough
            return "*ok*"

        if not test:
            doc = Document.get_by_address(payment_address)
            if not doc:
                return "error: couldn't find document"
            return self.process_payment(satoshis, doc)
        return "*ok*"
Exemple #13
0
    def run(self):
        result = {}
        x, y, y_actual = [], [], []
        for file in os.listdir(self.path):
            if file == 'truth.txt' or file == '.DS_Store':
                continue
            tree = ET.parse(os.path.join(self.path, file))
            root = tree.getroot()
            document = Document(language=root.get('lang'), name=root.get('id'))

            for d in root.findall('document'):
                document.add_document(d.text)
            x_test = [document]  # vector

            temp_result = {}
            for predictor in self.model:
                # print predictor

                if predictor.name == 'age_gender':
                    prediction = predictor.clf.predict(x_test)  # predict
                    temp_result.update(
                        predictor.label_extractor(list(predictor.label_encoder.inverse_transform(prediction))[0]))
                    document.author.gender = temp_result['gender']
                    document.author.age_group = temp_result['age_group']
                if predictor.name == 'personality':
                    target = predictor.label_encoder.classes_
                    prediction = list(predictor.clf.predict_proba(x_test))[0]
                    prediction = [change_range(p, 1.0, 0.0, 0.5, -0.5) for p in prediction]
                    temp_result.update(predictor.label_extractor(target, prediction))



            document.author.personality_traits.extroverted = temp_result['extroverted']
            document.author.personality_traits.agreeable = temp_result['agreeable']
            document.author.personality_traits.conscientious = temp_result['conscientious']
            document.author.personality_traits.stable = temp_result['stable']
            document.author.personality_traits.open = temp_result['open']

            result[os.path.splitext(file)[0]] = document
            # y.extend(prediction)
            # print y
            x.append(os.path.splitext(file)[0])
            # y_actual.append(predictor.label_extractor(self.truth[root.get('id')]))
        self.x_test = x_test
        # self.y_prediction = y
        # self.y_actual = self.label_encoder.transform(y_actual)
        self.result = result
def save_document(title, session):
    """Create a new document; return it."""
    ## XXX(alexr): need to handle source languages...
    document = Document(title, "bob", "en")
    session.add(document)
    session.commit()
    print("added document:", document)
    return document
 def get(self):
   actionable = Document.get_actionable()
   for d in actionable:
     ret = d.blockchain_certify()
     sender_address = "*****@*****.**"
     subject = "Document certified: %s %s" % (ret['success'], d.digest)
     body = subject + "\n\nmesage: %s" % (ret['message'])
     mail.send_mail(sender_address, ADMIN_EMAIL, subject, body)
Exemple #16
0
 def get(self):
     actionable = Document.get_actionable()
     for d in actionable:
         ret = d.blockchain_certify()
         sender_address = "*****@*****.**"
         subject = "Document certified: %s %s" % (ret['success'], d.digest)
         body = subject + "\n\nmesage: %s" % (ret['message'])
         mail.send_mail(sender_address, ADMIN_EMAIL, subject, body)
Exemple #17
0
        def store_media():
            """Store media object info in datastore.

      Also updates the user-info record to keep count of media objects.

      This function is run as a transaction.
      """
            user_info = UserInfo.get_by_key_name("user:%s" % user_email)
            if user_info is None:
                error_messages.append("User record has been deleted.  " "Try uploading again")
                return

            media = MediaObject(
                parent=user_info,
                owner=user_info,
                blob=blob_info.key(),
                creation=blob_info.creation,
                content_type=blob_info.content_type,
                filename=blob_info.filename,
                size=int(blob_info.size),
                lacks_document=True,
            )

            user_info.media_objects += 1
            db.put(user_info)
            db.put(media)

            if bool(is_doc) and is_doc != "0":
                tag_list = []
                if tags is not None:
                    tag_list = [x for x in re.split("\s*,\s*", tags) if x]

                doc = Document(
                    parent=user_info,
                    owner=user_info,
                    pages=[media.key()],
                    title=title,
                    description=description,
                    no_tags=(len(tag_list) == 0),
                    tags=tag_list,
                )
                db.put(doc)
                media.document = doc.key()
                media.lacks_document = False
                db.put(media)
Exemple #18
0
 def handle(self):
     digest = self.request.get("d")
     doc = Document.get_doc(digest)
     if not doc or not doc.tx:
         return {"success" : False, "error": "format"}
     # TODO: add check to prevent double timestamping
     tx, message = publish_data_old(doc)
     do_check_document(digest)
     return {"success" : True, "tx" : tx, "message" : message}
Exemple #19
0
    def store_media():
      """Store media object info in datastore.

      Also updates the user-info record to keep count of media objects.

      This function is run as a transaction.
      """
      user_info = UserInfo.get_by_key_name('user:%s' % user_email)
      if user_info is None:
        error_messages.append('User record has been deleted.  '
                              'Try uploading again')
        return

      media = MediaObject(
          parent=user_info,
          owner=user_info,
          blob=blob_info.key(),
          creation=blob_info.creation,
          content_type=blob_info.content_type,
          filename=blob_info.filename,
          size=int(blob_info.size),
          lacks_document=True)

      user_info.media_objects += 1
      db.put(user_info)
      db.put(media)

      if bool(is_doc) and is_doc != "0":
        tag_list = []
        if tags is not None:
          tag_list = [x for x in re.split('\s*,\s*', tags) if x]

        doc = Document(
            parent=user_info,
            owner=user_info,
            pages=[media.key()],
            title=title,
            description=description,
            no_tags=(len(tag_list)==0),
            tags=tag_list)
        db.put(doc)
        media.document = doc.key()
        media.lacks_document = False
        db.put(media)
 def create_doc(self, content, title, description='', author=''):
     """
     Creates an document.
     :param data: document's properties as json.
     :return:
     """
     doc = Document(content=content, title=title, description=description, author=author)
     self.db.session.add(doc)
     self.db.session.commit()
     return doc
Exemple #21
0
  def handle(self):
    digest = self.request.get("d")
    doc = Document.get_doc(digest)
    if not doc or not doc.payment_address:
      return {"success" : False, "error": "format"}
    
    if not doc.tx:
      return {"success" : False, "error": "no transaction"}

    return {"success" : True, "tx" : doc.tx}
Exemple #22
0
 def get_doc_by_pid(cls, pid):
     sql = """select * from doc where person_id=?"""
     data = (pid, )
     rows = cls.db.query(sql, data)
     records = []
     for row in rows:
         print(row)
         records.append(
             Document(row[0], row[1], row[2], cls.get_by_doc_id(row[0])))
     return records
Exemple #23
0
def makeMockDocument1():
  try: os.remove(testfile)
  except(OSError): pass
  
  
  document = Document(testfile)  
  
  # set up a mock example
  trn = document.database.transaction()
  with trn:
    for spandata in MockData1:
      span = Document.persistenceSchema.classes.Span(externalID = spandata['id'], info = spandata['info'])
      for tok in spandata['tokens']:
        span.addToken(Document.persistenceSchema.classes.Token(tok))
      document.addSpan(span)
    
  trn.commit()
  
  return document  
Exemple #24
0
    def post(self):
        user_info = get_user_info()
        if user_info is None:
            self.redirect('/?error_message=%s' % 'login required to view docs')
        docid = long(self.request.get("docid"))
        doc = Document.get_by_id(docid, parent=user_info)
        if doc is None:
            self.response.out.write("Docid %d not found." % (docid))
            return

        mode = self.request.get("mode")
        if mode == "break":
            break_and_delete_doc(user_info, doc)
            self.response.out.write(
                "[&lt;&lt; <a href='/'>Back</a>] Docid %d deleted and images broken out as un-annotated."
                % docid)
            return
        if mode == "delete":
            delete_doc_and_images(user_info, doc)
            self.response.out.write(
                "[&lt;&lt; <a href='/'>Back</a>] Docid %d and its images deleted."
                % docid)
            return

        # Simple properties:
        doc.physical_location = self.request.get("physical_location")
        doc.title = self.request.get("title")

        # Tags
        doc.tags = [
            x for x in re.split('\s*,\s*', self.request.get("tags")) if x
        ]
        doc.no_tags = (len(doc.tags) == 0)

        # Document Date
        date = self.request.get("date")
        if date:
            doc.doc_date = datetime.datetime.strptime(date, "%Y-%m-%d")
            doc.no_date = False
        else:
            doc.doc_date = None
            doc.no_date = True

        # Due date
        due_date_str = self.request.get("due_date")
        doc.due_date = None
        if due_date_str:
            doc.due_date = datetime.datetime.strptime(due_date_str, "%Y-%m-%d")

        def store():
            db.put(doc)

        db.run_in_transaction(store)
        self.redirect("/?saved_doc=" + str(docid))
Exemple #25
0
    def process_payment(self, satoshis, digest):
        secret = self.request.get("secret")
        if len(digest) != 64 or secret != CALLBACK_SECRET or satoshis < MIN_SATOSHIS_PAYMENT:
            return {"success" : False, "reason" : "format or payment below " + str(MIN_SATOSHIS_PAYMENT)}

        doc = Document.get_doc(digest)
        if not doc:
            return {"success" : False, "reason" : "Couldn't find document"}
        doc.pending = False
        doc.put()

        return {"success" : True}
 def get(self):
   archiveable = Document.get_archiveable()
   processed = False
   for d in archiveable:
     res = d.archive()
     self.response.write("%s %s<br />" % (d.digest, res))
     processed = True
   if processed:
     self.response.write("Running autoconsolidate<br />")
     auto_consolidate()
   else:
     self.response.write("Finished without operation<br />")
Exemple #27
0
 def get(self):
     archiveable = Document.get_archiveable()
     processed = False
     for d in archiveable:
         res = d.archive()
         self.response.write("%s %s<br />" % (d.digest, res))
         processed = True
     if processed:
         self.response.write("Running autoconsolidate<br />")
         auto_consolidate()
     else:
         self.response.write("Finished without operation<br />")
def make_from_json(jsonData, filename, removeOld=False):
    if removeOld:
        try:
            os.remove(filename)
        except (OSError):
            pass

    # create the document
    document = Document(filename)

    # set up a mock example
    trn = document.database.transaction()
    with trn:
        document.database.root['documentType'] = jsonData['documentType']
        document.database.root['variety'] = jsonData['variety']

        for i, spanData in enumerate(jsonData['spans']):
            span = Document.persistenceSchema.classes.Span(
                externalID=i, info=spanData['spanInfo'])
            for i, token in enumerate(spanData['tokens']):
                token = Document.persistenceSchema.classes.Token(token)
                try:
                    token.gloss = spanData['lemmas'][i]
                except:
                    pass

                span.addToken(token)

            document.addSpan(span)

    trn.commit()
    document.close()
Exemple #29
0
 def handle(self):
   digest = self.request.get("d")
   doc = Document.get_doc(digest)
   if not doc or doc.tx:
     return {"success" : False, "error": "format"}
   # TODO: add check to prevent double timestamping
   txid, message = publish_data(doc.digest.decode('hex'))
   if txid:
     doc.tx = txid
     doc.txstamp = datetime.datetime.now()
     LatestBlockchainDocuments.get_inst().add_document(digest)
     doc.put()
   return {"success" : txid is not None, "tx" : txid, "message" : message}
Exemple #30
0
 def handle(self):
     digest = self.request.get("d")
     doc = Document.get_doc(digest)
     if not doc or doc.tx:
         return {"success": False, "error": "format"}
     # TODO: add check to prevent double timestamping
     txid, message = publish_data(doc.digest.decode('hex'))
     if txid:
         doc.tx = txid
         doc.txstamp = datetime.datetime.now()
         LatestBlockchainDocuments.get_inst().add_document(digest)
         doc.put()
     return {"success": txid is not None, "tx": txid, "message": message}
Exemple #31
0
  def handle(self):
    digest = self.request.get("d")
    doc = Document.get_doc(digest)
    if not digest:
      return {"success": False, "reason": "format"}
    if not doc:
      return {"success": False, "reason": "nonexistent"}
    if doc.tx:
      return {"success": True, "status": "confirmed", "transaction": doc.tx}
    if doc.is_actionable():
      return {"success": True, "status": "pending"}

    return {"success": True, "status": "registered"}
Exemple #32
0
  def handle(self):
    digest = self.request.get("d")
    doc = Document.get_doc(digest)
    if not doc or not doc.payment_address:
      return {"success" : False, "error": "format"}
    
    txs = get_txs_for_addr(doc.payment_address)
    if not txs or len(txs) == 0:
      return {"success" : False, "error": "no transactions"}
    tx_hash, tx_timestamp = txs[0]
    doc.confirmed(tx_hash, tx_timestamp)

    LatestBlockchainDocuments.get_inst().add_document(doc)
    return {"success" : True, "tx" : doc.tx}
Exemple #33
0
    def post(self):
        user_info = get_user_info()
        if user_info is None:
            self.redirect("/?error_message=%s" % "login required to view docs")
        docid = long(self.request.get("docid"))
        doc = Document.get_by_id(docid, parent=user_info)
        if doc is None:
            self.response.out.write("Docid %d not found." % (docid))
            return

        mode = self.request.get("mode")
        if mode == "break":
            break_and_delete_doc(user_info, doc)
            self.response.out.write(
                "[&lt;&lt; <a href='/'>Back</a>] Docid %d deleted and images broken out as un-annotated." % docid
            )
            return
        if mode == "delete":
            delete_doc_and_images(user_info, doc)
            self.response.out.write("[&lt;&lt; <a href='/'>Back</a>] Docid %d and its images deleted." % docid)
            return

        # Simple properties:
        doc.physical_location = self.request.get("physical_location")
        doc.title = self.request.get("title")

        # Tags
        doc.tags = [x for x in re.split("\s*,\s*", self.request.get("tags")) if x]
        doc.no_tags = len(doc.tags) == 0

        # Document Date
        date = self.request.get("date")
        if date:
            doc.doc_date = datetime.datetime.strptime(date, "%Y-%m-%d")
            doc.no_date = False
        else:
            doc.doc_date = None
            doc.no_date = True

        # Due date
        due_date_str = self.request.get("due_date")
        doc.due_date = None
        if due_date_str:
            doc.due_date = datetime.datetime.strptime(due_date_str, "%Y-%m-%d")

        def store():
            db.put(doc)

        db.run_in_transaction(store)
        self.redirect("/?saved_doc=" + str(docid))
Exemple #34
0
 def show(self):
     window = self.window
     while True:
         event, values = window.read()
         if event == "submit":
             doc = Document(None, values['doc'], self.person.id, [])
             doc_id = self.service.add_doc(doc)
             if doc_id > 0:
                 self.make_doc_on_disk(self.person, doc)
             break
         elif event == "reset":
             window.Element("doc").Update(value="")
         else:
             break
     window.Close()
Exemple #35
0
def load_text(document, name, passcode, doc_owner):
    """ Load the text from the document into database, and create new db object """

    text = document.read()

    document = Document(text=text,
                        name=name,
                        passcode=passcode,
                        doc_owner=doc_owner)

    db.session.add(document)

    db.session.commit()

    return document
Exemple #36
0
def save_document(title, tags, segments):
    """Given a document title, tags (as a list of strings), all the segments
    (also a list of strings), create a new document and tag it."""
    session = get_session()
    document = Document(title, "bob", "es")
    session.add(document)
    session.commit()
    docid = document.id

    sentences = []
    for (segmentid, s) in segments:
        sent = Sentence(s.strip(), docid)
        sentences.append(sent)
    session.add_all(sentences)
    session.commit()
    for tag in tags:
        tag_document(document, tag)
Exemple #37
0
 def handle(self):
   digest = self.request.get("d")
   doc = Document.get_doc(digest)
   if not digest:
     return {"success": False, "reason": "format"}
   if not doc:
     return {"success": False, "reason": "nonexistent"}
   if doc.tx:
     return {"success": True, "status": "confirmed", "transaction": doc.tx, "txstamp": doc.txstamp}
   if doc.is_actionable():
     return {"success": True, "status": "pending"}
 
   return {"success": True, \
       "status": "registered", \
       "pay_address": doc.payment_address, \
       "price": MIN_SATOSHIS_PAYMENT \
       }
Exemple #38
0
  def get(self):
    if not users.is_current_user_admin():
      self.redirect('/?error_message=%s' % 'log-in required')

    used = set()
    for d in Document.all():
      used |= set(d.pages)

    dead = dict()
    for i in MediaObject.all():
      if i.key() not in used:
        dead[i.key()] = i

    for k in dead:
      dead[k].delete()

    self.redirect('/')
Exemple #39
0
def save_file(fn):
    session = Session()
    title = os.path.basename(fn)
    document = Document(title, "bob", "en")
    session.add(document)
    session.commit()

    docid = document.id

    with open(fn) as infile:
        sentences = []
        for line in infile:
            sent = Sentence(line.strip(), docid)
            sentences.append(sent)
    session.add_all(sentences)
    session.commit()
    print("added document:", document)
Exemple #40
0
  def get(self):
    self.response.headers['Cache-Control'] = "private"
    self.response.headers['Content-Type'] = "text/plain; charset=utf-8"

    user = UserInfo.get_by_key_name('user:[email protected]')

    docs = Document.all().filter('owner', user)
    docs = docs.fetch(10000)
    self.response.out.write("# got %d docs\n" % len(docs))
    for doc in docs:
      self.response.out.write("%s tags[%s] date[%s] title[%s] \n" % (doc.display_url, doc.tag_comma_separated, doc.date_yyyy_mm_dd, doc.title_or_empty_string))
      for page in doc.pages:
        self.response.out.write(" has_page: %d\n" % (page.id_or_name()))
    meds = MediaObject.all().filter('owner', user)
    meds = meds.fetch(10000)
    self.response.out.write("# got %d mediaobjects\n" % len(meds))
    for mo in meds:
      self.response.out.write("%s creation[%s] size[%d]\n" % (mo.url_path, str(mo.creation), mo.size))
Exemple #41
0
    def handle(self):
        digest = self.request.get("d")
        doc = Document.get_doc(digest)
        if not digest:
            return {"success": False, "reason": "format"}
        if not doc:
            return {"success": False, "reason": "nonexistent"}
        if doc.tx:
            return {"success": True, "status": "confirmed", "transaction": doc.tx, "txstamp": doc.txstamp}
        if doc.is_actionable():
            return {"success": True, "status": "pending"}

        return {
            "success": True,
            "status": "registered",
            "pay_address": doc.payment_address,
            "price": MIN_SATOSHIS_PAYMENT,
        }
Exemple #42
0
 def post(self):
   user_info = get_user_info()
   if user_info is None:
     self.redirect('/?error_message=%s' % 'log-in required')
   scan_ids = self.request.get_all("media_id")
   scans = MediaObject.get(scan_ids)
   doc = Document(
       parent=user_info,
       owner=user_info,
       pages=[scan.key() for scan in scans],
       title=None,
       description=None)
   def make_doc():
     db.put(doc)
     for scan in scans:
       scan.lacks_document = False
       scan.document = doc.key()
       db.put(scan)
   db.run_in_transaction(make_doc)
   self.redirect(doc.display_url + "?size=1200")
Exemple #43
0
 def get(self, docid):
   user_info = get_user_info()
   if user_info is None:
     self.redirect('/?error_message=%s' % 'login required to view docs')
   docid = long(docid)
   doc = Document.get_by_id(docid, parent=user_info)
   if doc is None:
     self.response.out.write("Docid %d not found." % (docid))
     return
   pages = MediaObject.get(doc.pages)
   size = self.request.get("size")
   if not size:
     size = 1200
   show_single_list = long(size) > 600
   self.response.out.write(template.render('doc.html',
                                           {"doc": doc,
                                            "pages": pages,
                                            "user_info": user_info,
                                            "size": size,
                                            "show_single_list": show_single_list},
                                           debug=True))
Exemple #44
0
    def handle(self):
        digest = self.request.get("d")
        doc = Document.get_doc(digest)
        if not doc or doc.tx:
            return {"success" : False, "error": "format"}
        
        ladd, radd = doc.get_address_repr()
        
        ltxs = get_txs_for_addr(ladd)
        rtxs = get_txs_for_addr(radd)
        if not ltxs or not rtxs:
            return {"success" : False, "error": "no transactions"}
        intersection = [tx for tx in ltxs if tx in rtxs]
        if len(intersection) == 0:
            return {"success" : False, "error": "no intersecting"}

        tx_hash, tx_timestamp = intersection[0]
        doc.confirmed(tx_hash, tx_timestamp)

        LatestBlockchainDocuments.get_inst().add_document(doc)
        return {"success" : True, "tx" : doc.tx}
    def get(self):
        self.response.headers['Cache-Control'] = "private"
        self.response.headers['Content-Type'] = "text/plain; charset=utf-8"

        user = UserInfo.get_by_key_name('user:[email protected]')

        docs = Document.all().filter('owner', user)
        docs = docs.fetch(10000)
        self.response.out.write("# got %d docs\n" % len(docs))
        for doc in docs:
            self.response.out.write(
                "%s tags[%s] date[%s] title[%s] \n" %
                (doc.display_url, doc.tag_comma_separated, doc.date_yyyy_mm_dd,
                 doc.title_or_empty_string))
            for page in doc.pages:
                self.response.out.write(" has_page: %d\n" %
                                        (page.id_or_name()))
        meds = MediaObject.all().filter('owner', user)
        meds = meds.fetch(10000)
        self.response.out.write("# got %d mediaobjects\n" % len(meds))
        for mo in meds:
            self.response.out.write("%s creation[%s] size[%d]\n" %
                                    (mo.url_path, str(mo.creation), mo.size))
 def get(self):
   digest = self.request.get('d')
   d = Document.get_doc(digest)
   if d and d.pending and d.has_balance():
     d.received_payment()
     self.response.write("%s %s<br />" % (d.digest, d.payment_address))
Exemple #47
0
 def handle(self):
   digest = self.request.get("d")
   doc = Document.get_doc(digest)
   if not doc:
     return {"success" : False, "error": "format"}
   return doc.blockchain_certify()
Exemple #48
0
 def get(self):
   actionable = Document.get_actionable()
   url = SECRET_ADMIN_PATH + '/autopay'
   for d in actionable:
     self.response.write('<a href="%s?d=%s">%s</a><br /><br />' % (url, d.digest, d.digest))
             'bc2ed3d70b2521dc828ebb32db8d8fd954c6b7a9fc63580e601b1863821bc34e'
             ),  # govtech_strong_password
        User(name='Admin',
             username='******',
             password=
             '******'
             ),  # super_duper_whitehacks_strong_password
        Module(code='IS200', name='Software Foundations'),
        Module(code='IS103', name='Computational Thinking'),
        Module(code='IS101', name='Seminar on Information Systems'),
        Module(code='WRIT001', name='Academic Writing'),
        Lesson(module_code='IS200', name='Lesson 01'),
        Lesson(module_code='IS103', name='Lesson 01'),
        Lesson(module_code='IS101', name='Lesson 01'),
        Lesson(module_code='WRIT001', name='Lesson 01'),
        Document(
            lesson_id=1,
            name='Document 01',
            is_draft=False,
            content=
            'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum'
        ),
        Document(lesson_id=4,
                 name='Document Flag',
                 is_draft=True,
                 content='NOTFLAG{youre_almost_there_try_harder}'),
    ]

    db.session.bulk_save_objects(obj)
    db.session.commit()
Exemple #50
0
  def handle(self):
    confirmed = self.request.get("confirmed")
    confirmed = confirmed and confirmed == "true"

    return [doc.to_dict() for doc in Document.get_latest(confirmed)]
Exemple #51
0
 def get(self):
     digest = self.request.get('d')
     d = Document.get_doc(digest)
     if d and d.pending and d.has_balance():
         d.received_payment()
         self.response.write("%s %s<br />" % (d.digest, d.payment_address))
Exemple #52
0
 def setUp(self):
     self.doc = Document()
Exemple #53
0
 def handle(self):
     digest = self.request.get("d")
     doc = Document.get_doc(digest)
     if not doc:
         return {"success": False, "error": "format"}
     return doc.blockchain_certify()
Exemple #54
0
 def handle(self):
   digest = self.request.get("d")
   doc = Document.get_doc(digest)
   if not doc:
     return {"success" : False, "error": "Document not found"}
   return {"success": True, "payment": doc.payment_received()}
Exemple #55
0
 def handle(self):
   digest = self.request.get("d")
   doc = Document.get_doc(digest)
   if not doc:
     return {"success" : False, "error": "Document not found"}
   return {"success": True, "doc": doc.to_dict()}
Exemple #56
0
 def get(self):
     pending = Document.get_pending()
     url = SECRET_ADMIN_PATH + '/autopay'
     for d in pending:
         self.response.write('<a href="%s?d=%s">%s</a><br /><br />' % (url, d.digest, d.digest))
Exemple #57
0
class DocumentTestCase(unittest.TestCase):
    def setUp(self):
        self.doc = Document()

    def test_insert(self):
        self.doc.insert(0, "here")
        self.assertEqual(self.doc.visible_text, "here")

        self.doc.insert(0, "again ")
        self.assertEqual(self.doc.visible_text, "again here")

        self.doc.insert(8, "1")
        self.assertEqual(self.doc.visible_text, "again he1re")

        self.doc.insert(8, "23")
        self.assertEqual(self.doc.visible_text, "again he231re")

        self.doc.insert(len(self.doc.visible_text), "ing")
        self.assertEqual(self.doc.visible_text, "again he231reing")

    def test_remove(self):
        self.doc.insert(0, "here is some stuff\nover a few lines")
        self.assertEqual(self.doc.visible_text, "here is some stuff\nover a few lines")

        self.doc.remove(0, 5)
        self.assertEqual(self.doc.visible_text, "is some stuff\nover a few lines")

        self.doc.remove(1, 1)
        self.assertEqual(self.doc.visible_text, "i some stuff\nover a few lines")

        self.doc.remove(len(self.doc.visible_text) - 4, 4)
        self.assertEqual(self.doc.visible_text, "i some stuff\nover a few l")

    def test_search(self):
        self.doc.insert(0, "hello there\nthis is a test\nof search")
        self.doc.search("hello")
        self.assertEqual(self.doc.current_search, "hello")

        self.assertEqual(self.doc.visible_text, "hello there")
        self.doc.search("o")
        self.assertEqual(self.doc.visible_text, "hello there\nof search")
        self.doc.search("th")
        self.assertEqual(self.doc.visible_text, "hello there\nthis is a test")
        self.doc.search("")
        self.assertEqual(self.doc.visible_text, "hello there\nthis is a test\nof search")

    def test_search_multiple_words(self):
        self.doc.insert(0, "hello there\nthis is a test\nof search")
        self.doc.search("this test")
        self.assertEqual(self.doc.current_search, "this test")

        self.assertEqual(self.doc.visible_text, "this is a test")

    def test_search_and_insert(self):
        self.doc.search("test")
        self.assertEqual(self.doc.visible_text, "")

        self.doc.search("")
        self.doc.insert(0, "hello there\nthis is a test\nof search")

        self.doc.search("test")
        self.assertEqual(self.doc.visible_text, "this is a test")
        self.doc.insert(0, "again ")
        self.assertEqual(self.doc.visible_text, "again this is a test")
        self.assertEqual(self.doc.text, "hello there\nagain this is a test\nof search")

        self.doc.search("")
        self.assertEqual(self.doc.visible_text, "hello there\nagain this is a test\nof search")

        self.doc.search("search")
        self.assertEqual(self.doc.visible_text, "of search")
        self.doc.insert(0, "hello ")
        self.assertEqual(self.doc.visible_text, "hello of search")

        self.doc.search("hello")
        self.assertEqual(self.doc.visible_text, "hello there\nhello of search")
        self.doc.insert(11, "go")
        self.assertEqual(self.doc.visible_text, "hello therego\nhello of search")
        self.assertEqual(self.doc.text, "hello therego\nagain this is a test\nhello of search")
        self.doc.insert(15, "23")
        self.assertEqual(self.doc.visible_text, "hello therego\nh23ello of search")
        self.assertEqual(self.doc.text, "hello therego\nagain this is a test\nh23ello of search")

    def test_search_and_remove(self):
        self.doc.search("")
        self.doc.insert(0, "hello there\nthis is a test\nof search\nhello there")

        self.doc.search("hello")
        self.assertEqual(self.doc.visible_text, "hello there\nhello there")

        self.doc.remove(0, 3)
        self.assertEqual(self.doc.visible_text, "lo there\nhello there")

        self.doc.remove(len(self.doc.visible_text) - 3, 3)
        self.assertEqual(self.doc.visible_text, "lo there\nhello th")

        # now remove accross multiple visible regions
        self.doc.remove(7, 3)
        self.assertEqual(self.doc.visible_text, "lo therello th")

        # finally check underlying text matches what we expect
        self.assertEqual(self.doc.text, "lo therello th\nthis is a test\nof search\n")

    def test_insert_remove(self):
        # in the simple case (without search) inserting them remove should
        # have no effect
        self.doc.insert(0, "hello there this is a test\n of inserting and removing\nsome text")
        self._check_inserting_and_removing()

    def test_insert_remove_while_searching(self):
        # when inserting and removing (in that order) with search nothing should change either
        self.doc.insert(0, "hello there this is a test\n of inserting and removing\nsome text")

        self.doc.search("te")
        self._check_inserting_and_removing()

        self.doc.search("hello")
        self._check_inserting_and_removing()

        self.doc.search("inserting")
        self._check_inserting_and_removing()

        self.doc.search("some")
        self._check_inserting_and_removing()

        # now try a search with no visible lines
        self.doc.search("876fdjfdsf87")
        self._check_inserting_and_removing()

    def _check_inserting_and_removing(self):
        for i in range(0, 200):
            text_length = len(self.doc.visible_text)
            offset = random.choice(range(text_length + 1))
            length = random.choice(range(text_length + 1)) + 1

            s = "".join(random.choice("gsgs8hkjefw98y") for i in range(length))

            visible_text_before = self.doc.visible_text
            text_before = self.doc.text

            self.doc.insert(offset, s)
            self.assert_(visible_text_before != self.doc.visible_text)
            self.assert_(text_before != self.doc.text)

            self.doc.remove(offset, length)
            self.assertEqual(self.doc.visible_text, visible_text_before)
            self.assertEqual(self.doc.text, text_before)