def pdf_to_db(filename):
def handle_voting_place(electoral_zone_code, county_code, county_name,
place_code, place_name):
place_db_id = place_cache.get(
f"{electoral_zone_code}#{county_code}#{place_code}")
if not place_db_id:
county_db_id = county_cache.get(
f"{electoral_zone_code}#{county_code}")
if not county_db_id:
electoral_zone_db_id = electoral_zone_cache.get(
electoral_zone_code)
if not electoral_zone_db_id:
electoral_zone = ElectoralZone.create(
code=electoral_zone_code)
electoral_zone_cache.update(
{electoral_zone_code: electoral_zone.id})
electoral_zone_db_id = electoral_zone.id
county = County.create(electoral_zone_id=electoral_zone_db_id,
code=county_code,
name=county_name)
county_cache.update(
{f"{electoral_zone_code}#{county_code}": county.id})
county_db_id = county.id
place = VotingPlace.create(county_id=county_db_id, name=place_name)
place_cache.update({
f"{electoral_zone_code}#{county_code}#{place_code}":
place.id
})
def handle_voter(name, birthdate, voter_id, section_code, place_code,
county_code, electoral_zone_code):
section_db_id = section_cache.get(
f"{electoral_zone_code}#{county_code}#{place_code}#{section_code}")
if not section_db_id:
place_db_id = place_cache.get(
f"{electoral_zone_code}#{county_code}#{place_code}")
section = VotingSection.create(voting_place_id=place_db_id,
code=section_code)
section_cache.update({
f"{electoral_zone_code}#{county_code}#{place_code}#{section_code}":
section.id
})
section_db_id = section.id
voters.append({
"voting_section_id": section_db_id,
"name": name.upper(),
"birthdate": birthdate,
"voter_id": voter_id
})
electoral_zone_cache = {}
county_cache = {}
place_cache = {}
section_cache = {}
voters = []
transform(filename, handle_voting_place, handle_voter)
db_batch_size = 100
with db.atomic():
for idx in range(0, len(voters), db_batch_size):
Voter.insert_many(voters[idx:idx + db_batch_size]).execute()
def new(cls, title, user, content=None):
with db.atomic():
ret = cls.create(title=title,
user=user,
time=int(time.time()),
content=content)
ret.weight = ret.id
ret.save()
return ret
def refresh_key(self):
count = 0
while count < 10:
with db.atomic():
try:
k = self.gen_key()
self.key = k['key']
self.key_time = k['key_time']
self.save()
return k
except DatabaseError:
count += 1
db.rollback()
raise ValueError("generate key failed")
def submit_edit():
""" handle file edit form """
# ... including student work pages.
# invoked from handle_post()
# the form text data is in the form dict key, i.e. request.form['edit_text']
# print_debug(' submit_edit: request.form : {}'.format(request.form.items()))
#
# request.form will look something like
# { 'submit_edit' : u'save page', # submit button
# 'edit_text' : u'page content\r\n===\r\n etc' # textarea
# }
#
# If this is a work page, then update the Work database object.
# (The Work object is created in Page.get_from_path)
if request.page.is_work:
now = str(Time()) # string with current time
with db.atomic():
if request.page.user_role.name == 'faculty':
page = request.page
page.work.faculty_modified = now
# If checkbox for 'submitted' and that didn't have a value,
# then set it - used to mark work as submitted,
# i.e. student emails work & faculty submits.
if "submitted_checkbox" in request.form \
and not page.work.submitted:
due = Time(page.work_due)
print_debug(" submit_edit: due = {}".format(str(due)))
if "on_time_checkbox" in request.form:
due.shift_minutes(-5)
else:
due.shift_minutes(5)
page.work.submitted = str(due)
print_debug(" submit_edit: submitted set to {}".format( \
page.work.submitted))
if 'grade' in request.form:
page.work.grade = str(request.form['grade'])
else:
request.page.work.student_modified = now
if not request.page.work.submitted:
request.page.work.submitted = now
request.page.work.save()
# Save the page content to a file and to git.
bytes_written = request.page.write_content(request.form['edit_text'])
print_debug(' submit_edit: bytes_written = {}'.format(bytes_written))
git.add_and_commit(request.page)
return request.base_url # ... and reload it without ?action=edit
def process_message(email_msg: Message, checksum: str, m_uid: str):
"""
Process an entire message object.
Split attachment files from rawmsg, create db entries for
each rawmsg, message meta and attachment.
"""
# We need to parse attachments first.
# They are extracted and removed from messages.
_attachments = [process_attachment(part) for part in email_msg.walk()]
attachments = list(filter(None, _attachments))
has_attachments = len(attachments) > 0
# Parse metadata
from_ = email_msg.get('From')
to = email_msg.get('To')
subject = email_msg.get('Subject')
date = email.utils.parsedate_to_datetime(
email_msg.get('Date')) if email_msg.get('Date') else None
with db.atomic():
rmsg = RawMsg.create(email_blob=email_msg.as_bytes(),
original_checksum=checksum)
if has_attachments:
for file_checksum, filename, content_type in attachments:
log.debug(file_checksum, filename, content_type)
att = Attachment.create(
file_checksum=file_checksum,
original_filename=filename,
content_type=content_type,
)
rmsg.attachments.add(att)
mmeta = MsgMeta.create(
rawmsg=rmsg,
imap_uid=m_uid,
from_=from_,
to=to,
subject=subject,
date=date,
has_attachments=has_attachments,
)
log.debug(m_uid, from_, to, subject)
def new(cls, title, user, board, content=None):
with db.atomic():
ret = cls.create(title=title, user=user, board=board, time=int(time.time()), content=content)
ret.weight = ret.id
ret.save()
return ret
def request_coin_price():
coin_prices = data_collector()
with db.atomic():
Crypto.insert_many(coin_prices).execute()
print(coin_prices)
def import_db():
with open('./search_dataset.csv') as f:
reader = csv.reader(f)
with db.atomic():
for row in reader:
Products.create(orig_id=row[0], product=row[1], brand=row[2])
