def validate(self):
from docassemble.webapp.daredis import r
from docassemble.base.logger import logmessage
from flask import request, abort
result = True
key = 'da:failedlogin:ip:' + str(request.remote_addr)
failed_attempts = r.get(key)
if failed_attempts is not None and int(failed_attempts) > daconfig['attempt limit']:
abort(404)
verification_key = 'da:phonelogin:'******':code'
verification_code = r.get(verification_key)
#r.delete(verification_key)
supplied_verification_code = re.sub(r'[^0-9]', '', self.verification_code.data)
logmessage("Supplied code is " + str(supplied_verification_code))
if verification_code is None:
logmessage("Verification code with " + str(verification_key) + " is None")
result = False
elif verification_code.decode() != supplied_verification_code:
logmessage("Verification code with " + str(verification_key) + " which is " + str(verification_code.decode()) + " does not match supplied code, which is " + str(self.verification_code.data))
result = False
else:
logmessage("Code matched")
if result is False:
logmessage("Problem with form")
r.incr(key)
r.expire(key, 86400)
elif failed_attempts is not None:
r.delete(key)
return result
def amount(self, period_to_use=1):
"""Returns the periodic value's amount for a full period,
or 0 if the value does not exist."""
if not self.exists:
return 0
logmessage("period is a " + str(type(self.period).__name__))
return (Decimal(self.value) * Decimal(self.period)) / Decimal(period_to_use)
def finalize(self):
if not S3_ENABLED:
return
if not self.fixed:
raise DAError("SavedFile: finalize called before fix")
existing_files = list()
for filename in os.listdir(self.directory):
existing_files.append(filename)
fullpath = os.path.join(self.directory, filename)
#logmessage("Found " + fullpath)
if os.path.isfile(fullpath):
save = True
if filename in self.keydict:
key = self.keydict[filename]
if self.modtimes[filename] == os.path.getmtime(fullpath):
save = False
else:
key = s3.new_key()
key.key = str(self.section) + '/' + str(self.file_number) + '/' + str(filename)
if filename == self.filename:
extension, mimetype = get_ext_and_mimetype(filename + '.' + self.extension)
key.content_type = mimetype
if save:
key.set_contents_from_filename(fullpath)
for filename, key in self.keydict.iteritems():
if filename not in existing_files:
logmessage("Deleting filename " + str(filename) + " from S3")
key.delete()
return
def edit_user_profile_page(id):
user = User.query.filter_by(id=id).first()
if user is None:
abort(404)
the_role_id = None
for role in user.roles:
the_role_id = role.id
form = EditUserProfileForm(request.form, user, role_id=the_role_id)
form.role_id.choices = [(r.id, r.name) for r in Role.query.order_by('name')]
logmessage("Setting default to " + str(the_role_id))
if request.method == 'POST' and form.validate():
form.populate_obj(user)
roles_to_remove = list()
for role in user.roles:
roles_to_remove.append(role)
for role in roles_to_remove:
user.roles.remove(role)
for role in Role.query.order_by('id'):
if role.id == form.role_id.data:
user.roles.append(role)
break
db.session.commit()
flash(word('The information was saved.'), 'success')
return redirect(url_for('user_list'))
return render_template('users/edit_user_profile_page.html', form=form)
def add_info_about_file(filename, basename, result):
if result['extension'] == 'pdf':
try:
reader = PyPDF2.PdfFileReader(open(filename, 'rb'))
result['pages'] = reader.getNumPages()
except:
result['pages'] = 1
elif os.path.isfile(basename + '.pdf'):
try:
reader = PyPDF2.PdfFileReader(open(basename + '.pdf', 'rb'))
result['pages'] = reader.getNumPages()
except:
result['pages'] = 1
elif result['extension'] in ['png', 'jpg', 'gif']:
im = Image.open(filename)
result['width'], result['height'] = im.size
elif result['extension'] == 'svg':
try:
tree = ET.parse(filename)
root = tree.getroot()
viewBox = root.attrib.get('viewBox', None)
if viewBox is not None:
dimen = viewBox.split(' ')
if len(dimen) == 4:
result['width'] = float(dimen[2]) - float(dimen[0])
result['height'] = float(dimen[3]) - float(dimen[1])
except:
raise Exception("problem reading " + str(filename))
logmessage('add_info_about_file: could not read ' + str(filename))
return
def word_to_pdf(in_file, in_format, out_file, pdfa=False, password=None, update_references=False):
tempdir = tempfile.mkdtemp()
from_file = os.path.join(tempdir, "file." + in_format)
to_file = os.path.join(tempdir, "file.pdf")
shutil.copyfile(in_file, from_file)
tries = 0
while tries < 5:
if update_references:
subprocess_arguments = [LIBREOFFICE_PATH, '--headless', '--invisible', 'macro:///Standard.Module1.PysIndexerPdf(' + from_file + ',' + to_file + ')']
else:
subprocess_arguments = [LIBREOFFICE_PATH, '--headless', '--convert-to', 'pdf', from_file]
p = subprocess.Popen(subprocess_arguments, cwd=tempdir)
result = p.wait()
if os.path.isfile(to_file):
break
result = 1
tries += 1
time.sleep(2 + tries*random.random())
logmessage("Retrying libreoffice with " + repr(subprocess_arguments))
continue
if result == 0:
if pdfa:
pdf_to_pdfa(to_file)
if password:
pdf_encrypt(to_file, password)
shutil.copyfile(to_file, out_file)
if tempdir is not None:
shutil.rmtree(tempdir)
if result != 0:
return False
return True
def read_fields(pdffile):
outfields = list()
fp = open(pdffile, 'rb')
id_to_page = dict()
parser = PDFParser(fp)
doc = PDFDocument(parser)
pageno = 1;
for page in PDFPage.create_pages(doc):
id_to_page[page.pageid] = pageno
pageno += 1
fields = resolve1(doc.catalog['AcroForm'])['Fields']
for i in fields:
field = resolve1(i)
name, value, rect, page, field_type = field.get('T'), field.get('V'), field.get('Rect'), field.get('P'), field.get('FT')
logmessage("name is " + str(name) + " and FT is |" + str(field_type) + "|")
if page is not None:
pageno = id_to_page[page.objid]
else:
pageno = 1
if str(field_type) == '/Btn':
if value == '/Yes':
default = "Yes"
else:
default = "No"
elif str(field_type) == '/Sig':
default = '${ user.signature }'
else:
if value is not None:
default = value
else:
default = word("something")
outfields.append((name, default, pageno, rect, field_type))
return outfields
def get_info_from_file_number(file_number, privileged=False, filename=None):
if current_user and current_user.is_authenticated and current_user.has_role('admin', 'developer', 'advocate', 'trainer'):
privileged = True
else:
if has_request_context() and 'uid' in session:
uid = session['uid']
else:
uid = docassemble.base.functions.get_uid()
#logmessage("get_info_from_file_number: privileged is " + str(privileged) + " and uid is " + str(uid))
result = dict()
if privileged:
upload = Uploads.query.filter_by(indexno=file_number).first()
else:
upload = Uploads.query.filter(and_(Uploads.indexno == file_number, or_(Uploads.key == uid, Uploads.private == False))).first()
if upload:
if filename is None:
result['filename'] = upload.filename
else:
result['filename'] = filename
result['extension'], result['mimetype'] = get_ext_and_mimetype(result['filename'])
sf = SavedFile(file_number, extension=result['extension'], fix=True)
result['path'] = sf.path
result['fullpath'] = result['path'] + '.' + result['extension']
result['private'] = upload.private
result['persistent'] = upload.persistent
#logmessage("fullpath is " + str(result['fullpath']))
if 'path' not in result:
logmessage("get_info_from_file_number: path is not in result for " + str(file_number))
return result
final_filename = result['path'] + '.' + result['extension']
if os.path.isfile(final_filename):
add_info_about_file(final_filename, result)
# else:
# logmessage("Filename " + final_filename + "did not exist.")
return(result)
def set_instance_name(self, thename):
if not self.has_nonrandom_instance_name:
self.instanceName = thename
self.has_nonrandom_instance_name = True
else:
logmessage("Not resetting name of " + self.instanceName)
return
def publish_package(pkgname, info, author_info, tz_name):
from flask_login import current_user
#raise Exception("email is " + repr(current_user.email) + " and pypi is " + repr(current_user.pypi_username))
directory = make_package_dir(pkgname, info, author_info, tz_name)
packagedir = os.path.join(directory, 'docassemble-' + str(pkgname))
output = "Publishing docassemble." + pkgname + " to PyPI . . .\n\n"
try:
output += subprocess.check_output(['python', 'setup.py', 'sdist'], cwd=packagedir, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as err:
output += err.output
dist_file = None
dist_dir = os.path.join(packagedir, 'dist')
if not os.path.isdir(dist_dir):
output += "dist directory " + str(dist_dir) + " did not exist after calling sdist"
else:
# for f in os.listdir(dist_dir):
# try:
# #output += str(['twine', 'register', '--repository', 'pypi', '--username', str(current_user.pypi_username), '--password', str(current_user.pypi_password), os.path.join('dist', f)]) + "\n"
# #raise Exception(repr(['twine', 'register', '--repository', 'pypi', '--username', str(current_user.pypi_username), '--password', str(current_user.pypi_password), os.path.join('dist', f)]))
# output += subprocess.check_output(['twine', 'register', '--repository', 'pypi', '--username', str(current_user.pypi_username), '--password', str(current_user.pypi_password), os.path.join('dist', f)], cwd=packagedir, stderr=subprocess.STDOUT)
# except subprocess.CalledProcessError as err:
# output += "Error calling twine register.\n"
# output += err.output
try:
#output += str(['twine', 'upload', '--repository', 'pypi', '--username', str(current_user.pypi_username), '--password', str(current_user.pypi_password), os.path.join('dist', '*')])
output += subprocess.check_output(['twine', 'upload', '--repository', 'pypi', '--username', str(current_user.pypi_username), '--password', str(current_user.pypi_password), os.path.join('dist', '*')], cwd=packagedir, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as err:
output += "Error calling twine upload.\n"
output += err.output
output = re.sub(r'\n', '<br>', output)
shutil.rmtree(directory)
logmessage(output)
return output
def set_instance_name(self, thename):
"""Sets the instanceName attribute, if it is not already set."""
if not self.has_nonrandom_instance_name:
self.instanceName = thename
self.has_nonrandom_instance_name = True
else:
logmessage("Not resetting name of " + self.instanceName)
return
def convert_length(length, unit):
value = pixels_in(length)
if unit in unit_multipliers:
size = float(value)/float(unit_multipliers[unit])
return(int(size))
else:
logmessage("Unit " + str(unit) + " is not a valid unit\n")
return(300)
def func(*args, **kwargs):
if this_thread.language in language_functions[term]:
return language_functions[term][this_thread.language](*args, **kwargs)
if '*' in language_functions[term]:
return language_functions[term]['*'](*args, **kwargs)
if 'en' in language_functions[term]:
logmessage("Term " + str(term) + " is not defined for language " + str(this_thread.language))
return language_functions[term]['en'](*args, **kwargs)
raise SystemError("term " + str(term) + " not defined in language_functions for English or *")
def fill_template(template, data_strings=[], data_names=[], hidden=[], readonly=[], images=[], pdf_url=''):
fdf = fdfgen.forge_fdf(pdf_url, data_strings, data_names, hidden, readonly)
fdf_file = tempfile.NamedTemporaryFile(mode="wb", suffix=".fdf", delete=False)
fdf_file.write(fdf)
fdf_file.close()
pdf_file = tempfile.NamedTemporaryFile(mode="wb", suffix=".pdf", delete=False)
subprocess_arguments = [PDFTK_PATH, template, 'fill_form', fdf_file.name,'output', pdf_file.name, 'flatten']
result = call(subprocess_arguments)
if result != 0:
logmessage("Failed to fill PDF form " + str(template))
raise DAError("Call to pdftk failed for template " + str(template) + " where arguments were " + " ".join(subprocess_arguments))
if len(images):
fields = dict()
for field, default, pageno, rect, field_type in read_fields(template):
if str(field_type) == '/Sig':
fields[field] = {'pageno': pageno, 'rect': rect}
for field, file_info in images:
if field not in fields:
logmessage("field name " + str(field) + " not found in PDF file")
continue
logmessage("Need to put image on page " + str(fields[field]['pageno']))
temp_png = tempfile.NamedTemporaryFile(mode="wb", suffix=".png")
args = ["convert", file_info['fullpath'], "-trim", "+repage", temp_png.name]
result = call(args)
if result == 1:
logmessage("failed to trim file: " + " ".join(args))
continue
im = Image.open(temp_png.name)
width, height = im.size
xone, yone, xtwo, ytwo = fields[field]['rect']
dppx = width/(xtwo-xone)
dppy = height/(ytwo-yone)
if (dppx > dppy):
dpp = dppx
else:
dpp = dppy
extent_x, extent_y = xone*dpp+width, yone*dpp+height
overlay_pdf_file = tempfile.NamedTemporaryFile(mode="wb", suffix=".pdf")
args = ["convert", temp_png.name, "-background", "none", "-density", str(int(dpp*72)), "-gravity", "NorthEast", "-extent", str(int(extent_x)) + 'x' + str(int(extent_y)), overlay_pdf_file.name]
result = call(args)
if result == 1:
logmessage("failed to make overlay: " + " ".join(args))
continue
new_pdf_file = tempfile.NamedTemporaryFile(mode="wb", suffix=".pdf")
with open(pdf_file.name, "rb") as inFile, open(overlay_pdf_file.name, "rb") as overlay:
original = pypdf.PdfFileReader(inFile)
background = original.getPage(fields[field]['pageno']-1)
foreground = pypdf.PdfFileReader(overlay).getPage(0)
background.mergePage(foreground)
writer = pypdf.PdfFileWriter()
for i in range(original.getNumPages()):
page = original.getPage(i)
writer.addPage(page)
with open(new_pdf_file.name, "wb") as outFile:
writer.write(outFile)
shutil.copyfile(new_pdf_file.name, pdf_file.name)
return pdf_file.name
def will_handle(self, problem=None, county=None):
logmessage("Testing " + str(problem) + " against " + str(self.handles))
if problem:
if not (hasattr(self, 'handles') and problem in self.handles):
return False
if county:
if not (hasattr(self, 'serves') and county in self.serves):
return False
return True
def cloud_custom(provider, config):
config_id = str(provider) + str(config)
if config_id in cloud_cache:
return cloud_cache[config_id]
the_config = daconfig.get(config, None)
if the_config is None or type(the_config) is not dict:
logmessage("cloud_custom: invalid cloud configuration")
return None
cloud_cache[config_id] = docassemble.webapp.cloud.get_custom_cloud(provider, the_config)
return cloud_cache[config_id]
def invite():
""" Allows users to send invitations to register an account """
user_manager = current_app.user_manager
next = request.args.get('next',
_endpoint_url(user_manager.after_invite_endpoint))
user_role = Role.query.filter_by(name='user').first()
invite_form = MyInviteForm(request.form)
invite_form.role_id.choices = [(text_type(r.id), text_type(r.name)) for r in db.session.query(Role).filter(and_(Role.name != 'cron', Role.name != 'admin')).order_by('name')]
invite_form.role_id.default = text_type(user_role.id)
if text_type(invite_form.role_id.data) == 'None':
invite_form.role_id.data = text_type(user_role.id)
if request.method=='POST' and invite_form.validate():
email = invite_form.email.data
the_role_id = None
for role in Role.query.order_by('id'):
if role.id == int(invite_form.role_id.data) and role.name != 'admin' and role.name != 'cron':
the_role_id = role.id
if the_role_id is None:
the_role_id = user_role.id
user, user_email = user_manager.find_user_by_email(email)
if user:
flash(word("A user with that e-mail has already registered"), "error")
return redirect(url_for('invite'))
else:
user_invite = MyUserInvitation(email=email, role_id=the_role_id, invited_by_user_id=current_user.id)
db.session.add(user_invite)
db.session.commit()
token = user_manager.generate_token(user_invite.id)
accept_invite_link = url_for('user.register',
token=token,
_external=True)
user_invite.token = token
db.session.commit()
#docassemble.webapp.daredis.clear_user_cache()
try:
logmessage("Trying to send e-mail to " + text_type(user_invite.email))
emails.send_invite_email(user_invite, accept_invite_link)
except Exception as e:
logmessage("Failed to send e-mail")
db.session.delete(user_invite)
db.session.commit()
#docassemble.webapp.daredis.clear_user_cache()
flash(word('Unable to send e-mail. Error was: ') + text_type(e), 'error')
return redirect(url_for('invite'))
flash(word('Invitation has been sent.'), 'success')
return redirect(next)
return render_template('flask_user/invite.html', version_warning=None, bodyclass='adminbody', page_title=word('Invite User'), tab_title=word('Invite User'), form=invite_form)
def a_preposition_b_default(a, b, **kwargs):
logmessage("Got here")
if hasattr(a, 'preposition'):
logmessage("Has preposition")
preposition = a.preposition
else:
preposition = word('in the')
if 'capitalize' in kwargs and kwargs['capitalize']:
return capitalize(unicode(a)) + unicode(' ' + preposition + ' ') + unicode(b)
else:
return unicode(a) + unicode(' ' + preposition + ' ') + unicode(b)
def will_handle(self, problem=None, county=None):
"""Returns True or False depending on whether the organization
serves the given county and/or handles the given problem."""
logmessage("Testing " + str(problem) + " against " + str(self.handles))
if problem:
if not (hasattr(self, 'handles') and problem in self.handles):
return False
if county:
if not (hasattr(self, 'serves') and county in self.serves):
return False
return True
def new_paragraph(self):
if self.still_new:
logmessage("new_paragraph is still new and style is " + self.style + " and indentation is " + text_type(self.indentation))
self.current_paragraph['params']['style'] = self.style
self.current_paragraph['params']['indentation'] = self.indentation
return
logmessage("new_paragraph where style is " + self.style + " and indentation is " + text_type(self.indentation))
self.current_paragraph = dict(params=dict(style=self.style, indentation=self.indentation), runs=[RichText('')])
self.paragraphs.append(self.current_paragraph)
self.run = self.current_paragraph['runs'][-1]
self.still_new = True
def flatten_pdf(filename):
#logmessage("flatten_pdf: running")
outfile = tempfile.NamedTemporaryFile(prefix="datemp", suffix=".pdf", delete=False)
subprocess_arguments = [PDFTK_PATH, filename, 'output', outfile.name, 'flatten']
#logmessage("Arguments are " + str(subprocess_arguments))
result = call(subprocess_arguments)
if result != 0:
logmessage("Failed to flatten PDF form " + str(template))
raise DAError("Call to pdftk failed for template " + str(template) + " where arguments were " + " ".join(subprocess_arguments))
commands = []
shutil.move(outfile.name, filename)
def pixels_in(length):
m = re.search(r"([0-9.]+) *([a-z]+)", str(length).lower())
if m:
value = float(m.group(1))
unit = m.group(2)
#logmessage("value is " + str(value) + " and unit is " + unit + "\n")
if unit in unit_multipliers:
size = float(unit_multipliers[unit]) * value
#logmessage("size is " + str(size) + "\n")
return(int(size))
logmessage("Could not read " + str(length) + "\n")
return(300)
def pdf_to_pdfa(filename):
logmessage("pdf_to_pdfa: running")
outfile = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False)
directory = tempfile.mkdtemp()
commands = ['gs', '-dPDFA', '-dBATCH', '-dNOPAUSE', '-sProcessColorModel=DeviceCMYK', '-sDEVICE=pdfwrite', '-sPDFACompatibilityPolicy=1', '-sOutputFile=' + outfile.name, filename]
try:
output = subprocess.check_output(commands, cwd=directory, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as err:
output = err.output
raise DAError("pdf_to_pdfa: error running ghostscript. " + output)
logmessage(output)
shutil.move(outfile.name, filename)
def get_installed_distributions():
from docassemble.webapp.config import daconfig
PACKAGE_DIRECTORY = daconfig.get('packages', '/usr/share/docassemble/local')
results = list()
output, err = subprocess.Popen([daconfig.get('pip', os.path.join(PACKAGE_DIRECTORY, 'bin', 'pip')), 'freeze'], stdin=subprocess.PIPE, stdout=subprocess.PIPE).communicate()
for line in output.split('\n'):
a = line.split("==")
if len(a) == 2:
results.append(Object(key=a[0].lower(), version=a[1]))
else:
logmessage("Did not understand line: " + str(line))
return results
def fix_ml_files(playground_number):
playground = SavedFile(playground_number, section='playgroundsources', fix=False)
changed = False
for filename in playground.list_of_files():
if re.match(r'^ml-.*\.json', filename):
playground.fix()
try:
if write_ml_source(playground, playground_number, filename, finalize=False):
changed = True
except:
logmessage("Error writing machine learning source file " + str(filename))
if changed:
playground.finalize()
def save_for_classification(self, indep, key=None, info=None):
self._initialize()
if key is None:
existing_entry = MachineLearning.query.filter_by(group_id=self.group_id, dependent=None, independent=codecs.encode(pickle.dumps(indep), 'base64').decode()).first()
else:
existing_entry = MachineLearning.query.filter_by(group_id=self.group_id, key=key, independent=codecs.encode(pickle.dumps(indep), 'base64').decode()).first()
if existing_entry is not None:
logmessage("entry is already there")
return existing_entry.id
new_entry = MachineLearning(group_id=self.group_id, independent=codecs.encode(pickle.dumps(indep), 'base64').decode(), create_time=datetime.datetime.utcnow(), active=False, key=key, info=codecs.encode(pickle.dumps(info), 'base64').decode() if info is not None else None)
db.session.add(new_entry)
db.session.commit()
return new_entry.id
def install_package(package):
if package.type == 'zip' and package.upload is None:
return 0, ''
logmessage('install_package: ' + package.name)
logfilecontents = ''
pip.utils.logging._log_state = threading.local()
pip.utils.logging._log_state.indentation = 0
pip_log = tempfile.NamedTemporaryFile()
if package.type == 'zip' and package.upload is not None:
saved_file = SavedFile(package.upload, extension='zip', fix=True)
commands = ['install', '--egg', '--no-index', '--src=' + tempfile.mkdtemp(), '--log-file=' + pip_log.name, '--upgrade', saved_file.path + '.zip']
elif package.type == 'git' and package.giturl is not None:
commands = ['install', '--egg', '--src=' + tempfile.mkdtemp(), '--upgrade', '--log-file=' + pip_log.name, 'git+' + package.giturl + '.git#egg=' + package.name]
elif package.type == 'pip':
if package.limitation is None:
limit = ""
else:
limit = str(package.limitation)
commands = ['install', '--src=' + tempfile.mkdtemp(), '--upgrade', '--log-file=' + pip_log.name, package.name + limit]
else:
return 1, 'Unable to recognize package type: ' + package.name
logmessage("Running pip " + " ".join(commands))
logfilecontents += "Running pip " + " ".join(commands) + "\n"
returnval = pip.main(commands)
with open(pip_log.name) as x:
logfilecontents += x.read()
logmessage(logfilecontents)
logmessage('install_package: done')
return returnval, logfilecontents
def _set_to_current(self):
logmessage("set to current")
if 'user' in this_thread.current_info and 'location' in this_thread.current_info['user'] and type(this_thread.current_info['user']['location']) is dict:
if 'latitude' in this_thread.current_info['user']['location'] and 'longitude' in this_thread.current_info['user']['location']:
self.latitude = this_thread.current_info['user']['location']['latitude']
self.longitude = this_thread.current_info['user']['location']['longitude']
self.known = True
#logmessage("known is true")
elif 'error' in this_thread.current_info['user']['location']:
self.error = this_thread.current_info['user']['location']['error']
self.known = False
#logmessage("known is false")
self.gathered = True
self.description = str(self)
return
def qr_include_string(match):
string = match.group(1)
try:
width = match.group(2)
width = re.sub(r'^(.*)px', convert_pixels, width)
if width == "full":
width = '\\textwidth'
except:
width = DEFAULT_IMAGE_WIDTH
im = qrcode.make(string)
the_image = tempfile.NamedTemporaryFile(suffix=".png", delete=False)
im.save(the_image.name)
output = '\\mbox{\\includegraphics[width=' + width + ']{' + the_image.name + '}}'
if width == '\\textwidth':
output = '\\clearpage ' + output + '\\clearpage '
logmessage("Output is " + output)
return(output)
def fill_template(template, data_strings=[], data_names=[], hidden=[], readonly=[], pdf_url=""):
fdf = fdfgen.forge_fdf(pdf_url, data_strings, data_names, hidden, readonly)
fdf_file = tempfile.NamedTemporaryFile(mode="wb", suffix=".fdf", delete=False)
fdf_file.write(fdf)
fdf_file.close()
pdf_file = tempfile.NamedTemporaryFile(mode="wb", suffix=".pdf", delete=False)
subprocess_arguments = [PDFTK_PATH, template, "fill_form", fdf_file.name, "output", pdf_file.name, "flatten"]
result = call(subprocess_arguments)
if result != 0:
logmessage("Failed to fill PDF form " + str(template))
raise DAError(
"Call to pdftk failed for template "
+ str(template)
+ " where arguments were "
+ " ".join(subprocess_arguments)
)
return pdf_file.name
def fix_names():
#from docassemble.webapp.app_object import app
from docassemble.webapp.db_object import db
from docassemble.webapp.packages.models import Package
from sqlalchemy import select
installed_packages = [
package.key for package in get_installed_distributions()
]
for package in db.session.execute(
select(Package).filter_by(
active=True).with_for_update()).scalars():
if package.name not in installed_packages:
pip_info = get_pip_info(package.name)
actual_name = pip_info['Name']
if actual_name is not None:
package.name = actual_name
else:
logmessage("fix_names: package " + package.name +
" does not appear to be installed")
db.session.commit()
def fix_ml_files(playground_number, current_project):
playground = SavedFile(playground_number,
section='playgroundsources',
fix=False)
changed = False
for filename in playground.list_of_files():
if re.match(r'^ml-.*\.json', filename):
playground.fix()
try:
if write_ml_source(playground,
playground_number,
current_project,
filename,
finalize=False):
changed = True
except:
logmessage("Error writing machine learning source file " +
str(filename))
if changed:
playground.finalize()
def update_references(filename):
initialize_libreoffice()
subprocess_arguments = [LIBREOFFICE_PATH, '--headless', '--invisible', 'macro:///Standard.Module1.PysIndexer(' + filename + ')']
tries = 0
while tries < 5:
docassemble.base.functions.server.applock('obtain', 'libreoffice')
try:
result = subprocess.run(subprocess_arguments, cwd=tempfile.gettempdir(), timeout=120).returncode
except subprocess.TimeoutExpired:
result = 1
tries = 5
docassemble.base.functions.server.applock('release', 'libreoffice')
if result == 0:
break
logmessage("update_references: call to LibreOffice returned non-zero response")
tries += 1
time.sleep(0.5 + tries*random.random())
if result != 0:
return False
return True
def concatenate_files(path_list, pdfa=False, password=None):
pdf_file = tempfile.NamedTemporaryFile(prefix="datemp", mode="wb", suffix=".pdf", delete=False)
subprocess_arguments = [PDFTK_PATH]
new_path_list = list()
for path in path_list:
mimetype, encoding = mimetypes.guess_type(path)
if mimetype.startswith('image'):
new_pdf_file = tempfile.NamedTemporaryFile(prefix="datemp", mode="wb", suffix=".pdf", delete=False)
args = ["convert", path, new_pdf_file.name]
result = call(args)
if result != 0:
logmessage("failed to convert image to PDF: " + " ".join(args))
continue
new_path_list.append(new_pdf_file.name)
elif mimetype in ('application/rtf', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'application/msword', 'application/vnd.oasis.opendocument.text'):
new_pdf_file = tempfile.NamedTemporaryFile(prefix="datemp", mode="wb", suffix=".pdf", delete=False)
if mimetype == 'application/rtf':
ext = 'rtf'
elif mimetype == 'application/vnd.openxmlformats-officedocument.wordprocessingml.document':
ext = 'docx'
elif mimetype == 'application/msword':
ext = 'doc'
elif mimetype == 'application/vnd.oasis.opendocument.text':
ext = 'odt'
word_to_pdf(path, ext, new_pdf_file.name, pdfa=False)
new_path_list.append(new_pdf_file.name)
elif mimetype == 'application/pdf':
new_path_list.append(path)
if len(new_path_list) == 0:
raise DAError("concatenate_files: no valid files to concatenate")
subprocess_arguments.extend(new_path_list)
subprocess_arguments.extend(['cat', 'output', pdf_file.name])
#logmessage("Arguments are " + str(subprocess_arguments))
result = call(subprocess_arguments)
if result != 0:
logmessage("Failed to concatenate PDF files")
raise DAError("Call to pdftk failed for concatenation where arguments were " + " ".join(subprocess_arguments))
if pdfa:
pdf_to_pdfa(pdf_file.name)
replicate_js_and_calculations(new_path_list[0], pdf_file.name, password)
return pdf_file.name
def get_info_from_file_number(file_number,
privileged=False,
filename=None,
uids=None):
if current_user and current_user.is_authenticated and current_user.has_role(
'admin', 'developer', 'advocate', 'trainer'):
privileged = True
elif uids is None:
try:
uids = [
docassemble.base.functions.this_thread.current_info['session']
]
except:
uids = []
result = dict()
upload = Uploads.query.filter_by(indexno=file_number).first()
if not privileged and upload is not None and upload.private and upload.key not in uids:
upload = None
if upload:
if filename is None:
result['filename'] = upload.filename
else:
result['filename'] = filename
result['extension'], result['mimetype'] = get_ext_and_mimetype(
result['filename'])
sf = SavedFile(file_number, extension=result['extension'], fix=True)
result['path'] = sf.path
result['fullpath'] = result['path'] + '.' + result['extension']
result['private'] = upload.private
result['persistent'] = upload.persistent
#logmessage("fullpath is " + str(result['fullpath']))
if 'path' not in result:
logmessage("get_info_from_file_number: path is not in result for " +
str(file_number))
return result
final_filename = result['path'] + '.' + result['extension']
if os.path.isfile(final_filename):
add_info_about_file(final_filename, result['path'], result)
# else:
# logmessage("Filename " + final_filename + "did not exist.")
return (result)
def add_info_about_file(filename, result):
if result['extension'] == 'pdf':
reader = pyPdf.PdfFileReader(open(filename))
result['pages'] = reader.getNumPages()
elif result['extension'] in ['png', 'jpg', 'gif']:
im = Image.open(filename)
result['width'], result['height'] = im.size
elif result['extension'] == 'svg':
try:
tree = ET.parse(filename)
root = tree.getroot()
viewBox = root.attrib.get('viewBox', None)
if viewBox is not None:
dimen = viewBox.split(' ')
if len(dimen) == 4:
result['width'] = float(dimen[2]) - float(dimen[0])
result['height'] = float(dimen[3]) - float(dimen[1])
except:
raise Exception("problem reading " + str(filename))
logmessage('add_info_about_file: could not read ' + str(filename))
return
def flatten_pdf(filename):
#logmessage("flatten_pdf: running")
outfile = tempfile.NamedTemporaryFile(prefix="datemp",
suffix=".pdf",
delete=False)
subprocess_arguments = [
PDFTK_PATH, filename, 'output', outfile.name, 'flatten'
]
#logmessage("Arguments are " + str(subprocess_arguments))
try:
result = subprocess.run(subprocess_arguments, timeout=60,
check=False).returncode
except subprocess.TimeoutExpired:
result = 1
logmessage("flatten_pdf: call to pdftk took too long")
if result != 0:
logmessage("Failed to flatten PDF form")
raise DAError(
"Call to pdftk failed for template where arguments were " +
" ".join(subprocess_arguments))
shutil.move(outfile.name, filename)
def send(self, message, envelope_from=None):
assert message.send_to, "No recipients have been added"
assert message.sender, (
"The message does not specify a sender and a default sender "
"has not been configured")
if message.has_bad_headers():
raise BadHeaderError
if message.date is None:
message.date = time.time()
data = {'to': ', '.join(list(sanitize_addresses(message.send_to)))}
if hasattr(message, 'mailgun_variables') and isinstance(
message.mailgun_variables, dict):
for key, val in message.mailgun_variables.items():
data['v:' + str(key)] = val
response = requests.post(
self.mail.api_url,
auth=HTTPBasicAuth('api', self.mail.api_key),
data=data,
files={'message': ('mime_message', message.as_string())})
if response.status_code >= 400:
logmessage("Mailgun status code: " + str(response.status_code))
logmessage("Mailgun response headers: " + repr(response.headers))
try:
logmessage(repr(response.body))
except:
pass
raise Exception("Failed to send e-mail message to " +
self.mail.api_url)
email_dispatched.send(message, app=current_app._get_current_object())
def get_pip_info(package_name, start_time=None):
if start_time is None:
start_time = time.time()
logmessage("get_pip_info: " + package_name + " after " +
str(time.time() - start_time) + " seconds")
try:
output = subprocess.check_output(['pip', 'show', package_name
]).decode('utf-8', 'ignore')
except subprocess.CalledProcessError as err:
output = ""
logmessage("get_pip_info: error. output was " +
err.output.decode('utf-8', 'ignore') + " after " +
str(time.time() - start_time) + " seconds")
# old_stdout = sys.stdout
# sys.stdout = saved_stdout = StringIO()
# pip.main(['show', package_name])
# sys.stdout = old_stdout
# output = saved_stdout.getvalue()
results = {}
if not isinstance(output, str):
output = output.decode('utf-8', 'ignore')
for line in output.split('\n'):
#logmessage("Found line " + str(line))
a = line.split(": ")
if len(a) == 2:
#logmessage("Found " + a[0] + " which was " + a[1])
results[a[0]] = a[1]
for key in ['Name', 'Home-page', 'Version']:
if key not in results:
results[key] = None
logmessage("get_pip_info: returning after " +
str(time.time() - start_time) + " seconds")
return results
def get_info_from_file_number(file_number, privileged=False, filename=None):
if current_user and current_user.is_authenticated and current_user.has_role(
'admin', 'developer', 'advocate', 'trainer'):
privileged = True
else:
if has_request_context() and 'uid' in session:
uid = session['uid']
else:
uid = docassemble.base.functions.get_uid()
#logmessage("get_info_from_file_number: privileged is " + str(privileged) + " and uid is " + str(uid))
result = dict()
if privileged:
upload = Uploads.query.filter_by(indexno=file_number).first()
else:
upload = Uploads.query.filter(
and_(Uploads.indexno == file_number,
or_(Uploads.key == uid, Uploads.private == False))).first()
if upload:
if filename is None:
result['filename'] = upload.filename
else:
result['filename'] = filename
result['extension'], result['mimetype'] = get_ext_and_mimetype(
result['filename'])
sf = SavedFile(file_number, extension=result['extension'], fix=True)
result['path'] = sf.path
result['fullpath'] = result['path'] + '.' + result['extension']
result['private'] = upload.private
result['persistent'] = upload.persistent
#logmessage("fullpath is " + str(result['fullpath']))
if 'path' not in result:
logmessage("get_info_from_file_number: path is not in result for " +
str(file_number))
return result
final_filename = result['path'] + '.' + result['extension']
if os.path.isfile(final_filename):
add_info_about_file(final_filename, result['path'], result)
# else:
# logmessage("Filename " + final_filename + "did not exist.")
return (result)
def initialize_libreoffice():
global LIBREOFFICE_INITIALIZED
if LIBREOFFICE_INITIALIZED:
return
LIBREOFFICE_INITIALIZED = True
if not os.path.isfile(LIBREOFFICE_MACRO_PATH):
logmessage("No LibreOffice macro path exists")
temp_file = tempfile.NamedTemporaryFile(prefix="datemp",
mode="wb",
suffix=".pdf")
word_file = docassemble.base.functions.package_template_filename(
'docassemble.demo:data/templates/template_test.docx')
word_to_pdf(word_file,
'docx',
temp_file.name,
pdfa=False,
password=None)
del temp_file
del word_file
try:
assert os.path.isdir(os.path.dirname(LIBREOFFICE_MACRO_PATH))
orig_path = docassemble.base.functions.package_template_filename(
'docassemble.base:data/macros/Module1.xba')
logmessage("Copying LibreOffice macro from " + orig_path)
shutil.copyfile(orig_path, LIBREOFFICE_MACRO_PATH)
except:
logmessage("Could not copy LibreOffice macro into place")
def apply_qpdf(filename):
try:
pypdf.PdfFileReader(open(filename, 'rb'), overwriteWarnings=False)
pdf_ok = True
except pypdf.utils.PdfReadError:
pdf_ok = False
if pdf_ok:
return
try:
new_file = tempfile.NamedTemporaryFile(prefix="datemp",
mode="wb",
suffix=".pdf",
delete=False)
qpdf_subprocess_arguments = [QPDF_PATH, filename, new_file.name]
try:
result = subprocess.run(qpdf_subprocess_arguments,
timeout=60,
check=False).returncode
except subprocess.TimeoutExpired:
result = 1
logmessage("apply_qpdf: call to qpdf took too long")
if result != 0:
logmessage("Failed to convert PDF " + str(filename))
logmessage("Call to qpdf failed for " + str(filename) +
" where arguments were " +
" ".join(qpdf_subprocess_arguments))
raise Exception("qpdf error")
pypdf.PdfFileReader(open(new_file.name, 'rb'), overwriteWarnings=False)
except:
raise DAError("Could not fix PDF")
shutil.copyfile(new_file.name, filename)
def set_to_current(self):
logmessage("set to current")
if 'user' in this_thread.current_info and 'location' in this_thread.current_info[
'user'] and type(
this_thread.current_info['user']['location']) is dict:
if 'latitude' in this_thread.current_info['user'][
'location'] and 'longitude' in this_thread.current_info[
'user']['location']:
self.latitude = this_thread.current_info['user']['location'][
'latitude']
self.longitude = this_thread.current_info['user']['location'][
'longitude']
self.known = True
#logmessage("known is true")
elif 'error' in this_thread.current_info['user']['location']:
self.error = this_thread.current_info['user']['location'][
'error']
self.known = False
#logmessage("known is false")
self.gathered = True
self.description = str(self)
return
def convert_file(in_file, out_file, input_extension, output_extension):
initialize_libreoffice()
tempdir = tempfile.mkdtemp()
from_file = os.path.join(tempdir, "file." + input_extension)
to_file = os.path.join(tempdir, "file." + output_extension)
shutil.copyfile(in_file, from_file)
subprocess_arguments = [
LIBREOFFICE_PATH, '--headless', '--invisible', '--convert-to',
output_extension, from_file, '--outdir', tempdir
]
#logmessage("convert_to: creating " + to_file + " by doing " + " ".join(subprocess_arguments))
tries = 0
while tries < 5:
docassemble.base.functions.server.applock('obtain', 'libreoffice')
try:
result = subprocess.run(subprocess_arguments,
cwd=tempdir,
timeout=120,
check=False).returncode
except subprocess.TimeoutExpired:
logmessage("convert_file: libreoffice took too long")
result = 1
tries = 5
docassemble.base.functions.server.applock('release', 'libreoffice')
if result != 0:
logmessage(
"convert_file: call to LibreOffice returned non-zero response")
if result == 0 and os.path.isfile(to_file):
break
result = 1
tries += 1
time.sleep(0.5 + tries * random.random())
if result == 0:
shutil.copyfile(to_file, out_file)
if tempdir is not None:
shutil.rmtree(tempdir)
if result != 0:
return False
return True
def publish_package(pkgname, info, author_info):
directory = make_package_dir(pkgname, info, author_info)
packagedir = os.path.join(directory, 'docassemble-' + str(pkgname))
output = "Publishing docassemble." + pkgname + " to PyPI . . .\n\n"
try:
output += subprocess.check_output(
['python', 'setup.py', 'register', '-r', 'pypi'],
cwd=packagedir,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as err:
output += err.output
try:
output += subprocess.check_output(
['python', 'setup.py', 'sdist', 'upload', '-r', 'pypi'],
cwd=packagedir,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as err:
output += err.output
output = re.sub(r'\n', '<br>', output)
shutil.rmtree(directory)
logmessage(output)
return output
def add_dependencies(user_id):
logmessage('add_dependencies: ' + str(user_id))
from docassemble.webapp.config import hostname
package_by_name = dict()
for package in Package.query.filter_by(active=True).all():
package_by_name[package.name] = package
installed_packages = get_installed_distributions()
for package in installed_packages:
if package.key in package_by_name:
continue
package_auth = PackageAuth(user_id=user_id)
if package.key in ['docassemble', 'docassemble.base', 'docassemble.webapp', 'docassemble.demo']:
package_entry = Package(name=package.key, package_auth=package_auth, giturl=docassemble_git_url, packageversion=package.version, gitsubdir=re.sub(r'\.', '-', package.key), type='git', core=True)
else:
package_entry = Package(name=package.key, package_auth=package_auth, type='pip', packageversion=package.version, dependency=True)
db.session.add(package_auth)
db.session.add(package_entry)
db.session.commit()
install = Install(hostname=hostname, packageversion=package_entry.packageversion, version=package_entry.version, package_id=package_entry.id)
db.session.add(install)
db.session.commit()
return
def validate(self):
#import redis
#import docassemble.base.util
from docassemble.webapp.daredis import r
from docassemble.base.logger import logmessage
from flask import request, abort
result = True
#r = redis.StrictRedis(host=docassemble.base.util.redis_server, db=0)
key = 'da:failedlogin:ip:' + str(request.remote_addr)
failed_attempts = r.get(key)
if failed_attempts is not None and int(
failed_attempts) > daconfig['attempt limit']:
abort(404)
verification_key = 'da:phonelogin:'******':code'
verification_code = r.get(verification_key)
#r.delete(verification_key)
supplied_verification_code = re.sub(r'[^0-9]', '',
self.verification_code.data)
logmessage("Supplied code is " + str(supplied_verification_code))
if verification_code is None:
logmessage("Verification code with " + str(verification_key) +
" is None")
result = False
elif verification_code != supplied_verification_code:
logmessage("Verification code with " + str(verification_key) +
" which is " + str(verification_code) +
" does not match supplied code, which is " +
str(self.verification_code.data))
result = False
else:
logmessage("Code matched")
if result is False:
logmessage("Problem with form")
r.incr(key)
r.expire(key, 86400)
elif failed_attempts is not None:
r.delete(key)
return result
def temp_url_for(self, **kwargs):
if kwargs.get('_attachment', False):
suffix = 'download'
else:
suffix = ''
filename = kwargs.get('filename', self.filename)
seconds = kwargs.get('seconds', None)
if isinstance(seconds, float):
seconds = int(seconds)
if not isinstance(seconds, int):
seconds = 30
if cloud is not None and daconfig.get('use cloud urls', False):
keyname = str(self.section) + '/' + str(
self.file_number) + '/' + path_to_key(filename)
key = cloud.get_key(keyname)
inline = not bool(kwargs.get('_attachment', False))
if key.does_exist:
return key.generate_url(
seconds,
display_filename=kwargs.get('display_filename', None),
inline=inline,
content_type=kwargs.get('content_type', None))
logmessage("key " + str(keyname) + " did not exist")
return 'about:blank'
r = docassemble.base.functions.server.server_redis
while True:
code = random_alphanumeric(32)
keyname = 'da:tempfile:' + code
if r.setnx(keyname,
str(self.section) + '^' + str(self.file_number)):
r.expire(keyname, seconds)
break
use_external = kwargs.get(
'_external',
bool('jsembed' in docassemble.base.functions.this_thread.misc))
url = url_for('rootindex', _external=use_external).rstrip('/')
url += '/tempfile' + suffix + '/' + code + '/' + path_to_key(
kwargs.get('display_filename', filename))
return url
def publish_package(pkgname, info, author_info, current_project='default'):
directory = make_package_dir(pkgname,
info,
author_info,
current_project=current_project)
packagedir = os.path.join(directory, 'docassemble-' + str(pkgname))
output = "Publishing docassemble." + pkgname + " to PyPI . . .\n\n"
try:
output += subprocess.check_output(['python', 'setup.py', 'sdist'],
cwd=packagedir,
stderr=subprocess.STDOUT).decode()
except subprocess.CalledProcessError as err:
output += err.output.decode()
dist_dir = os.path.join(packagedir, 'dist')
had_error = False
if not os.path.isdir(dist_dir):
output += "dist directory " + str(
dist_dir) + " did not exist after calling sdist"
had_error = True
else:
try:
output += subprocess.check_output(
[
'twine', 'upload', '--repository', 'pypi', '--username',
str(current_user.pypi_username), '--password',
str(current_user.pypi_password),
os.path.join('dist', '*')
],
cwd=packagedir,
stderr=subprocess.STDOUT).decode()
except subprocess.CalledProcessError as err:
output += "Error calling twine upload.\n"
output += err.output.decode()
had_error = True
output = re.sub(r'\n', '<br>', output)
shutil.rmtree(directory)
logmessage(output)
return (had_error, output)
def main():
container_role = ':' + os.environ.get('CONTAINERROLE', '') + ':'
errlog("checking to see if running create_tables if necessary")
if ':all:' in container_role or ':cron:' in container_role:
(redis_host, redis_port, redis_username, redis_password, redis_offset,
redis_cli, ssl_opts) = parse_redis_uri()
r = redis.StrictRedis(host=redis_host,
port=redis_port,
db=redis_offset,
password=redis_password,
username=redis_username,
**ssl_opts)
if r.get('da:skip_create_tables'):
logmessage("restart: skipping create_tables")
r.delete('da:skip_create_tables')
else:
errlog("running create_tables")
import docassemble.webapp.create_tables
docassemble.webapp.create_tables.main()
errlog("finished create_tables")
if ':cron:' in container_role:
r.delete('da:cron_restart')
webapp_path = daconfig.get(
'webapp', '/usr/share/docassemble/webapp/docassemble.wsgi')
cloud = get_cloud()
if cloud is not None:
key = cloud.get_key('config.yml')
if key.does_exist:
key.get_contents_to_filename(daconfig['config file'])
logmessage("Wrote config file based on copy on cloud")
wsgi_file = webapp_path
if os.path.isfile(wsgi_file):
errlog("touching wsgi file")
with open(wsgi_file, 'a', encoding='utf-8'):
os.utime(wsgi_file, None)
errlog("Restarted.")
sys.exit(0)
def update_versions():
logmessage("update_versions")
install_by_id = dict()
from docassemble.webapp.config import hostname
for install in Install.query.filter_by(hostname=hostname).all():
install_by_id[install.package_id] = install
package_by_name = dict()
for package in Package.query.filter_by(active=True).all():
package_by_name[package.name] = package
installed_packages = get_installed_distributions()
for package in installed_packages:
if package.key in package_by_name:
if package_by_name[
package.
key].id in install_by_id and package.version != install_by_id[
package_by_name[package.key].id].packageversion:
install_by_id[package_by_name[
package.key].id].packageversion = package.version
db.session.commit()
if package.version != package_by_name[package.key].packageversion:
package_by_name[package.key].packageversion = package.version
db.session.commit()
return
def markdown_to_docx(text, tpl):
if get_config('new markdown to docx', False):
source_code = docassemble.base.filter.markdown_to_html(text, do_terms=False)
source_code = re.sub("\n", ' ', source_code)
source_code = re.sub(">\s+<", '><', source_code)
soup = BeautifulSoup('<html>' + source_code + '</html>', 'html.parser')
parser = SoupParser(tpl)
for elem in soup.find_all(recursive=False):
parser.traverse(elem)
output = text_type(parser)
logmessage(output)
return output
else:
source_code = docassemble.base.filter.markdown_to_html(text, do_terms=False)
source_code = re.sub(r'(?<!\>)\n', ' ', source_code)
#source_code = re.sub("\n", ' ', source_code)
#source_code = re.sub(">\s+<", '><', source_code)
rt = RichText('')
soup = BeautifulSoup(source_code, 'lxml')
html_parsed = deque()
html_parsed = html_linear_parse(soup)
rt = add_to_rt(tpl, rt, html_parsed)
return rt
def safe_pypdf_reader(filename):
try:
return pypdf.PdfFileReader(open(filename, 'rb'),
overwriteWarnings=False)
except pypdf.utils.PdfReadError:
new_filename = tempfile.NamedTemporaryFile(prefix="datemp",
mode="wb",
suffix=".pdf",
delete=False)
qpdf_subprocess_arguments = [QPDF_PATH, filename, new_filename.name]
try:
result = subprocess.run(qpdf_subprocess_arguments,
timeout=60).returncode
except subprocess.TimeoutExpired:
result = 1
logmessage("fill_template: call to qpdf took too long")
if result != 0:
logmessage("Failed to convert PDF template " + str(filename))
raise DAError("Call to qpdf failed for template " + str(filename) +
" where arguments were " +
" ".join(qpdf_subprocess_arguments))
return pypdf.PdfFileReader(open(new_filename.name, 'rb'),
overwriteWarnings=False)
def upgrade_db(url, py_file, engine, name=None):
if name is None:
name = 'alembic'
packagedir = os.path.dirname(os.path.abspath(py_file))
alembic_path = os.path.join(packagedir, name)
if not os.path.isdir(alembic_path):
logmessage(name + " directory not found in package directory " +
packagedir)
return
ini_file = os.path.join(packagedir, name + '.ini')
if not os.path.isfile(ini_file):
logmessage(name + ".ini file not found at " + ini_file)
return
versions_path = os.path.join(alembic_path, 'versions')
if not os.path.isdir(versions_path):
os.makedirs(versions_path)
from alembic.config import Config
from alembic import command
alembic_cfg = Config(ini_file)
alembic_cfg.set_main_option("sqlalchemy.url", url)
alembic_cfg.set_main_option("script_location", alembic_path)
if not engine.has_table('alembic_version'):
command.stamp(alembic_cfg, "head")
command.upgrade(alembic_cfg, "head")
def publish_package(pkgname, info, author_info, tz_name, current_project='default'):
from flask_login import current_user
#raise Exception("email is " + repr(current_user.email) + " and pypi is " + repr(current_user.pypi_username))
directory = make_package_dir(pkgname, info, author_info, tz_name, current_project=current_project)
packagedir = os.path.join(directory, 'docassemble-' + str(pkgname))
output = "Publishing docassemble." + pkgname + " to PyPI . . .\n\n"
try:
output += subprocess.check_output(['python', 'setup.py', 'sdist'], cwd=packagedir, stderr=subprocess.STDOUT).decode()
except subprocess.CalledProcessError as err:
output += err.output.decode()
dist_file = None
dist_dir = os.path.join(packagedir, 'dist')
had_error = False
if not os.path.isdir(dist_dir):
output += "dist directory " + str(dist_dir) + " did not exist after calling sdist"
had_error = True
else:
# for f in os.listdir(dist_dir):
# try:
# #output += str(['twine', 'register', '--repository', 'pypi', '--username', str(current_user.pypi_username), '--password', str(current_user.pypi_password), os.path.join('dist', f)]) + "\n"
# #raise Exception(repr(['twine', 'register', '--repository', 'pypi', '--username', str(current_user.pypi_username), '--password', str(current_user.pypi_password), os.path.join('dist', f)]))
# output += subprocess.check_output(['twine', 'register', '--repository', 'pypi', '--username', str(current_user.pypi_username), '--password', str(current_user.pypi_password), os.path.join('dist', f)], cwd=packagedir, stderr=subprocess.STDOUT)
# except subprocess.CalledProcessError as err:
# output += "Error calling twine register.\n"
# output += err.output
try:
#output += str(['twine', 'upload', '--repository', 'pypi', '--username', str(current_user.pypi_username), '--password', str(current_user.pypi_password), os.path.join('dist', '*')])
output += subprocess.check_output(['twine', 'upload', '--repository', 'pypi', '--username', str(current_user.pypi_username), '--password', str(current_user.pypi_password), os.path.join('dist', '*')], cwd=packagedir, stderr=subprocess.STDOUT).decode()
except subprocess.CalledProcessError as err:
output += "Error calling twine upload.\n"
output += err.output.decode()
had_error = True
output = re.sub(r'\n', '<br>', output)
shutil.rmtree(directory)
logmessage(output)
return had_error, output
def __unicode__(self):
output = ''
list_number = 1
for para in self.paragraphs:
logmessage("Got a paragraph where style is " +
para['params']['style'] + " and indentation is " +
text_type(para['params']['indentation']))
output += '<w:p><w:pPr><w:pStyle w:val="Normal"/>'
if para['params']['style'] in ('ul', 'ol', 'blockquote'):
output += '<w:ind w:left="' + text_type(
36 * para['params']['indentation']
) + '" w:right="0" w:hanging="0"/>'
output += '<w:rPr></w:rPr></w:pPr>'
if para['params']['style'] == 'ul':
output += text_type(RichText("•\t"))
if para['params']['style'] == 'ol':
output += text_type(RichText(text_type(list_number) + ".\t"))
list_number += 1
else:
list_number = 1
for run in para['runs']:
output += text_type(run)
output += '</w:p>'
return output
def file_user_access(file_number, allow_user_id=None, allow_email=None, disallow_user_id=None, disallow_email=None, disallow_all=False):
something_added = False
if allow_user_id:
for user_id in set(allow_user_id):
existing_user = UserModel.query.filter_by(id=user_id).first()
if not existing_user:
logmessage("file_user_access: invalid user ID " + repr(user_id))
continue
if UploadsUserAuth.query.filter_by(uploads_indexno=file_number, user_id=user_id).first():
continue
new_auth_record = UploadsUserAuth(uploads_indexno=file_number, user_id=user_id)
db.session.add(new_auth_record)
something_added = True
if something_added:
db.session.commit()
something_added = False
if allow_email:
for email in set(allow_email):
existing_user = UserModel.query.filter_by(email=email).first()
if not existing_user:
logmessage("file_user_access: invalid email " + repr(email))
continue
if UploadsUserAuth.query.filter_by(uploads_indexno=file_number, user_id=existing_user.id).first():
continue
new_auth_record = UploadsUserAuth(uploads_indexno=file_number, user_id=existing_user.id)
db.session.add(new_auth_record)
something_added = True
if something_added:
db.session.commit()
if disallow_user_id:
for user_id in set(disallow_user_id):
UploadsUserAuth.query.filter_by(uploads_indexno=file_number, user_id=user_id).delete()
db.session.commit()
if disallow_email:
for email in set(disallow_email):
existing_user = UserModel.query.filter_by(email=email).first()
if not existing_user:
logmessage("file_user_access: invalid email " + repr(email))
continue
UploadsUserAuth.query.filter_by(uploads_indexno=file_number, user_id=existing_user.id).delete()
db.session.commit()
if disallow_all:
UploadsUserAuth.query.filter_by(uploads_indexno=file_number).delete()
if not (allow_user_id or allow_email or disallow_user_id or disallow_email or disallow_all):
result = dict(user_ids=list(), emails=list(), temp_user_ids=list())
for auth in db.session.query(UploadsUserAuth.user_id, UploadsUserAuth.temp_user_id, UserModel.email).outerjoin(UserModel, UploadsUserAuth.user_id == UserModel.id).filter(UploadsUserAuth.uploads_indexno == file_number).all():
if auth.user_id is not None:
result['user_ids'].append(auth.user_id)
if auth.temp_user_id is not None:
result['temp_user_ids'].append(auth.temp_user_id)
if auth.email:
result['emails'].append(auth.email)
return result
def install_package(package):
if package.type == 'zip' and package.upload is None:
return 0, ''
logmessage('install_package: ' + package.name)
logfilecontents = ''
pip.utils.logging._log_state = threading.local()
pip.utils.logging._log_state.indentation = 0
pip_log = tempfile.NamedTemporaryFile()
if package.type == 'zip' and package.upload is not None:
saved_file = SavedFile(package.upload, extension='zip', fix=True)
commands = [
'install', '--egg', '--no-index', '--src=' + tempfile.mkdtemp(),
'--log-file=' + pip_log.name, '--upgrade', saved_file.path + '.zip'
]
elif package.type == 'git' and package.giturl is not None:
commands = [
'install', '--egg', '--src=' + tempfile.mkdtemp(), '--upgrade',
'--log-file=' + pip_log.name,
'git+' + package.giturl + '.git#egg=' + package.name
]
elif package.type == 'pip':
if package.limitation is None:
limit = ""
else:
limit = str(package.limitation)
commands = [
'install', '--src=' + tempfile.mkdtemp(), '--upgrade',
'--log-file=' + pip_log.name, package.name + limit
]
else:
return 1, 'Unable to recognize package type: ' + package.name
logmessage("Running pip " + " ".join(commands))
logfilecontents += "Running pip " + " ".join(commands) + "\n"
returnval = pip.main(commands)
with open(pip_log.name) as x:
logfilecontents += x.read()
logmessage(logfilecontents)
logmessage('install_package: done')
return returnval, logfilecontents
def send(self, message, envelope_from=None):
assert message.send_to, "No recipients have been added"
assert message.sender, (
"The message does not specify a sender and a default sender "
"has not been configured")
if message.has_bad_headers():
raise BadHeaderError
if message.date is None:
message.date = time.time()
if not message.subject:
message.subject = word("(no subject)")
sgmessage = SGMail(
from_email=Email(message.sender),
to_emails=[
To(addressee)
for addressee in sanitize_addresses(message.recipients)
],
subject=message.subject,
plain_text_content=message.body,
html_content=message.html)
if message.reply_to:
sgmessage.reply_to = ReplyTo(message.reply_to)
if message.cc:
for recipient in list(sanitize_addresses(message.cc)):
sgmessage.add_cc(recipient)
if message.bcc:
for recipient in list(sanitize_addresses(message.bcc)):
sgmessage.add_bcc(recipient)
if message.attachments:
for flask_attachment in message.attachments:
attachment = Attachment()
attachment.file_content = FileContent(
base64.b64encode(flask_attachment.data).decode())
attachment.file_type = FileType(flask_attachment.content_type)
attachment.file_name = FileName(flask_attachment.filename)
attachment.disposition = Disposition(
flask_attachment.disposition)
sgmessage.add_attachment(attachment)
sg = SendGridAPIClient(self.mail.api_key)
response = sg.send(sgmessage)
if response.status_code >= 400:
logmessage("SendGrid status code: " + str(response.status_code))
logmessage("SendGrid response headers: " + repr(response.headers))
try:
logmessage(repr(response.body))
except:
pass
raise Exception("Failed to send e-mail message to SendGrid")
email_dispatched.send(message, app=current_app._get_current_object())
def geolocate(self):
if self.geolocated:
return self.geolocate_success
the_address = self.address_for_geolocation()
logmessage("Trying to geolocate " + str(the_address))
from pygeocoder import Geocoder
google_config = get_config('google')
if google_config and 'api key' in google_config:
my_geocoder = Geocoder(api_key=google_config['api key'])
else:
my_geocoder = Geocoder()
results = my_geocoder.geocode(the_address)
self.geolocated = True
if len(results):
self.geolocate_success = True
self.location.gathered = True
self.location.known = True
self.location.latitude = results[0].coordinates[0]
self.location.longitude = results[0].coordinates[1]
self.location.description = self.block()
self.geolocate_response = results.raw
geo_types = {
'administrative_area_level_2': 'county',
'neighborhood': 'neighborhood',
'postal_code': 'zip',
'country': 'country'
}
for geo_type, addr_type in geo_types.iteritems():
if hasattr(results[0],
geo_type) and not hasattr(self, addr_type):
logmessage("Setting " + str(addr_type) + " to " +
str(getattr(results[0], geo_type)) + " from " +
str(geo_type))
setattr(self, addr_type, getattr(results[0], geo_type))
#else:
#logmessage("Not setting " + addr_type + " from " + geo_type)
#logmessage(json.dumps(self.geolocate_response))
else:
logmessage("valid not ok: result count was " + str(len(results)))
self.geolocate_success = False
return self.geolocate_success