def add_dependencies(user_id):
#sys.stderr.write('add_dependencies: user_id is ' + str(user_id) + "\n")
sys.stderr.write("add_dependencies: starting\n")
from docassemble.base.config import hostname, daconfig
from docassemble.webapp.app_object import app
from docassemble.webapp.db_object import db
from docassemble.webapp.packages.models import Package, Install, PackageAuth
#docassemble_git_url = daconfig.get('docassemble git url', 'https://github.com/jhpyle/docassemble')
package_by_name = dict()
for package in Package.query.filter_by(active=True).order_by(
Package.name, Package.id.desc()).all():
if package.name in package_by_name:
continue
package_by_name[package.name] = package
installed_packages = get_installed_distributions()
for package in installed_packages:
if package.key in package_by_name:
continue
if package.key.startswith('mysqlclient') or package.key.startswith(
'mysql-connector') or package.key.startswith('MySQL-python'):
continue
pip_info = get_pip_info(package.key)
#sys.stderr.write("Home page of " + str(package.key) + " is " + str(pip_info['Home-page']) + "\n")
Package.query.filter_by(name=package.key).delete()
db.session.commit()
package_auth = PackageAuth(user_id=user_id)
if package.key.startswith('docassemble.') and pip_info[
'Home-page'] is not None and re.search(r'/github.com/',
pip_info['Home-page']):
package_entry = Package(name=package.key,
package_auth=package_auth,
type='git',
giturl=pip_info['Home-page'],
packageversion=package.version,
dependency=True)
else:
package_entry = Package(name=package.key,
package_auth=package_auth,
type='pip',
packageversion=package.version,
dependency=True)
db.session.add(package_entry)
db.session.commit()
install = Install(hostname=hostname,
packageversion=package_entry.packageversion,
version=package_entry.version,
package_id=package_entry.id)
db.session.add(install)
db.session.commit()
sys.stderr.write("add_dependencies: ending\n")
return
def add_dependencies(user_id, start_time=None):
if start_time is None:
start_time = time.time()
#sys.stderr.write('add_dependencies: user_id is ' + str(user_id) + "\n")
sys.stderr.write("add_dependencies: starting after " + str(time.time() - start_time) + " seconds\n")
from docassemble.base.config import hostname
from docassemble.webapp.app_object import app
from docassemble.webapp.db_object import db
from docassemble.webapp.packages.models import Package, Install, PackageAuth
from sqlalchemy import select, delete
packages_known = set()
for package in db.session.execute(select(Package.name).filter_by(active=True)):
packages_known.add(package.name)
installed_packages = get_installed_distributions(start_time=start_time)
home_pages = None
packages_to_add = list()
for package in installed_packages:
if package.key in packages_known:
continue
if package.key.startswith('mysqlclient') or package.key.startswith('mysql-connector') or package.key.startswith('MySQL-python'):
continue
db.session.execute(delete(Package).filter_by(name=package.key))
packages_to_add.append(package)
did_something = False
if len(packages_to_add):
did_something = True
db.session.commit()
for package in packages_to_add:
package_auth = PackageAuth(user_id=user_id)
if package.key.startswith('docassemble.'):
if home_pages is None:
home_pages = get_home_page_dict()
home_page = home_pages.get(package.key.lower(), None)
if home_page is not None and re.search(r'/github.com/', home_page):
package_entry = Package(name=package.key, package_auth=package_auth, type='git', giturl=home_page, packageversion=package.version, dependency=True)
else:
package_entry = Package(name=package.key, package_auth=package_auth, type='pip', packageversion=package.version, dependency=True)
else:
package_entry = Package(name=package.key, package_auth=package_auth, type='pip', packageversion=package.version, dependency=True)
db.session.add(package_entry)
db.session.commit()
install = Install(hostname=hostname, packageversion=package_entry.packageversion, version=package_entry.version, package_id=package_entry.id)
db.session.add(install)
db.session.commit()
sys.stderr.write("add_dependencies: ending after " + str(time.time() - start_time) + " seconds\n")
return did_something
def add_dependencies(user_id):
logmessage('add_dependencies: ' + str(user_id))
from docassemble.webapp.config import hostname
package_by_name = dict()
for package in Package.query.filter_by(active=True).all():
package_by_name[package.name] = package
installed_packages = get_installed_distributions()
for package in installed_packages:
if package.key in package_by_name:
continue
package_auth = PackageAuth(user_id=user_id)
if package.key in ['docassemble', 'docassemble.base', 'docassemble.webapp', 'docassemble.demo']:
package_entry = Package(name=package.key, package_auth=package_auth, giturl=docassemble_git_url, packageversion=package.version, gitsubdir=re.sub(r'\.', '-', package.key), type='git', core=True)
else:
package_entry = Package(name=package.key, package_auth=package_auth, type='pip', packageversion=package.version, dependency=True)
db.session.add(package_auth)
db.session.add(package_entry)
db.session.commit()
install = Install(hostname=hostname, packageversion=package_entry.packageversion, version=package_entry.version, package_id=package_entry.id)
db.session.add(install)
db.session.commit()
return
def check_for_updates(doing_startup=False,
start_time=None,
invalidate_cache=True,
full=True):
if start_time is None:
start_time = time.time()
sys.stderr.write("check_for_updates: starting after " +
str(time.time() - start_time) + " seconds\n")
if invalidate_cache:
invalidate_installed_distributions_cache()
from docassemble.base.config import hostname
from docassemble.webapp.app_object import app
from docassemble.webapp.db_object import db
from docassemble.webapp.packages.models import Package, Install, PackageAuth
ok = True
here_already = dict()
results = dict()
if full:
sys.stderr.write("check_for_updates: 0.5 after " +
str(time.time() - start_time) + " seconds\n")
for package_name in ('psycopg2', 'pdfminer', 'pdfminer3k', 'py-bcrypt',
'pycrypto', 'constraint', 'distutils2',
'azure-storage', 'Flask-User'):
num_deleted = Package.query.filter_by(name=package_name).delete()
if num_deleted > 0:
db.session.commit()
sys.stderr.write("check_for_updates: 1 after " +
str(time.time() - start_time) + " seconds\n")
installed_packages = get_installed_distributions(start_time=start_time)
for package in installed_packages:
here_already[package.key] = package.version
changed = False
if full:
if 'psycopg2' in here_already:
sys.stderr.write("check_for_updates: uninstalling psycopg2\n")
uninstall_package(DummyPackage('psycopg2'), start_time=start_time)
if 'psycopg2-binary' in here_already:
sys.stderr.write(
"check_for_updates: reinstalling psycopg2-binary\n")
uninstall_package(DummyPackage('psycopg2-binary'),
start_time=start_time)
install_package(DummyPackage('psycopg2-binary'),
start_time=start_time)
changed = True
if 'psycopg2-binary' not in here_already:
sys.stderr.write("check_for_updates: installing psycopg2-binary\n")
install_package(DummyPackage('psycopg2-binary'),
start_time=start_time)
change = True
if 'kombu' not in here_already or LooseVersion(
here_already['kombu']) <= LooseVersion('4.1.0'):
sys.stderr.write(
"check_for_updates: installing new kombu version\n")
install_package(DummyPackage('kombu'), start_time=start_time)
changed = True
if 'celery' not in here_already or LooseVersion(
here_already['celery']) <= LooseVersion('4.1.0'):
sys.stderr.write(
"check_for_updates: installing new celery version\n")
install_package(DummyPackage('celery'), start_time=start_time)
changed = True
if 'pycrypto' in here_already:
sys.stderr.write("check_for_updates: uninstalling pycrypto\n")
uninstall_package(DummyPackage('pycrypto'), start_time=start_time)
if 'pycryptodome' in here_already:
sys.stderr.write(
"check_for_updates: reinstalling pycryptodome\n")
uninstall_package(DummyPackage('pycryptodome'),
start_time=start_time)
install_package(DummyPackage('pycryptodome'),
start_time=start_time)
changed = True
if 'pycryptodome' not in here_already:
sys.stderr.write("check_for_updates: installing pycryptodome\n")
install_package(DummyPackage('pycryptodome'),
start_time=start_time)
changed = True
if 'pdfminer' in here_already:
sys.stderr.write("check_for_updates: uninstalling pdfminer\n")
uninstall_package(DummyPackage('pdfminer'), start_time=start_time)
changed = True
if 'pdfminer3k' in here_already:
sys.stderr.write("check_for_updates: uninstalling pdfminer3k\n")
uninstall_package(DummyPackage('pdfminer3k'),
start_time=start_time)
changed = True
if 'pdfminer.six' in here_already:
try:
from pdfminer.pdfparser import PDFParser
from pdfminer.pdfdocument import PDFDocument
except:
sys.stderr.write(
"check_for_updates: reinstalling pdfminer.six\n")
uninstall_package(DummyPackage('pdfminer.six'),
start_time=start_time)
install_package(DummyPackage('pdfminer.six'),
start_time=start_time)
else:
sys.stderr.write("check_for_updates: installing pdfminer.six\n")
install_package(DummyPackage('pdfminer.six'),
start_time=start_time)
changed = True
if 'py-bcrypt' in here_already:
sys.stderr.write("check_for_updates: uninstalling py-bcrypt\n")
uninstall_package(DummyPackage('py-bcrypt'), start_time=start_time)
changed = True
if 'bcrypt' in here_already:
sys.stderr.write("check_for_updates: reinstalling bcrypt\n")
uninstall_package(DummyPackage('bcrypt'),
start_time=start_time)
install_package(DummyPackage('bcrypt'), start_time=start_time)
changed = True
if 'bcrypt' not in here_already:
sys.stderr.write("check_for_updates: installing bcrypt\n")
install_package(DummyPackage('bcrypt'), start_time=start_time)
changed = True
if changed:
installed_packages = get_installed_distributions(
start_time=start_time)
here_already = dict()
for package in installed_packages:
here_already[package.key] = package.version
packages = dict()
installs = dict()
to_install = list()
to_uninstall = list()
uninstall_done = dict()
uninstalled_packages = dict()
logmessages = ''
package_by_name = dict()
sys.stderr.write("check_for_updates: 2 after " +
str(time.time() - start_time) + " seconds\n")
for package in Package.query.filter_by(active=True).all():
package_by_name[package.name] = package
#sys.stderr.write("check_for_updates: database includes a package called " + package.name + " after " + str(time.time() - start_time) + " seconds\n")
# packages is what is supposed to be installed
sys.stderr.write("check_for_updates: 3 after " +
str(time.time() - start_time) + " seconds\n")
for package in Package.query.filter_by(active=True).all():
if package.type is not None:
packages[package.id] = package
#sys.stderr.write("check_for_updates: database includes a package called " + package.name + " that has a type after " + str(time.time() - start_time) + " seconds\n")
#print("Found a package " + package.name)
sys.stderr.write("check_for_updates: 4 after " +
str(time.time() - start_time) + " seconds\n")
for package in Package.query.filter_by(active=False).all():
if package.name not in package_by_name:
#sys.stderr.write("check_for_updates: database says " + package.name + " should be uninstalled after " + str(time.time() - start_time) + " seconds\n")
uninstalled_packages[
package.
id] = package # this is what the database says should be uninstalled
sys.stderr.write("check_for_updates: 5 after " +
str(time.time() - start_time) + " seconds\n")
for install in Install.query.filter_by(hostname=hostname).all():
installs[
install.
package_id] = install # this is what the database says in installed on this server
if install.package_id in uninstalled_packages and uninstalled_packages[
install.package_id].name not in package_by_name:
sys.stderr.write("check_for_updates: " +
uninstalled_packages[install.package_id].name +
" will be uninstalled after " +
str(time.time() - start_time) + " seconds\n")
to_uninstall.append(uninstalled_packages[
install.package_id]) # uninstall if it is installed
changed = False
package_owner = dict()
sys.stderr.write("check_for_updates: 6 after " +
str(time.time() - start_time) + " seconds\n")
for auth in PackageAuth.query.filter_by(authtype='owner').all():
package_owner[auth.package_id] = auth.user_id
sys.stderr.write("check_for_updates: 7 after " +
str(time.time() - start_time) + " seconds\n")
for package in packages.values():
if package.id not in installs and package.name in here_already:
sys.stderr.write(
"check_for_updates: package " + package.name +
" here already. Writing an Install record for it.\n")
install = Install(hostname=hostname,
packageversion=here_already[package.name],
version=package.version,
package_id=package.id)
db.session.add(install)
installs[package.id] = install
changed = True
if changed:
db.session.commit()
sys.stderr.write("check_for_updates: 8 after " +
str(time.time() - start_time) + " seconds\n")
for package in packages.values():
#sys.stderr.write("check_for_updates: processing package id " + str(package.id) + "\n")
#sys.stderr.write("1: " + str(installs[package.id].packageversion) + " 2: " + str(package.packageversion) + "\n")
if (package.packageversion is not None and package.id in installs
and installs[package.id].packageversion is None
) or (package.packageversion is not None and package.id in installs
and installs[package.id].packageversion is not None
and LooseVersion(package.packageversion) > LooseVersion(
installs[package.id].packageversion)):
sys.stderr.write(
"check_for_updates: a new version of " + package.name +
" is needed because the necessary package version, " +
str(package.packageversion) +
", is ahead of the installed version, " +
str(installs[package.id].packageversion) + " after " +
str(time.time() - start_time) + " seconds\n")
new_version_needed = True
else:
new_version_needed = False
#sys.stderr.write("got here and new version is " + str(new_version_needed) + "\n")
# Check for missing local packages
if (package.name not in here_already) and (package.id in installs):
sys.stderr.write(
"check_for_updates: the package " + package.name +
" is supposed to be installed on this server, but was not detected after "
+ str(time.time() - start_time) + " seconds\n")
package_missing = True
else:
package_missing = False
if package.id in installs and package.version > installs[
package.id].version:
sys.stderr.write("check_for_updates: the package " + package.name +
" has internal version " + str(package.version) +
" but the installed version has version " +
str(installs[package.id].version) + " after " +
str(time.time() - start_time) + " seconds\n")
package_version_greater = True
else:
package_version_greater = False
if package.id not in installs:
sys.stderr.write(
"check_for_updates: the package " + package.name +
" is not in the table of installed packages for this server after "
+ str(time.time() - start_time) + " seconds\n")
if package.id not in installs or package_version_greater or new_version_needed or package_missing:
to_install.append(package)
#sys.stderr.write("done with that" + "\n")
sys.stderr.write("check_for_updates: 9 after " +
str(time.time() - start_time) + " seconds\n")
for package in to_uninstall:
#sys.stderr.write("Going to uninstall a package: " + package.name + "\n")
if package.name in uninstall_done:
sys.stderr.write("check_for_updates: skipping uninstallation of " +
str(package.name) +
" because already uninstalled after " +
str(time.time() - start_time) + " seconds" + "\n")
continue
if package.name not in here_already:
sys.stderr.write("check_for_updates: skipping uninstallation of " +
str(package.name) + " because not installed" +
" after " + str(time.time() - start_time) +
" seconds\n")
returnval = 1
newlog = ''
else:
sys.stderr.write(
"check_for_updates: calling uninstall_package on " +
package.name + "\n")
returnval, newlog = uninstall_package(package,
start_time=start_time)
uninstall_done[package.name] = 1
logmessages += newlog
if returnval == 0:
Install.query.filter_by(hostname=hostname,
package_id=package.id).delete()
results[
package.
name] = 'pip uninstall command returned success code. See log for details.'
elif returnval == 1:
Install.query.filter_by(hostname=hostname,
package_id=package.id).delete()
results[
package.
name] = 'pip uninstall was not run because the package was not installed.'
else:
results[
package.name] = 'pip uninstall command returned failure code'
ok = False
packages_to_delete = list()
sys.stderr.write("check_for_updates: 10 after " +
str(time.time() - start_time) + " seconds\n")
did_something = False
for package in to_install:
did_something = True
sys.stderr.write("check_for_updates: going to install a package: " +
package.name + " after " +
str(time.time() - start_time) + " seconds\n")
# if doing_startup and package.name.startswith('docassemble') and package.name in here_already:
# #adding this because of unpredictability of installing new versions of docassemble
# #just because of a system restart.
# sys.stderr.write("check_for_updates: skipping update on " + str(package.name) + "\n")
# continue
returnval, newlog = install_package(package, start_time=start_time)
logmessages += newlog
sys.stderr.write("check_for_updates: return value was " +
str(returnval) + " after " +
str(time.time() - start_time) + " seconds\n")
if returnval != 0:
sys.stderr.write("Return value was not good" + " after " +
str(time.time() - start_time) + " seconds\n")
ok = False
#pip._vendor.pkg_resources._initialize_master_working_set()
if full:
pip_info = get_pip_info(package.name, start_time=start_time)
real_name = pip_info['Name']
sys.stderr.write("check_for_updates: real name of package " +
str(package.name) + " is " + str(real_name) +
" after " + str(time.time() - start_time) +
" seconds\n")
else:
real_name = package.name
if real_name is None:
results[package.name] = 'install failed'
ok = False
if package.name not in here_already:
sys.stderr.write(
"check_for_updates: removing package entry for " +
package.name + " after " + str(time.time() - start_time) +
" seconds\n")
packages_to_delete.append(package)
elif returnval != 0:
results[package.name] = 'pip install command returned failure code'
else:
results[
package.
name] = 'pip install command returned success code. See log for details.'
if real_name != package.name:
sys.stderr.write("check_for_updates: changing name" +
" after " + str(time.time() - start_time) +
" seconds\n")
package.name = real_name
if package.id in installs:
install = installs[package.id]
install.version = package.version
else:
install = Install(hostname=hostname,
packageversion=package.packageversion,
version=package.version,
package_id=package.id)
db.session.add(install)
db.session.commit()
if did_something:
update_versions(start_time=start_time)
if full and add_dependencies(package_owner.get(package.id, 1),
start_time=start_time):
update_versions(start_time=start_time)
sys.stderr.write("check_for_updates: 11 after " +
str(time.time() - start_time) + " seconds\n")
for package in packages_to_delete:
db.session.delete(package)
sys.stderr.write("check_for_updates: 12 after " +
str(time.time() - start_time) + " seconds\n")
db.session.commit()
sys.stderr.write(
"check_for_updates: finished uninstalling and installing after " +
str(time.time() - start_time) + " seconds\n")
return ok, logmessages, results
def check_for_updates(doing_startup=False):
sys.stderr.write("check_for_updates: starting\n")
from docassemble.base.config import hostname
from docassemble.webapp.app_object import app
from docassemble.webapp.db_object import db
from docassemble.webapp.packages.models import Package, Install, PackageAuth
ok = True
here_already = dict()
results = dict()
sys.stderr.write("check_for_updates: 0.5\n")
num_deleted = Package.query.filter_by(name='psycopg2').delete()
if num_deleted > 0:
db.session.commit()
if PY2:
num_deleted = Package.query.filter_by(name='pycryptodome').delete()
if num_deleted > 0:
db.session.commit()
num_deleted = Package.query.filter_by(name='pdfminer3k').delete()
if num_deleted > 0:
db.session.commit()
else:
num_deleted = Package.query.filter_by(name='pdfminer').delete()
if num_deleted > 0:
db.session.commit()
num_deleted = Package.query.filter_by(name='py-bcrypt').delete()
if num_deleted > 0:
db.session.commit()
num_deleted = Package.query.filter_by(name='pycrypto').delete()
if num_deleted > 0:
db.session.commit()
num_deleted = Package.query.filter_by(name='constraint').delete()
if num_deleted > 0:
db.session.commit()
num_deleted = Package.query.filter_by(name='distutils2').delete()
if num_deleted > 0:
db.session.commit()
sys.stderr.write("check_for_updates: 1\n")
installed_packages = get_installed_distributions()
for package in installed_packages:
here_already[package.key] = package.version
changed = False
if 'pdfminer.six' not in here_already:
sys.stderr.write("check_for_updates: installing pdfminer.six\n")
install_package(DummyPackage('pdfminer.six'))
changed = True
if 'psycopg2' in here_already:
sys.stderr.write("check_for_updates: uninstalling psycopg2\n")
uninstall_package(DummyPackage('psycopg2'))
if 'psycopg2-binary' in here_already:
sys.stderr.write(
"check_for_updates: reinstalling psycopg2-binary\n")
uninstall_package(DummyPackage('psycopg2-binary'))
install_package(DummyPackage('psycopg2-binary'))
changed = True
if 'psycopg2-binary' not in here_already:
sys.stderr.write("check_for_updates: installing psycopg2-binary\n")
install_package(DummyPackage('psycopg2-binary'))
change = True
if PY2:
if 'pycryptodome' in here_already:
sys.stderr.write("check_for_updates: uninstalling pycryptodome\n")
uninstall_package(DummyPackage('pycryptodome'))
if 'pycrypto' in here_already:
sys.stderr.write("check_for_updates: reinstalling pycrypto\n")
uninstall_package(DummyPackage('pycrypto'))
install_package(DummyPackage('pycrypto'))
changed = True
if 'pathlib' not in here_already:
sys.stderr.write("check_for_updates: installing pathlib\n")
install_package(DummyPackage('pathlib'))
changed = True
if 'pycrypto' not in here_already:
sys.stderr.write("check_for_updates: installing pycrypto\n")
install_package(DummyPackage('pycrypto'))
changed = True
if 'kombu' not in here_already or LooseVersion(
here_already['kombu']) > LooseVersion('4.1.0'):
sys.stderr.write(
"check_for_updates: installing older kombu version\n")
kombu = DummyPackage('kombu')
kombu.limitation = '==4.1.0'
install_package(kombu)
changed = True
if 'celery' not in here_already or LooseVersion(
here_already['celery']) > LooseVersion('4.1.0'):
sys.stderr.write(
"check_for_updates: installing older celery version\n")
celery = DummyPackage('celery')
celery.limitation = '[redis]==4.1.0'
install_package(celery)
changed = True
if 'pdfminer' not in here_already:
sys.stderr.write("check_for_updates: installing pdfminer\n")
pdfminer = DummyPackage('pdfminer')
pdfminer.type = 'git'
pdfminer.giturl = 'https://github.com/euske/pdfminer'
pdfminer.gitsubdir = None
pdfminer.gitbranch = None
install_package(pdfminer)
changed = True
else:
if 'kombu' not in here_already or LooseVersion(
here_already['kombu']) <= LooseVersion('4.1.0'):
sys.stderr.write(
"check_for_updates: installing new kombu version\n")
install_package(DummyPackage('kombu'))
changed = True
if 'celery' not in here_already or LooseVersion(
here_already['celery']) <= LooseVersion('4.1.0'):
sys.stderr.write(
"check_for_updates: installing new celery version\n")
install_package(DummyPackage('celery'))
changed = True
if 'pycrypto' in here_already:
sys.stderr.write("check_for_updates: uninstalling pycrypto\n")
uninstall_package(DummyPackage('pycrypto'))
if 'pycryptodome' in here_already:
sys.stderr.write(
"check_for_updates: reinstalling pycryptodome\n")
uninstall_package(DummyPackage('pycryptodome'))
install_package(DummyPackage('pycryptodome'))
changed = True
if 'pycryptodome' not in here_already:
sys.stderr.write("check_for_updates: installing pycryptodome\n")
install_package(DummyPackage('pycryptodome'))
changed = True
if 'pdfminer' in here_already:
sys.stderr.write("check_for_updates: uninstalling pdfminer\n")
uninstall_package(DummyPackage('pdfminer'))
changed = True
if 'pdfminer3k' not in here_already:
sys.stderr.write("check_for_updates: installing pdfminer3k\n")
install_package(DummyPackage('pdfminer3k'))
changed = True
if 'py-bcrypt' in here_already:
sys.stderr.write("check_for_updates: uninstalling py-bcrypt\n")
uninstall_package(DummyPackage('py-bcrypt'))
changed = True
if 'bcrypt' in here_already:
sys.stderr.write("check_for_updates: reinstalling bcrypt\n")
uninstall_package(DummyPackage('bcrypt'))
install_package(DummyPackage('bcrypt'))
changed = True
if 'bcrypt' not in here_already:
sys.stderr.write("check_for_updates: installing bcrypt\n")
install_package(DummyPackage('bcrypt'))
changed = True
if changed:
installed_packages = get_installed_distributions()
here_already = dict()
for package in installed_packages:
here_already[package.key] = package.version
packages = dict()
installs = dict()
to_install = list()
to_uninstall = list()
uninstall_done = dict()
uninstalled_packages = dict()
logmessages = ''
package_by_name = dict()
sys.stderr.write("check_for_updates: 2\n")
for package in Package.query.filter_by(active=True).all():
package_by_name[package.name] = package
# packages is what is supposed to be installed
sys.stderr.write("check_for_updates: 3\n")
for package in Package.query.filter_by(active=True).all():
if package.type is not None:
packages[package.id] = package
#print("Found a package " + package.name)
sys.stderr.write("check_for_updates: 4\n")
for package in Package.query.filter_by(active=False).all():
if package.name not in package_by_name:
uninstalled_packages[
package.
id] = package # this is what the database says should be uninstalled
sys.stderr.write("check_for_updates: 5\n")
for install in Install.query.filter_by(hostname=hostname).all():
installs[
install.
package_id] = install # this is what the database says in installed on this server
if install.package_id in uninstalled_packages and uninstalled_packages[
install.package_id].name not in package_by_name:
to_uninstall.append(uninstalled_packages[
install.package_id]) # uninstall if it is installed
changed = False
package_owner = dict()
sys.stderr.write("check_for_updates: 6\n")
for auth in PackageAuth.query.filter_by(authtype='owner').all():
package_owner[auth.package_id] = auth.user_id
sys.stderr.write("check_for_updates: 7\n")
for package in packages.values():
if package.id not in installs and package.name in here_already:
sys.stderr.write("check_for_updates: package " + package.name +
" here already\n")
install = Install(hostname=hostname,
packageversion=here_already[package.name],
version=package.version,
package_id=package.id)
db.session.add(install)
installs[package.id] = install
changed = True
if changed:
db.session.commit()
sys.stderr.write("check_for_updates: 8\n")
for package in packages.values():
#sys.stderr.write("check_for_updates: processing package id " + str(package.id) + "\n")
#sys.stderr.write("1: " + str(installs[package.id].packageversion) + " 2: " + str(package.packageversion) + "\n")
if (package.packageversion is not None and package.id in installs
and installs[package.id].packageversion is None
) or (package.packageversion is not None and package.id in installs
and installs[package.id].packageversion is not None
and LooseVersion(package.packageversion) > LooseVersion(
installs[package.id].packageversion)):
new_version_needed = True
else:
new_version_needed = False
#sys.stderr.write("got here and new version is " + str(new_version_needed) + "\n")
# Check for missing local packages
if (package.name not in here_already) and (package.id in installs):
package_missing = True
else:
package_missing = False
if package.id not in installs or package.version > installs[
package.id].version or new_version_needed or package_missing:
to_install.append(package)
#sys.stderr.write("done with that" + "\n")
sys.stderr.write("check_for_updates: 9\n")
for package in to_uninstall:
#sys.stderr.write("Going to uninstall a package: " + package.name + "\n")
if package.name in uninstall_done:
sys.stderr.write("check_for_updates: skipping uninstallation of " +
str(package.name) +
" because already uninstalled" + "\n")
continue
if package.name not in here_already:
sys.stderr.write("check_for_updates: skipping uninstallation of " +
str(package.name) + " because not installed" +
"\n")
returnval = 1
newlog = ''
else:
returnval, newlog = uninstall_package(package)
uninstall_done[package.name] = 1
logmessages += newlog
if returnval == 0:
Install.query.filter_by(hostname=hostname,
package_id=package.id).delete()
results[
package.
name] = 'pip uninstall command returned success code. See log for details.'
elif returnval == 1:
Install.query.filter_by(hostname=hostname,
package_id=package.id).delete()
results[
package.
name] = 'pip uninstall was not run because the package was not installed.'
else:
results[
package.name] = 'pip uninstall command returned failure code'
ok = False
packages_to_delete = list()
sys.stderr.write("check_for_updates: 10\n")
for package in to_install:
sys.stderr.write("check_for_updates: going to install a package: " +
package.name + "\n")
# if doing_startup and package.name.startswith('docassemble') and package.name in here_already:
# #adding this because of unpredictability of installing new versions of docassemble
# #just because of a system restart.
# sys.stderr.write("check_for_updates: skipping update on " + str(package.name) + "\n")
# continue
returnval, newlog = install_package(package)
logmessages += newlog
sys.stderr.write("check_for_updates: return value was " +
str(returnval) + "\n")
if returnval != 0:
sys.stderr.write("Return value was not good" + "\n")
ok = False
#pip._vendor.pkg_resources._initialize_master_working_set()
pip_info = get_pip_info(package.name)
real_name = pip_info['Name']
sys.stderr.write("check_for_updates: real name of package " +
str(package.name) + " is " + str(real_name) + "\n")
if real_name is None:
results[package.name] = 'install failed'
ok = False
if package.name not in here_already:
sys.stderr.write(
"check_for_updates: removing package entry for " +
package.name + "\n")
packages_to_delete.append(package)
elif returnval != 0:
results[package.name] = 'pip install command returned failure code'
else:
results[
package.
name] = 'pip install command returned success code. See log for details.'
if real_name != package.name:
sys.stderr.write("check_for_updates: changing name" + "\n")
package.name = real_name
if package.id in installs:
install = installs[package.id]
install.version = package.version
else:
install = Install(hostname=hostname,
packageversion=package.packageversion,
version=package.version,
package_id=package.id)
db.session.add(install)
db.session.commit()
update_versions()
add_dependencies(package_owner.get(package.id, 1))
update_versions()
sys.stderr.write("check_for_updates: 11\n")
for package in packages_to_delete:
db.session.delete(package)
sys.stderr.write("check_for_updates: 12\n")
db.session.commit()
sys.stderr.write(
"check_for_updates: finished uninstalling and installing\n")
return ok, logmessages, results
def check_for_updates(doing_startup=False):
sys.stderr.write("check_for_updates: starting\n")
from docassemble.base.config import hostname
ok = True
here_already = dict()
results = dict()
sys.stderr.write("check_for_updates: 1\n")
installed_packages = get_installed_distributions()
for package in installed_packages:
here_already[package.key] = package.version
packages = dict()
installs = dict()
to_install = list()
to_uninstall = list()
uninstall_done = dict()
uninstalled_packages = dict()
logmessages = ''
package_by_name = dict()
sys.stderr.write("check_for_updates: 2\n")
for package in Package.query.filter_by(active=True).all():
package_by_name[package.name] = package
# packages is what is supposed to be installed
sys.stderr.write("check_for_updates: 3\n")
for package in Package.query.filter_by(active=True).all():
if package.type is not None:
packages[package.id] = package
#print "Found a package " + package.name
sys.stderr.write("check_for_updates: 4\n")
for package in Package.query.filter_by(active=False).all():
if package.name not in package_by_name:
uninstalled_packages[
package.
id] = package # this is what the database says should be uninstalled
sys.stderr.write("check_for_updates: 5\n")
for install in Install.query.filter_by(hostname=hostname).all():
installs[
install.
package_id] = install # this is what the database says in installed on this server
if install.package_id in uninstalled_packages and uninstalled_packages[
install.package_id].name not in package_by_name:
to_uninstall.append(uninstalled_packages[
install.package_id]) # uninstall if it is installed
changed = False
package_owner = dict()
sys.stderr.write("check_for_updates: 6\n")
for auth in PackageAuth.query.filter_by(authtype='owner').all():
package_owner[auth.package_id] = auth.user_id
sys.stderr.write("check_for_updates: 7\n")
for package in packages.itervalues():
if package.id not in installs and package.name in here_already:
sys.stderr.write("check_for_updates: package " + package.name +
" here already\n")
install = Install(hostname=hostname,
packageversion=here_already[package.name],
version=package.version,
package_id=package.id)
db.session.add(install)
installs[package.id] = install
changed = True
if changed:
db.session.commit()
sys.stderr.write("check_for_updates: 8\n")
for package in packages.itervalues():
#sys.stderr.write("check_for_updates: processing package id " + str(package.id) + "\n")
#sys.stderr.write("1: " + str(installs[package.id].packageversion) + " 2: " + str(package.packageversion) + "\n")
if (package.packageversion is not None and package.id in installs
and installs[package.id].packageversion is None
) or (package.packageversion is not None and package.id in installs
and installs[package.id].packageversion is not None
and LooseVersion(package.packageversion) > LooseVersion(
installs[package.id].packageversion)):
new_version_needed = True
else:
new_version_needed = False
#sys.stderr.write("got here and new version is " + str(new_version_needed) + "\n")
if package.id not in installs or package.version > installs[
package.id].version or new_version_needed:
to_install.append(package)
#sys.stderr.write("done with that" + "\n")
sys.stderr.write("check_for_updates: 9\n")
for package in to_uninstall:
#sys.stderr.write("Going to uninstall a package: " + package.name + "\n")
if package.name in uninstall_done:
sys.stderr.write("check_for_updates: skipping uninstallation of " +
str(package.name) +
" because already uninstalled" + "\n")
continue
returnval, newlog = uninstall_package(package)
uninstall_done[package.name] = 1
logmessages += newlog
if returnval == 0:
Install.query.filter_by(hostname=hostname,
package_id=package.id).delete()
results[package.name] = 'successfully uninstalled'
else:
results[package.name] = 'uninstall failed'
ok = False
packages_to_delete = list()
sys.stderr.write("check_for_updates: 10\n")
for package in to_install:
sys.stderr.write("check_for_updates: going to install a package: " +
package.name + "\n")
# if doing_startup and package.name.startswith('docassemble') and package.name in here_already:
# #adding this because of unpredictability of installing new versions of docassemble
# #just because of a system restart.
# sys.stderr.write("check_for_updates: skipping update on " + str(package.name) + "\n")
# continue
returnval, newlog = install_package(package)
logmessages += newlog
sys.stderr.write("check_for_updates: return value was " +
str(returnval) + "\n")
if returnval != 0:
sys.stderr.write("Return value was not good" + "\n")
ok = False
#pip._vendor.pkg_resources._initialize_master_working_set()
pip_info = get_pip_info(package.name)
real_name = pip_info['Name']
sys.stderr.write("check_for_updates: real name of package " +
str(package.name) + " is " + str(real_name) + "\n")
if real_name is None:
results[package.name] = 'install failed'
ok = False
if package.name not in here_already:
sys.stderr.write(
"check_for_updates: removing package entry for " +
package.name + "\n")
packages_to_delete.append(package)
elif returnval != 0:
results[package.name] = 'could not be upgraded'
else:
results[package.name] = 'successfully installed'
if real_name != package.name:
sys.stderr.write("check_for_updates: changing name" + "\n")
package.name = real_name
if package.id in installs:
install = installs[package.id]
install.version = package.version
else:
install = Install(hostname=hostname,
packageversion=package.packageversion,
version=package.version,
package_id=package.id)
db.session.add(install)
db.session.commit()
update_versions()
add_dependencies(package_owner.get(package.id, 1))
update_versions()
sys.stderr.write("check_for_updates: 11\n")
for package in packages_to_delete:
package.active = False
sys.stderr.write("check_for_updates: 12\n")
db.session.commit()
sys.stderr.write(
"check_for_updates: finished uninstalling and installing\n")
return ok, logmessages, results
def check_for_updates(start_time=None, invalidate_cache=True, full=True):
if start_time is None:
start_time = time.time()
sys.stderr.write("check_for_updates: starting after " + str(time.time() - start_time) + " seconds\n")
if invalidate_cache:
invalidate_installed_distributions_cache()
from docassemble.base.config import hostname
#from docassemble.webapp.app_object import app
from docassemble.webapp.db_object import db
from docassemble.webapp.packages.models import Package, Install, PackageAuth
from sqlalchemy import select, delete
ok = True # tracks whether there have been any errors
here_already = {} # packages that are installed on this server, based on pip list. package name -> package version
results = {} # result of actions taken on a package. package name -> message
if full:
sys.stderr.write("check_for_updates: 0.5 after " + str(time.time() - start_time) + " seconds\n")
for package_name in ('psycopg2', 'pdfminer', 'pdfminer3k', 'py-bcrypt', 'pycrypto', 'constraint', 'distutils2', 'azure-storage', 'Flask-User', 'Marisol'):
result = db.session.execute(delete(Package).filter_by(name=package_name))
if result.rowcount > 0:
db.session.commit()
sys.stderr.write("check_for_updates: 1 after " + str(time.time() - start_time) + " seconds\n")
installed_packages = get_installed_distributions(start_time=start_time)
for package in installed_packages:
here_already[package.key] = package.version
changed = False
# clean up old packages that cause problems.
if full:
if 'psycopg2' in here_already:
sys.stderr.write("check_for_updates: uninstalling psycopg2\n")
uninstall_package(DummyPackage('psycopg2'), start_time=start_time)
if 'psycopg2-binary' in here_already:
sys.stderr.write("check_for_updates: reinstalling psycopg2-binary\n")
uninstall_package(DummyPackage('psycopg2-binary'), start_time=start_time)
install_package(DummyPackage('psycopg2-binary'), start_time=start_time)
changed = True
if 'psycopg2-binary' not in here_already:
sys.stderr.write("check_for_updates: installing psycopg2-binary\n")
install_package(DummyPackage('psycopg2-binary'), start_time=start_time)
changed = True
if 'kombu' not in here_already or LooseVersion(here_already['kombu']) <= LooseVersion('4.1.0'):
sys.stderr.write("check_for_updates: installing new kombu version\n")
install_package(DummyPackage('kombu'), start_time=start_time)
changed = True
if 'celery' not in here_already or LooseVersion(here_already['celery']) <= LooseVersion('4.1.0'):
sys.stderr.write("check_for_updates: installing new celery version\n")
install_package(DummyPackage('celery'), start_time=start_time)
changed = True
if 'pycrypto' in here_already:
sys.stderr.write("check_for_updates: uninstalling pycrypto\n")
uninstall_package(DummyPackage('pycrypto'), start_time=start_time)
if 'pycryptodome' in here_already:
sys.stderr.write("check_for_updates: reinstalling pycryptodome\n")
uninstall_package(DummyPackage('pycryptodome'), start_time=start_time)
install_package(DummyPackage('pycryptodome'), start_time=start_time)
changed = True
if 'pycryptodome' not in here_already:
sys.stderr.write("check_for_updates: installing pycryptodome\n")
install_package(DummyPackage('pycryptodome'), start_time=start_time)
changed = True
if 'pdfminer' in here_already:
sys.stderr.write("check_for_updates: uninstalling pdfminer\n")
uninstall_package(DummyPackage('pdfminer'), start_time=start_time)
changed = True
if 'Marisol' in here_already:
sys.stderr.write("check_for_updates: uninstalling Marisol\n")
uninstall_package(DummyPackage('Marisol'), start_time=start_time)
changed = True
if 'pdfminer3k' in here_already:
sys.stderr.write("check_for_updates: uninstalling pdfminer3k\n")
uninstall_package(DummyPackage('pdfminer3k'), start_time=start_time)
changed = True
if 'pdfminer.six' in here_already:
try:
from pdfminer.pdfparser import PDFParser
from pdfminer.pdfdocument import PDFDocument
except:
sys.stderr.write("check_for_updates: reinstalling pdfminer.six\n")
uninstall_package(DummyPackage('pdfminer.six'), start_time=start_time)
install_package(DummyPackage('pdfminer.six'), start_time=start_time)
else:
sys.stderr.write("check_for_updates: installing pdfminer.six\n")
install_package(DummyPackage('pdfminer.six'), start_time=start_time)
changed = True
if 'py-bcrypt' in here_already:
sys.stderr.write("check_for_updates: uninstalling py-bcrypt\n")
uninstall_package(DummyPackage('py-bcrypt'), start_time=start_time)
changed = True
if 'bcrypt' in here_already:
sys.stderr.write("check_for_updates: reinstalling bcrypt\n")
uninstall_package(DummyPackage('bcrypt'), start_time=start_time)
install_package(DummyPackage('bcrypt'), start_time=start_time)
changed = True
if 'bcrypt' not in here_already:
sys.stderr.write("check_for_updates: installing bcrypt\n")
install_package(DummyPackage('bcrypt'), start_time=start_time)
changed = True
if changed:
installed_packages = get_installed_distributions(start_time=start_time)
here_already = {}
for package in installed_packages:
here_already[package.key] = package.version
logmessages = ''
packages = {} # packages that the database says are active and that have a type; package id -> package row
installs = {} # install rows representing what the database says in installed; package id -> install row
to_install = [] # package rows of packages to install
to_uninstall = [] # package rows of packages to uninstall
system_packages_to_fix = []
uninstall_done = set() # set of package names of packages already uninstalled
uninstalled_packages = {} # packages with active=False; package id -> package row
package_ids_to_delete = set() # set if IDs of package rows that have active=False but there is another
# row with active= True
package_by_name = {} # packages the database says are active; package name -> package row
sys.stderr.write("check_for_updates: 2 after " + str(time.time() - start_time) + " seconds\n")
# create dict package_by_name that maps a package name to a package database row,
# for packages that the database says should be installed
for package in db.session.execute(select(Package.name).filter_by(active=True)):
package_by_name[package.name] = package
#sys.stderr.write("check_for_updates: database includes a package called " + package.name + " after " + str(time.time() - start_time) + " seconds\n")
# packages is what is supposed to be installed
sys.stderr.write("check_for_updates: 3 after " + str(time.time() - start_time) + " seconds\n")
# create dict packages that maps a package ID to a package database row if the type is not null
for package in db.session.execute(select(Package).filter_by(active=True)).scalars():
if package.type is not None:
packages[package.id] = package
#sys.stderr.write("check_for_updates: database includes a package called " + package.name + " that has a type after " + str(time.time() - start_time) + " seconds\n")
#print("Found a package " + package.name)
sys.stderr.write("check_for_updates: 4 after " + str(time.time() - start_time) + " seconds\n")
# create dict uninstalled_packages that maps a package ID to a package database row if SQL wants the package deleted
for package in db.session.execute(select(Package).filter_by(active=False)).scalars():
# don't uninstall a system package
if package.name in system_packages:
logmessages += "Not uninstalling " + str(package.name) + " because it is a system package.\n"
system_packages_to_fix.append(package)
continue
if package.name in package_by_name:
# there are two Package entries for the same package, one with active=True, one with active=False
sys.stderr.write("check_for_updates: conflicting package entries for " + package.name + " after " + str(time.time() - start_time) + " seconds\n")
# let's delete the one with active=False
package_ids_to_delete.add(package.id)
else:
#sys.stderr.write("check_for_updates: database says " + package.name + " should be uninstalled after " + str(time.time() - start_time) + " seconds\n")
uninstalled_packages[package.id] = package # this is what the database says should be uninstalled
sys.stderr.write("check_for_updates: 5 after " + str(time.time() - start_time) + " seconds\n")
# build to_uninstall, which is the list of Package database entries targeted for uninstallation.
# Only add if there is an Install entry linked to the package id and there is no active Package row
# with the same name.
# Also build installs, which is a dict mapping package IDs to install rows, based on
# what the Install table says
for install in db.session.execute(select(Install).filter_by(hostname=hostname)).scalars():
installs[install.package_id] = install # this is what the database says in installed on this server
if install.package_id in uninstalled_packages and uninstalled_packages[install.package_id].name not in package_by_name:
sys.stderr.write("check_for_updates: " + uninstalled_packages[install.package_id].name + " will be uninstalled after " + str(time.time() - start_time) + " seconds\n")
to_uninstall.append(uninstalled_packages[install.package_id]) # uninstall if it is installed
changed = False
package_owner = {}
sys.stderr.write("check_for_updates: 6 after " + str(time.time() - start_time) + " seconds\n")
for auth in db.session.execute(select(PackageAuth).filter_by(authtype='owner')).scalars():
package_owner[auth.package_id] = auth.user_id
sys.stderr.write("check_for_updates: 7 after " + str(time.time() - start_time) + " seconds\n")
# Add Install rows for any active Package rows with non-null types if the package is present on the server
# Also add to the installs dict after adding.
for package in packages.values():
if package.id not in installs and package.name in here_already:
sys.stderr.write("check_for_updates: package " + package.name + " here already. Writing an Install record for it.\n")
install = Install(hostname=hostname, packageversion=here_already[package.name], version=package.version, package_id=package.id)
db.session.add(install)
installs[package.id] = install
changed = True
if changed:
db.session.commit()
sys.stderr.write("check_for_updates: 8 after " + str(time.time() - start_time) + " seconds\n")
for package in packages.values():
#sys.stderr.write("check_for_updates: processing package id " + str(package.id) + "\n")
#sys.stderr.write("1: " + str(installs[package.id].packageversion) + " 2: " + str(package.packageversion) + "\n")
if (package.packageversion is not None and package.id in installs and installs[package.id].packageversion is None) or (package.packageversion is not None and package.id in installs and installs[package.id].packageversion is not None and LooseVersion(package.packageversion) > LooseVersion(installs[package.id].packageversion)):
sys.stderr.write("check_for_updates: a new version of " + package.name + " is needed because the necessary package version, " + str(package.packageversion) + ", is ahead of the installed version, " + str(installs[package.id].packageversion) + " after " + str(time.time() - start_time) + " seconds\n")
new_version_needed = True
else:
new_version_needed = False
#sys.stderr.write("got here and new version is " + str(new_version_needed) + "\n")
# Check for missing local packages
if (package.name not in here_already) and (package.id in installs):
sys.stderr.write("check_for_updates: the package " + package.name + " is supposed to be installed on this server, but was not detected after " + str(time.time() - start_time) + " seconds\n")
package_missing = True
else:
package_missing = False
if package.id in installs and package.version > installs[package.id].version:
sys.stderr.write("check_for_updates: the package " + package.name + " has internal version " + str(package.version) + " but the installed version has version " + str(installs[package.id].version) + " after " + str(time.time() - start_time) + " seconds\n")
package_version_greater = True
else:
package_version_greater = False
if package.id not in installs:
sys.stderr.write("check_for_updates: the package " + package.name + " is not in the table of installed packages for this server after " + str(time.time() - start_time) + " seconds\n")
if package.id not in installs or package_version_greater or new_version_needed or package_missing:
if package.name in system_packages:
logmessages += "Not upgrading " + str(package.name) + " because it is a system package and its version needs to be consistent with the version of the package that is required by the docassemble.webapp package.\n"
sys.stderr.write("check_for_updates: the package " + package.name + " is a system package and cannot be updated except through docassemble.webapp after " + str(time.time() - start_time) + " seconds\n")
system_packages_to_fix.append(package)
else:
to_install.append(package)
#sys.stderr.write("done with that" + "\n")
sys.stderr.write("check_for_updates: 9 after " + str(time.time() - start_time) + " seconds\n")
for package in to_uninstall:
#sys.stderr.write("Going to uninstall a package: " + package.name + "\n")
if package.name in uninstall_done:
sys.stderr.write("check_for_updates: skipping uninstallation of " + str(package.name) + " because already uninstalled after " + str(time.time() - start_time) + " seconds" + "\n")
continue
if package.name not in here_already:
sys.stderr.write("check_for_updates: skipping uninstallation of " + str(package.name) + " because not installed" + " after " + str(time.time() - start_time) + " seconds\n")
returnval = 1
newlog = ''
else:
sys.stderr.write("check_for_updates: calling uninstall_package on " + package.name + "\n")
returnval, newlog = uninstall_package(package, start_time=start_time)
uninstall_done.add(package.name)
logmessages += newlog
if returnval == 0:
db.session.execute(delete(Install).filter_by(hostname=hostname, package_id=package.id))
results[package.name] = 'pip uninstall command returned success code. See log for details.'
elif returnval == 1:
db.session.execute(delete(Install).filter_by(hostname=hostname, package_id=package.id))
results[package.name] = 'pip uninstall was not run because the package was not installed.'
else:
results[package.name] = 'pip uninstall command returned failure code'
ok = False
packages_to_delete = []
sys.stderr.write("check_for_updates: 10 after " + str(time.time() - start_time) + " seconds\n")
did_something = False
for package in to_install:
did_something = True
sys.stderr.write("check_for_updates: going to install a package: " + package.name + " after " + str(time.time() - start_time) + " seconds\n")
# if doing_startup and package.name.startswith('docassemble') and package.name in here_already:
# #adding this because of unpredictability of installing new versions of docassemble
# #just because of a system restart.
# sys.stderr.write("check_for_updates: skipping update on " + str(package.name) + "\n")
# continue
returnval, newlog = install_package(package, start_time=start_time)
logmessages += newlog
sys.stderr.write("check_for_updates: return value was " + str(returnval) + " after " + str(time.time() - start_time) + " seconds\n")
if returnval != 0:
sys.stderr.write("Return value was not good" + " after " + str(time.time() - start_time) + " seconds\n")
ok = False
#pip._vendor.pkg_resources._initialize_master_working_set()
if full:
pip_info = get_pip_info(package.name, start_time=start_time)
real_name = pip_info['Name']
sys.stderr.write("check_for_updates: real name of package " + str(package.name) + " is " + str(real_name) + " after " + str(time.time() - start_time) + " seconds\n")
else:
real_name = package.name
if real_name is None:
results[package.name] = 'install failed'
ok = False
if package.name not in here_already:
sys.stderr.write("check_for_updates: removing package entry for " + package.name + " after " + str(time.time() - start_time) + " seconds\n")
packages_to_delete.append(package)
elif returnval != 0:
results[package.name] = 'pip install command returned failure code'
else:
results[package.name] = 'pip install command returned success code. See log for details.'
if real_name != package.name:
sys.stderr.write("check_for_updates: changing name" + " after " + str(time.time() - start_time) + " seconds\n")
package.name = real_name
if package.id in installs:
install = installs[package.id]
install.version = package.version
else:
install = Install(hostname=hostname, packageversion=package.packageversion, version=package.version, package_id=package.id)
db.session.add(install)
db.session.commit()
if did_something:
update_versions(start_time=start_time)
if full and add_dependencies(package_owner.get(package.id, 1), start_time=start_time):
update_versions(start_time=start_time)
sys.stderr.write("check_for_updates: 11 after " + str(time.time() - start_time) + " seconds\n")
for package in packages_to_delete:
db.session.delete(package)
sys.stderr.write("check_for_updates: 12 after " + str(time.time() - start_time) + " seconds\n")
db.session.commit()
sys.stderr.write("check_for_updates: finished uninstalling and installing after " + str(time.time() - start_time) + " seconds\n")
return ok, logmessages, results
def check_for_updates():
logmessage("check_for_update: starting")
from docassemble.webapp.config import hostname
ok = True
here_already = dict()
installed_packages = get_installed_distributions()
for package in installed_packages:
here_already[package.key] = package.version
packages = dict()
installs = dict()
to_install = list()
to_uninstall = list()
uninstalled_packages = dict()
logmessages = ''
for package in Package.query.filter_by(active=True).all():
if package.type is not None:
packages[package.id] = package
#print "Found a package " + package.name
for package in Package.query.filter_by(active=False).all():
uninstalled_packages[package.id] = package
for install in Install.query.filter_by(hostname=hostname).all():
installs[install.package_id] = install
if install.package_id in uninstalled_packages:
to_uninstall.append(uninstalled_packages[install.package_id])
changed = False
package_owner = dict()
for auth in PackageAuth.query.filter_by(authtype='owner').all():
package_owner[auth.package_id] = auth.user_id
for package in packages.itervalues():
if package.id not in installs and package.name in here_already:
logmessage("Package " + package.name + " here already")
install = Install(hostname=hostname, packageversion=here_already[package.name], version=package.version, package_id=package.id)
db.session.add(install)
installs[package.id] = install
changed = True
if changed:
db.session.commit()
for package in packages.itervalues():
#logmessage("Processing package id " + str(package.id))
#logmessage("1: " + str(installs[package.id].packageversion) + " 2: " + str(package.packageversion))
if (package.packageversion is not None and package.id in installs and installs[package.id].packageversion is None) or (package.packageversion is not None and package.id in installs and installs[package.id].packageversion is not None and LooseVersion(package.packageversion) > LooseVersion(installs[package.id].packageversion)):
new_version_needed = True
else:
new_version_needed = False
#logmessage("got here and new version is " + str(new_version_needed))
if package.id not in installs or package.version > installs[package.id].version or new_version_needed:
to_install.append(package)
#logmessage("done with that")
for package in to_uninstall:
#logmessage("Going to uninstall a package")
returnval, newlog = uninstall_package(package)
logmessages += newlog
if returnval == 0:
Install.query.filter_by(hostname=hostname, package_id=package.id).delete()
else:
ok = False
for package in to_install:
#logmessage("Going to install a package")
returnval, newlog = install_package(package)
logmessages += newlog
if returnval == 0:
if package.id in installs:
install = installs[package.id]
install.version = package.version
else:
install = Install(hostname=hostname, packageversion=package.packageversion, version=package.version, package_id=package.id)
db.session.add(install)
db.session.commit()
update_versions()
add_dependencies(package_owner[package.id])
update_versions()
else:
ok = False
db.session.commit()
#logmessage("check_for_update: finished uninstalling and installing")
return ok, logmessages
def check_for_updates():
logmessage("check_for_update: starting")
from docassemble.webapp.config import hostname
ok = True
here_already = dict()
installed_packages = get_installed_distributions()
for package in installed_packages:
here_already[package.key] = package.version
packages = dict()
installs = dict()
to_install = list()
to_uninstall = list()
uninstalled_packages = dict()
logmessages = ''
for package in Package.query.filter_by(active=True).all():
if package.type is not None:
packages[package.id] = package
#print "Found a package " + package.name
for package in Package.query.filter_by(active=False).all():
uninstalled_packages[package.id] = package
for install in Install.query.filter_by(hostname=hostname).all():
installs[install.package_id] = install
if install.package_id in uninstalled_packages:
to_uninstall.append(uninstalled_packages[install.package_id])
changed = False
package_owner = dict()
for auth in PackageAuth.query.filter_by(authtype='owner').all():
package_owner[auth.package_id] = auth.user_id
for package in packages.itervalues():
if package.id not in installs and package.name in here_already:
logmessage("Package " + package.name + " here already")
install = Install(hostname=hostname,
packageversion=here_already[package.name],
version=package.version,
package_id=package.id)
db.session.add(install)
installs[package.id] = install
changed = True
if changed:
db.session.commit()
for package in packages.itervalues():
#logmessage("Processing package id " + str(package.id))
#logmessage("1: " + str(installs[package.id].packageversion) + " 2: " + str(package.packageversion))
if (package.packageversion is not None and package.id in installs
and installs[package.id].packageversion is None
) or (package.packageversion is not None and package.id in installs
and installs[package.id].packageversion is not None
and LooseVersion(package.packageversion) > LooseVersion(
installs[package.id].packageversion)):
new_version_needed = True
else:
new_version_needed = False
#logmessage("got here and new version is " + str(new_version_needed))
if package.id not in installs or package.version > installs[
package.id].version or new_version_needed:
to_install.append(package)
#logmessage("done with that")
for package in to_uninstall:
#logmessage("Going to uninstall a package")
returnval, newlog = uninstall_package(package)
logmessages += newlog
if returnval == 0:
Install.query.filter_by(hostname=hostname,
package_id=package.id).delete()
else:
ok = False
for package in to_install:
#logmessage("Going to install a package")
returnval, newlog = install_package(package)
logmessages += newlog
if returnval == 0:
if package.id in installs:
install = installs[package.id]
install.version = package.version
else:
install = Install(hostname=hostname,
packageversion=package.packageversion,
version=package.version,
package_id=package.id)
db.session.add(install)
db.session.commit()
update_versions()
add_dependencies(package_owner[package.id])
update_versions()
else:
ok = False
db.session.commit()
#logmessage("check_for_update: finished uninstalling and installing")
return ok, logmessages
def check_for_updates(doing_startup=False):
sys.stderr.write("check_for_updates: starting\n")
from docassemble.base.config import hostname
ok = True
here_already = dict()
results = dict()
sys.stderr.write("check_for_updates: 1\n")
installed_packages = get_installed_distributions()
for package in installed_packages:
here_already[package.key] = package.version
packages = dict()
installs = dict()
to_install = list()
to_uninstall = list()
uninstall_done = dict()
uninstalled_packages = dict()
logmessages = ''
package_by_name = dict()
sys.stderr.write("check_for_updates: 2\n")
for package in Package.query.filter_by(active=True).all():
package_by_name[package.name] = package
# packages is what is supposed to be installed
sys.stderr.write("check_for_updates: 3\n")
for package in Package.query.filter_by(active=True).all():
if package.type is not None:
packages[package.id] = package
#print "Found a package " + package.name
sys.stderr.write("check_for_updates: 4\n")
for package in Package.query.filter_by(active=False).all():
if package.name not in package_by_name:
uninstalled_packages[package.id] = package # this is what the database says should be uninstalled
sys.stderr.write("check_for_updates: 5\n")
for install in Install.query.filter_by(hostname=hostname).all():
installs[install.package_id] = install # this is what the database says in installed on this server
if install.package_id in uninstalled_packages and uninstalled_packages[install.package_id].name not in package_by_name:
to_uninstall.append(uninstalled_packages[install.package_id]) # uninstall if it is installed
changed = False
package_owner = dict()
sys.stderr.write("check_for_updates: 6\n")
for auth in PackageAuth.query.filter_by(authtype='owner').all():
package_owner[auth.package_id] = auth.user_id
sys.stderr.write("check_for_updates: 7\n")
for package in packages.itervalues():
if package.id not in installs and package.name in here_already:
sys.stderr.write("check_for_updates: package " + package.name + " here already\n")
install = Install(hostname=hostname, packageversion=here_already[package.name], version=package.version, package_id=package.id)
db.session.add(install)
installs[package.id] = install
changed = True
if changed:
db.session.commit()
sys.stderr.write("check_for_updates: 8\n")
for package in packages.itervalues():
#sys.stderr.write("check_for_updates: processing package id " + str(package.id) + "\n")
#sys.stderr.write("1: " + str(installs[package.id].packageversion) + " 2: " + str(package.packageversion) + "\n")
if (package.packageversion is not None and package.id in installs and installs[package.id].packageversion is None) or (package.packageversion is not None and package.id in installs and installs[package.id].packageversion is not None and LooseVersion(package.packageversion) > LooseVersion(installs[package.id].packageversion)):
new_version_needed = True
else:
new_version_needed = False
#sys.stderr.write("got here and new version is " + str(new_version_needed) + "\n")
# Check for missing local packages
if (package.name not in here_already) and (package.id in installs):
package_missing = True
else:
package_missing = False
if package.id not in installs or package.version > installs[package.id].version or new_version_needed or package_missing:
to_install.append(package)
#sys.stderr.write("done with that" + "\n")
sys.stderr.write("check_for_updates: 9\n")
for package in to_uninstall:
#sys.stderr.write("Going to uninstall a package: " + package.name + "\n")
if package.name in uninstall_done:
sys.stderr.write("check_for_updates: skipping uninstallation of " + str(package.name) + " because already uninstalled" + "\n")
continue
returnval, newlog = uninstall_package(package)
uninstall_done[package.name] = 1
logmessages += newlog
if returnval == 0:
Install.query.filter_by(hostname=hostname, package_id=package.id).delete()
results[package.name] = 'successfully uninstalled'
else:
results[package.name] = 'uninstall failed'
ok = False
packages_to_delete = list()
sys.stderr.write("check_for_updates: 10\n")
for package in to_install:
sys.stderr.write("check_for_updates: going to install a package: " + package.name + "\n")
# if doing_startup and package.name.startswith('docassemble') and package.name in here_already:
# #adding this because of unpredictability of installing new versions of docassemble
# #just because of a system restart.
# sys.stderr.write("check_for_updates: skipping update on " + str(package.name) + "\n")
# continue
returnval, newlog = install_package(package)
logmessages += newlog
sys.stderr.write("check_for_updates: return value was " + str(returnval) + "\n")
if returnval != 0:
sys.stderr.write("Return value was not good" + "\n")
ok = False
#pip._vendor.pkg_resources._initialize_master_working_set()
pip_info = get_pip_info(package.name)
real_name = pip_info['Name']
sys.stderr.write("check_for_updates: real name of package " + str(package.name) + " is " + str(real_name) + "\n")
if real_name is None:
results[package.name] = 'install failed'
ok = False
if package.name not in here_already:
sys.stderr.write("check_for_updates: removing package entry for " + package.name + "\n")
packages_to_delete.append(package)
elif returnval != 0:
results[package.name] = 'could not be upgraded'
else:
results[package.name] = 'successfully installed'
if real_name != package.name:
sys.stderr.write("check_for_updates: changing name" + "\n")
package.name = real_name
if package.id in installs:
install = installs[package.id]
install.version = package.version
else:
install = Install(hostname=hostname, packageversion=package.packageversion, version=package.version, package_id=package.id)
db.session.add(install)
db.session.commit()
update_versions()
add_dependencies(package_owner.get(package.id, 1))
update_versions()
sys.stderr.write("check_for_updates: 11\n")
for package in packages_to_delete:
package.active = False
sys.stderr.write("check_for_updates: 12\n")
db.session.commit()
sys.stderr.write("check_for_updates: finished uninstalling and installing\n")
return ok, logmessages, results
def check_for_updates(doing_startup=False):
sys.stderr.write("check_for_updates: starting\n")
from docassemble.base.config import hostname
from docassemble.webapp.app_object import app
from docassemble.webapp.db_object import db
from docassemble.webapp.packages.models import Package, Install, PackageAuth
ok = True
here_already = dict()
results = dict()
sys.stderr.write("check_for_updates: 0.5\n")
num_deleted = Package.query.filter_by(name='psycopg2').delete()
if num_deleted > 0:
db.session.commit()
if PY2:
num_deleted = Package.query.filter_by(name='pycryptodome').delete()
if num_deleted > 0:
db.session.commit()
num_deleted = Package.query.filter_by(name='pdfminer3k').delete()
if num_deleted > 0:
db.session.commit()
else:
num_deleted = Package.query.filter_by(name='pdfminer').delete()
if num_deleted > 0:
db.session.commit()
num_deleted = Package.query.filter_by(name='py-bcrypt').delete()
if num_deleted > 0:
db.session.commit()
num_deleted = Package.query.filter_by(name='pycrypto').delete()
if num_deleted > 0:
db.session.commit()
num_deleted = Package.query.filter_by(name='constraint').delete()
if num_deleted > 0:
db.session.commit()
num_deleted = Package.query.filter_by(name='distutils2').delete()
if num_deleted > 0:
db.session.commit()
sys.stderr.write("check_for_updates: 1\n")
installed_packages = get_installed_distributions()
for package in installed_packages:
here_already[package.key] = package.version
changed = False
if 'pdfminer.six' not in here_already:
sys.stderr.write("check_for_updates: installing pdfminer.six\n")
install_package(DummyPackage('pdfminer.six'))
changed = True
if 'psycopg2' in here_already:
sys.stderr.write("check_for_updates: uninstalling psycopg2\n")
uninstall_package(DummyPackage('psycopg2'))
if 'psycopg2-binary' in here_already:
sys.stderr.write("check_for_updates: reinstalling psycopg2-binary\n")
uninstall_package(DummyPackage('psycopg2-binary'))
install_package(DummyPackage('psycopg2-binary'))
changed = True
if 'psycopg2-binary' not in here_already:
sys.stderr.write("check_for_updates: installing psycopg2-binary\n")
install_package(DummyPackage('psycopg2-binary'))
change = True
if PY2:
if 'pycryptodome' in here_already:
sys.stderr.write("check_for_updates: uninstalling pycryptodome\n")
uninstall_package(DummyPackage('pycryptodome'))
if 'pycrypto' in here_already:
sys.stderr.write("check_for_updates: reinstalling pycrypto\n")
uninstall_package(DummyPackage('pycrypto'))
install_package(DummyPackage('pycrypto'))
changed = True
if 'pathlib' not in here_already:
sys.stderr.write("check_for_updates: installing pathlib\n")
install_package(DummyPackage('pathlib'))
changed = True
if 'pycrypto' not in here_already:
sys.stderr.write("check_for_updates: installing pycrypto\n")
install_package(DummyPackage('pycrypto'))
changed = True
if 'kombu' not in here_already or LooseVersion(here_already['kombu']) > LooseVersion('4.1.0'):
sys.stderr.write("check_for_updates: installing older kombu version\n")
kombu = DummyPackage('kombu')
kombu.limitation = '==4.1.0'
install_package(kombu)
changed = True
if 'celery' not in here_already or LooseVersion(here_already['celery']) > LooseVersion('4.1.0'):
sys.stderr.write("check_for_updates: installing older celery version\n")
celery = DummyPackage('celery')
celery.limitation = '[redis]==4.1.0'
install_package(celery)
changed = True
if 'pdfminer' not in here_already:
sys.stderr.write("check_for_updates: installing pdfminer\n")
pdfminer = DummyPackage('pdfminer')
pdfminer.type = 'git'
pdfminer.giturl = 'https://github.com/euske/pdfminer'
pdfminer.gitsubdir = None
pdfminer.gitbranch = None
install_package(pdfminer)
changed = True
else:
if 'kombu' not in here_already or LooseVersion(here_already['kombu']) <= LooseVersion('4.1.0'):
sys.stderr.write("check_for_updates: installing new kombu version\n")
install_package(DummyPackage('kombu'))
changed = True
if 'celery' not in here_already or LooseVersion(here_already['celery']) <= LooseVersion('4.1.0'):
sys.stderr.write("check_for_updates: installing new celery version\n")
install_package(DummyPackage('celery'))
changed = True
if 'pycrypto' in here_already:
sys.stderr.write("check_for_updates: uninstalling pycrypto\n")
uninstall_package(DummyPackage('pycrypto'))
if 'pycryptodome' in here_already:
sys.stderr.write("check_for_updates: reinstalling pycryptodome\n")
uninstall_package(DummyPackage('pycryptodome'))
install_package(DummyPackage('pycryptodome'))
changed = True
if 'pycryptodome' not in here_already:
sys.stderr.write("check_for_updates: installing pycryptodome\n")
install_package(DummyPackage('pycryptodome'))
changed = True
if 'pdfminer' in here_already:
sys.stderr.write("check_for_updates: uninstalling pdfminer\n")
uninstall_package(DummyPackage('pdfminer'))
changed = True
if 'pdfminer3k' not in here_already:
sys.stderr.write("check_for_updates: installing pdfminer3k\n")
install_package(DummyPackage('pdfminer3k'))
changed = True
if 'py-bcrypt' in here_already:
sys.stderr.write("check_for_updates: uninstalling py-bcrypt\n")
uninstall_package(DummyPackage('py-bcrypt'))
changed = True
if 'bcrypt' in here_already:
sys.stderr.write("check_for_updates: reinstalling bcrypt\n")
uninstall_package(DummyPackage('bcrypt'))
install_package(DummyPackage('bcrypt'))
changed = True
if 'bcrypt' not in here_already:
sys.stderr.write("check_for_updates: installing bcrypt\n")
install_package(DummyPackage('bcrypt'))
changed = True
if changed:
installed_packages = get_installed_distributions()
here_already = dict()
for package in installed_packages:
here_already[package.key] = package.version
packages = dict()
installs = dict()
to_install = list()
to_uninstall = list()
uninstall_done = dict()
uninstalled_packages = dict()
logmessages = ''
package_by_name = dict()
sys.stderr.write("check_for_updates: 2\n")
for package in Package.query.filter_by(active=True).all():
package_by_name[package.name] = package
# packages is what is supposed to be installed
sys.stderr.write("check_for_updates: 3\n")
for package in Package.query.filter_by(active=True).all():
if package.type is not None:
packages[package.id] = package
#print("Found a package " + package.name)
sys.stderr.write("check_for_updates: 4\n")
for package in Package.query.filter_by(active=False).all():
if package.name not in package_by_name:
uninstalled_packages[package.id] = package # this is what the database says should be uninstalled
sys.stderr.write("check_for_updates: 5\n")
for install in Install.query.filter_by(hostname=hostname).all():
installs[install.package_id] = install # this is what the database says in installed on this server
if install.package_id in uninstalled_packages and uninstalled_packages[install.package_id].name not in package_by_name:
to_uninstall.append(uninstalled_packages[install.package_id]) # uninstall if it is installed
changed = False
package_owner = dict()
sys.stderr.write("check_for_updates: 6\n")
for auth in PackageAuth.query.filter_by(authtype='owner').all():
package_owner[auth.package_id] = auth.user_id
sys.stderr.write("check_for_updates: 7\n")
for package in packages.values():
if package.id not in installs and package.name in here_already:
sys.stderr.write("check_for_updates: package " + package.name + " here already\n")
install = Install(hostname=hostname, packageversion=here_already[package.name], version=package.version, package_id=package.id)
db.session.add(install)
installs[package.id] = install
changed = True
if changed:
db.session.commit()
sys.stderr.write("check_for_updates: 8\n")
for package in packages.values():
#sys.stderr.write("check_for_updates: processing package id " + str(package.id) + "\n")
#sys.stderr.write("1: " + str(installs[package.id].packageversion) + " 2: " + str(package.packageversion) + "\n")
if (package.packageversion is not None and package.id in installs and installs[package.id].packageversion is None) or (package.packageversion is not None and package.id in installs and installs[package.id].packageversion is not None and LooseVersion(package.packageversion) > LooseVersion(installs[package.id].packageversion)):
new_version_needed = True
else:
new_version_needed = False
#sys.stderr.write("got here and new version is " + str(new_version_needed) + "\n")
# Check for missing local packages
if (package.name not in here_already) and (package.id in installs):
package_missing = True
else:
package_missing = False
if package.id not in installs or package.version > installs[package.id].version or new_version_needed or package_missing:
to_install.append(package)
#sys.stderr.write("done with that" + "\n")
sys.stderr.write("check_for_updates: 9\n")
for package in to_uninstall:
#sys.stderr.write("Going to uninstall a package: " + package.name + "\n")
if package.name in uninstall_done:
sys.stderr.write("check_for_updates: skipping uninstallation of " + str(package.name) + " because already uninstalled" + "\n")
continue
if package.name not in here_already:
sys.stderr.write("check_for_updates: skipping uninstallation of " + str(package.name) + " because not installed" + "\n")
else:
returnval, newlog = uninstall_package(package)
uninstall_done[package.name] = 1
logmessages += newlog
if returnval == 0:
Install.query.filter_by(hostname=hostname, package_id=package.id).delete()
results[package.name] = 'pip uninstall command returned success code. See log for details.'
else:
results[package.name] = 'pip uninstall command returned failure code'
ok = False
packages_to_delete = list()
sys.stderr.write("check_for_updates: 10\n")
for package in to_install:
sys.stderr.write("check_for_updates: going to install a package: " + package.name + "\n")
# if doing_startup and package.name.startswith('docassemble') and package.name in here_already:
# #adding this because of unpredictability of installing new versions of docassemble
# #just because of a system restart.
# sys.stderr.write("check_for_updates: skipping update on " + str(package.name) + "\n")
# continue
returnval, newlog = install_package(package)
logmessages += newlog
sys.stderr.write("check_for_updates: return value was " + str(returnval) + "\n")
if returnval != 0:
sys.stderr.write("Return value was not good" + "\n")
ok = False
#pip._vendor.pkg_resources._initialize_master_working_set()
pip_info = get_pip_info(package.name)
real_name = pip_info['Name']
sys.stderr.write("check_for_updates: real name of package " + str(package.name) + " is " + str(real_name) + "\n")
if real_name is None:
results[package.name] = 'install failed'
ok = False
if package.name not in here_already:
sys.stderr.write("check_for_updates: removing package entry for " + package.name + "\n")
packages_to_delete.append(package)
elif returnval != 0:
results[package.name] = 'pip install command returned failure code'
else:
results[package.name] = 'pip install command returned success code. See log for details.'
if real_name != package.name:
sys.stderr.write("check_for_updates: changing name" + "\n")
package.name = real_name
if package.id in installs:
install = installs[package.id]
install.version = package.version
else:
install = Install(hostname=hostname, packageversion=package.packageversion, version=package.version, package_id=package.id)
db.session.add(install)
db.session.commit()
update_versions()
add_dependencies(package_owner.get(package.id, 1))
update_versions()
sys.stderr.write("check_for_updates: 11\n")
for package in packages_to_delete:
db.session.delete(package)
sys.stderr.write("check_for_updates: 12\n")
db.session.commit()
sys.stderr.write("check_for_updates: finished uninstalling and installing\n")
return ok, logmessages, results
