def add_dependencies(user_id): #sys.stderr.write('add_dependencies: user_id is ' + str(user_id) + "\n") sys.stderr.write("add_dependencies: starting\n") from docassemble.base.config import hostname, daconfig from docassemble.webapp.app_object import app from docassemble.webapp.db_object import db from docassemble.webapp.packages.models import Package, Install, PackageAuth #docassemble_git_url = daconfig.get('docassemble git url', 'https://github.com/jhpyle/docassemble') package_by_name = dict() for package in Package.query.filter_by(active=True).order_by( Package.name, Package.id.desc()).all(): if package.name in package_by_name: continue package_by_name[package.name] = package installed_packages = get_installed_distributions() for package in installed_packages: if package.key in package_by_name: continue if package.key.startswith('mysqlclient') or package.key.startswith( 'mysql-connector') or package.key.startswith('MySQL-python'): continue pip_info = get_pip_info(package.key) #sys.stderr.write("Home page of " + str(package.key) + " is " + str(pip_info['Home-page']) + "\n") Package.query.filter_by(name=package.key).delete() db.session.commit() package_auth = PackageAuth(user_id=user_id) if package.key.startswith('docassemble.') and pip_info[ 'Home-page'] is not None and re.search(r'/github.com/', pip_info['Home-page']): package_entry = Package(name=package.key, package_auth=package_auth, type='git', giturl=pip_info['Home-page'], packageversion=package.version, dependency=True) else: package_entry = Package(name=package.key, package_auth=package_auth, type='pip', packageversion=package.version, dependency=True) db.session.add(package_entry) db.session.commit() install = Install(hostname=hostname, packageversion=package_entry.packageversion, version=package_entry.version, package_id=package_entry.id) db.session.add(install) db.session.commit() sys.stderr.write("add_dependencies: ending\n") return
def add_dependencies(user_id, start_time=None): if start_time is None: start_time = time.time() #sys.stderr.write('add_dependencies: user_id is ' + str(user_id) + "\n") sys.stderr.write("add_dependencies: starting after " + str(time.time() - start_time) + " seconds\n") from docassemble.base.config import hostname from docassemble.webapp.app_object import app from docassemble.webapp.db_object import db from docassemble.webapp.packages.models import Package, Install, PackageAuth from sqlalchemy import select, delete packages_known = set() for package in db.session.execute(select(Package.name).filter_by(active=True)): packages_known.add(package.name) installed_packages = get_installed_distributions(start_time=start_time) home_pages = None packages_to_add = list() for package in installed_packages: if package.key in packages_known: continue if package.key.startswith('mysqlclient') or package.key.startswith('mysql-connector') or package.key.startswith('MySQL-python'): continue db.session.execute(delete(Package).filter_by(name=package.key)) packages_to_add.append(package) did_something = False if len(packages_to_add): did_something = True db.session.commit() for package in packages_to_add: package_auth = PackageAuth(user_id=user_id) if package.key.startswith('docassemble.'): if home_pages is None: home_pages = get_home_page_dict() home_page = home_pages.get(package.key.lower(), None) if home_page is not None and re.search(r'/github.com/', home_page): package_entry = Package(name=package.key, package_auth=package_auth, type='git', giturl=home_page, packageversion=package.version, dependency=True) else: package_entry = Package(name=package.key, package_auth=package_auth, type='pip', packageversion=package.version, dependency=True) else: package_entry = Package(name=package.key, package_auth=package_auth, type='pip', packageversion=package.version, dependency=True) db.session.add(package_entry) db.session.commit() install = Install(hostname=hostname, packageversion=package_entry.packageversion, version=package_entry.version, package_id=package_entry.id) db.session.add(install) db.session.commit() sys.stderr.write("add_dependencies: ending after " + str(time.time() - start_time) + " seconds\n") return did_something
def add_dependencies(user_id): logmessage('add_dependencies: ' + str(user_id)) from docassemble.webapp.config import hostname package_by_name = dict() for package in Package.query.filter_by(active=True).all(): package_by_name[package.name] = package installed_packages = get_installed_distributions() for package in installed_packages: if package.key in package_by_name: continue package_auth = PackageAuth(user_id=user_id) if package.key in ['docassemble', 'docassemble.base', 'docassemble.webapp', 'docassemble.demo']: package_entry = Package(name=package.key, package_auth=package_auth, giturl=docassemble_git_url, packageversion=package.version, gitsubdir=re.sub(r'\.', '-', package.key), type='git', core=True) else: package_entry = Package(name=package.key, package_auth=package_auth, type='pip', packageversion=package.version, dependency=True) db.session.add(package_auth) db.session.add(package_entry) db.session.commit() install = Install(hostname=hostname, packageversion=package_entry.packageversion, version=package_entry.version, package_id=package_entry.id) db.session.add(install) db.session.commit() return
def check_for_updates(doing_startup=False, start_time=None, invalidate_cache=True, full=True): if start_time is None: start_time = time.time() sys.stderr.write("check_for_updates: starting after " + str(time.time() - start_time) + " seconds\n") if invalidate_cache: invalidate_installed_distributions_cache() from docassemble.base.config import hostname from docassemble.webapp.app_object import app from docassemble.webapp.db_object import db from docassemble.webapp.packages.models import Package, Install, PackageAuth ok = True here_already = dict() results = dict() if full: sys.stderr.write("check_for_updates: 0.5 after " + str(time.time() - start_time) + " seconds\n") for package_name in ('psycopg2', 'pdfminer', 'pdfminer3k', 'py-bcrypt', 'pycrypto', 'constraint', 'distutils2', 'azure-storage', 'Flask-User'): num_deleted = Package.query.filter_by(name=package_name).delete() if num_deleted > 0: db.session.commit() sys.stderr.write("check_for_updates: 1 after " + str(time.time() - start_time) + " seconds\n") installed_packages = get_installed_distributions(start_time=start_time) for package in installed_packages: here_already[package.key] = package.version changed = False if full: if 'psycopg2' in here_already: sys.stderr.write("check_for_updates: uninstalling psycopg2\n") uninstall_package(DummyPackage('psycopg2'), start_time=start_time) if 'psycopg2-binary' in here_already: sys.stderr.write( "check_for_updates: reinstalling psycopg2-binary\n") uninstall_package(DummyPackage('psycopg2-binary'), start_time=start_time) install_package(DummyPackage('psycopg2-binary'), start_time=start_time) changed = True if 'psycopg2-binary' not in here_already: sys.stderr.write("check_for_updates: installing psycopg2-binary\n") install_package(DummyPackage('psycopg2-binary'), start_time=start_time) change = True if 'kombu' not in here_already or LooseVersion( here_already['kombu']) <= LooseVersion('4.1.0'): sys.stderr.write( "check_for_updates: installing new kombu version\n") install_package(DummyPackage('kombu'), start_time=start_time) changed = True if 'celery' not in here_already or LooseVersion( here_already['celery']) <= LooseVersion('4.1.0'): sys.stderr.write( "check_for_updates: installing new celery version\n") install_package(DummyPackage('celery'), start_time=start_time) changed = True if 'pycrypto' in here_already: sys.stderr.write("check_for_updates: uninstalling pycrypto\n") uninstall_package(DummyPackage('pycrypto'), start_time=start_time) if 'pycryptodome' in here_already: sys.stderr.write( "check_for_updates: reinstalling pycryptodome\n") uninstall_package(DummyPackage('pycryptodome'), start_time=start_time) install_package(DummyPackage('pycryptodome'), start_time=start_time) changed = True if 'pycryptodome' not in here_already: sys.stderr.write("check_for_updates: installing pycryptodome\n") install_package(DummyPackage('pycryptodome'), start_time=start_time) changed = True if 'pdfminer' in here_already: sys.stderr.write("check_for_updates: uninstalling pdfminer\n") uninstall_package(DummyPackage('pdfminer'), start_time=start_time) changed = True if 'pdfminer3k' in here_already: sys.stderr.write("check_for_updates: uninstalling pdfminer3k\n") uninstall_package(DummyPackage('pdfminer3k'), start_time=start_time) changed = True if 'pdfminer.six' in here_already: try: from pdfminer.pdfparser import PDFParser from pdfminer.pdfdocument import PDFDocument except: sys.stderr.write( "check_for_updates: reinstalling pdfminer.six\n") uninstall_package(DummyPackage('pdfminer.six'), start_time=start_time) install_package(DummyPackage('pdfminer.six'), start_time=start_time) else: sys.stderr.write("check_for_updates: installing pdfminer.six\n") install_package(DummyPackage('pdfminer.six'), start_time=start_time) changed = True if 'py-bcrypt' in here_already: sys.stderr.write("check_for_updates: uninstalling py-bcrypt\n") uninstall_package(DummyPackage('py-bcrypt'), start_time=start_time) changed = True if 'bcrypt' in here_already: sys.stderr.write("check_for_updates: reinstalling bcrypt\n") uninstall_package(DummyPackage('bcrypt'), start_time=start_time) install_package(DummyPackage('bcrypt'), start_time=start_time) changed = True if 'bcrypt' not in here_already: sys.stderr.write("check_for_updates: installing bcrypt\n") install_package(DummyPackage('bcrypt'), start_time=start_time) changed = True if changed: installed_packages = get_installed_distributions( start_time=start_time) here_already = dict() for package in installed_packages: here_already[package.key] = package.version packages = dict() installs = dict() to_install = list() to_uninstall = list() uninstall_done = dict() uninstalled_packages = dict() logmessages = '' package_by_name = dict() sys.stderr.write("check_for_updates: 2 after " + str(time.time() - start_time) + " seconds\n") for package in Package.query.filter_by(active=True).all(): package_by_name[package.name] = package #sys.stderr.write("check_for_updates: database includes a package called " + package.name + " after " + str(time.time() - start_time) + " seconds\n") # packages is what is supposed to be installed sys.stderr.write("check_for_updates: 3 after " + str(time.time() - start_time) + " seconds\n") for package in Package.query.filter_by(active=True).all(): if package.type is not None: packages[package.id] = package #sys.stderr.write("check_for_updates: database includes a package called " + package.name + " that has a type after " + str(time.time() - start_time) + " seconds\n") #print("Found a package " + package.name) sys.stderr.write("check_for_updates: 4 after " + str(time.time() - start_time) + " seconds\n") for package in Package.query.filter_by(active=False).all(): if package.name not in package_by_name: #sys.stderr.write("check_for_updates: database says " + package.name + " should be uninstalled after " + str(time.time() - start_time) + " seconds\n") uninstalled_packages[ package. id] = package # this is what the database says should be uninstalled sys.stderr.write("check_for_updates: 5 after " + str(time.time() - start_time) + " seconds\n") for install in Install.query.filter_by(hostname=hostname).all(): installs[ install. package_id] = install # this is what the database says in installed on this server if install.package_id in uninstalled_packages and uninstalled_packages[ install.package_id].name not in package_by_name: sys.stderr.write("check_for_updates: " + uninstalled_packages[install.package_id].name + " will be uninstalled after " + str(time.time() - start_time) + " seconds\n") to_uninstall.append(uninstalled_packages[ install.package_id]) # uninstall if it is installed changed = False package_owner = dict() sys.stderr.write("check_for_updates: 6 after " + str(time.time() - start_time) + " seconds\n") for auth in PackageAuth.query.filter_by(authtype='owner').all(): package_owner[auth.package_id] = auth.user_id sys.stderr.write("check_for_updates: 7 after " + str(time.time() - start_time) + " seconds\n") for package in packages.values(): if package.id not in installs and package.name in here_already: sys.stderr.write( "check_for_updates: package " + package.name + " here already. Writing an Install record for it.\n") install = Install(hostname=hostname, packageversion=here_already[package.name], version=package.version, package_id=package.id) db.session.add(install) installs[package.id] = install changed = True if changed: db.session.commit() sys.stderr.write("check_for_updates: 8 after " + str(time.time() - start_time) + " seconds\n") for package in packages.values(): #sys.stderr.write("check_for_updates: processing package id " + str(package.id) + "\n") #sys.stderr.write("1: " + str(installs[package.id].packageversion) + " 2: " + str(package.packageversion) + "\n") if (package.packageversion is not None and package.id in installs and installs[package.id].packageversion is None ) or (package.packageversion is not None and package.id in installs and installs[package.id].packageversion is not None and LooseVersion(package.packageversion) > LooseVersion( installs[package.id].packageversion)): sys.stderr.write( "check_for_updates: a new version of " + package.name + " is needed because the necessary package version, " + str(package.packageversion) + ", is ahead of the installed version, " + str(installs[package.id].packageversion) + " after " + str(time.time() - start_time) + " seconds\n") new_version_needed = True else: new_version_needed = False #sys.stderr.write("got here and new version is " + str(new_version_needed) + "\n") # Check for missing local packages if (package.name not in here_already) and (package.id in installs): sys.stderr.write( "check_for_updates: the package " + package.name + " is supposed to be installed on this server, but was not detected after " + str(time.time() - start_time) + " seconds\n") package_missing = True else: package_missing = False if package.id in installs and package.version > installs[ package.id].version: sys.stderr.write("check_for_updates: the package " + package.name + " has internal version " + str(package.version) + " but the installed version has version " + str(installs[package.id].version) + " after " + str(time.time() - start_time) + " seconds\n") package_version_greater = True else: package_version_greater = False if package.id not in installs: sys.stderr.write( "check_for_updates: the package " + package.name + " is not in the table of installed packages for this server after " + str(time.time() - start_time) + " seconds\n") if package.id not in installs or package_version_greater or new_version_needed or package_missing: to_install.append(package) #sys.stderr.write("done with that" + "\n") sys.stderr.write("check_for_updates: 9 after " + str(time.time() - start_time) + " seconds\n") for package in to_uninstall: #sys.stderr.write("Going to uninstall a package: " + package.name + "\n") if package.name in uninstall_done: sys.stderr.write("check_for_updates: skipping uninstallation of " + str(package.name) + " because already uninstalled after " + str(time.time() - start_time) + " seconds" + "\n") continue if package.name not in here_already: sys.stderr.write("check_for_updates: skipping uninstallation of " + str(package.name) + " because not installed" + " after " + str(time.time() - start_time) + " seconds\n") returnval = 1 newlog = '' else: sys.stderr.write( "check_for_updates: calling uninstall_package on " + package.name + "\n") returnval, newlog = uninstall_package(package, start_time=start_time) uninstall_done[package.name] = 1 logmessages += newlog if returnval == 0: Install.query.filter_by(hostname=hostname, package_id=package.id).delete() results[ package. name] = 'pip uninstall command returned success code. See log for details.' elif returnval == 1: Install.query.filter_by(hostname=hostname, package_id=package.id).delete() results[ package. name] = 'pip uninstall was not run because the package was not installed.' else: results[ package.name] = 'pip uninstall command returned failure code' ok = False packages_to_delete = list() sys.stderr.write("check_for_updates: 10 after " + str(time.time() - start_time) + " seconds\n") did_something = False for package in to_install: did_something = True sys.stderr.write("check_for_updates: going to install a package: " + package.name + " after " + str(time.time() - start_time) + " seconds\n") # if doing_startup and package.name.startswith('docassemble') and package.name in here_already: # #adding this because of unpredictability of installing new versions of docassemble # #just because of a system restart. # sys.stderr.write("check_for_updates: skipping update on " + str(package.name) + "\n") # continue returnval, newlog = install_package(package, start_time=start_time) logmessages += newlog sys.stderr.write("check_for_updates: return value was " + str(returnval) + " after " + str(time.time() - start_time) + " seconds\n") if returnval != 0: sys.stderr.write("Return value was not good" + " after " + str(time.time() - start_time) + " seconds\n") ok = False #pip._vendor.pkg_resources._initialize_master_working_set() if full: pip_info = get_pip_info(package.name, start_time=start_time) real_name = pip_info['Name'] sys.stderr.write("check_for_updates: real name of package " + str(package.name) + " is " + str(real_name) + " after " + str(time.time() - start_time) + " seconds\n") else: real_name = package.name if real_name is None: results[package.name] = 'install failed' ok = False if package.name not in here_already: sys.stderr.write( "check_for_updates: removing package entry for " + package.name + " after " + str(time.time() - start_time) + " seconds\n") packages_to_delete.append(package) elif returnval != 0: results[package.name] = 'pip install command returned failure code' else: results[ package. name] = 'pip install command returned success code. See log for details.' if real_name != package.name: sys.stderr.write("check_for_updates: changing name" + " after " + str(time.time() - start_time) + " seconds\n") package.name = real_name if package.id in installs: install = installs[package.id] install.version = package.version else: install = Install(hostname=hostname, packageversion=package.packageversion, version=package.version, package_id=package.id) db.session.add(install) db.session.commit() if did_something: update_versions(start_time=start_time) if full and add_dependencies(package_owner.get(package.id, 1), start_time=start_time): update_versions(start_time=start_time) sys.stderr.write("check_for_updates: 11 after " + str(time.time() - start_time) + " seconds\n") for package in packages_to_delete: db.session.delete(package) sys.stderr.write("check_for_updates: 12 after " + str(time.time() - start_time) + " seconds\n") db.session.commit() sys.stderr.write( "check_for_updates: finished uninstalling and installing after " + str(time.time() - start_time) + " seconds\n") return ok, logmessages, results
def check_for_updates(doing_startup=False): sys.stderr.write("check_for_updates: starting\n") from docassemble.base.config import hostname from docassemble.webapp.app_object import app from docassemble.webapp.db_object import db from docassemble.webapp.packages.models import Package, Install, PackageAuth ok = True here_already = dict() results = dict() sys.stderr.write("check_for_updates: 0.5\n") num_deleted = Package.query.filter_by(name='psycopg2').delete() if num_deleted > 0: db.session.commit() if PY2: num_deleted = Package.query.filter_by(name='pycryptodome').delete() if num_deleted > 0: db.session.commit() num_deleted = Package.query.filter_by(name='pdfminer3k').delete() if num_deleted > 0: db.session.commit() else: num_deleted = Package.query.filter_by(name='pdfminer').delete() if num_deleted > 0: db.session.commit() num_deleted = Package.query.filter_by(name='py-bcrypt').delete() if num_deleted > 0: db.session.commit() num_deleted = Package.query.filter_by(name='pycrypto').delete() if num_deleted > 0: db.session.commit() num_deleted = Package.query.filter_by(name='constraint').delete() if num_deleted > 0: db.session.commit() num_deleted = Package.query.filter_by(name='distutils2').delete() if num_deleted > 0: db.session.commit() sys.stderr.write("check_for_updates: 1\n") installed_packages = get_installed_distributions() for package in installed_packages: here_already[package.key] = package.version changed = False if 'pdfminer.six' not in here_already: sys.stderr.write("check_for_updates: installing pdfminer.six\n") install_package(DummyPackage('pdfminer.six')) changed = True if 'psycopg2' in here_already: sys.stderr.write("check_for_updates: uninstalling psycopg2\n") uninstall_package(DummyPackage('psycopg2')) if 'psycopg2-binary' in here_already: sys.stderr.write( "check_for_updates: reinstalling psycopg2-binary\n") uninstall_package(DummyPackage('psycopg2-binary')) install_package(DummyPackage('psycopg2-binary')) changed = True if 'psycopg2-binary' not in here_already: sys.stderr.write("check_for_updates: installing psycopg2-binary\n") install_package(DummyPackage('psycopg2-binary')) change = True if PY2: if 'pycryptodome' in here_already: sys.stderr.write("check_for_updates: uninstalling pycryptodome\n") uninstall_package(DummyPackage('pycryptodome')) if 'pycrypto' in here_already: sys.stderr.write("check_for_updates: reinstalling pycrypto\n") uninstall_package(DummyPackage('pycrypto')) install_package(DummyPackage('pycrypto')) changed = True if 'pathlib' not in here_already: sys.stderr.write("check_for_updates: installing pathlib\n") install_package(DummyPackage('pathlib')) changed = True if 'pycrypto' not in here_already: sys.stderr.write("check_for_updates: installing pycrypto\n") install_package(DummyPackage('pycrypto')) changed = True if 'kombu' not in here_already or LooseVersion( here_already['kombu']) > LooseVersion('4.1.0'): sys.stderr.write( "check_for_updates: installing older kombu version\n") kombu = DummyPackage('kombu') kombu.limitation = '==4.1.0' install_package(kombu) changed = True if 'celery' not in here_already or LooseVersion( here_already['celery']) > LooseVersion('4.1.0'): sys.stderr.write( "check_for_updates: installing older celery version\n") celery = DummyPackage('celery') celery.limitation = '[redis]==4.1.0' install_package(celery) changed = True if 'pdfminer' not in here_already: sys.stderr.write("check_for_updates: installing pdfminer\n") pdfminer = DummyPackage('pdfminer') pdfminer.type = 'git' pdfminer.giturl = 'https://github.com/euske/pdfminer' pdfminer.gitsubdir = None pdfminer.gitbranch = None install_package(pdfminer) changed = True else: if 'kombu' not in here_already or LooseVersion( here_already['kombu']) <= LooseVersion('4.1.0'): sys.stderr.write( "check_for_updates: installing new kombu version\n") install_package(DummyPackage('kombu')) changed = True if 'celery' not in here_already or LooseVersion( here_already['celery']) <= LooseVersion('4.1.0'): sys.stderr.write( "check_for_updates: installing new celery version\n") install_package(DummyPackage('celery')) changed = True if 'pycrypto' in here_already: sys.stderr.write("check_for_updates: uninstalling pycrypto\n") uninstall_package(DummyPackage('pycrypto')) if 'pycryptodome' in here_already: sys.stderr.write( "check_for_updates: reinstalling pycryptodome\n") uninstall_package(DummyPackage('pycryptodome')) install_package(DummyPackage('pycryptodome')) changed = True if 'pycryptodome' not in here_already: sys.stderr.write("check_for_updates: installing pycryptodome\n") install_package(DummyPackage('pycryptodome')) changed = True if 'pdfminer' in here_already: sys.stderr.write("check_for_updates: uninstalling pdfminer\n") uninstall_package(DummyPackage('pdfminer')) changed = True if 'pdfminer3k' not in here_already: sys.stderr.write("check_for_updates: installing pdfminer3k\n") install_package(DummyPackage('pdfminer3k')) changed = True if 'py-bcrypt' in here_already: sys.stderr.write("check_for_updates: uninstalling py-bcrypt\n") uninstall_package(DummyPackage('py-bcrypt')) changed = True if 'bcrypt' in here_already: sys.stderr.write("check_for_updates: reinstalling bcrypt\n") uninstall_package(DummyPackage('bcrypt')) install_package(DummyPackage('bcrypt')) changed = True if 'bcrypt' not in here_already: sys.stderr.write("check_for_updates: installing bcrypt\n") install_package(DummyPackage('bcrypt')) changed = True if changed: installed_packages = get_installed_distributions() here_already = dict() for package in installed_packages: here_already[package.key] = package.version packages = dict() installs = dict() to_install = list() to_uninstall = list() uninstall_done = dict() uninstalled_packages = dict() logmessages = '' package_by_name = dict() sys.stderr.write("check_for_updates: 2\n") for package in Package.query.filter_by(active=True).all(): package_by_name[package.name] = package # packages is what is supposed to be installed sys.stderr.write("check_for_updates: 3\n") for package in Package.query.filter_by(active=True).all(): if package.type is not None: packages[package.id] = package #print("Found a package " + package.name) sys.stderr.write("check_for_updates: 4\n") for package in Package.query.filter_by(active=False).all(): if package.name not in package_by_name: uninstalled_packages[ package. id] = package # this is what the database says should be uninstalled sys.stderr.write("check_for_updates: 5\n") for install in Install.query.filter_by(hostname=hostname).all(): installs[ install. package_id] = install # this is what the database says in installed on this server if install.package_id in uninstalled_packages and uninstalled_packages[ install.package_id].name not in package_by_name: to_uninstall.append(uninstalled_packages[ install.package_id]) # uninstall if it is installed changed = False package_owner = dict() sys.stderr.write("check_for_updates: 6\n") for auth in PackageAuth.query.filter_by(authtype='owner').all(): package_owner[auth.package_id] = auth.user_id sys.stderr.write("check_for_updates: 7\n") for package in packages.values(): if package.id not in installs and package.name in here_already: sys.stderr.write("check_for_updates: package " + package.name + " here already\n") install = Install(hostname=hostname, packageversion=here_already[package.name], version=package.version, package_id=package.id) db.session.add(install) installs[package.id] = install changed = True if changed: db.session.commit() sys.stderr.write("check_for_updates: 8\n") for package in packages.values(): #sys.stderr.write("check_for_updates: processing package id " + str(package.id) + "\n") #sys.stderr.write("1: " + str(installs[package.id].packageversion) + " 2: " + str(package.packageversion) + "\n") if (package.packageversion is not None and package.id in installs and installs[package.id].packageversion is None ) or (package.packageversion is not None and package.id in installs and installs[package.id].packageversion is not None and LooseVersion(package.packageversion) > LooseVersion( installs[package.id].packageversion)): new_version_needed = True else: new_version_needed = False #sys.stderr.write("got here and new version is " + str(new_version_needed) + "\n") # Check for missing local packages if (package.name not in here_already) and (package.id in installs): package_missing = True else: package_missing = False if package.id not in installs or package.version > installs[ package.id].version or new_version_needed or package_missing: to_install.append(package) #sys.stderr.write("done with that" + "\n") sys.stderr.write("check_for_updates: 9\n") for package in to_uninstall: #sys.stderr.write("Going to uninstall a package: " + package.name + "\n") if package.name in uninstall_done: sys.stderr.write("check_for_updates: skipping uninstallation of " + str(package.name) + " because already uninstalled" + "\n") continue if package.name not in here_already: sys.stderr.write("check_for_updates: skipping uninstallation of " + str(package.name) + " because not installed" + "\n") returnval = 1 newlog = '' else: returnval, newlog = uninstall_package(package) uninstall_done[package.name] = 1 logmessages += newlog if returnval == 0: Install.query.filter_by(hostname=hostname, package_id=package.id).delete() results[ package. name] = 'pip uninstall command returned success code. See log for details.' elif returnval == 1: Install.query.filter_by(hostname=hostname, package_id=package.id).delete() results[ package. name] = 'pip uninstall was not run because the package was not installed.' else: results[ package.name] = 'pip uninstall command returned failure code' ok = False packages_to_delete = list() sys.stderr.write("check_for_updates: 10\n") for package in to_install: sys.stderr.write("check_for_updates: going to install a package: " + package.name + "\n") # if doing_startup and package.name.startswith('docassemble') and package.name in here_already: # #adding this because of unpredictability of installing new versions of docassemble # #just because of a system restart. # sys.stderr.write("check_for_updates: skipping update on " + str(package.name) + "\n") # continue returnval, newlog = install_package(package) logmessages += newlog sys.stderr.write("check_for_updates: return value was " + str(returnval) + "\n") if returnval != 0: sys.stderr.write("Return value was not good" + "\n") ok = False #pip._vendor.pkg_resources._initialize_master_working_set() pip_info = get_pip_info(package.name) real_name = pip_info['Name'] sys.stderr.write("check_for_updates: real name of package " + str(package.name) + " is " + str(real_name) + "\n") if real_name is None: results[package.name] = 'install failed' ok = False if package.name not in here_already: sys.stderr.write( "check_for_updates: removing package entry for " + package.name + "\n") packages_to_delete.append(package) elif returnval != 0: results[package.name] = 'pip install command returned failure code' else: results[ package. name] = 'pip install command returned success code. See log for details.' if real_name != package.name: sys.stderr.write("check_for_updates: changing name" + "\n") package.name = real_name if package.id in installs: install = installs[package.id] install.version = package.version else: install = Install(hostname=hostname, packageversion=package.packageversion, version=package.version, package_id=package.id) db.session.add(install) db.session.commit() update_versions() add_dependencies(package_owner.get(package.id, 1)) update_versions() sys.stderr.write("check_for_updates: 11\n") for package in packages_to_delete: db.session.delete(package) sys.stderr.write("check_for_updates: 12\n") db.session.commit() sys.stderr.write( "check_for_updates: finished uninstalling and installing\n") return ok, logmessages, results
def check_for_updates(doing_startup=False): sys.stderr.write("check_for_updates: starting\n") from docassemble.base.config import hostname ok = True here_already = dict() results = dict() sys.stderr.write("check_for_updates: 1\n") installed_packages = get_installed_distributions() for package in installed_packages: here_already[package.key] = package.version packages = dict() installs = dict() to_install = list() to_uninstall = list() uninstall_done = dict() uninstalled_packages = dict() logmessages = '' package_by_name = dict() sys.stderr.write("check_for_updates: 2\n") for package in Package.query.filter_by(active=True).all(): package_by_name[package.name] = package # packages is what is supposed to be installed sys.stderr.write("check_for_updates: 3\n") for package in Package.query.filter_by(active=True).all(): if package.type is not None: packages[package.id] = package #print "Found a package " + package.name sys.stderr.write("check_for_updates: 4\n") for package in Package.query.filter_by(active=False).all(): if package.name not in package_by_name: uninstalled_packages[ package. id] = package # this is what the database says should be uninstalled sys.stderr.write("check_for_updates: 5\n") for install in Install.query.filter_by(hostname=hostname).all(): installs[ install. package_id] = install # this is what the database says in installed on this server if install.package_id in uninstalled_packages and uninstalled_packages[ install.package_id].name not in package_by_name: to_uninstall.append(uninstalled_packages[ install.package_id]) # uninstall if it is installed changed = False package_owner = dict() sys.stderr.write("check_for_updates: 6\n") for auth in PackageAuth.query.filter_by(authtype='owner').all(): package_owner[auth.package_id] = auth.user_id sys.stderr.write("check_for_updates: 7\n") for package in packages.itervalues(): if package.id not in installs and package.name in here_already: sys.stderr.write("check_for_updates: package " + package.name + " here already\n") install = Install(hostname=hostname, packageversion=here_already[package.name], version=package.version, package_id=package.id) db.session.add(install) installs[package.id] = install changed = True if changed: db.session.commit() sys.stderr.write("check_for_updates: 8\n") for package in packages.itervalues(): #sys.stderr.write("check_for_updates: processing package id " + str(package.id) + "\n") #sys.stderr.write("1: " + str(installs[package.id].packageversion) + " 2: " + str(package.packageversion) + "\n") if (package.packageversion is not None and package.id in installs and installs[package.id].packageversion is None ) or (package.packageversion is not None and package.id in installs and installs[package.id].packageversion is not None and LooseVersion(package.packageversion) > LooseVersion( installs[package.id].packageversion)): new_version_needed = True else: new_version_needed = False #sys.stderr.write("got here and new version is " + str(new_version_needed) + "\n") if package.id not in installs or package.version > installs[ package.id].version or new_version_needed: to_install.append(package) #sys.stderr.write("done with that" + "\n") sys.stderr.write("check_for_updates: 9\n") for package in to_uninstall: #sys.stderr.write("Going to uninstall a package: " + package.name + "\n") if package.name in uninstall_done: sys.stderr.write("check_for_updates: skipping uninstallation of " + str(package.name) + " because already uninstalled" + "\n") continue returnval, newlog = uninstall_package(package) uninstall_done[package.name] = 1 logmessages += newlog if returnval == 0: Install.query.filter_by(hostname=hostname, package_id=package.id).delete() results[package.name] = 'successfully uninstalled' else: results[package.name] = 'uninstall failed' ok = False packages_to_delete = list() sys.stderr.write("check_for_updates: 10\n") for package in to_install: sys.stderr.write("check_for_updates: going to install a package: " + package.name + "\n") # if doing_startup and package.name.startswith('docassemble') and package.name in here_already: # #adding this because of unpredictability of installing new versions of docassemble # #just because of a system restart. # sys.stderr.write("check_for_updates: skipping update on " + str(package.name) + "\n") # continue returnval, newlog = install_package(package) logmessages += newlog sys.stderr.write("check_for_updates: return value was " + str(returnval) + "\n") if returnval != 0: sys.stderr.write("Return value was not good" + "\n") ok = False #pip._vendor.pkg_resources._initialize_master_working_set() pip_info = get_pip_info(package.name) real_name = pip_info['Name'] sys.stderr.write("check_for_updates: real name of package " + str(package.name) + " is " + str(real_name) + "\n") if real_name is None: results[package.name] = 'install failed' ok = False if package.name not in here_already: sys.stderr.write( "check_for_updates: removing package entry for " + package.name + "\n") packages_to_delete.append(package) elif returnval != 0: results[package.name] = 'could not be upgraded' else: results[package.name] = 'successfully installed' if real_name != package.name: sys.stderr.write("check_for_updates: changing name" + "\n") package.name = real_name if package.id in installs: install = installs[package.id] install.version = package.version else: install = Install(hostname=hostname, packageversion=package.packageversion, version=package.version, package_id=package.id) db.session.add(install) db.session.commit() update_versions() add_dependencies(package_owner.get(package.id, 1)) update_versions() sys.stderr.write("check_for_updates: 11\n") for package in packages_to_delete: package.active = False sys.stderr.write("check_for_updates: 12\n") db.session.commit() sys.stderr.write( "check_for_updates: finished uninstalling and installing\n") return ok, logmessages, results
def check_for_updates(start_time=None, invalidate_cache=True, full=True): if start_time is None: start_time = time.time() sys.stderr.write("check_for_updates: starting after " + str(time.time() - start_time) + " seconds\n") if invalidate_cache: invalidate_installed_distributions_cache() from docassemble.base.config import hostname #from docassemble.webapp.app_object import app from docassemble.webapp.db_object import db from docassemble.webapp.packages.models import Package, Install, PackageAuth from sqlalchemy import select, delete ok = True # tracks whether there have been any errors here_already = {} # packages that are installed on this server, based on pip list. package name -> package version results = {} # result of actions taken on a package. package name -> message if full: sys.stderr.write("check_for_updates: 0.5 after " + str(time.time() - start_time) + " seconds\n") for package_name in ('psycopg2', 'pdfminer', 'pdfminer3k', 'py-bcrypt', 'pycrypto', 'constraint', 'distutils2', 'azure-storage', 'Flask-User', 'Marisol'): result = db.session.execute(delete(Package).filter_by(name=package_name)) if result.rowcount > 0: db.session.commit() sys.stderr.write("check_for_updates: 1 after " + str(time.time() - start_time) + " seconds\n") installed_packages = get_installed_distributions(start_time=start_time) for package in installed_packages: here_already[package.key] = package.version changed = False # clean up old packages that cause problems. if full: if 'psycopg2' in here_already: sys.stderr.write("check_for_updates: uninstalling psycopg2\n") uninstall_package(DummyPackage('psycopg2'), start_time=start_time) if 'psycopg2-binary' in here_already: sys.stderr.write("check_for_updates: reinstalling psycopg2-binary\n") uninstall_package(DummyPackage('psycopg2-binary'), start_time=start_time) install_package(DummyPackage('psycopg2-binary'), start_time=start_time) changed = True if 'psycopg2-binary' not in here_already: sys.stderr.write("check_for_updates: installing psycopg2-binary\n") install_package(DummyPackage('psycopg2-binary'), start_time=start_time) changed = True if 'kombu' not in here_already or LooseVersion(here_already['kombu']) <= LooseVersion('4.1.0'): sys.stderr.write("check_for_updates: installing new kombu version\n") install_package(DummyPackage('kombu'), start_time=start_time) changed = True if 'celery' not in here_already or LooseVersion(here_already['celery']) <= LooseVersion('4.1.0'): sys.stderr.write("check_for_updates: installing new celery version\n") install_package(DummyPackage('celery'), start_time=start_time) changed = True if 'pycrypto' in here_already: sys.stderr.write("check_for_updates: uninstalling pycrypto\n") uninstall_package(DummyPackage('pycrypto'), start_time=start_time) if 'pycryptodome' in here_already: sys.stderr.write("check_for_updates: reinstalling pycryptodome\n") uninstall_package(DummyPackage('pycryptodome'), start_time=start_time) install_package(DummyPackage('pycryptodome'), start_time=start_time) changed = True if 'pycryptodome' not in here_already: sys.stderr.write("check_for_updates: installing pycryptodome\n") install_package(DummyPackage('pycryptodome'), start_time=start_time) changed = True if 'pdfminer' in here_already: sys.stderr.write("check_for_updates: uninstalling pdfminer\n") uninstall_package(DummyPackage('pdfminer'), start_time=start_time) changed = True if 'Marisol' in here_already: sys.stderr.write("check_for_updates: uninstalling Marisol\n") uninstall_package(DummyPackage('Marisol'), start_time=start_time) changed = True if 'pdfminer3k' in here_already: sys.stderr.write("check_for_updates: uninstalling pdfminer3k\n") uninstall_package(DummyPackage('pdfminer3k'), start_time=start_time) changed = True if 'pdfminer.six' in here_already: try: from pdfminer.pdfparser import PDFParser from pdfminer.pdfdocument import PDFDocument except: sys.stderr.write("check_for_updates: reinstalling pdfminer.six\n") uninstall_package(DummyPackage('pdfminer.six'), start_time=start_time) install_package(DummyPackage('pdfminer.six'), start_time=start_time) else: sys.stderr.write("check_for_updates: installing pdfminer.six\n") install_package(DummyPackage('pdfminer.six'), start_time=start_time) changed = True if 'py-bcrypt' in here_already: sys.stderr.write("check_for_updates: uninstalling py-bcrypt\n") uninstall_package(DummyPackage('py-bcrypt'), start_time=start_time) changed = True if 'bcrypt' in here_already: sys.stderr.write("check_for_updates: reinstalling bcrypt\n") uninstall_package(DummyPackage('bcrypt'), start_time=start_time) install_package(DummyPackage('bcrypt'), start_time=start_time) changed = True if 'bcrypt' not in here_already: sys.stderr.write("check_for_updates: installing bcrypt\n") install_package(DummyPackage('bcrypt'), start_time=start_time) changed = True if changed: installed_packages = get_installed_distributions(start_time=start_time) here_already = {} for package in installed_packages: here_already[package.key] = package.version logmessages = '' packages = {} # packages that the database says are active and that have a type; package id -> package row installs = {} # install rows representing what the database says in installed; package id -> install row to_install = [] # package rows of packages to install to_uninstall = [] # package rows of packages to uninstall system_packages_to_fix = [] uninstall_done = set() # set of package names of packages already uninstalled uninstalled_packages = {} # packages with active=False; package id -> package row package_ids_to_delete = set() # set if IDs of package rows that have active=False but there is another # row with active= True package_by_name = {} # packages the database says are active; package name -> package row sys.stderr.write("check_for_updates: 2 after " + str(time.time() - start_time) + " seconds\n") # create dict package_by_name that maps a package name to a package database row, # for packages that the database says should be installed for package in db.session.execute(select(Package.name).filter_by(active=True)): package_by_name[package.name] = package #sys.stderr.write("check_for_updates: database includes a package called " + package.name + " after " + str(time.time() - start_time) + " seconds\n") # packages is what is supposed to be installed sys.stderr.write("check_for_updates: 3 after " + str(time.time() - start_time) + " seconds\n") # create dict packages that maps a package ID to a package database row if the type is not null for package in db.session.execute(select(Package).filter_by(active=True)).scalars(): if package.type is not None: packages[package.id] = package #sys.stderr.write("check_for_updates: database includes a package called " + package.name + " that has a type after " + str(time.time() - start_time) + " seconds\n") #print("Found a package " + package.name) sys.stderr.write("check_for_updates: 4 after " + str(time.time() - start_time) + " seconds\n") # create dict uninstalled_packages that maps a package ID to a package database row if SQL wants the package deleted for package in db.session.execute(select(Package).filter_by(active=False)).scalars(): # don't uninstall a system package if package.name in system_packages: logmessages += "Not uninstalling " + str(package.name) + " because it is a system package.\n" system_packages_to_fix.append(package) continue if package.name in package_by_name: # there are two Package entries for the same package, one with active=True, one with active=False sys.stderr.write("check_for_updates: conflicting package entries for " + package.name + " after " + str(time.time() - start_time) + " seconds\n") # let's delete the one with active=False package_ids_to_delete.add(package.id) else: #sys.stderr.write("check_for_updates: database says " + package.name + " should be uninstalled after " + str(time.time() - start_time) + " seconds\n") uninstalled_packages[package.id] = package # this is what the database says should be uninstalled sys.stderr.write("check_for_updates: 5 after " + str(time.time() - start_time) + " seconds\n") # build to_uninstall, which is the list of Package database entries targeted for uninstallation. # Only add if there is an Install entry linked to the package id and there is no active Package row # with the same name. # Also build installs, which is a dict mapping package IDs to install rows, based on # what the Install table says for install in db.session.execute(select(Install).filter_by(hostname=hostname)).scalars(): installs[install.package_id] = install # this is what the database says in installed on this server if install.package_id in uninstalled_packages and uninstalled_packages[install.package_id].name not in package_by_name: sys.stderr.write("check_for_updates: " + uninstalled_packages[install.package_id].name + " will be uninstalled after " + str(time.time() - start_time) + " seconds\n") to_uninstall.append(uninstalled_packages[install.package_id]) # uninstall if it is installed changed = False package_owner = {} sys.stderr.write("check_for_updates: 6 after " + str(time.time() - start_time) + " seconds\n") for auth in db.session.execute(select(PackageAuth).filter_by(authtype='owner')).scalars(): package_owner[auth.package_id] = auth.user_id sys.stderr.write("check_for_updates: 7 after " + str(time.time() - start_time) + " seconds\n") # Add Install rows for any active Package rows with non-null types if the package is present on the server # Also add to the installs dict after adding. for package in packages.values(): if package.id not in installs and package.name in here_already: sys.stderr.write("check_for_updates: package " + package.name + " here already. Writing an Install record for it.\n") install = Install(hostname=hostname, packageversion=here_already[package.name], version=package.version, package_id=package.id) db.session.add(install) installs[package.id] = install changed = True if changed: db.session.commit() sys.stderr.write("check_for_updates: 8 after " + str(time.time() - start_time) + " seconds\n") for package in packages.values(): #sys.stderr.write("check_for_updates: processing package id " + str(package.id) + "\n") #sys.stderr.write("1: " + str(installs[package.id].packageversion) + " 2: " + str(package.packageversion) + "\n") if (package.packageversion is not None and package.id in installs and installs[package.id].packageversion is None) or (package.packageversion is not None and package.id in installs and installs[package.id].packageversion is not None and LooseVersion(package.packageversion) > LooseVersion(installs[package.id].packageversion)): sys.stderr.write("check_for_updates: a new version of " + package.name + " is needed because the necessary package version, " + str(package.packageversion) + ", is ahead of the installed version, " + str(installs[package.id].packageversion) + " after " + str(time.time() - start_time) + " seconds\n") new_version_needed = True else: new_version_needed = False #sys.stderr.write("got here and new version is " + str(new_version_needed) + "\n") # Check for missing local packages if (package.name not in here_already) and (package.id in installs): sys.stderr.write("check_for_updates: the package " + package.name + " is supposed to be installed on this server, but was not detected after " + str(time.time() - start_time) + " seconds\n") package_missing = True else: package_missing = False if package.id in installs and package.version > installs[package.id].version: sys.stderr.write("check_for_updates: the package " + package.name + " has internal version " + str(package.version) + " but the installed version has version " + str(installs[package.id].version) + " after " + str(time.time() - start_time) + " seconds\n") package_version_greater = True else: package_version_greater = False if package.id not in installs: sys.stderr.write("check_for_updates: the package " + package.name + " is not in the table of installed packages for this server after " + str(time.time() - start_time) + " seconds\n") if package.id not in installs or package_version_greater or new_version_needed or package_missing: if package.name in system_packages: logmessages += "Not upgrading " + str(package.name) + " because it is a system package and its version needs to be consistent with the version of the package that is required by the docassemble.webapp package.\n" sys.stderr.write("check_for_updates: the package " + package.name + " is a system package and cannot be updated except through docassemble.webapp after " + str(time.time() - start_time) + " seconds\n") system_packages_to_fix.append(package) else: to_install.append(package) #sys.stderr.write("done with that" + "\n") sys.stderr.write("check_for_updates: 9 after " + str(time.time() - start_time) + " seconds\n") for package in to_uninstall: #sys.stderr.write("Going to uninstall a package: " + package.name + "\n") if package.name in uninstall_done: sys.stderr.write("check_for_updates: skipping uninstallation of " + str(package.name) + " because already uninstalled after " + str(time.time() - start_time) + " seconds" + "\n") continue if package.name not in here_already: sys.stderr.write("check_for_updates: skipping uninstallation of " + str(package.name) + " because not installed" + " after " + str(time.time() - start_time) + " seconds\n") returnval = 1 newlog = '' else: sys.stderr.write("check_for_updates: calling uninstall_package on " + package.name + "\n") returnval, newlog = uninstall_package(package, start_time=start_time) uninstall_done.add(package.name) logmessages += newlog if returnval == 0: db.session.execute(delete(Install).filter_by(hostname=hostname, package_id=package.id)) results[package.name] = 'pip uninstall command returned success code. See log for details.' elif returnval == 1: db.session.execute(delete(Install).filter_by(hostname=hostname, package_id=package.id)) results[package.name] = 'pip uninstall was not run because the package was not installed.' else: results[package.name] = 'pip uninstall command returned failure code' ok = False packages_to_delete = [] sys.stderr.write("check_for_updates: 10 after " + str(time.time() - start_time) + " seconds\n") did_something = False for package in to_install: did_something = True sys.stderr.write("check_for_updates: going to install a package: " + package.name + " after " + str(time.time() - start_time) + " seconds\n") # if doing_startup and package.name.startswith('docassemble') and package.name in here_already: # #adding this because of unpredictability of installing new versions of docassemble # #just because of a system restart. # sys.stderr.write("check_for_updates: skipping update on " + str(package.name) + "\n") # continue returnval, newlog = install_package(package, start_time=start_time) logmessages += newlog sys.stderr.write("check_for_updates: return value was " + str(returnval) + " after " + str(time.time() - start_time) + " seconds\n") if returnval != 0: sys.stderr.write("Return value was not good" + " after " + str(time.time() - start_time) + " seconds\n") ok = False #pip._vendor.pkg_resources._initialize_master_working_set() if full: pip_info = get_pip_info(package.name, start_time=start_time) real_name = pip_info['Name'] sys.stderr.write("check_for_updates: real name of package " + str(package.name) + " is " + str(real_name) + " after " + str(time.time() - start_time) + " seconds\n") else: real_name = package.name if real_name is None: results[package.name] = 'install failed' ok = False if package.name not in here_already: sys.stderr.write("check_for_updates: removing package entry for " + package.name + " after " + str(time.time() - start_time) + " seconds\n") packages_to_delete.append(package) elif returnval != 0: results[package.name] = 'pip install command returned failure code' else: results[package.name] = 'pip install command returned success code. See log for details.' if real_name != package.name: sys.stderr.write("check_for_updates: changing name" + " after " + str(time.time() - start_time) + " seconds\n") package.name = real_name if package.id in installs: install = installs[package.id] install.version = package.version else: install = Install(hostname=hostname, packageversion=package.packageversion, version=package.version, package_id=package.id) db.session.add(install) db.session.commit() if did_something: update_versions(start_time=start_time) if full and add_dependencies(package_owner.get(package.id, 1), start_time=start_time): update_versions(start_time=start_time) sys.stderr.write("check_for_updates: 11 after " + str(time.time() - start_time) + " seconds\n") for package in packages_to_delete: db.session.delete(package) sys.stderr.write("check_for_updates: 12 after " + str(time.time() - start_time) + " seconds\n") db.session.commit() sys.stderr.write("check_for_updates: finished uninstalling and installing after " + str(time.time() - start_time) + " seconds\n") return ok, logmessages, results
def check_for_updates(): logmessage("check_for_update: starting") from docassemble.webapp.config import hostname ok = True here_already = dict() installed_packages = get_installed_distributions() for package in installed_packages: here_already[package.key] = package.version packages = dict() installs = dict() to_install = list() to_uninstall = list() uninstalled_packages = dict() logmessages = '' for package in Package.query.filter_by(active=True).all(): if package.type is not None: packages[package.id] = package #print "Found a package " + package.name for package in Package.query.filter_by(active=False).all(): uninstalled_packages[package.id] = package for install in Install.query.filter_by(hostname=hostname).all(): installs[install.package_id] = install if install.package_id in uninstalled_packages: to_uninstall.append(uninstalled_packages[install.package_id]) changed = False package_owner = dict() for auth in PackageAuth.query.filter_by(authtype='owner').all(): package_owner[auth.package_id] = auth.user_id for package in packages.itervalues(): if package.id not in installs and package.name in here_already: logmessage("Package " + package.name + " here already") install = Install(hostname=hostname, packageversion=here_already[package.name], version=package.version, package_id=package.id) db.session.add(install) installs[package.id] = install changed = True if changed: db.session.commit() for package in packages.itervalues(): #logmessage("Processing package id " + str(package.id)) #logmessage("1: " + str(installs[package.id].packageversion) + " 2: " + str(package.packageversion)) if (package.packageversion is not None and package.id in installs and installs[package.id].packageversion is None) or (package.packageversion is not None and package.id in installs and installs[package.id].packageversion is not None and LooseVersion(package.packageversion) > LooseVersion(installs[package.id].packageversion)): new_version_needed = True else: new_version_needed = False #logmessage("got here and new version is " + str(new_version_needed)) if package.id not in installs or package.version > installs[package.id].version or new_version_needed: to_install.append(package) #logmessage("done with that") for package in to_uninstall: #logmessage("Going to uninstall a package") returnval, newlog = uninstall_package(package) logmessages += newlog if returnval == 0: Install.query.filter_by(hostname=hostname, package_id=package.id).delete() else: ok = False for package in to_install: #logmessage("Going to install a package") returnval, newlog = install_package(package) logmessages += newlog if returnval == 0: if package.id in installs: install = installs[package.id] install.version = package.version else: install = Install(hostname=hostname, packageversion=package.packageversion, version=package.version, package_id=package.id) db.session.add(install) db.session.commit() update_versions() add_dependencies(package_owner[package.id]) update_versions() else: ok = False db.session.commit() #logmessage("check_for_update: finished uninstalling and installing") return ok, logmessages
def check_for_updates(): logmessage("check_for_update: starting") from docassemble.webapp.config import hostname ok = True here_already = dict() installed_packages = get_installed_distributions() for package in installed_packages: here_already[package.key] = package.version packages = dict() installs = dict() to_install = list() to_uninstall = list() uninstalled_packages = dict() logmessages = '' for package in Package.query.filter_by(active=True).all(): if package.type is not None: packages[package.id] = package #print "Found a package " + package.name for package in Package.query.filter_by(active=False).all(): uninstalled_packages[package.id] = package for install in Install.query.filter_by(hostname=hostname).all(): installs[install.package_id] = install if install.package_id in uninstalled_packages: to_uninstall.append(uninstalled_packages[install.package_id]) changed = False package_owner = dict() for auth in PackageAuth.query.filter_by(authtype='owner').all(): package_owner[auth.package_id] = auth.user_id for package in packages.itervalues(): if package.id not in installs and package.name in here_already: logmessage("Package " + package.name + " here already") install = Install(hostname=hostname, packageversion=here_already[package.name], version=package.version, package_id=package.id) db.session.add(install) installs[package.id] = install changed = True if changed: db.session.commit() for package in packages.itervalues(): #logmessage("Processing package id " + str(package.id)) #logmessage("1: " + str(installs[package.id].packageversion) + " 2: " + str(package.packageversion)) if (package.packageversion is not None and package.id in installs and installs[package.id].packageversion is None ) or (package.packageversion is not None and package.id in installs and installs[package.id].packageversion is not None and LooseVersion(package.packageversion) > LooseVersion( installs[package.id].packageversion)): new_version_needed = True else: new_version_needed = False #logmessage("got here and new version is " + str(new_version_needed)) if package.id not in installs or package.version > installs[ package.id].version or new_version_needed: to_install.append(package) #logmessage("done with that") for package in to_uninstall: #logmessage("Going to uninstall a package") returnval, newlog = uninstall_package(package) logmessages += newlog if returnval == 0: Install.query.filter_by(hostname=hostname, package_id=package.id).delete() else: ok = False for package in to_install: #logmessage("Going to install a package") returnval, newlog = install_package(package) logmessages += newlog if returnval == 0: if package.id in installs: install = installs[package.id] install.version = package.version else: install = Install(hostname=hostname, packageversion=package.packageversion, version=package.version, package_id=package.id) db.session.add(install) db.session.commit() update_versions() add_dependencies(package_owner[package.id]) update_versions() else: ok = False db.session.commit() #logmessage("check_for_update: finished uninstalling and installing") return ok, logmessages
def check_for_updates(doing_startup=False): sys.stderr.write("check_for_updates: starting\n") from docassemble.base.config import hostname ok = True here_already = dict() results = dict() sys.stderr.write("check_for_updates: 1\n") installed_packages = get_installed_distributions() for package in installed_packages: here_already[package.key] = package.version packages = dict() installs = dict() to_install = list() to_uninstall = list() uninstall_done = dict() uninstalled_packages = dict() logmessages = '' package_by_name = dict() sys.stderr.write("check_for_updates: 2\n") for package in Package.query.filter_by(active=True).all(): package_by_name[package.name] = package # packages is what is supposed to be installed sys.stderr.write("check_for_updates: 3\n") for package in Package.query.filter_by(active=True).all(): if package.type is not None: packages[package.id] = package #print "Found a package " + package.name sys.stderr.write("check_for_updates: 4\n") for package in Package.query.filter_by(active=False).all(): if package.name not in package_by_name: uninstalled_packages[package.id] = package # this is what the database says should be uninstalled sys.stderr.write("check_for_updates: 5\n") for install in Install.query.filter_by(hostname=hostname).all(): installs[install.package_id] = install # this is what the database says in installed on this server if install.package_id in uninstalled_packages and uninstalled_packages[install.package_id].name not in package_by_name: to_uninstall.append(uninstalled_packages[install.package_id]) # uninstall if it is installed changed = False package_owner = dict() sys.stderr.write("check_for_updates: 6\n") for auth in PackageAuth.query.filter_by(authtype='owner').all(): package_owner[auth.package_id] = auth.user_id sys.stderr.write("check_for_updates: 7\n") for package in packages.itervalues(): if package.id not in installs and package.name in here_already: sys.stderr.write("check_for_updates: package " + package.name + " here already\n") install = Install(hostname=hostname, packageversion=here_already[package.name], version=package.version, package_id=package.id) db.session.add(install) installs[package.id] = install changed = True if changed: db.session.commit() sys.stderr.write("check_for_updates: 8\n") for package in packages.itervalues(): #sys.stderr.write("check_for_updates: processing package id " + str(package.id) + "\n") #sys.stderr.write("1: " + str(installs[package.id].packageversion) + " 2: " + str(package.packageversion) + "\n") if (package.packageversion is not None and package.id in installs and installs[package.id].packageversion is None) or (package.packageversion is not None and package.id in installs and installs[package.id].packageversion is not None and LooseVersion(package.packageversion) > LooseVersion(installs[package.id].packageversion)): new_version_needed = True else: new_version_needed = False #sys.stderr.write("got here and new version is " + str(new_version_needed) + "\n") # Check for missing local packages if (package.name not in here_already) and (package.id in installs): package_missing = True else: package_missing = False if package.id not in installs or package.version > installs[package.id].version or new_version_needed or package_missing: to_install.append(package) #sys.stderr.write("done with that" + "\n") sys.stderr.write("check_for_updates: 9\n") for package in to_uninstall: #sys.stderr.write("Going to uninstall a package: " + package.name + "\n") if package.name in uninstall_done: sys.stderr.write("check_for_updates: skipping uninstallation of " + str(package.name) + " because already uninstalled" + "\n") continue returnval, newlog = uninstall_package(package) uninstall_done[package.name] = 1 logmessages += newlog if returnval == 0: Install.query.filter_by(hostname=hostname, package_id=package.id).delete() results[package.name] = 'successfully uninstalled' else: results[package.name] = 'uninstall failed' ok = False packages_to_delete = list() sys.stderr.write("check_for_updates: 10\n") for package in to_install: sys.stderr.write("check_for_updates: going to install a package: " + package.name + "\n") # if doing_startup and package.name.startswith('docassemble') and package.name in here_already: # #adding this because of unpredictability of installing new versions of docassemble # #just because of a system restart. # sys.stderr.write("check_for_updates: skipping update on " + str(package.name) + "\n") # continue returnval, newlog = install_package(package) logmessages += newlog sys.stderr.write("check_for_updates: return value was " + str(returnval) + "\n") if returnval != 0: sys.stderr.write("Return value was not good" + "\n") ok = False #pip._vendor.pkg_resources._initialize_master_working_set() pip_info = get_pip_info(package.name) real_name = pip_info['Name'] sys.stderr.write("check_for_updates: real name of package " + str(package.name) + " is " + str(real_name) + "\n") if real_name is None: results[package.name] = 'install failed' ok = False if package.name not in here_already: sys.stderr.write("check_for_updates: removing package entry for " + package.name + "\n") packages_to_delete.append(package) elif returnval != 0: results[package.name] = 'could not be upgraded' else: results[package.name] = 'successfully installed' if real_name != package.name: sys.stderr.write("check_for_updates: changing name" + "\n") package.name = real_name if package.id in installs: install = installs[package.id] install.version = package.version else: install = Install(hostname=hostname, packageversion=package.packageversion, version=package.version, package_id=package.id) db.session.add(install) db.session.commit() update_versions() add_dependencies(package_owner.get(package.id, 1)) update_versions() sys.stderr.write("check_for_updates: 11\n") for package in packages_to_delete: package.active = False sys.stderr.write("check_for_updates: 12\n") db.session.commit() sys.stderr.write("check_for_updates: finished uninstalling and installing\n") return ok, logmessages, results
def check_for_updates(doing_startup=False): sys.stderr.write("check_for_updates: starting\n") from docassemble.base.config import hostname from docassemble.webapp.app_object import app from docassemble.webapp.db_object import db from docassemble.webapp.packages.models import Package, Install, PackageAuth ok = True here_already = dict() results = dict() sys.stderr.write("check_for_updates: 0.5\n") num_deleted = Package.query.filter_by(name='psycopg2').delete() if num_deleted > 0: db.session.commit() if PY2: num_deleted = Package.query.filter_by(name='pycryptodome').delete() if num_deleted > 0: db.session.commit() num_deleted = Package.query.filter_by(name='pdfminer3k').delete() if num_deleted > 0: db.session.commit() else: num_deleted = Package.query.filter_by(name='pdfminer').delete() if num_deleted > 0: db.session.commit() num_deleted = Package.query.filter_by(name='py-bcrypt').delete() if num_deleted > 0: db.session.commit() num_deleted = Package.query.filter_by(name='pycrypto').delete() if num_deleted > 0: db.session.commit() num_deleted = Package.query.filter_by(name='constraint').delete() if num_deleted > 0: db.session.commit() num_deleted = Package.query.filter_by(name='distutils2').delete() if num_deleted > 0: db.session.commit() sys.stderr.write("check_for_updates: 1\n") installed_packages = get_installed_distributions() for package in installed_packages: here_already[package.key] = package.version changed = False if 'pdfminer.six' not in here_already: sys.stderr.write("check_for_updates: installing pdfminer.six\n") install_package(DummyPackage('pdfminer.six')) changed = True if 'psycopg2' in here_already: sys.stderr.write("check_for_updates: uninstalling psycopg2\n") uninstall_package(DummyPackage('psycopg2')) if 'psycopg2-binary' in here_already: sys.stderr.write("check_for_updates: reinstalling psycopg2-binary\n") uninstall_package(DummyPackage('psycopg2-binary')) install_package(DummyPackage('psycopg2-binary')) changed = True if 'psycopg2-binary' not in here_already: sys.stderr.write("check_for_updates: installing psycopg2-binary\n") install_package(DummyPackage('psycopg2-binary')) change = True if PY2: if 'pycryptodome' in here_already: sys.stderr.write("check_for_updates: uninstalling pycryptodome\n") uninstall_package(DummyPackage('pycryptodome')) if 'pycrypto' in here_already: sys.stderr.write("check_for_updates: reinstalling pycrypto\n") uninstall_package(DummyPackage('pycrypto')) install_package(DummyPackage('pycrypto')) changed = True if 'pathlib' not in here_already: sys.stderr.write("check_for_updates: installing pathlib\n") install_package(DummyPackage('pathlib')) changed = True if 'pycrypto' not in here_already: sys.stderr.write("check_for_updates: installing pycrypto\n") install_package(DummyPackage('pycrypto')) changed = True if 'kombu' not in here_already or LooseVersion(here_already['kombu']) > LooseVersion('4.1.0'): sys.stderr.write("check_for_updates: installing older kombu version\n") kombu = DummyPackage('kombu') kombu.limitation = '==4.1.0' install_package(kombu) changed = True if 'celery' not in here_already or LooseVersion(here_already['celery']) > LooseVersion('4.1.0'): sys.stderr.write("check_for_updates: installing older celery version\n") celery = DummyPackage('celery') celery.limitation = '[redis]==4.1.0' install_package(celery) changed = True if 'pdfminer' not in here_already: sys.stderr.write("check_for_updates: installing pdfminer\n") pdfminer = DummyPackage('pdfminer') pdfminer.type = 'git' pdfminer.giturl = 'https://github.com/euske/pdfminer' pdfminer.gitsubdir = None pdfminer.gitbranch = None install_package(pdfminer) changed = True else: if 'kombu' not in here_already or LooseVersion(here_already['kombu']) <= LooseVersion('4.1.0'): sys.stderr.write("check_for_updates: installing new kombu version\n") install_package(DummyPackage('kombu')) changed = True if 'celery' not in here_already or LooseVersion(here_already['celery']) <= LooseVersion('4.1.0'): sys.stderr.write("check_for_updates: installing new celery version\n") install_package(DummyPackage('celery')) changed = True if 'pycrypto' in here_already: sys.stderr.write("check_for_updates: uninstalling pycrypto\n") uninstall_package(DummyPackage('pycrypto')) if 'pycryptodome' in here_already: sys.stderr.write("check_for_updates: reinstalling pycryptodome\n") uninstall_package(DummyPackage('pycryptodome')) install_package(DummyPackage('pycryptodome')) changed = True if 'pycryptodome' not in here_already: sys.stderr.write("check_for_updates: installing pycryptodome\n") install_package(DummyPackage('pycryptodome')) changed = True if 'pdfminer' in here_already: sys.stderr.write("check_for_updates: uninstalling pdfminer\n") uninstall_package(DummyPackage('pdfminer')) changed = True if 'pdfminer3k' not in here_already: sys.stderr.write("check_for_updates: installing pdfminer3k\n") install_package(DummyPackage('pdfminer3k')) changed = True if 'py-bcrypt' in here_already: sys.stderr.write("check_for_updates: uninstalling py-bcrypt\n") uninstall_package(DummyPackage('py-bcrypt')) changed = True if 'bcrypt' in here_already: sys.stderr.write("check_for_updates: reinstalling bcrypt\n") uninstall_package(DummyPackage('bcrypt')) install_package(DummyPackage('bcrypt')) changed = True if 'bcrypt' not in here_already: sys.stderr.write("check_for_updates: installing bcrypt\n") install_package(DummyPackage('bcrypt')) changed = True if changed: installed_packages = get_installed_distributions() here_already = dict() for package in installed_packages: here_already[package.key] = package.version packages = dict() installs = dict() to_install = list() to_uninstall = list() uninstall_done = dict() uninstalled_packages = dict() logmessages = '' package_by_name = dict() sys.stderr.write("check_for_updates: 2\n") for package in Package.query.filter_by(active=True).all(): package_by_name[package.name] = package # packages is what is supposed to be installed sys.stderr.write("check_for_updates: 3\n") for package in Package.query.filter_by(active=True).all(): if package.type is not None: packages[package.id] = package #print("Found a package " + package.name) sys.stderr.write("check_for_updates: 4\n") for package in Package.query.filter_by(active=False).all(): if package.name not in package_by_name: uninstalled_packages[package.id] = package # this is what the database says should be uninstalled sys.stderr.write("check_for_updates: 5\n") for install in Install.query.filter_by(hostname=hostname).all(): installs[install.package_id] = install # this is what the database says in installed on this server if install.package_id in uninstalled_packages and uninstalled_packages[install.package_id].name not in package_by_name: to_uninstall.append(uninstalled_packages[install.package_id]) # uninstall if it is installed changed = False package_owner = dict() sys.stderr.write("check_for_updates: 6\n") for auth in PackageAuth.query.filter_by(authtype='owner').all(): package_owner[auth.package_id] = auth.user_id sys.stderr.write("check_for_updates: 7\n") for package in packages.values(): if package.id not in installs and package.name in here_already: sys.stderr.write("check_for_updates: package " + package.name + " here already\n") install = Install(hostname=hostname, packageversion=here_already[package.name], version=package.version, package_id=package.id) db.session.add(install) installs[package.id] = install changed = True if changed: db.session.commit() sys.stderr.write("check_for_updates: 8\n") for package in packages.values(): #sys.stderr.write("check_for_updates: processing package id " + str(package.id) + "\n") #sys.stderr.write("1: " + str(installs[package.id].packageversion) + " 2: " + str(package.packageversion) + "\n") if (package.packageversion is not None and package.id in installs and installs[package.id].packageversion is None) or (package.packageversion is not None and package.id in installs and installs[package.id].packageversion is not None and LooseVersion(package.packageversion) > LooseVersion(installs[package.id].packageversion)): new_version_needed = True else: new_version_needed = False #sys.stderr.write("got here and new version is " + str(new_version_needed) + "\n") # Check for missing local packages if (package.name not in here_already) and (package.id in installs): package_missing = True else: package_missing = False if package.id not in installs or package.version > installs[package.id].version or new_version_needed or package_missing: to_install.append(package) #sys.stderr.write("done with that" + "\n") sys.stderr.write("check_for_updates: 9\n") for package in to_uninstall: #sys.stderr.write("Going to uninstall a package: " + package.name + "\n") if package.name in uninstall_done: sys.stderr.write("check_for_updates: skipping uninstallation of " + str(package.name) + " because already uninstalled" + "\n") continue if package.name not in here_already: sys.stderr.write("check_for_updates: skipping uninstallation of " + str(package.name) + " because not installed" + "\n") else: returnval, newlog = uninstall_package(package) uninstall_done[package.name] = 1 logmessages += newlog if returnval == 0: Install.query.filter_by(hostname=hostname, package_id=package.id).delete() results[package.name] = 'pip uninstall command returned success code. See log for details.' else: results[package.name] = 'pip uninstall command returned failure code' ok = False packages_to_delete = list() sys.stderr.write("check_for_updates: 10\n") for package in to_install: sys.stderr.write("check_for_updates: going to install a package: " + package.name + "\n") # if doing_startup and package.name.startswith('docassemble') and package.name in here_already: # #adding this because of unpredictability of installing new versions of docassemble # #just because of a system restart. # sys.stderr.write("check_for_updates: skipping update on " + str(package.name) + "\n") # continue returnval, newlog = install_package(package) logmessages += newlog sys.stderr.write("check_for_updates: return value was " + str(returnval) + "\n") if returnval != 0: sys.stderr.write("Return value was not good" + "\n") ok = False #pip._vendor.pkg_resources._initialize_master_working_set() pip_info = get_pip_info(package.name) real_name = pip_info['Name'] sys.stderr.write("check_for_updates: real name of package " + str(package.name) + " is " + str(real_name) + "\n") if real_name is None: results[package.name] = 'install failed' ok = False if package.name not in here_already: sys.stderr.write("check_for_updates: removing package entry for " + package.name + "\n") packages_to_delete.append(package) elif returnval != 0: results[package.name] = 'pip install command returned failure code' else: results[package.name] = 'pip install command returned success code. See log for details.' if real_name != package.name: sys.stderr.write("check_for_updates: changing name" + "\n") package.name = real_name if package.id in installs: install = installs[package.id] install.version = package.version else: install = Install(hostname=hostname, packageversion=package.packageversion, version=package.version, package_id=package.id) db.session.add(install) db.session.commit() update_versions() add_dependencies(package_owner.get(package.id, 1)) update_versions() sys.stderr.write("check_for_updates: 11\n") for package in packages_to_delete: db.session.delete(package) sys.stderr.write("check_for_updates: 12\n") db.session.commit() sys.stderr.write("check_for_updates: finished uninstalling and installing\n") return ok, logmessages, results