def run_calculations(deploy_settings): base_cmd_path = 'python /apps/demokratikollen/' commands = [base_cmd_path + 'compute_party_votes.py', base_cmd_path + 'calculations/party_covoting.py', base_cmd_path + 'calculations/sankey_data.py', base_cmd_path + 'calculations/election_data.py', base_cmd_path + 'calculations/search.py', base_cmd_path + 'calculations/cosigning.py', base_cmd_path + 'calculations/scb_best_party_gender.py', base_cmd_path + 'calculations/scb_best_party_education.py', base_cmd_path + 'calculations/scb_elections.py', base_cmd_path + 'calculations/scb_polls.py'] for cmd in commands: deploy_settings['log'].info("Starting calculation {0}".format(cmd)) container_utils.run_command_in_container('webapp-temp', cmd, log=deploy_settings['log'])
def update_webapp_src(deploy_settings, tag='latest'): if tag == 'latest': web_src_dir = os.path.join(deploy_settings['base_dir'],'demokratikollen') if tag == 'current': web_src_dir = os.path.join(deploy_settings['base_dir'],'demokratikollen_old') binds = {} binds[web_src_dir] = {'bind': '/mnt/websrc', 'ro': True} cont = cli.create_container(image='demokratikollen/webapp:'+tag, name='webapp-temp', command='/bin/sh -c "while true; do sleep 1; done"', volumes='/mnt/websrc') cli.start(cont['Id'], binds=binds, volumes_from='webapp-data') container_utils.run_command_in_container('webapp-temp', "bash -c 'rm -rf /apps/* && cp -r /mnt/websrc/demokratikollen /apps/'") cli.remove_container(cont['Id'], force=True, v=True)
def use_current_database_for_calculations(deploy_settings): data_dir = os.path.join(deploy_settings['base_dir'],'data/database_dumps') binds = {} binds[data_dir] = {'bind': '/data', 'ro': False} links = {} links['postgres-temp'] = 'postgres' postgres_path = '/data/demokratikollen_postgres_current.gz' cont = cli.create_container(image='demokratikollen/postgres:current', command='/bin/sh -c "while true; do sleep 1; done"', volumes='/data') cli.start(cont['Id'], binds=binds, links=links) container_utils.run_command_in_container(cont['Id'], "/bin/bash -c 'gunzip -c " + postgres_path + " | psql -h postgres -U demokratikollen demokratikollen'", deploy_settings['log']) cli.remove_container(cont['Id'], v=True, force=True)
def update_database_data(deploy_settings, tag='latest'): data_dir = os.path.join(deploy_settings['base_dir'],'data/database_dumps') binds = {} binds[data_dir] = {'bind': '/data', 'ro': False} links = {} links['postgres'] = 'postgres' links['mongo'] = 'mongo' if deploy_settings['deploy_extent'] == 'src': tag = 'current' if tag == 'latest' and os.path.isfile(os.path.join(data_dir, 'demokratikollen_postgres_latest.gz')): postgres_path = '/data/demokratikollen_postgres_latest.gz' else: postgres_path = '/data/demokratikollen_postgres_current.gz' if tag == 'latest' and os.path.isdir(os.path.join(data_dir, 'demokratikollen_mongo_latest')): mongo_path = '/data/demokratikollen_mongo_latest' else: mongo_path = '/data/demokratikollen_mongo_current' deploy_settings['log'].info("Updating the Postgres database.") cont = cli.create_container(image='demokratikollen/postgres:' + tag, command='/bin/sh -c "while true; do sleep 1; done"', volumes='/data') cli.start(cont['Id'], binds=binds, links=links) container_utils.run_command_in_container(cont['Id'], "/bin/bash -c 'gunzip -c " + postgres_path + " | psql -h postgres -U demokratikollen demokratikollen'", deploy_settings['log']) cli.remove_container(cont['Id'], v=True, force=True) deploy_settings['log'].info("Updating the mongo database.") cont = cli.create_container(image='demokratikollen/mongo:' + tag, command='/bin/sh -c "while true; do sleep 1; done"', volumes='/data') cli.start(cont['Id'], binds=binds, links=links) container_utils.run_command_in_container(cont['Id'], "mongorestore --host mongo --drop "+ mongo_path, deploy_settings['log']) cli.remove_container(cont['Id'], v=True, force=True)
def save_database_data(deploy_settings): data_dir = os.path.join(deploy_settings['base_dir'],'data/database_dumps') binds = {} binds[data_dir] = {'bind': '/data', 'ro': False} links = {} links['postgres-temp'] = 'postgres' links['mongo-temp'] = 'mongo' deploy_settings['log'].info("Saving Postgres database.") cont = cli.create_container(image='demokratikollen/postgres:latest', command='/bin/sh -c "while true; do sleep 1; done"', volumes='/data') cli.start(cont['Id'], binds=binds, links=links) container_utils.run_command_in_container(cont['Id'], "/bin/bash -c 'pg_dump -h postgres -U demokratikollen -c demokratikollen | gzip > /data/demokratikollen_postgres_latest.gz'") cli.remove_container(cont['Id'], v=True, force=True) deploy_settings['log'].info("Saving mongo database.") cont = cli.create_container(image='demokratikollen/mongo:latest', command='/bin/sh -c "while true; do sleep 1; done"', volumes='/data') cli.start(cont['Id'], binds=binds, links=links) container_utils.run_command_in_container(cont['Id'], "mongodump --host mongo --out /data/demokratikollen_mongo_latest") container_utils.run_command_in_container(cont['Id'], "chown -R {0}:{0} /data".format(deploy_settings['userid'])) cli.remove_container(cont['Id'], v=True, force=True)
def setup_containers_for_calculations(deploy_settings): deploy_settings['log'].info("Creating and starting temporary mongo and postgres containers.") cont = cli.create_container(image='demokratikollen/postgres:latest',name='postgres-temp') cli.start(cont['Id']) cont = cli.create_container(image='demokratikollen/mongo:latest', name='mongo-temp') cli.start(cont['Id']) deploy_settings['log'].info("Creating, starting and populating a temporary webapp container") webapp_dir = os.path.join(deploy_settings['base_dir'],'demokratikollen') data_dir = os.path.join(deploy_settings['base_dir'],'data') binds = {} binds[webapp_dir] = {'bind': '/mnt/demokratikollen', 'ro': False} binds[data_dir] = {'bind': '/data', 'ro': False} links = {} links['postgres-temp'] = 'postgres' links['mongo-temp'] = 'mongo' cont = cli.create_container(image='demokratikollen/webapp:latest', name='webapp-temp', command='python /etc/webapp/loop.py', volumes='/mnt/demokratikollen') cli.start(cont['Id'],binds=binds, links=links) container_utils.run_command_in_container(cont['Id'], 'cp -r /mnt/demokratikollen/demokratikollen /apps')
def populate_orm(deploy_settings): container_utils.run_command_in_container('webapp-temp', "python /apps/demokratikollen/populate_orm.py", log=deploy_settings['log'])
def populate_riksdagen(deploy_settings): container_utils.run_command_in_container('webapp-temp', "/bin/bash -c 'cd /apps/demokratikollen && python import_data.py auto data/urls.txt /data --wipe'", log=deploy_settings['log'])