def run_calculations(deploy_settings):
base_cmd_path = 'python /apps/demokratikollen/'
commands = [base_cmd_path + 'compute_party_votes.py',
base_cmd_path + 'calculations/party_covoting.py',
base_cmd_path + 'calculations/sankey_data.py',
base_cmd_path + 'calculations/election_data.py',
base_cmd_path + 'calculations/search.py',
base_cmd_path + 'calculations/cosigning.py',
base_cmd_path + 'calculations/scb_best_party_gender.py',
base_cmd_path + 'calculations/scb_best_party_education.py',
base_cmd_path + 'calculations/scb_elections.py',
base_cmd_path + 'calculations/scb_polls.py']
for cmd in commands:
deploy_settings['log'].info("Starting calculation {0}".format(cmd))
container_utils.run_command_in_container('webapp-temp', cmd, log=deploy_settings['log'])
def update_webapp_src(deploy_settings, tag='latest'):
if tag == 'latest':
web_src_dir = os.path.join(deploy_settings['base_dir'],'demokratikollen')
if tag == 'current':
web_src_dir = os.path.join(deploy_settings['base_dir'],'demokratikollen_old')
binds = {}
binds[web_src_dir] = {'bind': '/mnt/websrc', 'ro': True}
cont = cli.create_container(image='demokratikollen/webapp:'+tag,
name='webapp-temp',
command='/bin/sh -c "while true; do sleep 1; done"',
volumes='/mnt/websrc')
cli.start(cont['Id'], binds=binds, volumes_from='webapp-data')
container_utils.run_command_in_container('webapp-temp', "bash -c 'rm -rf /apps/* && cp -r /mnt/websrc/demokratikollen /apps/'")
cli.remove_container(cont['Id'], force=True, v=True)
def use_current_database_for_calculations(deploy_settings):
data_dir = os.path.join(deploy_settings['base_dir'],'data/database_dumps')
binds = {}
binds[data_dir] = {'bind': '/data', 'ro': False}
links = {}
links['postgres-temp'] = 'postgres'
postgres_path = '/data/demokratikollen_postgres_current.gz'
cont = cli.create_container(image='demokratikollen/postgres:current',
command='/bin/sh -c "while true; do sleep 1; done"',
volumes='/data')
cli.start(cont['Id'], binds=binds, links=links)
container_utils.run_command_in_container(cont['Id'],
"/bin/bash -c 'gunzip -c " + postgres_path + " | psql -h postgres -U demokratikollen demokratikollen'",
deploy_settings['log'])
cli.remove_container(cont['Id'], v=True, force=True)
def update_database_data(deploy_settings, tag='latest'):
data_dir = os.path.join(deploy_settings['base_dir'],'data/database_dumps')
binds = {}
binds[data_dir] = {'bind': '/data', 'ro': False}
links = {}
links['postgres'] = 'postgres'
links['mongo'] = 'mongo'
if deploy_settings['deploy_extent'] == 'src':
tag = 'current'
if tag == 'latest' and os.path.isfile(os.path.join(data_dir, 'demokratikollen_postgres_latest.gz')):
postgres_path = '/data/demokratikollen_postgres_latest.gz'
else:
postgres_path = '/data/demokratikollen_postgres_current.gz'
if tag == 'latest' and os.path.isdir(os.path.join(data_dir, 'demokratikollen_mongo_latest')):
mongo_path = '/data/demokratikollen_mongo_latest'
else:
mongo_path = '/data/demokratikollen_mongo_current'
deploy_settings['log'].info("Updating the Postgres database.")
cont = cli.create_container(image='demokratikollen/postgres:' + tag,
command='/bin/sh -c "while true; do sleep 1; done"',
volumes='/data')
cli.start(cont['Id'], binds=binds, links=links)
container_utils.run_command_in_container(cont['Id'],
"/bin/bash -c 'gunzip -c " + postgres_path + " | psql -h postgres -U demokratikollen demokratikollen'",
deploy_settings['log'])
cli.remove_container(cont['Id'], v=True, force=True)
deploy_settings['log'].info("Updating the mongo database.")
cont = cli.create_container(image='demokratikollen/mongo:' + tag,
command='/bin/sh -c "while true; do sleep 1; done"',
volumes='/data')
cli.start(cont['Id'], binds=binds, links=links)
container_utils.run_command_in_container(cont['Id'],
"mongorestore --host mongo --drop "+ mongo_path,
deploy_settings['log'])
cli.remove_container(cont['Id'], v=True, force=True)
def save_database_data(deploy_settings):
data_dir = os.path.join(deploy_settings['base_dir'],'data/database_dumps')
binds = {}
binds[data_dir] = {'bind': '/data', 'ro': False}
links = {}
links['postgres-temp'] = 'postgres'
links['mongo-temp'] = 'mongo'
deploy_settings['log'].info("Saving Postgres database.")
cont = cli.create_container(image='demokratikollen/postgres:latest',
command='/bin/sh -c "while true; do sleep 1; done"',
volumes='/data')
cli.start(cont['Id'], binds=binds, links=links)
container_utils.run_command_in_container(cont['Id'],
"/bin/bash -c 'pg_dump -h postgres -U demokratikollen -c demokratikollen | gzip > /data/demokratikollen_postgres_latest.gz'")
cli.remove_container(cont['Id'], v=True, force=True)
deploy_settings['log'].info("Saving mongo database.")
cont = cli.create_container(image='demokratikollen/mongo:latest',
command='/bin/sh -c "while true; do sleep 1; done"',
volumes='/data')
cli.start(cont['Id'], binds=binds, links=links)
container_utils.run_command_in_container(cont['Id'],
"mongodump --host mongo --out /data/demokratikollen_mongo_latest")
container_utils.run_command_in_container(cont['Id'],
"chown -R {0}:{0} /data".format(deploy_settings['userid']))
cli.remove_container(cont['Id'], v=True, force=True)
def setup_containers_for_calculations(deploy_settings):
deploy_settings['log'].info("Creating and starting temporary mongo and postgres containers.")
cont = cli.create_container(image='demokratikollen/postgres:latest',name='postgres-temp')
cli.start(cont['Id'])
cont = cli.create_container(image='demokratikollen/mongo:latest', name='mongo-temp')
cli.start(cont['Id'])
deploy_settings['log'].info("Creating, starting and populating a temporary webapp container")
webapp_dir = os.path.join(deploy_settings['base_dir'],'demokratikollen')
data_dir = os.path.join(deploy_settings['base_dir'],'data')
binds = {}
binds[webapp_dir] = {'bind': '/mnt/demokratikollen', 'ro': False}
binds[data_dir] = {'bind': '/data', 'ro': False}
links = {}
links['postgres-temp'] = 'postgres'
links['mongo-temp'] = 'mongo'
cont = cli.create_container(image='demokratikollen/webapp:latest',
name='webapp-temp',
command='python /etc/webapp/loop.py',
volumes='/mnt/demokratikollen')
cli.start(cont['Id'],binds=binds, links=links)
container_utils.run_command_in_container(cont['Id'], 'cp -r /mnt/demokratikollen/demokratikollen /apps')
def populate_orm(deploy_settings):
container_utils.run_command_in_container('webapp-temp', "python /apps/demokratikollen/populate_orm.py", log=deploy_settings['log'])
def populate_riksdagen(deploy_settings):
container_utils.run_command_in_container('webapp-temp', "/bin/bash -c 'cd /apps/demokratikollen && python import_data.py auto data/urls.txt /data --wipe'", log=deploy_settings['log'])