def install_gtfs_update_crontab(self):
'''Installs and starts a crontab to automatically dl and build a data bundle nightly.
'''
# prepare update script
refresh_settings = dict(gtfs_dl_file=unix_path_join(self.data_dir, 'google_transit.zip'),
gtfs_static_url=self.gtfs_conf.get('gtfs_static_url'),
gtfs_dl_logfile=unix_path_join(self.data_dir, 'nightly_dl.out'),
federation_builder_folder=self.federation_builder_folder,
bundle_dir=self.bundle_dir,
extra_args=self.gtfs_conf.get('extra_bundle_build_args'),
user=self.user,
cron_email=self.aws_conf.get('cron_email'),
from_mailer=env.host_string)
# check if script folders exists
if not exists(self.script_dir):
run('mkdir {0}'.format(self.script_dir))
self.populate_and_upload_template_file(refresh_settings, conf_helper, 'gtfs_refresh.sh', self.script_dir)
with cd(self.script_dir):
run('chmod 755 gtfs_refresh.sh')
# prepare update script
with open(os.path.join(CONFIG_TEMPLATE_DIR, 'gtfs_refresh_crontab')) as f:
refresh_cron_template = f.read()
cron_settings = dict(cron_email=self.aws_conf.get('cron_email'),
logfile=unix_path_join(self.data_dir, 'nightly_bundle.out'),
script_folder=self.script_dir)
gtfs_refresh_cron = refresh_cron_template.format(**cron_settings)
crontab_update(gtfs_refresh_cron, 'gtfs_refresh_cron')
def __init__(self, host_name, aws_conf, gtfs_conf=None, oba_conf=None):
AmazonLinuxFab.__init__(self, aws_conf, host_name)
self.aws_conf = aws_conf
self.gtfs_conf = gtfs_conf
self.oba_conf = oba_conf
self.data_dir = unix_path_join('/home', self.user, 'data')
self.bundle_dir = unix_path_join(self.data_dir, 'bundle')
self.script_dir = unix_path_join('/home', self.user, 'scripts')
self.config_dir = unix_path_join('/home', self.user, 'conf')
if oba_conf:
self.oba_base_folder = oba_conf.get('oba_base_folder')
def update_gtfs(self):
'''Uploads the downloaded gtfs zip file to the server and builds a new bundle.
'''
remote_gtfs_file = unix_path_join(self.data_dir, gtfs_file_name_raw)
# check if data folders exists
if not exists(self.data_dir):
run('mkdir {0}'.format(self.data_dir))
# remove old gtfs file (if needed)
if exists(remote_gtfs_file):
run('rm {0}'.format(remote_gtfs_file))
# upload new file
put(gtfs_file_name, 'data')
# create new bundle
bundle_main = '.'.join(['org',
'onebusaway',
'transit_data_federation',
'bundle',
'FederatedTransitDataBundleCreatorMain'])
more_args = self.gtfs_conf.get('extra_bundle_build_args')
with cd(self.federation_builder_folder):
run('java -classpath .:target/* {0} {1} {2} {3}'.format(bundle_main,
remote_gtfs_file,
self.bundle_dir,
more_args))
def venv(self, cmd, use_sudo=False):
pre_path = unix_path_join(self.user_home, 'otvia2-monitor', 'bin', 'activate')
with prefix('source {0}'.format(pre_path)):
if use_sudo:
sudo(cmd)
else:
run(cmd)
def deploy_all(self):
'''Deploys each webapp (copies to tomcat webapps).
'''
# copy the war files to tomcat for each webapp
tomcat_webapp_dir = unix_path_join('/home',
self.user,
'tomcat',
'webapps')
for webapp in ['onebusaway-transit-data-federation-webapp',
'onebusaway-api-webapp',
'onebusaway-sms-webapp',
'onebusaway-webapp']:
run('cp {0} {1}'.format(unix_path_join('/home',
self.user,
self.oba_base_folder,
webapp,
'target',
webapp + '.war'),
tomcat_webapp_dir))
def install_watchdog(self):
'''Configures and uploads watchdog script. Adds cron task to run it.
'''
# ensure watchdog .py file is in config directory
oba_script_file = os.path.join(CONFIG_DIR, 'check_oba.py')
if not os.path.exists(oba_script_file):
print('Watchdog python script does not exist in config directory.')
print('Please create it and set the appropriate location of the watchdog.ini file.')
return
# ensure script and config folder exists
if not exists(self.config_dir):
run('mkdir {0}'.format(self.config_dir))
if not exists(self.script_dir):
run('mkdir {0}'.format(self.script_dir))
# upload watchdog script (remove it if needed)
remote_script_file = unix_path_join(self.script_dir, 'check_oba.py')
if exists(remote_script_file):
sudo('rm -rf {0}'.format(remote_script_file))
put(oba_script_file, self.script_dir)
# upload watchdog config (remove it if needed)
remote_config_file = unix_path_join(self.config_dir, 'watchdog.ini')
if exists(remote_config_file):
sudo('rm -rf {0}'.format(remote_config_file))
put(os.path.join(CONFIG_DIR, 'watchdog.ini'), self.config_dir)
# update/insert cron to run script
with open(os.path.join(CONFIG_TEMPLATE_DIR, 'watchdog_crontab')) as f:
refresh_cron_template = f.read()
cron_settings = dict(cron_email=self.aws_conf.get('cron_email'),
watchdog_script=remote_script_file)
cron = refresh_cron_template.format(**cron_settings)
crontab_update(cron, 'watchdog_cron')
def install_federation_webapp(self):
'''Installs the transit-data-federation-webapp.
'''
transit_fed_config = self.make_params_from_datasource_file('transit-data-federation-webapp-data-sources.xml')
transit_fed_config['data_bundle_path'] = unix_path_join('/home', self.user, 'data', 'bundle')
for k in ['gtfs_rt_trip_updates_url', 'gtfs_rt_vehicle_positions_url', 'gtfs_rt_service_alerts_url']:
transit_fed_config[k] = self.gtfs_conf.get(k)
self.build_webapp(transit_fed_config,
'transit-data-federation-webapp-data-sources.xml',
'onebusaway-transit-data-federation-webapp')
def install_monitor(self):
run('git clone https://github.com/evansiroky/otvia2-monitor.git')
run('virtualenv -p /usr/bin/python otvia2-monitor')
# install gcc so pycrypto can compile
sudo('yum -y install gcc')
monitor_conf = ConfHelper.get_config('monitor')
with cd('otvia2-monitor'):
# upload monitoring config
run('mkdir config')
put(os.path.join(CONFIG_DIR, 'monitor.ini'), 'config/')
# install run scripts
self.venv('python setup.py develop')
# install node.js server
server_cfg = dict(server_admin_username=monitor_conf.get('server_admin_username'),
server_admin_password=monitor_conf.get('server_admin_password'),
server_access_username=monitor_conf.get('server_access_username'),
server_access_password=monitor_conf.get('server_access_password'))
put(ConfHelper.write_template(server_cfg, 'server.js'),
'config')
put(ConfHelper.write_template(dict(google_analytics_tracking_id=\
monitor_conf.get('google_analytics_tracking_id')),
'web.js'),
'config')
run('npm install')
run('npm run build')
# redirect port 80 to 3000 for node app
sudo('iptables -t nat -I PREROUTING -p tcp --dport 80 -j REDIRECT --to-port 3000')
sudo('service iptables save')
# start server
run('forever start -a --uid "otvia2-monitor" server/index.js')
# install cron to run monitor script
with open(os.path.join(TEMPLATE_DIR, 'monitor_crontab')) as f:
cron_template = f.read()
collect_script = unix_path_join(self.user_home, 'otvia2-monitor', 'bin', 'monitor')
cron_settings = dict(cron_email=self.conf.get('cron_email'),
path_to_monitor_script=collect_script)
cron = cron_template.format(**cron_settings)
crontab_update(cron, 'otvia2_data_collection')
def build_webapp(self, data_dict, config_template_file, webapp):
'''Build a webapp using maven.
Args:
data_dict (dict): A dict to set the stuff in the config template.
config_template_file (string): filename of the config template file.
webapp (string): The name of the webapp to build.
'''
# upload the data sources file to the project
destination = unix_path_join(self.oba_base_folder,
webapp,
'src',
'main',
'resources')
self.populate_and_upload_template_file(data_dict,
conf_helper,
config_template_file,
destination,
out_filename='data-sources.xml')
# build the project using maven
with cd(self.oba_base_folder):
run('/usr/local/maven/bin/mvn -am -pl {0} package'.format(webapp))
def __init__(self, host_name, aws_conf, gtfs_conf, oba_conf):
OBAFab.__init__(self, host_name, aws_conf, gtfs_conf, oba_conf)
self.federation_builder_folder = unix_path_join('/home',
self.user,
self.oba_base_folder,
'onebusaway-transit-data-federation-builder')