Exemple #1
0
 def install_gtfs_update_crontab(self):
     '''Installs and starts a crontab to automatically dl and build a data bundle nightly.
     '''
     
     # prepare update script
     refresh_settings = dict(gtfs_dl_file=unix_path_join(self.data_dir, 'google_transit.zip'),
                             gtfs_static_url=self.gtfs_conf.get('gtfs_static_url'),
                             gtfs_dl_logfile=unix_path_join(self.data_dir, 'nightly_dl.out'),
                             federation_builder_folder=self.federation_builder_folder,
                             bundle_dir=self.bundle_dir,
                             extra_args=self.gtfs_conf.get('extra_bundle_build_args'),
                             user=self.user,
                             cron_email=self.aws_conf.get('cron_email'),
                             from_mailer=env.host_string)
     
     # check if script folders exists
     if not exists(self.script_dir):
         run('mkdir {0}'.format(self.script_dir))
         
     self.populate_and_upload_template_file(refresh_settings, conf_helper, 'gtfs_refresh.sh', self.script_dir)
     with cd(self.script_dir):
         run('chmod 755 gtfs_refresh.sh')
             
     # prepare update script
     with open(os.path.join(CONFIG_TEMPLATE_DIR, 'gtfs_refresh_crontab')) as f:
         refresh_cron_template = f.read()
         
     cron_settings = dict(cron_email=self.aws_conf.get('cron_email'),
                          logfile=unix_path_join(self.data_dir, 'nightly_bundle.out'),
                          script_folder=self.script_dir)
     gtfs_refresh_cron = refresh_cron_template.format(**cron_settings)
         
     crontab_update(gtfs_refresh_cron, 'gtfs_refresh_cron')
Exemple #2
0
    def __init__(self, host_name, aws_conf, gtfs_conf=None, oba_conf=None):
        AmazonLinuxFab.__init__(self, aws_conf, host_name)
        self.aws_conf = aws_conf
        self.gtfs_conf = gtfs_conf
        self.oba_conf = oba_conf

        self.data_dir = unix_path_join('/home', self.user, 'data')
        self.bundle_dir = unix_path_join(self.data_dir, 'bundle')
        self.script_dir = unix_path_join('/home', self.user, 'scripts')
        self.config_dir = unix_path_join('/home', self.user, 'conf')

        if oba_conf:
            self.oba_base_folder = oba_conf.get('oba_base_folder')
Exemple #3
0
 def update_gtfs(self):
     '''Uploads the downloaded gtfs zip file to the server and builds a new bundle.
     '''
     
     remote_gtfs_file = unix_path_join(self.data_dir, gtfs_file_name_raw)
     
     # check if data folders exists
     if not exists(self.data_dir):
         run('mkdir {0}'.format(self.data_dir))
         
     # remove old gtfs file (if needed)
     if exists(remote_gtfs_file):
         run('rm {0}'.format(remote_gtfs_file))
     
     # upload new file
     put(gtfs_file_name, 'data')
     
     # create new bundle
     bundle_main = '.'.join(['org',
                             'onebusaway',
                             'transit_data_federation',
                             'bundle',
                             'FederatedTransitDataBundleCreatorMain'])
     more_args = self.gtfs_conf.get('extra_bundle_build_args')
     with cd(self.federation_builder_folder):
         run('java -classpath .:target/* {0} {1} {2} {3}'.format(bundle_main,
                                                                 remote_gtfs_file,
                                                                 self.bundle_dir,
                                                                 more_args))
Exemple #4
0
 def venv(self, cmd, use_sudo=False):
     pre_path = unix_path_join(self.user_home, 'otvia2-monitor', 'bin', 'activate')
     with prefix('source {0}'.format(pre_path)):
         if use_sudo:
             sudo(cmd)
         else:
             run(cmd)
Exemple #5
0
 def deploy_all(self):
     '''Deploys each webapp (copies to tomcat webapps).
     '''
     
     # copy the war files to tomcat for each webapp
     tomcat_webapp_dir = unix_path_join('/home',
                                        self.user,
                                        'tomcat',
                                        'webapps')
     for webapp in ['onebusaway-transit-data-federation-webapp',
                    'onebusaway-api-webapp',
                    'onebusaway-sms-webapp',
                    'onebusaway-webapp']:
         run('cp {0} {1}'.format(unix_path_join('/home',
                                                self.user,
                                                self.oba_base_folder,
                                                webapp,
                                                'target',
                                                webapp + '.war'),
                                 tomcat_webapp_dir))
Exemple #6
0
 def install_watchdog(self):
     '''Configures and uploads watchdog script.  Adds cron task to run it.
     '''
     
     # ensure watchdog .py file is in config directory
     oba_script_file = os.path.join(CONFIG_DIR, 'check_oba.py')
     if not os.path.exists(oba_script_file):
         print('Watchdog python script does not exist in config directory.')
         print('Please create it and set the appropriate location of the watchdog.ini file.')
         return
     
     # ensure script and config folder exists
     if not exists(self.config_dir):
         run('mkdir {0}'.format(self.config_dir))
         
     if not exists(self.script_dir):
         run('mkdir {0}'.format(self.script_dir))
         
     # upload watchdog script (remove it if needed)
     remote_script_file = unix_path_join(self.script_dir, 'check_oba.py')
     if exists(remote_script_file):
         sudo('rm -rf {0}'.format(remote_script_file))
         
     put(oba_script_file, self.script_dir)
     
     # upload watchdog config (remove it if needed)
     remote_config_file = unix_path_join(self.config_dir, 'watchdog.ini')
     if exists(remote_config_file):
         sudo('rm -rf {0}'.format(remote_config_file))
         
     put(os.path.join(CONFIG_DIR, 'watchdog.ini'), self.config_dir)
     
     # update/insert cron to run script
     with open(os.path.join(CONFIG_TEMPLATE_DIR, 'watchdog_crontab')) as f:
         refresh_cron_template = f.read()
         
     cron_settings = dict(cron_email=self.aws_conf.get('cron_email'),
                          watchdog_script=remote_script_file)
     cron = refresh_cron_template.format(**cron_settings)
         
     crontab_update(cron, 'watchdog_cron')        
Exemple #7
0
    def install_federation_webapp(self):
        '''Installs the transit-data-federation-webapp.
        '''

        transit_fed_config = self.make_params_from_datasource_file('transit-data-federation-webapp-data-sources.xml')
        
        transit_fed_config['data_bundle_path'] = unix_path_join('/home', self.user, 'data', 'bundle')
        for k in ['gtfs_rt_trip_updates_url', 'gtfs_rt_vehicle_positions_url', 'gtfs_rt_service_alerts_url']:
            transit_fed_config[k] = self.gtfs_conf.get(k)
        
        self.build_webapp(transit_fed_config, 
                          'transit-data-federation-webapp-data-sources.xml',
                          'onebusaway-transit-data-federation-webapp')
Exemple #8
0
    def install_monitor(self):
        run('git clone https://github.com/evansiroky/otvia2-monitor.git')
        run('virtualenv -p /usr/bin/python otvia2-monitor')
        # install gcc so pycrypto can compile
        sudo('yum -y install gcc')
        
        monitor_conf = ConfHelper.get_config('monitor')
        with cd('otvia2-monitor'):
            
            # upload monitoring config
            run('mkdir config')

            put(os.path.join(CONFIG_DIR, 'monitor.ini'), 'config/')

            # install run scripts
            self.venv('python setup.py develop')
            
            # install node.js server
            server_cfg = dict(server_admin_username=monitor_conf.get('server_admin_username'),
                              server_admin_password=monitor_conf.get('server_admin_password'),
                              server_access_username=monitor_conf.get('server_access_username'),
                              server_access_password=monitor_conf.get('server_access_password'))

            put(ConfHelper.write_template(server_cfg, 'server.js'),
                'config')

            put(ConfHelper.write_template(dict(google_analytics_tracking_id=\
                                               monitor_conf.get('google_analytics_tracking_id')),
                                          'web.js'),
                'config')

            run('npm install')
            run('npm run build')
            
            # redirect port 80 to 3000 for node app
            sudo('iptables -t nat -I PREROUTING -p tcp --dport 80 -j REDIRECT --to-port 3000')
            sudo('service iptables save')
            
            # start server
            run('forever start -a --uid "otvia2-monitor" server/index.js')
            
        # install cron to run monitor script
        with open(os.path.join(TEMPLATE_DIR, 'monitor_crontab')) as f:
            cron_template = f.read()
            
        collect_script = unix_path_join(self.user_home, 'otvia2-monitor', 'bin', 'monitor')

        cron_settings = dict(cron_email=self.conf.get('cron_email'),
                             path_to_monitor_script=collect_script)
        cron = cron_template.format(**cron_settings)
        crontab_update(cron, 'otvia2_data_collection')
Exemple #9
0
    def build_webapp(self, data_dict, config_template_file, webapp):
        '''Build a webapp using maven.
        
        Args:
            data_dict (dict): A dict to set the stuff in the config template.
            config_template_file (string): filename of the config template file.
            webapp (string): The name of the webapp to build.
        '''
                   
        # upload the data sources file to the project
        destination = unix_path_join(self.oba_base_folder,
                                     webapp,
                                     'src',
                                     'main',
                                     'resources')
        self.populate_and_upload_template_file(data_dict,
                                               conf_helper,
                                               config_template_file,
                                               destination,
                                               out_filename='data-sources.xml')

        # build the project using maven
        with cd(self.oba_base_folder):
            run('/usr/local/maven/bin/mvn -am -pl {0} package'.format(webapp))
Exemple #10
0
 def __init__(self, host_name, aws_conf, gtfs_conf, oba_conf):
     OBAFab.__init__(self, host_name, aws_conf, gtfs_conf, oba_conf)
     self.federation_builder_folder = unix_path_join('/home',
                                                     self.user,
                                                     self.oba_base_folder,
                                                     'onebusaway-transit-data-federation-builder')