def test_latest(self): svc = init_service("token.dat", "credentials.json") try: downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc)) downloader.latest() except Exception as e: assert False, e
def test_since(self): svc = init_service("token.dat", "credentials.json") downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc)) try: downloader.for_date(datetime.datetime.now() - datetime.timedelta(days=-30)) except Exception as e: assert False, e
def command(config_file): # Import ckan as it changes the dependent packages imported from dump_analysis import (get_run_info, TxtAnalysisFile, CsvAnalysisFile, DumpAnalysisOptions, DumpAnalysis) from pylons import config # settings ckan_instance_name = os.path.basename(config_file).replace('.ini', '') if ckan_instance_name not in ['development', 'dgutest']: default_dump_dir = '/var/lib/ckan/%s/static/dump' % ckan_instance_name default_analysis_dir = '/var/lib/ckan/%s/static/dump_analysis' % ckan_instance_name default_backup_dir = '/var/backups/ckan/%s' % ckan_instance_name default_openspending_reports_dir = '/var/lib/ckan/%s/openspending_reports' % ckan_instance_name else: # test purposes default_dump_dir = '~/dump' default_analysis_dir = '~/dump_analysis' default_backup_dir = '~/backups' default_openspending_reports_dir = '~/openspending_reports' dump_dir = os.path.expanduser(config.get('ckan.dump_dir', default_dump_dir)) analysis_dir = os.path.expanduser( config.get('ckan.dump_analysis_dir', default_analysis_dir)) backup_dir = os.path.expanduser( config.get('ckan.backup_dir', default_backup_dir)) openspending_reports_dir = os.path.expanduser( config.get('dgu.openspending_reports_dir', default_openspending_reports_dir)) ga_token_filepath = os.path.expanduser( config.get('googleanalytics.token.filepath', '')) dump_filebase = config.get('ckan.dump_filename_base', 'data.gov.uk-ckan-meta-data-%Y-%m-%d') dump_analysis_filebase = config.get('ckan.dump_analysis_base', 'data.gov.uk-analysis') backup_filebase = config.get('ckan.backup_filename_base', ckan_instance_name + '.%Y-%m-%d.pg_dump') tmp_filepath = config.get('ckan.temp_filepath', '/tmp/dump.tmp') openspending_reports_url = config.get( 'ckan.openspending_reports_url', 'http://data.etl.openspending.org/uk25k/report/') log = logging.getLogger('ckanext.dgu.bin.gov_daily') log.info('----------------------------') log.info('Starting daily script') start_time = datetime.datetime.today() import ckan.model as model import ckan.lib.dumper as dumper from ckanext.dgu.lib.inventory import inventory_dumper # Check database looks right num_packages_before = model.Session.query( model.Package).filter_by(state='active').count() log.info('Number of existing active packages: %i' % num_packages_before) if num_packages_before < 2: log.error('Expected more packages.') sys.exit(1) elif num_packages_before < 2500: log.warn('Expected more packages.') # Analytics try: if ga_token_filepath: if run_task('analytics'): log.info('Getting analytics for this month') from ckanext.ga_report.download_analytics import DownloadAnalytics from ckanext.ga_report.ga_auth import (init_service, get_profile_id) try: token, svc = init_service(ga_token_filepath, None) except TypeError: log.error( 'Could not complete authorization for Google Analytics.' 'Have you correctly run the getauthtoken task and ' 'specified the correct token file?') sys.exit(0) downloader = DownloadAnalytics(svc, token=token, profile_id=get_profile_id(svc), delete_first=False, skip_url_stats=False) downloader.latest() else: log.info( 'No token specified, so not downloading Google Analytics data') except Exception, exc_analytics: log.error("Failed to process Google Analytics data") log.exception(exc_analytics)