def test_latest(self):
svc = init_service("token.dat", "credentials.json")
try:
downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc))
downloader.latest()
except Exception as e:
assert False, e
def test_since(self):
svc = init_service("token.dat", "credentials.json")
downloader = DownloadAnalytics(svc, profile_id=get_profile_id(svc))
try:
downloader.for_date(datetime.datetime.now() -
datetime.timedelta(days=-30))
except Exception as e:
assert False, e
def command(config_file):
# Import ckan as it changes the dependent packages imported
from dump_analysis import (get_run_info, TxtAnalysisFile, CsvAnalysisFile,
DumpAnalysisOptions, DumpAnalysis)
from pylons import config
# settings
ckan_instance_name = os.path.basename(config_file).replace('.ini', '')
if ckan_instance_name not in ['development', 'dgutest']:
default_dump_dir = '/var/lib/ckan/%s/static/dump' % ckan_instance_name
default_analysis_dir = '/var/lib/ckan/%s/static/dump_analysis' % ckan_instance_name
default_backup_dir = '/var/backups/ckan/%s' % ckan_instance_name
default_openspending_reports_dir = '/var/lib/ckan/%s/openspending_reports' % ckan_instance_name
else:
# test purposes
default_dump_dir = '~/dump'
default_analysis_dir = '~/dump_analysis'
default_backup_dir = '~/backups'
default_openspending_reports_dir = '~/openspending_reports'
dump_dir = os.path.expanduser(config.get('ckan.dump_dir',
default_dump_dir))
analysis_dir = os.path.expanduser(
config.get('ckan.dump_analysis_dir', default_analysis_dir))
backup_dir = os.path.expanduser(
config.get('ckan.backup_dir', default_backup_dir))
openspending_reports_dir = os.path.expanduser(
config.get('dgu.openspending_reports_dir',
default_openspending_reports_dir))
ga_token_filepath = os.path.expanduser(
config.get('googleanalytics.token.filepath', ''))
dump_filebase = config.get('ckan.dump_filename_base',
'data.gov.uk-ckan-meta-data-%Y-%m-%d')
dump_analysis_filebase = config.get('ckan.dump_analysis_base',
'data.gov.uk-analysis')
backup_filebase = config.get('ckan.backup_filename_base',
ckan_instance_name + '.%Y-%m-%d.pg_dump')
tmp_filepath = config.get('ckan.temp_filepath', '/tmp/dump.tmp')
openspending_reports_url = config.get(
'ckan.openspending_reports_url',
'http://data.etl.openspending.org/uk25k/report/')
log = logging.getLogger('ckanext.dgu.bin.gov_daily')
log.info('----------------------------')
log.info('Starting daily script')
start_time = datetime.datetime.today()
import ckan.model as model
import ckan.lib.dumper as dumper
from ckanext.dgu.lib.inventory import inventory_dumper
# Check database looks right
num_packages_before = model.Session.query(
model.Package).filter_by(state='active').count()
log.info('Number of existing active packages: %i' % num_packages_before)
if num_packages_before < 2:
log.error('Expected more packages.')
sys.exit(1)
elif num_packages_before < 2500:
log.warn('Expected more packages.')
# Analytics
try:
if ga_token_filepath:
if run_task('analytics'):
log.info('Getting analytics for this month')
from ckanext.ga_report.download_analytics import DownloadAnalytics
from ckanext.ga_report.ga_auth import (init_service,
get_profile_id)
try:
token, svc = init_service(ga_token_filepath, None)
except TypeError:
log.error(
'Could not complete authorization for Google Analytics.'
'Have you correctly run the getauthtoken task and '
'specified the correct token file?')
sys.exit(0)
downloader = DownloadAnalytics(svc,
token=token,
profile_id=get_profile_id(svc),
delete_first=False,
skip_url_stats=False)
downloader.latest()
else:
log.info(
'No token specified, so not downloading Google Analytics data')
except Exception, exc_analytics:
log.error("Failed to process Google Analytics data")
log.exception(exc_analytics)
