def command(self):
"""
In this case we don't want a valid service, but rather just to
force the user through the auth flow. We allow this to complete to
act as a form of verification instead of just getting the token and
assuming it is correct.
"""
from ga_auth import init_service
init_service('token.dat', 'credentials.json')
def command(self):
"""
In this case we don't want a valid service, but rather just to
force the user through the auth flow. We allow this to complete to
act as a form of verification instead of just getting the token and
assuming it is correct.
"""
from ga_auth import init_service
init_service('token.dat', 'credentials.json')
def command(self):
self._load_config()
from download_analytics import DownloadAnalytics
from ga_auth import (init_service, get_profile_id)
ga_token_filepath = os.path.expanduser(
config.get('googleanalytics.token.filepath', ''))
if not ga_token_filepath:
print 'ERROR: In the CKAN config you need to specify the filepath of the ' \
'Google Analytics token file under key: googleanalytics.token.filepath'
return
try:
svc = init_service(ga_token_filepath)
except TypeError as e:
print('Unable to create a service: {0}'.format(e))
return
downloader = DownloadAnalytics(svc,
self.token,
profile_id=get_profile_id(svc),
delete_first=self.options.delete_first,
stat=self.options.stat,
print_progress=True)
time_period = self.args[0] if self.args else 'latest'
if time_period == 'all':
downloader.all_()
elif time_period == 'latest':
downloader.latest()
else:
# The month to use
for_date = datetime.datetime.strptime(time_period, '%Y-%m')
downloader.specific_month(for_date)
def parse_and_save(self):
"""Grab raw data from Google Analytics and save to the database"""
from ga_auth import (init_service, get_profile_id)
if len(self.args) == 0:
raise Exception("Missing token file")
tokenfile = self.args[0]
if not os.path.exists(tokenfile):
raise Exception('Cannot find the token file %s' % self.args[1])
try:
self.service = init_service(self.args[0])
except TypeError as e:
raise Exception('Unable to create a service: {0}'.format(e))
self.profile_id = get_profile_id(self.service)
if len(self.args) > 2:
raise Exception('Too many arguments')
given_start_date = None
if len(self.args) == 2:
given_start_date = datetime.datetime.strptime(self.args[1], '%Y-%m-%d').date()
packages_data = self.get_ga_data(start_date=given_start_date)
self.save_ga_data(packages_data)
log.info("Saved %s records from google" % len(packages_data))
def parse_and_save(self):
"""Grab raw data from Google Analytics and save to the database"""
from ga_auth import (init_service, get_profile_id)
tokenfile = self.args[0]
if not os.path.exists(tokenfile):
raise Exception('Cannot find the token file %s' % self.args[0])
try:
self.service = init_service(self.args[0], None)
except TypeError:
print ('Have you correctly run the getauthtoken task and '
'specified the correct file here')
raise Exception('Unable to create a service')
self.profile_id = get_profile_id(self.service)
if len(self.args) > 1:
if len(self.args) > 2 and self.args[1].lower() != 'internal':
raise Exception('Illegal argument %s' % self.args[1])
self.bulk_import()
else:
query = 'ga:pagePath=~%s,ga:pagePath=~%s' % \
(PACKAGE_URL, self.resource_url_tag)
packages_data = self.get_ga_data(query_filter=query)
self.save_ga_data(packages_data)
log.info("Saved %s records from google" % len(packages_data))
def getauthtoken(self, args):
"""
In this case we don't want a valid service, but rather just to
force the user through the auth flow. We allow this to complete to
act as a form of verification instead of just getting the token and
assuming it is correct.
"""
from ga_auth import init_service
if len(args) > 2:
raise Exception('Too many arguments')
credentials_file = None
if len(args) == 2:
credentails_file = args[1]
init_service(
'token.dat',
credentials_file if credentials_file else 'credentials.json')
def parse_and_save(self, args):
"""Grab raw data from Google Analytics and save to the database"""
from ga_auth import (init_service, get_profile_id)
if len(args) == 1:
raise Exception("Missing token file")
tokenfile = args[1]
if not os.path.exists(tokenfile):
raise Exception('Cannot find the token file %s' % args[1])
try:
self.service = init_service(args[1], None)
except TypeError:
print(
'Have you correctly run the getauthtoken task and '
'specified the correct file here')
raise Exception('Unable to create a service')
self.profile_id = get_profile_id(self.service)
if len(args) > 3:
raise Exception('Too many arguments')
given_start_date = None
if len(args) == 3:
given_start_date = datetime.datetime.strptime(args[2],
'%Y-%m-%d').date()
packages_data = self.get_ga_data(start_date=given_start_date)
self.save_ga_data(packages_data)
self.log.info("Saved %s records from google" % len(packages_data))
def parse_and_save(self):
"""Grab raw data from Google Analytics and save to the database"""
from ga_auth import (init_service, get_profile_id)
tokenfile = self.args[0]
if not os.path.exists(tokenfile):
raise Exception('Cannot find the token file %s' % self.args[0])
try:
self.service = init_service(self.args[0], None)
except TypeError:
print(
'Have you correctly run the getauthtoken task and '
'specified the correct file here')
raise Exception('Unable to create a service')
self.profile_id = get_profile_id(self.service)
if len(self.args) > 1:
if len(self.args) > 2 and self.args[1].lower() != 'internal':
raise Exception('Illegal argument %s' % self.args[1])
self.bulk_import()
else:
query = 'ga:pagePath=~%s,ga:pagePath=~%s' % \
(PACKAGE_URL, self.resource_url_tag)
packages_data = self.get_ga_data(query_filter=query)
self.save_ga_data(packages_data)
log.info("Saved %s records from google" % len(packages_data))
def command(self):
self._load_config()
from download_analytics import DownloadAnalytics
from ga_auth import (init_service, get_profile_id)
ga_token_filepath = os.path.expanduser(config.get('googleanalytics.token.filepath', ''))
if not ga_token_filepath:
print 'ERROR: In the CKAN config you need to specify the filepath of the ' \
'Google Analytics token file under key: googleanalytics.token.filepath'
return
try:
self.token, svc = init_service(ga_token_filepath, None)
except TypeError:
print ('Have you correctly run the getauthtoken task and '
'specified the correct token file in the CKAN config under '
'"googleanalytics.token.filepath"?')
return
downloader = DownloadAnalytics(svc, self.token, profile_id=get_profile_id(svc),
delete_first=self.options.delete_first,
skip_url_stats=self.options.skip_url_stats)
time_period = self.args[0] if self.args else 'latest'
if time_period == 'all':
downloader.all_()
elif time_period == 'latest':
downloader.latest()
else:
# The month to use
for_date = datetime.datetime.strptime(time_period, '%Y-%m')
downloader.specific_month(for_date)
def command(self):
self._load_config()
from download_analytics import DownloadAnalytics
from ga_auth import (init_service, get_profile_id)
ga_token_filepath = os.path.expanduser(config.get('googleanalytics.token.filepath', ''))
if not ga_token_filepath:
print 'ERROR: In the CKAN config you need to specify the filepath of the ' \
'Google Analytics token file under key: googleanalytics.token.filepath'
return
try:
self.token, svc = init_service(ga_token_filepath, None)
except TypeError:
print ('Have you correctly run the getauthtoken task and '
'specified the correct token file in the CKAN config under '
'"googleanalytics.token.filepath"?')
return
downloader = DownloadAnalytics(svc, self.token, profile_id=get_profile_id(svc),
delete_first=self.options.delete_first,
stat=self.options.stat,
print_progress=True)
time_period = self.args[0] if self.args else 'latest'
if time_period == 'all':
downloader.all_()
elif time_period == 'latest':
downloader.latest()
elif time_period == 'year':
year_date = self.args[1]
downloader.specific_year(year_date)
elif time_period == 'year-month':
year = self.args[1]
month = self.args[2]
downloader.specific_year_month(year, month)
elif time_period == 'year-step':
year_date = self.args[1]
downloader.specific_year_step(year_date)
elif time_period == 'only':
only_data = str(self.args[1])
selected_date = self.args[2]
for_date = datetime.datetime.strptime(selected_date, '%Y-%m')
downloader.specific_month_only(for_date, only_data)
else:
# The month to use
for_date = datetime.datetime.strptime(time_period, '%Y-%m')
downloader.specific_month(for_date)
def command(self):
"""
In this case we don't want a valid service, but rather just to
force the user through the auth flow. We allow this to complete to
act as a form of verification instead of just getting the token and
assuming it is correct.
"""
init = datetime.datetime.now()
s_args = (self.args if self.args else '(no args)')
print '[%s] - Init DgeGaReportGetAuthToken command with args: %s.' % (
init.strftime(DgeGaReportGetAuthToken.datetime_format), s_args)
try:
from ga_auth import init_service
init_service('token.dat',
self.args[0] if self.args else 'credentials.json')
except Exception as e:
print 'Exception %s' % e
sys.exit(1)
finally:
end = datetime.datetime.now()
print '[%s] - End DgeGaReportGetAuthToken command with args %s. Executed command in %s milliseconds.' % (
end.strftime(DgeGaReportGetAuthToken.datetime_format), s_args,
(end - init).total_seconds() * 1000)
sys.exit(0)
def _get_json(self, params, prev_fail=False):
ga_token_filepath = os.path.expanduser(config.get('googleanalytics.token.filepath', ''))
if not ga_token_filepath:
print 'ERROR: In the CKAN config you need to specify the filepath of the ' \
'Google Analytics token file under key: googleanalytics.token.filepath'
return
log.info("Trying to refresh our OAuth token")
try:
from ga_auth import init_service
self.token, svc = init_service(ga_token_filepath, None)
log.info("OAuth token refreshed")
except Exception, auth_exception:
log.error("Oauth refresh failed")
log.exception(auth_exception)
return
def _get_json(self, params, prev_fail=False):
ga_token_filepath = os.path.expanduser(config.get('googleanalytics.token.filepath', ''))
if not ga_token_filepath:
print 'ERROR: In the CKAN config you need to specify the filepath of the ' \
'Google Analytics token file under key: googleanalytics.token.filepath'
return
log.info("Trying to refresh our OAuth token")
try:
from ga_auth import init_service
self.token, svc = init_service(ga_token_filepath, None)
log.info("OAuth token refreshed")
except Exception, auth_exception:
log.error("Oauth refresh failed")
log.exception(auth_exception)
return
def init_service(self, args):
from ga_auth import init_service
if len(args) == 1:
raise Exception("Missing credentials file")
credentialsfile = args[1]
if not os.path.exists(credentialsfile):
raise Exception('Cannot find the credentials file %s' % credentialsfile)
try:
self.service = init_service(credentialsfile)
except TypeError:
print('Have you correctly run the init service task and '
'specified the correct file here')
raise Exception('Unable to create a service')
return self.service
def _get_ga_data_simple(self, params):
'''Returns the GA data specified in params.
Does all requests to the GA API.
Returns a dict with the data, or raises DownloadError if unsuccessful.
'''
ga_token_filepath = os.path.expanduser(
config.get('googleanalytics.token.filepath', ''))
if not ga_token_filepath:
log.error('In the CKAN config you need to specify the filepath '
'of the Google Analytics token file under key: '
'googleanalytics.token.filepath')
return
try:
from ga_auth import init_service
self.token, svc = init_service(ga_token_filepath, None)
except Exception, auth_exception:
log.error('OAuth refresh failed')
log.exception(auth_exception)
return dict(url=[])
def _get_ga_data_simple(self, params):
'''Returns the GA data specified in params.
Does all requests to the GA API.
Returns a dict with the data, or raises DownloadError if unsuccessful.
'''
ga_token_filepath = os.path.expanduser(
config.get('googleanalytics.token.filepath', ''))
if not ga_token_filepath:
log.error('In the CKAN config you need to specify the filepath '
'of the Google Analytics token file under key: '
'googleanalytics.token.filepath')
return
try:
from ga_auth import init_service
self.token, svc = init_service(ga_token_filepath, None)
except Exception, auth_exception:
log.error('OAuth refresh failed')
log.exception(auth_exception)
return dict(url=[])
def command(self):
self._load_config()
from download_analytics import DownloadAnalytics
from ga_auth import (init_service, get_profile_id)
ga_token_filepath = os.path.expanduser(
config.get('ds_stats.ga.token.filepath', ''))
if not ga_token_filepath:
print 'ERROR: In the CKAN config you need to specify the ' \
'filepath of the Google Analytics token file under ' \
'key: ds_stats.ga.token.filepath'
return
try:
self.token, svc = init_service(ga_token_filepath)
except TypeError:
print(
'Have you correctly run the getauthtoken task and '
'specified the correct token file in the CKAN config under '
'"ds_stats.ga.token.filepath"?')
return
downloader = DownloadAnalytics(
svc,
self.token,
profile_id=get_profile_id(svc),
delete_first=self.options.delete_first,
skip_url_stats=self.options.skip_url_stats)
time_period = self.args[0] if self.args else 'latest'
if time_period == 'all':
downloader.all_()
elif time_period == 'latest':
downloader.latest()
else:
# The month to use
for_date = datetime.datetime.strptime(time_period, '%Y-%m')
downloader.specific_month(for_date)
def command(self):
self._load_config()
from download_analytics import DownloadAnalytics
from ga_auth import init_service, get_profile_id
ga_token_filepath = os.path.expanduser(config.get("googleanalytics.token.filepath", ""))
if not ga_token_filepath:
print "ERROR: In the CKAN config you need to specify the filepath of the " "Google Analytics token file under key: googleanalytics.token.filepath"
return
try:
self.token, svc = init_service(ga_token_filepath, None)
except TypeError:
print (
"Have you correctly run the getauthtoken task and "
"specified the correct token file in the CKAN config under "
'"googleanalytics.token.filepath"?'
)
return
downloader = DownloadAnalytics(
svc,
self.token,
profile_id=get_profile_id(svc),
delete_first=self.options.delete_first,
stat=self.options.stat,
print_progress=True,
)
time_period = self.args[0] if self.args else "latest"
if time_period == "all":
downloader.all_()
elif time_period == "latest":
downloader.latest()
else:
# The month to use
for_date = datetime.datetime.strptime(time_period, "%Y-%m")
downloader.specific_month(for_date)
def parse_and_save(self):
"""Grab raw data from Google Analytics and save to the database"""
from ga_auth import (init_service, get_profile_id)
tokenfile = self.args[0]
if not os.path.exists(tokenfile):
raise Exception('Cannot find the token file %s' % self.args[0])
try:
self.service = init_service(self.args[0])
except TypeError as e:
raise Exception('Unable to create a service: {0}'.format(e))
self.profile_id = get_profile_id(self.service)
if len(self.args) > 1:
if len(self.args) > 2 and (self.args[1].lower() != 'internal' or self.args[1].lower() != 'downloads'):
raise Exception('Illegal argument %s' % self.args[1])
self.bulk_import(downloads=self.args[1].lower()=='downloads')
else:
query = 'ga:pagePath=~%s,ga:pagePath=~%s' % \
(PACKAGE_URL, self.resource_url_tag)
packages_data = self.get_ga_data(query_filter=query)
self.save_ga_data(packages_data)
log.info("Saved %s records from google" % len(packages_data))
def command(self):
"""Grab raw data from Google Analytics and save to the database"""
init = datetime.datetime.now()
s_args = (self.args if self.args else '(no args)')
print '[%s] - Init DgeGaReportLoadAnalytics command with args: %s.' % (
init.strftime(DgeGaReportLoadAnalytics.datetime_format), s_args)
try:
self._load_config()
from download_analytics import DownloadAnalytics
from ga_auth import (init_service, get_profile_id)
ga_token_filepath = config.get(
'ckanext-dge-ga-report.token.filepath', '')
if not ga_token_filepath or not os.path.exists(ga_token_filepath):
print 'ERROR: In the CKAN config you need to specify the filepath of the ' \
'Google Analytics token file under key: googleanalytics.token.filepath'
#return
sys.exit(1)
try:
self.token, svc = init_service(ga_token_filepath, None)
except TypeError:
print(
'Unable to create a service. Have you correctly run the getauthtoken task and '
'specified the correct token file in the CKAN config under '
'"ckanext-dge-ga-report.token.filepath"?')
#return
sys.exit(1)
save_print = self.args[0] if self.args else 'print'
save = True if save_print == 'save' else False
kind = self.args[1] if self.args else None
if kind is None or kind not in DownloadAnalytics.KIND_STATS:
print(
'A valid kind of statistics that you want to load must be '
'specified: %s' % DownloadAnalytics.KIND_STATS)
#return
sys.exit(1)
downloader = DownloadAnalytics(
svc,
self.token,
profile_id=get_profile_id(svc),
delete_first=self.options.delete_first,
stat=self.options.stat,
print_progress=True,
kind_stats=kind,
save_stats=save)
time_period = self.args[2] if self.args else 'latest'
if time_period == 'latest':
downloader.latest()
elif time_period == 'last_month':
now = datetime.datetime.now()
if now.month == 1:
last_month = datetime.datetime(now.year - 1, 12, 1, 0, 0,
0)
else:
last_month = datetime.datetime(now.year, now.month - 1, 1,
0, 0, 0)
downloader.specific_month(last_month)
else:
# The month to use
for_date = datetime.datetime.strptime(time_period, '%Y-%m')
downloader.specific_month(for_date)
except Exception as e:
print 'Exception %s' % e
sys.exit(1)
finally:
end = datetime.datetime.now()
print '[%s] - End DgeGaReportLoadAnalytics command with args %s. Executed command in %s milliseconds.' % (
end.strftime(DgeGaReportLoadAnalytics.datetime_format), s_args,
(end - init).total_seconds() * 1000)
sys.exit(0)
