def command(self): """ In this case we don't want a valid service, but rather just to force the user through the auth flow. We allow this to complete to act as a form of verification instead of just getting the token and assuming it is correct. """ from ga_auth import init_service init_service('token.dat', 'credentials.json')
def command(self): self._load_config() from download_analytics import DownloadAnalytics from ga_auth import (init_service, get_profile_id) ga_token_filepath = os.path.expanduser( config.get('googleanalytics.token.filepath', '')) if not ga_token_filepath: print 'ERROR: In the CKAN config you need to specify the filepath of the ' \ 'Google Analytics token file under key: googleanalytics.token.filepath' return try: svc = init_service(ga_token_filepath) except TypeError as e: print('Unable to create a service: {0}'.format(e)) return downloader = DownloadAnalytics(svc, self.token, profile_id=get_profile_id(svc), delete_first=self.options.delete_first, stat=self.options.stat, print_progress=True) time_period = self.args[0] if self.args else 'latest' if time_period == 'all': downloader.all_() elif time_period == 'latest': downloader.latest() else: # The month to use for_date = datetime.datetime.strptime(time_period, '%Y-%m') downloader.specific_month(for_date)
def parse_and_save(self): """Grab raw data from Google Analytics and save to the database""" from ga_auth import (init_service, get_profile_id) if len(self.args) == 0: raise Exception("Missing token file") tokenfile = self.args[0] if not os.path.exists(tokenfile): raise Exception('Cannot find the token file %s' % self.args[1]) try: self.service = init_service(self.args[0]) except TypeError as e: raise Exception('Unable to create a service: {0}'.format(e)) self.profile_id = get_profile_id(self.service) if len(self.args) > 2: raise Exception('Too many arguments') given_start_date = None if len(self.args) == 2: given_start_date = datetime.datetime.strptime(self.args[1], '%Y-%m-%d').date() packages_data = self.get_ga_data(start_date=given_start_date) self.save_ga_data(packages_data) log.info("Saved %s records from google" % len(packages_data))
def parse_and_save(self): """Grab raw data from Google Analytics and save to the database""" from ga_auth import (init_service, get_profile_id) tokenfile = self.args[0] if not os.path.exists(tokenfile): raise Exception('Cannot find the token file %s' % self.args[0]) try: self.service = init_service(self.args[0], None) except TypeError: print ('Have you correctly run the getauthtoken task and ' 'specified the correct file here') raise Exception('Unable to create a service') self.profile_id = get_profile_id(self.service) if len(self.args) > 1: if len(self.args) > 2 and self.args[1].lower() != 'internal': raise Exception('Illegal argument %s' % self.args[1]) self.bulk_import() else: query = 'ga:pagePath=~%s,ga:pagePath=~%s' % \ (PACKAGE_URL, self.resource_url_tag) packages_data = self.get_ga_data(query_filter=query) self.save_ga_data(packages_data) log.info("Saved %s records from google" % len(packages_data))
def getauthtoken(self, args): """ In this case we don't want a valid service, but rather just to force the user through the auth flow. We allow this to complete to act as a form of verification instead of just getting the token and assuming it is correct. """ from ga_auth import init_service if len(args) > 2: raise Exception('Too many arguments') credentials_file = None if len(args) == 2: credentails_file = args[1] init_service( 'token.dat', credentials_file if credentials_file else 'credentials.json')
def parse_and_save(self, args): """Grab raw data from Google Analytics and save to the database""" from ga_auth import (init_service, get_profile_id) if len(args) == 1: raise Exception("Missing token file") tokenfile = args[1] if not os.path.exists(tokenfile): raise Exception('Cannot find the token file %s' % args[1]) try: self.service = init_service(args[1], None) except TypeError: print( 'Have you correctly run the getauthtoken task and ' 'specified the correct file here') raise Exception('Unable to create a service') self.profile_id = get_profile_id(self.service) if len(args) > 3: raise Exception('Too many arguments') given_start_date = None if len(args) == 3: given_start_date = datetime.datetime.strptime(args[2], '%Y-%m-%d').date() packages_data = self.get_ga_data(start_date=given_start_date) self.save_ga_data(packages_data) self.log.info("Saved %s records from google" % len(packages_data))
def parse_and_save(self): """Grab raw data from Google Analytics and save to the database""" from ga_auth import (init_service, get_profile_id) tokenfile = self.args[0] if not os.path.exists(tokenfile): raise Exception('Cannot find the token file %s' % self.args[0]) try: self.service = init_service(self.args[0], None) except TypeError: print( 'Have you correctly run the getauthtoken task and ' 'specified the correct file here') raise Exception('Unable to create a service') self.profile_id = get_profile_id(self.service) if len(self.args) > 1: if len(self.args) > 2 and self.args[1].lower() != 'internal': raise Exception('Illegal argument %s' % self.args[1]) self.bulk_import() else: query = 'ga:pagePath=~%s,ga:pagePath=~%s' % \ (PACKAGE_URL, self.resource_url_tag) packages_data = self.get_ga_data(query_filter=query) self.save_ga_data(packages_data) log.info("Saved %s records from google" % len(packages_data))
def command(self): self._load_config() from download_analytics import DownloadAnalytics from ga_auth import (init_service, get_profile_id) ga_token_filepath = os.path.expanduser(config.get('googleanalytics.token.filepath', '')) if not ga_token_filepath: print 'ERROR: In the CKAN config you need to specify the filepath of the ' \ 'Google Analytics token file under key: googleanalytics.token.filepath' return try: self.token, svc = init_service(ga_token_filepath, None) except TypeError: print ('Have you correctly run the getauthtoken task and ' 'specified the correct token file in the CKAN config under ' '"googleanalytics.token.filepath"?') return downloader = DownloadAnalytics(svc, self.token, profile_id=get_profile_id(svc), delete_first=self.options.delete_first, skip_url_stats=self.options.skip_url_stats) time_period = self.args[0] if self.args else 'latest' if time_period == 'all': downloader.all_() elif time_period == 'latest': downloader.latest() else: # The month to use for_date = datetime.datetime.strptime(time_period, '%Y-%m') downloader.specific_month(for_date)
def command(self): self._load_config() from download_analytics import DownloadAnalytics from ga_auth import (init_service, get_profile_id) ga_token_filepath = os.path.expanduser(config.get('googleanalytics.token.filepath', '')) if not ga_token_filepath: print 'ERROR: In the CKAN config you need to specify the filepath of the ' \ 'Google Analytics token file under key: googleanalytics.token.filepath' return try: self.token, svc = init_service(ga_token_filepath, None) except TypeError: print ('Have you correctly run the getauthtoken task and ' 'specified the correct token file in the CKAN config under ' '"googleanalytics.token.filepath"?') return downloader = DownloadAnalytics(svc, self.token, profile_id=get_profile_id(svc), delete_first=self.options.delete_first, stat=self.options.stat, print_progress=True) time_period = self.args[0] if self.args else 'latest' if time_period == 'all': downloader.all_() elif time_period == 'latest': downloader.latest() elif time_period == 'year': year_date = self.args[1] downloader.specific_year(year_date) elif time_period == 'year-month': year = self.args[1] month = self.args[2] downloader.specific_year_month(year, month) elif time_period == 'year-step': year_date = self.args[1] downloader.specific_year_step(year_date) elif time_period == 'only': only_data = str(self.args[1]) selected_date = self.args[2] for_date = datetime.datetime.strptime(selected_date, '%Y-%m') downloader.specific_month_only(for_date, only_data) else: # The month to use for_date = datetime.datetime.strptime(time_period, '%Y-%m') downloader.specific_month(for_date)
def command(self): """ In this case we don't want a valid service, but rather just to force the user through the auth flow. We allow this to complete to act as a form of verification instead of just getting the token and assuming it is correct. """ init = datetime.datetime.now() s_args = (self.args if self.args else '(no args)') print '[%s] - Init DgeGaReportGetAuthToken command with args: %s.' % ( init.strftime(DgeGaReportGetAuthToken.datetime_format), s_args) try: from ga_auth import init_service init_service('token.dat', self.args[0] if self.args else 'credentials.json') except Exception as e: print 'Exception %s' % e sys.exit(1) finally: end = datetime.datetime.now() print '[%s] - End DgeGaReportGetAuthToken command with args %s. Executed command in %s milliseconds.' % ( end.strftime(DgeGaReportGetAuthToken.datetime_format), s_args, (end - init).total_seconds() * 1000) sys.exit(0)
def _get_json(self, params, prev_fail=False): ga_token_filepath = os.path.expanduser(config.get('googleanalytics.token.filepath', '')) if not ga_token_filepath: print 'ERROR: In the CKAN config you need to specify the filepath of the ' \ 'Google Analytics token file under key: googleanalytics.token.filepath' return log.info("Trying to refresh our OAuth token") try: from ga_auth import init_service self.token, svc = init_service(ga_token_filepath, None) log.info("OAuth token refreshed") except Exception, auth_exception: log.error("Oauth refresh failed") log.exception(auth_exception) return
def init_service(self, args): from ga_auth import init_service if len(args) == 1: raise Exception("Missing credentials file") credentialsfile = args[1] if not os.path.exists(credentialsfile): raise Exception('Cannot find the credentials file %s' % credentialsfile) try: self.service = init_service(credentialsfile) except TypeError: print('Have you correctly run the init service task and ' 'specified the correct file here') raise Exception('Unable to create a service') return self.service
def _get_ga_data_simple(self, params): '''Returns the GA data specified in params. Does all requests to the GA API. Returns a dict with the data, or raises DownloadError if unsuccessful. ''' ga_token_filepath = os.path.expanduser( config.get('googleanalytics.token.filepath', '')) if not ga_token_filepath: log.error('In the CKAN config you need to specify the filepath ' 'of the Google Analytics token file under key: ' 'googleanalytics.token.filepath') return try: from ga_auth import init_service self.token, svc = init_service(ga_token_filepath, None) except Exception, auth_exception: log.error('OAuth refresh failed') log.exception(auth_exception) return dict(url=[])
def command(self): self._load_config() from download_analytics import DownloadAnalytics from ga_auth import (init_service, get_profile_id) ga_token_filepath = os.path.expanduser( config.get('ds_stats.ga.token.filepath', '')) if not ga_token_filepath: print 'ERROR: In the CKAN config you need to specify the ' \ 'filepath of the Google Analytics token file under ' \ 'key: ds_stats.ga.token.filepath' return try: self.token, svc = init_service(ga_token_filepath) except TypeError: print( 'Have you correctly run the getauthtoken task and ' 'specified the correct token file in the CKAN config under ' '"ds_stats.ga.token.filepath"?') return downloader = DownloadAnalytics( svc, self.token, profile_id=get_profile_id(svc), delete_first=self.options.delete_first, skip_url_stats=self.options.skip_url_stats) time_period = self.args[0] if self.args else 'latest' if time_period == 'all': downloader.all_() elif time_period == 'latest': downloader.latest() else: # The month to use for_date = datetime.datetime.strptime(time_period, '%Y-%m') downloader.specific_month(for_date)
def command(self): self._load_config() from download_analytics import DownloadAnalytics from ga_auth import init_service, get_profile_id ga_token_filepath = os.path.expanduser(config.get("googleanalytics.token.filepath", "")) if not ga_token_filepath: print "ERROR: In the CKAN config you need to specify the filepath of the " "Google Analytics token file under key: googleanalytics.token.filepath" return try: self.token, svc = init_service(ga_token_filepath, None) except TypeError: print ( "Have you correctly run the getauthtoken task and " "specified the correct token file in the CKAN config under " '"googleanalytics.token.filepath"?' ) return downloader = DownloadAnalytics( svc, self.token, profile_id=get_profile_id(svc), delete_first=self.options.delete_first, stat=self.options.stat, print_progress=True, ) time_period = self.args[0] if self.args else "latest" if time_period == "all": downloader.all_() elif time_period == "latest": downloader.latest() else: # The month to use for_date = datetime.datetime.strptime(time_period, "%Y-%m") downloader.specific_month(for_date)
def parse_and_save(self): """Grab raw data from Google Analytics and save to the database""" from ga_auth import (init_service, get_profile_id) tokenfile = self.args[0] if not os.path.exists(tokenfile): raise Exception('Cannot find the token file %s' % self.args[0]) try: self.service = init_service(self.args[0]) except TypeError as e: raise Exception('Unable to create a service: {0}'.format(e)) self.profile_id = get_profile_id(self.service) if len(self.args) > 1: if len(self.args) > 2 and (self.args[1].lower() != 'internal' or self.args[1].lower() != 'downloads'): raise Exception('Illegal argument %s' % self.args[1]) self.bulk_import(downloads=self.args[1].lower()=='downloads') else: query = 'ga:pagePath=~%s,ga:pagePath=~%s' % \ (PACKAGE_URL, self.resource_url_tag) packages_data = self.get_ga_data(query_filter=query) self.save_ga_data(packages_data) log.info("Saved %s records from google" % len(packages_data))
def command(self): """Grab raw data from Google Analytics and save to the database""" init = datetime.datetime.now() s_args = (self.args if self.args else '(no args)') print '[%s] - Init DgeGaReportLoadAnalytics command with args: %s.' % ( init.strftime(DgeGaReportLoadAnalytics.datetime_format), s_args) try: self._load_config() from download_analytics import DownloadAnalytics from ga_auth import (init_service, get_profile_id) ga_token_filepath = config.get( 'ckanext-dge-ga-report.token.filepath', '') if not ga_token_filepath or not os.path.exists(ga_token_filepath): print 'ERROR: In the CKAN config you need to specify the filepath of the ' \ 'Google Analytics token file under key: googleanalytics.token.filepath' #return sys.exit(1) try: self.token, svc = init_service(ga_token_filepath, None) except TypeError: print( 'Unable to create a service. Have you correctly run the getauthtoken task and ' 'specified the correct token file in the CKAN config under ' '"ckanext-dge-ga-report.token.filepath"?') #return sys.exit(1) save_print = self.args[0] if self.args else 'print' save = True if save_print == 'save' else False kind = self.args[1] if self.args else None if kind is None or kind not in DownloadAnalytics.KIND_STATS: print( 'A valid kind of statistics that you want to load must be ' 'specified: %s' % DownloadAnalytics.KIND_STATS) #return sys.exit(1) downloader = DownloadAnalytics( svc, self.token, profile_id=get_profile_id(svc), delete_first=self.options.delete_first, stat=self.options.stat, print_progress=True, kind_stats=kind, save_stats=save) time_period = self.args[2] if self.args else 'latest' if time_period == 'latest': downloader.latest() elif time_period == 'last_month': now = datetime.datetime.now() if now.month == 1: last_month = datetime.datetime(now.year - 1, 12, 1, 0, 0, 0) else: last_month = datetime.datetime(now.year, now.month - 1, 1, 0, 0, 0) downloader.specific_month(last_month) else: # The month to use for_date = datetime.datetime.strptime(time_period, '%Y-%m') downloader.specific_month(for_date) except Exception as e: print 'Exception %s' % e sys.exit(1) finally: end = datetime.datetime.now() print '[%s] - End DgeGaReportLoadAnalytics command with args %s. Executed command in %s milliseconds.' % ( end.strftime(DgeGaReportLoadAnalytics.datetime_format), s_args, (end - init).total_seconds() * 1000) sys.exit(0)