def handle(self, *args, **options): pm_total = 0 pm_checked = 0 dup_removed = 0 with transaction.atomic(): for p in UserProject.objects.all(): for pm in ProjectMetrics.objects.filter(project=p, is_extension=False): pm.delete() for pm in ProjectMetrics.objects.filter(project=p, is_extension=True): if pm.urlmetrics.is_uptodate(): pm_checked += 1 pm.is_checked = True else: pm.is_checked = False for pd in p.projectdomain_set.filter(state=u'available'): ums = URLMetrics.objects.filter( query_url=pd.domain).order_by(u'-last_updated') if len(ums) == 0: um = URLMetrics(query_url=pd.domain) um.save() pm = ProjectMetrics(project=p, urlmetrics=um, is_checked=False) pm.save() else: if len(ums) > 1: for dum in ums[1:]: dup_removed += 1 dum.delete() um = ums[0] pm = ProjectMetrics(project=p, urlmetrics=um) if um.is_uptodate(): pm.is_checked = True pm_checked += 1 else: pm.is_checked = False pm.save() pm_total += 1 self.stdout.write('Statistics:') self.stdout.write(' Project metric links: %d' % pm_total) self.stdout.write(' Checked/up-to-date: %d' % pm_checked) self.stdout.write(' Duplicate metric records removed: %d' % dup_removed) self.stdout.write(' Total metric records (post-removal): %d' % (len(URLMetrics.objects.all())))
def update_project_metrics(project_id): """ Updates all the URLMetrics associated with the given project id through the Moz API. If the MozRank of a URL is over the set threshold, extension URLs are created and also checked. Args: project_id (int): The ID of the project to update. """ p = UserProject.objects.get(id=project_id) # Retrieve all fields available with free Moz API registration cols = URLMetrics.create_cols_bitflag([ 'Title', 'Canonical URL', 'External Links', 'Links', 'MozRank 10', 'MozRank Raw', 'Subdomain MozRank 10', 'Subdomain MozRank Raw', 'HTTP Status Code', 'Page Authority', 'Domain Authority']) wait_time = AdminSetting.get_moz_api_wait_time() mozrank_extension_threshold = AdminSetting.get_value('mozrank_extension_threshold') associate_project_metrics(p) pmetrics = ProjectMetrics.objects.filter(project=p, is_checked=False) for pm in pmetrics: with transaction.atomic(): if not pm.urlmetrics.is_uptodate(): check_moz_domain(pm.urlmetrics, cols, wait_time) if not pm.is_extension and pm.urlmetrics.mozrank_10 >= mozrank_extension_threshold: extensions = get_extensions(pm.urlmetrics) print u'Getting extensions (%d)' % len(extensions) for ex in extensions: print u' %s' % ex.query_url try: newpm = ProjectMetrics.objects.get(project=p, urlmetrics=ex) except ProjectMetrics.DoesNotExist: newpm = ProjectMetrics(project=p, urlmetrics=ex, is_checked=True, is_extension=True) if not ex.is_uptodate(): print u' Checking extension: %s' % ex.query_url check_moz_domain(ex, cols, wait_time) else: print u' Extension already checked: %s' % ex.query_url newpm.is_checked = True newpm.save() pm.is_checked=True pm.save() p.update_state() p.save()
def update_project_metrics(project_id): """ Updates all the URLMetrics associated with the given project id through the Moz API. If the MozRank of a URL is over the set threshold, extension URLs are created and also checked. Args: project_id (int): The ID of the project to update. """ p = UserProject.objects.get(id=project_id) # Retrieve all fields available with free Moz API registration cols = URLMetrics.create_cols_bitflag([ 'Title', 'Canonical URL', 'External Links', 'Links', 'MozRank 10', 'MozRank Raw', 'Subdomain MozRank 10', 'Subdomain MozRank Raw', 'HTTP Status Code', 'Page Authority', 'Domain Authority' ]) wait_time = AdminSetting.get_moz_api_wait_time() mozrank_extension_threshold = AdminSetting.get_value( 'mozrank_extension_threshold') associate_project_metrics(p) pmetrics = ProjectMetrics.objects.filter(project=p, is_checked=False) for pm in pmetrics: with transaction.atomic(): if not pm.urlmetrics.is_uptodate(): check_moz_domain(pm.urlmetrics, cols, wait_time) if not pm.is_extension and pm.urlmetrics.mozrank_10 >= mozrank_extension_threshold: extensions = get_extensions(pm.urlmetrics) print u'Getting extensions (%d)' % len(extensions) for ex in extensions: print u' %s' % ex.query_url try: newpm = ProjectMetrics.objects.get(project=p, urlmetrics=ex) except ProjectMetrics.DoesNotExist: newpm = ProjectMetrics(project=p, urlmetrics=ex, is_checked=True, is_extension=True) if not ex.is_uptodate(): print u' Checking extension: %s' % ex.query_url check_moz_domain(ex, cols, wait_time) else: print u' Extension already checked: %s' % ex.query_url newpm.is_checked = True newpm.save() pm.is_checked = True pm.save() p.update_state() p.save()