def index(request):
# Display list of recent names.
newest = Domain.all().order('-timestamp').fetch(10)
oldest = Domain.all().order('timestamp').fetch(5)
oldest.reverse()
domain_list = newest + [''] + oldest
# Recent statistics.
domain_stats = stats.KindStat.all().filter('kind_name', 'domains_domain')
domain_stats = domain_stats.order('-timestamp').fetch(3)
return render_to_response(request, 'domains/index.html', locals())
def upload_names(names, options):
timestamp = datetime.now()
domains = []
for name in names:
domain = Domain(key_name=name)
domain.before_put()
domains.append(domain)
lookups = lookup_names(names, options)
retry_objects(db.put, domains)
retry_objects(db.put, lookups)
def index(request):
# Display list of recent names.
newest = Domain.all().order('-timestamp').fetch(10)
oldest = Domain.all().order('timestamp').fetch(5)
oldest.reverse()
domain_list = newest + [''] + oldest
# Recent statistics.
domain_stats = stats.KindStat.all().filter('kind_name', 'domains_domain')
domain_stats = domain_stats.order('-timestamp').fetch(3)
return render_to_response(request, 'domains/index.html', locals())
def cron(request):
updated_domains = []
deleted_domains = []
query = Domain.all().order('timestamp')
update_description = 'oldest domain names'
domains = query.fetch(BATCH_SIZE_FETCH)
count_random = len(domains)
count_obsolete = 0
count_languages = 0
for domain in domains:
if (len(updated_domains) >= BATCH_SIZE_UPDATE
or len(deleted_domains) >= BATCH_SIZE_DELETE):
break
if len(domain.key().name()) > MAX_NAME_LENGTH:
deleted_domains.append(domain)
continue
domain.before_put()
if len(domain.key().name()) > 5 and not domain.score:
deleted_domains.append(domain)
continue
updated_domains.append(domain)
db.put(updated_domains)
db.delete(deleted_domains)
count_updated = len(updated_domains)
count_deleted = len(deleted_domains)
updated_domains.sort(key=lambda domain: -domain.score)
domain_list = updated_domains[:20] + [None] + deleted_domains[:10]
refresh_seconds = request.GET.get('refresh', 0)
return render_to_response(request, 'domains/index.html', locals())
def cron(request):
updated_domains = []
deleted_domains = []
query = Domain.all().order('timestamp')
update_description = 'oldest domain names'
domains = query.fetch(BATCH_SIZE_FETCH)
count_random = len(domains)
count_obsolete = 0
count_languages = 0
for domain in domains:
if (len(updated_domains) >= BATCH_SIZE_UPDATE or
len(deleted_domains) >= BATCH_SIZE_DELETE):
break
if len(domain.key().name()) > MAX_NAME_LENGTH:
deleted_domains.append(domain)
continue
domain.before_put()
if len(domain.key().name()) > 5 and not domain.score:
deleted_domains.append(domain)
continue
updated_domains.append(domain)
db.put(updated_domains)
db.delete(deleted_domains)
count_updated = len(updated_domains)
count_deleted = len(deleted_domains)
updated_domains.sort(key=lambda domain: -domain.score)
domain_list = updated_domains[:20] + [None] + deleted_domains[:10]
refresh_seconds = request.GET.get('refresh', 0)
return render_to_response(request, 'domains/index.html', locals())
def longest(request):
query = Domain.all(keys_only=True).order('-length')
query.filter('length > ', MAX_NAME_LENGTH)
names = [key.name() for key in query.fetch(100)]
db.delete([db.Key.from_path('domains_domain', name) for name in names])
db.delete([db.Key.from_path('dns_lookup', name) for name in names])
refresh_seconds = request.GET.get('refresh', 0)
return render_to_response(request, 'domains/longest.html', locals())
def longest(request):
query = Domain.all(keys_only=True).order('-length')
query.filter('length > ', MAX_NAME_LENGTH)
names = [key.name() for key in query.fetch(100)]
db.delete([db.Key.from_path('domains_domain', name) for name in names])
db.delete([db.Key.from_path('dns_lookup', name) for name in names])
refresh_seconds = request.GET.get('refresh', 0)
return render_to_response(request, 'domains/longest.html', locals())
def fetch_candidates(left, right, length):
query = Domain.all()
if left:
query.filter('left%d' % len(left), left)
if right:
query.filter('right%d' % len(right), right)
query.filter('length', length)
query.order('-score')
return query.fetch(JSON_FETCH_LIMIT)
def fetch_candidates(left, right, length):
query = Domain.all()
if left:
query.filter('left%d' % len(left), left)
if right:
query.filter('right%d' % len(right), right)
query.filter('length', length)
query.order('-score')
return query.fetch(JSON_FETCH_LIMIT)
def update_best_names(position, keyword, length, options):
print "Trying to fetch %d best names with" % options.batch,
if keyword and position == 'left':
print "prefix", keyword, "and",
if keyword and position == 'right':
print "suffix", keyword, "and",
print "length", length
query = Domain.all(keys_only=True)
if keyword:
query.filter('%s%d' % (position, len(keyword)), keyword)
query.filter('length', length)
query.order('-score')
keys = retry(query.fetch, options.batch)
if not keys:
return
lookups = lookup_names([key.name() for key in keys], options)
retry_objects(db.put, lookups)
def random_domains(keys_only=False,
length_choices=[1, 2, MAX_NAME_LENGTH],
position_choices=['left', 'right'],
order_choices=['length', '-english']):
query = Domain.all(keys_only=keys_only)
length = random.choice(length_choices)
name = random_prefix(length)
position = random.choice(position_choices)
if length == MAX_NAME_LENGTH:
description = "names that follow %s" % name
if position == 'left':
key = db.Key.from_path('domains_domain', name)
query.filter('__key__ >', key)
elif position == 'right':
query.filter('backwards >', name)
description += " backwards"
else:
order = random.choice(order_choices)
query.filter('%s%d' % (position, len(name)), name)
query.order(order)
description = "%s names that %s with %s" % (
ORDER_DESCRIPTIONS[order], POSITION_DESCRIPTIONS[position], name)
return query, description
def before_verify(request):
host_id = request.GET["hostid"]
domain_url = request.GET["domain"]
# domain_url = "dev.ecochecks.org"
host = get_object_or_404(Host, id=host_id)
# domain = domain.replace(r"-", r".")
if len(re.findall("www\\.", domain_url)) > 0:
domains = Domain.objects.filter(url__icontains=domain_url[4:], user=request.user)
else:
domains = Domain.objects.filter(url__icontains=domain_url, user=request.user)
if len(domains) > 0:
domain = domains[0]
else:
domain = Domain()
domain.user = request.user
domain.url = domain_url
domain.host = host
domain.save()
return redirect(reverse("verify_domain", args=[domain.id]))
