def get_url(request):
if request.method == 'POST':
form = URLForm(request.POST)
if form.is_valid():
url = form.cleaned_data["URL"]
try:
r = requests.get(url)
except requests.exceptions.RequestException as e:
print e
return HttpResponse("Connection failed")
else:
content = r.content
list_counters = content.split("<br \>")
for counter in list_counters:
if counter != "":
c = counter.split("=")
d = Counter(counter_name= c[0], counter_value= c[1])
d.save()
return HttpResponseRedirect('/counters_app/start')
else:
form = URLForm()
return render(request, 'counters_app/URL.html', {'form': form})
def home_view(request):
context = {}
context['user'] = request.user
if request.method == 'POST':
form = URLForm(request.POST)
if form.is_valid():
# Get url
url = form.cleaned_data['url']
# crawl
# Very poorly managed. This should have some safety features
# you should really just create a seperate crawl command to search important domains.
# It should be running whenever the server is running
found = mark_to_crawl(url)
if not found:
context['not_found'] = 'Site not found'
# get links
try:
site = Website.objects.get(url=urlparse(url).netloc)
context['pages'] = Webpage.objects.filter(website=site).all()[:10]
except Website.DoesNotExist:
# add 'please wait message to output'
pass
else:
form = URLForm() # An unbound form
context['form'] = form
return render_to_response('home.html', context, RequestContext(request))
def home_view(request):
context = {}
context['user'] = request.user
if request.method == 'POST':
form = URLForm(request.POST)
if form.is_valid():
# Get url
url = form.cleaned_data['url']
# crawl
# Very poorly managed. This should have some safety features
# you should really just create a seperate crawl command to search important domains.
# It should be running whenever the server is running
found = mark_to_crawl(url)
if not found:
context['not_found'] = 'Site not found'
# get links
try:
site = Website.objects.get(url=urlparse(url).netloc)
context['pages'] = Webpage.objects.filter(
website=site).all()[:10]
except Website.DoesNotExist:
# add 'please wait message to output'
pass
else:
form = URLForm() # An unbound form
context['form'] = form
return render_to_response('home.html', context, RequestContext(request))
def index(request):
if request.method == 'POST':
form = URLForm(request.POST)
if form.is_valid():
xml_url = form.cleaned_data["url"]
redirect_url = reverse("heartbeatmatch.views.track") + "?url=" + urllib2.quote(xml_url)
return HttpResponseRedirect(redirect_url)
else:
return render_to_response('hbm-index.html', {'URLForm': form}, context_instance=RequestContext(request))
else:
return render_to_response('hbm-index.html', {'URLForm': URLForm()}, context_instance=RequestContext(request))
def updateURL(request, key):
if not admin():
return HttpResponseRedirect(users.create_login_url("/blogs"))
form = URLForm(request.POST)
if form.is_valid():
friendlyURL = FriendlyURL.get(key)
friendlyURL.name = form.cleaned_data["name"]
friendlyURL.URL = form.cleaned_data["URL"]
friendlyURL.put()
return HttpResponseRedirect("/blogs")
else:
return editURL(request, key)
def createURL(request):
if not admin():
return HttpResponseRedirect(users.create_login_url('/blogs'))
form = URLForm(request.POST)
if form.is_valid():
friendlyURL = FriendlyURL()
friendlyURL.name = form.cleaned_data['name']
friendlyURL.URL = form.cleaned_data['URL']
friendlyURL.put()
return HttpResponseRedirect('/blogs')
else:
return newURL(request)
def get_url(request):
''' This function grabs the url that the user inputs in the landing page
Then scrapes the page using requests and loads the values into databse '''
if request.method == 'POST':
form = URLForm(request.POST)
if form.is_valid():
url = form.cleaned_data["URL"]
try:
r = requests.get(url)
except requests.exceptions.RequestException as e:
return HttpResponse("Connection failed ", e)
else:
u = url_hash()
u.url_hash = u._createHash(url)
content = r.content
list_counters = content.split("<br \>")
for counter in list_counters:
if counter != "":
c = counter.split("=")
app_counter = c[0].split("_")
_app_name = app_counter[0]
a = app_name(url_hash=u.url_hash, app_name=_app_name)
d = Counter(counter_name=app_counter[1],
counter_value=c[1],
app_name=a.app_name,
url_hash=u.url_hash,
pub_date=parser.parse(c[2]))
a.save()
u.save()
d.save()
return HttpResponseRedirect('/counters_app/start-apps/' +
str(u.url_hash))
else:
form = URLForm()
return render(request, 'counters_app/URL.html', {'form': form})
def get_url(request):
''' This function grabs the url that the user inputs in the landing page
Then scrapes the page using requests and loads the values into databse '''
if request.method == 'POST':
form = URLForm(request.POST)
if form.is_valid():
url = form.cleaned_data["URL"]
try:
r = requests.get(url)
except requests.exceptions.RequestException as e:
return HttpResponse("Connection failed ", e)
else:
u = url_hash()
u.url_hash = u._createHash(url)
content = r.content
list_counters = content.split("<br \>")
for counter in list_counters:
if counter != "":
c = counter.split("=")
app_counter = c[0].split("_")
_app_name = app_counter[0]
a = app_name(url_hash = u.url_hash, app_name = _app_name)
d = Counter(counter_name= app_counter[1], counter_value= c[1], app_name = a.app_name, url_hash = u.url_hash, pub_date = parser.parse(c[2]))
a.save()
u.save()
d.save()
return HttpResponseRedirect('/counters_app/start-apps/'+ str(u.url_hash))
else:
form = URLForm()
return render(request, 'counters_app/URL.html', {'form': form})
