def test_crawl_request_model(self):
"""
Test for the Crawl Request model.
"""
user = User.objects.create_user('testCrawlRequestUser',
'*****@*****.**',
'testpassword')
crawl_instance = CrawlRequest(user=user)
crawl_instance.name = 'crawl_model_test'
crawl_instance.type = 1
crawl_instance.domain = 'http://abc.com'
crawl_instance.urls = 'http://url.com'
crawl_instance.description = 'Test Crawl Request'
crawl_instance.docs_all = True
crawl_instance.docs_html = False
crawl_instance.docs_docx = False
crawl_instance.docs_pdf = False
crawl_instance.docs_collected = 10
crawl_instance.status = 1
crawl_instance.storage_location = 'http://storage.com'
crawl_instance.crawler_manager_endpoint = 'http://end.com'
crawl_instance.manifest = 'http://manifest.com'
crawl_instance.num_crawlers = 1
crawl_instance.save()
crawl_instance.get_absolute_url()
crawl_instances = CrawlRequest.objects.filter(name='crawl_model_test')
self.assertEqual(1, len(crawl_instances))
self.assertEqual("1 crawl_model_test", str(crawl_instances[0]))
def api_create_crawl(request):
"""
External api to submit a crawl request.
"""
logger.info('In api new job')
username = request.data['username']
user_obj = User.objects.get(username=username)
crawl_request = CrawlRequest(user=user_obj)
crawl_request.name = request.data['name']
crawl_request.domain = request.data['domain']
crawl_request.urls = request.data['urls']
crawl_request.save()
logger.info('NewJob created: %s', crawl_request.id)
logger.info('Received urls: %s', crawl_request.urls)
launch_crawler_manager(crawl_request, crawl_request.id)
payload = {}
payload['jobId'] = crawl_request.id
return Response(payload, status=status.HTTP_200_OK)