def process_scrape(spider_name):
"""
Process the results of a scrape for a spider. Will read the JSONLines file and make the appropriate
Raw objects in the database
:param spider_name: str name of the spider that produced the results
:return:
"""
try:
job = ScrapeJob.objects.latest_complete_job(spider_name)
except ScrapeJob.DoesNotExist:
logger.warn("No jobs found for spider {}".format(spider_name))
return
items_file = job.raw_response
# Disable SQL logging
if settings.DEBUG:
original = BaseDatabaseWrapper.make_debug_cursor
BaseDatabaseWrapper.make_debug_cursor = lambda self, cursor: CursorWrapper(
cursor, self)
# Get the processor and run it
processor = processors.get_processor_for_spider(spider_name)
logger.info('Processing file {} from ScrapeJob {}'.format(
items_file, job.id))
processor(items_file, job).process()
if settings.DEBUG:
BaseDatabaseWrapper.make_debug_cursor = original
# Log that the job was processed just now
job.last_fetched = datetime.now()
job.save()
return
def handle(self, *args, **options):
# Disable DB debug messages temporarily
if settings.DEBUG:
original = BaseDatabaseWrapper.make_debug_cursor
BaseDatabaseWrapper.make_debug_cursor = lambda self, cursor: CursorWrapper(cursor, self)
if len(args) == 0:
raise RuntimeError('You must provide a spider name to process the results.')
items_file = options.get('items_file', None)
if items_file is not None:
if not os.path.exists(items_file):
raise RuntimeError("The file {} does not seem to exist".format(items_file))
self._process_items_file(args[0], items_file)
else:
for spider in args:
self.handle_single_spider(spider, options.get('force', False))
if settings.DEBUG:
BaseDatabaseWrapper.make_debug_cursor = original
def disable_database_logging():
from django.db.backends import BaseDatabaseWrapper
from django.db.backends.util import CursorWrapper
BaseDatabaseWrapper.make_debug_cursor = lambda self, cursor: CursorWrapper(
cursor, self)
from .models import OauthRequest
from django.conf import settings
import simplejson as json
import requests
import traceback
from urllib import urlencode
import logging
#=================================================================
# Prevent sql commands from being printed out for this section...
from django.conf import settings
from django.db.backends import BaseDatabaseWrapper
from django.db.backends.util import CursorWrapper
if settings.DEBUG:
BaseDatabaseWrapper.make_debug_cursor = lambda self, cursor: CursorWrapper(
cursor, self)
#==================================================================
log = logging.getLogger("myclient.apps.client.views")
@login_required
def client(request, client_id):
"""
Show the client details, available codes and access tokens for a
given client.
"""
# XXX The client should eventually be decoupled from the user
# account information.
log.debug("In apps.client.views.client - entered")
from AadhaarAuth.data import AuthData
from AadhaarAuth.command import AuthConfig
from AadhaarAuth.response import AuthResponse
log = logging.getLogger("apps.aadhaar.authenticate")
#=================================================================
# Prevent sql commands from being printed out for this section...
from django.db.backends import BaseDatabaseWrapper
from django.db.backends.util import CursorWrapper
if settings.DEBUG:
BaseDatabaseWrapper.make_debug_cursor = \
lambda self, cursor: CursorWrapper(cursor, self)
#==================================================================
class AadhaarHelper(object):
_user = None
_credentials = {}
def __init__(self):
if not hasattr(settings, "AADHAAR_SETTINGS"):
setattr(settings, 'AADHAAR_SETTINGS', AadhaarSettings())
if not settings.AADHAAR_SETTINGS.is_valid():
raise Exception("Configuration file is invalid")
def get_credentials(self):
log.debug("authenticate: Returning credentials = %s " % self._credentials)
def disable_database_logging():
BaseDatabaseWrapper.make_debug_cursor = lambda self, cursor: CursorWrapper(
cursor, self)
def _deactivate_db_debug(self):
if settings.DEBUG:
self.original = BaseDatabaseWrapper.make_debug_cursor
BaseDatabaseWrapper.make_debug_cursor = lambda self, cursor: CursorWrapper(
cursor, self)
