def get_queryset(self): #get current user user = User(self.request.session['user']) # get max allowed jobs max_allowed = user.get_max_jobs() # get jobs jobs_cursor = user.get_scheduled_jobs(find_all=True) active_jobs = [] inactive_jobs = [] ar = None # reference to array to add jobs to for job in jobs_cursor: #get total urls job['total_urls'] = len(job['urls']) #take just last 3 urls job['first_urls'] = job['urls'][:3] #create id job['id'] = str(job['_id']) #split on active/inactive if job['is_active']: ar = active_jobs else: ar = inactive_jobs ar.append(job) return (max_allowed, active_jobs, inactive_jobs)
def create_user(self, data, ):
if 'email' in data:
alias = data.get('name','')
email = data.get('email','')
if 'plan_name' in data:
plan_name = data['plan_name']
u = User.create_user(alias=alias, email=email)
else:
u = {'error': 'Email has not been supplied'}
msg = self.tpl
if 'error' in u:
msg['status'] = 'error'
msg['errormessage'] = u['error']
else:
u = User.get_user(profile_id=u['profile_id'])
msg['status'] = 'ok'
msg['payload'] = {'name': u['alias']}
self._send_confirmation_email(secret_key=u.get('secret_key',''),alias=alias,email=email)
return HttpResponse( bsondumps(msg), content_type='application/json' )
def get_queryset(self):
#get date from which search
d = datetime.datetime.utcnow()
d = d - datetime.timedelta(days=1)
#get current user
user = User(self.request.session['user'])
#get task ids from db
ids_cur = user.get_task_ids(d)
#get total results
self.total_results = ids_cur.count()
itms = []
null = None #to make sure that eval('null') do not fire error
for id in ids_cur.limit(self.n):
tsk = celery_app.backend.get_task_meta(id['_id'])
res = tsk['result']
if tsk['status'].lower() == u'failure':
res = u'{0} error: {1}'.format(res['exc_type'], res['exc_message'])
tsk['err'] = res
else:
try:
res = res['url'] if res else u''
tsk['url'] = res
except KeyError:
tsk['err'] = json.dumps(res)
tsk["status_class"] = u"status_" + tsk["status"].lower().replace(u' ', u'_')
itms.append(tsk)
return itms
def query(self):
u = self.get_user( CustomAppSettings.get_vissbl_user_id() )
# Get stats
stats = User(u).get_user_stats_object()
data = stats.get_top_results()
# Get reference to the last date of ranks
last_date = data[-1] if len(data) > 0 else {'value':{'ranks':[]}}
return {"data": data, "last_date_ranks": last_date['value']['ranks'][:30]}
def _create_new_user(self, form):
#cl = CustomAppSettings.get_mongo()
#db = cl.webscr_profiles
##get plan
#plan = db.plans.find_one({'name': 'basic'})
##get db index
#db_ind = db.counters.find_and_modify(
# query={'_id': 'dbindex'},
# update={'$inc': {'seq': 1}},
# upsert=True,
# new=True
#)['seq']
#db_ind = int(db_ind)
##get alias
#alias = self.request.POST.get('alias')
##get new secret key
#secret_key = CustomAppSettings.get_new_secret_key()
## get new api key
#api_key = CustomAppSettings.get_new_api_key()
##try to write
#d = datetime.datetime.utcnow()
#db.users.insert({
# 'alias': alias,
# 'secret_key': secret_key,
# 'api_key': api_key,
# 'db_ind': db_ind,
# 'role': 'user',
# 'is_active': 0,
# 'created': d,
# 'actdeact': d,
# 'plan': {'id': plan['_id'], 'date': d},
# 'plans_history': [],
# 'contacts': {'email': '', 'cell': ''},
# 'points': 100,
#})
##create index on urls
#db = cl['webscr_' + str(db_ind)]
#db.urls.ensure_index('date_scraped', 1)
#db.task_meta.ensure_index('date_done', 1)
#cl.close()
User.create_user(alias=self.request.POST.get('alias'), plan='basic')
return super(AdminConsoleView, self).form_valid(form)
def execute_scheduled_jobs(self, ): from webscraper.entities import User users = User.get_users_with_schedules() for usr in users: usr.execute_scheduled_jobs()
def check_email(self, params, *args, **kwargs): email = params['email'] res = User.is_email_occupied(email) msg = self.tpl msg['payload']['email_occupied'] = res msg['status'] = 'ok' return HttpResponse( bsondumps(msg), content_type='application/json' )
def get_user(self, profile_id='', secret_key=''): if profile_id == '' and secret_key == '': if not self.u: session_usr = self.request.session['user'] self.u = User(session_usr) else: pass else: self.u = User.get_user(profile_id=profile_id, secret_key=secret_key) return self.u
def maintain(self):
from webscraper.entities import User
users = User.get_active_users()
#mongodb can have this limit on IN case
MAX_IN_LIMIT = 4000000
#count total deletes
tot = 0
#get celery_db connection
cl = CustomAppSettings.get_mongo()
celery_db = cl.celery_db
# remove old search results from each db
for user in users:
#get old task ids
old_task_ids = user.get_old_task_ids()[:MAX_IN_LIMIT]
count = len(old_task_ids)
tot += count
#db
db = user.get_user_db()
#remove old userdb.task_meta
db.task_meta.remove({'_id': {'$in': old_task_ids}})
#remove old userdb.urls
db.urls.remove({'task_id':{'$in': old_task_ids}})
#remove old celerydb.task_meta
celery_db.task_meta.remove({'_id':{'$in': old_task_ids}})
#write user profile statistics on removed
db = user.get_user_profiles_db()
col = db.cleaning_stats
d = datetime.datetime.utcnow()
doc = {'user': user.get_user_id(), 'date_done': d, 'total_removed': count}
col.insert(doc)
return {'total': tot}
def get_user_plan(self, params, *args, **kwargs): uid = params['userid'] msg = self.tpl u = User.get_user(profile_id=uid) if 'error'in u: msg['status'] = 'error' msg['errormessage'] = u['error'] return HttpResponse( bsondumps(msg), content_type='application/json' ) mongo_user = u plan = User(u).get_user_plan() msg['status'] = 'ok' msg['payload']['user'] = mongo_user msg['payload']['plan'] = plan return HttpResponse( bsondumps(msg), content_type='application/json' )
def get_user_and_plan(self, access_data, *args, **kwargs):
access_token = access_data.get('access_token', '')
msg = self.tpl
if not access_token:
msg['status'] = 'error'
msg['errormessage'] = 'Empty access token'
return HttpResponse( bsondumps(msg), content_type='application/json' )
url = 'https://graph.facebook.com/me?access_token={access_token}&fields=id,email,gender,link,locale,name,timezone,updated_time,verified'
url = url.format(access_token=access_token)
try:
response = urllib2.urlopen(url)
except urllib2.HTTPError:
fb_user={'error': {'message': 'Wrong api token'}}
except urllib2.URLError as e:
fb_user = {'error':{'message': 'Could not connect to Facebook'}}
else:
# All is fine
res = response.read()
fb_user = json.loads(res)
if 'error' in fb_user:
msg['status'] = 'error'
msg['errormessage'] = fb_user['error']['message']
return HttpResponse( bsondumps(msg), content_type='application/json' )
u = User.get_user(facebook_user=fb_user)
if 'error' in u:
msg['status'] = 'error'
msg['errormessage'] = u['error']
return HttpResponse( bsondumps(msg), content_type='application/json' )
mongo_user = u
plan = User(mongo_user).get_user_plan()
msg['status'] = 'ok'
msg['payload']['user'] = mongo_user
msg['payload']['plan'] = plan
return HttpResponse( bsondumps(msg), content_type='application/json' )
def get_user_plan(self, params, *args, **kwargs):
secret_key = params.get('secret_key','')
u = User.get_user(secret_key = secret_key)
msg = self.tpl
time.sleep(0.1)
if 'error' in u:
msg['status'] = 'error'
msg['errormessage'] = u['error']
return HttpResponse( bsondumps(msg), content_type='application/json' )
mongo_user = u
plan = User(u).get_user_plan()
msg['status'] = 'ok'
msg['payload']['user'] = mongo_user
msg['payload']['plan'] = plan
return HttpResponse( bsondumps(msg), content_type='application/json' )
def update_user(self, params, *args, **kwargs): uid = params['profile_id'] msg = self.tpl u = User.get_user(profile_id=uid) if 'error' in u: msg['status'] = 'error' msg['errormessage'] = u['error'] return HttpResponse( bsondumps(msg), content_type='application/json' ) res = User(u).delete() if res: msg['status'] = 'ok' msg['payload']['message'] = "user deleted successfully" else: msg['status'] = 'error' msg['errormessage'] = 'unable to delete the user' return HttpResponse( bsondumps(msg), content_type='application/json' )
def update_user(self, params, *args, **kwargs): uid = params['userid'] data = kwargs['data'] msg = self.tpl u = User.get_user(profile_id=uid) if 'error' in u: msg['status'] = 'error' msg['errormessage'] = u['error'] return HttpResponse( bsondumps(msg), content_type='application/json' ) res = User(u).update_info(data=data) if 'error' not in res: msg['status'] = 'ok' msg['payload']['user'] = u else: msg['status'] = 'error' msg['errormessage'] = res['error'] return HttpResponse( bsondumps(msg), content_type='application/json' )
def query(self, tld): u = self.get_user( CustomAppSettings.get_vissbl_user_id() ) stats = User(u).get_user_stats_object() res = stats.get_ranks_for_tld(tld) return res
