def get_queryset(self): #get current user user = User(self.request.session['user']) # get max allowed jobs max_allowed = user.get_max_jobs() # get jobs jobs_cursor = user.get_scheduled_jobs(find_all=True) active_jobs = [] inactive_jobs = [] ar = None # reference to array to add jobs to for job in jobs_cursor: #get total urls job['total_urls'] = len(job['urls']) #take just last 3 urls job['first_urls'] = job['urls'][:3] #create id job['id'] = str(job['_id']) #split on active/inactive if job['is_active']: ar = active_jobs else: ar = inactive_jobs ar.append(job) return (max_allowed, active_jobs, inactive_jobs)
def create_user(self, data, ): if 'email' in data: alias = data.get('name','') email = data.get('email','') if 'plan_name' in data: plan_name = data['plan_name'] u = User.create_user(alias=alias, email=email) else: u = {'error': 'Email has not been supplied'} msg = self.tpl if 'error' in u: msg['status'] = 'error' msg['errormessage'] = u['error'] else: u = User.get_user(profile_id=u['profile_id']) msg['status'] = 'ok' msg['payload'] = {'name': u['alias']} self._send_confirmation_email(secret_key=u.get('secret_key',''),alias=alias,email=email) return HttpResponse( bsondumps(msg), content_type='application/json' )
def get_queryset(self): #get date from which search d = datetime.datetime.utcnow() d = d - datetime.timedelta(days=1) #get current user user = User(self.request.session['user']) #get task ids from db ids_cur = user.get_task_ids(d) #get total results self.total_results = ids_cur.count() itms = [] null = None #to make sure that eval('null') do not fire error for id in ids_cur.limit(self.n): tsk = celery_app.backend.get_task_meta(id['_id']) res = tsk['result'] if tsk['status'].lower() == u'failure': res = u'{0} error: {1}'.format(res['exc_type'], res['exc_message']) tsk['err'] = res else: try: res = res['url'] if res else u'' tsk['url'] = res except KeyError: tsk['err'] = json.dumps(res) tsk["status_class"] = u"status_" + tsk["status"].lower().replace(u' ', u'_') itms.append(tsk) return itms
def query(self): u = self.get_user( CustomAppSettings.get_vissbl_user_id() ) # Get stats stats = User(u).get_user_stats_object() data = stats.get_top_results() # Get reference to the last date of ranks last_date = data[-1] if len(data) > 0 else {'value':{'ranks':[]}} return {"data": data, "last_date_ranks": last_date['value']['ranks'][:30]}
def _create_new_user(self, form): #cl = CustomAppSettings.get_mongo() #db = cl.webscr_profiles ##get plan #plan = db.plans.find_one({'name': 'basic'}) ##get db index #db_ind = db.counters.find_and_modify( # query={'_id': 'dbindex'}, # update={'$inc': {'seq': 1}}, # upsert=True, # new=True #)['seq'] #db_ind = int(db_ind) ##get alias #alias = self.request.POST.get('alias') ##get new secret key #secret_key = CustomAppSettings.get_new_secret_key() ## get new api key #api_key = CustomAppSettings.get_new_api_key() ##try to write #d = datetime.datetime.utcnow() #db.users.insert({ # 'alias': alias, # 'secret_key': secret_key, # 'api_key': api_key, # 'db_ind': db_ind, # 'role': 'user', # 'is_active': 0, # 'created': d, # 'actdeact': d, # 'plan': {'id': plan['_id'], 'date': d}, # 'plans_history': [], # 'contacts': {'email': '', 'cell': ''}, # 'points': 100, #}) ##create index on urls #db = cl['webscr_' + str(db_ind)] #db.urls.ensure_index('date_scraped', 1) #db.task_meta.ensure_index('date_done', 1) #cl.close() User.create_user(alias=self.request.POST.get('alias'), plan='basic') return super(AdminConsoleView, self).form_valid(form)
def execute_scheduled_jobs(self, ): from webscraper.entities import User users = User.get_users_with_schedules() for usr in users: usr.execute_scheduled_jobs()
def check_email(self, params, *args, **kwargs): email = params['email'] res = User.is_email_occupied(email) msg = self.tpl msg['payload']['email_occupied'] = res msg['status'] = 'ok' return HttpResponse( bsondumps(msg), content_type='application/json' )
def get_user(self, profile_id='', secret_key=''): if profile_id == '' and secret_key == '': if not self.u: session_usr = self.request.session['user'] self.u = User(session_usr) else: pass else: self.u = User.get_user(profile_id=profile_id, secret_key=secret_key) return self.u
def maintain(self): from webscraper.entities import User users = User.get_active_users() #mongodb can have this limit on IN case MAX_IN_LIMIT = 4000000 #count total deletes tot = 0 #get celery_db connection cl = CustomAppSettings.get_mongo() celery_db = cl.celery_db # remove old search results from each db for user in users: #get old task ids old_task_ids = user.get_old_task_ids()[:MAX_IN_LIMIT] count = len(old_task_ids) tot += count #db db = user.get_user_db() #remove old userdb.task_meta db.task_meta.remove({'_id': {'$in': old_task_ids}}) #remove old userdb.urls db.urls.remove({'task_id':{'$in': old_task_ids}}) #remove old celerydb.task_meta celery_db.task_meta.remove({'_id':{'$in': old_task_ids}}) #write user profile statistics on removed db = user.get_user_profiles_db() col = db.cleaning_stats d = datetime.datetime.utcnow() doc = {'user': user.get_user_id(), 'date_done': d, 'total_removed': count} col.insert(doc) return {'total': tot}
def get_user_plan(self, params, *args, **kwargs): uid = params['userid'] msg = self.tpl u = User.get_user(profile_id=uid) if 'error'in u: msg['status'] = 'error' msg['errormessage'] = u['error'] return HttpResponse( bsondumps(msg), content_type='application/json' ) mongo_user = u plan = User(u).get_user_plan() msg['status'] = 'ok' msg['payload']['user'] = mongo_user msg['payload']['plan'] = plan return HttpResponse( bsondumps(msg), content_type='application/json' )
def get_user_and_plan(self, access_data, *args, **kwargs): access_token = access_data.get('access_token', '') msg = self.tpl if not access_token: msg['status'] = 'error' msg['errormessage'] = 'Empty access token' return HttpResponse( bsondumps(msg), content_type='application/json' ) url = 'https://graph.facebook.com/me?access_token={access_token}&fields=id,email,gender,link,locale,name,timezone,updated_time,verified' url = url.format(access_token=access_token) try: response = urllib2.urlopen(url) except urllib2.HTTPError: fb_user={'error': {'message': 'Wrong api token'}} except urllib2.URLError as e: fb_user = {'error':{'message': 'Could not connect to Facebook'}} else: # All is fine res = response.read() fb_user = json.loads(res) if 'error' in fb_user: msg['status'] = 'error' msg['errormessage'] = fb_user['error']['message'] return HttpResponse( bsondumps(msg), content_type='application/json' ) u = User.get_user(facebook_user=fb_user) if 'error' in u: msg['status'] = 'error' msg['errormessage'] = u['error'] return HttpResponse( bsondumps(msg), content_type='application/json' ) mongo_user = u plan = User(mongo_user).get_user_plan() msg['status'] = 'ok' msg['payload']['user'] = mongo_user msg['payload']['plan'] = plan return HttpResponse( bsondumps(msg), content_type='application/json' )
def get_user_plan(self, params, *args, **kwargs): secret_key = params.get('secret_key','') u = User.get_user(secret_key = secret_key) msg = self.tpl time.sleep(0.1) if 'error' in u: msg['status'] = 'error' msg['errormessage'] = u['error'] return HttpResponse( bsondumps(msg), content_type='application/json' ) mongo_user = u plan = User(u).get_user_plan() msg['status'] = 'ok' msg['payload']['user'] = mongo_user msg['payload']['plan'] = plan return HttpResponse( bsondumps(msg), content_type='application/json' )
def update_user(self, params, *args, **kwargs): uid = params['profile_id'] msg = self.tpl u = User.get_user(profile_id=uid) if 'error' in u: msg['status'] = 'error' msg['errormessage'] = u['error'] return HttpResponse( bsondumps(msg), content_type='application/json' ) res = User(u).delete() if res: msg['status'] = 'ok' msg['payload']['message'] = "user deleted successfully" else: msg['status'] = 'error' msg['errormessage'] = 'unable to delete the user' return HttpResponse( bsondumps(msg), content_type='application/json' )
def update_user(self, params, *args, **kwargs): uid = params['userid'] data = kwargs['data'] msg = self.tpl u = User.get_user(profile_id=uid) if 'error' in u: msg['status'] = 'error' msg['errormessage'] = u['error'] return HttpResponse( bsondumps(msg), content_type='application/json' ) res = User(u).update_info(data=data) if 'error' not in res: msg['status'] = 'ok' msg['payload']['user'] = u else: msg['status'] = 'error' msg['errormessage'] = res['error'] return HttpResponse( bsondumps(msg), content_type='application/json' )
def query(self, tld): u = self.get_user( CustomAppSettings.get_vissbl_user_id() ) stats = User(u).get_user_stats_object() res = stats.get_ranks_for_tld(tld) return res