def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) REQ.get('https://facebook.com/') form = REQ.form(action='/login/') if form: data = { 'email': conf.FACEBOOK_USERNAME, 'pass': conf.FACEBOOK_PASSWORD, } REQ.submit_form(data=data, form=form) form = REQ.form(action='/login/') if form and 'validate-password' in form['url']: REQ.submit_form(data=data, form=form)
def _get(self, *args, **kwargs): page = REQ.get(*args, **kwargs) form = REQ.form(limit=2, selectors=['class="form-horizontal"']) if form: form['post'].update({ 'username': self._username, 'password': self._password, }) page = REQ.get(form['url'], post=form['post']) return page
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) page = REQ.get('https://auth.geeksforgeeks.org/') form = REQ.form(page=page, action=None, fid='Login') if form: REQ.get('https://auth.geeksforgeeks.org/setLoginToken.php') page = REQ.submit_form( url='https://auth.geeksforgeeks.org/auth.php', data={ 'user': conf.GEEKSFORGEEKS_USERNAME, 'pass': conf.GEEKSFORGEEKS_PASSWORD, }, form=form, )
def get_users_infos(users, resource, accounts, pbar=None): page = REQ.get(urljoin(resource.profile_url, Statistic.SETTINGS_URL_)) form = REQ.form(action=r'login.php\?action=login') if form: data = { 'username': conf.BESTCODER_AUTHORID, 'password': conf.BESTCODER_PASSWORD, 'remember': 'on', } page = REQ.submit_form(data=data, form=form) match = re.search('<select[^>]*id="country"[^>]*>.*?</select>', page, re.DOTALL) countries = dict( re.findall('<option[^>]*value="([0-9]+)"[^>]*>([^<]*)</option>', match.group(0))) @RateLimiter(max_calls=5, period=1) def fetch_user(user): url = resource.profile_url.format(account=user) page = REQ.get(url) info = {} matches = re.findall( r'<span[^>]*>([A-Z]+)</span>\s*<span[^>]*>([0-9]+)</span>', page) for k, v in matches: info[k.lower()] = int(v) match = re.search( '<img[^>]*src="[^"]*country[^"]*([0-9]+)[^"]*"[^>]*alt="country"[^>]*>', page) if match: info['country'] = countries.get(match.group(1)) match = re.search( '<img[^>]*class="img-circle"[^>]*src="([^"]*getAvatar.php[^"]*)"[^>]*>', page) if match: info['avatar_url'] = urljoin(url, match.group(1)) page = REQ.get(Statistic.USER_RATING_API_URL_.format(user)) data = json.loads(page) ratings = {} old_rating = None for stat in data: rating = ratings.setdefault(stat['contestid'], collections.OrderedDict()) new_rating = int(stat['rating']) if old_rating is not None: rating['old_rating'] = old_rating rating['rating_change'] = new_rating - old_rating rating['new_rating'] = new_rating old_rating = new_rating info['rating'] = new_rating if not ratings: info.pop('rating', None) return user, info, ratings with PoolExecutor(max_workers=8) as executor: for user, info, ratings in executor.map(fetch_user, users): if pbar: pbar.update() if not info: if info is None: yield {'info': None} else: yield {'skip': True} continue info = { 'info': info, 'contest_addition_update_params': { 'update': ratings, 'by': 'key', }, } yield info