def fix_url(url, session=None, cw=None): print_ = get_print(cw) if '&manga_name=' not in url: return url print_('fix url') qs = query_url(url) name = qs['manga_name'][0].replace('+', ' ') url_search = urljoin(url, '/bbs/search.php') url_search = update_url_query(url_search, {'stx': [name]}) print(url_search) html = read_html(url_search, session=session) soup = Soup(html) posts = soup.findAll('div', class_='post-row') print_(('posts:').format(len(posts))) if len(posts) != 1: return url for a in posts[0].findAll('a'): href = urljoin(url, a.attrs['href']) if 'manga_detail' in href: break else: raise Exception('Failed to find link') if cw is not None: cw.gal_num = href return href
def _call(self, url_api, referer='https://twitter.com', params=None): url_api = urljoin('https://api.twitter.com', url_api) if params: url_api = update_url_query(url_api, params) #print('call:', url_api) data = downloader.read_json(url_api, referer, session=self.session) return data
def get_query(query_hash, variables, session, cw=None): _wait(cw) print_ = get_print(cw) csrf_token = session.cookies.get('csrftoken', domain='.instagram.com') if not csrf_token: raise Exception('no csrftoken') hdr = { "X-CSRFToken": csrf_token, #2849 "X-IG-App-ID": "936619743392459", "X-IG-WWW-Claim": "0", "X-Requested-With": "XMLHttpRequest", } url_ = update_url_query('https://www.instagram.com/graphql/query/', { 'query_hash': query_hash, 'variables': json.dumps(variables) }) #print(len(edges), url_) r = session.get(url_, headers=hdr) try: j = json.loads(r.text) except Exception as e: print(e) j = {} if not j or j.get('status') == 'fail': msg = 'Fail: {} {}'.format( j.get('message') or 'Please wait a few minutes before you try again.', variables) print_(msg) sleep(60 * 30, cw) raise Exception(msg) return j
def _call(self, url_api, referer='https://twitter.com', params=None): url_api = urljoin('https://api.twitter.com', url_api) if params: url_api = update_url_query(url_api, params) #print('call:', url_api) r = self.session.get(url_api, headers={'Referer': referer}) csrf = r.cookies.get('ct0') if csrf: self.session.headers['x-csrf-token'] = csrf data = json.loads(r.text) return data
def call(self, path, qs, default_qs=True): if default_qs: qs_new = qs qs = self._qs.copy() qs.update(qs_new) url = self._url_base + path url = update_url_query(url, qs) r = self.session.get(url, headers=self._hdr) data = r.json() errs = data.get('errors', []) if errs: code = int(errs[0]['code']) if code == 0: raise Exception('Not found') elif code == 4012: raise errors.LoginRequired(errs[0]['detail']) r.raise_for_status() return data['response']