def _get_resource(resource_url: str) -> (str, bytes): """Download or reads a file (online or local). Parameters: resource_url (str): URL or path of resource to load Returns: str, bytes: Tuple containing the resource's MIME type and its data. Raises: NameError: If an HTTP request was made and ``requests`` is not available. ValueError: If ``resource_url``'s protocol is invalid. """ url_parsed = urlparse(resource_url) if url_parsed.scheme in ['http', 'https']: # Requests might not be installed if requests_get is not None: request = requests_get(resource_url) data = request.content if 'Content-Type' in request.headers: mimetype = request.headers['Content-Type'] else: mimetype = mimetypes.guess_type(resource_url) else: raise NameError("HTTP URL found but requests not available") elif url_parsed.scheme == '': # '' is local file with open(resource_url, 'rb') as f: data = f.read() mimetype, _ = mimetypes.guess_type(resource_url) elif url_parsed.scheme == 'data': raise ValueError("Resource path is a data URI", url_parsed.scheme) else: raise ValueError("Not local path or HTTP/HTTPS URL", url_parsed.scheme) return mimetype, data
def mef_person_update_index(sender, *args, **kwargs): """Index MEF person in ES.""" record = kwargs['record'] if 'documents' in record.get('$schema', ''): authors = record.get('authors', []) for author in authors: mef_url = author.get('$ref') if mef_url: mef_url = mef_url.replace( 'mef.rero.ch', current_app.config['RERO_ILS_MEF_HOST']) request = requests_get(url=mef_url, params=dict(resolve=1, sources=1)) if request.status_code == requests_codes.ok: data = request.json() id = data['id'] data = data.get('metadata') if data: data['id'] = id data['$schema'] = current_jsonschemas.path_to_url( current_app.config['RERO_ILS_PERSONS_MEF_SCHEMA']) indexer = RecordIndexer() index, doc_type = indexer.record_to_index(data) indexer.client.index( id=id, index=index, doc_type=doc_type, body=data, ) current_search.flush_and_refresh(index) else: current_app.logger.error( 'Mef resolver request error: {stat} {url}'.format( stat=request.status_code, url=mef_url)) raise Exception('unable to resolve')
def mef_person_delete(sender, *args, **kwargs): """Delete signal.""" record = kwargs['record'] if 'documents' in record.get('$schema', ''): authors = record.get('authors', []) for author in authors: mef_url = author.get('$ref') if mef_url: mef_url = mef_url.replace( 'mef.rero.ch', current_app.config['RERO_ILS_MEF_HOST']) request = requests_get(url=mef_url, params=dict(resolve=1, sources=1)) if request.status_code == requests_codes.ok: data = request.json() id = data['id'] data = data.get('metadata') if data: search = DocumentsSearch() count = search.filter( 'match', authors__pid=id).execute().hits.total if count == 1: indexer = RecordIndexer() index, doc_type = indexer.record_to_index(data) indexer.client.delete(id=id, index=index, doc_type=doc_type) current_search.flush_and_refresh(index) else: current_app.logger.error( 'Mef resolver request error: {result} {url}'.format( result=request.status_code, url=mef_url)) raise Exception('unable to resolve')
def generate_response(self, gfy_urls): carebox = [] textbody = '' for url in gfy_urls: fix_url = self.FIX_URL.format(url) result = requests_get(self.API_URL.format(url)) if result.ok: gfy_item = result.json()['gfyItem'] size = '{:.1f}'.format(gfy_item['gifSize'] / gfy_item['webmSize']) title = [fix_url, gfy_item['title']][gfy_item['title'] is True] reddit = gfy_item['redditId'] carebox.append({ 'fix_url': fix_url, 'size': size, 'title': title, 'reddit': reddit }) for gfycat in carebox: textbody += self.responses.gfycat_binding.format(**gfycat) if gfycat['reddit']: origin = self.session.get_submission( submission_id=gfycat['reddit']) caredict = { 'upvote': origin.upvote_ratio * 100, 'title': origin.title, 'url': 'https://np.reddit.com/{}/'.format(gfycat['reddit']) } textbody += self.responses.original_submission.format( **caredict) textbody = self.responses.intro + textbody + self.responses.outro return textbody.replace('\\n', '\n')
def parse_transcript_list(transcript_list_file): """ Takes a file containing a line separated list of ensembl transcripts, then returns a data structure containing the gene name, chromosome, and locus start and end """ if not isfile(transcript_list_file): print('Invalid File name') raise SystemExit transcript_list = [] with open(transcript_list_file, 'r') as f: r = f.readlines() for row in r: transcript_list.append(row.rstrip()) transcript_dict = {} for transcript in transcript_list: url = ('https://rest.ensembl.org/lookup/id/' + transcript + '?content-type=application/json') print(url) response = requests_get(url) data = response.json() if "error" in data: response = data["error"] raise ValueError(response) chromosome = data["seq_region_name"] locus_intervals = ("chr" + str(chromosome) + ":" + str(data["start"]) + "-" + str(data["end"])) if chromosome not in transcript_dict: transcript_dict[chromosome] = [] transcript_dict[chromosome].append((transcript, locus_intervals)) return transcript_dict
def login_github(): at_response = requests_post( 'https://github.com/login/oauth/access_token', data={ 'client_id': app.github_oauth_client_id, 'client_secret': app.github_oauth_client_secret, 'code': request.args['code'], 'accept': 'application/json' } ) assert at_response.status_code == 200 response_data = url_decode(at_response.text) access_token = response_data['access_token'] user_response = requests_get('https://api.github.com/user', params={'access_token': access_token}) assert user_response.status_code == 200 user_data = user_response.json() try: login = session.query(GithubLogin).filter_by( uid=user_data['login'] ).one() except NoResultFound: login = GithubLogin(user=User(), uid=user_data['login']) login.user.display_name = login.identifier() session.add(login) session.commit() login_user(login.user) return redirect(url_for('pages.index'))
def getHtml(self, word): self.word = word try: r = requests_get('http://www.iciba.com/' + self.word) return r.text except Exception as e: print(e)
def search_films(x): global j j = str for i in x: ght = requests_get(i) ght_1 = ght.json() k = str(ght_1["title"] + '(' + str(ght_1["episode_id"]) + ')') j = str(j) + k + '; '
def get_proxies(): global args if args.proxies: proxies=open(args.proxies,'r').read().strip().split('\n') else: proxies=requests_get('https://www.proxy-list.download/api/v1/get?type=https&anon=elite').content.decode().strip().split('\r\n') log('[INFO] %d proxies successfully loaded!'%len(proxies)) return proxies
def _get_current(self, _url_info, self_major, self_version): try: response = requests_get(_url_info.format(self_major), params={'v':self_version}) except ConnectionError, e: # We ignore ENETUNREACH because it simply means that we could not connect to the server, # which is fine, e.g. no Internet connectivity is allowed in that system. if e.errno != ENETUNREACH: raise
def GetStoredFiles(self): url = self._base_url + '/cgi/enumerate/' log('Request: GET ' + url) r = requests_get(url) self.CheckHttpError(r) files = r.json() files = sorted(files, key=lambda item: item['display_filename']) return files
def communityFilter(districtNameList, districtLinkList): """ // 按区县爬取小区名字和链接 """ districtCommunityDicList = [] """ // 分区县处理 """ for districtLink in districtLinkList: districtAllCommunity = {} """ // 每个区县所有小区列表是分页展示的 // 首先获取区县小区列表的第一页,获取页面信息 """ res = requests_get(districtLink) soup = BeautifulSoup(res.text, 'lxml') page_data = soup.select( 'div[class="page-box house-lst-page-box"]')[0]['page-data'] page_data = eval(page_data) totalPage = page_data["totalPage"] """ // 分页处理 """ for pageIndex in range(1, totalPage + 1): if (pageIndex > 1): pageLink = districtLink + 'pg' + str(pageIndex) + '/' res = requests_get(pageLink) soup = BeautifulSoup(res.text, 'lxml') communityResourceList = soup.select( 'li[class="clear xiaoquListItem"]') for communityResource in communityResourceList: communityLink = communityResource.a['href'] communityName = communityResource.img['alt'] districtAllCommunity[communityName] = communityLink districtCommunityDicList.append(districtAllCommunity) return districtCommunityDicList
def __api_request(service_label, params, index_name=None): """Wraps the access to the Nordlys API. It returns a 3-uple (results, total no. of results, pretty status message). :param service_label: a constant for the required service_label. :param params: request params. :param index_name: optional; name of index. :return: a list of docIDs. """ results = None # default init, it remains None if request returns error total = 0 msg = "" url = "/".join([PROTOCOL, SERVER_HOSTNAME_API, service_label]) if service_label == SERVICE_E_RETRIEVAL: url += "?q={}&model={}&start={}&1st_num_docs={}&fields_return={}".format( quote(params.get("q", "")), params.get("model", "lm"), params.get("start", 0), params.get("1st_num_docs", 100), params.get("fields_return", "abstract"), ) url += "&num_docs={}".format(params.get("num_docs", NUM_RESULTS)) elif service_label == SERVICE_E_LINKING: url += "?q={}".format(quote(params.get("q", ""))) elif service_label == SERVICE_TTI: url += "?q={}&method={}&num_docs={}&start={}&index={}&field={}".format( quote(params.get("q", "")), params.get("method", "tc"), params.get("num_docs", NUM_RESULTS), params.get("start", 0), params.get("index", TTI_INDEX_FALLBACK_2015_10), params.get("field", "_id")) try: print("Service request' URL: {}".format(url)) r = requests_get(url, timeout=REQUEST_TIMEOUT) print(r) results = j_loads(r.text) total = results.get("total_hits", 0) # Obtain postprocessed results to render, if needed entity_collection = MONGO_ENTITY_COLLECTIONS[0] if len( MONGO_ENTITY_COLLECTIONS) > 0 else "dbpedia-2015-10" results = process_results(results, service_label, protocol=PROTOCOL, server_hostname_api=SERVER_HOSTNAME_API, entity_collection=entity_collection, request_timeout=REQUEST_TIMEOUT) except ConnectionError: msg = "We're so sorry. There was a connection error :(" except Timeout: msg = "Timeout while trying to connect to the remote server, or while receiving data from it :(" except JSONDecodeError: msg = "There are no results for your query :(" return results, total, msg
def _api_call(self, call, **kwargs): url = self.__url + str(call) + '.json?' + '&'.join([str(key) + '=' + str(value) for key, value in kwargs.items() if value != ''])\ .replace('\'', '').replace('[', '').replace(']', '') response = requests_get(url, headers=self.__header) if response.status_code == 200: self.__connection = True return json_loads(response.text) else: return {response.status_code: response.text}
def _get_extract(self, page_id, page_title): """Get an extract from a wikipedia page of the given location.""" PARAMS_EXTRACT["titles"] = f"{page_title}" extract = requests_get(f"{ENDPOINT_WIKIMEDIA}", params=PARAMS_EXTRACT).json() self._extract = extract["query"]["pages"][f"{page_id}"]["extract"]
def getEN_mp3(self): url = 'https://dict.youdao.com/dictvoice?audio={word}&type=2'.format( word=self.word) mp3 = requests_get(url) with open('en.mp3', 'wb') as f: f.write(mp3.content) f.close() mp3.close()
def generate_response(self, massdrop_links): """Takes multiple links at once, iterates, generates a response appropriately. Idea is to take into account: Title, Price, Running Drop, Time left""" drop_field = [] textbody = "" fixed_urls = 0 time_ends_in = None will_update = False for url in massdrop_links: fix_url = "https://massdrop.com/buy/{}?mode=guest_open".format(url) api_url = self.API_URL.format(url) try: response = requests_get(api_url).json() if "error" in response: continue # API doesn't have response codes, responds with 200 OK always product_name = response["name"] # statusCode: 1 - Drop is active and running if response["statusCode"] == 1: current_price = "${:.2f}".format(response["currentPrice"]) prices = [x["price"] for x in response["steps"]] prices = self.massdrop_pricer(current_price, prices) time_ends_at = datetime.strptime(response["endAt"], "%Y-%m-%dT%H:%M:%S.%fZ") time_ends_in = time_ends_at - datetime.utcnow() time_left = " / " + self.time_formatter(time_ends_in) will_update = True else: time_left = "drop has ended" current_price, prices = "", "" drop_field.append( { "title": product_name, "current_price": current_price, "prices": prices, "time_left": time_left, "fix_url": fix_url, } ) fixed_urls += 1 except Exception as e: self.logger.error("Oh noes, an unexpected error happened: {} < {}".format(e.__class__.__name__, e)) if len(drop_field) == 0: return None, None # item is a dictionary that fits on the right binding - saves time, is short for item in drop_field: textbody += self.responses.product_binding.format(**item) update_string = ["", " ^| ^This ^comment ^updates ^every ^12 ^hours."][will_update] textbody = ( self.responses.intro_drop.format(products=("product", "products")[fixed_urls > 1]) + textbody + self.responses.outro_drop.format(update=update_string) ) return textbody.replace("\\n", "\n"), time_ends_in
def check_by_redirect(self, url: str, results: list): r = requests_get(url, headers=self.headers, verify=True, allow_redirects=False) if r.status_code == 302: return print(r.url) results.append(r.url)
def list_form_submissions(headers, form_id): '''List all submissions for a form.''' query_url = '{}/forms/{}/submissions'.format(api_url, form_id) response = requests_get(query_url, headers=headers) if response.status_code == 200: return response.json() else: raise Exception('API error (HTTP code {})'.format( response.status_code))
def download_img(icon_url: str): try: r = requests_get(icon_url, stream=True) if r.status_code == 200: with open(f"{DATA_PATH}icon.png", 'wb') as file: r.raw.decode_content = True copyfileobj(r.raw, file) except (ConnectionError): return
def run_begin_end_time(exp:str, runnum:int) : # returns a list of dicts per run with 'begin_time', 'end_time', 'run_num', 'run_type' if runnum>0 : resp = requests_get('https://pswww.slac.stanford.edu/prevlgbk/lgbk/%s/ws/runs' % exp).json() for d in resp : if d['run_num'] == runnum : return str(d['begin_time']), str(d['end_time']) logger.debug('begin and end time info is not found in mysql for run=%d. Uses default times.' %runnum) return '1000000000', '5000000000'
def get_html(url: str) -> str: """Return the html string for the given url.""" with requests_get(url, stream=True, headers=USER_AGENT_HEADER, timeout=15) as r: check_response_headers(r) content = next(r.iter_content(MAX_RESPONSE_LENGTH)) charset_match = CHARSET(content) return content.decode( charset_match[1].decode() if charset_match else r.encoding)
def main(): def search_planet(x): global a thy = requests_get(x) thy_1 = thy.json() a = thy_1["name"] def massiv(x): global m m = [] for i in x: m.append(i) def search_films(x): global j j = str for i in x: ght = requests_get(i) ght_1 = ght.json() k = str(ght_1["title"] + '(' + str(ght_1["episode_id"]) + ')') j = str(j) + k + '; ' name = requests_get('https://www.swapi.co/api/people') planet = requests_get('https://www.swapi.co/api/planets') data = name.json() data_1 = planet.json() z = 0 ch = [] for i in data['results']: pl = data['results'][z]['homeworld'] films = data['results'][z]['films'] massiv(films) search_films(m) search_planet(pl) b = 'Name: ' + (data['results'][z]['name'] ) + '; Planet: ' + a + '; Film: ' + str(j) ch.append(b) z += 1 for i in ch: print(i, '\n')
def check_updates(self): r = requests_get( 'https://api.github.com/repos/tampe125/dump-scraper/releases/latest' ) json_data = json.loads(r.content) if StrictVersion(json_data['tag_name']) > StrictVersion(self.version): logging.getLogger('dumpscraper').warn( "A new version is available, please download it from https://github.com/tampe125/dump-scraper/releases" )
def _get_http_get_request_from_url(self, url): try: r = requests_get(url, verify=self._ssl_verify) except RequestsConnectionError: raise HPNAConnectionError( "Failed to connect to HP Network Automation URL.") if r.status_code != 200: raise HPNAConnectionError( "Failed to connect to HP Network Automation URL.") return r.text
def __getToken(self): # 获取token认证 url_token = 'https://openapi.baidu.com/oauth/2.0/token' api_key = 'R26ZZakxixaQbIDGrPkUwOTc' secret_key = '10d76d90116385e126d95e1c277c538c' get_token_url = url_token+'?'+'grant_type=client_credentials&client_id=' + \ api_key+'&client_secret='+secret_key token = requests_get(get_token_url) r = jsonLoads(token.text) return r['access_token']
def get_records(self, from_date, max=0, file=None): """Get cantook records. from_date: record changed after this date to get max: maxium records to fetcher file: to save the fetched record """ self._count = 0 self._max = max url = self.get_request_url(start_date=from_date, page=1) request = requests_get(url) total_pages = int(request.headers.get('X-Total-Pages', 0)) total_items = int(request.headers.get('X-Total-Items', 0)) # per_pages = int(request.headers.get('X-Per-Page', 0)) current_page = int(request.headers.get('X-Current-Page', 0)) while (request.status_code == requests_codes.ok and current_page <= total_pages and (self._count < self._max or self._max == 0)): self.verbose_print( 'API page: {page} url: {url}'.format( page=current_page, url=url ) ) self.process_records(request.json().get('resources', [])) # get next page and update current_page url = self.get_request_url( start_date=from_date, page=current_page+1 ) request = requests_get(url) current_page = int(request.headers.get('X-Current-Page', 0)) if self.file: file.write(']') if ( (max != 0 and self._count != max) or (max == 0 and total_items != self._count) ): # we had an ERROR raise('ERROR not all records harvested') return total_items, self._count
def get_last_sent_id(): row = requests_get('{}/vindb/vin_records/last/'.format( settings.vindb_host), params={ 'data_source': settings.vindb_data_source, }, headers={ 'Content-Type': 'application/json' }).json() return row.get('uuid', 0)
def getEN_mp3(self): # sound = bs.find_all(class_='new-speak-step') # sound_url = sound[1].get('ms-on-mouseover').split("'") # mp3 = requests.get(sound_url[1]) url = 'https://dict.youdao.com/dictvoice?audio={word}&type=2'.format( word=self.word) mp3 = requests_get(url) with open('en.mp3', 'wb') as f: f.write(mp3.content) f.close()
def listSubnavi(self, path, hasitems, items_to_add=None): if hasitems == 'false': url = urllib_urljoin(self.base_url, path) html = requests_get(url).text soup = BeautifulSoup(html, 'html.parser') for item in soup('a', 'sdc-site-directory__content'): if items_to_add and item.get('href') not in items_to_add: continue label = item.span.string url = self.plugin.build_url({ 'action': 'showVideos', 'path': '{0}-videos'.format(item.get('href')), 'show_videos': 'false' }) self.addDir(label, url) else: items = None for nav_item in self.nav_json: if nav_item.get('path') == path: items = nav_item.get('children') if items: for item in items: action = item.get('action') if item.get( 'action', None) else 'showVideos' if action == 'listSubnavi': url = self.plugin.build_url({ 'action': action, 'path': item.get('path'), 'hasitems': 'true' if item.get('children', None) else 'false', 'items_to_add': item.get('includes') }) else: url = self.plugin.build_url({ 'action': action, 'path': item.get('path'), 'show_videos': 'true' if item.get('show_videos', None) is None or item.get('show_videos') == 'true' else 'false' }) self.addDir(item.get('label'), url) xbmcplugin.endOfDirectory(self.plugin.addon_handle, cacheToDisc=True)
def embed_line(self, i: int, line: List[str]) -> Optional[int]: try: r = requests_get(line[1], stream=True, timeout=self.timeout) image = Image.open(BytesIO(r.content)) image = self.transforms(image).unsqueeze(0) # Fake batch-size of 1 image = image.to(self.device) self.model(image) del image return i except Exception: return None
def __get_avg_rating_for_product_id(self, model_number): try: res = requests_get(url=self.AVG_RATING_FORMAT.format(reviews_addr=self.reviews_addr, model_number=model_number)) if res.status_code != 200: current_app.logger.warning("Could not fetch rating for product id: %s", model_number) return -1 data = json.loads(res.content) return data["rating"] except Exception as e: current_app.logger.exception("Error fetching rating for product id: %s", model_number) return -1
def __fetch_listings_for_product(self, product_id): try: res = requests_get(url=self.FETCH_LISTINGS_FORMAT.format(inventory_addr=self.inventory_addr, product_id=product_id)) if res.status_code != 200: current_app.logger.warning("Could not fetch rating for product id: %d", product_id) return -1 data = json.loads(res.content) return data except Exception as e: current_app.logger.exception("Error fetching listings for product id: %d", product_id) return []
def fetch_remote_hash(branch): if not branch: logging.error('No branch specified. Exiting.') return resp = requests_get( 'https://api.github.com/repos/screenly/screenly-ose/git/refs/heads/{}'.format(branch) ) if not resp.ok: logging.error('Invalid response from github: {}'.format(resp.content)) return False logging.debug('Got response from Github: {}'.format(resp.status_code)) latest_sha = resp.json()['object']['sha'] return latest_sha
def _request(uri, path): try: res = requests_get(uri, headers=__header, timeout=60) out.debug("res.headers", obj=dict(res.headers)) res.raise_for_status() except Exception as e: out.warn(f"HTTP error: {e}\n{color.red(format_exc())}") else: out.debug(f"HTTP {res.status_code} \"{res.reason}\" -- \"{uri}\"") if res.status_code == 204 or res.headers.get("Content-Length") == 0: out.error(f"HTTP {res.status_code} \"{res.reason}\"" f" -- \"{uri}\": no content") rw.writef(path, res.content) out.info(f"updated \"{path}\"" f", last modified on \"{res.headers.get('Last-Modified')}\"")
def update(self): """update(self) - Fill Queue with new Pastebin IDs""" new_pastes = [] raw = None while not raw: try: raw = requests_get('http://pastebin.com/archive').content if "Pastebin.com has blocked your IP" in raw: getLogger('dumpscraper').critical("Pastebin blocked your IP. Wait a couple of hours and try again") raise RunningError() except ConnectionError: getLogger('dumpscraper').warn('Connection error, trying again in 5 seconds') raw = None sleep(5) results = BeautifulSoup(raw, "lxml").findAll( lambda tag: tag.name == 'td' and tag.a and '/archive/' not in tag.a['href'] and tag.a['href'][1:]) for entry in results: paste = PastebinPaste(entry.a['href'][1:]) # Check to see if we found our last checked URL if paste.id == self.ref_id: break new_pastes.append(paste) # Don't cry if we don't have any results try: # Let's save the starting id, so I can skip already processed pastes self.ref_id = results[0].a['href'][1:] except IndexError: dump_logger = getLogger('dumpscraper') dump_logger.info("\tArchive links not found") dump_logger.debug('\t\tFetched page:') for row in results: dump_logger.debug('\t\t\t' + row) for entry in new_pastes[::-1]: self.put(entry)
def check_in(passcode): if passcode == "": redirect('/') user_ip = request.remote_addr #finds their ip print(user_ip) url = ("http://ip-api.com/json/{}?fields=country,city".format(user_ip)) #get info about country and city from their ip, in JSON format #more info is available from that site if needed r = requests_get(url) try: #try block in case... idk. something? location_data = json.loads(r.text) visitor_city = location_data['city'] visitor_country = location_data['country'] print("Someone is visiting from " + visitor_city,visitor_country) print("They're using the passcode " + passcode) conn = sqlite3.connect('people.db') #connect to the database c = conn.cursor() #finds the location for whoever uses this passcode stored_city = c.execute("SELECT location_city FROM people WHERE passcode = ?;", (passcode,)) print(stored_city.fetchall()) stored_country = c.execute("SELECT location_country FROM people WHERE passcode = ?;", (passcode,)) print(stored_country.fetchall()) #if the stored location is different to that of the current location #then we update it! #we check city first, because they might be in the same country, but have just #changed cities if visitor_city != stored_city: c.execute("UPDATE people SET location_city=? WHERE passcode = ?;",(visitor_city,passcode)) if (visitor_country != stored_city): c.execute("UPDATE people SET location_country=? WHERE passcode = ?;",(visitor_country,passcode)) conn.commit() c.close() #redirect("http://www.xkcd.com") except: pass return template('checkin')
def generate_response(self, gfy_urls): carebox = [] textbody = '' for url in gfy_urls: fix_url = self.FIX_URL.format(url) result = requests_get(self.API_URL.format(url)) if result.ok: gfy_item = result.json()['gfyItem'] size = '{:.1f}'.format(gfy_item['gifSize'] / gfy_item['webmSize']) title = [fix_url, gfy_item['title']][gfy_item['title'] is True] reddit = gfy_item['redditId'] carebox.append({'fix_url': fix_url, 'size': size, 'title': title, 'reddit': reddit}) for gfycat in carebox: textbody += self.responses.gfycat_binding.format(**gfycat) if gfycat['reddit']: origin = self.session.get_submission(submission_id=gfycat['reddit']) caredict = {'upvote': origin.upvote_ratio * 100, 'title': origin.title, 'url': 'https://np.reddit.com/{}/'.format(gfycat['reddit'])} textbody += self.responses.original_submission.format(**caredict) textbody = self.responses.intro + textbody + self.responses.outro return textbody.replace('\\n', '\n')
def remote_branch_available(branch): if not branch: logging.error('No branch specified. Exiting.') return try: resp = requests_get( 'https://api.github.com/repos/screenly/screenly-ose/branches', headers={ 'Accept': 'application/vnd.github.loki-preview+json', }, ) except exceptions.ConnectionError: logging.error('No internet connection.') return if not resp.ok: logging.error('Invalid response from Github: {}'.format(resp.content)) return for github_branch in resp.json(): if github_branch['name'] == branch: return True return False
def download_job_sid(self, sid, mode='json', filename=None): """ Wrapper for streaming results to a file instead of through sockets with the API. :param sid: sid of job :param mode: json, csv, or xml :return: local filename, False if failure """ # Only tested on 6.3, may need to mod this job_sid_url = 'https://{0}/en-US/api/search/jobs/{1}/results?isDownload=true&' \ 'timeFormat=%25FT%25T.%25Q%25%3Az&maxLines=0&count=0&filename=&outputMode={2}' \ '&spl_ctrl-limit=unlimited&spl_ctrl-count=50000'.format(self.host, sid, mode) if not filename: filename= '{0}.{1}'.format(sid, mode) cookie_builder = {} for l in self._auth_headers: for x in l[1].split('; '): q = x.split('=') cookie_builder[q[0]] = q[1] r = requests_get(job_sid_url, stream=True, cookies=cookie_builder, verify=False) cnt = 0 with open(filename, 'wb') as f: # I have the bandwidth to do this size, you may not. for chunk in r.iter_content(chunk_size=1024*1024*1024): if chunk: f.write(chunk) cnt += 1 if cnt % 1000 == 0: # Call control occasionally to keep the export stream alive requests_post(r'https://{0}/en-US/splunkd/__raw/services/search/jobs/{1}/control' .format(self.host, sid), data={ 'output_mode': mode, 'action': 'touch' }, cookies=cookie_builder, verify=False) return filename
def DownloadFile(self, url_path): url = self._base_url + url_path log('Request: GET ' + url) r = requests_get(url) self.CheckHttpError(r) return r.content
def check_updates(self): r = requests_get('https://api.github.com/repos/tampe125/dump-scraper/releases/latest') json_data = json.loads(r.content) if StrictVersion(json_data['tag_name']) > StrictVersion(self.version): logging.getLogger('dumpscraper').warn("A new version is available, please download it from https://github.com/tampe125/dump-scraper/releases")
def _get_current(self, _url_info, self_major, self_version): response = requests_get(_url_info.format(self_major), params={'v':self_version}) if response.status_code == OK: return bunchify(loads(response.text))