def fill(self, first, last):
last += relativedelta(years=7)
dfs = []
with enabled():
while first <= last:
response = requests.get(
url=
'https://www.theice.com/marketdata/ExpiryCalendar.shtml',
params={
'excel':
'',
'markets': ("ICE Futures U.S.", "ICE Futures Europe",
"ICE Futures Canada", "ICE OTC",
"ICE Trust U.S.", "ICE Clear Europe CDS",
"ICE Endex", "ICE Futures Singapore"),
'expirationEnabled':
"true",
'expirationDates':
("FTD", "LTD", "FDD", "LDD", "FND", "LND", "FSD"),
'dateFrom':
first.strftime('%d-%b-%Y')
},
headers={
'User-Agent': 'Mozilla/5.0',
})
first = dt.datetime.strptime(
response.text.splitlines()[2][4:],
'%d-%b-%Y').date() + dt.timedelta(days=1)
dfs.append(self.parse_csv(response.text))
return concat(dfs)
def process(input, entities):
output = {}
try:
video = entities['video'][0]['value']
with requests_cache.enabled('video_cache', backend='sqlite', expire_after=3600):
r = requests.get('https://www.googleapis.com/youtube/v3/search?part=snippet&maxResults=10&q=' + video + '&type=video&key=' + YOUTUBE_DATA_API_KEY)
data = r.json()
template = GenericTemplate()
for item in data['items']:
title = item['snippet']['title']
item_url = 'https://www.youtube.com/watch?v=' + item['id']['videoId']
image_url = item['snippet']['thumbnails']['high']['url']
subtitle = item['snippet']['channelTitle']
buttons = ButtonTemplate()
buttons.add_web_url('YouTube Link', 'https://www.youtube.com/watch?v=' + item['id']['videoId'])
buttons.add_web_url('Channel Link', 'https://www.youtube.com/channel/' + item['snippet']['channelId'])
template.add_element(title=title, item_url=item_url, image_url=image_url, subtitle=subtitle, buttons=buttons.get_buttons())
output['input'] = input
output['output'] = template.get_message()
output['success'] = True
except:
error_message = 'I couldn\'t find any videos matching your query.'
error_message += '\nPlease ask me something else, like:'
error_message += '\n - sia videos'
error_message += '\n - videos by eminem'
error_message += '\n - video coldplay'
output['error_msg'] = TextTemplate(error_message).get_message()
output['success'] = False
return output
def process(input, entities):
output = {}
try:
movie = entities['movie'][0]['value']
with requests_cache.enabled('movie_cache',
backend='sqlite',
expire_after=86400):
r = requests.get('http://www.omdbapi.com/?t=' + movie +
'&plot=full&r=json')
data = r.json()
output['input'] = input
template = TextTemplate('Title: ' + data['Title'] + '\nYear: ' +
data['Year'] + '\nIMDb Rating: ' +
data['imdbRating'] + ' / 10' + '\nPlot: ' +
data['Plot'])
text = template.get_text()
template = ButtonTemplate(text)
template.add_web_url(
'IMDb Link', 'http://www.imdb.com/title/' + data['imdbID'] + '/')
output['output'] = template.get_message()
output['success'] = True
except:
error_message = QUERY_ERROR.format('movie') + EXAMPLE_MOVIES
output['error_msg'] = TextTemplate(error_message).get_message()
output['success'] = False
return output
def process(input, entities):
output = {}
try:
video = entities['video'][0]['value']
with requests_cache.enabled('video_cache', backend='sqlite', expire_after=3600):
r = requests.get(
'https://www.googleapis.com/youtube/v3/search?part=snippet&maxResults=10&q=' + video + '&type=video&key=' + YOUTUBE_DATA_API_KEY)
data = r.json()
template = GenericTemplate()
for item in data['items']:
title = item['snippet']['title']
item_url = 'https://www.youtube.com/watch?v=' + item['id']['videoId']
image_url = item['snippet']['thumbnails']['high']['url']
subtitle = item['snippet']['channelTitle']
buttons = ButtonTemplate()
buttons.add_web_url('YouTube Link', 'https://www.youtube.com/watch?v=' + item['id']['videoId'])
buttons.add_web_url('Channel Link', 'https://www.youtube.com/channel/' + item['snippet']['channelId'])
template.add_element(title=title, item_url=item_url, image_url=image_url, subtitle=subtitle,
buttons=buttons.get_buttons())
output['input'] = input
output['output'] = template.get_message()
output['success'] = True
except:
error_message = 'I couldn\'t find any videos matching your query.'
error_message += '\nPlease ask me something else, like:'
error_message += '\n - sia videos'
error_message += '\n - videos by eminem'
error_message += '\n - video coldplay'
output['error_msg'] = TextTemplate(error_message).get_message()
output['success'] = False
return output
def get_detalhes_raw(papel='WEGE3'):
"""
Get RAW detailed data from fundamentus:
URL:
http://fundamentus.com.br/detalhes.php?papel=WEGE3
Output:
list of df
"""
##
## Busca avançada por empresa
##
url = 'http://fundamentus.com.br/detalhes.php?papel={}'.format(papel)
hdr = {
'User-agent':
'Mozilla/5.0 (Windows; U; Windows NT 6.1; rv:2.2) Gecko/20110201',
'Accept': 'text/html, text/plain, text/css, text/sgml, */*;q=0.01',
'Accept-Encoding': 'gzip, deflate',
}
with requests_cache.enabled():
content = requests.get(url, headers=hdr)
if content.from_cache:
logging.debug('.../detalhes.php?papel={}: [CACHED]'.format(papel))
else: # pragma: no cover
logging.debug(
'.../detalhes.php?papel={}: sleeping...'.format(papel))
time.sleep(.500) # 500 ms
## parse
tables_html = pd.read_html(content.text, decimal=",", thousands='.')
return tables_html
def list_papel_all():
"""
Get list of all companies, from 'detalhes' page
URL:
http://fundamentus.com.br/detalhes.php?papel=''
Output:
list
"""
##
url = 'http://fundamentus.com.br/detalhes.php?papel='
hdr = {
'User-agent':
'Mozilla/5.0 (Windows; U; Windows NT 6.1; rv:2.2) Gecko/20110201',
'Accept': 'text/html, text/plain, text/css, text/sgml, */*;q=0.01',
'Accept-Encoding': 'gzip, deflate',
}
with requests_cache.enabled():
content = requests.get(url, headers=hdr)
if content.from_cache:
logging.debug('list .../detalhes.php?papel= : [CACHED]')
else: # pragma: no cover
logging.debug('list .../detalhes.php?papel= : sleeping...')
time.sleep(.500) # 500 ms
## parse
df = pd.read_html(content.text, decimal=",", thousands='.')[0]
lst = list(df['Papel'])
logging.info('members in list = {}'.format(len(lst)))
return lst
def matches_preferred_time(venue):
if 'preferred_time' not in venue_list:
logging.debug('No preferred time set')
return True
preferred_time = venue_list['preferred_time']
with requests_cache.enabled(**long_cache):
venue_hours = foursq.venues.hours(venue['id'])
days_seen = []
days_accepted = []
try:
timeframes = venue_hours['popular']['timeframes']
except KeyError:
try:
timeframes = venue_hours['hours']['timeframes']
logging.debug('No popular hours available, using official hours')
except KeyError:
logging.debug('No hours information available')
return True
for timeframe in timeframes:
days_seen += timeframe['days']
matching = next((True for h in timeframe['open'] if int(h['start']) <= preferred_time <= int(h['end'])), False)
if matching:
days_accepted += timeframe['days']
if len(days_seen) != 7:
logging.warning('Days seen != 7: {}'.format(days_seen))
logging.debug('Open at {} on {} days (min 5)'.format(preferred_time, len(days_accepted)))
return len(days_accepted) >= 5
def fetch_etc_objects(self, **kwargs):
""" Function downloads all available :obj:`GenData` etc objects from DictyExpress database.
Returns:
:obj:`list`: :obj:`GenData` objects
"""
callback = kwargs.get("progress_callback", None)
with requests_cache.enabled(cache_name=cache_name,
backend=cache_backend):
try:
# Note: this is hardcoded for now. When we port this module to Resolwe platform
# data retrieval will be different
list_of_experiments = self._gen.api.data.get(
case_ids__contains='5535115cfad58d5e03006217',
status='done',
type__startswith='data:etc:')['objects']
if callback:
callback.emit()
except requests.exceptions.ConnectionError as e:
raise requests.exceptions.ConnectionError(
'Server not accessible, check your connection.') from e
store_experiments = [
GenData(exp, self._gen) for exp in list_of_experiments
]
if callback:
callback.emit()
return store_experiments
def download_etc_data(self, gen_data_id, **kwargs):
""" Function downloads etc data of a chosen experiment from the server.
Args:
gen_data_id (str): id of :obj:`GenData` object to download.
Returns:
:obj:`dict`: data in json like format
"""
callback = kwargs.get("progress_callback", None)
table_name = kwargs.get("table_name", '')
with requests_cache.enabled(cache_name=cache_name,
backend=cache_backend):
try:
response = next(
self._gen.download([gen_data_id], 'output.etcfile'))
# TODO: maybe edit Genesis module to support callback?
if callback:
callback.emit()
except requests.exceptions.ConnectionError as e:
raise requests.exceptions.ConnectionError(
'Server not accessible, check your connection.') from e
return response_to_json(response), table_name
def _fetch_access_token():
"""Make http POST request to obtain new API access token.
The access token is valid for 15 minutes, and the response is cached
for 14 minutes so that the token can be reused.
Returns
-------
str
Access token for https://api.wdpro.disney.go.com API.
"""
params = {
"grant_type": "assertion",
"assertion_type": "public",
"client_id": "WDPRO-MOBILE.MDX.WDW.ANDROID-PROD",
}
with requests_cache.enabled(
"token_cache", backend="memory", allowable_methods=("POST",), expire_after=840
):
r = requests.post("https://authorization.go.com/token", params=params)
if r.ok:
auth_data = r.json()
return f"{auth_data['token_type']} {auth_data['access_token']}"
else:
return None
def get_user_profiles(self, user_name_list):
logger.info(f'tw api profiles for {len(user_name_list)} users')
# group usernames in 100 lists
u_groups = [
user_name_list[n:n + 100]
for n in range(0, len(user_name_list), 100)
]
wait = 3
stream = []
for u_list in u_groups:
start_time = time.time()
with requests_cache.enabled(self.cache_path, expire_after=86400):
u_stream = self.api.request('users/lookup', {
'screen_name': u_list,
'include_entities': 'false'
})
for u in u_stream:
stream.append(self.parse_user(u))
elapsed = time.time() - start_time
pause = wait - elapsed
if pause > 0:
time.sleep(pause)
return stream
def get_country_levels(country):
"""Look up the admin levels available for a country from the iTOS service.
@param country: the ISO3 country code (e.g. "GIN")
@returns: a dict of HXL admin-level names and iTOS levels
"""
levels = {}
country = country.upper()
url = COUNTRY_URL_PATTERN.format(country=country)
with requests_cache.enabled(app.config.get('ITOS_CACHE_NAME', 'itos-in'),
backend=app.config.get('ITOS_CACHE_BACKEND',
'memory'),
expire_after=app.config.get(
'ITOS_CACHE_TIMEOUT', 604800)):
with requests.get(url) as result:
data = result.json()
if 'error' in data:
raise werkzeug.exceptions.NotFound(
"iTOS P-code service does not support country: {}".format(country))
for layer in result.json().get('layers'):
result = re.match(r'^Admin(\d)$', layer['name'])
if result:
id = layer['id']
l = result.group(1)
if result == 0:
levels['country'] = id
else:
levels['adm{}'.format(l)] = id
return levels
def process(input, entities):
output = {}
try:
anime = entities['anime'][0]['value']
with requests_cache.enabled('anime_cache',
backend='sqlite',
expire_after=86400):
r = requests.get('https://hummingbird.me/api/v1/search/anime',
params={'query': anime})
data = r.json()
template = TextTemplate()
template.set_text('Title: ' + data[0]['title'] + '\nSynopsis: ' +
data[0]['synopsis'])
template.set_post_text('\nCommunity Rating: ' +
str(round(data[0]['community_rating'], 2)) +
' / 5' + '\nStatus: ' + data[0]['status'])
text = template.get_text()
template = ButtonTemplate(text)
template.add_web_url('Hummingbird URL', data[0]['url'])
output['input'] = input
output['output'] = template.get_message()
output['success'] = True
except:
error_message = 'I couldn\'t find any anime matching your query.'
error_message += '\nPlease ask me something else, like:'
error_message += '\n - Death Note anime'
error_message += '\n - Dragon ball super anime status'
error_message += '\n - What is the anime rating of One Punch Man?'
output['error_msg'] = TextTemplate(error_message).get_message()
output['success'] = False
return output
def process(input, entities):
output = {}
try:
movie = entities['movie'][0]['value']
with requests_cache.enabled('movie_cache',
backend='sqlite',
expire_after=86400):
r = requests.get('http://www.omdbapi.com/?t=' + movie +
'&plot=full&r=json')
data = r.json()
output['input'] = input
template = TextTemplate('Title: ' + data['Title'] + '\nYear: ' +
data['Year'] + '\nIMDb Rating: ' +
data['imdbRating'] + ' / 10' + '\nPlot: ' +
data['Plot'])
text = template.get_text()
template = ButtonTemplate(text)
template.add_web_url(
'IMDb Link', 'http://www.imdb.com/title/' + data['imdbID'] + '/')
output['output'] = template.get_message()
output['success'] = True
except:
error_message = 'I couldn\'t find that movie.'
error_message += '\nPlease ask me something else, like:'
error_message += '\n - batman movie'
error_message += '\n - iron man 2 movie plot'
error_message += '\n - What is the rating of happyness movie?'
output['error_msg'] = TextTemplate(error_message).get_message()
output['success'] = False
return output
def download_etc_data(self, gen_data_id, **kwargs):
""" Function downloads etc data of a chosen experiment from the server.
:param gen_data_id: id of GeneData object
:type gen_data_id: str
:rtype: data in json like format
"""
callback = kwargs.get("progress_callback", None)
table_name = kwargs.get("table_name", '')
with requests_cache.enabled(cache_name=GENAPI_CACHE,
backend=CACHE_BACKEND):
try:
response = next(
self._gen.download([gen_data_id], 'output.etcfile'))
# TODO: maybe edit Genesis module to support callback?
if callback:
callback.emit()
except requests.exceptions.ConnectionError as e:
raise requests.exceptions.ConnectionError(
'Server not accessible, check your connection.') from e
return response_to_json(response), table_name
def process(input, entities=None):
output = {}
try:
with requests_cache.enabled('xkcd_cache', backend='sqlite', expire_after=3600):
# Get the latest comic
r = requests.get('http://xkcd.com/info.0.json')
data = r.json()
# Get a random comic between the first and the latest one
r = requests.get('http://xkcd.com/%d/info.0.json' % randint(1, data['num']))
data = r.json()
title = data['title']
item_url = 'http://xkcd.com/' + str(data['num']) + '/'
explanation_url = 'http://explainxkcd.com/' + str(data['num']) + '/'
image_url = data['img'].replace('\\', '')
subtitle = data['alt']
buttons = ButtonTemplate()
buttons.add_web_url('xkcd Link', item_url)
buttons.add_web_url('Explanation Link', explanation_url)
template = GenericTemplate()
template.add_element(title=title, item_url=item_url, image_url=image_url, subtitle=subtitle, buttons=buttons.get_buttons())
output['input'] = input
output['output'] = template.get_message()
output['success'] = True
except:
error_message = 'There was some error while retrieving data from xkcd.'
output['error_msg'] = TextTemplate(error_message).get_message()
output['success'] = False
return output
def process(input, entities):
output = {}
try:
music = entities['music'][0]['value']
with requests_cache.enabled('music_cache', backend='sqlite', expire_after=3600):
r = requests.get('https://api.spotify.com/v1/search?q=' + music + '&type=track')
data = r.json()
assert(len(data['tracks']['items']) > 0)
template = GenericTemplate()
for track in data['tracks']['items']:
title = track['name']
item_url = track['external_urls']['spotify']
image_url = track['album']['images'][0]['url']
artists = []
for artist in track['artists']:
artists.append(artist['name'])
duration = datetime.utcfromtimestamp(track['duration_ms'] / 1000).strftime('%M:%S')
subtitle = 'By ' + ', '.join(artists) + ' | ' + track['album']['name'] + ' | ' + duration
buttons = ButtonTemplate()
buttons.add_web_url('Preview Track', track['preview_url'])
buttons.add_web_url('Open in Spotify', 'https://embed.spotify.com/openspotify/?spuri=' + track['uri'])
template.add_element(title=title, item_url=item_url, image_url=image_url, subtitle=subtitle, buttons=buttons.get_buttons())
output['input'] = input
output['output'] = template.get_message()
output['success'] = True
except:
error_message = 'I couldn\'t find any music matching your query.'
error_message += '\nPlease ask me something else, like:'
error_message += '\n - hymn for the weekend song'
error_message += '\n - linkin park songs'
error_message += '\n - play hotel california'
output['error_msg'] = TextTemplate(error_message).get_message()
output['success'] = False
return output
def __init__(self, chain_id=ChainID.MAINNET, contract_address=None):
if contract_address is None:
contract_address = self.CONTRACT_ADDRESSES[chain_id]
self.contract_address = contract_address
self.chain_id = chain_id
self.contract_address = contract_address
# ethereum_tester = EthereumTester()
# self.provider = EthereumTesterProvider(ethereum_tester)
self.provider = HTTPProviderFactory.create(self.chain_id)
self.web3 = Web3(self.provider)
self.etherscan_api_key = get_etherscan_api_key()
ChainEtherscanContract = ChainEtherscanContractFactory.create(
self.chain_id)
self.ChainEtherscanAccount = ChainEtherscanAccountFactory.create(
self.chain_id)
# object construction needs to be within the context manager because
# the requests.Session object to be patched is initialized in the
# constructor
with requests_cache.enabled(**REQUESTS_CACHE_PARAMS):
self.etherscan_contract_api = ChainEtherscanContract(
address=self.contract_address, api_key=self.etherscan_api_key)
self.contract_abi = json.loads(
self.etherscan_contract_api.get_abi())
# contract_factory_class = ConciseContract
contract_factory_class = Contract
self.contract = self.web3.eth.contract(
abi=self.contract_abi,
address=self.contract_address,
ContractFactoryClass=contract_factory_class)
# retrieve signatures
self.events_signatures = self.get_events_signatures(self.contract_abi)
self.functions_signatures = self.get_functions_signatures(
self.contract_abi)
def list_papel_setor(setor=None):
"""
Setor: ...
Output:
List
"""
## GET: setor
url = 'http://www.fundamentus.com.br/resultado.php?setor={}'.format(setor)
hdr = {'User-agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; rv:2.2) Gecko/20110201' ,
'Accept': 'text/html, text/plain, text/css, text/sgml, */*;q=0.01' ,
'Accept-Encoding': 'gzip, deflate' ,
}
with requests_cache.enabled():
content = requests.get(url, headers=hdr)
if content.from_cache:
logging.debug('.../resultado.php?setor={}: [CACHED]'.format(setor))
else: # pragma: no cover
logging.debug('.../resultado.php?setor={}: sleeping...'.format(setor))
time.sleep(.500) # 500 ms
## parse + load
df = pd.read_html(content.text, decimal=",", thousands='.')[0]
##
return list(df['Papel'])
def process(input, entities):
output = {}
try:
anime = entities['anime'][0]['value']
with requests_cache.enabled('anime_cache', backend='sqlite', expire_after=86400):
r = requests.get('https://kitsu.io/api/edge/anime', params={
'filter[text]': anime
})
data = r.json()
top_anime = data['data'][0]['attributes']
template = TextTemplate()
template.set_text('Title: ' + top_anime['canonicalTitle'] + '\nSynopsis: ' + top_anime['synopsis'])
template.set_post_text('\nAverage Rating: ' + top_anime['averageRating'] + '%')
text = template.get_text()
template = ButtonTemplate(text)
template.add_web_url('Kitsu URL', 'https://kitsu.io/anime/' + top_anime['slug'])
output['input'] = input
output['output'] = template.get_message()
output['success'] = True
except:
error_message = 'I couldn\'t find any anime matching your query.'
error_message += '\nPlease ask me something else, like:'
error_message += '\n - Death Note anime'
error_message += '\n - Dragon ball super anime status'
error_message += '\n - What is the anime rating of One Punch Man?'
output['error_msg'] = TextTemplate(error_message).get_message()
output['success'] = False
return output
def process(input, entities):
output = {}
try:
word = entities['word'][0]['value']
with requests_cache.enabled('dictionary_cache',
backend='sqlite',
expire_after=86400):
r = requests.get('https://wordsapiv1.p.mashape.com/words/' + word +
'/definitions',
headers={'X-Mashape-Key': WORDS_API_KEY})
data = r.json()
output['input'] = input
output['output'] = TextTemplate(
'Definition of ' + word + ':\n' +
data['definitions'][0]['definition']).get_message()
output['success'] = True
except:
error_message = 'I couldn\'t find that definition.'
error_message += '\nPlease ask me something else, like:'
error_message += '\n - define comfort'
error_message += '\n - cloud definition'
error_message += '\n - what does an accolade mean?'
output['error_msg'] = TextTemplate(error_message).get_message()
output['success'] = False
return output
def fetch_etc_objects(self, *args, **kwargs):
"""Function downloads all available :obj:`GenData` etc objects from DictyExpress database.
:rtype: list of GenData objects
"""
with requests_cache.enabled(cache_name=GENAPI_CACHE,
backend=CACHE_BACKEND):
try:
# Note: this is hardcoded for now. When we port this module to Resolwe platform
# data retrieval will be different
list_of_experiments = self._gen.api.data.get(
case_ids__contains='5535115cfad58d5e03006217',
status='done',
type__startswith='data:etc:')['objects']
except requests.exceptions.ConnectionError as e:
raise requests.exceptions.ConnectionError(
'Server not accessible, check your connection.') from e
store_experiments = [
GenData(exp, self._gen) for exp in list_of_experiments
]
return store_experiments
def load_annotations(data_folder):
infile = os.path.join(data_folder, "litcovid2BioCJSON.gz")
assert os.path.exists(infile)
with open_anyfile(infile, mode='r') as file:
a = file.read()
data_list = json.loads(a)
# First item is a comment by provider
data = data_list[1]
doc_id_set = set()
with requests_cache.enabled('litcovid_cache', expire_after=expire_after):
logging.debug("requests_cache: %s",
requests_cache.get_cache().responses.filename)
for i, rec in enumerate(data, start=1):
# NCBI eutils API limits requests to 10/sec
if i % 100 == 0:
logging.info("litcovid.parser.load_annotations progress %s", i)
doc = getPubMedDataFor(rec["pmid"])
if not doc['from_cache']:
time.sleep(.2)
doc.pop('from_cache')
if doc['_id'] not in doc_id_set:
yield doc
doc_id_set.add(doc['_id'])
requests_cache.core.remove_expired_responses()
def test_as_context_manager(self):
url = HTTPBIN_URL + 'delay/2'
with requests_cache.enabled('test_cache', expire_after=10):
response = requests.get(url)
self.assertFalse(response.from_cache)
response = requests.get(url)
self.assertTrue(response.from_cache)
start = time.time()
response = requests.get(url)
end = time.time()
self.assertFalse(hasattr(response, 'from_cache'))
self.assertGreaterEqual(end - start, 1.5)
with requests_cache.enabled('test_cache'):
response = requests.get(url)
self.assertTrue(response.from_cache)
def process(input, entities):
output = {}
try:
movie = entities['movie'][0]['value']
with requests_cache.enabled('movie_cache', backend='sqlite', expire_after=86400):
# Make a search request to the API to get the movie's TMDb ID
r = requests.get('http://api.themoviedb.org/3/search/movie', params={
'api_key': TMDB_API_KEY,
'query': movie,
'include_adult': False
})
data = r.json()
assert (len(data['results']) > 0)
tmdb_id = str(data['results'][0]['id'])
# Make another request to the API using the movie's TMDb ID to get the movie's IMDb ID
r = requests.get('https://api.themoviedb.org/3/movie/' + tmdb_id, params={
'api_key': TMDB_API_KEY,
'append_to_response': 'videos'
})
data = r.json()
# Fetch movie rating from IMDb
ia = IMDb()
imdb_id = data['imdb_id']
imdb_movie = ia.get_movie(imdb_id[2:])
imdb_movie.fetch()
template = TextTemplate('Title: ' + data['title'] +
'\nYear: ' + data['release_date'][:4] +
'\nIMDb Rating: ' + str(imdb_movie.__dict__['rating']) + ' / 10' +
'\nOverview: ' + data['overview'])
text = template.get_text()
template = ButtonTemplate(text)
template.add_web_url('IMDb Link', 'https://www.imdb.com/title/' + data['imdb_id'] + '/')
videos = data['videos']['results']
# Append first Trailer URL if one exists
for video in videos:
if video['type'] == 'Trailer' and video['site'] == 'YouTube':
template.add_web_url('YouTube Trailer', YouTubeUtil.get_video_url(video['key']))
break
output['input'] = input
output['output'] = template.get_message()
output['success'] = True
except:
error_message = 'I couldn\'t find that movie.'
error_message += '\nPlease ask me something else, like:'
error_message += '\n - batman movie'
error_message += '\n - iron man 2 movie plot'
error_message += '\n - What is the rating of happyness movie?'
output['error_msg'] = TextTemplate(error_message).get_message()
output['success'] = False
return output
def _fetch(self, uri, cache=True):
if cache:
with requests_cache.enabled():
r = requests.get(uri, headers={'Accept': 'application/json'})
else:
with requests_cache.disabled():
r = requests.get(uri, headers={'Accept': 'application/json'})
return r.json
def process(input, entities):
output = {}
try:
movie = entities['movie'][0]['value']
with requests_cache.enabled('movie_cache',
backend='sqlite',
expire_after=86400):
# Make a search request to the API to get the movie's TMDb ID
r = requests.get('http://api.themoviedb.org/3/search/movie',
params={
'api_key': TMDB_API_KEY,
'query': movie,
'include_adult': False
})
data = r.json()
assert (len(data['results']) > 0)
tmdb_id = str(data['results'][0]['id'])
# Make another request to the API using the movie's TMDb ID to get the movie's IMDb ID
r = requests.get('https://api.themoviedb.org/3/movie/' + tmdb_id,
params={
'api_key': TMDB_API_KEY,
'append_to_response': 'videos'
})
data = r.json()
trailer = data['videos']['results'][0]['key']
youtube_link = youtube_link + str(trailer)
# Fetch movie rating from IMDb
ia = IMDb()
imdb_id = data['imdb_id']
imdb_movie = ia.get_movie(imdb_id[2:])
template = TextTemplate('Title: ' + data['title'] + '\nYear: ' +
data['release_date'][:4] + '\nIMDb Rating: ' +
str(imdb_movie['rating']) + ' / 10' +
'\nOverview: ' + data['overview'])
text = template.get_text()
template = ButtonTemplate(text)
template.add_web_url(
'IMDb Link', 'https://www.imdb.com/title/' + data['imdb_id'] + '/')
template.add_web_url('Trailer', youtube_link)
output['input'] = input
output['output'] = template.get_message()
output['success'] = True
except:
error_message = 'I couldn\'t find that movie.'
error_message += '\nPlease ask me something else, like:'
error_message += '\n - batman movie'
error_message += '\n - iron man 2 movie plot'
error_message += '\n - What is the rating of happyness movie?'
output['error_msg'] = TextTemplate(error_message).get_message()
output['success'] = False
return output
def api_search(auth_token, search_term):
with requests_cache.enabled('music_cache',
backend='sqlite',
expire_after=3600):
headers = {'Authorization': 'Bearer ' + auth_token}
r = requests.get('https://api.spotify.com/v1/search?q=' + search_term +
'&type=track',
headers=headers)
return r
def _fetch(self, uri, cache=True):
if cache:
with requests_cache.enabled():
r = requests.get(uri, headers={'Accept' : 'application/json'})
else:
with requests_cache.disabled():
r = requests.get(uri, headers={'Accept' : 'application/json'})
return r.json
def get_categories():
global _categories
if _categories:
return _categories
with requests_cache.enabled(**long_cache):
categories = foursq.venues.categories()
_flatten_categories([], categories['categories'])
return _categories
def test_as_context_manager(self):
url = 'https://httpbin.org/delay/2'
with requests_cache.enabled(session_factory=PerURLCacheSession,
expire_after=10):
response = requests.get(url)
assert not response.from_cache
response = requests.get(url)
assert response.from_cache
start = time.time()
response = requests.get(url)
end = time.time()
assert not hasattr(response, 'from_cache')
assert end - start >= 1.5
with requests_cache.enabled(session_factory=PerURLCacheSession):
response = requests.get(url)
assert response.from_cache
def get_resultado_raw():
"""
Get data from fundamentus:
URL:
http://fundamentus.com.br/resultado.php
RAW:
DataFrame preserves original HTML header names
Output:
DataFrame
"""
##
## Busca avançada por empresa
##
url = 'http://www.fundamentus.com.br/resultado.php'
hdr = {'User-agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; rv:2.2) Gecko/20110201',
'Accept': 'text/html, text/plain, text/css, text/sgml, */*;q=0.01',
'Accept-Encoding': 'gzip, deflate',
}
with requests_cache.enabled():
content = requests.get(url, headers=hdr)
if content.from_cache:
logging.debug('.../resultado.php: [CACHED]')
else: # pragma: no cover
logging.debug('.../resultado.php: sleeping...')
time.sleep(.500) # 500 ms
## parse + load
df = pd.read_html(content.text, decimal=",", thousands='.')[0]
## Fix: percent string
df['Div.Yield'] = utils.perc_to_float( df['Div.Yield'] )
df['Mrg Ebit'] = utils.perc_to_float( df['Mrg Ebit'] )
df['Mrg. Líq.'] = utils.perc_to_float( df['Mrg. Líq.'] )
df['ROIC'] = utils.perc_to_float( df['ROIC'] )
df['ROE'] = utils.perc_to_float( df['ROE'] )
df['Cresc. Rec.5a'] = utils.perc_to_float( df['Cresc. Rec.5a'] )
## index by 'Papel', instead of 'int'
df.index = df['Papel']
df.drop('Papel', axis='columns', inplace=True)
df.sort_index(inplace=True)
## naming
df.name = 'Fundamentus: HTML names'
df.columns.name = 'Multiples'
df.index.name = 'papel'
## return sorted by 'papel'
return df
def handle(self, *args, **options):
with requests_cache.enabled(**settings.REQUEST_CACHE):
_ = [ItemCollector(g).collect() for g in Genre.objects.all()]
_ = itertools.chain.from_iterable(_) # flatten
with transaction.atomic():
for i in _:
# http://stackoverflow.com/questions/7837033/valueerror-cannot-add-instance-is-on-database-default-value-is-on-databas/7999014
item = Item(**i.to_model_param())
item.save()
item.genres = i.genres()
item.save()
def handle(self, *args, **options):
with requests_cache.enabled(**settings.REQUEST_CACHE):
_ = [ItemCollector(g).collect() for g in Genre.objects.all()]
_ = itertools.chain.from_iterable(_) # flatten
with transaction.atomic():
for i in _:
# http://stackoverflow.com/questions/7837033/valueerror-cannot-add-instance-is-on-database-default-value-is-on-databas/7999014
item = Item(**i.to_model_param())
item.save()
item.genres = i.genres()
item.save()
def _get_csv_reader(id: str):
res = model.Resource.get(id)
if res and utils.is_dotstat_url(res.url):
with requests_cache.enabled(
CACHE_PATH,
expire_after=tk.asint(
tk.config.get(CONFIG_DOTSTAT_CACHE_AGE, 60)),
):
resp = requests.get(res.url, stream=True)
if resp.ok:
return csv.DictReader(resp.iter_lines(decode_unicode=True))
def process(input, entities):
output = {}
try:
movie = entities['movie'][0]['value']
with requests_cache.enabled('movie_cache',
backend='sqlite',
expire_after=86400):
# Make a search request to the API to get the movie's TMDb ID
r = requests.get('http://api.themoviedb.org/3/search/movie',
params={
'api_key': TMDB_API_KEY,
'query': movie,
'include_adult': False
})
data = r.json()
assert (len(data['results']) > 0)
tmdb_id = str(data['results'][0]['id'])
# Make another request to the API using the movie's TMDb ID to get the movie's IMDb ID
r = requests.get('https://api.themoviedb.org/3/movie/' + tmdb_id,
params={'api_key': TMDB_API_KEY})
data = r.json()
videoRequests = requests.get(
'https://api.themoviedb.org/3/movie/' + tmdb_id + '/videos',
params={'api_key': TMDB_API_KEY})
videoData = videoRequests.json()
with open('data.txt', 'w') as outfile:
json.dump(videoData, outfile)
template = TextTemplate('Title: ' + data['title'] + '\nYear: ' +
data['release_date'][:4] +
'\nAverage Rating: ' +
str(data['vote_average']) + ' / 10' +
'\nOverview: ' + data['overview'])
text = template.get_text()
template = ButtonTemplate(text)
template.add_web_url(
'IMDb Link', 'https://www.imdb.com/title/' + data['imdb_id'] + '/')
template.add_web_url(
'Trailer', 'https://www.youtube.com/watch?v=' +
videoData['results'][0]['key'])
output['input'] = input
output['output'] = template.get_message()
output['success'] = True
except:
error_message = 'I couldn\'t find that movie.'
error_message += '\nPlease ask me something else, like:'
error_message += '\n - batman movie'
error_message += '\n - iron man 2 movie plot'
error_message += '\n - What is the rating of happyness movie?'
output['error_msg'] = TextTemplate(error_message).get_message()
output['success'] = False
return output
def fetch(self, uri, cache=True):
if cache:
with requests_cache.enabled():
r = requests.get(uri, headers={'Accept' : 'application/json'})
else:
with requests_cache.disabled():
r = requests.get(uri, headers={'Accept' : 'application/json'})
# last uri-part is dataset name and dictionary key
key = uri.split('/')[-1]
return r.json.get(key)
def request_json_resource(url, params=None, retry=3, time_between=1):
with requests_cache.enabled(os.path.join(CACHE_DIR, 'lcs_static_cache')):
for i in range(retry):
response = requests.get(url, params=params, headers={'Origin': 'http://www.lolesports.com'})
if response.status_code == 200:
return response.json(object_pairs_hook=OrderedDict)
elif response.status_code == 404:
break
else:
time.sleep(time_between)
raise Exception('Unable to retrieve json recourse')
def test_enabled(self):
url = httpbin('get')
options = dict(cache_name=CACHE_NAME, backend=CACHE_BACKEND,
fast_save=FAST_SAVE)
with requests_cache.enabled(**options):
r = requests.get(url)
self.assertFalse(getattr(r, 'from_cache', False))
for i in range(2):
r = requests.get(url)
self.assertTrue(getattr(r, 'from_cache', False))
r = requests.get(url)
self.assertFalse(getattr(r, 'from_cache', False))
def fetch(self, uri, cache=True):
if cache:
with requests_cache.enabled():
r = requests.get(uri, headers={'Accept': 'application/json'})
else:
with requests_cache.disabled():
r = requests.get(uri, headers={'Accept': 'application/json'})
# last uri-part is dataset name and dictionary key
key = uri.split('/')[-1]
return r.json.get(key)
def test_enabled(self):
url = httpbin('get')
options = dict(cache_name=CACHE_NAME, backend=CACHE_BACKEND,
fast_save=FAST_SAVE)
with requests_cache.enabled(**options):
r = requests.get(url)
self.assertFalse(getattr(r, 'from_cache', False))
for i in range(2):
r = requests.get(url)
self.assertTrue(getattr(r, 'from_cache', False))
r = requests.get(url)
self.assertFalse(getattr(r, 'from_cache', False))
def get_cached_ids(self):
with requests_cache.enabled(cache_name=cache_name, backend=cache_backend):
cached_object = requests_cache.core.get_cache()
responses = [cached_object.get_response_and_time(response) for response in cached_object.responses]
gen_ids = []
for url in [response.url for response, _ in responses]:
gen_id = re.search(r'{}(.*?)/'.format(self._data_endpoint), url)
if gen_id is not None:
gen_ids.append(gen_id.group(1))
return gen_ids
def test_disabled_enabled(self):
delay = 1
url = 'http://httpbin.org/delay/%s' % delay
with requests_cache.disabled():
t = time.time()
n = 2
for i in range(n):
requests.get(url)
delta = time.time() - t
self.assertGreaterEqual(delta, delay * n)
with requests_cache.enabled():
t = time.time()
n = 5
for i in range(n):
requests.get(url)
delta = time.time() - t
self.assertLessEqual(delta, delay * n)
def process(input, entities):
output = {}
try:
word = entities['word'][0]['value']
with requests_cache.enabled('dictionary_cache', backend='sqlite', expire_after=86400):
r = requests.get('https://wordsapiv1.p.mashape.com/words/' + word + '/definitions', headers={
'X-Mashape-Key': WORDS_API_KEY
})
data = r.json()
output['input'] = input
output['output'] = TextTemplate(
'Definition of ' + word + ':\n' + data['definitions'][0]['definition']).get_message()
output['success'] = True
except:
error_message = 'I couldn\'t find that definition.'
error_message += '\nPlease ask me something else, like:'
error_message += '\n - define comfort'
error_message += '\n - cloud definition'
error_message += '\n - what does an accolade mean?'
output['error_msg'] = TextTemplate(error_message).get_message()
output['success'] = False
return output
def process(input, entities):
output = {}
try:
book_title = entities['book'][0]['value']
with requests_cache.enabled('book_cache', backend='sqlite', expire_after=86400):
response = requests.get(
'https://www.goodreads.com/book/title.xml?key=' + GOODREADS_ACCESS_TOKEN + '&title=' + book_title)
data = ElementTree.fromstring(response.content)
book_node = data.find('book')
author = book_node.find('authors').find('author').find('name').text
title = book_node.find('title').text
description = html2text(book_node.find('description').text)
average_rating = book_node.find('average_rating').text
link = book_node.find('link').text
goodreads_attribution = '- Powered by Goodreads'
template = TextTemplate()
template.set_text('Title: ' + title + '\nAuthor: ' + author + '\nDescription: ' + description)
template.set_post_text('\nAverage Rating: ' + average_rating + ' / 5' + '\n' + goodreads_attribution)
text = template.get_text()
template = ButtonTemplate(text)
template.add_web_url('Goodreads Link', link)
output['input'] = input
output['output'] = template.get_message()
output['success'] = True
except:
error_message = 'I couldn\'t find any book matching your query.'
error_message += '\nPlease ask me something else, like:'
error_message += '\n - book timeline'
error_message += '\n - harry potter book plot'
error_message += '\n - little women book rating'
output['error_msg'] = TextTemplate(error_message).get_message()
output['success'] = False
return output
def process(input, entities):
output = {}
try:
movie = entities['movie'][0]['value']
with requests_cache.enabled('movie_cache', backend='sqlite', expire_after=86400):
r = requests.get('http://www.omdbapi.com/?t=' + movie + '&plot=full&r=json')
data = r.json()
output['input'] = input
template = TextTemplate('Title: ' + data['Title'] + '\nYear: ' + data['Year'] + '\nIMDb Rating: ' + data['imdbRating'] + ' / 10' + '\nPlot: ' + data['Plot'])
text = template.get_text()
template = ButtonTemplate(text)
template.add_web_url('IMDb Link', 'http://www.imdb.com/title/' + data['imdbID'] + '/')
output['output'] = template.get_message()
output['success'] = True
except:
error_message = 'I couldn\'t find that movie.'
error_message += '\nPlease ask me something else, like:'
error_message += '\n - batman movie'
error_message += '\n - iron man 2 movie plot'
error_message += '\n - What is the rating of happyness movie?'
output['error_msg'] = TextTemplate(error_message).get_message()
output['success'] = False
return output
def get_venue(venue_id):
with requests_cache.enabled(**short_cache):
return foursq.venues(venue_id)['venue']
def api_search(auth_token, search_term):
with requests_cache.enabled('music_cache', backend='sqlite', expire_after=3600):
headers = {'Authorization': 'Bearer ' + auth_token}
r = requests.get('https://api.spotify.com/v1/search?q=' + search_term + '&type=track', headers=headers)
return r
from contextlib import contextmanager
import requests_cache
import requests
resp = requests.get("http://httpbin.org/redirect/4")
for r in resp.history:
print("url: {} request.url: {}".format(r.url, r.request.url))
@contextmanager
def cache_enabled(*args, **kwargs):
requests_cache.install_cache(*args, **kwargs)
try:
yield
finally:
requests_cache.uninstall_cache()
with requests_cache.enabled('test.db'):
resp = requests.get("http://httpbin.org/get")
print getattr(resp, 'from_cache', False)
s = requests.session()
for i in range(5):
s.get("http://httpbin.org/redirect/3")
resp = requests.get("http://httpbin.org/get")
print getattr(resp, 'from_cache', False)
def _request(url):
with requests_cache.enabled(os.path.join(CACHE_DIR, STATIC_CACHE)):
return requests.get(url, stream=True)
