def get(self):
y = yql.Public()
query = 'select * from xml where url="http://www.espncricinfo.com/rss/content/story/feeds/0.xml"'
res = y.execute(query)
#last notification
tweet = "A small Status"
#tweet = res.rows[0]['channel']['item'][0]['description']
#fetch from GAE datastore
fetched = model.fetch()
if not fetched:
model.add('INITIALIZATION')
self.response.write('INIT')
else:
fetched = str(list(r.tweet for r in fetched)[0])
self.response.write(' Fetching ')
if tweet == fetched:
self.response.write(' Same Tweet ')
pass
else:
model.add(tweet)
if len(tweet) > 140:
a, b = None, None
a, b = split(tweet)
update(a, b)
self.response.write(' Updated & Added ')
else:
update(tweet, None)
self.response.write(' Updated & Added ')
def test_placeholder_regex_five(self):
y = yql.Public()
query = """SELECT * from foo
where foo='bar' LIMIT
@foo"""
placeholders = y.get_placeholder_keys(query)
self.assertEqual(placeholders, ['foo'])
def getTwitterNameFromLanyrdPerson(lpersonpath):
y = yql.Public()
query = "select href from html where url='http://lanyrd.com"+lpersonpath+"' and xpath='//a[@class=\"icon twitter url nickname\"]'"
result = y.execute(query)
for row in result.rows:
return row.get('href')
return 'oops'
def test_check_env_var(self):
"""Testing env variable"""
y = yql.Public()
env = "http://datatables.org/alltables.env"
query = "SHOW tables;"
res = y.execute(query, env=env)
assert res.count >= 800
def test_placeholder_regex_five():
y = yql.Public()
query = """SELECT * from foo
where foo='bar' LIMIT
@foo"""
placeholders = y.get_placeholder_keys(query)
assert placeholders == ['foo']
def test_xpath_works(self):
y = yql.Public()
query = """SELECT * FROM html
WHERE url='http://google.co.uk'
AND xpath="//input[contains(@name, 'q')]"
LIMIT 10"""
res = y.execute(query)
assert res.rows[0].get("title") == "Google Search"
def page_not_found(request):
y = yql.Public()
response = y.execute(
'select * from flickr.photos.search where tags="openhackeu"')
photos = response['query']['results']['photo']
return render_to_response('404.html', {'photos': photos},
RequestContext(request))
def test_raises_error_if_status_is_not_200(self):
query = 'UPDATE foo'
expected_data = 'some data'
def request(url, http_method, headers, body):
return {'status': '500'}, json.dumps({'query': expected_data})
y = yql.Public(httplib2_inst=StubHttp(request_callback=request))
y_obj = y.execute(query)
def run_sparql_query(query, endpoint):
'''
# The following string replacement construction may be handy
query = 'select * from flickr.photos.search where text=@text limit 3';
y.execute(query, {"text": "panda"})
'''
y = yql.Public()
query = 'select * from sparql.search where query="' + query + '" and service="' + endpoint + '"'
env = "http://datatables.org/alltables.env"
return y.execute(query, env=env)
def getLanyrdPeopleXingEvent(twpl,year, event,typ='attendees'):
#attendees or trackers
y = yql.Public()
query="select href from html where url='http://lanyrd.com/"+str(year)+"/"+event+"/' and xpath='//div[@class=\""+typ+"-placeholder placeholder\"]/ul[@class=\"user-list\"]/li/a'"
result = y.execute(query)
for row in result.rows:
lpersonpath=row.get('href')
twurl=getTwitterNameFromLanyrdPerson(lpersonpath)
ret=(twurl.replace('http://twitter.com/',''),typ)
twpl.append(ret)
return twpl
def test_sends_content_type_and_serialized_data_on_update(self):
query = 'UPDATE foo'
expected_data = 'some data'
def request(url, http_method, headers, body):
self.assertEqual(headers, {"Content-Type": "application/json"})
self.assertEqual(body, json.dumps({'q': query}))
return {'status': '200'}, json.dumps({'query': expected_data})
y = yql.Public(httplib2_inst=StubHttp(request_callback=request))
y_obj = y.execute(query)
self.assertEqual(y_obj.raw, expected_data)
def get_raw_quote(self):
"""Get a list of quotes from the Yahoo YQL finance tables and return the result.
Given the code of the stock and a list containing the start and end dates of
the data.
"""
# Validate dates first
ret, date_range = validate_date_range(self.date_range)
if not ret:
# raise exception or just quit - validate_date_range will raise an exceptions
raise Exception('Date range is no valid')
start_date = date_range[0]
end_date = date_range[1]
# Create query object - must set the environment for community tables
y = yql.Public()
env = 'http://www.datatables.org/alltables.env'
# Determine the query columns
if self.fields == '*':
columns = '*'
else:
columns = [
self.get_column_from_field(field) for field in self.fields
]
# Join as a comma separated string
columns = ','.join(columns)
# Execute the query and get the response
query = 'select %(columns)s from yahoo.finance.historicaldata ' \
'where symbol = "%(code)s.%(exchange)s" ' \
'and startDate = "%(start_date)s" and endDate = "%(end_date)s"' \
% {
'code': self.code, 'exchange': self.exchange, 'columns': columns,
'start_date': start_date, 'end_date': end_date,
}
response = y.execute(query, env=env)
# If the response results are null there was an error
if response.results is None:
raise Exception('Error with results')
# Get the quote
quote = response.results['quote']
return quote
def get_yql_subjects(text):
registry = getUtility(IRegistry)
settings = registry.forInterface(ITagHelperSettingsSchema)
api_key = settings.yahoo_api_key
if api_key:
y = yql.Public(api_key)
query = '''select * from search.termextract where context ="%s"
''' % text.replace('"',"'")
try:
result = y.execute(query)
except:
return []
return result.rows
else:
return []
def get_yql_subjects_remote(url):
registry = getUtility(IRegistry)
settings = registry.forInterface(ITagHelperSettingsSchema)
api_key = settings.yahoo_api_key
if api_key:
y = yql.Public(api_key)
query = '''select * from search.termextract where context in (
select content from html where url="%s"
)''' % url
try:
result = y.execute(query)
except:
return []
return result.rows
else:
return []
def get_raw_quote(self):
"""Get a quote from the Yahoo YQL finance tables and return the result.
"""
# Error column name - save typing
error_column = 'ErrorIndicationreturnedforsymbolchangedinvalid'
# Create query object - must set the environment for community tables
y = yql.Public()
env = 'http://www.datatables.org/alltables.env'
# Determine the query columns
if self.fields == '*':
columns = '*'
else:
columns = [
self.get_column_from_field(field) for field in self.fields
]
# Ensure the error column in included
if not error_column in columns:
columns.append(error_column)
# Join as a comma separated string
columns = ','.join(columns)
# Execute the query and get the response
query = 'select %(columns)s from yahoo.finance.quotes where symbol = "%(code)s.%(exchange)s"' \
% {'code': self.code, 'exchange': self.exchange, 'columns': columns, }
response = y.execute(query, env=env)
# Get the quote and the error field
quote = response.results['quote']
error = quote[error_column]
# If no error return the quote or raise an exception
if error is None:
# Valid code and quote
return quote
raise Exception(error)
def fetch_answer(*question, **kwargs):
query="select * from answers.search where query='%s'"
y = yql.Public()
print('You asked: ',' '.join(*question))
result = y.execute(query % ' '.join(*question))
if len(result.rows)==0:
print('No answers found!')
return
limit = kwargs['limit']
if limit>0:
results = result.rows[:limit]
else:
results = result.rows
for item in results:
if item.get('type')=='Answered':
print(prepare_answer(item))
print('_'*82)
def get_current(tickers):
# Builds a string listing all tickers in the query format.
tickerString = ','.join(['"'+str(sym)+'"' for sym in tickers])
print(tickerString)
# The query to get the stock information TODO with...
query = 'select * from yahoo.finance.quotes where symbol in (' + tickerString + ')'
# Uses python-yql library to publicly connect to Yahoo API and execute the query.
y = yql.Public()
try:
result = y.execute(query, env = "http://datatables.org/alltables.env")
except YQLError:
return []
stocks = []
# Handle empty result set
if result['query']['count'] < 1:
return stocks
# Yahoo does not give a one-element list if our query only gave one result. Therefore put the one element in a list before treating.
if result['query']['count'] == 1:
raw = [result['query']['results']['quote']]
else:
raw = result['query']['results']['quote']
# Now transform to list of dictionaries with only lower case content and without html tags.
for row in raw:
stock = {}
for key in row.keys():
stock[str(key).lower()] = str(strip_tags(row[key]))
stocks.append(stock)
return stocks
def test_incorrect_type_raises_valueerror(self):
y = yql.Public()
query = "SELECT * from foo where dog=@test"
params = ('fail')
y.execute(query, params)
def test_json_response_from_file():
query = 'SELECT * from foo WHERE dog=@dog'
from httplib2 import Http
y = yql.Public(httplib2_inst=Http())
content = y.execute(query, {"dog": "fifi"})
assert content is not None
def start_location(data):
data.start()
return False
loop = gobject.MainLoop()
control = location.GPSDControl.get_default()
device = location.GPSDevice()
control.set_properties(preferred_method=location.METHOD_USER_SELECTED,
preferred_interval=location.INTERVAL_DEFAULT)
control.connect("error-verbose", on_error, loop)
device.connect("changed", on_changed, control)
i=0
QUERY = "USE 'http://www.meusgastos.com.br/odt/meuspostos.precos.coordenadas.xml'
as meuspostos.precos.coordenadas;
SELECT * FROM meuspostos.precos.coordenadas
WHERE lat=%f and lon=%f and sort=1" % device.fix[4:6]
y = yql.Public()
full_result = y.execute(QUERY)
for row in full_result['query']['results']['item']:
name_and_price = "%s (R$%s)" % (row['nome'],row['gasolina'])
gas_station_list.append(name_and_price)
control.connect("gpsd-stopped", on_stop, loop)
gobject.idle_add(start_location, control)
loop.run()
def test_empty_args_raises_valueerror(self):
y = yql.Public()
query = "SELECT * from foo where dog=@dog"
params = {}
y.execute(query, params)
def run_sparql_query(query, endpoint):
y = yql.Public()
query = 'select * from sparql.search where query="' + query + '" and service="' + endpoint + '"'
env = "http://datatables.org/alltables.env"
return y.execute(query, env=env)
def test_incorrect_args_raises_valueerror(self):
y = yql.Public()
query = "SELECT * from foo where dog=@dog"
params = {'test': 'fail'}
y.execute(query, params)
def test_params_raises_when_not_dict(self):
y = yql.Public()
query = "SELECT * from foo where dog=@dog"
params = ['test']
y.execute(query, params)
def test_unecessary_args_raises_valueerror(self):
y = yql.Public()
query = "SELECT * from foo where dog='test'"
params = {'test': 'fail'}
y.execute(query, params)
def test_json_response_from_file(self):
query = 'SELECT * from foo WHERE dog=@dog'
y = yql.Public(httplib2_inst=httplib2.Http())
content = y.execute(query, {"dog": "fifi"})
self.assertEqual(content.count, 3)
""" web.py - handy functions for web services """
import http, urlnorm
import json, urllib
import yql
short_url = "http://is.gd/create.php"
paste_url = "http://paste.dmptr.com"
yql_env = "http://datatables.org/alltables.env"
YQL = yql.Public()
class ShortenError(Exception):
def __init__(self, code, text):
self.code = code
self.text = text
def __str__(self):
return self.text
def isgd(url):
""" shortens a URL with the is.gd PAI """
url = urlnorm.normalize(url.encode('utf-8'), assume_scheme='http')
params = urllib.urlencode({'format': 'json', 'url': url})
request = http.get_json("http://is.gd/create.php?%s" % params)
if "errorcode" in request:
raise ShortenError(request["errorcode"], request["errormessage"])
else:
def test_placeholder_regex_one(self):
y = yql.Public()
query = "SELECT * from foo where email='*****@*****.**'"
placeholders = y.get_placeholder_keys(query)
self.assertEqual(placeholders, [])
def test_cannot_use_unrecognizable_endpoint(self):
y = yql.Public()
y.endpoint = 'some-strange-endpoint'
def test_placeholder_regex_three(self):
y = yql.Public()
query = "SELECT * from foo where email=@foo and test=@bar'"
placeholders = y.get_placeholder_keys(query)
self.assertEqual(placeholders, ['foo', 'bar'])