def get_application_access_token(app_id, app_secret):
''' Get an application access token, as described here:
http://developers.facebook.com/docs/authentication/#authenticating-as-an-application
Some graph API calls require one of thse.
'''
url = FB_GRAPH_BASE_URL
url += "/oauth/access_token?"
url += "client_id=%s" % app_id
url += "&client_secret=%s" % app_secret
url += "&grant_type=client_credentials"
s = net.get(url)
if s:
# in the body of the response will be access_token=
prefix = "access_token="
if not s.startswith(prefix):
logger.error("Got malformed app access token response: '%s'" % s)
x = len(prefix)
access_token = s[x:]
logger.debug("Got application access token: %s" % access_token)
return access_token
else:
logger.error("Failed to get application access token")
return none
def fetch(self, operation, operation_params):
''' fetch document, return file-like object as response '''
url = self.construct_url(operation, operation_params)
if self.printurl:
logger.info(url)
# Google App Engine supports Python 2.7 as of release 1.6.0. However, there is an outstanding bug (#6271) in their standard library facade over the urlfetch
# service, so use urlfetch directly if we're on GAE.
f = net.get(url)
return f
def get_fql(url):
""" Generic helper to execute an FQL request and handle error codes. """
# record time to get FQL data.
t1 = time.time()
s = net.get(url)
if not s:
raise NetException("Failed to execute FQL request, url=%s" % url)
# should have json. check for error or success now.
o = json.loads(s)
# logger.info("****FQL:")
# logger.info(o)
# logger.info("*************")
# type checking is ghetto, but FQL will return a JSON dictionary on error and a potentially a list otherwise. If it's not a dictionary,
# skip further error checking.
if isinstance(o, type({})):
try:
error_code = o["error_code"]
try:
error_msg = o["error_msg"]
except KeyError:
error_msg = ""
if error_code == 190: # Invalid OAuth 2.0 token
raise InvalidOAuthException("Invalid auth: error_msg='%s', url=%s" % (error_msg, url))
else:
raise Exception("FQL error code: %d, error_msg='%s', url=%s" % (error_code, error_msg, url))
except KeyError:
# no error. should have successfully retrieved friends json.
pass
# success
t2 = time.time()
x = t2 - t1
logger.info("get_fql: took %0.2f seconds" % x)
return o
def get_wall_posts(user_id, access_token):
''' requires read_stream to get non-public posts
Should return a json dictionary containing:
data - list of posts (maybe up to 50)
paging - links to get more "pages" of posts
'''
url = "FB_GRAPH_BASE_URL"
url += "/%s/feed?access_token=%s" % (user_id, access_token)
s = net.get(url)
logger.info("Got user's posts: %s" % s)
if s:
o = json.loads(s)
return o
else:
return None
def delete_test_user(uid, access_token):
''' Delete the test user '''
url = FB_GRAPH_BASE_URL
url += "/%s?method=delete&access_token=%s" % (uid, access_token)
s = net.get(url)
if s:
if s == 'true':
logger.debug("Deleted test user")
return True
else:
logger.error("Failed to delete test user, response='%s'" % s)
return None
else:
logger.error("Failed to delete test user")
return False
def list_test_users(app_id, app_access_token):
''' List test users associated with this application '''
url = "FB_GRAPH_BASE_URL"
url += "/%s/accounts/test-users" % app_id
url += "?access_token=%s" % app_access_token
s = net.get(url)
if s:
logger.debug("List test users, response: '%s'" % s)
o = json.loads(s)
users = o["data"]
return users
else:
logger.error("Failed to list test users")
return None
def _crawl(self):
item = self._site["task"]
md5 = item["md5"]
url = item["url"]
last_modified = item["last_modified"]
headers = {}
#有last_modified, 且已抓取过的，请求头要加上Last-Modified
if item["state"] == CONFIG.G_STATUS_CRAWLED and last_modified != "":
headers["Last-Modified"] = last_modified
(head, html) = net.get(url, headers, d_config=self._site["config"])
self._save_html(md5, html)
if int(head["code"]) == 200:
ret = self._analyze_html(url, html)
self._update_state(CONFIG.G_STATUS_CRAWLED, {"md5": md5})
return ret
else:
self._update_state(CONFIG.G_STATUS_ERROR, {"md5": md5})
return "spider.run: crawl %s %s" % (head["code"], url)
def _crawl(self):
item = self._site["task"]
md5 = item["md5"]
url = item["url"]
last_modified = item["last_modified"]
headers = {}
#有last_modified, 且已抓取过的，请求头要加上Last-Modified
if item["state"] == CONFIG.G_STATUS_CRAWLED and last_modified != "":
headers["Last-Modified"] = last_modified
(head, html) = net.get(url, headers, d_config=self._site["config"])
self._save_html(md5, html)
if int(head["code"]) == 200:
ret = self._analyze_html(url, html)
self._update_state(CONFIG.G_STATUS_CRAWLED, {"md5":md5})
return ret
else:
self._update_state(CONFIG.G_STATUS_ERROR, {"md5":md5})
return "spider.run: crawl %s %s"%(head["code"], url)
def get_friends_list(user_id, access_token, fields=["id"], offset=0, limit=None):
''' Get list of the user's friends
fields - list of fields to retrieve. e.g. (name, id)
'''
fields = ",".join(fields)
url = "FB_GRAPH_BASE_URL"
url += "/%s/friends?access_token=%s&fields=%s" % (user_id, access_token, fields)
if offset:
url += "&offset=%d" % offset
if limit:
url += "&limit=%d" % limit
s = net.get(url)
if s:
o = json.loads(s)
return o["data"] # data element contains a list of dictionary objects repesenting the friends
else:
return None
def _run(self, task_data):
url = task_data["url"]
domain = url.split("/")[2]
d_config = CONFIG.G_SITE[domain]
default_code = d_config["default_code"]
html = None
if CONFIG.G_IFSAVE_HTML == True:
html = self._html_from_db(task_data["md5"], CONFIG.G_TABLE_HTML)
if html:
default_code = "utf-8"
if not html:
(header, html) = net.get(url, d_config=d_config)
if not header["code"] == 200:
self._pick_state(task_data["md5"],\
CONFIG.G_STATE_ERROR, CONFIG.G_TABLE_LINK)
return (CONFIG.G_STATE_NET_ERROR, (0, 0))
count = self._pick(d_config, html, task_data, default_code)
return (CONFIG.G_STATE_PICKED, count)
def get_users(user_ids, access_token, fields=None, callback=None):
''' Get info about a list of users
callback - if specific, request will be done asynchronously and a handle to the rpc object will be returned
'''
user_ids = ",".join(user_ids)
if fields:
fields = ",".join(fields)
else:
fields = USER_COLUMN_CLAUSE
url = "FB_GRAPH_BASE_URL"
url += "/?ids=%s&access_token=%s&fields=%s" % (user_ids, access_token, fields)
logger.debug("get_users: url=%s" % url)
def result_callback(result):
if result:
o = json.loads(result)
return callback(o)
else:
return callback(None)
if callback:
# async
rpc = net.get_async(url, result_callback)
return rpc
else:
# blocking
s = net.get(url)
if s:
o = json.loads(s)
return o
else:
return None
def authenticate_app(app_id, app_secret, redirect_url, code):
''' Authenticate app. (last step of server-side flow authentication process) '''
url = "%s/oauth/access_token?client_id=%s&redirect_uri=%s&client_secret=%s&code=%s" % (FB_GRAPH_BASE_URL, app_id, redirect_url, app_secret, code)
logger.info("autenticate_app: code='%s'" % code)
logger.info("authenticate_app: url=%s" % url)
s = net.get(url)
logger.info("authenticate_app: Got '%s'" % s)
# s is a string of the form:
# 'access_token=AAAENJAf4kUUBAFnyAOxBpLD2O5nJvG5eD4X2ZBKAaZAl23ZA82qExyRZAtO6yILsVgyw963UPvfiOG9wUiHRCiyK5MXOSO5gbZBBdfaPSPu7liNJA9pG7&expires=5242'
# split on & to divide up the 2 useful bits.
(atok, etok) = s.split("&")
# split each on the equals:
access_token = atok.split("=")[1]
expires = etok.split("=")[1]
# expires == number of seconds until token expires.
return (access_token, expires)
def _run(self, task_data):
url = task_data["url"]
domain = url.split("/")[2]
d_config = CONFIG.G_SITE[domain]
default_code = d_config["default_code"]
html = None
if CONFIG.G_IFSAVE_HTML == True:
html = self._html_from_db(task_data["md5"], CONFIG.G_TABLE_HTML)
if html:
default_code = "utf-8"
if not html:
(header, html) = net.get(url, d_config=d_config)
if not header["code"] == 200:
self._pick_state(task_data["md5"],\
CONFIG.G_STATE_ERROR, CONFIG.G_TABLE_LINK)
return (CONFIG.G_STATE_NET_ERROR, (0, 0))
self.ext_before_pick(html, url)
count = self._pick(d_config, html, task_data, default_code)
return (CONFIG.G_STATE_PICKED, count)
from win10toast import ToastNotifier
import stocks.company
import net
base_url = 'http://www.tsetmc.com/tsev2/data/instinfodata.aspx?i={id}&c=67%20'
comLis = stocks.company.load_all()
toaster = ToastNotifier()
f = open('safe_res.txt', 'w', encoding='utf-8')
while True:
for v in comLis:
addr = base_url.format(id=v.id)
res = net.get(addr)
vec_data = res.text.split(',')
#
if len(vec_data) < 4:
continue
#
yesterday_price = int(vec_data[5])
last_price = int(vec_data[3])
if last_price <= yesterday_price:
continue
current_price = int(vec_data[2])
ratio = current_price / last_price - 1
if (ratio < -0.04):
#f.write(str(ratio) + ' : ' + v.name + '\n')
#f.flush()
toaster.show_toast("IR-Market", v.name)
print('--------------')
print(v.id, '|', ratio, ':', last_price, current_price)
import net
url_tmpl = 'https://rahavard365.com/asset/{}/%D8'
li = []
c = 0
for i in range(4, int(2e4)):
res = net.get(url_tmpl.format(i)).text
try:
loc = res.find('last_pb')
if loc != -1:
loc_e = res.find('date', loc)
#print(i, ':', res[loc+18:loc_e-2])
pb = float(res[loc + 18:loc_e - 2])
li.append((i, pb))
except:
print('err at', i)
c += 1
if c == 100:
print('.')
c = 0
li.sort(key=lambda x: x[1])
print(li)
with open('res.txt', 'w') as f:
col_width = max(len(str(word)) for row in li
for word in row) + 2 # padding
for tup in li:
f.write("".join(str(word).ljust(col_width)
for word in tup[:-1]) + str(tup[-1]) + '\n')
def had_constant_price(prices) -> bool:
if len(prices) == 0:
return True
pre_price = prices[0]
for price in prices:
if price != pre_price:
return False
pre_price = price
return True
final_list = []
for com in comLis:
try:
# tset
tset_vec_data = net.get(tset_base_url.format(id=com.id)).text.split(',')
last_price = int(tset_vec_data[3])
yesterday_price = int(tset_vec_data[5])
# tadbirrlc
url = base_url.format(symbol=com.symbol)
prices = [vec[3] for vec in net.get(url).json()['olst'][5]]
if had_constant_price(prices):
print(com.symbol)
final_list.append((last_price/yesterday_price, com))
except KeyboardInterrupt:
exit()
except:
continue
final_list.sort(key = lambda x: x[0])
