def run():
accom_db = db['accom']
for accom in accom_db.find():
print green(">> %s" % accom['name'])
url = "https://maps.googleapis.com/maps/api/place/nearbysearch/json?key=%s&location=%s&sensor=false&rankby=distance" % (
key,
"%s,%s" % ( accom['location'][0], accom['location'][1] )
)
accom['nearby'] = {}
for t in types.items():
print blue(">>> %s" % t[0])
t_url = "%s&types=%s" % (url, t[1])
response = common.get_json("%s" % t_url)
accom['nearby'][t[0]] = []
for place in response['results'][0:5]:
print yellow(">>>> %s" % place['name'])
# Really Google? Can't even return us a URL?
detail_url = "https://maps.googleapis.com/maps/api/place/details/json?sensor=false&key=%s&reference=%s" % (
key,
place['reference']
)
place = common.get_json(detail_url)
place = {
"name" : place['result']['name'],
"icon" : place['result']['icon'],
"url" : place['result']['url']
}
accom['nearby'][t[0]].append(place)
print green(">> OK")
accom_db.save(accom)
def main():
start_ts = int(time.time())
if 'psutil' in sys.modules:
mem1 = psutil.virtual_memory()
setup_environment()
list_file = "list.json"
names = common.get_json(list_file, default_value=[])
if '--generate' in sys.argv:
extended_names = extend_list(names)
json.dump(extended_names, sys.stdout, indent=4, sort_keys=True)
exit(0)
if len(sys.argv) > 1:
print("Invalid argument, exiting.")
exit(0)
snapshot_file = "snapshot.json"
snapshot = common.get_json(snapshot_file, default_value={})
execcount = snapshot.get("execcount", 0)
snapshot["execcount"] = execcount + 1
msg = setup_request_params(execcount)
extended_names = names if execcount % 5 != 1 else extend_list(names)
config = common.get_json("config.txt", default_value={})
processes = config.get("processes", 25)
msg += " using %d threads" % processes
timeout = config.get("timeout", 720)
global request_time
request_time = config.get("request_time", 15)
if isinstance(timeout, int):
time_left = timeout + start_ts - int(time.time())
else:
time_left = max(0, 3450 - int(time.time()) % 3600)
msg += ", timeout %d secs" % time_left
print_ts(msg)
results = download_all(extended_names,
time_left=time_left,
processes=processes)
news = set(extended_names).difference(set(names))
new_names = update_snapshot(snapshot, results, news)
update_stats(snapshot)
if len(new_names) > 0:
extended_names = sorted(list(set(new_names).union(set(names))))
with open(list_file, 'w') as outfile:
json.dump(extended_names, outfile, indent=4, sort_keys=True)
if 'psutil' in sys.modules:
mem2 = psutil.virtual_memory()
memory_msg = ", free memory: %.2fG -> %.2fG of %.2fG" % (
mem1.free / 1024.0**3, mem2.free / 1024.0**3,
mem2.total / 1024.0**3)
else:
memory_msg = ""
close_msg(start_ts, execcount, memory_msg)
def get_penguins(cpps=None, server=None, shape=None):
servers = common.get_json("servers")
shapes = common.get_json("shapes")
try:
cpps = common.get_cpps(servers, cpps)
server = common.get_server(servers, cpps, server)
shape = get_shape(shapes, shape)
except (KeyboardInterrupt, EOFError):
raise common.LoginError()
logger = logging.getLogger()
logger.addHandler(logging.NullHandler())
clients_offsets = [(common.get_client(servers, cpps, server, logger),
int(offset["x"]), int(offset["y"]))
for offset in shape["offsets"]]
return cpps, server, shape, clients_offsets
def list_method():
qs = get_json(request)
if qs.get('forum'):
key = "forum"
elif qs.get('user'):
key = "user"
else:
return json.dumps({"code": 2, "response": "No 'forum' key"}, indent=4)
key_value = qs.get(key)
since = qs.get('since', '')
order = qs.get('order', '')
limit = qs.get('limit', -1)
if key == "forum":
thread_list = get_thread_list(forum=key_value,
since=since,
order=order,
limit=limit)
else:
thread_list = get_thread_list(user=key_value,
since=since,
order=order,
limit=limit)
return json.dumps({"code": 0, "response": thread_list}, indent=4)
def details():
qs = get_json(request)
thread_id = qs.get('thread')
if not thread_id:
return json.dumps({"code": 2, "response": "No 'thread' key"}, indent=4)
thread = get_thread_by_id(thread_id)
if thread == list():
return json.dumps({"code": 1, "response": "Empty set"}, indent=4)
related_values = list()
qs_related = qs.get('related')
if type(qs_related) is list:
related_values.extend(qs_related)
elif type(qs_related) is str:
related_values.append(qs_related)
forum_related = False
user_related = False
for related_value in related_values:
if related_value == 'forum':
forum_related = True
elif related_value == 'user':
user_related = True
else:
return json.dumps({"code": 3, "response": "Wrong related value"}, indent=4)
if forum_related:
thread['forum'] = get_forum_dict(short_name=thread['forum'])
if user_related:
thread['user'] = get_user_dict(thread['user'])
return json.dumps({"code": 0, "response": thread}, indent=4)
def get_track(provider_id):
t = Track()
try:
######MediaNet######
# reqUrl = 'http://ie-api.mndigital.com?method=track.get&format=json&'\
# + urllib.urlencode({'mnetid': provider_id})\
# + '&ApiKey=%s' % MusicProvider._mnDigitalIntegrationAPIKey
reqUrl = "http://itunes.apple.com/lookup?" + urllib.urlencode({"id": provider_id})
# track_json = common.get_json(reqUrl)["Track"]
track_json = common.get_json(reqUrl)["results"][0]
t.provider_id = provider_id
t.length_seconds = 0
t.url = ""
# t.artist = track_json["Artist"]["Name"]
# t.title = track_json["Title"]
# t.length_friendly = track_json["Duration"]
# t.album = track_json["Album"]["Title"]
# t.art_url = track_json["Album"]["Images"]["Album150x150"]
t.artist = track_json["artistName"]
t.title = track_json["trackName"]
t.length_friendly = ""
t.album = track_json["collectionName"]
t.art_url = track_json["artworkUrl100"]
except Exception:
pass
return t
def list_method():
qs = get_json(request)
forum = qs.get('forum')
thread = qs.get('thread')
if not forum and not thread:
return json.dumps({
"code": 2,
"response": "No 'forum' or 'thread' key"
},
indent=4)
since = qs.get('since', '')
limit = qs.get('limit', -1)
order = qs.get('order', '')
if forum:
post_list = get_post_list(forum=forum,
since=since,
limit=limit,
order=order)
else:
post_list = get_post_list(thread=thread,
since=since,
limit=limit,
order=order)
return json.dumps({"code": 0, "response": post_list}, indent=4)
def get_data_price(date, code):
"""
일별 가격, 거래량을 수집한다.
"""
SEED = "http://m.stock.naver.com/api/item/getPriceDayList.nhn?code={code}&pageSize={page_size}&page={page}"
SEED = SEED.replace('{page_size}', '10')
rv = common.base_dict()
for page in range(1, 300):
url = SEED
url = url.replace('{code}', code)
url = url.replace('{page}', str(page))
j = common.get_json(url)
rows = j.get('result', dict()).get('list', [])
for row in rows:
if row.get('dt') == date:
rv['date'] = date
rv['price_max'] = int(row.get('hv', 0)) # 고가
rv['price_min'] = int(row.get('lv', 0)) # 저가
rv['price_start'] = int(row.get('ov', 0)) # 시가
rv['price_end'] = int(row.get('ncv', 0)) # 종가
rv['trade_cnt'] = int(row.get('aq', 0)) # 거래량
return rv
if row.get('dt') < date:
raise Exception('may be '+date+' is holiday')
raise Exception('not found data.')
def telekom_mobil_cartele(scraper_url: str):
response_body = get_json(scraper_url)
products_html = response_body['productsHtml']
soup = bs4.BeautifulSoup(products_html, 'html5lib')
feature_kw = OrderedDict([
('mobil_min_internat', 'min interna mobil'),
('mobil_min_nat mobil_sms_nat', 'minute sms naional'),
('mobil_date', 'trafic date'),
])
packages = []
extraoptiuni = soup.select('div.extraoptiuniTabWrapper')
for extraop in extraoptiuni:
detail_url = extraop.select_one('div.prepaidButtons a')['href']
abon_id = url_last_path(detail_url, scraper_url)
abon_name = extraop.select_one('div.prepaidTabTitle strong').text
abon_price = extraop.select_one('.prepaidPrice strong').text
features = extraop.select('.prepaidTabDescription .ptd_OfferRow')
features = [(feature.select_one('p').text,
feature.select_one('h4').text) for feature in features]
characteristics = extract_features(features, feature_kw, abon_name)
package = {
'name': abon_name.strip(),
'price': format_units(abon_price),
'scraper_id_hint': abon_id.strip(),
'characteristics': characteristics
}
packages.append(package)
return json.dumps({"packages": packages})
def get_room_id(self, room_id_or_name):
try:
return int(room_id_or_name)
except ValueError:
pass
for room_id, room_name in common.get_json("rooms").iteritems():
if room_name == room_id_or_name.lower():
return int(room_id)
self._error('Room "{}" not found'.format(room_id_or_name))
def list_followers_method(is_following):
qs = get_json(request)
email = qs.get('user')
if not email:
return json.dumps({"code": 2, "response": "No 'user' key"}, indent=4)
# Since part
since_id = qs.get('since_id', -1)
if since_id != -1:
since_sql = """AND User.user >= {}""".format(since_id)
else:
since_sql = ""
# Order part
order_sql = """ORDER BY User.name {}""".format(qs.get('order', 'desc'))
# Limit part
limit = qs.get('limit', -1)
if limit != -1:
try:
limit = int(limit)
except ValueError:
return json.dumps({"code": 3, "response": "Wrong limit value"}, indent=4)
if limit < 0:
return json.dumps({"code": 3, "response": "Wrong limit value"}, indent=4)
limit_sql = """LIMIT {}""".format(limit)
else:
limit_sql = ""
sql = """SELECT about, email, user, isAnonymous, name, username FROM User JOIN Follower ON """
if not is_following:
sql += """Follower.follower = User.email WHERE Follower.following"""
else:
sql += """Follower.following = User.email WHERE Follower.follower"""
sql += """ = %(email)s {since_value} {order_value} {limit_value};""".format(
since_value=since_sql, order_value=order_sql, limit_value=limit_sql)
user_list_sql = db.execute(sql, {'email': email})
if not user_list_sql:
return json.dumps({"code": 1, "response": "Empty set"}, indent=4)
user_list = list()
for user_sql in user_list_sql:
follower_email = str_to_json(user_sql[1])
user_list.append({'about': str_to_json(user_sql[0]),
'email': follower_email,
'id': str_to_json(user_sql[2]),
'isAnonymous': str_to_json(user_sql[3]),
'name': str_to_json(user_sql[4]),
'username': str_to_json(user_sql[5]),
'followers': get_followers_list(follower_email),
'following': get_following_list(follower_email),
'subscriptions': get_subscribed_threads_list(follower_email)})
return json.dumps({"code": 0, "response": user_list}, indent=4)
def update_players():
global PRO_PLAYERS
info = common.get_json(
"https://www.dota2.com/webapi/IDOTA2Fantasy/GetProPlayerInfo/v001")
if info is not None and "player_infos" in info:
PRO_PLAYERS = {}
for player in info["player_infos"]:
if "team_name" in player:
PRO_PLAYERS[player["account_id"]] = player["team_name"]
print("pro player updated:", len(PRO_PLAYERS))
def list_posts():
qs = get_json(request)
forum = qs.get('forum')
if not forum:
return json.dumps({"code": 2, "response": "No 'forum' key"}, indent=4)
# Related part
related_values = list()
qs_related = qs.get('related')
if type(qs_related) is list:
related_values.extend(qs_related)
elif type(qs_related) is str:
related_values.append(qs_related)
thread_related = False
forum_related = False
user_related = False
for related_value in related_values:
if related_value == 'thread':
thread_related = True
elif related_value == 'forum':
forum_related = True
elif related_value == 'user':
user_related = True
else:
return json.dumps({
"code": 3,
"response": "Wrong related value"
},
indent=4)
since = qs.get('since', '')
limit = qs.get('limit', -1)
sort = qs.get('sort', 'flat')
order = qs.get('order', 'desc')
post_list = get_post_list(forum=forum,
since=since,
limit=limit,
sort=sort,
order=order)
for post in post_list:
if user_related:
post['user'] = get_user_dict(post['user'])
if thread_related:
post['thread'] = get_thread_by_id(post['thread'])
if forum_related:
post['forum'] = get_forum_dict(short_name=post['forum'])
return json.dumps({"code": 0, "response": post_list}, indent=4)
def __init__(self, pro_teams):
opendota_pros = common.get_json(OPENDOTA_PROS)
if opendota_pros is None:
opendota_pros = []
self.chinese_pros = {}
for pro in opendota_pros:
country = pro.get("loccountrycode", pro.get("country_code", ""))
if country is not None and country.lower() == "cn":
self.chinese_pros[pro["account_id"]] = pro["name"]
self.pro_teams = pro_teams
def create_waybill():
#env_url = get_conf("env.conf", "default", "env_url")
api_res = "api/Order/Order/coHandle/"
uri = env_url+api_res
headers = get_json("header.json")
cookies = get_json("cookies.json")
# req = get_json("create_waybill.json")
req = get_json("data.json")
# req = change_waybill_number()
data ='from=pc&req='+ json.dumps(req)
print type(uri)
print type(cookies)
print type(headers)
r=requests.post(uri, data=data, headers=headers, cookies=cookies)
print r.text
if r.json()['errno'] != '0':
print "fail " + r.json()['errmsg']
else:
print "success"
def list_posts():
qs = get_json(request)
thread = qs.get('thread')
since = qs.get('since', '')
limit = qs.get('limit', -1)
order = qs.get('order', 'desc')
sort = qs.get('sort', 'flat')
post_list = get_post_list(thread=thread, since=since, limit=limit, sort=sort, order=order)
return json.dumps({"code": 0, "response": post_list}, indent=4)
def list_users():
qs = get_json(request)
if not qs.get('forum'):
return json.dumps({"code": 2, "response": "No 'forum' key"}, indent=4)
# Since id part
since_id = qs.get('since_id')
if since_id:
try:
since_id = int(since_id)
except ValueError:
return json.dumps({"code": 3, "response": "Wrong since_id value"}, indent=4)
since_id_sql = """AND User.user >= {}""".format(since_id)
else:
since_id_sql = ''
# Limit part
if qs.get('limit'):
limit = qs.get('limit')[0]
try:
limit = int(limit)
except ValueError:
return json.dumps({"code": 3, "response": "Wrong limit value"}, indent=4)
if limit < 0:
return json.dumps({"code": 3, "response": "Wrong limit value"}, indent=4)
limit_sql = """LIMIT {}""".format(limit)
else:
limit_sql = ''
# Order part
order = qs.get('order', 'desc')
order_sql = """ORDER BY User.name {}""".format(order)
sql = """SELECT User.user, User.email, User.name, User.username, User.isAnonymous, User.about FROM User \
WHERE User.email IN (SELECT DISTINCT user FROM Post WHERE forum = %(forum)s) {snc_sql} {ord_sql} \
{lim_sql};""".format(snc_sql=since_id_sql, lim_sql=limit_sql, ord_sql=order_sql)
user_list_sql = db.execute(sql, {'forum': qs.get('forum')})
user_list = list()
for user_sql in user_list_sql:
email = str_to_json(user_sql[1])
user_list.append({'id': str_to_json(user_sql[0]),
'email': email,
'name': str_to_json(user_sql[2]),
'username': str_to_json(user_sql[3]),
'isAnonymous': str_to_json(user_sql[4]),
'about': str_to_json(user_sql[5]),
'subscriptions': get_subscribed_threads_list(email)})
return json.dumps({"code": 0, "response": user_list}, indent=4)
def list_posts():
qs = get_json(request)
email = qs.get('user')
if not email:
return json.dumps({"code": 2, "response": "No 'user' key"}, indent=4)
since = qs.get('since', '')
limit = qs.get('limit', -1)
order = qs.get('order', 'desc')
post_list = get_post_list(user=email, since=since, limit=limit, order=order)
return json.dumps({"code": 0, "response": post_list}, indent=4)
def details():
qs = get_json(request)
email = qs.get('user')
if not email:
return json.dumps({"code": 2, "response": "No 'user' key"}, indent=4)
user = get_user_dict(email)
user['followers'] = get_followers_list(email)
user['following'] = get_following_list(email)
user['subscriptions'] = get_subscribed_threads_list(email)
return json.dumps({"code": 0, "response": user}, indent=4)
def details():
qs = get_json(request)
email = qs.get('user')
if not email:
return json.dumps({"code": 2, "response": "No 'user' key"}, indent=4)
user = get_user_dict(email)
user['followers'] = get_followers_list(email)
user['following'] = get_following_list(email)
user['subscriptions'] = get_subscribed_threads_list(email)
return json.dumps({"code": 0, "response": user}, indent=4)
def get_mastodon(config_file):
config = common.get_json(config_file)
if config == None:
print("File %s not found, exiting." % filename, file=sys.stderr)
sys.exit(1)
mastodon = Mastodon(
client_id=config["client_id"],
client_secret=config["client_secret"],
access_token=config["access_token"],
api_base_url='https://' +
config["mastodon_hostname"] # E.g., mastodon.social
)
return mastodon
def _error(self, message):
if isinstance(message, list):
code = int(message[4])
message = common.get_json("errors").get(str(code), "")
if self.logger is not None:
if message:
self.logger.error("Error #{}: {}".format(code, message))
else:
self.logger.error("Error #{}".format(code))
raise ClientError(self, message, code)
if self.logger is not None:
self.logger.error(message)
raise ClientError(self, message)
def telekom_mobil_abonamente(scraper_url: str):
response_body = get_json(scraper_url)
products_html = response_body['productsHtml']
soup = bs4.BeautifulSoup(products_html, 'html5lib')
feature_kw = OrderedDict([('mobil_min_internat', 'min internationale'),
('mobil_min_nat', 'min nationale'),
('mobil_date', 'trafic date'),
('mobil_sms_nat', 'sms nationale')])
packages = []
abonamente = soup.select('div.abonamenteTabWrapper')
for abon in abonamente:
detail_url = abon.select_one('div.abonamenteButtons a')['href']
if not urlparse(detail_url).netloc:
detail_url = urljoin(scraper_url, detail_url)
abon_name = abon.select_one('div.abonamenteTabTitle strong').text
abon_price = abon.select_one('.abonamentePrice strong').text
abon_id = urlparse(detail_url).path
abon_id = os.path.basename(os.path.normpath(abon_id))
abon_details = get_soup(detail_url)
features = abon_details.select('#tab-1 .tarrifs-table tbody tr')
characteristics = {}
for feature in features:
feature_name, feature_value = map(_extract_p_data,
feature.select('td'))
feature_words = re.sub('[^a-z]', '', feature_name.lower())
for alias, kws in feature_kw.items():
if all(kw in feature_words for kw in kws.split(' ')):
characteristics[alias] = format_units(feature_value)
break
if not all(alias in characteristics for alias in feature_kw.keys()):
missing = set(feature_kw.keys()) - set(characteristics.keys())
# raise ScraperError(f"{abon_name} missing values for [{', '.join(missing)}]")
print(f"{abon_name} missing values for [{', '.join(missing)}]")
package = {
'name': abon_name.strip(),
'price': format_units(abon_price),
'scraper_id_hint': abon_id.strip(),
'characteristics': characteristics
}
packages.append(package)
return json.dumps({"packages": packages})
def details():
qs = get_json(request)
short_name = qs.get('forum')
if not short_name:
return json.dumps({"code": 2, "response": "No 'forum' key"}, indent=4)
forum_dict = get_forum_dict(short_name=short_name)
if not forum_dict:
return json.dumps({"code": 1, "response": "Empty set"}, indent=4)
if qs.get('related', '') == 'user':
forum_dict['user'] = get_user_dict(forum_dict['user'])
return json.dumps({"code": 0, "response": forum_dict}, indent=4)
def details():
qs = get_json(request)
short_name = qs.get('forum')
if not short_name:
return json.dumps({"code": 2, "response": "No 'forum' key"}, indent=4)
forum_dict = get_forum_dict(short_name=short_name)
if not forum_dict:
return json.dumps({"code": 1, "response": "Empty set"}, indent=4)
if qs.get('related', '') == 'user':
forum_dict['user'] = get_user_dict(forum_dict['user'])
return json.dumps({"code": 0, "response": forum_dict}, indent=4)
def list_threads():
qs = get_json(request)
forum = qs.get('forum')
if not forum:
return json.dumps({"code": 2, "response": "No 'forum' key"}, indent=4)
since = qs.get('since', '')
order = qs.get('order', '')
limit = qs.get('limit', -1)
thread_list = get_thread_list(forum=forum,
since=since,
order=order,
limit=limit)
# Related part
related_values = list()
qs_related = qs.get('related')
if type(qs_related) is list:
related_values.extend(qs_related)
elif type(qs_related) is str:
related_values.append(qs_related)
forum_related = False
user_related = False
for related_value in related_values:
if related_value == 'forum':
forum_related = True
elif related_value == 'user':
user_related = True
else:
return json.dumps({
"code": 3,
"response": "Wrong related value"
},
indent=4)
for thread in thread_list:
if user_related:
thread['user'] = get_user_dict(thread['user'])
thread['user']['subscriptions'] = get_subscribed_threads_list(
thread['user']['email'])
if forum_related:
thread['forum'] = get_forum_dict(short_name=thread['forum'])
return json.dumps({"code": 0, "response": thread_list}, indent=4)
def list_posts():
qs = get_json(request)
email = qs.get('user')
if not email:
return json.dumps({"code": 2, "response": "No 'user' key"}, indent=4)
since = qs.get('since', '')
limit = qs.get('limit', -1)
order = qs.get('order', 'desc')
post_list = get_post_list(user=email,
since=since,
limit=limit,
order=order)
return json.dumps({"code": 0, "response": post_list}, indent=4)
def list_posts():
qs = get_json(request)
thread = qs.get('thread')
since = qs.get('since', '')
limit = qs.get('limit', -1)
order = qs.get('order', 'desc')
sort = qs.get('sort', 'flat')
post_list = get_post_list(thread=thread,
since=since,
limit=limit,
sort=sort,
order=order)
return json.dumps({"code": 0, "response": post_list}, indent=4)
def list_posts():
qs = get_json(request)
forum = qs.get('forum')
if not forum:
return json.dumps({"code": 2, "response": "No 'forum' key"}, indent=4)
# Related part
related_values = list()
qs_related = qs.get('related')
if type(qs_related) is list:
related_values.extend(qs_related)
elif type(qs_related) is str:
related_values.append(qs_related)
thread_related = False
forum_related = False
user_related = False
for related_value in related_values:
if related_value == 'thread':
thread_related = True
elif related_value == 'forum':
forum_related = True
elif related_value == 'user':
user_related = True
else:
return json.dumps({"code": 3, "response": "Wrong related value"}, indent=4)
since = qs.get('since', '')
limit = qs.get('limit', -1)
sort = qs.get('sort', 'flat')
order = qs.get('order', 'desc')
post_list = get_post_list(forum=forum, since=since, limit=limit, sort=sort, order=order)
for post in post_list:
if user_related:
post['user'] = get_user_dict(post['user'])
if thread_related:
post['thread'] = get_thread_by_id(post['thread'])
if forum_related:
post['forum'] = get_forum_dict(short_name=post['forum'])
return json.dumps({"code": 0, "response": post_list}, indent=4)
def get_matches():
uri = "https://api.steampowered.com/IDOTA2Match_570/GetLiveLeagueGames/v0001/?key=%s" % (STEAM_KEY)
result = common.get_json(uri)
if result is None or "result" not in result:
return "No notable teams detected in live matches.\n\n"
match_str = ""
for game in result["result"]["games"]:
if int(game["league_id"]) == LEAGUE_ID and _is_notable(game):
dire = game.get("dire_team", {}).get("team_name", "Dire")
radiant = game.get("radiant_team", {}).get("team_name", "Radiant")
match_str += "[**%s**](http://www.trackdota.com/matches/%s) \n\n" % ("%s vs. %s" % (dire, radiant), game["match_id"])
if len(match_str) == 0:
match_str = "No notable teams detected in live matches."
return match_str
def details():
qs = get_json(request)
post_id = qs.get('post')
if not post_id:
return json.dumps({"code": 2, "response": "No 'post' key"}, indent=4)
post = get_post_by_id(post_id)
if not post:
return json.dumps({"code": 1, "response": "Empty set"}, indent=4)
related_values = list()
qs_related = qs.get('related')
if type(qs_related) is list:
related_values.extend(qs_related)
elif type(qs_related) is str:
related_values.append(qs_related)
thread_related = False
forum_related = False
user_related = False
for related_value in related_values:
if related_value == 'forum':
forum_related = True
elif related_value == 'user':
user_related = True
elif related_value == 'thread':
thread_related = True
else:
return json.dumps({
"code": 3,
"response": "Wrong related value"
},
indent=4)
if thread_related:
post['thread'] = get_thread_by_id(post['thread'])
if forum_related:
post['forum'] = get_forum_dict(short_name=post['forum'])
if user_related:
post['user'] = get_user_dict(post['user'])
return json.dumps({"code": 0, "response": post}, indent=4)
def get_matches(self):
result = common.get_json(LIVE_GAMES)
if result is None or "result" not in result:
return ""
match_str = ""
live_games = [
game for game in result["result"]["games"]
if int(game["league_id"]) == LEAGUE_ID
]
print("found {} games".format(len(live_games)))
for game in live_games:
if len(live_games) < THRESHOLD or self._is_notable(game):
match_str += "[**%s**](http://www.trackdota.com/matches/%s) \n\n" % (
self._get_name(game), game["match_id"])
return match_str
def list_method():
qs = get_json(request)
if qs.get('forum'):
key = "forum"
elif qs.get('user'):
key = "user"
else:
return json.dumps({"code": 2, "response": "No 'forum' key"}, indent=4)
key_value = qs.get(key)
since = qs.get('since', '')
order = qs.get('order', '')
limit = qs.get('limit', -1)
if key == "forum":
thread_list = get_thread_list(forum=key_value, since=since, order=order, limit=limit)
else:
thread_list = get_thread_list(user=key_value, since=since, order=order, limit=limit)
return json.dumps({"code": 0, "response": thread_list}, indent=4)
def list_threads():
qs = get_json(request)
forum = qs.get('forum')
if not forum:
return json.dumps({"code": 2, "response": "No 'forum' key"}, indent=4)
since = qs.get('since', '')
order = qs.get('order', '')
limit = qs.get('limit', -1)
thread_list = get_thread_list(forum=forum, since=since, order=order, limit=limit)
# Related part
related_values = list()
qs_related = qs.get('related')
if type(qs_related) is list:
related_values.extend(qs_related)
elif type(qs_related) is str:
related_values.append(qs_related)
forum_related = False
user_related = False
for related_value in related_values:
if related_value == 'forum':
forum_related = True
elif related_value == 'user':
user_related = True
else:
return json.dumps({"code": 3, "response": "Wrong related value"}, indent=4)
for thread in thread_list:
if user_related:
thread['user'] = get_user_dict(thread['user'])
thread['user']['subscriptions'] = get_subscribed_threads_list(thread['user']['email'])
if forum_related:
thread['forum'] = get_forum_dict(short_name=thread['forum'])
return json.dumps({"code": 0, "response": thread_list}, indent=4)
def connect_clients(cpps, server, shape, clients_offsets, remember):
clients = [client for client, dx, dy in clients_offsets]
print "Logging in with {} penguin{}...".format(
len(clients), "s" if len(clients) > 1 else "")
if cpps == "cpr":
try:
import recaptcha
except ImportError:
raise common.LoginError(
"cefpython is not installed; Please install it using the following command and try again:\npip install cefpython3"
)
recaptcha.preload_tokens(len(clients))
penguins = common.get_json("penguins")
not_connected = auto_login(cpps, shape, penguins,
clients) if cpps in penguins else clients
connected = [client for client in clients if client not in not_connected]
try:
manual_login(cpps, server, shape, connected, not_connected, remember)
except common.LoginError:
logout(clients_offsets)
raise
print "All connected!"
def search(**kwargs):
######MediaNet######
qs_append = ""
for key in kwargs:
qs_append = qs_append + "&" + urllib.urlencode({key: kwargs[key]})
# reqUrl = 'http://ie-api.mndigital.com?method=Search.GetTracks&format=json'\
# + qs_append + '&page=1&pageSize=10&apiKey=%s' % MusicProvider._mnDigitalIntegrationAPIKey
reqUrl = "http://itunes.apple.com/search?media=music&limit=10&entity=musicTrack%s" % qs_append
r = common.get_json(reqUrl)
serialize_me = []
for t in r["results"]: # r["Tracks"]:
try:
dic = dict()
# MNDIGITAL
# dic["provider_id"] = t["MnetId"]
# dic["artist"] = t["Artist"]["Name"]
# dic["title"] = t["Title"]
# dic["album"] = t["Album"]["Title"]
# dic["art_url"] = t["Album"]["Images"]["Album150x150"]
# dic["length_friendly"] = t["Duration"]
dic["length_seconds"] = 0
dic["url"] = ""
# itunes
dic["provider_id"] = t["trackId"]
dic["artist"] = t["artistName"]
dic["title"] = t["trackName"]
dic["album"] = t["collectionName"]
dic["art_url"] = t["artworkUrl100"]
dic["length_friendly"] = ""
serialize_me.append(dic)
except:
pass
return serialize_me
def facebook_login(fbid, fbat):
u = User.from_fbid(fbid) #has this person logged in before?
if not u or User.current_id() != u.id: #skip if they are already logged in
profile_info = common.get_json('https://graph.facebook.com/me?fields=email,first_name,last_name&access_token=' + fbat)
if not "error" in profile_info:
if not u: #create a new user
u = User()
u.facebook_id = fbid
u.admin = False
u.photo_url = "http://graph.facebook.com/"+fbid+"/picture?type=normal"
u.created = str(datetime.now())
u.facebook_access_token = fbat
u.first_name = profile_info["first_name"]
u.last_name = profile_info["last_name"]
u.email = profile_info["email"]
u.login() #also does a save
else:
return None
return u
def _get(uri):
headers = {"Client-ID" : TWITCH_CLIENT_ID}
result = common.get_json(uri, headers=headers)
if result is None or "error" in result:
return None
return result["data"]
def main():
try:
clients_offsets = login()
except common.LoginError as e:
print e.message
sys.exit(1)
client, dx, dy = clients_offsets[0]
commands = common.Command.commands([
common.Command("help", show_help, []),
common.Command("internal", internal, []),
common.Command(
"id", get_id,
[lambda c: [penguin.name for penguin in c.penguins.itervalues()]],
common.Command.SINGLE_THREAD),
common.Command("name", name, [None], common.Command.SINGLE_THREAD),
common.Command("room", room, [common.get_json("rooms").values()]),
common.Command(
"igloo", igloo,
[lambda c: [penguin.name for penguin in c.penguins.itervalues()]]),
common.Command("color", color, [None]),
common.Command("head", head, [None]),
common.Command("face", face, [None]),
common.Command("neck", neck, [None]),
common.Command("body", body, [None]),
common.Command("hand", hand, [None]),
common.Command("feet", feet, [None]),
common.Command("pin", pin, [None]),
common.Command("background", background, [None]),
common.Command("inventory", inventory, []),
common.Command("walk", walk, [None, None]),
common.Command("dance", call_all("dance"), []),
common.Command("wave", call_all("wave"), []),
common.Command("sit", call_all("sit"),
[["s", "e", "w", "nw", "sw", "ne", "se", "n"]]),
common.Command("snowball", call_all("snowball"), [None, None]),
common.Command("say", say, []),
common.Command("joke", call_all("joke"), [None]),
common.Command("emote", call_all("emote"), [None]),
common.Command("buy", call_all("add_item"), [None],
common.Command.MULTI_THREADS),
common.Command("ai", call_all("add_item"), [None],
common.Command.MULTI_THREADS),
common.Command("coins", coins, [None]),
common.Command("ac", call_all("add_coins"), [None]),
common.Command("stamp", call_all("add_stamp"), [None],
common.Command.MULTI_THREADS),
common.Command("add_igloo", call_all("add_igloo"), [None],
common.Command.MULTI_THREADS),
common.Command("add_furniture", call_all("add_furniture"), [None],
common.Command.MULTI_THREADS),
common.Command("music", call_all("igloo_music"), [None]),
common.Command(
"buddy", buddy,
[lambda c: [penguin.name for penguin in c.penguins.itervalues()]],
common.Command.MULTI_THREADS),
common.Command(
"find", find,
[lambda c: [penguin.name for penguin in c.penguins.itervalues()]],
common.Command.MULTI_THREADS),
common.Command(
"follow", follow,
[lambda c: [penguin.name for penguin in c.penguins.itervalues()]]),
common.Command("unfollow", call_all("unfollow"), []),
common.Command("logout", logout, []),
common.Command("exit", logout, []),
common.Command("quit", logout, [])
])
while all(client.connected for client, dx, dy in clients_offsets):
try:
command, params = common.Command.read(client, commands)
except EOFError:
logout(clients_offsets)
break
command.execute(clients_offsets, params)
def main():
try:
client = login()
except common.LoginError as e:
print e
sys.exit(1)
commands = common.Command.index(
common.Command("log",
log,
common.Parameter.logging_level(required=False,
help="Logging level"),
help="Set logging level"),
common.Command("internal",
internal,
help="Get current internal room ID"),
common.Command("id",
get_id,
common.Parameter.penguin_name(required=False,
help="Penguin name"),
varargs=common.VarArgs.SINGLE_THREADED,
help="Get penguin ID"),
common.Command("name",
name,
common.Parameter.int_param("penguin_id",
required=False,
help="Penguin ID"),
varargs=common.VarArgs.SINGLE_THREADED,
help="Get penguin name"),
common.Command("room",
room,
common.Parameter("room_name",
common.get_json("rooms").values(),
required=False,
help="Room name"),
help="Go to room"),
common.Command("igloo",
igloo,
common.Parameter.penguin_name(required=False,
help="Penguin name"),
help="Go to igloo"),
common.Command("penguins",
penguins,
help="Get all penguins in current room"),
common.Command("color",
color,
common.Parameter.int_param("item_id",
required=False,
help="Color item ID"),
help="Get current color item ID / Equip color item"),
common.Command("head",
head,
common.Parameter.int_param("item_id",
required=False,
help="Head item ID"),
help="Get current head item ID / Equip head item"),
common.Command("face",
face,
common.Parameter.int_param("item_id",
required=False,
help="Face item ID"),
help="Get current face item ID / Equip face item"),
common.Command("neck",
neck,
common.Parameter.int_param("item_id",
required=False,
help="Neck item ID"),
help="Get current neck item ID / Equip neck item"),
common.Command("body",
body,
common.Parameter.int_param("item_id",
required=False,
help="Body item ID"),
help="Get current body item ID / Equip body item"),
common.Command("hand",
hand,
common.Parameter.int_param("item_id",
required=False,
help="Hand item ID"),
help="Get current hand item ID / Equip hand item"),
common.Command("feet",
feet,
common.Parameter.int_param("item_id",
required=False,
help="Feet item ID"),
help="Get current feet item ID / Equip feet item"),
common.Command("pin",
pin,
common.Parameter.int_param("item_id",
required=False,
help="Pin item ID"),
help="Get current pin item ID / Equip pin item"),
common.Command(
"background",
background,
common.Parameter.int_param("item_id",
required=False,
help="Background item ID"),
help="Get current background item ID / Equip background item"),
common.Command("clothes",
clothes,
common.Parameter.penguin_name(required=False,
help="Penguin name"),
varargs=common.VarArgs.SINGLE_THREADED,
help="Get all currently equipped item IDs"),
common.Command("inventory", inventory, help="Get current inventory"),
common.Command("buddies", buddies, help="Get all current buddies"),
common.Command("stamps",
stamps,
common.Parameter.penguin_name(required=False,
help="Penguin name"),
help="Get all currently earned stamps"),
common.Command("walk",
Client.walk,
common.Parameter.int_param("x", help="X"),
common.Parameter.int_param("y", help="Y"),
help="Walk"),
common.Command("dance", Client.dance, help="Dance"),
common.Command("wave", Client.wave, help="Wave"),
common.Command(
"sit",
Client.sit,
common.Parameter("direction",
["s", "e", "w", "nw", "sw", "ne", "se", "n"],
required=False,
help="Direction"),
help="Sit"),
common.Command("snowball",
Client.snowball,
common.Parameter.int_param("x", help="X"),
common.Parameter.int_param("y", help="Y"),
help="Throw snowball"),
common.Command("say",
say,
varargs=common.VarArgs.NORMAL,
help="Say message"),
common.Command("joke",
Client.joke,
common.Parameter.int_param("joke_id", help="Joke ID"),
help="Tell a joke"),
common.Command("emote",
Client.emote,
common.Parameter.int_param("emote_id", help="Emote ID"),
help="React an emote"),
common.Command(
"mail",
mail,
common.Parameter.other_penguin_name(help="Penguin name"),
common.Parameter.int_param("postcard_id", help="Postcard ID"),
help="Send a postcard"),
common.Command("buy",
Client.add_item,
common.Parameter.int_param("item_id", help="Item ID"),
varargs=common.VarArgs.MULTI_THREADED,
help="Buy item"),
common.Command("ai",
Client.add_item,
common.Parameter.int_param("item_id", help="Item ID"),
varargs=common.VarArgs.MULTI_THREADED,
help="Buy item"),
common.Command("coins",
coins,
common.Parameter.int_param("amount",
required=False,
help="Amount"),
help="Get current coins / Earn coins"),
common.Command("ac",
Client.add_coins,
common.Parameter.int_param("amount", help="Amount"),
help="Earn coins"),
common.Command("stamp",
Client.add_stamp,
common.Parameter.int_param("stamp_id", help="Stamp ID"),
varargs=common.VarArgs.MULTI_THREADED,
help="Earn stamp"),
common.Command("add_igloo",
Client.add_igloo,
common.Parameter.int_param("igloo_id", help="Igloo ID"),
varargs=common.VarArgs.MULTI_THREADED,
help="Buy igloo"),
common.Command("add_furniture",
Client.add_furniture,
common.Parameter.int_param("furniture_id",
help="Furniture ID"),
varargs=common.VarArgs.MULTI_THREADED,
help="Buy furniture"),
common.Command("music",
Client.igloo_music,
common.Parameter.int_param("music_id", help="Music ID"),
help="Set current igloo music"),
common.Command(
"buddy",
buddy,
common.Parameter.other_penguin_name(help="Penguin name"),
varargs=common.VarArgs.MULTI_THREADED,
help="Send buddy request"),
common.Command(
"find",
find,
common.Parameter.other_penguin_name(help="Penguin name"),
varargs=common.VarArgs.MULTI_THREADED,
help="Find buddy"),
common.Command(
"follow",
follow,
common.Parameter.other_penguin_name(help="Penguin name"),
help="Follow penguin"),
common.Command("unfollow", Client.unfollow, help="Stop following"),
common.Command("logout", logout, help="Logout"),
common.Command("exit", logout, help="Logout"),
common.Command("quit", logout, help="Logout"))
while client.connected:
try:
command, params = common.Command.read(client, commands)
except EOFError:
logout(client)
break
command.execute(client, params)
def _get(uri):
headers = {"Client-ID": TWITCH_CLIENT_ID}
result = common.get_json(uri, headers=headers)
if result is None or "error" in result:
return None
return result["data"]
def change_waybill_number():
req = get_json("create_waybill.json")
req['order_num'] = "dz17050058"
return req
def main():
try:
client = login()
except common.LoginError as e:
print e.message
sys.exit(1)
commands = common.Command.commands([
common.Command("help", show_help, []),
common.Command(
"log", log,
[["all", "debug", "info", "warning", "error", "critical"]]),
common.Command("internal", internal, []),
common.Command(
"id", get_id,
[lambda c: [penguin.name for penguin in c.penguins.itervalues()]],
common.Command.SINGLE_THREAD),
common.Command("name", name, [None], common.Command.SINGLE_THREAD),
common.Command("room", room, [common.get_json("rooms").values()]),
common.Command(
"igloo", igloo,
[lambda c: [penguin.name for penguin in c.penguins.itervalues()]]),
common.Command("penguins", penguins, []),
common.Command("color", color, [None]),
common.Command("head", head, [None]),
common.Command("face", face, [None]),
common.Command("neck", neck, [None]),
common.Command("body", body, [None]),
common.Command("hand", hand, [None]),
common.Command("feet", feet, [None]),
common.Command("pin", pin, [None]),
common.Command("background", background, [None]),
common.Command(
"clothes", clothes,
[lambda c: [penguin.name for penguin in c.penguins.itervalues()]],
common.Command.SINGLE_THREAD),
common.Command("inventory", inventory, []),
common.Command("buddies", buddies, []),
common.Command(
"stamps", stamps,
[lambda c: [penguin.name for penguin in c.penguins.itervalues()]],
common.Command.MULTI_THREADS),
common.Command("walk", Client.walk, [None, None]),
common.Command("dance", Client.dance, []),
common.Command("wave", Client.wave, []),
common.Command("sit", Client.sit,
[["s", "e", "w", "nw", "sw", "ne", "se", "n"]]),
common.Command("snowball", Client.snowball, [None, None]),
common.Command("say", say, []),
common.Command("joke", Client.joke, [None]),
common.Command("emote", Client.emote, [None]),
common.Command("mail", mail, [
lambda c: [penguin.name
for penguin in c.penguins.itervalues()], None
]),
common.Command("buy", Client.add_item, [None],
common.Command.MULTI_THREADS),
common.Command("ai", Client.add_item, [None],
common.Command.MULTI_THREADS),
common.Command("coins", coins, [None]),
common.Command("ac", Client.add_coins, [None]),
common.Command("stamp", Client.add_stamp, [None],
common.Command.MULTI_THREADS),
common.Command("add_igloo", Client.add_igloo, [None],
common.Command.MULTI_THREADS),
common.Command("add_furniture", Client.add_furniture, [None],
common.Command.MULTI_THREADS),
common.Command("music", Client.igloo_music, [None]),
common.Command(
"buddy", buddy,
[lambda c: [penguin.name for penguin in c.penguins.itervalues()]],
common.Command.MULTI_THREADS),
common.Command(
"find", find,
[lambda c: [penguin.name for penguin in c.penguins.itervalues()]],
common.Command.MULTI_THREADS),
common.Command(
"follow", follow,
[lambda c: [penguin.name for penguin in c.penguins.itervalues()]]),
common.Command("unfollow", Client.unfollow, []),
common.Command("logout", logout, []),
common.Command("exit", logout, []),
common.Command("quit", logout, [])
])
while client.connected:
try:
command, params = common.Command.read(client, commands)
except EOFError:
logout(client)
break
command.execute(client, params)
def list_users():
qs = get_json(request)
if not qs.get('forum'):
return json.dumps({"code": 2, "response": "No 'forum' key"}, indent=4)
# Since id part
since_id = qs.get('since_id')
if since_id:
try:
since_id = int(since_id)
except ValueError:
return json.dumps({
"code": 3,
"response": "Wrong since_id value"
},
indent=4)
since_id_sql = """AND User.user >= {}""".format(since_id)
else:
since_id_sql = ''
# Limit part
if qs.get('limit'):
limit = qs.get('limit')[0]
try:
limit = int(limit)
except ValueError:
return json.dumps({
"code": 3,
"response": "Wrong limit value"
},
indent=4)
if limit < 0:
return json.dumps({
"code": 3,
"response": "Wrong limit value"
},
indent=4)
limit_sql = """LIMIT {}""".format(limit)
else:
limit_sql = ''
# Order part
order = qs.get('order', 'desc')
order_sql = """ORDER BY User.name {}""".format(order)
sql = """SELECT User.user, User.email, User.name, User.username, User.isAnonymous, User.about FROM User \
WHERE User.email IN (SELECT DISTINCT user FROM Post WHERE forum = %(forum)s) {snc_sql} {ord_sql} \
{lim_sql};""".format(snc_sql=since_id_sql,
lim_sql=limit_sql,
ord_sql=order_sql)
user_list_sql = db.execute(sql, {'forum': qs.get('forum')})
user_list = list()
for user_sql in user_list_sql:
email = str_to_json(user_sql[1])
user_list.append({
'id': str_to_json(user_sql[0]),
'email': email,
'name': str_to_json(user_sql[2]),
'username': str_to_json(user_sql[3]),
'isAnonymous': str_to_json(user_sql[4]),
'about': str_to_json(user_sql[5]),
'subscriptions': get_subscribed_threads_list(email)
})
return json.dumps({"code": 0, "response": user_list}, indent=4)
def get_room_name(self, room_id):
room_id = self._require_int("room_id", room_id)
if room_id > 1000:
return "{}'s igloo".format(self.get_penguin(room_id - 1000).name)
return common.get_json("rooms").get(str(room_id),
"room {}".format(room_id))
def get_matches():
matches = common.get_json(FACEIT_API)
if matches is None:
return []
else:
return matches
def list_followers_method(is_following):
qs = get_json(request)
email = qs.get('user')
if not email:
return json.dumps({"code": 2, "response": "No 'user' key"}, indent=4)
# Since part
since_id = qs.get('since_id', -1)
if since_id != -1:
since_sql = """AND User.user >= {}""".format(since_id)
else:
since_sql = ""
# Order part
order_sql = """ORDER BY User.name {}""".format(qs.get('order', 'desc'))
# Limit part
limit = qs.get('limit', -1)
if limit != -1:
try:
limit = int(limit)
except ValueError:
return json.dumps({
"code": 3,
"response": "Wrong limit value"
},
indent=4)
if limit < 0:
return json.dumps({
"code": 3,
"response": "Wrong limit value"
},
indent=4)
limit_sql = """LIMIT {}""".format(limit)
else:
limit_sql = ""
sql = """SELECT about, email, user, isAnonymous, name, username FROM User JOIN Follower ON """
if not is_following:
sql += """Follower.follower = User.email WHERE Follower.following"""
else:
sql += """Follower.following = User.email WHERE Follower.follower"""
sql += """ = %(email)s {since_value} {order_value} {limit_value};""".format(
since_value=since_sql, order_value=order_sql, limit_value=limit_sql)
user_list_sql = db.execute(sql, {'email': email})
if not user_list_sql:
return json.dumps({"code": 1, "response": "Empty set"}, indent=4)
user_list = list()
for user_sql in user_list_sql:
follower_email = str_to_json(user_sql[1])
user_list.append({
'about':
str_to_json(user_sql[0]),
'email':
follower_email,
'id':
str_to_json(user_sql[2]),
'isAnonymous':
str_to_json(user_sql[3]),
'name':
str_to_json(user_sql[4]),
'username':
str_to_json(user_sql[5]),
'followers':
get_followers_list(follower_email),
'following':
get_following_list(follower_email),
'subscriptions':
get_subscribed_threads_list(follower_email)
})
return json.dumps({"code": 0, "response": user_list}, indent=4)
