def __init__(self, mobile, password, membername):
'''
:param mobile: 电话 int or string
:param password: 密码 string
:param membername: 成员全名 string
'''
self.mobile = mobile
self.password = password
self.membername = membername
self.dbtoken = MongoDB('Pocket48', 'dbtoken')
self.headers = {
'Host': 'pocketapi.48.cn',
'accept': '*/*',
'Accept-Language': 'zh-Hans-CN;q=1',
'User-Agent':
'PocketFans201807/6.0.0 (iPhone; iOS 12.2; Scale/2.00)',
'Accept-Encoding': 'gzip, deflate',
'appInfo': '{"vendor":"apple","deviceId":"0", \
"appVersion":"6.0.0","appBuild":"190409", \
"osVersion":"12.2.0","osType":"ios", \
"deviceName":"iphone","os":"ios"}',
'Content-Type': 'application/json;charset=utf-8',
'Connection': 'keep-alive',
}
dic_data = self.searchroom
self.ownerId, self.roomId = dic_data['ownerId'], dic_data['roomId']
def __init__(self):
'''
:param mobile: 电话 int or string
:param password: 密码 string
:param membername: 成员列表 string
'''
self.color = ['#FEEEED', '##CCFFFF', '#CCFFCC', '#E0FFFF', '#F0FFF0', '#F0F8FF', '#FFF0F5', '#FFFAFA',
'#F0FFFF', '#FFF5EE', '#E6E6FA', '#FFC0CB']
self.dbtoken = MongoDB('Pocket48', 'dbtoken')
self.key = b'1O8zgva3PlT_Evikm61A97wsWZ0JlTGSNEiRc0S7rCY='
self.Fernet = Fernet(self.key)
self.source_url = 'https://source.48.cn'
self.headers = {
'Host': 'pocketapi.48.cn',
'accept': '*/*',
'Accept-Language': 'zh-Hans-CN;q=1',
'User-Agent': 'PocketFans201807/6.0.0 (iPhone; iOS 12.2; Scale/2.00)',
'Accept-Encoding': 'gzip, deflate',
'appInfo': '{"vendor":"apple","deviceId":"0", \
"appVersion":"6.0.0","appBuild":"190409", \
"osVersion":"12.2.0","osType":"ios", \
"deviceName":"iphone","os":"ios"}',
'Content-Type': 'application/json;charset=utf-8',
'Connection': 'keep-alive',
}
def __init__(self, mobile, password, membername):
super().__init__(mobile, password, membername)
CQclient.__init__(self)
SCHEDULE.__init__(self)
self.dbchat = MongoDB('Poket48', str(self.roomId))
self.msgType = dict(TEXT=self.text, VIDEO=self.video,
IMAGE=self.image, AUDIO=self.audio)
def run(self):
ltp = Ltp.get_object2()
ltp.load_dict(ALL_DICT_PATH)
analyzer = EmotionAnalysis(ltp)
topic_collection = MongoDB.get_client()[MONGO_DB]['topic']
comment_collection = MongoDB.get_client()[MONGO_DB]['comment']
topic = topic_collection.find_one({'_id': ObjectId(self.id)})
count1 = 0
for weibo_id in topic['text_id_list']:
if MONGO_DB == 'weibo':
comments = comment_collection.find({'id': weibo_id})
else:
data = MongoDB.get_client()[MONGO_DB]['mid'].find_one(
{'id': weibo_id})
if not data:
continue
comments = comment_collection.find({'mid': data['mid']})
for data in comments:
if 'score' in data and data['score']:
continue
content = data['content'].strip()
if not content:
continue
data['score'] = analyzer.sent_sentiment_score(
data['content'].strip())
comment_collection.update_one({'_id': data['_id']},
{'$set': data}, True)
count1 += 1
self.count.emit(round((count1 / topic['text_num']) * 100))
self.stop.emit(True)
def sendPayload(self, payload):
try:
print("App 1 connecting on port 8080 using TLS")
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
ssl_sock = ssl.wrap_socket(s,
ca_certs="team1Server.crt",
cert_reqs=ssl.CERT_REQUIRED)
ssl_sock.connect(('localhost', 8080))
ssl_sock.sendall(payload)
print("JSON payload sent to _______ using TLS")
ssl_sock.close()
print(ssl_sock.cipher())
# Logging
MongoDB.mongoInstance("Test", "Sent to app2")
return True
except Exception as e:
client = MongoClient('localhost', 27017)
db = client.Team1
collection = db.logs
print(e)
#Logging
MongoDB.mongoInstance("Test", "Failed to send to app2")
return False
def searching_mongo(mongo_id):
"""
Search Mongo for a specific _id returned from elastic and returns the result.
"""
mongodb = MongoDB(host=os.getenv("MONGO_URI"))
mongo_result = mongodb.search_mongo(mongo_id)
return mongo_result
def readAndDecodeJSON(self):
try:
with urllib.request.urlopen(self.url + self.param) as payload:
jsonPayload = json.loads(payload.read().decode('utf-8'))
# Log to App5 Success
MongoDB.mongoInstance("Test", "Got Payload")
return jsonPayload
except Exception as e:
print("error: %s" % e)
# Log to App5 Failure
MongoDB.mongoInstance("Test", "Failed to get Payload")
def index():
logging.info("Received request for index page.")
players = {}
for key, value in configuration['user_ids'].items():
client = MongoDB(configuration, key)
points = 0
time = 0
work_time = 0
total_activities = 0
for habit in client.get_habits():
points = points + habit['points']
total_activities = total_activities + habit['total']
if habit['name'].lower() == 'work':
work_time = work_time + habit['time']
continue
time = time + habit['time']
for daily in client.get_dailies():
points = points + daily['points']
time = time + daily['time']
total_activities = total_activities + daily['total']
logging.debug("Calculated time to be: " + str(time))
total_time = work_time + time
# Gets the total number of days
days = divmod(time, 1440)
hours = divmod(days[1], 60)
minutes = hours[1]
work_days = divmod(work_time, 1440)
work_hours = divmod(work_days[1], 60)
players[key] = {"Username": value['username'], "Total Points": points, "Hobby Days": days[0], "Hobby Hours": hours[0], "Hobby Minutes": minutes, "Work Days": work_days[0], "Work Hours": work_hours[0], "Total Activities": total_activities}
logging.debug(pprint(players))
return render_template('index.html', players=players)
def call_crawler_Driver(query, c):
ecommerce = ['flipkart', 'paytm', 'amazon']
result = MongoDB.getData(query)
for i in result:
if i['url'].find('amazon') != -1:
ecommerce.remove('amazon')
elif i['url'].find('paytm') != -1:
ecommerce.remove('paytm')
elif i['url'].find('flipkart') != -1:
ecommerce.remove('flipkart')
print('Crawling for these e-commerce: '),
print(ecommerce)
if len(ecommerce) > 0 and c == 0:
my_process(ecommerce, query)
call_crawler_Driver(query, 1)
elif c == 1 or len(ecommerce) == 0:
for i in result:
print(i)
show_data.show(result)
class API:
def __init__(self):
self.rankdb = MongoDB('Javdb', 'rankdb')
@app.route('/<identity>', methods=['GET', 'POST'])
def identity(self, identity=None):
return str(self.rankdb.find({'identity': identity}))
def __init__(self):
""" METHOD INITIALIZES LOGGER, MONGO, GMAIL, EXCEPTION HOOK, ECT.
"""
# INSTANTIATE LOGGER
self.logger = Logger()
# CONNECT TO MONGO
self.mongo = MongoDB(self.logger)
# CONNECT TO GMAIL API
self.gmail = Gmail(self.mongo, self.logger)
# SET GMAIL AND MONGO ATTRIBUTE FOR LOGGER
self.logger.gmail = self.gmail
self.logger.mongo = self.mongo
self.traders = {}
self.accounts = []
self.sim_trader = SimTrader(self.mongo)
self.not_connected = []
def savePayload(self,payload):
try:
with open('jsonPayload.txt', 'w') as outFile:
outFile.write(json.dumps(payload))
#Logging
MongoDB.mongoInstance("test","Saved Payload")
return True
except Exception as e:
print("Error: %s" % e)
#Logging
MongoDB.mongoInstance("test","Failed to Save Payload")
return False
def __init__(self, mainwindow):
super().__init__()
client = MongoDB.get_client()
db = client[MONGO_DB]
self.collection = db['topic']
self.weibo_collection = db['weibo']
self.setupUi(mainwindow)
self.id = None
def __init__(self, pro_id):
'''
:param pro_id: 众筹项目编号 int
'''
super().__init__()
SCHEDULE.__init__(self)
self.pro_id = pro_id
self.url = 'https://zhongchou.modian.com/item/' + str(self.pro_id)
self.dbmodian = MongoDB('modian', str(self.pro_id))
# self.dbmodian.remove({})
self.headers = {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36'
}
self.url = 'https://m.modian.com/project/' + str(self.pro_id)
self.dataType = dict(orders=ORDERS_MESSAGE,
rank=RANK_MESSAGE,
detail=DETAIL_MESSAGE)
def create_emotion_pie(self):
"""
绘制情感倾向饼图
:return:
"""
figure = plt.figure(figsize=(6, 4), dpi=80, frameon=True)
data = [0, 0, 0]
topic = self.topic_collection.find_one({'_id': ObjectId(self.id)})
# if 'score' not in topic or not topic['score']:
for weibo_id in topic['text_id_list']:
if MONGO_DB == 'weibo':
comments = self.comment_collection.find({'id': weibo_id})
else:
mid = MongoDB.get_client()[MONGO_DB]['mid'].find_one(
{'id': weibo_id})
if not mid:
continue
comments = self.comment_collection.find({'mid': mid['mid']})
for comment in comments:
if not comment['content'].strip():
continue
if 'score' not in comment or not comment['score']:
continue
score = comment['score']
if score[0] - score[1] > 2:
data[0] += 1
elif score[0] - score[1] < 0:
data[1] += 1
else:
data[2] += 1
topic['score'] = data
self.topic_collection.update_one({'_id': topic['_id']},
{'$set': topic}, True)
# else:
# data = topic['score']
try:
data[0] += data[2] * 0.3 * data[0] / (data[0] + data[1])
data[1] += data[2] * 0.3 * data[1] / (data[0] + data[1])
data[2] *= 0.7
except:
pass
labels = ['积极', '消极', '其它']
colors = ['r', 'b', 'y']
fig1 = plt.subplot(1, 1, 1)
plt.xlim(0, 4)
plt.ylim(0, 4)
fig1.pie(x=data, labels=labels, colors=colors, autopct='%.1f%%')
fig1.set_title('话题舆情倾向度')
plt.xticks(())
plt.yticks(())
return figure
def __init__(self, mainwindow):
super().__init__()
client = MongoDB.get_client()
db = client[MONGO_DB]
self.collection = db['topic']
self.weibo_collection = db['weibo']
self.ltp = Ltp.get_object()
self.area = ''
self.keys = []
self.setupUi(mainwindow)
def __init__(self, id, parent=None):
super().__init__(parent)
self.parent = parent
self.ltp = Ltp.get_object()
client = MongoDB.get_client()
db = client[MONGO_DB]
self.topic_collection = db['topic']
self.weibo_collection = db['weibo']
self.comment_collection = db['comment']
self.id = id
self.setupUi()
def __init__(self, parent=None):
super().__init__(parent)
self.parent = parent
db = MongoDB.get_client()[MONGO_DB]
self.topic_collection = db['topic']
self.weibo_collection = db['weibo']
self.attention = False
self.emotion = False
self._time = False
self.ltp = Ltp.get_object()
self.setupUi()
def scraper_loop():
db = MongoDB(host=os.environ['DB_PORT_27017_TCP_ADDR'], port=27017)
# db = MongoDB(host='localhost', port=27017) # local flag
s = Scraper()
print("Starting scraper...")
while True:
try:
s.pull(n_pages=NPAGE) # number of altcound pages to scrape)
db.update(s)
update_static_page()
except KeyboardInterrupt:
print("Exiting....")
sys.exit(1)
except Exception as exc:
print("Error with the scraping:", sys.exc_info()[0])
traceback.print_exc()
else:
print("{}: Successfully finished scraping".format(s.timestamp))
time.sleep(SCRAPE_FREQ * 60)
def webhook():
habitica_update = request.json # type: dict
logging.info("Received update")
if habitica_update['direction'] != "down":
client = MongoDB(configuration, habitica_update['task']['userId'])
if habitica_update['task']['type'] == "daily":
logging.info("Received a daily task and we are now processing.")
client.update_habit(habitica_update['task'], False)
elif habitica_update['task']['type'] == "habit":
logging.info("Received a habit and we are now processing.")
client.update_habit(habitica_update['task'], True)
else:
logging.error("Received an unrecognized task type.")
es = Elasticsearch([{'host': configuration['elasticsearch_server'], 'port': configuration['elasticsearch_port']}])
logging.info("Pushing event to Elasticsearch")
res = es.index(index="test-index", doc_type='_doc', body=habitica_update)
logging.debug("Elasticsearch returned " + str(res['result']))
return '', 200
def __init__(self, parent, id):
super().__init__(parent)
self.id = id
client = MongoDB.get_client()
db = client[MONGO_DB]
self.topic_collection = db['topic']
self.weibo_collection = db['weibo']
self.comment_collection = db['comment']
self.horizontalLayout = QtWidgets.QHBoxLayout(self)
self.create_figure(self.create_heatline())
self.create_figure(self.create_emotion_pie())
plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False
def testReadAndDecodeJSON(self):
db = MongoDB()
payloadToCompare = {
"userId":
1,
"id":
1,
"title":
"sunt aut facere repellat provident occaecati excepturi optio reprehenderit",
"body":
"quia et suscipit\nsuscipit recusandae consequuntur expedita et cum\nreprehenderit molestiae ut ut quas totam\nnostrum rerum est autem sunt rem eveniet architecto"
}
self.assertEqual(PayloadRetriever().readAndDecodeJSON(db),
payloadToCompare)
def tls_connection(self):
try:
# Receive the secure payload using TLS
print("App 2 connecting on port 8080 using SSL (TLS)")
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s2.connect(('localhost', 9999))
s2.send(b'App 2 got connection from app 1')
s2.close()
ssl_socket = ssl.wrap_socket(s,
server_side=True,
certfile="team1tls.crt",
keyfile="team1tls.key")
# Listen on port 8080
ssl_socket.bind(('localhost', 8080))
ssl_socket.listen(5)
mongoDB = MongoDB()
condition = True
while condition:
print("Accept connections from outside")
(clientSocket, address) = ssl_socket.accept()
#print(clientSocket.recv(1024))
dataRecieved = clientSocket.recv(1024)
print(dataRecieved)
mongoDB.mongoInstance("Test", "Got Connection")
#if(clientSocket.recv(1024) is None):
if (dataRecieved is None):
print("data is none")
#condition = False
#print(dataRecieved)
#hash_payload_hmac(dataRecieved)
else:
condition = False
return dataRecieved
except Exception as e:
print(e)
def __init__(self, parent=None):
"""
初始化
:param parent: 调用浏览器引擎的父窗口
"""
super().__init__(parent)
self.superwindow = parent
self.setupUi()
self.client = MongoDB.get_client()
self.db = MongoDB.get_client()[MONGO_DB]
self.weibo = self.db['weibo']
self.topic = self.db['topic']
self.comment = self.db['comment']
self.user = self.db['user']
self.repost = self.db['repost']
self.flag = None
self.maparea = None
self.maphtml = None
self.barhtml = None
self.wordcloudhtml = None
self.graphtml = None
self.geohtml = None
self.id = None
def run(self):
ltp = Ltp.get_object()
tdt = Tdt.get_object()
model = Text2Vec.get_object()
weibo_collection = MongoDB.get_client()[MONGO_DB]['weibo']
count1 = 0
weibo_set = weibo_collection.find().sort('posted_at',
pymongo.ASCENDING)
for weibo in weibo_set:
tdt.single_pass(weibo, 'topic', ltp, model)
weibo_collection.update_one({'_id': weibo['_id']},
{'$set': weibo}, True)
count1 += 1
self.count.emit(count1)
self.stop.emit(True)
def read_update(pref):
db = MongoDB()
print(request.method)
if request.method == 'POST':
value = request.form['value']
db.update_db(pref, value)
user_info = db.get_info(pref)
return render_template('output.html', pref=pref, user=user_info)
else:
user_info = db.get_info(pref)
return render_template('output.html', pref=pref, user=user_info)
def main():
#timestamp = 1545730073
#startDtObject = date.fromtimestamp(timestamp)
#client = MongoClient('localhost', 27017)
#db = client.Team1
#collection = db.logs
startTime = time.time()
mongoDB = MongoDB()
# print(startDtObject)
print("Retrieving JSON payload from source.")
payload = PayloadRetriever().readAndDecodeJSON(mongoDB)
print("Sending payload to App2.")
SendPayload().sendPayload(payload, mongoDB)
print("Saving payload to text file.")
PayloadSaver().savePayload(payload, mongoDB)
RabbitReciever.getMessage(startTime)
endTime = time.time()
print(str(endTime - startTime) + " seconds")
def __init__(self):
self.cookie = ''
self.url = 'http://www.p26y.com/cn/'
self.torrent_url = 'https://www.bturl.at/search/'
self.video = {}
self.headers = {'Cookie': self.cookie,
'Host': 'www.p26y.com',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,'
'*/*;q=0.8',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, '
'like Gecko) Chrome/68.0.3440.75 Safari/537.36',
'If-None-Match': 'W/"3a18de34b39581014d560cc3522a80b7"',
'If-Modified-Since': 'Tue, 22 Jan 2019 00:00:00 GMT',
'Connection': 'keep-alive',
'Cache-Control': 'max-age=0',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Upgrade-Insecure-Requests': '1',
'Accept-Encoding': 'gzip, deflate'
}
self.allgirls = MongoDB('Javdb', 'girlsname') # 所有girl名称—>页面编码键值对
self.rankdb = MongoDB('Javdb', 'rankdb') # 最受受欢迎作品榜单
self.girlsindexdb = MongoDB('Javdb', 'girlsindexdb') # 单个girl的所有作品db
'''
author: Utpal Das
'''
import time
import stripe
from flask import Flask, request, jsonify
from flask_cors import CORS, cross_origin
from mongo import MongoDB
app = Flask(__name__)
data = dict()
mongo = MongoDB()
stripe.api_key = '<stripe secret key>'
@app.route('/',methods = ['POST'])
@cross_origin(origin='localhost',headers=['Content- Type','Authorization'])
def home():
if request.method == 'POST':
data = mongo.insertorupdate_transaction(request.data)
return data
@app.route('/<userid>/<date>',methods = ['GET'])
@cross_origin(origin='localhost',headers=['Content- Type','Authorization'])
def get_current_time(userid,date):
data = mongo.read_transaction(userid,date)
return data
@app.route('/create-checkout-session/<item>/<amount>', methods=['POST'])
@cross_origin(origin='localhost',headers=['Content- Type','Authorization'])
class API(object):
'''
口袋48 api
: func searchroom: 查找指定成员房间信息
: func login: 登录
: func chatroom: 查询房间消息
: func livedetail: 获取直播详情
'''
def __init__(self, mobile, password, membername):
'''
:param mobile: 电话 int or string
:param password: 密码 string
:param membername: 成员全名 string
'''
self.mobile = mobile
self.password = password
self.membername = membername
self.dbtoken = MongoDB('Pocket48', 'dbtoken')
self.headers = {
'Host': 'pocketapi.48.cn',
'accept': '*/*',
'Accept-Language': 'zh-Hans-CN;q=1',
'User-Agent':
'PocketFans201807/6.0.0 (iPhone; iOS 12.2; Scale/2.00)',
'Accept-Encoding': 'gzip, deflate',
'appInfo': '{"vendor":"apple","deviceId":"0", \
"appVersion":"6.0.0","appBuild":"190409", \
"osVersion":"12.2.0","osType":"ios", \
"deviceName":"iphone","os":"ios"}',
'Content-Type': 'application/json;charset=utf-8',
'Connection': 'keep-alive',
}
dic_data = self.searchroom
self.ownerId, self.roomId = dic_data['ownerId'], dic_data['roomId']
@property
def _token(self):
'''
获取token,超过1天则重新登录
: return: token string
'''
hastoken = list(self.dbtoken.find({'mobile': self.mobile}))
if not hastoken:
token = self.login
self.dbtoken.update(dict(timestamp=self._nowtime,
token=token,
mobile=self.mobile),
upsert=True)
elif self._nowtime - hastoken[0]['timestamp'] > 3600 * 24:
self.dbtoken.remove({'mobile': self.mobile})
token = self.login
self.dbtoken.update(dict(timestamp=self._nowtime,
token=token,
mobile=self.mobile),
upsert=True)
return list(self.dbtoken.find({'mobile': self.mobile}))[0]['token']
@property
def _nowtime(self):
'''
当前时间戳
: return: 当前时间戳 float
'''
return time.time()
@property
def searchroom(self):
'''
获取成员房间信息
: param membername: 成员全名
: return: 成员房间信息 json roomName 房间名, ownerName 成员名, roomId 房间名, ownerId 成员编号
'''
url = 'https://pocketapi.48.cn/im/api/v1/im/search'
data = {'name': self.membername}
try:
response = requests.post(url,
json=data,
headers=self.headers,
verify=False).json()['content']['data'][0]
return dict(roomName=response['targetName'],
ownerName=response['ownerName'],
ownerId=response['ownerId'],
roomId=response['targetId'])
except Exception as e:
raise e
@property
def login(self):
'''
登录
: param mobile: 手机号
: param password: 密码
: return token: 登录token string
'''
try:
url = 'https://pocketapi.48.cn/user/api/v1/login/app/mobile'
data = {
"pwd": str(self.password),
"mobile": str(self.mobile),
}
res = requests.post(url,
headers=self.headers,
json=data,
verify=False).json()
return res['content']['token']
except Exception as e:
raise e
@property
def chatroom(self):
'''
获取成员房间消息
: param roomId: 房间编号
: param ownerId: 成员编号
: return: 发言信息 json
'''
url = 'https://pocketapi.48.cn/im/api/v1/chatroom/msg/list/homeowner'
headers = self.headers
headers.update({'token': self._token})
data = dict(needTop1Msg=False,
roomId=str(self.roomId),
ownerId=str(self.ownerId))
try:
res = requests.post(url, headers=headers,
json=data).json()['content']['message']
return res
except Exception as e:
raise e
def livedetail(self, liveId):
'''
获取直播详情
:param liveId: 直播编号 int
:return: playStreamPath string, response
'''
url = "https://pocketapi.48.cn/live/api/v1/live/getLiveOne"
form = {"liveId": str(liveId)}
try:
response = requests.post(url, json=form,
headers=self.headers).json()
if response['status'] == 200:
playStreamPath = response['content']['playStreamPath']
return playStreamPath, response
else:
return False, False
except BaseException:
return False, False
