def __init__(self, mobile, password, membername): ''' :param mobile: 电话 int or string :param password: 密码 string :param membername: 成员全名 string ''' self.mobile = mobile self.password = password self.membername = membername self.dbtoken = MongoDB('Pocket48', 'dbtoken') self.headers = { 'Host': 'pocketapi.48.cn', 'accept': '*/*', 'Accept-Language': 'zh-Hans-CN;q=1', 'User-Agent': 'PocketFans201807/6.0.0 (iPhone; iOS 12.2; Scale/2.00)', 'Accept-Encoding': 'gzip, deflate', 'appInfo': '{"vendor":"apple","deviceId":"0", \ "appVersion":"6.0.0","appBuild":"190409", \ "osVersion":"12.2.0","osType":"ios", \ "deviceName":"iphone","os":"ios"}', 'Content-Type': 'application/json;charset=utf-8', 'Connection': 'keep-alive', } dic_data = self.searchroom self.ownerId, self.roomId = dic_data['ownerId'], dic_data['roomId']
def __init__(self): ''' :param mobile: 电话 int or string :param password: 密码 string :param membername: 成员列表 string ''' self.color = ['#FEEEED', '##CCFFFF', '#CCFFCC', '#E0FFFF', '#F0FFF0', '#F0F8FF', '#FFF0F5', '#FFFAFA', '#F0FFFF', '#FFF5EE', '#E6E6FA', '#FFC0CB'] self.dbtoken = MongoDB('Pocket48', 'dbtoken') self.key = b'1O8zgva3PlT_Evikm61A97wsWZ0JlTGSNEiRc0S7rCY=' self.Fernet = Fernet(self.key) self.source_url = 'https://source.48.cn' self.headers = { 'Host': 'pocketapi.48.cn', 'accept': '*/*', 'Accept-Language': 'zh-Hans-CN;q=1', 'User-Agent': 'PocketFans201807/6.0.0 (iPhone; iOS 12.2; Scale/2.00)', 'Accept-Encoding': 'gzip, deflate', 'appInfo': '{"vendor":"apple","deviceId":"0", \ "appVersion":"6.0.0","appBuild":"190409", \ "osVersion":"12.2.0","osType":"ios", \ "deviceName":"iphone","os":"ios"}', 'Content-Type': 'application/json;charset=utf-8', 'Connection': 'keep-alive', }
def __init__(self, mobile, password, membername): super().__init__(mobile, password, membername) CQclient.__init__(self) SCHEDULE.__init__(self) self.dbchat = MongoDB('Poket48', str(self.roomId)) self.msgType = dict(TEXT=self.text, VIDEO=self.video, IMAGE=self.image, AUDIO=self.audio)
def run(self): ltp = Ltp.get_object2() ltp.load_dict(ALL_DICT_PATH) analyzer = EmotionAnalysis(ltp) topic_collection = MongoDB.get_client()[MONGO_DB]['topic'] comment_collection = MongoDB.get_client()[MONGO_DB]['comment'] topic = topic_collection.find_one({'_id': ObjectId(self.id)}) count1 = 0 for weibo_id in topic['text_id_list']: if MONGO_DB == 'weibo': comments = comment_collection.find({'id': weibo_id}) else: data = MongoDB.get_client()[MONGO_DB]['mid'].find_one( {'id': weibo_id}) if not data: continue comments = comment_collection.find({'mid': data['mid']}) for data in comments: if 'score' in data and data['score']: continue content = data['content'].strip() if not content: continue data['score'] = analyzer.sent_sentiment_score( data['content'].strip()) comment_collection.update_one({'_id': data['_id']}, {'$set': data}, True) count1 += 1 self.count.emit(round((count1 / topic['text_num']) * 100)) self.stop.emit(True)
def sendPayload(self, payload): try: print("App 1 connecting on port 8080 using TLS") s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) ssl_sock = ssl.wrap_socket(s, ca_certs="team1Server.crt", cert_reqs=ssl.CERT_REQUIRED) ssl_sock.connect(('localhost', 8080)) ssl_sock.sendall(payload) print("JSON payload sent to _______ using TLS") ssl_sock.close() print(ssl_sock.cipher()) # Logging MongoDB.mongoInstance("Test", "Sent to app2") return True except Exception as e: client = MongoClient('localhost', 27017) db = client.Team1 collection = db.logs print(e) #Logging MongoDB.mongoInstance("Test", "Failed to send to app2") return False
def searching_mongo(mongo_id): """ Search Mongo for a specific _id returned from elastic and returns the result. """ mongodb = MongoDB(host=os.getenv("MONGO_URI")) mongo_result = mongodb.search_mongo(mongo_id) return mongo_result
def readAndDecodeJSON(self): try: with urllib.request.urlopen(self.url + self.param) as payload: jsonPayload = json.loads(payload.read().decode('utf-8')) # Log to App5 Success MongoDB.mongoInstance("Test", "Got Payload") return jsonPayload except Exception as e: print("error: %s" % e) # Log to App5 Failure MongoDB.mongoInstance("Test", "Failed to get Payload")
def index(): logging.info("Received request for index page.") players = {} for key, value in configuration['user_ids'].items(): client = MongoDB(configuration, key) points = 0 time = 0 work_time = 0 total_activities = 0 for habit in client.get_habits(): points = points + habit['points'] total_activities = total_activities + habit['total'] if habit['name'].lower() == 'work': work_time = work_time + habit['time'] continue time = time + habit['time'] for daily in client.get_dailies(): points = points + daily['points'] time = time + daily['time'] total_activities = total_activities + daily['total'] logging.debug("Calculated time to be: " + str(time)) total_time = work_time + time # Gets the total number of days days = divmod(time, 1440) hours = divmod(days[1], 60) minutes = hours[1] work_days = divmod(work_time, 1440) work_hours = divmod(work_days[1], 60) players[key] = {"Username": value['username'], "Total Points": points, "Hobby Days": days[0], "Hobby Hours": hours[0], "Hobby Minutes": minutes, "Work Days": work_days[0], "Work Hours": work_hours[0], "Total Activities": total_activities} logging.debug(pprint(players)) return render_template('index.html', players=players)
def call_crawler_Driver(query, c): ecommerce = ['flipkart', 'paytm', 'amazon'] result = MongoDB.getData(query) for i in result: if i['url'].find('amazon') != -1: ecommerce.remove('amazon') elif i['url'].find('paytm') != -1: ecommerce.remove('paytm') elif i['url'].find('flipkart') != -1: ecommerce.remove('flipkart') print('Crawling for these e-commerce: '), print(ecommerce) if len(ecommerce) > 0 and c == 0: my_process(ecommerce, query) call_crawler_Driver(query, 1) elif c == 1 or len(ecommerce) == 0: for i in result: print(i) show_data.show(result)
class API: def __init__(self): self.rankdb = MongoDB('Javdb', 'rankdb') @app.route('/<identity>', methods=['GET', 'POST']) def identity(self, identity=None): return str(self.rankdb.find({'identity': identity}))
def __init__(self): """ METHOD INITIALIZES LOGGER, MONGO, GMAIL, EXCEPTION HOOK, ECT. """ # INSTANTIATE LOGGER self.logger = Logger() # CONNECT TO MONGO self.mongo = MongoDB(self.logger) # CONNECT TO GMAIL API self.gmail = Gmail(self.mongo, self.logger) # SET GMAIL AND MONGO ATTRIBUTE FOR LOGGER self.logger.gmail = self.gmail self.logger.mongo = self.mongo self.traders = {} self.accounts = [] self.sim_trader = SimTrader(self.mongo) self.not_connected = []
def savePayload(self,payload): try: with open('jsonPayload.txt', 'w') as outFile: outFile.write(json.dumps(payload)) #Logging MongoDB.mongoInstance("test","Saved Payload") return True except Exception as e: print("Error: %s" % e) #Logging MongoDB.mongoInstance("test","Failed to Save Payload") return False
def __init__(self, mainwindow): super().__init__() client = MongoDB.get_client() db = client[MONGO_DB] self.collection = db['topic'] self.weibo_collection = db['weibo'] self.setupUi(mainwindow) self.id = None
def __init__(self, pro_id): ''' :param pro_id: 众筹项目编号 int ''' super().__init__() SCHEDULE.__init__(self) self.pro_id = pro_id self.url = 'https://zhongchou.modian.com/item/' + str(self.pro_id) self.dbmodian = MongoDB('modian', str(self.pro_id)) # self.dbmodian.remove({}) self.headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36' } self.url = 'https://m.modian.com/project/' + str(self.pro_id) self.dataType = dict(orders=ORDERS_MESSAGE, rank=RANK_MESSAGE, detail=DETAIL_MESSAGE)
def create_emotion_pie(self): """ 绘制情感倾向饼图 :return: """ figure = plt.figure(figsize=(6, 4), dpi=80, frameon=True) data = [0, 0, 0] topic = self.topic_collection.find_one({'_id': ObjectId(self.id)}) # if 'score' not in topic or not topic['score']: for weibo_id in topic['text_id_list']: if MONGO_DB == 'weibo': comments = self.comment_collection.find({'id': weibo_id}) else: mid = MongoDB.get_client()[MONGO_DB]['mid'].find_one( {'id': weibo_id}) if not mid: continue comments = self.comment_collection.find({'mid': mid['mid']}) for comment in comments: if not comment['content'].strip(): continue if 'score' not in comment or not comment['score']: continue score = comment['score'] if score[0] - score[1] > 2: data[0] += 1 elif score[0] - score[1] < 0: data[1] += 1 else: data[2] += 1 topic['score'] = data self.topic_collection.update_one({'_id': topic['_id']}, {'$set': topic}, True) # else: # data = topic['score'] try: data[0] += data[2] * 0.3 * data[0] / (data[0] + data[1]) data[1] += data[2] * 0.3 * data[1] / (data[0] + data[1]) data[2] *= 0.7 except: pass labels = ['积极', '消极', '其它'] colors = ['r', 'b', 'y'] fig1 = plt.subplot(1, 1, 1) plt.xlim(0, 4) plt.ylim(0, 4) fig1.pie(x=data, labels=labels, colors=colors, autopct='%.1f%%') fig1.set_title('话题舆情倾向度') plt.xticks(()) plt.yticks(()) return figure
def __init__(self, mainwindow): super().__init__() client = MongoDB.get_client() db = client[MONGO_DB] self.collection = db['topic'] self.weibo_collection = db['weibo'] self.ltp = Ltp.get_object() self.area = '' self.keys = [] self.setupUi(mainwindow)
def __init__(self, id, parent=None): super().__init__(parent) self.parent = parent self.ltp = Ltp.get_object() client = MongoDB.get_client() db = client[MONGO_DB] self.topic_collection = db['topic'] self.weibo_collection = db['weibo'] self.comment_collection = db['comment'] self.id = id self.setupUi()
def __init__(self, parent=None): super().__init__(parent) self.parent = parent db = MongoDB.get_client()[MONGO_DB] self.topic_collection = db['topic'] self.weibo_collection = db['weibo'] self.attention = False self.emotion = False self._time = False self.ltp = Ltp.get_object() self.setupUi()
def scraper_loop(): db = MongoDB(host=os.environ['DB_PORT_27017_TCP_ADDR'], port=27017) # db = MongoDB(host='localhost', port=27017) # local flag s = Scraper() print("Starting scraper...") while True: try: s.pull(n_pages=NPAGE) # number of altcound pages to scrape) db.update(s) update_static_page() except KeyboardInterrupt: print("Exiting....") sys.exit(1) except Exception as exc: print("Error with the scraping:", sys.exc_info()[0]) traceback.print_exc() else: print("{}: Successfully finished scraping".format(s.timestamp)) time.sleep(SCRAPE_FREQ * 60)
def webhook(): habitica_update = request.json # type: dict logging.info("Received update") if habitica_update['direction'] != "down": client = MongoDB(configuration, habitica_update['task']['userId']) if habitica_update['task']['type'] == "daily": logging.info("Received a daily task and we are now processing.") client.update_habit(habitica_update['task'], False) elif habitica_update['task']['type'] == "habit": logging.info("Received a habit and we are now processing.") client.update_habit(habitica_update['task'], True) else: logging.error("Received an unrecognized task type.") es = Elasticsearch([{'host': configuration['elasticsearch_server'], 'port': configuration['elasticsearch_port']}]) logging.info("Pushing event to Elasticsearch") res = es.index(index="test-index", doc_type='_doc', body=habitica_update) logging.debug("Elasticsearch returned " + str(res['result'])) return '', 200
def __init__(self, parent, id): super().__init__(parent) self.id = id client = MongoDB.get_client() db = client[MONGO_DB] self.topic_collection = db['topic'] self.weibo_collection = db['weibo'] self.comment_collection = db['comment'] self.horizontalLayout = QtWidgets.QHBoxLayout(self) self.create_figure(self.create_heatline()) self.create_figure(self.create_emotion_pie()) plt.rcParams['font.sans-serif'] = ['SimHei'] plt.rcParams['axes.unicode_minus'] = False
def testReadAndDecodeJSON(self): db = MongoDB() payloadToCompare = { "userId": 1, "id": 1, "title": "sunt aut facere repellat provident occaecati excepturi optio reprehenderit", "body": "quia et suscipit\nsuscipit recusandae consequuntur expedita et cum\nreprehenderit molestiae ut ut quas totam\nnostrum rerum est autem sunt rem eveniet architecto" } self.assertEqual(PayloadRetriever().readAndDecodeJSON(db), payloadToCompare)
def tls_connection(self): try: # Receive the secure payload using TLS print("App 2 connecting on port 8080 using SSL (TLS)") s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s2.connect(('localhost', 9999)) s2.send(b'App 2 got connection from app 1') s2.close() ssl_socket = ssl.wrap_socket(s, server_side=True, certfile="team1tls.crt", keyfile="team1tls.key") # Listen on port 8080 ssl_socket.bind(('localhost', 8080)) ssl_socket.listen(5) mongoDB = MongoDB() condition = True while condition: print("Accept connections from outside") (clientSocket, address) = ssl_socket.accept() #print(clientSocket.recv(1024)) dataRecieved = clientSocket.recv(1024) print(dataRecieved) mongoDB.mongoInstance("Test", "Got Connection") #if(clientSocket.recv(1024) is None): if (dataRecieved is None): print("data is none") #condition = False #print(dataRecieved) #hash_payload_hmac(dataRecieved) else: condition = False return dataRecieved except Exception as e: print(e)
def __init__(self, parent=None): """ 初始化 :param parent: 调用浏览器引擎的父窗口 """ super().__init__(parent) self.superwindow = parent self.setupUi() self.client = MongoDB.get_client() self.db = MongoDB.get_client()[MONGO_DB] self.weibo = self.db['weibo'] self.topic = self.db['topic'] self.comment = self.db['comment'] self.user = self.db['user'] self.repost = self.db['repost'] self.flag = None self.maparea = None self.maphtml = None self.barhtml = None self.wordcloudhtml = None self.graphtml = None self.geohtml = None self.id = None
def run(self): ltp = Ltp.get_object() tdt = Tdt.get_object() model = Text2Vec.get_object() weibo_collection = MongoDB.get_client()[MONGO_DB]['weibo'] count1 = 0 weibo_set = weibo_collection.find().sort('posted_at', pymongo.ASCENDING) for weibo in weibo_set: tdt.single_pass(weibo, 'topic', ltp, model) weibo_collection.update_one({'_id': weibo['_id']}, {'$set': weibo}, True) count1 += 1 self.count.emit(count1) self.stop.emit(True)
def read_update(pref): db = MongoDB() print(request.method) if request.method == 'POST': value = request.form['value'] db.update_db(pref, value) user_info = db.get_info(pref) return render_template('output.html', pref=pref, user=user_info) else: user_info = db.get_info(pref) return render_template('output.html', pref=pref, user=user_info)
def main(): #timestamp = 1545730073 #startDtObject = date.fromtimestamp(timestamp) #client = MongoClient('localhost', 27017) #db = client.Team1 #collection = db.logs startTime = time.time() mongoDB = MongoDB() # print(startDtObject) print("Retrieving JSON payload from source.") payload = PayloadRetriever().readAndDecodeJSON(mongoDB) print("Sending payload to App2.") SendPayload().sendPayload(payload, mongoDB) print("Saving payload to text file.") PayloadSaver().savePayload(payload, mongoDB) RabbitReciever.getMessage(startTime) endTime = time.time() print(str(endTime - startTime) + " seconds")
def __init__(self): self.cookie = '' self.url = 'http://www.p26y.com/cn/' self.torrent_url = 'https://www.bturl.at/search/' self.video = {} self.headers = {'Cookie': self.cookie, 'Host': 'www.p26y.com', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,' '*/*;q=0.8', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, ' 'like Gecko) Chrome/68.0.3440.75 Safari/537.36', 'If-None-Match': 'W/"3a18de34b39581014d560cc3522a80b7"', 'If-Modified-Since': 'Tue, 22 Jan 2019 00:00:00 GMT', 'Connection': 'keep-alive', 'Cache-Control': 'max-age=0', 'Accept-Language': 'zh-CN,zh;q=0.9', 'Upgrade-Insecure-Requests': '1', 'Accept-Encoding': 'gzip, deflate' } self.allgirls = MongoDB('Javdb', 'girlsname') # 所有girl名称—>页面编码键值对 self.rankdb = MongoDB('Javdb', 'rankdb') # 最受受欢迎作品榜单 self.girlsindexdb = MongoDB('Javdb', 'girlsindexdb') # 单个girl的所有作品db
''' author: Utpal Das ''' import time import stripe from flask import Flask, request, jsonify from flask_cors import CORS, cross_origin from mongo import MongoDB app = Flask(__name__) data = dict() mongo = MongoDB() stripe.api_key = '<stripe secret key>' @app.route('/',methods = ['POST']) @cross_origin(origin='localhost',headers=['Content- Type','Authorization']) def home(): if request.method == 'POST': data = mongo.insertorupdate_transaction(request.data) return data @app.route('/<userid>/<date>',methods = ['GET']) @cross_origin(origin='localhost',headers=['Content- Type','Authorization']) def get_current_time(userid,date): data = mongo.read_transaction(userid,date) return data @app.route('/create-checkout-session/<item>/<amount>', methods=['POST']) @cross_origin(origin='localhost',headers=['Content- Type','Authorization'])
class API(object): ''' 口袋48 api : func searchroom: 查找指定成员房间信息 : func login: 登录 : func chatroom: 查询房间消息 : func livedetail: 获取直播详情 ''' def __init__(self, mobile, password, membername): ''' :param mobile: 电话 int or string :param password: 密码 string :param membername: 成员全名 string ''' self.mobile = mobile self.password = password self.membername = membername self.dbtoken = MongoDB('Pocket48', 'dbtoken') self.headers = { 'Host': 'pocketapi.48.cn', 'accept': '*/*', 'Accept-Language': 'zh-Hans-CN;q=1', 'User-Agent': 'PocketFans201807/6.0.0 (iPhone; iOS 12.2; Scale/2.00)', 'Accept-Encoding': 'gzip, deflate', 'appInfo': '{"vendor":"apple","deviceId":"0", \ "appVersion":"6.0.0","appBuild":"190409", \ "osVersion":"12.2.0","osType":"ios", \ "deviceName":"iphone","os":"ios"}', 'Content-Type': 'application/json;charset=utf-8', 'Connection': 'keep-alive', } dic_data = self.searchroom self.ownerId, self.roomId = dic_data['ownerId'], dic_data['roomId'] @property def _token(self): ''' 获取token,超过1天则重新登录 : return: token string ''' hastoken = list(self.dbtoken.find({'mobile': self.mobile})) if not hastoken: token = self.login self.dbtoken.update(dict(timestamp=self._nowtime, token=token, mobile=self.mobile), upsert=True) elif self._nowtime - hastoken[0]['timestamp'] > 3600 * 24: self.dbtoken.remove({'mobile': self.mobile}) token = self.login self.dbtoken.update(dict(timestamp=self._nowtime, token=token, mobile=self.mobile), upsert=True) return list(self.dbtoken.find({'mobile': self.mobile}))[0]['token'] @property def _nowtime(self): ''' 当前时间戳 : return: 当前时间戳 float ''' return time.time() @property def searchroom(self): ''' 获取成员房间信息 : param membername: 成员全名 : return: 成员房间信息 json roomName 房间名, ownerName 成员名, roomId 房间名, ownerId 成员编号 ''' url = 'https://pocketapi.48.cn/im/api/v1/im/search' data = {'name': self.membername} try: response = requests.post(url, json=data, headers=self.headers, verify=False).json()['content']['data'][0] return dict(roomName=response['targetName'], ownerName=response['ownerName'], ownerId=response['ownerId'], roomId=response['targetId']) except Exception as e: raise e @property def login(self): ''' 登录 : param mobile: 手机号 : param password: 密码 : return token: 登录token string ''' try: url = 'https://pocketapi.48.cn/user/api/v1/login/app/mobile' data = { "pwd": str(self.password), "mobile": str(self.mobile), } res = requests.post(url, headers=self.headers, json=data, verify=False).json() return res['content']['token'] except Exception as e: raise e @property def chatroom(self): ''' 获取成员房间消息 : param roomId: 房间编号 : param ownerId: 成员编号 : return: 发言信息 json ''' url = 'https://pocketapi.48.cn/im/api/v1/chatroom/msg/list/homeowner' headers = self.headers headers.update({'token': self._token}) data = dict(needTop1Msg=False, roomId=str(self.roomId), ownerId=str(self.ownerId)) try: res = requests.post(url, headers=headers, json=data).json()['content']['message'] return res except Exception as e: raise e def livedetail(self, liveId): ''' 获取直播详情 :param liveId: 直播编号 int :return: playStreamPath string, response ''' url = "https://pocketapi.48.cn/live/api/v1/live/getLiveOne" form = {"liveId": str(liveId)} try: response = requests.post(url, json=form, headers=self.headers).json() if response['status'] == 200: playStreamPath = response['content']['playStreamPath'] return playStreamPath, response else: return False, False except BaseException: return False, False