def connect(self, address): self.__address = address p = urlparse.urlparse(self.ssServer) sshost, ssport, ssmethod, sspassword = (p.hostname, p.port, p.username, p.password) self.crypto = encrypt.Encryptor(sspassword, ssmethod) if not self.parentproxy: self._sock = create_connection((sshost, ssport), self.timeout) elif self.parentproxy.startswith('http://'): self._sock = create_connection((self.pproxyparse.hostname, self.pproxyparse.port or 80), self.timeout) s = 'CONNECT %s:%s HTTP/1.1\r\nHost: %s\r\n' % (sshost, ssport, sshost) if self.pproxyparse.username: a = '%s:%s' % (self.pproxyparse.username, self.pproxyparse.password) s += 'Proxy-Authorization: Basic %s\r\n' % base64.b64encode(a.encode()) s += '\r\n' self._sock.sendall(s.encode()) remoterfile = self._sock.makefile('rb', 0) data = remoterfile.readline() if b'200' not in data: raise IOError(0, 'bad response: %s' % data) while not data in (b'\r\n', b'\n', b''): data = remoterfile.readline() else: logging.error('sssocket does not support parent proxy server: %s for now' % self.parentproxy) return 1 self.setsockopt = self._sock.setsockopt self.fileno = self._sock.fileno
def _connect_via_proxy(self, netloc): timeout = None if self._proxylist else 20 logging.info('{} {} via {} client: {}'.format(self.command, self.shortpath or self.path, self.ppname, self.ssclient)) if not self.pproxy: return create_connection(netloc, timeout or 5) elif self.pproxy.startswith('http://'): return create_connection((self.pproxyparse.hostname, self.pproxyparse.port or 80), timeout or 10) elif self.pproxy.startswith('https://'): s = create_connection((self.pproxyparse.hostname, self.pproxyparse.port or 443), timeout or 10) s = ssl.wrap_socket(s) s.do_handshake() return s elif self.pproxy.startswith('ss://'): s = sssocket(self.pproxy, timeout, conf.parentdict.get('direct')[0]) s.connect(netloc) return s elif self.pproxy.startswith('socks5://'): s = create_connection((self.pproxyparse.hostname, self.pproxyparse.port or 1080), timeout or 10) s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) s.sendall(b"\x05\x02\x00\x02" if self.pproxyparse.username else b"\x05\x01\x00") data = s.recv(2) if data == b'\x05\x02': # basic auth s.sendall(b''.join([b"\x01", chr(len(self.pproxyparse.username)).encode(), self.pproxyparse.username.encode(), chr(len(self.pproxyparse.password)).encode(), self.pproxyparse.password.encode()])) data = s.recv(2) assert data[1] == b'\x00' # no auth needed or auth passed s.sendall(b''.join([b"\x05\x01\x00\x03", chr(len(netloc[0])).encode(), netloc[0].encode(), struct.pack(b">H", netloc[1])])) data = s.recv(4) assert data[1] == b'\x00' if data[3] == b'\x01': # read ipv4 addr s.recv(4) elif data[3] == b'\x03': # read host addr s.recv(ord(s.recv(1))) elif data[3] == b'\x04': # read ipv6 addr s.recv(16) s.recv(2) # read port s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 0) return s
def select_recent_tweet_id(diagnosis): con = create_connection() recent_tweet_id = -1 with con.cursor() as cursor: cursor.execute( "SELECT tweet_id FROM post WHERE diagnosis = %s ORDER BY created_at DESC LIMIT 1", diagnosis) recent_tweet_id = cursor.fetchone() if recent_tweet_id: recent_tweet_id = recent_tweet_id['tweet_id'] close_connection(con) return recent_tweet_id
def save(timelines, recent_tweet_id, diagnosis): for line in timelines['statuses']: if line['id_str'] == recent_tweet_id: print(diagnosis + ' は最新状態です: ' + recent_tweet_id) return False print(re.findall(r'端的に表すと「(.+?)」です。(.+)で例えると(.+?)。', line['text'])) fa = re.findall(r'端的に表すと「(.+?)」です。(.+)で例えると「(.+?)」。', line['text']) if not fa: continue else: con = create_connection() last_id = None with con.cursor() as cursor: cursor.execute( "INSERT IGNORE INTO post (tweet_id, created_at, message, species, sample,diagnosis) VALUES (%s,%s,%s,%s,%s,%s)", (line['id_str'], datetime.datetime.strptime(line['created_at'], '%a %b %d %H:%M:%S +0000 %Y'), fa[0][0], fa[0][1], fa[0][2], diagnosis)) last_id = cursor.lastrowid close_connection(con) if last_id: features = re.search(r'特性:(.+)', line['text']) if features: fs = features[1].replace('…', '').split(',') con = create_connection() with con.cursor() as cursor: for fn in fs: print(last_id, fn) cursor.execute( "INSERT INTO post_meta (post_id, name) VALUES (%s, %s)", (last_id, fn)) close_connection(con)
def run(): conn = util.create_connection("./db/news.db") site = util.getSiteByName(conn, "New York Times") site_url = site[0][2] site_id = site[0][0] t.init(visual_automation = True, chrome_browser = True) t.url(site_url) t.wait(10) df = catchContent() df = util.fixImgLink(df,"https://cf-templates-fghyux9ggb7t-ap-southeast-1.s3-ap-southeast-1.amazonaws.com/NewYorkTimes.png") df = util.fixSummary(df) t.wait(20) t.close() util.updateNews(conn, site_id, df)
def calc(): conn = create_connection() with conn.cursor() as cursor: cursor.execute("SELECT p1.message,(SELECT count(*) FROM post p2 where p2.diagnosis = p1.diagnosis AND p2.message = p1.message) as cnt, (SELECT group_concat(t.name separator ',') FROM (SELECT post_meta2.name AS name, COUNT(*) as cnt2 FROM post p3 INNER JOIN post_meta post_meta2 ON p3.id = post_meta2.post_id WHERE p3.message = p1.message GROUP BY post_meta2.name ORDER BY cnt2 DESC ) AS t ) AS features FROM post p1 INNER JOIN post_meta on post_meta.post_id = p1.id WHERE p1.diagnosis = '性格' GROUP BY p1.message ORDER BY cnt DESC") res = cursor.fetchall() # r = res.fetchall() for rs in res: messages[rs['message']] = rs['features'].split(',') close_connection(conn) # print(messages)s for i, i_val in messages.items(): for j, j_val in messages.items(): if not jaccord_map[i][j] or not jaccord_map[j][i]: jaccord_map[i][j] = calc_jaccord(i_val, j_val)
def run(): conn = util.create_connection("./db/news.db") site = util.getSiteByName(conn, "Today Online") site_url = site[0][2] site_id = site[0][0] t.init(visual_automation=True, chrome_browser=True) t.url(site_url) t.wait(2) t.hover('//div[@class="container footer-main"]') t.wait(6) df = catchContent() t.wait(20) t.close() util.updateNews(conn, site_id, df)
def run(): conn = util.create_connection("./db/news.db") summaries = util.getAllSummary(conn) text = "" for row in summaries: for item in row: item = item.encode("utf-8") text = text + item + " " wc = WordCloud(background_color="white", stopwords=STOPWORDS) wc.generate(text) wc.to_file('./web/static/cloud_word_summary_1.png') custom_mask = np.array(Image.open("./web/static/cloud.png")) wc = WordCloud(background_color="white", mask=custom_mask, contour_width=3, contour_color='white') wc.generate(text) wc.to_file('./web/static/cloud_word_summary_2.png') print("Image generated")
def run(): conn = util.create_connection("./db/news.db") site = util.getSiteByName(conn, "BBC News") site_url = site[0][2] site_id = site[0][0] results, url, number_BBC, Img_link = NewsFromBBC(site_url) df = pd.DataFrame(index=range(0,number_BBC-1), columns = ['Sno', 'Title', 'URL','Summary', 'Img_URL']) for i in range(1, number_BBC): summaries = SummarizeUrl(url[i]) df.iloc[i-1, 0] = i df.iloc[i-1, 1] = results[i] df.iloc[i-1, 2] = url[i] df.iloc[i-1, 3] = summaries df.iloc[i-1, 4] = Img_link[i] df = util.fixImgLink(df, "https://cf-templates-fghyux9ggb7t-ap-southeast-1.s3-ap-southeast-1.amazonaws.com/bbc.png") df = util.fixSummary(df) util.updateNews(conn, site_id, df)
def handle(self): try: self.connection.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) pskcipher = encrypt.Encryptor(self.server.PSK, self.server.method) self.connection.settimeout(self.timeout) data = self.rfile.read(pskcipher.iv_len) pskcipher.decrypt(data) while True: try: data = self.rfile.read(1) self.connection.settimeout(self.timeout) cmd = ord(pskcipher.decrypt(data)) except Exception as e: logging.error( 'cmd Exception: server %s %r from %s:%s' % (self.server.server_address[1], e, self.client_address[0], self.client_address[1])) break if cmd == 10: # client key exchange rint = random.randint(64, 255) req_len = pskcipher.decrypt(self.rfile.read(2)) req_len = struct.unpack('>H', req_len)[0] data = pskcipher.decrypt(self.rfile.read(req_len)) data = io.BytesIO(data) ts = data.read(4) if abs(struct.unpack('>I', ts)[0] - time.time()) > 120: # possible replay attack logging.error('bad timestamp. client_ip: %s' % self.client_address[0]) break pklen = ord(data.read(1)) client_pkey = data.read(pklen) client_auth = data.read(32) def _send(data): data = struct.pack('>H', len(data)) + data self.wfile.write(pskcipher.encrypt(data)) client = None for user, passwd in USER_PASS.items(): h = hmac.new(passwd.encode(), ts + client_pkey + user.encode(), hashlib.sha256).digest() if compare_digest(h, client_auth): client = user break else: logging.error('user not found. client_ip: %s' % self.client_address[0]) break pkey, passwd = KeyManager.create_key( client, client_pkey, pskcipher.key_len) if pkey: logging.info('new key exchange. client: %s, ip: %s' % (client, self.client_address[0])) h = hmac.new(passwd.encode(), client_pkey + pkey + client.encode(), hashlib.sha256).digest() scert = SERVER_CERT.get_pub_key() signature = SERVER_CERT.sign(h, self.server.hash_algo) data = chr(0) + chr(len(pkey)) + chr(len(scert)) + chr( len(signature) ) + pkey + h + scert + signature + os.urandom(rint) _send(data) continue else: logging.error( 'Private_key already registered. client: %s, ip: %s' % (client, self.client_address[0])) # KeyManager.del_key(hashlib.md5(client_pkey).digest()) break elif cmd == 11: # a connect request client_pkey = pskcipher.decrypt(self.rfile.read(16)) rint = random.randint(64, 2048) def _send(code, cipher): if code == 1: data = os.urandom(rint) data = pskcipher.encrypt(struct.pack('>H', rint)) + data self.wfile.write(data) else: ct = cipher.encrypt( chr(code) + os.urandom(rint - 1)) data = pskcipher.encrypt(struct.pack( '>H', len(ct))) + ct self.wfile.write(data) if KeyManager.check_key(client_pkey): logging.error( 'client key not exist or expired. client ip: %s' % self.client_address[0]) ctlen = struct.unpack( '>H', pskcipher.decrypt(self.rfile.read(2)))[0] self.rfile.read(ctlen) _send(1, None) continue user = KeyManager.get_user_by_pubkey(client_pkey) cipher = encrypt.AEncryptor( KeyManager.get_skey_by_pubkey(client_pkey), self.server.method, CTX) ctlen = struct.unpack( '>H', pskcipher.decrypt(self.rfile.read(2)))[0] ct = self.rfile.read(ctlen) data = cipher.decrypt(ct) buf = io.BytesIO(data) ts = buf.read(4) if abs(struct.unpack('>I', ts)[0] - time.time()) > 120: logging.error( 'bad timestamp, possible replay attrack. client ip: %s' % self.client_address[0]) # KeyManager.del_key(client_pkey) # _send(1, None) break host_len = ord(buf.read(1)) addr = buf.read(host_len) port = struct.unpack('>H', buf.read(2))[0] if self._request_is_loopback( (addr, port)) and port not in self.server.forward: logging.info( 'server %d access localhost:%d denied. from %s:%d, %s' % (self.server.server_address[1], port, self.client_address[0], self.client_address[1], user)) _send(2, cipher) continue try: logging.info('server %d request %s:%d from %s:%d, %s' % (self.server.server_address[1], addr, port, self.client_address[0], self.client_address[1], user)) remote = create_connection((addr, port), timeout=10, proxy=self.server.proxy) remote.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) _send(0, cipher) # self.remote.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) except (IOError, OSError) as e: # Connection refused logging.warning( 'server %s:%d %r on connecting %s:%d' % (self.server.server_address[0], self.server.server_address[1], e, addr, port)) _send(2, cipher) continue if self.forward_tcp(self.connection, remote, cipher, pskcipher, timeout=60): break self.connection.settimeout(60) logging.debug( 'hxsocks connect reusable, except next connection') elif cmd in (1, 3, 4, 17, 19, 20): # A shadowsocks request if not self.server.ss: logging.warning( 'shadowsocks not enabled for this server. port: %d' % self.server.server_address[1]) break ota = cmd & 16 if cmd & 15 == 1: _addr = pskcipher.decrypt(self.rfile.read(4)) addr = socket.inet_ntoa(_addr) elif cmd & 15 == 3: _addr = pskcipher.decrypt(self.rfile.read(1)) addr = pskcipher.decrypt(self.rfile.read(ord(_addr))) _addr += addr elif cmd & 15 == 4: _addr = socket.AF_INET6, pskcipher.decrypt( self.rfile.read(16)) addr = socket.inet_ntop(_addr) port = struct.unpack('>H', pskcipher.decrypt( self.rfile.read(2)))[0] # verify if ota: header = chr(cmd) + _addr + struct.pack('>H', port) self._ota_chunk_idx = 0 rmac = pskcipher.decrypt(self.rfile.read(10)) key = pskcipher.decipher_iv + pskcipher.key mac = hmac.new(key, header, hashlib.sha1).digest()[:10] if not compare_digest(rmac, mac): logging.error("OTA Failed!!") break if self._request_is_loopback( (addr, port)) and port not in self.server.forward: logging.info( 'server %d access localhost:%d denied. from %s:%d' % (self.server.server_address[1], port, self.client_address[0], self.client_address[1])) break try: remote = None logging.info( 'server %d SS request %s:%d from %s:%d %s' % (self.server.server_address[1], addr, port, self.client_address[0], self.client_address[1], 'with ota' if ota else '')) remote = create_connection((addr, port), timeout=10, proxy=self.server.proxy) if ota: return self.ssforward_tcp_ota(self.connection, remote, pskcipher, timeout=60) return self.ssforward_tcp(self.connection, remote, pskcipher, timeout=60) except (IOError, OSError) as e: # Connection refused logging.warn( 'server %s:%d %r on connecting %s:%d' % (self.server.server_address[0], self.server.server_address[1], e, addr, port)) return else: logging.warning('unknown cmd %d, bad encryption key?' % cmd) break ins, _, _ = select.select([self.connection], [], [], 1) while ins: data = self.connection.recv(self.bufsize) if not data: break ins, _, _ = select.select([self.connection], [], [], 1) except Exception as e: logging.error(repr(e)) logging.error(traceback.format_exc())
def handle(self): self.remote = None try: self.encryptor = encrypt.Encryptor(self.server.key, self.server.method, servermode=True) sock = self.connection # sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) iv_len = self.encryptor.iv_len() if iv_len: try: self.decrypt(self.rfile.read(iv_len)) except ValueError: logging.warn('server %s:%d iv reused, possible replay attrack. closing...' % self.server.server_address) return data = sock.recv(1) if not data: return addrtype = ord(self.decrypt(data)) if addrtype == 1: addr = socket.inet_ntoa(self.decrypt(self.rfile.read(4))) elif addrtype == 3: addr = self.decrypt(self.rfile.read(ord(self.decrypt(self.rfile.read(1))))) elif addrtype == 4: addr = socket.inet_ntop(socket.AF_INET6, self.decrypt(self.rfile.read(16))) else: # not supported logging.warn('server %s:%d addr_type not supported, maybe wrong password' % self.server.server_address) return port = struct.unpack('>H', self.decrypt(self.rfile.read(2)))[0] if self.server.aports and port not in self.server.aports: logging.info('server %s:%d port %d not allowed' % (self.server.server_address[0], self.server.server_address[1], port)) return if self._request_is_loopback((addr, port)): logging.info('server %s:%d localhost access denied' % self.server.server_address) return try: logging.info('server %s:%d request %s:%d from %s:%d' % (self.server.server_address[0], self.server.server_address[1], addr, port, self.client_address[0], self.client_address[1])) data = self.decrypt(sock.recv(self.bufsize)) if self.server.reverse: if data.startswith((b'GET', b'POST', b'HEAD', b'PUT', b'DELETE', b'TRACE', b'OPTIONS', b'PATCH', b'CONNECT')) and b'HTTP/1' in data and b'\r\n' in data: data = data.decode('latin1') data = data.replace('\r\n', '\r\nss-realip: %s:%s\r\nss-client: %s\r\n' % (self.client_address[0], self.client_address[1], self.server.key), 1) self.remote = create_connection(self.server.reverse, timeout=10) else: a = 'CONNECT %s:%d HTTP/1.0\r\nss-realip: %s:%s\r\nss-client: %s\r\n\r\n' % (addr, port, self.client_address[0], self.client_address[1], self.server.key) self.remote = create_connection(self.server.reverse, timeout=10) self.remote.sendall(a.encode('latin1')) remoterfile = self.remote.makefile('rb', 0) d = remoterfile.readline() while d not in (b'\r\n', b'\n', b'\r'): if not d: raise IOError(0, 'remote closed') d = remoterfile.readline() if not self.remote: self.remote = create_connection((addr, port), timeout=10) self.remote.sendall(data) # self.remote.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) except (IOError, OSError) as e: # Connection refused logging.warn('server %s:%d %r on connecting %s:%d' % (self.server.server_address[0], self.server.server_address[1], e, addr, port)) return self.handle_tcp(sock, self.remote) except socket.error as e: logging.warn('server %s:%d %r' % (self.server.server_address[0], self.server.server_address[1], e))
def work(): # create the user table database = input("please input your database name: ") conn = util.create_connection(database) with conn: cur = conn.cursor() cur.execute("SELECT id, user_title FROM user") rows = cur.fetchall() for each_user in rows: user_id = each_user[0] user_title = each_user[1] user_table[user_id] = user_title # create a database connection conn = util.create_connection(database) with conn: cur = conn.cursor() course_id = input("please input your course_id: ") message_to_be_executed = "SELECT id, title, inst_replied, starter FROM thread WHERE courseid = \"" message_to_be_executed += course_id message_to_be_executed += "\" ORDER BY posted_time" cur.execute(message_to_be_executed) threads = cur.fetchall() for each_thread in threads: text_to_be_written = "" thread_id = each_thread[0] thread_title = each_thread[1] thread_intervention = each_thread[2] thread_starter = each_thread[3] thread_intervened_time_dic[thread_id] = -1 if (thread_starter in user_table and (user_table[thread_starter] == 'Instructor' or user_table[thread_starter] == 'Staff')): continue text_to_be_written += thread_title # Detect the earliest intervention. if thread_intervention == 1: post_message = "SELECT user, post_time FROM post WHERE thread_id = \"" post_message += thread_id post_message += "\" ORDER BY post_time" cur.execute(post_message) all_posts = cur.fetchall() for each_post in all_posts: poster = each_post[0] post_time = each_post[1] if poster in user_table and ( user_table[poster] == 'Instructor' or user_table[poster] == 'Staff' ) and thread_intervened_time_dic[thread_id] == -1: thread_intervened_time_dic[thread_id] = post_time comment_message = "SELECT user, post_time FROM comment WHERE thread_id = \"" comment_message += thread_id comment_message += "\" ORDER BY post_time" cur.execute(comment_message) all_comments = cur.fetchall() for each_comment in all_comments: commenter = each_comment[0] comment_time = each_comment[1] if commenter in user_table and ( user_table[commenter] == 'Instructor' or user_table[commenter] == 'Staff'): if thread_intervened_time_dic[ thread_id] == -1 or comment_time < thread_intervened_time_dic[ thread_id]: thread_intervened_time_dic[ thread_id] = comment_time # Adding Texts. post_message = "SELECT id, post_text, user, post_time FROM post WHERE thread_id = \"" post_message += thread_id post_message += "\" ORDER BY post_time" cur.execute(post_message) all_posts = cur.fetchall() for each_post in all_posts: post_id = each_post[0] post_text = each_post[1] poster = each_post[2] post_time = each_post[3] if thread_intervention == 1 and thread_intervened_time_dic[ thread_id] != -1 and post_time >= thread_intervened_time_dic[ thread_id]: break else: text_to_be_written += '\n\n' text_to_be_written += post_text comment_message = "SELECT id, comment_text, user, post_time FROM comment WHERE post_id = \"" comment_message += post_id comment_message += "\" ORDER BY post_time" cur.execute(comment_message) all_comments = cur.fetchall() for each_comment in all_comments: comment_id = each_comment[0] comment_text = each_comment[1] commenter = each_comment[2] comment_time = each_comment[3] if thread_intervention == 1 and thread_intervened_time_dic[ thread_id] != -1 and comment_time >= thread_intervened_time_dic[ thread_id]: break else: text_to_be_written += '\n\n' text_to_be_written += comment_text file_name = str(thread_id) + '.txt' path = 'text/' + course_id + "/" + file_name with open(path, 'w') as f: print(text_to_be_written, file=f)