class Receiver(object):
def __init__(self, redis_host):
self.output_queue = RedisQueue(redis_host, "outqueue")
def run(self):
while True:
result = self.output_queue.get().data
pprint(result)
print "---"
def run_dump(self):
dumpfile = open("netcrawl.log", "w")
while True:
result = self.output_queue.get().data
pprint(result)
dumpfile.write(pformat(result) + "\n")
dumpfile.flush()
class RedisMessageProvider(MessageProvider):
def __init__(self, host, port, queue_name):
self.queue = RedisQueue(name=queue_name,
namespace='queue',
host=host,
port=port)
self.queue.wait_for()
def get_message(self):
return self.queue.get()
class FetcherWorker:
def __init__(self, in_queue_namespace, out_queue_namespace, apikey):
self.in_queue_namespace = in_queue_namespace
self.out_queue_namespace = out_queue_namespace
self.apikey = apikey
self.in_queue = RedisQueue(in_queue_namespace)
self.out_queue = RedisQueue(out_queue_namespace)
print "Fetcher loaded with apikey", self.apikey
def run(self):
while 1:
base_url = self.in_queue.get()
if base_url == "None":
# add end-of-queue markers for parsers
self.out_queue.put("None")
# ends program
break
url = base_url + self.apikey
t1 = time.time()
print "fetching try 1", url
resp = urllib2.urlopen(url)
if resp.code == 200:
text = resp.read()
self.out_queue.put(text)
else:
print 'failed once', url
time.sleep(10)
print "fetching try 2", url
resp = urllib2.urlopen(url)
if resp.code == 200:
text = resp.read()
self.out_queue.put(text)
print "done fetching"
# make sure we don't use the same API key within 2 seconds
t2 = time.time()
if t2 - t1 < 2.0:
time.sleep(2.0 - (t2 - t1))
class Crawler(object):
def __init__(self, redis_host, depth=10):
self.links_queue = RedisQueue(redis_host, "linksqueue")
self.pages_queue = RedisQueue(redis_host, "pagesqueue")
def run(self):
while True:
link = self.links_queue.get().data
try:
page = WebPage(requests.get(link).text, link, 80)
except:
print("Exception GETing {0}".format(link))
continue
self.pages_queue.put(page.to_dict())
class FetcherWorker:
def __init__(self, in_queue_namespace, out_queue_namespace, apikey):
self.in_queue_namespace = in_queue_namespace
self.out_queue_namespace = out_queue_namespace
self.apikey = apikey
self.in_queue = RedisQueue(in_queue_namespace)
self.out_queue = RedisQueue(out_queue_namespace)
print "Fetcher loaded with apikey", self.apikey
def run(self):
while 1:
base_url = self.in_queue.get()
if base_url == "None":
# add end-of-queue markers for parsers
self.out_queue.put("None")
# ends program
break
url = base_url + self.apikey
t1 = time.time()
print "fetching try 1", url
resp = urllib2.urlopen(url)
if resp.code == 200:
text = resp.read()
self.out_queue.put(text)
else:
print 'failed once', url
time.sleep(10)
print "fetching try 2", url
resp = urllib2.urlopen(url)
if resp.code == 200:
text = resp.read()
self.out_queue.put(text)
print "done fetching"
# make sure we don't use the same API key within 2 seconds
t2 = time.time()
if t2 - t1 < 2.0:
time.sleep(2.0 - (t2 - t1))
def process_request_origin(self, request, spider):
redis = RedisQueue('proxy_ip')
if not redis.empty():
proxy_ip = redis.get()
else:
proxy_ip = get_ip()
proxy_para = {'ip_port': proxy_ip, 'user_pass': ''}
request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
if proxy_para['user_pass'] is not None:
encoded_user_pass = base64.encodestring(proxy_para['user_pass'])
request.headers[
'Proxy-Authorization'] = 'Basic ' + encoded_user_pass
print "*********************** RedisProxyMiddleware Using proxy ip: %s *****" % proxy_para[
'ip_port']
redis.put(proxy_ip)
def process_request_origin(self, request, spider):
redis = RedisQueue('proxy_ip')
if not redis.empty():
proxy_ip = redis.get()
else:
proxy_ip = get_ip()
proxy_para = {
'ip_port': proxy_ip,
'user_pass': ''
}
request.meta['proxy'] = "http://%s" % proxy_para['ip_port']
if proxy_para['user_pass'] is not None:
encoded_user_pass = base64.encodestring(proxy_para['user_pass'])
request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass
print "*********************** RedisProxyMiddleware Using proxy ip: %s *****" % proxy_para['ip_port']
redis.put(proxy_ip)
class Indexer(object):
def __init__(self, redis_host, es_urls):
self.pages_queue = RedisQueue(redis_host, "pagesqueue") # take pages out of this queue
self.links_queue = RedisQueue(redis_host, "linksqueue") # put links into this queue
self.connection = pyelasticsearch.ElasticSearch(es_urls)
try:
self.connection.create_index("webpages")
except:
pass
def run(self):
while True:
result = self.pages_queue.get().data
result['tags'] = genTags(result['html'])
self.connection.index('webpages', 'webpage', result, id=result['ip'])
print('Indexed {0}'.format(result['ip']))
for link in result['links']:
self.links_queue.put(link)
class DatastoreWriterWorker:
def __init__(self, in_queue_namespace):
self.in_queue_namespace = in_queue_namespace
self.in_queue = RedisQueue(in_queue_namespace)
def run(self):
while 1:
json_doc = self.in_queue.get()
if json_doc == "None":
break
print "DatastoreWriterWorker got", json_doc
print
print "Write to KV store, Fluentd, and MySQL"
print
print
class DatastoreWriterWorker():
def __init__(self, in_queue_namespace):
self.in_queue_namespace = in_queue_namespace
self.in_queue = RedisQueue(in_queue_namespace)
def run(self):
while 1:
json_doc = self.in_queue.get()
if json_doc == "None":
break
print "DatastoreWriterWorker got", json_doc
print
print "Write to KV store, Fluentd, and MySQL"
print
print
def main():
done_que = RedisQueue('seed')
run_que = RedisQueue('run')
run_que.flushdb()
conn = sqlite3.connect('site_data.db')
conn.execute(
"create table if not exists mainpages (id integer primary key autoincrement, url TEXT,headers TEXT,content BLOB)"
)
spend = 0
cnt = 0
size = 0
while True:
data = cPickle.loads(done_que.get())
st = time.time()
urls = geturls(data['url'], data['content'])
if len(urls) == 0:
continue
for url in urls:
if url not in bfdone:
run_que.put(url)
gziphtml = sqlite3.Binary(gzip.zlib.compress(data['content']))
size += len(gziphtml)
conn.execute(
"insert into mainpages (url,headers,content) values (?,?,?)",
(data['url'], str(data['headers']), gziphtml))
et = time.time()
spend += (et - st)
cnt += 1
if cnt % 10 == 0:
print "cost:", spend / cnt, cnt, done_que.qsize(
), size / 1024 / 1024
conn.commit()
class ParserWorker():
def __init__(self, in_queue_namespace, out_queue_namespace):
self.in_queue_namespace = in_queue_namespace
self.out_queue_namespace = out_queue_namespace
self.in_queue = RedisQueue(in_queue_namespace)
self.out_queue = RedisQueue(out_queue_namespace)
print "Parser worker loaded"
def run(self):
while 1:
xml_text = self.in_queue.get()
print "Received XML"
if xml_text == "None":
self.out_queue.put("None")
break
json_doc = DataParser.parse_get_state_stats_resp(xml_text)
print "Made JSON"
self.out_queue.put(json_doc)
import struct
import os
# test.py filename
if __name__ == '__main__':
if (len(sys.argv) > 1):
filename = sys.argv[1]
else:
with open("/home/anmol/minor/mpu/outputs/next_name",
"r+") as name_file:
name = name_file.readline().split()[0]
name_file.seek(0)
name_file.write(str(int(name) + 1))
filename = name + ".csv"
f = open("/home/anmol/minor/mpu/outputs/" + filename, "w+")
f.write("Timestamp, Accel_x, Accel_y, Accel_z\n")
q = RedisQueue('test')
print("Starting saving script")
try:
while True:
data = q.get()
if data.decode('utf-8') == 'finished':
break
else:
f.write(data.decode('utf-8'))
finally:
f.close()
os.system("bzip2 /home/anmol/minor/mpu/outputs/{}".format(filename))
#!/usr/bin/python
from RedisQueue import RedisQueue
import subprocess
import json
import base64
q = RedisQueue('messages',
namespace='ansible',
host='internal-redis.ovmdvp.0001.use2.cache.amazonaws.com',
port=6379,
db=1)
while True:
res = q.get()
message = json.loads(res)
subprocess.Popen([
"/home/ubuntu/ansible-bot/message_bridge/run_ansible_controller.sh",
message['response_id'], message['playbook'],
base64.b64encode(res)
])
from RedisQueue import RedisQueue
q = RedisQueue('test')
while True:
item = q.get()
print item, "received from queue"
if item == "None":
break
class machine(multiprocessing.Process):
def __init__(self, df):
multiprocessing.Process.__init__(self)
self.df = df
def run(self):
sleep(10)
self.last_mean = .015
self.q = RedisQueue('test')
print('start')
self.conn = sqlite3.connect("data.db")
while not self.q.empty():
features = str(self.q.get())[3:-2].replace("'","").split(', ')
self.features = list(features)
for self.hold_time in ['_10']:
df = self.df[self.features+['stock_perc_change'+self.hold_time, 'abnormal_perc_change'+self.hold_time]]
targets = [self.df['stock_perc_change'+self.hold_time], self.df['abnormal_perc_change'+self.hold_time]]
positive_dfs = []
negative_dfs = []
for i in range(8):
a_train, a_test, b_train, b_test = train_test_split(df.ix[:,:-2], df.ix[:,-2:], test_size=.4)
self.train(a_train, b_train)
test_result, negative_df, positive_df = self.test(a_test, b_test)
if test_result:
positive_dfs.append(positive_df)
negative_dfs.append(negative_df)
else:
break
if test_result:
self.get_result(pd.concat(positive_dfs), pd.concat(negative_dfs))
def train(self, a_train, b_train):
self.clf = SVR(C=1.0, epsilon=0.2)
self.clf.fit(a_train, b_train['abnormal_perc_change'+self.hold_time])
def test(self, a_test, b_test):
a_test['Predicted'] = self.clf.predict(a_test)
a_test['Actual_stock_perc_change'+self.hold_time] = b_test['stock_perc_change'+self.hold_time]
a_test['Actual_abnormal_perc_change'+self.hold_time] = b_test['abnormal_perc_change'+self.hold_time]
if len(a_test['Predicted'].unique())<40:
return False, None, None
a_test = a_test.sort_values(by='Predicted')
return True, a_test.ix[:,-3:].head(20), a_test.ix[:,-3:].tail(20)
def get_result(self, df_p, df_n):
p_result = df_p.describe()
n_result = df_n.describe()
if p_result.ix['mean','Actual_abnormal_perc_change_10']<0 or n_result.ix['mean','Actual_abnormal_perc_change_10']>0:
return
if p_result.ix['50%','Actual_abnormal_perc_change_10']<0 or n_result.ix['50%','Actual_abnormal_perc_change_10']>0:
return
store_me = False
if p_result.ix['mean','Actual_abnormal_perc_change_10']>self.last_mean:
self.last_mean = p_result.ix['mean','Actual_abnormal_perc_change_10']
store_me = True
p_result.index = p_result.index+'_pos'
n_result.index = n_result.index+'_neg'
p_result = p_result.stack().reset_index()
p_result.index = p_result['level_1'] +'-'+ p_result['level_0']
p_result = p_result[0]
n_result = n_result.stack().reset_index()
n_result.index = n_result['level_1'] +'-'+ n_result['level_0']
n_result = n_result[0]
result = p_result.append(n_result)
result = pd.DataFrame(result).T
self.model_name = str(self.features)[1:-1]+'__'+self.hold_time[1:]
result['features'] = self.model_name
if store_me:
result.to_sql('results', self.conn, index = False, if_exists='append')
self.store_machine()
def store_machine(self):
df = self.df[self.features]
target = self.df['abnormal_perc_change'+self.hold_time]
self.clf = SVR(C=1.0, epsilon=0.2)
self.clf.fit(df, target)
from sklearn.externals import joblib
joblib.dump(self.clf, 'machines/'+self.model_name)
print_err(uname)
print_err("Something wrong when try to get user's followed topics")
time.sleep(random.uniform(0, 5))
return user_questions
if __name__ == '__main__':
q = RedisQueue('follow_question_queue')
sleep_time = 0
db = MongoClient().zhihu.zhihu_follow_questions
while 1:
if (q.empty()):
print('Finished at %s' % str(datetime.datetime.now()))
print('Waiting ...')
uname = q.get()
uname = uname.decode()
if db.find({'_id': uname}).count() > 0:
continue
try:
with timeout(seconds=40):
all_questions = get_user_questions(uname)
if all_questions == {}:
continue
elif all_questions is None:
sleep_time += random.uniform(1, 5)
print_err('Sleeping for %0.2f seconds' % sleep_time)
time.sleep(sleep_time)
else:
db.insert(all_questions)
# create the anomaly detection instance
print('Instantiating anomaly detection algorithm...')
anomaly_detection = AnomalyDetection()
anomaly_detection.load_model('anomaly_detection.csv')
anomaly_detection.train_forest()
print('Finished instantiating anomaly detection algorithm')
# initialize the forwards-backwards pipelines
print('Initializing pipelines...')
redis_packet_queue = RedisQueue('packet_worker_queue')
redis_results_queue = RedisQueue('packet_results_queue')
print('Pipelines initialized, awaiting messages...')
while True:
# fetch a packet
recv_info = redis_packet_queue.get()
if not recv_info:
continue
# load the packet as a JSON object and decode the features
parsed_json = json.loads(recv_info)
json_decoded_packet = parsed_json['packet']
json_decoded_packid = parsed_json['id']
json_decoded_src = parsed_json['src']
json_decoded_dst = parsed_json['dst']
# perform isolation, observe if outlier
result = anomaly_detection.predict_data(json_decoded_packet)
if isinstance(result, list):
q = RedisQueue('account_login', **redis_conn)
http = urllib3.PoolManager(num_pools=50)
def worker(value):
params = {}
params['account_login'] = base64.encodestring(value)
r = http.request('POST', author_login, params)
#服务器失败，重新压回队列
if r.status != 200:
q.put(value)
#IP白名单验证失败，重新压回队列
if r.data['status'] == 10002:
q.put(value)
print r.data
while 1:
# time.sleep(1);
if q.empty():
print 'empty queue'
break
s = q.qsize()
for i in range(0,s):
value = q.get()
t = threading.Thread(target=worker, args=(value,))
t.start()
if threading.active_count() >= 500:
time.sleep(1)
combinations = []
q = RedisQueue('test')
#for permute_length in range(3,7):
for permute_length in range(3,10):
for feature in list(itertools.combinations(k_best_features, r=permute_length)):
combinations.append(feature)
shuffle(combinations)
print('starting', len(combinations))
input()
# clear the queue
while not q.empty():
q.get()
print("empty")
for i in range(6):
x = machine(df)
print('starting...')
x.start()
for feature in combinations:
q.put(feature)
print('all put')
while not q.empty():
try:
from RedisQueue import RedisQueue
redis = RedisQueue('0', 'jandan')
def user_agent(url):
proxy_handler = urllib2.ProxyHandler({'http': '127.0.0.1:8080'})
opener = urllib2.build_opener(proxy_handler)
urllib2.install_opener(opener)
req_header = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0'
}
req_timeout = 20
req = urllib2.Request(url, None, req_header)
page = urllib2.urlopen(req, None, req_timeout)
html = page
return html
while True:
while not redis.empty():
down_url = redis.get()
print(down_url)
try:
data = user_agent(down_url).read()
with open('./' + down_url[-11:], 'wb') as code:
code.write(data)
redis.pop()
except:
pass
#!/usr/bin/env python
# UniPi Python Control Panel
# clean_redis_queue_2.py
# uses Python 3.5 up
# Author: Johannes Untiedt
# Version 10.0 vom 26.03.2018
import time
from RedisQueue import RedisQueue
if __name__ == '__main__':
print("clean_redis_queue_2 started")
q = RedisQueue('ws_2')
while not q.empty():
message = q.get()
print(message)
print("redis_queue_2 cleaned")
import urllib2
from RedisQueue import RedisQueue
redis = RedisQueue('jandan3')
def user_agent(url):
req_header = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0'}
req_timeout = 20
req = urllib2.Request(url,None,req_header)
page = urllib2.urlopen(req,None,req_timeout)
html = page
return html
while not redis.empty():
down_url = redis.get()
data = user_agent(down_url).read()
with open('D:/Python/picture'+'/'+down_url[-11:],'wb')as code:
code.write(data)
print down_url
from RedisQueue import RedisQueue
q=RedisQueue('test')
q.put("你好")
print(q.get().decode('utf-8'))
from ChatRoomApiManager import ChatRoomApiManager
from RedisQueue import RedisQueue
import time
time.sleep(1)
chatRoomApi = ChatRoomApiManager()
token = chatRoomApi.get_token()
q = RedisQueue('bot_response')
while 1:
time.sleep(3)
content = q.get()
if content is None:
continue
chatRoomApi.post_message(token, content)
# UniPi Python Control Panel
# UniPi_sender.py
# uses Python 3.5 up
# Author: Johannes Untiedt
# Version 10.0 vom 24.03.2018
from UniPi_interface_class import *
from RedisQueue import RedisQueue
q = RedisQueue('ws_3')
ui = unipi_interface("10.0.0.52","8080")
print("UniPi_sender started")
while True:
while not q.empty():
msg = q.get()
#print(msg, " ", type(msg))
msg = eval(msg)
#print(msg, " ", type(msg))
if (msg['event'] == 'change_R1' and msg['data'] == 'on'):
ui.set_relay(1,1,0)
elif msg['event'] == 'change_R1' and msg['data'] == 'off':
ui.set_relay(1,0,0)
elif msg['event'] == 'change_R2' and msg['data'] == 'on':
ui.set_relay(2,1,0)
elif msg['event'] == 'change_R2' and msg['data'] == 'off':
ui.set_relay(2,0,0)
elif msg['event'] == 'change_R3' and msg['data'] == 'on':
ui.set_relay(3,1,0)
elif msg['event'] == 'change_R3' and msg['data'] == 'off':
