def sync_code():
for i in seeds:
print i, get_local_ip()
if i == get_local_ip():
continue
with lcd(crawler_dir):
local("rsync -a crawler %s@%s:%s --exclude '.git'"%(env.user, i, crawler_dir))
def sync_code():
for i in seeds:
print i, get_local_ip()
if i == get_local_ip():
continue
with lcd(crawler_dir):
local("rsync -a crawler %s@%s:%s --exclude '.git'" %
(env.user, i, crawler_dir))
def __init__(self, id, ip = None):
self.ip = get_local_ip() if ip is None else ip
self.port = get_free_port(self.ip)
self.id = id
self.files = {}
self.files_backups = {}
self._files_backup_lock = Lock()
self.storage_path = "./storage/%s"%(str(self.id))
self._files_lock = Lock()
self.files_uplading = {}
# self.files_download_count = {}
self.file_tags = {}
self.file_tags_backups = {}
self._file_tags_lock = Lock()
self.tags = {}
self.tags_backups = {}
self._tags_lock = Lock()
try:
os.mkdir('storage')
except FileExistsError:
pass
try:
shutil.rmtree(self.storage_path)
except FileNotFoundError:
pass
try:
os.mkdir(self.storage_path)
except:
print("Creation of the directory %s failed" % self.storage_path)
self._start_thread = Thread(target = lambda: self._start_service(), daemon=True)
def __init__(self, id=None):
self._finger = [None] * M
self._predecessor = None
self.successor_list = []
self._lock = Lock()
self.ip = get_local_ip()
self.port = get_free_port(self.ip)
self.id = hash(self) if id is None else id
self._my_remote = RemoteObj(self.id, self.ip, self.port)
self.stabilize_thread = Thread(
target=lambda: self.stabilize_loop(interval=2.1), daemon=True)
self.stabilize_alive = False
self.fix_thread = Thread(target=lambda: self.fix_loop(interval=2.2),
daemon=True)
self.fix_alive = False
self.successors_thread = Thread(
target=lambda: self.successors_loop(interval=2.3), daemon=True)
self.successors_alive = False
self.start_service_thread = Thread(target=lambda: self.start_service(),
daemon=True)
self.storage = Storage(self.id)
self.remote_storage = RemoteObj(self.storage.id, self.storage.ip,
self.storage.port)
def add_node():
ip = utils.get_local_ip() #获得ip
r = metric.get_redis() #获得redis实例
seeds = mysettings.SEEDS.split(',') #添加的节点
if not seeds:
raise NoSeedsException()
for node in seeds:
seed = metric.get_redis(host=node) #获取相应节点的redis实例
try:
cluster_list = seed.smembers(mysettings.REDIS_CLUSTER_LIST_KEY
) #获得此节点上redis中保存的cluster_list
break
except:
pass
for node in cluster_list:
#对cluster_list中其他节点的cluster_list加入自己的IP，并且将cluster状态设置为需要重新Hash（其他节点会将任务重新Hash）；
#将所有节点及自己的IP加入到自己的cluster_list，并且将本机状态设置为normal
d = metric.get_redis(host=node)
d.sadd(mysettings.REDIS_CLUSTER_LIST_KEY, ip)
d.set(mysettings.REDIS_CLUSTER_STATUS_KEY, mysettings.STATUS_REHASHING)
r.sadd(mysettings.REDIS_CLUSTER_LIST_KEY, node)
r.sadd(mysettings.REDIS_CLUSTER_LIST_KEY, ip)
r.set(mysettings.REDIS_CLUSTER_STATUS_KEY, mysettings.STATUS_NORMAL)
def get_task_id(ips):
local_ip = utils.get_local_ip()
for i in range(len(ips)):
if ips[i] == local_ip:
return i
raise ValueError("Cannot find local_ip: %s in ips list: [%s]"
%(local_ip, ", ".join(ips)))
def start_crawl():
run("redis-cli set %s 0"%mysettings.EXIT_KEY)
with cd("%scrawler/twcrawler/twcrawler"%crawler_dir):
#run("nohup python add_node.py>/tmp/log&")
ip = get_local_ip()
for i in xrange(mysettings.PROCESS_NUM):
run("sh -c '(nohup scrapy crawl %s >/tmp/%s-%s-%s.log 2>&1 )&'"%\
(mysettings.SPIDER_NAME, ip, mysettings.SPIDER_NAME, i), pty=False)
def start_crawl():
run("redis-cli set %s 0" % mysettings.EXIT_KEY)
with cd("%scrawler/twcrawler/twcrawler" % crawler_dir):
#run("nohup python add_node.py>/tmp/log&")
ip = get_local_ip()
for i in xrange(mysettings.PROCESS_NUM):
run("sh -c '(nohup scrapy crawl %s >/tmp/%s-%s-%s.log 2>&1 )&'"%\
(mysettings.SPIDER_NAME, ip, mysettings.SPIDER_NAME, i), pty=False)
def register_ip(broker_ip):
socket = context.socket(zmq.DEALER)
socket.connect(broker_ip)
ip = get_local_ip()
msg = "REGISTER {}:{}".format(ip, HIDER_PORT)
socket.send(msg)
response = socket.recv()
def register_ip(broker_ip):
socket = context.socket(zmq.DEALER)
socket.connect(broker_ip)
ip = get_local_ip()
msg = "REGISTER {}:{}".format(ip, HIDER_PORT)
socket.send(msg)
response = socket.recv()
def init_peers(port):
global local_peer
local_peer = Peer(get_local_ip(), port)
add_peer(local_peer)
debug.output(debug.info, '[init_peers] start to look for peers')
find_peers()
debug.output(debug.info, '[init_peers] look for peers ended')
debug.output(debug.info, '[init_peers] broadcast known peers')
network.broadcast_message(messages.PeersMessage(get_known_peers()))
debug.output(debug.info, '[init_peers] broadcasted known peers')
def get_api_url(strip_http=False):
""" Determines the URL to use for API operations based on some socket
operations and settings from the settings.cfg. Defaults to using localhost
on the default API port defined in settings.cfg. """
fqdn = socket.getfqdn()
if fqdn == settings.get("api","prod_fqdn"):
output = settings.get("api","prod_url")
else:
# logger.debug("[API] host FQDN is '%s'. Backing off to dev API settings." % (fqdn))
output = "https://%s:%s/" % (get_local_ip(), settings.get("api","localhost_port"))
if strip_http:
return output[7:]
else:
return output
def main():
banner()
logger.info("Initializing...")
logger.info("Starting base docker")
container = create_container()
if not container:
return
host_port = docker_host_port(
CURRENT_CONTAINER)[HONEYPOT_DOCKER_SERVICE_PORT]
logger.info("Base docker has started")
logger.info("Creating initial iptables rules...")
local_ip = get_local_ip(INTERFACE)
out, ok = command("iptables -t nat -A PREROUTING -p tcp "
f"-d { local_ip } --dport { HONEYPOT_SERVICE_PORT } "
f"-j DNAT --to { local_ip }:{ host_port }")
if not ok:
return
out, ok = command(f"iptables -A INPUT -p tcp -i { INTERFACE } "
"--dport 22 -m state --state NEW,ESTABLISHED "
"-j ACCEPT")
if not ok:
return
out, ok = command(f"iptables -A OUTPUT -p tcp -o { INTERFACE } "
"--sport 22 -m state --state ESTABLISHED "
"-j ACCEPT")
if not ok:
return
out, ok = command(
f"iptables -A OUTPUT -p tcp --tcp-flags SYN,ACK SYN,ACK",
["-j", "LOG", "--log-prefix", "Connection established: "])
if not ok:
return
logger.info("Rules created. Honeydock is ready to go. :)")
handler = EventHandler(KERN_LOG_PATH)
watch_manager = WatchManager()
watch_manager.add_watch(handler.file_path, IN_MODIFY)
notifier = Notifier(watch_manager, handler)
notifier.loop()
def get_connection(self, dbconfig):
u'根据参数获取数据库的连接'
mysql_host_ip = utils.get_mysql_address()
# controller_host_ip = utils.get_controller_node_address()
mysql_passwd = utils.get_mysql_password()
local_ip = utils.get_local_ip()
if mysql_host_ip and local_ip and (mysql_host_ip == local_ip):
return MySQLdb.connect(host=dbconfig["host"],
port=dbconfig['port'],
user=dbconfig['user'],
charset=dbconfig['charset'])
else:
return MySQLdb.connect(host=mysql_host_ip,
port=dbconfig['port'],
user=dbconfig['user'],
passwd=mysql_passwd,
charset=dbconfig['charset'])
def create_reading_socket(self, ip=None, port=None):
sock = self._create_udp_socket()
ip = ip if ip or ip == '' else get_local_ip()
port = port if port else 0
sock.bind((ip, port))
return sock
def __init__(self):
self.context = zmq.Context()
def connect(self, addr):
self.addr = addr
self.socket = self.context.socket(zmq.DEALER)
self.socket.connect(self.addr)
def send(self, msg):
self.socket.send_multipart([msg])
return self.socket.recv()
if __name__ == '__main__':
parser = OptionParser()
parser.add_option(
"-a",
"--address",
dest="address",
default='tcp://127.0.0.1:5555'
)
(options, args) = parser.parse_args()
client = ZMQClient()
client.connect(options.address)
msg = '{0}:{1}'.format(
get_local_ip(),
options.address.split(':')[-1]
)
print client.send(msg)
def get_phone(cfg):
ip = cfg.ip
port = cfg.port
out_dir = cfg.out_dir
queue = cfg.queue
receive_queue = cfg.receive_queue
log = False
global phone_received_msgs
global phone_write_to_file
phone_write_to_file = torch.zeros(1)
phone_received_msgs = torch.zeros(1)
phone_write_to_file.share_memory_()
phone_received_msgs.share_memory_()
if ip == 0:
ip = get_local_ip()
plog("Found ip: {}".format(ip))
log_path = "{}/phone.log".format(out_dir)
out_file = open(os.path.join(log_path), "w")
plog("Star server on: {}:{}".format(ip, port))
# Configure message received
def message_received(client, server, message):
"""Called when a client sends a message"""
global phone_received_msgs
global phone_write_to_file
phone_received_msgs += 1
if log:
plog("Client(%d) said: %s" % (client['id'], message))
if phone_write_to_file[0] > 0:
out_file.write("{:0.6f};{};{}\n".format(time.time(), client['id'],
message))
# Start server:
proc = mp.Process(target=run_server,
args=(ip, port, message_received, new_client,
client_diconnected))
proc.start()
while phone_received_msgs[0] <= 0:
time.sleep(1)
# Wait for start command:
plog("Ready")
receive_queue.put(True)
resp = queue.get(block=True)
if resp:
plog("Start")
else:
return 1
phone_write_to_file[0] = 1
# Wait for closing command
while True:
res = get_nonblocking(queue)
time.sleep(1)
if res:
break
proc.terminate()
proc.join()
return 0
#!/usr/bin/env python
import argparse
import zmq
import utils
context = zmq.Context()
socket = context.socket(zmq.DEALER)
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--connect-address', default='tcp://127.0.0.1:5555')
parser.add_argument('-p', '--port', default=4444)
parser.add_argument('-l', '--list-only', action='store_true', default=False)
args = parser.parse_args()
myip = utils.get_local_ip(args.connect_address.split('://')[1].split(':')[0])
socket.connect(args.connect_address)
if args.list_only:
command = 'LIST'
else:
command = 'REGISTER {}:{}'.format(myip, args.port)
print command
socket.send(command)
print socket.recv()
#!/usr/bin/env python
import argparse
import zmq
import utils
context = zmq.Context()
socket = context.socket(zmq.DEALER)
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--connect-address', default='tcp://127.0.0.1:5555')
parser.add_argument('-p', '--port', default=4444)
parser.add_argument('-l', '--list-only', action='store_true', default=False)
args = parser.parse_args()
myip = utils.get_local_ip(args.connect_address.split('://')[1].split(':')[0])
socket.connect(args.connect_address)
if args.list_only:
command = 'LIST'
else:
command = 'REGISTER {}:{}'.format(myip, args.port)
print command
socket.send(command)
print socket.recv()
def __init__(self, ip = None):
self.ip = get_local_ip() if ip is None else ip
self.port = get_free_port(self.ip)
self.id = hash(self)
self.start_service_thread = Thread(target=lambda : self._start_service(), daemon=True)
self.last_list = []
def init_server(did):
global datacenterId, workerId
datacenterId = did
workerId = getWorkerIdFor(get_local_ip())
def process_IN_MODIFY(self, event):
global CURRENT_CONTAINER
global KERN_LOG_CONTENT
global KERN_LOG_LEN
local_ip = get_local_ip(INTERFACE)
loglines = open(self.file_path, 'r').readlines()
loglines_len = len(loglines)
diff = loglines_len - KERN_LOG_LEN
changes = loglines[-diff:]
KERN_LOG_CONTENT = open(KERN_LOG_PATH, 'r').readlines()
KERN_LOG_LEN = len(KERN_LOG_CONTENT)
for log in changes:
if "Connection established: " not in log:
continue
attacker_ip = (re.search(r'DST=(.*?) ', log)).group(1)
logger.debug("New connection was established!")
logger.debug(f"Connection detail: { log }")
logger.debug(f"Attacker ({ attacker_ip })")
if not attacker_exists(attacker_ip):
logger.info(
f"The IP: { attacker_ip } it's from a new attacker")
created = create_attacker(attacker_ip, CURRENT_CONTAINER)
if not created:
return
logger.info(
f"The IP: { attacker_ip } was added on attacker's table")
logger.info(
"Initiating proccess of attaching this IP to a docker instance"
)
logger.info("Removing main rule")
host_port = docker_host_port(
CURRENT_CONTAINER)[HONEYPOT_DOCKER_SERVICE_PORT]
out, ok = command(
"iptables -t nat -D PREROUTING -p tcp "
f"-d { local_ip } --dport { HONEYPOT_SERVICE_PORT } "
f"-j DNAT --to { local_ip }:{ host_port }")
if not ok:
return
logger.info("Attaching this IP to container")
out, ok = command(
"iptables -t nat -A PREROUTING -p tcp "
f"-s { attacker_ip } -d { local_ip } --dport { HONEYPOT_SERVICE_PORT } "
f"-j DNAT --to { local_ip }:{ host_port }")
if not ok:
return
logger.info("Creating a new docker instance")
container = create_container()
if not container:
return
host_port = docker_host_port(
CURRENT_CONTAINER)[HONEYPOT_DOCKER_SERVICE_PORT]
logger.info(
f"New docker container created on port { host_port }")
logger.info("Creating main rule to new docker")
out, ok = command(
"iptables -t nat -A PREROUTING -p tcp "
f"-d { local_ip } --dport { HONEYPOT_SERVICE_PORT } "
f"-j DNAT --to { local_ip }:{ host_port }")
if not ok:
return
logger.info("Done!")
else:
logger.info(
f"The IP: { attacker_ip } it's from a returning attacker. No action required"
)
def start_name_server():
ip = get_local_ip()
port = get_free_port(ip)
Pyro4.naming.startNSloop(ip, port)
def start_name_server():
ip = get_local_ip()
port = get_free_port(ip)
Pyro4.naming.startNSloop(ip, port)
if __name__ == '__main__':
remote_node = None
start_ns = True
try:
ns = Pyro4.locateNS()
for server in ns.list():
if "NameServer" in server:
ip, port = ns.list()[server].split('@')[1].split(':')
if ip == get_local_ip():
start_ns = False
print('Name server found on local pc')
if "NameServer" not in server and "run_node" not in server:
try:
with Pyro4.Proxy(ns.list()[server]) as p:
remote_node = p.get_info()
break
except:
pass
except Exception as x:
print(x)
if start_ns:
start_name_server_thread = Thread(target=lambda: start_name_server(),
daemon=True)
# coding=utf-8
import os
import sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import argparse
import time
import zmq
from utils import get_local_ip
context = zmq.Context()
socket = context.socket(zmq.DEALER)
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--connect-address', default='tcp://127.0.0.1:5555')
args = parser.parse_args()
socket.connect(args.connect_address)
while 1:
msg = "{}:{}".format(get_local_ip(), '5555')
print 'Sending address {} to dealer {}'.format(msg, args.connect_address)
socket.send(msg)
print socket.recv()
time.sleep(1)
def inject_information(record):
record.extra['ip'] = get_local_ip()
import zmq
from utils import get_local_ip
context = zmq.Context()
socket = context.socket(zmq.DEALER)
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--connect-address', default='tcp://127.0.0.1:5555')
parser.add_argument('-H', '--hider', default=False, action='store_true')
parser.add_argument('-C', '--city', default='Barcelona')
args = parser.parse_args()
socket.connect(args.connect_address)
hider = args.hider
city = args.hider
myip = get_local_ip()
address = args.connect_address
while 1:
if hider : command = "REGISTER {}:{}".format(myip, '5555')
else : command = "LIST"
print 'Sending command "{}" to dealer {}'.format(command, address)
socket.send(command)
print socket.recv()
time.sleep(1)
app.add_url_rule('/<path:p>', view_func=path_view_auth)
users.update({args.user: generate_password_hash(args.passwd)})
root = Path(args.dir).resolve().absolute()
port = str(args.port)
cert_file = temp_dir / "cert_file.crt"
key_file = temp_dir / "key_file.key"
create_self_signed_cert(cert_file, key_file)
upnp_add_port_mapping(internal_port=args.port,
external_port=args.port,
ip=get_local_ip(),
name=__name__)
print("\n\n")
print(f"Local address: https://{get_local_ip()}:{args.port}")
print(f"Global address: https://{get_global_ip()}:{args.port}")
print(f"User: {args.user}")
print(f"Password: {args.passwd}")
print("\n")
app.run(host=get_local_ip(),
port=port,
threaded=True,
debug=False,
ssl_context=(str(cert_file), str(key_file)))
self.port = port
def __str__(self):
return self.ip + ':' + str(self.port)
def __eq__(self, other):
return self.ip == other.ip and self.port == other.port
def __hash__(self):
return self.ip.__hash__() * 65536 + self.port
known_peers = set()
known_peers.add(Peer('127.0.0.1', 10086))
local_peer = Peer(get_local_ip(), 10086)
def get_local_peer():
return local_peer
def add_peer(peer):
if peer not in known_peers:
debug.output(debug.running, '[add_peer] new peer: ' + str(peer))
known_peers.add(peer)
def get_known_peers():
return known_peers
import argparse
import zmq
from utils import get_local_ip
context = zmq.Context()
socket = context.socket(zmq.DEALER)
parser = argparse.ArgumentParser()
parser.add_argument('-c',
'--connect-address',
default='tcp://172.16.16.188:5556')
parser.add_argument('-p', '--port', default=5555)
args = parser.parse_args()
myip = get_local_ip()
socket.connect(args.connect_address)
# First just register to the server
command = 'REGISTER {}:{}'.format(myip, args.port)
socket.send(command)
players = socket.recv_string().split()
#players = ['172.16.17.133:7777']
guesses = ['Ljubljana', 'Stockholm', 'Berlin']
print(players)
for player in players:
player_socket = context.socket(zmq.DEALER)
player_socket.connect('tcp://{}'.format(player))
def inject_information(record):
record.extra['ip'] = get_local_ip()
