Ejemplo n.º 1
0
def sync_code():
  for i in seeds:
    print i, get_local_ip()
    if i == get_local_ip():
      continue
    with lcd(crawler_dir):
      local("rsync -a crawler %s@%s:%s --exclude '.git'"%(env.user, i, crawler_dir))
Ejemplo n.º 2
0
def sync_code():
    for i in seeds:
        print i, get_local_ip()
        if i == get_local_ip():
            continue
        with lcd(crawler_dir):
            local("rsync -a crawler %s@%s:%s --exclude '.git'" %
                  (env.user, i, crawler_dir))
    def __init__(self, id, ip = None):
        self.ip = get_local_ip() if ip is None else ip
        self.port = get_free_port(self.ip)
        self.id = id
        self.files = {}
        self.files_backups = {}
        self._files_backup_lock = Lock()
        self.storage_path = "./storage/%s"%(str(self.id))
        self._files_lock = Lock()
        self.files_uplading = {}
        # self.files_download_count = {}

        self.file_tags = {}
        self.file_tags_backups = {}
        self._file_tags_lock = Lock()
        
        self.tags = {}
        self.tags_backups = {}
        self._tags_lock = Lock()

        try:
            os.mkdir('storage')
        except FileExistsError:
            pass
        try:
            shutil.rmtree(self.storage_path)
        except FileNotFoundError:
            pass
        try:
            os.mkdir(self.storage_path)
        except:
            print("Creation of the directory %s failed" % self.storage_path)

        self._start_thread = Thread(target = lambda: self._start_service(), daemon=True)
Ejemplo n.º 4
0
    def __init__(self, id=None):

        self._finger = [None] * M
        self._predecessor = None
        self.successor_list = []
        self._lock = Lock()
        self.ip = get_local_ip()
        self.port = get_free_port(self.ip)
        self.id = hash(self) if id is None else id
        self._my_remote = RemoteObj(self.id, self.ip, self.port)

        self.stabilize_thread = Thread(
            target=lambda: self.stabilize_loop(interval=2.1), daemon=True)
        self.stabilize_alive = False
        self.fix_thread = Thread(target=lambda: self.fix_loop(interval=2.2),
                                 daemon=True)
        self.fix_alive = False
        self.successors_thread = Thread(
            target=lambda: self.successors_loop(interval=2.3), daemon=True)
        self.successors_alive = False
        self.start_service_thread = Thread(target=lambda: self.start_service(),
                                           daemon=True)

        self.storage = Storage(self.id)
        self.remote_storage = RemoteObj(self.storage.id, self.storage.ip,
                                        self.storage.port)
Ejemplo n.º 5
0
def add_node():
    ip = utils.get_local_ip()  #获得ip
    r = metric.get_redis()  #获得redis实例

    seeds = mysettings.SEEDS.split(',')  #添加的节点
    if not seeds:
        raise NoSeedsException()

    for node in seeds:
        seed = metric.get_redis(host=node)  #获取相应节点的redis实例
        try:
            cluster_list = seed.smembers(mysettings.REDIS_CLUSTER_LIST_KEY
                                         )  #获得此节点上redis中保存的cluster_list
            break
        except:
            pass

    for node in cluster_list:
        #对cluster_list中其他节点的cluster_list加入自己的IP,并且将cluster状态设置为需要重新Hash(其他节点会将任务重新Hash);
        #将所有节点及自己的IP加入到自己的cluster_list,并且将本机状态设置为normal
        d = metric.get_redis(host=node)
        d.sadd(mysettings.REDIS_CLUSTER_LIST_KEY, ip)
        d.set(mysettings.REDIS_CLUSTER_STATUS_KEY, mysettings.STATUS_REHASHING)
        r.sadd(mysettings.REDIS_CLUSTER_LIST_KEY, node)

    r.sadd(mysettings.REDIS_CLUSTER_LIST_KEY, ip)
    r.set(mysettings.REDIS_CLUSTER_STATUS_KEY, mysettings.STATUS_NORMAL)
def get_task_id(ips):
    local_ip = utils.get_local_ip()
    for i in range(len(ips)):
        if ips[i] == local_ip:
            return i
    raise ValueError("Cannot find local_ip: %s in ips list: [%s]"
                     %(local_ip, ", ".join(ips)))
Ejemplo n.º 7
0
def start_crawl():
  run("redis-cli set %s 0"%mysettings.EXIT_KEY)
  with cd("%scrawler/twcrawler/twcrawler"%crawler_dir):
    #run("nohup python add_node.py>/tmp/log&")
    ip = get_local_ip()
    for i in xrange(mysettings.PROCESS_NUM):
      run("sh -c '(nohup scrapy crawl %s >/tmp/%s-%s-%s.log 2>&1 )&'"%\
        (mysettings.SPIDER_NAME, ip, mysettings.SPIDER_NAME, i),  pty=False)
Ejemplo n.º 8
0
def start_crawl():
    run("redis-cli set %s 0" % mysettings.EXIT_KEY)
    with cd("%scrawler/twcrawler/twcrawler" % crawler_dir):
        #run("nohup python add_node.py>/tmp/log&")
        ip = get_local_ip()
        for i in xrange(mysettings.PROCESS_NUM):
            run("sh -c '(nohup scrapy crawl %s >/tmp/%s-%s-%s.log 2>&1 )&'"%\
              (mysettings.SPIDER_NAME, ip, mysettings.SPIDER_NAME, i),  pty=False)
Ejemplo n.º 9
0
def register_ip(broker_ip):
    socket = context.socket(zmq.DEALER)

    socket.connect(broker_ip)
    ip = get_local_ip()

    msg = "REGISTER {}:{}".format(ip, HIDER_PORT)
    socket.send(msg)
    response = socket.recv()
Ejemplo n.º 10
0
def register_ip(broker_ip):
    socket = context.socket(zmq.DEALER)

    socket.connect(broker_ip)
    ip = get_local_ip()

    msg = "REGISTER {}:{}".format(ip, HIDER_PORT)
    socket.send(msg)
    response = socket.recv()
Ejemplo n.º 11
0
def init_peers(port):
    global local_peer
    local_peer = Peer(get_local_ip(), port)
    add_peer(local_peer)
    debug.output(debug.info, '[init_peers] start to look for peers')
    find_peers()
    debug.output(debug.info, '[init_peers] look for peers ended')
    debug.output(debug.info, '[init_peers] broadcast known peers')
    network.broadcast_message(messages.PeersMessage(get_known_peers()))
    debug.output(debug.info, '[init_peers] broadcasted known peers')
Ejemplo n.º 12
0
def get_api_url(strip_http=False):
    """ Determines the URL to use for API operations based on some socket
    operations and settings from the settings.cfg. Defaults to using localhost
    on the default API port defined in settings.cfg. """

    fqdn = socket.getfqdn()
    if fqdn == settings.get("api","prod_fqdn"):
        output = settings.get("api","prod_url")
    else:
#        logger.debug("[API] host FQDN is '%s'. Backing off to dev API settings." % (fqdn))
        output = "https://%s:%s/" % (get_local_ip(), settings.get("api","localhost_port"))

    if strip_http:
        return output[7:]
    else:
        return output
Ejemplo n.º 13
0
def main():
    banner()
    logger.info("Initializing...")

    logger.info("Starting base docker")
    container = create_container()
    if not container:
        return

    host_port = docker_host_port(
        CURRENT_CONTAINER)[HONEYPOT_DOCKER_SERVICE_PORT]
    logger.info("Base docker has started")

    logger.info("Creating initial iptables rules...")
    local_ip = get_local_ip(INTERFACE)

    out, ok = command("iptables -t nat -A PREROUTING -p tcp "
                      f"-d { local_ip } --dport { HONEYPOT_SERVICE_PORT } "
                      f"-j DNAT --to { local_ip }:{ host_port }")
    if not ok:
        return

    out, ok = command(f"iptables -A INPUT -p tcp -i { INTERFACE } "
                      "--dport 22 -m state --state NEW,ESTABLISHED "
                      "-j ACCEPT")
    if not ok:
        return

    out, ok = command(f"iptables -A OUTPUT -p tcp -o { INTERFACE } "
                      "--sport 22 -m state --state ESTABLISHED "
                      "-j ACCEPT")
    if not ok:
        return

    out, ok = command(
        f"iptables -A OUTPUT -p tcp --tcp-flags SYN,ACK SYN,ACK",
        ["-j", "LOG", "--log-prefix", "Connection established: "])
    if not ok:
        return

    logger.info("Rules created. Honeydock is ready to go. :)")

    handler = EventHandler(KERN_LOG_PATH)
    watch_manager = WatchManager()
    watch_manager.add_watch(handler.file_path, IN_MODIFY)
    notifier = Notifier(watch_manager, handler)
    notifier.loop()
Ejemplo n.º 14
0
    def get_connection(self, dbconfig):
        u'根据参数获取数据库的连接'
        mysql_host_ip = utils.get_mysql_address()
        #         controller_host_ip = utils.get_controller_node_address()
        mysql_passwd = utils.get_mysql_password()
        local_ip = utils.get_local_ip()

        if mysql_host_ip and local_ip and (mysql_host_ip == local_ip):
            return MySQLdb.connect(host=dbconfig["host"],
                                   port=dbconfig['port'],
                                   user=dbconfig['user'],
                                   charset=dbconfig['charset'])
        else:
            return MySQLdb.connect(host=mysql_host_ip,
                                   port=dbconfig['port'],
                                   user=dbconfig['user'],
                                   passwd=mysql_passwd,
                                   charset=dbconfig['charset'])
Ejemplo n.º 15
0
 def create_reading_socket(self, ip=None, port=None):
     sock = self._create_udp_socket()
     ip = ip if ip or ip == '' else get_local_ip()
     port = port if port else 0
     sock.bind((ip, port))
     return sock
Ejemplo n.º 16
0
    def __init__(self):
        self.context = zmq.Context()

    def connect(self, addr):
        self.addr = addr
        self.socket = self.context.socket(zmq.DEALER)
        self.socket.connect(self.addr)

    def send(self, msg):
        self.socket.send_multipart([msg])
        return self.socket.recv()

if __name__ == '__main__':
    parser = OptionParser()

    parser.add_option(
        "-a",
        "--address",
        dest="address",
        default='tcp://127.0.0.1:5555'
    )
    (options, args) = parser.parse_args()
    client = ZMQClient()
    client.connect(options.address)

    msg = '{0}:{1}'.format(
        get_local_ip(),
        options.address.split(':')[-1]
    )
    print client.send(msg)
Ejemplo n.º 17
0
def get_phone(cfg):
    ip = cfg.ip
    port = cfg.port
    out_dir = cfg.out_dir
    queue = cfg.queue
    receive_queue = cfg.receive_queue

    log = False
    global phone_received_msgs
    global phone_write_to_file
    phone_write_to_file = torch.zeros(1)
    phone_received_msgs = torch.zeros(1)
    phone_write_to_file.share_memory_()
    phone_received_msgs.share_memory_()

    if ip == 0:
        ip = get_local_ip()
        plog("Found ip: {}".format(ip))
    log_path = "{}/phone.log".format(out_dir)
    out_file = open(os.path.join(log_path), "w")

    plog("Star server on: {}:{}".format(ip, port))

    # Configure message received
    def message_received(client, server, message):
        """Called when a client sends a message"""
        global phone_received_msgs
        global phone_write_to_file
        phone_received_msgs += 1
        if log:
            plog("Client(%d) said: %s" % (client['id'], message))
        if phone_write_to_file[0] > 0:
            out_file.write("{:0.6f};{};{}\n".format(time.time(), client['id'],
                                                    message))

    # Start server:
    proc = mp.Process(target=run_server,
                      args=(ip, port, message_received, new_client,
                            client_diconnected))
    proc.start()

    while phone_received_msgs[0] <= 0:
        time.sleep(1)

    # Wait for start command:
    plog("Ready")
    receive_queue.put(True)
    resp = queue.get(block=True)
    if resp:
        plog("Start")
    else:
        return 1

    phone_write_to_file[0] = 1

    # Wait for closing command
    while True:
        res = get_nonblocking(queue)
        time.sleep(1)
        if res:
            break

    proc.terminate()
    proc.join()

    return 0
Ejemplo n.º 18
0
#!/usr/bin/env python

import argparse
import zmq
import utils

context = zmq.Context()

socket = context.socket(zmq.DEALER)

parser = argparse.ArgumentParser()
parser.add_argument('-c', '--connect-address', default='tcp://127.0.0.1:5555')
parser.add_argument('-p', '--port', default=4444)
parser.add_argument('-l', '--list-only', action='store_true', default=False)

args = parser.parse_args()

myip = utils.get_local_ip(args.connect_address.split('://')[1].split(':')[0])

socket.connect(args.connect_address)

if args.list_only:
  command = 'LIST'
else:
  command = 'REGISTER {}:{}'.format(myip, args.port)
print command
socket.send(command)
print socket.recv()
Ejemplo n.º 19
0
#!/usr/bin/env python

import argparse
import zmq
import utils

context = zmq.Context()

socket = context.socket(zmq.DEALER)

parser = argparse.ArgumentParser()
parser.add_argument('-c', '--connect-address', default='tcp://127.0.0.1:5555')
parser.add_argument('-p', '--port', default=4444)
parser.add_argument('-l', '--list-only', action='store_true', default=False)

args = parser.parse_args()

myip = utils.get_local_ip(args.connect_address.split('://')[1].split(':')[0])

socket.connect(args.connect_address)

if args.list_only:
    command = 'LIST'
else:
    command = 'REGISTER {}:{}'.format(myip, args.port)
print command
socket.send(command)
print socket.recv()
Ejemplo n.º 20
0
 def __init__(self, ip = None):
     self.ip = get_local_ip() if ip is None else ip
     self.port = get_free_port(self.ip)
     self.id = hash(self)
     self.start_service_thread = Thread(target=lambda : self._start_service(), daemon=True)
     self.last_list = []
Ejemplo n.º 21
0
def init_server(did):
    global datacenterId, workerId
    datacenterId = did 
    workerId = getWorkerIdFor(get_local_ip())
Ejemplo n.º 22
0
    def process_IN_MODIFY(self, event):
        global CURRENT_CONTAINER
        global KERN_LOG_CONTENT
        global KERN_LOG_LEN

        local_ip = get_local_ip(INTERFACE)
        loglines = open(self.file_path, 'r').readlines()
        loglines_len = len(loglines)
        diff = loglines_len - KERN_LOG_LEN
        changes = loglines[-diff:]

        KERN_LOG_CONTENT = open(KERN_LOG_PATH, 'r').readlines()
        KERN_LOG_LEN = len(KERN_LOG_CONTENT)

        for log in changes:
            if "Connection established: " not in log:
                continue

            attacker_ip = (re.search(r'DST=(.*?) ', log)).group(1)

            logger.debug("New connection was established!")
            logger.debug(f"Connection detail: { log }")
            logger.debug(f"Attacker ({ attacker_ip })")
            if not attacker_exists(attacker_ip):
                logger.info(
                    f"The IP: { attacker_ip } it's from a new attacker")
                created = create_attacker(attacker_ip, CURRENT_CONTAINER)
                if not created:
                    return

                logger.info(
                    f"The IP: { attacker_ip } was added on attacker's table")
                logger.info(
                    "Initiating proccess of attaching this IP to a docker instance"
                )
                logger.info("Removing main rule")
                host_port = docker_host_port(
                    CURRENT_CONTAINER)[HONEYPOT_DOCKER_SERVICE_PORT]
                out, ok = command(
                    "iptables -t nat -D PREROUTING -p tcp "
                    f"-d { local_ip } --dport { HONEYPOT_SERVICE_PORT } "
                    f"-j DNAT --to { local_ip }:{ host_port }")
                if not ok:
                    return

                logger.info("Attaching this IP to container")
                out, ok = command(
                    "iptables -t nat -A PREROUTING -p tcp "
                    f"-s { attacker_ip } -d { local_ip } --dport { HONEYPOT_SERVICE_PORT } "
                    f"-j DNAT --to { local_ip }:{ host_port }")
                if not ok:
                    return

                logger.info("Creating a new docker instance")
                container = create_container()
                if not container:
                    return

                host_port = docker_host_port(
                    CURRENT_CONTAINER)[HONEYPOT_DOCKER_SERVICE_PORT]
                logger.info(
                    f"New docker container created on port { host_port }")

                logger.info("Creating main rule to new docker")
                out, ok = command(
                    "iptables -t nat -A PREROUTING -p tcp "
                    f"-d { local_ip } --dport { HONEYPOT_SERVICE_PORT } "
                    f"-j DNAT --to { local_ip }:{ host_port }")
                if not ok:
                    return

                logger.info("Done!")
            else:
                logger.info(
                    f"The IP: { attacker_ip } it's from a returning attacker. No action required"
                )
Ejemplo n.º 23
0
def start_name_server():
    ip = get_local_ip()
    port = get_free_port(ip)
    Pyro4.naming.startNSloop(ip, port)
Ejemplo n.º 24
0
def start_name_server():
    ip = get_local_ip()
    port = get_free_port(ip)
    Pyro4.naming.startNSloop(ip, port)


if __name__ == '__main__':

    remote_node = None
    start_ns = True
    try:
        ns = Pyro4.locateNS()
        for server in ns.list():
            if "NameServer" in server:
                ip, port = ns.list()[server].split('@')[1].split(':')
                if ip == get_local_ip():
                    start_ns = False
                    print('Name server found on local pc')
            if "NameServer" not in server and "run_node" not in server:
                try:
                    with Pyro4.Proxy(ns.list()[server]) as p:
                        remote_node = p.get_info()
                    break
                except:
                    pass
    except Exception as x:
        print(x)

    if start_ns:
        start_name_server_thread = Thread(target=lambda: start_name_server(),
                                          daemon=True)
Ejemplo n.º 25
0
# coding=utf-8

import os
import sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import argparse
import time
import zmq


from utils import get_local_ip

context = zmq.Context()

socket = context.socket(zmq.DEALER)

parser = argparse.ArgumentParser()
parser.add_argument('-c', '--connect-address', default='tcp://127.0.0.1:5555')

args = parser.parse_args()

socket.connect(args.connect_address)

while 1:
    msg = "{}:{}".format(get_local_ip(), '5555')
    print 'Sending address {} to dealer {}'.format(msg, args.connect_address)
    socket.send(msg)
    print socket.recv()
    time.sleep(1)
Ejemplo n.º 26
0
def inject_information(record):
    record.extra['ip'] = get_local_ip()
Ejemplo n.º 27
0
import zmq


from utils import get_local_ip

context = zmq.Context()

socket = context.socket(zmq.DEALER)

parser = argparse.ArgumentParser()
parser.add_argument('-c', '--connect-address', default='tcp://127.0.0.1:5555')
parser.add_argument('-H', '--hider', default=False, action='store_true')
parser.add_argument('-C', '--city',  default='Barcelona')

args = parser.parse_args()

socket.connect(args.connect_address)

hider = args.hider
city = args.hider
myip = get_local_ip()
address = args.connect_address

while 1:
    if hider : command = "REGISTER {}:{}".format(myip, '5555')
    else     : command = "LIST"
    print 'Sending command "{}" to dealer {}'.format(command, address)
    socket.send(command)
    print socket.recv()
    time.sleep(1)
Ejemplo n.º 28
0
    app.add_url_rule('/<path:p>', view_func=path_view_auth)

    users.update({args.user: generate_password_hash(args.passwd)})

    root = Path(args.dir).resolve().absolute()

    port = str(args.port)

    cert_file = temp_dir / "cert_file.crt"
    key_file = temp_dir / "key_file.key"
    create_self_signed_cert(cert_file, key_file)

    upnp_add_port_mapping(internal_port=args.port,
                          external_port=args.port,
                          ip=get_local_ip(),
                          name=__name__)

    print("\n\n")
    print(f"Local address: https://{get_local_ip()}:{args.port}")
    print(f"Global address: https://{get_global_ip()}:{args.port}")

    print(f"User: {args.user}")
    print(f"Password: {args.passwd}")
    print("\n")

    app.run(host=get_local_ip(),
            port=port,
            threaded=True,
            debug=False,
            ssl_context=(str(cert_file), str(key_file)))
Ejemplo n.º 29
0
        self.port = port

    def __str__(self):
        return self.ip + ':' + str(self.port)

    def __eq__(self, other):
        return self.ip == other.ip and self.port == other.port

    def __hash__(self):
        return self.ip.__hash__() * 65536 + self.port


known_peers = set()
known_peers.add(Peer('127.0.0.1', 10086))

local_peer = Peer(get_local_ip(), 10086)


def get_local_peer():
    return local_peer


def add_peer(peer):
    if peer not in known_peers:
        debug.output(debug.running, '[add_peer] new peer: ' + str(peer))
        known_peers.add(peer)


def get_known_peers():
    return known_peers
Ejemplo n.º 30
0
import argparse
import zmq
from utils import get_local_ip

context = zmq.Context()

socket = context.socket(zmq.DEALER)

parser = argparse.ArgumentParser()
parser.add_argument('-c',
                    '--connect-address',
                    default='tcp://172.16.16.188:5556')
parser.add_argument('-p', '--port', default=5555)

args = parser.parse_args()
myip = get_local_ip()

socket.connect(args.connect_address)

# First just register to the server
command = 'REGISTER {}:{}'.format(myip, args.port)
socket.send(command)
players = socket.recv_string().split()
#players = ['172.16.17.133:7777']

guesses = ['Ljubljana', 'Stockholm', 'Berlin']

print(players)
for player in players:
    player_socket = context.socket(zmq.DEALER)
    player_socket.connect('tcp://{}'.format(player))
Ejemplo n.º 31
0
def inject_information(record):
    record.extra['ip'] = get_local_ip()