Esempio n. 1
0
    def select_highest_voted_to_chain(self):
        # ---- Add all ticks with same amount of votes to the dictionary ----
        # WARNING: This MUST happen less than 50% of the time and result in
        # usually only 1 winner, so that chain only branches occasionally
        # and thus doesn't become an exponentially growing tree.
        # This is the main condition to achieve network-wide consensus
        top_tick_refs = self.top_tick_refs()

        highest_ticks = self.get_ticks_by_ref(top_tick_refs)

        # TODO: Should always be > 0 but sometimes not, when unsynced...
        if len(highest_ticks) > 0:
            logger.debug("Top tick refs with " +
                         str(len(highest_ticks[0]['list'])) + " pings each:" +
                         str(top_tick_refs))

            tick_dict = {}
            for tick in highest_ticks:
                to_add = self.json_tick_to_chain_tick(tick)
                tick_dict = {**tick_dict, **to_add}

            # TODO: Is this atomic?
            if self.chain.full():
                # This removes earliest item from queue
                self.chain.get()

            self.chain.put(tick_dict)
        else:
            logger.info("Warning!! No ticks added to chain!!")

        self.restart_cycle()
Esempio n. 2
0
    def __init__(self):
        self.chain = Queue(maxsize=config['chain_max_length'])
        self.ping_pool = {}
        self.vote_pool = {}
        # Priority queue because we want to sort by cumulative continuity
        self.tick_pool = PriorityQueue()

        logger.debug("This node is " + credentials.addr)

        # TODO: Create valid genesis tick
        tick = {
            'pubkey':
            'pubkey',
            'nonce':
            68696043434,
            'list': [{
                'timestamp': 0,
                'pubkey': 'pubkey'
            }],
            'prev_tick':
            'prev_tick',
            'height':
            0,
            'this_tick':
            '55f5b323471532d860b11d4fc079ba38'
            '819567aa0915d83d4636d12e498a8f3e'
        }

        genesis_dict = self.json_tick_to_chain_tick(tick)
        self.chain.put(genesis_dict)
Esempio n. 3
0
def validate_ping(ping, ping_pool=None, vote=False):
    stage = 'vote' if vote else 'ping'
    if not validate_schema(ping, 'ping_schema.json'):
        logger.debug(stage + " failed schema validation")
        return False

    if ping_pool is not None:
        if vote:
            if pubkey_to_addr(ping['pubkey']) not in ping_pool:
                logger.debug("Voters's pubkey not found in pingpool")
                return False

            # Voting twice just overwrites your past vote!
        else:
            if pubkey_to_addr(ping['pubkey']) in ping_pool:
                logger.debug(stage + " was already in pool")
                return False

    # Check hash and sig, keeping in mind signature might be popped off
    if not validate_sig_hash(ping):
        logger.debug(stage + " failed sighash validation")
        return False

    # TODO: Do sanity check on a pings timestamp in relation to current time etc
    if not validate_ping_timestamp(ping):  # <-- empty stub function atm..
        logger.debug(stage + " failed sanity check on timestamp")
        return False

    return True
Esempio n. 4
0
    def send_mutual_add_requests(self, peerslist):
        successful_adds = 0
        # Mutual add peers
        for peer in peerslist:
            if peer not in self.peers and len(
                    self.peers) <= config['max_peers']:
                content = {"port": self.port, 'pubkey': credentials.pubkey}
                signature = sign(standard_encode(content), credentials.privkey)
                content['signature'] = signature
                status_code = None
                response = None
                result, success = attempt(requests.post,
                                          False,
                                          url=peer + '/mutual_add',
                                          json=content,
                                          timeout=config['timeout'])
                if success:
                    status_code = result.status_code
                    response = result.text
                else:
                    logger.debug("Couldn't connect to " + peer)

                if status_code in [201, 503]:
                    if status_code == 201:
                        logger.info("Adding peer " + str(peer))
                        peer_addr = response
                        self.register_peer(peer, peer_addr)
                        successful_adds += 1
                    if status_code == 503:
                        logger.info("Peer was at peer-maximum")

        return successful_adds
Esempio n. 5
0
    def train(self, train_loader, model, epoch):
        batch_time = AverageMeter()
        data_time = AverageMeter()
        losses = AverageMeter()
        acc = AverageMeter()

        # switch to train mode
        model.train()

        end = time.time()
        for batch_idx, (input, target) in enumerate(train_loader):
            data_time.update(time.time() - end)

            input_var = input.to(self.device)
            target_var = target.to(self.device)

            output = model(input_var)

            loss = self.criterion(output, target_var)

            self.optimizer.zero_grad()
            loss.backward()
            self.optimizer.step()

            with torch.no_grad():
                losses.update(loss.item())
                metric_val = self.metric(
                    output,
                    target_var)  # todo - add output dimention assertion
                acc.update(metric_val)
                self.watcher.display_every_iter(batch_idx, input_var, target,
                                                output)

            # measure elapsed time
            batch_time.update(time.time() - end)
            end = time.time()

        logger.debug('\rEpoch: {0}  [{1}/{2}]\t'
                     'ETA: {time:.0f}/{eta:.0f} s\t'
                     'data loading: {data_time.val:.3f} s\t'
                     'loss {loss.avg:.4f}\t'
                     'metric {acc.avg:.4f}\t'.format(epoch,
                                                     batch_idx,
                                                     len(train_loader),
                                                     eta=batch_time.avg *
                                                     len(train_loader),
                                                     time=batch_time.sum,
                                                     data_time=data_time,
                                                     loss=losses,
                                                     acc=acc))
        self.watcher.log_value(TRAIN_ACC_OUT, metric_val)
        self.watcher.log_value(TRAIN_LOSS_OUT, loss.item())
        return losses.avg, acc.avg
Esempio n. 6
0
    def register_peer(self, url, peer_addr):
        """
        Add a new peer to the list of peers

        :param url: <str> Address of peer. Eg. 'http://192.168.0.5:5000'
        :param peer_addr: <str> Mining addr of peer
        :return: <bool> Whether it was already in list or not
        """
        netloc = self.get_full_location(url)

        # Avoid adding self
        if peer_addr == credentials.addr:
            logger.debug("Cannot add self")
            return False

        self.peers[netloc] = peer_addr
        return True
Esempio n. 7
0
    def generate_and_process_tick(self):
        height = self.clockchain.current_height() + 1

        tick = {
            'list': list(self.clockchain.ping_pool.values()),
            'pubkey': credentials.pubkey,
            'prev_tick': self.clockchain.prev_tick_ref(),
            'height': height
        }

        this_tick, nonce = mine(tick)

        tick['nonce'] = nonce

        signature = sign(standard_encode(tick), credentials.privkey)
        tick['signature'] = signature

        # This is to keep track of the "name" of the tick as debug info
        # this_tick is not actually necessary according to tick schema
        tick['this_tick'] = this_tick

        prev_tick = self.clockchain.latest_selected_tick()

        possible_previous = self.clockchain.possible_previous_ticks()

        # Validate own tick
        retries = 0
        while retries < config['tick_retries']:
            if not validate_tick(
                    tick, prev_tick, possible_previous, verbose=False):
                retries = retries + 1
                time.sleep(config['tick_retries_sleep'])
            else:
                self.clockchain.add_to_tick_pool(tick)
                # Forward to peers (this must be after all validation)
                self.networker.forward(data_dict=tick,
                                       route='tick',
                                       origin=credentials.addr,
                                       redistribute=0)
                logger.debug("Forwarded own tick: " + str(tick))
                return True

        logger.debug(
            "Failed own tick validation too many times. not forwarded")
        return False
Esempio n. 8
0
    def ping_worker(self):
        while True:
            if self.networker.ready and not self.added_ping:

                self.networker.stage = "ping"

                logger.debug(
                    "Ping stage--------------------------------------")
                successful = \
                    self.generate_and_process_ping(
                        self.clockchain.prev_tick_ref())

                if not successful:
                    continue

                self.added_ping = True
            else:
                time.sleep(1)
Esempio n. 9
0
def build_app(g_port):
    if config['api_backend'] == "flask":
        logger.debug("Running in production mode")
        # Clockchain datastructure and an instance for network messaging
        g_clockchain = Clockchain()
        g_networker = Networker()

        # Timeminer handles all network validation, and API exposes messaging
        Timeminer(g_clockchain, g_networker)

        g_api = API(g_clockchain, g_networker)

        g_app = g_api.create_app()
        g_networker.activate(g_port)

        return g_app
    else:  # We don't intend to run Sanic with gunicorn
        return False
Esempio n. 10
0
    def get_sample_of_peers_from(peers, sample_size=config['max_peers']):
        peers_of_peers = set()
        # Get peers of peers and add to set (set has no duplicates)
        for peer in list(peers):
            result, success = attempt(requests.get,
                                      False,
                                      url=peer + '/info/peers',
                                      timeout=config['timeout'])
            if success:
                next_peers = json.loads(result.text)
                for next_peer in next_peers['peers']:
                    peers_of_peers.add(next_peer)
            else:
                logger.debug("Couldn't connect to " + peer)

        if sample_size > len(list(peers_of_peers)):
            sample_size = len(list(peers_of_peers))

        return random.sample(list(peers_of_peers), sample_size)
Esempio n. 11
0
    def generate_and_process_ping(self, reference, vote=False):
        # TODO: Code duplication between here and api.. where to put??
        # TODO: Can't be in helpers, and cant be in clockchain/networker..
        # Always construct ping in the following order:
        # 1) Init 2) Mine+nonce 3) Add signature
        # This is because the order of nonce and sig creation matters

        ping = {
            'pubkey': credentials.pubkey,
            'timestamp': utcnow(),
            'reference': reference
        }

        stage = 'vote' if vote else 'ping'

        _, nonce = mine(ping)
        ping['nonce'] = nonce

        signature = sign(standard_encode(ping), credentials.privkey)
        ping['signature'] = signature

        # Validate own ping
        if not validate_ping(ping, self.clockchain.ping_pool, vote):
            logger.debug("Failed own " + stage + " validation")
            return False

        if vote:
            self.clockchain.add_to_vote_pool(ping)
        else:
            self.clockchain.add_to_ping_pool(ping)

        route = 'vote' if vote else 'ping'

        # Forward to peers (this must be after all validation)
        self.networker.forward(data_dict=ping,
                               route=route,
                               origin=credentials.addr,
                               redistribute=0)

        logger.debug("Forwarded own " + route + ": " + str(ping))

        return True
Esempio n. 12
0
def validate_sig_hash(item):
    # The reason this is a combined check on sig+hash (instead of split methods)
    # Is that check must be atomic, as sig+hash mutate the tick in certain order

    # Deepcopy used to not modify instance we received
    item_copy = copy.deepcopy(item)
    signature = item_copy.pop('signature', None)

    if signature is None:
        logger.debug("Could not find signature in validate sighash..")
        return False

    # Check hash
    if not validate_difficulty(hasher(item_copy)):
        logger.debug("Invalid hash for item: " + str(item_copy) + " " +
                     hasher(item_copy))
        return False

    # Validate signature
    try:
        encoded_message = standard_encode(item_copy)
        if not verify(encoded_message, signature, item_copy['pubkey']):
            return False
    except ecdsa.BadSignatureError:
        # TODO : When new joiner joins, make sure peers relay latest hash
        logger.debug("Bad signature!" + str(item_copy) + " " + str(signature))
        return False

    return True
Esempio n. 13
0
    def top_tick_refs(self):
        highest_voted_ticks = []

        # Sort by value (amount of votes)
        sorted_votes = sorted(self.get_vote_counts().items(),
                              key=lambda x: x[1],
                              reverse=True)

        top_ref, top_score = sorted_votes.pop(0)
        highest_voted_ticks.append(top_ref)

        logger.debug("Highest amount of votes achieved was: " + str(top_score))

        # If any other refs share the same score, we return those too
        for vote in sorted_votes:
            next_ref, next_score = vote
            if next_score == top_score:
                highest_voted_ticks.append(next_ref)
            else:
                break

        return highest_voted_ticks
Esempio n. 14
0
    def join_network_worker(self):
        # Sleeping random amount to not have seed-clash (cannot do circular
        #  adding of peers at the exact same time as seeds)
        logger.debug("Running on port " + str(self.port))
        sleeptime = 2 + random.randrange(3000) / 1000.0
        logger.debug("Sleeping for " + str(int(sleeptime)) +
                     "s before network join")
        time.sleep(sleeptime)

        # First try to add seeds
        if self.port < 5003:
            self.send_mutual_add_requests(config['seeds'])

        # Then get random sample of peers from them
        peer_samples = self.get_sample_of_peers_from(config['seeds'])

        # Then add those peers
        self.send_mutual_add_requests(peer_samples)

        # TODO: Sync latest datastructures with peers (choosing the majority?)

        # Continuously try add new peers until my peerlist is above minimum size
        while True:
            time.sleep(4)  # TODO: Put in config
            if len(self.peers) < config['min_peers']:
                logger.debug(
                    "peerlist below minimum, trying to add more peers")
                peer_samples = self.get_sample_of_peers_from(self.peers)
                self.send_mutual_add_requests(peer_samples)
                self.ready = False
            else:
                self.ready = True
            if len(self.peers) < 1:
                logger.debug("no peers! adding seeds again")
                peer_samples = self.get_sample_of_peers_from(config['seeds'])
                self.send_mutual_add_requests(peer_samples)
Esempio n. 15
0
def validate_tick(tick,
                  previous_tick=None,
                  possible_previous_ticks=None,
                  verbose=True):
    # Doing validation on a copy so that the original keeps its "this_tick" ref
    # Otherwise the tick dict will be modified by any operations done here
    tick_copy = copy.deepcopy(tick)
    prev_tick_copy = copy.deepcopy(previous_tick)

    # This is used to keep track of the hash of the tick as debug information
    # Popping it off as it is not supposed to be an actual part of a tick
    tick_copy.pop('this_tick', None)

    if not validate_schema(tick_copy, 'tick_schema.json'):
        logger.debug("Tick failed schema validation")
        return False

    # Check hash and sig keeping in mind signature might be popped off
    if not validate_sig_hash(tick_copy):
        logger.debug("Tick failed signature and hash checking")
        return False

    if previous_tick is not None:
        if tick_copy['height'] != prev_tick_copy['height'] + 1:
            logger.debug("Tick failed height check")
            return False

    if possible_previous_ticks is not None:
        if not tick_copy['prev_tick'] in possible_previous_ticks:
            logger.debug(
                "Tick failed referencing any 1 of prev possible ticks")
            return False

    # TODO: This forces lower bound, but should also include upper bound?
    if not validate_tick_timediff(prev_tick_copy):  # Verbose: fails often
        logger.debug("Tick failed minimum timediff check") if verbose else None
        return False

    # Check all pings in list
    for ping in tick_copy['list']:
        # TODO: Check if tick's pings are in my own pool?
        # TODO: So they dont just send any random pings
        valid_ping = validate_ping(ping)
        if not valid_ping:
            logger.debug("tick invalid due to containing invalid ping")
            return False

    return True
Esempio n. 16
0
 async def logging_for_sanic(request, response):
     logger.debug(request.ip + " " + request.method + " " +
                  request.path + ": [" + str(response.status) + "] " +
                  response.body.decode('utf-8').replace('\\', ''))
Esempio n. 17
0
    def tick_worker(self):
        while True:
            # added_ping acts as a switch between "pingmode" and "tickmode"
            if self.networker.ready and self.added_ping:
                # Always construct tick in the following order:
                # 1) Init 2) Mine+nonce 3) Add signature
                # This is because the order of nonce and sig creation matters
                cycle_time = config['cycle_time']
                cycle_multiplier = config['cycle_time_multiplier']

                # Dynamic adjusting of sleeping time to match network lockstep
                prev_tick_ts = median_ts(
                    self.clockchain.latest_selected_tick())
                desired_ts = prev_tick_ts + cycle_multiplier * cycle_time

                wait_time = desired_ts - utcnow()

                logger.debug("Median ts: " + str(prev_tick_ts) + " min ts: " +
                             str(desired_ts) + " curr ts: " + str(utcnow()))

                overshoot = 0

                if wait_time < 0:
                    if self.clockchain.current_height(
                    ) != 0:  # If init, ignore
                        overshoot = -wait_time
                    logger.debug("Overshoot of " + str(int(overshoot)) + "s")
                    wait_time = 0

                logger.debug("Adjusted sleeping time: " + str(int(wait_time)))
                time.sleep(wait_time)  # Adjusting to follow network timing

                logger.debug(
                    "Tick stage--------------------------------------")
                start = time.time()  # Start and end time used to adjust sleep

                self.networker.stage = "tick"

                self.generate_and_process_tick()

                # All in all, there should be a total sleep of
                # 'cycle_time_multiplier' * 'cycle_time' in this thread.
                # Gets adjusted dynamically by wait_time mechanism above
                end = time.time()

                # Overshoot is used if we slept too long in ping stage,
                # then we compensate in this tick stage by speeding up sleep
                second_sleep = cycle_time - (end - start) - overshoot
                second_sleep = 0 if second_sleep < 0 else second_sleep

                time.sleep(second_sleep)  # 2nd sleep

                logger.debug(
                    "Vote stage--------------------------------------")
                start = time.time()

                self.networker.stage = "vote"
                # Use a ping to vote for highest continuity tick in tick_pool

                # TODO: What happens if I just selfishly vote for my own tick?
                active_tick_ref = self.clockchain.current_tick_ref()

                self.generate_and_process_ping(active_tick_ref, vote=True)

                logger.debug("Voted for: " + str(active_tick_ref))

                end = time.time()

                inbetween_sleep = cycle_time / 2
                time.sleep(inbetween_sleep)  # 2.5th sleep

                # Clearing ping_pool here already to possibly receive new pings
                self.clockchain.ping_pool = {}

                third_sleep = cycle_time - inbetween_sleep - (end - start)
                third_sleep = 0 if third_sleep < 0 else third_sleep
                time.sleep(third_sleep)  # 3rd sleep

                logger.debug(
                    "Select ticks stage------------------------------")
                self.networker.stage = "select"

                self.clockchain.select_highest_voted_to_chain()
                # TODO: If nothing was added to chain.. sth obv. wrong! Resync?

                self.added_ping = False
            else:
                time.sleep(1)
Esempio n. 18
0
def handle_exception(exception):
    logger.exception("Exception of type " + str(type(exception)) +
                     " occurred, see log for more details", exc_info=False)
    logger.debug(traceback.format_exc())
Esempio n. 19
0
 def after(response):
     logger.debug(request.remote_addr + " " + request.method + " " +
                  request.path + ": [" + str(response.status_code) +
                  "] " + response.get_data().decode("utf-8").rstrip())
     return response
Esempio n. 20
0
    def run(self):
        # Download the flow from NodeRed container and parse.
        s_time = time.time()
        while True:
            if self._parse_nodered_flow():
                break
            else:
                if time.time() - s_time > NODERED_FLOW_TIMEOUT * 60:
                    logger.critical('Failed to download flow, exiting...')
                    return
                else:
                    time.sleep(1)

        s_time = time.time()
        model_name = self.config.get('model_name')
        detect_mode = str(self.config.get('detect_mode', '')).lower()
        model_url = self.config.get('custom_model_url')
        while True:
            model_dir = download_model(
                model_name=model_name,
                device=detect_mode,
                model_url=model_url
                if self.config.get('public_model', True) is False else None)
            if model_dir:
                break
            else:
                if time.time() - s_time > DOWNLOAD_MODEL_TIMEOUT * 60:
                    logger.critical('Failed to download model, existing...')
                    return
                else:
                    time.sleep(.1)

        if detect_mode == 'gpu':
            if model_name == 'yolov3':
                from utils.object_detect_gpu_yolov3 import VisoGPUODYoloV3
                detector = VisoGPUODYoloV3(model_dir=model_dir)
            else:
                from utils.object_detect_gpu import VisoGPUOD
                detector = VisoGPUOD(model_dir=model_dir)
        else:
            if model_name == 'yolov3':
                from utils.openvino_detect_yolov3 import OpenVINODetectYOLOV3
                detector = OpenVINODetectYOLOV3(
                    model_dir=model_dir,
                    device='MYRIAD' if detect_mode == 'ncs' else 'CPU')
            else:
                if model_url:
                    from utils.object_detect_gpu import VisoGPUOD
                    detector = VisoGPUOD(model_dir=model_dir)
                else:
                    from utils.openvino_detect import OpenVinoObjectDetect
                    detector = OpenVinoObjectDetect(
                        model_dir=model_dir,
                        device='MYRIAD' if detect_mode == 'ncs' else 'CPU')

        tracking_mode = str(self.config.get('tracking_algorithm')).upper()
        tracking_quality = float(self.config.get('tracking_quality', 5))
        tracking_cycle = int(self.config.get('tracking_cycle', 2))

        trackers = []
        cnts = []
        for vid_src_id in range(len(self.sources)):
            trackers.append(
                ObjectTrack(trk_type=tracking_mode,
                            good_track_quality=tracking_quality))
            cnts.append(0)

        logger.info("Starting detection loop...")
        r = redis.StrictRedis()
        while True:
            for vid_src_id, src in enumerate(self.sources):
                str_frame = r.get(f"{REDIS_PREFIX}_{src.get('id')}")
                if str_frame:
                    str_frame = base64.b64decode(str_frame)
                    frame = cv2.imdecode(
                        np.fromstring(str_frame, dtype=np.uint8), -1)
                    h, w = frame.shape[:2]
                    if cnts[vid_src_id] % tracking_cycle == 0:
                        result = detector.detect_frame(frame)
                        filtered_objects = [
                            r for r in result
                            if r['label'] in self.config.get('labels', [])
                        ]
                        # FIXME: Remove this!
                        if filtered_objects:
                            logger.debug(filtered_objects)
                        cnts[vid_src_id] = 0
                        trackers[vid_src_id].upgrade_trackers(
                            dets=filtered_objects, trk_img=frame)
                    else:
                        trackers[vid_src_id].keep_trackers(trk_img=frame)
                    cnts[vid_src_id] += 1

                    result = trackers[vid_src_id].to_list()

                    roi_result = [
                        x for x in result if not src.get('roi_list', [])
                        or  # ROI is not defined?
                        any([
                            cv2.pointPolygonTest(
                                np.array([cnt], dtype=np.int32),
                                ((x['rect'][0] + x['rect'][2] // 2) * w,
                                 (x['rect'][1] + x['rect'][3] // 2) *
                                 h),  # Center point
                                False) >= 0 for cnt in src.get('roi_list', [])
                        ])
                    ]
                    if roi_result:
                        logger.info(
                            f'Detected Object from {src.get("id")}({src.get("name")}) - {roi_result}'
                        )
                        client.publish(
                            topic=f"{MQTT_PREFIX}_{self.config['id']}",
                            payload=json.dumps({
                                "camera_id": src.get("id"),
                                "result": roi_result
                            }))
                        if SAVE_IMAGE and time.time(
                        ) - self._last_saved_time[vid_src_id] > 60:
                            f_name = os.path.join(
                                SAVE_PATH,
                                f"{src.get('id')}_{datetime.datetime.now().isoformat()}.jpg"
                            )
                            logger.debug(f"Saving to a file - {f_name}")
                            for r in roi_result:
                                _x, _y, _w, _h = (
                                    np.array(r['rect']) *
                                    np.array([w, h, w, h])).astype(
                                        np.int).tolist()
                                cv2.rectangle(frame, (_x, _y),
                                              (_x + _w, _y + _h), (0, 255, 0),
                                              2)
                                cv2.putText(
                                    frame,
                                    f'{r["label"]} {round(r["confidence"] * 100, 1)} %',
                                    (_x, _y - 7), cv2.FONT_HERSHEY_COMPLEX,
                                    0.6, (0, 255, 0), 1)
                            cv2.imwrite(f_name, frame)
                            self._last_saved_time[vid_src_id] = time.time()