Пример #1
0
def insert_appliance(estimation, type_id, appliance_id, name, power):
    session.add(
        Appliance(appliance_id=safe_cast(int, appliance_id),
                  appliance_type_id=safe_cast(int, type_id),
                  appliance_name=name,
                  estimated_power=safe_cast(float, power),
                  estimation=estimation))
Пример #2
0
def insert_data(session, created_date, monitor_id, row):
    new_estimation = insert_estimation(session, created_date, monitor_id, row)
    row_len = len(row)
    counter = MIN_NUM_DATA_COL - 1
    while (counter + NUM_APPLIANCE_COL) < row_len:
        insert_appliance(
            new_estimation,
            safe_cast(int, row[counter]),  # Appliance Type ID
            safe_cast(int, row[counter + 1]),  # Appliance ID
            row[counter + 2],  # Appliance Name
            row[counter + 3]  # estimated power
        )
        counter += NUM_APPLIANCE_COL
def command_response(body, message):
    """
    Process a command received from an actuator
    :param body: command body
    :param message: complete message (headers, meta, ...)
    :return: None
    """
    log.info(msg=f'Message response received: {body}')
    headers = getattr(message, "headers", {})
    actuator = None

    if headers.get('error', False):
        correlation_ID = headers['source'].get('correlationID', '')
        opts = {
            '_coap_id' if isHex(correlation_ID) else 'command_id':
            correlation_ID
        }

        command = get_or_none(SentHistory, **opts)
        log.error(msg=f'Message Failure: cmd - {command.command_id}, {body}')

        response = {'error': body}

    else:
        act_host, act_port = headers.get('socket', '').split(':')[0:2]
        correlation_ID = headers.get('correlationID', '')
        opts = {
            '_coap_id' if isHex(correlation_ID) else 'command_id':
            correlation_ID
        }

        command = get_or_none(SentHistory, **opts)
        profile = headers.get('profile', '')

        encode = headers.get('encode', 'json')
        response = decode_msg(body, encode)

        actuator = get_or_none(
            model=Actuator,
            profile__iexact=profile,
            device__transport__host__iexact=act_host,
            device__transport__port=safe_cast(act_port, int),
            device__transport__protocol=get_or_none(Protocol,
                                                    name__iexact=headers.get(
                                                        'transport', '')))

        if hasattr(actuator, '__iter__'):
            log.warn(
                msg=
                f'Multiple actuators match for command response - {command.command_id}'
            )
            actuator = random.choice(actuator)

    try:
        cmd_rsp = ResponseHistory(command=command,
                                  actuator=actuator,
                                  response=response)
        cmd_rsp.save()
    except Exception as e:
        log.error(msg=f'Message response failed to save: {e}')
Пример #4
0
def get_dist2coast_list(filename='data/dist2coast.txt'):
    locations = list()
    with open(filename, 'r') as f:
        for line in f:
            words = line.split()
            if len(words) < 3: continue
            locations.append(safe_cast(words[2], int, 0))
    return locations
def insert_sensor(session, monitor_id, row):
    monitor = session.query(Monitor).get(monitor_id)
    try:
        session.add(
            EnvironmentalSensor(
                id=f'{monitor_id}_{row[0]}',
                timestamp=datetime.utcfromtimestamp(int(row[0])),
                time=datetime.strptime(row[1], '%Y/%m/%d %H:%M'),
                temperature=safe_cast(float, row[2]),
                humidity=safe_cast(int, row[3]),
                co2_concentration=safe_cast(int, row[4]),
                sound_pressure=safe_cast(int, row[5]),
                air_pressure=safe_cast(float, row[6]),
                monitor=monitor,
            ))
    except IntegrityError:
        session.rollback()
Пример #6
0
def _get_pagination_infos(article_infos, page_num):
    page_num = safe_cast(page_num, int, 1)

    return {
        'current_page': page_num,
        'total_page': article_infos['page_count'],
        'has_prev': 1 < page_num <= article_infos['page_count'],
        'has_next': 1 <= article_infos['page_count'] > page_num,
    }
Пример #7
0
 def on_StepPlotButton_clicked(self):
     self.logger.info('.on_StepPlotButton_clicked() entered')
     day = safe_cast(self.dayEdit.text().strip(), int, 0)
     if not (day == 0):
         dataset = self.get_dataset(self.repository, day)
         self.plot_instance = PlotForm(self.logger, dataset)
         self.plot_instance.show()
     else:
         self.logger.error('dayEdit.text is not int value!')
Пример #8
0
def grab_data_for_specific_ship(ships, mmsi):
    TYPE = 2
    FLAG = 1
    NAME = 0
    DESTINATION = 3
    NAV_STATUS = 4
    INFO_FOUND = 5
    NR_SHIP_INFO_FIELDS = 6

    url = "http://www.myshiptracking.com/requests/vesseldetails.php?type=json&mmsi="+str(mmsi)
    # print(url)

    ships['infoSearch'].remove(mmsi)
    ships['infoModified'].add(mmsi)
    results = [""] * NR_SHIP_INFO_FIELDS
    results[INFO_FOUND] = 0  # we add the ship to ship_info also if it is not found here, with zero `info_found` flag.
    ships['information'][mmsi] = results

    response = httpPool.request('GET', url)
    if response.status is not 200:
        print('myshiptracking ship_info: status is not 200. url: %s' % url, file=sys.stderr)
        return None
    try:
        j = json.loads(response.data.decode('utf-8'))
        # TODO: maybe try to change the sleep time here
    except Exception:
        print('myshiptracking ship_info: json parse error. url: %s' % url, file=sys.stderr)
        return None

    if type(j) != dict or "V" not in j or type(j["V"]) != dict:
        print('myshiptracking ship_info: json parse error. no `V`. url: %s' % url, file=sys.stderr)
        return None

    j = j["V"]
    try:
        results[TYPE] = safe_cast(j["VESSEL_TYPE"], str)
        results[FLAG] = safe_cast(j["FLAG"], str)
        results[NAME] = safe_cast(j["NAME"], str)
        results[DESTINATION] = safe_cast(j["DESTINATION"], str)
        results[NAV_STATUS] = safe_cast(j["NAV_STATUS"], str)
        results[INFO_FOUND] = 1
    except Exception:
        return None
Пример #9
0
def grab_data_from_shipfinder(ships):
    lastUpdate = ships['last_update']
    shipSet = ships['avail']
    # print(shipSet)
    url = "http://shipfinder.co/endpoints/shipDeltaUpdate.php"
    response = httpPool.request('GET', url)
    if response.status is not 200:
        print('shipfinder grab status no 200 err. url: %s'%url, file=sys.stderr)
        return None

    try:
        str = response.data.decode('utf-8')
        str = str[str.index('{'):]
        j = json.loads(str)
        # TODO: maybe try to change the sleep time here
    except Exception:
        print('shipfinder grab json parse err. url: %s'%url, file=sys.stderr)
        return None

    ships_data = j['ships']
    if ships_data is None or len(ships_data) < 1:
        return None

    # for statics
    nr_items = len(ships_data)
    nr_new_items = 0
    nr_updated_items = 0

    now = datetime.now(il_tz)
    for mmsi, vals in ships_data.items():
        mmsi = safe_cast(mmsi, int)
        if mmsi is None:
            print('shipfinder None mmsi . url: %s'%url, file=sys.stderr)
            continue
        if mmsi not in lastUpdate:
            nr_new_items += 1
            lastUpdate[mmsi] = dict()
        elif lastUpdate[mmsi][REPORTED_UPDATE_TIME] == safe_cast(vals[REPORTED_UPDATE_TIME], int):
            lastUpdate[mmsi][GRABBER_UPDATE_TIME] = now
            continue
        else:
            nr_updated_items += 1
        if mmsi not in shipSet:
            ships['avail'].add(mmsi)
            ships['infoSearch'].add(mmsi)

        ship = lastUpdate[mmsi]
        ship[LAT] = safe_cast(vals[LAT], float)
        ship[LONG] = safe_cast(vals[LONG], float)
        ship[SPEED] = safe_cast(vals[SPEED], float)
        ship[COURSE] = safe_cast(vals[COURSE], float)
        ship[REPORTED_UPDATE_TIME] = safe_cast(vals[REPORTED_UPDATE_TIME], int)
        ship[GRABBER_UPDATE_TIME] = now
        ship[DATA_SOURCE] = SHIPFINDER_DATA_SOURCE
        ships['modified'].add(mmsi)  # mark it as changed so a db update will occur
    return {'nr_items': nr_items, 'nr_new_items': nr_new_items, 'nr_updated_items': nr_updated_items}
Пример #10
0
 def convert_from(self, to_model, data):
     if safe_cast(data.get('user_id', 0), int):
         to_model.user_id = int(data['user_id'])
     else:
         to_model.user = None
     if safe_cast(data.get('article_id', 0), int):
         to_model.article_id = int(data['article_id'])
     else:
         to_model.article = None
     if safe_cast(data.get('parent_id', 0), int):
         to_model.parent_id = int(data['parent_id'])
     else:
         to_model.comment = None
     to_model.author = data.get('author', '')
     to_model.author_email = data.get('author_email', '')
     to_model.author_website = data.get('author_website', '')
     to_model.author_ip = data.get('author_ip', '')
     to_model.author_agent = data.get('author_agent', '')
     to_model.comment_date = timestamp2datetime(data.get('comment_date', 0), convert_to_local=True)
     to_model.content = data.get('content', '')
     to_model.status = 1
     return to_model
Пример #11
0
def number_factory(line):
    street_id, locality_id, numero, rep, lon, lat = line[:-1].split(SEPARATOR)
    street_id = int(street_id)
    locality_id = int(locality_id)
    numero = utils.safe_cast(numero, int, -1)  # fixme
    if numero > 2**16 - 1:
        raise Exception('Error : numero overflows 16bits')
    return (
        street_id,
        locality_id,
        numero,
        rep,
        utils.geohash(float(lon), float(lat)),
    )
Пример #12
0
 def convert_from(self, to_model, data):
     if safe_cast(data.get('user_id', 0), int):
         to_model.user_id = int(data['user_id'])
     else:
         to_model.user = None
     if safe_cast(data.get('article_id', 0), int):
         to_model.article_id = int(data['article_id'])
     else:
         to_model.article = None
     if safe_cast(data.get('parent_id', 0), int):
         to_model.parent_id = int(data['parent_id'])
     else:
         to_model.comment = None
     to_model.author = data.get('author', '')
     to_model.author_email = data.get('author_email', '')
     to_model.author_website = data.get('author_website', '')
     to_model.author_ip = data.get('author_ip', '')
     to_model.author_agent = data.get('author_agent', '')
     to_model.comment_date = timestamp2datetime(data.get('comment_date', 0),
                                                convert_to_local=True)
     to_model.content = data.get('content', '')
     to_model.status = 1
     return to_model
Пример #13
0
def listMoreVideo(google_api_request_url,label_name,type_):
    from domains import ClassYoutube
    from utils import safe_cast

    #https://www.googleapis.com/youtube/v3/playlistItems?part=snippet&type=video&maxResults=50&key=AIzaSyCqvYW8NI-OpMPaWR1DuZYW_llpmFdHRBI&playlistId=PLJ8cMiYb3G5eJquaiw6Xlyt3Qhe-3e7Xh
    yt=ClassYoutube(google_api_request_url)#this will be a broken class because it won't be able to parse

    just_numbers_from_label=re.sub("[^0-9]", "", label_name) #Removes all non-numeric characters  "Page 2" becomes "2"
    page_indicator=safe_cast(just_numbers_from_label,int,default=0)
    links=yt.get_video_list(None,None,google_api_request_url,page_indicator)
    yt.assemble_images_dictList(links)

    links_dictList=yt.dictList

    if links_dictList:
        dictlist_to_RelatedVideo_gui(links_dictList, google_api_request_url, 'more', label_name, type_)
    else:
        xbmc_notify('Nothing to list', google_api_request_url)
Пример #14
0
def action_send(usr: User, cmd: dict, actuator: str, channel: dict):
    """
    Process a command prior to sending it to the specified actuator(s)/profile
    :param usr: user sending command
    :param cmd: OpenC2 command
    :param actuator: actuator/profile receiving command
    :param channel: serialization & protocol to send the command
    :return: response Tuple(dict, int)
    """
    val = Validator(usr, cmd, actuator, channel)
    acts, protocol, serialization = val.validate()
    (actuators, fmt) = acts

    # Store command in db
    if "id" in cmd:
        cmd_id = cmd.get("id", uuid.uuid4())
        try:
            cmd_id = uuid.UUID(cmd_id, version=4)
        except ValueError:
            cmd_id = uuid.uuid4()
            cmd["id"] = str(cmd_id)
    else:
        cmd_id = uuid.uuid4()

    if get_or_none(SentHistory, command_id=cmd_id):
        return dict(
            command_id=[
                "This ID is used by another command."
            ]
        ), 400

    com = SentHistory(command_id=cmd_id, user=usr, command=cmd)
    try:
        com.save()
    except ValueError as e:
        return dict(
            detail="command error",
            response=str(e)
        ), 400

    # Process Actuators that should receive command
    processed_acts = set()

    # Process Protocols
    for proto in [protocol] if protocol else Protocol.objects.all():
        proto_acts = [a for a in actuators if a.device.transport.filter(protocol__name=proto.name).exists()]
        proto_acts = list(filter(lambda a: a.id not in processed_acts, proto_acts))
        processed_acts.update({act.id for act in proto_acts})

        if len(proto_acts) >= 1:
            if proto.name.lower() == "coap" and com.coap_id == b"":
                com.gen_coap_id()
                com.save()

            # Send command to transport
            log.info(usr=usr, msg=f"Send command {com.command_id}/{com.coap_id.hex()} to buffer")
            settings.MESSAGE_QUEUE.send(
                msg=json.dumps(cmd),
                headers=get_headers(proto, com, proto_acts, serialization, fmt),
                routing_key=proto.name.lower().replace(" ", "_")
            )

    wait = safe_cast(global_preferences.get("command__wait", 1), int, 1)
    rsp = None
    for _ in range(wait):
        rsp = get_or_none(ResponseHistory, command=com)
        if rsp:
            break
        time.sleep(1)

    rsp = [r.response for r in rsp] if hasattr(rsp, "__iter__") else ([rsp.response] if hasattr(rsp, "response") else None)

    return dict(
        detail=f"command {'received' if rsp is None else 'processed'}",
        response=rsp if rsp else "pending",
        command_id=com.command_id,
        command=com.command,
        wait=wait
    ), 200
Пример #15
0
def _get_start_index(page_num):
    page_num = safe_cast(page_num, int, 1)

    return settings.LIST_PER_PAGE * (page_num - 1)
Пример #16
0
def action_send(usr=None, cmd={}, actuator="", channel={}):
    """
    Process a command prior to sending it to the specified actuator(s)/profile
    :param usr: user sending command
    :param cmd: OpenC2 command
    :param actuator: actuator/profile receiving command
    :param channel: serialization & protocol to send the command
    :return: response dict
    """
    err = validate_usr(usr)
    if err:
        return err

    err = validate_cmd(usr, cmd)
    if err:
        return err

    actuators = None
    err = validate_actuator(usr, actuator)
    if err:
        if isinstance(err, (Actuator, list)):
            actuators = err
        else:
            return err

    protocol, serialization = validate_channel(actuators, channel)

    # Store command in db
    cmd_id = cmd.get("id", uuid.uuid4())
    if get_or_none(SentHistory, command_id=cmd_id):
        return dict(command_id=["This ID is used by another command."]), 400
    else:
        com = SentHistory(command_id=cmd_id, user=usr, command=cmd)
        try:
            com.save()
        except ValueError as e:
            return dict(detail="command error", response=str(e)), 400

    orc_ip = global_preferences.get("orchestrator__host", "127.0.0.1")
    orc_id = global_preferences.get("orchestrator__id", "")
    # Process Actuators that should receive command
    processed_acts = set()

    # Process Protocols
    protocols = [protocol] if protocol else Protocol.objects.all()
    for proto in protocols:
        proto_acts = [
            a for a in actuators
            if a.device.transport.filter(protocol__name=proto.name).exists()
        ]
        proto_acts = list(
            filter(lambda a: a.id not in processed_acts, proto_acts))
        processed_acts.update({act.id for act in proto_acts})

        if len(proto_acts) >= 1:
            if proto.name.lower() == "coap" and com.coap_id is b"":
                corr_id = com.gen_coap_id()
                com.save()
            else:
                corr_id = str(com.command_id)

            header = dict(source=dict(
                orchestratorID=orc_id,
                transport=dict(type=proto.name,
                               socket=f"{orc_ip}:{proto.port}"),
                correlationID=corr_id,
                date=f"{com.received_on:%a, %d %b %Y %X %Z}"),
                          destination=[])

            for act in proto_acts:
                com.actuators.add(act)
                trans = act.device.transport.filter(
                    protocol__name=proto.name).first()
                encoding = (serialization if serialization else
                            trans.serialization.first()).name.lower()

                dev = list(
                    filter(
                        lambda d: d["deviceID"] == str(act.device.device_id),
                        header["destination"]))
                profile = str(act.profile).lower()

                if len(dev) == 1:
                    idx = header["destination"].index(dev[0])
                    header["destination"][idx]["profile"].append(profile)

                else:
                    dest = dict(deviceID=str(act.device.device_id),
                                socket=f"{trans.host}:{trans.port}",
                                profile=[profile],
                                encoding=encoding)
                    if trans.protocol.pub_sub:
                        print("PUB SUB")
                    header["destination"].append(dest)

            # Send command to transport
            log.info(
                usr=usr,
                msg=
                f"Send command {com.command_id}/{com.coap_id.hex()} to buffer")
            settings.MESSAGE_QUEUE.send(msg=json.dumps(cmd),
                                        headers=header,
                                        routing_key=proto.name.lower().replace(
                                            " ", "_"))

    wait = safe_cast(global_preferences.get("command__wait", 1), int, 1)
    rsp = None
    for _ in range(wait):
        rsp = get_or_none(ResponseHistory, command=com)
        if rsp:
            break
        else:
            time.sleep(1)

    rsp = [r.response for r in rsp] if hasattr(rsp, "__iter__") else (
        [rsp.response] if hasattr(rsp, "response") else None)

    return dict(detail=f"command {'received' if rsp is None else 'processed'}",
                response=rsp if rsp else "pending",
                command_id=com.command_id,
                command=com.command,
                wait=wait), 200
Пример #17
0
def grab_data_from_myshiptracking_cell(ships, min_lat, max_lat, min_lon, max_lon, zoom):
    lastUpdate = ships['last_update']
    shipSet = ships['avail']

    url = 'http://www.myshiptracking.com/requests/vesselsonmap.php?type=json&minlat={}&maxlat={}&minlon={}&maxlon={}&zoom={}&mmsi=null'.format(
        min_lat, max_lat, min_lon, max_lon, zoom
    )
    response = httpPool.request('GET', url)
    if response.status is not 200:
        print('myshiptracking grab 200 err. url: %s'%url, file=sys.stderr)
        return None

    try:
        j = json.loads(response.data.decode('utf-8'))
        # TODO: maybe try to change the sleep time here
    except Exception:
        print('myshiptracking grab json parse err. url: %s'%url, file=sys.stderr)
        return None

    if type(j) != list or len(j) < 1:
        print('myshiptracking grab json parse err: no list. url: %s'%url, file=sys.stderr)
        return None

    if type(j[0]) != dict or 'DATA' not in j[0]:
        print('myshiptracking grab json parse err: no `DATA` key in list. url: %s'%url, file=sys.stderr)
        return None

    ships_data = j[0]['DATA']
    if ships_data is None or type(ships_data) != list:
        print('myshiptracking grab json err: no list in `DATA`. url: %s'%url, file=sys.stderr)
        return None

    if len(ships_data) < 1:
        #print('myshiptracking grab err: empty data', file=sys.stderr)
        return None

    # for statics
    nr_items = len(ships_data)
    nr_new_items = 0
    nr_updated_items = 0

    fields = {
        'COG': COURSE,
        'SOG': SPEED,
        'R': REPORTED_UPDATE_TIME,
        'LAT': LAT,
        'LNG': LONG
    }

    now = datetime.now(il_tz)
    for item in ships_data:
        if type(item) != dict:
            print('myshiptracking grab err: ship item is not a dict', file=sys.stderr)
            continue

        if 'MMSI' not in item:
            print('myshiptracking grab err: no mmsi for record', file=sys.stderr)
            continue
        mmsi = safe_cast(item['MMSI'], int)
        if mmsi is None:
            print('myshiptracking grab err: invalid mmsi (not int)', file=sys.stderr)
            continue

        if len(set(fields) - set(item)) > 0:
            print('myshiptracking grab err: no all req fields', file=sys.stderr)
            continue

        vals = [0]*NR_SHIP_FIELDS
        for field_name, to_idx in fields.items():
            vals[to_idx] = item[field_name]

        if mmsi not in lastUpdate:
            nr_new_items += 1
            lastUpdate[mmsi] = dict()
        elif lastUpdate[mmsi][REPORTED_UPDATE_TIME] == safe_cast(vals[REPORTED_UPDATE_TIME], int):
            lastUpdate[mmsi][GRABBER_UPDATE_TIME] = now
            continue
        else:
            nr_updated_items += 1
        if mmsi not in shipSet:
            ships['avail'].add(mmsi)
            ships['infoSearch'].add(mmsi)

        ship = lastUpdate[mmsi]
        ship[LAT] = safe_cast(vals[LAT], float)
        ship[LONG] = safe_cast(vals[LONG], float)
        ship[SPEED] = safe_cast(vals[SPEED], float)
        ship[COURSE] = safe_cast(vals[COURSE], float)
        ship[REPORTED_UPDATE_TIME] = safe_cast(vals[REPORTED_UPDATE_TIME], int)
        ship[GRABBER_UPDATE_TIME] = now
        ship[DATA_SOURCE] = MYSHIPTRACKING_DATA_SOURCE
        ships['modified'].add(mmsi)  # mark it as changed so a db update will occur
    return {'nr_items': nr_items, 'nr_new_items': nr_new_items, 'nr_updated_items': nr_updated_items}