def insert_appliance(estimation, type_id, appliance_id, name, power):
session.add(
Appliance(appliance_id=safe_cast(int, appliance_id),
appliance_type_id=safe_cast(int, type_id),
appliance_name=name,
estimated_power=safe_cast(float, power),
estimation=estimation))
def insert_data(session, created_date, monitor_id, row):
new_estimation = insert_estimation(session, created_date, monitor_id, row)
row_len = len(row)
counter = MIN_NUM_DATA_COL - 1
while (counter + NUM_APPLIANCE_COL) < row_len:
insert_appliance(
new_estimation,
safe_cast(int, row[counter]), # Appliance Type ID
safe_cast(int, row[counter + 1]), # Appliance ID
row[counter + 2], # Appliance Name
row[counter + 3] # estimated power
)
counter += NUM_APPLIANCE_COL
def command_response(body, message):
"""
Process a command received from an actuator
:param body: command body
:param message: complete message (headers, meta, ...)
:return: None
"""
log.info(msg=f'Message response received: {body}')
headers = getattr(message, "headers", {})
actuator = None
if headers.get('error', False):
correlation_ID = headers['source'].get('correlationID', '')
opts = {
'_coap_id' if isHex(correlation_ID) else 'command_id':
correlation_ID
}
command = get_or_none(SentHistory, **opts)
log.error(msg=f'Message Failure: cmd - {command.command_id}, {body}')
response = {'error': body}
else:
act_host, act_port = headers.get('socket', '').split(':')[0:2]
correlation_ID = headers.get('correlationID', '')
opts = {
'_coap_id' if isHex(correlation_ID) else 'command_id':
correlation_ID
}
command = get_or_none(SentHistory, **opts)
profile = headers.get('profile', '')
encode = headers.get('encode', 'json')
response = decode_msg(body, encode)
actuator = get_or_none(
model=Actuator,
profile__iexact=profile,
device__transport__host__iexact=act_host,
device__transport__port=safe_cast(act_port, int),
device__transport__protocol=get_or_none(Protocol,
name__iexact=headers.get(
'transport', '')))
if hasattr(actuator, '__iter__'):
log.warn(
msg=
f'Multiple actuators match for command response - {command.command_id}'
)
actuator = random.choice(actuator)
try:
cmd_rsp = ResponseHistory(command=command,
actuator=actuator,
response=response)
cmd_rsp.save()
except Exception as e:
log.error(msg=f'Message response failed to save: {e}')
def get_dist2coast_list(filename='data/dist2coast.txt'):
locations = list()
with open(filename, 'r') as f:
for line in f:
words = line.split()
if len(words) < 3: continue
locations.append(safe_cast(words[2], int, 0))
return locations
def insert_sensor(session, monitor_id, row):
monitor = session.query(Monitor).get(monitor_id)
try:
session.add(
EnvironmentalSensor(
id=f'{monitor_id}_{row[0]}',
timestamp=datetime.utcfromtimestamp(int(row[0])),
time=datetime.strptime(row[1], '%Y/%m/%d %H:%M'),
temperature=safe_cast(float, row[2]),
humidity=safe_cast(int, row[3]),
co2_concentration=safe_cast(int, row[4]),
sound_pressure=safe_cast(int, row[5]),
air_pressure=safe_cast(float, row[6]),
monitor=monitor,
))
except IntegrityError:
session.rollback()
def _get_pagination_infos(article_infos, page_num):
page_num = safe_cast(page_num, int, 1)
return {
'current_page': page_num,
'total_page': article_infos['page_count'],
'has_prev': 1 < page_num <= article_infos['page_count'],
'has_next': 1 <= article_infos['page_count'] > page_num,
}
def on_StepPlotButton_clicked(self):
self.logger.info('.on_StepPlotButton_clicked() entered')
day = safe_cast(self.dayEdit.text().strip(), int, 0)
if not (day == 0):
dataset = self.get_dataset(self.repository, day)
self.plot_instance = PlotForm(self.logger, dataset)
self.plot_instance.show()
else:
self.logger.error('dayEdit.text is not int value!')
def grab_data_for_specific_ship(ships, mmsi):
TYPE = 2
FLAG = 1
NAME = 0
DESTINATION = 3
NAV_STATUS = 4
INFO_FOUND = 5
NR_SHIP_INFO_FIELDS = 6
url = "http://www.myshiptracking.com/requests/vesseldetails.php?type=json&mmsi="+str(mmsi)
# print(url)
ships['infoSearch'].remove(mmsi)
ships['infoModified'].add(mmsi)
results = [""] * NR_SHIP_INFO_FIELDS
results[INFO_FOUND] = 0 # we add the ship to ship_info also if it is not found here, with zero `info_found` flag.
ships['information'][mmsi] = results
response = httpPool.request('GET', url)
if response.status is not 200:
print('myshiptracking ship_info: status is not 200. url: %s' % url, file=sys.stderr)
return None
try:
j = json.loads(response.data.decode('utf-8'))
# TODO: maybe try to change the sleep time here
except Exception:
print('myshiptracking ship_info: json parse error. url: %s' % url, file=sys.stderr)
return None
if type(j) != dict or "V" not in j or type(j["V"]) != dict:
print('myshiptracking ship_info: json parse error. no `V`. url: %s' % url, file=sys.stderr)
return None
j = j["V"]
try:
results[TYPE] = safe_cast(j["VESSEL_TYPE"], str)
results[FLAG] = safe_cast(j["FLAG"], str)
results[NAME] = safe_cast(j["NAME"], str)
results[DESTINATION] = safe_cast(j["DESTINATION"], str)
results[NAV_STATUS] = safe_cast(j["NAV_STATUS"], str)
results[INFO_FOUND] = 1
except Exception:
return None
def grab_data_from_shipfinder(ships):
lastUpdate = ships['last_update']
shipSet = ships['avail']
# print(shipSet)
url = "http://shipfinder.co/endpoints/shipDeltaUpdate.php"
response = httpPool.request('GET', url)
if response.status is not 200:
print('shipfinder grab status no 200 err. url: %s'%url, file=sys.stderr)
return None
try:
str = response.data.decode('utf-8')
str = str[str.index('{'):]
j = json.loads(str)
# TODO: maybe try to change the sleep time here
except Exception:
print('shipfinder grab json parse err. url: %s'%url, file=sys.stderr)
return None
ships_data = j['ships']
if ships_data is None or len(ships_data) < 1:
return None
# for statics
nr_items = len(ships_data)
nr_new_items = 0
nr_updated_items = 0
now = datetime.now(il_tz)
for mmsi, vals in ships_data.items():
mmsi = safe_cast(mmsi, int)
if mmsi is None:
print('shipfinder None mmsi . url: %s'%url, file=sys.stderr)
continue
if mmsi not in lastUpdate:
nr_new_items += 1
lastUpdate[mmsi] = dict()
elif lastUpdate[mmsi][REPORTED_UPDATE_TIME] == safe_cast(vals[REPORTED_UPDATE_TIME], int):
lastUpdate[mmsi][GRABBER_UPDATE_TIME] = now
continue
else:
nr_updated_items += 1
if mmsi not in shipSet:
ships['avail'].add(mmsi)
ships['infoSearch'].add(mmsi)
ship = lastUpdate[mmsi]
ship[LAT] = safe_cast(vals[LAT], float)
ship[LONG] = safe_cast(vals[LONG], float)
ship[SPEED] = safe_cast(vals[SPEED], float)
ship[COURSE] = safe_cast(vals[COURSE], float)
ship[REPORTED_UPDATE_TIME] = safe_cast(vals[REPORTED_UPDATE_TIME], int)
ship[GRABBER_UPDATE_TIME] = now
ship[DATA_SOURCE] = SHIPFINDER_DATA_SOURCE
ships['modified'].add(mmsi) # mark it as changed so a db update will occur
return {'nr_items': nr_items, 'nr_new_items': nr_new_items, 'nr_updated_items': nr_updated_items}
def convert_from(self, to_model, data):
if safe_cast(data.get('user_id', 0), int):
to_model.user_id = int(data['user_id'])
else:
to_model.user = None
if safe_cast(data.get('article_id', 0), int):
to_model.article_id = int(data['article_id'])
else:
to_model.article = None
if safe_cast(data.get('parent_id', 0), int):
to_model.parent_id = int(data['parent_id'])
else:
to_model.comment = None
to_model.author = data.get('author', '')
to_model.author_email = data.get('author_email', '')
to_model.author_website = data.get('author_website', '')
to_model.author_ip = data.get('author_ip', '')
to_model.author_agent = data.get('author_agent', '')
to_model.comment_date = timestamp2datetime(data.get('comment_date', 0), convert_to_local=True)
to_model.content = data.get('content', '')
to_model.status = 1
return to_model
def number_factory(line):
street_id, locality_id, numero, rep, lon, lat = line[:-1].split(SEPARATOR)
street_id = int(street_id)
locality_id = int(locality_id)
numero = utils.safe_cast(numero, int, -1) # fixme
if numero > 2**16 - 1:
raise Exception('Error : numero overflows 16bits')
return (
street_id,
locality_id,
numero,
rep,
utils.geohash(float(lon), float(lat)),
)
def convert_from(self, to_model, data):
if safe_cast(data.get('user_id', 0), int):
to_model.user_id = int(data['user_id'])
else:
to_model.user = None
if safe_cast(data.get('article_id', 0), int):
to_model.article_id = int(data['article_id'])
else:
to_model.article = None
if safe_cast(data.get('parent_id', 0), int):
to_model.parent_id = int(data['parent_id'])
else:
to_model.comment = None
to_model.author = data.get('author', '')
to_model.author_email = data.get('author_email', '')
to_model.author_website = data.get('author_website', '')
to_model.author_ip = data.get('author_ip', '')
to_model.author_agent = data.get('author_agent', '')
to_model.comment_date = timestamp2datetime(data.get('comment_date', 0),
convert_to_local=True)
to_model.content = data.get('content', '')
to_model.status = 1
return to_model
def listMoreVideo(google_api_request_url,label_name,type_):
from domains import ClassYoutube
from utils import safe_cast
#https://www.googleapis.com/youtube/v3/playlistItems?part=snippet&type=video&maxResults=50&key=AIzaSyCqvYW8NI-OpMPaWR1DuZYW_llpmFdHRBI&playlistId=PLJ8cMiYb3G5eJquaiw6Xlyt3Qhe-3e7Xh
yt=ClassYoutube(google_api_request_url)#this will be a broken class because it won't be able to parse
just_numbers_from_label=re.sub("[^0-9]", "", label_name) #Removes all non-numeric characters "Page 2" becomes "2"
page_indicator=safe_cast(just_numbers_from_label,int,default=0)
links=yt.get_video_list(None,None,google_api_request_url,page_indicator)
yt.assemble_images_dictList(links)
links_dictList=yt.dictList
if links_dictList:
dictlist_to_RelatedVideo_gui(links_dictList, google_api_request_url, 'more', label_name, type_)
else:
xbmc_notify('Nothing to list', google_api_request_url)
def action_send(usr: User, cmd: dict, actuator: str, channel: dict):
"""
Process a command prior to sending it to the specified actuator(s)/profile
:param usr: user sending command
:param cmd: OpenC2 command
:param actuator: actuator/profile receiving command
:param channel: serialization & protocol to send the command
:return: response Tuple(dict, int)
"""
val = Validator(usr, cmd, actuator, channel)
acts, protocol, serialization = val.validate()
(actuators, fmt) = acts
# Store command in db
if "id" in cmd:
cmd_id = cmd.get("id", uuid.uuid4())
try:
cmd_id = uuid.UUID(cmd_id, version=4)
except ValueError:
cmd_id = uuid.uuid4()
cmd["id"] = str(cmd_id)
else:
cmd_id = uuid.uuid4()
if get_or_none(SentHistory, command_id=cmd_id):
return dict(
command_id=[
"This ID is used by another command."
]
), 400
com = SentHistory(command_id=cmd_id, user=usr, command=cmd)
try:
com.save()
except ValueError as e:
return dict(
detail="command error",
response=str(e)
), 400
# Process Actuators that should receive command
processed_acts = set()
# Process Protocols
for proto in [protocol] if protocol else Protocol.objects.all():
proto_acts = [a for a in actuators if a.device.transport.filter(protocol__name=proto.name).exists()]
proto_acts = list(filter(lambda a: a.id not in processed_acts, proto_acts))
processed_acts.update({act.id for act in proto_acts})
if len(proto_acts) >= 1:
if proto.name.lower() == "coap" and com.coap_id == b"":
com.gen_coap_id()
com.save()
# Send command to transport
log.info(usr=usr, msg=f"Send command {com.command_id}/{com.coap_id.hex()} to buffer")
settings.MESSAGE_QUEUE.send(
msg=json.dumps(cmd),
headers=get_headers(proto, com, proto_acts, serialization, fmt),
routing_key=proto.name.lower().replace(" ", "_")
)
wait = safe_cast(global_preferences.get("command__wait", 1), int, 1)
rsp = None
for _ in range(wait):
rsp = get_or_none(ResponseHistory, command=com)
if rsp:
break
time.sleep(1)
rsp = [r.response for r in rsp] if hasattr(rsp, "__iter__") else ([rsp.response] if hasattr(rsp, "response") else None)
return dict(
detail=f"command {'received' if rsp is None else 'processed'}",
response=rsp if rsp else "pending",
command_id=com.command_id,
command=com.command,
wait=wait
), 200
def _get_start_index(page_num):
page_num = safe_cast(page_num, int, 1)
return settings.LIST_PER_PAGE * (page_num - 1)
def action_send(usr=None, cmd={}, actuator="", channel={}):
"""
Process a command prior to sending it to the specified actuator(s)/profile
:param usr: user sending command
:param cmd: OpenC2 command
:param actuator: actuator/profile receiving command
:param channel: serialization & protocol to send the command
:return: response dict
"""
err = validate_usr(usr)
if err:
return err
err = validate_cmd(usr, cmd)
if err:
return err
actuators = None
err = validate_actuator(usr, actuator)
if err:
if isinstance(err, (Actuator, list)):
actuators = err
else:
return err
protocol, serialization = validate_channel(actuators, channel)
# Store command in db
cmd_id = cmd.get("id", uuid.uuid4())
if get_or_none(SentHistory, command_id=cmd_id):
return dict(command_id=["This ID is used by another command."]), 400
else:
com = SentHistory(command_id=cmd_id, user=usr, command=cmd)
try:
com.save()
except ValueError as e:
return dict(detail="command error", response=str(e)), 400
orc_ip = global_preferences.get("orchestrator__host", "127.0.0.1")
orc_id = global_preferences.get("orchestrator__id", "")
# Process Actuators that should receive command
processed_acts = set()
# Process Protocols
protocols = [protocol] if protocol else Protocol.objects.all()
for proto in protocols:
proto_acts = [
a for a in actuators
if a.device.transport.filter(protocol__name=proto.name).exists()
]
proto_acts = list(
filter(lambda a: a.id not in processed_acts, proto_acts))
processed_acts.update({act.id for act in proto_acts})
if len(proto_acts) >= 1:
if proto.name.lower() == "coap" and com.coap_id is b"":
corr_id = com.gen_coap_id()
com.save()
else:
corr_id = str(com.command_id)
header = dict(source=dict(
orchestratorID=orc_id,
transport=dict(type=proto.name,
socket=f"{orc_ip}:{proto.port}"),
correlationID=corr_id,
date=f"{com.received_on:%a, %d %b %Y %X %Z}"),
destination=[])
for act in proto_acts:
com.actuators.add(act)
trans = act.device.transport.filter(
protocol__name=proto.name).first()
encoding = (serialization if serialization else
trans.serialization.first()).name.lower()
dev = list(
filter(
lambda d: d["deviceID"] == str(act.device.device_id),
header["destination"]))
profile = str(act.profile).lower()
if len(dev) == 1:
idx = header["destination"].index(dev[0])
header["destination"][idx]["profile"].append(profile)
else:
dest = dict(deviceID=str(act.device.device_id),
socket=f"{trans.host}:{trans.port}",
profile=[profile],
encoding=encoding)
if trans.protocol.pub_sub:
print("PUB SUB")
header["destination"].append(dest)
# Send command to transport
log.info(
usr=usr,
msg=
f"Send command {com.command_id}/{com.coap_id.hex()} to buffer")
settings.MESSAGE_QUEUE.send(msg=json.dumps(cmd),
headers=header,
routing_key=proto.name.lower().replace(
" ", "_"))
wait = safe_cast(global_preferences.get("command__wait", 1), int, 1)
rsp = None
for _ in range(wait):
rsp = get_or_none(ResponseHistory, command=com)
if rsp:
break
else:
time.sleep(1)
rsp = [r.response for r in rsp] if hasattr(rsp, "__iter__") else (
[rsp.response] if hasattr(rsp, "response") else None)
return dict(detail=f"command {'received' if rsp is None else 'processed'}",
response=rsp if rsp else "pending",
command_id=com.command_id,
command=com.command,
wait=wait), 200
def grab_data_from_myshiptracking_cell(ships, min_lat, max_lat, min_lon, max_lon, zoom):
lastUpdate = ships['last_update']
shipSet = ships['avail']
url = 'http://www.myshiptracking.com/requests/vesselsonmap.php?type=json&minlat={}&maxlat={}&minlon={}&maxlon={}&zoom={}&mmsi=null'.format(
min_lat, max_lat, min_lon, max_lon, zoom
)
response = httpPool.request('GET', url)
if response.status is not 200:
print('myshiptracking grab 200 err. url: %s'%url, file=sys.stderr)
return None
try:
j = json.loads(response.data.decode('utf-8'))
# TODO: maybe try to change the sleep time here
except Exception:
print('myshiptracking grab json parse err. url: %s'%url, file=sys.stderr)
return None
if type(j) != list or len(j) < 1:
print('myshiptracking grab json parse err: no list. url: %s'%url, file=sys.stderr)
return None
if type(j[0]) != dict or 'DATA' not in j[0]:
print('myshiptracking grab json parse err: no `DATA` key in list. url: %s'%url, file=sys.stderr)
return None
ships_data = j[0]['DATA']
if ships_data is None or type(ships_data) != list:
print('myshiptracking grab json err: no list in `DATA`. url: %s'%url, file=sys.stderr)
return None
if len(ships_data) < 1:
#print('myshiptracking grab err: empty data', file=sys.stderr)
return None
# for statics
nr_items = len(ships_data)
nr_new_items = 0
nr_updated_items = 0
fields = {
'COG': COURSE,
'SOG': SPEED,
'R': REPORTED_UPDATE_TIME,
'LAT': LAT,
'LNG': LONG
}
now = datetime.now(il_tz)
for item in ships_data:
if type(item) != dict:
print('myshiptracking grab err: ship item is not a dict', file=sys.stderr)
continue
if 'MMSI' not in item:
print('myshiptracking grab err: no mmsi for record', file=sys.stderr)
continue
mmsi = safe_cast(item['MMSI'], int)
if mmsi is None:
print('myshiptracking grab err: invalid mmsi (not int)', file=sys.stderr)
continue
if len(set(fields) - set(item)) > 0:
print('myshiptracking grab err: no all req fields', file=sys.stderr)
continue
vals = [0]*NR_SHIP_FIELDS
for field_name, to_idx in fields.items():
vals[to_idx] = item[field_name]
if mmsi not in lastUpdate:
nr_new_items += 1
lastUpdate[mmsi] = dict()
elif lastUpdate[mmsi][REPORTED_UPDATE_TIME] == safe_cast(vals[REPORTED_UPDATE_TIME], int):
lastUpdate[mmsi][GRABBER_UPDATE_TIME] = now
continue
else:
nr_updated_items += 1
if mmsi not in shipSet:
ships['avail'].add(mmsi)
ships['infoSearch'].add(mmsi)
ship = lastUpdate[mmsi]
ship[LAT] = safe_cast(vals[LAT], float)
ship[LONG] = safe_cast(vals[LONG], float)
ship[SPEED] = safe_cast(vals[SPEED], float)
ship[COURSE] = safe_cast(vals[COURSE], float)
ship[REPORTED_UPDATE_TIME] = safe_cast(vals[REPORTED_UPDATE_TIME], int)
ship[GRABBER_UPDATE_TIME] = now
ship[DATA_SOURCE] = MYSHIPTRACKING_DATA_SOURCE
ships['modified'].add(mmsi) # mark it as changed so a db update will occur
return {'nr_items': nr_items, 'nr_new_items': nr_new_items, 'nr_updated_items': nr_updated_items}