def eq(sms):
"""
- Process the sms message that fits for eq data.
:param sms: list data info of sms message .
:type sms: list
:returns: **Dataframe** - Return Dataframe structure output and if not return False for fail to parse message.
"""
pattern_matches = {}
# search for matches
for name in EQ_SMS_PATTERNS.keys():
search_results = EQ_SMS_PATTERNS[name].search(sms.msg)
if search_results:
pattern_matches[name] = search_results.group(0)
else:
print "No match for <%s> pattern." % (name),
print "Incomplete message not stored."
return False
print pattern_matches
# format date
datestr_init = pattern_matches["date"].upper()
pattern = ["%d%B%Y","%d%b%Y"]
datestr = None
for p in pattern:
try:
datestr = dt.strptime(datestr_init, p).strftime("%Y-%m-%d")
break
except:
pass
if datestr == None:
print ">> Error in datetime conversion for <%s>" % (datestr_init)
return False
# format time
timestr = pattern_matches["time"].replace(" ","").replace(".",":").upper()
try:
timestr = dt.strptime(timestr,"%I:%M%p").strftime("%H:%M:00")
except:
print ">> Error in datetime conversion", timestr
return False
del pattern_matches["date"]
del pattern_matches["time"]
pattern_matches["ts"] = "%s %s" % (datestr, timestr)
out = {}
for col_name in pattern_matches.keys():
out[col_name] = pattern_matches[col_name]
df = pd.DataFrame([out])
print df
return smsclass.DataTable("earthquake_events", df)
def stilt_parser(sms):
line = sms.msg
split_line = line.split('*')
logger_name = split_line[0]
indicator = split_line[1]
data = split_line[2]
data_split = data.split(',')
trans_data = pd.DataFrame(data_split).transpose()
trans_data = trans_data.rename(
columns={
0: "ac_x",
1: "ac_y",
2: "ac_z",
3: "mg_x",
4: "mg_y",
5: "mg_z",
6: "gr_x",
7: "gr_y",
8: "gr_z",
9: "temp",
10: "ts"
})
trans_data.ts[0] = dt.strptime(
trans_data.ts[0], '%y%m%d%H%M%S').strftime('%Y-%m-%d %H:%M:00')
trans_data["type"] = indicator
df = trans_data[[
"ts", "type", "ac_x", "ac_y", "ac_z", "mg_x", "mg_y", "mg_z", "gr_x",
"gr_y", "gr_z", "temp"
]]
name_df = "stilt_{}".format(logger_name.lower())
data = smsclass.DataTable(name_df, df)
return data
def ublox_parser(sms):
line = sms.msg
split_line = line.split('*')
data_line = split_line[0]
ts = dt.strptime(split_line[1],'%y%m%d%H%M%S').strftime('%Y-%m-%d %H:%M:00')
split_data = data_line.split(':')
logger_name = split_data[0]
ublox_data = split_data[1]
logger_data = split_data[2]
split_ublox_data = ublox_data.split(',')
split_logger_data = logger_data.split(',')
trans_ublox_data = pd.DataFrame(split_ublox_data).transpose()
trans_ublox_data = trans_ublox_data.rename(columns={0: "latitude",
1: "longitude",
2: "msl",
3: "prec"})
trans_ublox_data["ts"] = ts
trans_logger_data = pd.DataFrame(split_logger_data).transpose()
trans_logger_data = trans_logger_data.rename(columns={0: "temp", 1: "volt"})
df = pd.concat([trans_ublox_data, trans_logger_data],axis =1)
df = df[["ts","latitude","longitude", "msl", "prec","temp","volt"]]
name_df = "ublox_{}".format(logger_name.lower())
data = smsclass.DataTable(name_df,df)
return data
def uts(sms):
values = {}
print sms.msg
uts_name = re.search("^[A-Z]{5}(?=\*U\*)", sms.msg).group(0)
mc = mem.get_handle()
DF_EXTENSOMETERS = mc.get("DF_EXTENSOMETERS")
DATA_TABLE_NAME = "extensometer_uts_data"
x_id = DF_EXTENSOMETERS[DF_EXTENSOMETERS["extensometer_name"] ==
uts_name.lower()]["extensometer_type"]
x_id = int(x_id)
if x_id == 0:
return False
else:
values['extensometer_id'] = x_id
# uts_data = re.search("(?<=[A-Z]{5}\*U\*).*(?=\*[0-9]{12})", sms.msg).group(0).lower()
matches = re.findall('(?<=\*)[A-Z]{2}\:[0-9\.]*(?=\*)', sms.msg,
re.IGNORECASE)
MATCH_ITEMS = {
"LA": {
"name": "lag",
"fxn": int
},
"MX": {
"name": "abs_max_val",
"fxn": float
},
"MI": {
"name": "abs_max_index",
"fxn": int
},
"TP": {
"name": "temp_val",
"fxn": float
}
}
conversion_count = 0
for ma in matches:
identifier, value = tuple(ma.split(":"))
if identifier not in MATCH_ITEMS.keys():
print "Unknown identifier", identifier
continue
param = MATCH_ITEMS[identifier]
try:
values[param["name"]] = param["fxn"](value)
except ValueError:
print ">> Error: converting %s using %s" % (value, str(
param["fxn"]))
continue
conversion_count += 1
if conversion_count == 0:
print ">> Error: no successful conversion"
raise ValueError("No successful conversion of values")
try:
ts = re.search("(?<=\*)[0-9]{12}(?=$)", sms.msg).group(0)
ts = dt.strptime(ts, "%y%m%d%H%M%S").strftime("%Y-%m-%d %H:%M:%S")
except AttributeError:
raise ValueError("No valid timestamp recognized")
values["ts"] = ts
df_ext_values = pd.DataFrame([values])
print df_ext_values
return smsclass.DataTable(DATA_TABLE_NAME, df_ext_values)
def stilt_v2_parser(sms):
line = sms.msg
split_line = line.split('*')
logger_name = split_line[0]
data = split_line[1]
ts = dt.strptime(split_line[2],
'%y%m%d%H%M%S').strftime('%Y-%m-%d %H:%M:00')
data_split = data.split(';')
node0_data = data_split[0]
node0_data_split = re.split(':|,', node0_data)
for i in range(1, 11):
node0_data_split[i] = twos_comp(int(node0_data_split[i], 16), 16)
df0 = pd.DataFrame(node0_data_split).transpose()
df0 = df0.rename(
columns={
0: "node_id",
1: "ac_x",
2: "ac_y",
3: "ac_z",
4: "mg_x",
5: "mg_y",
6: "mg_z",
7: "gr_x",
8: "gr_y",
9: "gr_z",
10: "temp"
})
try:
df0 = df0.drop(columns=[11])
except:
print("no column 11")
try:
node1_data = data_split[1]
node1_data_split = re.split(':|,', node1_data)
for i in range(1, 11):
node1_data_split[i] = twos_comp(int(node1_data_split[i], 16), 16)
df1 = pd.DataFrame(node1_data_split).transpose()
df1 = df1.rename(
columns={
0: "node_id",
1: "ac_x",
2: "ac_y",
3: "ac_z",
4: "mg_x",
5: "mg_y",
6: "mg_z",
7: "gr_x",
8: "gr_y",
9: "gr_z",
10: "temp"
})
except:
print("No node 1")
df1 = pd.DataFrame()
logger_data = data_split[2]
logger_data_split = logger_data.split(',')
dflogger = pd.DataFrame(logger_data_split).transpose()
dflogger = dflogger.rename(columns={0: "taps", 1: "temp_rtc", 2: "volt"})
df_ts = pd.DataFrame({'ts': [ts]})
df = pd.concat([df0, df1], axis=0)
df = pd.concat([df_ts, df, dflogger], axis=1)
#
# df = trans_data[["ts","type","ac_x", "ac_y", "ac_z","mg_x","mg_y", "mg_z","gr_x","gr_y","gr_z","temp"]]
name_df = "stilt_{}".format(logger_name.lower())
data = smsclass.DataTable(name_df, df)
return data
def b64Parser(sms):
msg = sms.msg
print(msg)
if len(msg.split("*")) == 4:
msgsplit = msg.split('*')
tsm_name = msgsplit[0]
if len(tsm_name) != 5:
raise ValueError("length of tsm_name != 5")
dtype = msgsplit[1]
#print dtype
if len(dtype) == 2:
dtype = b64_to_dec(dtype)
else:
raise ValueError("length of dtype != 2")
datastr = msgsplit[2]
if len(datastr) == 0:
raise ValueError("length of data == 0")
ts = msgsplit[3]
ts_patterns = ['%y%m%d%H%M%S', '%Y-%m-%d %H:%M:%S']
timestamp = ''
if len(ts) not in [6, 12]:
raise ValueError("length of ts != 6 or 12")
for pattern in ts_patterns:
try:
timestamp = dt.strptime(ts,
pattern).strftime('%Y-%m-%d %H:%M:00')
break
except ValueError:
print("Error: wrong timestamp format", ts, "for pattern",
pattern)
outl = []
if dtype in [11, 12, 32, 33, 41, 42]:
name_df = 'tilt_' + tsm_name.lower()
n = 9 # 9 chars per node
sd = [datastr[i:i + n] for i in range(0, len(datastr), n)]
for piece in sd:
try:
ID = b64_to_dec(piece[0])
msgID = dtype
xd = b64_twos_comp(b64_to_dec(piece[1:3]))
yd = b64_twos_comp(b64_to_dec(piece[3:5]))
zd = b64_twos_comp(b64_to_dec(piece[5:7]))
bd = (b64_twos_comp(b64_to_dec(piece[7:9])) + 200) / 100.0
line = {
"ts": timestamp,
"node_id": ID,
"type_num": msgID,
"xval": xd,
"yval": yd,
"zval": zd,
"batt": bd
}
outl.append(line)
except ValueError:
print(
">> b64 Value Error detected.",
piece,
)
print("Piece of data to be ignored")
return
#elif dtype in [110,111,112,113,21,26,10,13]: # wala pang support for v2 bradcast soms
elif dtype in [110, 113, 10,
13]: # wala pang support for v2 bradcast soms
name_df = 'soms_' + tsm_name.lower()
n = 4
sd = [datastr[i:i + n] for i in range(0, len(datastr), n)]
for piece in sd:
try:
ID = b64_to_dec(piece[0])
msgID = dtype
soms = b64_to_dec(piece[1:4])
line = {
"ts": timestamp,
"node_id": ID,
"type_num": msgID,
"mval1": soms,
"mval2": 0
}
outl.append(line)
except ValueError:
print(
">> b64 Value Error detected.",
piece,
)
print("Piece of data to be ignored")
return
#for temp
elif dtype == 22:
name_df = 'temp_' + tsm_name.lower()
n = 3
sd = [datastr[i:i + n] for i in range(0, len(datastr), n)]
for piece in sd:
try:
ID = b64_to_dec(piece[0])
msgID = dtype
temp = b64_to_dec(piece[1:3])
line = {
"ts": timestamp,
"node_id": ID,
"type_num": msgID,
"temp_val": temp
}
outl.append(line)
except ValueError:
print(
">> b64 Value Error detected.",
piece,
)
print("Piece of data to be ignored")
return
else:
raise ValueError("dtype not recognized")
else:
raise ValueError("msg was not split into 3")
df = pd.DataFrame(outl)
data = smsclass.DataTable(name_df, df)
return data
def v1(sms):
"""
- The process of parsing version 1 subsurface data.
# :param sms: Dictionary of sms info.
# :type sms: obj
Args:
sms (obj): Dictionary of sms info.
Returns:
DataFrame: Dataframe output for success parsing and return
False if fails.
Example Output::
node_id xval yval zval
ts
2018-04-30 16:32:00 1 991 41 28
2018-04-30 16:32:00 2 964 17 109
2018-04-30 16:32:00 3 1015 20 -39
2018-04-30 16:32:00 4 1001 11 -2
2018-04-30 16:32:00 5 971 6 50
2018-04-30 16:32:00 6 -3015 106 -109
2018-04-30 16:32:00 7 1013 30 3
2018-04-30 16:32:00 8 1008 4 32
2018-04-30 16:32:00 9 1001 41 3
ts mval1 node_id
2018-04-30 16:32:00 2394 1
2018-04-30 16:32:00 2446 2
2018-04-30 16:32:00 2677 3
2018-04-30 16:32:00 2779 4
2018-04-30 16:32:00 2691 5
2018-04-30 16:32:00 2480 6
2018-04-30 16:32:00 2657 7
2018-04-30 16:32:00 2954 8
2018-04-30 16:32:00 2464 9
"""
data = sms.msg
data = data.replace("DUE", "")
data = data.replace(",", "*")
data = data.replace("/", "")
line = data[:-2]
tsm_name = line[0:4]
print('SITE: ' + tsm_name)
##msgdata = line[5:len(line)-11] #data is 6th char, last 10 char are date
try:
msgdata = (line.split('*'))[1]
except IndexError:
raise ValueError("Wrong message construction")
print('raw data: ' + msgdata)
#getting date and time
#msgdatetime = line[-10:]
try:
timestamp = (line.split('*'))[2][:10]
print('date & time: ' + timestamp)
except:
print('>> Date and time defaults to SMS not sensor data')
timestamp = sms.ts
# col_list = cfg.get("Misc","AdjustColumnTimeOf").split(',')
if tsm_name == 'PUGB':
timestamp = sms.ts
print("date & time adjusted " + timestamp)
else:
try:
timestamp = dt.strptime(timestamp,
'%y%m%d%H%M').strftime('%Y-%m-%d %H:%M:00')
except ValueError:
print(">> Error: date time conversion")
return False
print('date & time no change')
dlen = len(msgdata) #checks if data length is divisible by 15
#print 'data length: %d' %dlen
nodenum = dlen / 15
#print 'number of nodes: %d' %nodenum
if dlen == 0:
print('Error: There is NO data!')
return
elif ((dlen % 15) == 0):
#print 'Data has correct length!'
valid = dlen
else:
print('Warning: Excess data will be ignored!')
valid = nodenum * 15
outl_tilt = []
outl_soms = []
try:
i = 0
while i < valid:
#NODE ID
#parsed msg.data - NODE ID:
node_id = int('0x' + msgdata[i:i + 2], 16)
i = i + 2
#X VALUE
#parsed msg.data - TEMPX VALUE:
tempx = int('0x' + msgdata[i:i + 3], 16)
i = i + 3
#Y VALUE
#parsed msg.data - TEMPY VALUE:
tempy = int('0x' + msgdata[i:i + 3], 16)
i = i + 3
#Z VALUE
#parsed msg.data - ZVALUE:
tempz = int('0x' + msgdata[i:i + 3], 16)
i = i + 3
#M VALUE
#parsed msg.data - TEMPF VALUE:
tempf = int('0x' + msgdata[i:i + 4], 16)
i = i + 4
valueX = tempx
if valueX > 1024:
valueX = tempx - 4096
valueY = tempy
if valueY > 1024:
valueY = tempy - 4096
valueZ = tempz
if valueZ > 1024:
valueZ = tempz - 4096
valueF = tempf #is this the M VALUE?
tsm_name = tsm_name.lower()
line_tilt = {
"ts": timestamp,
"node_id": node_id,
"xval": valueX,
"yval": valueY,
"zval": valueZ
}
line_soms = {"ts": timestamp, "node_id": node_id, "mval1": valueF}
outl_tilt.append(line_tilt)
outl_soms.append(line_soms)
if len(outl_tilt) != 0:
df_tilt = smsclass.DataTable('tilt_' + tsm_name,
pd.DataFrame(outl_tilt))
df_soms = smsclass.DataTable('soms_' + tsm_name,
pd.DataFrame(outl_soms))
data = [df_tilt, df_soms]
return data
else:
print('\n>>Error: Error in Data format')
return
except KeyError:
print('\n>>Error: Error in Data format')
return
except KeyboardInterrupt:
print('\n>>Error: Unknown')
raise KeyboardInterrupt
return
except ValueError:
print('\n>>Error: Unknown')
return
def v2(sms):
"""
- The process of parsing version 2 and 3 subsurface data.
:param sms: Dictionary of sms info.
:type sms: obj
Returns:
DataFrame: Dataframe output for success parsing and return
False if fails.
Example Output::
BLCTA*x*010BFC3EEFEBF831D0BFF3D9FF2F82*180430163121
batt node_id type_num xval yval zval
ts
2018-04-30 16:31:00 3.31 1 11 1020 -18 -21
2018-04-30 16:31:00 3.30 29 11 1023 -39 -14
"""
msg = sms.msg
if len(msg.split(",")) == 3:
print(">> Editing old data format")
datafield = msg.split(",")[1]
dtype = datafield[2:4].upper()
if dtype == "20" or dtype == "0B":
dtypestr = "x"
elif dtype == "21" or dtype == "0C":
dtypestr = "y"
elif dtype == "6F" or dtype == "15":
dtypestr = "b"
elif dtype == "70" or dtype == "1A":
dtypestr = "c"
else:
raise ValueError(">> Data type" + dtype + "not recognized")
i = msg.find(",")
msg = msg[:i] + "*" + dtypestr + "*" + msg[i + 1:]
msg = msg.replace(",", "*").replace("/", "")
outl = []
msgsplit = msg.split('*')
tsm_name = msgsplit[0] # column id
if len(msgsplit) != 4:
print('wrong data format')
# print msg
return
if len(tsm_name) != 5:
print('wrong master name')
return
print(msg)
dtype = msgsplit[1].upper()
datastr = msgsplit[2]
if len(datastr) == 136:
datastr = datastr[0:72] + datastr[73:]
ts = msgsplit[3].strip()
if datastr == '':
datastr = '000000000000000'
print(">> Error: No parsed data in sms")
return
if len(ts) < 10:
print('>> Error in time value format: ')
return
ts_patterns = ['%y%m%d%H%M%S', '%Y-%m-%d %H:%M:%S']
timestamp = ''
ts = re.sub("[^0-9]", "", ts)
for pattern in ts_patterns:
try:
timestamp = dt.strptime(ts, pattern).strftime('%Y-%m-%d %H:%M:00')
break
except ValueError:
print("Error: wrong timestamp format", ts, "for pattern", pattern)
if timestamp == '':
raise ValueError(">> Error: Unrecognized timestamp pattern " + ts)
# update_sim_num_table(tsm_name,sender,timestamp[:8])
# PARTITION the message into n characters
if dtype == 'Y' or dtype == 'X':
n = 15
# PARTITION the message into n characters
sd = [datastr[i:i + n] for i in range(0, len(datastr), n)]
elif dtype == 'B':
# do parsing for datatype 'B' (SOMS RAW)
outl = soms_parser(msg, 1, 10, 0)
name_df = 'soms_' + tsm_name.lower()
# for piece in outl:
# print piece
elif dtype == 'C':
# do parsing for datatype 'C' (SOMS CALIB/NORMALIZED)
outl = soms_parser(msg, 2, 7, 0)
name_df = 'soms_' + tsm_name.lower()
# for piece in outl:
# print piece
elif dtype == 'D':
# do parsing for datatype 'D' TEMPERATURE
outl = []
n = 8
sd = [datastr[i:i + n] for i in range(0, len(datastr), n)]
for piece in sd:
try:
# print piece
ID = int(piece[0:2], 16)
msgID = int(piece[2:4], 16)
temp = int(piece[4:8], 16)
line = {
"ts": timestamp,
"node_id": ID,
"type_num": msgID,
"temp_val": temp
}
outl.append(line)
except:
return False
name_df = 'temp_' + tsm_name.lower()
else:
raise IndexError("Undefined data format " + dtype)
# do parsing for datatype 'X' or 'Y' (accel data)
if dtype.upper() == 'X' or dtype.upper() == 'Y':
outl = []
name_df = 'tilt_' + tsm_name.lower()
for piece in sd:
try:
# print piece
ID = int(piece[0:2], 16)
msgID = int(piece[2:4], 16)
xd = twos_comp(piece[4:7])
yd = twos_comp(piece[7:10])
zd = twos_comp(piece[10:13])
bd = (int(piece[13:15], 16) + 200) / 100.0
# line = [timestamp, ID, msgID, xd, yd, zd, bd]
line = {
"ts": timestamp,
"node_id": ID,
"type_num": msgID,
"xval": xd,
"yval": yd,
"zval": zd,
"batt": bd
}
outl.append(line)
except ValueError:
print(
">> Value Error detected.",
piece,
)
print("Piece of data to be ignored")
return
df = pd.DataFrame(outl)
data = smsclass.DataTable(name_df, df)
return data
def diagnostics(sms):
"""
- The process of parsing diagnostics of loggers.
:param sms: Dictionary of sms info.
:type sms: obj
Returns:
DataFrame: Dataframe output for success parsing and return
False if fails.
Example Input:
LTESA*m* 0.70*14.12*136.80*14.04*137.70*14.05* 0.90*14.02*210308150000
Example Output::
batt_volt curr_draw stat ts
0 14.12 0.70 0 2021-03-08 15:00:00
1 14.04 136.80 1 2021-03-08 15:00:00
2 14.05 137.70 2 2021-03-08 15:00:00
3 14.02 0.90 99 2021-03-08 15:00:00
"""
msg = sms.msg
msg = msg.replace("DUE", "")
msg = msg.replace("PZ", "")
split_msg = msg.split("*")
ts = split_msg[len(split_msg) - 1]
pattern = '%y%m%d%H%M%S'
timestamp = dt.strptime(ts, pattern).strftime('%Y-%m-%d %H:%M:00')
num_of_data = (len(split_msg) - 3) / 2
outl = []
for i in range(0, int(num_of_data)):
try:
curr_draw = split_msg[2 + (i) * 2]
batt_volt = split_msg[2 + (i) * 2 + 1]
if i == int(num_of_data) - 1:
stat = 99
last_str_split = batt_volt.split(">")
batt_volt = last_str_split[0]
try:
unsent_data = int(last_str_split[1])
except IndexError:
unsent_data = np.nan
else:
stat = i
line = {
"ts": timestamp,
"stat": stat,
"curr_draw": curr_draw,
"batt_volt": batt_volt
}
print(i)
print(stat, curr_draw, batt_volt)
line = {
"ts": timestamp,
"stat": stat,
"curr_draw": curr_draw,
"batt_volt": batt_volt
}
outl.append(line)
except:
print("kulang data")
tsm_name = split_msg[0].lower()
name_df = "volt_{}".format(tsm_name)
df = pd.DataFrame(outl)
volt = smsclass.DataTable(name_df, df)
#for unsent
out2 = []
tsm_sensors = mem.get("DF_TSM_SENSORS")
try:
tsm_id = tsm_sensors[tsm_sensors.tsm_name == tsm_name].tsm_id.values[0]
except:
tsm_id = np.nan
sender = sms.sim_num[-10:]
try:
query = (
"select mobile_id from logger_mobile "
"where sim_num like '%{}' order by date_activated desc limit 1".
format(sender))
mobile_id = db.read(query, resource="sms_data")[0][0]
except:
mobile_id = np.nan
line2 = {
"ts": timestamp,
"mobile_id": mobile_id,
"tsm_id": tsm_id,
"unsent": unsent_data
}
out2.append(line2)
df2 = pd.DataFrame(out2)
unsent = smsclass.DataTable("unsent", df2)
data = [volt, unsent]
return data
def lidar(sms):
msg = sms.msg
print(msg)
values = {}
matches = re.findall('(?<=\*)[A-Z]{2}\:[0-9\.]*(?=\*)', msg, re.IGNORECASE)
MATCH_ITEMS = {
"LR": {"name": "dist", "fxn": float},
"BV": {"name": "voltage", "fxn": float},
"BI": {"name": "current", "fxn": float},
"TP": {"name": "temp_val", "fxn": float}
}
table_name = "lidar_data"
conversion_count = 0
for ma in matches:
identifier, value = ma.split(":")
if identifier not in MATCH_ITEMS.keys():
print ("Unknown identifier", identifier)
continue
param = MATCH_ITEMS[identifier]
try:
values[param["name"]] = param["fxn"](value)
except ValueError:
print (">> Error: converting %s using %s" % (value, str(param["fxn"])))
continue
conversion_count += 1
if conversion_count == 0:
print (">> Error: no successful conversion")
raise ValueError("No successful conversion of values")
try:
ts = re.search("(?<=\*)[0-9]{12}(?=$)",msg).group(0)
ts = dt.strptime(ts,"%y%m%d%H%M%S").strftime("%Y-%m-%d %H:%M:%S")
except AttributeError:
raise ValueError("No valid timestamp recognized")
values["ts"] = ts
df_ext_values = pd.DataFrame([values])
line = msg
line = re.sub("(?<=\+) (?=\+)","NULL",line)
linesplit = line.split('*')
try:
x = (linesplit[5])
x = x.split(',')
x = x[0]
x = x.split(':')
x = x[1]
y = (linesplit[5])
y = y.split(',')
y = y[1]
z = (linesplit[5])
z = z.split(',')
z = z[2]
except IndexError:
raise ValueError("Incomplete data")
df_ac = [{'ac_xval':x,'ac_yval':y,
'ac_zval':z}]
df_ac = pd.DataFrame(df_ac)
line = re.sub("(?<=\+) (?=\+)","NULL",line)
linesplit = line.split('*')
try:
x = (linesplit[6])
x = x.split(',')
x = x[0]
x = x.split(':')
x = x[1]
y = (linesplit[6])
y = y.split(',')
y = y[1]
z = (linesplit[6])
z = z.split(',')
z = z[2]
except IndexError:
raise ValueError("Incomplete data")
# txtdatetime = dt.strptime(linesplit[9],
# '%y%m%d%H%M%S').strftime('%Y-%m-%d %H:%M:%S')
df_mg = [{'mg_xval':x,'mg_yval':y,
'mg_zval':z}]
df_mg = pd.DataFrame(df_mg)
line = re.sub("(?<=\+) (?=\+)","NULL",line)
linesplit = line.split('*')
try:
x = (linesplit[7])
x = x.split(',')
x = x[0]
x = x.split(':')
x = x[1]
y = (linesplit[7])
y = y.split(',')
y = y[1]
z = (linesplit[7])
z = z.split(',')
z = z[2]
except IndexError:
raise ValueError("Incomplete data")
df_gr = [{'gr_xval':x,'gr_yval':y,
'gr_zval':z}]
df_gr = pd.DataFrame(df_gr)
df_data = pd.concat([df_ext_values,df_ac,df_mg,df_gr], axis = 1)
return smsclass.DataTable(table_name,df_data)
## IMULA*L*LR:112.950*BV:8.45*BI:128.60*AC:9.5270,-0.1089,-0.3942*MG:0.0881,0.0755,-0.5267*GR:7.5512,9.0913,2.3975*TP:33.25*180807105005' ##
def eq(sms):
"""
- Process the sms message that fits for eq data.
:param sms: list data info of sms message .
:type sms: list
:returns: **Dataframe** - Return Dataframe structure output and if not return False for fail to parse message.
"""
pattern_matches = {}
print(sms.msg)
# search for matches
for name in EQ_SMS_PATTERNS.keys():
search_results = EQ_SMS_PATTERNS[name].search(sms.msg)
if search_results:
matched_pattern = search_results.group(0)
if name == 'depth':
int_pattern = re.compile(r"\d+", re.IGNORECASE)
matched_pattern = int_pattern.search(matched_pattern).group(0)
pattern_matches[name] = matched_pattern
else:
if name == 'issuer':
pattern_matches['issuer'] = np.nan
else:
print("No match for <%s> pattern." % (name), )
print("Incomplete message not stored.")
return False
print(pattern_matches)
# format date
datestr_init = pattern_matches["date"].upper()
datestr = None
try:
datestr = pd.to_datetime(datestr_init).date()
except:
pass
if datestr == None:
print(">> Error in datetime conversion for <%s>" % (datestr_init))
return False
# format time
timestr = pattern_matches["time"].replace(" ", "").replace(".",
":").upper()
try:
timestr = dt.strptime(timestr, "%I:%M%p").time()
except:
print(">> Error in datetime conversion", timestr)
return False
del pattern_matches["date"]
del pattern_matches["time"]
pattern_matches["ts"] = str(dt.combine(datestr, timestr))
out = {}
for col_name in pattern_matches.keys():
out[col_name] = pattern_matches[col_name]
df = pd.DataFrame([out])
print(df)
return smsclass.DataTable("earthquake_events", df)
def rain_arq(sms):
"""
- The process of parsing data of Arq message of rain.
:param sms: Dictionary of sms info.
:type sms: obj
Returns:
DataFrame: Dataframe output for success parsing and return
False if fails.
Example Output::
battery1 battery2 csq humidity rain temperature
ts
2018-04-26 13:30:58 4.143 4.158 9 69.8 0.0 30.0
"""
#msg = message
line = sms.msg
sender = sms.sim_num[-10:]
print('ARQ Weather data: ' + line)
line = re.sub("(?<=\+) (?=\+)", "NULL", line)
#table name
linesplit = line.split('+')
msgname = check_name_of_number(sender)
if msgname:
msgname = msgname.lower()
print(">> Number registered as", msgname)
# msgname_contact = msgname
else:
raise ValueError("Number not registered {}".format(sender))
try:
rain = int(linesplit[1]) * 0.5
batv1 = linesplit[3]
batv2 = linesplit[4]
csq = linesplit[9]
except IndexError:
raise ValueError("Incomplete data")
if csq == '':
csq = 'NULL'
try:
temp = linesplit[10]
hum = linesplit[11]
# flashp = linesplit[12]
except IndexError:
raise ValueError("Incomplete data")
txtdatetime = dt.strptime(linesplit[13],
'%y%m%d/%H%M%S').strftime('%Y-%m-%d %H:%M:%S')
try:
if csq != 'NULL' and csq != 'N/A':
df_data = [{
'ts': txtdatetime,
'rain': rain,
'temperature': temp,
'humidity': hum,
'battery1': batv1,
'battery2': batv2,
'csq': csq
}]
else:
df_data = [{
'ts': txtdatetime,
'rain': rain,
'temperature': temp,
'humidity': hum,
'battery1': batv1,
'battery2': batv2
}]
df_data = pd.DataFrame(df_data)
df_data = smsclass.DataTable('rain_' + msgname, df_data)
return df_data
except ValueError:
print('>> Error writing query string.', )
return None
def v3(sms):
"""
- The process of parsing data of v3 message of rain.
:param sms: Dictionary of sms info.
:type sms: obj
Returns:
DataFrame: Dataframe output for success parsing and return
False if fails.
Example Output::
battery1 battery2 csq humidity rain temperature
ts
2018-04-26 13:30:58 null null 15 null 0 null
"""
line = sms.msg
sender = sms.sim_num
#msg = message
line = re.sub("[^A-Z0-9,\/:\.\-]", "", line)
print('Weather data: ' + line)
if len(line.split(',')) > 9:
line = re.sub(",(?=$)", "", line)
line = re.sub("(?<=,)(?=(,|$))", "NULL", line)
line = re.sub("(?<=,)NULL(?=,)", "0.0", line)
# line = re.sub("(?<=,).*$","NULL",line)
print("line:", line)
try:
logger_name = check_name_of_number(sender)
logger_model = check_logger_model(logger_name)
print(logger_name, logger_model)
if logger_model in [23, 24, 25, 26]:
msgtable = logger_name
else:
msgtable = line.split(",")[0][:-1] + 'G'
# msgtable = check_name_of_number(sender)
msgdatetime = re.search("\d{02}\/\d{02}\/\d{02},\d{02}:\d{02}:\d{02}",
line).group(0)
txtdatetime = dt.strptime(msgdatetime, '%m/%d/%y,%H:%M:%S')
txtdatetime = txtdatetime.strftime('%Y-%m-%d %H:%M:%S')
# data = items.group(3)
rain = line.split(",")[6]
print(line)
csq = line.split(",")[8]
except (IndexError, AttributeError) as e:
print('\n>> Error: Rain message format is not recognized')
print(line)
return False
except ValueError:
print('\n>> Error: One of the values not correct')
print(line)
return False
except KeyboardInterrupt:
print('\n>>Error: Weather message format unknown ' + line)
return False
try:
# query = ("INSERT INTO rain_%s (ts,rain,csq) "
# "VALUES ('%s',%s,%s)") % (msgtable.lower(),txtdatetime,rain,csq)
# print query
if csq != 'NULL':
df_data = [{'ts': txtdatetime, 'rain': rain, 'csq': csq}]
else:
df_data = [{'ts': txtdatetime, 'rain': rain}]
df_data = pd.DataFrame(df_data)
df_data = smsclass.DataTable('rain_' + msgtable.lower(), df_data)
return df_data
except:
print('>> Error writing weather data to database. ' + line)
return
