def main():
log("Starting named entity recognition task")
log("Splitting training set into training and validation sets")
training, validation = Utils.create_training_validation_split(TRAINING_FILE_PATH)
log("Writing training and validation sets to file")
Utils.write_streams_to_file(training, "./input/training.txt")
Utils.write_streams_to_file(validation, "./input/validation.txt")
log("Reading training file")
token_stream, pos_stream, tag_stream = Utils.read_training_file(TRAINING_FILE_PATH)
log("Replacing low frequency tokens from training set")
token_stream, closed_vocabulary = Unknown.replace_low_frequency_tokens(token_stream)
log("Reading test file")
test_stream = Utils.read_test_file(TEST_FILE_PATH)
log("Replacing unknown tokens from test set")
test_stream = Unknown.replace_unknown_tokens(test_stream, closed_vocabulary)
log("Training most frequent class baseline")
baseline = MostFrequentClassBaseline(token_stream, tag_stream)
log("Predicting tags using baseline")
baseline_predictions = baseline.classify_test_stream(test_stream)
log("Writing predictions with baseline to file")
Utils.write_results_to_file(baseline_predictions, "../output/baseline_output.txt")
log("Training Hidden Markov Model")
hmm = HiddenMarkovModel(token_stream, tag_stream)
log("Predicting tags using HMM")
hmm_predictions = hmm.classify_test_stream(test_stream)
log("Writing predictions with HMM to file")
Utils.write_results_to_file(hmm_predictions, "../output/hmm_output.txt")
def test_ignores_raise_uninferable(self):
"""This is a docstring.
:raises NameError: Never
"""
raise Unknown("hi")
raise NameError("hi") # [unreachable]
def factory(type, baseval):
if type is None or baseval is None:
return Unknown("", "")
if type.strip().lower() == "local" and (baseval.count(':') >= 1
or baseval.count('|') >= 1):
baseval_list = re.split(':|', baseval, maxsplit=1)
type = baseval_list[0].strip().lower()
baseval = baseval_list[1].strip()
return PidFactory._getpidclass(type, baseval)
def factory(type, baseval):
baseval = baseval.strip().replace('\n', '')
# return eval(type + "("+baseval+")")
if type.strip().lower() == "dai" or type.lower() == "dai-nl" or 'dai' in type.lower(): return Dai(baseval)
if type.strip().lower() == "orcid": return Orcid(baseval)
if type.strip().lower() == "isni": return Isni(baseval)
if type.strip().lower() == "rid": return Rid(baseval)
if type.strip().lower() == "prs" or type.lower() == "nod-prs": return NodPrs(baseval)
if "crossref" in type.lower(): return Crossref(baseval)
if type.strip().lower() == "grid" or "grid" in type.lower(): return Grid(baseval)
return Unknown(type.strip().replace('\n', ''), baseval)
def _getpidclass(type, baseval):
baseval = baseval.strip().replace('\n', '')
if type.strip().lower() == "doi":
return Doi(baseval)
if type.strip().lower() == "urn" or type.strip().lower(
) == "urn:nbn" or type.strip().lower() == "urn:nbn:nl" or type.strip(
).lower() == "urnnbn" or type.strip().lower() == "nbn":
return Nbn(baseval)
if type.strip().lower() == "arxiv":
return ArXiv(baseval)
if type.strip().lower() == "href" or type.strip().lower(
) == "http" or type.strip().lower() == "url" or type.strip().lower(
) == "purl": # :mods:relatedItem/@xlink:href wordt met type 'url' opgezet...
if ("doi.org/" in baseval.lower()):
return Doi(baseval)
return Purl(baseval)
if type.strip().lower() == "issn":
return Issn(baseval)
if type.strip().lower() == "isbn":
return Isbn(baseval)
if type.strip().lower() == "pmid" or type.strip().lower() == "pubmed":
return Pmid(baseval)
if type.strip().lower() == "pure" or "pure" in type.lower():
return Pure(baseval)
if type.strip().lower() == "wos":
return Wos(baseval)
if type.strip().lower() == "scopus":
return Scopus(baseval)
if type.strip().lower() == "uri":
if ("issn" in baseval.lower()):
return Issn(baseval)
if ("isbn" in baseval.lower()):
return Isbn(baseval)
if ("doi" in baseval.lower()):
return Doi(baseval)
if (
"/handle/" in baseval.lower() or "hdl/" in baseval.lower()
or "handle.net" in baseval.lower()
): #handle may resolve locally, prefixed with local url-resolver, so return url type if given so.
return Handle(baseval)
if ("urn:nbn:nl" in baseval.lower()):
return Nbn(baseval)
if ("http" in baseval.lower() or "ftp" in baseval.lower()):
return Purl(baseval)
if type.strip().lower() == "handle" or type.strip().lower(
) == "handle.net" or type.strip().lower() == "hdl":
return Handle(baseval)
return Unknown(type.strip().replace('\n', ''), baseval)
def sysdescrparser(sysdescr):
"""SNMP sysDescr parsing.
Args:
:sysdescr(str): SNMP sysDescr raw string.
Returns:
:SysDescr sub-class instance: SysDescr is abstract super class.
Each vendor class extends Sysdescr class and following attributes.
:vendor(str): Vendor name.
:model(str): Product Model name.
:os(str): OS name.
:version(str): OS version name.
Example:
.. code-block:: python
>>> from sysdescrparser import sysdescrparser
>>> sysdescr = sysdescrparser('Juniper Networks, Inc. ...')
>>> sysdescr.vendor
'JUNIPER'
>>> sysdescr.model
'ex2200-48t-4g'
>>> sysdescr.os
'JUNOS'
>>> sysdescr.version
'10.2R1.8'
Support:
Currently supported Vendor and OS.
https://github.com/mtoshi/sysdescrparser/blob/master/samples/sample_data.json
See also:
https://github.com/mtoshi/sysdescrparser/blob/master/README.rst
"""
#
# cisco nxos
#
obj = CiscoNXOS(sysdescr)
if obj.parse():
return obj
#
# cisco iosxr
#
obj = CiscoIOSXR(sysdescr)
if obj.parse():
return obj
#
# cisco ios
#
obj = CiscoIOS(sysdescr)
if obj.parse():
return obj
#
# juniper junos
#
obj = JuniperJunos(sysdescr)
if obj.parse():
return obj
#
# juniper screenos
#
obj = JuniperScreenOS(sysdescr)
if obj.parse():
return obj
#
# brocade ironware
#
obj = BrocadeIronWare(sysdescr)
if obj.parse():
return obj
#
# brocade serveriron
#
obj = BrocadeServerIron(sysdescr)
if obj.parse():
return obj
#
# brocade networkos
#
obj = BrocadeNetworkOS(sysdescr)
if obj.parse():
return obj
#
# foundry ironware
#
obj = FoundryIronWare(sysdescr)
if obj.parse():
return obj
#
# arista eos
#
obj = AristaEOS(sysdescr)
if obj.parse():
return obj
#
# extreme xos
#
obj = ExtremeXOS(sysdescr)
if obj.parse():
return obj
#
# hp procurve
#
obj = HPProCurve(sysdescr)
if obj.parse():
return obj
#
# paloalto panos
#
obj = PaloAltoPANOS(sysdescr)
if obj.parse():
return obj
#
# a10 acos
#
obj = A10ACOS(sysdescr)
if obj.parse():
return obj
#
# citrix netscaler
#
obj = CitrixNetscaler(sysdescr)
if obj.parse():
return obj
#
# linux
#
obj = Linux(sysdescr)
if obj.parse():
return obj
#
# sun sunos
#
obj = SunSUNOS(sysdescr)
if obj.parse():
return obj
#
# freebsd
#
obj = FreeBSD(sysdescr)
if obj.parse():
return obj
#
# iij seil
#
obj = IIJSeil(sysdescr)
if obj.parse():
return obj
#
# yamaha rtx
#
obj = YamahaRTX(sysdescr)
if obj.parse():
return obj
#
# Unknown
#
obj = Unknown(sysdescr)
if obj.parse():
return obj
# pylint: disable=missing-docstring, expression-not-assigned, too-few-public-methods
# pylint: disable=no-member, import-error, line-too-long, useless-object-inheritance
# pylint: disable=unnecessary-comprehension, use-dict-literal, use-implicit-booleaness-not-comparison
from unknown import Unknown
class CustomClass(object):
def keys(self):
return []
for key in Unknown().keys():
pass
for key in Unknown.keys():
pass
for key in dict.keys():
pass
for key in {}.values():
pass
for key in {}.key():
pass
for key in CustomClass().keys():
pass
[key for key in {}.keys()] # [consider-iterating-dictionary]
(key for key in {}.keys()) # [consider-iterating-dictionary]
{key for key in {}.keys()} # [consider-iterating-dictionary]
{key: key for key in {}.keys()} # [consider-iterating-dictionary]
COMP1 = [key for key in {}.keys()] # [consider-iterating-dictionary]
COMP2 = (key for key in {}.keys()) # [consider-iterating-dictionary]
def factory():
from unknown import Unknown
return Unknown()
from animal import Animal
from cat import Cat
from unknown import Unknown
animal1 = Animal("unknown")
print(animal1.getName())
animal1.showInfo()
#animal1.eat();
print("\n")
jerry = Cat("jerry", 6, 9)
print("Hello, " + jerry.getName())
print("age: " + str(jerry.getAge()))
print("height: " + str(jerry.getHeight()))
jerry.showInfo()
jerry.eat()
unknown = Unknown("name", 9, 6)
unknown.showInfo()
# pylint: disable=missing-docstring, expression-not-assigned, too-few-public-methods, no-member, import-error, no-self-use
from unknown import Unknown
class CustomClass(object):
def keys(self):
return []
for key in Unknown().keys():
pass
for key in Unknown.keys():
pass
for key in dict.keys():
pass
for key in {}.values():
pass
for key in {}.key():
pass
for key in CustomClass().keys():
pass
[key for key in {}.keys()] # [consider-iterating-dictionary]
(key for key in {}.keys()) # [consider-iterating-dictionary]
{key for key in {}.keys()} # [consider-iterating-dictionary]
{key: key for key in {}.keys()} # [consider-iterating-dictionary]
for key in {}.keys(): # [consider-iterating-dictionary]
pass
async def parse_devices_data(self, parsed):
for i in parsed:
for endpoint in i["endpoints"]:
if endpoint["error"] == 0 and len(endpoint["data"]) > 0:
try:
attr_device = {}
class_name = "Unknown"
device_id = i["id"]
endpoint_id = endpoint["id"]
name_of_id = self.get_name_from_id(endpoint_id)
type_of_id = self.get_type_from_id(endpoint_id)
_LOGGER.debug("======[ DEVICE INFOS ]======")
_LOGGER.debug("ID {}".format(device_id))
_LOGGER.debug("ENDPOINT ID {}".format(endpoint_id))
_LOGGER.debug("Name {}".format(name_of_id))
_LOGGER.debug("Infos {}".format(endpoint["data"]))
_LOGGER.debug("Type {}".format(type_of_id))
_LOGGER.debug("==========================")
print_id = None
if len(name_of_id) != 0:
print_id = name_of_id
else:
print_id = device_id
attr_device['device_id'] = device_id
attr_device['endpoint_id'] = endpoint_id
attr_device['id'] = type_of_id + '_' + str(
device_id) + '_' + str(endpoint_id)
attr_device['name'] = print_id
attr_device['device_type'] = type_of_id
if type_of_id == 'light':
attr_device['light_name'] = print_id
class_name = "Light"
if type_of_id == 'shutter':
attr_device['cover_name'] = print_id
attr_device['device_type'] = 'cover'
class_name = "Cover"
if type_of_id == 'belmDoor':
attr_device['door_name'] = print_id
attr_device['device_type'] = 'sensor'
class_name = "Sensor"
if type_of_id == 'windowFrench' or type_of_id == 'window':
attr_device['door_name'] = print_id
attr_device['device_type'] = 'window'
class_name = "Window"
if type_of_id == 'boiler':
attr_device['device_type'] = 'climate'
class_name = "Boiler"
if type_of_id == 'electric' or type_of_id == 'hvac':
attr_device['device_type'] = 'climate'
class_name = "Electric"
if type_of_id == 'alarm':
attr_device['alarm_name'] = "Tyxal Alarm"
attr_device['device_type'] = 'alarm_control_panel'
class_name = "Alarm"
for elem in endpoint["data"]:
_LOGGER.debug("CURRENT ELEM={}".format(elem))
# endpoint_id = None
elementName = None
elementValue = None
elementValidity = None
elementName = elem["name"]
elementValue = elem["value"]
elementValidity = elem["validity"]
if elementValidity == 'upToDate':
attr_device[elementName] = elementValue
except Exception as e:
print('msg_data error in parsing !')
print(e)
if class_name == "Cover":
new_cover = Cover(tydom_attributes=attr_device,
mqtt=self.mqtt_client)
await new_cover.update()
elif class_name == "Sensor":
new_door = sensor(
elem_name='openState',
tydom_attributes_payload=attr_device,
attributes_topic_from_device='useless',
mqtt=self.mqtt_client)
await new_door.update()
elif class_name == "Window":
new_window = Window(tydom_attributes=attr_device,
tydom_client=self.tydom_client,
mqtt=self.mqtt_client)
await new_window.update()
elif class_name == "Unknown":
new_unknown = Unknown(tydom_attributes=attr_device,
tydom_client=self.tydom_client,
mqtt=self.mqtt_client)
await new_unknown.update()
elif class_name == "Light":
new_light = Light(tydom_attributes=attr_device,
mqtt=self.mqtt_client) #NEW METHOD
await new_light.update()
elif class_name == "Boiler":
new_boiler = Boiler(tydom_attributes=attr_device,
tydom_client=self.tydom_client,
mqtt=self.mqtt_client) #NEW METHOD
await new_boiler.update()
elif class_name == "Electric":
new_electric = Electric(
tydom_attributes=attr_device,
tydom_client=self.tydom_client,
mqtt=self.mqtt_client) #NEW METHOD
await new_electric.update()
# Get last known state (for alarm) # NEW METHOD
elif class_name == "Alarm":
# print(attr_alarm)
state = None
sos_state = False
maintenance_mode = False
out = None
try:
# {
# "name": "alarmState",
# "type": "string",
# "permission": "r",
# "enum_values": ["OFF", "DELAYED", "ON", "QUIET"]
# },
# {
# "name": "alarmMode",
# "type": "string",
# "permission": "r",
# "enum_values": ["OFF", "ON", "TEST", "ZONE", "MAINTENANCE"]
# }
if ('alarmState' in attr_device
and attr_device['alarmState'] == "ON") or (
'alarmState' in attr_device and
attr_device['alarmState']) == "QUIET":
state = "triggered"
elif 'alarmState' in attr_device and attr_device[
'alarmState'] == "DELAYED":
state = "pending"
if 'alarmSOS' in attr_device and attr_device[
'alarmSOS'] == "true":
state = "triggered"
sos_state = True
elif 'alarmMode' in attr_device and attr_device[
"alarmMode"] == "ON":
state = "armed_away"
elif 'alarmMode' in attr_device and attr_device[
"alarmMode"] == "ZONE":
state = "armed_home"
elif 'alarmMode' in attr_device and attr_device[
"alarmMode"] == "OFF":
state = "disarmed"
elif 'alarmMode' in attr_device and attr_device[
"alarmMode"] == "MAINTENANCE":
maintenance_mode = True
state = "disarmed"
if 'outTemperature' in attr_device:
out = attr_device["outTemperature"]
if (sos_state == True):
print("SOS !")
if not (state == None):
# print(state)
alarm = "alarm_tydom_" + str(endpoint_id)
# print("Alarm created / updated : "+alarm)
alarm = Alarm(current_state=state,
tydom_attributes=attr_device,
mqtt=self.mqtt_client)
await alarm.update()
except Exception as e:
print("Error in alarm parsing !")
print(e)
pass
else:
print("ERROR : can't change configuration")
pass
