def __init__(self, cmd, scope, debug=False):
self.debug = debug
if len(cmd.argv) > 0:
f = open(cmd.argv[0])
else:
f = sys.stdin
self.scope = scope
self.ffm = self.scope.FFM
self.pap = self.scope.GTW.OMP.PAP
self.networks = {}
self.parser = SQL_Parser(verbose=False, fix_double_encode=1)
self.parser.parse(f)
self.contents = self.parser.contents
self.tables = self.parser.tables
self.member_by_id = {}
self.person_by_id = {}
self.nicknames = {}
self.node_by_id = {}
self.phone_ids = {}
def __init__(self, cmd, scope, debug=False):
self.anonymize = cmd.anonymize
self.verbose = cmd.verbose
self.pers_exception = {}
try:
pf = open('pers.csv', 'r')
cr = csv.reader(pf, delimiter=';')
for line in cr:
self.pers_exception[int(line[0])] = (line[1], line[2])
except IOError:
print "WARN: Can't read additional person data"
self.debug = debug
if len(cmd.argv) > 0:
f = open(cmd.argv[0])
else:
f = sys.stdin
self.scope = scope
self.ffm = self.scope.FFM
self.pap = self.scope.GTW.OMP.PAP
self.networks = {}
self.parser = SQL_Parser(verbose=False, fix_double_encode=1)
self.parser.parse(f)
self.contents = self.parser.contents
self.tables = self.parser.tables
self.dev_by_id = {}
self.ffm_node = {}
self.member_by_id = {}
self.net_by_id = {}
self.nicknames = {}
self.nifin_by_id = {}
self.ntype_by_id = {}
self.node_by_id = {}
self.person_by_id = {}
self.phone_ids = {}
def __init__ (self, cmd, scope, debug = False) :
self.debug = debug
if len (cmd.argv) > 0 :
f = open (cmd.argv [0])
else :
f = sys.stdin
self.scope = scope
self.ffm = self.scope.FFM
self.pap = self.scope.GTW.OMP.PAP
self.networks = {}
self.parser = SQL_Parser (verbose = False, fix_double_encode = 1)
self.parser.parse (f)
self.contents = self.parser.contents
self.tables = self.parser.tables
self.member_by_id = {}
self.person_by_id = {}
self.nicknames = {}
self.node_by_id = {}
self.phone_ids = {}
def __init__ (self, cmd, scope, debug = False) :
self.anonymize = cmd.anonymize
self.verbose = cmd.verbose
self.pers_exception = {}
try :
pf = open ('pers.csv', 'r')
cr = csv.reader (pf, delimiter = ';')
for line in cr :
self.pers_exception [int (line [0])] = (line [1], line [2])
except IOError :
print ("WARN: Can't read additional person data")
self.debug = debug
if len (cmd.argv) > 0 :
f = open (cmd.argv [0])
else :
f = sys.stdin
self.scope = scope
self.ffw = self.scope.CNDB
self.pap = self.scope.GTW.OMP.PAP
self.networks = {}
self.parser = SQL_Parser (verbose = False, fix_double_encode = 1)
self.parser.parse (f)
self.contents = self.parser.contents
self.tables = self.parser.tables
self.dev_by_id = {}
self.ffw_node = {}
self.member_by_id = {}
self.net_by_id = {}
self.nicknames = {}
self.nifin_by_id = {}
self.ntype_by_id = {}
self.node_by_id = {}
self.person_by_id = {}
self.phone_ids = {}
self.node_pools = {}
class Convert(object):
# 54 : 655 # nick == email vorn?
person_dupes = \
{ 71 : 72
, 140 : 460
, 179 : 124
, 267 : 322
, 273 : 272 # probably same unnamed person, almost same nick
, 276 : 272 # yet another dupe with almost same nick
, 307 : 306
, 314 : 311
, 372 : 380 # same phone *and* password (!) different name (!)
, 424 : 418
, 440 : 838
, 617 : 623
, 669 : 1001 # same person?
, 686 : 687
, 708 : 707
, 709 : 707
, 718 : 860
, 729 : 728
, 741 : 740
, 755 : 754
, 776 : 301
, 799 : 807
, 820 : 359
, 821 : 359
, 831 : 836
, 852 : 851
, 892 : 706
, 903 : 904
, 973 : 544
}
person_ignore = dict.fromkeys \
(( 11
, 15
, 41
, 42
, 62
, 63
, 64
, 66
, 70
, 93
, 100
, 106
, 108
, 112
, 118
, 123
, 132
, 136
, 142
, 172
, 174
, 177
, 199
, 209
, 242
, 244
, 245
, 254
, 263
, 266
, 270
, 287
, 313
, 320
, 324
, 338
, 937
# Lastname missing:
, 34
, 69
, 107
, 166
, 168
, 200
, 251
, 260
, 323
# Bogus, only initials etc, ignore if no node:
, 102
, 103
, 185
, 233
, 299
, 455
, 530
, 845
, 872
, 919
, 955
))
def __init__(self, cmd, scope, debug=False):
self.anonymize = cmd.anonymize
self.verbose = cmd.verbose
self.pers_exception = {}
try:
pf = open('pers.csv', 'r')
cr = csv.reader(pf, delimiter=';')
for line in cr:
self.pers_exception[int(line[0])] = (line[1], line[2])
except IOError:
print "WARN: Can't read additional person data"
self.debug = debug
if len(cmd.argv) > 0:
f = open(cmd.argv[0])
else:
f = sys.stdin
self.scope = scope
self.ffm = self.scope.FFM
self.pap = self.scope.GTW.OMP.PAP
self.networks = {}
self.parser = SQL_Parser(verbose=False, fix_double_encode=1)
self.parser.parse(f)
self.contents = self.parser.contents
self.tables = self.parser.tables
self.dev_by_id = {}
self.ffm_node = {}
self.member_by_id = {}
self.net_by_id = {}
self.nicknames = {}
self.nifin_by_id = {}
self.ntype_by_id = {}
self.node_by_id = {}
self.person_by_id = {}
self.phone_ids = {}
# end def __init__
def create(self):
if self.anonymize:
self.fake_persons()
else:
self.create_persons()
self.create_nettypes()
self.scope.commit()
self.create_nodes()
self.scope.commit()
self.create_networks()
self.scope.commit()
self.create_devices()
self.scope.commit()
self.create_interfaces()
self.scope.commit()
self.create_dns_aliases()
self.scope.commit()
# end def create
def create_devices(self):
# ignore snmp_ip and snmp_lastseen (only used by three nodes)
dt = self.ffm.Net_Device_Type.instance(name='Generic', raw=True)
for d in sorted(self.contents['node'], key=lambda x: x.id):
if self.verbose:
print "INFO: Dev: %s Node: %s" % (d.id, d.location_id)
node = None
n = self.node_by_id.get(d.location_id)
if n:
if d.person_id and d.person_id != n.person_id:
print \
( "WARN: Device (node) %s, (loc) %s: "
"person mismatch d:%s n:%s"
% (d.id, n.id, d.person_id, n.person_id)
)
node = self.ffm_node.get(d.location_id)
if not node:
print "WARN: Node (location) %s for dev (node) %s missing" \
% (d.location_id, d.id)
continue
else:
mgr = self.person_by_id.get(d.person_id) or self.graz_admin
node = self.ffm.Node \
( name = d.name
, desc = 'Auto-created node (id: %s)' % d.location_id
, show_in_map = True
, manager = mgr
, raw = True
)
print "WARN: Manufacturing Node (loc: %s) for dev (node) %s" \
% (d.location_id, d.id)
dev = self.ffm.Net_Device \
( left = dt
, node = node
, name = d.name
, desc = d.comment
, raw = True
)
self.set_creation(dev, d.time)
self.dev_by_id[d.id] = dev
if len(self.scope.uncommitted_changes) > 10:
self.scope.commit()
# end def create_devices
def create_dns_aliases(self):
for dal in self.contents['dnsalias']:
if dal.ip_id not in self.nifin_by_id:
print 'WARN: ignoring dns_alias %s "%s" IP not found %s' \
% (dal.id, dal.name, dal.ip_id)
return
self.ffm.IP4_DNS_Alias \
(left = self.nifin_by_id [dal.ip_id], name = dal.name)
if len(self.scope.uncommitted_changes) > 10:
self.scope.commit()
# end def create_dns_aliases
def create_interfaces(self):
for iface in self.contents['ip']:
if iface.node_id not in self.dev_by_id:
print "WARN: Ignoring IP %s %s: no device (node) %s" \
% (iface.id, iface.ip, iface.node_id)
continue
dev = self.dev_by_id[iface.node_id]
net = self.net_by_id[iface.net_id]
ip = IP4_Address(iface.ip)
if ip not in net.net_address:
parent = self.ffm.IP4_Network.query \
( Q.net_address.CONTAINS (ip)
, ~ Q.electric
, sort_key = TFL.Sorted_By ("-net_address.mask_len")
).first ()
print "WARN: IP %s %s not in net %s %s found %s" \
% ( iface.id
, iface.ip
, iface.net_id
, net.net_address
, parent.net_address
)
net = parent
nw = net.reserve(ip, owner=dev.node.owner)
nif = self.ffm.Wired_Interface(left=dev, name=iface.name)
nii = self.ffm.Net_Interface_in_IP4_Network \
(nif, nw, mask_len = 32, name = iface.name)
self.nifin_by_id[iface.id] = nii
if len(self.scope.uncommitted_changes) > 10:
self.scope.commit()
# end def create_interfaces
def create_nettypes(self):
""" Network ranges for reservation
"""
by_mask = {}
# first group by netmask
for nw in self.contents['nettype']:
if not nw.comment:
print 'WARN: Ignoring nettype %s "%s"' % (nw.id, nw.name)
continue
for net_ip in nw.comment.split(','):
ip = IP4_Address(net_ip)
if ip.mask not in by_mask:
by_mask[ip.mask] = []
by_mask[ip.mask].append((ip, nw.name, nw.id))
typ = self.ffm.IP4_Network
for mask in sorted(by_mask):
for ip, name, id in by_mask[mask]:
if id not in self.ntype_by_id:
self.ntype_by_id[id] = []
r = typ.query \
( Q.net_address.CONTAINS (ip)
, ~ Q.electric
, sort_key = TFL.Sorted_By ("-net_address.mask_len")
).first ()
reserver = r.reserve if r else typ
network = reserver(ip, owner=self.graz_admin)
self.ntype_by_id[id].append(network)
if name:
network.set_raw(desc=name)
# end def create_nettypes
def create_networks(self):
for net in self.contents['net']:
parents = self.ntype_by_id.get(net.nettype_id, [])
node = self.ffm_node.get(net.location_id)
ip = IP4_Address(net.netip, net.netmask)
if node:
owner = node.owner
else:
print "WARN: Network %s %s Location %s missing" \
% (net.id, net.netip, net.location_id)
owner = self.graz_admin
parent = None
for p in parents:
if ip in p.net_address:
parent = p
break
else:
parent = None
for ps in self.ntype_by_id.itervalues():
for p in ps:
if ip in p.net_address:
parent = p
print "Got parent in ntype_by_id: %s" % parent
break
else:
parent = self.ffm.IP4_Network.query \
( Q.net_address.CONTAINS (ip)
, ~ Q.electric
, sort_key = TFL.Sorted_By ("-net_address.mask_len")
).first ()
if parent:
print "Got parent by network query: %s" % parent
if parent:
reserver = parent.reserve
else:
print "WARN: No parent: new network: %s" % ip
reserver = self.ffm.IP4_Network
network = reserver(ip, owner=owner)
self.net_by_id[net.id] = network
if node:
pool = self.ffm.IP4_Pool(left=network, node=node)
if net.comment:
network.set_raw(desc=net.comment)
if len(self.scope.uncommitted_changes) > 10:
self.scope.commit()
# end def create_networks
def create_nodes(self):
x_start = 4080
y_start = 4806
x_lon = 15.43844103813
y_lat = 47.07177327969
dx_lon = 50675.5176
dy_lat = 75505.521
for n in sorted(self.contents['location'], key=lambda x: x.id):
self.node_by_id[n.id] = n
person_id = self.person_dupes.get(n.person_id, n.person_id)
if person_id != 0 and person_id not in self.person_by_id:
# should not happen now
print "WARN: Location %s owner %s missing" % (n.id, person_id)
continue
if person_id == 0:
person = self.graz_admin
else:
person = self.person_by_id[person_id]
lat = lon = None
if n.pixel_x is not None and n.pixel_y is not None:
lon = "%f" % (x_lon + (n.pixel_x - x_start) / dx_lon)
lat = "%f" % (y_lat + (n.pixel_y - y_start) / dy_lat)
node = self.ffm.Node \
( name = n.name
, desc = n.comment.strip () or None
, show_in_map = not n.hidden
, manager = person
, position = dict (lat = lat, lon = lon)
, raw = True
)
self.ffm_node[n.id] = node
self.set_creation(node, n.time)
if n.street:
s = ' '.join(x for x in (n.street, n.streetnr) if x)
adr = self.pap.Address.instance_or_new \
( street = s
, zip = '8010'
, city = 'Graz'
, country = 'Austria'
)
node.set(address=adr)
if n.gallery_link:
MOM = self.scope.MOM
abs = 'http://gallery.funkfeuer.at/v/Graz/Knoten/%s/'
if n.gallery_link.startswith('http'):
abs = "%s"
url = MOM.Document \
(node, url = abs % n.gallery_link, type = 'Gallery')
if len(self.scope.uncommitted_changes) > 10:
self.scope.commit()
# end def create_nodes
def fake_persons(self):
self.graz_admin = self.pap.Person \
( first_name = 'Graz'
, last_name = 'Admin'
, raw = True
)
mail = '*****@*****.**'
email = self.pap.Email(address=mail)
self.pap.Person_has_Email(self.graz_admin, email)
auth = self.scope.Auth.Account.create_new_account_x \
( mail
, enabled = True
, suspended = True
, password = uuid.uuid4 ().hex
)
self.pap.Person_has_Account(self.graz_admin, auth)
for m in sorted(self.contents['person'], key=lambda x: x.id):
self.person_by_id[m.id] = self.graz_admin
# end def fake_persons
def create_persons(self):
for m in sorted(self.contents['person'], key=lambda x: x.id):
#print "%s: %r %r" % (m.id, m.firstname, m.lastname)
if m.id in self.person_ignore:
print "INFO: Ignoring anonymous without location: %s" % m.id
continue
if m.id in self.pers_exception:
pe = self.pers_exception[m.id]
fn, ln = (x.decode('utf-8') for x in pe)
else:
fn = m.firstname.strip()
ln = m.lastname.strip()
self.member_by_id[m.id] = m
if m.id in self.person_dupes:
print "INFO: Duplicate person: %s" % m.id
continue
if not fn or not ln:
print >> sys.stderr, "WARN: name missing: %s (%r/%r)" \
% (m.id, m.firstname, m.lastname)
if not fn and not ln:
if m.nick:
fn = m.nick
else:
fn = m.email.split('@')[0]
fn = fn or '?'
ln = ln or '?'
print "Person: %s %r/%r" % (m.id, fn, ln)
person = self.pap.Person \
( first_name = fn
, last_name = ln
, raw = True
)
self.person_by_id[m.id] = person
if m.nick:
self.try_insert_nick(m.nick, m.id, person)
if m.email:
mail = m.email.replace('[at]', '@')
email = self.pap.Email(address=mail)
self.pap.Person_has_Email(person, email)
if m.email == '*****@*****.**':
self.graz_admin = person
auth = self.scope.Auth.Account.create_new_account_x \
( mail
, enabled = True
, suspended = True
, password = uuid.uuid4 ().hex
)
self.pap.Person_has_Account(person, auth)
if m.tel:
self.try_insert_phone(m.tel, m.id, person)
if len(self.scope.uncommitted_changes) > 10:
self.scope.commit()
# get data from dupes
for d_id, m_id in self.person_dupes.iteritems():
# older version of db or dupe removed:
if d_id not in self.member_by_id or m_id not in self.person_by_id:
continue
d = self.member_by_id[d_id]
m = self.member_by_id[m_id]
person = self.person_by_id[m_id]
if d.email:
email = self.pap.Email(address=d.email)
self.pap.Person_has_Email(person, email)
if d.nick:
self.try_insert_nick(d.nick, m_id, person)
if d.tel:
self.try_insert_phone(d.tel, m_id, person)
if len(self.scope.uncommitted_changes) > 10:
self.scope.commit()
# end def create_persons
def set_creation(self, obj, create_time):
if not create_time:
return
if not isinstance(create_time, datetime):
create_time = datetime(*create_time.timetuple()[:3])
create_time = make_naive(create_time)
self.scope.ems.convert_creation_change(obj.pid, c_time=create_time)
# end def set_creation
def try_insert_phone(self, tel, id, person):
if tel.startswith('+430659'):
tel = '+43650' + tel[6:]
p = Phone(tel, city="Graz")
if p:
k = str(p)
t = self.pap.Phone.instance(*p)
if t:
eid = self.phone_ids[k]
prs = self.person_by_id[eid]
if eid != id:
print "WARN: %s/%s %s/%s: Duplicate Phone: %s" \
% (eid, prs.pid, id, person.pid, tel)
else:
self.phone_ids[k] = id
phone = self.pap.Phone(*p)
self.pap.Person_has_Phone(person, phone)
# end def try_insert_phone
def try_insert_nick(self, nick, id, person):
lnick = nick.lower()
if lnick in self.nicknames:
eid = self.nicknames[lnick]
prs = self.person_by_id[eid]
if eid != id:
print "WARN: %s/%s %s/%s Duplicate Nickname: %s" \
% (eid, prs.pid, id, person.pid, nick)
else:
n = self.pap.Nickname(nick, raw=True)
self.pap.Person_has_Nickname(person, n)
self.nicknames[lnick] = id
class Convert (object) :
# 54 : 655 # nick == email vorn?
person_dupes = \
{ 71 : 72
, 140 : 460
, 179 : 124
, 267 : 322
, 273 : 272 # probably same unnamed person, almost same nick
, 276 : 272 # yet another dupe with almost same nick
, 307 : 306
, 314 : 311
, 372 : 380 # same phone *and* password (!) different name (!)
, 424 : 418
, 440 : 838
, 617 : 623
, 669 : 1001 # same person?
, 686 : 687
, 708 : 707
, 709 : 707
, 718 : 860
, 729 : 728
, 741 : 740
, 755 : 754
, 776 : 301
, 799 : 807
, 820 : 359
, 821 : 359
, 831 : 836
, 852 : 851
, 892 : 706
, 903 : 904
, 973 : 544
, 1005 : 1007
}
person_ignore = dict.fromkeys \
(( 11
, 15
, 41
, 42
, 62
, 63
, 64
, 66
, 70
, 93
, 100
, 106
, 108
, 112
, 118
, 123
, 132
, 136
, 142
, 172
, 174
, 177
, 199
, 209
, 242
, 244
, 245
, 254
, 263
, 266
, 270
, 287
, 313
, 320
, 324
, 338
, 937
# Lastname missing:
, 34
, 69
, 107
, 166
, 168
, 200
, 251
, 260
, 323
# Bogus, only initials etc, ignore if no node:
, 102
, 103
, 185
, 233
, 299
, 455
, 530
, 845
, 872
, 919
, 955
))
def __init__ (self, cmd, scope, debug = False) :
self.anonymize = cmd.anonymize
self.verbose = cmd.verbose
self.pers_exception = {}
try :
pf = open ('pers.csv', 'r')
cr = csv.reader (pf, delimiter = ';')
for line in cr :
self.pers_exception [int (line [0])] = (line [1], line [2])
except IOError :
print ("WARN: Can't read additional person data")
self.debug = debug
if len (cmd.argv) > 0 :
f = open (cmd.argv [0])
else :
f = sys.stdin
self.scope = scope
self.ffw = self.scope.CNDB
self.pap = self.scope.GTW.OMP.PAP
self.networks = {}
self.parser = SQL_Parser (verbose = False, fix_double_encode = 1)
self.parser.parse (f)
self.contents = self.parser.contents
self.tables = self.parser.tables
self.dev_by_id = {}
self.ffw_node = {}
self.member_by_id = {}
self.net_by_id = {}
self.nicknames = {}
self.nifin_by_id = {}
self.ntype_by_id = {}
self.node_by_id = {}
self.person_by_id = {}
self.phone_ids = {}
self.node_pools = {}
# end def __init__
def create (self) :
self.fugru = self.pap.Adhoc_Group ("Funker Gruppe")
if self.anonymize :
self.fake_persons ()
else :
self.create_persons ()
self.create_nettypes ()
self.scope.commit ()
self.create_nodes ()
self.scope.commit ()
self.create_networks ()
self.scope.commit ()
self.create_devices ()
self.scope.commit ()
self.create_interfaces ()
self.scope.commit ()
self.create_dns_aliases ()
self.scope.commit ()
# end def create
def create_devices (self) :
# ignore snmp_ip and snmp_lastseen (only used by three nodes)
secondary_nodes = {}
dt = self.ffw.Net_Device_Type.instance (name = 'Generic', raw = True)
for d in sorted (self.contents ['node'], key = lambda x : x.id) :
if self.verbose :
print ("INFO: Dev: %s Node: %s" % (d.id, d.location_id))
node = None
n = self.node_by_id.get (d.location_id)
if n :
if ( d.person_id
and d.person_id != n.person_id
and d.person_id in self.person_by_id
) :
if (d.person_id, n.id) in secondary_nodes :
node = secondary_nodes [(d.person_id, n.id)]
print \
( "INFO: Device (node) %s, (loc) %s: "
"existing secondary node for d:%s/n:%s"
% (d.id, n.id, d.person_id, n.person_id)
)
else :
print \
( "INFO: Device (node) %s, (loc) %s: "
"manufacturing secondary node for d:%s/n:%s"
% (d.id, n.id, d.person_id, n.person_id)
)
pn = self.ffw_node.get (n.id)
mgr = self.person_by_id.get (d.person_id)
txt = 'Auto-created node (id: %s) for "%s"' \
% (d.location_id, n.name)
node = self.ffw.Node \
( name = '/'.join ((n.name, d.name))
, desc = txt
, manager = mgr
, raw = True
, show_in_map = pn.show_in_map
, position = pn.position
)
secondary_nodes [(d.person_id, n.id)] = node
else :
node = self.ffw_node.get (d.location_id)
if not node :
print \
( "WARN: Node (location) %s "
"for dev (node) %s missing" \
% (d.location_id, d.id)
)
continue
else :
mgr = self.person_by_id.get (d.person_id) or self.graz_admin
node = self.ffw.Node \
( name = d.name
, desc = 'Auto-created node (id: %s)' % d.location_id
, show_in_map = True
, manager = mgr
, raw = True
)
print \
( "INFO: Manufacturing Node (loc: %s) for dev (node) %s"
% (d.location_id, d.id)
)
dev = self.ffw.Net_Device \
( left = dt
, node = node
, name = d.name
, desc = d.comment
, raw = True
)
self.set_creation (dev, d.time)
self.dev_by_id [d.id] = dev
if len (self.scope.uncommitted_changes) > 10 :
self.scope.commit ()
# end def create_devices
def create_dns_aliases (self) :
for dal in self.contents ['dnsalias'] :
if dal.ip_id not in self.nifin_by_id :
print \
( 'WARN: ignoring dns_alias %s "%s" IP not found %s'
% (dal.id, dal.name, dal.ip_id)
)
return
self.ffw.IP4_DNS_Alias \
(left = self.nifin_by_id [dal.ip_id], name = dal.name)
if len (self.scope.uncommitted_changes) > 10 :
self.scope.commit ()
# end def create_dns_aliases
def create_interfaces (self) :
for iface in self.contents ['ip'] :
if iface.node_id not in self.dev_by_id :
print \
( "WARN: Ignoring IP %s %s: no device (node) %s"
% (iface.id, iface.ip, iface.node_id)
)
continue
dev = self.dev_by_id [iface.node_id]
ip = IP4_Address (iface.ip)
# Originally in a /32 reserved network
if ip in self.node_ips :
nw = self.node_ips [ip]
net = nw.reserve (ip)
assert ip in net.net_address
if iface.net_id not in nw.netids :
print \
( "WARN: Referenced wrong network: %s %s"
% (iface.net_id, ip)
)
else :
net = self.net_by_id [iface.net_id]
nw = net.reserve (ip, owner = dev.node.owner)
nif = self.ffw.Wired_Interface (left = dev, name = iface.name)
nii = self.ffw.Net_Interface_in_IP4_Network \
(nif, nw, mask_len = 32, name = iface.name)
self.nifin_by_id [iface.id] = nii
if len (self.scope.uncommitted_changes) > 10 :
self.scope.commit ()
for ip in self.node_ips :
self.node_ips [ip].reserve_network ()
# end def create_interfaces
def create_nettypes (self) :
""" Network ranges for reservation
"""
# Create all rfc1918 networks as top-level pools
for net_ip in '172.16.0.0/12', '10.0.0.0/8', '192.168.0.0/16' :
ip = IP4_Address (net_ip)
nw = self.ffw.IP4_Network (ip, owner = self.graz_admin)
pool = self.ffw.IP4_Pool (name = 'RFC 1918 %s' % net_ip)
self.ffw.IP4_Network_in_IP4_Pool (nw, pool)
by_mask = {}
pool_by_id = {}
# first group by netmask
for nw in self.contents ['nettype'] :
if not nw.comment :
print ('WARN: Ignoring nettype %s "%s"' % (nw.id, nw.name))
continue
pool = pool_by_id [nw.id] = self.ffw.IP4_Pool (name = nw.name)
if nw.name == 'GRAZ Client Subnet /29' :
self.ffw.IP4_Pool_permits_Group \
(pool, self.fugru, node_quota = 29)
netmask = None
try :
netmask = int (nw.name.split ('/') [-1])
except ValueError :
pass
if netmask :
pool.set (netmask_interval = (netmask,))
for net_ip in nw.comment.split (',') :
ip = IP4_Address (net_ip)
if ip.mask not in by_mask :
by_mask [ip.mask] = []
by_mask [ip.mask].append ((ip, nw.name, nw.id))
typ = self.ffw.IP4_Network
for mask in sorted (by_mask) :
for ip, name, id in by_mask [mask] :
if id not in self.ntype_by_id :
self.ntype_by_id [id] = []
r = typ.query \
( Q.net_address.CONTAINS (ip)
, ~ Q.electric
, sort_key = TFL.Sorted_By ("-net_address.mask_len")
).first ()
reserver = r.reserve if r else typ
network = reserver (ip, owner = self.graz_admin)
self.ntype_by_id [id].append (network)
self.ffw.IP4_Network_in_IP4_Pool (network, pool_by_id [id])
if name :
network.set_raw (desc = name)
# end def create_nettypes
def create_networks (self) :
self.node_ips = {}
ipdict = {}
self.ff_net_by_id = {}
for net in self.contents ['net'] :
self.ff_net_by_id [net.id] = net
parents = self.ntype_by_id.get (net.nettype_id, [])
node = self.ffw_node.get (net.location_id)
ip = IP4_Address (net.netip, net.netmask)
if parents and ip == parents [0].net_address :
self.net_by_id [net.id] = parents [0]
continue
if node :
owner = node.owner
else :
print \
( "WARN: Network %s %s Location %s missing"
% (net.id, net.netip, net.location_id)
)
owner = self.graz_admin
parent = None
for p in parents :
if ip in p.net_address :
parent = p
break
else :
parent = None
for ps in pyk.itervalues (self.ntype_by_id) :
for p in ps :
if ip in p.net_address :
parent = p
print ("Got parent in ntype_by_id: %s" % parent)
break
else :
parent = self.ffw.IP4_Network.query \
( Q.net_address.CONTAINS (ip)
, ~ Q.electric
, sort_key = TFL.Sorted_By ("-net_address.mask_len")
).first ()
if parent :
print ("Got parent by network query: %s" % parent)
ipdict [ip] = Network (self, ip, node, net.id, parent, owner)
mask = 32
while mask and ipdict :
# Dict is modified during iteration, use keys ()
for ip in ipdict.keys () :
# already found via sibling or network still too big
if ip.mask < mask or ip not in ipdict :
continue
assert ip.mask == mask
nw = ipdict [ip]
sib = next \
( k for k in ip.parent.subnets (ip.mask)
if ip.is_sibling (k)
)
# Sibling has same node? -> join to bigger net
# Only join networks that *have* a node
if ( sib in ipdict
and nw.node
and ipdict [sib].node == nw.node
) :
nw.join (ipdict [sib])
del ipdict [ip]
del ipdict [sib]
ipdict [ip.parent] = nw
else :
del ipdict [ip]
if nw.node and nw.ip.mask >= 24 :
# Don't reserve now, if allocated we don't want a pool
for i in nw.ip.subnets (32) :
self.node_ips [i] = nw
else :
nw.reserve_network ()
if len (self.scope.uncommitted_changes) > 10 :
self.scope.commit ()
mask -= 1
# end def create_networks
def create_nodes (self) :
x_start = 4080
y_start = 4806
x_lon = 15.43844103813
y_lat = 47.07177327969
dx_lon = 50675.5176
dy_lat = 75505.521
for n in sorted (self.contents ['location'], key = lambda x : x.id) :
self.node_by_id [n.id] = n
person_id = self.person_dupes.get (n.person_id, n.person_id)
if person_id != 0 and person_id not in self.person_by_id :
# should not happen now
print ("WARN: Location %s owner %s missing" % (n.id, person_id))
continue
if person_id == 0 :
person = self.graz_admin
else :
person = self.person_by_id [person_id]
lat = lon = None
if n.pixel_x is not None and n.pixel_y is not None :
lon = "%f" % (x_lon + (n.pixel_x - x_start) / dx_lon)
lat = "%f" % (y_lat + (n.pixel_y - y_start) / dy_lat)
node = self.ffw.Node \
( name = n.name
, desc = n.comment.strip () or None
, show_in_map = not n.hidden
, manager = person
, position = dict (lat = lat, lon = lon)
, raw = True
)
self.ffw_node [n.id] = node
self.set_creation (node, n.time)
if n.street :
s = ' '.join (x for x in (n.street, n.streetnr) if x)
adr = self.pap.Address.instance_or_new \
( street = s
, zip = '8010'
, city = 'Graz'
, country = 'Austria'
, raw = True
)
node.set (address = adr)
if n.gallery_link :
MOM = self.scope.MOM
abs = 'http://gallery.funkfeuer.at/v/Graz/Knoten/%s/'
if n.gallery_link.startswith ('http') :
abs = "%s"
url = MOM.Document \
(node, url = abs % n.gallery_link, type = 'Gallery')
if len (self.scope.uncommitted_changes) > 10 :
self.scope.commit ()
# end def create_nodes
def fake_persons (self) :
self.graz_admin = self.pap.Person \
( first_name = 'Graz'
, last_name = 'Admin'
, raw = True
)
mail = '*****@*****.**'
email = self.pap.Email (address = mail)
self.pap.Person_has_Email (self.graz_admin, email)
auth = self.scope.Auth.Account.create_new_account_x \
( mail
, enabled = True
, suspended = True
, password = uuid.uuid4 ().hex
)
self.pap.Person_has_Account (self.graz_admin, auth)
for m in sorted (self.contents ['person'], key = lambda x : x.id) :
self.person_by_id [m.id] = self.graz_admin
# end def fake_persons
def create_persons (self) :
for m in sorted (self.contents ['person'], key = lambda x : x.id) :
#print ("%s: %r %r" % (m.id, m.firstname, m.lastname))
if m.id in self.person_ignore :
print ("INFO: Ignoring anonymous without location: %s" % m.id)
continue
if m.id in self.pers_exception :
pe = self.pers_exception [m.id]
fn, ln = (pyk.decoded (x, 'utf-8') for x in pe)
else :
fn = m.firstname.strip ()
ln = m.lastname.strip ()
self.member_by_id [m.id] = m
if m.id in self.person_dupes :
print ("INFO: Duplicate person: %s" % m.id)
continue
if not fn or not ln :
print \
( "WARN: name missing: %s (%r/%r)"
% (m.id, m.firstname, m.lastname)
, file = sys.stderr
)
if not fn and not ln :
if m.nick :
fn = m.nick
else :
fn = m.email.split ('@') [0]
fn = fn or '?'
ln = ln or '?'
print ("Person: %s %r/%r" % (m.id, fn, ln))
person = self.pap.Person \
( first_name = fn
, last_name = ln
, raw = True
)
self.pap.Person_in_Group (person, self.fugru)
self.person_by_id [m.id] = person
if m.nick :
self.try_insert_nick (m.nick, m.id, person)
if m.email :
mail = m.email.replace ('[at]', '@')
email = self.pap.Email (address = mail)
self.pap.Person_has_Email (person, email)
if m.email == '*****@*****.**' :
g = self.pap.Adhoc_Group ('Admin Group Graz')
self.pap.Person_in_Group (person, g)
self.graz_admin = g
auth = self.scope.Auth.Account.create_new_account_x \
( mail
, enabled = True
, suspended = True
, password = uuid.uuid4 ().hex
)
self.pap.Person_has_Account (person, auth)
if m.tel :
self.try_insert_phone (m.tel, m.id, person)
if len (self.scope.uncommitted_changes) > 10 :
self.scope.commit ()
# get data from dupes
for d_id, m_id in pyk.iteritems (self.person_dupes) :
# older version of db or dupe removed:
if d_id not in self.member_by_id or m_id not in self.person_by_id :
continue
d = self.member_by_id [d_id]
m = self.member_by_id [m_id]
person = self.person_by_id [m_id]
if d.email :
email = self.pap.Email (address = d.email)
self.pap.Person_has_Email (person, email)
if d.nick :
self.try_insert_nick (d.nick, m_id, person)
if d.tel :
self.try_insert_phone (d.tel, m_id, person)
if len (self.scope.uncommitted_changes) > 10 :
self.scope.commit ()
# end def create_persons
def set_creation (self, obj, create_time) :
if not create_time :
return
if not isinstance (create_time, datetime) :
create_time = datetime (* create_time.timetuple () [:3])
create_time = make_naive (create_time)
self.scope.ems.convert_creation_change (obj.pid, c_time = create_time)
# end def set_creation
def try_insert_phone (self, tel, id, person) :
if tel.startswith ('+430659') :
tel = '+43650' + tel [6:]
p = Phone (tel, city = "Graz")
if p :
k = str (p)
t = self.pap.Phone.instance (*p)
if t :
eid = self.phone_ids [k]
prs = self.person_by_id [eid]
if eid != id :
print \
( "WARN: %s/%s %s/%s: Duplicate Phone: %s"
% (eid, prs.pid, id, person.pid, tel)
)
else :
self.phone_ids [k] = id
phone = self.pap.Phone (*p)
self.pap.Person_has_Phone (person, phone)
# end def try_insert_phone
def try_insert_nick (self, nick, id, person) :
lnick = nick.lower ()
if lnick in self.nicknames :
eid = self.nicknames [lnick]
prs = self.person_by_id [eid]
if eid != id :
print \
( "WARN: %s/%s %s/%s Duplicate Nickname: %s"
% (eid, prs.pid, id, person.pid, nick)
)
else :
n = self.pap.Nickname (nick, raw = True)
self.pap.Person_has_Nickname (person, n)
self.nicknames [lnick] = id
class Convert (object) :
person_dupes = \
{ 179 : 124
, 267 : 322
, 307 : 306
, 372 : 380 # same phone *and* password (!) different name (!)
, 424 : 418
, 617 : 623
, 686 : 687
, 708 : 707
, 709 : 707
, 718 : 860
, 729 : 728
, 741 : 740
, 755 : 754
, 776 : 301
, 799 : 807
, 820 : 359
, 821 : 359
, 831 : 836
, 852 : 851
, 892 : 706
, 903 : 904
}
def __init__ (self, cmd, scope, debug = False) :
self.debug = debug
if len (cmd.argv) > 0 :
f = open (cmd.argv [0])
else :
f = sys.stdin
self.scope = scope
self.ffm = self.scope.FFM
self.pap = self.scope.GTW.OMP.PAP
self.networks = {}
self.parser = SQL_Parser (verbose = False, fix_double_encode = 1)
self.parser.parse (f)
self.contents = self.parser.contents
self.tables = self.parser.tables
self.member_by_id = {}
self.person_by_id = {}
self.nicknames = {}
self.node_by_id = {}
self.phone_ids = {}
# end def __init__
def create (self) :
self.create_persons ()
self.create_nodes ()
self.create_devices ()
# end def create
def create_devices (self) :
# ignore snmp_ip and snmp_lastseen (only used by three nodes)
# FIXME: Use smokeping?
# FIXME: hastinc?
for d in self.contents ['node'] :
if d.location_id not in self.node_by_id :
print "WARN: Ignoring device (node) %s with location_id %s" \
% (d.id, d.location_id)
continue
n = self.node_by_id [d.location_id]
if d.person_id and d.person_id != n.person_id :
print "person %s: d:%s n:%s" % (d.id, d.person_id, n.person_id)
# end def create_devices
def create_nodes (self) :
x_start = 4080
y_start = 4806
x_lon = 15.43844103813
y_lat = 47.07177327969
dx_lon = 50675.5176
dy_lat = 75505.521
for n in sorted (self.contents ['location'], key = lambda x : x.id) :
self.node_by_id [n.id] = n
person_id = self.person_dupes.get (n.person_id, n.person_id)
if person_id not in self.person_by_id :
print "WARN: Location %s owner %s missing" % (n.id, person_id)
continue
person = self.person_by_id [person_id]
lat = lon = None
if n.pixel_x is not None and n.pixel_y is not None :
lon = "%f" % (x_lon + (n.pixel_x - x_start) / dx_lon)
lat = "%f" % (y_lat + (n.pixel_y - y_start) / dy_lat)
node = self.ffm.Node \
( name = n.name
, show_in_map = not n.hidden
, manager = person
, position = dict (lat = lat, lon = lon)
, raw = True
)
if n.street :
s = ' '.join (x for x in (n.street, n.streetnr) if x)
adr = self.pap.Address.instance_or_new \
( street = s
, zip = '8010'
, city = 'Graz'
, country = 'Austria'
)
self.pap.Node_has_Address (node, adr)
if n.gallery_link :
abs = 'http://gallery.funkfeuer.at/v/Graz/Knoten/%s/'
if n.gallery_link.startswith ('http') :
abs = "%s"
url = self.pap.Url.instance_or_new \
(abs % n.gallery_link, desc = 'Gallery')
self.pap.Node_has_Url (node, url)
#print "%4d %s %s %r %s %s %s %r" % \
# (n.id, n.hastinc, n.time, n.street, n.streetnr, n.gps_lon, n.gps_lat, n.comment)
# end def create_nodes
def create_persons (self) :
for m in sorted (self.contents ['person'], key = lambda x : x.id) :
#print "%s: %r %r" % (m.id, m.firstname, m.lastname)
self.member_by_id [m.id] = m
if m.id in self.person_dupes :
print "INFO: Duplicate person: %s" % m.id
continue
if not m.firstname or not m.lastname :
print >> sys.stderr, "WARN: name missing: %s (%s/%s)" \
% (m.id, m.firstname, m.lastname)
continue
person = self.pap.Person \
( first_name = m.firstname.strip ()
, last_name = m.lastname.strip ()
, raw = True
)
self.person_by_id [m.id] = person
if m.nick :
self.try_insert_nick (m.nick, m.id, person)
if m.email :
email = self.pap.Email (address = m.email)
self.pap.Person_has_Email (person, email)
if m.tel :
self.try_insert_phone (m.tel, m.id, person)
# get data from dupes
for d_id, m_id in self.person_dupes.iteritems () :
# older version of db or dupe removed:
if d_id not in self.member_by_id or m_id not in self.person_by_id :
continue
d = self.member_by_id [d_id]
m = self.member_by_id [m_id]
person = self.person_by_id [m_id]
if d.email :
email = self.pap.Email (address = d.email)
self.pap.Person_has_Email (person, email)
if d.nick :
self.try_insert_nick (d.nick, m_id, person)
if d.tel :
self.try_insert_phone (d.tel, m_id, person)
# end def create_persons
def set_creation (self, obj, create_time) :
create_time = make_naive (create_time)
self.scope.ems.convert_creation_change (obj.pid, c_time = create_time)
# end def set_creation
def try_insert_phone (self, tel, id, person) :
p = Phone (tel, city = "Graz")
if p :
k = str (p)
t = self.pap.Phone.instance (*p)
if t :
eid = self.phone_ids [k]
prs = self.person_by_id [eid]
if eid != id :
print "WARN: %s/%s %s/%s: Duplicate Phone: %s" \
% (eid, prs.pid, id, person.pid, tel)
else :
self.phone_ids [k] = id
phone = self.pap.Phone (*p)
self.pap.Person_has_Phone (person, phone)
# end def try_insert_phone
def try_insert_nick (self, nick, id, person) :
lnick = nick.lower ()
if lnick in self.nicknames :
eid = self.nicknames [lnick]
prs = self.person_by_id [eid]
if eid != id :
print "WARN: %s/%s %s/%s Duplicate Nickname: %s" \
% (eid, prs.pid, id, person.pid, nick)
else :
n = self.pap.Nickname (nick, raw = True)
self.pap.Person_has_Nickname (person, n)
self.nicknames [lnick] = id
class Convert (object) :
# 54 : 655 # nick == email vorn?
person_dupes = \
{ 71 : 72
, 140 : 460
, 179 : 124
, 267 : 322
, 273 : 272 # probably same unnamed person, almost same nick
, 276 : 272 # yet another dupe with almost same nick
, 307 : 306
, 314 : 311
, 372 : 380 # same phone *and* password (!) different name (!)
, 424 : 418
, 440 : 838
, 617 : 623
, 669 : 1001 # same person?
, 686 : 687
, 708 : 707
, 709 : 707
, 718 : 860
, 729 : 728
, 741 : 740
, 755 : 754
, 776 : 301
, 799 : 807
, 820 : 359
, 821 : 359
, 831 : 836
, 852 : 851
, 892 : 706
, 903 : 904
, 973 : 544
}
person_ignore = dict.fromkeys \
(( 11
, 15
, 41
, 42
, 62
, 63
, 64
, 66
, 70
, 93
, 100
, 106
, 108
, 112
, 118
, 123
, 132
, 136
, 142
, 172
, 174
, 177
, 199
, 209
, 242
, 244
, 245
, 254
, 263
, 266
, 270
, 287
, 313
, 320
, 324
, 338
, 937
# Lastname missing:
, 34
, 69
, 107
, 166
, 168
, 200
, 251
, 260
, 323
# Bogus, only initials etc, ignore if no node:
, 102
, 103
, 185
, 233
, 299
, 455
, 530
, 845
, 872
, 919
, 955
))
def __init__ (self, cmd, scope, debug = False) :
self.anonymize = cmd.anonymize
self.verbose = cmd.verbose
self.pers_exception = {}
try :
pf = open ('pers.csv', 'r')
cr = csv.reader (pf, delimiter = ';')
for line in cr :
self.pers_exception [int (line [0])] = (line [1], line [2])
except IOError :
print "WARN: Can't read additional person data"
self.debug = debug
if len (cmd.argv) > 0 :
f = open (cmd.argv [0])
else :
f = sys.stdin
self.scope = scope
self.ffm = self.scope.FFM
self.pap = self.scope.GTW.OMP.PAP
self.networks = {}
self.parser = SQL_Parser (verbose = False, fix_double_encode = 1)
self.parser.parse (f)
self.contents = self.parser.contents
self.tables = self.parser.tables
self.dev_by_id = {}
self.ffm_node = {}
self.member_by_id = {}
self.net_by_id = {}
self.nicknames = {}
self.nifin_by_id = {}
self.ntype_by_id = {}
self.node_by_id = {}
self.person_by_id = {}
self.phone_ids = {}
# end def __init__
def create (self) :
if self.anonymize :
self.fake_persons ()
else :
self.create_persons ()
self.create_nettypes ()
self.scope.commit ()
self.create_nodes ()
self.scope.commit ()
self.create_networks ()
self.scope.commit ()
self.create_devices ()
self.scope.commit ()
self.create_interfaces ()
self.scope.commit ()
self.create_dns_aliases ()
self.scope.commit ()
# end def create
def create_devices (self) :
# ignore snmp_ip and snmp_lastseen (only used by three nodes)
dt = self.ffm.Net_Device_Type.instance (name = 'Generic', raw = True)
for d in sorted (self.contents ['node'], key = lambda x : x.id) :
if self.verbose :
print "INFO: Dev: %s Node: %s" % (d.id, d.location_id)
node = None
n = self.node_by_id.get (d.location_id)
if n :
if d.person_id and d.person_id != n.person_id :
print \
( "WARN: Device (node) %s, (loc) %s: "
"person mismatch d:%s n:%s"
% (d.id, n.id, d.person_id, n.person_id)
)
node = self.ffm_node.get (d.location_id)
if not node :
print "WARN: Node (location) %s for dev (node) %s missing" \
% (d.location_id, d.id)
continue
else :
mgr = self.person_by_id.get (d.person_id) or self.graz_admin
node = self.ffm.Node \
( name = d.name
, desc = 'Auto-created node (id: %s)' % d.location_id
, show_in_map = True
, manager = mgr
, raw = True
)
print "WARN: Manufacturing Node (loc: %s) for dev (node) %s" \
% (d.location_id, d.id)
dev = self.ffm.Net_Device \
( left = dt
, node = node
, name = d.name
, desc = d.comment
, raw = True
)
self.set_creation (dev, d.time)
self.dev_by_id [d.id] = dev
if len (self.scope.uncommitted_changes) > 10 :
self.scope.commit ()
# end def create_devices
def create_dns_aliases (self) :
for dal in self.contents ['dnsalias'] :
if dal.ip_id not in self.nifin_by_id :
print 'WARN: ignoring dns_alias %s "%s" IP not found %s' \
% (dal.id, dal.name, dal.ip_id)
return
self.ffm.IP4_DNS_Alias \
(left = self.nifin_by_id [dal.ip_id], name = dal.name)
if len (self.scope.uncommitted_changes) > 10 :
self.scope.commit ()
# end def create_dns_aliases
def create_interfaces (self) :
for iface in self.contents ['ip'] :
if iface.node_id not in self.dev_by_id :
print "WARN: Ignoring IP %s %s: no device (node) %s" \
% (iface.id, iface.ip, iface.node_id)
continue
dev = self.dev_by_id [iface.node_id]
net = self.net_by_id [iface.net_id]
ip = IP4_Address (iface.ip)
if ip not in net.net_address :
parent = self.ffm.IP4_Network.query \
( Q.net_address.CONTAINS (ip)
, ~ Q.electric
, sort_key = TFL.Sorted_By ("-net_address.mask_len")
).first ()
print "WARN: IP %s %s not in net %s %s found %s" \
% ( iface.id
, iface.ip
, iface.net_id
, net.net_address
, parent.net_address
)
net = parent
nw = net.reserve (ip, owner = dev.node.owner)
nif = self.ffm.Wired_Interface (left = dev, name = iface.name)
nii = self.ffm.Net_Interface_in_IP4_Network \
(nif, nw, mask_len = 32, name = iface.name)
self.nifin_by_id [iface.id] = nii
if len (self.scope.uncommitted_changes) > 10 :
self.scope.commit ()
# end def create_interfaces
def create_nettypes (self) :
""" Network ranges for reservation
"""
by_mask = {}
# first group by netmask
for nw in self.contents ['nettype'] :
if not nw.comment :
print 'WARN: Ignoring nettype %s "%s"' % (nw.id, nw.name)
continue
for net_ip in nw.comment.split (',') :
ip = IP4_Address (net_ip)
if ip.mask not in by_mask :
by_mask [ip.mask] = []
by_mask [ip.mask].append ((ip, nw.name, nw.id))
typ = self.ffm.IP4_Network
for mask in sorted (by_mask) :
for ip, name, id in by_mask [mask] :
if id not in self.ntype_by_id :
self.ntype_by_id [id] = []
r = typ.query \
( Q.net_address.CONTAINS (ip)
, ~ Q.electric
, sort_key = TFL.Sorted_By ("-net_address.mask_len")
).first ()
reserver = r.reserve if r else typ
network = reserver (ip, owner = self.graz_admin)
self.ntype_by_id [id].append (network)
if name :
network.set_raw (desc = name)
# end def create_nettypes
def create_networks (self) :
for net in self.contents ['net'] :
parents = self.ntype_by_id.get (net.nettype_id, [])
node = self.ffm_node.get (net.location_id)
ip = IP4_Address (net.netip, net.netmask)
if node :
owner = node.owner
else :
print "WARN: Network %s %s Location %s missing" \
% (net.id, net.netip, net.location_id)
owner = self.graz_admin
parent = None
for p in parents :
if ip in p.net_address :
parent = p
break
else :
parent = None
for ps in self.ntype_by_id.itervalues () :
for p in ps :
if ip in p.net_address :
parent = p
print "Got parent in ntype_by_id: %s" % parent
break
else :
parent = self.ffm.IP4_Network.query \
( Q.net_address.CONTAINS (ip)
, ~ Q.electric
, sort_key = TFL.Sorted_By ("-net_address.mask_len")
).first ()
if parent :
print "Got parent by network query: %s" % parent
if parent :
reserver = parent.reserve
else :
print "WARN: No parent: new network: %s" % ip
reserver = self.ffm.IP4_Network
network = reserver (ip, owner = owner)
self.net_by_id [net.id] = network
if node :
pool = self.ffm.IP4_Pool (left = network, node = node)
if net.comment :
network.set_raw (desc = net.comment)
if len (self.scope.uncommitted_changes) > 10 :
self.scope.commit ()
# end def create_networks
def create_nodes (self) :
x_start = 4080
y_start = 4806
x_lon = 15.43844103813
y_lat = 47.07177327969
dx_lon = 50675.5176
dy_lat = 75505.521
for n in sorted (self.contents ['location'], key = lambda x : x.id) :
self.node_by_id [n.id] = n
person_id = self.person_dupes.get (n.person_id, n.person_id)
if person_id != 0 and person_id not in self.person_by_id :
# should not happen now
print "WARN: Location %s owner %s missing" % (n.id, person_id)
continue
if person_id == 0 :
person = self.graz_admin
else :
person = self.person_by_id [person_id]
lat = lon = None
if n.pixel_x is not None and n.pixel_y is not None :
lon = "%f" % (x_lon + (n.pixel_x - x_start) / dx_lon)
lat = "%f" % (y_lat + (n.pixel_y - y_start) / dy_lat)
node = self.ffm.Node \
( name = n.name
, desc = n.comment.strip () or None
, show_in_map = not n.hidden
, manager = person
, position = dict (lat = lat, lon = lon)
, raw = True
)
self.ffm_node [n.id] = node
self.set_creation (node, n.time)
if n.street :
s = ' '.join (x for x in (n.street, n.streetnr) if x)
adr = self.pap.Address.instance_or_new \
( street = s
, zip = '8010'
, city = 'Graz'
, country = 'Austria'
)
node.set (address = adr)
if n.gallery_link :
MOM = self.scope.MOM
abs = 'http://gallery.funkfeuer.at/v/Graz/Knoten/%s/'
if n.gallery_link.startswith ('http') :
abs = "%s"
url = MOM.Document \
(node, url = abs % n.gallery_link, type = 'Gallery')
if len (self.scope.uncommitted_changes) > 10 :
self.scope.commit ()
# end def create_nodes
def fake_persons (self) :
self.graz_admin = self.pap.Person \
( first_name = 'Graz'
, last_name = 'Admin'
, raw = True
)
mail = '*****@*****.**'
email = self.pap.Email (address = mail)
self.pap.Person_has_Email (self.graz_admin, email)
auth = self.scope.Auth.Account.create_new_account_x \
( mail
, enabled = True
, suspended = True
, password = uuid.uuid4 ().hex
)
self.pap.Person_has_Account (self.graz_admin, auth)
for m in sorted (self.contents ['person'], key = lambda x : x.id) :
self.person_by_id [m.id] = self.graz_admin
# end def fake_persons
def create_persons (self) :
for m in sorted (self.contents ['person'], key = lambda x : x.id) :
#print "%s: %r %r" % (m.id, m.firstname, m.lastname)
if m.id in self.person_ignore :
print "INFO: Ignoring anonymous without location: %s" % m.id
continue
if m.id in self.pers_exception :
pe = self.pers_exception [m.id]
fn, ln = (x.decode ('utf-8') for x in pe)
else :
fn = m.firstname.strip ()
ln = m.lastname.strip ()
self.member_by_id [m.id] = m
if m.id in self.person_dupes :
print "INFO: Duplicate person: %s" % m.id
continue
if not fn or not ln :
print >> sys.stderr, "WARN: name missing: %s (%r/%r)" \
% (m.id, m.firstname, m.lastname)
if not fn and not ln :
if m.nick :
fn = m.nick
else :
fn = m.email.split ('@') [0]
fn = fn or '?'
ln = ln or '?'
print "Person: %s %r/%r" % (m.id, fn, ln)
person = self.pap.Person \
( first_name = fn
, last_name = ln
, raw = True
)
self.person_by_id [m.id] = person
if m.nick :
self.try_insert_nick (m.nick, m.id, person)
if m.email :
mail = m.email.replace ('[at]', '@')
email = self.pap.Email (address = mail)
self.pap.Person_has_Email (person, email)
if m.email == '*****@*****.**' :
self.graz_admin = person
auth = self.scope.Auth.Account.create_new_account_x \
( mail
, enabled = True
, suspended = True
, password = uuid.uuid4 ().hex
)
self.pap.Person_has_Account (person, auth)
if m.tel :
self.try_insert_phone (m.tel, m.id, person)
if len (self.scope.uncommitted_changes) > 10 :
self.scope.commit ()
# get data from dupes
for d_id, m_id in self.person_dupes.iteritems () :
# older version of db or dupe removed:
if d_id not in self.member_by_id or m_id not in self.person_by_id :
continue
d = self.member_by_id [d_id]
m = self.member_by_id [m_id]
person = self.person_by_id [m_id]
if d.email :
email = self.pap.Email (address = d.email)
self.pap.Person_has_Email (person, email)
if d.nick :
self.try_insert_nick (d.nick, m_id, person)
if d.tel :
self.try_insert_phone (d.tel, m_id, person)
if len (self.scope.uncommitted_changes) > 10 :
self.scope.commit ()
# end def create_persons
def set_creation (self, obj, create_time) :
if not create_time :
return
if not isinstance (create_time, datetime) :
create_time = datetime (* create_time.timetuple () [:3])
create_time = make_naive (create_time)
self.scope.ems.convert_creation_change (obj.pid, c_time = create_time)
# end def set_creation
def try_insert_phone (self, tel, id, person) :
if tel.startswith ('+430659') :
tel = '+43650' + tel [6:]
p = Phone (tel, city = "Graz")
if p :
k = str (p)
t = self.pap.Phone.instance (*p)
if t :
eid = self.phone_ids [k]
prs = self.person_by_id [eid]
if eid != id :
print "WARN: %s/%s %s/%s: Duplicate Phone: %s" \
% (eid, prs.pid, id, person.pid, tel)
else :
self.phone_ids [k] = id
phone = self.pap.Phone (*p)
self.pap.Person_has_Phone (person, phone)
# end def try_insert_phone
def try_insert_nick (self, nick, id, person) :
lnick = nick.lower ()
if lnick in self.nicknames :
eid = self.nicknames [lnick]
prs = self.person_by_id [eid]
if eid != id :
print "WARN: %s/%s %s/%s Duplicate Nickname: %s" \
% (eid, prs.pid, id, person.pid, nick)
else :
n = self.pap.Nickname (nick, raw = True)
self.pap.Person_has_Nickname (person, n)
self.nicknames [lnick] = id
def __init__ (self, cmd, scope, debug = False) :
self.debug = debug
self.verbose = cmd.verbose
self.anonymize = cmd.anonymize
if len (cmd.argv) > 0 :
f = open (cmd.argv [0])
else :
f = sys.stdin
self.ip4nets = {}
self.ip6nets = {}
if cmd.network :
for n in cmd.network :
ip, comment = n.split (';', 1)
if ':' in ip :
ip = IP6_Address (ip)
self.ip6nets [ip] = comment
else :
ip = IP4_Address (ip)
self.ip4nets [ip] = comment
olsr = get_olsr_container (cmd.olsr_file)
self.olsr_nodes = {}
for t in olsr.topo.forward.iterkeys () :
self.olsr_nodes [t] = True
for t in olsr.topo.reverse.iterkeys () :
self.olsr_nodes [t] = True
self.olsr_mid = olsr.mid.by_ip
self.olsr_hna = olsr.hna
self.rev_mid = {}
for k, v in self.olsr_mid.iteritems () :
if k not in self.olsr_nodes :
pyk.fprint ("WARN: MIB %s: not in OLSR Topology" % k)
#assert k in self.olsr_nodes
for mid in v :
assert mid not in self.rev_mid
self.rev_mid [mid] = True
self.spider_info = pickle.load (open (cmd.spider_dump, 'rb'))
self.spider_devs = {}
self.spider_iface = {}
for ip, dev in self.spider_info.iteritems () :
if self.verbose :
pyk.fprint ("IP:", ip)
# ignore spider errors
if not isinstance (dev, Guess) :
continue
dev.mainip = ip
dev.done = False
for iface in dev.interfaces.itervalues () :
iface.done = False
for ip4 in iface.inet4 :
i4 = ip4.ip
# ignore rfc1918, link local, localnet
if unroutable (i4) :
continue
if ( i4 in self.spider_devs
and self.spider_devs [i4] != dev
) :
pyk.fprint ("WARN: Device %s/%s not equal:" % (ip, i4))
pyk.fprint ("=" * 60)
pyk.fprint (dev.verbose_repr ())
pyk.fprint ("-" * 60)
pyk.fprint (self.spider_devs [i4].verbose_repr ())
pyk.fprint ("=" * 60)
elif ( i4 in self.spider_iface
and self.spider_iface [i4] != iface
) :
assert dev == self.spider_devs [i4]
spif = self.spider_iface [i4]
pyk.fprint \
( "WARN: Interfaces %s/%s of dev-ip %s share ip %s"
% (iface.name, spif.name, ip, i4)
)
spif.names.append (iface.name)
if iface.is_wlan :
spif.is_wlan = iface.is_wlan
spif.wlan_info = getattr (iface, 'wlan_info', None)
if self.verbose :
pyk.fprint ("=" * 60)
pyk.fprint (iface)
pyk.fprint (spif)
pyk.fprint ("-" * 60)
pyk.fprint (dev.verbose_repr ())
pyk.fprint ("=" * 60)
iface = spif
self.spider_devs [i4] = dev
self.spider_iface [i4] = iface
iface.device = dev
if ip not in self.spider_devs :
pyk.fprint ("WARN: ip %s not in dev" % ip)
if self.verbose :
pyk.fprint ("=" * 60)
pyk.fprint (dev.verbose_repr ())
pyk.fprint ("=" * 60)
name = 'unknown'
assert name not in dev.interfaces
iface = Interface (4711, name)
iface.done = False
dev.interfaces [name] = iface
iface.device = dev
iface.append_inet4 (Inet4 (ip, None, None, iface = name))
self.spider_iface [ip] = iface
self.spider_devs [ip] = dev
self.scope = scope
self.ffm = self.scope.FFM
self.pap = self.scope.GTW.OMP.PAP
self.mentor = {}
self.rsrvd_nets = {}
self.ffm_node_by_id = {}
self.node_by_id = {}
self.ip_by_ip = {}
self.email_ids = {}
self.phone_ids = {}
self.person_by_id = {}
self.member_by_id = {}
self.dev_by_node = {}
self.cons_dev = {}
self.parser = SQL_Parser \
(verbose = False, fix_double_encode = True)
self.parser.parse (f)
self.contents = self.parser.contents
self.tables = self.parser.tables
class Convert (object) :
def __init__ (self, cmd, scope, debug = False) :
self.debug = debug
self.verbose = cmd.verbose
self.anonymize = cmd.anonymize
if len (cmd.argv) > 0 :
f = open (cmd.argv [0])
else :
f = sys.stdin
self.ip4nets = {}
self.ip6nets = {}
if cmd.network :
for n in cmd.network :
ip, comment = n.split (';', 1)
if ':' in ip :
ip = IP6_Address (ip)
self.ip6nets [ip] = comment
else :
ip = IP4_Address (ip)
self.ip4nets [ip] = comment
olsr = get_olsr_container (cmd.olsr_file)
self.olsr_nodes = {}
for t in olsr.topo.forward.iterkeys () :
self.olsr_nodes [t] = True
for t in olsr.topo.reverse.iterkeys () :
self.olsr_nodes [t] = True
self.olsr_mid = olsr.mid.by_ip
self.olsr_hna = olsr.hna
self.rev_mid = {}
for k, v in self.olsr_mid.iteritems () :
if k not in self.olsr_nodes :
pyk.fprint ("WARN: MIB %s: not in OLSR Topology" % k)
#assert k in self.olsr_nodes
for mid in v :
assert mid not in self.rev_mid
self.rev_mid [mid] = True
self.spider_info = pickle.load (open (cmd.spider_dump, 'rb'))
self.spider_devs = {}
self.spider_iface = {}
for ip, dev in self.spider_info.iteritems () :
if self.verbose :
pyk.fprint ("IP:", ip)
# ignore spider errors
if not isinstance (dev, Guess) :
continue
dev.mainip = ip
dev.done = False
for iface in dev.interfaces.itervalues () :
iface.done = False
for ip4 in iface.inet4 :
i4 = ip4.ip
# ignore rfc1918, link local, localnet
if unroutable (i4) :
continue
if ( i4 in self.spider_devs
and self.spider_devs [i4] != dev
) :
pyk.fprint ("WARN: Device %s/%s not equal:" % (ip, i4))
pyk.fprint ("=" * 60)
pyk.fprint (dev.verbose_repr ())
pyk.fprint ("-" * 60)
pyk.fprint (self.spider_devs [i4].verbose_repr ())
pyk.fprint ("=" * 60)
elif ( i4 in self.spider_iface
and self.spider_iface [i4] != iface
) :
assert dev == self.spider_devs [i4]
spif = self.spider_iface [i4]
pyk.fprint \
( "WARN: Interfaces %s/%s of dev-ip %s share ip %s"
% (iface.name, spif.name, ip, i4)
)
spif.names.append (iface.name)
if iface.is_wlan :
spif.is_wlan = iface.is_wlan
spif.wlan_info = getattr (iface, 'wlan_info', None)
if self.verbose :
pyk.fprint ("=" * 60)
pyk.fprint (iface)
pyk.fprint (spif)
pyk.fprint ("-" * 60)
pyk.fprint (dev.verbose_repr ())
pyk.fprint ("=" * 60)
iface = spif
self.spider_devs [i4] = dev
self.spider_iface [i4] = iface
iface.device = dev
if ip not in self.spider_devs :
pyk.fprint ("WARN: ip %s not in dev" % ip)
if self.verbose :
pyk.fprint ("=" * 60)
pyk.fprint (dev.verbose_repr ())
pyk.fprint ("=" * 60)
name = 'unknown'
assert name not in dev.interfaces
iface = Interface (4711, name)
iface.done = False
dev.interfaces [name] = iface
iface.device = dev
iface.append_inet4 (Inet4 (ip, None, None, iface = name))
self.spider_iface [ip] = iface
self.spider_devs [ip] = dev
self.scope = scope
self.ffm = self.scope.FFM
self.pap = self.scope.GTW.OMP.PAP
self.mentor = {}
self.rsrvd_nets = {}
self.ffm_node_by_id = {}
self.node_by_id = {}
self.ip_by_ip = {}
self.email_ids = {}
self.phone_ids = {}
self.person_by_id = {}
self.member_by_id = {}
self.dev_by_node = {}
self.cons_dev = {}
self.parser = SQL_Parser \
(verbose = False, fix_double_encode = True)
self.parser.parse (f)
self.contents = self.parser.contents
self.tables = self.parser.tables
# end def __init__
def set_last_change (self, obj, change_time, create_time) :
change_time = make_naive (change_time)
create_time = make_naive (create_time)
self.scope.ems.convert_creation_change \
(obj.pid, c_time = create_time, time = change_time or create_time)
#pyk.fprint (obj, obj.creation_date, obj.last_changed)
# end def set_last_change
def create_nodes (self) :
scope = self.scope
for n in self.contents ['nodes'] :
if len (scope.uncommitted_changes) > 100 :
scope.commit ()
gps = None
if n.gps_lat_deg is None :
assert n.gps_lat_min is None
assert n.gps_lat_sec is None
assert n.gps_lon_deg is None
assert n.gps_lon_min is None
assert n.gps_lon_sec is None
elif n.gps_lat_min is None :
assert n.gps_lat_sec is None
assert n.gps_lon_min is None
assert n.gps_lon_sec is None
if self.anonymize :
lat = "%2.2f" % n.gps_lat_deg
lon = "%2.2f" % n.gps_lon_deg
else :
lat = "%f" % n.gps_lat_deg
lon = "%f" % n.gps_lon_deg
gps = dict (lat = lat, lon = lon)
else :
assert n.gps_lat_deg == int (n.gps_lat_deg)
assert n.gps_lat_min == int (n.gps_lat_min)
assert n.gps_lon_deg == int (n.gps_lon_deg)
assert n.gps_lon_min == int (n.gps_lon_min)
lat = "%d d %d m" % (int (n.gps_lat_deg), int (n.gps_lat_min))
lon = "%d d %d m" % (int (n.gps_lon_deg), int (n.gps_lon_min))
if n.gps_lat_sec is not None :
lat = lat + " %f s" % n.gps_lat_sec
if n.gps_lon_sec is not None :
lon = lon + " %f s" % n.gps_lon_sec
gps = dict (lat = lat, lon = lon)
if self.anonymize :
lat = n.gps_lat_deg + n.gps_lat_min + (n.gps_lat_sec or 0)
lon = n.gps_lon_deg + n.gps_lon_min + (n.gps_lon_sec or 0)
gps = dict (lat = "%2.2f" % lat, lon = "%2.2f" % lon)
id = self.person_dupes.get (n.id_members, n.id_members)
owner = self.person_by_id.get (id)
if self.anonymize :
manager = owner
owner = None
elif not isinstance (owner, self.pap.Person) :
assert len (owner.actor) == 1
manager = iter (owner.actor).next ()
elif id in self.companies or id in self.associations :
manager = owner
assert len (manager.acts_for) == 1
owner = iter (manager.acts_for).next ()
elif n.id_tech_c and n.id_tech_c != n.id_members :
manager = self.person_by_id.get (n.id_tech_c)
assert (manager)
pyk.fprint ("INFO: Tech contact found: %s" % n.id_tech_c)
else :
manager = owner
owner = None
# node with missing manager has devices, use 0xff admin as owner
if not manager and n.id in self.dev_by_node :
manager = self.person_by_id.get (1)
pyk.fprint \
( "WARN: Node %s: member %s not found, using 1"
% (n.id, n.id_members)
)
if manager :
node = self.ffm.Node \
( name = n.name
, position = gps
, show_in_map = n.map
, manager = manager
, owner = owner
, raw = True
)
self.set_last_change (node, n.changed, n.created)
assert (node)
self.ffm_node_by_id [n.id] = node
else :
pyk.fprint \
( "ERR: Node %s: member %s not found"
% (n.id, n.id_members)
)
# end def create_nodes
# first id is the one to remove, the second one is the correct one
person_dupes = dict (( (373, 551) # checked, real dupe
, (338, 109) # checked, real dupe
, (189, 281) # checked, 189 contains almost no data
# and 189 has no nodes
, (285, 284) # checked, real dupe
, (299, 297) # checked, real dupe
, (300, 462) # checked, real dupe
, (542, 586) # checked, real dupe
, (251, 344) # checked, real dupe
, (188, 614) # checked, real dupe
, (177, 421) # checked, real dupe
, (432, 433) # checked, real dupe, merge addresses
, ( 26, 480) # probably: almost same nick, merge adrs
, ( 90, 499) # FIXME: same person? merge adrs?
# 90 has node 1110
# 499 has node 1105 and 812
# two accounts, one for HTL, one private?
# maybe create company?
# make company owner of 1110 and
# 499 tech-c of all nodes?
, (505, 507) # checked, real dupe
, (410, 547) # checked, real dupe
, (712, 680) # checked, real dupe
, (230, 729) # checked, real dupe
, (375, 743) # checked, real dupe
, (755, 175) # checked, real dupe
, (219, 759) # Probably same (nick similar), merge adr
, (453, 454) # checked, real dupe
, (803, 804) # checked, real dupe
, (295, 556) # same gmx address, merge adr
, (697, 814) # checked, real dupe
, (476, 854) # checked, real dupe
, (312, 307) # checked, real dupe
, (351, 355) # checked, real dupe
, (401, 309) # checked, real dupe
, (871, 870) # checked, real dupe
, (580, 898) # checked, real dupe
, (894, 896) # checked, real dupe
, (910, 766) # checked, real dupe
, (926, 927) # checked, real dupe
, (938, 939) # checked, real dupe
, (584, 939) # not entirely sure but all
# lowercase in both records
# indicates same person
, (756, 758) # checked, real dupe
, ( 0, 1) # ignore Funkfeuer Parkplatz
))
rev_person_dupes = dict ((v, k) for k, v in person_dupes.iteritems ())
merge_adr = dict.fromkeys ((432, 26, 759, 295))
phone_bogus = dict.fromkeys \
(( '01111111'
, '1234567'
, '0048334961656'
, '001123456789'
, '+972 1234567'
, '003468110524227'
, '1234'
, '0'
, '-'
, '+49 1 35738755'
, '974 5517 9729'
, '0525001340'
, '59780'
, '1013'
))
companies = dict.fromkeys ((112, ))
associations = dict.fromkeys ((146, 176, 318, 438, 737, 809))
company_actor = {134 : 37}
association_actor = { 1 : 15, 838 : 671}
person_disable = dict.fromkeys ((263, 385, 612, 621))
person_remove = dict.fromkeys ((549, 608))
def try_insert_phone (self, person, m, x, c) :
if x :
p = None
if x in self.phone_bogus :
return
try :
p = Phone (x, m.town, c)
except ValueError, err :
if str (err).startswith ('WARN') :
pyk.fprint (err)
return
if not p :
return
t = self.pap.Phone.instance (* p)
k = str (p)
if t :
eid = self.phone_ids [k]
prs = self.person_by_id [eid]
if ( prs.pid == person.pid
or self.pap.Subject_has_Phone.instance (person, t)
) :
return # don't insert twice
pyk.fprint \
( "WARN: %s/%s %s/%s: Duplicate phone: %s"
% (eid, prs.pid, m.id, person.pid, x)
)
else :
t = self.pap.Phone (* p)
self.phone_ids [k] = m.id
self.pap.Subject_has_Phone (person, t)
def __init__(self, cmd, scope, debug=False):
self.debug = debug
self.verbose = cmd.verbose
self.anonymize = cmd.anonymize
if len(cmd.argv) > 0:
f = open(cmd.argv[0])
else:
f = sys.stdin
self.ip4nets = {}
self.ip6nets = {}
if cmd.network:
for n in cmd.network:
ip, comment = n.split(';', 1)
if ':' in ip:
ip = IP6_Address(ip)
self.ip6nets[ip] = comment
else:
ip = IP4_Address(ip)
self.ip4nets[ip] = comment
olsr = get_olsr_container(cmd.olsr_file)
self.olsr_nodes = {}
for t in olsr.topo.forward.iterkeys():
self.olsr_nodes[t] = True
for t in olsr.topo.reverse.iterkeys():
self.olsr_nodes[t] = True
self.olsr_mid = olsr.mid.by_ip
self.olsr_hna = olsr.hna
self.rev_mid = {}
for k, v in self.olsr_mid.iteritems():
if k not in self.olsr_nodes:
pyk.fprint("WARN: MIB %s: not in OLSR Topology" % k)
#assert k in self.olsr_nodes
for mid in v:
assert mid not in self.rev_mid
self.rev_mid[mid] = True
self.spider_info = pickle.load(open(cmd.spider_dump, 'rb'))
self.spider_devs = {}
self.spider_iface = {}
for ip, dev in self.spider_info.iteritems():
if self.verbose:
pyk.fprint("IP:", ip)
# ignore spider errors
if not isinstance(dev, Guess):
continue
dev.mainip = ip
dev.done = False
for iface in dev.interfaces.itervalues():
iface.done = False
for ip4 in iface.inet4:
i4 = ip4.ip
# ignore rfc1918, link local, localnet
if unroutable(i4):
continue
if (i4 in self.spider_devs
and self.spider_devs[i4] != dev):
pyk.fprint("WARN: Device %s/%s not equal:" % (ip, i4))
pyk.fprint("=" * 60)
pyk.fprint(dev.verbose_repr())
pyk.fprint("-" * 60)
pyk.fprint(self.spider_devs[i4].verbose_repr())
pyk.fprint("=" * 60)
elif (i4 in self.spider_iface
and self.spider_iface[i4] != iface):
assert dev == self.spider_devs[i4]
spif = self.spider_iface[i4]
pyk.fprint \
( "WARN: Interfaces %s/%s of dev-ip %s share ip %s"
% (iface.name, spif.name, ip, i4)
)
spif.names.append(iface.name)
if iface.is_wlan:
spif.is_wlan = iface.is_wlan
spif.wlan_info = getattr(iface, 'wlan_info', None)
if self.verbose:
pyk.fprint("=" * 60)
pyk.fprint(iface)
pyk.fprint(spif)
pyk.fprint("-" * 60)
pyk.fprint(dev.verbose_repr())
pyk.fprint("=" * 60)
iface = spif
self.spider_devs[i4] = dev
self.spider_iface[i4] = iface
iface.device = dev
if ip not in self.spider_devs:
pyk.fprint("WARN: ip %s not in dev" % ip)
if self.verbose:
pyk.fprint("=" * 60)
pyk.fprint(dev.verbose_repr())
pyk.fprint("=" * 60)
name = 'unknown'
assert name not in dev.interfaces
iface = Interface(4711, name)
iface.done = False
dev.interfaces[name] = iface
iface.device = dev
iface.append_inet4(Inet4(ip, None, None, iface=name))
self.spider_iface[ip] = iface
self.spider_devs[ip] = dev
self.scope = scope
self.ffm = self.scope.FFM
self.pap = self.scope.GTW.OMP.PAP
self.mentor = {}
self.rsrvd_nets = {}
self.ffm_node_by_id = {}
self.node_by_id = {}
self.ip_by_ip = {}
self.email_ids = {}
self.phone_ids = {}
self.person_by_id = {}
self.member_by_id = {}
self.dev_by_node = {}
self.cons_dev = {}
self.parser = SQL_Parser \
(verbose = False, fix_double_encode = True)
self.parser.parse(f)
self.contents = self.parser.contents
self.tables = self.parser.tables
class Convert(object):
def __init__(self, cmd, scope, debug=False):
self.debug = debug
self.verbose = cmd.verbose
self.anonymize = cmd.anonymize
if len(cmd.argv) > 0:
f = open(cmd.argv[0])
else:
f = sys.stdin
self.ip4nets = {}
self.ip6nets = {}
if cmd.network:
for n in cmd.network:
ip, comment = n.split(';', 1)
if ':' in ip:
ip = IP6_Address(ip)
self.ip6nets[ip] = comment
else:
ip = IP4_Address(ip)
self.ip4nets[ip] = comment
olsr = get_olsr_container(cmd.olsr_file)
self.olsr_nodes = {}
for t in olsr.topo.forward.iterkeys():
self.olsr_nodes[t] = True
for t in olsr.topo.reverse.iterkeys():
self.olsr_nodes[t] = True
self.olsr_mid = olsr.mid.by_ip
self.olsr_hna = olsr.hna
self.rev_mid = {}
for k, v in self.olsr_mid.iteritems():
if k not in self.olsr_nodes:
pyk.fprint("WARN: MIB %s: not in OLSR Topology" % k)
#assert k in self.olsr_nodes
for mid in v:
assert mid not in self.rev_mid
self.rev_mid[mid] = True
self.spider_info = pickle.load(open(cmd.spider_dump, 'rb'))
self.spider_devs = {}
self.spider_iface = {}
for ip, dev in self.spider_info.iteritems():
if self.verbose:
pyk.fprint("IP:", ip)
# ignore spider errors
if not isinstance(dev, Guess):
continue
dev.mainip = ip
dev.done = False
for iface in dev.interfaces.itervalues():
iface.done = False
for ip4 in iface.inet4:
i4 = ip4.ip
# ignore rfc1918, link local, localnet
if unroutable(i4):
continue
if (i4 in self.spider_devs
and self.spider_devs[i4] != dev):
pyk.fprint("WARN: Device %s/%s not equal:" % (ip, i4))
pyk.fprint("=" * 60)
pyk.fprint(dev.verbose_repr())
pyk.fprint("-" * 60)
pyk.fprint(self.spider_devs[i4].verbose_repr())
pyk.fprint("=" * 60)
elif (i4 in self.spider_iface
and self.spider_iface[i4] != iface):
assert dev == self.spider_devs[i4]
spif = self.spider_iface[i4]
pyk.fprint \
( "WARN: Interfaces %s/%s of dev-ip %s share ip %s"
% (iface.name, spif.name, ip, i4)
)
spif.names.append(iface.name)
if iface.is_wlan:
spif.is_wlan = iface.is_wlan
spif.wlan_info = getattr(iface, 'wlan_info', None)
if self.verbose:
pyk.fprint("=" * 60)
pyk.fprint(iface)
pyk.fprint(spif)
pyk.fprint("-" * 60)
pyk.fprint(dev.verbose_repr())
pyk.fprint("=" * 60)
iface = spif
self.spider_devs[i4] = dev
self.spider_iface[i4] = iface
iface.device = dev
if ip not in self.spider_devs:
pyk.fprint("WARN: ip %s not in dev" % ip)
if self.verbose:
pyk.fprint("=" * 60)
pyk.fprint(dev.verbose_repr())
pyk.fprint("=" * 60)
name = 'unknown'
assert name not in dev.interfaces
iface = Interface(4711, name)
iface.done = False
dev.interfaces[name] = iface
iface.device = dev
iface.append_inet4(Inet4(ip, None, None, iface=name))
self.spider_iface[ip] = iface
self.spider_devs[ip] = dev
self.scope = scope
self.ffm = self.scope.FFM
self.pap = self.scope.GTW.OMP.PAP
self.mentor = {}
self.rsrvd_nets = {}
self.ffm_node_by_id = {}
self.node_by_id = {}
self.ip_by_ip = {}
self.email_ids = {}
self.phone_ids = {}
self.person_by_id = {}
self.member_by_id = {}
self.dev_by_node = {}
self.cons_dev = {}
self.parser = SQL_Parser \
(verbose = False, fix_double_encode = True)
self.parser.parse(f)
self.contents = self.parser.contents
self.tables = self.parser.tables
# end def __init__
def set_last_change(self, obj, change_time, create_time):
change_time = make_naive(change_time)
create_time = make_naive(create_time)
self.scope.ems.convert_creation_change \
(obj.pid, c_time = create_time, time = change_time or create_time)
#pyk.fprint (obj, obj.creation_date, obj.last_changed)
# end def set_last_change
def create_nodes(self):
scope = self.scope
for n in self.contents['nodes']:
if len(scope.uncommitted_changes) > 100:
scope.commit()
gps = None
if n.gps_lat_deg is None:
assert n.gps_lat_min is None
assert n.gps_lat_sec is None
assert n.gps_lon_deg is None
assert n.gps_lon_min is None
assert n.gps_lon_sec is None
elif n.gps_lat_min is None:
assert n.gps_lat_sec is None
assert n.gps_lon_min is None
assert n.gps_lon_sec is None
if self.anonymize:
lat = "%2.2f" % n.gps_lat_deg
lon = "%2.2f" % n.gps_lon_deg
else:
lat = "%f" % n.gps_lat_deg
lon = "%f" % n.gps_lon_deg
gps = dict(lat=lat, lon=lon)
else:
assert n.gps_lat_deg == int(n.gps_lat_deg)
assert n.gps_lat_min == int(n.gps_lat_min)
assert n.gps_lon_deg == int(n.gps_lon_deg)
assert n.gps_lon_min == int(n.gps_lon_min)
lat = "%d d %d m" % (int(n.gps_lat_deg), int(n.gps_lat_min))
lon = "%d d %d m" % (int(n.gps_lon_deg), int(n.gps_lon_min))
if n.gps_lat_sec is not None:
lat = lat + " %f s" % n.gps_lat_sec
if n.gps_lon_sec is not None:
lon = lon + " %f s" % n.gps_lon_sec
gps = dict(lat=lat, lon=lon)
if self.anonymize:
lat = n.gps_lat_deg + n.gps_lat_min + (n.gps_lat_sec or 0)
lon = n.gps_lon_deg + n.gps_lon_min + (n.gps_lon_sec or 0)
gps = dict(lat="%2.2f" % lat, lon="%2.2f" % lon)
id = self.person_dupes.get(n.id_members, n.id_members)
owner = self.person_by_id.get(id)
if self.anonymize:
manager = owner
owner = None
elif not isinstance(owner, self.pap.Person):
assert len(owner.actor) == 1
manager = iter(owner.actor).next()
elif id in self.companies or id in self.associations:
manager = owner
assert len(manager.acts_for) == 1
owner = iter(manager.acts_for).next()
elif n.id_tech_c and n.id_tech_c != n.id_members:
manager = self.person_by_id.get(n.id_tech_c)
assert (manager)
pyk.fprint("INFO: Tech contact found: %s" % n.id_tech_c)
else:
manager = owner
owner = None
# node with missing manager has devices, use 0xff admin as owner
if not manager and n.id in self.dev_by_node:
manager = self.person_by_id.get(1)
pyk.fprint \
( "WARN: Node %s: member %s not found, using 1"
% (n.id, n.id_members)
)
if manager:
node = self.ffm.Node \
( name = n.name
, position = gps
, show_in_map = n.map
, manager = manager
, owner = owner
, raw = True
)
self.set_last_change(node, n.changed, n.created)
assert (node)
self.ffm_node_by_id[n.id] = node
else:
pyk.fprint \
( "ERR: Node %s: member %s not found"
% (n.id, n.id_members)
)
# end def create_nodes
# first id is the one to remove, the second one is the correct one
person_dupes = dict((
(373, 551) # checked, real dupe
,
(338, 109) # checked, real dupe
,
(189, 281) # checked, 189 contains almost no data
# and 189 has no nodes
,
(285, 284) # checked, real dupe
,
(299, 297) # checked, real dupe
,
(300, 462) # checked, real dupe
,
(542, 586) # checked, real dupe
,
(251, 344) # checked, real dupe
,
(188, 614) # checked, real dupe
,
(177, 421) # checked, real dupe
,
(432, 433) # checked, real dupe, merge addresses
,
(26, 480) # probably: almost same nick, merge adrs
,
(90, 499) # FIXME: same person? merge adrs?
# 90 has node 1110
# 499 has node 1105 and 812
# two accounts, one for HTL, one private?
# maybe create company?
# make company owner of 1110 and
# 499 tech-c of all nodes?
,
(505, 507) # checked, real dupe
,
(410, 547) # checked, real dupe
,
(712, 680) # checked, real dupe
,
(230, 729) # checked, real dupe
,
(375, 743) # checked, real dupe
,
(755, 175) # checked, real dupe
,
(219, 759) # Probably same (nick similar), merge adr
,
(453, 454) # checked, real dupe
,
(803, 804) # checked, real dupe
,
(295, 556) # same gmx address, merge adr
,
(697, 814) # checked, real dupe
,
(476, 854) # checked, real dupe
,
(312, 307) # checked, real dupe
,
(351, 355) # checked, real dupe
,
(401, 309) # checked, real dupe
,
(871, 870) # checked, real dupe
,
(580, 898) # checked, real dupe
,
(894, 896) # checked, real dupe
,
(910, 766) # checked, real dupe
,
(926, 927) # checked, real dupe
,
(938, 939) # checked, real dupe
,
(584, 939) # not entirely sure but all
# lowercase in both records
# indicates same person
,
(756, 758) # checked, real dupe
,
(0, 1) # ignore Funkfeuer Parkplatz
))
rev_person_dupes = dict((v, k) for k, v in person_dupes.iteritems())
merge_adr = dict.fromkeys((432, 26, 759, 295))
phone_bogus = dict.fromkeys \
(( '01111111'
, '1234567'
, '0048334961656'
, '001123456789'
, '+972 1234567'
, '003468110524227'
, '1234'
, '0'
, '-'
, '+49 1 35738755'
, '974 5517 9729'
, '0525001340'
, '59780'
, '1013'
))
companies = dict.fromkeys((112, ))
associations = dict.fromkeys((146, 176, 318, 438, 737, 809))
company_actor = {134: 37}
association_actor = {1: 15, 838: 671}
person_disable = dict.fromkeys((263, 385, 612, 621))
person_remove = dict.fromkeys((549, 608))
def try_insert_phone(self, person, m, x, c):
if x:
p = None
if x in self.phone_bogus:
return
try:
p = Phone(x, m.town, c)
except ValueError, err:
if str(err).startswith('WARN'):
pyk.fprint(err)
return
if not p:
return
t = self.pap.Phone.instance(*p)
k = str(p)
if t:
eid = self.phone_ids[k]
prs = self.person_by_id[eid]
if (prs.pid == person.pid
or self.pap.Subject_has_Phone.instance(person, t)):
return # don't insert twice
pyk.fprint \
( "WARN: %s/%s %s/%s: Duplicate phone: %s"
% (eid, prs.pid, m.id, person.pid, x)
)
else:
t = self.pap.Phone(*p)
self.phone_ids[k] = m.id
self.pap.Subject_has_Phone(person, t)
class Convert(object):
person_dupes = \
{ 179 : 124
, 267 : 322
, 307 : 306
, 372 : 380 # same phone *and* password (!) different name (!)
, 424 : 418
, 617 : 623
, 686 : 687
, 708 : 707
, 709 : 707
, 718 : 860
, 729 : 728
, 741 : 740
, 755 : 754
, 776 : 301
, 799 : 807
, 820 : 359
, 821 : 359
, 831 : 836
, 852 : 851
, 892 : 706
, 903 : 904
}
def __init__(self, cmd, scope, debug=False):
self.debug = debug
if len(cmd.argv) > 0:
f = open(cmd.argv[0])
else:
f = sys.stdin
self.scope = scope
self.ffm = self.scope.FFM
self.pap = self.scope.GTW.OMP.PAP
self.networks = {}
self.parser = SQL_Parser(verbose=False, fix_double_encode=1)
self.parser.parse(f)
self.contents = self.parser.contents
self.tables = self.parser.tables
self.member_by_id = {}
self.person_by_id = {}
self.nicknames = {}
self.node_by_id = {}
self.phone_ids = {}
# end def __init__
def create(self):
self.create_persons()
self.create_nodes()
self.create_devices()
# end def create
def create_devices(self):
# ignore snmp_ip and snmp_lastseen (only used by three nodes)
# FIXME: Use smokeping?
# FIXME: hastinc?
for d in self.contents['node']:
if d.location_id not in self.node_by_id:
print "WARN: Ignoring device (node) %s with location_id %s" \
% (d.id, d.location_id)
continue
n = self.node_by_id[d.location_id]
if d.person_id and d.person_id != n.person_id:
print "person %s: d:%s n:%s" % (d.id, d.person_id, n.person_id)
# end def create_devices
def create_nodes(self):
x_start = 4080
y_start = 4806
x_lon = 15.43844103813
y_lat = 47.07177327969
dx_lon = 50675.5176
dy_lat = 75505.521
for n in sorted(self.contents['location'], key=lambda x: x.id):
self.node_by_id[n.id] = n
person_id = self.person_dupes.get(n.person_id, n.person_id)
if person_id not in self.person_by_id:
print "WARN: Location %s owner %s missing" % (n.id, person_id)
continue
person = self.person_by_id[person_id]
lat = lon = None
if n.pixel_x is not None and n.pixel_y is not None:
lon = "%f" % (x_lon + (n.pixel_x - x_start) / dx_lon)
lat = "%f" % (y_lat + (n.pixel_y - y_start) / dy_lat)
node = self.ffm.Node \
( name = n.name
, show_in_map = not n.hidden
, manager = person
, position = dict (lat = lat, lon = lon)
, raw = True
)
if n.street:
s = ' '.join(x for x in (n.street, n.streetnr) if x)
adr = self.pap.Address.instance_or_new \
( street = s
, zip = '8010'
, city = 'Graz'
, country = 'Austria'
)
self.pap.Node_has_Address(node, adr)
if n.gallery_link:
abs = 'http://gallery.funkfeuer.at/v/Graz/Knoten/%s/'
if n.gallery_link.startswith('http'):
abs = "%s"
url = self.pap.Url.instance_or_new \
(abs % n.gallery_link, desc = 'Gallery')
self.pap.Node_has_Url(node, url)
#print "%4d %s %s %r %s %s %s %r" % \
# (n.id, n.hastinc, n.time, n.street, n.streetnr, n.gps_lon, n.gps_lat, n.comment)
# end def create_nodes
def create_persons(self):
for m in sorted(self.contents['person'], key=lambda x: x.id):
#print "%s: %r %r" % (m.id, m.firstname, m.lastname)
self.member_by_id[m.id] = m
if m.id in self.person_dupes:
print "INFO: Duplicate person: %s" % m.id
continue
if not m.firstname or not m.lastname:
print >> sys.stderr, "WARN: name missing: %s (%s/%s)" \
% (m.id, m.firstname, m.lastname)
continue
person = self.pap.Person \
( first_name = m.firstname.strip ()
, last_name = m.lastname.strip ()
, raw = True
)
self.person_by_id[m.id] = person
if m.nick:
self.try_insert_nick(m.nick, m.id, person)
if m.email:
email = self.pap.Email(address=m.email)
self.pap.Person_has_Email(person, email)
if m.tel:
self.try_insert_phone(m.tel, m.id, person)
# get data from dupes
for d_id, m_id in self.person_dupes.iteritems():
# older version of db or dupe removed:
if d_id not in self.member_by_id or m_id not in self.person_by_id:
continue
d = self.member_by_id[d_id]
m = self.member_by_id[m_id]
person = self.person_by_id[m_id]
if d.email:
email = self.pap.Email(address=d.email)
self.pap.Person_has_Email(person, email)
if d.nick:
self.try_insert_nick(d.nick, m_id, person)
if d.tel:
self.try_insert_phone(d.tel, m_id, person)
# end def create_persons
def set_creation(self, obj, create_time):
create_time = make_naive(create_time)
self.scope.ems.convert_creation_change(obj.pid, c_time=create_time)
# end def set_creation
def try_insert_phone(self, tel, id, person):
p = Phone(tel, city="Graz")
if p:
k = str(p)
t = self.pap.Phone.instance(*p)
if t:
eid = self.phone_ids[k]
prs = self.person_by_id[eid]
if eid != id:
print "WARN: %s/%s %s/%s: Duplicate Phone: %s" \
% (eid, prs.pid, id, person.pid, tel)
else:
self.phone_ids[k] = id
phone = self.pap.Phone(*p)
self.pap.Person_has_Phone(person, phone)
# end def try_insert_phone
def try_insert_nick(self, nick, id, person):
lnick = nick.lower()
if lnick in self.nicknames:
eid = self.nicknames[lnick]
prs = self.person_by_id[eid]
if eid != id:
print "WARN: %s/%s %s/%s Duplicate Nickname: %s" \
% (eid, prs.pid, id, person.pid, nick)
else:
n = self.pap.Nickname(nick, raw=True)
self.pap.Person_has_Nickname(person, n)
self.nicknames[lnick] = id
