def discoverMeters(self):
# find all the AcquiSuite boxes
response = requests.get(self.bmoroot + self.statuspage, auth=self.auth)
soup = bs(response.content, features="html.parser")
for tr in soup.findAll('tr'):
tds = tr('td')
if len(tds) != 6:
continue
name = tds[0].a.string
self.devices[name] = {
'ip' : self.remove_nbsp(tds[3].string),
'href' : tds[0].a['href'],
}
# look at all the meters hanging off each of them
for location in self.devices:
response = requests.get(self.bmoroot + self.devices[location]['href'], auth=self.auth)
soup = bs(response.content, features="html.parser")
subdevices = []
for tr in soup.findAll('tr'):
tds = tr('td')
if len(tds) != 5 or tds[3].a != None:
continue
subdevices.append({
'address' : re.sub("<.*?>", "", str(tds[0])),
'status': self.remove_nbsp(tds[1].string),
'name' : self.remove_nbsp(tds[2].string),
'type' : self.remove_nbsp(tds[3].string),
'firmware': self.remove_nbsp(tds[4].string)
})
self.devices[location]['subdevices'] = subdevices
for location, devs in self.devices.items():
params = urllib.parse.parse_qs(urllib.parse.urlsplit(devs['href']).query)
if "AS" not in params or "DB" not in params:
continue
if location in self.auth:
continue
thisconf = {}
for dev in devs['subdevices']:
if sensordb.get_map(dev['type'], location) != None:
dlurl = self.bmoroot + 'mbdev_export.php/' + params['AS'][0] + '_' + \
dev['address'] + '.csv' + "?DB=" + params['DB'][0] + '&AS=' + \
params['AS'][0] + '&MB=' + dev['address'] + '&DOWNLOAD=YES' + \
"&COLNAMES=ON&EXPORTTIMEZONE=UTC&DELIMITER=COMMA" + \
'&DATE_RANGE_STARTTIME={}&DATE_RANGE_ENDTIME={}'
dlurl = dlurl.replace(" ", "")
thisconf[dev['name']] = (
dev['type'],
dlurl)
if thisconf:
self.conf[location] = thisconf
def make_field_idxs(type, header, location=None):
paths = [None]
map_ = sensordb.get_map(type, header=header, location=location)
for t in header[1:]:
paths.append(None)
for channel in map_['sensors'] + map_['meters']:
if t.strip().startswith(channel[0]):
paths[-1] = (channel[2], channel[3])
break
ddups = {}
for elt in paths:
if elt:
name = '-'.join(elt)
ddups[name] = ddups.get(name, 0) + 1
for k, v in ddups.iteritems():
if v > 1:
print "WARNING:", v, "matching channels for", k
print header
print paths
print ddups
return paths, map_
def crawler(start, end, getDevices, getConfig):
starttime = start.split("-")
endtime = end.split("-")
datadir = "data"
createFolder(datadir)
# find all the AcquiSuite boxes
devices = {}
response = requests.get(BMOROOT + STATUSPAGE, auth=AUTH)
soup = bs(response.content, features="html.parser")
for tr in soup.findAll('tr'):
tds = tr('td')
if len(tds) != 6:
continue
name = tds[0].a.string
devices[name] = {
'ip': remove_nbsp(tds[3].string),
'href': tds[0].a['href'],
}
# look at all the meters hanging off each of them
for location in devices:
print("Location: ", location, " URL: ",
BMOROOT + devices[location]['href'])
response = requests.get(BMOROOT + devices[location]['href'], auth=AUTH)
soup = bs(response.content, features="html.parser")
subdevices = []
for tr in soup.findAll('tr'):
tds = tr('td')
if len(tds) != 5 or tds[3].a != None:
continue
subdevices.append({
'address': re.sub("<.*?>", "", str(tds[0])),
'status': remove_nbsp(tds[1].string),
'name': remove_nbsp(tds[2].string),
'type': remove_nbsp(tds[3].string),
'firmware': remove_nbsp(tds[4].string)
})
devices[location]['subdevices'] = subdevices
if getDevices:
devices_json = open("devices.json", "w")
with open("devices.json", "w") as out:
out.write(json.dumps(devices))
devices_json.close()
print("Exported Device Data to JSON")
conf = {}
for location, devs in devices.items():
params = urllib.parse.parse_qs(
urllib.parse.urlsplit(devs['href']).query)
if "AS" not in params or "DB" not in params:
continue
if location in AUTH:
continue
thisconf = {}
for dev in devs['subdevices']:
if sensordb.get_map(dev['type'], location) != None:
dlurl = BMOROOT + 'mbdev_export.php/' + params['AS'][0] + '_' + \
dev['address'] + '.csv' + "?DB=" + params['DB'][0] + '&AS=' + \
params['AS'][0] + '&MB=' + dev['address'] + '&DOWNLOAD=YES' + \
"&COLNAMES=ON&EXPORTTIMEZONE=UTC&DELIMITER=COMMA" + \
'&DATE_RANGE_STARTTIME={}&DATE_RANGE_ENDTIME={}'
dlurl = dlurl.replace(" ", "")
thisconf[dev['name']] = (dev['type'], dlurl)
if thisconf:
conf[location] = thisconf
# generate config file
cf = configparser.RawConfigParser('', ordereddict.OrderedDict)
cf.optionxform = str
cf.add_section('server')
cf.set('server', 'SuggestThreadPool', '20')
cf.set('server', 'Port', '9051')
cf.add_section('/')
cf.set('/', 'Metadata/Location/Campus', 'UCB')
cf.set('/', 'Metadata/SourceName', 'buildingmanageronline archive')
cf.set('/', 'uuid', '91dde108-d02b-11e0-8542-0026bb56ec92')
requests_total = 0
requests_failed = 0
requests_nodata = 0
for building in conf:
building_path = '/' + to_pathname(building)
cf.add_section(building_path)
cf.set(building_path, 'type', 'Collection')
for metername in conf[building].keys():
metertype, url = conf[building][metername]
building_name = building
if "New" in building_name:
building_name = building_name[:building_name.index("New")]
if "NEW" in building_name:
building_name = building_name[:building_name.index("NEW")]
meter_path = building_path + '/' + to_pathname(metername)
cf.add_section(meter_path)
cf.set(meter_path, 'Metadata/Extra/MeterName', metername)
cf.set(meter_path, 'Metadata/Instrument/Model',
'"' + metertype + '"')
cf.set(meter_path, 'Metadata/Location/Building', building_name)
cf.set(meter_path, 'Url', url)
# add any extra config options specific to this meter type
sensor_map = sensordb.get_map(metertype, building_name)
if 'extra' in sensor_map:
for k, val in sensor_map['extra'].items():
cf.set(meter_path, k, val)
req_url = url.format(start, end) \
+ "&mnuStartMonth=" + starttime[1] \
+ "&mnuStartDay=" + starttime[2] \
+ "&mnuStartYear=" + starttime[0] \
+ "&mnuStartTime=0%3A0" \
+ "&mnuEndMonth=" + endtime[1] \
+ "&mnuEndDay=" + endtime[2] \
+ "&mnuEndYear=" + endtime[0] \
+ "&mnuEndTime=23%3A59"
response = requests.get(url=req_url, auth=AUTH)
requests_total += 1
if response:
data = response.content.decode('utf-8')
if "No data found within range" in data:
print("[Warning] No data found for " + meter_path +
" within given date range")
requests_nodata += 1
continue
filename = meter_path.replace(r"[\[\]]", "")
filename = datadir + "/" + filename.strip("/").replace(
"/", "-")
filename += "-" + start + "to" + end
filename += ".csv"
try:
csv_df = pd.read_csv(io.StringIO(data))
csv_df.to_csv(filename, index=False)
print("Downloaded Data for", meter_path)
except pd.errors.ParserError as error:
print("[Error] Failed to download file " + meter_path +
". Error: " + error)
requests_failed += 1
else:
requests_failed += 1
print("Received", response.status_code,
"response for request:", req_url)
if getConfig:
config_file = open("config.ini", "w")
cf.write(config_file)
config_file.close()
print("\n-------------")
print("# of Requests - No Data: ", requests_nodata)
print("# of Requests - Failed: ", requests_failed)
print("# of Requests - Total: ", requests_total)
no_data_percent = float(requests_nodata) / float(requests_total) * 100
failed_percent = float(requests_failed) / float(requests_total) * 100
print(f"Percent of Requests - No Data: {no_data_percent:.3f}")
print(f"Percent of Requests - Failed: {failed_percent:.3f}")
conf.optionxform = str
conf.add_section('/')
conf.set('/', 'Metadata/Location/Campus', 'UCB')
conf.set('/', 'type', 'Collection')
for location, devs in devices.iteritems():
if not location in auth.AUTH: continue
parent_sec = make_section((location, ))
conf.add_section(parent_sec)
conf.set(parent_sec, 'type', 'Collection')
conf.set(parent_sec, 'Metadata/Location/Building', location)
sec = None
for d in devs['subdevices']:
map = sensordb.get_map(d['type'], location)
if map != None:
sec = make_section((location, d['name']))
conf.add_section(sec)
conf.set(sec, 'type', 'smap.drivers.obvius.obvius.Driver')
conf.set(sec, 'Username', auth.AUTH[location][0])
conf.set(sec, 'Password', auth.AUTH[location][1])
conf.set(sec, 'Url', 'http://' + devs['ip'] +
'/setup/devicexml.cgi?ADDRESS=%s&TYPE=DATA' % d['address'],)
conf.set(sec, 'ObviousType', d['type'])
if sec == None:
conf.remove_section(parent_sec)
conf.write(sys.stderr)
elif opts.load:
conf = {}
