def handle(self, *args, **options):
cruncher = SDW_Data_Cruncher()
states = [
"AL", "AK", "AS", "AZ", "AR", "CA", "CO", "CT", "DE", "DC", "FM",
"FL", "GA", "GU", "HI", "ID", "IL", "IN", "IA", "KS", "KY", "LA",
"ME", "MH", "MD", "MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV",
"NH", "NJ", "NM", "NY", "NC", "ND", "MP", "OH", "OK", "OR", "PW",
"PA", "PR", "RI", "SC", "SD", "TN", "TX", "UT", "VT", "VI", "VA",
"WA", "WV", "WI", "WY"
]
for state in states:
areas = cruncher.calc_state_scores(state, print_test=True)
for area in areas:
location = app_models.location.objects.filter(
fips_county=area['county_fips']).first()
if area['score'] == 0:
if not app_models.data.objects.filter(
location=location, score=area['score']).exists():
data = app_models.data()
data.location = location
data.score = area['score']
data.save()
log(
'%s, %s [%s]' % (location.major_city,
location.state, area['score']),
'success')
else:
log(
'%s, %s [%s]' % (location.major_city,
location.state, area['score']),
'info')
else:
if area['score'] - 0.05 < 0:
min_score = 0
else:
min_score = area['score'] - 0.05
max_score = area['score'] + 0.05
if not app_models.data.objects.filter(
location=location,
score__gte=min_score,
score__lte=max_score).exists():
data = app_models.data()
data.location = location
data.score = area['score']
data.save()
log(
'%s, %s [%s]' % (location.major_city,
location.state, area['score']),
'success')
else:
log(
'%s, %s [%s]' % (location.major_city,
location.state, area['score']),
'info')
def remote_water_quality_monitoring_network_task():
log('Running Remote Water Quality Monitoring Network Parser','info')
crawler = remote_water_quality_monitoring_network()
for node in crawler.get():
station = get_object_or_None(models.node, name = node['station'])
if not station:
station = models.node()
station.name = node['station']
station.sourcetype = 'live'
station.devicealias = node['station'].replace(' ','_').lower()
station.save()
log('Station: %s (%s) created' % (station.name, station.devicealias),'info')
# insert sensor data
for sensor_type, unit in _SENSOR_UNITS.iteritems():
if sensor_type in ['orp']:
continue
data = models.data()
data.source = station.sourcetype
data.node_name = station.name
data.node_alias =station.devicealias
data.sensor_type = sensor_type
data.sensor_units = unit
data.lat = station.lat
data.long = station.long
if node[sensor_type]:
data.value = round(float(node[sensor_type]), 2)
else:
data.value = 0.0
data.save()
log('Station: %s data updated' % (station.name),'info')
log('Done','success')
def upload(request):
if request.method == 'GET':
id = request.user.username
instance = data.objects.filter(user=id)
instance = serializers.serialize('json', instance)
dat = instance.replace("'", "\"")
d = json.loads(dat)
return render(request, 'app/upload.html', {'file_added': d})
elif request.method == 'POST':
# return JsonResponse({'success': 'true'})
try:
myfile = request.FILES['file']
except:
pass
id = request.user.username
if data.objects.filter(user=id,
inputfile_path=myfile.name).count() == 0:
file_name = default_storage.save(
'app/data/' + request.user.username + '/' + myfile.name,
myfile)
instance = data(user=request.user.username,
inputfile_path=myfile.name)
instance.save()
else:
print("file already exist")
return HttpResponseRedirect("upload")
def nodesim():
log('Running Node Sim','info')
for node in models.node.objects.filter(active = True):
nodesim = get_object_or_None(models.sim_node, devicealias = node.devicealias)
if nodesim:
for sensor in nodesim.sensors.all():
d = noise(mean = sensor.mean , std = sensor.std, samples = 1)
data = models.data()
data.source = 'sim'
data.value = d.next()
data.node_name = nodesim.name
data.node_alias = node.devicealias
data.sensor_name = sensor.name
data.sensor_type = sensor.type
data.sensor_units = sensor.units
data.lat = node.lat
data.long = node.long
data.save()
log('Sampled Node %s, Sensor %s, Value %s' % (nodesim.name, sensor.name, data.value) ,'success')
def transform_view():
f = request.files['data_file']
if not f:
return "No file"
# get data from form
stream = io.StringIO(f.stream.read().decode("UTF8"), newline=None)
stuff = csv_input = csv.reader(stream)
# add to DB
x = 0
for ary in csv_input:
u = models.data(sn=ary[0], vendor=ary[1], device=ary[2], disc=ary[3])
db.session.add(u)
x += 1
db.session.commit()
return render_template('CSVdone.html',
title='CSV_Loaded',
env=os.environ,
x=x,
data=stuff)
def handle(self, *args, **options):
cruncher = SDW_Data_Cruncher()
states = [
"AL", "AK", "AS", "AZ", "AR", "CA", "CO", "CT",
"DE", "DC", "FM", "FL", "GA", "GU", "HI", "ID",
"IL", "IN", "IA", "KS", "KY", "LA", "ME", "MH",
"MD", "MA", "MI", "MN", "MS", "MO", "MT", "NE",
"NV", "NH", "NJ", "NM", "NY", "NC", "ND", "MP",
"OH", "OK", "OR", "PW", "PA", "PR", "RI", "SC",
"SD", "TN", "TX", "UT", "VT", "VI", "VA", "WA",
"WV", "WI", "WY"
]
for state in states:
areas = cruncher.calc_state_scores(state, print_test=True)
for area in areas.itertuples():
location = app_models.Location.objects.filter(
fips_county=area.fips_county).first()
systems = raw_models.EpaFacilitySystem.objects.filter(
FacFIPSCode=area.fips_county).values()
systems_df = read_frame(systems)
if systems_df.shape[0] != 0:
systems_df['in_violation'] = systems_df.apply(
lambda x: x['CurrVioFlag'] == 1, axis=1)
systems_df = systems_df[[
'PWSId', 'FacName', 'FacLong', 'FacLat', 'in_violation']]
if area.score == 0:
if not app_models.data.objects.filter(location=location, score=area.score).exists():
data = app_models.data()
# todo: need to insert facilities here
data.location = location
data.score = area.score
try:
data.save()
log('%s, %s [%s]' % (location.major_city,
location.state, area.score), 'success')
except:
print(location)
else:
log('%s, %s [%s]' % (location.major_city,
location.state, area.score), 'info')
else:
if area.score - 0.05 < 0:
min_score = 0
else:
min_score = area.score - 0.05
max_score = area.score + 0.05
if not app_models.data.objects.filter(location=location, score__gte=min_score, score__lte=max_score).exists():
data = app_models.data()
# todo: need to insert facilities here
data.location = location
data.score = area.score
try:
data.save()
log('%s, %s [%s]' % (location.major_city,
location.state, area.score), 'success')
except:
print(location)
else:
log('%s, %s [%s]' % (location.major_city,
location.state, area.score), 'info')
