def handle(self, *args, **options):
# get admin user
user = User.objects.get(username='******')
# dict with airpressure
baros = {}
# set of well to update
wells = set()
logger.info('Updating time series')
# loop over screens with Ellitrack loggers
for screen in Screen.objects.filter(
loggerpos__logger__model__istartswith='etd').distinct():
logger.info(screen)
register_screen(screen)
# get or create compensated series
name = '%s COMP' % unicode(screen)
series, created = screen.mloc.series_set.get_or_create(
name=name, defaults={
'user': user,
'timezone': 'Etc/GMT-1'
})
if created:
logger.debug('Created timeseries {}'.format(series))
last = screen.last_measurement()
start = last.date if last else None
# update set of source files for all installations with Ellitrack loggers
new_files = 0
for lp in screen.loggerpos_set.filter(
logger__model__istartswith='etd'):
new_files += lp.update_files()
if new_files > 0:
old_count = series.aantal()
inserted = recomp(screen, series, start, baros)
new_count = series.aantal()
logger.debug('{} data points added to timeseries {}'.format(
new_count - old_count, series))
if inserted > 0:
wells.add(screen.well)
make_wellcharts(None, None, wells)
def handle(self, *args, **options):
# get the first superuser
# user=User.objects.filter(is_superuser=True).first()
fname = options.get('fname')
if not fname:
raise ('csv file missing')
net = Network.objects.first()
with open(fname, 'r') as f:
reader = csv.DictReader(f)
for row in reader:
name = row['label']
filter = 1
x = float(row['x'])
y = float(row['y'])
datum = row['plaatsing [date]'].strip()
if len(datum):
datum = datetime.strptime(datum, '%m/%d/%Y').date()
bkb = row['bovenkant peilbuis [mNAP]'] or None
top = row['filter bovenkant [mNAP]'] or None
bottom = row['filter onderkant [mNAP]'] or None
dia = row['Filter-diameter'] or None
loc = Point(x=x, y=y, srid=28992)
loc.transform(4326)
print(name, loc.x, loc.y)
well, created = net.well_set.update_or_create(name=name,
defaults={
'location':
loc,
'date':
datum or None
})
register_well(well)
screen, created = well.screen_set.get_or_create(nr=filter,
defaults={
'top': top,
'bottom':
bottom,
'diameter':
dia,
'refpnt':
bkb
})
register_screen(screen)
def recomp_screens(modeladmin, request, queryset):
for screen in queryset:
register_screen(screen)
series = screen.find_series()
if not series:
name = '%s COMP' % unicode(screen)
series, created = Series.objects.update_or_create(name=name,
defaults={
'user':
request.user,
'mlocatie':
screen.mloc,
'timezone':
'Etc/GMT-1'
})
recomp(screen, series)
series.validate(reset=True, accept=True, user=request.user)
make_screencharts(modeladmin, request, queryset)
def handle(self, *args, **options):
# get the first superuser
# user=User.objects.filter(is_superuser=True).first()
fname = options.get('fname')
if not fname:
raise('csv file missing')
net = Network.objects.first()
with open(fname,'r') as f:
reader = csv.DictReader(f)
for row in reader:
name = row['LOCATIE']
filter = int(row['FILTER'])
x = float(row['X'])
y = float(row['Y'])
nitg = row['NITGCODE']
loc = Point(x=x,y=y,srid=28992)
loc.transform(4326)
print(name,loc.x,loc.y)
well,created = net.well_set.update_or_create(name=name,nitg=nitg,defaults={'location':loc})
register_well(well)
screen,created = well.screen_set.get_or_create(nr=filter)
register_screen(screen)
def handle(self, *args, **options):
files = options['files']
net = Network.objects.first()
ellitrack = Generator.objects.get(name='Ellitrack')
admin = User.objects.get(username='******')
tz = pytz.timezone('Europe/Amsterdam')
for fname in files:
logger.info('Importing wells from {}'.format(fname))
with open(fname) as f:
numcreated = 0
numupdated = 0
reader = csv.DictReader(f)
for row in reader:
name = row['ID'].strip()
logger.info('Importing {}'.format(name))
x = asfloat(row['X'])
y = asfloat(row['Y'])
if x is None or y is None:
logger.info('Well {} skipped: No coordinates'.format(name))
continue
try:
defaults = {
'location': Point(x,y,srid=RDNEW),
'postcode': row['Postcode'],
'straat': row['Straat'],
'huisnummer': row['Huisnummer'],
'plaats': row['Plaats'],
'description': unicode(row['Opmerkingen'],'iso8859'),
'maaiveld': asfloat(row['Maaiveld']),
'date': row['Constructiedatum'],
}
well, created = Well.objects.update_or_create(network=net,name=name,defaults=defaults)
register_well(well)
# curdir = os.path.dirname(fname)
# fotoname = row['path foto']
# if fotoname:
# fotopath = os.path.join(curdir,fotoname)
# if os.path.exists(fotopath):
# with open(fotopath,'rb') as f:
# well.add_photo(fotoname,f)
defaults = {
'top': asfloat(row['Bovenkant filter']),
'bottom': asfloat(row['Onderkant filter']),
'refpnt': asfloat(row['Bovenkant buis']),
# 'depth': asfloat(row['Diepte'])
}
screen, created = well.screen_set.update_or_create(nr=1,defaults=defaults)
register_screen(screen)
if created:
logger.info('Added {screen}'.format(screen=str(screen)))
numcreated += 1
serial = row['Logger ID']
if serial:
datalogger, created = Datalogger.objects.get_or_create(serial=serial,defaults={'model':'etd'})
if created:
logger.info('Created logger {}'.format(serial))
start = row.get('Datum peiling','2018-10-30')
time = row.get('Tijd peiling', '12:00')
start = start + ' ' + time
start_date = datetime.strptime(start,'%Y-%m-%d %H:%M')
defaults = {
'screen': screen,
'start_date' : tz.localize(start_date),
'refpnt': screen.refpnt,
'depth': asfloat(row['Kabellengte']),
}
pos, created = datalogger.loggerpos_set.update_or_create(logger=datalogger,defaults=defaults)
if created:
logger.info('Installed {}'.format(pos))
ds, created = LoggerDatasource.objects.update_or_create(
logger = datalogger,
name = serial,
defaults={'description': 'Ellitrack datalogger {}'.format(serial),
'meetlocatie': screen.mloc,
'timezone': 'Europe/Amsterdam',
'user': admin,
'generator': ellitrack,
'url': settings.FTP_URL,
'username': settings.FTP_USERNAME,
'password': settings.FTP_PASSWORD,
})
if created:
ds.locations.add(screen.mloc)
logger.info('Created datasource {}'.format(ds))
result = ds.download()
if result:
# download succeeded, create timeseries
ds.update_parameters()
for p in ds.parameter_set.all():
series, created = p.series_set.get_or_create(
mlocatie = screen.mloc,
name = p.name,
defaults = {'description': p.description,
'unit': p.unit,
'user': admin})
if created:
logger.info('Created timeseries {}'.format(series))
else:
logger.info('Updating timeseries {}'.format(series))
series.update()
else:
logger.info('No data for {}'.format(screen))
except Exception as e:
logger.error('{}: {}'.format(name,e))
logger.info('Import completed, {} created, {} updated'.format(numcreated,numupdated))
def create_well(self, form):
# create well
network = form.cleaned_data['network']
name = form.cleaned_data['name']
srid = form.cleaned_data['srid']
x = form.cleaned_data['xcoord']
y = form.cleaned_data['ycoord']
z = form.cleaned_data['zcoord']
date = form.cleaned_data['date']
well = Well(network=network,
name=name,
location=Point(x, y, srid=int(srid)),
maaiveld=z,
date=date)
set_well_address(well)
register_well(well)
# create screen
ref = form.cleaned_data['refpnt']
top = form.cleaned_data['top']
bottom = form.cleaned_data['bottom']
depth = form.cleaned_data['depth']
screen = Screen(well=well,
nr=1,
refpnt=ref,
top=top,
bottom=bottom,
depth=depth)
register_screen(screen)
# create logger
serial = form.cleaned_data['serial']
type = form.cleaned_data['logger_type']
logger, created = Datalogger.objects.get_or_create(
serial=serial,
defaults={'model': '14' if type == 'Diver' else 'etd2'})
#create datasource
options = {'user': self.request.user}
if type == 'Diver':
options.update({
'generator':
Generator.objects.get(name__iexact='Schlumberger'),
'autoupdate':
False,
'timezone':
'CET',
})
else:
options.update({
'generator':
Generator.objects.get(name__iexact='Ellitrack'),
'url':
settings.FTP_URL,
'username':
settings.FTP_USERNAME,
'password':
settings.FTP_PASSWORD,
'timezone':
'Europe/Amsterdam',
})
datasource, created = logger.datasources.get_or_create(
name=serial, meetlocatie=screen.mloc, defaults=options)
# create logger installation
logger_depth = form.cleaned_data['logger_depth']
logger.loggerpos_set.create(screen=screen,
refpnt=ref,
start_date=date,
depth=logger_depth)
def update():
tz = pytz.timezone('CET')
start = tz.localize(
datetime.datetime(date.year, date.month, date.day))
datasource.download(start)
generate_datasource_series(None, self.request, [datasource])
make_wellcharts(None, self.request, [well])
# retrieve data in background
from threading import Thread
t = Thread(target=update)
t.start()
def handle(self, *args, **options):
files = options['files']
net = Network.objects.first()
prj = Project.objects.first()
ellitrack = Generator.objects.get(name='Ellitrack')
admin = User.objects.get(username='******')
for fname in files:
logger.info('Importing wells from {}'.format(fname))
with open(fname) as f:
numcreated = 0
numupdated = 0
reader = csv.DictReader(f)
for row in reader:
x = asfloat(row['X'])
y = asfloat(row['Y'])
name = row['peilbuisnr']
try:
loc = Point(x,y,srid=RDNEW)
description = unicode(row['Beschrijving'],'iso8859')
well, created = Well.objects.update_or_create(name=name,defaults={
'network': net,
'location': loc,
'description':description,
'date':datetime(2017,9,1)
})
if created:
register_well(well)
bottom = asfloat(row['peilbuisdiepte hand'] or row['peilbuisdiepte'])
top = bottom - 1 if bottom else None
refpnt = well.maaiveld or well.ahn # moet bovenkant buis worden!!
screen, created = well.screen_set.update_or_create(nr=1,defaults={'top': top, 'bottom': bottom, 'refpnt': refpnt})
if created:
register_screen(screen)
logger.info('Added {screen}'.format(screen=str(screen)))
numcreated += 1
else:
logger.info('Updated {screen}'.format(screen=str(screen)))
numupdated += 1
serial = row['serienummer']
if serial:
datalogger, created = Datalogger.objects.get_or_create(serial=serial,defaults={'model':'etd'})
if created:
logger.info('Created logger {}'.format(serial))
install = row['Geinstalleerd op'] or '9/10/2017'
date = datetime.strptime(install,'%d/%M/%Y')
depth = asfloat(row['kabellengte'])
pos, created = datalogger.loggerpos_set.update_or_create(logger=datalogger,defaults={'screen': screen, 'start_date': date,'depth':depth})
if created:
logger.info('Installed {}'.format(pos))
else:
logger.info('Updated {}'.format(pos))
ds, created = LoggerDatasource.objects.update_or_create(
logger = datalogger,
name = serial,
defaults={'description': 'Ellitrack datalogger serienummer {}'.format(serial),
'meetlocatie': screen.mloc,
'timezone': 'Europe/Amsterdam',
'user': admin,
'generator': ellitrack,
'url': settings.FTP_URL,
'username': settings.FTP_USERNAME,
'password': settings.FTP_PASSWORD,
})
if created:
ds.locations.add(screen.mloc)
logger.info('Created datasource {}'.format(ds))
result = ds.download()
if result:
# download succeeded, create timeseries
ds.update_parameters()
for p in ds.parameter_set.all():
series, created = p.series_set.get_or_create(
mlocatie = screen.mloc,
name = p.name,
defaults = {'description': p.description,
'unit': p.unit,
'user': admin})
if created:
logger.info('Created timeseries {}'.format(series))
series.update()
except Exception as e:
logger.error('{}: {}'.format(name,e))
logger.info('Import completed, {} created, {} updated'.format(numcreated,numupdated))
def handle(self, *args, **options):
files = options['files']
net = Network.objects.first()
ellitrack = Generator.objects.get(name='Ellitrack')
admin = User.objects.get(username='******')
tz = pytz.timezone('Europe/Amsterdam')
for fname in files:
logger.info('Importing wells from {}'.format(fname))
with open(fname) as f:
reader = csv.DictReader(f)
for row in reader:
name = row['ID'].strip()
logger.info('Importing {}'.format(name))
x = asfloat(row['X'])
y = asfloat(row['Y'])
if x is None or y is None:
logger.info(
'Well {} skipped: No coordinates'.format(name))
continue
location = Point(x, y, srid=RDNEW)
location.transform(WGS84)
try:
defaults = {
'location':
location,
'postcode':
row['Postcode'],
'straat':
row['Straat'],
'huisnummer':
row['Huisnummer'],
'plaats':
row.get('Plaats', 'Hendrik-Ido-Ambacht'),
'description':
unicode(row['Opmerkingen locatie'], 'iso8859'),
'maaiveld':
asfloat(row['Maaiveld']),
'date':
asdate(row.get('Constructiedatum'))
}
well, created = Well.objects.update_or_create(
network=net, name=name, defaults=defaults)
register_well(well)
# fotodir = os.path.join(os.path.dirname(fname),'fotos')
# for nr in range(1,6):
# fotoname = row['Foto %d'%nr]
# if fotoname:
# fotopath = os.path.join(fotodir,fotoname)
# if os.path.exists(fotopath):
# with open(fotopath,'rb') as f:
# well.add_photo(fotoname,f)
# else:
# break
#
# logdir = os.path.join(os.path.dirname(fname),'boorstaten')
# logname = row['Boorstaat']
# if logname:
# logpath = os.path.join(logdir,logname)
# if os.path.exists(logpath):
# with open(logpath,'rb') as f:
# well.set_log(logname,f)
# #continue # only update well data, photos and logs
defaults = {
'top': asfloat(row['Bovenkant filter m-MV']),
'bottom': asfloat(row['Onderkant filter m-MV']),
'refpnt': asfloat(row['Bovenkant buis']),
'depth': asfloat(row['Diepte']),
'diameter': asfloat(row['Diameter buis'])
}
screen, created = well.screen_set.update_or_create(
nr=1, defaults=defaults)
register_screen(screen)
if created:
logger.info(
'Added {screen}'.format(screen=str(screen)))
hand = Handpeilingen.objects.update_or_create(
screen=screen,
defaults={
'refpnt': 'bkb',
'user': admin,
'mlocatie': screen.mloc,
'name': '{} HAND'.format(screen),
'unit': 'm',
'type': 'scatter',
'timezone': settings.TIME_ZONE
})
serial = row['Logger ID']
if serial:
datalogger, created = Datalogger.objects.get_or_create(
serial=serial, defaults={'model': 'etd2'})
if created:
logger.info('Created logger {}'.format(serial))
defaults = {
'screen': screen,
'start_date':
asdate(row.get('Datum installatie')),
'refpnt': screen.refpnt,
'depth': asfloat(row['Kabellengte']),
}
pos, created = datalogger.loggerpos_set.update_or_create(
logger=datalogger, defaults=defaults)
if created:
logger.info('Installed {}'.format(pos))
ds, created = LoggerDatasource.objects.update_or_create(
logger=datalogger,
name=serial,
defaults={
'description':
'Ellitrack datalogger {}'.format(serial),
'meetlocatie':
screen.mloc,
'timezone':
'Europe/Amsterdam',
'user':
admin,
'generator':
ellitrack,
'url':
settings.FTP_URL,
'username':
settings.FTP_USERNAME,
'password':
settings.FTP_PASSWORD,
})
if created:
ds.locations.add(screen.mloc)
logger.info('Created datasource {}'.format(ds))
# result = ds.download()
# if result:
# # download succeeded, create timeseries
# ds.update_parameters()
# for p in ds.parameter_set.all():
# series, created = p.series_set.get_or_create(
# mlocatie = screen.mloc,
# name = p.name,
# defaults = {'description': p.description,
# 'unit': p.unit,
# 'user': admin})
# if created:
# logger.info('Created timeseries {}'.format(series))
# else:
# logger.info('Updating timeseries {}'.format(series))
# series.update()
# else:
# logger.info('No data for {}'.format(screen))
except Exception as e:
logger.error('{}: {}'.format(name, e))
logger.info('Import completed')
def register_screens(modeladmin, request, queryset):
from util import register_screen
for screen in queryset:
register_screen(screen)