def lookupFlightInfo(utcStamp, timezone, vehicle, defaultTrackName):
#
# Get flight and track info and find location of sample if available
#
flight = None
try:
flight = getFlight(utcStamp, vehicle.vehicle)
if flight:
trackName = flight.name
elif vehicle:
trackName = vehicle.name
else:
trackName = defaultTrackName
track = BasaltTrack.getTrackByName(trackName)
if not track:
if trackName == defaultTrackName:
track = BasaltTrack(name=defaultTrackName)
else:
track = BasaltTrack(name=defaultTrackName,
vehicle=vehicle,
timezone=timezone)
sampleLocation = getClosestPosition(track=track,
timestamp=utcStamp,
vehicle=vehicle)
except:
sampleLocation = None
return (flight, sampleLocation)
def doImportNotes(request, sourceFile, tz, resource):
dictreader = csv.DictReader(sourceFile)
for row in dictreader:
row['author'] = request.user
if row['content'] or row['tags']:
if 'first_name' in row and 'last_name' in row:
if row['first_name'] and row['last_name']:
try:
row['author'] = User.objects.get(first_name=row['first_name'], last_name=row['last_name'])
del row['first_name']
del row['last_name']
except:
pass
if row['event_time']:
event_time = dateparser(row['event_time'])
if tz != pytz.utc:
localized_time = tz.localize(event_time)
event_time = TimeUtil.timeZoneToUtc(localized_time)
row['event_time'] = event_time
try:
# TODO implement tags when ready
del row['tags']
except:
pass
NOTE_MODEL = Note.get()
note = NOTE_MODEL(**row)
note.creation_time = datetime.now(pytz.utc)
note.modification_time = datetime.now(pytz.utc)
if resource:
note.position = getClosestPosition(timestamp=note.event_time, resource=resource)
note.save()
def lookupPosition(self):
# IMPORTANT this should not be used across multitudes of notes, it is designed to be used during construction.
if not self.position:
track=None
if hasattr(self, 'flight') and self.flight:
if hasattr(self.flight, 'track'):
track = self.flight.track
self.position = getClosestPosition(track=track, timestamp=self.event_time)
return self.position
def getPosition(self):
# IMPORTANT this should not be used across multitudes of notes, it is designed to be used during construction.
if not self.position and self.location_found == None:
resource = None
if self.flight:
if self.flight.vehicle:
resource = self.flight.vehicle.basaltresource
self.position = getClosestPosition(timestamp=self.event_time,
resource=resource)
if self.position:
self.location_found = True
else:
self.location_found = False
self.save()
return self.position
def getPosition(self):
from geocamTrack.utils import getClosestPosition
if self.position:
return self.position
elif self.position_not_found == None and self.getEventTime():
# populate the position
timestamp = self.getEventTime()
if not timestamp:
return None
foundPosition = getClosestPosition(timestamp=timestamp)
if foundPosition:
self.position = foundPosition
self.position_not_found = False
else:
self.position_not_found = True
self.save()
return self.position
def getPosition(self):
from geocamTrack.utils import getClosestPosition
if self.position:
return self.position
elif self.position_not_found == None and self.getEventTime():
# populate the position
timestamp = self.getEventTime()
if not timestamp:
return None
foundPosition = getClosestPosition(timestamp=timestamp)
if foundPosition:
self.position = foundPosition
self.position_not_found = False
else:
self.position_not_found = True
self.save()
return self.position
def save(self, commit=True):
instance = super(SampleForm, self).save(commit=False)
instance.collection_time = self.cleaned_data['collection_time']
if instance.resource and instance.collection_time:
instance.track_position = getClosestPosition(timestamp=instance.collection_time, resource=instance.resource)
if (('lat' in self.changed_data) and ('lon' in self.changed_data)) or ('altitude' in self.changed_data):
if instance.user_position is None:
instance.user_position = LOCATION_MODEL.get().objects.create(serverTimestamp = datetime.datetime.now(pytz.utc),
timestamp = instance.collection_time,
latitude = self.cleaned_data['lat'],
longitude = self.cleaned_data['lon'],
altitude = self.cleaned_data['altitude'])
else:
instance.user_position.latitude = self.cleaned_data['lat']
instance.user_position.longitude = self.cleaned_data['lon']
instance.user_position.altitude = self.cleaned_data['altitude']
if ('collector_name' in self.changed_data):
fullName = self.cleaned_data['collector_name']
try:
splitName = fullName.split(' ')
firstAndLast = [x for x in splitName if x.strip()]
collector = User.objects.filter(first_name=firstAndLast[0]).filter(last_name=firstAndLast[1])[0]
instance.collector = collector
except:
self.errors['error'] = "Save Failed. User %s does not exist." % fullName
return instance
# if fields changed, validate against the name
needsNameRebuild = False
for field in SAMPLE_MODEL.get().getFieldsForName():
if field in self.changed_data:
needsNameRebuild = True
break
if needsNameRebuild:
builtName = instance.buildName()
if instance.name != builtName:
instance.name = builtName
if commit:
instance.save()
return instance
import pytz
from basaltApp.models import *
from geocamTrack.utils import getClosestPosition
missingPositionSampleList = []
startTime = datetime.datetime(2016, 10, 30, 0, 6, 0, tzinfo=pytz.utc)
samples = BasaltSample.objects.filter(
collection_time__gte=startTime).order_by("collection_time")
sampleCount = 0
missingPositionCount = 0
for sample in samples:
if (not sample.track_position) and (not sample.user_position):
if sample.resource:
print "**** Processing: %s ****" % sample.name
sample.track_position = getClosestPosition(
timestamp=sample.collection_time, resource=sample.resource)
if not sample.track_position:
sample.track_position = getClosestPosition(
timestamp=sample.collection_time, resource=sample.resource)
if not sample.track_position:
print " No location found - adding to list of samples w/o position"
missingPositionSampleList.append(sample.name)
missingPositionCount += 1
print " Date: %s, Resource: %s, Location: %s\n" % (
sample.collection_time, sample.resource, sample.track_position)
sample.save()
sampleCount += 1
# if sample.resource is None:
# print "**** Processing: %s ****" % sample.name
# sample.resource = BasaltResource.objects.get(name=settings.XGDS_SAMPLE_DEFAULT_COLLECTOR)
# sample.track_position = getClosestPosition(timestamp=sample.collection_time, resource=sample.resource)
def create_image_set(file,
filename,
author,
vehicle,
camera,
width_height=None,
form_tz=None,
form_tz_name=None,
exif_data=None,
exif_time=None,
object_id=None,
time_mark=None,
latlon=None):
"""
Create image set from one image, and save image set to database
:param file:
:param filename:
:param width_height:
:param author:
:param vehicle:
:param camera:
:param form_tz:
:param form_tz_name:
:param exif_data:
:param exif_time:
:param object_id:
:param time_mark:
:param latlon:
:return:
"""
if not exif_time:
exif_time = datetime.now(pytz.utc)
# create a new image set instance
if object_id:
new_image_set = IMAGE_SET_MODEL.get()(pk=int(object_id))
else:
new_image_set = IMAGE_SET_MODEL.get()()
new_image_set.acquisition_time = exif_time
new_image_set.name = filename
if isinstance(camera, CAMERA_MODEL.get()):
new_image_set.camera = camera
new_image_set.flight = getFlight(new_image_set.acquisition_time, vehicle)
track = None
if new_image_set.flight:
if not form_tz_name:
form_tz_name = new_image_set.flight.timezone
if not form_tz and form_tz_name:
form_tz = pytz.timezone(form_tz_name)
try:
track = new_image_set.flight.track
except:
# the flight may not have a track
pass
new_image_set.acquisition_timezone = form_tz_name
if track:
new_image_set.track_position = getClosestPosition(track=track,
timestamp=exif_time,
vehicle=vehicle)
if exif_data:
new_image_set.exif_position = buildExifPosition(
exif_data, new_image_set.camera, vehicle, exif_time, form_tz)
elif latlon:
new_image_set.exif_position = buildExifPositionFromLatLon(
latlon, exif_time)
new_image_set.author = author
# timing stats
if time_mark:
now_time = time.time()
upload_and_save_time = now_time - time_mark
new_image_set.uploadAndSaveTime = upload_and_save_time
overall_start_time = cache.get("imageAutoloadGlobalTimeMark", None)
if overall_start_time:
total_time_since_notify = now_time - float(overall_start_time)
new_image_set.totalTimeSinceNotify = total_time_since_notify
# end timing stats
new_image_set.save()
# build the metadata for the single image
single_image_metadata = {'imageSet': new_image_set}
try:
if exif_data:
single_image_metadata['width'] = int(
getExifValue(exif_data, 'ExifImageWidth'))
single_image_metadata['height'] = int(
getExifValue(exif_data, 'ExifImageHeight'))
else:
if not width_height:
dt = Image.open(file)
width_height = dt.size
single_image_metadata['width'] = width_height[0]
single_image_metadata['height'] = width_height[1]
except:
pass
# convert the image if needed
converted_file = convert_to_jpg_if_needed(file)
if converted_file:
# create the single image for the source
single_image_metadata['fileSizeBytes'] = get_file_size(file)
single_image_metadata['file'] = file
single_image_metadata['imageType'] = ImageType.source.value
single_image_metadata['raw'] = False
single_image_metadata['thumbnail'] = False
source_single_image = SINGLE_IMAGE_MODEL.get().objects.create(
**single_image_metadata)
file = converted_file
# create the single image for the raw / full / renderable
try:
single_image_metadata['fileSizeBytes'] = get_file_size(file)
single_image_metadata['file'] = file
single_image_metadata['imageType'] = ImageType.full.value
single_image_metadata['raw'] = True
single_image_metadata['thumbnail'] = False
except:
pass
new_single_image = SINGLE_IMAGE_MODEL.get().objects.create(
**single_image_metadata)
# relay was here
createThumbnail(new_single_image, new_image_set)
# TODO: replace this with a BoundedSemaphore
# TODO: we are pretty sure this was causing the fail in tiling and in importing images because many deepzoom
# threads are kicked off at the same time yet this code uses just one flag. #FIX
# TODO: suggest putting a single flag for each image we are tiling into REDIS
# dbServer = couchdb.Server(settings.COUCHDB_URL)
# db = dbServer[settings.COUCHDB_FILESTORE_NAME]
# if 'create_deepzoom_thread' in db:
# myFlag = db['create_deepzoom_thread']
# myFlag['active'] = True
# db['create_deepzoom_thread'] = myFlag
# else:
# db['create_deepzoom_thread'] = {'active': True}
createDeepzoomTiles(new_image_set)
return new_image_set
def importNirvssSpectra(filename):
"""
Import NIRVSS spectra from a CSV file and write them to the database
:param filename: the name of the CSV file
:return: the number of spectra imported
"""
num_imported = 0
num_rejected_noflight = 0
num_rejected_exists = 0
# The only current way to know which instrument this is from is the filename
if 'LW' in filename:
instrumentName = 'NIRVSS LW'
elif 'SW' in filename:
instrumentName = 'NIRVSS SW'
instrument = ScienceInstrument.getInstrument(instrumentName)
# Use this to store objects and bulk create in groups
queue = []
reader = DictReader(open(filename, 'r'))
for row in reader:
epochTime = datetime.datetime.utcfromtimestamp(float(
row['Epoch Time'])).replace(tzinfo=pytz.UTC)
flight = getFlight(epochTime, None)
if flight is None:
num_rejected_noflight += 1
if num_rejected_noflight < 10:
print 'No flight for', row
continue
# Check for existing database entries with this same instrument and acquisition time
existingRecords = NirvssSpectrometerDataProduct.objects.filter(
acquisition_time=epochTime, instrument=instrument)
if len(existingRecords) > 0:
num_rejected_exists += 1
if num_rejected_exists < 10:
print 'This spectrum is already imported as:'
for record in existingRecords:
print ' %s' % record
continue
track_position = None
if flight:
track_position = getClosestPosition(epochTime)
# No existing records, so add this one
nsdp = NirvssSpectrometerDataProduct()
nsdp.description = ''
nsdp.manufacturer_data_file = None
nsdp.manufacturer_mime_type = 'text/plain'
nsdp.portable_data_file = None
nsdp.portable_mime_type = 'text/plain'
nsdp.portable_file_format_name = 'ASCII'
nsdp.acquisition_time = epochTime
nsdp.acquisition_timezone = 'UTC'
nsdp.creation_time = datetime.datetime.utcnow().replace(
tzinfo=pytz.utc)
nsdp.track_position = track_position
nsdp.user_position = None
nsdp.collector = None
nsdp.creator = None
nsdp.instrument = instrument
nsdp.name = nsdp.__unicode__()
nsdp.flight = flight
nsdp.save()
num_imported += 1
# Add all of the (wavelength,reflectance) values for the spectrum
for column_label, value in row.iteritems():
# NIRVSS csv header has channel names like "R2311", "R2323",
# etc. which indicate wavelength, and the data rows contain
# float values which are the radiance values
match = re.search('R(\d+)', column_label)
if match:
# If the column header is R#### store the corresponding
# radiance values as samples
datapoint = NirvssSpectrometerSample()
datapoint.dataProduct = nsdp
datapoint.wavelength = int(match.group(1))
datapoint.reflectance = float(value)
queue.append(datapoint)
NirvssSpectrometerSample.objects.bulk_create(queue)
queue = []
if flight is not None:
# for this flight, create one band depth time series for all existing band depth definitions
create_band_depth_time_series(flight=flight)
# from each generated band depth time series, create a band depth geojson
create_geojson_for_all_bdd(flight=flight)
stats = {
'num_imported': num_imported,
'num_rejected_noflight': num_rejected_noflight,
'num_rejected_exists': num_rejected_exists
}
return stats
from datetime import datetime
import pytz
django.setup()
from basaltApp.models import BasaltImageSet, BasaltSingleImage, BasaltVehicle
from geocamTrack.utils import getClosestPosition
hawaiiStandardTime = pytz.timezone('US/Hawaii')
startTime = datetime(2016, 11, 8, 0, 0, 0, tzinfo=hawaiiStandardTime)
endTime = datetime(2016, 11, 9, 0, 0, 0, tzinfo=hawaiiStandardTime)
ev1Vehicle = BasaltVehicle.objects.get(name="EV1")
imgList = BasaltImageSet.objects.filter(acquisition_time__gte=startTime).filter(acquisition_time__lte=endTime)
print "Found %d images." % imgList.count()
for img in imgList:
position = getClosestPosition(vehicle=ev1Vehicle, timestamp=img.acquisition_time)
if img.flight:
print "F: %s, N: %s, P: %s" % (img.flight.name, img.name, position)
singleImages = img.images
print " Images:"
for si in singleImages.all():
print " Thumb: %s (%s x %s)" % (si.thumbnail, si.width, si.height)
else:
position = getClosestPosition(vehicle=ev1Vehicle, timestamp=img.acquisition_time)
print "F: %s, N: %s, P: %s" % ("<none>", img.name, position)
singleImages = img.images
print " Images:"
for si in singleImages.all():
print " Thumb: %s (%s x %s)" % (si.thumbnail, si.width, si.height)
def getTrackPosition(timestamp, resource):
'''
Look up and return the closest tracked position if there is one.
'''
return getClosestPosition(timestamp=timestamp, resource=resource)
def getTrackPosition(timestamp, resource):
'''
Look up and return the closest tracked position if there is one.
'''
return getClosestPosition(timestamp=timestamp, resource=resource)
