def lookupFlightInfo(utcStamp, timezone, vehicle, defaultTrackName): # # Get flight and track info and find location of sample if available # flight = None try: flight = getFlight(utcStamp, vehicle.vehicle) if flight: trackName = flight.name elif vehicle: trackName = vehicle.name else: trackName = defaultTrackName track = BasaltTrack.getTrackByName(trackName) if not track: if trackName == defaultTrackName: track = BasaltTrack(name=defaultTrackName) else: track = BasaltTrack(name=defaultTrackName, vehicle=vehicle, timezone=timezone) sampleLocation = getClosestPosition(track=track, timestamp=utcStamp, vehicle=vehicle) except: sampleLocation = None return (flight, sampleLocation)
def doImportNotes(request, sourceFile, tz, resource): dictreader = csv.DictReader(sourceFile) for row in dictreader: row['author'] = request.user if row['content'] or row['tags']: if 'first_name' in row and 'last_name' in row: if row['first_name'] and row['last_name']: try: row['author'] = User.objects.get(first_name=row['first_name'], last_name=row['last_name']) del row['first_name'] del row['last_name'] except: pass if row['event_time']: event_time = dateparser(row['event_time']) if tz != pytz.utc: localized_time = tz.localize(event_time) event_time = TimeUtil.timeZoneToUtc(localized_time) row['event_time'] = event_time try: # TODO implement tags when ready del row['tags'] except: pass NOTE_MODEL = Note.get() note = NOTE_MODEL(**row) note.creation_time = datetime.now(pytz.utc) note.modification_time = datetime.now(pytz.utc) if resource: note.position = getClosestPosition(timestamp=note.event_time, resource=resource) note.save()
def lookupPosition(self): # IMPORTANT this should not be used across multitudes of notes, it is designed to be used during construction. if not self.position: track=None if hasattr(self, 'flight') and self.flight: if hasattr(self.flight, 'track'): track = self.flight.track self.position = getClosestPosition(track=track, timestamp=self.event_time) return self.position
def getPosition(self): # IMPORTANT this should not be used across multitudes of notes, it is designed to be used during construction. if not self.position and self.location_found == None: resource = None if self.flight: if self.flight.vehicle: resource = self.flight.vehicle.basaltresource self.position = getClosestPosition(timestamp=self.event_time, resource=resource) if self.position: self.location_found = True else: self.location_found = False self.save() return self.position
def getPosition(self): from geocamTrack.utils import getClosestPosition if self.position: return self.position elif self.position_not_found == None and self.getEventTime(): # populate the position timestamp = self.getEventTime() if not timestamp: return None foundPosition = getClosestPosition(timestamp=timestamp) if foundPosition: self.position = foundPosition self.position_not_found = False else: self.position_not_found = True self.save() return self.position
def save(self, commit=True): instance = super(SampleForm, self).save(commit=False) instance.collection_time = self.cleaned_data['collection_time'] if instance.resource and instance.collection_time: instance.track_position = getClosestPosition(timestamp=instance.collection_time, resource=instance.resource) if (('lat' in self.changed_data) and ('lon' in self.changed_data)) or ('altitude' in self.changed_data): if instance.user_position is None: instance.user_position = LOCATION_MODEL.get().objects.create(serverTimestamp = datetime.datetime.now(pytz.utc), timestamp = instance.collection_time, latitude = self.cleaned_data['lat'], longitude = self.cleaned_data['lon'], altitude = self.cleaned_data['altitude']) else: instance.user_position.latitude = self.cleaned_data['lat'] instance.user_position.longitude = self.cleaned_data['lon'] instance.user_position.altitude = self.cleaned_data['altitude'] if ('collector_name' in self.changed_data): fullName = self.cleaned_data['collector_name'] try: splitName = fullName.split(' ') firstAndLast = [x for x in splitName if x.strip()] collector = User.objects.filter(first_name=firstAndLast[0]).filter(last_name=firstAndLast[1])[0] instance.collector = collector except: self.errors['error'] = "Save Failed. User %s does not exist." % fullName return instance # if fields changed, validate against the name needsNameRebuild = False for field in SAMPLE_MODEL.get().getFieldsForName(): if field in self.changed_data: needsNameRebuild = True break if needsNameRebuild: builtName = instance.buildName() if instance.name != builtName: instance.name = builtName if commit: instance.save() return instance
import pytz from basaltApp.models import * from geocamTrack.utils import getClosestPosition missingPositionSampleList = [] startTime = datetime.datetime(2016, 10, 30, 0, 6, 0, tzinfo=pytz.utc) samples = BasaltSample.objects.filter( collection_time__gte=startTime).order_by("collection_time") sampleCount = 0 missingPositionCount = 0 for sample in samples: if (not sample.track_position) and (not sample.user_position): if sample.resource: print "**** Processing: %s ****" % sample.name sample.track_position = getClosestPosition( timestamp=sample.collection_time, resource=sample.resource) if not sample.track_position: sample.track_position = getClosestPosition( timestamp=sample.collection_time, resource=sample.resource) if not sample.track_position: print " No location found - adding to list of samples w/o position" missingPositionSampleList.append(sample.name) missingPositionCount += 1 print " Date: %s, Resource: %s, Location: %s\n" % ( sample.collection_time, sample.resource, sample.track_position) sample.save() sampleCount += 1 # if sample.resource is None: # print "**** Processing: %s ****" % sample.name # sample.resource = BasaltResource.objects.get(name=settings.XGDS_SAMPLE_DEFAULT_COLLECTOR) # sample.track_position = getClosestPosition(timestamp=sample.collection_time, resource=sample.resource)
def create_image_set(file, filename, author, vehicle, camera, width_height=None, form_tz=None, form_tz_name=None, exif_data=None, exif_time=None, object_id=None, time_mark=None, latlon=None): """ Create image set from one image, and save image set to database :param file: :param filename: :param width_height: :param author: :param vehicle: :param camera: :param form_tz: :param form_tz_name: :param exif_data: :param exif_time: :param object_id: :param time_mark: :param latlon: :return: """ if not exif_time: exif_time = datetime.now(pytz.utc) # create a new image set instance if object_id: new_image_set = IMAGE_SET_MODEL.get()(pk=int(object_id)) else: new_image_set = IMAGE_SET_MODEL.get()() new_image_set.acquisition_time = exif_time new_image_set.name = filename if isinstance(camera, CAMERA_MODEL.get()): new_image_set.camera = camera new_image_set.flight = getFlight(new_image_set.acquisition_time, vehicle) track = None if new_image_set.flight: if not form_tz_name: form_tz_name = new_image_set.flight.timezone if not form_tz and form_tz_name: form_tz = pytz.timezone(form_tz_name) try: track = new_image_set.flight.track except: # the flight may not have a track pass new_image_set.acquisition_timezone = form_tz_name if track: new_image_set.track_position = getClosestPosition(track=track, timestamp=exif_time, vehicle=vehicle) if exif_data: new_image_set.exif_position = buildExifPosition( exif_data, new_image_set.camera, vehicle, exif_time, form_tz) elif latlon: new_image_set.exif_position = buildExifPositionFromLatLon( latlon, exif_time) new_image_set.author = author # timing stats if time_mark: now_time = time.time() upload_and_save_time = now_time - time_mark new_image_set.uploadAndSaveTime = upload_and_save_time overall_start_time = cache.get("imageAutoloadGlobalTimeMark", None) if overall_start_time: total_time_since_notify = now_time - float(overall_start_time) new_image_set.totalTimeSinceNotify = total_time_since_notify # end timing stats new_image_set.save() # build the metadata for the single image single_image_metadata = {'imageSet': new_image_set} try: if exif_data: single_image_metadata['width'] = int( getExifValue(exif_data, 'ExifImageWidth')) single_image_metadata['height'] = int( getExifValue(exif_data, 'ExifImageHeight')) else: if not width_height: dt = Image.open(file) width_height = dt.size single_image_metadata['width'] = width_height[0] single_image_metadata['height'] = width_height[1] except: pass # convert the image if needed converted_file = convert_to_jpg_if_needed(file) if converted_file: # create the single image for the source single_image_metadata['fileSizeBytes'] = get_file_size(file) single_image_metadata['file'] = file single_image_metadata['imageType'] = ImageType.source.value single_image_metadata['raw'] = False single_image_metadata['thumbnail'] = False source_single_image = SINGLE_IMAGE_MODEL.get().objects.create( **single_image_metadata) file = converted_file # create the single image for the raw / full / renderable try: single_image_metadata['fileSizeBytes'] = get_file_size(file) single_image_metadata['file'] = file single_image_metadata['imageType'] = ImageType.full.value single_image_metadata['raw'] = True single_image_metadata['thumbnail'] = False except: pass new_single_image = SINGLE_IMAGE_MODEL.get().objects.create( **single_image_metadata) # relay was here createThumbnail(new_single_image, new_image_set) # TODO: replace this with a BoundedSemaphore # TODO: we are pretty sure this was causing the fail in tiling and in importing images because many deepzoom # threads are kicked off at the same time yet this code uses just one flag. #FIX # TODO: suggest putting a single flag for each image we are tiling into REDIS # dbServer = couchdb.Server(settings.COUCHDB_URL) # db = dbServer[settings.COUCHDB_FILESTORE_NAME] # if 'create_deepzoom_thread' in db: # myFlag = db['create_deepzoom_thread'] # myFlag['active'] = True # db['create_deepzoom_thread'] = myFlag # else: # db['create_deepzoom_thread'] = {'active': True} createDeepzoomTiles(new_image_set) return new_image_set
def importNirvssSpectra(filename): """ Import NIRVSS spectra from a CSV file and write them to the database :param filename: the name of the CSV file :return: the number of spectra imported """ num_imported = 0 num_rejected_noflight = 0 num_rejected_exists = 0 # The only current way to know which instrument this is from is the filename if 'LW' in filename: instrumentName = 'NIRVSS LW' elif 'SW' in filename: instrumentName = 'NIRVSS SW' instrument = ScienceInstrument.getInstrument(instrumentName) # Use this to store objects and bulk create in groups queue = [] reader = DictReader(open(filename, 'r')) for row in reader: epochTime = datetime.datetime.utcfromtimestamp(float( row['Epoch Time'])).replace(tzinfo=pytz.UTC) flight = getFlight(epochTime, None) if flight is None: num_rejected_noflight += 1 if num_rejected_noflight < 10: print 'No flight for', row continue # Check for existing database entries with this same instrument and acquisition time existingRecords = NirvssSpectrometerDataProduct.objects.filter( acquisition_time=epochTime, instrument=instrument) if len(existingRecords) > 0: num_rejected_exists += 1 if num_rejected_exists < 10: print 'This spectrum is already imported as:' for record in existingRecords: print ' %s' % record continue track_position = None if flight: track_position = getClosestPosition(epochTime) # No existing records, so add this one nsdp = NirvssSpectrometerDataProduct() nsdp.description = '' nsdp.manufacturer_data_file = None nsdp.manufacturer_mime_type = 'text/plain' nsdp.portable_data_file = None nsdp.portable_mime_type = 'text/plain' nsdp.portable_file_format_name = 'ASCII' nsdp.acquisition_time = epochTime nsdp.acquisition_timezone = 'UTC' nsdp.creation_time = datetime.datetime.utcnow().replace( tzinfo=pytz.utc) nsdp.track_position = track_position nsdp.user_position = None nsdp.collector = None nsdp.creator = None nsdp.instrument = instrument nsdp.name = nsdp.__unicode__() nsdp.flight = flight nsdp.save() num_imported += 1 # Add all of the (wavelength,reflectance) values for the spectrum for column_label, value in row.iteritems(): # NIRVSS csv header has channel names like "R2311", "R2323", # etc. which indicate wavelength, and the data rows contain # float values which are the radiance values match = re.search('R(\d+)', column_label) if match: # If the column header is R#### store the corresponding # radiance values as samples datapoint = NirvssSpectrometerSample() datapoint.dataProduct = nsdp datapoint.wavelength = int(match.group(1)) datapoint.reflectance = float(value) queue.append(datapoint) NirvssSpectrometerSample.objects.bulk_create(queue) queue = [] if flight is not None: # for this flight, create one band depth time series for all existing band depth definitions create_band_depth_time_series(flight=flight) # from each generated band depth time series, create a band depth geojson create_geojson_for_all_bdd(flight=flight) stats = { 'num_imported': num_imported, 'num_rejected_noflight': num_rejected_noflight, 'num_rejected_exists': num_rejected_exists } return stats
from datetime import datetime import pytz django.setup() from basaltApp.models import BasaltImageSet, BasaltSingleImage, BasaltVehicle from geocamTrack.utils import getClosestPosition hawaiiStandardTime = pytz.timezone('US/Hawaii') startTime = datetime(2016, 11, 8, 0, 0, 0, tzinfo=hawaiiStandardTime) endTime = datetime(2016, 11, 9, 0, 0, 0, tzinfo=hawaiiStandardTime) ev1Vehicle = BasaltVehicle.objects.get(name="EV1") imgList = BasaltImageSet.objects.filter(acquisition_time__gte=startTime).filter(acquisition_time__lte=endTime) print "Found %d images." % imgList.count() for img in imgList: position = getClosestPosition(vehicle=ev1Vehicle, timestamp=img.acquisition_time) if img.flight: print "F: %s, N: %s, P: %s" % (img.flight.name, img.name, position) singleImages = img.images print " Images:" for si in singleImages.all(): print " Thumb: %s (%s x %s)" % (si.thumbnail, si.width, si.height) else: position = getClosestPosition(vehicle=ev1Vehicle, timestamp=img.acquisition_time) print "F: %s, N: %s, P: %s" % ("<none>", img.name, position) singleImages = img.images print " Images:" for si in singleImages.all(): print " Thumb: %s (%s x %s)" % (si.thumbnail, si.width, si.height)
def getTrackPosition(timestamp, resource): ''' Look up and return the closest tracked position if there is one. ''' return getClosestPosition(timestamp=timestamp, resource=resource)