def lookupFlightInfo(utcStamp, timezone, vehicle, defaultTrackName):
#
# Get flight and track info and find location of sample if available
#
flight = None
try:
flight = getFlight(utcStamp, vehicle.vehicle)
if flight:
trackName = flight.name
elif vehicle:
trackName = vehicle.name
else:
trackName = defaultTrackName
track = BasaltTrack.getTrackByName(trackName)
if not track:
if trackName == defaultTrackName:
track = BasaltTrack(name=defaultTrackName)
else:
track = BasaltTrack(name=defaultTrackName,
vehicle=vehicle,
timezone=timezone)
sampleLocation = getClosestPosition(track=track,
timestamp=utcStamp,
vehicle=vehicle)
except:
sampleLocation = None
return (flight, sampleLocation)
def grab_frame_from_time(
grab_time,
vehicle,
author=None,
prefix=settings.XGDS_IMAGE_FRAME_GRAB_FILENAME_PREFIX):
"""
Direct method to grab the frame given a time, without requests
:param grab_time: the grab time datetime
:param vehicle: the vehicle model
:return: the ImageSet that was created
"""
flight = getFlight(grab_time, vehicle)
if not flight:
raise Exception("No flight for time %s" % str(grab_time))
# TODO we currently name episodes after group flights this may change.
# Ideally have a link
source = SOURCE_MODEL.get().objects.get(vehicle=vehicle)
grab_info = prepare_grab_frame(flight.group.name, source.shortName,
grab_time)
# print('grab info is ')
print(grab_info)
if grab_info:
SAVE_IMAGE_FUNCTION = getClassByName('xgds_image.views.do_grab_frame')
# TODO we are assuming there is a camera named for the vehicle
camera = CAMERA_MODEL.get().objects.get(name=vehicle.name)
if not author:
# TODO assuming there is a user named camera
author = User.objects.get(username='******')
return SAVE_IMAGE_FUNCTION(grab_info['segment'].startTime, grab_time,
grab_info['file_path'], prefix, camera,
author, vehicle,
grab_info['index_file_path'])
def clean_flight(self, row):
"""
Updates the row by looking up the correct flight id by the name.
Hardcoding to Hercules vehicle
:param row:
:return: the updated row
"""
row['flight'] = getFlight(row['event_time'], self.hercules)
return row
def link_station_searches(filename):
times_file = open(filename, "r")
line = times_file.readline()
while line:
if line.startswith('Timestamps for'):
if not line.endswith('_CONT\n'):
re_matches = re.findall(r"(SCIENCE_\w*)", line)
station_name = ""
if len(re_matches) > 0:
station_name = re_matches[0]
start_time_line = times_file.readline()
first_data_line = times_file.readline()
last_data_line = times_file.readline()
# get start time
start_time_string = start_time_line.split(" ")[-1][0:-1]
start_time = dateparser(start_time_string).replace(
tzinfo=pytz.UTC)
formatted_start_time = start_time.strftime(
'%Y-%m-%dT%H%M%S.%f')
global flight
if not flight:
flight = getFlight(start_time)
# get end time
end_time_string = last_data_line.split(" ")[-1][0:-1]
end_time = dateparser(end_time_string).replace(tzinfo=pytz.UTC)
formatted_end_time = end_time.strftime('%Y-%m-%dT%H%M%S.%f')
photos_url = "https://%s/xgds_map_server/search/Photo/flight__group:%d,flight__vehicle:1,acquisition_timezone:utc,min_acquisition_time:%s,max_acquisition_time:%s" % \
(URL_PREFIX, flight.group.id, formatted_start_time, formatted_end_time)
photos_link = NamedURL(name='%s Photos' % station_name,
url=photos_url,
content_object=flight)
print str(photos_link)
photos_link.save()
nirvss_url = "https://%s/xgds_map_server/search/Spectrometer/flight__group:%d,flight__vehicle:1,acquisition_timezone:utc,min_acquisition_time:%s,max_acquisition_time:%s" % \
(URL_PREFIX, flight.group.id, formatted_start_time, formatted_end_time)
nirvss_link = NamedURL(name='%s NIRVSS' % station_name,
url=nirvss_url,
content_object=flight)
print str(nirvss_link)
nirvss_link.save()
line = times_file.readline()
def populateNoteData(request, form):
data, tags, errors = xgds_notes2_views.populateNoteData(request, form)
if 'flight_id' in data:
flight = BasaltFlight.objects.get(id=data['flight_id'])
data.pop('flight_id')
data['flight'] = flight
try:
data.pop('vehicle')
except:
pass
# look up the flight
elif 'vehicle' in data:
vehicle = None
vehicle = data['vehicle']
data.pop('vehicle')
flight = getFlight(data['event_time'], vehicle)
if flight:
data['flight'] = flight
return data, tags, errors
def create_image_set(file,
filename,
author,
vehicle,
camera,
width_height=None,
form_tz=None,
form_tz_name=None,
exif_data=None,
exif_time=None,
object_id=None,
time_mark=None,
latlon=None):
"""
Create image set from one image, and save image set to database
:param file:
:param filename:
:param width_height:
:param author:
:param vehicle:
:param camera:
:param form_tz:
:param form_tz_name:
:param exif_data:
:param exif_time:
:param object_id:
:param time_mark:
:param latlon:
:return:
"""
if not exif_time:
exif_time = datetime.now(pytz.utc)
# create a new image set instance
if object_id:
new_image_set = IMAGE_SET_MODEL.get()(pk=int(object_id))
else:
new_image_set = IMAGE_SET_MODEL.get()()
new_image_set.acquisition_time = exif_time
new_image_set.name = filename
if isinstance(camera, CAMERA_MODEL.get()):
new_image_set.camera = camera
new_image_set.flight = getFlight(new_image_set.acquisition_time, vehicle)
track = None
if new_image_set.flight:
if not form_tz_name:
form_tz_name = new_image_set.flight.timezone
if not form_tz and form_tz_name:
form_tz = pytz.timezone(form_tz_name)
try:
track = new_image_set.flight.track
except:
# the flight may not have a track
pass
new_image_set.acquisition_timezone = form_tz_name
if track:
new_image_set.track_position = getClosestPosition(track=track,
timestamp=exif_time,
vehicle=vehicle)
if exif_data:
new_image_set.exif_position = buildExifPosition(
exif_data, new_image_set.camera, vehicle, exif_time, form_tz)
elif latlon:
new_image_set.exif_position = buildExifPositionFromLatLon(
latlon, exif_time)
new_image_set.author = author
# timing stats
if time_mark:
now_time = time.time()
upload_and_save_time = now_time - time_mark
new_image_set.uploadAndSaveTime = upload_and_save_time
overall_start_time = cache.get("imageAutoloadGlobalTimeMark", None)
if overall_start_time:
total_time_since_notify = now_time - float(overall_start_time)
new_image_set.totalTimeSinceNotify = total_time_since_notify
# end timing stats
new_image_set.save()
# build the metadata for the single image
single_image_metadata = {'imageSet': new_image_set}
try:
if exif_data:
single_image_metadata['width'] = int(
getExifValue(exif_data, 'ExifImageWidth'))
single_image_metadata['height'] = int(
getExifValue(exif_data, 'ExifImageHeight'))
else:
if not width_height:
dt = Image.open(file)
width_height = dt.size
single_image_metadata['width'] = width_height[0]
single_image_metadata['height'] = width_height[1]
except:
pass
# convert the image if needed
converted_file = convert_to_jpg_if_needed(file)
if converted_file:
# create the single image for the source
single_image_metadata['fileSizeBytes'] = get_file_size(file)
single_image_metadata['file'] = file
single_image_metadata['imageType'] = ImageType.source.value
single_image_metadata['raw'] = False
single_image_metadata['thumbnail'] = False
source_single_image = SINGLE_IMAGE_MODEL.get().objects.create(
**single_image_metadata)
file = converted_file
# create the single image for the raw / full / renderable
try:
single_image_metadata['fileSizeBytes'] = get_file_size(file)
single_image_metadata['file'] = file
single_image_metadata['imageType'] = ImageType.full.value
single_image_metadata['raw'] = True
single_image_metadata['thumbnail'] = False
except:
pass
new_single_image = SINGLE_IMAGE_MODEL.get().objects.create(
**single_image_metadata)
# relay was here
createThumbnail(new_single_image, new_image_set)
# TODO: replace this with a BoundedSemaphore
# TODO: we are pretty sure this was causing the fail in tiling and in importing images because many deepzoom
# threads are kicked off at the same time yet this code uses just one flag. #FIX
# TODO: suggest putting a single flag for each image we are tiling into REDIS
# dbServer = couchdb.Server(settings.COUCHDB_URL)
# db = dbServer[settings.COUCHDB_FILESTORE_NAME]
# if 'create_deepzoom_thread' in db:
# myFlag = db['create_deepzoom_thread']
# myFlag['active'] = True
# db['create_deepzoom_thread'] = myFlag
# else:
# db['create_deepzoom_thread'] = {'active': True}
createDeepzoomTiles(new_image_set)
return new_image_set
def importNirvssSpectra(filename):
"""
Import NIRVSS spectra from a CSV file and write them to the database
:param filename: the name of the CSV file
:return: the number of spectra imported
"""
num_imported = 0
num_rejected_noflight = 0
num_rejected_exists = 0
# The only current way to know which instrument this is from is the filename
if 'LW' in filename:
instrumentName = 'NIRVSS LW'
elif 'SW' in filename:
instrumentName = 'NIRVSS SW'
instrument = ScienceInstrument.getInstrument(instrumentName)
# Use this to store objects and bulk create in groups
queue = []
reader = DictReader(open(filename, 'r'))
for row in reader:
epochTime = datetime.datetime.utcfromtimestamp(float(
row['Epoch Time'])).replace(tzinfo=pytz.UTC)
flight = getFlight(epochTime, None)
if flight is None:
num_rejected_noflight += 1
if num_rejected_noflight < 10:
print 'No flight for', row
continue
# Check for existing database entries with this same instrument and acquisition time
existingRecords = NirvssSpectrometerDataProduct.objects.filter(
acquisition_time=epochTime, instrument=instrument)
if len(existingRecords) > 0:
num_rejected_exists += 1
if num_rejected_exists < 10:
print 'This spectrum is already imported as:'
for record in existingRecords:
print ' %s' % record
continue
track_position = None
if flight:
track_position = getClosestPosition(epochTime)
# No existing records, so add this one
nsdp = NirvssSpectrometerDataProduct()
nsdp.description = ''
nsdp.manufacturer_data_file = None
nsdp.manufacturer_mime_type = 'text/plain'
nsdp.portable_data_file = None
nsdp.portable_mime_type = 'text/plain'
nsdp.portable_file_format_name = 'ASCII'
nsdp.acquisition_time = epochTime
nsdp.acquisition_timezone = 'UTC'
nsdp.creation_time = datetime.datetime.utcnow().replace(
tzinfo=pytz.utc)
nsdp.track_position = track_position
nsdp.user_position = None
nsdp.collector = None
nsdp.creator = None
nsdp.instrument = instrument
nsdp.name = nsdp.__unicode__()
nsdp.flight = flight
nsdp.save()
num_imported += 1
# Add all of the (wavelength,reflectance) values for the spectrum
for column_label, value in row.iteritems():
# NIRVSS csv header has channel names like "R2311", "R2323",
# etc. which indicate wavelength, and the data rows contain
# float values which are the radiance values
match = re.search('R(\d+)', column_label)
if match:
# If the column header is R#### store the corresponding
# radiance values as samples
datapoint = NirvssSpectrometerSample()
datapoint.dataProduct = nsdp
datapoint.wavelength = int(match.group(1))
datapoint.reflectance = float(value)
queue.append(datapoint)
NirvssSpectrometerSample.objects.bulk_create(queue)
queue = []
if flight is not None:
# for this flight, create one band depth time series for all existing band depth definitions
create_band_depth_time_series(flight=flight)
# from each generated band depth time series, create a band depth geojson
create_geojson_for_all_bdd(flight=flight)
stats = {
'num_imported': num_imported,
'num_rejected_noflight': num_rejected_noflight,
'num_rejected_exists': num_rejected_exists
}
return stats