def getExifTime(request, exifTimeString, uploadedFile, form_tz): # correct the timezone, we store time in utc if 'UTC' in form_tz.zone: form_tz = pytz.utc if exifTimeString: exifTime = dateparser(str(exifTimeString)) if (form_tz != pytz.utc) and exifTime and exifTime.tzinfo is None: localized_time = form_tz.localize(exifTime) exifTime = TimeUtil.timeZoneToUtc(localized_time) else: exifTime = exifTime.replace(tzinfo=pytz.utc) else: # read the time from the last modified time that we pushed in from imageUpload.js if 'HTTP_LASTMOD' in request.META: modtimesString = request.META['HTTP_LASTMOD'] if modtimesString: modtime = None theImages = modtimesString.split(',') for i in theImages: k, v = i.split('||') if k == str(uploadedFile.name): modtime = datetime.fromtimestamp(int(v) / 1000) break if modtime: localized_time = form_tz.localize(modtime) exifTime = TimeUtil.timeZoneToUtc(localized_time) return exifTime
def buildExifPosition(exif, camera, vehicle, exifTime, form_tz): """ Given the image's exif data and a camera object, creates a new position object that contains the lat and lon information. """ gpsLatLon = getLatLon(exif) gpsTimeStamp = getGPSDatetime(exif) if gpsTimeStamp: gpsTimeStamp = form_tz.localize(gpsTimeStamp) gpsTimeStamp = TimeUtil.timeZoneToUtc(gpsTimeStamp) else: gpsTimeStamp = exifTime if gpsTimeStamp and gpsLatLon[0] and gpsLatLon[1]: #TODO this requires that the position model has heading and altitude ... #TODO right now this is hardcoded, instead there should be a classmethod to build a position dict # given some incoming data and the classmethod should be on the position model position_class = POSITION_MODEL.get() position_dict = { 'serverTimestamp': gpsTimeStamp, 'timestamp': gpsTimeStamp, 'latitude': gpsLatLon[0], 'longitude': gpsLatLon[1] } if hasattr(position_class, 'yaw'): position_dict['yaw'] = getHeading(exif) elif hasattr(position_class, 'heading'): position_dict['heading'] = getHeading(exif) if hasattr(position_class, 'altitude'): position_dict['altitude'] = getAltitude(exif) position = POSITION_MODEL.get().objects.create(**position_dict) return position return None
def buildExifPosition(exif, camera, resource, exifTime, form_tz): """ Given the image's exif data and a camera object, creates a new position object that contains the lat and lon information. """ gpsLatLon = getLatLon(exif) gpsTimeStamp = getGPSDatetime(exif) if gpsTimeStamp: gpsTimeStamp = form_tz.localize(gpsTimeStamp) gpsTimeStamp = TimeUtil.timeZoneToUtc(gpsTimeStamp) else: gpsTimeStamp = exifTime if gpsTimeStamp and gpsLatLon[0] and gpsLatLon[1]: # TODO this requires that the position model has heading and altitude ... position = POSITION_MODEL.get().objects.create( serverTimestamp=gpsTimeStamp, timestamp=gpsTimeStamp, latitude=gpsLatLon[0], longitude=gpsLatLon[1], heading=getHeading(exif), altitude=getAltitude(exif), ) return position return None
def doImportNotes(request, sourceFile, tz, resource): dictreader = csv.DictReader(sourceFile) for row in dictreader: row['author'] = request.user if row['content'] or row['tags']: if 'first_name' in row and 'last_name' in row: if row['first_name'] and row['last_name']: try: row['author'] = User.objects.get(first_name=row['first_name'], last_name=row['last_name']) del row['first_name'] del row['last_name'] except: pass if row['event_time']: event_time = dateparser(row['event_time']) if tz != pytz.utc: localized_time = tz.localize(event_time) event_time = TimeUtil.timeZoneToUtc(localized_time) row['event_time'] = event_time try: # TODO implement tags when ready del row['tags'] except: pass NOTE_MODEL = Note.get() note = NOTE_MODEL(**row) note.creation_time = datetime.now(pytz.utc) note.modification_time = datetime.now(pytz.utc) if resource: note.position = getClosestPosition(timestamp=note.event_time, resource=resource) note.save()
def buildExifPosition(exif, camera, resource, exifTime, form_tz): ''' Given the image's exif data and a camera object, creates a new position object that contains the lat and lon information. ''' gpsLatLon = getLatLon(exif) gpsTimeStamp = getGPSDatetime(exif) if gpsTimeStamp: gpsTimeStamp = form_tz.localize(gpsTimeStamp) gpsTimeStamp = TimeUtil.timeZoneToUtc(gpsTimeStamp) else: gpsTimeStamp = exifTime if gpsTimeStamp and gpsLatLon[0] and gpsLatLon[1]: #TODO this requires that the position model has heading and altitude ... position = POSITION_MODEL.get().objects.create( serverTimestamp=gpsTimeStamp, timestamp=gpsTimeStamp, latitude=gpsLatLon[0], longitude=gpsLatLon[1], heading=getHeading(exif), altitude=getAltitude(exif)) return position return None
def filterFieldBefore(self, query, clause, field, term, negated): if negated: raise BadQuery("Oops, can't use minus sign with BEFORE in clause '%s' of search '%s'" % (clause, query)) try: # intervalStart=False to get end of specified interval # (inclusive before) utcDT = TimeUtil.stringToUtcDT(term, intervalStart=False) except ValueError, msg: raise BadQuery("Oops, %s in clause '%s' of search '%s'" % (msg, clause, query))
def filterFieldAfter(self, query, clause, field, term, negated): if negated: raise BadQuery("Oops, can't use minus sign with AFTER in clause '%s' of search '%s'" % (clause, query)) try: # intervalStart=True to get start of specified interval # (inclusive after) utcDT = TimeUtil.stringToUtcDT(term, intervalStart=True) except ValueError, msg: raise BadQuery("Oops, %s in clause '%s' of search '%s'" % (msg, clause, query))
def getTimestamp(self, obj): utcDt = getattr(obj, self.timestampField) posixTimeSecs = TimeUtil.utcDateTimeToPosix(utcDt) posixTimeMs = posixTimeSecs * 1000 if self.timestampMicrosecondsField is not None: microseconds = getattr(obj, self.timestampMicrosecondsField) assert microseconds <= 1e+6 posixTimeMs += microseconds * 1e-3 elif self.timestampNanosecondsField is not None: nanoseconds = getattr(obj, self.timestampNanosecondsField) assert nanoseconds <= 1e+9 posixTimeMs += nanoseconds * 1e-6 return posixTimeMs
def indexRecord(self, obj): if (self.queueMode and (self.status['numSamples'] % BATCH_SLEEP_NUM_SAMPLES) == 0): batchProcessDuration = time.time() - self.batchProcessStartTime if settings.XGDS_PLOT_BATCH_SLEEP_TIME_FACTOR > 0: sleepTime = batchProcessDuration * settings.XGDS_PLOT_BATCH_SLEEP_TIME_FACTOR print 'sleeping for %.3f seconds to avoid overloading server' % sleepTime time.sleep(sleepTime) self.batchProcessStartTime = time.time() posixTimeMs = self.queryManager.getTimestamp(obj) maxTime = self.status['maxTime'] or -99e+20 if posixTimeMs < maxTime: print( 'skipping old (duplicate?) record: posixTimeMs %.3f <= maxTime %.3f' % (posixTimeMs, self.status['maxTime'])) return self.status['maxTime'] = max(maxTime, posixTimeMs) minTime = self.status['minTime'] or 99e+20 self.status['minTime'] = min(minTime, posixTimeMs) self.status['numSamples'] += 1 if self.status['numSamples'] % 100 == 0: print '%d %s tile update' % (self.status['numSamples'], self.valueCode) pos = self.poseCollector.getLastPositionBeforePosixTimeMs(posixTimeMs) if pos is None: print( 'skipping record at time %s, no preceding positions available' % TimeUtil.posixToUtcDateTime(posixTimeMs / 1000.0)) return timestampUtc = datetime.datetime.utcfromtimestamp(1e-3 * posixTimeMs) timestampLocal = pytz.utc.localize(timestampUtc).astimezone( OPS_TIME_ZONE) dayCode = timestampLocal.strftime('%Y%m%d') for tileParams in tile.getTilesOverlappingBounds( dayCode, (pos.longitude, pos.latitude, pos.longitude, pos.latitude)): val = self.valueManager.getValue(obj) self.addSample(tileParams, pos, val) self.parent.delayBox.addJob((self.valueCode, tileParams))
def indexRecord(self, obj): if (self.queueMode and (self.status['numSamples'] % BATCH_SLEEP_NUM_SAMPLES) == 0): batchProcessDuration = time.time() - self.batchProcessStartTime if settings.XGDS_PLOT_BATCH_SLEEP_TIME_FACTOR > 0: sleepTime = batchProcessDuration * settings.XGDS_PLOT_BATCH_SLEEP_TIME_FACTOR print 'sleeping for %.3f seconds to avoid overloading server' % sleepTime time.sleep(sleepTime) self.batchProcessStartTime = time.time() posixTimeMs = self.queryManager.getTimestamp(obj) maxTime = self.status['maxTime'] or -99e+20 if posixTimeMs < maxTime: print ('skipping old (duplicate?) record: posixTimeMs %.3f <= maxTime %.3f' % (posixTimeMs, self.status['maxTime'])) return self.status['maxTime'] = max(maxTime, posixTimeMs) minTime = self.status['minTime'] or 99e+20 self.status['minTime'] = min(minTime, posixTimeMs) self.status['numSamples'] += 1 if self.status['numSamples'] % 100 == 0: print '%d %s tile update' % (self.status['numSamples'], self.valueCode) pos = self.poseCollector.getLastPositionBeforePosixTimeMs(posixTimeMs) if pos is None: print ('skipping record at time %s, no preceding positions available' % TimeUtil.posixToUtcDateTime(posixTimeMs / 1000.0)) return timestampUtc = datetime.datetime.utcfromtimestamp(1e-3 * posixTimeMs) timestampLocal = pytz.utc.localize(timestampUtc).astimezone(OPS_TIME_ZONE) dayCode = timestampLocal.strftime('%Y%m%d') for tileParams in tile.getTilesOverlappingBounds(dayCode, (pos.longitude, pos.latitude, pos.longitude, pos.latitude)): val = self.valueManager.getValue(obj) self.addSample(tileParams, pos, val) self.parent.delayBox.addJob((self.valueCode, tileParams))
def writeCurrentKml(self, out, pos, iconStyle=None, urlFn=None): if iconStyle is None: iconStyle = self.getIconStyle(pos) ageStr = '' if settings.GEOCAM_TRACK_SHOW_CURRENT_POSITION_AGE: now = datetime.datetime.now(pytz.utc) diff = now - pos.timestamp diffSecs = diff.days * 24 * 60 * 60 + diff.seconds if diffSecs >= settings.GEOCAM_TRACK_CURRENT_POSITION_AGE_MIN_SECONDS: age = TimeUtil.getTimeShort(pos.timestamp) ageStr = ' (%s)' % age ageStr += ' %s' % getTimeSpinner(datetime.datetime.now(pytz.utc)) label = ('%s%s%s' % (self.getLabelName(pos), self.getLabelExtra(pos), ageStr)) out.write(""" <Placemark> <name>%(label)s</name> """ % dict(label=label)) if iconStyle: out.write("<Style>\n") iconStyle.writeKml(out, pos.getHeading(), urlFn=urlFn, color=self.getIconColor(pos)) out.write("</Style>\n") out.write(""" <Point> <coordinates> """) pos.writeCoordinatesKml(out) out.write(""" </coordinates> </Point> </Placemark> """)
def utc_to_django(value): return TimeUtil.utcToTimeZone(value, settings.TIME_ZONE)
def grab_frame_save_image(request): """ Grab a frame from video and save it as an Image Set. Expecting the following in request.POST start time: start time of the video, in UTC grab time: grab time of the image to grab, in UTC vehicle: the name of the vehicle, or it will use default camera: the name of the camera (to associate with the ImageSet) filename_prefix: the prefix to use for the filename, defaults to 'Framegrab' :param request: POST DICTIONARY must contain above values :return: json response of the newly created image set """ try: start = request.POST.get('start_time') start_time = TimeUtil.convert_time_with_zone(dateparser(start), 'UTC') except: result_dict = {'status': 'error', 'error': 'Invalid video start time.'} return JsonResponse(json.dumps(result_dict), status=httplib.NOT_ACCEPTABLE, safe=False) try: grab = request.POST.get('grab_time') grab_time = TimeUtil.convert_time_with_zone(dateparser(grab), 'UTC') except: result_dict = {'status': 'error', 'error': 'Invalid video grab time.'} return JsonResponse(json.dumps(result_dict), status=httplib.NOT_ACCEPTABLE, safe=False) author = computeAuthor(request) vehicle_name = request.POST.get('vehicle', None) vehicle = lookup_vehicle(vehicle_name) cam_name = request.POST.get('camera') if not cam_name: cam_name = vehicle_name camera = CAMERA_MODEL.get().objects.get(name=cam_name) try: # request.POST.get('index_file_path') has the m3u8 new_image_set = do_grab_frame( start_time, grab_time, request.POST.get('path'), request.POST.get('filename_prefix', settings.XGDS_IMAGE_FRAME_GRAB_FILENAME_PREFIX), camera, author, vehicle, request.POST.get('index_file_path')) if new_image_set is None: result_dict = { 'status': 'error', 'error': 'Unable to create new ImageSet.' } return JsonResponse(json.dumps(result_dict), status=httplib.INTERNAL_SERVER_ERROR, safe=False) # This should really happen in do_grab_frame but it does nothing by default and needs a request so ... new_image_set.finish_initialization(request) # pass the image set to the client as json. return JsonResponse( { 'success': 'true', 'json': new_image_set.toMapDict() }, encoder=DatetimeJsonEncoder, safe=False) except Exception as e: result_dict = {'status': 'error', 'error': str(e)} return JsonResponse(json.dumps(result_dict), status=httplib.INTERNAL_SERVER_ERROR, safe=False)
def saveImage(request): """ Image drag and drop, saves the files and to the database. """ if request.method == "POST": timeMark = time.time() form = UploadFileForm(request.POST, request.FILES) if form.is_valid(): # create and save a single image obj uploadedFile = request.FILES["file"] newImage = SINGLE_IMAGE_MODEL.get()(file=uploadedFile) form_tz = form.getTimezone() resource = form.getResource() exifData = getExifData(newImage) # save image dimensions and file size try: newImage.width = int(getExifValue(exifData, "ExifImageWidth")) newImage.height = int(getExifValue(exifData, "ExifImageHeight")) except: pass newImage.fileSizeBytes = uploadedFile.size # get exif time exifTime = None exifTimeString = getExifValue(exifData, "DateTimeOriginal") if not exifTimeString: exifTimeString = getExifValue(exifData, "DateTime") if exifTimeString: exifTime = datetime.strptime(str(exifTimeString), "%Y:%m:%d %H:%M:%S") if (form_tz != pytz.utc) and exifTime: localized_time = form_tz.localize(exifTime) exifTime = TimeUtil.timeZoneToUtc(localized_time) else: exifTime = exifTime.replace(tzinfo=pytz.utc) else: # read the time from the last modified time that we pushed in from imageUpload.js if "HTTP_LASTMOD" in request.META: modtimesString = request.META["HTTP_LASTMOD"] if modtimesString: modtime = None theImages = modtimesString.split(",") for i in theImages: k, v = i.split("||") if k == str(uploadedFile.name): modtime = datetime.fromtimestamp(int(v) / 1000) break if modtime: localized_time = form_tz.localize(modtime) exifTime = TimeUtil.timeZoneToUtc(localized_time) if not exifTime: exifTime = datetime.now(pytz.utc) # create a new image set instance author = None if request.user.is_authenticated(): author = request.user # set user as image author elif "username" in request.POST: try: username = str(request.POST["username"]) author = User.objects.get(username=username) except: author = User.objects.get(username="******") newImageSet = IMAGE_SET_MODEL.get()() newImageSet.acquisition_time = exifTime newImageSet.acquisition_timezone = form.getTimezoneName() fileName = uploadedFile.name newImageSet.name = fileName newImageSet.camera = getCameraObject(exifData) newImageSet.track_position = getTrackPosition(exifTime, resource) newImageSet.exif_position = buildExifPosition(exifData, newImageSet.camera, resource, exifTime, form_tz) newImageSet.author = author newImageSet.resource = resource newImageSet.finish_initialization(request) nowTime = time.time() uploadAndSaveTime = nowTime - timeMark newImageSet.uploadAndSaveTime = uploadAndSaveTime overallStartTime = cache.get("imageAutoloadGlobalTimeMark", None) if overallStartTime: totalTimeSinceNotify = nowTime - float(overallStartTime) newImageSet.totalTimeSinceNotify = totalTimeSinceNotify newImageSet.save() # link the "image set" to "image". newImage.imageSet = newImageSet newImage.save() # create a thumbnail thumbnailStream = createThumbnailFile(newImage.file) SINGLE_IMAGE_MODEL.get().objects.create( file=thumbnailStream, raw=False, thumbnail=True, imageSet=newImageSet ) # create deep zoom tiles for viewing in openseadragon. if newImageSet.create_deepzoom: deepzoomTilingThread = Thread(target=newImageSet.create_deepzoom_image) deepzoomTilingThread.start() # newImageSet.create_deepzoom_image() imageSetDict = newImageSet.toMapDict() # pass the image set to the client as json. return HttpResponse( json.dumps({"success": "true", "json": imageSetDict}, cls=DatetimeJsonEncoder), content_type="application/json", ) else: return HttpResponse( json.dumps({"error": "Imported image is not valid", "details": form.errors}), content_type="application/json", )
def posixTimeMsToUtcDateTime(posixTimeMs): return TimeUtil.posixToUtcDateTime(posixTimeMs / 1000.0)
def saveImage(request): """ Image drag and drop, saves the files and to the database. """ if request.method == 'POST': timeMark = time.time() form = UploadFileForm(request.POST, request.FILES) if form.is_valid(): # create and save a single image obj uploadedFile = request.FILES['file'] newSingleImage = SINGLE_IMAGE_MODEL.get()(file=uploadedFile) form_tz = form.getTimezone() resource = form.getResource() exifData = getExifData(newSingleImage) # save image dimensions and file size try: newSingleImage.width = int( getExifValue(exifData, 'ExifImageWidth')) newSingleImage.height = int( getExifValue(exifData, 'ExifImageHeight')) except: pass newSingleImage.fileSizeBytes = uploadedFile.size # get exif time exifTime = None exifTimeString = getExifValue(exifData, 'DateTimeOriginal') if not exifTimeString: exifTimeString = getExifValue(exifData, 'DateTime') if exifTimeString: exifTime = datetime.strptime(str(exifTimeString), '%Y:%m:%d %H:%M:%S') if (form_tz != pytz.utc) and exifTime: localized_time = form_tz.localize(exifTime) exifTime = TimeUtil.timeZoneToUtc(localized_time) else: exifTime = exifTime.replace(tzinfo=pytz.utc) else: # read the time from the last modified time that we pushed in from imageUpload.js if 'HTTP_LASTMOD' in request.META: modtimesString = request.META['HTTP_LASTMOD'] if modtimesString: modtime = None theImages = modtimesString.split(',') for i in theImages: k, v = i.split('||') if k == str(uploadedFile.name): modtime = datetime.fromtimestamp(int(v) / 1000) break if modtime: localized_time = form_tz.localize(modtime) exifTime = TimeUtil.timeZoneToUtc(localized_time) if not exifTime: exifTime = datetime.now(pytz.utc) # create a new image set instance author = None if request.user.is_authenticated(): author = request.user # set user as image author elif 'username' in request.POST: try: username = str(request.POST['username']) author = User.objects.get(username=username) except: author = User.objects.get(username='******') if 'object_id' in request.POST: newImageSet = IMAGE_SET_MODEL.get()( pk=int(request.POST['object_id'])) else: newImageSet = IMAGE_SET_MODEL.get()() newImageSet.acquisition_time = exifTime newImageSet.acquisition_timezone = form.getTimezoneName() fileName = uploadedFile.name newImageSet.name = fileName newImageSet.camera = getCameraObject(exifData) newImageSet.track_position = getTrackPosition(exifTime, resource) newImageSet.exif_position = buildExifPosition( exifData, newImageSet.camera, resource, exifTime, form_tz) newImageSet.author = author newImageSet.resource = resource newImageSet.finish_initialization(request) nowTime = time.time() uploadAndSaveTime = nowTime - timeMark newImageSet.uploadAndSaveTime = uploadAndSaveTime overallStartTime = cache.get("imageAutoloadGlobalTimeMark", None) if overallStartTime: totalTimeSinceNotify = nowTime - float(overallStartTime) newImageSet.totalTimeSinceNotify = totalTimeSinceNotify newImageSet.save() # link the "image set" to "image". newSingleImage.imageSet = newImageSet newSingleImage.save() # relay if needed if 'relay' in request.POST: # create the record for the datum # fire a message for new data deletePostKey(request.POST, 'relay') addRelay(newImageSet, request.FILES, json.dumps(request.POST), reverse('xgds_save_image')) # create a thumbnail thumbnailStream = createThumbnailFile(newSingleImage.file) SINGLE_IMAGE_MODEL.get().objects.create(file=thumbnailStream, raw=False, thumbnail=True, imageSet=newImageSet) # create deep zoom tiles for viewing in openseadragon. if (newImageSet.create_deepzoom): deepzoomTilingThread = Thread( target=newImageSet.create_deepzoom_image) deepzoomTilingThread.start() # newImageSet.create_deepzoom_image() imageSetDict = newImageSet.toMapDict() # pass the image set to the client as json. return HttpResponse(json.dumps( { 'success': 'true', 'json': imageSetDict }, cls=DatetimeJsonEncoder), content_type='application/json') else: return HttpResponse(json.dumps({ 'error': 'Imported image is not valid', 'details': form.errors }), content_type='application/json')
def clean_acquisition_timezone(self): return TimeUtil.clean_timezone( self.cleaned_data['acquisition_timezone'])