def __init__(self, vehicle, slack, args):
Thread.__init__(self)
self.q = queue.Queue(maxsize=0)
self.vehicle = vehicle
self.slack = slack
self.args = args
self.pw = lrauvNc4ToNetcdf.InterpolatorWriter()
# Assume that the database has already been created with description and terrain information,
# so use minimal arguments in constructor
self.cl = CANONLoader(args.database, args.campaign)
self.cl.dbAlias = args.database
self.cl.campaignName = args.campaign
import os
import sys
import datetime # needed for glider data
import time # for startdate, enddate args
os.environ['DJANGO_SETTINGS_MODULE']='settings'
project_dir = os.path.dirname(__file__)
# the next line makes it possible to find CANON
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../")) # this makes it possible to find CANON, one directory up
from CANON import CANONLoader
# building input data sources object
#cl = CANONLoader('stoqs_september2011', 'CANON - September 2011')
cl = CANONLoader('stoqs_september2013', 'CANON - September 2013')
# default location of thredds and dods data:
cl.tdsBase = 'http://odss.mbari.org/thredds/'
cl.dodsBase = cl.tdsBase + 'dodsC/'
######################################################################
# GLIDERS
######################################################################
# Set start and end dates for all glider loads
# startdate is 24hours from now
ts=time.time()-(12.2*60*60)
st=datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M')
t=time.strptime(st,"%Y-%m-%d %H:%M")
#t =time.strptime("2013-09-03 20:01", "%Y-%m-%d %H:%M")
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
import timing
# building input data sources object
cl = CANONLoader(
'stoqs_cn13id_oct2013',
'CN13ID - October 2013',
description=
'Warden cruise on Western Flyer into the California Current System off Monterey Bay',
x3dTerrains={
'https://stoqs.mbari.org/x3d/Globe_1m_bath_10x/Globe_1m_bath_10x_scene.x3d':
{
'position': '14051448.48336 -15407886.51486 6184041.22775',
'orientation': '0.83940 0.33030 0.43164 1.44880',
'centerOfRotation': '0 0 0',
'VerticalExaggeration': '10',
}
},
grdTerrain=os.path.join(parentDir, 'Globe_1m_bath.grd'))
# Set start and end dates for all loads from sources that contain data
# beyond the temporal bounds of the campaign
startdate = datetime.datetime(2013, 10, 6) # Fixed start
enddate = datetime.datetime(2013, 10, 18) # Fixed end
# default location of thredds and dods data:
cl.tdsBase = 'http://odss.mbari.org/thredds/'
start = dt
else:
start = None
if args.end is not None:
dt = datetime.strptime(args.end, '%Y%m%dT%H%M%S')
end = dt
else:
end = None
if args.post:
token = os.environ['SLACKTOKEN']
slack = Slacker(token)
# Assume that the database has already been created with description and terrain information, so use minimal arguments in constructor
cl = CANONLoader(args.database, args.campaign)
cl.dbAlias = args.database
cl.campaignName = args.campaign
# Get directory list from sites
url, files = args.inUrl.rsplit('/', 1)
logger.info("Crawling %s for %s files", url, files)
skips = Crawl.SKIPS + [
".*Courier*", ".*Express*", ".*Normal*, '.*Priority*", ".*.cfg$",
".*.js$", ".*.kml$", ".*.log$"
]
c = Crawl(os.path.join(url, 'catalog.xml'),
select=[files],
debug=False,
skip=skips)
from CANON import CANONLoader
import timing
cl = CANONLoader(
'stoqs_all_dorado',
'Plankton Proxies - All Dorado Data',
description='All Dorado survey data from 2003 through 2019 and beyond',
x3dTerrains={
'https://stoqs.mbari.org/x3d/Monterey25_10x/Monterey25_10x_scene.x3d':
{
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation':
'-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
},
'https://stoqs.mbari.org/x3d/Monterey25_1x/Monterey25_1x_src_scene.x3d':
{
'name': 'Monterey25_1x',
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation':
'-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '1',
},
},
grdTerrain=os.path.join(parentDir, 'Monterey25.grd'))
startdate = datetime.datetime(2003, 1, 1)
enddate = datetime.datetime(2029, 12, 31)
import urllib2
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
from thredds_crawler.crawl import Crawl
cl = CANONLoader(
"stoqs_os2015",
"CANON-ECOHAB - Off Season 2015",
description="CANON Off Season 2015 Experiment in Monterey Bay",
x3dTerrains={
"http://dods.mbari.org/terrain/x3d/Monterey25_10x/Monterey25_10x_scene.x3d": {
"position": "-2822317.31255 -4438600.53640 3786150.85474",
"orientation": "0.89575 -0.31076 -0.31791 1.63772",
"centerOfRotation": "-2711557.9403829873 -4331414.329506527 3801353.4691465236",
"VerticalExaggeration": "10",
"speed": "0.1",
}
},
grdTerrain=os.path.join(parentDir, "Monterey25.grd"),
)
# Set start and end dates for all loads from sources that contain data
# beyond the temporal bounds of the campaign
#
# startdate = datetime.datetime(2015, 7, 31) # Fixed start
startdate = datetime.datetime(2015, 6, 6) # Fixed start
# enddate = datetime.datetime(2015, 9, 30) # Fixed end
'''
import os
import sys
import datetime # needed for glider data
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
project_dir = os.path.dirname(__file__)
# the next line makes it possible to find CANON
sys.path.insert(0, os.path.join(os.path.dirname(__file__),
"../")) # settings.py is one dir up
from CANON import CANONLoader
# building input data sources object
cl = CANONLoader('stoqs_september2011', 'CANON - September 2011')
# default location of thredds and dods data:
cl.tdsBase = 'http://odss.mbari.org/thredds/'
cl.dodsBase = cl.tdsBase + 'dodsC/'
# special location for dorado data
cl.dorado_base = 'http://dods.mbari.org/opendap/data/auvctd/surveys/2011/netcdf/'
cl.dorado_files = [
'Dorado389_2011_249_00_249_00_decim.nc',
'Dorado389_2011_250_01_250_01_decim.nc',
'Dorado389_2011_255_00_255_00_decim.nc',
'Dorado389_2011_257_00_257_00_decim.nc',
'Dorado389_2011_262_00_262_00_decim.nc',
'Dorado389_2011_263_00_263_00_decim.nc',
'Dorado389_2011_264_00_264_00_decim.nc',
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
project_dir = os.path.dirname(__file__)
# the next line makes it possible to find CANON
sys.path.insert(0, os.path.join(
os.path.dirname(__file__),
"../")) # this makes it possible to find CANON, one directory up
from CANON import CANONLoader
# building input data sources object
from socket import gethostname
hostname = gethostname()
print(hostname)
if hostname == 'odss-test.shore.mbari.org':
cl = CANONLoader('stoqs_september2011', 'CANON - September 2011')
else:
cl = CANONLoader('stoqs_september2013', 'CANON - September 2013')
# default location of thredds and dods data:
cl.tdsBase = 'http://odss.mbari.org/thredds/'
cl.dodsBase = cl.tdsBase + 'dodsC/'
######################################################################
# GLIDERS
######################################################################
# Set start and end dates for all glider loads
# startdate is 24hours from now
ts = time.time() - (2.2 * 60 * 60)
st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M')
t = time.strptime(st, "%Y-%m-%d %H:%M")
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
from thredds_crawler.etree import etree
from thredds_crawler.crawl import Crawl
cl = CANONLoader('stoqs_canon_september2015', 'CANON - September-October 2015',
description = 'Fall 2015 Front Identification in northern Monterey Bay',
x3dTerrains = {
'/static/x3d/Monterey25_10x/Monterey25_10x_scene.x3d': {
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation': '-2711557.94 -4331414.32 3801353.46',
'VerticalExaggeration': '10',
'speed': '.1',
}
},
grdTerrain = os.path.join(parentDir, 'Monterey25.grd')
)
# Set start and end dates for all loads from sources that contain data
# beyond the temporal bounds of the campaign
startdate = datetime.datetime(2015, 9, 17) # Fixed start Thursday Sep 17 2015 per F. Chavez
enddate = datetime.datetime(2015, 10, 16) # Fixed end two days after end of CANON cruises
# default location of thredds and dods data:
cl.tdsBase = 'http://odss.mbari.org/thredds/'
cl.dodsBase = cl.tdsBase + 'dodsC/'
import sys
import datetime
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
from SampleLoaders import SubSamplesLoader
cl = CANONLoader('stoqs_february2012', 'GOC - February 2012',
description = 'Western Flyer profile and underway CTD data from Monterey to Gulf of California with Sample data from BOG',
x3dTerrains = {
'http://dods.mbari.org/terrain/x3d/Globe_1m_bath_10x/Globe_1m_bath_10x_scene.x3d': {
'position': '14051448.48336 -15407886.51486 6184041.22775',
'orientation': '0.83940 0.33030 0.43164 1.44880',
'centerOfRotation': '0 0 0',
'VerticalExaggeration': '10',
}
},
grdTerrain = os.path.join(parentDir, 'Globe_1m_bath.grd')
)
# Base OPenDAP server - if aboard a ship change to the local odss server
cl.tdsBase = 'http://odss.mbari.org/thredds/'
cl.dodsBase = cl.tdsBase + 'dodsC/'
# Western Flyer Underway CTD
cl.wfuctd_base = cl.dodsBase + 'GOC_february2012/wf/uctd/'
cl.wfuctd_files = [
'goc12m01.nc',
import datetime # needed for glider data
import time # for startdate, enddate args
os.environ['DJANGO_SETTINGS_MODULE']='settings'
project_dir = os.path.dirname(__file__)
# the next line makes it possible to find CANON
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../")) # this makes it possible to find CANON, one directory up
from CANON import CANONLoader
# building input data sources object
from socket import gethostname
hostname=gethostname()
print hostname
if hostname=='odss-test.shore.mbari.org':
cl = CANONLoader('stoqs_september2011', 'CANON - September 2011')
else:
cl = CANONLoader('stoqs_september2013', 'CANON - September 2013')
# default location of thredds and dods data:
cl.tdsBase = 'http://odss.mbari.org/thredds/'
cl.dodsBase = cl.tdsBase + 'dodsC/'
######################################################################
# GLIDERS
######################################################################
# Set start and end dates for all glider loads
# startdate is 24hours from now
ts=time.time()-(2.2*60*60)
st=datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M')
import os
import sys
import datetime
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
cl = CANONLoader('stoqs_canon_april2014', 'CANON-ECOHAB - April 2014',
description = 'Spring 2014 ECOHAB in San Pedro Bay',
x3dTerrains= { '/stoqs/static/x3d/SanPedroBasin50/SanPedroBasin50_10x-pop.x3d': {
'position': '-2523652.5 -4726093.2 3499413.2',
'orientation': '0.96902 -0.20915 -0.13134 1.74597',
'centerOfRotation': '-2505293.6 -4686937.5 3513055.2',
'VerticalExaggeration': '10',
}
},
grdTerrain = os.path.join(parentDir, 'SanPedroBasin50.grd')
)
# Aboard the Carson use zuma
##cl.tdsBase = 'http://zuma.rc.mbari.org/thredds/'
cl.tdsBase = 'http://odss.mbari.org/thredds/' # Use this on shore
cl.dodsBase = cl.tdsBase + 'dodsC/'
# Decimated dorado data
cl.dorado_base = cl.dodsBase + 'CANON/2014_Apr/Platforms/AUVs/Dorado/'
cl.dorado_files = [
'Dorado389_2014_102_00_102_00_decim.nc', 'Dorado389_2014_103_00_103_00_decim.nc',
import datetime # needed for glider data
import time # for startdate, enddate args
os.environ['DJANGO_SETTINGS_MODULE']='settings'
project_dir = os.path.dirname(__file__)
# the next line makes it possible to find CANON
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../")) # this makes it possible to find CANON, one directory up
from CANON import CANONLoader
# building input data sources object
from socket import gethostname
hostname=gethostname()
print hostname
if hostname=='odss-test.shore.mbari.org':
cl = CANONLoader('stoqs_september2011', 'CANON - September 2011')
else:
cl = CANONLoader('stoqs_september2013', 'CANON - September 2013')
# default location of thredds and dods data:
cl.tdsBase = 'http://odss.mbari.org/thredds/'
cl.dodsBase = cl.tdsBase + 'dodsC/'
# Set start and end dates for mooring, twice per day. In the morning and afternoon.
##t =time.strptime("2013-09-09 0:01", "%Y-%m-%d %H:%M")
##startdate=t[:6]
ts=time.time()-(33*60*60)
st=datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M')
t= time.strptime("2013-09-18 0:01", "%Y-%m-%d %H:%M") #deployed 09/18/13
#t=time.strptime(st,"%Y-%m-%d %H:%M")
from CANON import CANONLoader
from loaders import FileNotFound
from thredds_crawler.crawl import Crawl
from thredds_crawler.etree import etree
cl = CANONLoader('stoqs_ps2017', 'post Season 2017',
description='post season 2017 observations in Monterey Bay',
x3dTerrains={
'http://dods.mbari.org/terrain/x3d/Monterey25_10x/Monterey25_10x_scene.x3d': {
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation': '-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
},
'http://stoqs.mbari.org/x3d/Monterey25_1x/Monterey25_1x_src_scene.x3d': {
'name': 'Monterey25_1x',
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation': '-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '1',
},
},
grdTerrain=os.path.join(parentDir, 'Monterey25.grd')
)
# Set start and end dates for all loads from sources that contain data
# beyond the temporal bounds of the campaign
#
startdate = datetime.datetime(2017, 5, 16) # Fixed start. May 15, 2017
enddate = datetime.datetime(2017, 9, 17) # Fixed end. September 17, 2017
import os
import sys
import datetime
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
cl = CANONLoader('stoqs_simz_oct2013', 'Sampling and Identification of Marine Zooplankton - October 2013',
description = 'Rachel Carson and Dorado surveys in Northern Monterey Bay',
x3dTerrains = {
'http://dods.mbari.org/terrain/x3d/Monterey25_10x/Monterey25_10x_scene.x3d': {
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation': '-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
}
},
grdTerrain = os.path.join(parentDir, 'Monterey25.grd')
)
startDatetime = datetime.datetime(2013, 10, 22)
endDatetime = datetime.datetime(2013, 10, 29)
# Aboard the Carson use zuma:
##cl.tdsBase = 'http://zuma.rc.mbari.org/thredds/'
# On shore, use the odss server:
cl.tdsBase = 'http://odss.mbari.org/thredds/'
cl.dodsBase = cl.tdsBase + 'dodsC/'
import os
import sys
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
# Assign input data sources
cl = CANONLoader('stoqs_june2011', 'CANON - June 2011',
description = 'Front detection Dorado and Tethys surveys in Monterey Bay',
x3dTerrains = {
'http://dods.mbari.org/terrain/x3d/Monterey25_10x/Monterey25_10x_scene.x3d': {
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation': '-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
'speed': '.1',
}
},
grdTerrain = os.path.join(parentDir, 'Monterey25.grd')
)
cl.dorado_base = 'http://dods.mbari.org/opendap/data/auvctd/surveys/2011/netcdf/'
cl.dorado_files = [ 'Dorado389_2011_164_05_164_05_decim.nc',
'Dorado389_2011_165_00_165_00_decim.nc',
'Dorado389_2011_166_00_166_00_decim.nc',
'Dorado389_2011_171_01_171_01_decim.nc',
]
cl.dorado_parms = [ 'temperature', 'oxygen', 'nitrate', 'bbp420', 'bbp700',
'fl700_uncorr', 'salinity', 'biolume',
import csv
import urllib.request, urllib.error, urllib.parse
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
import timing
cl = CANONLoader('stoqs_september2014', 'CANON-ECOHAB - September 2014',
description = 'Fall 2014 Dye Release Experiment in Monterey Bay',
x3dTerrains = {
'https://stoqs.mbari.org/x3d/Monterey25_10x/Monterey25_10x_scene.x3d': {
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation': '-2711557.94 -4331414.32 3801353.46',
'VerticalExaggeration': '10',
'speed': '.1',
}
},
grdTerrain = os.path.join(parentDir, 'Monterey25.grd')
)
# Set start and end dates for all loads from sources that contain data
# beyond the temporal bounds of the campaign
startdate = datetime.datetime(2014, 9, 21) # Fixed start
enddate = datetime.datetime(2014, 10, 12) # Fixed end
# default location of thredds and dods data:
cl.tdsBase = 'http://odss.mbari.org/thredds/'
cl.dodsBase = cl.tdsBase + 'dodsC/'
import sys
import datetime # needed for glider data
import time # for startdate, enddate args
project_dir = os.path.dirname(__file__)
# the next line makes it possible to find CANON
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../")) # this makes it possible to find CANON, one directory up
from CANON import CANONLoader
# building input data sources object
from socket import gethostname
hostname=gethostname()
print(hostname)
if hostname=='odss-test.shore.mbari.org':
cl = CANONLoader('stoqs_september2011', 'CANON - September 2011')
else:
cl = CANONLoader('stoqs_september2013', 'CANON - September 2013')
# default location of thredds and dods data:
cl.tdsBase = 'http://odss.mbari.org/thredds/'
cl.dodsBase = cl.tdsBase + 'dodsC/'
######################################################################
# GLIDERS
######################################################################
# Set start and end dates for all glider loads
# startdate is 24hours from now
ts=time.time()-(12.2*60*60)
import os
import sys
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # to find CANON, one directory up
from CANON import CANONLoader
# Assign input data sources
cl = CANONLoader(
'stoqs_september2010',
'CANON - September 2010',
description='ESP Drift with Dorado circling outside Monterey Bay',
x3dTerrains={
'http://dods.mbari.org/terrain/x3d/Monterey25_10x/Monterey25_10x_scene.x3d':
{
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation':
'-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
'speed': '1',
}
},
grdTerrain=os.path.join(parentDir, 'Monterey25.grd'))
cl.dorado_base = 'http://dods.mbari.org/opendap/data/auvctd/surveys/2010/netcdf/'
cl.dorado_files = [
'Dorado389_2010_257_01_258_04_decim.nc',
'Dorado389_2010_258_05_258_08_decim.nc',
'Dorado389_2010_259_00_259_03_decim.nc',
'Dorado389_2010_260_00_260_00_decim.nc',
'Dorado389_2010_261_00_261_00_decim.nc'
import datetime # needed for glider data
import time # for startdate, enddate args
os.environ['DJANGO_SETTINGS_MODULE']='settings'
project_dir = os.path.dirname(__file__)
# the next line makes it possible to find CANON
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../")) # this makes it possible to find CANON, one directory up
from CANON import CANONLoader
# building input data sources object
from socket import gethostname
hostname=gethostname()
print(hostname)
if hostname=='odss-test.shore.mbari.org':
cl = CANONLoader('stoqs_september2011', 'CANON - September 2011')
else:
cl = CANONLoader('stoqs_september2013', 'CANON - September 2013')
#25613JMC01.nc 100% 3172 3.1KB/s 00:00
#25613JMC02.nc 100% 6268 6.1KB/s 00:00
#25613JMC03.nc 100% 5692 5.6KB/s 00:00
#25613JMC04.nc 100% 6340 6.2KB/s 00:00
#25613JMC05.nc 100% 4468 4.4KB/s 00:00
#26013JMC01.nc 100% 6484 6.3KB/s 00:00
#26013JMC02.nc 100% 3460 3.4KB/s 00:00
#26013JMC03.nc 100% 6484 6.3KB/s 00:00
#26013JMC04.nc 100% 5692 5.6KB/s 00:00
#uctd/25413JMm01.txt
#uctd/25613JMm01.txt
import requests
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
from thredds_crawler.crawl import Crawl
from thredds_crawler.etree import etree
cl = CANONLoader('stoqs_os2015', 'CANON-ECOHAB - Off Season 2015',
description = 'CANON Off Season 2015 Experiment in Monterey Bay',
x3dTerrains = {
'http://dods.mbari.org/terrain/x3d/Monterey25_10x/Monterey25_10x_scene.x3d': {
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation': '-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
'speed': '0.1',
}
},
grdTerrain = os.path.join(parentDir, 'Monterey25.grd')
)
# Set start and end dates for all loads from sources that contain data
# beyond the temporal bounds of the campaign
#
#startdate = datetime.datetime(2015, 7, 31) # Fixed start
startdate = datetime.datetime(2015, 6, 6) # Fixed start
#enddate = datetime.datetime(2015, 9, 30) # Fixed end
enddate = datetime.datetime(2015, 12, 31) # Fixed end. Extend "offseason" to end of year
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
from loaders import FileNotFound
from thredds_crawler.crawl import Crawl
from thredds_crawler.etree import etree
cl = CANONLoader('stoqs_canon_september2016', 'CANON - September 2016',
description = 'CANON September 2016 Experiment in Monterey Bay',
x3dTerrains = {
'http://dods.mbari.org/terrain/x3d/Monterey25_10x/Monterey25_10x_scene.x3d': {
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation': '-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
'speed': '0.1',
}
},
grdTerrain = os.path.join(parentDir, 'Monterey25.grd')
)
# Set start and end dates for all loads from sources that contain data
# beyond the temporal bounds of the campaign
#
startdate = datetime.datetime(2016, 7, 11) # Fixed start July 11 2016, deployment of nps34
enddate = datetime.datetime(2016, 10, 7) # Fixed end. October 7 2016, a few days after end of CANON cruise
# default location of thredds and dods data:
cl.tdsBase = 'http://odss.mbari.org/thredds/'
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
import timing
cl = CANONLoader(
'stoqs_september2012',
'CANON - September 2012',
description=
'Western Flyer and Tethys following drifting ESP off of Big Sur',
x3dTerrains={
'https://stoqs.mbari.org/x3d/Monterey25_10x/Monterey25_10x_scene.x3d':
{
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation':
'-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
'speed': '.1',
}
},
grdTerrain=os.path.join(parentDir, 'Monterey25.grd'))
# Aboard the Flyer use malibu's VSAT IP address:
cl.tdsBase = 'http://odss.mbari.org/thredds/'
cl.dodsBase = cl.tdsBase + 'dodsC/'
# 2-second decimated dorado data
# http://192.168.111.177:8080/thredds/dodsC/CANON_september2012/dorado/Dorado389_2012_258_00_258_00_decim.nc
'''
import os
import sys
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
# Assign input data sources
cl = CANONLoader('stoqs_september2010', 'CANON - September 2010',
description = 'ESP Drift with Dorado circling outside Monterey Bay',
x3dTerrains = {
'http://dods.mbari.org/terrain/x3d/Monterey25_10x/Monterey25_10x_scene.x3d': {
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation': '-2711557.94 -4331414.32 3801353.46',
'VerticalExaggeration': '10',
'speed': '.1',
}
},
grdTerrain = os.path.join(parentDir, 'Monterey25.grd')
)
cl.dorado_base = 'http://dods.mbari.org/opendap/data/auvctd/surveys/2010/netcdf/'
cl.dorado_files = [ 'Dorado389_2010_257_01_258_04_decim.nc',
'Dorado389_2010_258_05_258_08_decim.nc',
'Dorado389_2010_259_00_259_03_decim.nc',
'Dorado389_2010_260_00_260_00_decim.nc',
'Dorado389_2010_261_00_261_00_decim.nc',
]
cl.dorado_parms = [ 'temperature', 'oxygen', 'nitrate', 'bbp420', 'bbp700',
import datetime # needed for glider data
import time # for startdate, enddate args
os.environ['DJANGO_SETTINGS_MODULE']='settings'
project_dir = os.path.dirname(__file__)
# the next line makes it possible to find CANON
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../")) # this makes it possible to find CANON, one directory up
from CANON import CANONLoader
# building input data sources object
from socket import gethostname
hostname=gethostname()
print hostname
if hostname=='odss-test.shore.mbari.org':
cl = CANONLoader('stoqs_september2011', 'CANON - September 2011')
else:
cl = CANONLoader('stoqs_september2013', 'CANON - September 2013')
# default location of thredds and dods data:
cl.tdsBase = 'http://odss.mbari.org/thredds/'
cl.dodsBase = cl.tdsBase + 'dodsC/'
######################################################################
# Set start and end dates for all glider loads
#t =time.strptime("2013-09-11 0:01", "%Y-%m-%d %H:%M")
ts=time.time()-(11*60*60)
st=datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M')
startdate=t[:6]
t =time.strptime("2013-09-29 0:01", "%Y-%m-%d %H:%M")
import datetime
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
import timing
cl = CANONLoader(
'stoqs_simz_spring2014',
'Sampling and Identification of Marine Zooplankton - Spring 2014',
description='Month-long investigation of water in northern Monterey Bay',
x3dTerrains={
'https://stoqs.mbari.org/x3d/Monterey25_10x/Monterey25_10x_scene.x3d':
{
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation':
'-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
'speed': '.1',
}
},
# Do not check in .grd files to the repository, keep them in the loaders directory
grdTerrain=os.path.join(parentDir, 'Monterey25.grd'),
)
startDatetime = datetime.datetime(2014, 2, 18)
endDatetime = datetime.datetime(2014, 3, 18)
# Aboard the Carson use zuma:
##cl.tdsBase = 'http://zuma.rc.mbari.org/thredds/'
# On shore, use the odss server:
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
import datetime
import timing
# Assign input data sources
cl = CANONLoader(
'stoqs_june2011',
'CANON - June 2011',
description='Front detection Dorado and Tethys surveys in Monterey Bay',
x3dTerrains={
'https://stoqs.mbari.org/x3d/Monterey25_10x/Monterey25_10x_scene.x3d':
{
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation':
'-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
'speed': '.1',
}
},
grdTerrain=os.path.join(parentDir, 'Monterey25.grd'))
startdate = datetime.datetime(2011, 6, 10)
enddate = datetime.datetime(2011, 6, 24)
cl.dorado_base = 'http://dods.mbari.org/opendap/data/auvctd/surveys/2011/netcdf/'
cl.dorado_files = [
'Dorado389_2011_164_05_164_05_decim.nc',
'Dorado389_2011_165_00_165_00_decim.nc',
import datetime # needed for glider data
import time # for startdate, enddate args
os.environ['DJANGO_SETTINGS_MODULE']='settings'
project_dir = os.path.dirname(__file__)
# the next line makes it possible to find CANON
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../")) # this makes it possible to find CANON, one directory up
from CANON import CANONLoader
# building input data sources object
from socket import gethostname
hostname=gethostname()
print hostname
if hostname=='odss-test.shore.mbari.org':
cl = CANONLoader('stoqs_september2011', 'CANON - September 2011')
else:
cl = CANONLoader('stoqs_september2013', 'CANON - September 2013')
# default location of thredds and dods data:
cl.tdsBase = 'http://odss.mbari.org/thredds/'
cl.dodsBase = cl.tdsBase + 'dodsC/'
######################################################################
# GLIDERS
######################################################################
# Set start and end dates for all glider loads
# startdate is 24hours from now
ts=time.time()-(12.2*60*60)
'''
import os
import sys
import datetime # needed for glider data
import time # for startdate, enddate args
os.environ['DJANGO_SETTINGS_MODULE']='settings'
project_dir = os.path.dirname(__file__)
# the next line makes it possible to find CANON
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../")) # this makes it possible to find CANON, one directory up
from CANON import CANONLoader
# building input data sources object
cl = CANONLoader('stoqs_september2013', 'CANON - September 2013')
# default location of thredds and dods data:
cl.tdsBase = 'http://odss.mbari.org/thredds/'
cl.dodsBase = cl.tdsBase + 'dodsC/'
#####################################################################
# DORADO
#####################################################################
# special location for dorado data
cl.dorado_base = 'http://dods.mbari.org/opendap/data/auvctd/surveys/2013/netcdf/'
cl.dorado_files = [# 'Dorado389_2013_228_00_228_00_decim.nc',
# 'Dorado389_2013_228_01_228_01_decim.nc'
# 'Dorado389_2013_259_00_259_00_decim.nc' #Sep 16
# 'Dorado389_2013_259_00_259_01_decim.nc'
# 'Dorado389_2013_260_00_260_00_decim.nc' #Sep 17
project_dir = os.path.dirname(__file__)
# the next line makes it possible to find CANON
sys.path.insert(0, os.path.join(
os.path.dirname(__file__),
"../")) # this makes it possible to find CANON, one directory up
from CANON import CANONLoader
# building input data sources object
from socket import gethostname
hostname = gethostname()
print(hostname)
if hostname == 'odss-test.shore.mbari.org':
cl = CANONLoader('stoqs_september2011', 'CANON - September 2011')
else:
cl = CANONLoader('stoqs_september2013', 'CANON - September 2013')
# default location of thredds and dods data:
cl.tdsBase = 'http://odss.mbari.org/thredds/'
cl.dodsBase = cl.tdsBase + 'dodsC/'
#####################################################################
# DORADO
#####################################################################
# special location for dorado data
cl.dorado_base = 'http://dods.mbari.org/opendap/data/auvctd/surveys/2013/netcdf/'
cl.dorado_files = [ # 'Dorado389_2013_259_00_259_00_decim.nc', #Sep 16 Loaded
# 'Dorado389_2013_261_01_261_01_decim.nc',
# 'Dorado389_2013_262_00_262_00_decim.nc', #Sep 19 Dorado389_2013_262_00_262_00
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
import timing
cl = CANONLoader('stoqs_canon_september2017', 'CANON - September 2017',
description='September 2017 campaign observations in Monterey Bay',
x3dTerrains={
'https://stoqs.mbari.org/x3d/Monterey25_10x/Monterey25_10x_scene.x3d': {
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation': '-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
},
'https://stoqs.mbari.org/x3d/Monterey25_1x/Monterey25_1x_src_scene.x3d': {
'name': 'Monterey25_1x',
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation': '-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '1',
},
},
grdTerrain=os.path.join(parentDir, 'Monterey25.grd')
)
# Set start and end dates for all loads from sources that contain data
# beyond the temporal bounds of the campaign
#
startdate = datetime.datetime(2017, 9, 18) # Fixed start. September 18, 2017
enddate = datetime.datetime(2017, 12, 31) # Fixed end. October 7, 2017. Extended to end of year.
from CANON import CANONLoader
import timing
cl = CANONLoader(
'stoqs_canon_april2017',
'KISS CANON Spring 2017',
description='KISS CANON Spring 2017 Experiment in Monterey Bay',
x3dTerrains={
'https://stoqs.mbari.org/x3d/Monterey25_10x/Monterey25_10x_scene.x3d':
{
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation':
'-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
},
'https://stoqs.mbari.org/x3d/Monterey25_1x/Monterey25_1x_src_scene.x3d':
{
'name': 'Monterey25_1x',
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation':
'-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '1',
},
},
grdTerrain=os.path.join(parentDir, 'Monterey25.grd'))
# Set start and end dates for all loads from sources that contain data
# beyond the temporal bounds of the campaign
#
@status: __status__
@license: __license__
'''
import os
import datetime
from CANON import CANONLoader
# Assign input data sources - use locally served x3d terrain data
cl = CANONLoader('default', 'Initial Test Database',
description = 'Post-setup load of a variety of data to use for testing',
x3dTerrains = {
'/static/x3d/Monterey25_10x/Monterey25_10x_scene.x3d': {
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation': '-2711557.94 -4331414.32 3801353.46',
'VerticalExaggeration': '10',
'speed': '.1',
}
},
# Terrain data file is expected to be in loaders directory
grdTerrain = os.path.join(os.path.dirname(__file__), 'Monterey25.grd')
)
# Assign input data sets from OPeNDAP URLs pointing to Discrete Sampling Geometry CF-NetCDF sources
# - Trajectory
cl.dorado_base = 'http://dods.mbari.org/opendap/data/auvctd/surveys/2010/netcdf/'
cl.dorado_files = [ 'Dorado389_2010_300_00_300_00_decim.nc' ]
cl.dorado_parms = [ 'temperature', 'oxygen', 'nitrate', 'bbp420', 'bbp700',
'fl700_uncorr', 'salinity', 'biolume', 'roll', 'pitch', 'yaw',
'sepCountList', 'mepCountList']
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
from SampleLoaders import SubSamplesLoader
import timing
cl = CANONLoader(
'stoqs_february2012',
'GOC - February 2012',
description=
'Western Flyer profile and underway CTD data from Monterey to Gulf of California with Sample data from BOG',
x3dTerrains={
'https://stoqs.mbari.org/x3d/Globe_1m_bath_10x/Globe_1m_bath_10x_scene.x3d':
{
'position': '14051448.48336 -15407886.51486 6184041.22775',
'orientation': '0.83940 0.33030 0.43164 1.44880',
'centerOfRotation': '0 0 0',
'VerticalExaggeration': '10',
}
},
grdTerrain=os.path.join(parentDir, 'Globe_1m_bath.grd'))
# Base OPenDAP server - if aboard a ship change to the local odss server
cl.tdsBase = 'http://odss.mbari.org/thredds/'
cl.dodsBase = cl.tdsBase + 'dodsC/'
# Western Flyer Underway CTD
cl.wfuctd_base = cl.dodsBase + 'GOC_february2012/wf/uctd/'
cl.wfuctd_files = [
import sys
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
import datetime
import timing
cl = CANONLoader('stoqs_october2010', 'CANON - October 2010',
description = 'Bloomex observing campaign in Monterey Bay',
x3dTerrains = {
'https://stoqs.mbari.org/x3d/Monterey25_10x/Monterey25_10x_scene.x3d': {
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation': '-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
'speed': '.1',
}
},
grdTerrain = os.path.join(parentDir, 'Monterey25.grd')
)
startdate = datetime.datetime(2010, 10, 4)
enddate = datetime.datetime(2010, 10, 29)
# Dorado data - 2 second gridded
cl.dorado_base = 'http://dods.mbari.org/opendap/data/auvctd/surveys/2010/netcdf/'
cl.dorado_files = [ 'Dorado389_2010_277_01_277_01_decim.nc',
'Dorado389_2010_278_01_278_01_decim.nc',
'Dorado389_2010_279_02_279_02_decim.nc',
from CANON import CANONLoader
import timing
cl = CANONLoader(
'stoqs_canon_may2019',
'CANON - May 2019',
description='May 2019 coordinated campaign observations in Monterey Bay',
x3dTerrains={
'https://stoqs.mbari.org/x3d/Monterey25_10x/Monterey25_10x_scene.x3d':
{
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation':
'-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
},
'https://stoqs.mbari.org/x3d/Monterey25_1x/Monterey25_1x_src_scene.x3d':
{
'name': 'Monterey25_1x',
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation':
'-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '1',
},
},
grdTerrain=os.path.join(parentDir, 'Monterey25.grd'))
# Set start and end dates for all loads from sources that contain data
# beyond the temporal bounds of the campaign
#
import sys
import datetime # needed for glider data
import time # for startdate, enddate args
project_dir = os.path.dirname(__file__)
# the next line makes it possible to find CANON
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../")) # this makes it possible to find CANON, one directory up
from CANON import CANONLoader
# building input data sources object
from socket import gethostname
hostname=gethostname()
print(hostname)
if hostname=='odss-test.shore.mbari.org':
cl = CANONLoader('stoqs_september2011', 'CANON - September 2011')
else:
cl = CANONLoader('stoqs_september2013', 'CANON - September 2013')
# default location of thredds and dods data:
cl.tdsBase = 'http://odss.mbari.org/thredds/'
cl.dodsBase = cl.tdsBase + 'dodsC/'
######################################################################
# Set start and end dates for all glider loads
#t =time.strptime("2013-09-11 0:01", "%Y-%m-%d %H:%M")
ts=time.time()-(11*60*60)
st=datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M')
startdate=t[:6]
t =time.strptime("2013-09-29 0:01", "%Y-%m-%d %H:%M")
import time # for startdate, enddate args
os.environ['DJANGO_SETTINGS_MODULE']='settings'
project_dir = os.path.dirname(__file__)
# the next line makes it possible to find CANON
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../")) # this makes it possible to find CANON, one directory up
from CANON import CANONLoader
# building input data sources object
from socket import gethostname
hostname=gethostname()
print(hostname)
if hostname=='odss-test.shore.mbari.org':
cl = CANONLoader('stoqs_september2011', 'CANON - September 2011')
else:
cl = CANONLoader('stoqs_september2013', 'CANON - September 2013')
# default location of thredds and dods data:
cl.tdsBase = 'http://odss.mbari.org/thredds/'
cl.dodsBase = cl.tdsBase + 'dodsC/'
#####################################################################
# DORADO
#####################################################################
# special location for dorado data
cl.dorado_base = 'http://dods.mbari.org/opendap/data/auvctd/surveys/2013/netcdf/'
cl.dorado_files = [# 'Dorado389_2013_259_00_259_00_decim.nc', #Sep 16 Loaded
# 'Dorado389_2013_261_01_261_01_decim.nc',
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
import timing
cl = CANONLoader('stoqs_simz_oct2014', 'Sampling and Identification of Marine Zooplankton - October 2014',
description = 'Rachel Carson and Dorado surveys in Northern Monterey Bay',
# Assign a GeoOrigin in the middle of the terrain that is an appropriate
# location for rotateYUp, making fly navigation work well. All other GCC
# positions are in the GeoOrigin with rotateYUp='true' coordinate system
x3dTerrains = {
'https://stoqs.mbari.org/x3d/Monterey25_10x/Monterey25_10x_scene.x3d': {
'position': '-5334.11754 77527.85269 57495.84643',
'orientation': '-0.99840 0.05415 0.01651 0.88794',
'centerOfRotation': '1973.702 -553.761 -10885.8333',
'VerticalExaggeration': '10',
'geoOrigin': '36.75 -122 0',
'speed': '0.1',
}
},
grdTerrain = os.path.join(parentDir, 'Monterey25.grd')
)
startDatetime = datetime.datetime(2014, 10, 15)
endDatetime = datetime.datetime(2014, 10, 23)
# Aboard the Carson use zuma:
##cl.tdsBase = 'http://zuma.rc.mbari.org/thredds/'
# On shore, use the odss server:
#!/usr/bin/env python
'''
Script to append a small amount of mooring data to an activity already
in the default database. This is meant for testing the --append option
for our load scripts.
Mike McCann
MBARI 6 September 2018
'''
import datetime
from CANON import CANONLoader
import timing
# Assign input data sources - use locally served x3d terrain data
cl = CANONLoader('default', 'Initial Test Database')
# Assign input data sets from OPeNDAP URLs pointing to Discrete Sampling Geometry CF-NetCDF sources
# - TimeSeries and TimeSeriesProfile
cl.m1_base = 'http://dods.mbari.org/opendap/data/ssdsdata/deployments/m1/'
cl.m1_files = [
'201010/OS_M1_20101027hourly_CMSTV.nc',
'201010/m1_hs2_20101027.nc',
]
cl.m1_parms = [
'northward_sea_water_velocity_HR', 'SEA_WATER_SALINITY_HR',
'SEA_WATER_TEMPERATURE_HR', 'AIR_TEMPERATURE_HR', 'bb470', 'fl676'
]
# Use the same startDatetime as in loadTestData.py so that the Activity is named the same
# Load 1 more hour of data beyond the endDatetime in loadTestData.py
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
from thredds_crawler.etree import etree
from thredds_crawler.crawl import Crawl
cl = CANONLoader(
'stoqs_canon_september2015',
'CANON - September-October 2015',
description='Fall 2015 Front Identification in northern Monterey Bay',
x3dTerrains={
'/static/x3d/Monterey25_10x/Monterey25_10x_scene.x3d': {
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation': '-2711557.94 -4331414.32 3801353.46',
'VerticalExaggeration': '10',
'speed': '.1',
}
},
grdTerrain=os.path.join(parentDir, 'Monterey25.grd'))
# Set start and end dates for all loads from sources that contain data
# beyond the temporal bounds of the campaign
startdate = datetime.datetime(
2015, 9, 8) # Changed to 8th to include pre CANON LRAUV test data
enddate = datetime.datetime(
2015, 10, 16) # Fixed end two days after end of CANON cruises
# default location of thredds and dods data:
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
from thredds_crawler.crawl import Crawl
from thredds_crawler.etree import etree
cl = CANONLoader(
'stoqs_os2015',
'CANON-ECOHAB - Off Season 2015',
description='CANON Off Season 2015 Experiment in Monterey Bay',
x3dTerrains={
'http://dods.mbari.org/terrain/x3d/Monterey25_10x/Monterey25_10x_scene.x3d':
{
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation':
'-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
'speed': '0.1',
}
},
grdTerrain=os.path.join(parentDir, 'Monterey25.grd'))
# Set start and end dates for all loads from sources that contain data
# beyond the temporal bounds of the campaign
#
#startdate = datetime.datetime(2015, 7, 31) # Fixed start
startdate = datetime.datetime(2015, 6, 6) # Fixed start
#enddate = datetime.datetime(2015, 9, 30) # Fixed end
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
import timing
cl = CANONLoader('stoqs_all_dorado', 'Plankton Proxies - All Dorado Data',
description='All Dorado survey data from 2003 through 2019 and beyond',
x3dTerrains={
'http://dods.mbari.org/terrain/x3d/Monterey25_10x/Monterey25_10x_scene.x3d': {
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation': '-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
},
'http://stoqs.mbari.org/x3d/Monterey25_1x/Monterey25_1x_src_scene.x3d': {
'name': 'Monterey25_1x',
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation': '-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '1',
},
},
grdTerrain=os.path.join(parentDir, 'Monterey25.grd')
)
startdate = datetime.datetime(2003, 1, 1)
enddate = datetime.datetime(2019, 12, 31)
# Execute the load
cl.process_command_line()
from datetime import datetime
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir)
from CANON import CANONLoader
import timing
cl = CANONLoader(
'stoqs_canon_july2020',
'CANON - July 2020',
description='July 2020 shipless campaign in Monterey Bay (CN20S)',
x3dTerrains={
'https://stoqs.mbari.org/x3d/Monterey25_10x/Monterey25_10x_scene.x3d':
{
'name': 'Monterey25_10x',
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation':
'-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
},
},
grdTerrain=os.path.join(parentDir, 'Monterey25.grd'))
startdate = datetime(2020, 7, 15)
enddate = datetime(2020, 8, 5)
# default location of thredds and dods data:
cl.tdsBase = 'http://odss.mbari.org/thredds/'
cl.dodsBase = cl.tdsBase + 'dodsC/'
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
import timing
cl = CANONLoader(
'stoqs_dorado2011',
'Dorado - All 2011 missions',
description=
'In Monterey Bay and Santa Monica Basin - includes processed Gulper Samples',
x3dTerrains={
'https://stoqs.mbari.org/x3d/Monterey25_10x/Monterey25_10x_scene.x3d':
{
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation':
'-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
'speed': '.1',
}
},
grdTerrain=os.path.join(parentDir, 'Monterey25.grd'))
# Dorado surveys in 2011
cl.dorado_base = 'http://dods.mbari.org/opendap/data/auvctd/surveys/2011/netcdf/'
cl.dorado_files = [
'Dorado389_2011_060_01_060_01_decim.nc',
'Dorado389_2011_061_00_061_00_decim.nc',
'Dorado389_2011_062_05_062_05_decim.nc',
project_dir = os.path.dirname(__file__)
# the next line makes it possible to find CANON
sys.path.insert(0, os.path.join(
os.path.dirname(__file__),
"../")) # this makes it possible to find CANON, one directory up
from CANON import CANONLoader
# building input data sources object
from socket import gethostname
hostname = gethostname()
print hostname
if hostname == 'odss-test.shore.mbari.org':
cl = CANONLoader('stoqs_september2011', 'CANON - September 2011')
else:
cl = CANONLoader('stoqs_september2013', 'CANON - September 2013')
# default location of thredds and dods data:
cl.tdsBase = 'http://odss.mbari.org/thredds/'
cl.dodsBase = cl.tdsBase + 'dodsC/'
######################################################################
# GLIDERS
######################################################################
# Set start and end dates for all glider loads
# startdate is 24hours from now
ts = time.time() - (13.2 * 60 * 60)
st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M')
t = time.strptime(st, "%Y-%m-%d %H:%M")
import os
import sys
import datetime
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
cl = CANONLoader('stoqs_simz_jul2014', 'Sampling and Identification of Marine Zooplankton - July 2014',
description = 'Rachel Carson and Dorado surveys, Bodega Bay region',
x3dTerrains = {
'http://dods.mbari.org/terrain/x3d/Monterey25_10x/Monterey25_10x_scene.x3d': {
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation': '-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
'speed': '0.1',
}
},
grdTerrain = os.path.join(parentDir, 'Monterey25.grd')
)
startDatetime = datetime.datetime(2014, 7, 28)
endDatetime = datetime.datetime(2014, 7, 31)
# Aboard the Carson use zuma:
cl.tdsBase = 'http://zuma.rc.mbari.org/thredds/'
# On shore, use the odss server:
##cl.tdsBase = 'http://odss.mbari.org/thredds/'
cl.dodsBase = cl.tdsBase + 'dodsC/'
import os
import sys
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
from datetime import datetime
# Assign input data sources
cl = CANONLoader('stoqs_september2010', 'CANON - September 2010',
description = 'ESP Drift with Dorado circling outside Monterey Bay',
x3dTerrains = {
'http://dods.mbari.org/terrain/x3d/Monterey25_10x/Monterey25_10x_scene.x3d': {
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation': '-2711557.94 -4331414.32 3801353.46',
'VerticalExaggeration': '10',
'speed': '.1',
}
},
grdTerrain = os.path.join(parentDir, 'Monterey25.grd')
)
# AUV
cl.dorado_base = 'http://dods.mbari.org/opendap/data/auvctd/surveys/2010/netcdf/'
cl.dorado_files = [ 'Dorado389_2010_257_01_258_04_decim.nc',
'Dorado389_2010_258_05_258_08_decim.nc',
'Dorado389_2010_259_00_259_03_decim.nc',
'Dorado389_2010_260_00_260_00_decim.nc',
'Dorado389_2010_261_00_261_00_decim.nc',
]
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
from loaders import FileNotFound
from thredds_crawler.crawl import Crawl
from thredds_crawler.etree import etree
cl = CANONLoader('stoqs_canon_september2016', 'CANON - September 2016',
description = 'CANON September 2016 Experiment in Monterey Bay',
x3dTerrains = {
'http://dods.mbari.org/terrain/x3d/Monterey25_10x/Monterey25_10x_scene.x3d': {
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation': '-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
'speed': '0.1',
}
},
grdTerrain = os.path.join(parentDir, 'Monterey25.grd')
)
# Set start and end dates for all loads from sources that contain data
# beyond the temporal bounds of the campaign
#
startdate = datetime.datetime(2016, 7, 11) # Fixed start July 11 2016, deployment of nps34
enddate = datetime.datetime(2016, 10, 7) # Fixed end. October 7 2016, a few days after end of CANON cruise
# default location of thredds and dods data:
cl.tdsBase = 'http://odss.mbari.org/thredds/'
start = dt
else:
start = None
if args.end is not None:
dt = datetime.strptime(args.end, '%Y%m%dT%H%M%S')
end = dt
else:
end = None
if args.post:
token = os.environ['SLACKTOKEN']
slack = Slacker(token)
# Assume that the database has already been created with description and terrain information, so use minimal arguments in constructor
cl = CANONLoader(args.database, args.campaign)
cl.dbAlias = args.database
cl.campaignName = args.campaign
# Get directory list from sites
s = args.inUrl.rsplit('/',1)
files = s[1]
url = s[0]
logger.info("Crawling %s for %s files" % (url, files))
c = Crawl(os.path.join(url, 'catalog.xml'), select=[files], debug=False)
for d in c.datasets:
logger.debug('Found %s' % d.id)
urls = [s.get("url") for d in c.datasets for s in d.services if s.get("service").lower() == "opendap"]
import os
import sys
import datetime
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
cl = CANONLoader('stoqs_may2012', 'CANON - May 2012',
description = 'Front detection AUV and Glider surveys in Monterey Bay',
x3dTerrains = {
'/static/x3d/Monterey25_10x/Monterey25_10x_scene.x3d': {
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation': '-2711557.94 -4331414.32 3801353.46',
'VerticalExaggeration': '10',
'speed': '.1',
}
},
grdTerrain = os.path.join(parentDir, 'Monterey25.grd')
)
startdate = datetime.datetime(2012, 5, 15)
enddate = datetime.datetime(2012, 6, 30)
# 2-second decimated dorado data
cl.dorado_base = 'http://dods.mbari.org/opendap/data/auvctd/surveys/2012/netcdf/'
cl.dorado_files = [
'Dorado389_2012_142_01_142_01_decim.nc',
'Dorado389_2012_142_02_142_02_decim.nc',
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
import timing
cl = CANONLoader('stoqs_os2017', 'CANON - Off Season 2017',
description='CANON Off Season 2017 Experiment in Monterey Bay',
x3dTerrains={
'https://stoqs.mbari.org/x3d/Monterey25_10x/Monterey25_10x_scene.x3d': {
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation': '-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
},
'https://stoqs.mbari.org/x3d/Monterey25_1x/Monterey25_1x_src_scene.x3d': {
'name': 'Monterey25_1x',
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation': '-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '1',
},
},
grdTerrain=os.path.join(parentDir, 'Monterey25.grd')
)
# Set start and end dates for all loads from sources that contain data
# beyond the temporal bounds of the campaign
#
startdate = datetime.datetime(2017, 1, 1) # Fixed start
enddate = datetime.datetime(2017, 12, 31) # Fixed end. Extend "offseason" to end of year
import datetime # needed for glider data
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
import timing
cl = CANONLoader(
'stoqs_canon_september2016',
'CANON - September 2016',
description='CANON September 2016 Experiment in Monterey Bay',
x3dTerrains={
'https://stoqs.mbari.org/x3d/Monterey25_10x/Monterey25_10x_scene.x3d':
{
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation':
'-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
'speed': '0.1',
}
},
grdTerrain=os.path.join(parentDir, 'Monterey25.grd'))
# Set start and end dates for all loads from sources that contain data
# beyond the temporal bounds of the campaign
#
startdate = datetime.datetime(
2016, 7, 11) # Fixed start July 11 2016, deployment of nps34
enddate = datetime.datetime(
2016, 10,
import sys
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
# Assign input data sources
cl = CANONLoader(
'stoqs_june2011',
'CANON - June 2011',
description='Front detection Dorado and Tethys surveys in Monterey Bay',
x3dTerrains={
'http://dods.mbari.org/terrain/x3d/Monterey25_10x/Monterey25_10x_scene.x3d':
{
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation':
'-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
'speed': '.1',
}
},
grdTerrain=os.path.join(parentDir, 'Monterey25.grd'))
cl.dorado_base = 'http://dods.mbari.org/opendap/data/auvctd/surveys/2011/netcdf/'
cl.dorado_files = [
'Dorado389_2011_164_05_164_05_decim.nc',
'Dorado389_2011_165_00_165_00_decim.nc',
'Dorado389_2011_166_00_166_00_decim.nc',
'Dorado389_2011_171_01_171_01_decim.nc',
]
import sys
import datetime # needed for glider data
import time # for startdate, enddate args
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
project_dir = os.path.dirname(__file__)
# the next line makes it possible to find CANON
sys.path.insert(0, os.path.join(
os.path.dirname(__file__),
"../")) # this makes it possible to find CANON, one directory up
from CANON import CANONLoader
# building input data sources object
#cl = CANONLoader('stoqs_september2011', 'CANON - September 2011')
cl = CANONLoader('stoqs_september2013', 'CANON - September 2013')
# default location of thredds and dods data:
cl.tdsBase = 'http://odss.mbari.org/thredds/'
cl.dodsBase = cl.tdsBase + 'dodsC/'
######################################################################
# GLIDERS
######################################################################
# Set start and end dates for all glider loads
# startdate is 24hours from now
ts = time.time() - (12.2 * 60 * 60)
st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M')
t = time.strptime(st, "%Y-%m-%d %H:%M")
#t =time.strptime("2013-09-03 20:01", "%Y-%m-%d %H:%M")
startdate = t[:6]
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
import timing
cl = CANONLoader('stoqs_canon_may2018', 'CANON - May June 2018',
description='May June 2018 campaign observations in Monterey Bay',
x3dTerrains={
'http://dods.mbari.org/terrain/x3d/Monterey25_10x/Monterey25_10x_scene.x3d': {
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation': '-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
},
'http://stoqs.mbari.org/x3d/Monterey25_1x/Monterey25_1x_src_scene.x3d': {
'name': 'Monterey25_1x',
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation': '-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '1',
},
},
grdTerrain=os.path.join(parentDir, 'Monterey25.grd')
)
# Set start and end dates for all loads from sources that contain data
# beyond the temporal bounds of the campaign
#
startdate = datetime.datetime(2018, 5, 15) # Fixed start. May 15, 2018
enddate = datetime.datetime(2018, 6, 15) # Fixed end. June 15, 2018.
import csv
import urllib2
parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found
from CANON import CANONLoader
cl = CANONLoader(
'stoqs_september2014',
'CANON-ECOHAB - September 2014',
description='Fall 2014 Dye Release Experiment in Monterey Bay',
x3dTerrains={
'http://dods.mbari.org/terrain/x3d/Monterey25_10x/Monterey25_10x_scene.x3d':
{
'position': '-2822317.31255 -4438600.53640 3786150.85474',
'orientation': '0.89575 -0.31076 -0.31791 1.63772',
'centerOfRotation':
'-2711557.9403829873 -4331414.329506527 3801353.4691465236',
'VerticalExaggeration': '10',
'speed': '0.1',
}
},
grdTerrain=os.path.join(parentDir, 'Monterey25.grd'))
# Set start and end dates for all loads from sources that contain data
# beyond the temporal bounds of the campaign
startdate = datetime.datetime(2014, 9, 21) # Fixed start
enddate = datetime.datetime(2014, 10, 12) # Fixed end
# default location of thredds and dods data:
cl.tdsBase = 'http://odss.mbari.org/thredds/'
