def main(argv):
openVDM = openvdm.OpenVDM()
time.sleep(5)
tasks = openVDM.getTasks()
for task in tasks:
openVDM.setIdle_task(task['taskID'])
collectionSystemTransfers = openVDM.getCollectionSystemTransfers()
for collectionSystemTransfer in collectionSystemTransfers:
if not collectionSystemTransfer['status'] == '3':
openVDM.setIdle_collectionSystemTransfer(
collectionSystemTransfer['collectionSystemTransferID'])
cruiseDataTransfers = openVDM.getCruiseDataTransfers()
for cruiseDataTransfer in cruiseDataTransfers:
if not cruiseDataTransfer['status'] == '3':
openVDM.setIdle_cruiseDataTransfer(
cruiseDataTransfer['cruiseDataTransferID'])
requiredCruiseDataTransfers = openVDM.getRequiredCruiseDataTransfers()
for requiredCruiseDataTransfer in requiredCruiseDataTransfers:
if not requiredCruiseDataTransfer['status'] == '3':
openVDM.setIdle_cruiseDataTransfer(
requiredCruiseDataTransfer['cruiseDataTransferID'])
openVDM.clearGearmanJobsFromDB()
def __init__(self, host_list=None):
self.stop = False
self.quit = False
self.OVDM = openvdm.OpenVDM()
self.cruiseID = ''
self.taskID = '0'
super(OVDMGearmanWorker, self).__init__(host_list=[self.OVDM.getGearmanServer()])
def __init__(self, host_list=None):
self.stop = False
self.quit = False
self.OVDM = openvdm.OpenVDM()
self.cruiseID = ''
self.collectionSystemTransfer = {}
self.files = {}
super(OVDMGearmanWorker, self).__init__(host_list=[self.OVDM.getGearmanServer()])
def __init__(self):
self.OVDM = openvdm.OpenVDM()
self.cruiseID = ''
self.systemStatus = ''
self.startTime = time.gmtime(0)
self.cruiseDataTransfer = {}
self.shipboardDataWarehouseConfig = {}
super(OVDMGearmanWorker,
self).__init__(host_list=[self.OVDM.getGearmanServer()])
def __init__(self, host_list=None):
self.stop = False
self.quit = False
self.OVDM = openvdm.OpenVDM()
self.cruiseID = ''
self.systemStatus = ''
self.cruiseDataTransfer = {}
self.shipboardDataWarehouseConfig = {}
super(OVDMGearmanWorker,
self).__init__(host_list=[self.OVDM.getGearmanServer()])
def main(argv):
time.sleep(10)
openVDM = openvdm.OpenVDM()
parser = argparse.ArgumentParser(
description='OpenVDM Data Transfer Scheduler')
#parser.add_argument('--siteRoot', default=openVDM.getSiteRoot(), metavar='siteRoot', help='the base URL for this OpenVDM installation')
parser.add_argument('--interval',
default=openVDM.getTransferInterval(),
metavar='interval',
type=int,
help='Delay in minutes')
args = parser.parse_args()
#print args.siteRoot
#parsed_url = urlparse.urlparse(args.siteRoot)
#print parsed_url
#if not bool(parsed_url.scheme) or not bool(parsed_url.netloc):
# print args.siteRoot + " is not a valid URL"
# sys.exit(1)
gm_client = gearman.GearmanClient([openVDM.getGearmanServer()])
while True:
current_sec = 0
while True:
t = time.gmtime()
if current_sec < t.tm_sec:
current_sec = t.tm_sec
time.sleep(60 - t.tm_sec)
else:
break
collectionSystemTransfers = openVDM.getCollectionSystemTransfers()
for collectionSystemTransfer in collectionSystemTransfers:
print 'Submitting collection system transfer job for: ' + collectionSystemTransfer[
'longName']
gmData = {}
gmData['collectionSystemTransfer'] = {}
gmData['collectionSystemTransfer'][
'collectionSystemTransferID'] = collectionSystemTransfer[
'collectionSystemTransferID']
#print json.dumps(gmData, indent=2)
completed_job_request = gm_client.submit_job(
"runCollectionSystemTransfer",
json.dumps(gmData),
background=True)
#resultsObj = json.loads(completed_job_request.result)
time.sleep(2)
cruiseDataTransfers = openVDM.getCruiseDataTransfers()
for cruiseDataTransfer in cruiseDataTransfers:
print 'Submitting cruise data transfer job for: ' + cruiseDataTransfer[
'longName']
gmData = {}
gmData['cruiseDataTransfer'] = {}
gmData['cruiseDataTransfer'][
'cruiseDataTransferID'] = cruiseDataTransfer[
'cruiseDataTransferID']
#print json.dumps(gmData, indent=2)
completed_job_request = gm_client.submit_job(
"runCruiseDataTransfer", json.dumps(gmData), background=True)
#resultsObj = json.loads(completed_job_request.result)
time.sleep(2)
requiredCruiseDataTransfers = openVDM.getRequiredCruiseDataTransfers()
for requiredCruiseDataTransfer in requiredCruiseDataTransfers:
if requiredCruiseDataTransfer['name'] == 'SSDW':
print 'Submitting cruise data transfer job for: ' + requiredCruiseDataTransfer[
'longName']
gmData = {}
completed_job_request = gm_client.submit_job(
"runShipToShoreTransfer",
json.dumps(gmData),
background=True)
#resultsObj = json.loads(completed_job_request.result)
time.sleep(2)
delay = args.interval * 60 - len(collectionSystemTransfers) * 2 - len(
cruiseDataTransfers) * 2 - 2
#print delay
time.sleep(delay)
def main(argv):
# Object that defines the GGA dashboardData datasets first by Collection System and
# then by device. In this example only the SCS collections system is recording GGA
# data.
#
# In this example there is a dashboardData plugin for the SCS collection system
# that created geoJSON dashboardData files from two sensors, the POSMV and CNAV.
# These datasets are defined within the "GPSSources" array.
#
# Each element within the "GPSSources" array includes the name of the device (used
# in the final filename) and a regex string specifying where the dashboardData
# are located within the top-level directory for SCS data within the dashboardData
# directory specified within OpenVDM. In most cases the dashboardData files reside
# in the exact same directory structure as the raw data files.
#
# i.e. Raw Files: /cruiseID/SCS/NAV/POSMV-GGA_*.Raw
# DashboardData Files: /cruiseID/OpenVDM/DashboardData/SCS/NAV/POSMV-GGA_*.json
#
AllGPSSources = [{
"CollectionSystem":
"SCS",
"GPSSources": [{
"device": "POSMV",
"regex": "NAV/POSMV-GGA_*.json"
}, {
"device": "CNAV",
"regex": "NAV/CNAV-GGA_*.json"
}]
}]
# Define the command-line structure
parser = argparse.ArgumentParser(
description='build cruise tracklines post-dashboard processing')
parser.add_argument('-c',
dest='cruiseID',
metavar='cruiseID',
help='the cruiseID to process')
parser.add_argument(
'collectionSystem',
help='json-formatted array of files to NOT include for processing')
# Process the command-line argumants
args = parser.parse_args()
# build an OpenVDM object
openVDM = openvdm.OpenVDM()
# define the cruiseID as the current cruiseID
cruiseID = openVDM.getCruiseID()
# if a cruiseID was declared in the command-line aruments, redefine the cruiseID variable
if not args.cruiseID is None:
print "Setting CruiseID to " + args.cruiseID
cruiseID = args.cruiseID
# Retrieve the shipboard data warehouse configuration
shipboardDataWarehouseConfig = openVDM.getShipboardDataWarehouseConfig()
# Construct the full path to the cruise data directory
cruiseDir = shipboardDataWarehouseConfig[
'shipboardDataWarehouseBaseDir'] + '/' + cruiseID
# Verify the cruise data directory exists
if not os.path.isdir(cruiseDir):
print "ERROR: Cruise Data Directory: '" + cruiseDir + "' not found!"
return -1
# Retrieve the information for the collection system defined in the command-line argument
collectionSystem = openVDM.getCollectionSystemTransferByName(
args.collectionSystem)
if not collectionSystem:
print "ERROR: Collection System: '" + args.collectionSystem + "' not found!"
return -1
#get the name of the Products directory
productsDirectory = openVDM.getExtraDirectoryByName('Products')
if not productsDirectory:
print "ERROR: 'Products' directory not found!"
return -1
else:
# Verify the Products directory exists
if not os.path.isdir(cruiseDir + '/' + productsDirectory['destDir']):
print "ERROR: Products Directory: '" + cruiseDir + '/' + productsDirectory[
'destDir'] + "' not found!"
return -1
#get the name of the Dashboard Data directory
dashboardDataDirectory = openVDM.getRequiredExtraDirectoryByName(
'Dashboard Data')
if not dashboardDataDirectory:
print "ERROR: 'Dashboard Data' directory not found!"
return -1
else:
# Verify the Dashboard Data directory for the specified collection system exists
if not os.path.isdir(cruiseDir + '/' +
dashboardDataDirectory['destDir'] + '/' +
collectionSystem['destDir']):
print "ERROR: Dashboard Data Directory for " + args.collectionSystem + ": '" + cruiseDir + '/' + dashboardDataDirectory[
'destDir'] + '/' + collectionSystem['destDir'] + "' not found!"
return -1
# Create the Tracklines directory within Products directory if it does not already exist.
# If the directory needs to be created, also set the ownership permissions
try:
os.mkdir(
cruiseDir + '/' + productsDirectory['destDir'] + '/Tracklines',
0755)
os.chown(
cruiseDir + '/' + productsDirectory['destDir'] + '/Tracklines',
pwd.getpwnam(
shipboardDataWarehouseConfig['shipboardDataWarehouseUsername']
).pw_uid,
grp.getgrnam(
shipboardDataWarehouseConfig['shipboardDataWarehouseUsername']
).gr_gid)
except OSError:
print "Tracklines directory already exists"
# Loop through the AllGPSSources object
for GPSSources in AllGPSSources:
# If the collection system name matches the one in the command-line argrument
if GPSSources['CollectionSystem'] == args.collectionSystem:
#Build a geoJSON and kml cruisetrack for each GGA Device
for GPSSource in GPSSources['GPSSources']:
print "Processing " + GPSSource['device']
# Build the list of files coorsponding to the current device based on the regex provided
files = glob.glob(cruiseDir + '/' +
dashboardDataDirectory['destDir'] + '/' +
collectionSystem['destDir'] + '/' +
GPSSource['regex'])
# Combind the geoJSON objects
combineGeoJsonObj = combineGeoJsonFiles(
files, cruiseID, GPSSource['device'])
# If there was a problem, exit
if not combineGeoJsonObj:
return -1
# If the combine was successful and there is data
if combineGeoJsonObj is not None:
# Save the combined geoJSON object to file
writeToFile(
json.dumps(combineGeoJsonObj),
cruiseDir + '/' + productsDirectory['destDir'] + '/' +
'Tracklines' + '/' + cruiseID + '_' +
GPSSource['device'] + '_Trackline.json',
shipboardDataWarehouseConfig[
'shipboardDataWarehouseUsername'])
# Convert the combined geoJSON object to kml and save to file
writeToFile(
convToKML(combineGeoJsonObj),
cruiseDir + '/' + productsDirectory['destDir'] + '/' +
'Tracklines' + '/' + cruiseID + '_' +
GPSSource['device'] + '_Trackline.kml',
shipboardDataWarehouseConfig[
'shipboardDataWarehouseUsername'])
# No need to proceed to another collectionSystem
break
def __init__(self, host_list=None):
self.OVDM = openvdm.OpenVDM()
self.jobPID = ''
super(OVDMGearmanWorker,
self).__init__(host_list=[self.OVDM.getGearmanServer()])
def main(argv):
AllGPSSources = [{
"CollectionSystem":
"SCS",
"GPSSources": [{
"device": "POSMV",
"regex": "NAV/POSMV-GGA_*.Raw"
}, {
"device": "CNAV",
"regex": "NAV/CNAV-GGA_*.Raw"
}]
}, {
"CollectionSystem":
"USBL",
"GPSSources": [{
"device": "POSMV",
"regex": "USBL-GGA_*.Raw"
}]
}]
openVDM = openvdm.OpenVDM()
parser = argparse.ArgumentParser(
description='r2rNavManager post-transfer processing script')
parser.add_argument('-c',
default=openVDM.getCruiseID(),
metavar='cruiseID',
help='the cruiseID to process')
parser.add_argument(
'-s',
metavar='skip',
help='json-formatted array of files to NOT include for processing')
parser.add_argument(
'collectionSystem',
help='json-formatted array of files to NOT include for processing')
args = parser.parse_args()
#get the warehouse config
shipboardDataWarehouseConfig = openVDM.getShipboardDataWarehouseConfig()
# construct the root of the cruise data directory
cruiseDir = shipboardDataWarehouseConfig[
'shipboardDataWarehouseBaseDir'] + '/' + openVDM.getCruiseID()
#get the name of the SCS directory
collectionSystem = openVDM.getCollectionSystemTransferByName(
args.collectionSystem)
if not collectionSystem:
print "ERROR: Collection System: '" + args.collectionSystem + "' not found"
return -1
#get the name of the r2r directory
r2rDirectory = openVDM.getExtraDirectoryByName('r2r')
if not r2rDirectory:
print "ERROR: 'r2r' directory not found"
return -1
#if NavManager directory in r2r directory does not exist, create it
try:
os.mkdir(cruiseDir + '/' + r2rDirectory['destDir'] + '/NavManager')
except OSError:
print "NavManager directory already exists"
for GPSSources in AllGPSSources:
if GPSSources['CollectionSystem'] == args.collectionSystem:
#create a temp directory
tmpdir = tempfile.mkdtemp()
#foreach GGA Source Device
for GPSSource in GPSSources['GPSSources']:
print "Processing " + GPSSource['device']
#copy GGA Files to tmpdir
files = glob.glob(cruiseDir + '/' +
collectionSystem['destDir'] + '/' +
GPSSource['regex'])
#if there are no files, continue to the next GGA type
if len(files) > 0:
for file in files:
#print file
shutil.copy(file, tmpdir)
else:
continue
#if device directory does not exists in r2r/NavManager directory, create it
deviceDir = cruiseDir + '/' + r2rDirectory[
'destDir'] + '/NavManager' + '/' + GPSSource['device']
try:
os.mkdir(deviceDir)
except OSError:
print "NavManager/" + GPSSource[
'device'] + " directory already exists"
print "Running navcopy.php..."
command = [
'php', '/usr/local/bin/NavManager/bin/navcopy.php', '-f',
'nav2', '-d', tmpdir, '-o',
deviceDir + '/bestres_raw.r2rnav'
]
proc = subprocess.Popen(command,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
out, err = proc.communicate()
print "Running navinfo.php..."
command = [
'php', '/usr/local/bin/NavManager/bin/navinfo.php', '-i',
deviceDir + '/bestres_raw.r2rnav', '-l',
deviceDir + '/navinfo_report.txt'
]
proc = subprocess.Popen(command,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
out, err = proc.communicate()
print "Running navqa.php..."
command = [
'php', '/usr/local/bin/NavManager/bin/navqa.php', '-i',
deviceDir + '/bestres_raw.r2rnav', '-l',
deviceDir + '/navqa_report.txt'
]
proc = subprocess.Popen(command,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
out, err = proc.communicate()
print "Running navqc.php..."
command = [
'php', '/usr/local/bin/NavManager/bin/navqc.php', '-i',
deviceDir + '/bestres_raw.r2rnav', '-o',
deviceDir + '/bestres_qc.r2rnav', '-l',
deviceDir + '/navqc_report.txt'
]
proc = subprocess.Popen(command,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
out, err = proc.communicate()
#cleanup
shutil.rmtree(tmpdir)
