def main(argv): openVDM = openvdm.OpenVDM() time.sleep(5) tasks = openVDM.getTasks() for task in tasks: openVDM.setIdle_task(task['taskID']) collectionSystemTransfers = openVDM.getCollectionSystemTransfers() for collectionSystemTransfer in collectionSystemTransfers: if not collectionSystemTransfer['status'] == '3': openVDM.setIdle_collectionSystemTransfer( collectionSystemTransfer['collectionSystemTransferID']) cruiseDataTransfers = openVDM.getCruiseDataTransfers() for cruiseDataTransfer in cruiseDataTransfers: if not cruiseDataTransfer['status'] == '3': openVDM.setIdle_cruiseDataTransfer( cruiseDataTransfer['cruiseDataTransferID']) requiredCruiseDataTransfers = openVDM.getRequiredCruiseDataTransfers() for requiredCruiseDataTransfer in requiredCruiseDataTransfers: if not requiredCruiseDataTransfer['status'] == '3': openVDM.setIdle_cruiseDataTransfer( requiredCruiseDataTransfer['cruiseDataTransferID']) openVDM.clearGearmanJobsFromDB()
def __init__(self, host_list=None): self.stop = False self.quit = False self.OVDM = openvdm.OpenVDM() self.cruiseID = '' self.taskID = '0' super(OVDMGearmanWorker, self).__init__(host_list=[self.OVDM.getGearmanServer()])
def __init__(self, host_list=None): self.stop = False self.quit = False self.OVDM = openvdm.OpenVDM() self.cruiseID = '' self.collectionSystemTransfer = {} self.files = {} super(OVDMGearmanWorker, self).__init__(host_list=[self.OVDM.getGearmanServer()])
def __init__(self): self.OVDM = openvdm.OpenVDM() self.cruiseID = '' self.systemStatus = '' self.startTime = time.gmtime(0) self.cruiseDataTransfer = {} self.shipboardDataWarehouseConfig = {} super(OVDMGearmanWorker, self).__init__(host_list=[self.OVDM.getGearmanServer()])
def __init__(self, host_list=None): self.stop = False self.quit = False self.OVDM = openvdm.OpenVDM() self.cruiseID = '' self.systemStatus = '' self.cruiseDataTransfer = {} self.shipboardDataWarehouseConfig = {} super(OVDMGearmanWorker, self).__init__(host_list=[self.OVDM.getGearmanServer()])
def main(argv): time.sleep(10) openVDM = openvdm.OpenVDM() parser = argparse.ArgumentParser( description='OpenVDM Data Transfer Scheduler') #parser.add_argument('--siteRoot', default=openVDM.getSiteRoot(), metavar='siteRoot', help='the base URL for this OpenVDM installation') parser.add_argument('--interval', default=openVDM.getTransferInterval(), metavar='interval', type=int, help='Delay in minutes') args = parser.parse_args() #print args.siteRoot #parsed_url = urlparse.urlparse(args.siteRoot) #print parsed_url #if not bool(parsed_url.scheme) or not bool(parsed_url.netloc): # print args.siteRoot + " is not a valid URL" # sys.exit(1) gm_client = gearman.GearmanClient([openVDM.getGearmanServer()]) while True: current_sec = 0 while True: t = time.gmtime() if current_sec < t.tm_sec: current_sec = t.tm_sec time.sleep(60 - t.tm_sec) else: break collectionSystemTransfers = openVDM.getCollectionSystemTransfers() for collectionSystemTransfer in collectionSystemTransfers: print 'Submitting collection system transfer job for: ' + collectionSystemTransfer[ 'longName'] gmData = {} gmData['collectionSystemTransfer'] = {} gmData['collectionSystemTransfer'][ 'collectionSystemTransferID'] = collectionSystemTransfer[ 'collectionSystemTransferID'] #print json.dumps(gmData, indent=2) completed_job_request = gm_client.submit_job( "runCollectionSystemTransfer", json.dumps(gmData), background=True) #resultsObj = json.loads(completed_job_request.result) time.sleep(2) cruiseDataTransfers = openVDM.getCruiseDataTransfers() for cruiseDataTransfer in cruiseDataTransfers: print 'Submitting cruise data transfer job for: ' + cruiseDataTransfer[ 'longName'] gmData = {} gmData['cruiseDataTransfer'] = {} gmData['cruiseDataTransfer'][ 'cruiseDataTransferID'] = cruiseDataTransfer[ 'cruiseDataTransferID'] #print json.dumps(gmData, indent=2) completed_job_request = gm_client.submit_job( "runCruiseDataTransfer", json.dumps(gmData), background=True) #resultsObj = json.loads(completed_job_request.result) time.sleep(2) requiredCruiseDataTransfers = openVDM.getRequiredCruiseDataTransfers() for requiredCruiseDataTransfer in requiredCruiseDataTransfers: if requiredCruiseDataTransfer['name'] == 'SSDW': print 'Submitting cruise data transfer job for: ' + requiredCruiseDataTransfer[ 'longName'] gmData = {} completed_job_request = gm_client.submit_job( "runShipToShoreTransfer", json.dumps(gmData), background=True) #resultsObj = json.loads(completed_job_request.result) time.sleep(2) delay = args.interval * 60 - len(collectionSystemTransfers) * 2 - len( cruiseDataTransfers) * 2 - 2 #print delay time.sleep(delay)
def main(argv): # Object that defines the GGA dashboardData datasets first by Collection System and # then by device. In this example only the SCS collections system is recording GGA # data. # # In this example there is a dashboardData plugin for the SCS collection system # that created geoJSON dashboardData files from two sensors, the POSMV and CNAV. # These datasets are defined within the "GPSSources" array. # # Each element within the "GPSSources" array includes the name of the device (used # in the final filename) and a regex string specifying where the dashboardData # are located within the top-level directory for SCS data within the dashboardData # directory specified within OpenVDM. In most cases the dashboardData files reside # in the exact same directory structure as the raw data files. # # i.e. Raw Files: /cruiseID/SCS/NAV/POSMV-GGA_*.Raw # DashboardData Files: /cruiseID/OpenVDM/DashboardData/SCS/NAV/POSMV-GGA_*.json # AllGPSSources = [{ "CollectionSystem": "SCS", "GPSSources": [{ "device": "POSMV", "regex": "NAV/POSMV-GGA_*.json" }, { "device": "CNAV", "regex": "NAV/CNAV-GGA_*.json" }] }] # Define the command-line structure parser = argparse.ArgumentParser( description='build cruise tracklines post-dashboard processing') parser.add_argument('-c', dest='cruiseID', metavar='cruiseID', help='the cruiseID to process') parser.add_argument( 'collectionSystem', help='json-formatted array of files to NOT include for processing') # Process the command-line argumants args = parser.parse_args() # build an OpenVDM object openVDM = openvdm.OpenVDM() # define the cruiseID as the current cruiseID cruiseID = openVDM.getCruiseID() # if a cruiseID was declared in the command-line aruments, redefine the cruiseID variable if not args.cruiseID is None: print "Setting CruiseID to " + args.cruiseID cruiseID = args.cruiseID # Retrieve the shipboard data warehouse configuration shipboardDataWarehouseConfig = openVDM.getShipboardDataWarehouseConfig() # Construct the full path to the cruise data directory cruiseDir = shipboardDataWarehouseConfig[ 'shipboardDataWarehouseBaseDir'] + '/' + cruiseID # Verify the cruise data directory exists if not os.path.isdir(cruiseDir): print "ERROR: Cruise Data Directory: '" + cruiseDir + "' not found!" return -1 # Retrieve the information for the collection system defined in the command-line argument collectionSystem = openVDM.getCollectionSystemTransferByName( args.collectionSystem) if not collectionSystem: print "ERROR: Collection System: '" + args.collectionSystem + "' not found!" return -1 #get the name of the Products directory productsDirectory = openVDM.getExtraDirectoryByName('Products') if not productsDirectory: print "ERROR: 'Products' directory not found!" return -1 else: # Verify the Products directory exists if not os.path.isdir(cruiseDir + '/' + productsDirectory['destDir']): print "ERROR: Products Directory: '" + cruiseDir + '/' + productsDirectory[ 'destDir'] + "' not found!" return -1 #get the name of the Dashboard Data directory dashboardDataDirectory = openVDM.getRequiredExtraDirectoryByName( 'Dashboard Data') if not dashboardDataDirectory: print "ERROR: 'Dashboard Data' directory not found!" return -1 else: # Verify the Dashboard Data directory for the specified collection system exists if not os.path.isdir(cruiseDir + '/' + dashboardDataDirectory['destDir'] + '/' + collectionSystem['destDir']): print "ERROR: Dashboard Data Directory for " + args.collectionSystem + ": '" + cruiseDir + '/' + dashboardDataDirectory[ 'destDir'] + '/' + collectionSystem['destDir'] + "' not found!" return -1 # Create the Tracklines directory within Products directory if it does not already exist. # If the directory needs to be created, also set the ownership permissions try: os.mkdir( cruiseDir + '/' + productsDirectory['destDir'] + '/Tracklines', 0755) os.chown( cruiseDir + '/' + productsDirectory['destDir'] + '/Tracklines', pwd.getpwnam( shipboardDataWarehouseConfig['shipboardDataWarehouseUsername'] ).pw_uid, grp.getgrnam( shipboardDataWarehouseConfig['shipboardDataWarehouseUsername'] ).gr_gid) except OSError: print "Tracklines directory already exists" # Loop through the AllGPSSources object for GPSSources in AllGPSSources: # If the collection system name matches the one in the command-line argrument if GPSSources['CollectionSystem'] == args.collectionSystem: #Build a geoJSON and kml cruisetrack for each GGA Device for GPSSource in GPSSources['GPSSources']: print "Processing " + GPSSource['device'] # Build the list of files coorsponding to the current device based on the regex provided files = glob.glob(cruiseDir + '/' + dashboardDataDirectory['destDir'] + '/' + collectionSystem['destDir'] + '/' + GPSSource['regex']) # Combind the geoJSON objects combineGeoJsonObj = combineGeoJsonFiles( files, cruiseID, GPSSource['device']) # If there was a problem, exit if not combineGeoJsonObj: return -1 # If the combine was successful and there is data if combineGeoJsonObj is not None: # Save the combined geoJSON object to file writeToFile( json.dumps(combineGeoJsonObj), cruiseDir + '/' + productsDirectory['destDir'] + '/' + 'Tracklines' + '/' + cruiseID + '_' + GPSSource['device'] + '_Trackline.json', shipboardDataWarehouseConfig[ 'shipboardDataWarehouseUsername']) # Convert the combined geoJSON object to kml and save to file writeToFile( convToKML(combineGeoJsonObj), cruiseDir + '/' + productsDirectory['destDir'] + '/' + 'Tracklines' + '/' + cruiseID + '_' + GPSSource['device'] + '_Trackline.kml', shipboardDataWarehouseConfig[ 'shipboardDataWarehouseUsername']) # No need to proceed to another collectionSystem break
def __init__(self, host_list=None): self.OVDM = openvdm.OpenVDM() self.jobPID = '' super(OVDMGearmanWorker, self).__init__(host_list=[self.OVDM.getGearmanServer()])
def main(argv): AllGPSSources = [{ "CollectionSystem": "SCS", "GPSSources": [{ "device": "POSMV", "regex": "NAV/POSMV-GGA_*.Raw" }, { "device": "CNAV", "regex": "NAV/CNAV-GGA_*.Raw" }] }, { "CollectionSystem": "USBL", "GPSSources": [{ "device": "POSMV", "regex": "USBL-GGA_*.Raw" }] }] openVDM = openvdm.OpenVDM() parser = argparse.ArgumentParser( description='r2rNavManager post-transfer processing script') parser.add_argument('-c', default=openVDM.getCruiseID(), metavar='cruiseID', help='the cruiseID to process') parser.add_argument( '-s', metavar='skip', help='json-formatted array of files to NOT include for processing') parser.add_argument( 'collectionSystem', help='json-formatted array of files to NOT include for processing') args = parser.parse_args() #get the warehouse config shipboardDataWarehouseConfig = openVDM.getShipboardDataWarehouseConfig() # construct the root of the cruise data directory cruiseDir = shipboardDataWarehouseConfig[ 'shipboardDataWarehouseBaseDir'] + '/' + openVDM.getCruiseID() #get the name of the SCS directory collectionSystem = openVDM.getCollectionSystemTransferByName( args.collectionSystem) if not collectionSystem: print "ERROR: Collection System: '" + args.collectionSystem + "' not found" return -1 #get the name of the r2r directory r2rDirectory = openVDM.getExtraDirectoryByName('r2r') if not r2rDirectory: print "ERROR: 'r2r' directory not found" return -1 #if NavManager directory in r2r directory does not exist, create it try: os.mkdir(cruiseDir + '/' + r2rDirectory['destDir'] + '/NavManager') except OSError: print "NavManager directory already exists" for GPSSources in AllGPSSources: if GPSSources['CollectionSystem'] == args.collectionSystem: #create a temp directory tmpdir = tempfile.mkdtemp() #foreach GGA Source Device for GPSSource in GPSSources['GPSSources']: print "Processing " + GPSSource['device'] #copy GGA Files to tmpdir files = glob.glob(cruiseDir + '/' + collectionSystem['destDir'] + '/' + GPSSource['regex']) #if there are no files, continue to the next GGA type if len(files) > 0: for file in files: #print file shutil.copy(file, tmpdir) else: continue #if device directory does not exists in r2r/NavManager directory, create it deviceDir = cruiseDir + '/' + r2rDirectory[ 'destDir'] + '/NavManager' + '/' + GPSSource['device'] try: os.mkdir(deviceDir) except OSError: print "NavManager/" + GPSSource[ 'device'] + " directory already exists" print "Running navcopy.php..." command = [ 'php', '/usr/local/bin/NavManager/bin/navcopy.php', '-f', 'nav2', '-d', tmpdir, '-o', deviceDir + '/bestres_raw.r2rnav' ] proc = subprocess.Popen(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE) out, err = proc.communicate() print "Running navinfo.php..." command = [ 'php', '/usr/local/bin/NavManager/bin/navinfo.php', '-i', deviceDir + '/bestres_raw.r2rnav', '-l', deviceDir + '/navinfo_report.txt' ] proc = subprocess.Popen(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE) out, err = proc.communicate() print "Running navqa.php..." command = [ 'php', '/usr/local/bin/NavManager/bin/navqa.php', '-i', deviceDir + '/bestres_raw.r2rnav', '-l', deviceDir + '/navqa_report.txt' ] proc = subprocess.Popen(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE) out, err = proc.communicate() print "Running navqc.php..." command = [ 'php', '/usr/local/bin/NavManager/bin/navqc.php', '-i', deviceDir + '/bestres_raw.r2rnav', '-o', deviceDir + '/bestres_qc.r2rnav', '-l', deviceDir + '/navqc_report.txt' ] proc = subprocess.Popen(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE) out, err = proc.communicate() #cleanup shutil.rmtree(tmpdir)