def task_set_lowering_data_directory_permissions(gearman_worker, gearman_job): """ Set the permissions for the specified lowering ID """ job_results = {'parts': []} payload_obj = json.loads(gearman_job.data) logging.debug("Payload: %s", json.dumps(payload_obj, indent=2)) gearman_worker.send_job_status(gearman_job, 5, 10) if os.path.isdir(gearman_worker.lowering_dir): logging.info("Clear read permissions") set_owner_group_permissions( gearman_worker. shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], gearman_worker.lowering_dir) job_results['parts'].append({ "partName": "Set Directory Permissions for current lowering", "result": "Pass" }) job_results['parts'].append({ "partName": "Set LoweringData Directory Permissions", "result": "Pass" }) gearman_worker.send_job_status(gearman_job, 10, 10) return json.dumps(job_results)
def task_set_cruise_data_directory_permissions(gearman_worker, gearman_job): """ Set the permissions for the specified cruise ID """ job_results = {'parts': []} payload_obj = json.loads(gearman_job.data) logging.debug("Payload: %s", json.dumps(payload_obj, indent=2)) gearman_worker.send_job_status(gearman_job, 5, 10) if gearman_worker.ovdm.show_only_current_cruise_dir() is True: logging.info( "Clear read permissions for all directories within CruiseData") lockdown_directory( gearman_worker. shipboard_data_warehouse_config['shipboardDataWarehouseBaseDir'], gearman_worker.cruise_dir) job_results['parts'].append({ "partName": "Clear CruiseData Directory Read Permissions", "result": "Pass" }) gearman_worker.send_job_status(gearman_job, 8, 10) if os.path.isdir(gearman_worker.cruise_dir): logging.info( "Set ownership and read/write permissions for current cruise directory within CruiseData" ) set_owner_group_permissions( gearman_worker. shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], gearman_worker.cruise_dir) job_results['parts'].append({ "partName": "Set Directory Permissions for current cruise", "result": "Pass" }) job_results['parts'].append({ "partName": "Set CruiseData Directory Permissions", "result": "Pass" }) gearman_worker.send_job_status(gearman_job, 10, 10) return json.dumps(job_results)
def task_setup_new_lowering(gearman_worker, gearman_job): """ Setup a new lowering """ job_results = {'parts':[]} payload_obj = json.loads(gearman_job.data) logging.debug("Payload: %s", json.dumps(payload_obj, indent=2)) lowering_config_filepath = os.path.join(gearman_worker.lowering_dir, DEFAULT_LOWERING_CONFIG_FN) gearman_worker.send_job_status(gearman_job, 1, 10) gm_client = python3_gearman.GearmanClient([gearman_worker.ovdm.get_gearman_server()]) logging.info("Creating lowering data directory") completed_job_request = gm_client.submit_job("createLoweringDirectory", gearman_job.data) result_obj = json.loads(completed_job_request.result) if result_obj['parts'][-1]['result'] == "Pass": # Final Verdict job_results['parts'].append({"partName": "Create lowering data directory structure", "result": "Pass"}) else: logging.error("Failed to create lowering data directory") job_results['parts'].append({"partName": "Create lowering data directory structure", "result": "Fail", "reason": result_obj['parts'][-1]['reason']}) return json.dumps(job_results) gearman_worker.send_job_status(gearman_job, 5, 10) #build lowering Config file logging.info("Exporting Lowering Configuration") output_results = export_lowering_config(gearman_worker, lowering_config_filepath) if output_results['verdict']: job_results['parts'].append({"partName": "Export lowering config data to file", "result": "Pass"}) else: job_results['parts'].append({"partName": "Export lowering config data to file", "result": "Fail", "reason": output_results['reason']}) return json.dumps(job_results) output_results = set_owner_group_permissions(gearman_worker.shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], lowering_config_filepath) if not output_results['verdict']: job_results['parts'].append({"partName": "Set lowering config file ownership/permissions", "result": "Fail", "reason": output_results['reason']}) return json.dumps(job_results) gearman_worker.send_job_status(gearman_job, 9, 10) logging.info("Updating Lowering Size") lowering_size_proc = subprocess.run(['du','-sb', gearman_worker.lowering_dir], capture_output=True, text=True, check=False) if lowering_size_proc.returncode == 0: logging.info("Cruise Size: %s", lowering_size_proc.stdout.split()[0]) gearman_worker.ovdm.set_lowering_size(lowering_size_proc.stdout.split()[0]) else: gearman_worker.ovdm.set_lowering_size("0") gearman_worker.send_job_status(gearman_job, 10, 10) return json.dumps(job_results)
def task_export_ovdm_config(gearman_worker, gearman_job): """ Export the OpenVDM configuration to file """ job_results = {'parts':[]} ovdm_config_file_path = os.path.join(gearman_worker.cruise_dir, DEFAULT_CRUISE_CONFIG_FN) gearman_worker.send_job_status(gearman_job, 1, 10) logging.info("Verifying cruise directory exists") if os.path.exists(gearman_worker.cruise_dir): job_results['parts'].append({"partName": "Verify cruise directory exists", "result": "Pass"}) else: job_results['parts'].append({"partName": "Verify cruise directory exists", "result": "Fail", "reason": "Unable to locate the cruise directory: " + gearman_worker.cruise_dir}) return json.dumps(job_results) gearman_worker.send_job_status(gearman_job, 3, 10) #build OpenVDM Config file logging.info("Exporting OpenVDM Configuration") output_results = export_ovdm_config(gearman_worker, ovdm_config_file_path) if output_results['verdict']: job_results['parts'].append({"partName": "Export OpenVDM config data to file", "result": "Pass"}) else: job_results['parts'].append({"partName": "Export OpenVDM config data to file", "result": "Fail", "reason": output_results['reason']}) return json.dumps(job_results) gearman_worker.send_job_status(gearman_job, 6, 10) logging.info("Setting file ownership/permissions") output_results = set_owner_group_permissions(gearman_worker.shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], ovdm_config_file_path) if output_results['verdict']: job_results['parts'].append({"partName": "Set file ownership/permissions", "result": "Pass"}) else: job_results['parts'].append({"partName": "Set file ownership/permissions", "result": "Fail", "reason": output_results['reason']}) return json.dumps(job_results) gearman_worker.send_job_status(gearman_job, 10, 10) return json.dumps(job_results)
def task_create_lowering_directory(gearman_worker, gearman_job): """ Setup the lowering directory for the specified lowering ID """ job_results = {'parts': []} payload_obj = json.loads(gearman_job.data) logging.debug("Payload: %s", json.dumps(payload_obj, indent=2)) gearman_worker.send_job_status(gearman_job, 1, 10) cruise_dir = os.path.join( gearman_worker. shipboard_data_warehouse_config['shipboardDataWarehouseBaseDir'], gearman_worker.cruise_id) lowering_data_base_dir = os.path.join( cruise_dir, gearman_worker.shipboard_data_warehouse_config['loweringDataBaseDir']) if os.path.exists(cruise_dir): job_results['parts'].append({ "partName": "Verify Cruise Directory exists", "result": "Pass" }) else: logging.error("Failed to find cruise directory: %s", cruise_dir) job_results['parts'].append({ "partName": "Verify Cruise Directory exists", "result": "Fail", "reason": "Unable to find cruise directory: " + cruise_dir }) return json.dumps(job_results) if os.path.exists(lowering_data_base_dir): job_results['parts'].append({ "partName": "Verify Lowering Data Directory exists", "result": "Pass" }) else: logging.error("Lowering Data Directory doesn not exist: %s", lowering_data_base_dir) job_results['parts'].append({ "partName": "Verify Lowering Data Directory exists", "result": "Fail", "reason": "Unable to find lowering data base directory: " + lowering_data_base_dir }) return json.dumps(job_results) if not os.path.exists(gearman_worker.lowering_dir): job_results['parts'].append({ "partName": "Verify Lowering Directory does not exists", "result": "Pass" }) else: logging.error("Lowering directory already exists: %s", gearman_worker.lowering_dir) job_results['parts'].append({ "partName": "Verify Lowering Directory does not exists", "result": "Fail", "reason": "Lowering directory " + gearman_worker.lowering_dir + " already exists" }) return json.dumps(job_results) gearman_worker.send_job_status(gearman_job, 2, 10) directorylist = build_directorylist(gearman_worker) logging.debug("Directory List: %s", json.dumps(directorylist, indent=2)) if len(directorylist) > 0: job_results['parts'].append({ "partName": "Build Directory List", "result": "Pass" }) else: logging.warning("Directory list is empty") job_results['parts'].append({ "partName": "Build Directory List", "result": "Fail", "reason": "Empty list of directories to create" }) return json.dumps(job_results) gearman_worker.send_job_status(gearman_job, 5, 10) output_results = create_directories(directorylist) if output_results['verdict']: job_results['parts'].append({ "partName": "Create Directories", "result": "Pass" }) else: logging.error( "Failed to create any/all of the lowering data directory structure" ) job_results['parts'].append({ "partName": "Create Directories", "result": "Fail", "reason": output_results['reason'] }) gearman_worker.send_job_status(gearman_job, 8, 10) output_results = set_owner_group_permissions( gearman_worker. shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], gearman_worker.lowering_dir) if output_results['verdict']: job_results['parts'].append({ "partName": "Set cruise directory ownership/permissions", "result": "Pass" }) else: job_results['parts'].append({ "partName": "Set cruise directory ownership/permissions", "result": "Fail", "reason": output_results['reason'] }) return json.dumps(job_results) gearman_worker.send_job_status(gearman_job, 10, 10) return json.dumps(job_results)
def transfer_local_dest_dir(gearman_worker, gearman_job): # pylint: disable=too-many-locals,too-many-statements """ Copy cruise data to a local directory """ logging.debug("Transfer to Local Directory") cruise_dir = os.path.join( gearman_worker. shipboard_data_warehouse_config['shipboardDataWarehouseBaseDir'], gearman_worker.cruise_id) dest_dir = gearman_worker.cruise_data_transfer['destDir'].rstrip('/') logging.debug('Destination Dir: %s', dest_dir) logging.debug("Building file list") files = build_filelist(gearman_worker, cruise_dir) # Create temp directory tmpdir = tempfile.mkdtemp() rsync_exclude_list_filepath = os.path.join(tmpdir, 'rsyncExcludeList.txt') try: with open(rsync_exclude_list_filepath, 'w') as rsync_excludelist_file: rsync_excludelist_file.write('\n'.join(files['exclude'])) except IOError: logging.error("Error Saving temporary rsync filelist file") # Cleanup shutil.rmtree(tmpdir) return False file_index = 0 file_count = 1 # avoids divide by 0 error command = [ 'rsync', '-trimnv', '--stats', '--exclude-from=' + rsync_exclude_list_filepath, cruise_dir, dest_dir ] logging.debug('File count Command: %s', ' '.join(command)) proc = subprocess.run(command, capture_output=True, text=True, check=False) for line in proc.stdout: if line.startswith('Number of regular files transferred:'): file_count = int(line.split(':')[1]) bandwidth_imit = '--bwlimit=' + gearman_worker.cruise_data_transfer[ 'bandwidthLimit'] if gearman_worker.cruise_data_transfer[ 'bandwidthLimit'] != '0' else '--bwlimit=20000000' # 20GB/s a.k.a. stupid big command = [ 'rsync', '-trimv', bandwidth_imit, '--exclude-from=' + rsync_exclude_list_filepath, cruise_dir, dest_dir ] logging.debug('Transfer Command: %s', ' '.join(command)) proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) while True: line = proc.stdout.readline().rstrip('\n') err_line = proc.stderr.readline().rstrip('\n') if err_line: logging.warning("Err Line: %s", err_line) if line: logging.debug("Line: %s", line) if proc.poll() is not None: break # if not line: # continue if line.startswith('>f+++++++++'): filename = line.split(' ', 1)[1] files['new'].append(filename) gearman_worker.send_job_status( gearman_job, int(20 + 70 * float(file_index) / float(file_count)), 100) file_index += 1 elif line.startswith('>f.'): filename = line.split(' ', 1)[1] files['updated'].append(filename) gearman_worker.send_job_status( gearman_job, int(20 + 70 * float(file_index) / float(file_count)), 100) file_index += 1 if gearman_worker.stop: logging.debug("Stopping") proc.terminate() break # files['new'] = [os.path.join('/', gearman_worker.cruise_id, filename) for filename in files['new']] # files['updated'] = [os.path.join('/', gearman_worker.cruise_id, filename) for filename in files['updated']] logging.info("Setting file permissions") output_results = set_owner_group_permissions( gearman_worker. shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], os.path.join(dest_dir, gearman_worker.cruise_id)) # Cleanup shutil.rmtree(tmpdir) if not output_results['verdict']: logging.error( "Error setting ownership/permissions for cruise data at destination: %s", os.path.join(dest_dir, gearman_worker.cruise_id)) return output_results return {'verdict': True, 'files': files}
def task_rebuild_data_dashboard(gearman_worker, gearman_job): # pylint: disable=too-many-locals,too-many-branches,too-many-statements """ Rebuild the existing dashboard files """ job_results = {'parts': [], 'files': {'new': [], 'updated': []}} payload_obj = json.loads(gearman_job.data) logging.debug('Payload: %s', json.dumps(payload_obj, indent=2)) if os.path.exists(gearman_worker.data_dashboard_dir): job_results['parts'].append({ "partName": "Verify Data Dashboard Directory exists", "result": "Pass" }) else: logging.error("Data dashboard directory not found: %s", gearman_worker.data_dashboard_dir) job_results['parts'].append({ "partName": "Verify Data Dashboard Directory exists", "result": "Fail", "reason": "Unable to locate the data dashboard directory: " + gearman_worker.data_dashboard_dir }) return json.dumps(job_results) collection_system_transfers = gearman_worker.ovdm.get_active_collection_system_transfers( ) gearman_worker.send_job_status(gearman_job, 1, 100) new_manifest_entries = [] collection_system_transfer_count = len(collection_system_transfers) collection_system_transfer_index = 0 for collection_system_transfer in collection_system_transfers: # pylint: disable=too-many-nested-blocks logging.info('Processing data from: %s', collection_system_transfer['name']) processing_script_filename = os.path.join( gearman_worker.ovdm.get_plugin_dir(), collection_system_transfer['name'].lower() + gearman_worker.ovdm.get_plugin_suffix()) logging.debug("Processing Script Filename: %s", processing_script_filename) if not os.path.isfile(processing_script_filename): logging.warning( "Processing script for collection system %s not found, moving on.", collection_system_transfer['name']) gearman_worker.send_job_status( gearman_job, int(10 + (80 * float(collection_system_transfer_index) / float(collection_system_transfer_count))), 100) collection_system_transfer_index += 1 continue # collection_system_transferOutputDir = os.path.join(gearman_worker.data_dashboard_dir, collection_system_transfer['destDir']) #build filelist filelist = [] if collection_system_transfer['cruiseOrLowering'] == "0": collection_system_transfer_input_dir = os.path.join( gearman_worker.cruise_dir, collection_system_transfer['destDir']) filelist.extend( build_filelist(collection_system_transfer_input_dir)) filelist = [ os.path.join(collection_system_transfer['destDir'], filename) for filename in filelist ] else: lowerings = gearman_worker.ovdm.get_lowerings() lowering_base_dir = gearman_worker.shipboard_data_warehouse_config[ 'loweringDataBaseDir'] for lowering in lowerings: collection_system_transfer_input_dir = os.path.join( gearman_worker.cruise_dir, lowering_base_dir, lowering, collection_system_transfer['destDir']) lowering_filelist = build_filelist( collection_system_transfer_input_dir) filelist.extend([ os.path.join(lowering_base_dir, lowering, collection_system_transfer['destDir'], filename) for filename in lowering_filelist ]) logging.debug("FileList: %s", json.dumps(filelist, indent=2)) file_count = len(filelist) file_index = 0 logging.info("%s file(s) to process", file_count) for filename in filelist: if gearman_worker.stop: break logging.info("Processing file: %s", filename) json_filename = os.path.splitext(filename)[0] + '.json' logging.debug("jsonFileName: %s", json_filename) raw_filepath = os.path.join(gearman_worker.cruise_dir, filename) logging.debug("rawFilePath: %s", raw_filepath) json_filepath = os.path.join(gearman_worker.data_dashboard_dir, json_filename) logging.debug("jsonFilePath: %s", json_filepath) if os.stat(raw_filepath).st_size == 0: logging.warning("File %s is empty", filename) continue command = [ PYTHON_BINARY, processing_script_filename, '--dataType', raw_filepath ] logging.debug("Get Datatype Command: %s", ' '.join(command)) datatype_proc = subprocess.run(command, capture_output=True, text=True, check=False) if datatype_proc.stdout: dd_type = datatype_proc.stdout.rstrip('\n') logging.debug("Found to be type: %s", dd_type) command = [ PYTHON_BINARY, processing_script_filename, raw_filepath ] logging.debug("Processing Command: %s", ' '.join(command)) data_proc = subprocess.run(command, capture_output=True, text=True, check=False) if data_proc.stdout: try: logging.debug("Parsing output") # logging.debug(data_proc.stdout) out_obj = json.loads(data_proc.stdout) except Exception as err: logging.error(str(err)) error_title = 'Error parsing output' error_body = 'Invalid JSON output recieved from processing. Command: ' + ' '.join( command) logging.error("%s: %s", error_title, error_body) gearman_worker.ovdm.send_msg(error_title, error_body) job_results['parts'].append({ "partName": "Parsing JSON output " + filename, "result": "Fail", "reason": error_title + ':' + error_body }) else: if out_obj is None: error_title = 'Error processing file' error_body = 'No JSON output recieved from file. Processing Command: ' + ' '.join( command) logging.error("%s: %s", error_title, error_body) gearman_worker.ovdm.send_msg( error_title, error_body) job_results['parts'].append({ "partName": "Parsing JSON output from file " + filename, "result": "Fail", "reason": error_title + ': ' + error_body }) if data_proc.stderr: logging.error('err: %s', data_proc.stderr) elif 'error' in out_obj: error_title = 'Error processing file' error_body = out_obj['error'] logging.error("%s: %s", error_title, error_body) gearman_worker.ovdm.send_msg( error_title, error_body) job_results['parts'].append({ "partName": "Processing Datafile " + filename, "result": "Fail", "reason": error_title + ':' + error_body }) else: #job_results['parts'].append({"partName": "Processing Datafile " + filename, "result": "Pass"}) output_results = output_json_data_to_file( json_filepath, out_obj) if output_results['verdict']: job_results['parts'].append({ "partName": "Writing DashboardData file: " + filename, "result": "Pass" }) else: error_title = 'Error writing file' error_body = "Error Writing DashboardData file: " + filename logging.error("%s: %s", error_title, error_body) gearman_worker.ovdm.send_msg( error_title, error_body) job_results['parts'].append({ "partName": "Writing Dashboard file: " + filename, "result": "Fail", "reason": output_results['verdict'] }) new_manifest_entries.append({ "type": dd_type, "dd_json": json_filepath.replace( gearman_worker. shipboard_data_warehouse_config[ 'shipboardDataWarehouseBaseDir'] + '/', ''), "raw_data": raw_filepath.replace( gearman_worker. shipboard_data_warehouse_config[ 'shipboardDataWarehouseBaseDir'] + '/', '') }) else: error_title = 'Error processing file' error_body = 'No JSON output recieved from file. Processing Command: ' + ' '.join( command) logging.error("%s: %s", error_title, error_body) gearman_worker.ovdm.send_msg(error_title, error_body) job_results['parts'].append({ "partName": "Parsing JSON output from file " + filename, "result": "Fail", "reason": error_title + ': ' + error_body }) if data_proc.stderr: logging.error('err: %s', data_proc.stderr) else: logging.warning("File is of unknown datatype, moving on") if datatype_proc.stderr: logging.error('err: %s', datatype_proc.stderr) gearman_worker.send_job_status( gearman_job, int(10 + 70 * float(file_index) / float(file_count)), 100) file_index += 1 collection_system_transfer_index += 1 gearman_worker.send_job_status(gearman_job, 90, 100) logging.info("Update Dashboard Manifest file") output_results = output_json_data_to_file( gearman_worker.data_dashboard_manifest_file_path, new_manifest_entries) if output_results['verdict']: job_results['parts'].append({ "partName": "Updating manifest file", "result": "Pass" }) else: logging.error("Error updating manifest file %s", gearman_worker.data_dashboard_manifest_file_path) job_results['parts'].append({ "partName": "Updating manifest file", "result": "Fail", "reason": output_results['reason'] }) return json.dumps(job_results) gearman_worker.send_job_status(gearman_job, 95, 100) logging.info("Setting file ownership/permissions") output_results = set_owner_group_permissions( gearman_worker. shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], gearman_worker.data_dashboard_dir) if output_results['verdict']: job_results['parts'].append({ "partName": "Setting file/directory ownership", "result": "Pass" }) else: logging.error("Error Setting file/directory ownership/permissions") job_results['parts'].append({ "partName": "Setting file/directory ownership", "result": "Fail", "reason": output_results['reason'] }) return json.dumps(job_results) gearman_worker.send_job_status(gearman_job, 99, 100) data_dashboard_dest_dir = gearman_worker.ovdm.get_required_extra_directory_by_name( 'Dashboard_Data')['destDir'] job_results['files']['updated'] = [ os.path.join(data_dashboard_dest_dir, filepath) for filepath in build_filelist(gearman_worker.data_dashboard_dir) ] # might need to remove cruise_dir from begining of filepaths gearman_worker.send_job_status(gearman_job, 10, 10) return json.dumps(job_results)
def task_update_data_dashboard(gearman_worker, gearman_job): # pylint: disable=too-many-locals,too-many-branches,too-many-statements """ Update the existing dashboard files with new/updated raw data """ job_results = {'parts': [], 'files': {'new': [], 'updated': []}} payload_obj = json.loads(gearman_job.data) logging.debug('Payload: %s', json.dumps(payload_obj, indent=2)) gearman_worker.send_job_status(gearman_job, 5, 100) logging.info('Collection System Transfer: %s', gearman_worker.collection_system_transfer['name']) new_manifest_entries = [] remove_manifest_entries = [] #check for processing file processing_script_filename = os.path.join( gearman_worker.ovdm.get_plugin_dir(), gearman_worker.collection_system_transfer['name'].lower() + gearman_worker.ovdm.get_plugin_suffix()) logging.debug("Processing Script Filename: %s", processing_script_filename) if os.path.isfile(processing_script_filename): job_results['parts'].append({ "partName": "Dashboard Processing File Located", "result": "Pass" }) else: logging.warning("Processing script not found: %s", processing_script_filename) return json.dumps(job_results) gearman_worker.send_job_status(gearman_job, 10, 100) #build filelist filelist = [] if payload_obj['files']['new'] or payload_obj['files']['updated']: filelist = payload_obj['files']['new'] filelist += payload_obj['files']['updated'] logging.debug('File List: %s', json.dumps(filelist, indent=2)) job_results['parts'].append({ "partName": "Retrieve Filelist", "result": "Pass" }) else: logging.warning("No new or updated files to process") job_results['parts'].append({ "partName": "Retrieve Filelist", "result": "Pass" }) return json.dumps(job_results) file_count = len(filelist) file_index = 0 for filename in filelist: # pylint: disable=too-many-nested-blocks if gearman_worker.stop: break logging.info("Processing file: %s", filename) json_filename = os.path.splitext(filename)[0] + '.json' raw_filepath = os.path.join(gearman_worker.cruise_dir, filename) json_filepath = os.path.join(gearman_worker.data_dashboard_dir, json_filename) if not os.path.isfile(raw_filepath): job_results['parts'].append({ "partName": "Verify data file exists", "result": "Fail", "reason": "Unable to find data file: " + filename }) logging.warning("File not found %s, skipping", filename) continue if os.stat(raw_filepath).st_size == 0: logging.warning("File is empty %s, skipping", filename) continue command = [ PYTHON_BINARY, processing_script_filename, '--dataType', raw_filepath ] logging.debug("DataType Retrieval Command: %s", ' '.join(command)) datatype_proc = subprocess.run(command, capture_output=True, text=True, check=False) if datatype_proc.stdout: dd_type = datatype_proc.stdout.rstrip('\n') logging.debug("DataType found to be: %s", dd_type) command = [PYTHON_BINARY, processing_script_filename, raw_filepath] logging.debug("Data Processing Command: %s", ' '.join(command)) data_proc = subprocess.run(command, capture_output=True, text=True, check=False) if data_proc.stdout: try: logging.debug("Verifying output") out_obj = json.loads(data_proc.stdout) except Exception as err: logging.error("Error parsing JSON output from file: %s", filename) logging.debug(str(err)) job_results['parts'].append({ "partName": "Parsing JSON output from file " + filename, "result": "Fail", "reason": "Error parsing JSON output from file: " + filename }) continue else: if not out_obj: error_title = 'Datafile Parsing error' error_body = "Parser returned no output. Parsing command: {}", ' '.join( command) logging.error("%s: %s", error_title, error_body) gearman_worker.ovdm.send_msg(error_title, error_body) elif 'error' in out_obj: error_title = 'Datafile Parsing error' error_body = out_obj['error'] logging.error("%s: %s", error_title, error_body) gearman_worker.ovdm.send_msg(error_title, error_body) else: output_results = output_json_data_to_file( json_filepath, out_obj) if output_results['verdict']: job_results['parts'].append({ "partName": "Writing DashboardData file: " + filename, "result": "Pass" }) else: error_title = 'Data Dashboard Processing failed' error_body = "Error Writing DashboardData file: " + filename + ". Reason: " + output_results[ 'reason'] logging.error("%s: %s", error_title, error_body) gearman_worker.ovdm.send_msg( error_title, error_body) job_results['parts'].append({ "partName": "Writing Dashboard file: " + filename, "result": "Fail", "reason": output_results['reason'] }) new_manifest_entries.append({ "type": dd_type, "dd_json": json_filepath.replace( gearman_worker.shipboard_data_warehouse_config[ 'shipboardDataWarehouseBaseDir'] + '/', ''), "raw_data": raw_filepath.replace( gearman_worker.shipboard_data_warehouse_config[ 'shipboardDataWarehouseBaseDir'] + '/', '') }) else: error_title = 'Data Dashboard Processing failed' error_body = 'No JSON output recieved from file. Parsing Command: ' + ' '.join( command) logging.error("%s: %s", error_title, error_body) gearman_worker.ovdm.send_msg(error_title, error_body) remove_manifest_entries.append({ "dd_json": json_filepath.replace( gearman_worker.shipboard_data_warehouse_config[ 'shipboardDataWarehouseBaseDir'] + '/', ''), "raw_data": raw_filepath.replace( gearman_worker.shipboard_data_warehouse_config[ 'shipboardDataWarehouseBaseDir'] + '/', '') }) #job_results['parts'].append({"partName": "Parsing JSON output from file " + filename, "result": "Fail"}) if data_proc.stderr: logging.error("Err: %s", data_proc.stderr) else: logging.warning("File is of unknown datatype: %s", raw_filepath) remove_manifest_entries.append({ "dd_json": json_filepath.replace( gearman_worker.shipboard_data_warehouse_config[ 'shipboardDataWarehouseBaseDir'] + '/', ''), "raw_data": raw_filepath.replace( gearman_worker.shipboard_data_warehouse_config[ 'shipboardDataWarehouseBaseDir'] + '/', '') }) if datatype_proc.stderr: logging.error("Err: %s", datatype_proc.stderr) gearman_worker.send_job_status( gearman_job, int(10 + 70 * float(file_index) / float(file_count)), 100) file_index += 1 gearman_worker.send_job_status(gearman_job, 8, 10) if len(new_manifest_entries) > 0: logging.info("Updating Manifest file: %s", gearman_worker.data_dashboard_manifest_file_path) rows_removed = 0 existing_manifest_entries = [] try: with open(gearman_worker.data_dashboard_manifest_file_path, 'r') as dashboard_manifest_file: existing_manifest_entries = json.load(dashboard_manifest_file) job_results['parts'].append({ "partName": "Reading pre-existing Dashboard manifest file", "result": "Pass" }) except IOError: logging.warning("Error Reading Dashboard Manifest file %s", gearman_worker.data_dashboard_manifest_file_path) pass except Exception as err: logging.error(str(err)) job_results['parts'].append({ "partName": "Reading pre-existing Dashboard manifest file", "result": "Fail", "reason": "Error reading dashboard manifest file: " + gearman_worker.data_dashboard_manifest_file_path }) return json.dumps(job_results) logging.debug("Entries to remove: %s", json.dumps(remove_manifest_entries, indent=2)) for remove_entry in remove_manifest_entries: for idx, existing_entry in enumerate(existing_manifest_entries): if remove_entry['raw_data'] == existing_entry['raw_data']: del existing_manifest_entries[idx] rows_removed += 1 if os.path.isfile( os.path.join( gearman_worker.shipboard_data_warehouse_config[ 'shipboardDataWarehouseBaseDir'], remove_entry['dd_json'])): logging.info( "Deleting orphaned dd_json file %s", os.path.join( gearman_worker.shipboard_data_warehouse_config[ 'shipboardDataWarehouseBaseDir'], remove_entry['dd_json'])) os.remove( os.path.join( gearman_worker.shipboard_data_warehouse_config[ 'shipboardDataWarehouseBaseDir'], remove_entry['dd_json'])) break logging.debug("Entries to add/update: %s", json.dumps(new_manifest_entries, indent=2)) for new_entry in new_manifest_entries: updated = False for existing_entry in existing_manifest_entries: if new_entry['raw_data'] == existing_entry['raw_data']: updated = True job_results['files']['updated'].append( new_entry['dd_json'].replace( gearman_worker.cruise_id + '/', '')) break if not updated: #added job_results['files']['new'].append( new_entry['dd_json'].replace( gearman_worker.cruise_id + '/', '')) existing_manifest_entries.append(new_entry) if len(job_results['files']['new']) > 0: logging.info("%s row(s) added", len(job_results['files']['new'])) if len(job_results['files']['updated']) > 0: logging.info("%s row(s) updated", len(job_results['files']['updated'])) if rows_removed: logging.info("%s row(s) removed", rows_removed) output_results = output_json_data_to_file( gearman_worker.data_dashboard_manifest_file_path, existing_manifest_entries) if not output_results['verdict']: logging.error("Error Writing Dashboard manifest file: %s", gearman_worker.data_dashboard_manifest_file_path) job_results['parts'].append({ "partName": "Writing Dashboard manifest file", "result": "Fail", "reason": output_results['reason'] }) return json.dumps(job_results) job_results['parts'].append({ "partName": "Writing Dashboard manifest file", "result": "Pass" }) job_results['files']['updated'].append( os.path.join( gearman_worker.ovdm.get_required_extra_directory_by_name( 'Dashboard_Data')['destDir'], DEFAULT_DATA_DASHBOARD_MANIFEST_FN)) gearman_worker.send_job_status(gearman_job, 9, 10) logging.info("Setting file ownership/permissions") output_results = set_owner_group_permissions( gearman_worker. shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], gearman_worker.data_dashboard_dir) if output_results['verdict']: job_results['parts'].append({ "partName": "Set file/directory ownership", "result": "Pass" }) else: job_results['parts'].append({ "partName": "Set file/directory ownership", "result": "Fail", "reason": output_results['reason'] }) return json.dumps(job_results) gearman_worker.send_job_status(gearman_job, 10, 10) return json.dumps(job_results)
def task_rsync_publicdata_to_cruise_data(gearman_worker, gearman_job): """ Sync the contents of the PublicData share to the from_PublicData Extra Directory """ job_results = {'parts':[]} publicdata_dir = gearman_worker.shipboard_data_warehouse_config['shipboardDataWarehousePublicDataDir'] from_publicdata_dir = os.path.join(gearman_worker.cruise_dir, gearman_worker.ovdm.get_required_extra_directory_by_name('From_PublicData')['destDir']) gearman_worker.send_job_status(gearman_job, 1, 10) if os.path.exists(from_publicdata_dir): job_results['parts'].append({"partName": "Verify From_PublicData directory exists", "result": "Pass"}) else: job_results['parts'].append({"partName": "Verify From_PublicData directory exists", "result": "Fail", "reason": "Unable to locate the From_PublicData directory: " + from_publicdata_dir}) return json.dumps(job_results) if os.path.exists(publicdata_dir): job_results['parts'].append({"partName": "Verify PublicData directory exists", "result": "Pass"}) else: job_results['parts'].append({"partName": "Verify PublicData directory exists", "result": "Fail", "reason": "Unable to locate the PublicData directory: " + publicdata_dir}) return json.dumps(job_results) gearman_worker.send_job_status(gearman_job, 5, 10) logging.info("Transferring files from PublicData to the cruise data directory") output_results = transfer_publicdata_dir(gearman_worker, gearman_job) if not output_results['verdict']: job_results['parts'].append({"partName": "Transfer files", "result": "Fail", "reason": output_results['reason']}) return json.dumps(job_results) job_results['parts'].append({"partName": "Transfer files", "result": "Pass"}) files = output_results['files'] logging.debug("Files Transferred: %s",json.dumps(files, indent=2)) gearman_worker.send_job_status(gearman_job, 8, 10) if len(files['new']) > 0 or len(files['updated']) > 0: logging.info("Setting file permissions") output_results = set_owner_group_permissions(gearman_worker.shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], from_publicdata_dir) if output_results['verdict']: job_results['parts'].append({"partName": "Set file/directory ownership/permissions", "result": "Pass"}) else: job_results['parts'].append({"partName": "Set file/directory ownership/permissions", "result": "Fail", "reason": output_results['reason']}) return json.dumps(job_results) gearman_worker.send_job_status(gearman_job, 9, 100) logging.info("Initiating MD5 Summary Task") gm_client = python3_gearman.GearmanClient([gearman_worker.ovdm.get_gearman_server()]) gm_data = {} gm_data['cruiseID'] = gearman_worker.cruise_id gm_data['files'] = files gm_data['files']['new'] = [os.path.join(from_publicdata_dir,filename) for filename in gm_data['files']['new']] gm_data['files']['updated'] = [os.path.join(from_publicdata_dir,filename) for filename in gm_data['files']['updated']] gm_client.submit_job("updateMD5Summary", json.dumps(gm_data)) logging.info("MD5 Summary Task Complete") # need to verify update MD5 completed successfully gearman_worker.send_job_status(gearman_job, 10, 10) return json.dumps(job_results)
def task_finalize_current_lowering(gearman_worker, gearman_job): """ Finalize the current lowering """ job_results = {'parts':[]} gearman_worker.send_job_status(gearman_job, 1, 10) lowering_config_filepath = os.path.join(gearman_worker.lowering_dir, DEFAULT_LOWERING_CONFIG_FN) if os.path.exists(gearman_worker.lowering_dir) and (gearman_worker.lowering_id != ''): job_results['parts'].append({"partName": "Verify Lowering Directory exists", "result": "Pass"}) else: job_results['parts'].append({"partName": "Verify Lowering Directory exists", "result": "Fail", "reason": "Lowering directory: " + gearman_worker.lowering_dir + " could not be found"}) return json.dumps(job_results) gearman_worker.send_job_status(gearman_job, 3, 10) logging.info("Queuing Collection System Transfers") gm_client = python3_gearman.GearmanClient([gearman_worker.ovdm.get_gearman_server()]) gm_data = { 'loweringID': gearman_worker.lowering_id, 'loweringStartDate': gearman_worker.lowering_start_date, 'systemStatus': "On", 'collectionSystemTransfer': {} } collection_system_transfer_jobs = [] collection_system_transfers = gearman_worker.ovdm.get_active_collection_system_transfers(cruise=False) for collection_system_transfer in collection_system_transfers: logging.debug("Queuing runCollectionSystemTransfer job for %s", collection_system_transfer['name']) gm_data['collectionSystemTransfer']['collectionSystemTransferID'] = collection_system_transfer['collectionSystemTransferID'] collection_system_transfer_jobs.append( {"task": "runCollectionSystemTransfer", "data": json.dumps(gm_data)} ) gearman_worker.send_job_status(gearman_job, 5, 10) if len(collection_system_transfer_jobs) > 0: logging.info("Submitting runCollectionSystemTransfer jobs") submitted_job_request = gm_client.submit_multiple_jobs(collection_system_transfer_jobs, background=False, wait_until_complete=False) gearman_worker.send_job_status(gearman_job, 7, 10) time.sleep(1) gm_client.wait_until_jobs_completed(submitted_job_request) logging.info("Completed runCollectionSystemTransfers jobs") gearman_worker.send_job_status(gearman_job, 9, 10) #build Lowering Config file logging.info("Exporting Lowering Configuration") output_results = export_lowering_config(gearman_worker, lowering_config_filepath, finalize=True) if output_results['verdict']: job_results['parts'].append({"partName": "Export Lowering config data to file", "result": "Pass"}) else: job_results['parts'].append({"partName": "Export Lowering config data to file", "result": "Fail", "reason": output_results['reason']}) return json.dumps(job_results) output_results = set_owner_group_permissions(gearman_worker.shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], lowering_config_filepath) if output_results['verdict']: job_results['parts'].append({"partName": "Set Lowering config file ownership/permissions", "result": "Pass"}) else: job_results['parts'].append({"partName": "Set Lowering config file ownership/permissions", "result": "Fail", "reason": output_results['reason']}) return json.dumps(job_results) # need to add code for updating MD5 gearman_worker.send_job_status(gearman_job, 10, 10) return json.dumps(job_results)
def task_setup_new_cruise(gearman_worker, gearman_job): # pylint: disable=too-many-return-statements,too-many-statements """ Setup a new cruise """ job_results = {'parts':[]} payload_obj = json.loads(gearman_job.data) logging.debug("Payload: %s", json.dumps(payload_obj, indent=2)) ovdm_config_file_path = os.path.join(gearman_worker.cruise_dir, DEFAULT_CRUISE_CONFIG_FN) gearman_worker.send_job_status(gearman_job, 1, 10) gm_client = python3_gearman.GearmanClient([gearman_worker.ovdm.get_gearman_server()]) logging.info("Set ownership/permissions for the CruiseData directory") completed_job_request = gm_client.submit_job("setCruiseDataDirectoryPermissions", gearman_job.data) result_obj = json.loads(completed_job_request.result) if result_obj['parts'][-1]['result'] != "Pass": # Final Verdict logging.error("Failed to lockdown the CruiseData directory") job_results['parts'].append({"partName": "Set ownership/permissions for CruiseData directory", "result": "Fail", "reason": result_obj['parts'][-1]['reason']}) return json.dumps(job_results) job_results['parts'].append({"partName": "Set ownership/permissions for CruiseData directory", "result": "Pass"}) logging.info("Creating cruise data directory") completed_job_request = gm_client.submit_job("createCruiseDirectory", gearman_job.data) result_obj = json.loads(completed_job_request.result) if result_obj['parts'][-1]['result'] != "Pass": # Final Verdict logging.error("Failed to create cruise data directory") job_results['parts'].append({"partName": "Create cruise data directory structure", "result": "Fail", "reason": result_obj['parts'][-1]['reason']}) return json.dumps(job_results) job_results['parts'].append({"partName": "Create cruise data directory structure", "result": "Pass"}) gearman_worker.send_job_status(gearman_job, 5, 10) #build OpenVDM Config file logging.info("Exporting Cruise Configuration") output_results = export_ovdm_config(gearman_worker, ovdm_config_file_path) if not output_results['verdict']: job_results['parts'].append({"partName": "Export OpenVDM config data to file", "result": "Fail", "reason": output_results['reason']}) return json.dumps(job_results) job_results['parts'].append({"partName": "Export OpenVDM config data to file", "result": "Pass"}) output_results = set_owner_group_permissions(gearman_worker.shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], ovdm_config_file_path) if not output_results['verdict']: job_results['parts'].append({"partName": "Set OpenVDM config file ownership/permissions", "result": "Fail", "reason": output_results['reason']}) return json.dumps(job_results) gearman_worker.send_job_status(gearman_job, 7, 10) logging.info("Creating MD5 summary files") completed_job_request = gm_client.submit_job("rebuildMD5Summary", gearman_job.data) result_obj = json.loads(completed_job_request.result) if result_obj['parts'][-1]['result'] != "Pass": # Final Verdict logging.error("Failed to create MD5 summary files") job_results['parts'].append({"partName": "Create MD5 summary files", "result": "Fail", "reason": result_obj['parts'][-1]['reason']}) return json.dumps(job_results) job_results['parts'].append({"partName": "Create MD5 summary files", "result": "Pass"}) gearman_worker.send_job_status(gearman_job, 8, 10) logging.info("Creating data dashboard directory structure and manifest file") completed_job_request = gm_client.submit_job("rebuildDataDashboard", gearman_job.data) result_obj = json.loads(completed_job_request.result) if result_obj['parts'][-1]['result'] != "Pass": # Final Verdict logging.error("Failed to create data dashboard directory structure and/or manifest file") job_results['parts'].append({"partName": "Create data dashboard directory structure and manifest file", "result": "Fail", "reason": result_obj['parts'][-1]['reason']}) return json.dumps(job_results) job_results['parts'].append({"partName": "Create data dashboard directory structure and manifest file", "result": "Pass"}) gearman_worker.send_job_status(gearman_job, 9, 10) logging.info("Updating Cruise Size") cruise_size_proc = subprocess.run(['du','-sb', gearman_worker.cruise_dir], capture_output=True, text=True, check=False) if cruise_size_proc.returncode == 0: logging.info("Cruise Size: %s", cruise_size_proc.stdout.split()[0]) gearman_worker.ovdm.set_cruise_size(cruise_size_proc.stdout.split()[0]) else: gearman_worker.ovdm.set_cruise_size("0") gearman_worker.ovdm.set_lowering_size("0") gearman_worker.send_job_status(gearman_job, 10, 10) return json.dumps(job_results)
def task_run_ship_to_shore_transfer(gearman_worker, current_job): # pylint: disable=too-many-statements """ Perform the ship-to-shore transfer """ time.sleep(randint(0, 2)) job_results = { 'parts': [{ "partName": "Transfer In-Progress", "result": "Pass" }, { "partName": "Transfer Enabled", "result": "Pass" }], 'files': {} } logging.debug("Setting transfer status to 'Running'") gearman_worker.ovdm.set_running_cruise_data_transfer( gearman_worker.cruise_data_transfer['cruiseDataTransferID'], os.getpid(), current_job.handle) logging.info("Testing configuration") gearman_worker.send_job_status(current_job, 1, 10) gm_client = python3_gearman.GearmanClient( [gearman_worker.ovdm.get_gearman_server()]) gm_data = { 'cruiseDataTransfer': gearman_worker.cruise_data_transfer, 'cruiseID': gearman_worker.cruise_id } completed_job_request = gm_client.submit_job("testCruiseDataTransfer", json.dumps(gm_data)) results_obj = json.loads(completed_job_request.result) logging.debug('Connection Test Results: %s', json.dumps(results_obj, indent=2)) if results_obj['parts'][-1]['result'] == "Pass": # Final Verdict logging.debug("Connection test passed") job_results['parts'].append({ "partName": "Connection Test", "result": "Pass" }) else: logging.warning("Connection test failed, quitting job") job_results['parts'].append({ "partName": "Connection Test", "result": "Fail", "reason": results_obj['parts'][-1]['reason'] }) return json.dumps(job_results) gearman_worker.send_job_status(current_job, 2, 10) logging.info("Transferring files") output_results = None if gearman_worker.cruise_data_transfer[ 'transferType'] == "4": # SSH Server output_results = transfer_ssh_dest_dir(gearman_worker, current_job) else: logging.error("Unknown Transfer Type") job_results['parts'].append({ "partName": "Transfer Files", "result": "Fail", "reason": "Unknown transfer type" }) return json.dumps(job_results) if not output_results['verdict']: logging.error("Transfer of remote files failed: %s", output_results['reason']) job_results['parts'].append({ "partName": "Transfer Files", "result": "Fail", "reason": output_results['reason'] }) return job_results logging.debug("Transfer completed successfully") job_results['files'] = output_results['files'] job_results['parts'].append({ "partName": "Transfer Files", "result": "Pass" }) if len(job_results['files']['new']) > 0: logging.debug("%s file(s) added", len(job_results['files']['new'])) if len(job_results['files']['updated']) > 0: logging.debug("%s file(s) updated", len(job_results['files']['updated'])) if len(job_results['files']['exclude']) > 0: logging.debug("%s file(s) intentionally skipped", len(job_results['files']['exclude'])) gearman_worker.send_job_status(current_job, 9, 10) if job_results['files']['new'] or job_results['files']['updated']: logging.debug("Building logfiles") logfile_filename = gearman_worker.cruise_data_transfer[ 'name'] + '_' + gearman_worker.transfer_start_date + '.log' log_contents = { 'files': { 'new': job_results['files']['new'], 'updated': job_results['files']['updated'] } } output_results = output_json_data_to_file( os.path.join(build_logfile_dirpath(gearman_worker), logfile_filename), log_contents['files']) if output_results['verdict']: job_results['parts'].append({ "partName": "Write transfer logfile", "result": "Pass" }) else: logging.error("Error writing transfer logfile: %s", logfile_filename) job_results['parts'].append({ "partName": "Write transfer logfile", "result": "Fail", "reason": output_results['reason'] }) return json.dumps(job_results) output_results = set_owner_group_permissions( gearman_worker. shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], os.path.join(build_logfile_dirpath(gearman_worker), logfile_filename)) if not output_results['verdict']: job_results['parts'].append({ "partName": "Set OpenVDM config file ownership/permissions", "result": "Fail", "reason": output_results['reason'] }) return json.dumps(job_results) gearman_worker.send_job_status(current_job, 10, 10) time.sleep(2) return json.dumps(job_results)
def task_create_cruise_directory(gearman_worker, gearman_job): """ Setup the cruise directory for the specified cruise ID """ job_results = {'parts': []} payload_obj = json.loads(gearman_job.data) logging.debug("Payload: %s", json.dumps(payload_obj, indent=2)) gearman_worker.send_job_status(gearman_job, 1, 10) if os.path.exists( gearman_worker. shipboard_data_warehouse_config['shipboardDataWarehouseBaseDir']): job_results['parts'].append({ "partName": "Verify Base Directory exists", "result": "Pass" }) else: logging.error( "Failed to find base directory: %s", gearman_worker. shipboard_data_warehouse_config['shipboardDataWarehouseBaseDir']) job_results['parts'].append({ "partName": "Verify Base Directory exists", "result": "Fail", "reason": "Failed to find base directory: " + gearman_worker. shipboard_data_warehouse_config['shipboardDataWarehouseBaseDir'] }) return json.dumps(job_results) if not os.path.exists(gearman_worker.cruise_dir): job_results['parts'].append({ "partName": "Verify Cruise Directory does not exists", "result": "Pass" }) else: logging.error("Cruise directory already exists: %s", gearman_worker.cruise_dir) job_results['parts'].append({ "partName": "Verify Cruise Directory does not exists", "result": "Fail", "reason": "Cruise directory " + gearman_worker.cruise_dir + " already exists" }) return json.dumps(job_results) gearman_worker.send_job_status(gearman_job, 2, 10) directorylist = build_directorylist(gearman_worker) logging.debug("Directory List: %s", json.dumps(directorylist, indent=2)) if len(directorylist) > 0: job_results['parts'].append({ "partName": "Build Directory List", "result": "Pass" }) else: logging.warning("Directory list is empty") job_results['parts'].append({ "partName": "Build Directory List", "result": "Fail", "reason": "Empty list of directories to create" }) return json.dumps(job_results) gearman_worker.send_job_status(gearman_job, 5, 10) output_results = create_directories(directorylist) if output_results['verdict']: job_results['parts'].append({ "partName": "Create Directories", "result": "Pass" }) else: logging.error( "Failed to create any/all of the cruise data directory structure") job_results['parts'].append({ "partName": "Create Directories", "result": "Fail", "reason": output_results['reason'] }) gearman_worker.send_job_status(gearman_job, 7, 10) if gearman_worker.ovdm.show_only_current_cruise_dir() is True: logging.info("Clear read permissions for all cruise directories") lockdown_directory( gearman_worker. shipboard_data_warehouse_config['shipboardDataWarehouseBaseDir'], gearman_worker.cruise_dir) job_results['parts'].append({ "partName": "Clear CruiseData Directory Read Permissions", "result": "Pass" }) gearman_worker.send_job_status(gearman_job, 8, 10) output_results = set_owner_group_permissions( gearman_worker. shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], gearman_worker.cruise_dir) if output_results['verdict']: job_results['parts'].append({ "partName": "Set cruise directory ownership/permissions", "result": "Pass" }) else: job_results['parts'].append({ "partName": "Set cruise directory ownership/permissions", "result": "Fail", "reason": output_results['reason'] }) return json.dumps(job_results) gearman_worker.send_job_status(gearman_job, 10, 10) return json.dumps(job_results)
def task_rebuild_md5_summary(gearman_worker, gearman_job): # pylint: disable=too-many-statements """ Rebuild the existing MD5 summary files """ job_results = {'parts':[]} payload_obj = json.loads(gearman_job.data) logging.debug("Payload: %s", json.dumps(payload_obj, indent=2)) gearman_worker.send_job_status(gearman_job, 1, 10) if os.path.exists(gearman_worker.cruise_dir): job_results['parts'].append({"partName": "Verify Cruise Directory exists", "result": "Pass"}) else: logging.error("Cruise directory not found") job_results['parts'].append({"partName": "Verify Cruise Directory exists", "result": "Fail", "reason": "Unable to locate the cruise directory: " + gearman_worker.cruise_dir}) return json.dumps(job_results) logging.info("Building filelist") filelist = build_filelist(gearman_worker.cruise_dir) logging.debug('Filelist: %s', json.dumps(filelist, indent=2)) job_results['parts'].append({"partName": "Retrieve Filelist", "result": "Pass"}) gearman_worker.send_job_status(gearman_job, 2, 10) logging.info("Building hashes") new_hashes = build_md5_hashes(gearman_worker, gearman_job, filelist) logging.debug("Hashes: %s", json.dumps(new_hashes, indent=2)) if gearman_worker.stop: job_results['parts'].append({"partName": "Calculate Hashes", "result": "Fail", "reason": "Job was stopped by user"}) return json.dumps(job_results) job_results['parts'].append({"partName": "Calculate Hashes", "result": "Pass"}) gearman_worker.send_job_status(gearman_job, 85, 100) logging.debug("Sorting Hashes") sorted_hashes = sorted(new_hashes, key=lambda hashes: hashes['filename']) gearman_worker.send_job_status(gearman_job, 9, 10) logging.info("Building MD5 Summary file") try: #logging.debug("Saving new MD5 Summary file") with open(gearman_worker.md5_summary_filepath, 'w') as md5_summary_file: for filehash in sorted_hashes: md5_summary_file.write(filehash['hash'] + ' ' + filehash['filename'] + '\n') job_results['parts'].append({"partName": "Writing MD5 Summary file", "result": "Pass"}) except IOError: logging.error("Error saving MD5 Summary file: %s", gearman_worker.md5_summary_filepath) job_results['parts'].append({"partName": "Writing MD5 Summary file", "result": "Fail", "reason": "Error saving MD5 Summary file: " + gearman_worker.md5_summary_filepath}) return json.dumps(job_results) output_results = set_owner_group_permissions(gearman_worker.shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], gearman_worker.md5_summary_filepath) if output_results['verdict']: job_results['parts'].append({"partName": "Set MD5 Summary file ownership/permissions", "result": "Pass"}) else: logging.error("Failed to set directory ownership") job_results['parts'].append({"partName": "Set MD5 Summary file ownership/permissions", "result": "Fail", "reason": output_results['reason']}) gearman_worker.send_job_status(gearman_job, 95, 100) logging.info("Building MD5 Summary MD5 file") output_results = build_md5_summary_md5(gearman_worker) if output_results['verdict']: job_results['parts'].append({"partName": "Writing MD5 Summary MD5 file", "result": "Pass"}) else: job_results['parts'].append({"partName": "Writing MD5 Summary MD5 file", "result": "Fail", "reason": output_results['reason']}) return json.dumps(job_results) output_results = set_owner_group_permissions(gearman_worker.shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], gearman_worker.md5_summary_md5_filepath) if output_results['verdict']: job_results['parts'].append({"partName": "Set MD5 Summary MD5 file ownership/permissions", "result": "Pass"}) else: logging.error("Failed to set directory ownership") job_results['parts'].append({"partName": "Set MD5 Summary MD5 file ownership/permissions", "result": "Fail", "reason": output_results['reason']}) return json.dumps(job_results) gearman_worker.send_job_status(gearman_job, 10, 10) return json.dumps(job_results)
def task_update_md5_summary(gearman_worker, gearman_job): # pylint: disable=too-many-branches,too-many-statements,too-many-locals """ Update the existing MD5 summary files """ job_results = {'parts':[]} payload_obj = json.loads(gearman_job.data) logging.debug("Payload: %s", json.dumps(payload_obj, indent=2)) gearman_worker.send_job_status(gearman_job, 1, 10) logging.debug("Building filelist") filelist = [] job_results['parts'].append({"partName": "Retrieve Filelist", "result": "Pass"}) if payload_obj['files']['new'] or payload_obj['files']['updated']: filelist.extend(payload_obj['files']['new']) filelist.extend(payload_obj['files']['updated']) else: return json.dumps(job_results) #filelist = [os.path.join(gearman_worker.cruiseID, filename) for filename in filelist] logging.debug('Filelist: %s', json.dumps(filelist, indent=2)) gearman_worker.send_job_status(gearman_job, 2, 10) logging.debug("Building hashes") new_hashes = build_md5_hashes(gearman_worker, gearman_job, filelist) logging.debug('Hashes: %s', json.dumps(new_hashes, indent=2)) gearman_worker.send_job_status(gearman_job, 8, 10) if gearman_worker.stop: return json.dumps(job_results) job_results['parts'].append({"partName": "Calculate Hashes", "result": "Pass"}) existing_hashes = [] logging.debug("Processing existing MD5 summary file") try: with open(gearman_worker.md5_summary_filepath, 'r') as md5_summary_file: for line in md5_summary_file: (md5_hash, filename) = line.split(' ', 1) existing_hashes.append({'hash': md5_hash, 'filename': filename.rstrip('\n')}) except IOError: logging.error("Error Reading pre-existing MD5 Summary file: %s", gearman_worker.md5_summary_filepath) job_results['parts'].append({"partName": "Reading pre-existing MD5 Summary file", "result": "Fail", "reason": "Error Reading pre-existing MD5 Summary file: " + gearman_worker.md5_summary_filepath}) return json.dumps(job_results) #logging.debug('Existing Hashes:', json.dumps(existing_hashes, indent=2)) job_results['parts'].append({"partName": "Reading pre-existing MD5 Summary file", "result": "Pass"}) row_added = 0 row_updated = 0 for new_hash in new_hashes: updated = False for existing_hash in existing_hashes: if new_hash['filename'] == existing_hash['filename']: existing_hash['hash'] = new_hash['hash'] updated = True row_updated += 1 break if not updated: existing_hashes.append({'hash': new_hash['hash'], 'filename': new_hash['filename']}) row_added += 1 if row_added > 0: logging.debug("%s row(s) added", row_added) if row_updated > 0: logging.debug("%s row(s) updated", row_updated) gearman_worker.send_job_status(gearman_job, 85, 100) #logging.debug("Sorting hashes") sorted_hashes = sorted(existing_hashes, key=lambda hashes: hashes['filename']) logging.debug("Building MD5 Summary file") try: with open(gearman_worker.md5_summary_filepath, 'w') as md5_summary_file: for filehash in sorted_hashes: md5_summary_file.write(filehash['hash'] + ' ' + filehash['filename'] + '\n') job_results['parts'].append({"partName": "Writing MD5 Summary file", "result": "Pass"}) except IOError: logging.error("Error updating MD5 Summary file: %s", gearman_worker.md5_summary_filepath) job_results['parts'].append({"partName": "Writing MD5 Summary file", "result": "Fail", "reason": "Error updating MD5 Summary file: " + gearman_worker.md5_summary_filepath}) return json.dumps(job_results) output_results = set_owner_group_permissions(gearman_worker.shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], gearman_worker.md5_summary_filepath) if output_results['verdict']: job_results['parts'].append({"partName": "Set MD5 Summary file ownership/permissions", "result": "Pass"}) else: logging.error("Failed to set directory ownership") job_results['parts'].append({"partName": "Set MD5 Summary file ownership/permissions", "result": "Fail", "reason": output_results['reason']}) gearman_worker.send_job_status(gearman_job, 9, 10) logging.debug("Building MD5 Summary MD5 file") output_results = build_md5_summary_md5(gearman_worker) if output_results['verdict']: job_results['parts'].append({"partName": "Writing MD5 Summary MD5 file", "result": "Pass"}) else: job_results['parts'].append({"partName": "Writing MD5 Summary MD5 file", "result": "Fail", "reason": output_results['reason']}) output_results = set_owner_group_permissions(gearman_worker.shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], gearman_worker.md5_summary_md5_filepath) if output_results['verdict']: job_results['parts'].append({"partName": "Set MD5 Summary MD5 file ownership/permissions", "result": "Pass"}) else: logging.error("Failed to set directory ownership") job_results['parts'].append({"partName": "Set MD5 Summary MD5 file ownership/permissions", "result": "Fail", "reason": output_results['reason']}) gearman_worker.send_job_status(gearman_job, 10, 10) return json.dumps(job_results)
def task_rebuild_lowering_directory(gearman_worker, gearman_job): """ Verify and create if necessary all the lowering sub-directories """ job_results = {'parts': []} payload_obj = json.loads(gearman_job.data) logging.debug("Payload: %s", json.dumps(payload_obj, indent=2)) gearman_worker.send_job_status(gearman_job, 1, 10) if not os.path.exists(gearman_worker.lowering_dir): logging.error("Lowering directory not found") job_results['parts'].append({ "partName": "Verify Lowering Directory exists", "result": "Fail", "reason": "Unable to find lowering directory: " + gearman_worker.lowering_dir }) return json.dumps(job_results) job_results['parts'].append({ "partName": "Verify Lowering Directory exists", "result": "Pass" }) gearman_worker.send_job_status(gearman_job, 2, 10) logging.info("Build directory list") directorylist = build_directorylist(gearman_worker) logging.debug("Directory List: %s", json.dumps(directorylist, indent=2)) if len(directorylist) == 0: logging.error("Directory list is empty") job_results['parts'].append({ "partName": "Build Directory List", "result": "Fail", "reason": "Empty list of directories to create" }) return json.dumps(job_results) job_results['parts'].append({ "partName": "Build Directory List", "result": "Pass" }) gearman_worker.send_job_status(gearman_job, 5, 10) logging.info("Create directories") output_results = create_directories(directorylist) if output_results['verdict']: job_results['parts'].append({ "partName": "Create Directories", "result": "Pass" }) else: logging.error( "Unable to create any/all of the lowering data directory structure" ) job_results['parts'].append({ "partName": "Create Directories", "result": "Fail", "reason": output_results['reason'] }) gearman_worker.send_job_status(gearman_job, 7, 10) logging.info("Set directory ownership/permissions") output_results = set_owner_group_permissions( gearman_worker. shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], gearman_worker.lowering_dir) if output_results['verdict']: job_results['parts'].append({ "partName": "Set Directory ownership/permissions", "result": "Pass" }) else: logging.error("Failed to set directory ownership") job_results['parts'].append({ "partName": "Set Directory ownership/permissions", "result": "Fail", "reason": output_results['reason'] }) gearman_worker.send_job_status(gearman_job, 10, 10) return json.dumps(job_results)
def task_finalize_current_cruise(gearman_worker, gearman_job): # pylint: disable=too-many-return-statements,too-many-statements """ Finalize the current cruise """ job_results = {'parts':[]} gearman_worker.send_job_status(gearman_job, 1, 10) publicdata_dir = gearman_worker.shipboard_data_warehouse_config['shipboardDataWarehousePublicDataDir'] from_publicdata_dir = os.path.join(gearman_worker.cruise_dir, gearman_worker.ovdm.get_required_extra_directory_by_name('From_PublicData')['destDir']) ovdm_config_file_path = os.path.join(gearman_worker.cruise_dir, DEFAULT_CRUISE_CONFIG_FN) if not os.path.exists(gearman_worker.cruise_dir): job_results['parts'].append({"partName": "Verify cruise directory exists", "result": "Fail", "reason": "Cruise directory: " + gearman_worker.cruise_dir + " could not be found"}) return json.dumps(job_results) job_results['parts'].append({"partName": "Verify cruise directory exists", "result": "Pass"}) gearman_worker.send_job_status(gearman_job, 2, 10) logging.info("Queuing Collection System Transfers") gm_client = python3_gearman.GearmanClient([gearman_worker.ovdm.get_gearman_server()]) gm_data = { 'cruiseID': gearman_worker.cruise_id, 'cruiseStartDate': gearman_worker.cruise_start_date, 'systemStatus': "On", 'collectionSystemTransfer': {} } collection_system_transfer_jobs = [] collection_system_transfers = gearman_worker.ovdm.get_active_collection_system_transfers(lowering=False) for collection_system_transfer in collection_system_transfers: logging.debug("Queuing runCollectionSystemTransfer job for %s", collection_system_transfer['name']) gm_data['collectionSystemTransfer']['collectionSystemTransferID'] = collection_system_transfer['collectionSystemTransferID'] collection_system_transfer_jobs.append( {"task": "runCollectionSystemTransfer", "data": json.dumps(gm_data)} ) gearman_worker.send_job_status(gearman_job, 3, 10) if len(collection_system_transfer_jobs) > 0: logging.info("Submitting runCollectionSystemTransfer jobs") submitted_job_request = gm_client.submit_multiple_jobs(collection_system_transfer_jobs, background=False, wait_until_complete=False) gearman_worker.send_job_status(gearman_job, 4, 10) time.sleep(1) gm_client.wait_until_jobs_completed(submitted_job_request) logging.info("Completed runCollectionSystemTransfers jobs") gearman_worker.send_job_status(gearman_job, 5, 10) if gearman_worker.ovdm.get_transfer_public_data(): logging.info("Transferring files from PublicData to the cruise data directory") logging.debug("Verify From_PublicData directory exists within the cruise data directory") if not os.path.exists(from_publicdata_dir): job_results['parts'].append({"partName": "Verify From_PublicData directory exists", "result": "Fail", "reason": "From_PublicData directory: " + from_publicdata_dir + " could not be found"}) return json.dumps(job_results) job_results['parts'].append({"partName": "Verify From_PublicData directory exists", "result": "Pass"}) logging.debug("Verify PublicData Directory exists") if not os.path.exists(publicdata_dir): job_results['parts'].append({"partName": "Verify PublicData directory exists", "result": "Fail", "reason": "PublicData directory: " + publicdata_dir + " could not be found"}) return json.dumps(job_results) job_results['parts'].append({"partName": "Verify PublicData directory exists", "result": "Pass"}) logging.debug("Transferring files") output_results = transfer_publicdata_dir(gearman_worker, gearman_job) logging.debug("Transfer Complete") if not output_results['verdict']: job_results['parts'].append({"partName": "Transfer PublicData files", "result": "Fail", "reason": output_results['reason']}) return json.dumps(job_results) job_results['parts'].append({"partName": "Transfer PublicData files", "result": "Pass"}) files = output_results['files'] logging.debug("PublicData Files Transferred: %s", json.dumps(files, indent=2)) logging.info("Clearing files from PublicData") output_results = clear_directory(publicdata_dir) logging.debug("Clearing Complete") if not output_results['verdict']: job_results['parts'].append({"partName": "Clear out PublicData files", "result": "Fail", "reason": output_results['reason']}) return json.dumps(job_results) job_results['parts'].append({"partName": "Clear out PublicData files", "result": "Pass"}) gearman_worker.send_job_status(gearman_job, 9, 10) if len(files['new']) > 0 or len(files['updated']) > 0: output_results = set_owner_group_permissions(gearman_worker.shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], from_publicdata_dir) if not output_results['verdict']: job_results['parts'].append({"partName": "Set file/directory ownership/permissions", "result": "Fail", "reason": output_results['reason']}) return json.dumps(job_results) job_results['parts'].append({"partName": "Set file/directory ownership/permissions", "result": "Pass"}) else: files = { 'include':[], 'exclude':[], 'new':[], 'updated':[] } gearman_worker.send_job_status(gearman_job, 95, 100) #build OpenVDM Config file logging.info("Exporting OpenVDM Configuration") output_results = export_ovdm_config(gearman_worker, ovdm_config_file_path, finalize=True) if not output_results['verdict']: job_results['parts'].append({"partName": "Export OpenVDM config data to file", "result": "Fail", "reason": output_results['reason']}) return json.dumps(job_results) job_results['parts'].append({"partName": "Export OpenVDM config data to file", "result": "Pass"}) output_results = set_owner_group_permissions(gearman_worker.shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], ovdm_config_file_path) if not output_results['verdict']: job_results['parts'].append({"partName": "Set OpenVDM config file ownership/permissions", "result": "Fail", "reason": output_results['reason']}) return json.dumps(job_results) job_results['parts'].append({"partName": "Set OpenVDM config file ownership/permissions", "result": "Pass"}) logging.info("Initiating MD5 Summary Task") gm_data = { 'cruiseID': gearman_worker.cruise_id, 'files': files } # gm_data['files']['new'] = [from_publicdata_dir.replace(gearman_worker.cruise_dir, '') + '/' + filename for filename in gm_data['files']['new']] # gm_data['files']['updated'] = [from_publicdata_dir.replace(gearman_worker.cruise_dir, '') + '/' + filename for filename in gm_data['files']['updated']] gm_data['files']['updated'].append(DEFAULT_CRUISE_CONFIG_FN) gm_client.submit_job("updateMD5Summary", json.dumps(gm_data)) logging.debug("MD5 Summary Task Complete") # need to add code for cruise data transfers gearman_worker.send_job_status(gearman_job, 10, 10) return json.dumps(job_results)