def SGEPluginJob(start_json): """ Spawn a thread that will start a SGE job, and wait for it to return after it has return it will make a PUT the API using to update the status of the run args is a dict of all the args that are needed by for the plugin to run """ try: os.umask(0000) # Query some essential values plugin = start_json['runinfo']['plugin'] resultpk = start_json['runinfo']['pk'] analysis_name = os.path.basename(start_json['runinfo']['analysis_dir']) plugin_output = start_json['runinfo']['results_dir'] logger.debug("Getting ready for queue : %s %s", resultpk, plugin['name']) # Update pluginresult status h = httplib2.Http() headers = { "Content-type": "application/json", "Accept": "application/json" } url = 'http://localhost' + '/rundb/api/v1/results/%s/plugin/' % resultpk data = { plugin['name']: "Pending", 'owner': start_json.get('owner', None), } resp, content = h.request(url, "PUT", body=json.dumps(data), headers=headers) logger.debug("PluginResult Marked Queued : %s %s" % (resp, content)) #Make sure the dirs exist if not os.path.exists(plugin_output): os.makedirs(plugin_output, 0775) # Write start_json to startplugin.json file. startpluginjson = os.path.join(plugin_output, 'startplugin.json') with open(startpluginjson, "w") as fp: json.dump(start_json, fp, indent=2) # Branch for launch.sh vs 3.0 plugin class logger.debug("Finding plugin definition: '%s':'%s'", plugin['name'], plugin['path']) (launch, isCompat) = pluginmanager.find_pluginscript(plugin['path'], plugin['name']) if not launch or not os.path.exists(launch): logger.error( "Analysis: %s. Path to plugin script: '%s' Does Not Exist!", analysis_name, launch) return 1 # Create individual launch script from template and plugin launch.sh launcher = PluginRunner() if isCompat: launchScript = launcher.createPluginWrapper(launch, start_json) else: start_json.update({'command': ["python %s -vv" % launch]}) launchScript = launcher.createPluginWrapper(None, start_json) launchWrapper = launcher.writePluginLauncher(plugin_output, plugin['name'], launchScript) launcher.writePluginJson(start_json) # Prepare drmaa Job - SGE/gridengine only jt = _session.createJobTemplate() jt.nativeSpecification = " -q %s" % ("plugin.q") if constants.Feature.EXPORT in plugin['features']: jt.nativeSpecification += ' -l ep=1 ' hold = plugin.get('hold_jid', []) if hold: jt.nativeSpecification += " -hold_jid " + ','.join( str(jobid) for jobid in hold) jt.workingDirectory = plugin_output jt.outputPath = ":" + os.path.join(plugin_output, "drmaa_stdout.txt") jt.joinFiles = True # Merge stdout and stderr # Plugin command is: ion_pluginname_launch.sh -j startplugin.json jt.remoteCommand = launchWrapper jt.args = ["-j", startpluginjson] # Submit the job to drmaa jobid = _session.runJob(jt) # Update pluginresult status h = httplib2.Http() headers = { "Content-type": "application/json", "Accept": "application/json" } url = 'http://localhost' + '/rundb/api/v1/results/%s/plugin/' % resultpk resp, content = h.request(url, "PUT", body=json.dumps({plugin['name']: "Queued"}), headers=headers) logger.debug("PluginResult Marked Queued : %s %s" % (resp, content)) logger.info("Analysis: %s. Plugin: %s. Job: %s Queued." % (analysis_name, plugin['name'], jobid)) _session.deleteJobTemplate(jt) return jobid except: logger.critical("SGEPluginJob method failure") logger.critical(traceback.format_exc())
def unzipPlugin(zipfile, logger=None): if not logger: logger = logging.getLogger(__name__) ## Extract plugin to scratch folder. When complete, move to final location. plugin_path, ext = os.path.splitext(zipfile) plugin_name = os.path.basename(plugin_path) # ZIP file must named with plugin name - fragile # FIXME - handle (1) additions (common for multiple downloads via browser) # FIXME - handle version string in ZIP archive name scratch_path = os.path.join(settings.PLUGIN_PATH, "scratch", "install-temp", plugin_name) (prefix, files) = extract_zip(zipfile, scratch_path, auto_prefix=True, logger=logger) if prefix: plugin_name = os.path.basename(prefix) plugin_temp_home = os.path.join(scratch_path, prefix) try: # Convert script into PluginClass, get info by introspection from iondb.plugins.manager import pluginmanager script, islaunch = pluginmanager.find_pluginscript( plugin_temp_home, plugin_name) logger.debug("Got script: %s", script) from ion.plugin.loader import cache ret = cache.load_module(plugin_name, script) cls = cache.get_plugin(plugin_name) p = cls() final_name = p.name # what the plugin calls itself, regardless of ZIP file name logger.info("Plugin calls itself: '%s'", final_name) except: logger.exception("Unable to interrogate plugin name from: '%s'", zipfile) final_name = plugin_name #move to the plugin dir # New extract_zip removes prefix from extracted files. # But still writes to file_name try: final_install_dir = os.path.join(settings.PLUGIN_PATH, final_name) if os.path.exists(final_install_dir) and (final_install_dir != settings.PLUGIN_PATH): logger.info("Deleting old copy of plugin at '%s'", final_install_dir) delete_that_folder( final_install_dir, "Error Deleting old copy of plugin at '%s'" % final_install_dir) parent_folder = os.path.dirname(final_install_dir) if not os.path.exists(parent_folder): logger.info("Creating path for plugin '%s' for '%s'", parent_folder, final_install_dir) os.makedirs(parent_folder, 0555) logger.info( "Moving plugin from temp extract folder '%s' to final location: '%s'", plugin_temp_home, final_install_dir) shutil.move(plugin_temp_home, final_install_dir) delete_that_folder(scratch_path, "Deleting plugin install scratch folder") except (IOError, OSError): logger.exception( "Failed to move plugin from temp extract folder '%s' to final location: '%s'", plugin_temp_home, final_install_dir) raise # Now that it has been downloaded, # convert pre-plugin into real db plugin object try: from iondb.plugins.manager import pluginmanager (new_plugin, updated) = pluginmanager.install(final_name, final_install_dir) except ValueError: logger.exception("Failed to install plugin") #delete_that_folder(final_install_dir) return { "plugin": final_name, "path": final_install_dir, "files": files, }
def unzipPlugin(zipfile, logger=None): if not logger: logger = logging.getLogger(__name__) ## Extract plugin to scratch folder. When complete, move to final location. plugin_path, ext = os.path.splitext(zipfile) plugin_name = os.path.basename(plugin_path) # ZIP file must named with plugin name - fragile # FIXME - handle (1) additions (common for multiple downloads via browser) # FIXME - handle version string in ZIP archive name scratch_path = os.path.join(settings.PLUGIN_PATH,"scratch","install-temp",plugin_name) (prefix, files) = extract_zip(zipfile, scratch_path, auto_prefix=True, logger=logger) if prefix: plugin_name = os.path.basename(prefix) plugin_temp_home = os.path.join(scratch_path, prefix) try: # Convert script into PluginClass, get info by introspection from iondb.plugins.manager import pluginmanager script, islaunch = pluginmanager.find_pluginscript(plugin_temp_home, plugin_name) logger.debug("Got script: %s", script) from ion.plugin.loader import cache ret = cache.load_module(plugin_name, script) cls = cache.get_plugin(plugin_name) p = cls() final_name = p.name # what the plugin calls itself, regardless of ZIP file name logger.info("Plugin calls itself: '%s'", final_name) except: logger.exception("Unable to interrogate plugin name from: '%s'", zipfile) final_name = plugin_name #move to the plugin dir # New extract_zip removes prefix from extracted files. # But still writes to file_name try: final_install_dir = os.path.join(settings.PLUGIN_PATH, final_name) if os.path.exists(final_install_dir) and (final_install_dir != settings.PLUGIN_PATH): logger.info("Deleting old copy of plugin at '%s'", final_install_dir) delete_that_folder(final_install_dir, "Error Deleting old copy of plugin at '%s'" % final_install_dir) parent_folder = os.path.dirname(final_install_dir) if not os.path.exists(parent_folder): logger.info("Creating path for plugin '%s' for '%s'", parent_folder, final_install_dir) os.makedirs(parent_folder, 0555) logger.info("Moving plugin from temp extract folder '%s' to final location: '%s'", plugin_temp_home, final_install_dir) shutil.move(plugin_temp_home, final_install_dir) delete_that_folder(scratch_path, "Deleting plugin install scratch folder") except (IOError, OSError): logger.exception("Failed to move plugin from temp extract folder '%s' to final location: '%s'", plugin_temp_home, final_install_dir) raise # Now that it has been downloaded, # convert pre-plugin into real db plugin object try: from iondb.plugins.manager import pluginmanager (new_plugin, updated) = pluginmanager.install(final_name, final_install_dir) except ValueError: logger.exception("Failed to install plugin") #delete_that_folder(final_install_dir) return { "plugin": final_name, "path": final_install_dir, "files": files, }
def SGEPluginJob(start_json, hold=False): """ Spawn a thread that will start a SGE job, and wait for it to return after it has return it will make a PUT the API using to update the status of the run args is a dict of all the args that are needed by for the plugin to run """ try: os.umask(0000) plugin_output = start_json['runinfo']['results_dir'] #Make sure the dirs exist if not os.path.exists(plugin_output): os.makedirs(plugin_output, 0775) plugin = start_json['runinfo']['plugin'] logger.info("Preparing for SGE submission - plugin %s --v%s on result %s (%s)", plugin['name'], plugin['version'], start_json['expmeta']['results_name'], start_json['runinfo']['pk']) # Branch for launch.sh vs 3.0 plugin class #logger.debug("Finding plugin definition: '%s':'%s'", plugin['name'], plugin['path']) (launch, isCompat) = pluginmanager.find_pluginscript(plugin['path'], plugin['name']) analysis_name = os.path.basename(start_json['runinfo']['analysis_dir']) if not launch or not os.path.exists(launch): logger.error("Analysis: %s. Path to plugin script: '%s' Does Not Exist!", analysis_name, launch) return 1 # Create individual launch script from template and plugin launch.sh launcher = PluginRunner() if isCompat: launchScript = launcher.createPluginWrapper(launch, start_json) else: start_json.update({'command': ["python %s -vv" % launch]}) launchScript = launcher.createPluginWrapper(None, start_json) launchWrapper = launcher.writePluginLauncher(plugin_output, plugin['name'], launchScript) # Returns filename of startpluginjson file, passed to script below startpluginjson = launcher.writePluginJson(start_json) # Prepare drmaa Job - SGE/gridengine only jt = _session.createJobTemplate() jt.nativeSpecification = " -q %s" % ("plugin.q") if Feature.EXPORT in plugin['features']: jt.nativeSpecification += ' -l ep=1 ' hold_jid = plugin.get('hold_jid', []) if hold_jid: jt.nativeSpecification += " -hold_jid " + ','.join(str(jobid) for jobid in hold_jid) jt.workingDirectory = plugin_output jt.outputPath = ":" + os.path.join(plugin_output, "drmaa_stdout.txt") jt.joinFiles = True # Merge stdout and stderr # Plugin command is: ion_pluginname_launch.sh -j startplugin.json jt.remoteCommand = launchWrapper jt.args = [ "-j", startpluginjson ] if hold: jt.jobSubmissionState = drmaa.JobSubmissionState.HOLD_STATE # Submit the job to drmaa jobid = _session.runJob(jt) logger.info("Analysis: %s. Plugin: %s. Job: %s Queued." % (analysis_name, plugin['name'], jobid)) _session.deleteJobTemplate(jt) return jobid except: logger.exception("SGEPluginJob method failure") raise
def SGEPluginJob(start_json): """ Spawn a thread that will start a SGE job, and wait for it to return after it has return it will make a PUT the API using to update the status of the run args is a dict of all the args that are needed by for the plugin to run """ try: os.umask(0000) # Query some essential values plugin = start_json['runinfo']['plugin'] resultpk = start_json['runinfo']['pk'] analysis_name = os.path.basename(start_json['runinfo']['analysis_dir']) plugin_output = start_json['runinfo']['results_dir'] logger.debug("Getting ready for queue : %s %s", resultpk, plugin['name']) #Make sure the dirs exist if not os.path.exists(plugin_output): os.makedirs(plugin_output, 0755) # Write start_json to startplugin.json file. startpluginjson = os.path.join(plugin_output,'startplugin.json') with open(startpluginjson,"w") as fp: json.dump(start_json,fp,indent=2) # Branch for launch.sh vs 3.0 plugin class logger.debug("Finding plugin definition: '%s':'%s'", plugin['name'], plugin['path']) (launch, isCompat) = pluginmanager.find_pluginscript(plugin['path'], plugin['name']) if not launch or not os.path.exists(launch): logger.error("Analysis: %s. Path to plugin script: '%s' Does Not Exist!", analysis_name, launch) return 1 # Create individual launch script from template and plugin launch.sh launcher = PluginRunner() if isCompat: launchScript = launcher.createPluginWrapper(launch, start_json) else: start_json.update({'command': ["python %s -vv" % launch]}) launchScript = launcher.createPluginWrapper(None, start_json) launchWrapper = launcher.writePluginLauncher(plugin_output, plugin['name'], launchScript) launcher.writePluginJson(start_json) # Prepare drmaa Job - SGE/gridengine only jt = _session.createJobTemplate() jt.nativeSpecification = " -q %s" % ("plugin.q") hold = plugin.get('hold_jid', []) if hold: jt.nativeSpecification += " -hold_jid " + ','.join(str(jobid) for jobid in hold) jt.workingDirectory = plugin_output jt.outputPath = ":" + os.path.join(plugin_output, "drmaa_stdout.txt") jt.joinFiles = True # Merge stdout and stderr # Plugin command is: ion_pluginname_launch.sh -j startplugin.json jt.remoteCommand = launchWrapper jt.args = [ "-j", startpluginjson ] # Submit the job to drmaa jobid = _session.runJob(jt) # Update pluginresult status h = httplib2.Http() headers = {"Content-type": "application/json","Accept": "application/json"} url = 'http://localhost' + '/rundb/api/v1/results/%s/plugin/' % resultpk resp, content = h.request(url, "PUT", body=json.dumps({plugin['name']:"Queued"}), headers=headers) logger.debug("PluginResult Marked Queued : %s %s" % (resp, content)) logger.info("Analysis: %s. Plugin: %s. Job: %s Queued." % (analysis_name, plugin['name'], jobid)) _session.deleteJobTemplate(jt) return jobid except: logger.critical("SGEPluginJob method failure") logger.critical(traceback.format_exc())