def init_pygdb(logpath=None): """Initialize pygdb with logging NOTE: 1) A pygdb instance is needed at the top-most-level of the debugged python program 2) When debugging NON-daemons make sure to use 'cgi-bin' USAGE cgi-bin: 1) At top-most-level: cgi-bin/X.py: from mig.shared.debug import init_pygdb pygdb = init_pygdb() 2) In any of the descendant modules: import pygdb.breakpoint pygdb.breakpoint.set() """ configuration = get_configuration_object(skip_log=True) if not hasattr(configuration, 'gdb_logger'): if not logpath: logpath = os.path.join(configuration.log_dir, "gdb.log") logger = configuration.gdb_logger = daemon_logger( "gdb", level=configuration.loglevel, path=logpath) else: logger = configuration.gdb_logger if not pygdb: msg = "The python pygdb module is missing" logger.error(msg) raise RuntimeError(msg) pygdb.breakpoint.enable(logger=logger) return pygdb
def init_cgiscript_possibly_with_cert(print_header=True, content_type='text/html'): """Prepare for CGI script with optional client certificate. Only used from some of the cgi scripts still on the legacy-form like requestnewjob and put. I.e. scripts where certs are not required due to use of sessionid. """ # Always rely on os.environ here since only called from cgi scripts environ = os.environ if print_header: cgiscript_header(content_type=content_type) configuration = get_configuration_object() logger = configuration.logger out = CGIOutput(logger) # get DN of user currently logged in client_id = extract_client_id(configuration, environ) if not client_id: logger.debug('(No client ID available in SSL session)') logger.info('script: %s cert: %s' % (requested_page(), client_id)) return (logger, configuration, client_id, out)
def signature(): defaults = {'job_id': REJECT_UNSET} configuration = get_configuration_object() external_dict = mrslkeywords.get_keywords_dict(configuration) for (key, value_dict) in external_dict.iteritems(): if key not in defaults: # do not overwrite defaults[key] = [] return ['html_form', defaults]
def main(): configuration = get_configuration_object() # Overwrite default logger logger = configuration.logger = get_logger(logging.INFO) logger = configuration.logger = get_logger(logging.INFO) vgrids_dict = unpickle(TRIGGER_DICT_FILE, logger) vgrid_list = get_vgrids_dict(vgrids_dict) for name in vgrid_list: print(name)
def main(): status = True configuration = get_configuration_object() logger = configuration.logger = get_logger(logging.INFO) # Overwrite default logger argc = len(sys.argv) - 1 user_vgrid_list = None if argc == 1: user_vgrid_list = [vgrid.strip() for vgrid in sys.argv[1].split(',')] logger.info('Using custom vgrid_list: %s' % user_vgrid_list) vgrids_dict = unpickle(TRIGGER_DICT_FILE, logger) update_trigger_dict = None if vgrids_dict: (vgrids_dict, vgrid_list) = filter_vgrids_dict(configuration, vgrids_dict, user_vgrid_list) else: status = False logger.error("Missing vgrid dict file: '%s'" % TRIGGER_DICT_FILE) if status: status = backup_trigger_files(configuration, vgrid_list) if status: status = backup_imagesettings_files(configuration, vgrid_list) if status: status = backup_paraview_links(configuration, vgrid_list) if status: update_trigger_dict = \ get_update_trigger_dict_and_check_for_unique_clientid(configuration, vgrids_dict) if update_trigger_dict is None: status = False if status: status = remove_triggers(configuration, vgrids_dict) if status: status = update_backend(configuration, update_trigger_dict) if status: return 0 else: return 1
def signature(): defaults = {} configuration = get_configuration_object() show_fields = get_job_specs(configuration) for (key, specs) in show_fields: if key not in defaults: # make sure required fields are set but do not overwrite if specs['Required']: defaults[key] = REJECT_UNSET else: defaults[key] = [] return ['jobobj', defaults]
def init_cgi_script(environ, delayed_input=None): """Shared init""" configuration = get_configuration_object() logger = configuration.logger # get and log ID of user currently logged in client_id = extract_client_id(configuration, environ) logger.info('script: %s cert: %s' % (requested_page(), client_id)) if not delayed_input: fieldstorage = cgi.FieldStorage() user_arguments_dict = fieldstorage_to_dict(fieldstorage) else: user_arguments_dict = {'__DELAYED_INPUT__': delayed_input} return (configuration, logger, client_id, user_arguments_dict)
def bailout_title(configuration=None, title_text=""): """Helper to handle very basic title output in a failsafe way""" # Hide menu to avoid message truncation title = { 'object_type': 'title', 'text': title_text, 'skipmenu': True, 'style': '', 'script': '' } _logger = None try: if not configuration: from mig.shared.conf import get_configuration_object configuration = get_configuration_object() _logger = configuration.logger from mig.shared.html import themed_styles, themed_scripts title['style'] = themed_styles(configuration) title['script'] = themed_scripts(configuration, logged_in=False) except Exception as exc: if _logger: _logger.error("failed to provide even basic styling for title") return title
def get_resource_config_dict(configuration, config_file): """Find and return configuration dictionary in provided conf file""" if not configuration: configuration = get_configuration_object() result = parse(config_file) external_dict = resconf_get_keywords_dict(configuration) # The Configfile has the right structure # Check if the types are correct too (status, msg) = check_types(result, external_dict, configuration) if not status: return (False, 'Parse failed (typecheck) ' + msg, external_dict) global_dict = {} for (key, value_dict) in external_dict.iteritems(): global_dict[key] = value_dict['Value'] return (status, msg, global_dict)
def main(): configuration = get_configuration_object() # Overwrite default logger logger = configuration.logger = get_logger(logging.INFO) vgrids_dict = {} logger.info('==================== Filling vgrids ====================') fh = open(SETTINGS_LIST) for line in fh: line = line.strip() if len(line) > 0: status = fill_vgrids(configuration, line, vgrids_dict) if not status: break fh.close() if status: logger.info( '==================== Filling triggers ====================') status = fill_triggers(configuration, vgrids_dict) if status: logger.info( '==================== Writing triggers dict ====================') logger.info("'Pickle to file: '%s'" % VGRID_DICT_FILE) status = pickle(vgrids_dict, VGRID_DICT_FILE, logger) if status: return 0 else: return 1
if not fnmatch.fnmatch("%s" % val, entry[name].lstrip('0')): _logger.debug("cron_match failed on %s: %s vs %s" % (name, val, entry[name])) return False return True def at_remain(configuration, at_time, entry): """Return the number of minutes remaining before entry should run""" _logger = configuration.logger return int((entry['time_stamp'] - at_time).total_seconds() / 60) if __name__ == '__main__': from mig.shared.conf import get_configuration_object conf = get_configuration_object() client_id = '/C=DK/ST=NA/L=NA/O=NBI/OU=NA/CN=Jonas Bardino/[email protected]' now = datetime.datetime.now() now = now.replace(second=0, microsecond=0) trigger_rule = { 'templates': [], 'run_as': client_id, 'rate_limit': '', 'vgrid_name': 'eScience', 'rule_id': 'test-dummy', 'match_dirs': False, 'match_files': True, 'arguments': ['+TRIGGERPATH+'], 'settle_time': '', 'path': '*.txt*', 'changes': ['modified'], 'action': 'trigger-created', 'match_recursive': True} trigger_samples = [('abc.txt', 'modified'), ('subdir/def.txt', 'modified')] print("Test trigger event map:") for (path, change) in trigger_samples: print("Expanded path vars for %s %s:" % (path, change)) expanded = get_path_expand_map(path, trigger_rule, change) for (key, val) in expanded.items():
""" html += \ '<!-- begin raw footer: this line is used by showvgridmonitor -->' html += get_xgi_html_footer(configuration, '') try: file_handle = open(html_file, 'w') file_handle.write(html) file_handle.close() except Exception as exc: print('Could not write monitor page %s: %s' % (html_file, exc)) if __name__ == '__main__': # Force no log init since we use separate logger configuration = get_configuration_object(skip_log=True) log_level = configuration.loglevel if sys.argv[1:] and sys.argv[1] in ['debug', 'info', 'warning', 'error']: log_level = sys.argv[1] # Use separate logger logger = daemon_logger("monitor", configuration.user_monitor_log, log_level) configuration.logger = logger # Allow e.g. logrotate to force log re-open after rotates register_hangup_handler(configuration) if not configuration.site_enable_jobs: err_msg = "Job support is disabled in configuration!"
elif opt == '-v': verbose = True else: print('Error: %s not supported!' % opt) if conf_path and not os.path.isfile(conf_path): print('Failed to read configuration file: %s' % conf_path) sys.exit(1) if verbose: if conf_path: print('using configuration in %s' % conf_path) else: print('using configuration from MIG_CONF (or default)') configuration = get_configuration_object(config_file=conf_path, skip_log=True) if user_id and args: print('Error: Only one kind of user specification allowed at a time') usage() sys.exit(1) if args: user_dict['full_name'] = args[0] try: user_dict['organization'] = args[1] user_dict['state'] = args[2] user_dict['country'] = args[3] user_dict['email'] = args[4] except IndexError:
usage() sys.exit(0) if conf_path and not os.path.isfile(conf_path): print('Failed to read configuration file: %s' % conf_path) sys.exit(1) if verbose: if conf_path: if verbose: print('using configuration in %s' % conf_path) else: if verbose: print('using configuration from MIG_CONF (or default)') configuration = get_configuration_object(config_file=conf_path) logger = configuration.logger if user_file and args: print('Error: Only one kind of user specification allowed at a time') usage() sys.exit(1) if args: try: peer_dict['full_name'] = args[0] peer_dict['organization'] = args[1] peer_dict['state'] = args[2] peer_dict['country'] = args[3] peer_dict['email'] = args[4] peer_dict['comment'] = args[5] except IndexError:
def run(configuration, localfile_spaces, unique_resource_name, outfile='AUTOMATIC'): """Parse configuration in localfile_spaces and write results to outfile if non-empty. The keyword AUTOMATIC is replaced by the expected resource configuration path. """ if not configuration: configuration = get_configuration_object() (status, msg, conf) = get_resource_config_dict(configuration, localfile_spaces) if not status: return (False, msg) # verify runtime environments are specified correctly if 'RUNTIMEENVIRONMENT' in conf: for re in conf['RUNTIMEENVIRONMENT']: try: (name, value) = re except Exception as err: return (False, 'Runtime environment error: %s' % err) if not is_runtime_environment(name, configuration): return ( False, "Non existing runtime environment specified ('%s'), please create the runtime environment before specifying it in resource configurations." % name) (re_dict, msg) = get_re_dict(name, configuration) if not re_dict: return (False, 'Runtime environment error, could not open (%s) %s' % (name, msg)) if 'ENVIRONMENTVARIABLE' not in re_dict: if value: # res conf has envs, but according to the template it should not return ( False, "%s should not have any environments and you specified '%s'. Details about the runtime environment <a href=showre.py?re_name=%s>here</a>" % (re, value, name)) else: continue re_dict_environments = re_dict['ENVIRONMENTVARIABLE'] re_dict_environment_names = [] for re_environment in re_dict_environments: re_dict_environment_names.append(re_environment['name']) if not len(value) == len(re_dict_environments): return ( False, "You have specified %s environments, but the runtime environment '%s' requires %s. Details about the runtime environment <a href='showre.py?re_name=%s'>here.</a>" % (len(value), name, len(re_dict_environments), name)) # we now know that the number of environments are # correct, verify that there are no name duplicates used_envnames = [] for env in value: try: (envname, _) = env if envname in used_envnames: # same envname used twice return ( False, "You have specified the environment '%s' more than once for the '%s' runtime environment." % (envname, name)) used_envnames.append(envname) except Exception as err: return ( False, 'Runtimeenvironment error: Name and value not found in env: %s' % err) # verify environment names are correct according to the # runtime environment definition do this by comparing # list of names specified for runtime environment and # res. conf. # re_dict_environment_names and used_envnames should # have the same entries! for n in re_dict_environment_names: # any build-in list comparison functionality? if not n in used_envnames: return ( False, "You have not specified an environment named '%s' which is required by the '%s' runtime environment. Details about the runtime environment <a href=showre.py?re_name=%s>here.</a>" % (n, name, name)) # check VGrid access vgrid_label = configuration.site_vgrid_label for (unit_config, unit_name) in (('EXECONFIG', '+EXENAME+'), ('STORECONFIG', '+STORENAME+')): for res_unit in conf[unit_config]: # replace unit_name with specified res_unit name for res_unit_key in res_unit.keys(): if type(res_unit[res_unit_key]) == type(''): res_unit[res_unit_key] = res_unit[res_unit_key].replace( unit_name, res_unit['name']) # verify resource is in specified vgrid vgrid_name = res_unit['vgrid'] # print "vgrid_name in res_unit" + vgrid_name if vgrid_name == '': # ok pass else: if type(vgrid_name) == type([]): # list for vgrid in vgrid_name: if not vgrid_is_default(vgrid) and not \ vgrid_is_resource(vgrid, unique_resource_name, configuration): return (False, """Your resource is not allowed in the %s '%s' specified in the configuation for the '%s' resource unit. Please contact the %s owner and ask if you can be included in the %s.""" % (vgrid_label, vgrid, res_unit['name'], vgrid_label, vgrid_label)) else: # string if not vgrid_is_default(vgrid) and not vgrid_is_resource( vgrid_name, unique_resource_name, configuration): return (False, """Your resource is not allowed in the %s '%s' specified in the configuation for the '%s' resource unit. Please contact the %s owner and ask if you can be included in the %s.""" % (vgrid_label, vgrid_name, res_unit['name'], vgrid_label, vgrid_label)) # save dictionary to a file if outfile == 'AUTOMATIC': # save configuration as python dictionary in the resource' private directory filename = configuration.resource_home + unique_resource_name\ + '/config' elif outfile: # outfile specified (DumpConfig) filename = outfile else: return (True, 'Everything ok') try: fsock = open(filename, 'w') st = dumps(conf, 0) fsock.write(st) fsock.close() except Exception as err: return (False, "Fatal error: could not open '" + filename + "' for writing!" + '\n Msg: ' + str(err)) return (True, 'Everything ok, config updated')