def init_pygdb(logpath=None):
"""Initialize pygdb with logging
NOTE: 1) A pygdb instance is needed at the top-most-level
of the debugged python program
2) When debugging NON-daemons make sure to use 'cgi-bin'
USAGE cgi-bin:
1) At top-most-level: cgi-bin/X.py:
from mig.shared.debug import init_pygdb
pygdb = init_pygdb()
2) In any of the descendant modules:
import pygdb.breakpoint
pygdb.breakpoint.set()
"""
configuration = get_configuration_object(skip_log=True)
if not hasattr(configuration, 'gdb_logger'):
if not logpath:
logpath = os.path.join(configuration.log_dir, "gdb.log")
logger = configuration.gdb_logger = daemon_logger(
"gdb", level=configuration.loglevel, path=logpath)
else:
logger = configuration.gdb_logger
if not pygdb:
msg = "The python pygdb module is missing"
logger.error(msg)
raise RuntimeError(msg)
pygdb.breakpoint.enable(logger=logger)
return pygdb
def init_cgiscript_possibly_with_cert(print_header=True,
content_type='text/html'):
"""Prepare for CGI script with optional client certificate. Only used from
some of the cgi scripts still on the legacy-form like requestnewjob and
put. I.e. scripts where certs are not required due to use of sessionid.
"""
# Always rely on os.environ here since only called from cgi scripts
environ = os.environ
if print_header:
cgiscript_header(content_type=content_type)
configuration = get_configuration_object()
logger = configuration.logger
out = CGIOutput(logger)
# get DN of user currently logged in
client_id = extract_client_id(configuration, environ)
if not client_id:
logger.debug('(No client ID available in SSL session)')
logger.info('script: %s cert: %s' % (requested_page(), client_id))
return (logger, configuration, client_id, out)
def signature():
defaults = {'job_id': REJECT_UNSET}
configuration = get_configuration_object()
external_dict = mrslkeywords.get_keywords_dict(configuration)
for (key, value_dict) in external_dict.iteritems():
if key not in defaults:
# do not overwrite
defaults[key] = []
return ['html_form', defaults]
def main():
configuration = get_configuration_object()
# Overwrite default logger
logger = configuration.logger = get_logger(logging.INFO)
logger = configuration.logger = get_logger(logging.INFO)
vgrids_dict = unpickle(TRIGGER_DICT_FILE, logger)
vgrid_list = get_vgrids_dict(vgrids_dict)
for name in vgrid_list:
print(name)
def main():
status = True
configuration = get_configuration_object()
logger = configuration.logger = get_logger(logging.INFO)
# Overwrite default logger
argc = len(sys.argv) - 1
user_vgrid_list = None
if argc == 1:
user_vgrid_list = [vgrid.strip() for vgrid in sys.argv[1].split(',')]
logger.info('Using custom vgrid_list: %s' % user_vgrid_list)
vgrids_dict = unpickle(TRIGGER_DICT_FILE, logger)
update_trigger_dict = None
if vgrids_dict:
(vgrids_dict, vgrid_list) = filter_vgrids_dict(configuration,
vgrids_dict,
user_vgrid_list)
else:
status = False
logger.error("Missing vgrid dict file: '%s'" % TRIGGER_DICT_FILE)
if status:
status = backup_trigger_files(configuration, vgrid_list)
if status:
status = backup_imagesettings_files(configuration, vgrid_list)
if status:
status = backup_paraview_links(configuration, vgrid_list)
if status:
update_trigger_dict = \
get_update_trigger_dict_and_check_for_unique_clientid(configuration,
vgrids_dict)
if update_trigger_dict is None:
status = False
if status:
status = remove_triggers(configuration, vgrids_dict)
if status:
status = update_backend(configuration, update_trigger_dict)
if status:
return 0
else:
return 1
def signature():
defaults = {}
configuration = get_configuration_object()
show_fields = get_job_specs(configuration)
for (key, specs) in show_fields:
if key not in defaults:
# make sure required fields are set but do not overwrite
if specs['Required']:
defaults[key] = REJECT_UNSET
else:
defaults[key] = []
return ['jobobj', defaults]
def init_cgi_script(environ, delayed_input=None):
"""Shared init"""
configuration = get_configuration_object()
logger = configuration.logger
# get and log ID of user currently logged in
client_id = extract_client_id(configuration, environ)
logger.info('script: %s cert: %s' % (requested_page(), client_id))
if not delayed_input:
fieldstorage = cgi.FieldStorage()
user_arguments_dict = fieldstorage_to_dict(fieldstorage)
else:
user_arguments_dict = {'__DELAYED_INPUT__': delayed_input}
return (configuration, logger, client_id, user_arguments_dict)
def bailout_title(configuration=None, title_text=""):
"""Helper to handle very basic title output in a failsafe way"""
# Hide menu to avoid message truncation
title = {
'object_type': 'title',
'text': title_text,
'skipmenu': True,
'style': '',
'script': ''
}
_logger = None
try:
if not configuration:
from mig.shared.conf import get_configuration_object
configuration = get_configuration_object()
_logger = configuration.logger
from mig.shared.html import themed_styles, themed_scripts
title['style'] = themed_styles(configuration)
title['script'] = themed_scripts(configuration, logged_in=False)
except Exception as exc:
if _logger:
_logger.error("failed to provide even basic styling for title")
return title
def get_resource_config_dict(configuration, config_file):
"""Find and return configuration dictionary in provided
conf file"""
if not configuration:
configuration = get_configuration_object()
result = parse(config_file)
external_dict = resconf_get_keywords_dict(configuration)
# The Configfile has the right structure
# Check if the types are correct too
(status, msg) = check_types(result, external_dict, configuration)
if not status:
return (False, 'Parse failed (typecheck) ' + msg, external_dict)
global_dict = {}
for (key, value_dict) in external_dict.iteritems():
global_dict[key] = value_dict['Value']
return (status, msg, global_dict)
def main():
configuration = get_configuration_object()
# Overwrite default logger
logger = configuration.logger = get_logger(logging.INFO)
vgrids_dict = {}
logger.info('==================== Filling vgrids ====================')
fh = open(SETTINGS_LIST)
for line in fh:
line = line.strip()
if len(line) > 0:
status = fill_vgrids(configuration, line, vgrids_dict)
if not status:
break
fh.close()
if status:
logger.info(
'==================== Filling triggers ====================')
status = fill_triggers(configuration, vgrids_dict)
if status:
logger.info(
'==================== Writing triggers dict ====================')
logger.info("'Pickle to file: '%s'" % VGRID_DICT_FILE)
status = pickle(vgrids_dict, VGRID_DICT_FILE, logger)
if status:
return 0
else:
return 1
if not fnmatch.fnmatch("%s" % val, entry[name].lstrip('0')):
_logger.debug("cron_match failed on %s: %s vs %s" %
(name, val, entry[name]))
return False
return True
def at_remain(configuration, at_time, entry):
"""Return the number of minutes remaining before entry should run"""
_logger = configuration.logger
return int((entry['time_stamp'] - at_time).total_seconds() / 60)
if __name__ == '__main__':
from mig.shared.conf import get_configuration_object
conf = get_configuration_object()
client_id = '/C=DK/ST=NA/L=NA/O=NBI/OU=NA/CN=Jonas Bardino/[email protected]'
now = datetime.datetime.now()
now = now.replace(second=0, microsecond=0)
trigger_rule = {
'templates': [], 'run_as': client_id, 'rate_limit': '',
'vgrid_name': 'eScience', 'rule_id': 'test-dummy', 'match_dirs': False,
'match_files': True, 'arguments': ['+TRIGGERPATH+'], 'settle_time': '',
'path': '*.txt*', 'changes': ['modified'], 'action': 'trigger-created',
'match_recursive': True}
trigger_samples = [('abc.txt', 'modified'), ('subdir/def.txt', 'modified')]
print("Test trigger event map:")
for (path, change) in trigger_samples:
print("Expanded path vars for %s %s:" % (path, change))
expanded = get_path_expand_map(path, trigger_rule, change)
for (key, val) in expanded.items():
"""
html += \
'<!-- begin raw footer: this line is used by showvgridmonitor -->'
html += get_xgi_html_footer(configuration, '')
try:
file_handle = open(html_file, 'w')
file_handle.write(html)
file_handle.close()
except Exception as exc:
print('Could not write monitor page %s: %s' % (html_file, exc))
if __name__ == '__main__':
# Force no log init since we use separate logger
configuration = get_configuration_object(skip_log=True)
log_level = configuration.loglevel
if sys.argv[1:] and sys.argv[1] in ['debug', 'info', 'warning', 'error']:
log_level = sys.argv[1]
# Use separate logger
logger = daemon_logger("monitor", configuration.user_monitor_log,
log_level)
configuration.logger = logger
# Allow e.g. logrotate to force log re-open after rotates
register_hangup_handler(configuration)
if not configuration.site_enable_jobs:
err_msg = "Job support is disabled in configuration!"
elif opt == '-v':
verbose = True
else:
print('Error: %s not supported!' % opt)
if conf_path and not os.path.isfile(conf_path):
print('Failed to read configuration file: %s' % conf_path)
sys.exit(1)
if verbose:
if conf_path:
print('using configuration in %s' % conf_path)
else:
print('using configuration from MIG_CONF (or default)')
configuration = get_configuration_object(config_file=conf_path,
skip_log=True)
if user_id and args:
print('Error: Only one kind of user specification allowed at a time')
usage()
sys.exit(1)
if args:
user_dict['full_name'] = args[0]
try:
user_dict['organization'] = args[1]
user_dict['state'] = args[2]
user_dict['country'] = args[3]
user_dict['email'] = args[4]
except IndexError:
usage()
sys.exit(0)
if conf_path and not os.path.isfile(conf_path):
print('Failed to read configuration file: %s' % conf_path)
sys.exit(1)
if verbose:
if conf_path:
if verbose:
print('using configuration in %s' % conf_path)
else:
if verbose:
print('using configuration from MIG_CONF (or default)')
configuration = get_configuration_object(config_file=conf_path)
logger = configuration.logger
if user_file and args:
print('Error: Only one kind of user specification allowed at a time')
usage()
sys.exit(1)
if args:
try:
peer_dict['full_name'] = args[0]
peer_dict['organization'] = args[1]
peer_dict['state'] = args[2]
peer_dict['country'] = args[3]
peer_dict['email'] = args[4]
peer_dict['comment'] = args[5]
except IndexError:
def run(configuration,
localfile_spaces,
unique_resource_name,
outfile='AUTOMATIC'):
"""Parse configuration in localfile_spaces and write results to outfile
if non-empty. The keyword AUTOMATIC is replaced by the expected resource
configuration path.
"""
if not configuration:
configuration = get_configuration_object()
(status, msg, conf) = get_resource_config_dict(configuration,
localfile_spaces)
if not status:
return (False, msg)
# verify runtime environments are specified correctly
if 'RUNTIMEENVIRONMENT' in conf:
for re in conf['RUNTIMEENVIRONMENT']:
try:
(name, value) = re
except Exception as err:
return (False, 'Runtime environment error: %s' % err)
if not is_runtime_environment(name, configuration):
return (
False,
"Non existing runtime environment specified ('%s'), please create the runtime environment before specifying it in resource configurations."
% name)
(re_dict, msg) = get_re_dict(name, configuration)
if not re_dict:
return (False,
'Runtime environment error, could not open (%s) %s' %
(name, msg))
if 'ENVIRONMENTVARIABLE' not in re_dict:
if value:
# res conf has envs, but according to the template it should not
return (
False,
"%s should not have any environments and you specified '%s'. Details about the runtime environment <a href=showre.py?re_name=%s>here</a>"
% (re, value, name))
else:
continue
re_dict_environments = re_dict['ENVIRONMENTVARIABLE']
re_dict_environment_names = []
for re_environment in re_dict_environments:
re_dict_environment_names.append(re_environment['name'])
if not len(value) == len(re_dict_environments):
return (
False,
"You have specified %s environments, but the runtime environment '%s' requires %s. Details about the runtime environment <a href='showre.py?re_name=%s'>here.</a>"
% (len(value), name, len(re_dict_environments), name))
# we now know that the number of environments are
# correct, verify that there are no name duplicates
used_envnames = []
for env in value:
try:
(envname, _) = env
if envname in used_envnames:
# same envname used twice
return (
False,
"You have specified the environment '%s' more than once for the '%s' runtime environment."
% (envname, name))
used_envnames.append(envname)
except Exception as err:
return (
False,
'Runtimeenvironment error: Name and value not found in env: %s'
% err)
# verify environment names are correct according to the
# runtime environment definition do this by comparing
# list of names specified for runtime environment and
# res. conf.
# re_dict_environment_names and used_envnames should
# have the same entries!
for n in re_dict_environment_names:
# any build-in list comparison functionality?
if not n in used_envnames:
return (
False,
"You have not specified an environment named '%s' which is required by the '%s' runtime environment. Details about the runtime environment <a href=showre.py?re_name=%s>here.</a>"
% (n, name, name))
# check VGrid access
vgrid_label = configuration.site_vgrid_label
for (unit_config, unit_name) in (('EXECONFIG', '+EXENAME+'),
('STORECONFIG', '+STORENAME+')):
for res_unit in conf[unit_config]:
# replace unit_name with specified res_unit name
for res_unit_key in res_unit.keys():
if type(res_unit[res_unit_key]) == type(''):
res_unit[res_unit_key] = res_unit[res_unit_key].replace(
unit_name, res_unit['name'])
# verify resource is in specified vgrid
vgrid_name = res_unit['vgrid']
# print "vgrid_name in res_unit" + vgrid_name
if vgrid_name == '':
# ok
pass
else:
if type(vgrid_name) == type([]):
# list
for vgrid in vgrid_name:
if not vgrid_is_default(vgrid) and not \
vgrid_is_resource(vgrid, unique_resource_name,
configuration):
return (False,
"""Your resource is not allowed in the %s
'%s' specified in the configuation for the '%s' resource unit. Please contact
the %s owner and ask if you can be included in the %s.""" %
(vgrid_label, vgrid, res_unit['name'],
vgrid_label, vgrid_label))
else:
# string
if not vgrid_is_default(vgrid) and not vgrid_is_resource(
vgrid_name, unique_resource_name, configuration):
return (False,
"""Your resource is not allowed in the %s '%s'
specified in the configuation for the '%s' resource unit. Please contact the %s
owner and ask if you can be included in the %s.""" %
(vgrid_label, vgrid_name, res_unit['name'],
vgrid_label, vgrid_label))
# save dictionary to a file
if outfile == 'AUTOMATIC':
# save configuration as python dictionary in the resource' private directory
filename = configuration.resource_home + unique_resource_name\
+ '/config'
elif outfile:
# outfile specified (DumpConfig)
filename = outfile
else:
return (True, 'Everything ok')
try:
fsock = open(filename, 'w')
st = dumps(conf, 0)
fsock.write(st)
fsock.close()
except Exception as err:
return (False, "Fatal error: could not open '" + filename +
"' for writing!" + '\n Msg: ' + str(err))
return (True, 'Everything ok, config updated')
