def load_and_validate_config_files(self):
config_files = [
self.get_server_config_path(),
self.get_version_path(),
self.get_hosts_config_path(),
self.get_host_access_control_config_path(),
get_irods_environment_path()
]
if os.path.exists(self.get_database_config_path()):
config_files.append(self.get_database_config_path())
config_dicts = dict([(path, load_json_config_file(path))
for path in config_files])
try:
server_config_dict = config_dicts[self.get_server_config_path()]
base_uri = server_config_dict['schema_validation_base_uri']
except KeyError:
base_uri = None
if self.verbose:
print('{0} did not contain \'{1}\''.format(
self.get_server_config_path(),
'schema_validation_base_uri'),
file=sys.stderr)
try:
version_dict = config_dicts[self.get_version_path()]
uri_version = version_dict['configuration_schema_version']
except KeyError:
uri_version = None
if self.verbose:
print('{0} did not contain \'{1}\''.format(
self.get_version_path(), 'configuration_schema_version'),
file=sys.stderr)
if base_uri and uri_version:
validation_uri_prefix = '/'.join(
[base_uri, 'v{0}'.format(uri_version)])
for path, json_dict in config_dicts.items():
schema_uri = '/'.join(
[validation_uri_prefix,
os.path.basename(path)])
try:
validate_json.validate_dict(json_dict,
schema_uri,
name=path,
verbose=self.verbose)
except validate_json.ValidationWarning as e:
if self.verbose:
print(e, file=sys.stderr)
except validate_json.ValidationError as e:
irods_six.reraise(IrodsControllerError, e,
sys.exc_info()[2])
elif self.verbose:
print('Preflight Check problem:',
'JSON Configuration Validation failed.',
sep='\n\t',
file=sys.stderr)
return config_dicts
def stop(self, timeout=20):
if self.verbose:
print('Stopping iRODS server... ', end='')
try:
if self.get_binary_to_pids_dict([self.get_server_executable()]):
try:
self.irods_grid_shutdown(timeout=timeout)
except Exception as e:
if self.verbose:
print('Error encountered in graceful shutdown.',
e,
sep='\n',
file=sys.stderr)
else:
if self.verbose:
print('No iRODS servers running. ',
file=sys.stderr,
end='')
# kill servers first to stop spawning of other processes
server_pids_dict = self.get_binary_to_pids_dict(
[self.get_server_executable()])
if server_pids_dict:
if self.verbose:
print(
'iRODS server processes remain after "irods-grid shutdown".',
format_binary_to_pids_dict(server_pids_dict),
'Killing forcefully...',
sep='\n',
file=sys.stderr)
for pid in server_pids_dict[self.get_server_executable()]:
try:
kill_pid(pid)
except psutil.NoSuchProcess:
pass
delete_cache_files_by_pid(pid)
binary_to_pids_dict = self.get_binary_to_pids_dict()
if binary_to_pids_dict:
if self.verbose:
print(
'iRODS child processes remain after "irods-grid shutdown".',
format_binary_to_pids_dict(binary_to_pids_dict),
'Killing forcefully...',
sep='\n',
file=sys.stderr)
for binary, pids in binary_to_pids_dict.items():
for pid in pids:
try:
kill_pid(pid)
except psutil.NoSuchProcess:
pass
delete_cache_files_by_pid(pid)
except IrodsControllerError as e:
if self.verbose:
print('Failure')
irods_six.reraise(IrodsControllerError, e, sys.exc_info()[2])
if self.verbose:
print('Success')
def validate_dict(config_dict, schema_uri, name=None, verbose=False):
if name is None:
name = schema_uri.rpartition('/')[2]
try:
e = jsonschema.exceptions
except AttributeError:
irods_six.reraise(ValidationWarning, ValidationWarning(
'WARNING: Validation failed for {0} -- jsonschema too old v[{1}]'.format(
name, jsonschema.__version__)),
sys.exc_info()[2])
except NameError:
irods_six.reraise(ValidationWarning, ValidationWarning(
'WARNING: Validation failed for {0} -- jsonschema not installed'.format(
name)),
sys.exc_info()[2])
try:
# load the schema url
try:
response = requests.get(schema_uri)
except NameError:
irods_six.reraise(ValidationError, ValidationError(
'WARNING: Validation failed for {0} -- requests not installed'.format(
name)),
sys.exc_info()[2])
# check response values
try:
# modern requests
schema = json.loads(response.text)
except AttributeError:
# requests pre-v1.0.0
response.encoding = 'utf8'
schema = json.loads(response.content)
# validate
jsonschema.validate(config_dict, schema)
except (
jsonschema.exceptions.RefResolutionError, # could not resolve recursive schema $ref
ValueError # most network errors and 404s
) as e:
irods_six.reraise(ValidationWarning, ValidationWarning('\n\t'.join([
'WARNING: Validation Failed for [{0}]:'.format(name),
'against [{0}]'.format(schema_uri),
'{0}: {1}'.format(e.__class__.__name__, e)])),
sys.exc_info()[2])
except (
jsonschema.exceptions.ValidationError,
jsonschema.exceptions.SchemaError,
BaseException
) as e:
irods_six.reraise(ValidationError, ValidationError('\n\t'.join([
'ERROR: Validation Failed for [{0}]:'.format(name),
'against [{0}]'.format(schema_uri),
'{0}: {1}'.format(e.__class__.__name__, e)])),
sys.exc_info()[2])
if verbose and name:
print("Validating [{0}]... Success".format(name))
def load_and_validate_config_files(self):
config_files = [
self.get_server_config_path(),
self.get_version_path(),
self.get_hosts_config_path(),
self.get_host_access_control_config_path(),
get_irods_environment_path()]
if os.path.exists(self.get_database_config_path()):
config_files.append(self.get_database_config_path())
config_dicts = dict([(path, load_json_config_file(path)) for path in config_files])
try :
server_config_dict = config_dicts[self.get_server_config_path()]
base_uri = server_config_dict['schema_validation_base_uri']
except KeyError:
base_uri = None
if self.verbose:
print( '{0} did not contain \'{1}\''.format(
self.get_server_config_path(),
'schema_validation_base_uri'),
file=sys.stderr)
try :
version_dict = config_dicts[self.get_version_path()]
uri_version = version_dict['configuration_schema_version']
except KeyError:
uri_version = None
if self.verbose:
print( '{0} did not contain \'{1}\''.format(
self.get_version_path(),
'configuration_schema_version'),
file=sys.stderr)
if base_uri and uri_version:
validation_uri_prefix = '/'.join([
base_uri,
'v{0}'.format(uri_version)])
for path, json_dict in config_dicts.items():
schema_uri = '/'.join([
validation_uri_prefix,
os.path.basename(path)])
try :
validate_json.validate_dict(
json_dict,
schema_uri,
name=path,
verbose=self.verbose)
except validate_json.ValidationWarning as e:
if self.verbose:
print(e, file=sys.stderr)
except validate_json.ValidationError as e:
irods_six.reraise(IrodsControllerError, e, sys.exc_info()[2])
elif self.verbose:
print( 'Preflight Check problem:',
'JSON Configuration Validation failed.',
sep='\n\t', file=sys.stderr)
return config_dicts
def stop(self, timeout=20):
l = logging.getLogger(__name__)
l.debug('Calling stop on IrodsController')
l.info('Stopping iRODS server...')
try:
if self.get_binary_to_pids_dict([self.get_server_executable()]):
try:
self.irods_grid_shutdown(timeout=timeout)
except Exception as e:
l.error('Error encountered in graceful shutdown.',
exc_info=True)
else:
l.warning('No iRODS servers running.')
# kill servers first to stop spawning of other processes
server_pids_dict = self.get_binary_to_pids_dict(
[self.get_server_executable()])
if server_pids_dict:
l.warning(
'iRODS server processes remain after "irods-grid shutdown".'
)
l.warning(format_binary_to_pids_dict(server_pids_dict))
l.warning('Killing forcefully...')
for pid in server_pids_dict[self.get_server_executable()]:
l.warning('Killing %s, pid %s',
self.get_server_executable(), pid)
try:
kill_pid(pid)
except psutil.NoSuchProcess:
pass
delete_cache_files_by_pid(pid)
binary_to_pids_dict = self.get_binary_to_pids_dict()
if binary_to_pids_dict:
l.warning(
'iRODS child processes remain after "irods-grid shutdown".'
)
l.warning(format_binary_to_pids_dict(binary_to_pids_dict))
l.warning('Killing forcefully...')
for binary, pids in binary_to_pids_dict.items():
for pid in pids:
l.warning('Killing %s, pid %s', binary, pid)
try:
kill_pid(pid)
except psutil.NoSuchProcess:
pass
delete_cache_files_by_pid(pid)
except IrodsControllerError as e:
l.info('Failure')
irods_six.reraise(IrodsControllerError, e, sys.exc_info()[2])
l.info('Success')
def load_and_validate(config_file, schema_uri, verbose=False):
try:
# load configuration file
with open(config_file, 'r') as f:
config_dict = json.load(f)
except BaseException as e:
irods_six.reraise(ValidationError, ValidationError('\n\t'.join([
'ERROR: Validation Failed for [{0}]:'.format(config_file),
'against [{0}]'.format(schema_uri),
'{0}: {1}'.format(e.__class__.__name__, e)])),
sys.exc_info()[2])
validate_dict(config_dict, schema_uri, name=config_file, verbose=verbose)
return config_dict
def stop(self, timeout=20):
if self.verbose:
print('Stopping iRODS server... ', end='')
try:
if self.get_binary_to_pids_dict([self.get_server_executable()]):
try:
self.irods_grid_shutdown(timeout=timeout)
except Exception as e:
if self.verbose:
print('Error encountered in graceful shutdown.', e,
sep='\n', file=sys.stderr)
else:
if self.verbose:
print('No iRODS servers running. ', file=sys.stderr, end='')
# kill servers first to stop spawning of other processes
server_pids_dict = self.get_binary_to_pids_dict([self.get_server_executable()])
if server_pids_dict:
if self.verbose:
print('iRODS server processes remain after "irods-grid shutdown".',
format_binary_to_pids_dict(server_pids_dict),
'Killing forcefully...',
sep='\n', file=sys.stderr)
for pid in server_pids_dict[self.get_server_executable()]:
try:
kill_pid(pid)
except psutil.NoSuchProcess:
pass
delete_cache_files_by_pid(pid)
binary_to_pids_dict = self.get_binary_to_pids_dict()
if binary_to_pids_dict:
if self.verbose:
print('iRODS child processes remain after "irods-grid shutdown".',
format_binary_to_pids_dict(binary_to_pids_dict),
'Killing forcefully...',
sep='\n', file=sys.stderr)
for binary, pids in binary_to_pids_dict.items():
for pid in pids:
try:
kill_pid(pid)
except psutil.NoSuchProcess:
pass
delete_cache_files_by_pid(pid)
except IrodsControllerError as e:
if self.verbose:
print('Failure')
irods_six.reraise(IrodsControllerError, e, sys.exc_info()[2])
if self.verbose:
print('Success')
def load_and_validate(config_file, schema_uri):
l = logging.getLogger(__name__)
try:
# load configuration file
with open(config_file, 'r') as f:
config_dict = json.load(f)
except BaseException as e:
irods_six.reraise(ValidationError, ValidationError('\n\t'.join([
'ERROR: Validation Failed for [{0}]:'.format(config_file),
'against [{0}]'.format(schema_uri),
'{0}: {1}'.format(e.__class__.__name__, e)])),
sys.exc_info()[2])
validate_dict(config_dict, schema_uri, name=config_file)
return config_dict
def get_pids_executing_binary_file(binary_file_path):
out, err, returncode = execute_command_permissive(
['lsof', '-F', 'pf', binary_file_path])
out = out if returncode == 0 else ''
parsed_out = parse_formatted_lsof_output(out)
try:
# we only want pids in executing state
return [int(d['p']) for d in parsed_out if d['f'] == 'txt']
except (ValueError, KeyError):
irods_six.reraise(IrodsControllerError, IrodsControllerError('\n\t'.join([
'non-conforming lsof output:',
'{0}'.format(out),
'{0}'.format(err)])),
sys.exc_info()[2])
def load_json_config_file(path):
if os.path.exists(path):
with open(path, 'r') as f:
try:
return json.load(f)
except ValueError as e:
irods_six.reraise(
IrodsControllerError,
IrodsControllerError('\n\t'.join([
'JSON load failed for [{0}]:'.format(path),
'Invalid JSON.',
'{0}: {1}'.format(e.__class__.__name__, e)
])),
sys.exc_info()[2])
else:
raise IrodsControllerError('File {0} does not exist.'.format(path))
def get_pids_executing_binary_file(binary_file_path):
# get lsof listing of pids
p = subprocess.Popen(['lsof', '-F', 'pf', binary_file_path],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = p.communicate()
out = out.decode() if p.returncode == 0 else ''
parsed_out = parse_formatted_lsof_output(out)
try:
# we only want pids in executing state
return [int(d['p']) for d in parsed_out if d['f'] == 'txt']
except (ValueError, KeyError):
irods_six.reraise(IrodsControllerError, IrodsControllerError('\n\t'.join([
'non-conforming lsof output:',
'{0}'.format(out),
'{0}'.format(err)])),
sys.exc_info()[2])
def load_json_config_file(path):
if os.path.exists(path):
with open(path, 'r') as f:
try :
return json.load(f)
except ValueError as e:
irods_six.reraise(IrodsControllerError, IrodsControllerError('\n\t'.join([
'JSON load failed for [{0}]:'.format(
path),
'Invalid JSON.',
'{0}: {1}'.format(
e.__class__.__name__, e)])),
sys.exc_info()[2])
else:
raise IrodsControllerError(
'File {0} does not exist.'.format(
path))
def stop(self, timeout=20):
l = logging.getLogger(__name__)
l.debug('Calling stop on IrodsController')
l.info('Stopping iRODS server...')
try:
if self.get_binary_to_pids_dict([self.get_server_executable()]):
try:
self.irods_grid_shutdown(timeout=timeout)
except Exception as e:
l.error('Error encountered in graceful shutdown.', exc_info=True)
else:
l.warning('No iRODS servers running.')
# kill servers first to stop spawning of other processes
server_pids_dict = self.get_binary_to_pids_dict([self.get_server_executable()])
if server_pids_dict:
l.warning('iRODS server processes remain after "irods-grid shutdown".')
l.warning(format_binary_to_pids_dict(server_pids_dict))
l.warning('Killing forcefully...')
for pid in server_pids_dict[self.get_server_executable()]:
l.warning('Killing %s, pid %s', self.get_server_executable(), pid)
try:
kill_pid(pid)
except psutil.NoSuchProcess:
pass
delete_cache_files_by_pid(pid)
binary_to_pids_dict = self.get_binary_to_pids_dict()
if binary_to_pids_dict:
l.warning('iRODS child processes remain after "irods-grid shutdown".')
l.warning(format_binary_to_pids_dict(binary_to_pids_dict))
l.warning('Killing forcefully...')
for binary, pids in binary_to_pids_dict.items():
for pid in pids:
l.warning('Killing %s, pid %s', binary, pid)
try:
kill_pid(pid)
except psutil.NoSuchProcess:
pass
delete_cache_files_by_pid(pid)
except IrodsControllerError as e:
l.info('Failure')
irods_six.reraise(IrodsControllerError, e, sys.exc_info()[2])
l.info('Success')
def get_initial_schema_from_web(schema_uri):
try:
response = requests.get(schema_uri, timeout=5)
except NameError:
irods_six.reraise(ValidationError, ValidationError(
'WARNING: Validation Failed for {0} -- requests not installed'.format(
name)),
sys.exc_info()[2])
# check response values
try:
# modern requests
schema = json.loads(response.text)
except AttributeError:
# requests pre-v1.0.0
response.encoding = 'utf8'
schema = json.loads(response.content)
return schema
def get_pids_executing_binary_file(binary_file_path):
# get lsof listing of pids
p = subprocess.Popen(['lsof', '-F', 'pf', binary_file_path],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = p.communicate()
out = out.decode() if p.returncode == 0 else ''
parsed_out = parse_formatted_lsof_output(out)
try:
# we only want pids in executing state
return [int(d['p']) for d in parsed_out if d['f'] == 'txt']
except (ValueError, KeyError):
irods_six.reraise(
IrodsControllerError,
IrodsControllerError('\n\t'.join([
'non-conforming lsof output:', '{0}'.format(out),
'{0}'.format(err)
])),
sys.exc_info()[2])
def execute_command_nonblocking(args, **kwargs):
l = logging.getLogger(__name__)
l.debug('Calling %s with options:', args)
if 'env' in kwargs:
kwargs_without_env = copy.copy(kwargs)
kwargs_without_env['env'] = 'HIDDEN'
l.debug(pprint.pformat(kwargs_without_env))
else :
kwargs_without_env = kwargs
l.debug(pprint.pformat(kwargs_without_env))
try :
return subprocess.Popen(args, **kwargs)
except OSError as e:
irods_six.reraise(IrodsControllerError,
IrodsControllerError('\n'.join([
'Call to open process with {0} failed:'.format(
args),
indent(
'Could not find the requested executable \'{0}\'; '
'please ensure \'{0}\' is installed and in the path.'.format(
args[0]))])),
sys.exc_info()[2])
def execute_command_nonblocking(args, **kwargs):
l = logging.getLogger(__name__)
l.debug('Calling %s with options:', args)
if 'env' in kwargs:
kwargs_without_env = copy.copy(kwargs)
kwargs_without_env['env'] = 'HIDDEN'
l.debug(pprint.pformat(kwargs_without_env))
else:
kwargs_without_env = kwargs
l.debug(pprint.pformat(kwargs_without_env))
try:
return subprocess.Popen(args, **kwargs)
except OSError as e:
irods_six.reraise(
IrodsControllerError,
IrodsControllerError('\n'.join([
'Call to open process with {0} failed:'.format(args),
indent('Could not find the requested executable \'{0}\'; '
'please ensure \'{0}\' is installed and in the path.'.
format(args[0]))
])),
sys.exc_info()[2])
def validate_dict(config_dict, schema_uri, name=None):
l = logging.getLogger(__name__)
if name is None:
name = schema_uri.rpartition('/')[2]
try:
e = jsonschema.exceptions
except AttributeError:
irods_six.reraise(ValidationWarning, ValidationWarning(
'WARNING: Validation Failed for {0} -- jsonschema too old v[{1}]'.format(
name, jsonschema.__version__)),
sys.exc_info()[2])
except NameError:
irods_six.reraise(ValidationWarning, ValidationWarning(
'WARNING: Validation Failed for {0} -- jsonschema not installed'.format(
name)),
sys.exc_info()[2])
try:
schema = get_initial_schema(schema_uri)
l.debug('Validating %s against json schema:', name)
l.debug(pprint.pformat(schema))
jsonschema.validate(config_dict, schema)
except (jsonschema.exceptions.RefResolutionError, # could not resolve recursive schema $ref
ValueError, # 404s and bad JSON
requests.exceptions.ConnectionError, # network connection error
requests.exceptions.Timeout # timeout
) as e:
irods_six.reraise(ValidationWarning, ValidationWarning('\n\t'.join([
'WARNING: Validation Failed for [{0}]:'.format(name),
'against [{0}]'.format(schema_uri),
'{0}: {1}'.format(e.__class__.__name__, e)])),
sys.exc_info()[2])
except (jsonschema.exceptions.ValidationError, # validation error
jsonschema.exceptions.SchemaError, # schema error
BaseException # catch all
) as e:
irods_six.reraise(ValidationError, ValidationError('\n\t'.join([
'ERROR: Validation Failed for [{0}]:'.format(name),
'against [{0}]'.format(schema_uri),
'{0}: {1}'.format(e.__class__.__name__, e)])),
sys.exc_info()[2])
l.info("Validating [%s]... Success", name)
def start(self, execution_environment={}, insert_behavior=True):
l = logging.getLogger(__name__)
l.debug('Calling start on IrodsController')
if not os.path.exists(self.get_server_executable()):
raise IrodsControllerError('\n\t'.join([
'Configuration problem:',
'The \'{0}\' application could not be found. Have the'.format(
os.path.basename(self.get_server_executable())),
'iRODS servers been compiled?']),
sys.exc_info()[2])
try:
(test_file_handle, test_file_name) = tempfile.mkstemp(
dir=self.get_log_directory())
os.close(test_file_handle)
os.unlink(test_file_name)
except (IOError, OSError):
irods_six.reraise(IrodsControllerError, IrodsControllerError('\n\t'.join([
'Configuration problem:',
'The server log directory, \'{0}\''.format(
self.get_log_directory()),
'is not writeable. Please change its permissions',
'and retry.'])),
sys.exc_info()[2])
new_execution_environment = {}
if insert_behavior:
for key, value in os.environ.items():
new_execution_environment[key] = value
new_execution_environment['irodsConfigDir'] = self.get_config_directory()
new_execution_environment['PWD'] = self.get_server_bin_directory()
for key, value in self.static_execution_environment.items():
new_execution_environment[key] = value
for key, value in execution_environment.items():
new_execution_environment[key] = value
config_dicts = self.load_and_validate_config_files(new_execution_environment, insert_behavior)
if insert_behavior:
if 'environment_variables' in config_dicts[self.get_server_config_path()]:
for key, value in config_dicts[self.get_server_config_path()]['environment_variables'].items():
if key not in new_execution_environment:
new_execution_environment[key] = value
if self.get_database_config_path() in config_dicts:
schema_version_in_file = config_dicts[self.get_version_path()]['catalog_schema_version']
if not self.check_database_schema_version(
schema_version_in_file=schema_version_in_file):
raise IrodsControllerError('\n\t'.join([
'Preflight Check problem:',
'Database Schema in the database is ahead',
'of {0} - Please upgrade.'.format(
os.path.basename(self.get_version_path()))]))
try:
irods_port = int(config_dicts[self.get_server_config_path()]['zone_port'])
l.debug('Attempting to bind socket %s', irods_port)
with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
try:
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(('127.0.0.1', irods_port))
except socket.error:
irods_six.reraise(IrodsControllerError,
IrodsControllerError('Could not bind port {0}.'.format(irods_port)),
sys.exc_info()[2])
l.debug('Socket %s bound and released successfully.')
l.info('Starting iRODS server...')
execute_command(
[self.get_server_executable()],
cwd=self.get_server_bin_directory(),
env=new_execution_environment)
retry_count = 100
while True:
l.debug('Attempting to connect to iRODS server on port %s. Attempt #%s',
irods_port, 101 - retry_count)
with contextlib.closing(socket.socket(
socket.AF_INET, socket.SOCK_STREAM)) as s:
if s.connect_ex(('127.0.0.1', irods_port)) == 0:
if get_pids_executing_binary_file(
self.get_server_executable()):
l.debug('Successfully connected to port %s.', irods_port)
break
else:
retry_count = 0
if retry_count <= 0:
raise IrodsControllerError('iRODS server failed to start.')
retry_count = retry_count - 1
time.sleep(1)
except IrodsControllerError as e:
l.info('Failure')
irods_six.reraise(IrodsControllerError, e, sys.exc_info()[2])
l.info('Success')
def start(self, execution_environment={}, insert_behavior=True):
new_execution_environment = {}
for key, value in execution_environment.items():
new_execution_environment[key] = value
if insert_behavior:
for key, value in self.static_execution_environment.items():
if key not in new_execution_environment:
new_execution_environment[key] = value
if 'irodsHomeDir' not in new_execution_environment:
new_execution_environment['irodsHomeDir'] = self.get_irods_directory()
if 'irodsConfigDir' not in new_execution_environment:
new_execution_environment['irodsConfigDir'] = self.get_config_directory()
if 'PWD' not in new_execution_environment:
new_execution_environment['PWD'] = self.get_server_bin_directory()
for key, value in os.environ.items():
if key not in new_execution_environment:
new_execution_environment[key] = value
if not os.path.exists(self.get_server_executable()):
raise IrodsControllerError('\n\t'.join([
'Configuration problem:',
'The \'{0}\' application could not be found. Have the'.format(
os.path.basename(self.get_server_executable())),
'iRODS servers been compiled?']),
sys.exc_info()[2])
try:
(test_file_handle, test_file_name) = tempfile.mkstemp(
dir=self.get_log_directory())
os.close(test_file_handle)
os.unlink(test_file_name)
except (IOError, OSError):
irods_six.reraise(IrodsControllerError, IrodsControllerError('\n\t'.join([
'Configuration problem:',
'The server log directory, \'{0}\''.format(
self.get_log_directory()),
'is not writeable. Please change its permissions',
'and retry.'])),
sys.exc_info()[2])
config_dicts = self.load_and_validate_config_files()
if self.get_database_config_path() in config_dicts:
schema_version_in_file = config_dicts[self.get_version_path()]['catalog_schema_version']
if not self.check_database_schema_version(
schema_version_in_file=schema_version_in_file):
raise IrodsControllerError('\n\t'.join([
'Preflight Check problem:',
'Database Schema in the database is ahead',
'of {0} - Please upgrade.'.format(
os.path.basename(self.get_version_path()))]))
if self.verbose:
print('Starting iRODS server...', end=' ')
try:
irods_port = int(config_dicts[self.get_server_config_path()]['zone_port'])
with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
try:
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(('127.0.0.1', irods_port))
except socket.error:
irods_six.reraise(IrodsControllerError,
IrodsControllerError('Could not bind port {0}.'.format(irods_port)),
sys.exc_info()[2])
p = subprocess.Popen(
[self.get_server_executable()],
cwd=self.get_server_bin_directory(),
env=new_execution_environment,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, _ = p.communicate()
if p.returncode != 0:
raise IrodsControllerError('\n\t'.join([
'iRODS server failed to start.',
out.decode()]))
retry_count = 100
while True:
with contextlib.closing(socket.socket(
socket.AF_INET, socket.SOCK_STREAM)) as s:
if s.connect_ex(('127.0.0.1', irods_port)) == 0:
if get_pids_executing_binary_file(
self.get_server_executable()):
break
else:
retry_count = 0
if retry_count <= 0:
raise IrodsControllerError('iRODS server failed to start.')
retry_count = retry_count - 1
time.sleep(1)
except IrodsControllerError as e:
if self.verbose:
print('Failure')
irods_six.reraise(IrodsControllerError, e, sys.exc_info()[2])
if self.verbose:
print('Success')
def load_and_validate_config_files(self, execution_environment=None, insert_behavior=True):
l = logging.getLogger(__name__)
config_files = [
self.get_server_config_path(),
self.get_version_path(),
self.get_hosts_config_path(),
self.get_host_access_control_config_path()]
if os.path.exists(self.get_database_config_path()):
config_files.append(self.get_database_config_path())
else:
l.debug('The database config file, \'%s\', does not exist.', self.get_database_config_path())
config_dicts = {}
for path in config_files:
l.debug('Loading %s into dictionary', path)
config_dicts[path] = load_json_config_file(path)
if execution_environment is not None and 'IRODS_ENVIRONMENT_FILE' in execution_environment:
irods_environment_path = execution_environment['IRODS_ENVIRONMENT_FILE']
elif insert_behavior and 'environment_variables' in config_dicts[self.get_server_config_path()] and 'IRODS_ENVIRONMENT_FILE' in config_dicts[self.get_server_config_path()]['environment_variables']:
irods_environment_path = config_dicts[self.get_server_config_path()]['environment_variables']['IRODS_ENVIRONMENT_FILE']
else:
irods_environment_path = get_irods_environment_path()
l.debug('Loading %s into dictionary', irods_environment_path)
config_dicts[irods_environment_path] = load_json_config_file(irods_environment_path)
l.debug('Attempting to construct schema URI...')
try :
server_config_dict = config_dicts[self.get_server_config_path()]
key = 'schema_validation_base_uri'
base_uri = server_config_dict[key]
except KeyError:
base_uri = None
l.warning('%s did not contain \'%s\'',
self.get_server_config_path(), key)
try :
version_dict = config_dicts[self.get_version_path()]
key = 'configuration_schema_version'
uri_version = version_dict[key]
except KeyError:
uri_version = None
l.warning('%s did not contain \'%s\'',
self.get_version_path(), key)
if base_uri and uri_version:
validation_uri_prefix = '/'.join([
base_uri,
'v{0}'.format(uri_version)])
l.debug('Successfully constructed schema URI.')
for path in config_files:
json_dict = config_dicts[path]
schema_uri = '/'.join([
validation_uri_prefix,
os.path.basename(path)])
l.debug('Attempting to validate %s against %s', path, schema_uri)
try :
validate_json.validate_dict(
json_dict,
schema_uri,
name=path)
except validate_json.ValidationWarning as e:
l.warning('Error encountered in validate_json', exc_info=True)
except validate_json.ValidationError as e:
irods_six.reraise(IrodsControllerError, e, sys.exc_info()[2])
json_dict = config_dicts[irods_environment_path]
schema_uri = '/'.join([
validation_uri_prefix,
os.path.basename(get_irods_environment_path())])
l.debug('Attempting to validate %s against %s', irods_environment_path, schema_uri)
try :
validate_json.validate_dict(
json_dict,
schema_uri,
name=irods_environment_path)
except validate_json.ValidationWarning as e:
l.warning('Error encountered in validate_json', exc_info=True)
except validate_json.ValidationError as e:
irods_six.reraise(IrodsControllerError, e, sys.exc_info()[2])
else:
l.debug('Failed to construct schema URI')
l.warning('%s\n%s',
'Preflight Check problem:',
indent('JSON Configuration Validation failed.'))
return config_dicts
def start(self, execution_environment={}, insert_behavior=True):
new_execution_environment = {}
for key, value in execution_environment.items():
new_execution_environment[key] = value
if insert_behavior:
for key, value in self.static_execution_environment.items():
if key not in new_execution_environment:
new_execution_environment[key] = value
if 'irodsHomeDir' not in new_execution_environment:
new_execution_environment[
'irodsHomeDir'] = self.get_irods_directory()
if 'irodsConfigDir' not in new_execution_environment:
new_execution_environment[
'irodsConfigDir'] = self.get_config_directory()
if 'PWD' not in new_execution_environment:
new_execution_environment[
'PWD'] = self.get_server_bin_directory()
for key, value in os.environ.items():
if key not in new_execution_environment:
new_execution_environment[key] = value
if not os.path.exists(self.get_server_executable()):
raise IrodsControllerError(
'\n\t'.join([
'Configuration problem:',
'The \'{0}\' application could not be found. Have the'.
format(os.path.basename(self.get_server_executable())),
'iRODS servers been compiled?'
]),
sys.exc_info()[2])
try:
(test_file_handle,
test_file_name) = tempfile.mkstemp(dir=self.get_log_directory())
os.close(test_file_handle)
os.unlink(test_file_name)
except (IOError, OSError):
irods_six.reraise(
IrodsControllerError,
IrodsControllerError('\n\t'.join([
'Configuration problem:',
'The server log directory, \'{0}\''.format(
self.get_log_directory()),
'is not writeable. Please change its permissions',
'and retry.'
])),
sys.exc_info()[2])
config_dicts = self.load_and_validate_config_files()
if self.get_database_config_path() in config_dicts:
schema_version_in_file = config_dicts[
self.get_version_path()]['catalog_schema_version']
if not self.check_database_schema_version(
schema_version_in_file=schema_version_in_file):
raise IrodsControllerError('\n\t'.join([
'Preflight Check problem:',
'Database Schema in the database is ahead',
'of {0} - Please upgrade.'.format(
os.path.basename(self.get_version_path()))
]))
if self.verbose:
print('Starting iRODS server...', end=' ')
try:
irods_port = int(
config_dicts[self.get_server_config_path()]['zone_port'])
with contextlib.closing(
socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
try:
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(('127.0.0.1', irods_port))
except socket.error:
irods_six.reraise(
IrodsControllerError,
IrodsControllerError(
'Could not bind port {0}.'.format(irods_port)),
sys.exc_info()[2])
p = subprocess.Popen([self.get_server_executable()],
cwd=self.get_server_bin_directory(),
env=new_execution_environment,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, _ = p.communicate()
if p.returncode != 0:
raise IrodsControllerError('\n\t'.join(
['iRODS server failed to start.',
out.decode()]))
retry_count = 100
while True:
with contextlib.closing(
socket.socket(socket.AF_INET,
socket.SOCK_STREAM)) as s:
if s.connect_ex(('127.0.0.1', irods_port)) == 0:
if get_pids_executing_binary_file(
self.get_server_executable()):
break
else:
retry_count = 0
if retry_count <= 0:
raise IrodsControllerError('iRODS server failed to start.')
retry_count = retry_count - 1
time.sleep(1)
except IrodsControllerError as e:
if self.verbose:
print('Failure')
irods_six.reraise(IrodsControllerError, e, sys.exc_info()[2])
if self.verbose:
print('Success')
def load_and_validate_config_files(self,
execution_environment=None,
insert_behavior=True):
l = logging.getLogger(__name__)
config_files = [
self.get_server_config_path(),
self.get_version_path(),
self.get_hosts_config_path(),
self.get_host_access_control_config_path()
]
if os.path.exists(self.get_database_config_path()):
config_files.append(self.get_database_config_path())
else:
l.debug('The database config file, \'%s\', does not exist.',
self.get_database_config_path())
config_dicts = {}
for path in config_files:
l.debug('Loading %s into dictionary', path)
config_dicts[path] = load_json_config_file(path)
if execution_environment is not None and 'IRODS_ENVIRONMENT_FILE' in execution_environment:
irods_environment_path = execution_environment[
'IRODS_ENVIRONMENT_FILE']
elif insert_behavior and 'environment_variables' in config_dicts[
self.get_server_config_path(
)] and 'IRODS_ENVIRONMENT_FILE' in config_dicts[
self.get_server_config_path()]['environment_variables']:
irods_environment_path = config_dicts[self.get_server_config_path(
)]['environment_variables']['IRODS_ENVIRONMENT_FILE']
else:
irods_environment_path = get_irods_environment_path()
l.debug('Loading %s into dictionary', irods_environment_path)
config_dicts[irods_environment_path] = load_json_config_file(
irods_environment_path)
l.debug('Attempting to construct schema URI...')
try:
server_config_dict = config_dicts[self.get_server_config_path()]
key = 'schema_validation_base_uri'
base_uri = server_config_dict[key]
except KeyError:
base_uri = None
l.warning('%s did not contain \'%s\'',
self.get_server_config_path(), key)
try:
version_dict = config_dicts[self.get_version_path()]
key = 'configuration_schema_version'
uri_version = version_dict[key]
except KeyError:
uri_version = None
l.warning('%s did not contain \'%s\'', self.get_version_path(),
key)
if base_uri and uri_version:
validation_uri_prefix = '/'.join(
[base_uri, 'v{0}'.format(uri_version)])
l.debug('Successfully constructed schema URI.')
for path in config_files:
json_dict = config_dicts[path]
schema_uri = '/'.join(
[validation_uri_prefix,
os.path.basename(path)])
l.debug('Attempting to validate %s against %s', path,
schema_uri)
try:
validate_json.validate_dict(json_dict,
schema_uri,
name=path)
except validate_json.ValidationWarning as e:
l.warning('Error encountered in validate_json',
exc_info=True)
except validate_json.ValidationError as e:
irods_six.reraise(IrodsControllerError, e,
sys.exc_info()[2])
json_dict = config_dicts[irods_environment_path]
schema_uri = '/'.join([
validation_uri_prefix,
os.path.basename(get_irods_environment_path())
])
l.debug('Attempting to validate %s against %s',
irods_environment_path, schema_uri)
try:
validate_json.validate_dict(json_dict,
schema_uri,
name=irods_environment_path)
except validate_json.ValidationWarning as e:
l.warning('Error encountered in validate_json', exc_info=True)
except validate_json.ValidationError as e:
irods_six.reraise(IrodsControllerError, e, sys.exc_info()[2])
else:
l.debug('Failed to construct schema URI')
l.warning('%s\n%s', 'Preflight Check problem:',
indent('JSON Configuration Validation failed.'))
return config_dicts
def start(self, execution_environment={}, insert_behavior=True):
l = logging.getLogger(__name__)
l.debug('Calling start on IrodsController')
if not os.path.exists(self.get_server_executable()):
raise IrodsControllerError(
'\n\t'.join([
'Configuration problem:',
'The \'{0}\' application could not be found. Have the'.
format(os.path.basename(self.get_server_executable())),
'iRODS servers been compiled?'
]),
sys.exc_info()[2])
try:
(test_file_handle,
test_file_name) = tempfile.mkstemp(dir=self.get_log_directory())
os.close(test_file_handle)
os.unlink(test_file_name)
except (IOError, OSError):
irods_six.reraise(
IrodsControllerError,
IrodsControllerError('\n\t'.join([
'Configuration problem:',
'The server log directory, \'{0}\''.format(
self.get_log_directory()),
'is not writeable. Please change its permissions',
'and retry.'
])),
sys.exc_info()[2])
new_execution_environment = {}
if insert_behavior:
for key, value in os.environ.items():
new_execution_environment[key] = value
new_execution_environment[
'irodsConfigDir'] = self.get_config_directory()
new_execution_environment['PWD'] = self.get_server_bin_directory()
for key, value in self.static_execution_environment.items():
new_execution_environment[key] = value
for key, value in execution_environment.items():
new_execution_environment[key] = value
config_dicts = self.load_and_validate_config_files(
new_execution_environment, insert_behavior)
if insert_behavior:
if 'environment_variables' in config_dicts[
self.get_server_config_path()]:
for key, value in config_dicts[self.get_server_config_path(
)]['environment_variables'].items():
if key not in new_execution_environment:
new_execution_environment[key] = value
if self.get_database_config_path() in config_dicts:
schema_version_in_file = config_dicts[
self.get_version_path()]['catalog_schema_version']
if not self.check_database_schema_version(
schema_version_in_file=schema_version_in_file):
raise IrodsControllerError('\n\t'.join([
'Preflight Check problem:',
'Database Schema in the database is ahead',
'of {0} - Please upgrade.'.format(
os.path.basename(self.get_version_path()))
]))
try:
irods_port = int(
config_dicts[self.get_server_config_path()]['zone_port'])
l.debug('Attempting to bind socket %s', irods_port)
with contextlib.closing(
socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
try:
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(('127.0.0.1', irods_port))
except socket.error:
irods_six.reraise(
IrodsControllerError,
IrodsControllerError(
'Could not bind port {0}.'.format(irods_port)),
sys.exc_info()[2])
l.debug('Socket %s bound and released successfully.')
l.info('Starting iRODS server...')
execute_command([self.get_server_executable()],
cwd=self.get_server_bin_directory(),
env=new_execution_environment)
retry_count = 100
while True:
l.debug(
'Attempting to connect to iRODS server on port %s. Attempt #%s',
irods_port, 101 - retry_count)
with contextlib.closing(
socket.socket(socket.AF_INET,
socket.SOCK_STREAM)) as s:
if s.connect_ex(('127.0.0.1', irods_port)) == 0:
if get_pids_executing_binary_file(
self.get_server_executable()):
l.debug('Successfully connected to port %s.',
irods_port)
break
else:
retry_count = 0
if retry_count <= 0:
raise IrodsControllerError('iRODS server failed to start.')
retry_count = retry_count - 1
time.sleep(1)
except IrodsControllerError as e:
l.info('Failure')
irods_six.reraise(IrodsControllerError, e, sys.exc_info()[2])
l.info('Success')
