def test_filemanager_all_data_local(self):
sta = True
types = ['atm', 'ice', 'ocn', 'rest', 'streams.cice', 'streams.ocean']
database = 'test.db'
simstart = 51
simend = 60
experiment = '20171011.beta2_FCT2-icedeep_branch.A_WCYCL1850S.ne30_oECv3_ICG.edison'
if os.path.exists(self.local_path):
shutil.rmtree(self.local_path)
filemanager = FileManager(mutex=self.mutex,
sta=sta,
types=types,
database=database,
remote_endpoint=self.remote_endpoint,
remote_path=self.remote_path,
local_endpoint=self.local_endpoint,
local_path=self.local_path)
filemanager.populate_file_list(simstart=simstart,
simend=simend,
experiment=experiment)
filemanager.update_local_status()
self.assertFalse(filemanager.all_data_local())
self.mutex.acquire()
for df in DataFile.select():
name = df.name
head, tail = os.path.split(df.local_path)
if not os.path.exists(head):
os.makedirs(head)
with open(df.local_path, 'w') as fp:
fp.write('this is a test file')
size = os.path.getsize(df.local_path)
df.remote_size = size
df.local_size = size
df.save()
if self.mutex.locked():
self.mutex.release()
filemanager.update_local_status()
self.assertTrue(filemanager.all_data_local())
def initialize(argv, **kwargs):
"""
Parse the commandline arguments, and setup the master config dict
Parameters:
argv (list): a list of arguments
event_list (EventList): The main list of events
kill_event (threading.Event): An event used to kill all running threads
__version__ (str): the current version number for processflow
__branch__ (str): the branch this version was built from
"""
# Setup the parser
pargs = parse_args(argv=argv)
if pargs.version:
msg = 'Processflow version {}'.format(kwargs['version'])
print msg
sys.exit(0)
if not pargs.config:
parse_args(print_help=True)
return False, False, False
event_list = kwargs['event_list']
event = kwargs['kill_event']
print_line(line='Entering setup', event_list=event_list)
# check if globus config is valid, else remove it
globus_config = os.path.join(os.path.expanduser('~'), '.globus.cfg')
if os.path.exists(globus_config):
try:
conf = ConfigObj(globus_config)
except:
os.remove(globus_config)
if not os.path.exists(pargs.config):
print "Invalid config, {} does not exist".format(pargs.config)
return False, False, False
# Check that there are no white space errors in the config file
line_index = check_config_white_space(pargs.config)
if line_index != 0:
print '''
ERROR: line {num} does not have a space after the \'=\', white space is required.
Please add a space and run again.'''.format(num=line_index)
return False, False, False
# read the config file and setup the config dict
try:
config = ConfigObj(pargs.config)
except Exception as e:
print_debug(e)
print "Error parsing config file {}".format(pargs.config)
parse_args(print_help=True)
return False, False, False
# run validator for config file
messages = verify_config(config)
if messages:
for message in messages:
print_message(message)
return False, False, False
try:
setup_directories(pargs, config)
except Exception as e:
print_message('Failed to setup directories')
print_debug(e)
sys.exit(1)
if pargs.resource_path:
config['global']['resource_path'] = os.path.abspath(
pargs.resource_path)
else:
config['global']['resource_path'] = os.path.join(
sys.prefix, 'share', 'processflow', 'resources')
# Setup boolean config flags
config['global']['host'] = True if config.get('img_hosting') else False
config['global']['always_copy'] = True if pargs.always_copy else False
config['global']['dryrun'] = True if pargs.dryrun else False
config['global']['debug'] = True if pargs.debug else False
config['global']['verify'] = True if pargs.verify else False
config['global']['max_jobs'] = pargs.max_jobs if pargs.max_jobs else False
# setup logging
if pargs.log:
log_path = pargs.log
else:
log_path = os.path.join(config['global']['project_path'], 'output',
'processflow.log')
print_line(line='Log saved to {}'.format(log_path), event_list=event_list)
if not kwargs.get('testing'):
from imp import reload
reload(logging)
config['global']['log_path'] = log_path
if os.path.exists(log_path):
logbak = log_path + '.bak'
if os.path.exists(logbak):
os.remove(logbak)
copyfile(log_path, log_path + '.bak')
log_level = logging.DEBUG if pargs.debug else logging.INFO
logging.basicConfig(format='%(asctime)s:%(levelname)s: %(message)s',
datefmt='%m/%d/%Y %I:%M:%S %p',
filename=log_path,
filemode='w',
level=log_level)
logging.getLogger('globus_sdk').setLevel(logging.ERROR)
logging.getLogger('globus_cli').setLevel(logging.ERROR)
logging.info("Running with config:")
msg = json.dumps(config, sort_keys=False, indent=4)
logging.info(msg)
if pargs.max_jobs:
print_line(line="running with maximum {} jobs".format(pargs.max_jobs),
event_list=event_list)
if not config['global']['host'] or not config.get('img_hosting'):
print_line(line='Not hosting img output', event_list=event_list)
msg = 'processflow version {} branch {}'.format(kwargs['version'],
kwargs['branch'])
logging.info(msg)
# Copy the config into the input directory for safe keeping
input_config_path = os.path.join(config['global']['project_path'], 'input',
'run.cfg')
try:
copy(pargs.config, input_config_path)
except:
pass
if config['global']['always_copy']:
msg = 'Running in forced-copy mode, previously hosted diagnostic output will be replaced'
else:
msg = 'Running without forced-copy, previous hosted output will be preserved'
print_line(line=msg, event_list=event_list)
# initialize the filemanager
db = os.path.join(config['global'].get('project_path'), 'output',
'processflow.db')
msg = 'Initializing file manager'
print_line(msg, event_list)
filemanager = FileManager(database=db,
event_list=event_list,
config=config)
filemanager.populate_file_list()
msg = 'Starting local status update'
print_line(msg, event_list)
filemanager.update_local_status()
msg = 'Local status update complete'
print_line(msg, event_list)
msg = filemanager.report_files_local()
print_line(msg, event_list)
filemanager.write_database()
all_data = filemanager.all_data_local()
if all_data:
msg = 'all data is local'
else:
msg = 'Additional data needed'
print_line(msg, event_list)
logging.info("FileManager setup complete")
logging.info(str(filemanager))
if all_data:
print_line(line="skipping globus setup", event_list=event_list)
else:
if config['global'].get('local_globus_uuid'):
endpoints = [endpoint for endpoint in filemanager.get_endpoints()]
local_endpoint = config['global'].get('local_globus_uuid')
if local_endpoint:
endpoints.append(local_endpoint)
msg = 'Checking authentication for {} endpoints'.format(endpoints)
print_line(line=msg, event_list=event_list)
setup_success = setup_globus(endpoints=endpoints,
event_list=event_list)
if not setup_success:
print "Globus setup error"
return False, False, False
else:
print_line(line='Globus authentication complete',
event_list=event_list)
# setup the runmanager
runmanager = RunManager(event_list=event_list,
event=event,
config=config,
filemanager=filemanager)
runmanager.setup_cases()
runmanager.setup_jobs()
runmanager.write_job_sets(
os.path.join(config['global']['project_path'], 'output', 'state.txt'))
return config, filemanager, runmanager
