def wait_for_file_to_be_written_to(fpath,
total_timeout=100000,
time_frame=0.05):
'''
fpath: str
path to file to check
total_timeout: number
total number of seconds before aborting the wait command
time_frame: number
number of seconds to wait between each check of file size.
Purpose: Wait until a file that exists to have its filesize remains constant in
a given time frame. It will not be constant if it is currently being written to.
'''
start_time = time_utils.gtime()
while True:
try:
fsize = os.path.getsize(fpath)
break
except FileNotFoundError:
pass
if time_utils.gtime() - start_time > total_timeout:
raise Exception(
'file ' + fpath +
' still not done being written to after a total of ' +
str(total_timeout) + ' seconds')
time_utils.sleep(time_frame)
while fsize != os.path.getsize(fpath) and fsize != 0:
fsize = os.path.getsize(fpath)
time_utils.sleep(time_frame)
if time_utils.gtime() - start_time > total_timeout:
raise Exception(
'file ' + fpath +
' still not done being written to after a total of ' +
str(total_timeout) + ' seconds')
def rm_file_with_message(fpath, message):
while os.path.exists(fpath):
try:
with open(fpath) as f:
read_message = f.read()
except:
return
if read_message == message:
rm(fpath)
else:
return
time_utils.sleep(0.1)
def safe_np_load(npy_fpath,
total_timeout=10000,
time_frame=0.05,
verbose=False,
check_file_done_being_written_to=True):
'''
npy_fpath: str
Path to file that is loadable by np.load()
total_timeout: number
total number of seconds before aborting the wait command
time_frame: number
number of seconds to wait between each check of file size.
verbose: bool
Whether to print some log info
check_file_done_being_written_to: bool
Whether to check file size to determine if the file is being written to
and thus unsafe to load.
Return: np.array
The contents of npy_fpath as loaded by np.load()
Purpose: Check to make sure file exists before loading it. If DNE, wait until
it does exist or your timeout is reached.
'''
start_time = time_utils.gtime()
if check_file_done_being_written_to:
wait_for_file_to_exist_and_written_to(npy_fpath,
total_timeout=total_timeout,
time_frame=time_frame)
else:
wait_for_file_to_exist(npy_fpath,
total_timeout=total_timeout,
time_frame=time_frame)
if verbose:
print(
'took {} seconds to wait for file to exist and written to according to the function wait_for_file_to_exist_and_written_to'
.format(time_utils.gtime() - start_time))
start_time_load = time_utils.gtime()
while time_utils.gtime() - start_time < total_timeout:
try:
npy = np.load(npy_fpath)
if verbose:
print('took {} seconds after file {} exists to load it'.format(
time_utils.gtime() - start_time_load, npy_fpath))
return npy
except ValueError:
time_utils.sleep(time_frame)
raise TimeoutError('total_timeout was reached in save_np_load')
def read_fragile_csv(fpath):
wait_for_file_to_be_written_to(fpath, total_timeout=1000, time_frame=0.1)
read_success = False
start_time = time_utils.gtime()
while not read_success:
try:
df = pd.read_csv(fpath)
read_success = True
except:
time_utils.sleep(0.1)
if time_utils.gtime() - start_time > 1000:
raise Exception(
'Took more than 1000 seconds to try to read', tasks_fpath,
'\nExpected the file to be existant and non-empty.')
return df
def wait_for_file_to_exist(fpath, total_timeout=100000, time_frame=0.05):
'''
fpath: str
path to file to check
total_timeout: number
total number of seconds before aborting the wait command
time_frame: number
number of seconds to wait between each check of file size.
Purpose: Wait until file exists for up to total_timeout seconds.
'''
start_time = time_utils.gtime()
while not os.path.exists(fpath):
if time_utils.gtime() - start_time > total_timeout:
raise Exception('file ' + fpath + ' still DNE after a total of ' +
str(total_timeout) + ' seconds')
time_utils.sleep(time_frame)
def wait_for_file_to_vanish(fpath,
total_timeout=100000,
time_frame=0.05,
go_ahead_if_out_of_time=False):
start_time = time_utils.gtime()
if time_frame == 0:
while os.path.exists(fpath):
if time_utils.gtime(
) - start_time > total_timeout and not go_ahead_if_out_of_time:
raise Exception('file ' + fpath +
' still exists after a total of ' +
str(total_timeout) + ' seconds')
return
#wait until a file is removed by some other process
while os.path.exists(fpath):
#sleep a random amount of time to help prevent clashing (if multiple ranks)
time_utils.sleep(random.uniform(time_frame, 1.1 * time_frame))
if time_utils.gtime(
) - start_time > total_timeout and not go_ahead_if_out_of_time:
raise Exception('file ' + fpath +
' still exists after a total of ' +
str(total_timeout) + ' seconds')
def lock_file(fpath,
lockfile_message='locked',
total_timeout=100000,
time_frame=0.05,
go_ahead_if_out_of_time=False):
start_time = time_utils.gtime()
wait_for_file_to_vanish(fpath,
total_timeout=total_timeout,
time_frame=time_frame,
go_ahead_if_out_of_time=go_ahead_if_out_of_time)
read_lockfile_message = 'Nonelkjlkj'
while read_lockfile_message != lockfile_message:
with open(fpath, 'w') as f:
f.write(lockfile_message)
time_utils.sleep(0.05)
try:
with open(fpath) as f:
read_lockfile_message = f.read()
except:
pass
if time_utils.gtime(
) - start_time > total_timeout and not go_ahead_if_out_of_time:
raise Exception('Took longer than total_timeout =', total_timeout,
'seconds to acquire lock file.')
def file_system_barrier(comm):
'''
comm: MPI.COMM_WORLD from mpi4py
MPI communicator from mpi4py
Purpose: mpi4py's comm.barrier() doesn't always work. This
function creates a file for each rank in the comm and
each rank in the comm will wait until every other rank
has created a file. It will then remove these tmp files.
Notes: Currently only supports the world communicator.
'''
output_fpath_prefix = 'barrier_world_rank_'
output_fpath_search_str = output_fpath_prefix + '*'
file_utils.output_from_rank(message_args=('Here'),
rank=comm.rank,
output_fpath_prefix=output_fpath_prefix)
while len(file_utils.glob(output_fpath_search_str)) < comm.size:
time_utils.sleep(0.5)
time_utils.sleep(1.5)
if comm.rank == 0:
file_utils.rm(output_fpath_search_str)
else:
time_utils.sleep(0.4)