def ExecuteGamit(Sessions, JobServer, dry_run=False):
gamit_pbar = tqdm(total=len(
[GamitSession for GamitSession in Sessions if not GamitSession.ready]),
desc=' >> GAMIT sessions completion',
ncols=100) # type: tqdm
tqdm.write(' >> Initializing %i GAMIT sessions' % (len(Sessions)))
# For debugging parallel python runs
# console = logging.FileHandler('pp.log')
# console.setLevel(logging.DEBUG)
# JobServer.job_server.logger.setLevel(logging.DEBUG)
# formatter = logging.Formatter('%(asctime)s %(name)s: %(levelname)-8s %(message)s')
# console.setFormatter(formatter)
# JobServer.job_server.logger.addHandler(console)
modules = ('pyRinex', 'datetime', 'os', 'shutil', 'pyBrdc', 'pySp3',
'subprocess', 're', 'pyETM', 'glob', 'platform', 'traceback')
JobServer.create_cluster(run_gamit_session, (pyGamitTask.GamitTask, ),
callback_handle,
gamit_pbar,
modules=modules)
gtasks = []
for GamitSession in Sessions:
if not GamitSession.ready:
# do not submit the task if the session is ready!
GamitSession.initialize()
task = pyGamitTask.GamitTask(GamitSession.remote_pwd,
GamitSession.params,
GamitSession.solution_pwd)
JobServer.submit(task, task.params['DirName'], task.date.year,
task.date.doy, dry_run)
# save it just in case...
gtasks.append(task)
else:
tqdm.write(' -- Session already processed: ' +
GamitSession.NetName + ' ' +
GamitSession.date.yyyyddd())
tqdm.write(' -- Done initializing and submitting GAMIT sessions')
JobServer.wait()
gamit_pbar.close()
JobServer.close_cluster()
def ExecuteGamit(cnn,
JobServer,
GamitConfig,
stations,
check_stations,
ignore_missing,
dates,
dry_run=False,
create_kml=False):
modules = ('pyRinex', 'datetime', 'os', 'shutil', 'pyBrdc', 'pySp3',
'subprocess', 're', 'pyETM', 'glob', 'platform', 'traceback')
tqdm.write(
' >> %s Creating GAMIT session instances and executing GAMIT, please wait...'
% print_datetime())
sessions = []
archive = pyArchiveStruct.RinexStruct(
cnn) # type: pyArchiveStruct.RinexStruct
for date in tqdm(dates, ncols=80, disable=None):
# make the dir for these sessions
# this avoids a racing condition when starting each process
pwd = GamitConfig.gamitopt['solutions_dir'].rstrip(
'/') + '/' + date.yyyy() + '/' + date.ddd()
if not os.path.exists(pwd):
os.makedirs(pwd)
net_object = Network(cnn, archive, GamitConfig, stations, date,
check_stations, ignore_missing)
sessions += net_object.sessions
# Network outputs the sessions to be processed
# submit them if they are not ready
tqdm.write(
' -- %s %i GAMIT sessions to submit (%i already processed)' %
(print_datetime(),
len([sess for sess in net_object.sessions if not sess.ready]),
len([sess for sess in net_object.sessions if sess.ready])))
pbar = tqdm(total=len(sessions),
disable=None,
desc=' >> GAMIT sessions completion',
ncols=100)
# create the cluster for the run
JobServer.create_cluster(run_gamit_session, (pyGamitTask.GamitTask, ),
gamit_callback,
pbar,
modules=modules)
for GamitSession in sessions:
if not GamitSession.ready:
# do not submit the task if the session is ready!
# tqdm.write(' >> %s Init' % (datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
GamitSession.initialize()
# tqdm.write(' >> %s Done Init' % (datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
task = pyGamitTask.GamitTask(GamitSession.remote_pwd,
GamitSession.params,
GamitSession.solution_pwd)
# tqdm.write(' >> %s Done task' % (datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
JobServer.submit(task, task.params['DirName'], task.date.year,
task.date.doy, dry_run)
msg = 'Submitting for processing'
else:
msg = 'Session already processed'
pbar.update()
tqdm.write(
' -- %s %s %s %s%02i -> %s' %
(print_datetime(), GamitSession.NetName,
GamitSession.date.yyyyddd(), GamitSession.org, GamitSession.subnet
if GamitSession.subnet is not None else 0, msg))
if create_kml:
# generate a KML of the sessions
generate_kml(dates, sessions, GamitConfig)
tqdm.write(' -- %s Done initializing and submitting GAMIT sessions' %
print_datetime())
# DDG: because of problems with keeping the database connection open (in some platforms), we invoke a class
# that just performs a select on the database
timer = DbAlive(cnn, 120)
JobServer.wait()
pbar.close()
timer.stop()
JobServer.close_cluster()
return sessions