def create_detector_dir_name(args):
'''Create a directory in the detector computer.
'''
exp_name = make_directory_name(args)
# detector_path = Path(args.detector_top_dir).joinpath(args.year_month,exp_name)
detector_path = Path(args.detector_top_dir).joinpath(exp_name)
log.info('Directory on analysis machine: {:s}'.format(str(detector_path)))
return str(detector_path)
def show_endpoints(args, ac, tc):
log.info(
'Show all endpoints shared and owned by my globus user credentials')
log.info("*** Endpoints shared with me:")
for ep in tc.endpoint_search(filter_scope="shared-with-me"):
log.info("*** *** [{}] {}".format(ep["id"], ep["display_name"]))
log.info("*** Endpoints owned with me::")
for ep in tc.endpoint_search(filter_scope="my-endpoints"):
log.info("*** *** [{}] {}".format(ep["id"], ep["display_name"]))
def make_username_list(args):
'''Make a list of the usernames from the current DM experiment.
'''
log.info('Making a list of DM system usernames from current DM experiment')
exp_name = directories.make_directory_name(args)
try:
exp_obj = exp_api.getExperimentByName(exp_name)
return exp_obj['experimentUsernameList']
except:
log.error('No such experiment in the DM system: {:s}'.format(exp_name))
log.error(' Have you run globus dm_init yet?')
return []
def mkdir(remote_server, remote_dir):
'''Create a directory on the analysis and detector computers.
These computers are generally located in the same local network at the beamline.
'''
log.info('Creating directory on server %s:%s' % (remote_server, remote_dir))
path_parts = Path(remote_dir).parts
parent_path = Path(path_parts[0])
for p in path_parts[1:]:
parent_path = parent_path.joinpath(p)
ret = check_local_directory(remote_server , str(parent_path))
if ret == 2:
iret = create_local_directory(remote_server, str(parent_path))
def add_users(exp_obj, username_list):
'''Add a list of users to a DM experiment
'''
existing_unames = exp_obj['experimentUsernameList']
for uname in username_list:
user_obj = user_api.getUserByUsername(uname)
if uname in existing_unames:
log.warning(
' User {:s} is already a user for the experiment'.format(
make_pretty_user_name(user_obj)))
continue
user_api.addUserExperimentRole(uname, 'User', exp_obj['name'])
log.info(' Added user {0:s} to the DM experiment'.format(
make_pretty_user_name(user_obj)))
def create_dir(
directory, # Subdirectory name under top to be created
args,
ac, # Authorize client
tc): # Transfer client
new_dir_path = args.globus_server_top_dir + directory + '/'
try:
response = tc.operation_mkdir(args.globus_server_uuid,
path=new_dir_path)
log.info('*** Created folder: %s' % new_dir_path)
return True
except:
log.warning('*** Path %s already exists' % new_dir_path)
return False
def create_local_directory(remote_server, remote_dir):
'''Create directory and all necessary parent directories, on the
analysis and detector computers. These computers are generally
located in the same local network at the beamline.
'''
cmd = 'mkdir -m 777 ' + remote_dir
try:
# log.info(' *** sending command %s' % (cmd))
log.info(' *** creating remote directory %s' % (remote_dir))
subprocess.check_call(['ssh', remote_server, cmd])
log.info(' *** creating remote directory %s: Done!' % (remote_dir))
return 0
except subprocess.CalledProcessError as e:
log.error(' *** Error while creating remote directory. Error code: %d' % (e.returncode))
return -1
def send_email(args):
log.info("email will contain %s download data link",
args.globus_server_name)
log.info("send email to users?")
if not yes_or_no(' *** Yes or No'):
log.info(' ')
log.warning(' *** Message not not sent')
return False
users = dm.list_users_this_dm_exp(args)
emails = dm.make_user_email_list(users)
#users = scheduling.get_current_users(args)
#emails = scheduling.get_current_emails(users, exclude_pi=False)
emails.append(args.primary_beamline_contact_email)
emails.append(args.secondary_beamline_contact_email)
if (args.globus_server_name == 'voyager'):
s = smtplib.SMTP('mailhost.anl.gov')
for em in emails:
if args.msg['To'] is None:
args.msg['To'] = em
else:
args.msg.replace_header('To', em)
log.info(' Sending informational message to {:s}'.format(em))
s.send_message(args.msg)
s.quit()
elif (args.globus_server_name == 'petrel'):
# # see https://globus-sdk-python.readthedocs.io/en/stable/tutorial/#step-1-get-a-client
# # to create your project app_id. Once is set put it in globus.config app-id field
ac, tc = globus.create_clients(args)
log.info('Creating user directories on server %s:%s' %
(args.globus_server_uuid, args.globus_server_top_dir))
# try to create the directories to share on the globus server in case after the globus init the pi last name was manually changed
globus.create_globus_dir(args, ac, tc)
new_dir = args.year_month + '/' + args.pi_last_name
for email in emails:
args.pi_email = email
log.warning('Sharing %s%s with %s' %
(args.globus_server_top_dir, new_dir, args.pi_email))
globus.share_globus_dir(args, ac, tc)
else:
log.error("%s is not a supported globus server" %
args.globus_server_name)
def list_users_this_dm_exp(args):
'''Provide a list of user names for this DM expt
in the form "d(badge#)"
'''
log.info('Listing the users on the DM experiment')
exp_name = directories.make_directory_name(args)
try:
exp_obj = exp_api.getExperimentByName(exp_name)
except:
log.error(' No appropriate DM experiment found.')
return None
username_list = exp_obj['experimentUsernameList']
if len(username_list) == 0:
log.info(' No users for this experiment')
return None
else:
print(username_list)
return username_list
def show_config(args):
"""Log all values set in the args namespace.
Arguments are grouped according to their section and logged alphabetically
using the DEBUG log level thus --verbose is required.
"""
args = args.__dict__
log.warning('Globus status start')
for section, name in zip(SECTIONS, NICE_NAMES):
entries = sorted((k for k in args.keys()
if k.replace('_', '-') in SECTIONS[section]))
if entries:
for entry in entries:
value = args[entry] if args[entry] != None else "-"
log.info(" {:<16} {}".format(entry, value))
log.warning('Globus status end')
def remove_user(args):
'''Remove a user from the DM experiment.
'''
exp_name = directories.make_directory_name(args)
dm_username = '******'.format(args.edit_user_badge)
try:
user_to_remove = user_api.getUserByUsername(dm_username)
except:
log.error(
' Problem retrieving user information. Check the badge number')
return
log.info('Removing user {0:s} from experiment {1:s}'.format(
make_pretty_user_name(user_to_remove), exp_name))
try:
exp_obj = exp_api.getExperimentByName(exp_name)
except:
log.error(' No appropriate DM experiment found.')
return
try:
user_api.deleteUserExperimentRole(dm_username, 'User', exp_name)
except:
log.error(' Problem removing the user. Check the badge number')
def start_daq(args):
'''Starts the data managememnt (DM) data acquisition (DAQ) system.
In this mode of operation, the DM system will monitor specified data directory
for incoming files, and will transfer data automatically.
Alternative is to upload files after experiment is done.
'''
exp_name = directories.make_directory_name(args)
analysis_dir_name = directories.create_analysis_dir_name(args)
log.info('Check that the directory exists on the analysis machine')
dir_check = directories.check_local_directory(args.analysis,
analysis_dir_name)
if dir_check == 2:
log.info(' Need to make the analysis machine directory')
mkdir_response = directories.create_local_directory(
args.analysis, analysis_dir_name)
if mkdir_response:
log.error(
' Unknown response when creating analysis machine directory. Exiting'
)
return
elif dir_check == 0:
log.info(' Directory already exists')
else:
log.warning(
' Unknown response when checking for analysis machine directory. Exiting'
)
return
dm_dir_name = "@{0:s}:{1:s}".format(args.analysis, analysis_dir_name)
log.info('Check to make sure the appropriate DAQ is not already running.')
current_daqs = daq_api.listDaqs()
for d in current_daqs:
if (d['experimentName'] == exp_name and d['status'] == 'running'
and d['dataDirectory'] == dm_dir_name):
log.warning(' DAQ is already running. Returning.')
return
log.info('Add a DAQ to experiment {:s}'.format(exp_name))
daq_obj = daq_api.startDaq(exp_name, dm_dir_name)
def create_globus_dir(
args,
ac, # Authorize client
tc): # Transfer client
date_dir_path = args.globus_server_top_dir + args.year_month + '/'
pi_last_name_dir_path = args.globus_server_top_dir + args.year_month + '/' + args.pi_last_name + '/'
try:
response = tc.operation_mkdir(args.globus_server_uuid,
path=date_dir_path)
log.info('*** Created folder: %s' % date_dir_path)
except:
log.warning('*** Path %s already exists' % (date_dir_path))
try:
response = tc.operation_mkdir(args.globus_server_uuid,
path=pi_last_name_dir_path)
log.info('*** Created folder: %s' % pi_last_name_dir_path)
return True
except:
log.warning('*** Path %s already exists' % (pi_last_name_dir_path))
return False
def make_dm_username_list(args):
'''Make a list of DM usernames 'd+badge#' from the current proposal (GUP number).
'''
log.info('Making a list of DM system usernames from target proposal')
target_prop = bss_api.getProposal(str(args.prop_number))
users = target_prop['experimenters']
log.info(' Adding the primary beamline contact')
user_ids = {'d' + str(args.primary_beamline_contact_badge)}
for u in users:
log.info(' Adding user {0}, {1}, badge {2}'.format(
u['lastName'], u['firstName'], u['badge']))
user_ids.add('d' + str(u['badge']))
return user_ids
def stop_daq(args):
'''Stops the currently running DM DAQ.
'''
exp_name = directories.make_directory_name(args)
log.info('Stopping all DM DAQs for experiment {:s}'.format(exp_name))
daqs = daq_api.listDaqs()
removed_daq_counter = 0
for d in daqs:
if d['experimentName'] == exp_name and d['status'] == 'running':
log.info(
' Found running DAQ for this experiment. Stopping now.')
daq_api.stopDaq(d['experimentName'], d['dataDirectory'])
removed_daq_counter += 1
if removed_daq_counter == 0:
log.info(' No active DAQs for this experiment were found')
def list_users(args):
'''Lists the users on the current experiment in a nice format.
'''
log.info('Listing the users on the DM experiment')
exp_name = directories.make_directory_name(args)
try:
exp_obj = exp_api.getExperimentByName(exp_name)
except:
log.error(' No appropriate DM experiment found.')
return
username_list = exp_obj['experimentUsernameList']
if len(username_list) == 0:
log.info(' No users for this experiment')
return
for uname in username_list:
user_obj = user_api.getUserByUsername(uname)
log.info(' User {0:s}, badge {1:s} is on the DM experiment'.format(
make_pretty_user_name(user_obj), user_obj['badge']))
def create_experiment(args):
'''Creates a new DM experiment on Voyager.
Parameters
----------
args : list
args is used to extract current year_month, pi_last_name, prop_number,
prop_title and generate a unique DM experiment name in the form of
year-month-PILastName-ProposalNumber
Returns
-------
Experiment object
'''
dir_name = directories.make_directory_name(args)
log.info('See if there is already a DM experiment')
try:
old_exp = exp_api.getExperimentByName(dir_name)
log.warning(' Experiment already exists')
return old_exp
except:
log.info('Creating new DM experiment: {0:s}/{1:s}'.format(
args.year_month, dir_name))
target_prop = bss_api.getProposal(str(args.prop_number))
start_datetime = datetime.datetime.strptime(target_prop['startTime'],
'%Y-%m-%d %H:%M:%S')
end_datetime = datetime.datetime.strptime(target_prop['endTime'],
'%Y-%m-%d %H:%M:%S')
new_exp = exp_api.addExperiment(
dir_name,
typeName=args.experiment_type,
description=args.prop_title,
rootPath=args.year_month,
startDate=start_datetime.strftime('%d-%b-%y'),
endDate=end_datetime.strftime('%d-%b-%y'))
log.info(' Experiment successfully created!')
return new_exp
