def share_globus_dir(
args,
ac, # Authorize client
tc): # Transfer client
# Query to Auth Client to verify if a globus user ID is associated to the user email address, if not one is generated
response = ac.get("/v2/api/identities?usernames=" + args.pi_email)
# Get user id from user email
r = ac.get_identities(usernames=args.pi_email)
user_id = r['identities'][0]['id']
# log.info(r, user_id)
directory_full_path = args.globus_server_top_dir + args.year_month + '/' + args.pi_last_name + '/'
# Set access control and notify user
rule_data = {
'DATA_TYPE': 'access',
'principal_type': 'identity',
'principal': user_id,
'path': directory_full_path,
'permissions': 'r',
'notify_email': args.pi_email,
'notify_message': args.msg.as_string()
}
try:
response = tc.add_endpoint_acl_rule(args.globus_server_uuid, rule_data)
log.info('*** Path %s has been shared with %s' %
(directory_full_path, args.pi_email))
return True
except:
log.warning('*** Path %s is already shared with %s' %
(directory_full_path, args.pi_email))
return False
def make_user_email_list(username_list):
'''Make a list of e-mail addresses from a list of DM usernames.
Parameters
----------
username_list : list
list of DM usernames, each of which is in the form 'd+badge#'.
Returns
-------
list
e-mail addresses.
'''
email_list = []
for u in username_list:
try:
user_obj = user_api.getUserByUsername(u)
email_list.append(user_obj['email'])
log.info(' Added email {:s} for user {:s}'.format(
email_list[-1], u))
except:
log.warning(' Problem loading email for user {:s}'.format(u))
return email_list
def yes_or_no(question):
answer = str(input(question + " (Y/N): ")).lower().strip()
while not (answer == "y" or answer == "yes" or answer == "n"
or answer == "no"):
log.warning("Input yes or no")
answer = str(input(question + "(Y/N): ")).lower().strip()
if answer[0] == "y":
return True
else:
return False
def add_users(exp_obj, username_list):
'''Add a list of users to a DM experiment
'''
existing_unames = exp_obj['experimentUsernameList']
for uname in username_list:
user_obj = user_api.getUserByUsername(uname)
if uname in existing_unames:
log.warning(
' User {:s} is already a user for the experiment'.format(
make_pretty_user_name(user_obj)))
continue
user_api.addUserExperimentRole(uname, 'User', exp_obj['name'])
log.info(' Added user {0:s} to the DM experiment'.format(
make_pretty_user_name(user_obj)))
def create_dir(
directory, # Subdirectory name under top to be created
args,
ac, # Authorize client
tc): # Transfer client
new_dir_path = args.globus_server_top_dir + directory + '/'
try:
response = tc.operation_mkdir(args.globus_server_uuid,
path=new_dir_path)
log.info('*** Created folder: %s' % new_dir_path)
return True
except:
log.warning('*** Path %s already exists' % new_dir_path)
return False
def send_email(args):
log.info("email will contain %s download data link",
args.globus_server_name)
log.info("send email to users?")
if not yes_or_no(' *** Yes or No'):
log.info(' ')
log.warning(' *** Message not not sent')
return False
users = dm.list_users_this_dm_exp(args)
emails = dm.make_user_email_list(users)
#users = scheduling.get_current_users(args)
#emails = scheduling.get_current_emails(users, exclude_pi=False)
emails.append(args.primary_beamline_contact_email)
emails.append(args.secondary_beamline_contact_email)
if (args.globus_server_name == 'voyager'):
s = smtplib.SMTP('mailhost.anl.gov')
for em in emails:
if args.msg['To'] is None:
args.msg['To'] = em
else:
args.msg.replace_header('To', em)
log.info(' Sending informational message to {:s}'.format(em))
s.send_message(args.msg)
s.quit()
elif (args.globus_server_name == 'petrel'):
# # see https://globus-sdk-python.readthedocs.io/en/stable/tutorial/#step-1-get-a-client
# # to create your project app_id. Once is set put it in globus.config app-id field
ac, tc = globus.create_clients(args)
log.info('Creating user directories on server %s:%s' %
(args.globus_server_uuid, args.globus_server_top_dir))
# try to create the directories to share on the globus server in case after the globus init the pi last name was manually changed
globus.create_globus_dir(args, ac, tc)
new_dir = args.year_month + '/' + args.pi_last_name
for email in emails:
args.pi_email = email
log.warning('Sharing %s%s with %s' %
(args.globus_server_top_dir, new_dir, args.pi_email))
globus.share_globus_dir(args, ac, tc)
else:
log.error("%s is not a supported globus server" %
args.globus_server_name)
def show_config(args):
"""Log all values set in the args namespace.
Arguments are grouped according to their section and logged alphabetically
using the DEBUG log level thus --verbose is required.
"""
args = args.__dict__
log.warning('Globus status start')
for section, name in zip(SECTIONS, NICE_NAMES):
entries = sorted((k for k in args.keys()
if k.replace('_', '-') in SECTIONS[section]))
if entries:
for entry in entries:
value = args[entry] if args[entry] != None else "-"
log.info(" {:<16} {}".format(entry, value))
log.warning('Globus status end')
def create_clients(args):
"""
Create authorize and transfer clients
Parameters
----------
globus_app_id : App UUID
Returns
-------
ac : Authorize client
tc : Transfer client
"""
token_response = refresh_globus_token(args)
# let's get stuff for the Globus Transfer service
globus_transfer_data = token_response.by_resource_server[
'transfer.api.globus.org']
# the refresh token and access token, often abbr. as RT and AT
transfer_rt = globus_transfer_data['refresh_token']
transfer_at = globus_transfer_data['access_token']
expires_at_s = globus_transfer_data['expires_at_seconds']
globus_token_life = expires_at_s - time.time()
log.warning("Globus access token will expire in %2.2f hours",
(globus_token_life / 3600))
# see https://globus-sdk-python.readthedocs.io/en/stable/tutorial/#step-1-get-a-client
# to create your project app_id. Once is set put it in globus.config app-id field
globus_app_id = args.globus_app_uuid
client = globus_sdk.NativeAppAuthClient(globus_app_id)
client.oauth2_start_flow(refresh_tokens=True)
# Now we've got the data we need, but what do we do?
# That "GlobusAuthorizer" from before is about to come to the rescue
authorizer = globus_sdk.RefreshTokenAuthorizer(transfer_rt,
client,
access_token=transfer_at,
expires_at=expires_at_s)
# and try using `tc` to make TransferClient calls. Everything should just
# work -- for days and days, months and months, even years
ac = globus_sdk.AuthClient(authorizer=authorizer)
tc = globus_sdk.TransferClient(authorizer=authorizer)
return ac, tc
def check_local_directory(remote_server, remote_dir):
'''Check if a directory exists on the analysis or on the detector
computers. These computers are generally located in the same local
network at the beamline.
'''
try:
rcmd = 'ls ' + remote_dir
# rcmd is the command used to check if the remote directory exists
subprocess.check_call(['ssh', remote_server, rcmd], stderr=open(os.devnull, 'wb'), stdout=open(os.devnull, 'wb'))
log.warning(' *** remote directory %s exists' % (remote_dir))
return 0
except subprocess.CalledProcessError as e:
# log.info(' *** return code = %d' % (e.returncode))
log.warning(' *** remote directory %s does not exist' % (remote_dir))
if e.returncode == 2:
return e.returncode
else:
log.error(' *** Unknown error code returned: %d' % (e.returncode))
return -1
def create_experiment(args):
'''Creates a new DM experiment on Voyager.
Parameters
----------
args : list
args is used to extract current year_month, pi_last_name, prop_number,
prop_title and generate a unique DM experiment name in the form of
year-month-PILastName-ProposalNumber
Returns
-------
Experiment object
'''
dir_name = directories.make_directory_name(args)
log.info('See if there is already a DM experiment')
try:
old_exp = exp_api.getExperimentByName(dir_name)
log.warning(' Experiment already exists')
return old_exp
except:
log.info('Creating new DM experiment: {0:s}/{1:s}'.format(
args.year_month, dir_name))
target_prop = bss_api.getProposal(str(args.prop_number))
start_datetime = datetime.datetime.strptime(target_prop['startTime'],
'%Y-%m-%d %H:%M:%S')
end_datetime = datetime.datetime.strptime(target_prop['endTime'],
'%Y-%m-%d %H:%M:%S')
new_exp = exp_api.addExperiment(
dir_name,
typeName=args.experiment_type,
description=args.prop_title,
rootPath=args.year_month,
startDate=start_datetime.strftime('%d-%b-%y'),
endDate=end_datetime.strftime('%d-%b-%y'))
log.info(' Experiment successfully created!')
return new_exp
def start_daq(args):
'''Starts the data managememnt (DM) data acquisition (DAQ) system.
In this mode of operation, the DM system will monitor specified data directory
for incoming files, and will transfer data automatically.
Alternative is to upload files after experiment is done.
'''
exp_name = directories.make_directory_name(args)
analysis_dir_name = directories.create_analysis_dir_name(args)
log.info('Check that the directory exists on the analysis machine')
dir_check = directories.check_local_directory(args.analysis,
analysis_dir_name)
if dir_check == 2:
log.info(' Need to make the analysis machine directory')
mkdir_response = directories.create_local_directory(
args.analysis, analysis_dir_name)
if mkdir_response:
log.error(
' Unknown response when creating analysis machine directory. Exiting'
)
return
elif dir_check == 0:
log.info(' Directory already exists')
else:
log.warning(
' Unknown response when checking for analysis machine directory. Exiting'
)
return
dm_dir_name = "@{0:s}:{1:s}".format(args.analysis, analysis_dir_name)
log.info('Check to make sure the appropriate DAQ is not already running.')
current_daqs = daq_api.listDaqs()
for d in current_daqs:
if (d['experimentName'] == exp_name and d['status'] == 'running'
and d['dataDirectory'] == dm_dir_name):
log.warning(' DAQ is already running. Returning.')
return
log.info('Add a DAQ to experiment {:s}'.format(exp_name))
daq_obj = daq_api.startDaq(exp_name, dm_dir_name)
def create_globus_dir(
args,
ac, # Authorize client
tc): # Transfer client
date_dir_path = args.globus_server_top_dir + args.year_month + '/'
pi_last_name_dir_path = args.globus_server_top_dir + args.year_month + '/' + args.pi_last_name + '/'
try:
response = tc.operation_mkdir(args.globus_server_uuid,
path=date_dir_path)
log.info('*** Created folder: %s' % date_dir_path)
except:
log.warning('*** Path %s already exists' % (date_dir_path))
try:
response = tc.operation_mkdir(args.globus_server_uuid,
path=pi_last_name_dir_path)
log.info('*** Created folder: %s' % pi_last_name_dir_path)
return True
except:
log.warning('*** Path %s already exists' % (pi_last_name_dir_path))
return False
def refresh_globus_token(args):
"""
Create and save a globus token. The token is valid for 48h.
Parameters
----------
globus_app_id : App UUID
"""
try:
token_response = np.load(args.globus_token_file,
allow_pickle='TRUE').item()
except FileNotFoundError:
log.error('Globus token is missing. Creating one')
# Creating new token
# --------------------------------------------
globus_app_id = args.globus_app_uuid
client = globus_sdk.NativeAppAuthClient(globus_app_id)
client.oauth2_start_flow(refresh_tokens=True)
log.warning('Please go to this URL and login: {0}'.format(
client.oauth2_get_authorize_url()))
get_input = getattr(__builtins__, 'raw_input', input)
auth_code = get_input(
'Please enter the code you get after login here: ').strip(
) # pythn 3
# auth_code = raw_input('Please enter the code you get after login here: ').strip() # python 2.7
token_response = client.oauth2_exchange_code_for_tokens(auth_code)
# --------------------------------------------
np.save(args.globus_token_file, token_response)
# let's get stuff for the Globus Transfer service
globus_transfer_data = token_response.by_resource_server[
'transfer.api.globus.org']
# the refresh token and access token, often abbr. as RT and AT
transfer_rt = globus_transfer_data['refresh_token']
transfer_at = globus_transfer_data['access_token']
expires_at_s = globus_transfer_data['expires_at_seconds']
globus_token_life = expires_at_s - time.time()
if (globus_token_life < 0):
# Creating new token
# --------------------------------------------
globus_app_id = args.globus_app_uuid
client = globus_sdk.NativeAppAuthClient(globus_app_id)
client.oauth2_start_flow(refresh_tokens=True)
log.warning('Please go to this URL and login: {0}'.format(
client.oauth2_get_authorize_url()))
get_input = getattr(__builtins__, 'raw_input', input)
auth_code = get_input(
'Please enter the code you get after login here: ').strip(
) # pythn 3
# auth_code = raw_input('Please enter the code you get after login here: ').strip() # python 2.7
token_response = client.oauth2_exchange_code_for_tokens(auth_code)
# --------------------------------------------
np.save(args.globus_token_file, token_response)
return token_response
