def push():
pc = commands.get_pilot_client()
pc.context.push()
click.secho(
'Global projects have been updated. Users will be notified '
'within 24 hours.',
fg='green')
def describe(path, output_json, limit, relative, path_is_sub):
pc = commands.get_pilot_client()
entry = pc.get_search_entry(path,
relative=relative,
path_is_sub=path_is_sub)
if not entry:
click.echo('Unable to find entry')
return
if output_json:
click.echo(json.dumps(entry, indent=4))
return
single_file_entry = get_matching_file(pc.get_globus_http_url(path), entry)
if single_file_entry:
cols = [
'title', 'authors', 'publisher', 'subjects', 'dates', 'data',
'dataframe', 'rows', 'columns', 'formats', 'version', 'size',
'description'
]
output = '\n'.join(
get_formatted_fields(entry, cols, limit=limit) + [''] +
get_single_file_info(entry, single_file_entry) + ['', ''] +
get_location_info(entry))
else:
cols = [
'title', 'authors', 'publisher', 'subjects', 'dates', 'formats',
'version', 'combined_size', 'description', 'files'
]
output = '\n'.join(
get_formatted_fields(entry, cols, limit=limit) + [''] +
get_location_info(entry))
click.echo(output)
def add():
pc = commands.get_pilot_client()
order = ['title', 'short_name', 'description']
base_path = pc.context.get_value('projects_base_path')
PROJECT_QUERIES['short_name']['prompt'] = \
PROJECT_QUERIES['short_name']['prompt'].format(base_path)
titles = [p['title'] for p in pc.project.load_all().values()]
PROJECT_QUERIES['title']['current'] = titles
iv = input_validation.InputValidator(queries=PROJECT_QUERIES, order=order)
project = iv.ask_all()
project.update({
'search_index':
pc.context.get_value('projects_default_search_index'),
'resource_server':
pc.context.get_value('projects_default_resource_server')
})
project['endpoint'] = pc.context.get_value('projects_endpoint')
project['group'] = pc.context.get_value('projects_group')
short_name = project.pop('short_name')
project['base_path'] = os.path.join(base_path, short_name)
click.secho('Updating global project list... ', nl=False)
pc.project.add_project(short_name, project)
tc = pc.get_transfer_client()
tc.operation_mkdir(project['endpoint'], project['base_path'])
pc.context.push()
click.secho('Success', fg='green')
pc.project.set_project(short_name)
click.secho('Switched to project {}'.format(short_name))
click.secho('Your new project "{}" has been added! '
'Users will be notified within 24 hours next time they use '
'this tool.'.format(project['title']),
fg='green')
def edit(project=None):
pc = commands.get_pilot_client()
current = pc.project.current if pc.project.is_set() else None
project = project or current
if project is None:
click.echo('Use "pilot project info <project>" to list info about a '
'project.')
return
info = pc.project.get_info(project)
queries = {
'title': PROJECT_QUERIES['title'].copy(),
'description': PROJECT_QUERIES['description'].copy()
}
titles = [
p['title'] for p in pc.project.load_all().values()
if p['title'] != info['title']
]
queries['title']['current'] = titles
for key in queries:
queries[key]['default'] = info[key]
iv = input_validation.InputValidator(queries=queries,
order=['title', 'description'])
new_info = iv.ask_all()
info.update(new_info)
pc.project.set_project(project, info)
pc.context.push()
click.secho('Project "{}" has been updated'.format(project), fg='green')
def set_command(project):
pc = commands.get_pilot_client()
try:
pc.project.current = project
click.echo(f'Current project set to {project}')
except ValueError as ve:
click.secho(str(ve), fg='red')
def status_command(number):
pc = get_pilot_client()
ordered_tlogs = []
tlog_order = ['id', 'dataframe', 'status', 'start_time', 'task_id']
# Fetch a limmited set of logs by the most recent entries
tlogs = pc.transfer_log.get_log()[:number]
pending_tasks = [t for t in tlogs if t['status'] not in INACTIVE_STATES]
if pending_tasks:
click.secho('Updating tasks...', fg='green')
update_tasks(pending_tasks)
tlogs = pc.transfer_log.get_log()[:number]
for tlog in tlogs:
tlog['id'] = str(tlog['id'])
tlog['start_time'] = tlog['start_time'].strftime('%Y-%m-%d %H:%M')
ordered_tlogs.append([tlog[item] for item in tlog_order])
fmt = '{:4.3}{:30.29}{:10.11}{:18.17}{:37.36}'
header_names = ['ID', 'Dataframe', 'Status', 'Start Time', 'Task ID']
headers = fmt.format(*header_names)
output = '\n'.join([fmt.format(*items) for items in ordered_tlogs])
click.echo(headers)
click.echo(output)
def get_location_info(entry):
short_path = os.path.basename(get_common_path(entry))
pc = commands.get_pilot_client()
return [
'Location Information',
'{:21.20}{}'.format('Subject', pc.get_subject_url(short_path)),
'{:21.20}{}'.format('Portal', pc.get_portal_url(short_path))
]
def validate_is_globus_endpoint(v, entity):
pc = commands.get_pilot_client()
tc = pc.get_transfer_client()
log.debug(f'Checking ep: {entity}')
try:
tc.get_endpoint(entity)
except globus_sdk.exc.TransferAPIError:
log.debug(f'Failed to fetch endpoint {entity}', exc_info=True)
raise exc.PilotValidator(
f'Failed to get endpoint {entity}, please choose '
'a valid Globus Endpoint') from None
def list_command(path, output_json, limit, relative, all_recs):
# Should require login if there are publicly visible records
pc = commands.get_pilot_client()
project = pc.project.current
search_params = {'limit': limit}
if all_recs:
search_params['filters'], relative = [], False
search_results = pc.search(project=project, custom_params=search_params)
log.debug(search_results)
if output_json:
click.echo(json.dumps(search_results, indent=4))
return
path_sub = pc.get_subject_url(path) if relative else path
curated_results = [
r for r in search_results['gmeta'] if path_sub in r['subject']
]
results = 'Showing {}/{} of total results for "{}"'.format(
len(curated_results), search_results['total'], project)
items = ['title', 'data', 'dataframe', 'rows', 'columns', 'size']
titles = get_titles(items) + ['Path']
fmt = '{:21.20}{:11.10}{:10.9}{:7.6}{:7.6}{:7.6}{}'
output = []
for result in curated_results:
# If this path refers to a result in a different base location, skip
# it, it isn't part of this project
if relative and pc.get_path('') not in result['subject']:
log.debug('Skipping result {}'.format(result['subject']))
continue
data = dict(parse_result(result['content'][0], items))
parsed = [data.get(name) for name in items]
parsed += [get_short_path(result) if relative else result['subject']]
parsed = [str(p) for p in parsed]
output.append(fmt.format(*parsed))
if not output:
click.secho('No results to display for "{}"'.format(path or '/'),
fg='yellow')
else:
output = [results, fmt.format(*titles)] + output
click.echo('\n'.join(output))
result_names = [os.path.basename(r['subject']) for r in curated_results]
path_info = pc.ls(path, extended=True)
dirs = [
name for name, info in path_info.items()
if info['type'] == 'dir' and name not in result_names
]
if dirs:
click.echo('\nDirectories:\n\t{}'.format('\n\t'.join(dirs)))
else:
click.echo('\nNo Directories in {}'.format(path or '/'))
def index_command(ctx):
pc = commands.get_pilot_client()
if ctx.invoked_subcommand is None:
click.echo('Set index with "pilot index set <index_uuid>|'
'<index_name>"')
contexts = pc.context.load_all()
fmt = '{} {}'
for context in contexts:
if context == pc.context.current:
click.secho(fmt.format('*', context), fg='green')
else:
click.echo(fmt.format(' ', context))
def setup():
"""Setup a brand new pilot config for a new index.
NOTE: Paths with tilde '~' DO NOT WORK. These cause problems for resolving
paths that look like /~/foo/bar, which sometimes translate as ~/foo/bar
instead. These are disabled to prevent that from happening.
"""
PROJECT_QUERIES = {
'projects_endpoint': {
'prompt': 'Set a Globus UUID where your data should be stored.',
'default': '',
'help': 'visit "https://app.globus.org/file-manager/collections" '
'to find Globus endpoint to store your data.',
'validation': [input_validation.validate_is_uuid,
input_validation.validate_is_globus_endpoint],
},
'projects_base_path': {
'prompt': 'Pick a base path.',
'default': '/',
'help': 'All data will be saved under this directory',
'validation': [input_validation.validate_no_spaces,
input_validation.validate_absolute_path,
input_validation.validate_no_tilde],
},
'projects_group': {
'prompt': 'Pick a Globus Group to secure Globus Search records',
'default': 'public',
'help': 'The group determines who can view records in search. '
'People not in this group will not see records in Globus '
'Search. "public" allows anyone to see these records.',
'validation': [input_validation.validate_is_valid_globus_group],
},
}
pc = commands.get_pilot_client()
projects = pc.project.load_all().keys()
if projects:
click.secho(
f'Index is already setup with the following projects: {projects}. '
f'Please delete them before setting up your index.', fg='red')
return
order = ['projects_endpoint', 'projects_base_path', 'projects_group']
iv = input_validation.InputValidator(queries=PROJECT_QUERIES, order=order)
new_ctx = iv.ask_all()
pc.context.update_context(new_ctx)
pc.context.push()
click.secho('Your index has been setup successfully. Now see '
'`pilot project add`.', fg='green')
return
def validate_slug_to_path_unique(v, slug):
pc = commands.get_pilot_client()
tc = pc.get_transfer_client()
try:
ctx = pc.context.get_context()
ep, path = ctx.get('projects_endpoint'), ctx.get('projects_base_path')
log.debug('Checking ep: {} path: {}'.format(ep, path))
response = tc.operation_ls(ep, path=path)
existing = [f['name'] for f in response.data['DATA']]
if slug in existing:
raise exc.PilotValidator('"{}" is not available, please choose '
'another'.format(slug))
except globus_sdk.exc.TransferAPIError as tapie:
log.exception(tapie.message)
raise exc.PilotValidator('An error occurred, please try a different '
'value or notify a system administrator.')
def info(index=None):
pc = commands.get_pilot_client()
index = index or pc.context.current
if index is None:
click.echo('Use "pilot index info <context>" to list info about a '
'project.')
return
try:
info = pc.context.get_context(index)
except exc.PilotInvalidProject as pip:
click.secho(str(pip), fg='red')
return
fmt = '{:40.39}{}'
log.debug(info)
output = '\n'.join([fmt.format(*i) for i in info.items()])
click.echo(output)
click.echo()
def cli(ctx):
pc = commands.get_pilot_client()
if not pc.config.is_migrated():
click.secho('Old config detected, upgrading... ', fg='yellow',
nl=False)
try:
pc.config.migrate()
click.secho('Success!', fg='green')
except Exception:
click.secho(f'Failed! Try removing '
f'{pc.config_file} and logging in '
f'again.', fg='red')
if pc.is_logged_in():
if pc.context.is_cache_stale():
log.debug('Cache is stale! Updating...')
try:
if any(pc.context.update_with_diff(dry_run=True).values()):
click.secho('Projects have updated. Use '
'"pilot project update"'
' to get the newest changes.', fg='yellow')
log.debug('Update was successful.')
except globus_sdk.exc.SearchAPIError as sapie:
if not sapie.code == 'NotFound.Generic':
log.error('This error is unexpected!')
log.exception(sapie)
click.secho(
'No manifest exists on this index. You can add '
'one by running `pilot context push`', fg='red')
pcommands = ['delete', 'describe', 'download', 'list', 'mkdir',
'upload']
if not pc.project.is_set() and ctx.invoked_subcommand in pcommands:
click.secho('No project set, use "pilot project" to list projects '
'and "pilot project set <myproject>" '
'to set your current project.', fg='yellow')
sys.exit(exc.ExitCodes.INVALID_CLIENT_CONFIGURATION)
else:
if (ctx.invoked_subcommand and
ctx.invoked_subcommand not in INVOKABLE_WITHOUT_LOGIN):
click.echo('You are not logged in.')
sys.exit(exc.ExitCodes.NOT_LOGGED_IN)
if ctx.invoked_subcommand is None:
click.echo(ctx.get_help())
def update(dry_run, update_groups_cache):
pc = commands.get_pilot_client()
try:
output = pc.context.update_with_diff(
dry_run=dry_run, update_groups_cache=update_groups_cache)
if not any(output.values()):
click.secho('Project is up to date', fg='green')
return
for group, changes in output.items():
if not changes:
continue
click.echo('{}:'.format(group))
for change_type, items in changes.items():
click.echo('\t{}:'.format(change_type))
for name, value in items.items():
click.echo('\t\t{} -> {}'.format(name, value))
except globus_sdk.exc.SearchAPIError as sapie:
click.secho(str(sapie), fg='red')
click.secho(
'You can create the manifest for this index with `pilot '
'context push`',
fg='blue')
def project_command(ctx):
pc = commands.get_pilot_client()
if not pc.is_logged_in():
click.echo('You are not logged in.')
return
invalid_with_pending_update = ['delete', 'add']
if ctx.invoked_subcommand in invalid_with_pending_update:
if any(pc.context.update_with_diff(dry_run=True).values()):
click.secho(
'There is an update for projects, please update '
'("pilot project update") before adding a new project',
fg='red')
sys.exit()
if ctx.invoked_subcommand is None:
click.echo('Set project with "pilot project set <myproject>"')
projects = pc.project.load_all()
current = pc.project.current if pc.project.is_set() else None
fmt = '{} {}'
for project in projects:
if project == current:
click.secho(fmt.format('*', project), fg='green')
else:
click.echo(fmt.format(' ', project))
def update_tasks(transfer_tasks):
"""
Update pending Globus Transfer tasks, and save the resulting status to
the config. transfer tasks is a list of dicts as returned by
config.get_pc.transfer_log()
User must be logged in!
"""
pc = get_pilot_client()
auth = pc.get_authorizers()['transfer.api.globus.org']
tc = globus_sdk.TransferClient(authorizer=auth)
user_tasks = {
r.data['task_id']: r.data
for r in tc.task_list(num_results=100).data
}
for task in transfer_tasks:
task_data = user_tasks.get(task['task_id'])
if not task_data:
click.secho('Unable to update status for {}'.format(task['id']),
fg='yellow')
else:
status = task_data.get('nice_status') or task_data.get('status')
pc.transfer_log.update_log(task['task_id'], status)
def info(project=None):
pc = commands.get_pilot_client()
current = pc.project.current if pc.project.is_set() else None
project = project or current
if project is None:
click.echo('Use "pilot project info <project>" to list info about a '
'project.')
return
try:
info = pc.project.get_info(project)
except exc.PilotInvalidProject as pip:
click.secho(str(pip), fg='red')
return
ep_name = info['endpoint']
try:
tc = pc.get_transfer_client()
ep = tc.get_endpoint(info['endpoint']).data
ep_name = ep['display_name'] or ep['canonical_name'] or ep_name
except globus_sdk.exc.TransferAPIError:
click.echo(
'Failed to lookup endpoint {}, please ensure it is active.'.format(
info['endpoint']))
dinfo = [
(info['title'], ''),
('Endpoint', ep_name),
('Group', pc.project.lookup_group(info['group'])),
('Base Path', info['base_path']),
]
fmt = '{:25.24}{}'
log.debug(info)
output = '\n'.join([fmt.format(*i) for i in dinfo])
click.echo(output)
click.echo()
click.echo(info['description'])
def validate_project_path_unique(v, path):
pc = commands.get_pilot_client()
paths = [p['base_path'] for p in pc.project.load_all().values()]
if path in paths:
raise exc.PilotValidator('Path must be unique. (Other paths include '
f'{", ".join(paths)})')
def set_index(ctx, index_name):
pc = commands.get_pilot_client()
if not pc.context.get_context(index_name):
log.debug(f'No local context "{index_name}", attempting lookup...')
try:
# Ensure we're using the index UUID, lookup by name is no longer
# possible
uuid.UUID(index_name)
index_uuid = index_name
click.secho('looking up {}...'.format(index_uuid))
sc = pc.get_search_client()
index_info = sc.get_index(index_uuid)
display_name = index_info['display_name']
pc.context.add_context(display_name, {
'client_id': 'e4d82438-00df-4dbd-ab90-b6258933c335',
'app_name': '{} app'.format(display_name),
'manifest_index': index_uuid,
'manifest_subject': 'globus://project-manifest.json',
'scopes': pc.DEFAULT_SCOPES,
'projects_cache_timeout': DEFAULT_PROJECTS_CACHE_TIMEOUT,
'projects_endpoint': '',
'projects_base_path': '',
'projects_group': '',
'projects_default_search_index': index_uuid,
'projects_default_resource_server': 'petrel_https_server',
})
index_name = display_name
except ValueError:
click.secho(f'"{index_name}": must be a UUID when adding a new '
f'un-tracked search index.', fg='red')
sys.exit(1)
except Exception as e:
log.exception(e)
click.secho('Unable to find index {}'.format(index_name))
sys.exit(2)
pc.context.set_context(index_name)
try:
log.debug('Updating index...')
pc.context.update()
except exc.NoManifestException:
click.secho('Pilot has not been setup for this index. Run '
'"pilot index setup" to do so.', fg='red')
sys.exit(3)
# If there is only one project, automatically set pilot to that one
if len(pc.project.load_all()) == 1:
projects = pc.project.load_all()
pc.project.current = list(projects.keys())[0]
log.debug('Context set! Fetching general info for user...')
ctx.invoke(info, index=index_name)
ctx.invoke(project_command)
try:
pc.load_tokens(requested_scopes=pc.context.get_value('scopes'))
except ScopesMismatch:
click.secho('Scopes do not match for this index, please login '
'again.', fg='red')
log.debug(pc.context.get_value('projects_default_search_index'))
if not pc.project.load_all():
example = '''
[[default-project]]
base_path = /
description =
endpoint =
group =
resource_server = petrel_https_server
search_index = {}
title = default-project
'''.format(pc.context.get_value('projects_default_search_index'))
click.secho('No projects in this index, you can bootstrap one in the '
'config under [projects]\n{}'.format(example))
def delete(project, keep_context):
pc = commands.get_pilot_client()
if project not in pc.project.load_all():
click.secho('{} is not a valid project'.format(project), fg='red')
return 1
pc.project.current = project
results = pc.search(project=project)
pinfo = pc.project.get_info(project)
search_query = {
'q': '*',
'filters': {
'field_name': 'project_metadata.project-slug',
'type': 'match_all',
'values': [project or pc.project.current]
}
}
project_base_path = pc.get_path('', project=project)
search_client = pc.get_search_client()
transfer_client = pc.get_transfer_client()
log.debug('Base path for delete is: {}'.format(project_base_path))
dz = '\n{}\nDANGER ZONE\n{}'.format('/' * 80, '/' * 80)
click.secho('{dz}\n'
'This will delete all data and search results in your '
'project.\n{tot} datasets will be deleted for {project}\n\n'
'Base Directory "{project_base_path}" will be deleted.'
'{dz}'
''.format(dz=dz,
tot=results['total'],
project=project,
project_base_path=project_base_path),
bg='red')
click.echo(
'Please type the name ({}) of your project to delete it> '.format(
project),
nl=False)
if input() != project:
click.secho('Names do not match, aborting...')
return 1
click.echo(f'Deleting Data: {project_base_path}')
try:
ddata = globus_sdk.DeleteData(transfer_client,
pinfo['endpoint'],
recursive=True,
notify_on_succeeded=False)
ddata.add_item(project_base_path)
transfer_client.submit_delete(ddata)
except globus_sdk.exc.TransferAPIError as tapie:
log.debug('Error deleting base folder', exc_info=tapie)
click.secho(
f'Error deleting project base folder {project_base_path}: '
f'{str(tapie)}',
fg='red')
try:
click.echo(f'Deleting Search Records: {pinfo["search_index"]}')
log.debug(f'Search Query: {search_query}')
search_client.delete_by_query(pinfo['search_index'], search_query)
except globus_sdk.exc.SearchAPIError as sapie:
log.debug('Error deleting test data', exc_info=sapie)
click.secho(
f'Error deleting search data {pinfo["search_index"]}: '
f'{str(sapie)}',
fg='red')
if keep_context:
click.secho('Keeping now empty pilot project {}'.format(project),
fg='blue')
pc.mkdir('/')
else:
click.echo('Removing project...')
pc.project.delete_project(project)
pc.context.push()
click.secho('Project {} has been deleted successfully.'.format(project),
fg='green')
def validate_project_endpoint(v, ep):
pc = commands.get_pilot_client()
if ep not in pc.project.ENDPOINTS.keys():
raise exc.PilotValidator('Endpoint must be one of: "{}"'.format(
', '.join(pc.project.ENDPOINTS.keys())))
def get_short_path(result):
pc = commands.get_pilot_client()
sub = urllib.parse.urlparse(result['subject'])
return sub.path.replace(pc.get_path(''), '').lstrip('/')
def validate_project_slug_unique(v, slug):
pc = commands.get_pilot_client()
slugs = pc.project.load_all().keys()
if slug in slugs:
raise exc.PilotValidator(f'Slug must be unique from {", ".join(slug)}')
