def cleanup():
user_identity_name = request.form.get('user_identity_name')
dependent_tokens = get_dependent_tokens(g.req_token)
transfer_token = dependent_tokens.by_resource_server[
'transfer.api.globus.org']['access_token']
dest_ep = app.config['GRAPH_ENDPOINT_ID']
dest_base = app.config['GRAPH_ENDPOINT_BASE']
dest_path = '%sGraphs for %s/' % (dest_base, user_identity_name)
transfer = TransferClient(authorizer=AccessTokenAuthorizer(transfer_token))
transfer.endpoint_autoactivate(dest_ep)
try:
acl = next(acl for acl in transfer.endpoint_acl_list(dest_ep)
if dest_path == acl['path'])
except StopIteration:
pass
except TransferAPIError as ex:
# PermissionDenied can happen if a new Portal client is swapped
# in and it doesn't have endpoint manager on the dest_ep.
# The /portal/processed directory has been set to to writeable
# for all users so the delete task will succeed even if an ACL
# can't be set.
if ex.code == 'PermissionDenied':
pass
else:
transfer.delete_endpoint_acl_rule(dest_ep, acl['id'])
delete_request = DeleteData(transfer_client=transfer,
endpoint=dest_ep,
label="Delete Graphs from the Service Demo",
recursive=True)
delete_request.add_item(dest_path)
try:
task = transfer.submit_delete(delete_request)
except TransferAPIError as ex:
raise InternalServerError(message=ex.message)
else:
return jsonify(dict(task_id=task['task_id']))
def cleanup():
user_identity_name = request.form.get('user_identity_name')
dependent_tokens = get_dependent_tokens(g.req_token)
transfer_token = dependent_tokens.by_resource_server[
'transfer.api.globus.org']['access_token']
dest_ep = app.config['GRAPH_ENDPOINT_ID']
dest_base = app.config['GRAPH_ENDPOINT_BASE']
dest_path = '%sGraphs for %s/' % (dest_base, user_identity_name)
transfer = TransferClient(authorizer=AccessTokenAuthorizer(transfer_token))
transfer.endpoint_autoactivate(dest_ep)
try:
acl = next(acl for acl in transfer.endpoint_acl_list(dest_ep)
if dest_path == acl['path'])
except StopIteration:
pass
except TransferAPIError as ex:
# PermissionDenied can happen if a new Portal client is swapped
# in and it doesn't have endpoint manager on the dest_ep.
# The /portal/processed directory has been set to to writeable
# for all users so the delete task will succeed even if an ACL
# can't be set.
if ex.code == 'PermissionDenied':
pass
else:
transfer.delete_endpoint_acl_rule(dest_ep, acl['id'])
delete_request = DeleteData(transfer_client=transfer, endpoint=dest_ep,
label="Delete Graphs from the Service Demo",
recursive=True)
delete_request.add_item(dest_path)
try:
task = transfer.submit_delete(delete_request)
except TransferAPIError as ex:
raise InternalServerError(message=ex.message)
else:
return jsonify(dict(task_id=task['task_id']))
class TestGlobus(TestZstash):
def preactivate_globus(self):
"""
Read the local globus endpoint UUID from ~/.zstash.ini.
If the ini file does not exist, create an ini file with empty values,
and try to find the local endpoint UUID based on the FQDN
"""
local_endpoint = None
ini_path = os.path.expanduser("~/.zstash.ini")
ini = configparser.ConfigParser()
if ini.read(ini_path):
if "local" in ini.sections():
local_endpoint = ini["local"].get("globus_endpoint_uuid")
else:
ini["local"] = {"globus_endpoint_uuid": ""}
try:
with open(ini_path, "w") as f:
ini.write(f)
except Exception as e:
self.fail(e)
if not local_endpoint:
fqdn = socket.getfqdn()
for pattern in regex_endpoint_map.keys():
if re.fullmatch(pattern, fqdn):
local_endpoint = regex_endpoint_map.get(pattern)
break
if not local_endpoint:
# self.fail("{} does not have the local Globus endpoint set".format(ini_path))
self.skipTest(
"{} does not have the local Globus endpoint set".format(
ini_path))
native_client = NativeClient(
client_id="6c1629cf-446c-49e7-af95-323c6412397f",
app_name="Zstash",
default_scopes=
"openid urn:globus:auth:scope:transfer.api.globus.org:all",
)
native_client.login(no_local_server=True, refresh_tokens=True)
transfer_authorizer = native_client.get_authorizers().get(
"transfer.api.globus.org")
self.transfer_client = TransferClient(transfer_authorizer)
for ep_id in [hpss_globus_endpoint, local_endpoint]:
r = self.transfer_client.endpoint_autoactivate(ep_id,
if_expires_in=600)
if r.get("code") == "AutoActivationFailed":
self.fail(
"The {} endpoint is not activated or the current activation expires soon. Please go to https://app.globus.org/file-manager/collections/{} and (re)-activate the endpoint."
.format(ep_id, ep_id))
def delete_files_globus(self):
ep_id = hpss_globus_endpoint
r = self.transfer_client.endpoint_autoactivate(ep_id, if_expires_in=60)
if r.get("code") == "AutoActivationFailed":
self.fail(
"The {} endpoint is not activated. Please go to https://app.globus.org/file-manager/collections/{} and activate the endpoint."
.format(ep_id, ep_id))
ddata = DeleteData(self.transfer_client,
hpss_globus_endpoint,
recursive=True)
ddata.add_item("/~/zstash_test/")
try:
task = self.transfer_client.submit_delete(ddata)
task_id = task.get("task_id")
"""
A Globus transfer job (task) can be in one of the three states:
ACTIVE, SUCCEEDED, FAILED. The script every 5 seconds polls a
status of the transfer job (task) from the Globus Transfer service,
with 5 second timeout limit. If the task is ACTIVE after time runs
out 'task_wait' returns False, and True otherwise.
"""
while not self.transfer_client.task_wait(task_id, 5, 5):
task = self.transfer_client.get_task(task_id)
if task.get("is_paused"):
break
"""
The Globus transfer job (task) has been finished (SUCCEEDED or FAILED),
or is still active (ACTIVE). Check if the transfer SUCCEEDED or FAILED.
"""
task = self.transfer_client.get_task(task_id)
if task["status"] == "SUCCEEDED":
pass
elif task.get("status") == "ACTIVE":
if task.get("is_paused"):
pause_info = self.transfer_client.task_pause_info(task_id)
paused_rules = pause_info.get("pause_rules")
reason = paused_rules[0].get("message")
message = "The task was paused. Reason: {}".format(reason)
print(message)
else:
message = "The task reached a {} second deadline\n".format(
24 * 3600)
print(message)
self.transfer_client.cancel_task(task_id)
else:
print("Globus delete FAILED")
except TransferAPIError as e:
if e.code == "NoCredException":
self.fail(
"{}. Please go to https://app.globus.org/endpoints and activate the endpoint."
.format(e.message))
else:
self.fail(e)
except Exception as e:
self.fail("{} - exception: {}".format(self, e))
def tearDown(self):
"""
Tear down a test. This is run after every test method.
After the script has failed or completed, remove all created files, even those on the HPSS repo.
"""
os.chdir(TOP_LEVEL)
print("Removing test files, both locally and at the HPSS repo")
# self.cache may appear in any of these directories,
# but should not appear at the same level as these.
# Therefore, there is no need to explicitly remove it.
for d in [self.test_dir, self.backup_dir]:
if os.path.exists(d):
shutil.rmtree(d)
if self.hpss_path and self.hpss_path.lower().startswith("globus:"):
self.delete_files_globus()
def helperLsGlobus(self,
test_name,
hpss_path,
cache=None,
zstash_path=ZSTASH_PATH):
"""
Test `zstash ls --hpss=globus://...`.
"""
self.preactivate_globus()
self.hpss_path = hpss_path
if cache:
# Override default cache
self.cache = cache
cache_option = " --cache={}".format(self.cache)
else:
cache_option = ""
use_hpss = self.setupDirs(test_name)
self.create(use_hpss, zstash_path, cache=self.cache)
self.assertWorkspace()
os.chdir(self.test_dir)
for option in ["", "-v", "-l"]:
print_starred("Testing zstash ls {}".format(option))
cmd = "{}zstash ls{} {} --hpss={}".format(zstash_path,
cache_option, option,
self.hpss_path)
output, err = run_cmd(cmd)
self.check_strings(cmd, output + err, ["file0.txt"], ["ERROR"])
os.chdir(TOP_LEVEL)
def testLs(self):
self.helperLsGlobus("testLsGlobus",
f"globus://{hpss_globus_endpoint}/~/zstash_test/")
