def transfer_needed(self, event_list, event, remote_endpoint, ui,
display_event, emailaddr, thread_list):
"""
Start a transfer job for any files that arent local, but do exist remotely
Globus user must already be logged in
Parameters:
event_list (EventList): the list to push information into
event (threadding.event): the thread event to trigger a cancel
"""
if self.active_transfers >= 2:
return False
# required files dont exist locally, do exist remotely
# or if they do exist locally have a different local and remote size
self.mutex.acquire()
try:
required_files = [
x for x in DataFile.select().where(
(DataFile.remote_status == filestatus['EXISTS'])
& (DataFile.local_status != filestatus['IN_TRANSIT'])
& ((DataFile.local_status == filestatus['NOT_EXIST'])
| (DataFile.local_size != DataFile.remote_size)))
]
if len(required_files) == 0:
return False
target_files = []
target_size = 1e11 # 100 GB
total_size = 0
for file in required_files:
if total_size + file.remote_size < target_size:
target_files.append({
'name': file.name,
'local_size': file.local_size,
'local_path': file.local_path,
'local_status': file.local_status,
'remote_size': file.remote_size,
'remote_path': file.remote_path,
'remote_status': file.remote_status
})
total_size += file.remote_size
else:
break
except Exception as e:
print_debug(e)
return False
finally:
if self.mutex.locked():
self.mutex.release()
logging.info('Transfering required files')
print 'total transfer size {size} gigabytes for {nfiles} files'.format(
size=(total_size / 1e9), nfiles=len(target_files))
transfer_config = {
'file_list': target_files,
'source_endpoint': self.remote_endpoint,
'destination_endpoint': self.local_endpoint,
'source_path': self.remote_path,
'destination_path': self.local_path,
'source_email': emailaddr,
'display_event': display_event,
'ui': ui,
}
transfer = Transfer(config=transfer_config, event_list=event_list)
print 'starting transfer for:'
transfer_names = [x['name'] for x in transfer.file_list]
for file in transfer.file_list:
print ' ' + file['name']
logging.info(file['name'])
self.mutex.acquire()
try:
DataFile.update(local_status=filestatus['IN_TRANSIT']).where(
DataFile.name << transfer_names).execute()
print 'following files are in transit'
for df in DataFile.select():
if df.local_status == filestatus['IN_TRANSIT']:
print ' ' + df.name
except Exception as e:
print_debug(e)
return False
finally:
if self.mutex.locked():
self.mutex.release()
args = (transfer, event, event_list)
thread = threading.Thread(target=self._handle_transfer,
name='filemanager_transfer',
args=args)
thread_list.append(thread)
thread.start()
return True
def transfer_needed(self, event_list, event):
"""
Start a transfer job for any files that arent local, but do exist remotely
Globus user must already be logged in
"""
# required files dont exist locally, do exist remotely
# or if they do exist locally have a different local and remote size
target_files = list()
self._mutex.acquire()
try:
q = (DataFile.select(DataFile.case).where(
DataFile.local_status == FileStatus.NOT_PRESENT.value))
caselist = [x.case for x in q.execute()]
if not caselist or len(caselist) == 0:
return
cases = list()
for case in caselist:
if case not in cases:
cases.append(case)
for case in cases:
q = (DataFile.select().where((DataFile.case == case) & (
DataFile.local_status == FileStatus.NOT_PRESENT.value)))
required_files = [x for x in q.execute()]
for file in required_files:
if file.transfer_type == 'local':
required_files.remove(file)
if not required_files:
msg = 'ERROR: all missing files are marked as local'
print_line(msg, self._event_list)
return
# mark files as in-transit so we dont double-copy
q = (DataFile.update({
DataFile.local_status:
FileStatus.IN_TRANSIT
}).where(DataFile.name << [x.name for x in required_files]))
q.execute()
for file in required_files:
target_files.append({
'local_path': file.local_path,
'remote_path': file.remote_path,
})
if required_files[0].transfer_type == 'globus':
msg = 'Starting globus file transfer of {} files'.format(
len(required_files))
print_line(msg, self._event_list)
msg = 'See https://www.globus.org/app/activity for transfer details'
print_line(msg, self._event_list)
client = get_client()
remote_uuid = required_files[0].remote_uuid
local_uuid = self._config['global']['local_globus_uuid']
thread_name = '{}_globus_transfer'.format(
required_files[0].case)
_args = (client, remote_uuid, local_uuid, target_files,
self.kill_event)
thread = Thread(target=globus_transfer,
name=thread_name,
args=_args)
self.thread_list.append(thread)
thread.start()
elif required_files[0].transfer_type == 'sftp':
msg = 'Starting sftp file transfer of {} files'.format(
len(required_files))
print_line(msg, self._event_list)
client = get_ssh_client(required_files[0].remote_hostname)
thread_name = '{}_sftp_transfer'.format(
required_files[0].case)
_args = (target_files, client, self.kill_event)
thread = Thread(target=self._ssh_transfer,
name=thread_name,
args=_args)
self.thread_list.append(thread)
thread.start()
except Exception as e:
print_debug(e)
return False
finally:
if self._mutex.locked():
self._mutex.release()
def update_remote_status(self, client):
"""
Check remote location for existance of the files on our list
If they exist, update their status in the DB
Parameters:
client (globus_sdk.client): the globus client to use for remote query
"""
result = client.endpoint_autoactivate(self.remote_endpoint,
if_expires_in=2880)
if result['code'] == "AutoActivationFailed":
return False
if self.sta:
for _type in self.types:
if _type == 'rest':
if not self.updated_rest:
self.mutex.acquire()
name, path, size = self.update_remote_rest_sta_path(
client)
DataFile.update(
remote_status=filestatus['EXISTS'],
remote_size=size,
remote_path=path,
name=name).where(
DataFile.datatype == 'rest').execute()
if self.mutex.locked():
self.mutex.release()
self.updated_rest = True
continue
elif _type in [
'streams.ocean', 'streams.cice', 'mpas-o_in',
'mpas-cice_in'
]:
remote_path = os.path.join(self.remote_path, 'run')
elif _type == 'meridionalHeatTransport':
remote_path = os.path.join(self.remote_path, 'archive',
'ocn', 'hist')
else:
remote_path = os.path.join(self.remote_path, 'archive',
_type, 'hist')
print 'Querying globus for {}'.format(_type)
res = self._get_ls(client=client, path=remote_path)
self.mutex.acquire()
try:
names = [
x.name for x in DataFile.select().where(
DataFile.datatype == _type)
]
to_update_name = [
x['name'] for x in res if x['name'] in names
]
to_update_size = [
x['size'] for x in res if x['name'] in names
]
q = DataFile.update(
remote_status=filestatus['EXISTS'],
remote_size=to_update_size[to_update_name.index(
DataFile.name)]).where(
(DataFile.name << to_update_name)
& (DataFile.datatype == _type))
n = q.execute()
except Exception as e:
print_debug(e)
print "Do you have the correct start and end dates?"
finally:
if self.mutex.locked():
self.mutex.release()
else:
remote_path = self.remote_path
res = self._get_ls(client=client, path=remote_path)
self.mutex.acquire()
try:
for _type in self.types:
names = [
x.name for x in DataFile.select().where(
DataFile.datatype == _type)
]
to_update_name = [
x['name'] for x in res if x['name'] in names
]
to_update_size = [
x['size'] for x in res if x['name'] in names
]
q = DataFile.update(
remote_status=filestatus['EXISTS'],
remote_size=to_update_size[to_update_name.index(
DataFile.name)]).where(
(DataFile.name << to_update_name)
& (DataFile.datatype == _type))
n = q.execute()
print 'updated {} records'.format(n)
except Exception as e:
print_debug(e)
finally:
if self.mutex.locked():
self.mutex.release()
