def test_download_files(): def _requests_mock(method, url, **kwargs): response = Mock() response.headers = { 'Content-Disposition': 'attachment; ' 'filename={}'.format(url.split('/')[-1])} return response def _download_file_mock(url, file_name, **kwargs): return file_name alma = Alma() alma._request = Mock(side_effect=_requests_mock) alma._download_file = Mock(side_effect=_download_file_mock) downloaded_files = alma.download_files(['https://location/file1']) assert len(downloaded_files) == 1 assert downloaded_files[0].endswith('file1') alma._request.reset_mock() alma._download_file.reset_mock() downloaded_files = alma.download_files(['https://location/file1', 'https://location/file2']) assert len(downloaded_files) == 2 # error cases alma._request = Mock() # no Content-Disposition results in no downloaded file alma._request.return_value = Mock(headers={}) result = alma.download_files(['https://location/file1']) assert not result
from astroquery.alma import Alma import os import shutil import time from bs4 import BeautifulSoup alma = Alma() rslt = alma.query({'project_code': '2019.1.00092.S'}, public=False) index = "https://bulk.cv.nrao.edu/almadata/proprietary/keflavich/" alma.login('keflavich') ind = alma._request('GET', index, cache=False) soup = BeautifulSoup(ind.text) urllist = [index+'/'+xx.attrs['href'].strip("/") for xx in soup.findAll('a')] ['https://bulk.cv.nrao.edu/almadata/proprietary/keflavich/X3a53', 'https://bulk.cv.nrao.edu/almadata/proprietary/keflavich/X3a5f', 'https://bulk.cv.nrao.edu/almadata/proprietary/keflavich/X3a57', 'https://bulk.cv.nrao.edu/almadata/proprietary/keflavich/X3a63', 'https://bulk.cv.nrao.edu/almadata/proprietary/keflavich/X3a33'] for baseurl in urllist: nm = baseurl.split("/")[-1] if os.path.exists(f'{nm}_calibrated_final.ms.tgz'): print(f"Skipped {baseurl} b/c it's done.") continue if 'X' not in nm: # simply skip... continue alma = Alma()