def test_copy(self, computed_md5_mock):
# the md5sum of the file being copied
md5sum = 'd41d8cd98f00b204e9800998ecf84eee'
# patch the compute_md5 function in vos to return the above value
computed_md5_mock.return_value = md5sum
#mock the props of the corresponding node
props = MagicMock()
props.get.return_value = md5sum
#add props to the mocked node
node = MagicMock(spec=Node)
node.props = props
# mock one by one the chain of connection.session.response.headers
conn = MagicMock(spec=Connection)
session = MagicMock()
response = MagicMock()
headers = MagicMock()
headers.get.return_value = md5sum
response.headers = headers
session.get.return_value = response
conn.session = session
test_client = Client()
# use the mocked connection instead of the real one
test_client.conn = conn
get_node_url_mock = Mock(
return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
test_client.get_node_url = get_node_url_mock
#patch Client.get_node to return our mocked node
get_node_mock = Mock(return_value=node)
test_client.get_node = get_node_mock
# time to test...
vospaceLocation = 'vos://test/foo'
osLocation = '/tmp/foo'
# copy from vospace
test_client.copy(vospaceLocation, osLocation)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None,
view='data')
computed_md5_mock.assert_called_once_with(osLocation)
get_node_mock.assert_called_once_with(vospaceLocation)
# copy to vospace
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
test_client.copy(osLocation, vospaceLocation)
get_node_url_mock.assert_called_once_with(vospaceLocation, 'PUT')
computed_md5_mock.assert_called_once_with(osLocation)
# error tests - md5sum mismatch
computed_md5_mock.return_value = '000bad000'
with self.assertRaises(OSError):
test_client.copy(vospaceLocation, osLocation)
with self.assertRaises(OSError):
test_client.copy(osLocation, vospaceLocation)
def test_copy(self, computed_md5_mock):
# the md5sum of the file being copied
md5sum = 'd41d8cd98f00b204e9800998ecf84eee'
# patch the compute_md5 function in vos to return the above value
computed_md5_mock.return_value = md5sum
# mock the props of the corresponding node
props = MagicMock()
props.get.return_value = md5sum
# add props to the mocked node
node = MagicMock(spec=Node)
node.props = props
# mock one by one the chain of connection.session.response.headers
conn = MagicMock(spec=Connection)
session = MagicMock()
response = MagicMock()
headers = MagicMock()
headers.get.return_value = md5sum
response.headers = headers
session.get.return_value = response
conn.session = session
test_client = Client()
# use the mocked connection instead of the real one
test_client.conn = conn
get_node_url_mock = Mock(
return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
test_client.get_node_url = get_node_url_mock
mock_update = Mock()
test_client.update = mock_update
# patch Client.get_node to return our mocked node
get_node_mock = Mock(return_value=node)
test_client.get_node = get_node_mock
# time to test...
vospaceLocation = 'vos://test/foo'
osLocation = '/tmp/foo'
if os.path.isfile(osLocation):
os.remove(osLocation)
# copy from vospace
test_client.copy(vospaceLocation, osLocation)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None,
view='data')
computed_md5_mock.assert_called_once_with(osLocation)
assert get_node_mock.called
# repeat - local file and vospace file are now the same -> only
# get_node is called to get the md5 of remote file
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
get_node_mock.reset_mock()
test_client.copy(vospaceLocation, osLocation)
assert not get_node_url_mock.called
computed_md5_mock.assert_called_once_with(osLocation)
get_node_mock.assert_called_once_with(vospaceLocation)
# change the content of local files to trigger a new copy
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
computed_md5_mock.side_effect = ['d002233', md5sum]
get_node_mock.reset_mock()
test_client.copy(vospaceLocation, osLocation)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None,
view='data')
computed_md5_mock.assert_called_with(osLocation)
get_node_mock.assert_called_once_with(vospaceLocation)
# copy to vospace when md5 sums are the same -> only update occurs
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
computed_md5_mock.side_effect = None
computed_md5_mock.return_value = md5sum
test_client.copy(osLocation, vospaceLocation)
mock_update.assert_called_once()
assert not get_node_url_mock.called
# make md5 different
get_node_url_mock.reset_mock()
get_node_url_mock.return_value =\
['http://cadc.ca/test', 'http://cadc.ca/test']
computed_md5_mock.reset_mock()
mock_update.reset_mock()
props.get.side_effect = ['d00223344', md5sum]
test_client.copy(osLocation, vospaceLocation)
assert not mock_update.called
get_node_url_mock.assert_called_once_with(vospaceLocation, 'PUT')
computed_md5_mock.assert_called_once_with(osLocation)
# copy 0 size file -> delete and create on client but no bytes
# transferred
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
computed_md5_mock.return_value = vos.ZERO_MD5
props.get.side_effect = [md5sum]
mock_delete = Mock()
mock_create = Mock()
test_client.delete = mock_delete
test_client.create = mock_create
test_client.copy(osLocation, vospaceLocation)
mock_create.assert_called_once_with(vospaceLocation)
mock_delete.assert_called_once_with(vospaceLocation)
assert not get_node_url_mock.called
# copy new 0 size file -> reate on client but no bytes transferred
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
mock_delete.reset_mock()
mock_create.reset_mock()
computed_md5_mock.return_value = vos.ZERO_MD5
props.get.side_effect = [None]
mock_delete = Mock()
mock_create = Mock()
test_client.delete = mock_delete
test_client.create = mock_create
test_client.copy(osLocation, vospaceLocation)
mock_create.assert_called_once_with(vospaceLocation)
assert not mock_delete.called
assert not get_node_url_mock.called
# error tests - md5sum mismatch
props.get.side_effect = [md5sum]
computed_md5_mock.return_value = '000bad000'
with self.assertRaises(OSError):
test_client.copy(vospaceLocation, osLocation)
with self.assertRaises(OSError):
test_client.copy(osLocation, vospaceLocation)
# requests just the headers
props.get.side_effect = [None]
get_node_url_mock = Mock(
return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
test_client.get_node_url = get_node_url_mock
computed_md5_mock.reset_mock()
computed_md5_mock.side_effect = ['d002233', md5sum]
get_node_mock.reset_mock()
test_client.copy(vospaceLocation, osLocation, head=True)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None,
view='header')
def test_copy(self, computed_md5_mock):
# the md5sum of the file being copied
md5sum = 'd41d8cd98f00b204e9800998ecf84eee'
# patch the compute_md5 function in vos to return the above value
computed_md5_mock.return_value = md5sum
# mock the props of the corresponding node
props = MagicMock()
props.get.return_value = md5sum
# add props to the mocked node
node = MagicMock(spec=Node)
node.props = props
# mock one by one the chain of connection.session.response.headers
conn = MagicMock(spec=Connection)
session = MagicMock()
response = MagicMock()
headers = MagicMock()
headers.get.return_value = md5sum
response.headers = headers
session.get.return_value = response
conn.session = session
test_client = Client()
# use the mocked connection instead of the real one
test_client.conn = conn
get_node_url_mock = Mock(
return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
test_client.get_node_url = get_node_url_mock
mock_update = Mock()
test_client.update = mock_update
# patch Client.get_node to return our mocked node
get_node_mock = Mock(return_value=node)
test_client.get_node = get_node_mock
# time to test...
vospaceLocation = 'vos://test/foo'
osLocation = '/tmp/foo'
if os.path.isfile(osLocation):
os.remove(osLocation)
# copy from vospace
test_client.copy(vospaceLocation, osLocation)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None, view='data')
computed_md5_mock.assert_called_once_with(osLocation)
assert get_node_mock.called
# repeat - local file and vospace file are now the same -> only
# get_node is called to get the md5 of remote file
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
get_node_mock.reset_mock()
test_client.copy(vospaceLocation, osLocation)
assert not get_node_url_mock.called
computed_md5_mock.assert_called_once_with(osLocation)
get_node_mock.assert_called_once_with(vospaceLocation)
# change the content of local files to trigger a new copy
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
computed_md5_mock.side_effect = ['d002233', md5sum]
get_node_mock.reset_mock()
test_client.copy(vospaceLocation, osLocation)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None, view='data')
computed_md5_mock.assert_called_with(osLocation)
get_node_mock.assert_called_once_with(vospaceLocation)
# copy to vospace when md5 sums are the same -> only update occurs
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
computed_md5_mock.side_effect = None
computed_md5_mock.return_value = md5sum
test_client.copy(osLocation, vospaceLocation)
mock_update.assert_called_once()
assert not get_node_url_mock.called
# make md5 different
get_node_url_mock.reset_mock()
get_node_url_mock.return_value =\
['http://cadc.ca/test', 'http://cadc.ca/test']
computed_md5_mock.reset_mock()
mock_update.reset_mock()
props.get.side_effect = ['d00223344', md5sum]
test_client.copy(osLocation, vospaceLocation)
assert not mock_update.called
get_node_url_mock.assert_called_once_with(vospaceLocation, 'PUT')
computed_md5_mock.assert_called_once_with(osLocation)
# copy 0 size file -> delete and create on client but no bytes
# transferred
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
computed_md5_mock.return_value = vos.ZERO_MD5
props.get.side_effect = [md5sum]
mock_delete = Mock()
mock_create = Mock()
test_client.delete = mock_delete
test_client.create = mock_create
test_client.copy(osLocation, vospaceLocation)
mock_create.assert_called_once_with(vospaceLocation)
mock_delete.assert_called_once_with(vospaceLocation)
assert not get_node_url_mock.called
# copy new 0 size file -> reate on client but no bytes transferred
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
mock_delete.reset_mock()
mock_create.reset_mock()
computed_md5_mock.return_value = vos.ZERO_MD5
props.get.side_effect = [None]
mock_delete = Mock()
mock_create = Mock()
test_client.delete = mock_delete
test_client.create = mock_create
test_client.copy(osLocation, vospaceLocation)
mock_create.assert_called_once_with(vospaceLocation)
assert not mock_delete.called
assert not get_node_url_mock.called
# error tests - md5sum mismatch
props.get.side_effect = [md5sum]
computed_md5_mock.return_value = '000bad000'
with self.assertRaises(OSError):
test_client.copy(vospaceLocation, osLocation)
with self.assertRaises(OSError):
test_client.copy(osLocation, vospaceLocation)
# requests just the headers
props.get.side_effect = [None]
get_node_url_mock = Mock(
return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
test_client.get_node_url = get_node_url_mock
computed_md5_mock.reset_mock()
computed_md5_mock.side_effect = ['d002233', md5sum]
get_node_mock.reset_mock()
test_client.copy(vospaceLocation, osLocation, head=True)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None, view='header')
def test_copy(self, computed_md5_mock):
file_content = 'File content'.encode('utf-8')
# the md5sum of the file being copied
transfer_md5 = hashlib.md5()
transfer_md5.update(file_content)
md5sum = transfer_md5.hexdigest()
# patch the compute_md5 function in vos to return the above value
computed_md5_mock.return_value = md5sum
# mock the props of the corresponding node
props = MagicMock()
props.get.return_value = md5sum
# add props to the mocked node
node = MagicMock(spec=Node)
node.props = {'MD5': md5sum, 'length': 12}
# mock one by one the chain of connection.session.response.headers
session = MagicMock()
response = MagicMock()
headers = MagicMock()
headers.get.return_value = md5sum
response.headers = headers
session.get.return_value = response
response.iter_content.return_value = BytesIO(file_content)
test_client = Client()
test_client.get_session = Mock(return_value=session)
# use the mocked connection instead of the real one
get_node_url_mock = Mock(
return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
test_client.get_node_url = get_node_url_mock
mock_update = Mock()
test_client.update = mock_update
# patch Client.get_node to return our mocked node
get_node_mock = Mock(return_value=node)
test_client.get_node = get_node_mock
# time to test...
vospaceLocation = 'vos://test/foo'
osLocation = '/tmp/foo'
if os.path.isfile(osLocation):
os.remove(osLocation)
# copy from vospace
test_client.copy(vospaceLocation, osLocation)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None, view='data')
assert not computed_md5_mock.called,\
'MD5 should be computed on the fly'
assert get_node_mock.called
# repeat - local file and vospace file are now the same -> only
# get_node is called to get the md5 of remote file
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
get_node_mock.reset_mock()
props.reset_mock()
props.get.return_value = md5sum
test_client.copy(vospaceLocation, osLocation)
assert not get_node_url_mock.called
computed_md5_mock.assert_called_once_with(osLocation)
get_node_mock.assert_called_once_with(vospaceLocation, force=True)
# change the content of local files to trigger a new copy
get_node_url_mock.reset_mock()
get_node_mock.reset_mock()
computed_md5_mock.reset_mock()
computed_md5_mock.return_value = 'd002233'
response.iter_content.return_value = BytesIO(file_content)
test_client.copy(vospaceLocation, osLocation)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None, view='data')
computed_md5_mock.assert_called_with(osLocation)
get_node_mock.assert_called_once_with(vospaceLocation, force=True)
# change the content of local files to trigger a new copy
get_node_url_mock.reset_mock()
get_node_url_mock.return_value = \
['https://mysite.com/node/node123/cutout']
computed_md5_mock.reset_mock()
computed_md5_mock.return_value = 'd002233'
# computed_md5_mock.side_effect = ['d002233', md5sum]
get_node_mock.reset_mock()
response.iter_content.return_value = BytesIO(file_content)
session.get.return_value = response
test_client.get_session = Mock(return_value=session)
test_client.copy('{}{}'.format(vospaceLocation,
'[1][10:60]'), osLocation)
get_node_url_mock.assert_called_once_with(
vospaceLocation, method='GET', cutout='[1][10:60]', view='cutout')
# copy to vospace when md5 sums are the same -> only update occurs
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
computed_md5_mock.side_effect = None
computed_md5_mock.return_value = md5sum
test_client.copy(osLocation, vospaceLocation)
mock_update.assert_called_once()
assert not get_node_url_mock.called
# make md5 different on destination
get_node_url_mock.reset_mock()
get_node_url_mock.return_value =\
['http://cadc.ca/test', 'http://cadc.ca/test']
computed_md5_mock.reset_mock()
mock_update.reset_mock()
computed_md5_mock.return_value = md5sum
to_update_node = MagicMock()
to_update_node.props = {'MD5': 'abcde', 'length': 12}
test_client.get_node = Mock(side_effect=[to_update_node, node])
test_client.copy(osLocation, vospaceLocation)
assert not mock_update.called
get_node_url_mock.assert_called_once_with(vospaceLocation, 'PUT',
content_length=12,
md5_checksum='abcde')
computed_md5_mock.assert_called_once_with(osLocation)
# copy 0 size file -> delete and create node but no bytes
# transferred
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
test_client.get_node = Mock(return_value=node)
node.props['length'] = 0
mock_delete = Mock()
mock_create = Mock()
test_client.delete = mock_delete
test_client.create = mock_create
with patch('vos.vos.os.stat', Mock()) as stat_mock:
stat_mock.return_value = Mock(st_size=0)
test_client.copy(osLocation, vospaceLocation)
mock_create.assert_called_once_with(vospaceLocation)
mock_delete.assert_called_once_with(vospaceLocation)
assert not get_node_url_mock.called
# copy new 0 size file -> create node but no bytes transferred
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
mock_delete.reset_mock()
mock_create.reset_mock()
test_client.get_node = Mock(side_effect=[Exception(), node])
mock_delete = Mock()
mock_create = Mock()
test_client.delete = mock_delete
test_client.create = mock_create
with patch('vos.vos.os.stat', Mock()) as stat_mock:
stat_mock.return_value = Mock(st_size=0)
test_client.copy(osLocation, vospaceLocation)
mock_create.assert_called_once_with(vospaceLocation)
assert not mock_delete.called
assert not get_node_url_mock.called
# error tests - md5sum mismatch
node.props['length'] = 12
computed_md5_mock.return_value = '000bad000'
test_client.get_node = Mock(return_value=node)
with self.assertRaises(OSError):
test_client.copy(vospaceLocation, osLocation)
# existing file
mock_delete.reset_mock()
with self.assertRaises(OSError):
with patch('vos.vos.os.stat', Mock()) as stat_mock:
stat_mock.return_value = Mock(st_size=12)
test_client.copy(osLocation, vospaceLocation)
assert not mock_delete.called # server takes care of cleanup
# new file
mock_delete.reset_mock()
with self.assertRaises(OSError):
with patch('vos.vos.os.stat', Mock()) as stat_mock:
stat_mock.return_value = Mock(st_size=12)
node.props['MD5'] = None
test_client.copy(osLocation, vospaceLocation)
assert mock_delete.called # cleanup required
# requests just the headers when md5 not provided in the header
props.get.side_effect = [None]
get_node_url_mock = Mock(
return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
test_client.get_node_url = get_node_url_mock
get_node_mock.reset_mock()
headers.get.return_value = None
test_client.copy(vospaceLocation, osLocation, head=True)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None, view='header')
# repeat headers request when md5 provided in the header
props.get.side_effect = md5sum
get_node_url_mock = Mock(
return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
test_client.get_node_url = get_node_url_mock
get_node_mock.reset_mock()
response.iter_content.return_value = BytesIO(file_content)
headers.get.return_value = None
test_client.copy(vospaceLocation, osLocation, head=True)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None, view='header')
def main(orbit=None, **kwargs):
"""
This is the driver program. Gets the images from VOSpace or the local filesystem.
expected kwargs:
pointing, index, chip, rate, angle, ra, dec, p_name, discovery, nstk, zpt, dbimages
:param kwargs: these are the 'args' sent on the commandline or in an input file.
:param orbit: the orbit of the object that should be in the frames reference by kwargs provided.
:type orbit: BKOrbit
:return: list of observations
:rtype list(ObsRecod)
"""
pointing = kwargs['pointing']
chip = kwargs['chip']
index = kwargs['index']
rate = kwargs['rate']
angle = kwargs['angle']
ra = kwargs['ra']
dec = kwargs['dec']
p_name = kwargs['p_name']
discovery = kwargs['discovery']
nstk = kwargs['nstk']
rejected = kwargs.get('rejected', False)
zpt = kwargs.get('zpt', 26.9)
dbimages = kwargs.get('dbimages', 'vos:NewHorizons/dbimages/')
client = Client()
int_rate = int(rate * 10)
int_angle = int((angle % 360) * 10)
images = []
# Load the 3 images associated with this point/chip/rate/angle set.
epoch = None
for idx in range(nstk):
expnum = f'{int(pointing)}{int_rate:02d}{int_angle:04d}{idx}'
image = f'{expnum}p{chip:02d}.fits'
url = f'{dbimages}/{pointing:05d}/{chip:03d}/{index:04d}/{image}'
logging.info(f"Looking for image at {url}")
try:
if os.access(url, os.R_OK):
image = url
elif not os.access(image, os.R_OK):
# get from VOSpace is not already on disk
client.copy(url, image)
except Exception as ex:
logging.error(str(ex))
# Return empty set on VOSpace copy error.
return {}
images.append(image)
epoch = Time(fits.open(images[len(images) // 2])[0].header['DATE-AVG'],
scale='tai').utc
regions = f'{dbimages}/{pointing:05d}.reg'
try:
if not os.access(regions, os.R_OK):
regions = client.copy(regions, '.', disposition=True)
except Exception as ex:
logging.debug(f"{ex}")
regions = None
wcs_dict = {}
if orbit is not None:
epoch = orbit.epoch.mjd
if epoch is None:
epoch = Time("2020-05-22T00:00:00")
epoch = Time(kwargs.get('epoch', epoch), format='mjd')
load_images(images,
ra,
dec,
wcs_dict,
orbit,
dra=rate * math.cos(math.radians(angle)),
ddec=rate * math.sin(math.radians(angle)),
regions=regions,
rejected=rejected,
basedate=epoch)
obs = measure_image(p_name, images, wcs_dict, discovery=discovery, zpt=zpt)
return obs