def test_copy(self, computed_md5_mock): # the md5sum of the file being copied md5sum = 'd41d8cd98f00b204e9800998ecf84eee' # patch the compute_md5 function in vos to return the above value computed_md5_mock.return_value = md5sum #mock the props of the corresponding node props = MagicMock() props.get.return_value = md5sum #add props to the mocked node node = MagicMock(spec=Node) node.props = props # mock one by one the chain of connection.session.response.headers conn = MagicMock(spec=Connection) session = MagicMock() response = MagicMock() headers = MagicMock() headers.get.return_value = md5sum response.headers = headers session.get.return_value = response conn.session = session test_client = Client() # use the mocked connection instead of the real one test_client.conn = conn get_node_url_mock = Mock( return_value=['http://cadc.ca/test', 'http://cadc.ca/test']) test_client.get_node_url = get_node_url_mock #patch Client.get_node to return our mocked node get_node_mock = Mock(return_value=node) test_client.get_node = get_node_mock # time to test... vospaceLocation = 'vos://test/foo' osLocation = '/tmp/foo' # copy from vospace test_client.copy(vospaceLocation, osLocation) get_node_url_mock.assert_called_once_with(vospaceLocation, method='GET', cutout=None, view='data') computed_md5_mock.assert_called_once_with(osLocation) get_node_mock.assert_called_once_with(vospaceLocation) # copy to vospace get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() test_client.copy(osLocation, vospaceLocation) get_node_url_mock.assert_called_once_with(vospaceLocation, 'PUT') computed_md5_mock.assert_called_once_with(osLocation) # error tests - md5sum mismatch computed_md5_mock.return_value = '000bad000' with self.assertRaises(OSError): test_client.copy(vospaceLocation, osLocation) with self.assertRaises(OSError): test_client.copy(osLocation, vospaceLocation)
def test_copy(self, computed_md5_mock): # the md5sum of the file being copied md5sum = 'd41d8cd98f00b204e9800998ecf84eee' # patch the compute_md5 function in vos to return the above value computed_md5_mock.return_value = md5sum # mock the props of the corresponding node props = MagicMock() props.get.return_value = md5sum # add props to the mocked node node = MagicMock(spec=Node) node.props = props # mock one by one the chain of connection.session.response.headers conn = MagicMock(spec=Connection) session = MagicMock() response = MagicMock() headers = MagicMock() headers.get.return_value = md5sum response.headers = headers session.get.return_value = response conn.session = session test_client = Client() # use the mocked connection instead of the real one test_client.conn = conn get_node_url_mock = Mock( return_value=['http://cadc.ca/test', 'http://cadc.ca/test']) test_client.get_node_url = get_node_url_mock mock_update = Mock() test_client.update = mock_update # patch Client.get_node to return our mocked node get_node_mock = Mock(return_value=node) test_client.get_node = get_node_mock # time to test... vospaceLocation = 'vos://test/foo' osLocation = '/tmp/foo' if os.path.isfile(osLocation): os.remove(osLocation) # copy from vospace test_client.copy(vospaceLocation, osLocation) get_node_url_mock.assert_called_once_with(vospaceLocation, method='GET', cutout=None, view='data') computed_md5_mock.assert_called_once_with(osLocation) assert get_node_mock.called # repeat - local file and vospace file are now the same -> only # get_node is called to get the md5 of remote file get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() get_node_mock.reset_mock() test_client.copy(vospaceLocation, osLocation) assert not get_node_url_mock.called computed_md5_mock.assert_called_once_with(osLocation) get_node_mock.assert_called_once_with(vospaceLocation) # change the content of local files to trigger a new copy get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() computed_md5_mock.side_effect = ['d002233', md5sum] get_node_mock.reset_mock() test_client.copy(vospaceLocation, osLocation) get_node_url_mock.assert_called_once_with(vospaceLocation, method='GET', cutout=None, view='data') computed_md5_mock.assert_called_with(osLocation) get_node_mock.assert_called_once_with(vospaceLocation) # copy to vospace when md5 sums are the same -> only update occurs get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() computed_md5_mock.side_effect = None computed_md5_mock.return_value = md5sum test_client.copy(osLocation, vospaceLocation) mock_update.assert_called_once() assert not get_node_url_mock.called # make md5 different get_node_url_mock.reset_mock() get_node_url_mock.return_value =\ ['http://cadc.ca/test', 'http://cadc.ca/test'] computed_md5_mock.reset_mock() mock_update.reset_mock() props.get.side_effect = ['d00223344', md5sum] test_client.copy(osLocation, vospaceLocation) assert not mock_update.called get_node_url_mock.assert_called_once_with(vospaceLocation, 'PUT') computed_md5_mock.assert_called_once_with(osLocation) # copy 0 size file -> delete and create on client but no bytes # transferred get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() computed_md5_mock.return_value = vos.ZERO_MD5 props.get.side_effect = [md5sum] mock_delete = Mock() mock_create = Mock() test_client.delete = mock_delete test_client.create = mock_create test_client.copy(osLocation, vospaceLocation) mock_create.assert_called_once_with(vospaceLocation) mock_delete.assert_called_once_with(vospaceLocation) assert not get_node_url_mock.called # copy new 0 size file -> reate on client but no bytes transferred get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() mock_delete.reset_mock() mock_create.reset_mock() computed_md5_mock.return_value = vos.ZERO_MD5 props.get.side_effect = [None] mock_delete = Mock() mock_create = Mock() test_client.delete = mock_delete test_client.create = mock_create test_client.copy(osLocation, vospaceLocation) mock_create.assert_called_once_with(vospaceLocation) assert not mock_delete.called assert not get_node_url_mock.called # error tests - md5sum mismatch props.get.side_effect = [md5sum] computed_md5_mock.return_value = '000bad000' with self.assertRaises(OSError): test_client.copy(vospaceLocation, osLocation) with self.assertRaises(OSError): test_client.copy(osLocation, vospaceLocation) # requests just the headers props.get.side_effect = [None] get_node_url_mock = Mock( return_value=['http://cadc.ca/test', 'http://cadc.ca/test']) test_client.get_node_url = get_node_url_mock computed_md5_mock.reset_mock() computed_md5_mock.side_effect = ['d002233', md5sum] get_node_mock.reset_mock() test_client.copy(vospaceLocation, osLocation, head=True) get_node_url_mock.assert_called_once_with(vospaceLocation, method='GET', cutout=None, view='header')
def test_copy(self, computed_md5_mock): # the md5sum of the file being copied md5sum = 'd41d8cd98f00b204e9800998ecf84eee' # patch the compute_md5 function in vos to return the above value computed_md5_mock.return_value = md5sum # mock the props of the corresponding node props = MagicMock() props.get.return_value = md5sum # add props to the mocked node node = MagicMock(spec=Node) node.props = props # mock one by one the chain of connection.session.response.headers conn = MagicMock(spec=Connection) session = MagicMock() response = MagicMock() headers = MagicMock() headers.get.return_value = md5sum response.headers = headers session.get.return_value = response conn.session = session test_client = Client() # use the mocked connection instead of the real one test_client.conn = conn get_node_url_mock = Mock( return_value=['http://cadc.ca/test', 'http://cadc.ca/test']) test_client.get_node_url = get_node_url_mock mock_update = Mock() test_client.update = mock_update # patch Client.get_node to return our mocked node get_node_mock = Mock(return_value=node) test_client.get_node = get_node_mock # time to test... vospaceLocation = 'vos://test/foo' osLocation = '/tmp/foo' if os.path.isfile(osLocation): os.remove(osLocation) # copy from vospace test_client.copy(vospaceLocation, osLocation) get_node_url_mock.assert_called_once_with(vospaceLocation, method='GET', cutout=None, view='data') computed_md5_mock.assert_called_once_with(osLocation) assert get_node_mock.called # repeat - local file and vospace file are now the same -> only # get_node is called to get the md5 of remote file get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() get_node_mock.reset_mock() test_client.copy(vospaceLocation, osLocation) assert not get_node_url_mock.called computed_md5_mock.assert_called_once_with(osLocation) get_node_mock.assert_called_once_with(vospaceLocation) # change the content of local files to trigger a new copy get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() computed_md5_mock.side_effect = ['d002233', md5sum] get_node_mock.reset_mock() test_client.copy(vospaceLocation, osLocation) get_node_url_mock.assert_called_once_with(vospaceLocation, method='GET', cutout=None, view='data') computed_md5_mock.assert_called_with(osLocation) get_node_mock.assert_called_once_with(vospaceLocation) # copy to vospace when md5 sums are the same -> only update occurs get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() computed_md5_mock.side_effect = None computed_md5_mock.return_value = md5sum test_client.copy(osLocation, vospaceLocation) mock_update.assert_called_once() assert not get_node_url_mock.called # make md5 different get_node_url_mock.reset_mock() get_node_url_mock.return_value =\ ['http://cadc.ca/test', 'http://cadc.ca/test'] computed_md5_mock.reset_mock() mock_update.reset_mock() props.get.side_effect = ['d00223344', md5sum] test_client.copy(osLocation, vospaceLocation) assert not mock_update.called get_node_url_mock.assert_called_once_with(vospaceLocation, 'PUT') computed_md5_mock.assert_called_once_with(osLocation) # copy 0 size file -> delete and create on client but no bytes # transferred get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() computed_md5_mock.return_value = vos.ZERO_MD5 props.get.side_effect = [md5sum] mock_delete = Mock() mock_create = Mock() test_client.delete = mock_delete test_client.create = mock_create test_client.copy(osLocation, vospaceLocation) mock_create.assert_called_once_with(vospaceLocation) mock_delete.assert_called_once_with(vospaceLocation) assert not get_node_url_mock.called # copy new 0 size file -> reate on client but no bytes transferred get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() mock_delete.reset_mock() mock_create.reset_mock() computed_md5_mock.return_value = vos.ZERO_MD5 props.get.side_effect = [None] mock_delete = Mock() mock_create = Mock() test_client.delete = mock_delete test_client.create = mock_create test_client.copy(osLocation, vospaceLocation) mock_create.assert_called_once_with(vospaceLocation) assert not mock_delete.called assert not get_node_url_mock.called # error tests - md5sum mismatch props.get.side_effect = [md5sum] computed_md5_mock.return_value = '000bad000' with self.assertRaises(OSError): test_client.copy(vospaceLocation, osLocation) with self.assertRaises(OSError): test_client.copy(osLocation, vospaceLocation) # requests just the headers props.get.side_effect = [None] get_node_url_mock = Mock( return_value=['http://cadc.ca/test', 'http://cadc.ca/test']) test_client.get_node_url = get_node_url_mock computed_md5_mock.reset_mock() computed_md5_mock.side_effect = ['d002233', md5sum] get_node_mock.reset_mock() test_client.copy(vospaceLocation, osLocation, head=True) get_node_url_mock.assert_called_once_with(vospaceLocation, method='GET', cutout=None, view='header')
def test_copy(self, computed_md5_mock): file_content = 'File content'.encode('utf-8') # the md5sum of the file being copied transfer_md5 = hashlib.md5() transfer_md5.update(file_content) md5sum = transfer_md5.hexdigest() # patch the compute_md5 function in vos to return the above value computed_md5_mock.return_value = md5sum # mock the props of the corresponding node props = MagicMock() props.get.return_value = md5sum # add props to the mocked node node = MagicMock(spec=Node) node.props = {'MD5': md5sum, 'length': 12} # mock one by one the chain of connection.session.response.headers session = MagicMock() response = MagicMock() headers = MagicMock() headers.get.return_value = md5sum response.headers = headers session.get.return_value = response response.iter_content.return_value = BytesIO(file_content) test_client = Client() test_client.get_session = Mock(return_value=session) # use the mocked connection instead of the real one get_node_url_mock = Mock( return_value=['http://cadc.ca/test', 'http://cadc.ca/test']) test_client.get_node_url = get_node_url_mock mock_update = Mock() test_client.update = mock_update # patch Client.get_node to return our mocked node get_node_mock = Mock(return_value=node) test_client.get_node = get_node_mock # time to test... vospaceLocation = 'vos://test/foo' osLocation = '/tmp/foo' if os.path.isfile(osLocation): os.remove(osLocation) # copy from vospace test_client.copy(vospaceLocation, osLocation) get_node_url_mock.assert_called_once_with(vospaceLocation, method='GET', cutout=None, view='data') assert not computed_md5_mock.called,\ 'MD5 should be computed on the fly' assert get_node_mock.called # repeat - local file and vospace file are now the same -> only # get_node is called to get the md5 of remote file get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() get_node_mock.reset_mock() props.reset_mock() props.get.return_value = md5sum test_client.copy(vospaceLocation, osLocation) assert not get_node_url_mock.called computed_md5_mock.assert_called_once_with(osLocation) get_node_mock.assert_called_once_with(vospaceLocation, force=True) # change the content of local files to trigger a new copy get_node_url_mock.reset_mock() get_node_mock.reset_mock() computed_md5_mock.reset_mock() computed_md5_mock.return_value = 'd002233' response.iter_content.return_value = BytesIO(file_content) test_client.copy(vospaceLocation, osLocation) get_node_url_mock.assert_called_once_with(vospaceLocation, method='GET', cutout=None, view='data') computed_md5_mock.assert_called_with(osLocation) get_node_mock.assert_called_once_with(vospaceLocation, force=True) # change the content of local files to trigger a new copy get_node_url_mock.reset_mock() get_node_url_mock.return_value = \ ['https://mysite.com/node/node123/cutout'] computed_md5_mock.reset_mock() computed_md5_mock.return_value = 'd002233' # computed_md5_mock.side_effect = ['d002233', md5sum] get_node_mock.reset_mock() response.iter_content.return_value = BytesIO(file_content) session.get.return_value = response test_client.get_session = Mock(return_value=session) test_client.copy('{}{}'.format(vospaceLocation, '[1][10:60]'), osLocation) get_node_url_mock.assert_called_once_with( vospaceLocation, method='GET', cutout='[1][10:60]', view='cutout') # copy to vospace when md5 sums are the same -> only update occurs get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() computed_md5_mock.side_effect = None computed_md5_mock.return_value = md5sum test_client.copy(osLocation, vospaceLocation) mock_update.assert_called_once() assert not get_node_url_mock.called # make md5 different on destination get_node_url_mock.reset_mock() get_node_url_mock.return_value =\ ['http://cadc.ca/test', 'http://cadc.ca/test'] computed_md5_mock.reset_mock() mock_update.reset_mock() computed_md5_mock.return_value = md5sum to_update_node = MagicMock() to_update_node.props = {'MD5': 'abcde', 'length': 12} test_client.get_node = Mock(side_effect=[to_update_node, node]) test_client.copy(osLocation, vospaceLocation) assert not mock_update.called get_node_url_mock.assert_called_once_with(vospaceLocation, 'PUT', content_length=12, md5_checksum='abcde') computed_md5_mock.assert_called_once_with(osLocation) # copy 0 size file -> delete and create node but no bytes # transferred get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() test_client.get_node = Mock(return_value=node) node.props['length'] = 0 mock_delete = Mock() mock_create = Mock() test_client.delete = mock_delete test_client.create = mock_create with patch('vos.vos.os.stat', Mock()) as stat_mock: stat_mock.return_value = Mock(st_size=0) test_client.copy(osLocation, vospaceLocation) mock_create.assert_called_once_with(vospaceLocation) mock_delete.assert_called_once_with(vospaceLocation) assert not get_node_url_mock.called # copy new 0 size file -> create node but no bytes transferred get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() mock_delete.reset_mock() mock_create.reset_mock() test_client.get_node = Mock(side_effect=[Exception(), node]) mock_delete = Mock() mock_create = Mock() test_client.delete = mock_delete test_client.create = mock_create with patch('vos.vos.os.stat', Mock()) as stat_mock: stat_mock.return_value = Mock(st_size=0) test_client.copy(osLocation, vospaceLocation) mock_create.assert_called_once_with(vospaceLocation) assert not mock_delete.called assert not get_node_url_mock.called # error tests - md5sum mismatch node.props['length'] = 12 computed_md5_mock.return_value = '000bad000' test_client.get_node = Mock(return_value=node) with self.assertRaises(OSError): test_client.copy(vospaceLocation, osLocation) # existing file mock_delete.reset_mock() with self.assertRaises(OSError): with patch('vos.vos.os.stat', Mock()) as stat_mock: stat_mock.return_value = Mock(st_size=12) test_client.copy(osLocation, vospaceLocation) assert not mock_delete.called # server takes care of cleanup # new file mock_delete.reset_mock() with self.assertRaises(OSError): with patch('vos.vos.os.stat', Mock()) as stat_mock: stat_mock.return_value = Mock(st_size=12) node.props['MD5'] = None test_client.copy(osLocation, vospaceLocation) assert mock_delete.called # cleanup required # requests just the headers when md5 not provided in the header props.get.side_effect = [None] get_node_url_mock = Mock( return_value=['http://cadc.ca/test', 'http://cadc.ca/test']) test_client.get_node_url = get_node_url_mock get_node_mock.reset_mock() headers.get.return_value = None test_client.copy(vospaceLocation, osLocation, head=True) get_node_url_mock.assert_called_once_with(vospaceLocation, method='GET', cutout=None, view='header') # repeat headers request when md5 provided in the header props.get.side_effect = md5sum get_node_url_mock = Mock( return_value=['http://cadc.ca/test', 'http://cadc.ca/test']) test_client.get_node_url = get_node_url_mock get_node_mock.reset_mock() response.iter_content.return_value = BytesIO(file_content) headers.get.return_value = None test_client.copy(vospaceLocation, osLocation, head=True) get_node_url_mock.assert_called_once_with(vospaceLocation, method='GET', cutout=None, view='header')
def main(orbit=None, **kwargs): """ This is the driver program. Gets the images from VOSpace or the local filesystem. expected kwargs: pointing, index, chip, rate, angle, ra, dec, p_name, discovery, nstk, zpt, dbimages :param kwargs: these are the 'args' sent on the commandline or in an input file. :param orbit: the orbit of the object that should be in the frames reference by kwargs provided. :type orbit: BKOrbit :return: list of observations :rtype list(ObsRecod) """ pointing = kwargs['pointing'] chip = kwargs['chip'] index = kwargs['index'] rate = kwargs['rate'] angle = kwargs['angle'] ra = kwargs['ra'] dec = kwargs['dec'] p_name = kwargs['p_name'] discovery = kwargs['discovery'] nstk = kwargs['nstk'] rejected = kwargs.get('rejected', False) zpt = kwargs.get('zpt', 26.9) dbimages = kwargs.get('dbimages', 'vos:NewHorizons/dbimages/') client = Client() int_rate = int(rate * 10) int_angle = int((angle % 360) * 10) images = [] # Load the 3 images associated with this point/chip/rate/angle set. epoch = None for idx in range(nstk): expnum = f'{int(pointing)}{int_rate:02d}{int_angle:04d}{idx}' image = f'{expnum}p{chip:02d}.fits' url = f'{dbimages}/{pointing:05d}/{chip:03d}/{index:04d}/{image}' logging.info(f"Looking for image at {url}") try: if os.access(url, os.R_OK): image = url elif not os.access(image, os.R_OK): # get from VOSpace is not already on disk client.copy(url, image) except Exception as ex: logging.error(str(ex)) # Return empty set on VOSpace copy error. return {} images.append(image) epoch = Time(fits.open(images[len(images) // 2])[0].header['DATE-AVG'], scale='tai').utc regions = f'{dbimages}/{pointing:05d}.reg' try: if not os.access(regions, os.R_OK): regions = client.copy(regions, '.', disposition=True) except Exception as ex: logging.debug(f"{ex}") regions = None wcs_dict = {} if orbit is not None: epoch = orbit.epoch.mjd if epoch is None: epoch = Time("2020-05-22T00:00:00") epoch = Time(kwargs.get('epoch', epoch), format='mjd') load_images(images, ra, dec, wcs_dict, orbit, dra=rate * math.cos(math.radians(angle)), ddec=rate * math.sin(math.radians(angle)), regions=regions, rejected=rejected, basedate=epoch) obs = measure_image(p_name, images, wcs_dict, discovery=discovery, zpt=zpt) return obs