def test_init_client(self):
# No parameters uses cert in ~/.ssl giving authenticated / https
# create a fake pem file
certfile = '/tmp/some-cert-file.pem'
open(certfile, 'w+')
Client.VOSPACE_CERTFILE = "some-cert-file.pem"
with patch('os.access'):
client = Client(vospace_certfile=certfile)
self.assertTrue(client.conn.subject.certificate)
self.assertFalse(client.conn.vo_token)
# Supplying an empty string for certfile implies anonymous / http
client = Client(vospace_certfile='')
self.assertTrue(client.conn.subject.anon)
self.assertFalse(client.conn.vo_token)
# Specifying a token implies authenticated / http
client = Client(vospace_token='a_token_string')
self.assertTrue(client.conn.subject.anon)
self.assertTrue(client.conn.vo_token)
# Specifying both a certfile and token implies token (auth) / http
with patch('os.access'):
client = Client(vospace_certfile=certfile,
vospace_token='a_token_string')
self.assertTrue(client.conn.subject.anon)
self.assertTrue(client.conn.vo_token)
def test_nodetype(self):
mock_node = MagicMock(id=333)
mock_node.type = 'vos:ContainerNode'
client = Client()
client.get_node = Mock(return_value=mock_node)
self.assertEquals('vos:ContainerNode',
client._node_type('vos:/somenode'))
self.assertTrue(client.isdir('vos:/somenode'))
mock_node.type = 'vos:DataNode'
self.assertEquals('vos:DataNode', client._node_type('vos:/somenode'))
self.assertTrue(client.isfile('vos:/somenode'))
# through a link
mock_node.type = 'vos:ContainerNode'
mock_link_node = Mock(type='vos:LinkNode')
mock_link_node.target = 'vos:/somefile'
client.get_node = Mock(
side_effect=[mock_link_node, mock_node, mock_link_node, mock_node])
self.assertEquals('vos:ContainerNode',
client._node_type('vos:/somenode'))
self.assertTrue(client.isdir('vos:/somenode'))
# through an external link - not sure why the type is DataNode in this case???
mock_link_node.target = '/somefile'
client.get_node = Mock(side_effect=[mock_link_node, mock_link_node])
self.assertEquals('vos:DataNode', client._node_type('vos:/somenode'))
self.assertTrue(client.isfile('vos:/somenode'))
def test_update(self):
node = Node(ElementTree.fromstring(NODE_XML))
resp = Mock()
resp.headers.get = Mock(return_value="https://www.canfar.phys.uvic.ca/vospace")
client = Client()
client.get_node_url = Mock(return_value='https://www.canfar.phys.uvic.ca/vospace')
client.conn = Mock()
client.conn.session.post = Mock(return_value=resp)
client.get_transfer_error = Mock()
client.protocol = 'https'
data = str(node)
property_url = 'https://www.canfar.phys.uvic.ca/vospace/nodeprops'
result = client.update(node, False)
self.assertEqual(result, 0)
client.conn.session.post.assert_called_with('https://www.canfar.phys.uvic.ca/vospace',
data=data, allow_redirects=False)
call1 = call(property_url, allow_redirects=False, data=data,
headers={'Content-type': 'text/xml'})
call2 = call('https://www.canfar.phys.uvic.ca/vospace/phase', allow_redirects=False, data="PHASE=RUN",
headers={'Content-type': "text/text"})
calls = [call1, call2]
client.conn = Mock()
client.conn.session.post = Mock(return_value=resp)
with patch('vos.vos.EndPoints.properties', property_url):
result = client.update(node, True)
self.assertEqual(result, 0)
client.conn.session.post.assert_has_calls(calls)
def test_copy(self, computed_md5_mock):
# the md5sum of the file being copied
md5sum = 'd41d8cd98f00b204e9800998ecf84eee'
# patch the compute_md5 function in vos to return the above value
computed_md5_mock.return_value = md5sum
#mock the props of the corresponding node
props = MagicMock()
props.get.return_value = md5sum
#add props to the mocked node
node = MagicMock(spec=Node)
node.props = props
# mock one by one the chain of connection.session.response.headers
conn = MagicMock(spec=Connection)
session = MagicMock()
response = MagicMock()
headers = MagicMock()
headers.get.return_value = md5sum
response.headers = headers
session.get.return_value = response
conn.session = session
test_client = Client()
# use the mocked connection instead of the real one
test_client.conn = conn
get_node_url_mock = Mock(
return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
test_client.get_node_url = get_node_url_mock
#patch Client.get_node to return our mocked node
get_node_mock = Mock(return_value=node)
test_client.get_node = get_node_mock
# time to test...
vospaceLocation = 'vos://test/foo'
osLocation = '/tmp/foo'
# copy from vospace
test_client.copy(vospaceLocation, osLocation)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None,
view='data')
computed_md5_mock.assert_called_once_with(osLocation)
get_node_mock.assert_called_once_with(vospaceLocation)
# copy to vospace
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
test_client.copy(osLocation, vospaceLocation)
get_node_url_mock.assert_called_once_with(vospaceLocation, 'PUT')
computed_md5_mock.assert_called_once_with(osLocation)
# error tests - md5sum mismatch
computed_md5_mock.return_value = '000bad000'
with self.assertRaises(OSError):
test_client.copy(vospaceLocation, osLocation)
with self.assertRaises(OSError):
test_client.copy(osLocation, vospaceLocation)
def test_open(self):
# Invalid mode raises OSError
with self.assertRaises(OSError):
client = Client()
client.open('vos://foo/bar', mode=-1)
with self.assertRaises(OSError):
client = Client()
client.get_node_url = Mock(return_value=None)
client.open(None, url=None)
conn = Connection()
mock_vofile = VOFile(['http://foo.com/bar'], conn, 'GET')
client = Client()
client.get_node_url = Mock(return_value=mock_vofile)
vofile = client.open(None, url=None)
self.assertEquals(vofile.url.URLs[0], 'http://foo.com/bar')
def test_init_client(self):
# No parameters uses cert in ~/.ssl giving authenticated / https
# create a fake pem file
certfile = '/tmp/some-cert-file.pem'
open(certfile, 'w+')
Client.VOSPACE_CERTFILE = "some-cert-file.pem"
with patch('os.access'):
client = Client(vospace_certfile=certfile)
client.get_session(uri='ivo://cadc.nrc.ca/vault')
conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn
self.assertTrue(conn.subject.certificate)
self.assertFalse(conn.vo_token)
# Supplying an empty string for certfile implies anonymous / http
client = Client(vospace_certfile='')
client.get_session(uri='ivo://cadc.nrc.ca/vault')
conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn
self.assertTrue(conn.subject.anon)
self.assertFalse(conn.vo_token)
# Specifying a token implies authenticated / http
client = Client(vospace_token='a_token_string')
client.get_session(uri='ivo://cadc.nrc.ca/vault')
conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn
self.assertTrue(conn.subject.anon)
self.assertTrue(conn.vo_token)
# Specifying both a certfile and token implies token (auth) / http
with patch('os.access'):
client = Client(vospace_certfile=certfile,
vospace_token='a_token_string')
client.get_session(uri='ivo://cadc.nrc.ca/vault')
conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn
self.assertTrue(conn.subject.anon)
self.assertTrue(conn.vo_token)
# update auth for specific service
with patch('os.access'):
client.set_auth(uri='ivo://cadc.nrc.ca/vault',
vospace_certfile=certfile)
conn = client._endpoints['ivo://cadc.nrc.ca/vault'].conn
self.assertTrue(conn.subject.certificate)
self.assertFalse(conn.vo_token)
def test_delete(self):
certfile = '/tmp/SomeCert.pem'
open(certfile, 'w+')
with patch('os.access'):
client = Client(vospace_certfile=certfile)
uri1 = 'vos://cadc.nrc.ca!vospace/nosuchfile1'
url = 'https://www.canfar.phys.uvic.ca/vospace/nodes/nosuchfile1'
client.conn.session.delete = Mock()
client.delete(uri1)
client.conn.session.delete.assert_called_once_with(url)
def test_open(self):
# Invalid mode raises OSError
with self.assertRaises(OSError):
client = Client()
client.open('vos://foo/bar', mode=-10000)
with self.assertRaises(OSError):
client = Client()
client.get_node_url = Mock(return_value=None)
client.open(None, url=None)
conn = Connection(resource_id='ivo://cadc.nrc.ca/vault')
mock_vofile = VOFile(['http://foo.com/bar'], conn, 'GET')
client = Client()
client.get_node_url = Mock(return_value=mock_vofile)
endpoints_mock = Mock(conn=conn)
client.get_endpoints = Mock(return_value=endpoints_mock)
vofile = client.open(None, url=None)
self.assertEqual(vofile.url.URLs[0], 'http://foo.com/bar')
def test_create(self):
uri = 'vos://create.vospace.auth!vospace/bar'
client = Client()
node = Node(client.fix_uri(uri))
node2 = Node(str(node))
self.assertEqual(node, node2)
data = str(node)
headers = {'size': str(len(data))}
client = Client()
# client.get_node_url = Mock(return_value='http://foo.com/bar')
session_mock = MagicMock()
client.get_session = Mock(return_value=session_mock)
session_mock.put.return_value = Mock(content=str(node))
result = client.create(uri)
self.assertEqual(node, result)
session_mock.put.assert_called_with(
'http://www.canfar.phys.uvic.ca/vospace/nodes/bar',
headers=headers, data=data)
def test_transfer_error(self, mock_vofile):
vofile = MagicMock()
mock_vofile.return_value = vofile
vospace_url = 'https://somevospace.server/vospace'
session = Mock()
session.get.side_effect = [Mock(content='COMPLETED')]
conn = Mock(spec=Connection)
conn.session = session
test_client = Client()
# use the mocked connection instead of the real one
test_client.conn = conn
# job successfully completed
vofile.read.side_effect = [b'QUEUED', b'COMPLETED']
self.assertFalse(test_client.get_transfer_error(
vospace_url +'/results/transferDetails', 'vos://vospace'))
session.get.assert_called_once_with(vospace_url + '/phase', allow_redirects=False)
# job suspended
session.reset_mock()
session.get.side_effect = [Mock(content=b'COMPLETED')]
vofile.read.side_effect = [b'QUEUED', b'SUSPENDED']
with self.assertRaises(OSError):
test_client.get_transfer_error(
vospace_url +'/results/transferDetails', 'vos://vospace')
#check arguments for session.get calls
self.assertEquals([call(vospace_url + '/phase', allow_redirects=False)],
session.get.call_args_list )
# job encountered an internal error
session.reset_mock()
vofile.read.side_effect = Mock(side_effect=[b'QUEUED', b'ERROR'])
session.get.side_effect = [Mock(content=b'COMPLETED'), Mock(text='InternalFault')]
with self.assertRaises(OSError):
test_client.get_transfer_error(
vospace_url +'/results/transferDetails', 'vos://vospace')
self.assertEquals([call(vospace_url + '/phase', allow_redirects=False),
call(vospace_url + '/error')],
session.get.call_args_list )
# job encountered an unsupported link error
session.reset_mock()
link_file = 'testlink.fits'
vofile.read.side_effect = Mock(side_effect=[b'QUEUED', b'ERROR'])
session.get.side_effect = [Mock(content=b'COMPLETED'),
Mock(text="Unsupported link target: " + link_file)]
self.assertEquals(link_file, test_client.get_transfer_error(
vospace_url +'/results/transferDetails', 'vos://vospace'))
self.assertEquals([call(vospace_url + '/phase', allow_redirects=False),
call(vospace_url + '/error')],
session.get.call_args_list )
def test_get_info_list(self):
# list tuples of a LinkNode
mock_node = MagicMock(type='vos:DataNode')
mock_node.return_value = mock_node
mock_node.name = 'testnode'
mock_node.get_info.return_value = {'name': 'aa'}
mock_link_node = Mock(type='vos:LinkNode')
mock_link_node.target = 'vos:/somefile'
client = Client()
client.get_node = MagicMock(side_effect=[mock_link_node, mock_node])
self.assertEquals([mock_node],
client.get_children_info('vos:/somenode'))
def off_quota(self):
"""
Test that a 413 raised by the server gets a reasonable error to the user.
@return:
"""
with patch('vos.vos.VOFile') as mockVOFile:
mockVOFile.open = Mock()
mockVOFile.read = Mock()
mockVOFile.write = Mock()
client = Client()
client.conn = Mock()
client.transfer(uri='vos:test', direction="pushToVoSpace")
def test_move(self):
mock_resp_403 = Mock(name="mock_resp_303")
mock_resp_403.status_code = 403
conn = Connection()
conn.session.post = Mock(return_value=mock_resp_403)
client = Client(conn=conn)
uri1 = 'notvos://cadc.nrc.ca!vospace/nosuchfile1'
uri2 = 'notvos://cadc.nrc.ca!vospace/nosuchfile2'
with self.assertRaises(OSError):
client.move(uri1, uri2)
def rs():
config = mc.Config()
config.get_executors()
vos_client = Client(vospace_certfile=config.proxy_fqn)
builder = GemProcBuilder(config)
storage_name = builder.build(sys.argv[1])
store_transfer = tc.VoFitsTransfer(vos_client)
rc.run_single(
config=config,
storage_name=storage_name,
command_name='gemProc2caom2',
meta_visitors=[],
data_visitors=[provenance_augmentation, preview_augmentation],
store_transfer=store_transfer,
)
def test_getNode(self):
"""
@return:
"""
uri = "vos://foo.com!vospace/bar"
nodes = (' <vos:nodes>\n'
'<vos:node uri="vos://cadc.nrc.ca!vospace/mydir/file123" '
'xs:type="vos:DataNode">\n'
' <vos:properties>\n'
' <vos:property '
'uri="ivo://ivoa.net/vospace/core#date">2016-05-10T09:52:13'
'</vos:property>\n'
' </vos:properties>\n'
'</vos:node>\n'
'<vos:node uri="vos://cadc.nrc.ca!vospace/mydir/file456" '
'xs:type="vos:DataNode">\n'
' <vos:properties>\n'
' <vos:property uri="ivo://ivoa.net/vospace/core#date">'
'2016-05-19T09:52:14</vos:property>\n'
' </vos:properties>\n'
'</vos:node>\n'
'</vos:nodes>\n')
mock_vofile = Mock()
client = Client()
client.open = Mock(return_value=mock_vofile)
mock_vofile.read = Mock(
return_value=NODE_XML.format(uri, '').encode('UTF-8'))
my_node = client.get_node(uri, limit=0, force=False)
self.assertEqual(uri, my_node.uri)
self.assertEqual(len(my_node.node_list), 0)
mock_vofile.read = Mock(
return_value=NODE_XML.format(uri, nodes).encode('UTF-8'))
my_node = client.get_node(uri, limit=2, force=True)
self.assertEqual(uri, my_node.uri)
self.assertEqual(len(my_node.node_list), 2)
my_node = client.get_node(uri, limit=2, force=False)
self.assertEqual(uri, my_node.uri)
self.assertEqual(len(my_node.node_list), 2)
def bad_exposure_list():
"""
Retreieve the list of bad exposrures from S. Gwyn's VOSpace listing.
:return: list of bad exposures
:rtype list
"""
Client().copy(u'vos:sgwyn/tkBAD', u'tkBAD')
bad_expnum_list = []
for lines in open('tkBAD').readlines():
# noinspection PyBroadException
try:
bad_expnum_list.append(lines.split()[0])
except:
sys.stderr.write(lines)
return bad_expnum_list
def test_add_props(self):
old_node = Node(ElementTree.fromstring(NODE_XML))
new_node = Node(ElementTree.fromstring(NODE_XML))
new_node.props['quota'] = '1000'
new_node.create = Mock(return_value=new_node.node)
data = str(new_node)
headers = {'size': str(len(data))}
client = Client()
client.get_node = Mock(return_value=old_node)
client.get_node_url = Mock(return_value='http://foo.com/bar')
client.conn = Mock()
with patch('vos.Client', client) as mock:
mock.add_props(new_node)
mock.conn.session.post.assert_called_with('http://foo.com/bar',
headers=headers, data=data)
def test_update(self):
node = Node(ElementTree.fromstring(NODE_XML))
resp = Mock()
resp.headers.get = Mock(
return_value="https://www.canfar.phys.uvic.ca/vospace")
conn = Mock(spec=vos.Connection)
conn.session.post = Mock(return_value=resp)
client = Client(conn=conn)
client.get_node_url = Mock(
return_value='https://www.canfar.phys.uvic.ca/vospace')
client.get_transfer_error = Mock()
client.protocol = 'https'
data = str(node)
property_url = 'https://www.canfar.phys.uvic.ca/vospace/nodeprops'
endpoints_mock = Mock()
endpoints_mock.properties = property_url
client.get_endpoints = Mock(return_value=endpoints_mock)
result = client.update(node, False)
self.assertEqual(result, 0)
client.conn.session.post.assert_called_with(
'https://www.canfar.phys.uvic.ca/vospace',
data=data,
allow_redirects=False)
call1 = call(property_url,
allow_redirects=False,
data=data,
headers={'Content-type': 'text/xml'})
call2 = call(
'https://www.canfar.phys.uvic.ca/vospace/phase',
allow_redirects=False,
data="PHASE=RUN",
headers={'Content-type': 'application/x-www-form-urlencoded'})
calls = [call1, call2]
client.conn = Mock(spec=vos.Connection)
client.conn.session.post = Mock(return_value=resp)
result = client.update(node, True)
self.assertEqual(result, 0)
client.conn.session.post.assert_has_calls(calls)
def test_get_node_url():
client = Client()
with pytest.raises(TypeError):
client.get_node_url('vos://cadc.nrc.ca!vospace/auser', sort='Blah')
with pytest.raises(ValueError):
client.get_node_url('vos://cadc.nrc.ca!vospace/auser', order='Blah')
response = Mock(spec=requests.Response)
response.status_code = 303
resource_id = 'ivo://cadc.nrc.ca/vospace'
session_mock = Mock(spec=requests.Session, get=Mock(return_value=response))
session_mock.headers = Mock()
client._endpoints[resource_id] = vos.EndPoints(resource_id_uri=resource_id)
client._endpoints[resource_id].conn.ws_client._session = session_mock
equery = urlparse(client.get_node_url('vos://cadc.nrc.ca!vospace/auser',
sort=vos.SortNodeProperty.DATE)).query
assert(urllib.parse.unquote(equery) ==
'sort={}'.format(vos.SortNodeProperty.DATE.value))
equery = urlparse(client.get_node_url('vos://cadc.nrc.ca!vospace/auser',
sort=vos.SortNodeProperty.LENGTH, order='asc')).query
args = urllib.parse.unquote(equery).split('&')
assert(2 == len(args))
assert('order=asc' in args)
assert('sort={}'.format(vos.SortNodeProperty.LENGTH.value) in args)
equery = urlparse(client.get_node_url('vos://cadc.nrc.ca!vospace/auser',
order='desc')).query
assert('order=desc' == urllib.parse.unquote(equery))
# test header view
transfer_url = 'https://some.location/some/headers'
response.headers = {'Location': transfer_url}
client._endpoints[resource_id].conn.ws_client._session.get = \
Mock(return_value=response)
assert transfer_url == \
client.get_node_url('vos://cadc.nrc.ca!vospace/auser',
view='header')[0]
# get the argument lists for client.conn.session.get
args, kwargs = client._endpoints[
resource_id].conn.ws_client._session.get.call_args_list[0]
# check head is amongst the other parameters
assert kwargs['params']['view'] == 'header'
def _run_remote():
"""
Uses a todo file to identify the work to be done.
:return 0 if successful, -1 if there's any sort of failure. Return status
is used by airflow for task instance management and reporting.
"""
config = mc.Config()
config.get_executors()
name_builder = nbc.FileNameBuilder(GemProcName)
vos_client = Client(vospace_certfile=config.proxy_fqn)
store_transfer = tc.VoFitsTransfer(vos_client)
data_source = dsc.VaultListDirDataSource(vos_client, config)
return rc.run_by_todo(config=config,
name_builder=name_builder,
command_name=APPLICATION,
source=data_source,
meta_visitors=META_VISITORS,
data_visitors=DATA_VISITORS,
store_transfer=store_transfer)
def test_add_props(self):
old_node = Node(ElementTree.fromstring(NODE_XML))
old_node.uri = 'vos:sometest'
new_node = Node(ElementTree.fromstring(NODE_XML))
new_node.props['quota'] = '1000'
new_node.create = Mock(return_value=new_node.node)
data = str(new_node)
headers = {'size': str(len(data))}
client = Client()
client.get_node = Mock(return_value=old_node)
client.get_node_url = Mock(return_value='http://foo.com/bar')
mock_session = Mock()
client.get_session = Mock(return_value=mock_session)
client.add_props(new_node)
client.get_session.assert_called_with('vos://foo.com!vospace/bar')
mock_session.post.assert_called_with('http://foo.com/bar',
headers=headers,
data=data)
def _run_vo():
"""
Uses a VOS listdir to identify the work to be done.
:return 0 if successful, -1 if there's any sort of failure. Return status
is used by airflow for task instance management and reporting.
"""
config, clients, name_builder, metadata_reader = _common()
vos_client = Client(vospace_certfile=config.proxy_file_name)
clients.vo_client = vos_client
source = data_source.DAOVaultDataSource(
config, clients.vo_client, clients.data_client
)
store_transferrer = transfer.VoFitsCleanupTransfer(vos_client, config)
return rc.run_by_todo(
name_builder=name_builder,
meta_visitors=META_VISITORS,
data_visitors=DATA_VISITORS,
source=source,
clients=clients,
store_transfer=store_transferrer,
metadata_reader=metadata_reader,
)
def main(orbit=None, **kwargs):
"""
This is the driver program. Gets the images from VOSpace or the local filesystem.
expected kwargs:
pointing, index, chip, rate, angle, ra, dec, p_name, discovery, nstk, zpt, dbimages
:param kwargs: these are the 'args' sent on the commandline or in an input file.
:param orbit: the orbit of the object that should be in the frames reference by kwargs provided.
:type orbit: BKOrbit
:return: list of observations
:rtype list(ObsRecod)
"""
pointing = kwargs['pointing']
chip = kwargs['chip']
index = kwargs['index']
rate = kwargs['rate']
angle = kwargs['angle']
ra = kwargs['ra']
dec = kwargs['dec']
p_name = kwargs['p_name']
discovery = kwargs['discovery']
nstk = kwargs['nstk']
rejected = kwargs.get('rejected', False)
zpt = kwargs.get('zpt', 26.9)
dbimages = kwargs.get('dbimages', 'vos:NewHorizons/dbimages/')
client = Client()
int_rate = int(rate * 10)
int_angle = int((angle % 360) * 10)
images = []
# Load the 3 images associated with this point/chip/rate/angle set.
epoch = None
for idx in range(nstk):
expnum = f'{int(pointing)}{int_rate:02d}{int_angle:04d}{idx}'
image = f'{expnum}p{chip:02d}.fits'
url = f'{dbimages}/{pointing:05d}/{chip:03d}/{index:04d}/{image}'
logging.info(f"Looking for image at {url}")
try:
if os.access(url, os.R_OK):
image = url
elif not os.access(image, os.R_OK):
# get from VOSpace is not already on disk
client.copy(url, image)
except Exception as ex:
logging.error(str(ex))
# Return empty set on VOSpace copy error.
return {}
images.append(image)
epoch = Time(fits.open(images[len(images) // 2])[0].header['DATE-AVG'],
scale='tai').utc
regions = f'{dbimages}/{pointing:05d}.reg'
try:
if not os.access(regions, os.R_OK):
regions = client.copy(regions, '.', disposition=True)
except Exception as ex:
logging.debug(f"{ex}")
regions = None
wcs_dict = {}
if orbit is not None:
epoch = orbit.epoch.mjd
if epoch is None:
epoch = Time("2020-05-22T00:00:00")
epoch = Time(kwargs.get('epoch', epoch), format='mjd')
load_images(images,
ra,
dec,
wcs_dict,
orbit,
dra=rate * math.cos(math.radians(angle)),
ddec=rate * math.sin(math.radians(angle)),
regions=regions,
rejected=rejected,
basedate=epoch)
obs = measure_image(p_name, images, wcs_dict, discovery=discovery, zpt=zpt)
return obs
from vos import Client
c = Client()
#for expnum in c.listdir('vos:cfis/solar_system/dbimages', force=True):
for expnum in open('expnum_list.txt').readlines():
expnum = expnum.strip()
n = c.get_node('vos:cfis/solar_system/dbimages/{}'.format(expnum), force=True)
print n.uri
for prop in n.props:
if 'stationary' in prop:
n.props[prop] = None
c.add_props(n)
print("Done")
#!python
from multiprocessing import Pool
from ossos import mop_file, storage
import os
from astropy.io import fits
import numpy
from vos import Client
import sys
from ossos import junk_keywords
from ossos import util
import argparse
import logging
client = Client()
RA = 'RA_J2000'
DEC = 'DE_J2000'
del_keyword_list = junk_keywords.bad_keywords_list
task = 'slow'
dependency = 'step1'
def run(expnum, ccd, version='p', prefix='', dry_run=False, force=False):
message = 'success'
if storage.get_status(task, prefix, expnum, version=version,
ccd=ccd) and not force:
logging.info("{} completed successfully for {} {} {} {}".format(
task, prefix, expnum, version, ccd))
return
def test_copy(self, computed_md5_mock):
file_content = 'File content'.encode('utf-8')
# the md5sum of the file being copied
transfer_md5 = hashlib.md5()
transfer_md5.update(file_content)
md5sum = transfer_md5.hexdigest()
# patch the compute_md5 function in vos to return the above value
computed_md5_mock.return_value = md5sum
# mock the props of the corresponding node
props = MagicMock()
props.get.return_value = md5sum
# add props to the mocked node
node = MagicMock(spec=Node)
node.props = {'MD5': md5sum, 'length': 12}
# mock one by one the chain of connection.session.response.headers
session = MagicMock()
response = MagicMock()
headers = MagicMock()
headers.get.return_value = md5sum
response.headers = headers
session.get.return_value = response
response.iter_content.return_value = BytesIO(file_content)
test_client = Client()
test_client.get_session = Mock(return_value=session)
# use the mocked connection instead of the real one
get_node_url_mock = Mock(
return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
test_client.get_node_url = get_node_url_mock
mock_update = Mock()
test_client.update = mock_update
# patch Client.get_node to return our mocked node
get_node_mock = Mock(return_value=node)
test_client.get_node = get_node_mock
# time to test...
vospaceLocation = 'vos://test/foo'
osLocation = '/tmp/foo'
if os.path.isfile(osLocation):
os.remove(osLocation)
# copy from vospace
test_client.copy(vospaceLocation, osLocation)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None, view='data')
assert not computed_md5_mock.called,\
'MD5 should be computed on the fly'
assert get_node_mock.called
# repeat - local file and vospace file are now the same -> only
# get_node is called to get the md5 of remote file
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
get_node_mock.reset_mock()
props.reset_mock()
props.get.return_value = md5sum
test_client.copy(vospaceLocation, osLocation)
assert not get_node_url_mock.called
computed_md5_mock.assert_called_once_with(osLocation)
get_node_mock.assert_called_once_with(vospaceLocation, force=True)
# change the content of local files to trigger a new copy
get_node_url_mock.reset_mock()
get_node_mock.reset_mock()
computed_md5_mock.reset_mock()
computed_md5_mock.return_value = 'd002233'
response.iter_content.return_value = BytesIO(file_content)
test_client.copy(vospaceLocation, osLocation)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None, view='data')
computed_md5_mock.assert_called_with(osLocation)
get_node_mock.assert_called_once_with(vospaceLocation, force=True)
# change the content of local files to trigger a new copy
get_node_url_mock.reset_mock()
get_node_url_mock.return_value = \
['https://mysite.com/node/node123/cutout']
computed_md5_mock.reset_mock()
computed_md5_mock.return_value = 'd002233'
# computed_md5_mock.side_effect = ['d002233', md5sum]
get_node_mock.reset_mock()
response.iter_content.return_value = BytesIO(file_content)
session.get.return_value = response
test_client.get_session = Mock(return_value=session)
test_client.copy('{}{}'.format(vospaceLocation,
'[1][10:60]'), osLocation)
get_node_url_mock.assert_called_once_with(
vospaceLocation, method='GET', cutout='[1][10:60]', view='cutout')
# copy to vospace when md5 sums are the same -> only update occurs
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
computed_md5_mock.side_effect = None
computed_md5_mock.return_value = md5sum
test_client.copy(osLocation, vospaceLocation)
mock_update.assert_called_once()
assert not get_node_url_mock.called
# make md5 different on destination
get_node_url_mock.reset_mock()
get_node_url_mock.return_value =\
['http://cadc.ca/test', 'http://cadc.ca/test']
computed_md5_mock.reset_mock()
mock_update.reset_mock()
computed_md5_mock.return_value = md5sum
to_update_node = MagicMock()
to_update_node.props = {'MD5': 'abcde', 'length': 12}
test_client.get_node = Mock(side_effect=[to_update_node, node])
test_client.copy(osLocation, vospaceLocation)
assert not mock_update.called
get_node_url_mock.assert_called_once_with(vospaceLocation, 'PUT',
content_length=12,
md5_checksum='abcde')
computed_md5_mock.assert_called_once_with(osLocation)
# copy 0 size file -> delete and create node but no bytes
# transferred
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
test_client.get_node = Mock(return_value=node)
node.props['length'] = 0
mock_delete = Mock()
mock_create = Mock()
test_client.delete = mock_delete
test_client.create = mock_create
with patch('vos.vos.os.stat', Mock()) as stat_mock:
stat_mock.return_value = Mock(st_size=0)
test_client.copy(osLocation, vospaceLocation)
mock_create.assert_called_once_with(vospaceLocation)
mock_delete.assert_called_once_with(vospaceLocation)
assert not get_node_url_mock.called
# copy new 0 size file -> create node but no bytes transferred
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
mock_delete.reset_mock()
mock_create.reset_mock()
test_client.get_node = Mock(side_effect=[Exception(), node])
mock_delete = Mock()
mock_create = Mock()
test_client.delete = mock_delete
test_client.create = mock_create
with patch('vos.vos.os.stat', Mock()) as stat_mock:
stat_mock.return_value = Mock(st_size=0)
test_client.copy(osLocation, vospaceLocation)
mock_create.assert_called_once_with(vospaceLocation)
assert not mock_delete.called
assert not get_node_url_mock.called
# error tests - md5sum mismatch
node.props['length'] = 12
computed_md5_mock.return_value = '000bad000'
test_client.get_node = Mock(return_value=node)
with self.assertRaises(OSError):
test_client.copy(vospaceLocation, osLocation)
# existing file
mock_delete.reset_mock()
with self.assertRaises(OSError):
with patch('vos.vos.os.stat', Mock()) as stat_mock:
stat_mock.return_value = Mock(st_size=12)
test_client.copy(osLocation, vospaceLocation)
assert not mock_delete.called # server takes care of cleanup
# new file
mock_delete.reset_mock()
with self.assertRaises(OSError):
with patch('vos.vos.os.stat', Mock()) as stat_mock:
stat_mock.return_value = Mock(st_size=12)
node.props['MD5'] = None
test_client.copy(osLocation, vospaceLocation)
assert mock_delete.called # cleanup required
# requests just the headers when md5 not provided in the header
props.get.side_effect = [None]
get_node_url_mock = Mock(
return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
test_client.get_node_url = get_node_url_mock
get_node_mock.reset_mock()
headers.get.return_value = None
test_client.copy(vospaceLocation, osLocation, head=True)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None, view='header')
# repeat headers request when md5 provided in the header
props.get.side_effect = md5sum
get_node_url_mock = Mock(
return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
test_client.get_node_url = get_node_url_mock
get_node_mock.reset_mock()
response.iter_content.return_value = BytesIO(file_content)
headers.get.return_value = None
test_client.copy(vospaceLocation, osLocation, head=True)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None, view='header')
def test_copy(self, computed_md5_mock):
# the md5sum of the file being copied
md5sum = 'd41d8cd98f00b204e9800998ecf84eee'
# patch the compute_md5 function in vos to return the above value
computed_md5_mock.return_value = md5sum
# mock the props of the corresponding node
props = MagicMock()
props.get.return_value = md5sum
# add props to the mocked node
node = MagicMock(spec=Node)
node.props = props
# mock one by one the chain of connection.session.response.headers
conn = MagicMock(spec=Connection)
session = MagicMock()
response = MagicMock()
headers = MagicMock()
headers.get.return_value = md5sum
response.headers = headers
session.get.return_value = response
conn.session = session
test_client = Client()
# use the mocked connection instead of the real one
test_client.conn = conn
get_node_url_mock = Mock(
return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
test_client.get_node_url = get_node_url_mock
mock_update = Mock()
test_client.update = mock_update
# patch Client.get_node to return our mocked node
get_node_mock = Mock(return_value=node)
test_client.get_node = get_node_mock
# time to test...
vospaceLocation = 'vos://test/foo'
osLocation = '/tmp/foo'
if os.path.isfile(osLocation):
os.remove(osLocation)
# copy from vospace
test_client.copy(vospaceLocation, osLocation)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None,
view='data')
computed_md5_mock.assert_called_once_with(osLocation)
assert get_node_mock.called
# repeat - local file and vospace file are now the same -> only
# get_node is called to get the md5 of remote file
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
get_node_mock.reset_mock()
test_client.copy(vospaceLocation, osLocation)
assert not get_node_url_mock.called
computed_md5_mock.assert_called_once_with(osLocation)
get_node_mock.assert_called_once_with(vospaceLocation)
# change the content of local files to trigger a new copy
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
computed_md5_mock.side_effect = ['d002233', md5sum]
get_node_mock.reset_mock()
test_client.copy(vospaceLocation, osLocation)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None,
view='data')
computed_md5_mock.assert_called_with(osLocation)
get_node_mock.assert_called_once_with(vospaceLocation)
# copy to vospace when md5 sums are the same -> only update occurs
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
computed_md5_mock.side_effect = None
computed_md5_mock.return_value = md5sum
test_client.copy(osLocation, vospaceLocation)
mock_update.assert_called_once()
assert not get_node_url_mock.called
# make md5 different
get_node_url_mock.reset_mock()
get_node_url_mock.return_value =\
['http://cadc.ca/test', 'http://cadc.ca/test']
computed_md5_mock.reset_mock()
mock_update.reset_mock()
props.get.side_effect = ['d00223344', md5sum]
test_client.copy(osLocation, vospaceLocation)
assert not mock_update.called
get_node_url_mock.assert_called_once_with(vospaceLocation, 'PUT')
computed_md5_mock.assert_called_once_with(osLocation)
# copy 0 size file -> delete and create on client but no bytes
# transferred
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
computed_md5_mock.return_value = vos.ZERO_MD5
props.get.side_effect = [md5sum]
mock_delete = Mock()
mock_create = Mock()
test_client.delete = mock_delete
test_client.create = mock_create
test_client.copy(osLocation, vospaceLocation)
mock_create.assert_called_once_with(vospaceLocation)
mock_delete.assert_called_once_with(vospaceLocation)
assert not get_node_url_mock.called
# copy new 0 size file -> reate on client but no bytes transferred
get_node_url_mock.reset_mock()
computed_md5_mock.reset_mock()
mock_delete.reset_mock()
mock_create.reset_mock()
computed_md5_mock.return_value = vos.ZERO_MD5
props.get.side_effect = [None]
mock_delete = Mock()
mock_create = Mock()
test_client.delete = mock_delete
test_client.create = mock_create
test_client.copy(osLocation, vospaceLocation)
mock_create.assert_called_once_with(vospaceLocation)
assert not mock_delete.called
assert not get_node_url_mock.called
# error tests - md5sum mismatch
props.get.side_effect = [md5sum]
computed_md5_mock.return_value = '000bad000'
with self.assertRaises(OSError):
test_client.copy(vospaceLocation, osLocation)
with self.assertRaises(OSError):
test_client.copy(osLocation, vospaceLocation)
# requests just the headers
props.get.side_effect = [None]
get_node_url_mock = Mock(
return_value=['http://cadc.ca/test', 'http://cadc.ca/test'])
test_client.get_node_url = get_node_url_mock
computed_md5_mock.reset_mock()
computed_md5_mock.side_effect = ['d002233', md5sum]
get_node_mock.reset_mock()
test_client.copy(vospaceLocation, osLocation, head=True)
get_node_url_mock.assert_called_once_with(vospaceLocation,
method='GET',
cutout=None,
view='header')
def test_glob(self):
# test the pattern matches in directories and file names
# simple test for listing of directory, no wild cards
# NOTE: Mock class also has a 'name' attribute so we cannot
# instantiate a mock node with Mock(name='blah'). There are
# two other wasy to do it as seen below
mock_node = MagicMock(type='vos:ContainerNode')
mock_node.configure_mock(name='anode')
client = Client()
client.get_node = Mock(return_value=mock_node)
self.assertEquals(['vos:/anode/'], client.glob('vos:/anode/'))
# simple test for file listing of file
mock_node = MagicMock(type='vos:DataNode')
mock_node.configure_mock(name='afile')
client = Client()
client.get_node = Mock(return_value=mock_node)
self.assertEquals(['vos:/afile'], client.glob('vos:/afile'))
# create a mock directory structure on the form
# /anode/abc /anode/def - > anode/a* should return
# /anode/adc
mock_node = MagicMock(type='vos:ContainerNode')
mock_node.configure_mock(name='anode')
mock_child_node1 = Mock(type='vos:DataNode')
mock_child_node1.name = 'abc'
mock_child_node2 = Mock(type='vos:DataNode')
mock_child_node2.name = 'def'
# because we use wild characters in the root node,
# we need to create a corresponding node for the base node
mock_base_node = Mock(type='vos:ContainerNode')
mock_base_node.name = 'vos:'
mock_base_node.node_list = [mock_node]
mock_node.node_list = [
mock_base_node, mock_child_node1, mock_child_node2
]
client = Client()
client.get_node = Mock(
side_effect=[mock_node, mock_base_node, mock_node])
self.assertEquals(['vos:/anode/abc'], client.glob('vos:/anode/a*'))
self.assertEquals(['vos:/anode/abc'], client.glob('vos:/*node/abc'))
# test nodes:
# /anode/.test1 /bnode/sometests /bnode/blah
# /[a,c]node/*test* should return /bnode/somtests (.test1 is filtered
# out as a special file)
mock_child_node1 = Mock(type='vos:DataNode')
mock_child_node1.name = '.test1'
mock_node1 = MagicMock(type='vos:ContainerNode')
mock_node1.configure_mock(name='anode')
mock_node1.node_list = [mock_child_node1]
mock_child_node2 = Mock(type='vos:DataNode')
mock_child_node2.name = 'sometests'
mock_child_node3 = Mock(type='vos:DataNode')
mock_child_node3.name = 'blah'
mock_node2 = MagicMock(type='vos:ContainerNode')
mock_node2.configure_mock(name='bnode')
mock_node2.node_list = [mock_child_node2, mock_child_node3]
# because we use wild characters in the root node,
# we need to create a corresponding node for the base node
mock_base_node = Mock(type='vos:DataNode')
mock_base_node.name = 'vos:'
mock_base_node.node_list = [mock_node1, mock_node2]
client = Client()
client.get_node = Mock(
side_effect=[mock_base_node, mock_node1, mock_node2])
self.assertEquals(['vos:/bnode/sometests'],
client.glob('vos:/[a,b]node/*test*'))