def testSubject(self, os_mock):
# anon subject
subject = auth.Subject()
self.assertTrue(subject.anon)
self.assertEqual(None, subject.certificate)
self.assertEqual({}, subject._hosts_auth)
self.assertEqual(None, subject.get_auth('realm1'))
# cert subject
cert = 'somecert'
subject = auth.Subject(certificate=cert)
self.assertFalse(subject.anon)
self.assertEqual(cert, subject.certificate)
self.assertEqual({}, subject._hosts_auth)
self.assertEqual(None, subject.get_auth('realm1'))
# empty netrc subject
m = mock_open()
with patch('six.moves.builtins.open', m, create=True):
subject = auth.Subject(netrc='somefile')
self.assertFalse(subject.anon)
self.assertEqual(None, subject.certificate)
self.assertEqual({}, subject._hosts_auth)
self.assertEqual(None, subject.get_auth('realm1'))
# netrc with content
netrc_content = {'realm1': ('user1', None, 'pass1'),
'realm2': ('user1', None, 'pass2')}
expected_host_auth = {'realm1': ('user1', 'pass1'),
'realm2': ('user1', 'pass2')}
os_mock.path.join.return_value = '/home/myhome/.netrc'
with patch('cadcutils.net.auth.netrclib') as netrclib_mock:
netrclib_mock.netrc.return_value.hosts = netrc_content
subject = auth.Subject(netrc=True)
self.assertFalse(subject.anon)
self.assertEqual(None, subject.certificate)
self.assertEqual('/home/myhome/.netrc', subject.netrc)
self.assertEqual(expected_host_auth, subject._hosts_auth)
self.assertEqual(('user1', 'pass1'), subject.get_auth('realm1'))
self.assertEqual(('user1', 'pass2'), subject.get_auth('realm2'))
self.assertEqual(None, subject.get_auth('realm3'))
# subject with username
username = '******'
passwd = 'passwd1'
subject = auth.Subject(username=username)
self.assertFalse(subject.anon)
self.assertEqual(None, subject.certificate)
self.assertEqual({}, subject._hosts_auth)
with patch('cadcutils.net.auth.getpass') as getpass_mock:
getpass_mock.getpass.return_value = passwd
self.assertEqual((username, passwd), subject.get_auth('realm1'))
parser = get_base_parser(subparsers=False)
args = parser.parse_args(['--resource-id', 'blah'])
subject = auth.Subject.from_cmd_line_args(args)
self.assertTrue(subject.anon)
sys.argv = ['cadc-client', '--resource-id', 'blah', '--cert',
'mycert.pem']
args = parser.parse_args()
subject = auth.Subject.from_cmd_line_args(args)
self.assertFalse(subject.anon)
self.assertEqual('mycert.pem', subject.certificate)
def main_app():
parser = util.get_base_parser(version=version.version,
default_resource_id=DEFAULT_RESOURCE_ID)
parser.description = (
'Client for a CAOM2 repo. In addition to CRUD (Create, Read, Update '
'and Delete) operations it also implements a visitor operation that '
'allows for updating multiple observations in a collection')
parser.add_argument("-s", "--server", help='URL of the CAOM2 repo server')
parser.formatter_class = argparse.RawTextHelpFormatter
subparsers = parser.add_subparsers(dest='cmd')
create_parser = subparsers.add_parser(
'create',
description='Create a new observation',
help='Create a new observation')
create_parser.add_argument('observation',
help='XML file containing the observation',
type=argparse.FileType('r'))
read_parser = subparsers.add_parser(
'read',
description='Read an existing observation',
help='Read an existing observation')
read_parser.add_argument('--output',
'-o',
help='destination file',
required=False)
read_parser.add_argument('collection',
help='collection name in CAOM2 repo')
read_parser.add_argument('observationID', help='observation identifier')
update_parser = subparsers.add_parser(
'update',
description='Update an existing observation',
help='Update an existing observation')
update_parser.add_argument('observation',
help='XML file containing the observation',
type=argparse.FileType('r'))
delete_parser = subparsers.add_parser(
'delete',
description='Delete an existing observation',
help='Delete an existing observation')
delete_parser.add_argument('collection',
help='collection name in CAOM2 repo')
delete_parser.add_argument('observationID', help='observation identifier')
# Note: RawTextHelpFormatter allows for the use of newline in epilog
visit_parser = subparsers.add_parser(
'visit',
formatter_class=argparse.RawTextHelpFormatter,
description='Visit observations in a collection',
help='Visit observations in a collection')
visit_parser.add_argument('--plugin',
required=True,
type=argparse.FileType('r'),
help='plugin class to update each observation')
visit_parser.add_argument('--start',
type=str2date,
help=('earliest observation to visit '
'(UTC IVOA format: YYYY-mm-ddTH:M:S)'))
visit_parser.add_argument(
'--end',
type=str2date,
help='latest observation to visit (UTC IVOA format: YYYY-mm-ddTH:M:S)')
visit_parser.add_argument(
'--obs_file',
help='file containing observations to be visited',
type=argparse.FileType('r'))
visit_parser.add_argument(
'--threads',
type=int,
choices=xrange(2, 10),
help='number of working threads used by the visitor when getting ' +
'observations, range is 2 to 10')
visit_parser.add_argument(
'--halt-on-error',
action='store_true',
help='stop visitor on first update exception raised by plugin')
visit_parser.add_argument('collection',
help='data collection in CAOM2 repo')
visit_parser.epilog = \
"""
Minimum plugin file format:
----
from caom2 import Observation
class ObservationUpdater:
def update(self, observation, **kwargs):
assert isinstance(observation, Observation), (
'observation {} is not an Observation'.format(observation))
# custom code to update the observation
# other arguments passed by the calling code to the update
# method:
# kwargs['subject'] - user authentication that caom2repo was
# invoked with
----
"""
args = parser.parse_args()
if len(sys.argv) < 2:
parser.print_usage(file=sys.stderr)
sys.stderr.write("{}: error: too few arguments\n".format(APP_NAME))
sys.exit(-1)
if args.verbose:
level = logging.INFO
elif args.debug:
level = logging.DEBUG
else:
level = logging.WARN
subject = net.Subject.from_cmd_line_args(args)
server = None
if args.server:
server = args.server
multiprocessing.Manager()
logging.basicConfig(
format='%(asctime)s %(process)d %(levelname)-8s %(name)-12s ' +
'%(funcName)s %(message)s',
level=level,
stream=sys.stdout)
logger = logging.getLogger('main_app')
client = CAOM2RepoClient(subject, level, args.resource_id, host=server)
if args.cmd == 'visit':
print("Visit")
logger.debug(
"Call visitor with plugin={}, start={}, end={}, collection={}, " +
"obs_file={}, threads={}".format(args.plugin.name, args.start,
args.end, args.collection,
args.obs_file, args.threads))
try:
(visited, updated, skipped, failed) = \
client.visit(args.plugin.name, args.collection,
start=args.start,
end=args.end, obs_file=args.obs_file,
nthreads=args.threads,
halt_on_error=args.halt_on_error)
finally:
if args.obs_file is not None:
args.obs_file.close()
logger.info(
'Visitor stats: visited/updated/skipped/errors: {}/{}/{}/{}'.
format(len(visited), len(updated), len(skipped), len(failed)))
elif args.cmd == 'create':
logger.info("Create")
obs_reader = ObservationReader()
client.put_observation(obs_reader.read(args.observation))
elif args.cmd == 'read':
logger.info("Read")
observation = client.get_observation(args.collection,
args.observationID)
observation_writer = ObservationWriter()
if args.output:
observation_writer.write(observation, args.output)
else:
observation_writer.write(observation, sys.stdout)
elif args.cmd == 'update':
logger.info("Update")
obs_reader = ObservationReader()
# TODO not sure if need to read in string first
client.post_observation(obs_reader.read(args.observation))
else:
logger.info("Delete")
client.delete_observation(collection=args.collection,
observation_id=args.observationID)
logger.info("DONE")
def main_app():
parser = util.get_base_parser(version=version.version,
default_resource_id=DEFAULT_RESOURCE_ID)
parser.description = (
'Client for accessing the data Web Service at the Canadian Astronomy '
'Data Centre (www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/data)')
subparsers = parser.add_subparsers(
dest='cmd',
help='supported commands. Use the -h|--help argument of a command '
'for more details')
get_parser = subparsers.add_parser(
'get',
description='Retrieve files from a CADC archive',
help='Retrieve files from a CADC archive')
get_parser.add_argument('-o',
'--output',
help='space-separated list of destination files '
'(quotes required for multiple elements)',
required=False)
get_parser.add_argument(
'--cutout',
nargs='*',
help=('specify one or multiple extension and/or pixel range cutout '
'operations to be performed. Use cfitsio syntax'),
required=False)
get_parser.add_argument(
'--nomd5',
action='store_true',
required=False,
help='do not perform md5 check at the end of transfer')
get_parser.add_argument('-z',
'--decompress',
help='decompress the data (gzip only)',
action='store_true',
required=False)
get_parser.add_argument(
'--wcs',
help='return the World Coordinate System (WCS) information',
action='store_true',
required=False)
get_parser.add_argument('--fhead',
help='return the FITS header information',
action='store_true',
required=False)
get_parser.add_argument('archive', help='CADC archive')
get_parser.add_argument('filename',
help='the name of the file in the archive',
nargs='+')
get_parser.epilog = (
'Examples:\n'
'- Anonymously getting a public file:\n'
' cadc-data get -v GEMINI 00aug02_002.fits\n'
'- Use certificate to get a cutout and save it to a file:\n'
' cadc-data get --cert ~/.ssl/cadcproxy.pem -o '
'/tmp/700000o-cutout.fits --cutout [1] CFHT 700000o\n'
'- Use default netrc file ($HOME/.netrc) to get FITS header of a '
'file:\n'
' cadc-data get -v -n --fhead GEMINI 00aug02_002.fits\n'
'- Use a different netrc file to download wcs information:\n'
' cadc-data get -d --netrc ~/mynetrc -o /tmp/700000o-wcs.fits '
'--wcs CFHT 700000o\n'
'- Connect as user to download two files and uncompress them '
'(prompt for password if user not in $HOME/.netrc):\n'
' cadc-data get -v -u auser -z GEMINI 00aug02_002.fits '
'00aug02_003.fits')
put_parser = subparsers.add_parser(
'put',
description='Upload files into a CADC archive',
help='Upload files into a CADC archive')
put_parser.add_argument('-t',
'--type',
help="MIME type to set in archive. If missing, the"
" application will try to deduce it",
required=False)
put_parser.add_argument('-e',
'--encoding',
help='MIME encoding to set in archive. If missing,'
' the application will try to deduce it',
required=False)
put_parser.add_argument('-s',
'--archive-stream',
help='specific archive stream to add the file to',
required=False)
put_parser.add_argument(
'-i',
'--input',
help='space-separated list of input name to use in archive - '
'overrides the actual file names in source. (quotes required for '
'multiple elements)',
required=False)
put_parser.add_argument('archive', help='CADC archive')
put_parser.add_argument(
'--nomd5',
action='store_true',
required=False,
help='do not perform md5 check at the end of transfer')
put_parser.add_argument(
'source',
help='file or directory containing the files to be put',
nargs='+')
put_parser.epilog = (
'Examples:\n'
'- Use certificate to put a file in an archive stream under a '
'different name :\n'
' cadc-data put --cert ~/.ssl/cadcproxy.pem -as default \n'
' -t "application/gzip" -i newfilename.fits.gz '
'TEST myfile.fits.gz\n'
'- Use default netrc file ($HOME/.netrc) to put two files:\n'
' cadc-data put -v -n TEST myfile1.fits.gz myfile2.fits.gz\n'
'- Use a different netrc file to put files from a directory:\n'
' cadc-data put -d --netrc ~/mynetrc TEST dir\n'
'- Connect as user to put files from multiple sources (prompt for '
'password if user not in $HOME/.netrc):\n'
' cadc-data put -v -u auser TEST myfile.fits.gz dir1 dir2')
info_parser = subparsers.add_parser(
'info',
description=(
'Get information regarding files in a '
'CADC archive on the form:\n'
'File:\n'
'\t -name\n'
'\t -size\n'
'\t -md5sum\n'
'\t -encoding\n'
'\t -type\n'
'\t -usize\n'
'\t -umd5sum\n'
# '\t -ingest_date\n'
'\t -lastmod'),
help='Get information regarding files in a CADC archive')
info_parser.add_argument('archive', help='CADC archive')
info_parser.add_argument('filename',
help='the name of the file in the archive',
nargs='+')
info_parser.epilog = (
'Examples:\n'
'- Anonymously getting information about a public file:\n'
' cadc-data info GEMINI 00aug02_002.fits\n'
'- Use certificate to get information about a file:\n'
' cadc-data info --cert ~/.ssl/cadcproxy.pem CFHT 700000o\n'
'- Use default netrc file ($HOME/.netrc) to get information about '
'a file:\n'
' cadc-data info -n GEMINI 00aug02_002.fits\n'
'- Use a different netrc file to get information about a file:\n'
' cadc-data info --netrc ~/mynetrc CFHT 700000o\n'
'- Connect as user to get information about two files '
'(prompt for password if user not in $HOME/.netrc):\n'
' cadc-data info -u auser GEMINI 00aug02_002.fits '
'00aug02_003.fits')
# handle errors
errors = [0]
def handle_error(msg, exit_after=True):
"""
Prints error message and exit (by default)
:param msg: error message to print
:param exit_after: True if log error message and exit,
False if log error message and return
:return:
"""
errors[0] += 1
logger.error(msg)
if exit_after:
sys.exit(-1) # TODO use different error codes?
args = parser.parse_args()
if len(sys.argv) < 2:
parser.print_usage(file=sys.stderr)
sys.stderr.write("{}: error: too few arguments\n".format(APP_NAME))
sys.exit(-1)
if args.verbose:
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
elif args.debug:
logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
else:
logging.basicConfig(level=logging.WARN, stream=sys.stdout)
subject = net.Subject.from_cmd_line_args(args)
client = CadcDataClient(subject, args.resource_id, host=args.host)
try:
if args.cmd == 'get':
logger.info('get')
archive = args.archive
file_names = args.filename
if args.output is not None:
files = args.output.split()
if len(files) != len(file_names):
handle_error(
'Different size of destination files list ({}) '
'and list of file names ({})'.format(
files, file_names))
for f, fname in list(zip(files, file_names)):
try:
client.get_file(archive,
fname,
f,
decompress=args.decompress,
fhead=args.fhead,
wcs=args.wcs,
cutout=args.cutout,
md5_check=(not args.nomd5))
except exceptions.NotFoundException:
handle_error('File name {} not found'.format(fname),
exit_after=False)
else:
for fname in file_names:
try:
client.get_file(archive,
fname,
None,
decompress=args.decompress,
fhead=args.fhead,
wcs=args.wcs,
cutout=args.cutout,
md5_check=(not args.nomd5))
except exceptions.NotFoundException:
handle_error('File name not found {}'.format(fname),
exit_after=False)
elif args.cmd == 'info':
logger.info('info')
archive = args.archive
for file_name in args.filename:
try:
file_info = client.get_file_info(archive, file_name)
except exceptions.NotFoundException:
handle_error('File name {} not found in archive {}'.format(
file_name, archive),
exit_after=False)
continue
print('File {}:'.format(file_name))
for field in sorted(file_info):
print('\t {:>10}: {}'.format(field, file_info[field]))
else:
logger.info('put')
archive = args.archive
sources = args.source
files = []
for file1 in sources:
if os.path.isfile(file1):
files.append(file1)
elif os.path.isdir(file1):
for f in sorted(os.listdir(file1)):
if os.path.isfile(os.path.join(file1, f)):
files.append(os.path.join(file1, f))
else:
logger.warning(
'{} not added to the list of files to '
'put'.format(f))
logger.debug('Files to put: {}'.format(files))
if len(files) == 0:
handle_error('No files found to put')
if args.input:
input_names = args.input.split()
if len(input_names) != len(files):
handle_error('The number of input names does not match'
' the number of sources: {} vs {}'.format(
len(input_names), len(files)))
for f, input_name in list(zip(files, input_names)):
client.put_file(archive,
f,
archive_stream=args.archive_stream,
mime_type=args.type,
mime_encoding=args.encoding,
md5_check=(not args.nomd5),
input_name=input_name)
else:
for f in files:
client.put_file(archive,
f,
archive_stream=args.archive_stream,
mime_type=args.type,
mime_encoding=args.encoding,
md5_check=(not args.nomd5))
except exceptions.UnauthorizedException:
if subject.anon:
handle_error('Operation cannot be performed anonymously. '
'Use one of the available methods to authenticate')
else:
handle_error('Unexpected authentication problem')
except exceptions.ForbiddenException:
handle_error('Unauthorized to perform operation')
except exceptions.UnexpectedException as e:
handle_error('Unexpected server error: {}'.format(str(e)))
except Exception as e:
handle_error(str(e))
if errors[0] > 0:
logger.error('Finished with {} error(s)'.format(errors[0]))
sys.exit(-1)
else:
logger.info("DONE")
def main_app(command='cadc-tap query'):
parser = util.get_base_parser(version=version.version,
default_resource_id=DEFAULT_SERVICE_ID)
_customize_parser(parser)
parser.description = (
'Client for accessing databases using TAP protocol at the Canadian '
'Astronomy Data Centre (www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca)')
subparsers = parser.add_subparsers(
dest='cmd',
help='supported commands. Use the -h|--help argument of a command '
'for more details')
schema_parser = subparsers.add_parser(
'schema',
description=('Print the tables available for querying.'),
help='Print the tables available for querying.')
query_parser = subparsers.add_parser('query',
description=('Run an adql query'),
help='Run an adql query')
query_parser.add_argument(
'-o',
'--output-file',
default=None,
help='write query results to file (default is to STDOUT)',
required=False)
options_parser = query_parser.add_mutually_exclusive_group(required=True)
options_parser.add_argument(
'QUERY',
default=None,
nargs='?',
help='ADQL query to run, format is a string with quotes around it, '
'for example "SELECT observationURI FROM caom2.Observation"')
options_parser.add_argument(
'-i',
'--input-file',
default=None,
help='read query string from file (default is from STDIN),'
' location of file')
# Maybe adding async option later
"""
query_parser.add_argument(
'-a', '--async-job',
action='store_true',
help='issue an asynchronous query (default is synchronous'
' which only outputs the top 2000 results)',
required=False)
"""
query_parser.add_argument(
'-f',
'--format',
default='VOTable',
choices=['VOTable', 'csv', 'tsv'],
help='output format, either tsv, csv, fits (TBD), or votable(default)',
required=False)
query_parser.add_argument(
'-t',
'--tmptable',
default=None,
help='Temp table upload, the value is in format: '
'"tablename:/path/to/table". In query to reference the table'
' use tap_upload.tablename',
required=False)
query_parser.epilog = (
'Examples:\n'
'- Anonymously run a query string:\n'
' {0} "SELECT TOP 10 type FROM caom2.Observation"\n'
'- Use certificate to run a query from a file:\n'
' {0} -i /data/query.sql --cert ~/.ssl/cadcproxy.pem\n'
'- Use username/password to run a query on the tap service:\n'
' {0} -s ivo://cadc.nrc.ca/tap '
'"SELECT TOP 10 type FROM caom2.Observation"'
' -u <username>\n'
'- Use netrc file to run a query on the ams/mast service'
' :\n'
' {0} -i data/query.sql -n -s ivo://cadc.nrc.ca/ams/mast\n'.
format(command))
create_parser = subparsers.add_parser('create',
description='Create a table',
help='Create a table')
create_parser.add_argument('-f',
'--format',
choices=sorted(ALLOWED_TB_DEF_TYPES.keys()),
required=False,
help='Format of the table definition file')
create_parser.add_argument(
'TABLENAME',
help='name of the table (<schema.table>) in the tap service')
create_parser.add_argument(
'TABLEDEFINITION',
help='file containing the definition of the table or "-" if definition'
' in stdin')
delete_parser = subparsers.add_parser('delete',
description='Delete a table',
help='delete a table')
delete_parser.add_argument('TABLENAME',
help='name of the table (<schema.table)'
'in the tap service to be deleted')
index_parser = subparsers.add_parser('index',
description='Create a table index',
help='Create a table index')
index_parser.add_argument('-U',
'--unique',
action='store_true',
help='index is unique')
index_parser.add_argument(
'TABLENAME',
help='name of the table in the tap service to create the index for')
index_parser.add_argument(
'COLUMN', help='name of the column to create the index for')
load_parser = subparsers.add_parser('load',
description='Load data to a table',
help='Load data to a table')
load_parser.add_argument('-f',
'--format',
choices=sorted(ALLOWED_CONTENT_TYPES.keys()),
required=False,
default='tsv',
help='Format of the data file')
load_parser.add_argument(
'TABLENAME', help='name of the table (<schema.table>) to load data to')
load_parser.add_argument(
'SOURCE',
nargs='+',
help='source of the data. It can be files or "-" for stdout.')
# handle errors
errors = [0]
def handle_error(msg, exit_after=True):
"""
Prints error message and exit (by default)
:param msg: error message to print
:param exit_after: True if log error message and exit,
False if log error message and return
:return:
"""
errors[0] += 1
logger.error(msg)
if exit_after:
sys.exit(-1) # TODO use different error codes?
_customize_parser(schema_parser)
_customize_parser(query_parser)
_customize_parser(create_parser)
_customize_parser(delete_parser)
_customize_parser(index_parser)
_customize_parser(load_parser)
args = parser.parse_args()
if len(sys.argv) < 2:
parser.print_usage(file=sys.stderr)
sys.stderr.write("{}: error: too few arguments\n".format(APP_NAME))
sys.exit(-1)
if args.verbose:
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
elif args.debug:
logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
else:
logging.basicConfig(level=logging.WARN, stream=sys.stdout)
subject = net.Subject.from_cmd_line_args(args)
client = CadcTapClient(subject, resource_id=args.service)
if args.cmd == 'create':
client.create_table(args.TABLENAME, args.TABLEDEFINITION, args.format)
elif args.cmd == 'delete':
reply = input('You are about to delete table {} and its content... '
'Continue? [yes/no] '.format(args.TABLENAME))
while True:
if reply == 'yes':
client.delete_table(args.TABLENAME)
break
elif reply == 'no':
logger.warn('Table {} not deleted.'.format(args.TABLENAME))
sys.exit(-1)
else:
reply = input('Please reply with yes or no: ')
elif args.cmd == 'index':
client.create_index(args.TABLENAME, args.COLUMN, args.unique)
elif args.cmd == 'load':
client.load(args.TABLENAME, args.SOURCE, args.format)
elif args.cmd == 'query':
if args.input_file is not None:
with open(args.input_file) as f:
query = f.read().strip()
else:
query = args.QUERY
client.query(query, args.output_file, args.format, args.tmptable)
elif args.cmd == 'schema':
client.schema()
print('DONE')
def get_cert_main():
""" Client to download an X509 certificate and save it in users home
directory"""
def _signal_handler(signal, frame):
sys.stderr.write("\n")
sys.exit(-1)
signal.signal(signal.SIGINT, _signal_handler)
parser = util.get_base_parser(subparsers=False, version=GET_CERT_VERSION,
default_resource_id=CRED_RESOURCE_ID,
auth_required=True)
parser.description = ('Retrieve a security certificate for interaction '
'with a Web service such as VOSpace. Certificate '
'will be valid for days-valid and stored as local '
'file cert_filename.')
parser.add_argument('--cert-filename',
default=os.path.join(os.getenv('HOME', '/tmp'),
'.ssl/cadcproxy.pem'),
help=('filesystem location to store the proxy '
'certificate. (default: {})'.
format(os.path.join(os.getenv('HOME', '/tmp'),
'.ssl/cadcproxy.pem'))))
parser.add_argument('--days-valid', type=int, default=10,
help='number of days the certificate should be valid.')
args = parser.parse_args()
dirname = os.path.dirname(args.cert_filename)
try:
os.makedirs(dirname)
except OSError as oex:
if os.path.isdir(dirname):
pass
elif oex.errno == 20 or oex.errno == 17:
sys.stderr.write("%s : %s\n" % (str(oex), dirname))
sys.stderr.write("Expected %s to be a directory.\n" % dirname)
sys.exit(oex.errno)
else:
raise oex
try:
subject = Subject.from_cmd_line_args(args)
cert = get_cert(subject, days_valid=args.days_valid)
with open(args.cert_filename, 'w') as w:
w.write(cert)
print('DONE. {} day certificate saved in {}'.format(
args.days_valid, args.cert_filename))
except OSError as ose:
sys.stderr.write("FAILED to retrieved {} day certificate\n".format(
args.days_valid))
if ose.errno != 401:
sys.stderr.write(html2text.html2text(str(ose)))
return getattr(ose, 'errno', 1)
else:
sys.stderr.write("Access denied\n")
except Exception as ex:
sys.stderr.write("FAILED to retrieved {} day certificate\n".format(
args.days_valid))
sys.stderr.write('{}'.format(html2text.html2text(str(ex))))
return getattr(ex, 'errno', 1)
def main_app():
parser = util.get_base_parser(version=version.version,
default_resource_id=DEFAULT_RESOURCE_ID)
parser.description = (
'Client for a CAOM2 repo. In addition to CRUD (Create, Read, Update and Delete) '
'operations it also implements a visitor operation that allows for updating '
'multiple observations in a collection')
parser.add_argument("-s", "--server", help='URL of the CAOM2 repo server')
parser.formatter_class = argparse.RawTextHelpFormatter
subparsers = parser.add_subparsers(dest='cmd')
create_parser = subparsers.add_parser(
'create',
description='Create a new observation',
help='Create a new observation')
create_parser.add_argument('observation',
help='XML file containing the observation',
type=argparse.FileType('r'))
read_parser = subparsers.add_parser(
'read',
description='Read an existing observation',
help='Read an existing observation')
read_parser.add_argument('--output',
'-o',
help='destination file',
required=False)
read_parser.add_argument('collection',
help='collection name in CAOM2 repo')
read_parser.add_argument('observationID', help='observation identifier')
update_parser = subparsers.add_parser(
'update',
description='Update an existing observation',
help='Update an existing observation')
update_parser.add_argument('observation',
help='XML file containing the observation',
type=argparse.FileType('r'))
delete_parser = subparsers.add_parser(
'delete',
description='Delete an existing observation',
help='Delete an existing observation')
delete_parser.add_argument('collection',
help='collection name in CAOM2 repo')
delete_parser.add_argument('observationID', help='observation identifier')
# Note: RawTextHelpFormatter allows for the use of newline in epilog
visit_parser = subparsers.add_parser(
'visit',
formatter_class=argparse.RawTextHelpFormatter,
description='Visit observations in a collection',
help='Visit observations in a collection')
visit_parser.add_argument('--plugin',
required=True,
type=argparse.FileType('r'),
help='plugin class to update each observation')
visit_parser.add_argument(
'--start',
type=str2date,
help='earliest observation to visit (UTC IVOA format: YYYY-mm-ddTH:M:S)'
)
visit_parser.add_argument(
'--end',
type=str2date,
help='latest observation to visit (UTC IVOA format: YYYY-mm-ddTH:M:S)')
visit_parser.add_argument(
'--halt-on-error',
action='store_true',
help='stop visitor on first update exception raised by plugin')
visit_parser.add_argument('collection',
help='data collection in CAOM2 repo')
visit_parser.epilog =\
"""
Minimum plugin file format:
----
from caom2 import Observation
class ObservationUpdater:
def update(self, observation):
assert isinstance(observation, Observation), (
'observation {} is not an Observation'.format(observation))
# custom code to update the observation
----
"""
args = parser.parse_args()
if args.verbose:
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
elif args.debug:
logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
else:
logging.basicConfig(level=logging.WARN, stream=sys.stdout)
subject = net.Subject.from_cmd_line_args(args)
server = None
if args.server:
server = args.server
client = CAOM2RepoClient(subject, args.resource_id, host=server)
if args.cmd == 'visit':
print("Visit")
logging.debug(
"Call visitor with plugin={}, start={}, end={}, collection={}".
format(args.plugin.name, args.start, args.end, args.collection))
(visited, updated, skipped, failed) = \
client.visit(args.plugin.name, args.collection, start=args.start, end=args.end,
halt_on_error=args.halt_on_error)
logging.info(
'Visitor stats: visited/updated/skipped/errors: {}/{}/{}/{}'.
format(len(visited), len(updated), len(skipped), len(failed)))
elif args.cmd == 'create':
logging.info("Create")
obs_reader = ObservationReader()
client.put_observation(obs_reader.read(args.observation))
elif args.cmd == 'read':
logging.info("Read")
observation = client.get_observation(args.collection,
args.observationID)
observation_writer = ObservationWriter()
if args.output:
with open(args.output, 'wb') as obsfile:
observation_writer.write(observation, obsfile)
else:
observation_writer.write(observation, sys.stdout)
elif args.cmd == 'update':
logging.info("Update")
obs_reader = ObservationReader()
# TODO not sure if need to read in string first
client.post_observation(obs_reader.read(args.observation))
else:
logging.info("Delete")
client.delete_observation(collection=args.collection,
observation_id=args.observationID)
logging.info("DONE")
def main_app():
parser = util.get_base_parser(version=version.version,
default_resource_id=False)
parser.description = (
'Application for transferring data and metadata electronically '
'to the Canadian Astronomy Data Centre.\n'
'It uses the config information in '
'~/.config/cadc-etrans to get the execution context '
'and configuration.')
subparsers = parser.add_subparsers(
dest='cmd',
help='Supported commands. Use the -h|--help argument of a command '
'for more details')
data_parser = subparsers.add_parser(
'data', description='Transfer data to a CADC archive',
help='Transfer data to a CADC archive.')
data_parser.add_argument('-c', '--check-filename',
help='Namecheck file to check file names against',
required=False)
data_parser.add_argument(
'-s', '--streamname', choices=allowed_streams,
help='Process only files in this input stream [new, replace, any]')
data_parser.add_argument(
'--dryrun', help=('Perform all steps with the exception of the actual '
'file transfer to the CADC'),
action='store_true', required=False)
data_parser.add_argument('source',
help='Source directory where the files are '
'located.')
data_parser.epilog = (
'Note:\nTo ensure that a file is fully received before attempting to\n'
'transfer it, it must spend a minimum amount of time (5min) in the\n'
'input directory without being modified/updated prior to its\n'
'processing.\n\n'
'Examples:\n'
'- Use default netrc file ($HOME/.netrc) to transfer FITS files in\n'
' the "current" dir: \n'
' cadc-etrans data -v -n current\n'
'- Use a different netrc file transfer the files in dryrun mode:\n'
' cadc-etrans data -d --netrc ~/mynetrc --dryrun workdir ')
status_parser = subparsers.\
add_parser('status',
description='Displays the status of the system',
help='Display the status of the system')
status_parser.add_argument('-b', '--backup',
help='sends status and local logs to a backup '
'location specified in the config file',
action='store_true', required=False)
status_parser.add_argument('source',
help='Source directory where the files are '
'located.')
# handle errors
errors = [0]
def handle_error(msg, exit_after=True):
"""
Prints error message and exit (by default)
:param msg: error message to print
:param exit_after: True if log error message and exit,
False if log error message and return
:return:
"""
errors[0] += 1
logger.error(msg)
if exit_after:
sys.exit(-1) # TODO use different error codes?
args = parser.parse_args()
if len(sys.argv) < 2:
parser.print_usage(file=sys.stderr)
sys.stderr.write("{}: error: too few arguments\n".format(APP_NAME))
sys.exit(-1)
if args.verbose:
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
elif args.debug:
logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
else:
if (args.cmd != 'status') and args.dryrun:
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
else:
logging.basicConfig(level=logging.WARN, stream=sys.stderr)
if args.cmd == 'meta':
raise NotImplementedError('meta command not implemented yet')
subject = net.Subject.from_cmd_line_args(args)
if args.cmd == 'status':
if args.backup:
update_backup(subject, args.source)
else:
print_status(args.source)
elif args.cmd == 'data':
try:
clean_up(args.source, dry_run=args.dryrun)
transfer(args.source, stream_name=args.streamname,
dry_run=args.dryrun, subject=subject,
namecheck_file=args.check_filename)
except CommandError as e:
logger.error('{}'.format(str(e)))
else:
print('ERROR - unknown command')
sys.exit(-1)
print('DONE')