def create_ascii_table(obsTable, outfile):
"""Given a table of observations create an ascii log file for easy parsing.
Store the result in outfile (could/should be a vospace dataNode)
obsTable: astropy.votable.array object
outfile: str (name of the vospace dataNode to store the result to)
"""
logging.info("writing text log to %s" % (outfile))
stamp = "#\n# Last Updated: " + time.asctime() + "\n#\n"
header = "| %20s | %20s | %20s | %20s | %20s | %20s | %20s |\n" % (
"EXPNUM", "OBS-DATE", "FIELD", "EXPTIME(s)", "RA", "DEC", "RUNID")
bar = "=" * (len(header) - 1) + "\n"
if outfile[0:4] == "vos:":
fout = vos.Client().open(outfile, mode=os.O_WRONLY)
else:
fout = open(outfile, 'w')
t2 = None
fout.write(bar + stamp + bar + header)
populated = vos.Client().listdir('vos:OSSOS/dbimages')
for i in range(len(obsTable) - 1, -1, -1):
row = obsTable.data[i]
if row['dataset_name'] not in populated:
storage.populate(row['dataset_name'])
sDate = str(
ephem.date(row.StartDate + 2400000.5 -
ephem.julian_date(ephem.date(0))))[:20]
t1 = time.strptime(sDate, "%Y/%m/%d %H:%M:%S")
if t2 is None or math.fabs(time.mktime(t2) -
time.mktime(t1)) > 3 * 3600.0:
fout.write(bar)
t2 = t1
ra = str(ephem.hours(math.radians(row.RA)))
dec = str(ephem.degrees(math.radians(row.DEC)))
line = "| %20s | %20s | %20s | %20.1f | %20s | %20s | %20s |\n" % (
str(row.dataset_name),
str(
ephem.date(row.StartDate + 2400000.5 -
ephem.julian_date(ephem.date(0))))[:20],
row.TargetName[:20], row.ExposureTime, ra[:20], dec[:20],
row.ProposalID[:20])
fout.write(line)
fout.write(bar)
fout.close()
return
def update_backup(subject, dirname):
"""
Logs an update of the status of processing and rejecting files and
transfers the latest logging files to the configured (vos) backup
:param subject:
:param dirname:
:return:
"""
backup_dir = etrans_config.get(ETRANS_SECTION, 'backup_dir')
# this should come from the config instance
config_file = os.path.join(utils._CONFIG_PATH, 'cadc-etrans-config')
if not backup_dir:
raise RuntimeError('backup_dir must be specified in {}'.
format(config_file))
if not backup_dir.startswith('vos:'):
raise RuntimeError(not 'Only back_dir of form vos: supported in {}'.
format(config_file))
rfiles = get_rejected_files(dirname)
tfiles = get_trans_files(dirname)
_get_transfer_log().info('{} - {}'.
format(utils.LOG_STATUS_LABEL,
json.dumps({'rejected': rfiles,
'transferring': tfiles})))
logger.debug('Rsyncing the transfer logs')
translog = etrans_config.get(ETRANS_SECTION, 'transfer_log_dir')
client = vos.Client(vospace_certfile=subject.certificate,
transfer_shortcut=True)
for f in glob.glob(
'{}*'.format(os.path.join(translog, TRANS_ROOT_LOGNAME))):
client.copy(f, '{}/{}'.format(backup_dir, os.path.basename(f)))
def __init__(self, slice_rows=None, slice_cols=None, vosclient=None):
"""
Constructor.
Args:
resolver:
Resolves source readings to the URI's from which they can be
retrieved.
slice_rows, slice_cols: int
The number of rows and columns (pixels) to slice out around the
source. Leave as None to use default configuration values.
"""
# If not provided, read defaults from application config file
if slice_rows is None:
slice_rows = config.read("IMG_RETRIEVAL.DEFAULT_SLICE_ROWS")
if slice_cols is None:
slice_cols = config.read("IMG_RETRIEVAL.DEFAULT_SLICE_COLS")
self.slice_rows = slice_rows
self.slice_cols = slice_cols
if vosclient is None:
self.vosclient = vos.Client(cadc_short_cut=True)
else:
self.vosclient = vosclient
self.cutout_calculator = CutoutCalculator(slice_rows, slice_cols)
def get_vos_file(vos_uri, cutout=None):
vos_client = vos.Client()
if cutout is None:
view = 'data'
else:
view = 'cutout'
return vos_client.open(vos_uri, view=view, cutout=cutout)
def client(self):
"""
Send back the client we were sent, or construct a default one.
"""
if self._client is not None:
return self._client
self._client = vos.Client()
return self._client
def test_open(self):
client = vos.Client()
filename = tempfile.NamedTemporaryFile()
uri = client.rootNode+'/'+filename
vofile = client.open(uri)
vofile.write(data)
vofile.close()
vofile = client.open(uri, view="data")
assert data==vofile.read()
def get_fits_header(uri):
filename = os.path.basename(uri)
if os.access(filename, os.R_OK):
header = fits.open(filename)[0].header
else:
c = vos.Client()
fobj = c.open(uri, view='header')
header = fits.Header.fromtextfile(BytesIO(fobj.read()))
fobj.close()
return header
def get_fits_cutout(uri, ra, dec, rad):
filename = os.path.basename(uri)
if os.access(filename, os.R_OK):
hdulist = fits.open(filename)
else:
c = vos.Client()
fobj = c.open(uri,
view='cutout',
cutout=f'CIRCLE ICRS {ra} {dec} {rad}')
hdulist = fits.open(BytesIO(fobj.read()))
fobj.close()
return hdulist
def open(self, filename):
"""Open the file for reading"""
# is this a vospace file reference?
parts = urlparse.urlparse(filename)
if parts.scheme == VOS_SCHEME:
self.fobj = vos.Client().open(filename)
elif parts.scheme == FILE_SCHEME or parts.scheme == '':
self.fobj = open(filename)
else:
raise IOError("unknown scheme: %s" % (parts.scheme))
return
def __init__(self,
root,
cache_dir,
options,
conn=None,
cache_limit=1024,
cache_nodes=False,
cache_max_flush_threads=10,
secure_get=False):
"""Initialize the VOFS.
cache_limit is in MB.
The style here is to use dictionaries to contain information
about the Node. The full VOSpace path is used as the Key for
most of these dictionaries."""
self.cache_nodes = cache_nodes
# Standard attributes of the Node
# Where in the file system this Node is currently located
self.loading_dir = {}
# Command line options.
self.opt = options
# What is the 'root' of the VOSpace? (eg vos:MyVOSpace)
self.root = root
# VOSpace is a bit slow so we do some caching.
self.cache = Cache(cache_dir,
cache_limit,
False,
VOFS.cacheTimeout,
maxFlushThreads=cache_max_flush_threads)
# All communication with the VOSpace goes through this client
# connection.
try:
self.client = vos.Client(root_node=root,
conn=conn,
transfer_shortcut=True,
secure_get=secure_get)
except Exception as e:
e = FuseOSError(getattr(e, 'errno', EIO))
e.filename = root
e.strerror = getattr(e, 'strerror', 'failed while making mount')
raise e
# Create a condition variable to get rid of those nasty sleeps
self.condition = CacheCondition(lock=None, timeout=VOFS.cacheTimeout)
def __init__(self, server=None, endpoint=None, **kwargs):
self.keep_columns = [
'Job_ID', 'User', 'Started_on', 'Status', 'VM_Type', 'Command'
]
self.filter = {'User': None, 'Status': None}
if server is None:
server = CANFAR_PROC_SERVER
if endpoint is None:
endpoint = CANFAR_PROC_ENDPOINT
self.server = server
self.endpoint = endpoint
self.client = vos.Client(**kwargs)
def vospace_connector():
# images are in http://www.canfar.phys.uvic.ca/vosui/#/OSSOS/dbimages/
# eg. (symbolic link, need to be connected/certificate etc)
# http://www.canfar.phys.uvic.ca/vosui/#/OSSOS/dbimages/1607615/1607615o.head
vospace = vos.Client(
) #cadc_short_cut=True) # shortcut doesn't work due to symbolic links
stem = 'vos:OSSOS/dbimages/'
# pull back each directory in vos:OSSOS/dbimages/ and extract the .head file in it
# could use listDir?
vos_images = [n[0] for n in vospace.getInfoList(stem) if n[0].isdigit()]
vos_images.sort()
return vospace, stem, vos_images
def test_getNodeURL_cadcshortcut(self):
# Create client with different params for this test
shortcut_url = "shortcut_url"
http_con = Mock(spec=httplib.HTTPConnection)
http_response = Mock(status=303, spec=httplib.HTTPResponse)
http_response.getheader.return_value = shortcut_url
http_con.getresponse.return_value = http_response
conn = Mock(spec=vos.Connection)
conn.get_connection.return_value = http_con
self.undertest = vos.Client(vospace_certfile="", conn=conn, cadc_short_cut=True)
self.undertest.transfer = Mock(return_value=TEST_ENDPOINT)
url = self.undertest.get_node_url(TEST_URI_2, view="data")
assert_that(not self.undertest.transfer.called)
http_response.getheader.assert_called_once_with("Location", ANY)
assert_that(url, equal_to(shortcut_url))
def export_discovery_triplet(self, field):
# write discovery_triplet to a file in VOSpace
triplet = self.discovery_triplet(field)
if triplet:
# try:
stem = 'vos:OSSOS/triplets/'
vospace = vos.Client(certFile=CERTFILE)
# TESTING TESTING REMOVE BEFORE FLIGHT
tmpfile = 'E_13A_discovery_expnums.txt'
blockuri = stem + tmpfile
# does a file for this block already exist in VOSpace? If so, copy it back.
if vospace.access(blockuri): # Does this work this way?
# vospace.create(blockuri)
vospace.copy(blockuri, './' + tmpfile)
print tmpfile, '<-', blockuri
# the field is already present
with open(tmpfile, 'r+') as scratch:
lines = scratch.readlines()
for line in lines:
if line.split(
)[3] == field: # It's present already. Add updating later
return
scratch.write(
'%s %s %s %s\n' %
(triplet[0][0], triplet[0][1], triplet[0][2], field))
else:
with open(tmpfile, 'w') as scratch:
# 3-line file: id id id field
scratch.write(
'%s %s %s %s\n' %
(triplet[0][0], triplet[0][1], triplet[0][2], field))
vospace.copy(tmpfile, blockuri)
print tmpfile, '->', blockuri
# except Exception, e:
# print e
return
def main():
args = parse_args()
dataset = h5py.File(args.output_hdf5, 'a')
dataset_patches = dataset.require_group('patches')
vosdir = os.path.join('vos:jonathansick', args.vodir)
c = vos.Client('vos:jonathansick')
file_names = c.listdir(vosdir)
for name in file_names:
print name
if not name.endswith('.hdf5'):
continue
patch_name = os.path.splitext(os.path.basename(name))[0]
if patch_name in dataset_patches:
print "{0} already in dataset".format(patch_name)
continue
try:
c.copy(os.path.join(vosdir, name), name)
except:
print "Could not download {0}".format(name)
continue
# open the patch's HDF5 file
try:
patch_data = h5py.File(name, 'r')
except IOError:
continue
patch_serial = patch_data.attrs['patch']
patch_data.copy(patch_data, dataset_patches, name=patch_serial)
patch_data.close()
os.remove(name)
validate_coords(dataset_patches)
reduce_sfh_table(dataset, dataset_patches)
dataset.flush()
dataset.close()
def main():
ra = 287.58987
dec = -20.80109
rad = 10 / 3600.0
ra_rate = -2.41 / 3600.0
dec_rate = -0.22 / 3600.0
logging.basicConfig(
level=logging.DEBUG,
format=
'%(relativeCreated)d %(pathname)s %(funcName)s %(lineno)d %(message)s',
datefmt='%I:%M:%S')
base_dir = "HSC_May25-lsst"
rerun = "diff"
pointing = "03071"
filter = "HSC-R2"
vos_dir = os.path.join(VOS_BASE_URI, base_dir, 'rerun', rerun, pointing,
filter)
c = vos.Client()
logging.getLogger("urllib3").setLevel(logging.DEBUG)
filenames = c.listdir(vos_dir)
inputs = {}
ref_date = None
for filename in filenames:
logging.debug(f'Reading {filename}')
if filename not in inputs:
inputs[filename] = {}
inputs[filename]['uri'] = os.path.join(vos_dir, filename)
inputs[filename]['header'] = get_fits_header(inputs[filename]['uri'])
inputs[filename]['ref_date'] = exp_date(inputs[filename]['header'])
if ref_date is None:
ref_date = inputs[filename]['ref_date']
dt = (inputs[filename]['ref_date'] - ref_date).to('hour').value
this_ra, this_dec = shift_ra_dec(ra, dec, ra_rate, dec_rate, ref_date,
inputs[filename]['ref_date'])
logging.debug(
f'{filename} {ref_date} {dt} {ra} {dec} {this_ra} {this_dec}')
inputs[filename]['hdulist'] = get_fits_cutout(inputs[filename]['uri'],
this_ra, this_dec, rad)
w = wcs.WCS(inputs[filename]['hdulist'][1].header)
x, y = w.wcs_world2pix(ra, dec, 0)
logging.debug(f'{filename} {ra},{dec}->{x},{y}')
inputs[filename]['centre'] = x, y
inputs[filename]['data'] = inputs[filename]['hdulist'][
1].data / inputs[filename]['hdulist'][3].data
if not os.access(filename, os.F_OK):
inputs[filename]['hdulist'].writeto(filename)
# fits.ImageHDU(data=stack_inputs(inputs)).writeto('stack.fits')
peak = 0
for xoffset in numpy.arange(-10, 11, 1):
for yoffset in numpy.arange(-10, 11, 1):
for xrate in numpy.arange(-10, 11, 1):
for yrate in numpy.arange(-10, 11, 1):
current = metric(inputs, xoffset, yoffset, xrate, yrate,
ref_date)
if current > peak:
solution = (xoffset, yoffset, xrate, yrate)
peak = current
diag_plot(inputs, solution, ref_date)
logging.info(
f'Best fit (xo, yo, xr, yr) -> {solution}: scale: {peak}'
)
MAXCOUNT = 30000
CERTFILE = os.path.join(os.getenv('HOME'), '.ssl', 'cadcproxy.pem')
DBIMAGES = 'vos:OSSOS/dbimages'
MEASURE3 = 'vos:OSSOS/measure3'
POSTAGE_STAMPS = 'vos:OSSOS/postage_stamps'
DATA_WEB_SERVICE = 'https://www.canfar.phys.uvic.ca/data/pub/'
VOSPACE_WEB_SERVICE = 'https://www.canfar.phys.uvic.ca/vospace/nodes/'
TAP_WEB_SERVICE = 'http://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/tap/sync'
OSSOS_TAG_URI_BASE = 'ivo://canfar.uvic.ca/ossos'
OBJECT_COUNT = "object_count"
vospace = vos.Client()
SUCCESS = 'success'
# ## some cache holders.
mopheaders = {}
astheaders = {}
APCOR_EXT = "apcor"
ZEROPOINT_USED_EXT = "zeropoint.used"
PSF_EXT = "psf.fits"
def cone_search(ra,
dec,
dra=0.01,
def get_flat_list(repo='vos:OSSOS/dbimages/calibrators'):
vospace = vos.Client()
flats = {}
for filename in vospace.listdir(repo):
flats[filename[0:5]] = filename
return flats
import requests
from io import StringIO
from astropy.table import Table
from astropy.io import ascii
import tempfile
import vos
parser = argparse.ArgumentParser()
parser.add_argument('-o',
'--output',
action="store_true",
help="Store to file instead of mailing list")
args = parser.parse_args()
current_list = vos.Client().listdir('vos:OSSOS/dbimages')
recon_exposures = Table.read(StringIO(
requests.get(url, params=params).content),
format='ascii.csv')
print(recon_exposures)
if args.output:
with open('list.txt', 'w') as fout:
cond = [str(g) not in current_list for g in recon_exposures['expnum']]
ascii.write(recon_exposures[cond], fout, format='no_header')
if not args.output:
you = '*****@*****.**'
you = '*****@*****.**'
me = '*****@*****.**'
def starnet_download_file(filename):
vclient = vos.Client()
vclient.copy('vos:starnet/public/' + filename, datadir + filename)
print(filename + ' downloaded')
def __init__(self, type="Component"):
self.client = vos.Client("/cirada")
# self.catalogues = client.listdir("vos:/cirada/data/EMU/catalogues")
self.catalogues = list(
client.iglob("vos:/cirada/data/EMU/catalogues/AS101*{type}*csv"))
def setUp(self):
self.conn = Mock(spec=vos.Connection)
# Set certFile to blank because person running this may or may not
# have a certification file.
self.undertest = vos.Client(vospace_certfile="", conn=self.conn)
def refresh_vos_client(self):
"""
If we have gotten a new certfile we have to create a new Client
object before it will get used.
"""
self.vosclient = vos.Client(cadc_short_cut=True)
def run_transfer(self, transdir, filenames, dry_run=False):
"""
Function to launch data transfer.
:param transdir: directory containing files to transfer.
:param filenames: list of files to transfer.
:param dry_run: specifies dry run mode.
"""
logger.debug('Constructing VOS client')
vos_client = vos.Client()
if not vos_client.isdir(self.vos_base):
raise Exception('VOS base directory does not exist (or is file)')
vos_cache = {}
for file_ in (filenames):
# Determine local file information.
file_path = os.path.join(transdir, file_)
file_md5 = get_md5sum(file_path)
# Determine VO space information.
vos_sub_dir = self.determine_vos_directory(transdir, file_)
if vos_sub_dir is None:
# This indicates that the file should be skipped.
continue
logger.debug('VOS directory for {0}: {1}'.format(
file_path, vos_sub_dir))
vos_dir = '/'.join([self.vos_base, vos_sub_dir])
vos_file = '/'.join([vos_dir, file_])
# Get directory listing -- this creates the directory
# if not in dry-run mode.
if vos_dir in vos_cache:
vos_dir_info = vos_cache[vos_dir]
else:
vos_dir_info = self.get_vos_directory_entries(vos_client,
vos_dir,
dry_run=dry_run)
vos_cache[vos_dir] = vos_dir_info
# Perform storage, if file changed (and not in dry-run mode).
vos_md5 = vos_dir_info.get(file_, ())
if (vos_md5 != ()) and (vos_md5 == file_md5):
logger.info('Skipped storing {0} as {1} [UNCHANGED]'.format(
file_path, vos_file))
elif dry_run:
logger.info('Skipped storing {0} as {1} [DRY-RUN]'.format(
file_path, vos_file))
else:
if vos_md5 != ():
logger.debug('Deleting existing file {0}'.format(vos_file))
vos_client.delete(vos_file)
logger.info('Storing {0} as {1}'.format(file_path, vos_file))
vos_client.copy(file_path, vos_file)
def resolve(pointing, chip, rate, angle, x, y, vos_basedir):
"""
:param pointing: pointing that source is found on
:param chip: chip that source is found on
:param rate: rate of motion that maximized source signal
:param angle: angle of motion that maximized source signal
:param x: x pixel value of source location in pointing/chip/rate/angle stack
:param y: y pixel value
:param vos_basedir: base location of VOSpace stacks, e.g. 'vos:NewHorizons/S20A-OT04/STACKS_V4'
:return: stack, ra, dec, mjd
"""
# Resolve retrieves the headers from the VOSpace repo with the stacks.
client = vos.Client()
# Standard pattern used for DAOMOP Shift-and-Stack
# Get a list of stack files and match to the one whose rate is closest to what we need.
stack_dir_uri = f"{vos_basedir}/{pointing:05d}/{chip:03d}"
stacks = client.listdir(stack_dir_uri)
stack = None
min_offset = None
min_idx = None
for idx, stack in enumerate(stacks):
stack_angle = stack_rate = 0
try:
match = STACK_PATTERN.search(stack)
if match is not None:
stack_rate = float(match.group('rate'))
stack_angle = float(match.group('angle'))
except Exception as ex:
# logging.debug(f"Failed to match {stack}\n")
logging.error(str(ex))
continue
# use cos/sin rule to compute vector difference between requested rate and stack rate.
da = math.radians(math.fabs(stack_angle - angle))
dr = rate**2 + stack_rate**2 - 2 * rate * stack_rate * math.cos(da)
# select the minimum vector difference as the stack of interest.
if min_offset is None or min_offset > dr:
min_offset = dr
min_idx = idx
if min_idx is None:
logging.error(f"No match for {pointing} {chip} {rate} {angle} {x} {y}")
logging.error(f"Checked: {stacks} in {stack_dir_uri}")
return None
# Found the STACK in the VOSpace storage area that matches the pointing/chip/rate/angle of interest.
# Now pull the 'cutout' with the WCS and compute the RA/DEC from the x/y location.
stack_image = stacks[min_idx]
logging.info(
f"pointing:{pointing:05d}, chip:{chip:03d}, rate:{rate:5.1f}, angle:{angle:5.1f} "
f"matched stack:{stack_image} in {vos_basedir}")
uri = f"{stack_dir_uri}/{stack_image}"
# noinspection PyBroadException
# get the header of the matching stack and compute RA/DEC of the source using that header
try:
fobj = BytesIO(client.open(uri, cutout=CUTOUT, view='cutout').read())
with fits.open(fobj) as hdu:
ra, dec = WCS(hdu[0].header).all_pix2world(x, y, 1)
fobj = BytesIO(client.open(uri, cutout='[0]', view='cutout').read())
with fits.open(fobj) as hdu:
epoch = hdu[0].header.get('MIDMJD',
hdu[0].header.get('MJD-STR', -1.0))
except Exception as ex:
logging.error(
"Failed while measuring {pointing} {chip} {x} {y} on {stack}")
logging.error("{ex}")
return None
return {'stack': stack_image, 'ra': ra, 'dec': dec, 'epoch': epoch}
def create_sky_plot(obstable, outfile, night_count=1, stack=True):
"""Given a VOTable that describes the observation coverage provide a PDF of the skycoverge.
obstable: vostable.arrary
stack: BOOL (true: stack all the observations in a series of plots)
"""
# camera dimensions
width = 0.98
height = 0.98
if outfile[0:4] == 'vos:':
tmpFile = tempfile.NamedTemporaryFile(suffix='.pdf')
pdf = PdfPages(tmpFile.name)
else:
pdf = PdfPages(outfile)
ra_min = obstable['RA'].max() + 1.5
ra_max = obstable['RA'].min() - 1.5
dec_min = obstable['DEC'].min() - 1.5
dec_max = obstable['DEC'].max() + 1.5
saturn = ephem.Saturn()
subplots = []
t2 = None
count = 0
fig = None
for row in reversed(obstable.data):
date = ephem.date(row.StartDate + 2400000.5 -
ephem.julian_date(ephem.date(0)))
sDate = str(date)
saturn.compute(date)
sra = math.degrees(saturn.ra)
sdec = math.degrees(saturn.dec)
t1 = time.strptime(sDate, "%Y/%m/%d %H:%M:%S")
if t2 is None or (math.fabs(time.mktime(t2) - time.mktime(t1)) >
3 * 3600.0 and opt.stack):
if fig is not None:
pdf.savefig()
close()
fig = figure(figsize=(7, 2))
ax = fig.add_subplot(111, aspect='equal')
ax.set_title("Data taken on %s-%s-%s" %
(t1.tm_year, t1.tm_mon, t1.tm_mday),
fontdict={'fontsize': 8})
ax.axis((245, 200, -20, 0))
ax.grid()
ax.set_xlabel("RA (deg)", fontdict={'fontsize': 8})
ax.set_ylabel("DEC (deg)", fontdict={'fontsize': 8})
t2 = t1
ra = row.RA - width / 2.0
dec = row.DEC - height / 2.0
color = 'b'
if 'W' in row['TargetName']:
color = 'g'
ax.add_artist(
Rectangle(xy=(ra, dec),
height=height,
width=width,
edgecolor=color,
facecolor=color,
lw=0.5,
fill='g',
alpha=0.33))
ax.add_artist(
Rectangle(xy=(sra, sdec),
height=0.3,
width=0.3,
edgecolor='r',
facecolor='r',
lw=0.5,
fill='k',
alpha=0.33))
if ax is not None:
ax.axis((270, 215, -20, 0))
pdf.savefig()
close()
pdf.close()
if outfile[0:4] == "vos:":
vos.Client().copy(tmpFile.name, outfile)
tmpFile.close()
return
default=None,
help="Distribution the output by header/chip?")
parser.add_option("--ccd",
action="store",
default=36,
type="int",
help="CCD to process [do all be default]")
parser.add_option("--dbimages",
action="store",
default="vos:OSSOS/dbimages",
help="VOSpace location of the dbimages directory")
### get the bias frames from the archive.
(opt, file_ids) = parser.parse_args()
vos_client = vos.Client()
if opt.combine and opt.normal and False:
## only take one image from each pointing
t = {}
for file_id in file_ids:
filename = os.path.join(opt.dbimages,
"%s/%so.head" % (file_id, file_id))
f = pyfits.open(
StringIO(vos_client.open(filename, view='data').read()))
t[f[0].header['OBJECT']] = file_id
f.close()
f = None
file_ids = []
for object in t:
file_ids.append(t[object])
phdr[0].header['UTEND'] = (lcdata[len(lcdata)-1][0],'End date of observations') bhdr = fits.BinTableHDU(data=lcdata.as_array()) phdulist = fits.HDUList([phdr[0],bhdr]) phdulist.writeto(os.path.join(os.getcwd(),os.path.basename(outfname)),clobber='True')#write locally first #now copy it to vospace vC.copy(os.path.join(os.getcwd(),os.path.basename(outfname)),outfname) print 'File written: '+outfname #cmd = 'getCert --daysValid=25 --dest=$HOME/.ssl/cadcproxy.pem' #os.system(cmd) vC = vos.Client() voMOST = 'vos://cadc.nrc.ca~vospace/MOST' voHome = os.path.join(voMOST,'most2caomlc') #vospace data server dws = 'http://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/data/pub' #Collect files; Read the list of directories from the 'lightcurve_data.txt'. #lclist = os.path.join(voHome,'resource/lightcurve_data.txt') lclistpath = os.path.join(voHome,'resource','lightcurve_data.txt') lclist = vC.open(lclistpath,mode='r',view='data') #with open(lclist,'r') as f:
import subprocess
import os
import vos
import logging
import urllib
import errno
from astropy.io import fits
import urlparse
CERTFILE = os.path.join(os.getenv('HOME'), '.ssl', 'cadcproxy.pem')
DBIMAGES = 'vos:OSSOS/dbimages'
DATA_WEB_SERVICE = 'https://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/data/pub/'
OSSOS_TAG_URI_BASE = 'ivo://canfar.uvic.ca/ossos'
vospace = vos.Client(cadc_short_cut=True, certFile=CERTFILE)
SUCCESS = 'success'
def populate(dataset_name, data_web_service_url=DATA_WEB_SERVICE + "CFHT"):
"""Given a dataset_name created the desired dbimages directories
and links to the raw data files stored at CADC."""
data_dest = get_uri(dataset_name, version='o', ext='fits.fz')
data_source = "%s/%so.fits.fz" % (data_web_service_url, dataset_name)
c = vospace
try:
c.mkdir(os.path.dirname(data_dest))
except IOError as e:
