def create_ascii_table(obsTable, outfile): """Given a table of observations create an ascii log file for easy parsing. Store the result in outfile (could/should be a vospace dataNode) obsTable: astropy.votable.array object outfile: str (name of the vospace dataNode to store the result to) """ logging.info("writing text log to %s" % (outfile)) stamp = "#\n# Last Updated: " + time.asctime() + "\n#\n" header = "| %20s | %20s | %20s | %20s | %20s | %20s | %20s |\n" % ( "EXPNUM", "OBS-DATE", "FIELD", "EXPTIME(s)", "RA", "DEC", "RUNID") bar = "=" * (len(header) - 1) + "\n" if outfile[0:4] == "vos:": fout = vos.Client().open(outfile, mode=os.O_WRONLY) else: fout = open(outfile, 'w') t2 = None fout.write(bar + stamp + bar + header) populated = vos.Client().listdir('vos:OSSOS/dbimages') for i in range(len(obsTable) - 1, -1, -1): row = obsTable.data[i] if row['dataset_name'] not in populated: storage.populate(row['dataset_name']) sDate = str( ephem.date(row.StartDate + 2400000.5 - ephem.julian_date(ephem.date(0))))[:20] t1 = time.strptime(sDate, "%Y/%m/%d %H:%M:%S") if t2 is None or math.fabs(time.mktime(t2) - time.mktime(t1)) > 3 * 3600.0: fout.write(bar) t2 = t1 ra = str(ephem.hours(math.radians(row.RA))) dec = str(ephem.degrees(math.radians(row.DEC))) line = "| %20s | %20s | %20s | %20.1f | %20s | %20s | %20s |\n" % ( str(row.dataset_name), str( ephem.date(row.StartDate + 2400000.5 - ephem.julian_date(ephem.date(0))))[:20], row.TargetName[:20], row.ExposureTime, ra[:20], dec[:20], row.ProposalID[:20]) fout.write(line) fout.write(bar) fout.close() return
def update_backup(subject, dirname): """ Logs an update of the status of processing and rejecting files and transfers the latest logging files to the configured (vos) backup :param subject: :param dirname: :return: """ backup_dir = etrans_config.get(ETRANS_SECTION, 'backup_dir') # this should come from the config instance config_file = os.path.join(utils._CONFIG_PATH, 'cadc-etrans-config') if not backup_dir: raise RuntimeError('backup_dir must be specified in {}'. format(config_file)) if not backup_dir.startswith('vos:'): raise RuntimeError(not 'Only back_dir of form vos: supported in {}'. format(config_file)) rfiles = get_rejected_files(dirname) tfiles = get_trans_files(dirname) _get_transfer_log().info('{} - {}'. format(utils.LOG_STATUS_LABEL, json.dumps({'rejected': rfiles, 'transferring': tfiles}))) logger.debug('Rsyncing the transfer logs') translog = etrans_config.get(ETRANS_SECTION, 'transfer_log_dir') client = vos.Client(vospace_certfile=subject.certificate, transfer_shortcut=True) for f in glob.glob( '{}*'.format(os.path.join(translog, TRANS_ROOT_LOGNAME))): client.copy(f, '{}/{}'.format(backup_dir, os.path.basename(f)))
def __init__(self, slice_rows=None, slice_cols=None, vosclient=None): """ Constructor. Args: resolver: Resolves source readings to the URI's from which they can be retrieved. slice_rows, slice_cols: int The number of rows and columns (pixels) to slice out around the source. Leave as None to use default configuration values. """ # If not provided, read defaults from application config file if slice_rows is None: slice_rows = config.read("IMG_RETRIEVAL.DEFAULT_SLICE_ROWS") if slice_cols is None: slice_cols = config.read("IMG_RETRIEVAL.DEFAULT_SLICE_COLS") self.slice_rows = slice_rows self.slice_cols = slice_cols if vosclient is None: self.vosclient = vos.Client(cadc_short_cut=True) else: self.vosclient = vosclient self.cutout_calculator = CutoutCalculator(slice_rows, slice_cols)
def get_vos_file(vos_uri, cutout=None): vos_client = vos.Client() if cutout is None: view = 'data' else: view = 'cutout' return vos_client.open(vos_uri, view=view, cutout=cutout)
def client(self): """ Send back the client we were sent, or construct a default one. """ if self._client is not None: return self._client self._client = vos.Client() return self._client
def test_open(self): client = vos.Client() filename = tempfile.NamedTemporaryFile() uri = client.rootNode+'/'+filename vofile = client.open(uri) vofile.write(data) vofile.close() vofile = client.open(uri, view="data") assert data==vofile.read()
def get_fits_header(uri): filename = os.path.basename(uri) if os.access(filename, os.R_OK): header = fits.open(filename)[0].header else: c = vos.Client() fobj = c.open(uri, view='header') header = fits.Header.fromtextfile(BytesIO(fobj.read())) fobj.close() return header
def get_fits_cutout(uri, ra, dec, rad): filename = os.path.basename(uri) if os.access(filename, os.R_OK): hdulist = fits.open(filename) else: c = vos.Client() fobj = c.open(uri, view='cutout', cutout=f'CIRCLE ICRS {ra} {dec} {rad}') hdulist = fits.open(BytesIO(fobj.read())) fobj.close() return hdulist
def open(self, filename): """Open the file for reading""" # is this a vospace file reference? parts = urlparse.urlparse(filename) if parts.scheme == VOS_SCHEME: self.fobj = vos.Client().open(filename) elif parts.scheme == FILE_SCHEME or parts.scheme == '': self.fobj = open(filename) else: raise IOError("unknown scheme: %s" % (parts.scheme)) return
def __init__(self, root, cache_dir, options, conn=None, cache_limit=1024, cache_nodes=False, cache_max_flush_threads=10, secure_get=False): """Initialize the VOFS. cache_limit is in MB. The style here is to use dictionaries to contain information about the Node. The full VOSpace path is used as the Key for most of these dictionaries.""" self.cache_nodes = cache_nodes # Standard attributes of the Node # Where in the file system this Node is currently located self.loading_dir = {} # Command line options. self.opt = options # What is the 'root' of the VOSpace? (eg vos:MyVOSpace) self.root = root # VOSpace is a bit slow so we do some caching. self.cache = Cache(cache_dir, cache_limit, False, VOFS.cacheTimeout, maxFlushThreads=cache_max_flush_threads) # All communication with the VOSpace goes through this client # connection. try: self.client = vos.Client(root_node=root, conn=conn, transfer_shortcut=True, secure_get=secure_get) except Exception as e: e = FuseOSError(getattr(e, 'errno', EIO)) e.filename = root e.strerror = getattr(e, 'strerror', 'failed while making mount') raise e # Create a condition variable to get rid of those nasty sleeps self.condition = CacheCondition(lock=None, timeout=VOFS.cacheTimeout)
def __init__(self, server=None, endpoint=None, **kwargs): self.keep_columns = [ 'Job_ID', 'User', 'Started_on', 'Status', 'VM_Type', 'Command' ] self.filter = {'User': None, 'Status': None} if server is None: server = CANFAR_PROC_SERVER if endpoint is None: endpoint = CANFAR_PROC_ENDPOINT self.server = server self.endpoint = endpoint self.client = vos.Client(**kwargs)
def vospace_connector(): # images are in http://www.canfar.phys.uvic.ca/vosui/#/OSSOS/dbimages/ # eg. (symbolic link, need to be connected/certificate etc) # http://www.canfar.phys.uvic.ca/vosui/#/OSSOS/dbimages/1607615/1607615o.head vospace = vos.Client( ) #cadc_short_cut=True) # shortcut doesn't work due to symbolic links stem = 'vos:OSSOS/dbimages/' # pull back each directory in vos:OSSOS/dbimages/ and extract the .head file in it # could use listDir? vos_images = [n[0] for n in vospace.getInfoList(stem) if n[0].isdigit()] vos_images.sort() return vospace, stem, vos_images
def test_getNodeURL_cadcshortcut(self): # Create client with different params for this test shortcut_url = "shortcut_url" http_con = Mock(spec=httplib.HTTPConnection) http_response = Mock(status=303, spec=httplib.HTTPResponse) http_response.getheader.return_value = shortcut_url http_con.getresponse.return_value = http_response conn = Mock(spec=vos.Connection) conn.get_connection.return_value = http_con self.undertest = vos.Client(vospace_certfile="", conn=conn, cadc_short_cut=True) self.undertest.transfer = Mock(return_value=TEST_ENDPOINT) url = self.undertest.get_node_url(TEST_URI_2, view="data") assert_that(not self.undertest.transfer.called) http_response.getheader.assert_called_once_with("Location", ANY) assert_that(url, equal_to(shortcut_url))
def export_discovery_triplet(self, field): # write discovery_triplet to a file in VOSpace triplet = self.discovery_triplet(field) if triplet: # try: stem = 'vos:OSSOS/triplets/' vospace = vos.Client(certFile=CERTFILE) # TESTING TESTING REMOVE BEFORE FLIGHT tmpfile = 'E_13A_discovery_expnums.txt' blockuri = stem + tmpfile # does a file for this block already exist in VOSpace? If so, copy it back. if vospace.access(blockuri): # Does this work this way? # vospace.create(blockuri) vospace.copy(blockuri, './' + tmpfile) print tmpfile, '<-', blockuri # the field is already present with open(tmpfile, 'r+') as scratch: lines = scratch.readlines() for line in lines: if line.split( )[3] == field: # It's present already. Add updating later return scratch.write( '%s %s %s %s\n' % (triplet[0][0], triplet[0][1], triplet[0][2], field)) else: with open(tmpfile, 'w') as scratch: # 3-line file: id id id field scratch.write( '%s %s %s %s\n' % (triplet[0][0], triplet[0][1], triplet[0][2], field)) vospace.copy(tmpfile, blockuri) print tmpfile, '->', blockuri # except Exception, e: # print e return
def main(): args = parse_args() dataset = h5py.File(args.output_hdf5, 'a') dataset_patches = dataset.require_group('patches') vosdir = os.path.join('vos:jonathansick', args.vodir) c = vos.Client('vos:jonathansick') file_names = c.listdir(vosdir) for name in file_names: print name if not name.endswith('.hdf5'): continue patch_name = os.path.splitext(os.path.basename(name))[0] if patch_name in dataset_patches: print "{0} already in dataset".format(patch_name) continue try: c.copy(os.path.join(vosdir, name), name) except: print "Could not download {0}".format(name) continue # open the patch's HDF5 file try: patch_data = h5py.File(name, 'r') except IOError: continue patch_serial = patch_data.attrs['patch'] patch_data.copy(patch_data, dataset_patches, name=patch_serial) patch_data.close() os.remove(name) validate_coords(dataset_patches) reduce_sfh_table(dataset, dataset_patches) dataset.flush() dataset.close()
def main(): ra = 287.58987 dec = -20.80109 rad = 10 / 3600.0 ra_rate = -2.41 / 3600.0 dec_rate = -0.22 / 3600.0 logging.basicConfig( level=logging.DEBUG, format= '%(relativeCreated)d %(pathname)s %(funcName)s %(lineno)d %(message)s', datefmt='%I:%M:%S') base_dir = "HSC_May25-lsst" rerun = "diff" pointing = "03071" filter = "HSC-R2" vos_dir = os.path.join(VOS_BASE_URI, base_dir, 'rerun', rerun, pointing, filter) c = vos.Client() logging.getLogger("urllib3").setLevel(logging.DEBUG) filenames = c.listdir(vos_dir) inputs = {} ref_date = None for filename in filenames: logging.debug(f'Reading {filename}') if filename not in inputs: inputs[filename] = {} inputs[filename]['uri'] = os.path.join(vos_dir, filename) inputs[filename]['header'] = get_fits_header(inputs[filename]['uri']) inputs[filename]['ref_date'] = exp_date(inputs[filename]['header']) if ref_date is None: ref_date = inputs[filename]['ref_date'] dt = (inputs[filename]['ref_date'] - ref_date).to('hour').value this_ra, this_dec = shift_ra_dec(ra, dec, ra_rate, dec_rate, ref_date, inputs[filename]['ref_date']) logging.debug( f'{filename} {ref_date} {dt} {ra} {dec} {this_ra} {this_dec}') inputs[filename]['hdulist'] = get_fits_cutout(inputs[filename]['uri'], this_ra, this_dec, rad) w = wcs.WCS(inputs[filename]['hdulist'][1].header) x, y = w.wcs_world2pix(ra, dec, 0) logging.debug(f'{filename} {ra},{dec}->{x},{y}') inputs[filename]['centre'] = x, y inputs[filename]['data'] = inputs[filename]['hdulist'][ 1].data / inputs[filename]['hdulist'][3].data if not os.access(filename, os.F_OK): inputs[filename]['hdulist'].writeto(filename) # fits.ImageHDU(data=stack_inputs(inputs)).writeto('stack.fits') peak = 0 for xoffset in numpy.arange(-10, 11, 1): for yoffset in numpy.arange(-10, 11, 1): for xrate in numpy.arange(-10, 11, 1): for yrate in numpy.arange(-10, 11, 1): current = metric(inputs, xoffset, yoffset, xrate, yrate, ref_date) if current > peak: solution = (xoffset, yoffset, xrate, yrate) peak = current diag_plot(inputs, solution, ref_date) logging.info( f'Best fit (xo, yo, xr, yr) -> {solution}: scale: {peak}' )
MAXCOUNT = 30000 CERTFILE = os.path.join(os.getenv('HOME'), '.ssl', 'cadcproxy.pem') DBIMAGES = 'vos:OSSOS/dbimages' MEASURE3 = 'vos:OSSOS/measure3' POSTAGE_STAMPS = 'vos:OSSOS/postage_stamps' DATA_WEB_SERVICE = 'https://www.canfar.phys.uvic.ca/data/pub/' VOSPACE_WEB_SERVICE = 'https://www.canfar.phys.uvic.ca/vospace/nodes/' TAP_WEB_SERVICE = 'http://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/tap/sync' OSSOS_TAG_URI_BASE = 'ivo://canfar.uvic.ca/ossos' OBJECT_COUNT = "object_count" vospace = vos.Client() SUCCESS = 'success' # ## some cache holders. mopheaders = {} astheaders = {} APCOR_EXT = "apcor" ZEROPOINT_USED_EXT = "zeropoint.used" PSF_EXT = "psf.fits" def cone_search(ra, dec, dra=0.01,
def get_flat_list(repo='vos:OSSOS/dbimages/calibrators'): vospace = vos.Client() flats = {} for filename in vospace.listdir(repo): flats[filename[0:5]] = filename return flats
import requests from io import StringIO from astropy.table import Table from astropy.io import ascii import tempfile import vos parser = argparse.ArgumentParser() parser.add_argument('-o', '--output', action="store_true", help="Store to file instead of mailing list") args = parser.parse_args() current_list = vos.Client().listdir('vos:OSSOS/dbimages') recon_exposures = Table.read(StringIO( requests.get(url, params=params).content), format='ascii.csv') print(recon_exposures) if args.output: with open('list.txt', 'w') as fout: cond = [str(g) not in current_list for g in recon_exposures['expnum']] ascii.write(recon_exposures[cond], fout, format='no_header') if not args.output: you = '*****@*****.**' you = '*****@*****.**' me = '*****@*****.**'
def starnet_download_file(filename): vclient = vos.Client() vclient.copy('vos:starnet/public/' + filename, datadir + filename) print(filename + ' downloaded')
def __init__(self, type="Component"): self.client = vos.Client("/cirada") # self.catalogues = client.listdir("vos:/cirada/data/EMU/catalogues") self.catalogues = list( client.iglob("vos:/cirada/data/EMU/catalogues/AS101*{type}*csv"))
def setUp(self): self.conn = Mock(spec=vos.Connection) # Set certFile to blank because person running this may or may not # have a certification file. self.undertest = vos.Client(vospace_certfile="", conn=self.conn)
def refresh_vos_client(self): """ If we have gotten a new certfile we have to create a new Client object before it will get used. """ self.vosclient = vos.Client(cadc_short_cut=True)
def run_transfer(self, transdir, filenames, dry_run=False): """ Function to launch data transfer. :param transdir: directory containing files to transfer. :param filenames: list of files to transfer. :param dry_run: specifies dry run mode. """ logger.debug('Constructing VOS client') vos_client = vos.Client() if not vos_client.isdir(self.vos_base): raise Exception('VOS base directory does not exist (or is file)') vos_cache = {} for file_ in (filenames): # Determine local file information. file_path = os.path.join(transdir, file_) file_md5 = get_md5sum(file_path) # Determine VO space information. vos_sub_dir = self.determine_vos_directory(transdir, file_) if vos_sub_dir is None: # This indicates that the file should be skipped. continue logger.debug('VOS directory for {0}: {1}'.format( file_path, vos_sub_dir)) vos_dir = '/'.join([self.vos_base, vos_sub_dir]) vos_file = '/'.join([vos_dir, file_]) # Get directory listing -- this creates the directory # if not in dry-run mode. if vos_dir in vos_cache: vos_dir_info = vos_cache[vos_dir] else: vos_dir_info = self.get_vos_directory_entries(vos_client, vos_dir, dry_run=dry_run) vos_cache[vos_dir] = vos_dir_info # Perform storage, if file changed (and not in dry-run mode). vos_md5 = vos_dir_info.get(file_, ()) if (vos_md5 != ()) and (vos_md5 == file_md5): logger.info('Skipped storing {0} as {1} [UNCHANGED]'.format( file_path, vos_file)) elif dry_run: logger.info('Skipped storing {0} as {1} [DRY-RUN]'.format( file_path, vos_file)) else: if vos_md5 != (): logger.debug('Deleting existing file {0}'.format(vos_file)) vos_client.delete(vos_file) logger.info('Storing {0} as {1}'.format(file_path, vos_file)) vos_client.copy(file_path, vos_file)
def resolve(pointing, chip, rate, angle, x, y, vos_basedir): """ :param pointing: pointing that source is found on :param chip: chip that source is found on :param rate: rate of motion that maximized source signal :param angle: angle of motion that maximized source signal :param x: x pixel value of source location in pointing/chip/rate/angle stack :param y: y pixel value :param vos_basedir: base location of VOSpace stacks, e.g. 'vos:NewHorizons/S20A-OT04/STACKS_V4' :return: stack, ra, dec, mjd """ # Resolve retrieves the headers from the VOSpace repo with the stacks. client = vos.Client() # Standard pattern used for DAOMOP Shift-and-Stack # Get a list of stack files and match to the one whose rate is closest to what we need. stack_dir_uri = f"{vos_basedir}/{pointing:05d}/{chip:03d}" stacks = client.listdir(stack_dir_uri) stack = None min_offset = None min_idx = None for idx, stack in enumerate(stacks): stack_angle = stack_rate = 0 try: match = STACK_PATTERN.search(stack) if match is not None: stack_rate = float(match.group('rate')) stack_angle = float(match.group('angle')) except Exception as ex: # logging.debug(f"Failed to match {stack}\n") logging.error(str(ex)) continue # use cos/sin rule to compute vector difference between requested rate and stack rate. da = math.radians(math.fabs(stack_angle - angle)) dr = rate**2 + stack_rate**2 - 2 * rate * stack_rate * math.cos(da) # select the minimum vector difference as the stack of interest. if min_offset is None or min_offset > dr: min_offset = dr min_idx = idx if min_idx is None: logging.error(f"No match for {pointing} {chip} {rate} {angle} {x} {y}") logging.error(f"Checked: {stacks} in {stack_dir_uri}") return None # Found the STACK in the VOSpace storage area that matches the pointing/chip/rate/angle of interest. # Now pull the 'cutout' with the WCS and compute the RA/DEC from the x/y location. stack_image = stacks[min_idx] logging.info( f"pointing:{pointing:05d}, chip:{chip:03d}, rate:{rate:5.1f}, angle:{angle:5.1f} " f"matched stack:{stack_image} in {vos_basedir}") uri = f"{stack_dir_uri}/{stack_image}" # noinspection PyBroadException # get the header of the matching stack and compute RA/DEC of the source using that header try: fobj = BytesIO(client.open(uri, cutout=CUTOUT, view='cutout').read()) with fits.open(fobj) as hdu: ra, dec = WCS(hdu[0].header).all_pix2world(x, y, 1) fobj = BytesIO(client.open(uri, cutout='[0]', view='cutout').read()) with fits.open(fobj) as hdu: epoch = hdu[0].header.get('MIDMJD', hdu[0].header.get('MJD-STR', -1.0)) except Exception as ex: logging.error( "Failed while measuring {pointing} {chip} {x} {y} on {stack}") logging.error("{ex}") return None return {'stack': stack_image, 'ra': ra, 'dec': dec, 'epoch': epoch}
def create_sky_plot(obstable, outfile, night_count=1, stack=True): """Given a VOTable that describes the observation coverage provide a PDF of the skycoverge. obstable: vostable.arrary stack: BOOL (true: stack all the observations in a series of plots) """ # camera dimensions width = 0.98 height = 0.98 if outfile[0:4] == 'vos:': tmpFile = tempfile.NamedTemporaryFile(suffix='.pdf') pdf = PdfPages(tmpFile.name) else: pdf = PdfPages(outfile) ra_min = obstable['RA'].max() + 1.5 ra_max = obstable['RA'].min() - 1.5 dec_min = obstable['DEC'].min() - 1.5 dec_max = obstable['DEC'].max() + 1.5 saturn = ephem.Saturn() subplots = [] t2 = None count = 0 fig = None for row in reversed(obstable.data): date = ephem.date(row.StartDate + 2400000.5 - ephem.julian_date(ephem.date(0))) sDate = str(date) saturn.compute(date) sra = math.degrees(saturn.ra) sdec = math.degrees(saturn.dec) t1 = time.strptime(sDate, "%Y/%m/%d %H:%M:%S") if t2 is None or (math.fabs(time.mktime(t2) - time.mktime(t1)) > 3 * 3600.0 and opt.stack): if fig is not None: pdf.savefig() close() fig = figure(figsize=(7, 2)) ax = fig.add_subplot(111, aspect='equal') ax.set_title("Data taken on %s-%s-%s" % (t1.tm_year, t1.tm_mon, t1.tm_mday), fontdict={'fontsize': 8}) ax.axis((245, 200, -20, 0)) ax.grid() ax.set_xlabel("RA (deg)", fontdict={'fontsize': 8}) ax.set_ylabel("DEC (deg)", fontdict={'fontsize': 8}) t2 = t1 ra = row.RA - width / 2.0 dec = row.DEC - height / 2.0 color = 'b' if 'W' in row['TargetName']: color = 'g' ax.add_artist( Rectangle(xy=(ra, dec), height=height, width=width, edgecolor=color, facecolor=color, lw=0.5, fill='g', alpha=0.33)) ax.add_artist( Rectangle(xy=(sra, sdec), height=0.3, width=0.3, edgecolor='r', facecolor='r', lw=0.5, fill='k', alpha=0.33)) if ax is not None: ax.axis((270, 215, -20, 0)) pdf.savefig() close() pdf.close() if outfile[0:4] == "vos:": vos.Client().copy(tmpFile.name, outfile) tmpFile.close() return
default=None, help="Distribution the output by header/chip?") parser.add_option("--ccd", action="store", default=36, type="int", help="CCD to process [do all be default]") parser.add_option("--dbimages", action="store", default="vos:OSSOS/dbimages", help="VOSpace location of the dbimages directory") ### get the bias frames from the archive. (opt, file_ids) = parser.parse_args() vos_client = vos.Client() if opt.combine and opt.normal and False: ## only take one image from each pointing t = {} for file_id in file_ids: filename = os.path.join(opt.dbimages, "%s/%so.head" % (file_id, file_id)) f = pyfits.open( StringIO(vos_client.open(filename, view='data').read())) t[f[0].header['OBJECT']] = file_id f.close() f = None file_ids = [] for object in t: file_ids.append(t[object])
phdr[0].header['UTEND'] = (lcdata[len(lcdata)-1][0],'End date of observations') bhdr = fits.BinTableHDU(data=lcdata.as_array()) phdulist = fits.HDUList([phdr[0],bhdr]) phdulist.writeto(os.path.join(os.getcwd(),os.path.basename(outfname)),clobber='True')#write locally first #now copy it to vospace vC.copy(os.path.join(os.getcwd(),os.path.basename(outfname)),outfname) print 'File written: '+outfname #cmd = 'getCert --daysValid=25 --dest=$HOME/.ssl/cadcproxy.pem' #os.system(cmd) vC = vos.Client() voMOST = 'vos://cadc.nrc.ca~vospace/MOST' voHome = os.path.join(voMOST,'most2caomlc') #vospace data server dws = 'http://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/data/pub' #Collect files; Read the list of directories from the 'lightcurve_data.txt'. #lclist = os.path.join(voHome,'resource/lightcurve_data.txt') lclistpath = os.path.join(voHome,'resource','lightcurve_data.txt') lclist = vC.open(lclistpath,mode='r',view='data') #with open(lclist,'r') as f:
import subprocess import os import vos import logging import urllib import errno from astropy.io import fits import urlparse CERTFILE = os.path.join(os.getenv('HOME'), '.ssl', 'cadcproxy.pem') DBIMAGES = 'vos:OSSOS/dbimages' DATA_WEB_SERVICE = 'https://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/data/pub/' OSSOS_TAG_URI_BASE = 'ivo://canfar.uvic.ca/ossos' vospace = vos.Client(cadc_short_cut=True, certFile=CERTFILE) SUCCESS = 'success' def populate(dataset_name, data_web_service_url=DATA_WEB_SERVICE + "CFHT"): """Given a dataset_name created the desired dbimages directories and links to the raw data files stored at CADC.""" data_dest = get_uri(dataset_name, version='o', ext='fits.fz') data_source = "%s/%so.fits.fz" % (data_web_service_url, dataset_name) c = vospace try: c.mkdir(os.path.dirname(data_dest)) except IOError as e: