def table_from_scratch(): from astropy.io.votable.tree import VOTableFile, Resource, Table, Field # Create a new VOTable file... votable = VOTableFile() # ...with one resource... resource = Resource() votable.resources.append(resource) # ... with one table table = Table(votable) resource.tables.append(table) # Define some fields table.fields.extend([ Field(votable, ID="filename", datatype="char"), Field(votable, ID="matrix", datatype="double", arraysize="2x2")]) # Now, use those field definitions to create the numpy record arrays, with # the given number of rows table.create_arrays(2) # Now table.array can be filled with data table.array[0] = ('test1.xml', [[1, 0], [0, 1]]) table.array[1] = ('test2.xml', [[0.5, 0.3], [0.2, 0.1]]) # Now write the whole thing to a file. # Note, we have to use the top-level votable file object out = io.StringIO() votable.to_xml(out)
def test_exec_sync(): # save results in a file # create the VOTable result # example from http://docs.astropy.org/en/stable/io/votable/ votable = VOTableFile() resource = Resource() votable.resources.append(resource) table = Table(votable) resource.tables.append(table) table.fields.extend([ Field(votable, name="filename", datatype="char", arraysize="*"), Field(votable, name="matrix", datatype="double", arraysize="2x2")]) table.create_arrays(2) table.array[0] = ('test1.xml', [[1, 0], [0, 1]]) table.array[1] = ('test2.xml', [[0.5, 0.3], [0.2, 0.1]]) buffer = BytesIO() votable.to_xml(buffer) cadc = Cadc(auth_session=requests.Session()) response = Mock() response.to_table.return_value = buffer.getvalue() cadc.cadctap.search = Mock(return_value=response) output_file = '{}/test_vooutput.xml'.format(tempfile.tempdir) cadc.exec_sync('some query', output_file=output_file) actual = parse(output_file) assert len(votable.resources) == len(actual.resources) == 1 assert len(votable.resources[0].tables) ==\ len(actual.resources[0].tables) == 1 actual_table = actual.resources[0].tables[0] try: # TODO remove when astropy LTS upgraded from astropy.utils.diff import report_diff_values assert report_diff_values(table, actual_table, fileobj=sys.stdout) except ImportError: pass
def createTableFromObject(data, path="", names=[], dtypes=[], sizes=[]): path_tmp = "/Users/cjimenez/Documents/PHD/data/tmp/" # Create a new VOTable file... votable = VOTableFile() # ...with one resource... resource = Resource() votable.resources.append(resource) # ... with one table table = Table(votable) resource.tables.append(table) # Define some fields fields = [] for idx, val in enumerate(names): fields.append(Field(votable, name=val, datatype=dtypes[idx])) table.fields.extend(fields) # Now, use those field definitions to create the numpy record arrays, with # the given number of rows table.create_arrays(len(data)) # Now table.array can be filled with data for idx, val in enumerate(data): table.array[idx] = val # Now write the whole thing to a file. # Note, we have to use the top-level votable file object votable.to_xml(path_tmp + path)
def exportData( dataTuple ,name): dataK, labelK, data_errorK, dataL, labelL = dataTuple # concatenate all arrays from dataTuple and add labels for errors data_full = np.concatenate( (np.concatenate( ( dataK, data_errorK ) , axis = 1 ), dataL), axis = 1 ) error_label = [] for i in labelK: error_label += [i + "_error"] label_full = labelK + error_label + labelL votable = VOTableFile() resource = Resource() votable.resources.append(resource) table = Table(votable) resource.tables.append(table) fields = [] for i in range( data_full.shape[1] ): fields += [Field( votable, name = label_full[i], datatype='float' )] table.fields.extend( fields ) table.create_arrays( data_full.shape[0] ) for i in range( data_full.shape[0] ): table.array[i] = tuple( data_full[i,:] ) votable.to_xml(name +".xml")
def votable_xml_string(version): votable_file = VOTableFile(version=version) votable_file.resources.append(Resource()) xml_bytes = io.BytesIO() votable_file.to_xml(xml_bytes) xml_bytes.seek(0) bstring = xml_bytes.read() s = bstring.decode("utf-8") return s
def _run_test_from_scratch_example(): from astropy.io.votable.tree import VOTableFile, Resource, Table, Field # Create a new VOTable file... votable = VOTableFile() # ...with one resource... resource = Resource() votable.resources.append(resource) # ... with one table table = Table(votable) resource.tables.append(table) # Define some fields table.fields.extend([ Field(votable, name="filename", datatype="char", arraysize="*"), Field(votable, name="matrix", datatype="double", arraysize="2x2")]) # Now, use those field definitions to create the numpy record arrays, with # the given number of rows table.create_arrays(2) # Now table.array can be filled with data table.array[0] = ('test1.xml', [[1, 0], [0, 1]]) table.array[1] = ('test2.xml', [[0.5, 0.3], [0.2, 0.1]]) assert table.array[0][0] == 'test1.xml'
def write_set(self, filename, votype='ascii', overwrite=False): ''' Write all tables to a VOT file Required Arguments: *filename*: [ string ] The VOT file to write the tables to Optional Keyword Arguments: *votype*: [ 'ascii' | 'binary' ] Whether to write the tables as ASCII or binary tables ''' if os.path.exists(filename): if overwrite: os.remove(filename) else: raise Exception("File exists: %s" % filename) vo_table = VOTableFile() resource = Resource() vo_table.resources.append(resource) for table_key in self.tables: resource.tables.append(_to_table(self.tables[table_key], vo_table)) if votype is 'binary': vo_table.get_first_table().format = 'binary' vo_table.set_all_tables_format('binary') vo_table.to_xml(filename)
def format_votable(self): """ Return ------- Return data in VOTable format. """ names_column = ['obj','ra','dec','mjd','mag','magerr','filter','catalog'] descriptions_column = ['Id of object in catalog the original catalog', 'Right ascension of source','Declination of source', 'Julian Day','Magnitude','Magnitude Error', 'Filter code','Original Catalog'] #dtype_column = [] # dtype=dtype_column t = Table(rows=self.data_np,names=names_column,descriptions=descriptions_column) votable= VOTableFile.from_table(t) buf = io.BytesIO() writeto(votable,buf) return buf.getvalue().decode("utf-8")
def _votablefile(): table = Table( [[23, 42, 1337], [b'Illuminatus', b"Don't panic, and always carry a towel", b'Elite']], names=('1', '2')) table['1'].meta['ucd'] = 'foo;bar' table['2'].meta['utype'] = 'foobar' votablefile = VOTableFile.from_table(table) info = Info(name='QUERY_STATUS', value='OK') info.content = 'OK' votablefile.resources[0].infos.append(info) return votablefile
def empty_vo(rtype='results'): """ Generate an empty VO Table with one resource Parameters ---------- rtype : str, optional Type of resouuce Returns ------- evotable : VOTable """ evotable = VOTableFile() resource = Resource(type=rtype) evotable.resources.append(resource) return evotable
def test_version(): """ VOTableFile.__init__ allows versions of '1.0', '1.1', '1.2', '1.3' and '1.4'. The '1.0' is curious since other checks in parse() and the version setter do not allow '1.0'. This test confirms that behavior for now. A future change may remove the '1.0'. """ # Exercise the checks in __init__ with pytest.warns(AstropyDeprecationWarning): VOTableFile(version='1.0') for version in ('1.1', '1.2', '1.3', '1.4'): VOTableFile(version=version) for version in ('0.9', '2.0'): with pytest.raises( ValueError, match=r"should be in \('1.0', '1.1', '1.2', '1.3', '1.4'\)."): VOTableFile(version=version) # Exercise the checks in the setter vot = VOTableFile() for version in ('1.1', '1.2', '1.3', '1.4'): vot.version = version for version in ('1.0', '2.0'): with pytest.raises( ValueError, match=r"supports VOTable versions '1.1', '1.2', '1.3', '1.4'$" ): vot.version = version # Exercise the checks in the parser. begin = b'<?xml version="1.0" encoding="utf-8"?><VOTABLE version="' middle = b'" xmlns="http://www.ivoa.net/xml/VOTable/v' end = b'" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><RESOURCE/></VOTABLE>' # Valid versions for bversion in (b'1.1', b'1.2', b'1.3'): parse(io.BytesIO(begin + bversion + middle + bversion + end), verify='exception') parse(io.BytesIO(begin + b'1.4' + middle + b'1.3' + end), verify='exception') # Invalid versions for bversion in (b'1.0', b'2.0'): with pytest.warns(W21): parse(io.BytesIO(begin + bversion + middle + bversion + end), verify='exception')
def __spectrum_to_votable(self, spectrum): # Create a new VOTable file... votable = VOTableFile() # ...with one resource... resource = Resource() votable.resources.append(resource) # ... with one table table = Table(votable) resource.tables.append(table) # Define some fields waveobs = Field(votable, name="waveobs", datatype="double", unit="nm", ucd="em.wl") flux = Field(votable, name="flux", datatype="double", unit="Jy", ucd="phot.flux") err = Field(votable, name="err", datatype="double", ucd="stat.error;phot.flux") table.fields.extend([waveobs, flux, err]) table.groups.extend([Group([flux, err])]) #import ipdb #ipdb.set_trace() # Now, use those field definitions to create the numpy record arrays, with # the given number of rows table.create_arrays(len(spectrum)) # Now table.array can be filled with data table.array['waveobs'] = spectrum['waveobs'] table.array['flux'] = spectrum['flux'] table.array['err'] = spectrum['err'] #votable.set_all_tables_format('binary') # VOSpec does not understand binary format return votable
def votablefile_dataset(): table = Table([[ 'image/fits', 'application/x-votable+xml', 'application/x-votable+xml;content=datalink' ], [ b'http://example.com/querydata/image.fits', b'http://example.com/querydata/votable.xml', b'http://example.com/querydata/votable-datalink.xml' ]], names=('dataformat', 'dataurl')) table['dataformat'].meta['ucd'] = 'meta.code.mime' table['dataurl'].meta['utype'] = 'Access.Reference' table['dataurl'].meta['ucd'] = 'meta.dataset;meta.ref.url' votablefile = VOTableFile.from_table(table) info = Info(name='QUERY_STATUS', value='OK') info.content = 'OK' votablefile.resources[0].infos.append(info) return votablefile
def VOGetCapabilities(filename, outfile): votable = VOTableFile() parser = ElementTree.XMLParser(recover=True) f = codecs.open(filename, 'r', 'utf-8') string = f.read() string = bytes(bytearray( string, encoding='utf-8')) ## force utf-8 encoding even if xml says otherwise root = ElementTree.fromstring(string, parser) for element in root: res = element.tag resid = str(res).split('}')[-1:][ 0] # MM, note that we have to convert res to string first # logger.debug(resid) ## debug # if (res!=ElementTree.Comment): if (resid != str(ElementTree.Comment)): # MM # resid=re.sub(':','_',res) resid = re.sub(':', '_', resid) resource = Resource(name=res, ID=resid) votable.resources.append(resource) table = Table(votable) resource.tables.append(table) for param in element: par = param.tag parid = str(par).split('}')[-1:][0] # MM value = param.text if value is None: listvalue = param.items() value = listvalue dt = type(value).__name__ if (dt == "str"): dt = "char" arraysize = "*" table.params.extend([ # Param(votable, name=par, datatype=dt, value=value)]) Param(votable, name=parid, datatype=dt, value=value) ]) elif (dt == "list"): if (len(value) > 0): for tup in value: table.params.extend([ # Param(votable, name=tup[0], datatype="char", value=tup[1])]) Param(votable, name=tup[0].split('}')[-1:][0], datatype="char", value=tup[1]) ]) # if (par=="Layer"): if (parid == "Layer"): # lresource = Resource(name=par,ID=par) lresource = Resource(name=par, ID=parid) # MM votable.resources.append(lresource) ltable = Table(votable) lresource.tables.append(ltable) j = 0 data = [] for layer in param: lay = layer.tag layid = str(lay).split('}')[-1:][0] # MM datalay = [] # if (lay=="Layer"): if (layid == "Layer"): # MM for field in layer: fieldid = str(field.tag).split('}')[-1:][0] if (j == 0): ltable.fields.extend([ # Field(votable, name=field.tag, datatype="char", arraysize="*")]) Field(votable, name=fieldid, datatype="char", arraysize="*") ]) else: try: # ltable.get_field_by_id_or_name(field.tag) ltable.get_field_by_id_or_name(fieldid) except: ltable.fields.extend([ # Field(votable, name=field.tag, datatype="char", arraysize="*")]) Field(votable, name=fieldid, datatype="char", arraysize="*") ]) if field.text == None: field.text = "Empty" datalay.append(field.text) try: i = i + 1 except: i = 0 j = 1 if datalay != []: data.append(datalay) l = len(ltable.fields) # nl = int(math.ceil((i)/l)+1) #this will only work if there are no nested layers # print(nl) # so i'm commenting this out and replacing with simple len(data) nl = len(data) #MM # logger.debug(nl) ## debug dim = nl * l - 1 for x in range(0, nl): while len(data[x]) < l: data[x].append('Empty') try: ltable.create_arrays(dim) ltable.array = (np.ma.asarray(data, dtype='str')) ltable.array.mask = False except: raise votable.to_xml(outfile)
fields = tab.fields col_names = table.dtype.names print table.dtype[1] print len(table[sys.argv[2]]) print len(table[sys.argv[3]]) i_r = removeDupes (table[sys.argv[2]], table[sys.argv[3]]) print np.array(table[sys.argv[2]])[i_r] print i_r from astropy.io.votable.tree import VOTableFile, Resource, Table, Field # Create a new VOTable file... votable = VOTableFile() # ...with one resource... resource = Resource() votable.resources.append(resource) # ... with one table table_out = Table(votable) resource.tables.append(table_out) table_out.fields.extend(fields) table_out.create_arrays(len(i_r)) for n in col_names: table_out.array[n] = table[n][i_r]
def visibility(request): now = datetime.datetime.now() html = "<html><body>It is now %s.</body></html>" % now # Create a new VOTable file... votable = VOTableFile() # ...with one resource... resource = Resource() votable.resources.append(resource) # ... with one table table = Table(votable) resource.tables.append(table) resource.description ="European Space Astronomy Centre. INTEGRAL SOC - " \ "Object Visibility Simple Access Protocol (ObjVisSAP)" resource.infos.append(Info(name="QUERY_STATUS", value="OK")) resource.infos.append(Info(name="SERVICE PROTOCOL", value="1.0")) resource.infos.append(Info(name="REQUEST", value="queryData")) resource.infos.append( Info(name="s_ra", value="%s" % request.GET.get("s_ra"))) resource.infos.append( Info(name="s_dec", value="%s" % request.GET.get("s_dec"))) resource.infos.append( Info(name="t_min", value="%s" % request.GET.get("t_min"))) resource.infos.append( Info(name="t_max", value="%s" % request.GET.get("t_max"))) # Define some fields # table.fields.extend([ # Field(votable, name="filename", datatype="char", arraysize="*"), # Field(votable, name="matrix", datatype="double", arraysize="2x2")]) table.fields.extend([ Field(votable, name="t_start", datatype="double", ucd="time.start", utype="Char.TimeAxis.Coverage.Bounds.Limits.StartTime"), Field(votable, name="t_stop", datatype="double", ucd="time.start", utype="Char.TimeAxis.Coverage.Bounds.Limits.StartTime"), Field(votable, name="t_visibility", datatype="double", ucd="time.start", utype="Char.TimeAxis.Coverage.Bounds.Limits.StartTime"), ]) results = VisibilityCalculator.getVisibilityIntervals( request.GET.get("s_ra"), request.GET.get("s_dec"), request.GET.get("t_min"), request.GET.get("t_max")) number_of_intervals = len(results) table.create_arrays(number_of_intervals) for i in range(0, number_of_intervals): table.array[i] = (results[i][0], results[i][1], results[i][2]) # Now write the whole thing to a file to be streamed # Note, we have to use the top-level votable file object xml_now = "/tmp/new_votable_%s.xml" % now votable.to_xml(xml_now) stream = open(xml_now).read() os.remove(xml_now) return HttpResponse(stream, content_type='text/xml')
ColumnNameList.append(Fieldname) Description = Text[34:] DescriptionList.append(Description) if Section == 5: # Row data Row = [] for c in range(0, len(StartPos)): #print (ColumnName[c],Text[int(StartPos[c])-1:int(FinishPos[c])]) Result = Text[int(StartPos[c]) - 1:int(FinishPos[c])] Row.append(Result) # Write row to CSV file Output.writerow(Row) List.append(Row) votable = VOTableFile() #coosys=CooSys(ID="J2000", equinox="J2000", system="eq_FK5") #votable.coordinate_systems.append(coosys) resource = Resource() votable.resources.append(resource) table = Table(votable) resource.tables.append(table) table.params.extend([ Param(votable, name="imageFile", ucd="meta.file;meta.fits", datatype="char", arraysize="255", value="atlas-cdfs.fits"), Param(votable, name="Reference frequency",
raise RuntimeError('Error reading input file ' + args.input_fits.name + ' as FITS.') from exc # Right now we only know how to handle a very specific file format from the EXES instrument. # Not entirely clear how to test for it up front. if len(hdulist) != 1: print(sys.argv[0], 'can only process FITS files with a single, primary HDU.', file=sys.stderr) sys.exit(1) # Process the FITS headers into a <GROUP> in the first <TABLE>. hdr = hdulist[0].header # Set up the main structure of the VOTable file: vt = VOTableFile() # Create a single <RESOURCE> in the file r = Resource() vt.resources.append(r) # Place an empty (for now) <TABLE> in the file t = Table(vt) r.tables.append(t) # Create the special <GROUP> for the raw FITS headers g = make_fits_header_group(hdr, t) t.groups.append(g) # Access the actual file data. We can only have a primary HDU - this means it must be an IMAGE HDU. pixel_data = hdulist[0].data
def save_table(table, tabname): results = VOTableFile() resource = Resource() results.resources.append(resource) resource.tables.append(from_table(table).get_first_table()) results.to_xml(tabname)
def test_data_file(self): wh().get().RESULTS_PATH = os.path.dirname(os.path.realpath(__file__)) test_log("Testing the initialization...", self) job_id = 1 fname = "test_data.dat" model = results_model.objects.filter(job_id=job_id) self.assertFalse(model) d = data_file(1) model = results_model.objects.filter(job_id=job_id) self.assertEqual(len(model), 1) test_log("Testing a simple file creation...", self) file_name = os.path.join(wh().get().RESULTS_PATH, fname) f = d.file(fname) f.write("test") f.close() model = results_model.objects.filter(job_id=job_id) self.assertEqual(wh().get().RESULTS_PATH, model[0].resources.all().filter(name=fname)[0].path) self.assertEqual(fname, model[0].resources.all().filter(name=fname)[0].name) self.assertTrue(os.path.isfile(file_name)) f = open(file_name) fdata = f.read() f.close() self.assertEqual(fdata, "test") os.remove(file_name) test_log("Testing a plot addition...", self) plot = plot_model.objects.create(name="test_plot", job_id=job_id, alg_name="test", script="", html="") self.assertFalse(model[0].plots.all().filter(name="test_plot")) d.add_plot(plot) self.assertEqual(len(model[0].plots.all().filter(name="test_plot")), 1) test_log("Testing a FITS file storage...", self) n = np.arange(100.0) hdu = fits.PrimaryHDU(n) hdul = fits.HDUList([hdu]) d.save_fits(fname, hdul) self.assertEqual(len(model[0].resources.all().filter(name=fname)), 2) self.assertTrue(os.path.isfile(file_name)) os.remove(file_name) test_log("Testing a VOTable file storage...", self) votable = VOTableFile() resource = Resource() votable.resources.append(resource) table = Table(votable) resource.tables.append(table) table.fields.extend([ Field(votable, name="filename", datatype="char", arraysize="*"), Field(votable, name="matrix", datatype="double", arraysize="2x2") ]) table.create_arrays(2) table.array[0] = ('test_1', [[1, 0], [0, 1]]) table.array[1] = ('test_2', [[0.5, 0.3], [0.2, 0.1]]) d.save_vot(fname, votable) self.assertEqual(len(model[0].resources.all().filter(name=fname)), 3) self.assertTrue(os.path.isfile(file_name)) os.remove(file_name)
#Comparing with external catalogues. externalsurveys = [] firstsurvey, nvsssurvey, wensssurvey, sumsssurvey = survey('FIRST'), survey( 'NVSS'), survey('WENSS'), survey('SUMSS') firstdata = external_cat(firstsurvey, catalogue, FIRSTcompare) nvssdata = external_cat(nvsssurvey, catalogue, NVSScompare) wenssdata = external_cat(wensssurvey, catalogue, WENSScompare) sumssdata = external_cat(sumsssurvey, catalogue, SUMSScompare) for i in firstdata, nvssdata, wenssdata, sumssdata: if len(i.rawcatdata) > 0: externalsurveys.append(i) #VO output.. if keepvo > 0: votable = VOTableFile() resource = Resource() votable.resources.append(resource) resource.tables.append( catalogue.makevo(votable, description=telescope + ' Data')) for surveydata in externalsurveys: resource.tables.append( surveydata.makevo(votable, description=surveydata.survey.name + ' Data')) votable.to_xml(options.outputdir + '/' + inname + '.vo') outfiles.append(options.outputdir + '/' + inname + '.vo') print '**** vo catalogue containg all available source lists is in ' + inname + '.vo' #Make the required plots for each survey. for surveydata in externalsurveys: outfiles = outfiles + surveydata.docomparison(options.outputdir + '/' +
def sort_write(sortname, spect, fitsdict, filesort, space=3): """ Write out an xml and ascii file that contains the details of the file sorting. By default, the filename is printed first, followed by the filetype. After these, all parameters listed in the 'keyword' item in the settings file will be printed Parameters ---------- sortname : str The filename to be used to save the list of sorted files spect : dict Properties of the spectrograph. fitsdict : dict Contains relevant information from fits header files filesort : dict Details of the sorted files space : int Keyword to set how many blank spaces to place between keywords """ msgs.info("Preparing to write out the data sorting details") nfiles = fitsdict['filename'].size # Specify which keywords to print after 'filename' and 'filetype' prord = ['filename', 'frametype', 'target', 'exptime', 'naxis0', 'naxis1', 'filter1', 'filter2'] prdtp = ["char", "char", "char", "double", "int", "int", "char", "char"] # Now insert the remaining keywords: fkey = spect['keyword'].keys() for i in fkey: if i not in prord: prord.append(i) # Append the type of value this keyword holds typv = type(fitsdict[i][0]) if typv is int or typv is np.int_: prdtp.append("int") elif isinstance(fitsdict[i][0], basestring) or typv is np.string_: prdtp.append("char") elif typv is float or typv is np.float_: prdtp.append("double") else: msgs.bug("I didn't expect useful headers to contain type {0:s}".format(typv).replace('<type ', '').replace('>', '')) # Open a VOTable for writing votable = VOTableFile() resource = Resource() votable.resources.append(resource) table = Table(votable) resource.tables.append(table) # Define VOTable fields tabarr=[] # Insert the filename and filetype first for i in xrange(len(prord)): tabarr.append(Field(votable, name=prord[i], datatype=prdtp[i], arraysize="*")) table.fields.extend(tabarr) table.create_arrays(nfiles) filtyp = filesort.keys() for i in xrange(nfiles): values = () for pr in prord: if pr == 'frametype': addval = "" for ft in filtyp: if i in filesort[ft]: if len(addval) != 0: addval += "," addval += ft addval = (addval,) else: addval = (fitsdict[pr][i],) values = values + addval table.array[i] = values #osspl = sortname.split('.') #if len(osspl) > 1: # fname = sortname #else: fname = sortname+'.xml' votable.to_xml(fname) msgs.info("Successfully written sorted data information file:"+msgs.newline() + "{0:s}".format(fname)) # ASCII file (JXP) jxpord = ['filename', 'date', 'frametype', 'target', 'exptime', 'binning', 'dichroic', 'disperser', 'cdangle', 'decker'] # Generate the columns clms = [] for pr in jxpord: try: lidx = prord.index(pr) except ValueError: msgs.warn('{:s} keyword not used'.format(pr)) else: clm = [] for i in xrange(nfiles): clm.append(table.array[i][lidx]) clms.append(Column(clm, name=pr)) # Create Table jxp_tbl = tTable(clms) # Write jxp_name = fname.replace('.xml', '.lst') jxp_tbl.write(jxp_name, format='ascii.fixed_width') return