def test_fetch_separate_filenames():
# Setup
db = Database('sqlite:///')
download_query = [
vso.attrs.Time('2012-08-05', '2012-08-05 00:00:05'),
vso.attrs.Instrument('AIA')
]
tmp_test_dir = os.path.join(sunpy.config.get('downloads', 'download_dir'),
'tmp_test_dir/')
if not os.path.isdir(tmp_test_dir):
os.mkdir(tmp_test_dir)
path = tmp_test_dir + '{file}'
db.fetch(*download_query, path=path)
# Test
assert len(db) == 4
dir_contents = os.listdir(tmp_test_dir)
assert 'aia_lev1_335a_2012_08_05t00_00_02_62z_image_lev1.fits' in dir_contents
assert 'aia_lev1_94a_2012_08_05t00_00_01_12z_image_lev1.fits' in dir_contents
assert os.path.isfile(
os.path.join(tmp_test_dir,
'aia_lev1_335a_2012_08_05t00_00_02_62z_image_lev1.fits'))
assert os.path.isfile(
os.path.join(tmp_test_dir,
'aia_lev1_94a_2012_08_05t00_00_01_12z_image_lev1.fits'))
# Teardown
shutil.rmtree(tmp_test_dir)
def test_fetch_separate_filenames():
# Setup
db = Database("sqlite:///")
download_query = [vso.attrs.Time("2012-08-05", "2012-08-05 00:00:05"), vso.attrs.Instrument("AIA")]
tmp_test_dir = os.path.join(sunpy.config.get("downloads", "download_dir"), "tmp_test_dir/")
if not os.path.isdir(tmp_test_dir):
os.mkdir(tmp_test_dir)
path = tmp_test_dir + "{file}"
db.fetch(*download_query, path=path)
# Test
assert len(db) == 8
dir_contents = os.listdir(tmp_test_dir)
assert "aia_lev1_335a_2012_08_05t00_00_02_62z_image_lev1.fits" in dir_contents
assert "aia_lev1_94a_2012_08_05t00_00_01_12z_image_lev1.fits" in dir_contents
assert os.path.isfile(os.path.join(tmp_test_dir, "aia_lev1_335a_2012_08_05t00_00_02_62z_image_lev1.fits"))
assert os.path.isfile(os.path.join(tmp_test_dir, "aia_lev1_94a_2012_08_05t00_00_01_12z_image_lev1.fits"))
# Teardown
shutil.rmtree(tmp_test_dir)
def __init__(self):
self._view = QtWidgets.QWidget()
self._ui = Ui_DataManager()
self._ui.setupUi(self._view)
self._ui.refresh_button.setIcon(self._view.style().standardIcon(
QStyle.SP_BrowserReload))
self._ui.remove_button.setIcon(self._view.style().standardIcon(
QStyle.SP_DialogNoButton))
self._ui.add_button.setIcon(self._view.style().standardIcon(
QStyle.SP_DialogYesButton))
self.dlg = DataManagerFilterDialog()
db = QtSql.QSqlDatabase.addDatabase('QSQLITE')
db.setDatabaseName(
sunpy.config.get("database", "url").replace("sqlite:///", ""))
model = QtSql.QSqlTableModel()
model.setTable("data")
model.setEditStrategy(QtSql.QSqlTableModel.OnFieldChange)
model.select()
self._ui.data_table.setModel(model)
self.initTableHeader(model)
self._ui.add_button.clicked.connect(lambda x: self.onAdd())
self._ui.remove_button.clicked.connect(lambda x: self.onRemove())
self._ui.open_button.clicked.connect(lambda x: self.onOpen())
self._ui.refresh_button.clicked.connect(lambda x: self.model.select())
self._ui.filter_button.clicked.connect(lambda x: self.onFilter())
self.sunpy_db = Database()
self.model = model
def test_fetch_separate_filenames():
# Setup
db = Database('sqlite:///')
download_query = [
vso.attrs.Time('2012-08-05', '2012-08-05 00:00:05'),
vso.attrs.Instrument('AIA')
]
tmp_test_dir = os.path.join(
sunpy.config.get('downloads', 'download_dir'),
'tmp_test_dir/'
)
if not os.path.isdir(tmp_test_dir):
os.makedirs(tmp_test_dir)
path = tmp_test_dir + '{file}'
db.fetch(*download_query, path=path)
# Test
assert len(db) == 4
dir_contents = os.listdir(tmp_test_dir)
assert 'aia_lev1_335a_2012_08_05t00_00_02_62z_image_lev1.fits' in dir_contents
assert 'aia_lev1_94a_2012_08_05t00_00_01_12z_image_lev1.fits' in dir_contents
assert os.path.isfile(os.path.join(tmp_test_dir, 'aia_lev1_335a_2012_08_05t00_00_02_62z_image_lev1.fits'))
assert os.path.isfile(os.path.join(tmp_test_dir, 'aia_lev1_94a_2012_08_05t00_00_01_12z_image_lev1.fits'))
# Teardown
shutil.rmtree(tmp_test_dir)
def filled_database():
database = Database('sqlite:///:memory:')
for i in xrange(1, 11):
entry = DatabaseEntry()
database.add(entry)
# every fourth entry gets the tag 'foo'
if i % 4 == 0:
database.tag(entry, 'foo')
# every fifth entry gets the tag 'bar'
if i % 5 == 0:
database.tag(entry, 'bar')
database.commit()
return database
def test_config_url(monkeypatch):
monkeypatch.setattr("sunpy.config", ConfigParser.SafeConfigParser())
url = 'sqlite:///'
sunpy.config.add_section('database')
sunpy.config.set('database', 'url', url)
database = Database()
assert database.url == url
def filled_database():
database = Database('sqlite:///:memory:')
for i in range(1, 11):
entry = DatabaseEntry()
database.add(entry)
# every fourth entry gets the tag 'foo'
if i % 4 == 0:
database.tag(entry, 'foo')
# every fifth entry gets the tag 'bar'
if i % 5 == 0:
database.tag(entry, 'bar')
database.commit()
return database
def __init__(self):
self._view = QtWidgets.QWidget()
self._ui = Ui_DownloadResult()
self._ui.setupUi(self._view)
self._ui.tabs.clear()
self._ui.tabs.tabCloseRequested.connect(self._onRemoveTab)
self.database = Database()
self.tabs = {}
self.queries = {}
self.query_id = 0
self.loading = []
self.loaded = {entry.fileid: entry.path for entry in list(self.database)}
self._ui.download_button.clicked.connect(lambda evt: self._onDownloadSelected())
self._ui.open_button.clicked.connect(lambda evt: self._onOpenSelected())
def connect_db(self, conn_string=''):
'''
This function creates a database connection based on the parameters
specified in the GUI.
Parameters: conn_string (string of the connection for the object) Optional
'''
def_wavelength = self.default_wavelength.get_widget().currentText()
try:
if conn_string == '':
conn_string = sunpy.config.get('database', 'url')
global database
database = Database(conn_string, default_waveunit=def_wavelength)
except:
conn_string = self.get_conn_string()
global database
database = Database(conn_string, default_waveunit=def_wavelength)
if self.set_default_box.get_state():
self.set_default_db(conn_string)
self.set_info("Database Connected at %s" % conn_string)
return database
def split_function_database():
"""
Generates a custom database to test the split_database function
"""
database = Database('sqlite:///:memory:')
for i in range(1, 11):
entry = DatabaseEntry()
database.add(entry)
# every fourth entry gets the instrument 'EIA'
if i % 4 == 0:
database.edit(entry, instrument='EIA')
# every fifth entry gets the instrument 'AIA_3'
elif i % 5 == 0:
database.edit(entry, instrument='AIA_3')
# every other entry gets instrument 'RHESSI'
else:
database.edit(entry, instrument='RHESSI')
# all entries have provider 'xyz'
database.edit(entry, provider='xyz')
database.commit()
return database
def test_create_display_table_empty_db():
with pytest.raises(TypeError):
_create_display_table(Database('sqlite:///'), ['id'])
from sunpy.net import vso
from sunpy.database import Database
database = Database("sqlite:///")
database.download(vso.attrs.Time("2012-08-05", "2012-08-05 00:00:05"),
vso.attrs.Instrument('AIA'))
len(database)
from sunpy.database.tables import display_entries
print display_entries(database,
["id", "observation_time_start", "wavemin", "wavemax"])
hmiC.plot(cmap=cmap, annotate=False)
plt.clim(0,300)
plt.show()
# Now compare your awesome image with the HMI Intensitygram on https://sdo.gsfc.nasa.gov/data/.
# Intensitygrams show how bright the sun is over it's surface. You'll also see a flattened version on the SDO website.
# The flattened image takes into account the fact that most of the light from the sides of the sun as we see it is not directed towards us.
# This makes the sun look darker near the edges like in our image. This effect is called limb darkening.
## Multiwavelength Images
# We know that the sun emits many wavelengths of electromagnetic radiation lets get some images for the different parts of the EM spectrum.
# We've already seen a continuum or white light image of the sun that we colourised at the begining of the Notebook.
# To store our new data we are going to start a database which wil help us manage the observations we download:
db = Database('sqlite:///sunpydata.sqlite')
db.default_waveunit = 'angstrom'
# You'll notice a file called sunpydata.sqlite has appeared in our directory.
# This is our database. Let's download something to fill it.
# Science ready data from spacecraft is not available instantly so we will have to get information from a while back.
result = Fido.search(a.Time('2019/05/06 12:00', '2019/05/06 12:01'),a.Instrument('aia'),a.vso.Sample(2*u.minute),a.Wavelength(94*u.angstrom)|a.Wavelength(131*u.angstrom)|a.Wavelength(171*u.angstrom)|a.Wavelength(193*u.angstrom)|a.Wavelength(211*u.angstrom)|a.Wavelength(304*u.angstrom)|a.Wavelength(335*u.angstrom)|a.Wavelength(1600*u.angstrom)|a.Wavelength(4500*u.angstrom))
print(result)
# You should see a load of tables with the results from our search.
# Each table is for a different wavelength we specified in Angstrom (10E-10 m).
# We searched images from the SDO instrument AIA, and set a.vso.Sample() greater than the time period we searched.
# This meant we only get one image for each wavelength in that time period (which is only 2 minutes to begin with!).
# We can now download the data.
from sunpy.physics.differential_rotation import _calc_P_B0_SD
import matplotlib.pyplot as plt
plt.ioff()
import matplotlib.animation as an
import pandas as pd
import pickle
from glob import glob
from sunpy.database import Database
from sunpy.net import vso
from datetime import timedelta
stereo_ttest = timedelta(minutes=60)
#set up the database of stereo files
db = Database('sqlite:///sunpydb.sqlite', default_waveunit=u.AA)
# read in the stereo files
files = glob('/storage2/EUVI/preped/*.fts')
# read in the eis pointing data
eis_pdata = ascii.read("eis_pointing_table.txt")
eis_pdata['XCEN'].unit = u.arcsec
eis_pdata['YCEN'].unit = u.arcsec
eis_pdata['FOVX'].unit = u.arcsec
eis_pdata['FOVY'].unit = u.arcsec
eis_pdata.sort('DATE_OBS')
times = [sunpy.time.parse_time(tt.strip()) for tt in eis_pdata['DATE_OBS']]
def database():
return Database('sqlite:///:memory:')
def database_using_lfucache():
return Database('sqlite:///:memory:', LFUCache, cache_size=3)
class DownloadResultController(ToolController):
queries = {}
app_ctrl: AppController = RequiredFeature(AppController.__name__)
content_ctrl: ContentController = RequiredFeature(ContentController.name)
def __init__(self):
self._view = QtWidgets.QWidget()
self._ui = Ui_DownloadResult()
self._ui.setupUi(self._view)
self._ui.tabs.clear()
self._ui.tabs.tabCloseRequested.connect(self._onRemoveTab)
self.database = Database()
self.tabs = {}
self.queries = {}
self.query_id = 0
self.loading = []
self.loaded = {
entry.fileid: entry.path
for entry in list(self.database)
}
self._ui.download_button.clicked.connect(
lambda evt: self._onDownloadSelected())
self._ui.open_button.clicked.connect(
lambda evt: self._onOpenSelected())
def query(self, attrs):
# open tool if not already opened
self.app_ctrl.openController(self.name)
self.query_id += 1
# add pending tab
tab = ResultTab(self.query_id)
self.tabs[self.query_id] = tab
index = self._ui.tabs.addTab(tab, "Query " + str(self.query_id))
self._ui.tabs.setCurrentIndex(index)
# start query
executeTask(Fido.search, attrs, self._onQueryResult, [self.query_id])
# register events
tab.download.connect(
lambda f_id, q_id=self.query_id: self.download(q_id, f_id))
tab.open.connect(lambda f_id: self._onOpen(f_id))
def _onQueryResult(self, query, id):
if id not in self.tabs:
return
query_model = self._convertQuery(query)
self.tabs[id].loadQuery(query_model)
self.tabs[id].setLoading(self.loading)
self.tabs[id].setLoaded(self.loaded.keys())
self.queries[id] = query
def download(self, q_id, f_id):
req = copy.copy(self.queries[q_id])
req._list = [copy.copy(r) for r in req]
for resp in req:
resp[:] = [item for item in resp if item.fileid == f_id]
self._addLoading([f_id])
executeTask(Fido.fetch, [req], self._onDownloadResult, [f_id, req])
def _onDownloadResult(self, paths, f_id, request):
path = paths[0]
entry = list(
tables.entries_from_fido_search_result(
request, self.database.default_waveunit))[0]
entry.path = path
self.database.add(entry)
self.database.commit()
self._addLoaded({f_id: path})
def _onOpen(self, f_id):
viewer = MapViewerController.fromFile(self.loaded[f_id])
self.content_ctrl.addViewerController(viewer)
def _onRemoveTab(self, index):
tab = self._ui.tabs.widget(index)
self._ui.tabs.removeTab(index)
self.tabs.pop(tab.q_id)
def _onDownloadSelected(self):
tab = self._ui.tabs.currentWidget()
f_ids = tab.getSelectedFIds()
for f_id in f_ids:
if f_id in self.loading or f_id in self.loaded:
continue
self.download(tab.q_id, f_id)
def _onOpenSelected(self):
tab = self._ui.tabs.currentWidget()
f_ids = tab.getSelectedFIds()
for f_id in f_ids:
if f_id not in self.loaded:
continue
self._onOpen(f_id)
def _convertQuery(self, query):
items = [item for response in query for item in response]
return [[c[1](item) for c in columns] for item in items]
def _addLoading(self, f_ids):
self.loading.extend(f_ids)
for tab in self.tabs.values():
tab.setLoading(f_ids)
def _addLoaded(self, dict):
self.loading = [
f_id for f_id in self.loading if f_id not in dict.keys()
]
self.loaded.update(dict)
for tab in self.tabs.values():
tab.setLoaded(dict.keys())
@property
def item_config(self) -> ItemConfig:
return ItemConfig().setTitle("Download Results").setOrientation(
QtCore.Qt.BottomDockWidgetArea)
@property
def view(self) -> QtWidgets:
return self._view
def split_function_database():
"""
Generates a custom database to test the split_database function
"""
database = Database('sqlite:///:memory:')
for i in range(1, 11):
entry = DatabaseEntry()
database.add(entry)
# every fourth entry gets the instrument 'EIA'
if i % 4 == 0:
database.edit(entry, instrument='EIA')
# every fifth entry gets the instrument 'AIA_3'
elif i % 5 == 0:
database.edit(entry, instrument='AIA_3')
# every other entry gets instrument 'RHESSI'
else:
database.edit(entry, instrument='RHESSI')
# all entries have provider 'xyz'
database.edit(entry, provider='xyz')
database.commit()
return database
def default_waveunit_database():
unit_database = Database('sqlite:///:memory:',
default_waveunit=units.meter)
str_database = Database('sqlite:///:memory:', default_waveunit="m")
return unit_database, str_database
from sunpy.net import vso
from sunpy.database import Database
database = Database("sqlite:///")
database.download(
vso.attrs.Time("2012-08-05", "2012-08-05 00:00:05"),
vso.attrs.Instrument('AIA'))
len(database)
from sunpy.database.tables import display_entries
print display_entries(
database,
["id", "observation_time_start", "wavemin", "wavemax"])
class DataManagerController(ToolController):
content_ctrl: ContentController = RequiredFeature(ContentController.name)
def __init__(self):
self._view = QtWidgets.QWidget()
self._ui = Ui_DataManager()
self._ui.setupUi(self._view)
self._ui.refresh_button.setIcon(self._view.style().standardIcon(
QStyle.SP_BrowserReload))
self._ui.remove_button.setIcon(self._view.style().standardIcon(
QStyle.SP_DialogNoButton))
self._ui.add_button.setIcon(self._view.style().standardIcon(
QStyle.SP_DialogYesButton))
self.dlg = DataManagerFilterDialog()
db = QtSql.QSqlDatabase.addDatabase('QSQLITE')
db.setDatabaseName(
sunpy.config.get("database", "url").replace("sqlite:///", ""))
model = QtSql.QSqlTableModel()
model.setTable("data")
model.setEditStrategy(QtSql.QSqlTableModel.OnFieldChange)
model.select()
self._ui.data_table.setModel(model)
self.initTableHeader(model)
self._ui.add_button.clicked.connect(lambda x: self.onAdd())
self._ui.remove_button.clicked.connect(lambda x: self.onRemove())
self._ui.open_button.clicked.connect(lambda x: self.onOpen())
self._ui.refresh_button.clicked.connect(lambda x: self.model.select())
self._ui.filter_button.clicked.connect(lambda x: self.onFilter())
self.sunpy_db = Database()
self.model = model
def initTableHeader(self, model):
header = self._ui.data_table.horizontalHeader()
for i in range(model.columnCount()):
header.setSectionResizeMode(i,
QtWidgets.QHeaderView.ResizeToContents)
header.setContextMenuPolicy(QtCore.Qt.CustomContextMenu)
header.customContextMenuRequested.connect(self.onHeaderMenu)
self._ui.data_table.setContextMenuPolicy(QtCore.Qt.CustomContextMenu)
self._ui.data_table.customContextMenuRequested.connect(
self.onHeaderMenu)
self._ui.data_table.hideColumn(0)
self._ui.data_table.hideColumn(2)
self._ui.data_table.hideColumn(3)
self._ui.data_table.hideColumn(4)
self._ui.data_table.hideColumn(8)
self._ui.data_table.hideColumn(11)
self._ui.data_table.hideColumn(12)
self._ui.data_table.hideColumn(13)
self._ui.data_table.hideColumn(14)
def onAdd(self):
paths, _ = QtWidgets.QFileDialog.getOpenFileNames(
None, filter="FITS files (*.fits; *.fit; *.fts)")
for p in paths:
self.sunpy_db.add_from_file(p)
self.sunpy_db.commit()
self.model.select()
def onRemove(self):
rows = set([i.row() for i in self._ui.data_table.selectedIndexes()])
for r in rows:
self.model.removeRow(r)
self.model.submitAll()
self.model.select()
def onOpen(self):
rows = set([i.row() for i in self._ui.data_table.selectedIndexes()])
paths = [
self.model.index(row, self.model.fieldIndex("path")).data()
for row in rows
]
for path in paths:
viewer_ctrl = MapViewerController.fromFile(path)
self.content_ctrl.addViewerController(viewer_ctrl)
def onFilter(self):
if self.dlg.exec_():
self.model.setFilter(self.dlg.getFilter())
self.model.select()
def onHeaderMenu(self, point):
menu = QMenu(self._view)
actions = []
for column in range(self.model.columnCount()):
label = self.model.headerData(column, QtCore.Qt.Horizontal)
action = QtWidgets.QAction(label)
action.setCheckable(True)
action.setChecked(not self._ui.data_table.isColumnHidden(column))
event = lambda checked, c=column: self._ui.data_table.showColumn(
c) if checked else self._ui.data_table.hideColumn(c)
action.triggered.connect(event)
actions.append(action)
menu.addActions(actions)
menu.exec_(QCursor.pos())
@property
def item_config(self) -> ItemConfig:
return ItemConfig().setMenuPath("File/Data Manager").setTitle(
"Data Manager")
@property
def view(self) -> QWidget:
self.model.select()
return self._view
# This is how it would query sunpy the data mapped in 'gathering_data_information.txt'
# At this time the vso client seems unstable and sunpy integration could not be complete
from sunpy.database import Database
from sunpy.net import vso
#Sets the connection to database, in this case, it is in postgresql but it can be others too
database = Database('postgresql+psycopg2://user:password@host/dbname')
#downloads and add to database
def downloadData(q_provider, q_source, q_instrument, q_physobs):
client = vso.VSOClient()
qr = client.search(
vso.attrs.Time('2016-05-07 00:00:00', '2016-05-08 00:00:00'))
print(qr)
database.add_from_vso_query_result(qr)
database.commit()
#downloads MAgnetogram vector from vso client
def dl_MagnetogramVector():
q_provider = 'JSOC'
q_source = 'SDO'
q_instrument = 'HMI'
q_physobs = 'vector_magnetic_field'
downloadData(q_provider, q_source, q_instrument, q_physobs)
#downloads XRay data from vso client
def dl_XRay():
def test_config_url_none(monkeypatch):
monkeypatch.setattr("sunpy.config", ConfigParser.SafeConfigParser())
with pytest.raises(ConfigParser.NoSectionError):
Database()
def test_display_entries_empty_db():
with pytest.raises(TypeError):
display_entries(Database('sqlite:///'), ['id'])
from sunpy.time import TimeRange, parse_time
from sunpy.net import hek, Fido, attrs as a
from bokeh.embed import file_html
from bokeh.resources import CDN
from sunpy.database import Database
from sunpy.instr.goes import flux_to_flareclass
import astropy.units as u
from flask import Flask, render_template, request, make_response
from authomatic.adapters import WerkzeugAdapter
from authomatic import Authomatic
from config import CONFIG
database = Database('sqlite:///sunpydata.sqlite')
class Server():
def __init__(self):
print("Start server")
self.printers = []
self.printer_class = {}
self.db = pymongo.MongoClient('localhost', 27017)['SunAlert']
self.authomatic = Authomatic(CONFIG,
'your secret string',
report_errors=False)
self.app = Flask('SunAlert')
