def import_data(ID, setup='omicron'):
db = Cinfdata(setup, use_caching=True)
data = db.get_data(ID)
try:
data[:,0] = data[:,0]/1000
except:
meta = db.get_metadata(ID)
print('"{}" data for ID {} is empty and set to "None"'.format(meta['mass_label'], ID))
data = None
return data
def __init__(self, timestamp, caching=True):
"""Save all data in 'DATA' dict object"""
# Connect to database
db = Cinfdata('omicron',
use_caching=caching,
grouping_column='time',
)
# Get data in unit seconds
group_data = db.get_data_group(timestamp, scaling_factors=(1E-3, None))
group_meta = db.get_metadata_group(timestamp)
# Get a list of labels and group data
self.name = group_meta[list(group_meta.keys())[0]]['Comment']
self.labels = [group_meta[key]['mass_label'] for key in group_data.keys()]
self.data = {group_meta[key]['mass_label']: group_data[key] for key in group_data.keys()}
print('Loaded data from Experiment: "{}"'.format(self.name))
class MainCarousel(Carousel):
"""Main cinfdata class.
Contains the current settings and provides an interface to the cinfdata
information
"""
def __init__(self, **kwargs):
super(MainCarousel, self).__init__(**kwargs)
# Initiate cinfdata and bind properties
self.cinfdata = Cinfdata(self, creds.username, creds.password)
# Add a reference to cinfdata to page selection
self.ids.page_selection.cinfdata = self.cinfdata
self.ids.main_image.cinfdata = self.cinfdata
# Initiate date plot options and add cinfdata reference
#self.dateplot_options = DatePlotOptions()
#self.dateplot_options.cinfdata = self.cinfdata
self.waiting_png = CoreImage('data/waiting.png')
def on_index(self, *args):
"""Update image when switcing into the middle carousel part
on_index is called the caroudel is turned into a new position
"""
super(MainCarousel, self).on_index(*args)
Logger.debug('MainCarousel: index: {}'.format(args[1]))
if args[1] == 1:
# First time on_index is called, there is no cinfdata object yet
if not hasattr(self, 'cinfdata'):
print("no cindata yet")
return
# Also we switch back to
if self.cinfdata.selected_plot is None:
return
self.ids.main_image.update_image(self.waiting_png)
Clock.schedule_once(self._get_image_and_update)
def _get_image_and_update(self, _):
"""Get image from cinfdata and update main image"""
data = self.cinfdata.get_plot()
self.ids.main_image.update_image(data)
def change_plot(self, setup_and_link):
"""Change the plot settings widget when a new plot is selected"""
#Logger.debug("Change plot %s", setup_and_link[0])
#Logger.debug("Change plot %s", setup_and_link[1])
_, link = setup_and_link
if link['pagetype'] == 'dateplot':
self.ids.plot_settings.clear_widgets()
dateplot_options = DatePlotOptions(setup_and_link)
dateplot_options.cinfdata = self.cinfdata
self.ids.plot_settings.add_widget(dateplot_options)
else:
message = 'Support for the \'{}\' plot type is not yet implemented'\
.format(link['pagetype'])
raise NotImplementedError(message)
def _after_password(self, username, password, _):
"""After password has been received, initialize cinfdata"""
# Initiate cinfdata and bind properties
self.cinfdata = Cinfdata(self, username, password)
# FIXME, check password somehow and think about how to come
# back to _after_init
# Add a reference to cinfdata to page selection
self.ids.page_selection.cinfdata = self.cinfdata
self.ids.main_image.cinfdata = self.cinfdata
def __init__(self, timestamp, caching=True):
"""Save all data in 'DATA' dict object"""
# Connect to database
db = Cinfdata(
'omicron',
use_caching=caching,
grouping_column='time',
)
# Get data in unit seconds
group_data = db.get_data_group(timestamp, scaling_factors=(1E-3, None))
group_meta = db.get_metadata_group(timestamp)
# Get a list of labels and group data
self.name = group_meta[list(group_meta.keys())[0]]['Comment']
self.labels = [
group_meta[key]['mass_label'] for key in group_data.keys()
]
self.data = {
group_meta[key]['mass_label']: group_data[key]
for key in group_data.keys()
}
print('Loaded data from Experiment: "{}"'.format(self.name))
def __init__(self, **kwargs):
super(MainCarousel, self).__init__(**kwargs)
# Initiate cinfdata and bind properties
self.cinfdata = Cinfdata(self, creds.username, creds.password)
# Add a reference to cinfdata to page selection
self.ids.page_selection.cinfdata = self.cinfdata
self.ids.main_image.cinfdata = self.cinfdata
# Initiate date plot options and add cinfdata reference
#self.dateplot_options = DatePlotOptions()
#self.dateplot_options.cinfdata = self.cinfdata
self.waiting_png = CoreImage('data/waiting.png')
class MainCarousel(Carousel):
"""Main cinfdata class.
Contains the current settings and provides an interface to the cinfdata
information
"""
def __init__(self, **kwargs):
super(MainCarousel, self).__init__(**kwargs)
self.waiting_png = CoreImage('data/waiting.png')
Clock.schedule_once(self._after_init)
def _after_init(self, _):
"""Get password"""
username = password = None
if creds is not None:
username, password = creds.username, creds.password
# Implement using password managers
if username is None:
# username, password = password.get_creds()
pass
# If we git creds from creds module or password managers, set callback
if username is not None:
callback = partial(self._after_password, username, password)
Clock.schedule_once(callback)
return
# Otherwise, ask for them (PasswordOpen will call _after_password)
pw = PasswordPopup(self)
pw.open()
def _after_password(self, username, password, _):
"""After password has been received, initialize cinfdata"""
# Initiate cinfdata and bind properties
self.cinfdata = Cinfdata(self, username, password)
# FIXME, check password somehow and think about how to come
# back to _after_init
# Add a reference to cinfdata to page selection
self.ids.page_selection.cinfdata = self.cinfdata
self.ids.main_image.cinfdata = self.cinfdata
def on_index(self, *args):
"""Update image when switcing into the middle carousel part
on_index is called the caroudel is turned into a new position
"""
super(MainCarousel, self).on_index(*args)
Logger.debug('MainCarousel: index: {}'.format(args[1]))
if args[1] == 1:
# First time on_index is called, there is no cinfdata object yet
if not hasattr(self, 'cinfdata'):
print("no cindata yet")
return
# Also we switch back to
if self.cinfdata.selected_plot is None:
return
self.ids.main_image.update_image(self.waiting_png)
Clock.schedule_once(self._get_image_and_update)
def _get_image_and_update(self, _):
"""Get image from cinfdata and update main image"""
data = self.cinfdata.get_plot()
self.ids.main_image.update_image(data)
def change_plot(self, setup_and_link):
"""Change the plot settings widget when a new plot is selected"""
#Logger.debug("Change plot %s", setup_and_link[0])
#Logger.debug("Change plot %s", setup_and_link[1])
_, link = setup_and_link
if link['pagetype'] == 'dateplot':
self.ids.plot_settings.clear_widgets()
dateplot_options = DatePlotOptions(setup_and_link)
dateplot_options.cinfdata = self.cinfdata
self.ids.plot_settings.add_widget(dateplot_options)
else:
message = 'Support for the \'{}\' plot type is not yet implemented'\
.format(link['pagetype'])
raise NotImplementedError(message)
from cinfdata import Cinfdata
db = Cinfdata('stm312', use_caching=True)
spectrum = db.get_data(6688)
metadata = db.get_metadata(6688)
'tof_R1_voltage': None,
'sem_voltage': 1798.83,
u'unixtime': 1464261944L,
'type': 4L,
'pass_energy': None,
'time': datetime.datetime(2016, 5, 26, 13, 25, 44),
'sample_temperature': None,
'tof_focus_voltage': None,
'tof_pulse_voltage': None,
'tof_emission_current': None,
'tof_deflection_voltage': None,
'tof_p1_2': None,
'tof_p1_0': None,
'tof_p1_1': None,
'tof_lens_E': None,
'tof_lens_D': None,
'tof_lens_A': None,
'tof_lens_C': None,
'tof_lens_B': None
}
cinfdata = Cinfdata('tof', use_caching=True)
# Get from database
assert np.isclose(cinfdata.get_data(5417).sum(), DATA_CHECKSUM)
assert cinfdata.get_metadata(5417) == METADATA
print('Test database OK')
# And now fetch from cache
assert np.isclose(cinfdata.get_data(5417).sum(), DATA_CHECKSUM)
assert cinfdata.get_metadata(5417) == METADATA
print('Test cache OK')
remaining = target_number - integral
time_left = remaining / dep_rate
time_hr = int(time_left / 3600)
time_min = int((time_left - time_hr * 3600) / 60)
time_sec = int(time_left - time_hr * 3600 - time_min * 60)
print(msg.format(target_coverage, time_hr, time_min, time_sec))
################################################################
### MAIN ###
################################################################
if __name__ == '__main__':
#import rcparam
from cinfdata import Cinfdata
db = Cinfdata('stm312', use_caching=False) # pylint: disable=invalid-name
ID = 13602
STRING = '\
TARGET=10.0;\
MODEL=NP;\
SA_DENSITY=1;\
LEAK_CURRENT=1e-12;\
PARTICLE_DIAMETER=1.3;\
APERTURE_DIAMETER=7;\
TIME=[0,9200]'
try:
SESSION = IntegrateCurrent(STRING, db.get_data(ID))
SESSION.integrate()
if SESSION.plot:
SESSION.plt.show()
from cinfdata import Cinfdata
from matplotlib import pyplot as plt
db = Cinfdata('stm312')
spectrum = db.get_data(6688)
metadata = db.get_metadata(6688)
plt.plot(spectrum[:, 0], spectrum[:, 1])
plt.title(metadata['Comment'])
plt.show()
self.dbg2.axvline(x=self.data[offset + counter, 0],
color='r')
break
self.std2 = self.data[1:offset + counter, 1].std()
print('STDs: ', std_gradient, self.std2)
return std_gradient
################################################################
### MAIN ###
################################################################
if __name__ == '__main__':
#import rcparam
from cinfdata import Cinfdata
db = Cinfdata('omicron', use_caching=False) # pylint: disable=invalid-name
#ID = 18851
#ID = 12299 # NP NiFe
ID = 18982 # GG 29
#ID = 17601 # SA1
#ID = 17585 # SA2
ID = 14273 # 70% 6nm Pt
STRING = '\
TARGET=5.0;\
MODEL=NP;\
SA_DENSITY=1.58426e19;\
PARTICLE_DIAMETER=6.0;\
APERTURE_DIAMETER=12.41;\
FIRST_LIMIT=20;\
SENSITIVITY_LIMIT=.7;SENSITIVITY_FILTER=1.;\
TIME=[];\
mode = 'plot'
#mode = 'save'
settings = {
#9489: 220,
#9494: 400,
#9504: 1500,
#9507: 800,
#9509: 1000,
#9512: 600,
#9516: 1000,
#9518: 500,
#9527: 600,
9545: 350,
}
db = Cinfdata('dummy', use_caching=False)
for ID, LEVEL in settings.items():
data = db.get_data(ID)
time, signal = data[:, 0], data[:, 1]
smooth = ct.smooth(signal)
diff = np.abs(np.diff(signal-smooth))
index = np.where(diff > LEVEL)[0] + 1
i = np.where(time > 0)[0]
if mode in ['test', 'plot']:
plt.title(str(ID))
plt.plot(data[:, 0], data[:, 1], 'b-', label='Raw data')
plt.plot(time[index], signal[index], 'mo', markerfacecolor='w')
import datetime
import numpy as np
from cinfdata import Cinfdata
DATA_CHECKSUM = 12525.0000008
METADATA = {
'comment': '', 'tof_liner_voltage': None, 'tof_iterations': None,
'timestep': 0.1, 'id': 5417L, 'preamp_range': -7, 'tof_pulse_width': None,
'pre_wait_time': None, 'tof_ion_energy': None, 'tof_R2_voltage': None,
'mass_label': None, 'tof_R1_voltage': None, 'sem_voltage': 1798.83,
u'unixtime': 1464261944L, 'type': 4L, 'pass_energy': None,
'time': datetime.datetime(2016, 5, 26, 13, 25, 44),
'sample_temperature': None, 'tof_focus_voltage': None,
'tof_pulse_voltage': None, 'tof_emission_current': None,
'tof_deflection_voltage': None, 'tof_p1_2': None, 'tof_p1_0': None,
'tof_p1_1': None, 'tof_lens_E': None, 'tof_lens_D': None,
'tof_lens_A': None, 'tof_lens_C': None, 'tof_lens_B': None
}
cinfdata = Cinfdata('tof', use_caching=True)
# Get from database
assert np.isclose(cinfdata.get_data(5417).sum(), DATA_CHECKSUM)
assert cinfdata.get_metadata(5417) == METADATA
print('Test database OK')
# And now fetch from cache
assert np.isclose(cinfdata.get_data(5417).sum(), DATA_CHECKSUM)
assert cinfdata.get_metadata(5417) == METADATA
print('Test cache OK')
def download_cinfdata_set(setup='sniffer',
group_id=None,
grouping_column=None,
**kwargs):
if grouping_column is None:
grouping_column, group_id = kwargs.popitem()
from .Combining import synchronize
try:
from cinfdata import Cinfdata
except ImportError:
print(
'the cinfdata module must be on your python path. It\'s here: \n' +
'https://github.com/CINF/cinf_database/blob/master/cinfdata.py')
try:
cinfd = Cinfdata(setup,
grouping_column=grouping_column,
allow_wildcards=True,
label_column='mass_label')
except:
raise #untill I know exactly which error I'm trying to catch.
print('couldn\'t connect. You should run gstm')
#os.system('gstm')
raise RuntimeError('Couldn\'t connect to cinfdata!')
#obj = cinfd.get_metadata_group('2018-03-30 14:13:17')
#all_datasets = cinfd.get_metadata_group('%')
#the_list = [(ID, d['time'], d['comment']) for ID, d in all_datasets.items()]
#print(the_list)
obj = cinfd.get_metadata_group(group_id)
#print(str(obj)) #
idlists = {
} # keys will be time as string. values will be corresponding id's
for key, value in obj.items():
#label = value['mass_label']
#print(label)
timestamp = str(value['time'])
if timestamp not in idlists:
idlists[timestamp] = []
idlists[timestamp] += [value['id']]
datasets = {}
for timestamp, idlist in idlists.items():
if len(idlist) == 0:
print('No data associated with timestamp \'' + timestamp + '\'.')
continue
dataset = {'title': timestamp, 'data_type': 'MS'}
metadatas = dict([(i, cinfd.get_metadata(i)) for i in idlist])
unixtimes = [metadatas[i]['unixtime'] for i in idlist]
if len(set(unixtimes)) > 1:
msg = 'unix times don\'t match for timestamp \'' + timestamp + '\'!'
raise ValueError(msg)
dataset['tstamp'] = unixtimes[0]
dataset['timestamp'] = metadatas[idlist[0]]['time'].strftime(
'%H:%M:%S')
labels = [metadatas[i]['mass_label'] for i in idlist]
if 'Mass Scan' in labels:
dataset['scan_type'] = 'mass'
else:
dataset['scan_type'] = 'time'
dataset['data_cols'] = set()
dataset['timecols'] = {}
for i in idlist: #avoiding id since it's got a builtin meaning
data = cinfd.get_data(i)
label = metadatas[i]['mass_label']
if len(data.shape) == 1:
dataset[label] = data
dataset['data_cols'].add(label)
elif data.shape[1] == 2:
x = data[:, 0]
y = data[:, 1]
x_label = label + '-x'
y_label = label + '-y'
dataset['timecols'][x_label] = y_label
dataset[x_label] = x * 1e-3 # cinfdata saves time in ms!!!
dataset[y_label] = y
dataset['data_cols'].add(x_label)
dataset['data_cols'].add(y_label)
datasets[timestamp] = dataset
timescans = [
dataset for dataset in datasets.values()
if dataset['scan_type'] == 'time'
]
combined = synchronize(timescans, t_zero='first')
return combined
def download_cinfdata_set(
setup="sniffer", group_id=None, grouping_column=None, **kwargs
):
if grouping_column is None:
grouping_column, group_id = kwargs.popitem()
from .Combining import synchronize
try:
from cinfdata import Cinfdata
except ImportError:
print(
"the cinfdata module must be on your python path. It's here: \n"
+ "https://github.com/CINF/cinf_database/blob/master/cinfdata.py"
)
try:
cinfd = Cinfdata(
setup,
grouping_column=grouping_column,
allow_wildcards=True,
label_column="mass_label",
)
except:
raise # untill I know exactly which error I'm trying to catch.
print("couldn't connect. You should run gstm")
# os.system('gstm')
raise RuntimeError("Couldn't connect to cinfdata!")
# obj = cinfd.get_metadata_group('2018-03-30 14:13:17')
# all_datasets = cinfd.get_metadata_group('%')
# the_list = [(ID, d['time'], d['comment']) for ID, d in all_datasets.items()]
# print(the_list)
obj = cinfd.get_metadata_group(group_id)
# print(str(obj)) #
idlists = {} # keys will be time as string. values will be corresponding id's
for key, value in obj.items():
# label = value['mass_label']
# print(label)
timestamp = str(value["time"])
if timestamp not in idlists:
idlists[timestamp] = []
idlists[timestamp] += [value["id"]]
datasets = {}
for timestamp, idlist in idlists.items():
if len(idlist) == 0:
print("No data associated with timestamp '" + timestamp + "'.")
continue
dataset = {"title": timestamp, "data_type": "MS"}
metadatas = dict([(i, cinfd.get_metadata(i)) for i in idlist])
unixtimes = [metadatas[i]["unixtime"] for i in idlist]
if len(set(unixtimes)) > 1:
msg = "unix times don't match for timestamp '" + timestamp + "'!"
raise ValueError(msg)
dataset["tstamp"] = unixtimes[0]
dataset["timestamp"] = metadatas[idlist[0]]["time"].strftime("%H:%M:%S")
labels = [metadatas[i]["mass_label"] for i in idlist]
if "Mass Scan" in labels:
dataset["scan_type"] = "mass"
else:
dataset["scan_type"] = "time"
dataset["data_cols"] = set()
dataset["timecols"] = {}
for i in idlist: # avoiding id since it's got a builtin meaning
data = cinfd.get_data(i)
label = metadatas[i]["mass_label"]
if len(data.shape) == 1:
dataset[label] = data
dataset["data_cols"].add(label)
elif data.shape[1] == 2:
x = data[:, 0]
y = data[:, 1]
x_label = label + "-x"
y_label = label + "-y"
dataset["timecols"][y_label] = x_label # Fixed 20B26!!!
dataset[x_label] = x * 1e-3 # cinfdata saves time in ms!!!
dataset[y_label] = y
dataset["data_cols"].add(x_label)
dataset["data_cols"].add(y_label)
datasets[timestamp] = dataset
timescans = [
dataset for dataset in datasets.values() if dataset["scan_type"] == "time"
]
combined = synchronize(timescans, t_zero="first")
return combined
class MainCarousel(Carousel):
"""Main cinfdata class.
Contains the current settings and provides an interface to the cinfdata
information
"""
def __init__(self, **kwargs):
super(MainCarousel, self).__init__(**kwargs)
self.waiting_png = CoreImage('data/waiting.png')
Clock.schedule_once(self._after_init)
def _after_init(self, _):
"""Get password"""
username = password = None
if creds is not None:
username, password = creds.username, creds.password
# Implement using password managers
if username is None:
# username, password = password.get_creds()
pass
# If we git creds from creds module or password managers, set callback
if username is not None:
callback = partial(self._after_password, username, password)
Clock.schedule_once(callback)
return
# Otherwise, ask for them (PasswordOpen will call _after_password)
pw = PasswordPopup(self)
pw.open()
def _after_password(self, username, password, _):
"""After password has been received, initialize cinfdata"""
# Initiate cinfdata and bind properties
self.cinfdata = Cinfdata(self, username, password)
# FIXME, check password somehow and think about how to come
# back to _after_init
# Add a reference to cinfdata to page selection
self.ids.page_selection.cinfdata = self.cinfdata
self.ids.main_image.cinfdata = self.cinfdata
def on_index(self, *args):
"""Update image when switcing into the middle carousel part
on_index is called the caroudel is turned into a new position
"""
super(MainCarousel, self).on_index(*args)
Logger.debug('MainCarousel: index: {}'.format(args[1]))
if args[1] == 1:
# First time on_index is called, there is no cinfdata object yet
if not hasattr(self, 'cinfdata'):
print("no cindata yet")
return
# Also we switch back to
if self.cinfdata.selected_plot is None:
return
self.ids.main_image.update_image(self.waiting_png)
Clock.schedule_once(self._get_image_and_update)
def _get_image_and_update(self, _):
"""Get image from cinfdata and update main image"""
data = self.cinfdata.get_plot()
self.ids.main_image.update_image(data)
def change_plot(self, setup_and_link):
"""Change the plot settings widget when a new plot is selected"""
#Logger.debug("Change plot %s", setup_and_link[0])
#Logger.debug("Change plot %s", setup_and_link[1])
_, link = setup_and_link
if link['pagetype'] == 'dateplot':
self.ids.plot_settings.clear_widgets()
dateplot_options = DatePlotOptions(setup_and_link)
dateplot_options.cinfdata = self.cinfdata
self.ids.plot_settings.add_widget(dateplot_options)
else:
message = 'Support for the \'{}\' plot type is not yet implemented'\
.format(link['pagetype'])
raise NotImplementedError(message)
