def __init__(self, doc):
############### Initiating Variables ###################
self.doc = doc
self.uploaded_projects_path = join(dirname(__file__),"..","data","uploaded_projects")
self.extracted_projects_path = join(dirname(__file__),"..","data","extracted_projects")
self.current_uploaded_file = "No file loaded"
self.current_extracted_file = "No Project"
self.full_uploaded_project_path = "empty"
self.full_extracted_project_path = "empty"
self.download_file_path = "No file to download"
self.tables = []
############### Creating Controls ###################
#self.fileinput = FileInput(accept=".xer", name="fileinput", css_classes=["myinput"])
self.fileinput = FileInput(accept=".xer", name="fileinput", css_classes=["myinput"])
self.textarea = TextAreaInput(value="XER file contents will show here!", rows = 10, name="textarea", css_classes=["mytext"])
self.extract = Button(label="Extract Data", name="extract", css_classes=["mybutton"])
self.download = Div(text="""No file to download""", name="download", css_classes=["mylink"])
self.panel_tabs = self.init_panels()
############### Defining Responses ###################
self.fileinput.on_change('value', self.upload_init)
self.extract.on_click(self.extract_init)
############### Exporting Controls ###################
self.doc.add_root(self.fileinput)
self.doc.add_root(self.textarea)
self.doc.add_root(self.extract)
self.doc.add_root(self.download)
self.doc.add_root(self.panel_tabs)
def __init__(self):
# extend __init__ super class
IPlugin.__init__(self)
# instantiate main components
self.tx = TxData()
self.prod_master = ProductMaster()
self.cust_master = CustomerMaster()
self.quote = Quote()
self.price_model = PriceModel()
# cds_plot lives here rather than quote because they draw on several main components
self.cds_plot = ColumnDataSource(data=dict(x=[0], y=[0], attr_label=[''], attr_name=['']))
# plot_filters lives here because it's drawing from several main components
self.plot_filters = [MultiSelect(title='Product_filter', value=[''], options=[''])]
self.plot_filters[0].on_change('value', self.update_plot)
# configuration backup settings
self.backup_loc = TextInput(value=getcwd())
self.backup_config_button = Button(label='Output config', button_type='success')
self.backup_config_button.on_click(self.output_config)
self.config_data_input = FileInput(accept=".json")
self.config_data_input.on_change('value', self.load_config)
self.datafile_selector = RadioButtonGroup(labels=['Tx data', 'Product master', 'Customer master'], active=0)
self.datafile_selector.on_change('active', self.refresh_view)
# define and initialize views (and their layout attributes)
self.view_quote = QuoteView(self)
self.view_price_mgt = PriceMgtView(self)
self.view_price_model_config = PriceModelConfigView(self)
self.view_attr_settings = AttrSettingsView(self)
self.view_settings = SettingsView(self)
self.tabs = Tabs(
tabs=[Panel(child=self.view_quote.layout, title='Quote'),
Panel(child=self.view_price_mgt.layout, title='Price Management'),
Panel(child=self.view_price_model_config.layout, title='Price adjustments'),
Panel(child=self.view_attr_settings.layout, title='Attribute Settings'),
Panel(child=self.view_settings.layout, title='Settings')])
# define some on_change events
self.price_model.price_group_selector.on_change('active', self.refresh_view)
self.price_model.num_prod_groups.on_change('value', self.refresh_view, self.update_on_change)
self.price_model.num_customer_groups.on_change('value', self.refresh_view)
self.tx.num_columns.on_change('value', self.refresh_view)
self.prod_master.num_columns.on_change('value', self.refresh_view)
self.cust_master.num_columns.on_change('value', self.refresh_view)
for pg in self.price_model.price_groups:
pg.num_attributes.on_change('value', self.refresh_view, self.update_on_change)
for attr in pg.attributes:
attr.sourcefile.on_change('value', partial(attr.update_sourcecolumns_options, cpq=self))
attr.num_configs.on_change('value', self.refresh_view, self.update_on_change, self.update_quote_filters)
def create_file_input(self, para=None):
file_input = FileInput(accept=".ens,.bin")
callback = CustomJS(args=dict(para=para, file_input=file_input), code="""
para.text = "<p><b>filename:</b> " + file_input.filename + \
"<p><b>number of lines:</b> " + atob(file_input.value).split('\\n').length
""")
#file_input.js_on_change('value', callback)
file_input.on_change('value', self.select_file_handler)
return file_input
def __init__(self):
# extend __init__ super class
_DataSource.__init__(self)
# overwrite base class filepath with FileInput widget
self.filepath = FileInput(accept=".xlsx")
self.filepath.on_change('value', self.update_data)
# add columndatasource and table
self.cds = ColumnDataSource(data=dict(values=['']))
self.table = DataTable(source=self.cds, columns=[TableColumn(field='', title='')], editable=True)
def startserver(doc):
file_input = FileInput(accept=".ulg, .csv")
file_input.on_change('value', upload_new_data_sim)
file_input2 = FileInput(accept=".ulg, .csv")
file_input2.on_change('value', upload_new_data_real)
intro_text = Div(text="""<H2>Sim/Real Thiel Coefficient Calculator</H2>""",
width=500,
height=100,
align="center")
sim_upload_text = Paragraph(text="Upload a simulator datalog:",
width=500,
height=15)
real_upload_text = Paragraph(
text="Upload a corresponding real-world datalog:",
width=500,
height=15)
#checkbox_group = CheckboxGroup(labels=["x", "y", "vx","vy","lat","lon"], active=[0, 1])
sim_reverse_button = RadioButtonGroup(labels=["Sim Default", "Reversed"],
active=0)
sim_reverse_button.on_change('active',
lambda attr, old, new: reverse_sim())
real_reverse_button = RadioButtonGroup(labels=["Real Default", "Reversed"],
active=0)
real_reverse_button.on_change('active',
lambda attr, old, new: reverse_real())
simsource_static.selected.on_change('indices', simselection_change)
# The below are in case you want to see the x axis range change as you pan. Poorly documented elsewhere!
#ts1.x_range.on_change('end', lambda attr, old, new: print ("TS1 X range = ", ts1.x_range.start, ts1.x_range.end))
#ts2.x_range.on_change('end', lambda attr, old, new: print ("TS2 X range = ", ts2.x_range.start, ts2.x_range.end))
ts1.x_range.on_change(
'end', lambda attr, old, new: change_sim_scale(ts1.x_range.start))
ts2.x_range.on_change(
'end', lambda attr, old, new: change_real_scale(ts2.x_range.start))
# set up layout
widgets = column(datatype, stats)
sim_button = column(sim_reverse_button)
real_button = column(real_reverse_button)
main_row = row(widgets)
series = column(ts1, sim_button, ts2, real_button)
layout = column(main_row, series)
# initialize
update()
doc.add_root(intro_text)
doc.add_root(sim_upload_text)
doc.add_root(file_input)
doc.add_root(real_upload_text)
doc.add_root(file_input2)
doc.add_root(layout)
doc.title = "Flight data"
def create_disp():
##### 各画面要素作成
# テーブル作成
df_columns = target_data.columns.tolist()
datatable_source = ColumnDataSource(data=target_data)
table_columns = create_datatable_columns(df_columns)
data_table = DataTable(source=datatable_source,
selectable=True,
columns=table_columns,
sortable=True)
# カラムが変わった時に呼び出す関数リスト
column_change_functions = []
# 表示を切り替えるplotリスト
visible_plots = []
cm_operation_area, cm_plot_area = create_cm(column_change_functions,
visible_plots,
datatable_source)
# upload button がクリックされた時の処理
def upload_button_callback(event):
global target_data
global user
csv_str = base64.b64decode(csv_input.value)
target_data = pd.read_csv(io.BytesIO(csv_str))
df_columns = target_data.columns.tolist()
call_column_change(column_change_functions, df_columns)
table_columns = create_datatable_columns(df_columns)
data_table.columns = table_columns
datatable_source.data = target_data
data_table.update()
try:
user.csv_data = target_data.to_json()
db.session.commit()
except Exception as e:
warn = str(e)
# csvアップロード実行ボタン
upload_button = Button(label="Upload", button_type="success")
upload_button.on_event(ButtonClick, upload_button_callback)
# ファイル選択ボックス
csv_input = FileInput()
scatter_operation_area, scatter_plot_area = create_scatter(
column_change_functions, visible_plots, datatable_source)
# サブオペレーションエリア
sub_operation_area = Row(cm_operation_area, scatter_operation_area)
operation_area = Column(csv_input, upload_button, sub_operation_area,
data_table)
graph_area = Column(cm_plot_area, scatter_plot_area)
layout_disp = Row(graph_area, operation_area)
return layout_disp
class Quote(_DataSource):
def __init__(self):
# extend __init__ super class
_DataSource.__init__(self)
# overwrite base class filepath with FileInput widget
self.filepath = FileInput(accept=".xlsx")
self.filepath.on_change('value', self.update_data)
# add columndatasource and table
self.cds = ColumnDataSource(data=dict(values=['']))
self.table = DataTable(source=self.cds, columns=[TableColumn(field='', title='')], editable=True)
def import_data(self):
"""
This overrides the base class method
:return:
"""
# # decode file (received as b64 encoded) and write to xlsx locally
decoded = base64.b64decode(self.filepath.value)
# stream data to pandas df without saving locally
toread = io.BytesIO()
toread.write(decoded) # pass your `decrypted` string as the argument here
toread.seek(0) # reset the pointer
self.df = get_one_table(toread, 'quote_table').dropna(axis=0, how='all')
cols = [TableColumn(field=_, title=_) for _ in self.df.columns] # bokeh columns
self.table.columns = cols
self.cds.data = self.df
def calc_model_prices(self, pricemodel):
# TODO: calculate model price
self.df['Model price'] = 1
def make_layout():
"""Inclui o gráficos e os controles no curdoc do Bokeh Server."""
plot = Plot(DISTS)
numbins_slider = Slider(title="Number of bins",
value=10,
start=5,
end=15,
step=1)
div = Div(text="<b>Goodness of fit tests (p-values)</b>")
table = fit_table([dist.name for dist in DISTS])
def numbins_slider_callback(attr, old, new): # pylint: disable=unused-argument
"""Função que atualiza o histograma e a CDF dos observações."""
hist, edges = np.histogram(plot.values, density=True, bins=new)
plot.update_histogram(hist, edges)
data = plot.cumulative_source.data
chi = [chi_squared(plot.values, dist.cdf, new) for dist in DISTS]
ks = [kolmogorov_smirnov(plot.values, dist.cdf) for dist in DISTS]
wms = [weverton(dist.cdf, data["x"], data["y"]) for dist in DISTS]
table.source.data.update(ks=ks, chi=chi, wms=wms) # pylint: disable=no-member
def upload_callback(attr, old, new):
"""Função que atualiza os dados do plot com os dados do arquivo enviado."""
file_contents = base64.b64decode(new)
file_contents_bytes = io.BytesIO(file_contents)
observed = np.loadtxt(file_contents_bytes)
plot.update_data(observed)
numbins_slider.value = plot.num_bins
numbins_slider.end = len(observed) // 3
numbins_slider_callback(attr, old, numbins_slider.value)
button = FileInput()
button.on_change("value", upload_callback)
numbins_slider.on_change("value", numbins_slider_callback)
controls = column(button, numbins_slider, div, table, width=300)
return row(controls, plot.layout, sizing_mode="scale_height")
class projectdata:
''' ProjectData class is the core function of '/ProjectData' application page
ProjectData receives 'doc' argument from the caller init module which is generated by Bokeh Application
ProjectData is responsible of creating the controls which is displayed on the application page, initiate the controls, and response according to user events
ProjectData provides the following functionalities:
Upload XER file from user environment
Display XER file contents on the display screen
Save XER file to disegnated folder on server
Covert XER file to Excel-book
Display some of Excel-book tables on panel tabs
Provide a download link of the Excel-book
Inform the user of the work being currently done through the display screen'''
# Initiating class
# 'doc' is generated by the caller (Bokeh Application)
def __init__(self, doc):
############### Initiating Variables ###################
self.doc = doc
self.uploaded_projects_path = join(dirname(__file__),"..","data","uploaded_projects")
self.extracted_projects_path = join(dirname(__file__),"..","data","extracted_projects")
self.current_uploaded_file = "No file loaded"
self.current_extracted_file = "No Project"
self.full_uploaded_project_path = "empty"
self.full_extracted_project_path = "empty"
self.download_file_path = "No file to download"
self.tables = []
############### Creating Controls ###################
#self.fileinput = FileInput(accept=".xer", name="fileinput", css_classes=["myinput"])
self.fileinput = FileInput(accept=".xer", name="fileinput", css_classes=["myinput"])
self.textarea = TextAreaInput(value="XER file contents will show here!", rows = 10, name="textarea", css_classes=["mytext"])
self.extract = Button(label="Extract Data", name="extract", css_classes=["mybutton"])
self.download = Div(text="""No file to download""", name="download", css_classes=["mylink"])
self.panel_tabs = self.init_panels()
############### Defining Responses ###################
self.fileinput.on_change('value', self.upload_init)
self.extract.on_click(self.extract_init)
############### Exporting Controls ###################
self.doc.add_root(self.fileinput)
self.doc.add_root(self.textarea)
self.doc.add_root(self.extract)
self.doc.add_root(self.download)
self.doc.add_root(self.panel_tabs)
def init_panels(self):
# Load Primavera partial schema
schema = loadschema.loadschema()
# Create list of empty Bokeh DataTables
self.tables = self.initiate_data_tables(schema)
# Initiate list of tabs
datatables = []
# For each Bokeh DataTable....
for index, table in enumerate(self.tables):
# Assign Bokeh DataTable to new tab
tab = Panel(child=table, title=table.name)
# Append tab to list of tabs
datatables.append(tab)
# Create the panel of tabs and return. CSS Class name 'mypanel' will be used by the styling sheet
return Tabs(tabs=datatables, name="panel", css_classes=["mypanel"])
# 'schema' is generated by loadschema
def initiate_data_tables(self, schema):
# Initiate return container
tables = []
# for each Dictionary in the schema
for dic in schema:
# Initiate a new List for Bokeh TableColumn object which represent a table column
table_columns = []
# Initiate an empty data source (Dictionary) for the table, to be updated when the actual project is loaded
table_source = {}
# for each column name in the columns' names list
for column in list(dic.values())[0]:
# Create a new Bokeh TableColumn object and append to column's list
table_columns.append(TableColumn(field=column, title=column))
# Create Dictionary entry for each column, where key = column's name, and value = list of empty values
table_source[column] = [' ']*15
# Use empty data source dictionary to create Bokeh ColumnDataSource which is the data source for Bokeh DataTable
source = ColumnDataSource(data=table_source)
# Create Bokeh DataTable with Bokeh ColumnDataSource and Bokeh TableColumn objects, table name = key of schema dictionary
# CSS Class name 'mytable' will be used by the styling sheet since the table will be rendered with the assigned class name
datatable = DataTable(source = source, columns = table_columns, width=780, fit_columns = False, name=list(dic.keys())[0], css_classes=["mytable"], reorderable=False)
# Append the DataTable to the return container
tables.append(datatable)
# Return
return tables
def update_data_tables(self):
# Prepare the Excel-book file name to open
file_name = str(self.current_uploaded_file).split(sep='xer')[0] + 'xlsx'
# Generate on server full path to the Excel-book
project = join(self.extracted_projects_path, file_name)
# open Excel-book
with pd.ExcelFile(project) as xl:
# Access each sheet in the Excel-book
for index, sheet in enumerate(xl.sheet_names):
# Load the sheet into Pandas DataFrame
data = xl.parse(sheet)
# Create data source
source = ColumnDataSource(data.head(15))
# Access each Bokeh DataTable
for table in self.tables:
# If DataTable name matches current Excel-book sheet...
if table.name==sheet:
# Update DataTable source
table.source = source
def upload_init(self, attr, old, new):
# Because Bokeh state only synchronizes with the browser when the callback ends
# Uploading has to be divided to steps using add_next_tick_callback(step) and time.sleep
# Uploading uses 'upload_core' and 'upload_finalize' functions
# Assign current project to uploaded file
self.current_uploaded_file = self.fileinput.filename
# Initiate file types
mimetypes.init()
# Check for uploaded file type
file_type, file_encoding = mimetypes.guess_type(self.current_uploaded_file, strict=False)
# If the file is not XER file
if file_type == 'application/xer':
# Assign full path for uploaded file
self.full_uploaded_project_path = join(self.uploaded_projects_path, self.current_uploaded_file)
# Clear previous messages and Inform user about the start of uploading process
self.textarea.value = "\n" + "Uploading..." + "\n"
# Execute uploading on next server cycle
self.doc.add_next_tick_callback(self.upload_core)
else:
# Warn the user and do nothing
self.textarea.value = "\n" + "Please select XER file to upload" + "\n"
def upload_core(self):
# Postpond the execution a little bit
time.sleep(0.1)
# Upload file and decode the contents into bytes
file_contents = base64.b64decode(self.fileinput.value)
# Decode the uploaded bytes to ANSI text
self._string_file = file_contents.decode("ANSI")
# Inform user with file contents
self.textarea.value = "\n" + self._string_file + "\n" + self.textarea.value
# Execute saving on next server cycle
self.doc.add_next_tick_callback(self.upload_finalize)
# Inform user that file is being saved
self.textarea.value = "\n" + "Saving ..." + "\n" + self.textarea.value
def upload_finalize(self):
# Postpond the execution a little bit
time.sleep(0.1)
# Create text file at server
with open(self.full_uploaded_project_path,"w") as save_file:
# Save loaded file on server
save_file.write(self._string_file)
# Inform user that uploading is finished
self.textarea.value = "\n" + "Uploading Finished" + "\n" + self.textarea.value
def extract_init(self):
# Because Bokeh state only synchronizes with the browser when the callback ends
# Extraction has to be divided to steps using add_next_tick_callback(step) and time.sleep
# Extraction uses 'extract_core' and 'extract_finalize' functions
# Check if XER file exists on server
if os.path.isfile(self.full_uploaded_project_path):
# Clear previous messages and Inform user about the start of extractig process
self.textarea.value = "\n" + "Extracting..." + "\n"
# Execute extraction on next server cycle
self.doc.add_next_tick_callback(self.extract_core)
else:
# Inform the user and do nothing
self.textarea.value = "\n" + "Please upload XER file before extracting" + "\n"
def extract_core(self):
# Postpond the execution a little bit
time.sleep(0.1)
# Assign extracted porject
self.current_extracted_file = self.current_uploaded_file.split(sep=".xer")[0]+".xlsx"
self.full_extracted_project_path = join(self.extracted_projects_path, self.current_extracted_file)
# Extract XER file tables and save it to Excel-book, validate the extracted file
status = readxer.readxer(self.current_uploaded_file,self.uploaded_projects_path,self.extracted_projects_path)
# If sucess, continue
if status=="sucess":
# Inform user about progress
self.textarea.value = "\n" + "Loading Tables..." + "\n" + self.textarea.value
# Execute viewing extracted data on next server cycle
self.doc.add_next_tick_callback(self.extract_finalize)
# If empty project, inform the user and do nothing
elif status=="No_Activities":
self.textarea.value = "\n" + "This project has no activities" + "\n" + self.textarea.value
self.reset_porject()
# If no project, inform the user and do nothing
elif status=="No_Project":
self.textarea.value = "\n" + "There is no project to load" + "\n" + self.textarea.value
self.reset_porject()
def extract_finalize(self):
# Postpond the execution a little bit
time.sleep(0.1)
# Update Bokeh DataTables
self.update_data_tables()
# Inform user about progress
self.textarea.value = "\n" + "Loading Tables Finished" + "\n" + self.textarea.value
# Prepare download link href
self.download_file_path = "/data/extracted_projects/"+self.current_extracted_file
# Prepare download link display text (file name)
download_file = self.current_extracted_file
# Generate download link
self.download.text = """<a href="""+'"'+self.download_file_path+'" '+"""download>"""+download_file+"""</a>"""
# Inform user that download is available
self.textarea.value = "\n" + download_file + " is available to download" + "\n" + self.textarea.value
def reset_porject(self):
os.remove(self.full_extracted_project_path)
os.remove(self.full_uploaded_project_path)
self.current_uploaded_file = "No file loaded"
self.current_extracted_file = "No Project"
self.full_uploaded_project_path = "empty"
self.full_extracted_project_path = "empty"
class PyPricer(IPlugin):
def __init__(self):
# extend __init__ super class
IPlugin.__init__(self)
# instantiate main components
self.tx = TxData()
self.prod_master = ProductMaster()
self.cust_master = CustomerMaster()
self.quote = Quote()
self.price_model = PriceModel()
# cds_plot lives here rather than quote because they draw on several main components
self.cds_plot = ColumnDataSource(data=dict(x=[0], y=[0], attr_label=[''], attr_name=['']))
# plot_filters lives here because it's drawing from several main components
self.plot_filters = [MultiSelect(title='Product_filter', value=[''], options=[''])]
self.plot_filters[0].on_change('value', self.update_plot)
# configuration backup settings
self.backup_loc = TextInput(value=getcwd())
self.backup_config_button = Button(label='Output config', button_type='success')
self.backup_config_button.on_click(self.output_config)
self.config_data_input = FileInput(accept=".json")
self.config_data_input.on_change('value', self.load_config)
self.datafile_selector = RadioButtonGroup(labels=['Tx data', 'Product master', 'Customer master'], active=0)
self.datafile_selector.on_change('active', self.refresh_view)
# define and initialize views (and their layout attributes)
self.view_quote = QuoteView(self)
self.view_price_mgt = PriceMgtView(self)
self.view_price_model_config = PriceModelConfigView(self)
self.view_attr_settings = AttrSettingsView(self)
self.view_settings = SettingsView(self)
self.tabs = Tabs(
tabs=[Panel(child=self.view_quote.layout, title='Quote'),
Panel(child=self.view_price_mgt.layout, title='Price Management'),
Panel(child=self.view_price_model_config.layout, title='Price adjustments'),
Panel(child=self.view_attr_settings.layout, title='Attribute Settings'),
Panel(child=self.view_settings.layout, title='Settings')])
# define some on_change events
self.price_model.price_group_selector.on_change('active', self.refresh_view)
self.price_model.num_prod_groups.on_change('value', self.refresh_view, self.update_on_change)
self.price_model.num_customer_groups.on_change('value', self.refresh_view)
self.tx.num_columns.on_change('value', self.refresh_view)
self.prod_master.num_columns.on_change('value', self.refresh_view)
self.cust_master.num_columns.on_change('value', self.refresh_view)
for pg in self.price_model.price_groups:
pg.num_attributes.on_change('value', self.refresh_view, self.update_on_change)
for attr in pg.attributes:
attr.sourcefile.on_change('value', partial(attr.update_sourcecolumns_options, cpq=self))
attr.num_configs.on_change('value', self.refresh_view, self.update_on_change, self.update_quote_filters)
def update_quote_filters(self, attr, old, new):
self.set_plot_filters()
def update_on_change(self, attr, old, new):
for pg in self.price_model.price_groups:
pg.num_attributes.on_change('value', self.refresh_view, self.update_on_change)
for attr in pg.attributes:
attr.sourcefile.on_change('value', partial(attr.update_sourcecolumns_options, cpq=self))
attr.num_configs.on_change('value', self.refresh_view, self.update_on_change)
def create_plot(self):
tooltips = [
("Item Number", "@item_number"),
("Price", "@price"),
("CM%", "@cm_percent"),
("MM%", "@mm_percent"),
("Net Sales", "@net_sales")
]
plot = figure(title='Price vs margin', plot_height=500, plot_width=500, x_range=(0, 5), y_range=[0, 1],
tools="save, hover, box_zoom, pan, reset, undo, redo, wheel_zoom, box_select", tooltips=tooltips)
plot.circle('x', 'y', source=self.cds_plot, alpha=0.5)
# clean up figure
plot.toolbar.logo = None
return plot
def update_plot(self, attr, old, new):
self.set_data_for_quote_plot()
def output_config(self, event):
output_dict = {}
output_dict['tx_filepath'] = self.tx.filepath.value
output_dict['prod_master_filepath'] = self.prod_master.filepath.value
output_dict['cust_master_filepath'] = self.cust_master.filepath.value
output_dict['num_prod_groups'] = self.price_model.num_prod_groups.value
output_dict['num_customer_groups'] = self.price_model.num_customer_groups.value
output_dict['price_groups'] = {}
for pg in self.price_model.price_groups:
pg_dict = {}
pg_dict['name'] = pg.name.value
pg_dict['num_attributes'] = pg.num_attributes.value
pg_dict['attributes'] = {}
for attr in pg.attributes:
attr_dict = {}
attr_dict['name'] = attr.name.value
attr_dict['source_file'] = attr.sourcefile.value
attr_dict['source_col'] = attr.sourcecolumns.value
attr_dict['include_in_filter'] = attr.include_in_filters.active
attr_dict['num_configs'] = attr.num_configs.value
attr_dict['configurations'] = {}
for config in attr.configurations:
config_dict = {}
config_dict['name'] = config.name.value
config_dict['coefficient'] = config.coefficient.value
config_dict['abs_or_pct'] = config.abs_or_pct.value
attr_dict['configurations'][config.name.value] = config_dict
pg_dict['attributes'][attr.name.value] = attr_dict
output_dict['price_groups'][pg.name.value] = pg_dict
folder = Path(self.backup_loc.value)
file = 'cpq_config_' + time.strftime("%Y%m%d_%H%M%S") + '.json'
with open(folder / file, 'w') as f:
json.dump(output_dict, f)
print('output to', folder / file)
def load_config(self, attr, old, new):
# decode file (received as b64 encoded) and write to xlsx locally
decoded = json.loads(base64.b64decode(self.config_data_input.value))
# reset main component instances to make sure we're starting fresh
self.tx = TxData()
self.prod_master = ProductMaster()
self.cust_master = CustomerMaster()
self.quote = Quote()
self.price_model = PriceModel()
# data source settings
self.tx.filepath.value = decoded['tx_filepath']
self.prod_master.filepath.value = decoded['prod_master_filepath']
self.cust_master.filepath.value = decoded['cust_master_filepath']
self.price_model.num_prod_groups.value = decoded['num_prod_groups']
self.price_model.num_customer_groups.value = decoded['num_customer_groups']
for ((pg_name, pg_dict), pg) in zip(decoded['price_groups'].items(), self.price_model.price_groups):
pg.name.value = pg_dict['name']
pg.num_attributes.value = pg_dict['num_attributes']
for ((attr_name, attr_dict), attr) in zip(pg_dict['attributes'].items(), pg.attributes):
attr.name.value = attr_dict['name']
attr.sourcefile.value = attr_dict['source_file']
attr.sourcecolumns.value = attr_dict['source_col']
attr.include_in_filters.active = attr_dict['include_in_filter']
attr.num_configs.value = attr_dict['num_configs']
for ((config_name, config_dict), config) in \
zip(attr_dict['configurations'].items(), attr.configurations):
config.name.value = config_dict['name']
config.coefficient.value = config_dict['coefficient']
config.abs_or_pct.value = config_dict['abs_or_pct']
self.price_model.price_group_selector.on_change('active', self.refresh_view)
self.price_model.num_prod_groups.on_change('value', self.refresh_view, self.update_on_change)
self.price_model.num_customer_groups.on_change('value', self.refresh_view)
self.tx.num_columns.on_change('value', self.refresh_view)
self.prod_master.num_columns.on_change('value', self.refresh_view)
self.cust_master.num_columns.on_change('value', self.refresh_view)
for pg in self.price_model.price_groups:
pg.num_attributes.on_change('value', self.refresh_view, self.update_on_change)
pg.attribute_selector.on_change('active', self.refresh_view, self.update_on_change)
for attr in pg.attributes:
attr.sourcefile.on_change('value', partial(attr.update_sourcecolumns_options, cpq=self))
attr.num_configs.on_change('value', self.refresh_view, self.update_on_change, self.update_quote_filters)
self.set_plot_filters()
self.view_quote.refresh(self)
self.view_price_model_config.refresh(self)
self.view_attr_settings.refresh(self)
self.view_settings.refresh(self)
def set_plot_filters(self):
# TODO: Select active price group based on selected item in quote table
selected_pg_num = 0 # self.price_model.price_group_selector.active
selected_pg = self.price_model.price_groups[selected_pg_num]
self.plot_filters = []
for attr in selected_pg.attributes:
title = attr.name.value
values = [config.name.value for config in attr.configurations]
self.plot_filters.append(MultiSelect(title=title, value=values, options=values))
self.view_quote.refresh(cpq=self)
def set_data_for_quote_plot(self):
# For date filter, apply on Tx data
df_item_lvl = \
pd.pivot_table(
self.tx.df,
index=['Item number', 'Rollup Customer Name'],
values=['Net Sales', 'Volume (MSI)', 'Contribution Margin $', 'Material Margin $'],
aggfunc=np.sum
).reset_index()
print('before')
print(df_item_lvl)
# For customer and product filters, apply on pivoted data
# Step 1: get list of items that meet all criteria
prod_filters = []
for f in self.plot_filters:
print('f.value = ', f.value)
prod_filters.append(self.prod_master.df['Item Category Code'].isin(f.value))
final_prod_filter = np.logical_and.reduce(prod_filters)
included_items = self.prod_master.df.loc[final_prod_filter, 'Item Code']
print(included_items)
# Step 2: get list of customers that meet all criteria
included_cust = self.cust_master.df['Parent Name']
# Step 3; get pivoted data where items are in item list and customers are in customer list
df_item_lvl = df_item_lvl.loc[
df_item_lvl['Item number'].astype('str').isin(included_items.astype('str').tolist())]
print('after')
print(df_item_lvl)
df_item_lvl['Net Price ($/MSI)'] = df_item_lvl['Net Sales'] / df_item_lvl['Volume (MSI)']
df_item_lvl['CM%'] = df_item_lvl['Contribution Margin $'] / df_item_lvl['Net Sales']
df_item_lvl['MM%'] = df_item_lvl['Material Margin $'] / df_item_lvl['Net Sales']
df_item_lvl = df_item_lvl[
(df_item_lvl['Net Price ($/MSI)'] > 0) &
(df_item_lvl['Net Price ($/MSI)'] < 5) &
(df_item_lvl['Contribution Margin $'] > -0.5) &
(df_item_lvl['Contribution Margin $'] < 1)]
self.cds_plot.data = \
dict(x=df_item_lvl['Net Price ($/MSI)'],
y=df_item_lvl['CM%'],
item_number=df_item_lvl['Item number'],
price=df_item_lvl['Net Price ($/MSI)'],
cm_percent=df_item_lvl['CM%'],
mm_percent=df_item_lvl['Material Margin $'],
net_sales=df_item_lvl['Net Sales'],
vol_msi=df_item_lvl['Volume (MSI)'],
sizes=df_item_lvl['Net Sales'] / df_item_lvl['Net Sales'].max() * 0.1)
def refresh_view(self, attr, old, new):
self.view_price_model_config.refresh(self)
self.view_attr_settings.refresh(self)
self.view_settings.refresh(self)
def bkapp(self, doc):
doc.add_root(self.tabs)
def __init__(self, type):
self.test = False
self.report_type = type
self.utc = TimezoneInfo()
self.kp_zone = TimezoneInfo(utc_offset=-7*u.hour)
self.zones = [self.utc, self.kp_zone]
self.datefmt = DateFormatter(format="%m/%d/%Y %H:%M:%S")
self.inst_style = {'font-size':'150%'}
self.subt_style = {'font-size':'200%','font-style':'bold'}
self.title_style = {'font-size':'250%','font-style':'bold'}
self.alert_style = {'font-size':'150%','color':'red'}
self.nl_file = None
self.intro_subtitle = Div(text="Connect to Night Log",css_classes=['subt-style'])
self.time_note = Div(text="<b> Note: </b> Enter all times as HHMM (1818 = 18:18 = 6:18pm) in Kitt Peak local time. Either enter the time or hit the <b> Now </b> button if it just occured.", css_classes=['inst-style'])
hostname = socket.gethostname()
ip_address = socket.gethostbyname(hostname)
if 'desi' in hostname:
self.location = 'kpno'
#self.conn = psycopg2.connect(host="desi-db", port="5442", database="desi_dev", user="******", password="******")
elif 'app' in hostname: #this is not true. Needs to change.
self.location = 'nersc'
else:
self.location = 'other'
nw_dirs = {'nersc':'/global/cfs/cdirs/desi/spectro/nightwatch/nersc/', 'kpno':'/exposures/nightwatch/', 'other':None}
self.nw_dir = nw_dirs[self.location]
self.nl_dir = os.environ['NL_DIR']
self.your_name = TextInput(title ='Your Name', placeholder = 'John Smith')
self.os_name_1 = TextInput(title ='Observing Scientist 1', placeholder = 'Ruth Bader Ginsberg')
self.os_name_2 = TextInput(title ='Observing Scientist 2', placeholder = "Sandra Day O'Connor")
self.lo_names = ['None ','Liz Buckley-Geer','Ann Elliott','Parker Fagrelius','Satya Gontcho A Gontcho','James Lasker','Martin Landriau','Claire Poppett','Michael Schubnell','Luke Tyas','Other ']
self.oa_names = ['None ','Karen Butler','Amy Robertson','Anthony Paat','Dave Summers','Doug Williams','Other ']
self.intro_txt = Div(text=' ')
self.comment_txt = Div(text=" ", css_classes=['inst-style'], width=1000)
self.date_init = Select(title="Existing Night Logs")
self.time_title = Paragraph(text='Time* (Kitt Peak local time)', align='center')
self.now_btn = Button(label='Now', css_classes=['now_button'], width=50)
days = [d for d in os.listdir(self.nl_dir) if os.path.isdir(os.path.join(self.nl_dir, d))]
init_nl_list = np.sort([day for day in days if 'nightlog_meta.json' in os.listdir(os.path.join(self.nl_dir,day))])[::-1][0:10]
self.date_init.options = list(init_nl_list)
self.date_init.value = init_nl_list[0]
self.connect_txt = Div(text=' ', css_classes=['alert-style'])
self.connect_bt = Button(label="Connect to Existing Night Log", css_classes=['connect_button'])
self.exp_info = Div(text="Fill In Only Relevant Data. Mandatory fields have an asterisk*.", css_classes=['inst-style'],width=500)
self.exp_comment = TextAreaInput(title ='Comment/Remark', placeholder = 'Humidity high for calibration lamps',value=None,rows=10, cols=5,width=800,max_length=5000)
self.exp_time = TextInput(placeholder = '20:07',value=None, width=100) #title ='Time in Kitt Peak local time*',
self.exp_btn = Button(label='Add', css_classes=['add_button'])
self.exp_type = Select(title="Exposure Type", value = None, options=['None','Zero','Focus','Dark','Arc','FVC','DESI'])
self.exp_alert = Div(text=' ', css_classes=['alert-style'])
self.exp_exposure_start = TextInput(title='Exposure Number: First', placeholder='12345', value=None)
self.exp_exposure_finish = TextInput(title='Exposure Number: Last', placeholder='12346', value=None)
self.nl_subtitle = Div(text="Current DESI Night Log: {}".format(self.nl_file), css_classes=['subt-style'])
self.nl_btn = Button(label='Get Current DESI Night Log', css_classes=['connect_button'])
self.nl_text = Div(text=" ", css_classes=['inst-style'], width=1000)
self.nl_alert = Div(text='You must be connected to a Night Log', css_classes=['alert-style'], width=500)
self.nl_info = Div(text="Night Log Info:", css_classes=['inst-style'], width=500)
self.exptable_alert = Div(text=" ",css_classes=['alert-style'], width=500)
self.checklist = CheckboxGroup(labels=[])
self.check_time = TextInput(placeholder = '20:07', value=None) #title ='Time in Kitt Peak local time*',
self.check_alert = Div(text=" ", css_classes=['alert-style'])
self.check_btn = Button(label='Submit', css_classes=['add_button'])
self.check_comment = TextAreaInput(title='Comment', placeholder='comment if necessary', rows=3, cols=3)
self.prob_subtitle = Div(text="Problems", css_classes=['subt-style'])
self.prob_inst = Div(text="Describe problems as they come up and at what time they occur. If there is an Alarm ID associated with the problem, include it, but leave blank if not. If possible, include a description of the remedy.", css_classes=['inst-style'], width=1000)
self.prob_time = TextInput(placeholder = '20:07', value=None, width=100) #title ='Time in Kitt Peak local time*',
self.prob_input = TextAreaInput(placeholder="NightWatch not plotting raw data", rows=10, cols=3, title="Problem Description*:")
self.prob_alarm = TextInput(title='Alarm ID', placeholder='12', value=None, width=100)
self.prob_action = TextAreaInput(title='Resolution/Action',placeholder='description',rows=10, cols=3)
self.prob_btn = Button(label='Add', css_classes=['add_button'])
self.prob_alert = Div(text=' ', css_classes=['alert-style'])
self.img_subtitle = Div(text="Images", css_classes=['subt-style'])
self.img_upinst = Div(text="Include images in the Night Log by uploading a png image from your local computer. Select file, write a comment and click Add", css_classes=['inst-style'], width=1000)
self.img_upinst2 = Div(text=" Choose image to include with comment: ", css_classes=['inst-style'])
self.img_upload = FileInput(accept=".png")
self.img_upload.on_change('value', self.upload_image)
self.img_upload_comments = FileInput(accept=".png")
self.img_upload_comments.on_change('value', self.upload_image_comments)
self.img_upload_comments_os = FileInput(accept=".png")
self.img_upload_comments_os.on_change('value', self.upload_image_comments_os)
self.img_upload_problems = FileInput(accept=".png")
self.img_upload_problems.on_change('value', self.upload_image_problems)
self.img_inst = Div(text="Include images in the Night Log by entering the location of the images on the desi cluster", css_classes=['inst-style'], width=1000)
self.img_input = TextInput(title='image file location on desi cluster', placeholder='/n/home/desiobserver/image.png',value=None)
self.img_comment = TextAreaInput(placeholder='comment about image', rows=8, cols=3, title='Image caption')
self.img_btn = Button(label='Add', css_classes=['add_button'])
self.img_alert = Div(text=" ",width=1000)
self.plot_subtitle = Div(text="Telemetry Plots", css_classes=['subt-style'])
self.DESI_Log = None
self.save_telem_plots = False
y_units="screen",
text="prout",
render_mode="css",
**STATIC.label)
plot.add_layout(nodes_info)
nodes_info.visible = False
CACHE.plot.nodes_info = nodes_info
CACHE.plot.p = plot
Setter.nodes_metrics(True)
# ---------- Set all callbacks and buttons ---------- #
file_input = FileInput(
accept=".csv"
) # https://docs.bokeh.org/en/latest/docs/reference/models/widgets.inputs.html#bokeh.models.widgets.inputs.FileInput
file_input.on_change('value', FileInputHandler.callback)
CACHE.widgets.file_input = file_input
renderer_button = Button(button_type="warning",
label="Statistics visualisation")
renderer_button.on_click(VisualizerHandler.renderer_visualisation_callback)
CACHE.widgets.renderer_button = renderer_button
timestep_slider = Slider(title="Timestep",
start=CACHE.graph_attr.min_timestep,
end=CACHE.graph_attr.timesteps,
value=CACHE.plot.timestep,
step=1,
**STATIC.widget.slider)
def __init__(self, controller):
"""Method sets up object. First part of two-part initialization."""
self.controller = controller
# Header for section to get file
self.file_header =\
Div(text="""<span style='font-weight:bold'>"""
"""Choose the file to display</span>""",
sizing_mode='stretch_width')
# Selects the data file to read into the system
self.select_file = FileInput(accept=".csv",
sizing_mode='stretch_width')
# Shows summary and status for data read in.
self.status = Div(text="""No file connected""",
sizing_mode='stretch_width')
# Chart to show temperature and/or humidity.
self.temphumidity = Figure(x_axis_type='datetime',
title="Humidity & temperature by datetime",
x_axis_label='Datetime',
y_axis_label='Temperature (C)')
df = pd.DataFrame({
'Timestamp': [pd.to_datetime('4/12/2016 8:15:33 AM')],
'Temperature (C)': [25.0],
'Relative humidity (%)': [40.0]
})
self.cds = ColumnDataSource(df)
self.temphumidity.line(x='Timestamp',
y='Temperature (C)',
line_color='red',
legend_label='Temperature (C)',
line_width=2,
line_alpha=0.5,
source=self.cds)
self.temphumidity.extra_y_ranges = \
{"humidity": Range1d(start=0, end=100)}
self.temphumidity.add_layout(
LinearAxis(y_range_name="humidity", axis_label='Humidity (%)'),
'right')
self.temphumidity.line(x='Timestamp',
y='Relative humidity (%)',
legend_label='Relative humidity (%)',
line_color='blue',
line_width=2,
line_alpha=0.5,
source=self.cds,
y_range_name="humidity")
self.temphumidity.legend.click_policy = "hide"
self.temphumidity.title.text_font_size = '20px'
self.temphumidity.xaxis.axis_label_text_font_size = '15px'
self.temphumidity.xaxis.major_label_text_font_size = '15px'
self.temphumidity.yaxis.axis_label_text_font_size = '15px'
self.temphumidity.yaxis.major_label_text_font_size = '15px'
# Layout
self.layout = row(children=[
column(children=[self.file_header, self.select_file, self.status],
sizing_mode='fixed',
width=250,
height=80),
column(self.temphumidity, sizing_mode='stretch_both')
],
sizing_mode='stretch_both')
self.panel = Panel(child=self.layout, title='Read & display')
def get_thiel_analysis_plots(ulog, px4_ulog, db_data, vehicle_data,
link_to_main_plots):
global dfdata, simsource, simsource_static, realsource, realsource_static, usimsource, ts1, ts2, x, y
"""
get all bokeh plots shown on the Thiel analysis page
:return: list of bokeh plots
"""
def _resample(time_array, data, desired_time):
""" resample data at a given time to a vector of desired_time """
data_f = interp1d(time_array, data, fill_value='extrapolate')
return data_f(desired_time)
sim = False
if link_to_main_plots.find("sim") is not -1:
temp_link_to_main_plots = link_to_main_plots.replace('sim', '')
sim = True
if sim:
print("do some sim stuff")
else:
print("do regular stuff")
page_intro = """
<p>
This page shows the correspondance between a simulated and a real flight log.
</p>
"""
# curdoc().template_variables['title_html'] = get_heading_html(
# ulog, px4_ulog,db_data, None, [('Open Main Plots', link_to_main_plots,)],
# 'Thiel Analysis') + page_intro
curdoc().template_variables['title_html'] = get_heading_html(
ulog,
px4_ulog,
db_data,
None,
additional_links=[(
'Open Main Plots',
link_to_main_plots,
), ("Open Matching Simulation Log", '/browse?search=sim')])
# set up plots
simsource = ColumnDataSource(data=dict(x=[], y=[]))
simsource_static = ColumnDataSource(data=dict(x=[], y=[]))
realsource = ColumnDataSource(data=dict(realx=[], realy=[]))
realsource_static = ColumnDataSource(data=dict(realx=[], realy=[]))
cur_dataset = ulog.get_dataset('vehicle_local_position')
dfdata = pd.DataFrame(cur_dataset.data)
print(dfdata['x'])
keys = []
data = ulog.data_list
for d in data:
data_keys = [f.field_name for f in d.field_data]
data_keys.remove('timestamp')
# print (data_keys)
keys.append(data_keys)
t = cur_dataset.data['timestamp']
x = cur_dataset.data['x']
y = cur_dataset.data['y']
usimsource = ColumnDataSource(data=dict(x=t, y=y))
usimsource_static = ColumnDataSource(data=dict(x=t, y=y))
simtools = 'xpan,wheel_zoom,xbox_select,reset'
realtools = 'xpan,wheel_zoom,reset'
ts1 = figure(plot_width=900,
plot_height=200,
tools=simtools,
x_axis_type='linear',
active_drag="xbox_select")
ts1.line('x', 'y', source=simsource, line_width=2)
ts1.circle('x',
'y',
size=1,
source=simsource_static,
color=None,
selection_color="orange")
ts2 = figure(plot_width=900,
plot_height=200,
tools=realtools,
x_axis_type='linear')
# to adjust ranges, add something like this: x_range=Range1d(0, 1000), y_range = None,
# ts2.x_range = ts1.x_range
ts2.line('realx', 'realy', source=realsource, line_width=2)
ts2.circle('realx',
'realy',
size=1,
source=realsource_static,
color="orange")
plots = []
flight_mode_changes = get_flight_mode_changes(ulog)
x_range_offset = (ulog.last_timestamp - ulog.start_timestamp) * 0.05
x_range = Range1d(ulog.start_timestamp - x_range_offset,
ulog.last_timestamp + x_range_offset)
# cur_dataset = {}
# cur_dataset = ulog.get_dataset('vehicle_gps_position')
# # x = cur_dataset.data['vehicle_local_position']
# # y = cur_dataset.data['y']
# # FIXME: bokeh should be able to handle np.nan values properly, but
# # we still get a ValueError('Out of range float values are not JSON
# # compliant'), if x or y contains nan
# non_nan_indexes = np.logical_not(np.logical_or(np.isnan(x), np.isnan(y)))
# x = x[non_nan_indexes]
# y = y[non_nan_indexes]
# if check_if_all_zero:
# if np.count_nonzero(x) == 0 and np.count_nonzero(y) == 0:
# raise ValueError()
# data_source = ColumnDataSource(data=dict(x=x, y=y))
# data_set['timestamp'] = cur_dataset.data['timestamp']
# plot positions
# datatype = Select(value='XY', options=DEFAULT_FIELDS)
datatype = Select(value='XY', options=keys[0])
datatype.on_change('value', sim_change)
file_input = FileInput(accept=".ulg")
file_input.on_change('value', upload_new_data_sim)
file_input2 = FileInput(accept=".ulg")
file_input2.on_change('value', upload_new_data_real)
intro_text = Div(
text="""<H2>Sim/Real Thiel Coefficient Calculator</H2>""",
width=500,
height=100,
align="center")
sim_upload_text = Paragraph(text="Upload a simulator datalog:",
width=500,
height=15)
real_upload_text = Paragraph(
text="Upload a corresponding real-world datalog:",
width=500,
height=15)
choose_field_text = Paragraph(text="Choose a data field to compare:",
width=500,
height=15)
#checkbox_group = CheckboxGroup(labels=["x", "y", "vx","vy","lat","lon"], active=[0, 1])
simsource_static.selected.on_change('indices', simselection_change)
# The below are in case you want to see the x axis range change as you pan. Poorly documented elsewhere!
#ts1.x_range.on_change('end', lambda attr, old, new: print ("TS1 X range = ", ts1.x_range.start, ts1.x_range.end))
#ts2.x_range.on_change('end', lambda attr, old, new: print ("TS2 X range = ", ts2.x_range.start, ts2.x_range.end))
ts1.x_range.on_change(
'end', lambda attr, old, new: change_sim_scale(ts1.x_range.start))
ts2.x_range.on_change(
'end', lambda attr, old, new: change_real_scale(ts2.x_range.start))
# set up layout
widgets = column(datatype, stats)
sim_button = column(sim_reverse_button)
real_button = column(real_reverse_button)
main_row = row(widgets)
series = column(ts1, sim_button, ts2, real_button)
layout = column(main_row, series)
# initialize
update()
curdoc().add_root(intro_text)
curdoc().add_root(sim_upload_text)
curdoc().add_root(file_input)
curdoc().add_root(real_upload_text)
curdoc().add_root(file_input2)
curdoc().add_root(choose_field_text)
curdoc().add_root(layout)
curdoc().title = "Flight data"
# plot_config['custom_tools'] = 'xpan,wheel_zoom,xbox_select,reset'
# Local position
for axis in ['x', 'y', 'z']:
data_plot = DataPlot(data,
plot_config,
'vehicle_local_position',
y_axis_label='[m]',
title='Local Position ' + axis.upper(),
plot_height='small',
x_range=x_range)
data_plot.add_graph([axis],
colors2[0:1], [axis.upper() + ' Estimated'],
mark_nan=True)
data_plot.change_dataset('vehicle_local_position_setpoint')
data_plot.add_graph([axis],
colors2[1:2], [axis.upper() + ' Setpoint'],
use_step_lines=True)
plot_flight_modes_background(data_plot, flight_mode_changes)
if data_plot.finalize() is not None: plots.append(data_plot)
x_range_offset = (ulog.last_timestamp - ulog.start_timestamp) * 0.05
x_range = Range1d(ulog.start_timestamp - x_range_offset,
ulog.last_timestamp + x_range_offset)
# exchange all DataPlots with the bokeh_plot
jinja_plot_data = []
for i in range(len(plots)):
if plots[i] is None:
plots[i] = column(param_changes_button,
width=int(plot_width * 0.99))
if isinstance(plots[i], DataPlot):
if plots[i].param_change_label is not None:
param_change_labels.append(plots[i].param_change_label)
plot_title = plots[i].title
plots[i] = plots[i].bokeh_plot
fragment = 'Nav-'+plot_title.replace(' ', '-') \
.replace('&', '_').replace('(', '').replace(')', '')
jinja_plot_data.append({
'model_id': plots[i].ref['id'],
'fragment': fragment,
'title': plot_title
})
curdoc().template_variables['plots'] = jinja_plot_data
return plots
fig = figure(name=('p'+str(i)), title=currChromPoints[plotLoc][0] + " " + str(currChrom[plotLoc]), y_range=list(OrderedDict.fromkeys(readsFiltered['readID'])),
plot_width = plotWidth, plot_height=len(list(set(readsFiltered['readID'])))*15+220,
x_range=(0, 1), tools=tools, toolbar_location=None, min_border_bottom=100, min_border_top=100)
fig.yaxis.visible = False
figList.append(fig)
figList[0].yaxis.visible = True
figList[-1].toolbar_location = "right"
return figList
else:
return [figure(), figure()]
pageTitle = Div(text='Long read gene fusion visualization', style={'font-size': '24px', 'text-decoration':'underline'})
fusionUploadTitle = Div(text='Fusion file (.tsv)', name='fusionUploadText')
readsUploadTitle = Div(text='Reads file (.bed)')
fusionUpload = FileInput(accept=".tsv", name='fusion')
fusionUpload.on_change('value', upload_table_data)
readsUpload = FileInput(accept=".bed", name='reads')
readsUpload.on_change('value', upload_reads_data)
fastaUploadTitle = Div(text='Fusion fragments aligned to original reads file (.bed, optional)')
fastaUpload = FileInput(accept=".bed", name='fareads')
informUser = Div(text='updates will appear here', name='informUser')
tableSource.selected.on_change('indices', table_click_callback)
buttonSource = ColumnDataSource(data=dict(press=[0], figs=[2]))
def buttonClick(attr):
hiddenButtonSource.data.update(ColumnDataSource({"selected":[attr]}).data)
table_click_callback(0, 0, 0)
def saveClick(attr):
# currLayout = curdoc().get_model_by_name('mainLayout').children
# curdoc().get_model_by_name()
from bokeh.io import output_file, show
from bokeh.models.widgets import FileInput
output_file("file_input.html")
file_input = FileInput()
show(file_input)
decoded = b64decode(file_input.value)
f = io.BytesIO(decoded)
image = Image.open(f)
print(image.size)
print(type(image))
img_array = np.array(image, dtype=np.uint8)
os.chdir(directory)
imsave("image.jpeg", img_array)
imsave("image1.jpeg", img_array)
os.chdir(directory1)
img_path = 'server_folder/static/image1.jpeg'
url = img_path
source.data = dict(url=[url])
file_input = FileInput(accept=".jpeg")
file_input.on_change('value', upload_data)
########################################################################
btn4 = Button(label="Download Image", button_type="success")
btn4.js_on_click(
CustomJS(args=dict(source=source),
code="""
var data = source.data;
value1=data['url'];
var file = new Blob([value1], {type: 'image/jpeg'});
var elem = window.document.createElement('output');
elem.href = window.URL.createObjectURL(file);
elem.download = 'downloaded_image.jpeg';
document.body.appendChild(elem);
from bokeh.plotting import figure, output_file, show
def upload_fit_data(attr, old, new):
print("fit data upload succeeded")
decoded = b64decode(new)
f = io.BytesIO(decoded)
global df
print(type(new))
df = pd.read_csv(f)
#print(new_df)
file_input = FileInput(accept=".csv,.json,.txt,.pdf,.xls")
file_input.on_change('value', upload_fit_data)
p = figure(
# x_range=(minlng,maxlng),
# y_range=(minlat, maxlat),
#x_axis_type=map_repr,
#y_axis_type=map_repr,
#title='IDLE Vehicles Map',
match_aspect=True,
tools="pan,wheel_zoom,box_zoom,tap,box_select,reset,save"
#tools='tap'
)
p.circle(
x=[],
'left_table': file1,
'right_table': file2,
'left_keys': left_keys,
'right_keys': right_keys,
'metric_lists': metric_pairs
}
running_info.text = '<b>Status Bar</b>: Datasources are being analyzed ->->-> <b>Please Wait</b>'
harmonize(data_dict)
running_info.text = '<b>Status Bar</b>: Please check your <b>downloads folder</b> for ->->-> <b>Comparison_File.xlsx</b>'
"""Widget Tab 1"""
file_input1 = FileInput(accept='.csv')
file_1 = Paragraph(text='First File:')
file_input2 = FileInput(accept='.csv')
file_2 = Paragraph(text='Second File:')
#lock model parameters
update_button = Button(label="Update Data Files",
css_classes=['upload_button'],
width=300)
update_button.on_click(update_datasource)
file_inputs = column(hello,
welcome,
Spacer(max_height=10),
row(file_1, file_input1),
input_file = pd.read_csv(f)
print(len(input_file))
pre.text='<h4 style="border-top: 2px solid #778899;width: 1600px"><br><b style="color:slategray">Upload completed!....</b><br></h4>'
def file_upload_click(attr,old,new):
print(new)
pre.text='<h4 style="border-top: 2px solid #778899;width: 1600px"><br><b style="color:slategray">Upload in Progress....</b><br></h4>'
fu= Button(label='Upload File')
#fu.on_click(file_upload_click)
from bokeh.events import ButtonClick
file_input = FileInput(accept=".csv,.json,.txt,.pdf,.xls")
file_input.on_change('filename',file_upload_click)
file_input.on_change('value',file_upload)
"""
WIDGETS
"""
sdate_input = TextInput(value="2020-01-03 00:00:00", title="Start Date: (YYYY-MM-DD HH:MM:SS)")
#sdate_input.on_change("value", my_slider_handler)
edate_input = TextInput(value="2020-01-03 01:00:00", title="End Date: (YYYY-MM-DD HH:MM:SS)")
#edate_input.on_change("value", my_slider_handler)
count_threshold = TextInput(value="1", title="Count Threshold Value")
resolution_slider=Slider(start=1, end=15, value=7, step=1, title="Hex Resolution")
def create_layout():
"""Inclui o gráficos e os controles no curdoc do Bokeh Server."""
plot_html = ColumnDataSource(dict(file_html=["Title"], html=[""]))
plot = figure(
title="Title",
sizing_mode="stretch_both",
tools="wheel_zoom,box_zoom,pan,,crosshair,reset,save",
# tooltips=[("y", "$y"), ("x", "$x")],
active_scroll="wheel_zoom",
toolbar_location="above",
output_backend="webgl",
)
plot.xaxis.axis_label = "X"
plot.yaxis.axis_label = "Y"
plot.add_layout(Legend())
plot.legend.click_policy = "hide"
plot.legend.background_fill_alpha = 0.6
table_source = ColumnDataSource()
side_controls = column(width=400, height=200)
series_source_controls = row(sizing_mode="scale_width")
tabs = Tabs()
models.COLORS = cycle(Category10_10)
models.Series.labels = []
def upload_callback(attr, old, new):
"""Função que atualiza os dados do arquivo aberto."""
file_contents = base64.b64decode(new)
file_contents_bytes = io.BytesIO(file_contents)
table_source.data = pd.read_csv(file_contents_bytes).to_dict("list")
colmmns = list(table_source.data.keys())
for widget in list(values_selectors.values()):
widget.options = colmmns
datatable.columns = [TableColumn(field=_, title=_) for _ in colmmns]
def update_plot_html(event):
title = plot.title.text
file_name = title + ".html"
# output_file(file_name, title=title, mode="inline")
# save(plot)
html = file_html(plot, INLINE, title)
plot_html.data = dict(file_name=[file_name], html=[html])
# Widgets (controls) ======================================================
# Plot controls
plot_title = TextInput(title="Plot title", value="Title")
x_title = TextInput(title="X title", value="X")
y_title = TextInput(title="Y title", value="Y")
positions = [_.replace("_", "-") for _ in list(LegendLocation)]
legend_position = Select(title="Legend position", options=positions)
download_button = Button(label="Download saved",
button_type="success",
align="end")
save_button = Button(label="Save", align="end")
save_button.on_click(update_plot_html)
with open("bokeh_chart_maker/download.js") as f:
callback = CustomJS(args=dict(source=plot_html), code=f.read())
download_button.js_on_click(callback)
plot_controls = row(plot_title, x_title, y_title, legend_position,
save_button, download_button)
def update_plot(attr, old, new):
plot.title.text = plot_title.value
plot.xaxis.axis_label = x_title.value
plot.yaxis.axis_label = y_title.value
plot.legend.location = legend_position.value.replace("-", "_")
for widget in plot_controls.children:
if hasattr(widget, "value"):
widget.on_change("value", update_plot)
legend_position.value = "top_left"
# Series controls
upload_button = FileInput(accept=".csv")
upload_button.on_change("value", upload_callback)
columns = [
TableColumn(field=_, title=_, width=10)
for _ in table_source.data.keys()
]
datatable = DataTable(source=table_source,
columns=columns,
width=390,
height=200)
values_selectors = {}
glyph_type = Select(title="Glyph type",
options=["line", "scatter"],
width=100)
def update_series_source_controls(attr, old, new):
colmmns = list(table_source.data.keys())
kw = dict(options=colmmns, width=80)
values_selectors["x"] = Select(title=f"x-values", **kw)
values_selectors["y"] = Select(title=f"y-values", **kw)
series_source_controls.children = list(values_selectors.values())
glyph_type.on_change("value", update_series_source_controls)
glyph_type.value = "line"
add_button = Button(label="Add glyph",
button_type="success",
width=50,
align="end")
side_controls.children = [
Div(text="<h3>Load file</h3>", height=35),
upload_button,
datatable,
row(glyph_type, series_source_controls, add_button, width=390),
Div(text="<h3>Glyphs</h3>", height=35),
tabs,
]
def add_series(event):
any_col = list(table_source.data.keys())[0]
source = ColumnDataSource()
for key, selector in values_selectors.items():
column_name = selector.value or any_col
source.data[key] = table_source.data[column_name]
vars_map = dict((k, str(k)) for k in values_selectors.keys())
if glyph_type.value == "line":
series = models.line_series(plot, source=source, **vars_map)
elif glyph_type.value == "scatter":
series = models.scatter_series(plot, source=source, **vars_map)
def delete_series(event):
plot.renderers = [
r for r in plot.renderers if r.glyph != series.glyph
]
legend_items = list(plot.legend.items)
plot.legend.items = [
item for item in legend_items
if series.glyph != item.renderers[0].glyph
]
tabs.tabs = [panel for panel in tabs.tabs if panel != series.panel]
series.delete_button.on_click(delete_series)
tabs.tabs.append(series.panel)
add_button.on_click(add_series)
return [
side_controls,
column(plot_controls, plot, sizing_mode="stretch_width")
]
data_table_panel = {}
source = {}
table_row_1 = {}
table_row_2 = {}
table_row_3 = {}
table_row_4 = {}
table_bloc = {}
grid = {}
tab = {}
#age max for job offer age
age_max = "100000"
#build all widget objects and aggregate them in a grid plot for each panel (according to job board origin).
for idx, origin in enumerate(origin_source):
cv_input[origin] = FileInput(accept='.txt')
location_select[origin] = RadioButtonGroup(
labels=location_choice,
active=0,
css_classes=['custom_group_button_bokeh'])
text = template_div.format(site_name=origin,
number=total_job_numbers[idx],
total=total_job_numbers[idx])
div_job_number[origin] = Div(text=text, height=50)
select_1[origin] = Dropdown(value=age_max,
label='Publication date',
css_classes=['custom_button_bokeh'],
menu=[("All", age_max),
("Less than 1 day", "1"),
("Less than 3 days", "3"),
("Less than 7 days", "7"),
TableColumn(field="index", title="Distribution"),
TableColumn(field="sumsquare_error", title="Error")
]
tabla1.source = source
#####################################
# Update text
conclusion_text.text = "The parameters for the best distribution in " + targetVar.value + " are"
conclusion.text = str(distfit_var.get_best())
#dir(conclusion)
################################################
#For upload file
file_input = FileInput(accept=".csv")
file_input.on_change('value', upload_fit_data)
#############################
#For Checkbox
#Function that enables the checkboxlist
def check_box_populate(attr, old, new):
global file2
if new != "No Filter":
labels = file2[new].unique()
radioButtom_group.labels = list(map(str, labels))
def compute_summary(attr, old, new):
if filterSelect.value != 'No Filter':
def change_sim_scale(shift):
global simx_offset, new_data
simx_offset = shift
new_data = True
update()
def change_real_scale(shift):
global realx_offset, new_data
realx_offset = shift
new_data = True
update()
file_input = FileInput(accept=".ulg, .csv")
file_input.on_change('value', upload_new_data_sim)
file_input2 = FileInput(accept=".ulg, .csv")
file_input2.on_change('value', upload_new_data_real)
intro_text = Div(text="""<H2>Sim/Real Thiel Coefficient Calculator</H2>""",
width=500,
height=100,
align="center")
sim_upload_text = Paragraph(text="Upload a simulator datalog:",
width=500,
height=15)
real_upload_text = Paragraph(text="Upload a corresponding real-world datalog:",
width=500,
height=15)
#checkbox_group = CheckboxGroup(labels=["x", "y", "vx","vy","lat","lon"], active=[0, 1])
def post(self, *args, **kwargs):
""" POST request callback """
if self.multipart_streamer:
try:
file_input = FileInput(accept=".ulg, .csv")
file_input.on_change('value', upload_new_data_sim)
file_input2 = FileInput(accept=".ulg, .csv")
file_input2.on_change('value', upload_new_data_real)
curdoc().template_variables['title_html'] = get_heading_html(
ulog, px4_ulog, db_data, None, [
('Open Main Plots', link_to_main_plots)
], 'PID Analysis') + page_intro
intro_text = Div(
text="""<H2>Sim/Real Thiel Coefficient Calculator</H2>""",
width=500,
height=100,
align="center")
sim_upload_text = Div(
text=
"""Upload a simulator datalog: <a href="/tornado_handlers/browse">
this script</a>""",
width=500,
height=15)
real_upload_text = Paragraph(
text="Upload a corresponding real-world datalog:",
width=500,
height=15)
#checkbox_group = CheckboxGroup(labels=["x", "y", "vx","vy","lat","lon"], active=[0, 1])
sim_reverse_button = RadioButtonGroup(
labels=["Sim Default", "Reversed"], active=0)
sim_reverse_button.on_change(
'active', lambda attr, old, new: reverse_sim())
real_reverse_button = RadioButtonGroup(
labels=["Real Default", "Reversed"], active=0)
real_reverse_button.on_change(
'active', lambda attr, old, new: reverse_real())
simsource_static.selected.on_change('indices',
simselection_change)
# The below are in case you want to see the x axis range change as you pan. Poorly documented elsewhere!
#ts1.x_range.on_change('end', lambda attr, old, new: print ("TS1 X range = ", ts1.x_range.start, ts1.x_range.end))
#ts2.x_range.on_change('end', lambda attr, old, new: print ("TS2 X range = ", ts2.x_range.start, ts2.x_range.end))
ts1.x_range.on_change(
'end',
lambda attr, old, new: change_sim_scale(ts1.x_range.start))
ts2.x_range.on_change(
'end', lambda attr, old, new: change_real_scale(ts2.x_range
.start))
# set up layout
widgets = column(datatype, stats)
sim_button = column(sim_reverse_button)
real_button = column(real_reverse_button)
main_row = row(widgets)
series = column(ts1, sim_button, ts2, real_button)
layout = column(main_row, series)
# initialize
update()
doc.add_root(intro_text)
doc.add_root(sim_upload_text)
doc.add_root(file_input)
doc.add_root(real_upload_text)
doc.add_root(file_input2)
doc.add_root(layout)
doc.title = "Flight data"
except CustomHTTPError:
raise
except ULogException:
raise CustomHTTPError(
400,
'Failed to parse the file. It is most likely corrupt.')
except:
print('Error when handling POST data',
sys.exc_info()[0],
sys.exc_info()[1])
raise CustomHTTPError(500)
finally:
self.multipart_streamer.release_parts()
data["rel_diff_x_new"] = data["rel_x_new"] - data["rel_y_new"]
data["rel_diff_y_new"] = data["rel_y_new"] - data["rel_x_new"]
data = pd.concat([data, pd.DataFrame(data.apply(chisq, axis=1))], axis=1)
data.columns = data.columns.astype(str)
data = data.rename(index=str, columns={"0": "LL", "1": "pval"})
data.LL = round(data.LL, 2)
data.pval = round(data.pval, 2)
data = data[~data["word"].isin(ss[0:int(stopwords_1.value)])]
return data, df1, df2
#ticker1 = Select(value='twain-huckleberry', options=nix('hemingway-farewell', DEFAULT_TICKERS))
#ticker2 = Select(value='hemingway-farewell', options=nix('twain-huckleberry', DEFAULT_TICKERS))
ticker1 = Select(value='twain-huckleberry', options=DEFAULT_TICKERS)
ticker2 = Select(value='hemingway-farewell', options=DEFAULT_TICKERS)
user1 = FileInput(accept='.csv,.txt,.doc,.docx')
user2 = FileInput(accept='.csv,.txt,.doc,.docx')
stopwords_1 = Select(title="Remove most frequent words:",
value="0",
options=["0", "10", "20", "50", "100", "200"])
source = ColumnDataSource(data=dict(word=[],
rank_x=[],
rank_x_new=[],
rank_y=[],
rank_y_new=[],
freq_x=[],
freq_y=[],
sum_x=[],
sum_y=[],
class ReadDisplay():
"""Reads data saved to file into the system and displays it."""
# %%
def __init__(self, controller):
"""Method sets up object. First part of two-part initialization."""
self.controller = controller
# Header for section to get file
self.file_header =\
Div(text="""<span style='font-weight:bold'>"""
"""Choose the file to display</span>""",
sizing_mode='stretch_width')
# Selects the data file to read into the system
self.select_file = FileInput(accept=".csv",
sizing_mode='stretch_width')
# Shows summary and status for data read in.
self.status = Div(text="""No file connected""",
sizing_mode='stretch_width')
# Chart to show temperature and/or humidity.
self.temphumidity = Figure(x_axis_type='datetime',
title="Humidity & temperature by datetime",
x_axis_label='Datetime',
y_axis_label='Temperature (C)')
df = pd.DataFrame({
'Timestamp': [pd.to_datetime('4/12/2016 8:15:33 AM')],
'Temperature (C)': [25.0],
'Relative humidity (%)': [40.0]
})
self.cds = ColumnDataSource(df)
self.temphumidity.line(x='Timestamp',
y='Temperature (C)',
line_color='red',
legend_label='Temperature (C)',
line_width=2,
line_alpha=0.5,
source=self.cds)
self.temphumidity.extra_y_ranges = \
{"humidity": Range1d(start=0, end=100)}
self.temphumidity.add_layout(
LinearAxis(y_range_name="humidity", axis_label='Humidity (%)'),
'right')
self.temphumidity.line(x='Timestamp',
y='Relative humidity (%)',
legend_label='Relative humidity (%)',
line_color='blue',
line_width=2,
line_alpha=0.5,
source=self.cds,
y_range_name="humidity")
self.temphumidity.legend.click_policy = "hide"
self.temphumidity.title.text_font_size = '20px'
self.temphumidity.xaxis.axis_label_text_font_size = '15px'
self.temphumidity.xaxis.major_label_text_font_size = '15px'
self.temphumidity.yaxis.axis_label_text_font_size = '15px'
self.temphumidity.yaxis.major_label_text_font_size = '15px'
# Layout
self.layout = row(children=[
column(children=[self.file_header, self.select_file, self.status],
sizing_mode='fixed',
width=250,
height=80),
column(self.temphumidity, sizing_mode='stretch_both')
],
sizing_mode='stretch_both')
self.panel = Panel(child=self.layout, title='Read & display')
# %%
def setup(self):
"""Method sets up object. Second part of two-part initialization."""
self.select_file.on_change("value", self.callback_select_file)
# %%
def update(self):
"""Method updates object."""
pass
# %%
def callback_select_file(self, attrname, old, new):
"""Callback method for select file"""
self.status.text = 'Reading in the data file....'
# Convert the data to a Pandas dataframe
convert = BytesIO(base64.b64decode(self.select_file.value))
df = pd.read_csv(convert)
# Check the Pandas dataframe has the correct fields
if set(df.columns) != set([
'Timestamp', 'Temperature (C)', 'Relative humidity (%)',
'Pressure (Pa)'
]):
self.status.text = (
"""The file {0} has the columns {1} """
"""when it should have the columns {2} """.format(
self.select_file.filename, set(df.columns),
set([
'Timestamp', 'Temperature (C)',
'Relative humidity (%)', 'Pressure (Pa)'
])))
return
# Make sure the data types are correct
df['Timestamp'] = pd.to_datetime(df['Timestamp'])
self.cds.data = {
'Timestamp': df['Timestamp'],
'Temperature (C)': df['Temperature (C)'],
'Relative humidity (%)': df['Relative humidity (%)']
}
self.status.text = 'Read in the data file correctly.'
class Report():
def __init__(self, type):
self.test = False
self.report_type = type
self.utc = TimezoneInfo()
self.kp_zone = TimezoneInfo(utc_offset=-7*u.hour)
self.zones = [self.utc, self.kp_zone]
self.datefmt = DateFormatter(format="%m/%d/%Y %H:%M:%S")
self.inst_style = {'font-size':'150%'}
self.subt_style = {'font-size':'200%','font-style':'bold'}
self.title_style = {'font-size':'250%','font-style':'bold'}
self.alert_style = {'font-size':'150%','color':'red'}
self.nl_file = None
self.intro_subtitle = Div(text="Connect to Night Log",css_classes=['subt-style'])
self.time_note = Div(text="<b> Note: </b> Enter all times as HHMM (1818 = 18:18 = 6:18pm) in Kitt Peak local time. Either enter the time or hit the <b> Now </b> button if it just occured.", css_classes=['inst-style'])
hostname = socket.gethostname()
ip_address = socket.gethostbyname(hostname)
if 'desi' in hostname:
self.location = 'kpno'
#self.conn = psycopg2.connect(host="desi-db", port="5442", database="desi_dev", user="******", password="******")
elif 'app' in hostname: #this is not true. Needs to change.
self.location = 'nersc'
else:
self.location = 'other'
nw_dirs = {'nersc':'/global/cfs/cdirs/desi/spectro/nightwatch/nersc/', 'kpno':'/exposures/nightwatch/', 'other':None}
self.nw_dir = nw_dirs[self.location]
self.nl_dir = os.environ['NL_DIR']
self.your_name = TextInput(title ='Your Name', placeholder = 'John Smith')
self.os_name_1 = TextInput(title ='Observing Scientist 1', placeholder = 'Ruth Bader Ginsberg')
self.os_name_2 = TextInput(title ='Observing Scientist 2', placeholder = "Sandra Day O'Connor")
self.lo_names = ['None ','Liz Buckley-Geer','Ann Elliott','Parker Fagrelius','Satya Gontcho A Gontcho','James Lasker','Martin Landriau','Claire Poppett','Michael Schubnell','Luke Tyas','Other ']
self.oa_names = ['None ','Karen Butler','Amy Robertson','Anthony Paat','Dave Summers','Doug Williams','Other ']
self.intro_txt = Div(text=' ')
self.comment_txt = Div(text=" ", css_classes=['inst-style'], width=1000)
self.date_init = Select(title="Existing Night Logs")
self.time_title = Paragraph(text='Time* (Kitt Peak local time)', align='center')
self.now_btn = Button(label='Now', css_classes=['now_button'], width=50)
days = [d for d in os.listdir(self.nl_dir) if os.path.isdir(os.path.join(self.nl_dir, d))]
init_nl_list = np.sort([day for day in days if 'nightlog_meta.json' in os.listdir(os.path.join(self.nl_dir,day))])[::-1][0:10]
self.date_init.options = list(init_nl_list)
self.date_init.value = init_nl_list[0]
self.connect_txt = Div(text=' ', css_classes=['alert-style'])
self.connect_bt = Button(label="Connect to Existing Night Log", css_classes=['connect_button'])
self.exp_info = Div(text="Fill In Only Relevant Data. Mandatory fields have an asterisk*.", css_classes=['inst-style'],width=500)
self.exp_comment = TextAreaInput(title ='Comment/Remark', placeholder = 'Humidity high for calibration lamps',value=None,rows=10, cols=5,width=800,max_length=5000)
self.exp_time = TextInput(placeholder = '20:07',value=None, width=100) #title ='Time in Kitt Peak local time*',
self.exp_btn = Button(label='Add', css_classes=['add_button'])
self.exp_type = Select(title="Exposure Type", value = None, options=['None','Zero','Focus','Dark','Arc','FVC','DESI'])
self.exp_alert = Div(text=' ', css_classes=['alert-style'])
self.exp_exposure_start = TextInput(title='Exposure Number: First', placeholder='12345', value=None)
self.exp_exposure_finish = TextInput(title='Exposure Number: Last', placeholder='12346', value=None)
self.nl_subtitle = Div(text="Current DESI Night Log: {}".format(self.nl_file), css_classes=['subt-style'])
self.nl_btn = Button(label='Get Current DESI Night Log', css_classes=['connect_button'])
self.nl_text = Div(text=" ", css_classes=['inst-style'], width=1000)
self.nl_alert = Div(text='You must be connected to a Night Log', css_classes=['alert-style'], width=500)
self.nl_info = Div(text="Night Log Info:", css_classes=['inst-style'], width=500)
self.exptable_alert = Div(text=" ",css_classes=['alert-style'], width=500)
self.checklist = CheckboxGroup(labels=[])
self.check_time = TextInput(placeholder = '20:07', value=None) #title ='Time in Kitt Peak local time*',
self.check_alert = Div(text=" ", css_classes=['alert-style'])
self.check_btn = Button(label='Submit', css_classes=['add_button'])
self.check_comment = TextAreaInput(title='Comment', placeholder='comment if necessary', rows=3, cols=3)
self.prob_subtitle = Div(text="Problems", css_classes=['subt-style'])
self.prob_inst = Div(text="Describe problems as they come up and at what time they occur. If there is an Alarm ID associated with the problem, include it, but leave blank if not. If possible, include a description of the remedy.", css_classes=['inst-style'], width=1000)
self.prob_time = TextInput(placeholder = '20:07', value=None, width=100) #title ='Time in Kitt Peak local time*',
self.prob_input = TextAreaInput(placeholder="NightWatch not plotting raw data", rows=10, cols=3, title="Problem Description*:")
self.prob_alarm = TextInput(title='Alarm ID', placeholder='12', value=None, width=100)
self.prob_action = TextAreaInput(title='Resolution/Action',placeholder='description',rows=10, cols=3)
self.prob_btn = Button(label='Add', css_classes=['add_button'])
self.prob_alert = Div(text=' ', css_classes=['alert-style'])
self.img_subtitle = Div(text="Images", css_classes=['subt-style'])
self.img_upinst = Div(text="Include images in the Night Log by uploading a png image from your local computer. Select file, write a comment and click Add", css_classes=['inst-style'], width=1000)
self.img_upinst2 = Div(text=" Choose image to include with comment: ", css_classes=['inst-style'])
self.img_upload = FileInput(accept=".png")
self.img_upload.on_change('value', self.upload_image)
self.img_upload_comments = FileInput(accept=".png")
self.img_upload_comments.on_change('value', self.upload_image_comments)
self.img_upload_comments_os = FileInput(accept=".png")
self.img_upload_comments_os.on_change('value', self.upload_image_comments_os)
self.img_upload_problems = FileInput(accept=".png")
self.img_upload_problems.on_change('value', self.upload_image_problems)
self.img_inst = Div(text="Include images in the Night Log by entering the location of the images on the desi cluster", css_classes=['inst-style'], width=1000)
self.img_input = TextInput(title='image file location on desi cluster', placeholder='/n/home/desiobserver/image.png',value=None)
self.img_comment = TextAreaInput(placeholder='comment about image', rows=8, cols=3, title='Image caption')
self.img_btn = Button(label='Add', css_classes=['add_button'])
self.img_alert = Div(text=" ",width=1000)
self.plot_subtitle = Div(text="Telemetry Plots", css_classes=['subt-style'])
self.DESI_Log = None
self.save_telem_plots = False
def clear_input(self, items):
"""
After submitting something to the log, this will clear the form.
"""
if isinstance(items, list):
for item in items:
item.value = None
else:
items.value = None
def get_intro_layout(self):
intro_layout = layout([self.title,
[self.page_logo, self.instructions],
self.intro_subtitle,
[self.date_init, self.your_name],
[self.connect_bt],
self.connect_txt,
self.nl_info,
self.intro_txt], width=1000)
self.intro_tab = Panel(child=intro_layout, title="Initialization")
def get_checklist_layout(self):
checklist_layout = layout(self.title,
self.check_subtitle,
self.checklist_inst,
self.checklist,
self.check_comment,
[self.check_btn],
self.check_alert, width=1000)
self.check_tab = Panel(child=checklist_layout, title="DQS Checklist")
def get_prob_layout(self):
prob_layout = layout([self.title,
self.prob_subtitle,
self.prob_inst,
self.time_note,
[self.time_title, self.prob_time, self.now_btn, self.now_btn, self.img_upinst2, self.img_upload_problems],
self.prob_alarm,
[self.prob_input, self.prob_action],
[self.prob_btn],
self.prob_alert], width=1000)
self.prob_tab = Panel(child=prob_layout, title="Problems")
def get_plots_layout(self):
telem_data = pd.DataFrame(columns = ['tel_time','tower_time','exp_time','exp','mirror_temp','truss_temp','air_temp','humidity','wind_speed','airmass','exptime','seeing'])
self.telem_source = ColumnDataSource(telem_data)
plot_tools = 'pan,wheel_zoom,lasso_select,reset,undo,save'
p1 = figure(plot_width=800, plot_height=300, x_axis_label='UTC Time', y_axis_label='Temp (C)',x_axis_type="datetime", tools=plot_tools)
p2 = figure(plot_width=800, plot_height=300, x_axis_label='UTC Time', y_axis_label='Humidity (%)', x_axis_type="datetime",tools=plot_tools)
p3 = figure(plot_width=800, plot_height=300, x_axis_label='UTC Time', y_axis_label='Wind Speed (mph)', x_axis_type="datetime",tools=plot_tools)
p4 = figure(plot_width=800, plot_height=300, x_axis_label='UTC Time', y_axis_label='Airmass', x_axis_type="datetime",tools=plot_tools)
p5 = figure(plot_width=800, plot_height=300, x_axis_label='UTC Time', y_axis_label='Exptime (sec)', x_axis_type="datetime",tools=plot_tools)
p6 = figure(plot_width=800, plot_height=300, x_axis_label='Exposure', y_axis_label='Seeing (arcsec)', tools=plot_tools)
p1.circle(x = 'tel_time',y='mirror_temp',source=self.telem_source,color='orange', legend_label = 'Mirror', size=10, alpha=0.5)
p1.circle(x = 'tel_time',y='truss_temp',source=self.telem_source, legend_label = 'Truss', size=10, alpha=0.5)
p1.circle(x = 'tel_time',y='air_temp',source=self.telem_source, color='green',legend_label = 'Air', size=10, alpha=0.5)
p1.legend.location = "top_right"
p2.circle(x = 'tower_time',y='humidity',source=self.telem_source, size=10, alpha=0.5)
p3.circle(x = 'tower_time',y='wind_speed',source=self.telem_source, size=10, alpha=0.5)
p4.circle(x = 'exp_time',y='airmass',source=self.telem_source, size=10, alpha=0.5)
p5.circle(x = 'exp_time',y='exptime',source=self.telem_source, size=10, alpha=0.5)
p6.circle(x = 'exp',y='seeing',source=self.telem_source, size=10, alpha=0.5)
plot_layout = layout([self.title,
self.plot_subtitle,
p6,p1,p2,p3,p4,p5], width=1000)
self.plot_tab = Panel(child=plot_layout, title="Telemetry Plots")
def get_nl_layout(self):
exp_data = pd.DataFrame(columns = ['date_obs','id','program','sequence','flavor','exptime'])
self.explist_source = ColumnDataSource(exp_data)
columns = [TableColumn(field='date_obs', title='Time (UTC)', width=50, formatter=self.datefmt),
TableColumn(field='id', title='Exposure', width=50),
TableColumn(field='sequence', title='Sequence', width=100),
TableColumn(field='flavor', title='Flavor', width=50),
TableColumn(field='exptime', title='Exptime', width=50),
TableColumn(field='program', title='Program', width=300)]
self.exp_table = DataTable(source=self.explist_source, columns=columns, width=1000)
nl_layout = layout([self.title,
self.nl_subtitle,
self.nl_alert,
self.nl_text,
self.exptable_alert,
self.exp_table], width=1000)
self.nl_tab = Panel(child=nl_layout, title="Current DESI Night Log")
def get_img_layout(self):
img_layout = layout([self.title,
self.img_subtitle,
self.img_upinst,
self.img_upload,
self.img_inst,
self.img_input,
self.img_comment,
self.img_btn,
self.img_alert], width=1000)
self.img_tab = Panel(child=img_layout, title='Images')
def short_time(self, str_time):
"""Returns %H%M in whichever time zone selected
"""
try:
t = datetime.strptime(str_time, "%Y%m%dT%H:%M")
zone = self.kp_zone #zones[time_select.active]
time = datetime(t.year, t.month, t.day, t.hour, t.minute, tzinfo = zone)
return "{}:{}".format(str(time.hour).zfill(2), str(time.minute).zfill(2))
except:
return str_time
def get_time(self, time):
"""Returns strptime with utc. Takes time zone selection
"""
date = self.date_init.value
zone = self.kp_zone #zones[time_select.active]
try:
t = datetime.strptime(date+":"+time,'%Y%m%d:%H%M')
except:
try:
t = datetime.strptime(date+":"+time,'%Y%m%d:%I:%M%p')
except:
try:
t = datetime.strptime(date+":"+time,'%Y%m%d:%H:%M')
except:
pass #print("need format %H%M, %H:%M, %H:%M%p")
try:
tt = datetime(t.year, t.month, t.day, t.hour, t.minute, tzinfo = zone)
return tt.strftime("%Y%m%dT%H:%M")
except:
return time
def get_strftime(self, time):
date = self.date_init.value
year, month, day = int(date[0:4]), int(date[4:6]), int(date[6:8])
d = datetime(year, month, day)
dt = datetime.combine(d,time)
return dt.strftime("%Y%m%dT%H:%M")
def connect_log(self):
"""
Initialize Night Log with Input Date
"""
try:
date = datetime.strptime(self.date_init.value, '%Y%m%d')
except:
date = datetime.now()
self.night = str(date.year)+str(date.month).zfill(2)+str(date.day).zfill(2)
self.DESI_Log=nl.NightLog(str(date.year),str(date.month).zfill(2),str(date.day).zfill(2))
exists = self.DESI_Log.check_exists()
your_firstname, your_lastname = self.your_name.value.split(' ')[0], ' '.join(self.your_name.value.split(' ')[1:])
if exists:
self.connect_txt.text = 'Connected to Night Log for {}'.format(self.date_init.value)
meta_dict = self.DESI_Log.get_meta_data()
if self.report_type == 'DQS':
self.DESI_Log.add_dqs_observer(your_firstname, your_lastname)
self.your_name.value = meta_dict['{}_1'.format(self.report_type.lower())]+' '+meta_dict['{}_last'.format(self.report_type.lower())]
elif self.report_type == 'OS':
self.os_name_1.value = meta_dict['{}_1_first'.format(self.report_type.lower())]+' '+meta_dict['{}_1_last'.format(self.report_type.lower())]
self.os_name_2.value = meta_dict['{}_2_first'.format(self.report_type.lower())]+' '+meta_dict['{}_2_last'.format(self.report_type.lower())]
self.current_header()
#if self.location == 'nersc':
self.nl_file = os.path.join(self.DESI_Log.root_dir,'nightlog.html')
# else:
# self.nl_file = os.getcwd()+'/'+self.DESI_Log.root_dir+'nightlog.html'
self.nl_subtitle.text = "Current DESI Night Log: {}".format(self.nl_file)
if self.report_type == 'OS':
plan_txt_text="https://desi.lbl.gov/trac/wiki/DESIOperations/ObservingPlans/OpsPlan{}{}{}".format(date.year,str(date.month).zfill(2),str(date.day).zfill(2))
self.plan_txt.text = '<a href={}>Tonights Plan Here</a>'.format(plan_txt_text)
self.LO.value = meta_dict['os_lo_1']+' '+meta_dict['os_lo_last']
self.OA.value = meta_dict['os_oa_1']+' '+meta_dict['os_oa_last']
try:
self.weather_source.data = new_data
new_data = pd.read_csv(self.DESI_Log.weather_file)
new_data = new_data[['time','desc','temp','wind','humidity']]
except:
pass
if os.path.exists(self.DESI_Log.contributer_file):
cont_txt = ''
f = open(self.DESI_Log.contributer_file, "r")
for line in f:
cont_txt += line
self.contributer_list.value = cont_txt
if os.path.exists(self.DESI_Log.weather_file):
data = pd.read_csv(self.DESI_Log.weather_file)[['time','desc','temp','wind','humidity']]
self.weather_source.data = data
self.current_nl()
else:
self.connect_txt.text = 'The Night Log for this {} is not yet initialized.'.format(self.date_init.value)
def initialize_log(self):
"""
Initialize Night Log with Input Date
"""
date = datetime.now()
LO_firstname, LO_lastname = self.LO.value.split(' ')[0], ' '.join(self.LO.value.split(' ')[1:])
OA_firstname, OA_lastname = self.OA.value.split(' ')[0], ' '.join(self.OA.value.split(' ')[1:])
os_1_firstname, os_1_lastname = self.os_name_1.value.split(' ')[0], ' '.join(self.os_name_1.value.split(' ')[1:])
os_2_firstname, os_2_lastname = self.os_name_2.value.split(' ')[0], ' '.join(self.os_name_2.value.split(' ')[1:])
eph = sky_calendar()
time_sunset = self.get_strftime(eph['sunset'])
time_sunrise = self.get_strftime(eph['sunrise'])
time_moonrise = self.get_strftime(eph['moonrise'])
time_moonset = self.get_strftime(eph['moonset'])
illumination = eph['illumination']
dusk_18_deg = self.get_strftime(eph['dusk_astronomical'])
dawn_18_deg = self.get_strftime(eph['dawn_astronomical'])
self.DESI_Log=nl.NightLog(str(date.year),str(date.month).zfill(2),str(date.day).zfill(2))
self.DESI_Log.initializing()
self.DESI_Log.get_started_os(os_1_firstname,os_1_lastname,os_2_firstname,os_2_lastname,LO_firstname,LO_lastname,
OA_firstname,OA_lastname,time_sunset,dusk_18_deg,dawn_18_deg,time_sunrise,time_moonrise,time_moonset,illumination)
#update_weather_source_data()
self.connect_txt.text = 'Night Log is Initialized'
self.current_header()
self.current_nl()
days = os.listdir(self.nl_dir)
init_nl_list = np.sort([day for day in days if 'nightlog_meta.json' in os.listdir(os.path.join(self.nl_dir,day))])[::-1][0:10]
self.date_init.options = list(init_nl_list)
self.date_init.value = init_nl_list[0]
def current_header(self):
self.DESI_Log.write_intro()
path = os.path.join(self.DESI_Log.root_dir,"header.html")
nl_file = open(path,'r')
intro = ''
for line in nl_file:
intro = intro + line + '\n'
self.intro_txt.text = intro
nl_file.closed
def current_nl(self):
try:
now = datetime.now()
self.DESI_Log.finish_the_night()
path = os.path.join(self.DESI_Log.root_dir,"nightlog.html")
nl_file = open(path,'r')
nl_txt = ''
for line in nl_file:
nl_txt = nl_txt + line + '\n'
self.nl_text.text = nl_txt
nl_file.closed
self.nl_alert.text = 'Last Updated on this page: {}'.format(now)
self.nl_subtitle.text = "Current DESI Night Log: {}".format(path)
# DB connection
if self.location == 'kpno':
if not hasattr(self, 'conn') or self.conn is None:
self.conn = psycopg2.connect(host="desi-db", port="5442", database="desi_dev", user="******", password="******")
self.get_exp_list()
self.get_seeing()
try:
self.make_telem_plots()
except:
#print('Something wrong with making telemetry plots')
pass
if self.location == 'kpno':
if hasattr(self, 'conn') and self.conn is not None:
try:
self.conn.close()
except:
pass
self.conn = None
return True
except Exception as e:
print('current_nl Exception: %s' % str(e))
self.nl_alert.text = 'You are not connected to a Night Log'
return False
def get_exp_list(self):
if self.location == 'kpno':
exp_df = pd.read_sql_query(f"SELECT * FROM exposure WHERE night = '{self.night}'", self.conn)
if len(exp_df.date_obs) != 0:
time = exp_df.date_obs.dt.tz_convert('US/Arizona')
exp_df['date_obs'] = time
self.explist_source.data = exp_df[['date_obs','id','program','sequence','flavor','exptime']].sort_values(by='id',ascending=False)
exp_df = exp_df.sort_values(by='id')
exp_df.to_csv(self.DESI_Log.explist_file, index=False)
else:
self.exptable_alert.text = f'No exposures available for night {self.night}'
else:
self.exptable_alert.text = 'Cannot connect to Exposure Data Base'
def get_seeing(self):
#self.seeing_df = pd.DataFrame()
seeing = []
exps = []
exposures = pd.DataFrame(self.explist_source.data)['id']
for exp in list(exposures):
try:
folder = '/data/platemaker/test/{}/'.format(int(exp))
filen = os.path.join(folder,'qc-gfaproc-{}.?.info'.format(int(exp)))
f = glob.glob(filen)[-1]
try:
lines = open(f,'r').readlines()
line = lines[5].strip()
x0 = line.find('=')
x1 = line.find(',')
s = float(line[x0+1:x1])
except:
s = np.nan
seeing.append(s)
exps.append(exp)
except:
pass
self.seeing_df = pd.DataFrame()
self.seeing_df['Seeing'] = seeing
self.seeing_df['Exps'] = exps
self.seeing_df.to_csv(os.path.join(self.DESI_Log.root_dir,'seeing.csv'),index=False)
# KH added to avoid issues with X11 and Display (we only need the png)
figure = plt.figure()
plt.plot(self.seeing_df.Exps, self.seeing_df.Seeing,'o')
plt.xlabel("Exposure")
plt.ylabel("Seeing (arcsec)")
plt.savefig(os.path.join(self.DESI_Log.root_dir,'seeing.png'))
plt.close(figure)
def make_telem_plots(self):
start_utc = '{} {}'.format(int(self.night)+1, '00:00:00')
end_utc = '{} {}'.format(int(self.night)+1, '13:00:00')
tel_df = pd.read_sql_query(f"SELECT * FROM environmentmonitor_telescope WHERE time_recorded > '{start_utc}' AND time_recorded < '{end_utc}'", self.conn)
exp_df = pd.read_sql_query(f"SELECT * FROM exposure WHERE night = '{self.night}'", self.conn)
tower_df = pd.read_sql_query(f"SELECT * FROM environmentmonitor_tower WHERE time_recorded > '{start_utc}' AND time_recorded < '{end_utc}'", self.conn)
#self.get_seeing()
telem_data = pd.DataFrame(columns = ['tel_time','tower_time','exp_time','exp','mirror_temp','truss_temp','air_temp','humidity','wind_speed','airmass','exptime','seeing'])
telem_data.tel_time = tel_df.time_recorded.dt.tz_convert('US/Arizona')
telem_data.tower_time = tower_df.time_recorded.dt.tz_convert('US/Arizona')
telem_data.exp_time = exp_df.date_obs.dt.tz_convert('US/Arizona')
telem_data.exp = self.seeing_df.Exps
telem_data.mirror_temp = tel_df.mirror_temp
telem_data.truss_temp = tel_df.truss_temp
telem_data.air_temp = tel_df.air_temp
telem_data.humidity = tower_df.humidity
telem_data.wind_speed = tower_df.wind_speed
telem_data.airmass = exp_df.airmass
telem_data.exptime = exp_df.exptime
telem_data.seeing = self.seeing_df.Seeing
self.telem_source.data = telem_data
if self.save_telem_plots:
if set(list(exp_df.seeing)) == set([None]):
sky_monitor = False
fig = plt.figure(figsize= (8,15))
#fig, axarr = plt.subplots(5, 1, figsize = (8,15), sharex=True)
else:
sky_monitor = True
fig, axarr = plt.subplots(9, 1, figsize = (8,20), sharex=True)
ax1 = fig.add_subplot(6,1,1)
ax1.scatter(telem_data.exp, telem_data.seeing, s=5, label='Seeing')
ax1.set_ylabel("Seeing (arcsec)")
ax1.grid(True)
ax1.set_xlabel("Exposure")
#ax = axarr.ravel()
ax2 = fig.add_subplot(6,1,2)
ax2.scatter(tel_df.time_recorded.dt.tz_convert('US/Arizona'), tel_df.mirror_temp, s=5, label='mirror temp')
ax2.scatter(tel_df.time_recorded.dt.tz_convert('US/Arizona'), tel_df.truss_temp, s=5, label='truss temp')
ax2.scatter(tel_df.time_recorded.dt.tz_convert('US/Arizona'), tel_df.air_temp, s=5, label='air temp')
ax2.set_ylabel("Temperature (C)")
ax2.legend()
ax2.grid(True)
ax2.tick_params(labelbottom=False)
ax3 = fig.add_subplot(6,1,3, sharex = ax2)
ax3.scatter(tower_df.time_recorded.dt.tz_convert('US/Arizona'), tower_df.humidity, s=5, label='humidity')
ax3.set_ylabel("Humidity %")
ax3.grid(True)
ax3.tick_params(labelbottom=False)
ax4 = fig.add_subplot(6,1,4, sharex=ax2)
ax4.scatter(tower_df.time_recorded.dt.tz_convert('US/Arizona'), tower_df.wind_speed, s=5, label='wind speed')
ax4.set_ylabel("Wind Speed (mph)")
ax4.grid(True)
ax4.tick_params(labelbottom=False)
ax5 = fig.add_subplot(6,1,5, sharex=ax2)
ax5.scatter(exp_df.date_obs.dt.tz_convert('US/Arizona'), exp_df.airmass, s=5, label='airmass')
ax5.set_ylabel("Airmass")
ax5.grid(True)
ax5.tick_params(labelbottom=False)
ax6 = fig.add_subplot(6,1,6, sharex=ax2)
ax6.scatter(exp_df.date_obs.dt.tz_convert('US/Arizona'), exp_df.exptime, s=5, label='exptime')
ax6.set_ylabel("Exposure time (s)")
ax6.grid(True)
#ax1 = fig.add_subplot(6,1,6)
#ax1.scatter(telem_data.exp, telem_data.seeing, s=5, label='Seeing')
#ax1.set_ylabel("Seeing (arcsec)")
if sky_monitor:
ax[6].scatter(exp_df.date_obs.dt.tz_convert('US/Arizona'), exp_df.seeing, s=5, label='seeing')
ax[6].set_ylabel("Seeing")
ax[7].scatter(exp_df.date_obs.dt.tz_convert('US/Arizona'), exp_df.transpar, s=5, label='transparency')
ax[7].set_ylabel("Transparency (%)")
ax[8].scatter(exp_df.date_obs.dt.tz_convert('US/Arizona'), exp_df.skylevel, s=5, label='Sky Level')
ax[8].set_ylabel("Sky level (AB/arcsec^2)")
#for i in range(len(ax)):
# ax[i].grid(True)
#X label ticks as local time
ax6.set_xlabel("Local Time (MST)")
ax6.xaxis.set_major_formatter(mdates.DateFormatter('%m/%d %H:%M', tz=pytz.timezone("US/Arizona")))
ax6.tick_params(labelrotation=45)
fig.suptitle("Telemetry for obsday={}".format(self.night))
fig.tight_layout()
plt.savefig(self.DESI_Log.telem_plots_file)
def exp_to_html(self):
exp_df = pd.read_csv(self.DESI_Log.explist_file)
exp_df = exp_df[['night','id','program','sequence','flavor','exptime']]
exp_html = exp_df.to_html()
return exp_html
def check_add(self):
"""add checklist time to Night Log
"""
complete = self.checklist.active
check_time = datetime.now().strftime("%Y%m%dT%H:%M")
if len(complete) == len(self.checklist.labels):
self.DESI_Log.add_to_checklist(check_time, self.check_comment.value, self.report_type)
self.check_alert.text = "Checklist last submitted at {}".format(check_time[-5:])
else:
self.check_alert.text = "Must complete all tasks before submitting checklist"
self.clear_input(self.check_comment)
self.checklist.active = []
def prob_add(self):
"""Adds problem to nightlog
"""
# Was an image uploaded?
if hasattr(self, 'img_upload_problems') and self.img_upload_problems.filename not in [None, '']:
img_data = self.img_upload_problems.value.encode('utf-8')
img_name = str(self.img_upload_problems.filename)
self.img_upload_problems.filename = None
else:
img_data = None
img_name = None
if self.report_type == 'Other':
self.DESI_Log.add_problem(self.get_time(self.prob_time.value), self.prob_input.value, self.prob_alarm.value,
self.prob_action.value,self.report_type, self.your_name.value,
img_name=img_name, img_data=img_data)
else:
self.DESI_Log.add_problem(self.get_time(self.prob_time.value), self.prob_input.value, self.prob_alarm.value,
self.prob_action.value,self.report_type,
img_name=img_name, img_data=img_data)
# Preview
if img_name != None:
# move to class initialization
image_location_on_server = f'http://desi-www.kpno.noao.edu:8090/nightlogs/{self.night}/images/{img_name}'
preview = '<img src="{}" style="width:300px;height:300px;">'.format(image_location_on_server)
preview += "<br>"
preview += "Last Problem Input: '{}' at {}".format(self.prob_input.value, self.prob_time.value)
self.prob_alert.text = preview
else:
self.prob_alert.text = "Last Problem Input: '{}' at {}".format(self.prob_input.value, self.prob_time.value)
self.clear_input([self.prob_time, self.prob_input, self.prob_alarm, self.prob_action])
def plan_add(self):
self.DESI_Log.add_plan_os([self.plan_order.value, self.plan_input.value])
self.plan_alert.text = 'Last item input: {}'.format(self.plan_input.value)
self.clear_input([self.plan_order, self.plan_input])
def milestone_add(self):
now = datetime.now()
self.DESI_Log.add_milestone_os([self.milestone_input.value, self.milestone_exp_start.value, self.milestone_exp_end.value, self.milestone_exp_excl.value])
self.milestone_alert.text = 'Last Milestone Entered: {} at {}'.format(self.milestone_input.value, now)
self.clear_input([self.milestone_input, self.milestone_exp_start, self.milestone_exp_end, self.milestone_exp_excl])
def weather_add(self):
"""Adds table to Night Log
"""
new_data = pd.DataFrame([[self.weather_time.value, self.weather_desc.value, self.weather_temp.value, self.weather_wind.value, self.weather_humidity.value]],
columns = ['time','desc','temp','wind','humidity'])
old_data = pd.DataFrame(self.weather_source.data)[['time','desc','temp','wind','humidity']]
data = pd.concat([old_data, new_data])
data.drop_duplicates(subset=['time'], keep='last',inplace=True)
self.weather_source.data = data
self.DESI_Log.add_weather_os(data)
self.clear_input([self.weather_time, self.weather_desc, self.weather_temp, self.weather_wind, self.weather_humidity])
def progress_add(self):
if self.exp_time.value not in [None, 'None'," ", ""]:
data = [self.hdr_type.value, self.get_time(self.exp_time.value), self.exp_comment.value, self.exp_exposure_start.value, self.exp_exposure_finish.value,self.exp_type.value, self.exp_script.value, self.get_time(self.exp_time_end.value), self.exp_focus_trim.value, self.exp_tile.value, self.exp_tile_type.value]
# For comments allow image uploads
has_image = False
if self.exp_comment.value not in [None, ''] and hasattr(self, 'img_upload_comments_os') and self.img_upload_comments_os.filename not in [None,'']:
if self.img_upload_comments_os.filename not in [None, '']:
img_data = self.img_upload_comments_os.value.encode('utf-8')
img_name = str(self.img_upload_comments_os.filename)
has_image = True
self.DESI_Log.add_progress(data, img_name = img_name, img_data = img_data)
# move to class initialization
image_location_on_server = f'http://desi-www.kpno.noao.edu:8090/nightlogs/{self.night}/images/{img_name}'
preview = '<img src="{}" style="width:300px;height:300px;">'.format(image_location_on_server)
preview += "<br>"
preview += "A comment was added at {}".format(self.exp_time.value)
self.exp_alert.text = preview
self.img_upload_comments_os.filename=None
if not has_image:
self.DESI_Log.add_progress(data)
self.exp_alert.text = 'Last Input was for Observation Type *{}* at {}'.format(self.hdr_type.value, self.exp_time.value)
self.clear_input([self.exp_time, self.exp_comment, self.exp_exposure_start, self.exp_exposure_finish, self.exp_type,
self.exp_script,self.exp_time_end, self.exp_focus_trim, self.exp_tile, self.exp_tile_type])
else:
self.exp_alert.text = 'Could not submit entry for Observation Type *{}* because not all mandatory fields were filled.'.format(self.hdr_type.value)
def comment_add(self):
if self.your_name.value in [None,' ','']:
self.comment_alert.text = 'You need to enter your name on first page before submitting a comment'
else:
has_image = False
if hasattr(self, 'img_upload_comments'):
if self.img_upload_comments.filename not in [None, '']:
img_data = self.img_upload_comments.value.encode('utf-8')
image_name = str(self.img_upload_comments.filename)
has_image = True
self.DESI_Log.upload_image_comments(img_data, self.get_time(self.exp_time.value), self.exp_comment.value,
self.your_name.value, image_name)
# move to class initialization
image_location_on_server = f'http://desi-www.kpno.noao.edu:8090/nightlogs/{self.night}/images/{image_name}'
preview = '<img src="{}" style="width:300px;height:300px;">'.format(image_location_on_server)
preview += "<br>"
preview += "A comment was added at {}".format(self.exp_time.value)
self.comment_alert.text = preview
self.img_upload_comments.filename=None
self.clear_input([self.exp_time, self.exp_comment])
if not has_image:
self.DESI_Log.add_comment_other(self.get_time(self.exp_time.value), self.exp_comment.value, self.your_name.value)
self.comment_alert.text = "A comment was added at {}".format(self.exp_time.value)
self.clear_input([self.exp_time, self.exp_comment])
def add_contributer_list(self):
cont_list = self.contributer_list.value
self.DESI_Log.add_contributer_list(cont_list)
def add_summary(self):
now = datetime.now()
summary = self.summary.value
self.DESI_Log.add_summary(summary)
self.clear_input([self.summary])
self.milestone_alert.text = 'Summary Entered at {}'.format(self.milestone_input.value, now)
def upload_image(self, attr, old, new):
print(f'Local image file upload: {self.img_upload.filename}')
def upload_image_comments_os(self, attr, old, new):
print(f'Local image file upload (OS comments): {self.img_upload_comments_os.filename}')
def upload_image_comments(self, attr, old, new):
print(f'Local image file upload (Other comments): {self.img_upload_comments.filename}')
def upload_image_problems(self, attr, old, new):
print(f'Local image file upload (Other comments): {self.img_upload_problems.filename}')
def image_add(self):
"""Copies image from the input location to the image folder for the nightlog.
Then calls add_image() from nightlog.py which writes it to the html file
Then gives preview of image of last image.
"""
image_loc = self.img_input.value
if image_loc in [None, '']:
# got a local upload
img_data = self.img_upload.value.encode('utf-8')
self.DESI_Log.upload_image(img_data, self.img_comment.value, self.img_upload.filename)
# move to class initialization
image_location_on_server = f'http://desi-www.kpno.noao.edu:8090/nightlogs/{self.night}/images/{self.img_upload.filename}'
preview = '<img src="{}" style="width:300px;height:300px;">'.format(image_location_on_server)
preview += "<br>"
preview += "{}".format(self.img_comment.value)
self.img_alert.text = preview
return
image_name = os.path.split(image_loc)[1]
image_type = os.path.splitext(image_name)[1]
bashCommand1 = "cp {} {}".format(image_loc,self.DESI_Log.image_dir)
bashCommand2 = "cp {} {}".format(image_loc,self.report_type+"_Report/static/images/tmp_img{}".format(image_type))
results = subprocess.run(bashCommand1.split(), text=True, stdout=subprocess.PIPE, check=True)
results = subprocess.run(bashCommand2.split(), text=True, stdout=subprocess.PIPE, check=True)
self.DESI_Log.add_image(os.path.join(self.DESI_Log.image_dir,image_name), self.img_comment.value)
#KH: this is something you need to reconsider. It is bad practice to write into the source code directory
preview = '<img src="{}_Report/static/images/tmp_img{}" style="width:300px;height:300px;">'.format(self.report_type,image_type)
preview += "\n"
preview += "{}".format(self.img_comment.value)
self.img_alert.text = preview
self.clear_input([self.img_input, self.img_comment])
def time_is_now(self):
now = datetime.now().astimezone(tz=self.kp_zone)
now_time = self.short_time(datetime.strftime(now, "%Y%m%dT%H:%M"))
tab = self.layout.active
time_input = self.time_tabs[tab]
time_input.value = now_time
def nl_submit(self):
if not self.current_nl():
self.nl_text.text = 'You cannot submit a Night Log to the eLog until you have connected to an existing Night Log or initialized tonights Night Log'
else:
try:
from ECLAPI import ECLConnection, ECLEntry
except ImportError:
ECLConnection = None
self.nl_text.text = "Can't connect to eLog"
f = self.nl_file[:-5]
nl_file=open(f,'r')
lines = nl_file.readlines()
nl_html = ' '
for line in lines:
nl_html += line
e = ECLEntry('Synopsis_Night', text=nl_html, textile=True)
subject = 'Night Summary {}-{}-{}'.format(self.date_init.value[0:4], self.date_init.value[4:6], self.date_init.value[6:])
e.addSubject(subject)
url = 'http://desi-www.kpno.noao.edu:8090/ECL/desi'
user = '******'
pw = 'dosuser'
if self.test:
pass
else:
elconn = ECLConnection(url, user, pw)
response = elconn.post(e)
elconn.close()
if response[0] != 200:
raise Exception(response)
self.nl_text.text = "You cannot post to the eLog on this machine"
self.save_telem_plots = True
self.current_nl()
nl_text = "Night Log posted to eLog and emailed to collaboration" + '</br>'
self.nl_text.text = nl_text
if self.test:
self.email_nightsum(user_email = ["*****@*****.**","*****@*****.**"])
else:
self.email_nightsum(user_email = ["*****@*****.**","*****@*****.**","*****@*****.**"])
def email_nightsum(self,user_email = None):
if self.location == 'kpno':
try:
self.make_telem_plots()
except:
print("Something wrong with telem plots")
sender = "*****@*****.**"
# Create message container - the correct MIME type is multipart/alternative.
msg = MIMEMultipart('html')
msg['Subject'] = "Night Summary %s" % self.date_init.value #mjd2iso(mjd)
msg['From'] = sender
if len(user_email) == 1:
msg['To'] = user_email[0]
else:
msg['To'] = ', '.join(user_email)
# Create the body of the message (a plain-text and an HTML version).
f = self.nl_file
nl_file=open(f,'r')
lines = nl_file.readlines()
nl_html = ' '
img_names = []
for line in lines:
nl_html += line
# Add images
x = nl_html.find('Images')
nl_html = nl_html[:x]
nl_html += "<h3 id='images'>Images</h3>"
nl_html += '\n'
if os.path.exists(self.DESI_Log.image_file):
images = os.listdir(self.DESI_Log.image_dir)
images = [s for s in images if os.path.splitext(s)[1] != '']
f = open(self.DESI_Log.image_file,'r')
image_lines = f.readlines()
for ii, line in enumerate(image_lines):
for i, img in enumerate(images):
if img in line:
nl_html += '<img src="cid:image{}" style="width:300px;height:300px;">'.format(i)
nl_html += '\n'
nl_html += '{}'.format(image_lines[ii+1])
nl_html += '\n'
# Add exposures
if os.path.exists(self.DESI_Log.explist_file):
exp_list = self.exp_to_html()
nl_html += ("<h3 id='exposures'>Exposures</h3>")
for line in exp_list:
nl_html += line
# Add telem plots
nl_html += "<h3 id='telem_plots'>Night Telemetry</h3>"
nl_html += '\n'
if os.path.exists(self.DESI_Log.telem_plots_file):
nl_html += '<img src="{}">'.format(self.DESI_Log.telem_plots_file)
nl_html += '\n'
Html_file = open(os.path.join(self.DESI_Log.root_dir,'NightSummary{}'.format(self.night)),"w")
Html_file.write(nl_html)
Html_file.close()
# Record the MIME types of both parts - text/plain and text/html.
part1 = MIMEText(nl_html, 'plain')
part2 = MIMEText(nl_html, 'html')
# Attach parts into message container.
# According to RFC 2046, the last part of a multipart message, in this case
# the HTML message, is best and preferred.
#msg.attach(part1)
msg.attach(part2)
# Add images
if os.path.exists(self.DESI_Log.image_file):
for i, img in enumerate(images):
fp = open(os.path.join(self.DESI_Log.image_dir,img), 'rb')
msgImage = MIMEImage(fp.read())
fp.close()
msgImage.add_header('Content-ID', '<image{}>'.format(i))
msg.attach(msgImage)
if os.path.exists(self.DESI_Log.telem_plots_file):
fp = open(self.DESI_Log.telem_plots_file, 'rb')
msgImage = MIMEImage(fp.read())
fp.close()
msg.attach(msgImage)
text = msg.as_string()
# Send the message via local SMTP server.
#yag = yagmail.SMTP(sender)
#yag.send("*****@*****.**",nl_html,self.DESI_Log.telem_plots_file)
s = smtplib.SMTP('localhost')
s.sendmail(sender, user_email, text)
s.quit()
skipinitialspace=True)
metadata.columns = ['Product ID', 'Description', 'price', 'Category']
def upload_dataset(attr, old, new):
global dataset
dataset = pd.read_csv(io.BytesIO(base64.b64decode(
dataset_file_input.value)),
skipinitialspace=True)
dataset.columns = ['asin', 'reviewerID', 'overall', 'unixReviewTime']
pre_dataset = PreText(text="""Reviews Dataset CSV File""",
width=200,
height=10)
dataset_file_input = FileInput(accept=".csv")
dataset_file_input.on_change('value', upload_dataset)
def upload_metadata(attr, old, new):
global metadata
metadata = pd.read_csv(io.BytesIO(
base64.b64decode(metadata_file_input.value)),
skipinitialspace=True)
metadata.columns = ['Product ID', 'Description', 'price', 'Category']
pre_metadata = PreText(text="""Metadata CSV File""", width=150, height=10)
metadata_file_input = FileInput(accept=".csv")
metadata_file_input.on_change('value', upload_metadata)
error_message.text+="<br>Error: Title is too long!"
if title_input.value == "":
errors=True
error_message.text+="<br>Error: Abstract cannot be empty"
if len(abstract_input.value) > 10000:
errors=True
error_message.text += "<br>Error: Abstract is too long!"
return errors
button = Button(label='Predict citations', button_type='success', aspect_ratio=3)
# button.js_on_click(CustomJS(code="console.log('button: click!', this.toString())"))
button.on_click(update_output)
file_input = FileInput(accept=".txt,.pdf", name="Import paper here")
controls = [file_input, title_input, author_number, abstract_input, button, error_message]
displays = [p, grid]
inputs = column(controls, width=500, height=1000)
outputs = column(displays, width=500, height=100)
inputs2 = column(Div(text="<h1>Inputs</h1>"), inputs)
outputs2 = column(Div(text="<h1>Outputs</h1>"), outputs)
final_row = row(inputs2, outputs2)
pan_oi = Panel(child=final_row, title="Citations")
tabs = Tabs(tabs=[pan_oi])
curdoc().add_root(tabs)
