def get_data_source(user_id, data_id, tz_str=None):
d = get_data_page(user_id, data_id)
d.name = 'data'
if tz_str is not None:
offset_s = tz_offset_seconds(tz_str)
log.debug("offseting date with {} seconds for {}".format(
offset_s, tz_str))
d = d.tshift(offset_s, freq='s')
source = ColumnDataSource(pd.DataFrame(d))
def on_latest_change(attr, old, new):
log.info('on_latest_change {} old={} new={}'.format(attr, old, new))
source.on_change('data', on_latest_change)
return source
class Plot:
WIDTH = 800
HEIGHT = 600
def __init__(self, shifts, multiplicities, deviations):
self.shifts = shifts
self.multiplicities = multiplicities
self.deviations = deviations
xr = Range1d(start=220, end=-20)
self.plot = figure(x_axis_label="ppm", x_range=xr, tools="save,reset", plot_width=self.WIDTH, plot_height=self.HEIGHT)
self.lineSource = ColumnDataSource(data=dict(x=[0], y=[0]))
self.plot.line('x', 'y', source=self.lineSource, line_width=2)
# Remove grid from plot
self.plot.xgrid.grid_line_color = None
self.plot.ygrid.grid_line_color = None
# Remove bokeh logo
self.plot.toolbar.logo = None
horizontalBoxZoomTool = HorizontalBoxZoomTool()
self.plot.add_tools(horizontalBoxZoomTool)
fixedZoomOutTool = FixedZoomOutTool(factor=0.4)
self.plot.add_tools(fixedZoomOutTool)
self.plot.extra_y_ranges['box'] = Range1d(start=0, end=0.1)
self.selectionSource = ColumnDataSource(data=dict(left=[], right=[]), callback=CustomJS(code="""
if ('data' in this.selected) {
window.top.location.href = "http://localhost:8080?" + this.selected.data.query + "style=plot";
}
"""))
self.selectionSource.on_change('selected', lambda attr, old, new: self.delete(new['1d']['indices']))
rect = HBar(
left='left',
right='right',
y=0.5,
height=1,
line_alpha=0.2,
fill_alpha=0.2,
line_color="red",
fill_color="red"
)
renderer = self.plot.add_glyph(self.selectionSource, rect)
renderer.y_range_name = "box"
self.labelSource = ColumnDataSource(data=dict(x=[], y=[], text=[]))
label = Text(
x='x',
y='y',
text='text',
text_align='center',
text_font_size="10pt"
)
renderer = self.plot.add_glyph(self.labelSource, label)
renderer.y_range_name = "box"
removeTool = CustomTapTool()
self.plot.add_tools(removeTool)
self.plot.toolbar.active_tap = None
callback = CustomJS(args=dict(), code="""
/// get BoxSelectTool dimensions from cb_data parameter of Callback
var geometry = cb_data['geometry'];
var shift = (geometry['x0'] + geometry['x1']) / 2;
var deviation = Math.abs(geometry['x1'] - shift);
var query = cb_data['query'] + "shift%5B%5D=" + shift + '&multiplicity%5B%5D=any&deviation%5B%5D=' + deviation + '&style=plot';
window.top.location.href = "http://localhost:8080?" + query;
""")
selectTool = CustomBoxSelectTool(
tool_name="Select Area",
dimensions = "width",
callback = callback,
query=self.paramsToQuery()
)
self.plot.add_tools(selectTool)
self.initSelect()
self.drawButton = CustomButton(id='myButton')
self.drawButton.on_click(self.drawPlot)
curdoc().add_root(
row(
column(row(self.plot)),
column(CustomRow(column(self.drawButton), hide=True))
)
)
def paramsToQuery(self):
query = ""
for (shift, multiplicity, deviation) in zip(self.shifts, self.multiplicities, self.deviations):
query += "shift%5B%5D={}&multiplicity%5B%5D={}&deviation%5B%5D={}&".format(shift, multiplicity, deviation)
return query
def initSelect(self):
left, right = [], []
labelX, labelY, labelText = [], [], []
for (shift, multiplicity, deviation) in zip(self.shifts, self.multiplicities, self.deviations):
left.append(shift - deviation)
right.append(shift + deviation)
labelX.append(shift)
labelY.append(0.08)
labelText.append(multiplicity)
self.selectionSource.stream({
'left': left,
'right': right
})
self.labelSource.stream({
'x': labelX,
'y': labelY,
'text': labelText
})
def drawPlot(self, data):
dic, _ = ng.bruker.read("../data/{}".format(data['id']))
_, pdata = ng.bruker.read_pdata("../data/{}/pdata/1/".format(data['id']))
udic = ng.bruker.guess_udic(dic, pdata)
uc = ng.fileiobase.uc_from_udic(udic)
ppmScale = uc.ppm_scale()
self.plot.y_range = None
self.lineSource.data = {
'x': ppmScale,
'y': pdata
}
def delete(self, ids):
if ids:
left = list(self.selectionSource.data['left'])
right = list(self.selectionSource.data['right'])
for i in ids:
left.pop(i)
right.pop(i)
self.shifts.pop(i)
self.multiplicities.pop(i)
self.deviations.pop(i)
self.selectionSource.data = {
'left': left,
'right': right
}
# Deselect all
self.selectionSource.selected = {
'0d': {'glyph': None, 'indices': []},
'1d': {'indices': []},
'2d': {'indices': {}},
'data': {'query': self.paramsToQuery()}
}
def selectArea(self, dimensions):
patch = {
'left': [dimensions['x0']],
'right': [dimensions['x1']]
}
self.selectionSource.stream(patch)
class RDFToGeoJSON:
def __init__(self, layout):
self.__layout = layout
self.__selected = []
self.__file_list = self.__get_eurostats()
self.__rdf_table_source = ColumnDataSource(
dict(id=[f['id'] for f in self.__file_list]))
self.__rdf_table_source.on_change('selected', self.__on_select)
rdf_table_columns = [TableColumn(field='id', title='RDF ID')]
self.rdf_table = DataTable(source=self.__rdf_table_source,
columns=rdf_table_columns,
width=300,
height=500,
selectable=True)
geojson_data = {'id': [], 'lvl': []}
for file in self.__file_list:
if file['geojson']['nuts1']['exists']:
geojson_data['id'].append(file['id'])
geojson_data['lvl'].append(1)
if file['geojson']['nuts2']['exists']:
geojson_data['id'].append(file['id'])
geojson_data['lvl'].append(2)
if file['geojson']['nuts3']['exists']:
geojson_data['id'].append(file['id'])
geojson_data['lvl'].append(3)
self.__geojson_table_source = ColumnDataSource(geojson_data)
geojson_table_columns = [
TableColumn(field='lvl', title='NUTS Level'),
TableColumn(field='id', title='ID')
]
self.geojson_table = DataTable(source=self.__geojson_table_source,
columns=geojson_table_columns,
width=300,
height=500,
selectable=True)
convert_button = Button(label="Convert to GeoJSON",
button_type="success")
convert_button.on_click(self.transform)
# todo: creates bug - empty table
#self.__layout.children[1] = column(widgetbox(self.rdf_table), convert_button)
self.__layout.children[2] = column(self.geojson_table)
def __on_select(self, attr, old, new):
self.__selected = [
self.__file_list[index] for index in new['1d']['indices']
]
def transform(self):
print("converting")
for f in self.__selected:
print(".")
f['results'] = self.__extract_observations(f['rdf'])
print("writing")
self.__write_geojson(self.__selected)
print("done converting")
self.__file_list = self.__get_eurostats()
geojson_data = {'id': [], 'lvl': []}
for file in self.__file_list:
if file['geojson']['nuts1']['exists']:
geojson_data['id'].append(file['id'])
geojson_data['lvl'].append(1)
if file['geojson']['nuts2']['exists']:
geojson_data['id'].append(file['id'])
geojson_data['lvl'].append(2)
if file['geojson']['nuts3']['exists']:
geojson_data['id'].append(file['id'])
geojson_data['lvl'].append(3)
self.__geojson_table_source = ColumnDataSource(geojson_data)
geojson_table_columns = [
TableColumn(field='lvl', title='NUTS Level'),
TableColumn(field='id', title='ID')
]
self.geojson_table = DataTable(source=self.__geojson_table_source,
columns=geojson_table_columns,
width=300,
height=500,
selectable=True)
self.__layout.children[2] = column(self.geojson_table)
def __get_eurostats(self):
"""
This function generates the file names for every RDF in the "data/rdf/eurostats" subdirectory.
:return: A list of dictionaries containing the id and file names of the RDFs found.
"""
rdf_path_prefix = "data/rdf/eurostats/"
geojson_path_prefix = "data/geojson/eurostats/"
observation_list = []
nuts_files = [
[
str(os.path.basename(file).split('.')[0])
for file in os.listdir(geojson_path_prefix + "nuts_1/")
],
[
str(os.path.basename(file).split('.')[0])
for file in os.listdir(geojson_path_prefix + "nuts_2/")
],
[
str(os.path.basename(file).split('.')[0])
for file in os.listdir(geojson_path_prefix + "nuts_3/")
]
]
for file in os.listdir(rdf_path_prefix):
observation = {}
observation_name = str(os.path.basename(file).split('.')[0])
observation['id'] = observation_name
observation['rdf'] = rdf_path_prefix + file
observation['geojson'] = {
'nuts1': {
'path': geojson_path_prefix + "nuts_1/" +
observation_name + ".geojson",
'exists': observation_name in nuts_files[0]
},
'nuts2': {
'path': geojson_path_prefix + "nuts_2/" +
observation_name + ".geojson",
'exists': observation_name in nuts_files[1]
},
'nuts3': {
'path': geojson_path_prefix + "nuts_3/" +
observation_name + ".geojson",
'exists': observation_name in nuts_files[2]
},
}
observation_list.append(observation)
return observation_list
def __extract_observations(self, file):
g = rdf.Graph()
g.parse(file, format="xml")
return g.query("""
prefix obs: <http://purl.org/linked-data/sdmx/2009/measure#>
prefix prop: <http://eurostat.linked-statistics.org/property#>
prefix qb: <http://purl.org/linked-data/cube#>
prefix sdmx-dimension: <http://purl.org/linked-data/sdmx/2009/dimension#>
select distinct ?designation ?time ?value ?unit
where {
?observation a qb:Observation.
?observation prop:geo ?designation.
?observation prop:unit ?unit.
?observation sdmx-dimension:timePeriod ?time.
?observation obs:obsValue ?value.
}
""")
def __write_geojson(self, file_list):
with open(
'data/geojson/eurostats/nuts_rg_60M_2013_lvl_3.geojson') as f:
nuts3 = json.load(f)
with open(
'data/geojson/eurostats/nuts_rg_60M_2013_lvl_2.geojson') as f:
nuts2 = json.load(f)
with open(
'data/geojson/eurostats/nuts_rg_60M_2013_lvl_1.geojson') as f:
nuts1 = json.load(f)
nuts = [nuts1, nuts2, nuts3]
for file in file_list:
geojson = copy.deepcopy(nuts)
self.__process_file(file, geojson)
for lvl in range(0, len(geojson)):
with open(file['geojson']['nuts{}'.format(lvl + 1)]['path'],
'w') as outfile:
json.dump(geojson[lvl], fp=outfile, indent=4)
def __process_file(self, file, nuts):
for row in file['results']:
# recover uncluttered information from the sparql result
geo = row[0].split('#')[1]
time = row[1]
value = row[2]
unit = row[3].split('#')[1]
# search for the NUTS_ID (geo) in the NUTS level 1 to 3
index = -1
nuts_lvl = -1
found = False
while not found:
index += 1
done = []
# prepare break condition
for lvl in range(0, len(nuts)):
done.append(False)
# check if the ID matches in any of the NUTS levels
for lvl in range(0, len(nuts)):
if index < len(nuts[lvl]['features']):
if nuts[lvl]['features'][index]['properties'][
'NUTS_ID'] == geo:
nuts_lvl = lvl
found = True
break
else:
done[lvl] = True
if all(done):
break
if nuts_lvl != -1:
observation = {'period': time, 'unit': unit, 'value': value}
# check if any of the nested elements in the JSON already exist
if 'OBSERVATIONS' in nuts[nuts_lvl]['features'][index][
'properties']:
if file['id'] in nuts[nuts_lvl]['features'][index][
'properties']['OBSERVATIONS']:
duplicate = False
for observations in nuts[nuts_lvl]['features'][index][
'properties']['OBSERVATIONS'][file['id']]:
if observations['period'] == observation['period']:
duplicate = True
break
if not duplicate:
nuts[nuts_lvl]['features'][index]['properties'][
'OBSERVATIONS'][file['id']].append(observation)
else:
nuts[nuts_lvl]['features'][index]['properties'][
'OBSERVATIONS'][file['id']] = [observation]
else:
nuts[nuts_lvl]['features'][index]['properties'][
'OBSERVATIONS'] = {
file['id']: [observation]
}
from bokeh.io import curdoc
from bokeh.layouts import column
from bokeh.plotting import figure
from bokeh.models.callbacks import CustomJS
from bokeh.models.sources import ColumnDataSource
from bokeh.models.widgets import Slider
# this is the real callback that we want to happen on slider mouseup
def cb(attr, old, new):
print("UPDATE", source.data['value'])
#p.x_range=range(0, int(source.data['value']))
# This data source is just used to communicate / trigger the real callback
source = ColumnDataSource(data=dict(value=[]))
source.on_change('data', cb)
# a figure, just for example
p = figure(x_range=(0,1), y_range=(0,1))
# add a slider with a CustomJS callback and a mouseup policy to update the source
slider = Slider(start=1, end=10, value=1, step=0.1, callback_policy='mouseup')
slider.callback = CustomJS(args=dict(source=source), code="""
source.data = { value: [cb_obj.value] }
""")
curdoc().add_root(column(slider, p))
# make sure to add the source explicitly
curdoc().add_root(source)
