def get_ncdump_html(self):
"""
Generates html code to display the contents of the ncdump text file. The generated html
is used for netcdf file type metadata view and edit modes.
:return:
"""
nc_dump_div = div()
nc_dump_res_file = None
for f in self.logical_file.files.all():
if f.extension == ".txt":
nc_dump_res_file = f
break
if nc_dump_res_file is not None:
nc_dump_div = div(style="clear: both", cls="col-xs-12")
with nc_dump_div:
legend("NetCDF Header Information")
p(nc_dump_res_file.full_path[33:])
header_info = nc_dump_res_file.resource_file.read()
header_info = header_info.decode('utf-8')
textarea(header_info,
readonly="",
rows="15",
cls="input-xlarge",
style="min-width: 100%")
return nc_dump_div
def get_json_file_data_html(self):
"""
Generates html code to display the contents of the json file. The generated html
is used for ref timeseries file type metadata in the view and edit modes.
:return:
"""
json_res_file = self.logical_file.files.first()
json_file_content_div = div(style="clear:both;",
cls="col-xs-12 content-block")
with json_file_content_div:
legend("Reference Time Series JSON File Content",
style="margin-top:20px;")
p(json_res_file.full_path[33:])
header_info = self.json_file_content
if isinstance(header_info, str):
header_info = unicode(header_info, 'utf-8')
textarea(header_info,
readonly="",
rows="15",
cls="input-xlarge",
style="min-width: 100%")
return json_file_content_div
def get_abstract_form(self):
form_action = "/hsapi/_internal/{}/update-reftimeseries-abstract/"
form_action = form_action.format(self.logical_file.id)
root_div = div(cls="col-xs-12")
# if json file contains abstract then we don't need this form since abstract can't be
# edited in that case
if self.has_abstract_in_json:
return
with root_div:
with form(action=form_action, id="filetype-abstract",
method="post", enctype="multipart/form-data"):
div("{% csrf_token %}")
with div(cls="form-group"):
with div(cls="control-group"):
legend('Abstract')
with div(cls="controls"):
textarea(self.abstract,
cls="form-control input-sm textinput textInput",
id="file_abstract", cols=40, rows=5,
name="abstract")
with div(cls="row", style="margin-top:10px;"):
with div(cls="col-md-offset-10 col-xs-offset-6 col-md-2 col-xs-6"):
button("Save changes", cls="btn btn-primary pull-right btn-form-submit",
style="display: none;", type="button")
return root_div
def get_abstract_form(self):
form_action = "/hsapi/_internal/{}/update-reftimeseries-abstract/"
form_action = form_action.format(self.logical_file.id)
root_div = div(cls="col-xs-12")
# if json file contains abstract then we don't need this form since abstract can't be
# edited in that case at the aggregation level
if self.has_abstract_in_json:
return
with root_div:
with form(action=form_action, id="filetype-abstract",
method="post", enctype="multipart/form-data"):
div("{% csrf_token %}")
with div(cls="form-group"):
with div(cls="control-group"):
legend('Abstract')
with div(cls="controls"):
abstract = ""
if self.abstract:
abstract = self.abstract
textarea(abstract,
cls="form-control input-sm textinput textInput",
id="file_abstract", cols=40, rows=5,
name="abstract")
with div(cls="row", style="margin-top:10px;"):
with div(cls="col-md-offset-10 col-xs-offset-6 col-md-2 col-xs-6"):
button("Save changes", cls="btn btn-primary pull-right btn-form-submit",
style="display: none;", type="button")
return root_div
def get_json_file_data_html(self):
"""
Generates html code to display the contents of the json file. The generated html
is used for ref timeseries file type metadata in the view and edit modes.
:return:
"""
json_res_file = self.logical_file.files.first()
json_file_content_div = div(cls="content-block")
with json_file_content_div:
legend("Reference Time Series JSON File Content")
p(json_res_file.full_path[33:])
header_info = self.json_file_content
if isinstance(header_info, str):
header_info = unicode(header_info, 'utf-8')
textarea(header_info, readonly="", rows="15",
cls="input-xlarge", style="min-width: 100%; resize: vertical;")
return json_file_content_div
def get_ncdump_html(self):
"""
Generates html code to display the contents of the ncdump text file. The generated html
is used for netcdf file type metadata view and edit modes.
:return:
"""
nc_dump_div = div()
nc_dump_res_file = None
for f in self.logical_file.files.all():
if f.extension == ".txt":
nc_dump_res_file = f
break
if nc_dump_res_file is not None:
nc_dump_div = div(style="clear: both", cls="content-block")
with nc_dump_div:
legend("NetCDF Header Information")
p(nc_dump_res_file.full_path[33:])
header_info = nc_dump_res_file.resource_file.read()
header_info = header_info.decode('utf-8')
textarea(header_info, readonly="", rows="15",
cls="input-xlarge", style="min-width: 100%; resize: vertical;")
return nc_dump_div
def render(r):
ta = dt.textarea(src.read_text(), id="cool-textarea")
render.preview.send(dt.div(dt.h5("hello"), ta))
return []
def _get_delete_key_value_modal_forms(self):
form_action = "/hsapi/_internal/{0}/{1}/delete-file-keyvalue-metadata/"
form_action = form_action.format(self.logical_file.__class__.__name__,
self.logical_file.id)
counter = 0
root_div = div(id="delete-keyvalue-filetype-modals")
with root_div:
for k, v in self.extra_metadata.iteritems():
counter += 1
modal_div = div(
cls="modal fade",
id="delete-keyvalue-filetype-modal-{}".format(counter),
tabindex="-1",
role="dialog",
aria_labelledby="delete-key-value-metadata",
aria_hidden="true")
with modal_div:
with div(cls="modal-dialog", role="document"):
with div(cls="modal-content"):
form_id = "delete-keyvalue-filetype-metadata-{}".format(
counter)
with form(action=form_action,
id=form_id,
method="post",
enctype="multipart/form-data"):
div("{% csrf_token %}")
with div(cls="modal-header"):
button("x",
type="button",
cls="close",
data_dismiss="modal",
aria_hidden="true")
h4("Confirm to Delete Key/Value Metadata",
cls="modal-title",
id="delete-key-value-metadata")
with div(cls="modal-body"):
with div(cls="form-group"):
with div(cls="control-group"):
label(
"Key",
cls=
"control-label requiredField",
fr="file_extra_meta_name")
with div(cls="controls"):
input(
cls=
"form-control input-sm textinput "
"textInput",
value=k,
id="file_extra_meta_key",
maxlength="100",
name="key",
type="text",
readonly="readonly")
with div(cls="control-group"):
label(
"Value",
cls=
"control-label requiredField",
fr="file_extra_meta_value")
with div(cls="controls"):
textarea(
v,
cls=
"form-control input-sm textarea",
cols="40",
rows="10",
id="file_extra_meta_value",
style="resize: vertical;",
name="value",
type="text",
readonly="readonly")
with div(cls="modal-footer"):
button("Cancel",
type="button",
cls="btn btn-default",
data_dismiss="modal")
button("Delete",
type="button",
cls="btn btn-danger",
id="btn-delete-key-value"
) # TODO: TESTING
return root_div
def _get_add_key_value_modal_form(self):
form_action = "/hsapi/_internal/{0}/{1}/update-file-keyvalue-metadata/"
form_action = form_action.format(self.logical_file.__class__.__name__,
self.logical_file.id)
modal_div = div(cls="modal fade",
id="add-keyvalue-filetype-modal",
tabindex="-1",
role="dialog",
aria_labelledby="add-key-value-metadata",
aria_hidden="true")
with modal_div:
with div(cls="modal-dialog", role="document"):
with div(cls="modal-content"):
with form(action=form_action,
id="add-keyvalue-filetype-metadata",
method="post",
enctype="multipart/form-data"):
div("{% csrf_token %}")
with div(cls="modal-header"):
button("x",
type="button",
cls="close",
data_dismiss="modal",
aria_hidden="true")
h4("Add Key/Value Metadata",
cls="modal-title",
id="add-key-value-metadata")
with div(cls="modal-body"):
with div(cls="form-group"):
with div(cls="control-group"):
label("Key",
cls="control-label requiredField",
fr="file_extra_meta_name")
with div(cls="controls"):
input(
cls=
"form-control input-sm textinput textInput",
id="file_extra_meta_name",
maxlength="100",
name="name",
type="text")
with div(cls="control-group"):
label("Value",
cls="control-label requiredField",
fr="file_extra_meta_value")
with div(cls="controls"):
textarea(
cls=
"form-control input-sm textarea",
cols="40",
rows="10",
id="file_extra_meta_value",
style="resize: vertical;",
name="value",
type="text")
with div(cls="modal-footer"):
button("Cancel",
type="button",
cls="btn btn-default",
data_dismiss="modal")
button(
"OK",
type="button",
cls="btn btn-primary",
id="btn-confirm-add-metadata") # TODO: TESTING
return modal_div
def _get_edit_key_value_modal_forms(self):
# TODO: See if can use one modal dialog to edit any pair of key/value
form_action = "/hsapi/_internal/{0}/{1}/update-file-keyvalue-metadata/"
form_action = form_action.format(self.logical_file.__class__.__name__,
self.logical_file.id)
counter = 0
root_div = div(id="edit-keyvalue-filetype-modals")
with root_div:
for k, v in self.extra_metadata.iteritems():
counter += 1
modal_div = div(
cls="modal fade",
id="edit-keyvalue-filetype-modal-{}".format(counter),
tabindex="-1",
role="dialog",
aria_labelledby="edit-key-value-metadata",
aria_hidden="true")
with modal_div:
with div(cls="modal-dialog", role="document"):
with div(cls="modal-content"):
form_id = "edit-keyvalue-filetype-metadata-{}".format(
counter)
with form(action=form_action,
id=form_id,
data_counter="{}".format(counter),
method="post",
enctype="multipart/form-data"):
div("{% csrf_token %}")
with div(cls="modal-header"):
button("x",
type="button",
cls="close",
data_dismiss="modal",
aria_hidden="true")
h4("Update Key/Value Metadata",
cls="modal-title",
id="edit-key-value-metadata")
with div(cls="modal-body"):
with div(cls="form-group"):
with div(cls="control-group"):
label(
"Key(Original)",
cls=
"control-label requiredField",
fr=
"file_extra_meta_key_original")
with div(cls="controls"):
input(
value=k,
readonly="readonly",
cls=
"form-control input-sm textinput "
"textInput",
id=
"file_extra_meta_key_original",
maxlength="100",
name="key_original",
type="text")
with div(cls="control-group"):
label(
"Key",
cls=
"control-label requiredField",
fr="file_extra_meta_key")
with div(cls="controls"):
input(
value=k,
cls=
"form-control input-sm textinput "
"textInput",
id="file_extra_meta_key",
maxlength="100",
name="key",
type="text")
with div(cls="control-group"):
label(
"Value",
cls=
"control-label requiredField",
fr="file_extra_meta_value")
with div(cls="controls"):
textarea(
v,
cls=
"form-control input-sm textarea",
cols="40",
rows="10",
id="file_extra_meta_value",
name="value",
type="text")
with div(cls="modal-footer"):
button("Cancel",
type="button",
cls="btn btn-default",
data_dismiss="modal")
button("OK",
id="btn-confirm-edit-key-value",
type="button",
cls="btn btn-primary")
return root_div
def render_latest(repo: Path, digest, rendered: Path):
logger.info('processing %s', repo)
rtype = get_result_type(repo)
Format = FormatTrait.for_(rtype)
Ignore = IgnoreTrait.for_(rtype)
import pytz
NOW = datetime.now(tz=pytz.utc)
name = repo.stem
doc = dominate.document(
title=f'axol results for {name}, rendered at {fdate(NOW)}')
with doc.head:
T.style(STYLE)
raw_script(JS)
T.link(
rel='stylesheet',
href=
"https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.48.2/codemirror.min.css"
)
T.script(
src=
'https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.48.2/codemirror.js'
) # TODO use min?
citems: Iterator[Tuple[datetime, Item]] = chain.from_iterable(
((d, x) for x in zz) for d, zz in digest.changes.items())
# group according to link, so we can display already occuring items along with newer occurences
items2: List[Sequence[Tuple[datetime, Item]]] = [
grp for _, grp in group_by_key(citems,
key=lambda p: f'{p[1].link}').items()
]
# TODO sort within each group?
def min_dt(group: Sequence[Tuple[datetime, Item]]) -> datetime:
return min(g[0] for g in group)
# TODO ok, this is def too many types here...
items3: Mapping[datetime,
List[Sequence[Tuple[datetime,
Item]]]] = group_by_key(items2,
key=min_dt)
rss = True
if rss:
# pip3 install feedgen
from feedgen.feed import FeedGenerator # type: ignore
fg = FeedGenerator()
# TODO memorize items?
fg.title(name)
fg.id('axol/' + name)
first = True
for d, items in sorted(items3.items()):
litems = list(items)
logger.info('%s %s: atom, dumping %d items', name, d, len(litems))
if first:
logger.info("SKIPPING first batch to prevent RSS bloat")
first = False
continue
for zz in litems:
fe = fg.add_entry()
# TODO not sure about css?
# TODO not sure which date should use? I gues crawling date makes more sense..
_d, z = zz[0] # TODO meh!
id_ = z.uid # TODO FIXME!!
fe.id(id_)
title = Format.title(zz) or '<no title>' # meh
fe.title(title)
fe.link(href=Format.link(zz))
# TODO not sure if it's a reasonable date to use...
fe.published(published=d)
fe.author(author={'name': z.user}) # TODO maybe, concat users?
ignored = Ignore.ignore_group(zz)
if ignored is not None:
# TODO not sure if it highlights with read or something?
content = ignored
else:
content = Format.format(zz)
# eh, XML was complaining at some non-utf characters
content = str(content)
# https://stackoverflow.com/a/25920392/706389 make lxml happy...
content = re.sub(
u'[^\u0020-\uD7FF\u0009\u000A\u000D\uE000-\uFFFD\U00010000-\U0010FFFF]+',
'', content)
fe.content(content=content, type='CDATA')
# fe.updated(updated=NOW)
# TODO assemble a summary similar to HTML?
# fe.summary()
atomfeed = fg.atom_str(pretty=True)
# eh, my feed reader (miniflux) can't handle it if it's 'cdata'
# not sure which one is right
# ugh, that didn't work because escaping desicion is based on CDATA attribute...
atomfeed = atomfeed.replace(b'type="CDATA"', b'type="html"')
# fe._FeedEntry__atom_content['type'] = 'html'
atomdir = rendered / 'atom'
atomdir.mkdir(parents=True, exist_ok=True)
(atomdir / (name + '.xml')).write_bytes(atomfeed)
with doc:
with T.div(id='sidebar'):
T.label('Blacklisted:', for_='blacklisted')
T.div(id='blacklisted')
T.textarea(id='blacklist-edit', rows=10)
T.button('apply', id='blacklist-apply')
odd = True
for d, items in sorted(items3.items(), reverse=True):
litems = list(items)
odd = not odd
logger.info('%s %s: dumping %d items', name, d, len(litems))
with T.div(cls='day-changes'):
with T.div():
T.b(fdate(d))
T.span(f'{len(litems)} items')
with T.div(
cls=f'day-changes-inner {"odd" if odd else "even"}'):
for i in items:
# TODO FIXME use getattr to specialise trait?
# TODO FIXME ignore should be at changes collecting stage?
ignored = Ignore.ignore_group(i)
if ignored is not None:
# TODO maybe let format result handle that... not sure
T.div(ignored, cls='item ignored')
# TODO log maybe?
# TODO eh. need to handle in cumulatives...
else:
fi = Format.format(i)
T.div(fi, cls='item')
# f*****g hell.. didn't manage to render content inside iframe no matter how I tried..
# with T.iframe(id='blacklist', src=''):
# pass
# TODO perhaps needs to be iterative...
rf = rendered / (name + '.html')
with rf.open('w') as fo:
fo.write(str(doc))
return rf
