def qdes_render_date_with_offset(date_value_utc, time=True): if not date_value_utc: return '' offset = render_datetime(date_value_utc, date_format='%z') if time: return render_datetime( date_value_utc, date_format='%Y-%m-%dT%H:%M:%S') + offset[:3] + ':' + offset[-2:] return render_datetime(date_value_utc, date_format='%Y-%m-%d')
def _render_datetime(self, datetime_, date_format=None, with_hours=False): datetime_ = helpers._datestamp_to_datetime(datetime_) if datetime_ == None: return False else: jstTime = datetime_ + datetime.timedelta(hours=9) return helpers.render_datetime(jstTime, date_format, with_hours)
def get_last_revision_timestamp_group(group_id): activity_objects = model.activity.group_activity_list( group_id, limit=1, offset=0) if len(activity_objects) > 0: activity = activity_objects[0] return h.render_datetime(activity.timestamp) return None
def dset_render_datetime(dateTimeString): '''Render a timestamp string using a preferred format (e.g. September 30, 2000, 08:00 PM) If timestamp is badly formatted, then THE ORIGINAL string is returned. ''' dateFormat = '%B %d, %Y, %I:%M %p' try: dateTimeObject = dateutil.parser.parse(dateTimeString, ignoretz=True) renderedDateTime = h.render_datetime(dateTimeObject, dateFormat, True) except ValueError: renderedDateTime = dateTimeString return renderedDateTime
def dataset_footnote(pkg_dict): dataset_url = url_for(controller='package', action='read', id=pkg_dict.get('id'), qualified=True) source_str = 'Source: {0}'.format(dataset_url) dataset_date = date_str_to_datetime(pkg_dict.get('metadata_modified')) date_str = 'Last updated at {0} : {1}'.format( url_for(controller='home', action='index', qualified=True), render_datetime(dataset_date, '%Y-%m-%d')) return '\n\n{0} \r\n{1}'.format(source_str, date_str)
def render_datetime(datetime_, date_format=None, with_hours=False): '''Render a datetime object or timestamp string as a pretty string (Y-m-d H:m). If timestamp is badly formatted, then a blank string is returned. This is a wrapper on the CKAN one which has an American date_format. ''' if not date_format: date_format = '%d %b %Y' if with_hours: date_format += ' %H:%M' from ckan.lib.helpers import render_datetime return render_datetime(datetime_, date_format)
def get_latest_update_for_package(pkg_dict, date_format=None): ''' Returns the most recent update datetime (formatted as a string) for the package and its resources. If there is no update time found then 'unknown' is returned. If there is datetime found then it is rendered using the standard ckan helper. :param pkg_dict: the package dict :param date_format: date format for the return datetime :return: 'unknown' or a string containing the rendered datetime ''' latest_date, _ = _get_latest_update(itertools.chain([pkg_dict], pkg_dict.get(u'resources', []))) if latest_date is not None: return h.render_datetime(latest_date, date_format=date_format) else: return _(u'unknown')
def get_latest_update_for_package_resources(pkg_dict, date_format=None): ''' Returns the most recent update datetime (formatted as a string) across all resources in this package. If there is no update time found then 'unknown' is returned. If there is datetime found then it is rendered using the standard ckan helper. :param pkg_dict: the package dict :param date_format: date format for the return datetime :return: 'unknown' or a string containing the rendered datetime and the resource name ''' latest_date, latest_resource = _get_latest_update(pkg_dict.get(u'resources', [])) if latest_date is not None: name = latest_resource[u'name'] return u'{} ({})'.format(h.render_datetime(latest_date, date_format=date_format), name) # there is no available update so we return 'unknown' return _(u'unknown')
def render_new_dataset(template, activity): """Return the HTML snippet describing a user creating a new dataset""" # Provide a html link to the user's page actor = link_to_user(activity['user_id']) actor = '%s' % actor # Provide a html link to the new group object = h.dataset_link(activity['data']['package']) if object: object = '%s' % object # Build the entire message and return the html. date = '%s' % h.render_datetime(activity['timestamp']) template = template.format(actor=actor, date=date, object=object) template = '%s %s' % (template, date) # It is important to return it as a literal or it will be converted to text return html.literal(template)
def render_new_group(template, activity): """Return the HTML snippet describing a user creating a new group""" # Provide a html link to the user's page actor = link_to_user(activity['user_id']) actor = '%s' % actor # Provide a html link to the new group object = h.group_link(activity['data']['group']) if object: object = '%s' % object # Build the entire message and return the html. date = '%s' % h.render_datetime(activity['timestamp']) template = template.format(actor=actor, date=date, object=object) template = '%s %s' % (template, date) # It is important to return it as a literal or it will be converted to text return html.literal(template)
def qdes_generate_csv(title, rows): u""" Create a csv file to ./tmp directory and return the filename. """ filename = '' if rows: date = render_datetime(datetime.utcnow(), date_format='%Y-%m-%d') filename = 'audit-' + str(date) + '-' + title + '.csv' fieldnames = [] for key in rows[0]: fieldnames.append(key) with open(constants.TMP_PATH + '/' + filename, mode='w') as csv_file: csv_writer = csv.DictWriter(csv_file, fieldnames=fieldnames) csv_writer.writeheader() for row in rows: csv_writer.writerow(row) return filename
def get_reference_date(date_str): ''' Gets a reference date extra created by the harvesters and formats it nicely for the UI. Examples: [{"type": "creation", "value": "1977"}, {"type": "revision", "value": "1981-05-15"}] [{"type": "publication", "value": "1977"}] [{"type": "publication", "value": "NaN-NaN-NaN"}] Results 1977 (creation), May 15, 1981 (revision) 1977 (publication) NaN-NaN-NaN (publication) ''' try: out = [] for date in h.json.loads(date_str): value = h.render_datetime(date['value']) or date['value'] out.append('{0} ({1})'.format(value, date['type'])) return ', '.join(out) except (ValueError, TypeError): return date_str
def get_reference_dates_ec(date_str): ''' Gets a reference date extra created by the harvesters and formats it nicely for the UI. Examples: [{"type": "creation", "value": "1977"}, {"type": "revision", "value": "1981-05-15"}] [{"type": "publication", "value": "1977"}] [{"type": "publication", "value": "NaN-NaN-NaN"}] Results 1977 (creation), May 15, 1981 (revision) 1977 (publication) NaN-NaN-NaN (publication) ''' try: out = {} for date in h.json.loads(date_str): value = h.render_datetime(date['value']) or date['value'] out[date['type']] = value return out except (ValueError, TypeError): return None
def test_blank(self): data = None eq_(h.render_datetime(data), '')
def test_with_seconds(self): data = datetime.datetime(2008, 4, 13, 20, 40, 59, 123456) eq_(h.render_datetime(data, with_seconds=True), 'April 13, 2008, 20:40:59 (UTC)')
def test_render_datetime_blank(self): res = h.render_datetime(None) assert_equal(res, '')
def test_render_datetime(self): res = h.render_datetime(datetime.datetime(2008, 4, 13, 20, 40, 20, 123456)) assert_equal(res, 'Apr 13, 2008')
def render_datetime_datagov(date_str): try: value = h.render_datetime(date_str) except (ValueError, TypeError): return date_str return value
def test_render_datetime_but_from_string(self): res = h.render_datetime('2008-04-13T20:40:20.123456') assert_equal(res, 'Apr 13, 2008')
def test_from_string(self): data = '2008-04-13T20:40:20.123456' eq_(h.render_datetime(data), 'April 13, 2008')
def feedback_report(publisher, include_sub_publishers=False, include_published=False, use_cache=False): """ For the publisher provided (and optionally for sub-publishers) this function will generate a report on the feedback for that publisher. """ import collections import datetime import ckan.lib.helpers as helpers from ckanext.dgu.lib.publisher import go_down_tree from ckanext.dgu.model.feedback import Feedback from operator import itemgetter from sqlalchemy.util import OrderedDict publisher_name = '__all__' if publisher: publisher_name = publisher.name if use_cache: key = 'feedback-report' if include_published: key = 'feedback-all-report' if include_sub_publishers: key = "".join([key, '-withsubpub']) cache = model.DataCache.get_fresh(publisher_name, key) if cache is None: log.info("Did not find cached report - %s/%s" % (publisher_name,key,)) else: log.info("Found feedback report in cache") return cache if publisher: group_ids = [publisher.id] if include_sub_publishers: groups = sorted([x for x in go_down_tree(publisher)], key=lambda x: x.title) group_ids = [x.id for x in groups] memberships = model.Session.query(model.Member)\ .join(model.Package, model.Package.id==model.Member.table_id)\ .filter(model.Member.state == 'active')\ .filter(model.Member.group_id.in_(group_ids))\ .filter(model.Member.table_name == 'package')\ .filter(model.Package.state == 'active') else: memberships = model.Session.query(model.Member)\ .join(model.Package, model.Package.id==model.Member.table_id)\ .filter(model.Member.state == 'active')\ .filter(model.Member.table_name == 'package')\ .filter(model.Package.state == 'active') results = [] for member in memberships.all(): pkg = model.Package.get(member.table_id) # For now we will skip over unpublished items if not include_published and not pkg.extras.get('unpublished', False): continue key = pkg.name data = collections.defaultdict(int) data['publisher-name'] = member.group.name data['generated-at'] = helpers.render_datetime(datetime.datetime.now(), "%d/%m/%Y %H:%M") data['publisher-title'] = member.group.title data['package-name'] = pkg.name data['package-title'] = pkg.title data['publish-date'] = pkg.extras.get('publish-date', '') for item in model.Session.query(Feedback).filter(Feedback.visible == True)\ .filter(Feedback.package_id == member.table_id )\ .filter(Feedback.active == True ): if item.economic: data['economic'] += 1 if item.social: data['social'] += 1 if item.linked: data['linked'] += 1 if item.other: data['other'] += 1 if item.effective: data['effective'] += 1 data['total-comments'] = sum([data['economic'], data['social'], data['linked'], data['other'], data['effective']]) results.append(data) return sorted(results, key=itemgetter('package-title'))
def hxl_preview_iframe_url_show(context, data_dict): resource_dict = data_dict.get('resource') resource_view_dict = data_dict.get('resource_view') # hxl_preview_mode = data_dict.get('hxl_preview_mode') # # only_view_mode = 'false' # start_edit_mode = 'false' # # if hxl_preview_mode == 'onlyView': # only_view_mode = 'true' # start_edit_mode = 'false' # elif hxl_preview_mode == 'edit': # only_view_mode = 'false' # start_edit_mode = 'true' # only_view_mode = 'true' if hxl_preview_mode == 'onlyView' else 'false' # start_edit_mode = 'true' if hxl_preview_mode == 'edit' else 'false' # start_edit_mode = 'true' if self.__is_allowed_to_edit(resource_dict) and \ # not self.__is_hxl_preview_config_saved(resource_view_dict) else 'false' has_modify_permission = 'true' if context.get('has_modify_permission', False) else 'false' package_id = resource_dict.get('package_id') package_dict = _get_action('package_show')(context, {'id': package_id}) package_source = package_dict.get('dataset_source', '') package_url = h.url_for(controller='package', action='read', id=package_id, qualified=True) resource_last_modified = h.render_datetime( resource_dict.get('last_modified') or resource_dict.get('created')) params = { 'hxl_preview_app': config.get('hdx.hxl_preview_app.url'), 'resource_url': urllib.urlencode({'url': resource_dict.get('url')}), 'resource_view_id': urllib.urlencode({'resource_view_id': resource_view_dict.get('id')}), 'hdx_domain': urllib.urlencode({'hdx_domain': __get_ckan_domain_without_protocol()}), 'has_modify_permission': urllib.urlencode({'has_modify_permission': has_modify_permission}), 'embedded_source': urllib.urlencode( {'embeddedSource': urllib.quote(package_source.encode('utf-8'))}), 'embedded_url': urllib.urlencode({'embeddedUrl': package_url}), 'embedded_date': urllib.urlencode({ 'embeddedDate': urllib.quote(resource_last_modified.encode('utf-8')) }) # 'edit_mode': urllib.urlencode({'editMode': start_edit_mode}), # 'only_view_mode': urllib.urlencode({'onlyViewMode': only_view_mode}), } url = '{hxl_preview_app}/show;{resource_url};{hdx_domain};{embedded_source};{embedded_url};{embedded_date};{resource_view_id};{has_modify_permission}'.format( **params) return url
def friendly_date(date_str): return render_datetime(date_str, '%d, %B, %Y')
def test_date(self): data = datetime.datetime(2008, 4, 13, 20, 40, 59, 123456) eq_(h.render_datetime(data), 'April 13, 2008')
def test_render_datetime_year_before_1900(self): res = h.render_datetime('1875-04-13T20:40:20.123456', date_format='%Y') assert_equal(res, '1875') res = h.render_datetime('1875-04-13T20:40:20.123456', date_format='%y') assert_equal(res, '75')
def test_before_1900_with_2_digit_year(self): data = '1875-04-13T20:40:20.123456' eq_(h.render_datetime(data, date_format='%y'), '75')
def report_timestamps_split(timestamps): return [render_datetime(timestamp) for timestamp in timestamps.split(' ')]
def test_render_datetime_but_from_string(self): res = h.render_datetime("2008-04-13T20:40:20.123456") assert_equal(res, "April 13, 2008")
def test_render_datetime(self): res = h.render_datetime( datetime.datetime(2008, 4, 13, 20, 40, 20, 123456)) assert_equal(res, 'Apr 13, 2008')
def feedback_report(publisher, include_sub_publishers=False, include_published=False, use_cache=False): """ For the publisher provided (and optionally for sub-publishers) this function will generate a report on the feedback for that publisher. """ import collections import datetime import ckan.lib.helpers as helpers from ckanext.dgu.lib.publisher import go_down_tree from ckanext.dgu.model.feedback import Feedback from operator import itemgetter from sqlalchemy.util import OrderedDict publisher_name = '__all__' if publisher: publisher_name = publisher.name if use_cache: key = 'feedback-report' if include_published: key = 'feedback-all-report' if include_sub_publishers: key = "".join([key, '-withsub']) report, report_date = model.DataCache.get_if_fresh(publisher_name, key) if report is None: log.info("Did not find cached report - %s/%s" % (publisher_name, key)) else: log.info("Found feedback report in cache") return report, report_date if publisher: group_ids = [publisher.id] if include_sub_publishers: groups = sorted([x for x in go_down_tree(publisher)], key=lambda x: x.title) group_ids = [x.id for x in groups] memberships = model.Session.query(model.Member)\ .join(model.Package, model.Package.id==model.Member.table_id)\ .filter(model.Member.state == 'active')\ .filter(model.Member.group_id.in_(group_ids))\ .filter(model.Member.table_name == 'package')\ .filter(model.Package.state == 'active') else: memberships = model.Session.query(model.Member)\ .join(model.Package, model.Package.id==model.Member.table_id)\ .filter(model.Member.state == 'active')\ .filter(model.Member.table_name == 'package')\ .filter(model.Package.state == 'active') results = [] for member in memberships.all(): pkg = model.Package.get(member.table_id) # For now we will skip over unpublished items if not include_published and not pkg.extras.get('unpublished', False): continue key = pkg.name data = collections.defaultdict(int) data['publisher-name'] = member.group.name data['generated-at'] = helpers.render_datetime(datetime.datetime.now(), "%d/%m/%Y %H:%M") data['publisher-title'] = member.group.title data['package-name'] = pkg.name data['package-title'] = pkg.title data['publish-date'] = pkg.extras.get('publish-date', '') for item in model.Session.query(Feedback).filter(Feedback.visible == True)\ .filter(Feedback.package_id == member.table_id )\ .filter(Feedback.active == True ): if item.economic: data['economic'] += 1 if item.social: data['social'] += 1 if item.linked: data['linked'] += 1 if item.other: data['other'] += 1 if item.effective: data['effective'] += 1 data['total-comments'] = sum([ data['economic'], data['social'], data['linked'], data['other'], data['effective'] ]) results.append(data) log.info( 'Report generated: feedback_report publishers=%s subpub=%s published=%s', publisher.name if publisher else 'all', include_sub_publishers, include_published) return sorted(results, key=itemgetter('package-title')), 'just now'
def feedback_report(organization=None, include_sub_organizations=False, include_published=False): """ For the publisher provided (and optionally for sub-publishers) this function will generate a report on the feedback for that publisher. """ import ckan.lib.helpers as helpers from ckanext.dgu.model.feedback import Feedback if organization: organization = model.Group.by_name(organization) if not organization: raise p.toolkit.ObjectNotFound() else: organization = None # Get packages for these organization(s) memberships = model.Session.query(model.Member)\ .join(model.Package, model.Package.id==model.Member.table_id)\ .filter(model.Member.state == 'active') memberships = lib.filter_by_organizations(memberships, organization, include_sub_organizations)\ .filter(model.Member.table_name == 'package')\ .filter(model.Package.state == 'active') # For each package, count the feedback comments results = [] num_pkgs_with_feedback = 0 for member in memberships.all(): pkg = model.Package.get(member.table_id) # Skip unpublished datasets if that's asked for if not include_published and not pkg.extras.get('unpublished', False): continue pkg_data = collections.defaultdict(int) pkg_data['organization-name'] = member.group.name pkg_data['generated-at'] = helpers.render_datetime(datetime.datetime.now(), "%d/%m/%Y %H:%M") pkg_data['organization-title'] = member.group.title pkg_data['package-name'] = pkg.name pkg_data['package-title'] = pkg.title pkg_data['publish-date'] = pkg.extras.get('publish-date', '') for feedback in model.Session.query(Feedback).filter(Feedback.visible == True)\ .filter(Feedback.package_id == member.table_id )\ .filter(Feedback.active == True ): if feedback.economic: pkg_data['economic'] += 1 if feedback.social: pkg_data['social'] += 1 if feedback.linked: pkg_data['linked'] += 1 if feedback.other: pkg_data['other'] += 1 if feedback.effective: pkg_data['effective'] += 1 pkg_data['total-comments'] = sum([pkg_data['economic'], pkg_data['social'], pkg_data['linked'], pkg_data['other'], pkg_data['effective']]) results.append(pkg_data) if pkg_data['total-comments'] > 0: num_pkgs_with_feedback += 1 return {'table': sorted(results, key=lambda x: -x.get('total-comments')), 'dataset_count': len(results), 'dataset_count_with_feedback': num_pkgs_with_feedback, }
def test_render_datetime(date, extra, exp): assert h.render_datetime(date, **extra) == exp
def feedback_report(organization=None, include_sub_organizations=False, include_published=False): """ For the publisher provided (and optionally for sub-publishers) this function will generate a report on the feedback for that publisher. """ import ckan.lib.helpers as helpers from ckanext.dgu.model.feedback import Feedback if organization: organization = model.Group.by_name(organization) if not organization: raise p.toolkit.ObjectNotFound() else: organization = None # Get packages for these organization(s) memberships = model.Session.query(model.Member)\ .join(model.Package, model.Package.id==model.Member.table_id)\ .filter(model.Member.state == 'active') memberships = lib.filter_by_organizations(memberships, organization, include_sub_organizations)\ .filter(model.Member.table_name == 'package')\ .filter(model.Package.state == 'active') # For each package, count the feedback comments results = [] num_pkgs_with_feedback = 0 for member in memberships.all(): pkg = model.Package.get(member.table_id) # Skip unpublished datasets if that's asked for if not include_published and not pkg.extras.get('unpublished', False): continue pkg_data = collections.defaultdict(int) pkg_data['organization-name'] = member.group.name pkg_data['generated-at'] = helpers.render_datetime( datetime.datetime.now(), "%d/%m/%Y %H:%M") pkg_data['organization-title'] = member.group.title pkg_data['package-name'] = pkg.name pkg_data['package-title'] = pkg.title pkg_data['publish-date'] = pkg.extras.get('publish-date', '') for feedback in model.Session.query(Feedback).filter(Feedback.visible == True)\ .filter(Feedback.package_id == member.table_id )\ .filter(Feedback.active == True ): if feedback.economic: pkg_data['economic'] += 1 if feedback.social: pkg_data['social'] += 1 if feedback.linked: pkg_data['linked'] += 1 if feedback.other: pkg_data['other'] += 1 if feedback.effective: pkg_data['effective'] += 1 pkg_data['total-comments'] = sum([ pkg_data['economic'], pkg_data['social'], pkg_data['linked'], pkg_data['other'], pkg_data['effective'] ]) results.append(pkg_data) if pkg_data['total-comments'] > 0: num_pkgs_with_feedback += 1 return { 'table': sorted(results, key=lambda x: -x.get('total-comments')), 'dataset_count': len(results), 'dataset_count_with_feedback': num_pkgs_with_feedback, }
def test_render_datetime(self): res = h.render_datetime(datetime.datetime(2008, 4, 13, 20, 40, 20, 123456)) assert_equal(res, '2008-04-13 20:40')
def test_before_1900(self): data = '1875-04-13T20:40:20.123456' eq_(h.render_datetime(data, date_format='%Y'), '1875')
def test_escaped_percent(self): data = '2008-04-13T20:40:20.123456' eq_(h.render_datetime(data, date_format='%%%Y'), '%2008')
def test_render_datetime_year_before_1900_escape_percent(self): res = h.render_datetime('1875-04-13', date_format='%%%y') assert_equal(res, '%75') res = h.render_datetime('1875-04-13', date_format='%%%Y') assert_equal(res, '%1875')
def get_val_date(date): return h.render_datetime(h.date_str_to_datetime(date))
def test_render_datetime_with_hours(self): res = h.render_datetime(datetime.datetime(2008, 4, 13, 20, 40, 20, 123456), with_hours=True) assert_equal(res, 'April 13, 2008, 20:40 (UTC)')