def convert_datelikes_to_isoformat(record):
for ix, value in enumerate(record):
if isinstance(value, datetime):
record[ix] = isodate.datetime_isoformat(value)
elif isinstance(value, date):
record[ix] = isodate.date_isoformat(value)
elif isinstance(value, time):
record[ix] = isodate.date_isoformat(value)
def beaconinfo_creator():
byc = initialize_service("collations_creator")
b_info = { "date": date_isoformat(datetime.datetime.now()), "datasets": { } }
print("=> updating entry {} in {}.{}".format(b_info[ "date" ], byc[ "config" ][ "info_db" ], byc[ "config" ][ "beacon_info_coll"]) )
mongo_client = MongoClient( )
dbs = MongoClient().list_database_names()
for ds_id in byc["dataset_definitions"].keys():
if not ds_id in dbs:
print("¡¡¡ Dataset "+ds_id+" doesn't exist !!!")
else:
# elif ds_id == "cellosaurus":
b_info["datasets"].update( { ds_id: _dataset_update_counts(byc["dataset_definitions"][ds_id], **byc) } )
# else:
# continue
info_db = mongo_client[ byc[ "config" ][ "info_db" ] ]
info_coll = info_db[ byc[ "config" ][ "beacon_info_coll"] ]
info_coll.delete_many( { "date": b_info["date"] } ) #, upsert=True
info_coll.insert_one( b_info ) #, upsert=True
print("=> updated entry {} in {}.{}".format(b_info["date"], byc[ "config" ][ "info_db" ], byc[ "config" ][ "beacon_info_coll"]) )
def db_ebay_got_seller_list_date(self):
"""
Provides a method for preventing the `get_seller_list` command
from being ran multiple times a day
Returns True when there is either no date, or the date was in
the past. Returns False when there is a date that is today or
in the future
"""
query = "SELECT value FROM ebay_internals WHERE key = 'got_seller_list_date'"
self.__execute(query)
last_date = self.__fetchone('value', 'no')
if last_date != 'no' and isodate.parse_date(last_date) >= datetime.date.today():
msg = 'We already ran get_seller_list today (or in the future). Wait until tomorrow'
self.log.warning(msg)
return False
if last_date == 'no' or isodate.parse_date(last_date) < datetime.date.today():
query = "UPDATE ebay_internals SET value = :isodate WHERE key = 'got_seller_list_date'"
today = isodate.date_isoformat(datetime.date.today())
self.__execute(query, {'isodate': today})
self.db_ebay_zero_request_counter()
return True
def call(self, **kwargs):
today = date.today()
chores_static = self.data_source.get_entries_for_stat_on_date(
"chores", "static")[0]
chores_data = self.data_source.get_entries_for_stat_over_range(
"chores", "earliest", "latest")
chores = [Chore(x) for x in chores_static['data']['chores']]
for chore_date in chores_data:
for chore in chores:
chore.parse_date_entry(chore_date)
# Sort chores into categories
categorised_chores = dict()
for chore in chores:
if chore.category not in categorised_chores:
categorised_chores[chore.category] = []
categorised_chores[chore.category].append(chore)
# Get layout info
layout = chores_static['data']['layout']
# Return json
return flask.jsonify({
"today": isodate.date_isoformat(today),
"chores": {
k: [x.to_json() for x in v]
for k, v in categorised_chores.items()
},
"layout": layout
})
def update_publications():
args = _get_args()
if args.test:
print("¡¡¡ TEST MODE - no db update !!!")
# Read annotation table:
rows = read_annotation_table(args)
print("=> {} publications will be looked up".format(len(rows)))
# Connect to MongoDB and load publication collection
client = MongoClient()
cl = client['progenetix'].publications
ids = cl.distinct("id")
# Update the database
for row in rows:
post = create_progenetix_post(row)
if post["id"] in ids:
print(post["id"], ": skipped - already in progenetix.publications")
else:
print(post["id"], ": inserting this into progenetix.publications")
post.update({"updated": date_isoformat(datetime.datetime.now())})
if not args.test:
result = cl.insert_one(post)
result.inserted_id
else:
jprint(post)
def locate(self):
operating_day = isodate.date_isoformat(date.today())
journey = trias.TripInfoRequest(self.journey_id,
operating_day).execute()
visits = journey["TripInfoResponse"]["TripInfoResult"][
"PreviousCall"] + journey["TripInfoResponse"]["TripInfoResult"][
"OnwardCall"]
visits = [Visit(visit) for visit in visits]
def __repr__(self):
return "Cx(%s, %d, %d, %d:%s)" % (
self.outbreak,
self.training,
self.horizon,
self.now,
isodate.date_isoformat(self.now_date),
)
def test_format(self):
"""
Take date object and create ISO string from it.
This is the reverse test to test_parse.
"""
if expectation is None:
self.assertRaises(AttributeError, date_isoformat, expectation, format, yeardigits)
else:
self.assertEqual(date_isoformat(expectation, format, yeardigits), datestring)
def to_json(self):
recommended_period = self.recommended_period
if recommended_period is not None:
recommended_period = isodate.duration_isoformat(recommended_period)
latest_done = self.latest_done
if isinstance(latest_done, date):
latest_done = isodate.date_isoformat(latest_done)
next_date = self.get_next_date()
if isinstance(next_date, date):
next_date = isodate.date_isoformat(next_date)
return {
"id": self.id,
"display_name": self.display_name,
"category": self.category,
"recommended_period": recommended_period,
"latest_done": latest_done,
"next_date": next_date,
"is_overdue": self.is_overdue(),
}
def _dump_default(self, obj):
if isinstance(obj, datetime.datetime):
return isodate.datetime_isoformat(obj)
elif isinstance(obj, datetime.date):
return isodate.date_isoformat(obj)
elif isinstance(obj, datetime.time):
return isodate.time_isoformat(obj)
elif isinstance(obj, datetime.timedelta):
return obj.total_seconds()
else:
raise TypeError(obj)
def _dump_default(self, obj):
if isinstance(obj, datetime.datetime):
return isodate.datetime_isoformat(obj)
elif isinstance(obj, datetime.date):
return isodate.date_isoformat(obj)
elif isinstance(obj, datetime.time):
return isodate.time_isoformat(obj)
elif isinstance(obj, datetime.timedelta):
return obj.total_seconds()
else:
raise TypeError(obj)
def to_raw(self, value, context=None):
if isinstance(value, datetime.date):
return isodate.date_isoformat(value)
if isinstance(value, six.string_types):
self.to_python(value, context=context)
return value
raise exceptions.ValidationError(
"Invalid date value '%s' and type %s" % (value, type(value)),
value=value,
invalid='type',
against='date')
def to_raw(self, value, context=None):
if isinstance(value, datetime.date):
return isodate.date_isoformat(value)
if isinstance(value, six.string_types):
self.to_python(value, context=context)
return value
raise exceptions.ValidationError(
"Invalid date value '%s' and type %s" % (value, type(value)),
value=value,
invalid='type',
against='date'
)
def to_datetimes(prop_type, value):
if prop_type == ndb.DateProperty:
return isodate.date_isoformat(value)
elif prop_type == ndb.TimeProperty:
return isodate.time_isoformat(value)
elif prop_type == ndb.DateTimeProperty:
return isodate.datetime_isoformat(value) + 'Z'
elif prop_type == TimeDeltaProperty:
return str(value)
def test_format(self):
'''
Take date object and create ISO string from it.
This is the reverse test to test_parse.
'''
if expectation is None:
self.assertRaises(AttributeError, date_isoformat, expectation,
format, yeardigits)
else:
self.assertEqual(
date_isoformat(expectation, format, yeardigits),
datestring)
def get_field(request, person, field):
if not hasattr(person, field):
raise ContentExportError(_(u"Field '${name}' does not exist", mapping={
'name': field
}))
value = get_value(request, person, field)
if value is None:
return u''
if isinstance(value, bool):
return _(u'Yes') if value is True else _(u'No')
if isinstance(value, basestring):
return unicode(value)
if field in person.membership_fields.keys():
organizations = (api.content.get(UID=i).title for i in value.keys())
return u', '.join(organizations)
if isinstance(value, (list, tuple, set)):
return u', '.join(value)
if isinstance(value, dict):
parts = []
for key in sorted(value):
parts.append(u'{}: {}'.format(key, value[key]))
return u', '.join(parts)
if isinstance(value, date):
return unicode(isodate.date_isoformat(value))
if isinstance(value, datetime):
return unicode(isodate.datetime_isoformat(value))
if isinstance(value, time):
return unicode(isodate.time_isoformat(value))
if isinstance(value, RichTextValue):
return unicode(value.output)
if isinstance(value, (NamedImage, NamedBlobImage)):
base = person.absolute_url()
if value.filename:
return u'{}/@@download/{}/{}'.format(base, field, value.filename)
else:
return u'{}/@@download/{}'.format(base, field)
return value
def default(self, obj):
"""Tests the input object, obj, to encode as JSON."""
if hasattr(obj, '__json__'):
return getattr(obj, '__json__')()
import datetime
import isodate
if isinstance(obj, datetime.datetime):
return isodate.datetime_isoformat(obj)
elif isinstance(obj, datetime.date):
return isodate.date_isoformat(obj)
elif isinstance(obj, datetime.time):
return isodate.time_isoformat(obj)
elif isinstance(obj, datetime.timedelta):
return isodate.duration_isoformat(obj)
return simplejson.JSONEncoder.default(self, obj)
def get_isodate(date_str):
"""convert the given date_str string into an iso 8601 date"""
iso_date = None
if not date_str:
return None
#first, is it already a valid isodate?
try:
isodate.parse_date(date_str)
return date_str
except isodate.ISO8601Error, e:
# if not, try to parse it
try:
iso_date = isodate.date_isoformat(timelib.strtodatetime(date_str))
except Exception, e:
log.msg(e.message, level=log.WARNING)
return None
def get_isodate(date_str):
"""convert the given date_str string into an iso 8601 date"""
iso_date = None
if not date_str:
return None
# first, is it already a valid isodate?
try:
isodate.parse_date(date_str)
return date_str
except isodate.ISO8601Error, e:
# if not, try to parse it
try:
iso_date = isodate.date_isoformat(timelib.strtodatetime(date_str))
except Exception, e:
log.msg(e.message, level=log.WARNING)
return None
def default(self, obj):
try:
if isinstance(obj, uuid.UUID):
return str(obj)
elif isinstance(obj, decimal.Decimal):
return float(obj)
elif isinstance(obj, datetime.datetime):
return datetime_isoformat(obj)
elif isinstance(obj, datetime.date):
return date_isoformat(obj)
elif isinstance(obj, pathlib.Path):
return str(obj)
elif isinstance(obj, LooseVersion):
return str(obj)
iterable = iter(obj)
except TypeError:
pass
else:
return list(iterable)
return JSONEncoder.default(self, obj)
def ensure_term_value(self, value):
""" """
if value is EMPTY_VALUE or IFhirPrimitiveType.providedBy(value):
return value
if isinstance(value, list):
value = list([NonFhirTerm.ensure_value_type(val) for val in value])
else:
if isinstance(value, bool):
value = FhirBoolean(value is True and "true" or "false")
elif isinstance(value, int):
value = FhirInteger(value)
elif isinstance(value, float):
value = FhirDecimal(value)
elif isinstance(value, datetime.date):
value = FhirDate(isodate.date_isoformat(value))
elif isinstance(value, datetime.datetime):
value = FhirDateTime(isodate.datetime_isoformat(value))
else:
value = FhirString(value)
return value
def jsonable_dict(self):
output = {}
o_dict = db.to_dict(self)
for k,v in o_dict.items():
if not v or isinstance(
v, (int, long, float, bool, dict, basestring,)):
output[k]=v
elif isinstance(v, datetime.datetime):
output[k] = str(isodate.datetime_isoformat(v))
elif isinstance(v, datetime.date):
output[k] = str(isodate.date_isoformat(v))
elif isinstance(v, db.GeoPt):
output[k] = {'lat': v.lat, 'lon': v.lon}
elif isinstance(v, list):
#This will need better logic but for now addresses lists of Keys
output[k] = [item if type(item)==unicode else str(item)
for item in v]
elif isinstance(v, JsonableModel):
output[k] = v.jsonable_dict()
output[k]['key'] = unicode(v.key())
else:
raise ValueError('cannot encode ' + repr(k))
return output
def default(self, obj):
if isinstance(obj, set):
return list(obj)
if isinstance(obj, datetime):
return u'__datetime__@%s' % isodate.datetime_isoformat(obj)
if isinstance(obj, date):
return u'__date__@%s' % isodate.date_isoformat(obj)
if isinstance(obj, datetime_time):
return u'__time__@%s' % isodate.time_isoformat(obj)
if isinstance(obj, RichTextValue):
return u'__richtext__@%s' % base64.b64encode(
json.dumps(
dict(raw=obj.raw,
encoding=obj.encoding,
mime=obj.mimeType,
output_mime=obj.outputMimeType)))
return json.JSONEncoder.default(self, obj)
def default(self, obj):
if isinstance(obj, set):
return list(obj)
if isinstance(obj, datetime):
return u'__datetime__@%s' % isodate.datetime_isoformat(obj)
if isinstance(obj, date):
return u'__date__@%s' % isodate.date_isoformat(obj)
if isinstance(obj, datetime_time):
return u'__time__@%s' % isodate.time_isoformat(obj)
if isinstance(obj, RichTextValue):
return u'__richtext__@%s' % base64.b64encode(json.dumps(dict(
raw=obj.raw,
encoding=obj.encoding,
mime=obj.mimeType,
output_mime=obj.outputMimeType
)))
return json.JSONEncoder.default(self, obj)
def _assign_publication_defaults(publication):
publication.update({
"updated": date_isoformat(datetime.datetime.now()),
"provenance": {
"geo_location": {
"type": 'Feature',
"geometry": { "type": 'Point', "coordinates": [ 0, 0 ] },
"properties": {
"label": 'Atlantis, Null Island',
"city": 'Atlantis',
"country": 'Null Island',
"continent": 'Africa',
"latitude": 0,
"longitude": 0,
"ISO3166alpha3": 'AAA',
"precision": 'city'
}
}
},
"counts": { "ccgh": 0, "acgh": 0, "wes": 0, "wgs": 0, "ngs": 0, "genomes": 0, "progenetix": 0, "arraymap": 0 },
})
return publication
def isostring(self):
if self.date is None:
return None
if isinstance(self.date, datetime.datetime):
return isodate.datetime_isoformat(self.date)
return isodate.date_isoformat(self.date)
def isostring(self):
if self.date is None:
return None
if isinstance(self.date, datetime.datetime):
return isodate.datetime_isoformat(self.date)
return isodate.date_isoformat(self.date)
def dateago(date, format=None):
return mark_safe(
'<time class="timeago" pubdate="" datetime="%s">%s</time>' % (
date_isoformat(date),
datefilter(date, format)))
def str_encode(self, value):
"""Encode the value into string representation."""
self.validate(value)
if value is not None:
value = isodate.date_isoformat(value)
return value
def toExternalObject(self, **unused_kwargs):
return isodate.date_isoformat(self.date)
def _create_collations_from_dataset(ds_id, **byc):
coll_types = byc["these_prefs"]["collationed"]
# coll_types = { "NCIT": { } }
# coll_types = { "PMID": { } }
for pre in coll_types.keys():
pre_h_f = path.join(byc["pkg_path"], "byconeer", "rsrc", pre,
"numbered-hierarchies.tsv")
if path.exists(pre_h_f):
print("Creating hierarchy for " + pre)
hier = get_prefix_hierarchy(ds_id, pre, pre_h_f, **byc)
elif "PMID" in pre:
hier = _make_dummy_publication_hierarchy(**byc)
else:
# create /retrieve hierarchy tree; method to be developed
print("Creating dummy hierarchy for " + pre)
hier = _get_dummy_hierarchy(ds_id, pre, **byc)
coll_client = MongoClient()
coll_coll = coll_client[ds_id][byc["config"]["collations_coll"]]
data_client = MongoClient()
data_db = data_client[ds_id]
data_coll = data_db[byc["config"]["collations_source"]]
data_key = byc["filter_definitions"][pre]["db_key"]
data_pat = byc["filter_definitions"][pre]["pattern_strict"]
onto_ids = _get_ids_for_prefix(data_coll, data_key, data_pat)
sel_hiers = []
# get the set of all parents for sample codes
data_parents = set()
for o_id in onto_ids:
if o_id in hier:
data_parents.update(hier[o_id]["parent_terms"])
no = len(hier.keys())
matched = 0
if not byc["args"].test:
bar = Bar("Writing " + pre,
max=no,
suffix='%(percent)d%%' + " of " + str(no))
for count, code in enumerate(hier.keys(), start=1):
if not byc["args"].test:
bar.next()
children = list(set(hier[code]["child_terms"]) & set(data_parents))
hier[code].update({"child_terms": children})
if len(children) < 1:
if byc["args"].test:
print(code + " w/o children")
continue
code_no = data_coll.count_documents({data_key: code})
if len(children) < 2:
child_no = code_no
else:
child_no = data_coll.count_documents(
{data_key: {
"$in": children
}})
if child_no > 0:
hier[code].update({
"code_matches":
code_no,
"count":
child_no,
"date":
date_isoformat(datetime.datetime.now())
})
matched += 1
if not byc["args"].test:
sel_hiers.append(hier[code])
else:
print("{}:\t{} ({} deep) samples - {} / {} {}".format(
code, code_no, child_no, count, no, pre))
if not byc["args"].test:
bar.finish()
print("==> Updating database ...")
if matched > 0:
coll_clean_q = {"id": {"$regex": "^" + pre}}
coll_coll.delete_many(coll_clean_q)
coll_coll.insert_many(sel_hiers)
print("===> Found {} of {} {} codes & added them to {}.{} <===".format(
matched, no, pre, ds_id, byc["config"]["collations_coll"]))
def format_value(self, value):
if isinstance(value, date):
return isodate.date_isoformat(value, self.format, self.yeardigits)
return value
def __repr__(self):
return 'Cx(%s, %d, %d, %d:%s)' % (
self.outbreak, self.training, self.horizon, self.now,
isodate.date_isoformat(self.now_date))
def model_to_json(self, model_value):
# RFC 3339 string in JSON
return date_isoformat(model_value)
def dateago(date, format=None):
return mark_safe(
'<time class="timeago" pubdate="" datetime="%s">%s</time>' %
(date_isoformat(date), datefilter(date, format)))
REGISTRY = [ ]
def polymorphic_handler(obj):
found = False
val = None
for ri in REGISTRY:
typ, func = ri
if isinstance(obj, typ):
found = True
val = func(obj)
break
if found:
return val
raise TypeError, 'Object of type %s with value of %s is not JSON serializable' % (type(obj), repr(obj))
# Practical example: add support for datetime serialization
from datetime import datetime, date
import isodate
REGISTRY.append( (datetime, lambda o: isodate.datetime_isoformat(o)) )
REGISTRY.append( (date, lambda o: isodate.date_isoformat(o)) )
# add Decimal support
import decimal
REGISTRY.append( (decimal.Decimal, lambda o: str(o)) )
def _format_value(self, value):
if isinstance(value, date):
return isodate.date_isoformat(value, self.format, self.yeardigits)
return value
def to_url(self, value):
return date_isoformat(value)
