def print_chgsets_files(self, db, last_update):
'''Generate two files for each cset, a geojson file containing changets changes
and one containing bbox of changeset'''
files = [] # List of files we generated
if db:
for c in db.chgsets_find(state=db.STATE_DONE, after=last_update, sort=False):
try:
fn = self.geojson.format(id=c['cid'])
logger.info("Export cset {} diff to file '{}', last update {}".format(c['cid'], fn, last_update))
cset = osm.changeset.Changeset(id=c['cid'], api=None)
meta = db.chgset_get_meta(c['cid'])
cset.meta = meta
info = db.chgset_get_info(c['cid'])
cset.data_import(info)
with tempfilewriter.TempFileWriter(join(self.path, fn)) as f:
json.dump(cset.getGeoJsonDiff(), f)
files.append(fn)
b = '{},{},{},{}'.format(cset.meta['min_lat'], cset.meta['min_lon'],
cset.meta['max_lat'], cset.meta['max_lon'])
fn = self.bbox.format(id=c['cid'])
logger.info("Export cset {} bounds to file '{}'".format(c['cid'], fn))
with tempfilewriter.TempFileWriter(join(self.path, fn)) as f:
f.write(b)
files.append(fn)
except Exception as e:
exc_info = sys.exc_info()
logger.error('Error exporting cid {}: {}'.format(c['cid'], exc_info))
logger.error('Cset data: {}'.format(c))
#raise exc_info[1], None, exc_info[2]
db_show_brief(None, db, False)
return files
def print_chgsets_bbox(self, db):
'''Generate a single file with all bboxes of all changesets'''
geoj = { "type": "FeatureCollection",
"features": [] }
if db:
for c in db.chgsets_find(state=db.STATE_DONE, sort=False):
self.add_cset_bbox(c, db, geoj)
logger.debug('Data sent to json file={}'.format(geoj))
with tempfilewriter.TempFileWriter(join(self.path, self.list_fname)) as f:
f.write(json.dumps(geoj))
def print_state(self, db, update_reason):
if update_reason != 'new_generation.osmtracker':
logger.debug('Not updating, reason {}'.format(update_reason))
return
now = datetime.datetime.now()
#if now.day != self.last_update.day:
# print('Cycler - new day: {} {}'.format(now.day, self.last_update.day))
#period = now-state.cset_start_time
#if period.total_seconds() > self.config.get('horizon_hours', 'tracker')*3600:
# os.rename(self.list_fname, self.list_fname_old)
# print('Cycler')
# state.clear_csets()
self.last_update = datetime.datetime.now()
if self.generation != db.generation:
self.generation = db.generation
template = self.env.get_template(self.template_name)
ctx = {
'csets': [],
'csets_err': [],
'csetmeta': {},
'csetinfo': {},
'show_details': self.show_details,
'show_comments': self.show_comments
}
# All Backend config with 'cfg_' prefix
for k, v in self.cfg.iteritems():
ctx['cfg_' + k] = v
notes = 0
csets_total = 0
csets_w_notes = 0
csets_w_addr_changes = 0
for c in db.chgsets_find(state=[
db.STATE_CLOSED, db.STATE_OPEN, db.STATE_ANALYSING2,
db.STATE_REANALYSING, db.STATE_DONE, db.STATE_NEW
]):
cid = c['cid']
logger.debug('Cset {}={}'.format(cid, c))
logger.debug('Backend labels {}, cset labels {}'.format(
self.labels, c['labels']))
if self.labels and not set(c['labels']).intersection(
self.labels):
logger.warn('Skipping cid {}, labels {}/{}'.format(
cid, self.labels, c['labels']))
continue
info = db.chgset_get_info(cid)
meta = db.chgset_get_meta(cid)
meta, info = schemacheck.check(c, meta, info)
ctx['csetmeta'][cid] = meta
ctx['csetinfo'][cid] = info
if c['state'] == db.STATE_QUARANTINED:
ctx['csets_err'].append(c)
else:
ctx['csets'].append(c)
csets_total += 1
#if meta['open'] or (info and 'truncated' in info['state']):
# logger.warn('Skipping cid {}'.format(cid))
# continue
notecnt = int(
meta['comments_count']
) # FIXME: This is duplicate w. BackendHtmlSummary.py
if notecnt > 0:
notes += int(meta['comments_count'])
csets_w_notes += 1
#if c['state'] != db.STATE_DONE:
# continue
if 'address-node-change' in c[
'labels']: #FIXME: does not belong here - this is configuration
csets_w_addr_changes += 1
ctx['csets_num_total'] = csets_total
ctx['csets_with_notes'] = csets_w_notes
ctx['csets_with_addr_changes'] = csets_w_addr_changes
logger.debug('Data passed to template: {}'.format(ctx))
with tempfilewriter.TempFileWriter(self.list_fname) as f:
f.write(
'<!-- Generated by OpenStreetMap Analytic Difference Engine -->\n'
)
f.write(template.render(ctx))
def print_state(self, db, update_reason):
force = True # Because otherwise 'summary_created' timestamp below is not updated
if not db or self.generation != db.generation or force:
if not db or not db.pointer:
return
template = self.env.get_template(self.template_name)
self.generation = db.generation
now = datetime.datetime.utcnow().replace(tzinfo=pytz.utc)
dbptr = db.pointer
data = {
'csets': [],
'track_starttime':
dbptr['first_pointer']['timestamp'],
'track_endtime':
dbptr['timestamp'],
'tracked_hours':
(dbptr['timestamp'] -
dbptr['first_pointer']['timestamp']).total_seconds() / 3600,
'summary_created':
now,
'pointer_timestamp':
dbptr['timestamp'],
'first_seqno':
dbptr['first_pointer']['seqno'],
'latest_seqno':
dbptr['seqno'] - 1,
'generation':
self.generation
}
cset_tracked_hours = (
dbptr['timestamp'] -
dbptr['first_pointer']['timestamp']).total_seconds() / 3600
cset_tracked_hours = min(cset_tracked_hours, self.horizon_hours)
users = {}
notes = 0
csets_w_notes = 0
csets_w_addr_changes = 0
for c in db.chgsets_find(state=[
db.STATE_CLOSED, db.STATE_OPEN, db.STATE_ANALYSING2,
db.STATE_REANALYSING, db.STATE_DONE
],
sort=False):
data['csets'].append(c)
cid = c['cid']
meta = db.chgset_get_meta(cid)
info = db.chgset_get_info(cid)
meta, info = schemacheck.check(c, meta, info)
user = meta['user']
users[user] = users.get(user, 0) + 1
if meta['open'] or (info and 'truncated' in info['state']):
continue
notecnt = int(meta['comments_count'])
if notecnt > 0:
notes += int(meta['comments_count'])
csets_w_notes += 1
if c['state'] != db.STATE_DONE:
continue
if 'address-node-change' in c[
'labels']: #FIXME: does not belong here - this is configuration
csets_w_addr_changes += 1
data['csets_with_notes'] = csets_w_notes
data['csets_with_addr_changes'] = csets_w_addr_changes
# Summarize edits and mileage - we don't do this incrementally
# since csets can be split over multiple diffs
edits = {
'node': {
'create': 0,
'modify': 0,
'delete': 0
},
'way': {
'create': 0,
'modify': 0,
'delete': 0
},
'relation': {
'create': 0,
'modify': 0,
'delete': 0
}
}
mileage = {}
for c in db.chgsets_find(state=db.STATE_DONE, sort=False):
cid = c['cid']
info = db.chgset_get_info(cid)
_, info = schemacheck.check(c, None, info)
if 'truncated' in info['state']:
continue
summary = info['summary']
for action in ['create', 'modify', 'delete']:
if summary['_' + action] > 0:
for type in ['node', 'way', 'relation']:
edits[type][action] += summary[action][type]
self.merge_int_dict(mileage, info['mileage_m'])
data['edits'] = edits
data['users'] = users
data['notes'] = notes
logger.debug('Accumulated mileage: {}'.format(mileage))
mileage_bytype = []
if mileage:
sum = 0
num_items = 0
by_type = mileage['by_type']
for cat in by_type.keys():
mi = [(t, int(by_type[cat][t]))
for t in by_type[cat].keys()]
mi = sorted(mi, key=lambda x: x[1], reverse=True)
for typ, m in mi:
if num_items < 13:
mileage_bytype.append((cat, typ, self._i2s(m)))
sum += m
num_items += 1
data['mileage_bytype'] = mileage_bytype
data['mileage_meter'] = self._i2s(sum)
# Rounding means that if cset_tracked_hours=0, then we pretend the current
# metrics are for one hours
data['mileage_meter_per_hour'] = self._i2s(
int(sum / max(1, cset_tracked_hours)))
# Export info about analytics queue
now = datetime.datetime.utcnow().replace(tzinfo=pytz.utc)
oldest_ts = None
processing_cnt = 0
for c in db.chgsets_find(state=[
db.STATE_NEW, db.STATE_BOUNDS_CHECK,
db.STATE_BOUNDS_CHECKED, db.STATE_ANALYSING1,
db.STATE_ANALYSING2
],
sort=False):
age_s = (now - c['queued']).total_seconds()
if oldest_ts is None or age_s > oldest:
oldest = age_s
oldest_cid = c['cid']
oldest_ts = c['queued']
processing_cnt += 1
data['processing_outstanding_cset_cnt'] = processing_cnt
if oldest_ts:
logging.info(
'Processing lag governed by cset {}, timestamp {}'.format(
oldest_cid, oldest_ts))
data['processing_oldest_outstanding_cset'] = oldest_ts
#if hasattr(state, 'pointer'): # FIXME
# lag = now-state.pointer.timestamp()
# data['lag_seconds'] = int(lag.seconds)
logger.debug('Data passed to template: {}'.format(data))
with tempfilewriter.TempFileWriter(self.list_fname) as f:
f.write(
'<!-- Generated by OpenStreetMap Analytic Difference Engine -->'
)
f.write(template.render(data))