def run(sequence, time='hour', changesets=True, objects=True, users=True):
"""
"""
import osmdt
if not changesets and not objects and not users:
raise Exception('No output variables are specified')
data_stream = osmdt.fetch(sequence, time=time)
data_object = osmdt.process(data_stream)
if changesets:
changeset_dict = osmdt.extract_changesets(data_object)
else:
changeset_dict = {}
if objects:
object_dict = osmdt.extract_objects(data_object)
else:
object_dict = {}
if users:
user_dict = osmdt.extract_users(data_object)
else:
user_dict = {}
return changeset_dict, object_dict, user_dict
def run(sequence,time='hour',changesets=True,objects=True,users=True):
"""
"""
import osmdt
if not changesets and not objects and not users:
raise Exception('No output variables are specified')
data_stream = osmdt.fetch(sequence,time=time)
data_object = osmdt.process(data_stream)
if changesets:
changeset_dict = osmdt.extract_changesets(data_object)
else:
changeset_dict = {}
if objects:
object_dict = osmdt.extract_objects(data_object)
else:
object_dict = {}
if users:
user_dict = osmdt.extract_users(data_object)
else:
user_dict = {}
return changeset_dict, object_dict, user_dict
def test_extract_users():
f = gzip.GzipFile('tests/040.osc.gz')
data = osmdt.process(f)
users = osmdt.extract_users(data)
assert 'sammuell_imports' in users
assert len(users) == 8
while True:
sequence = osmhm.fetch.fetch_last_read()
if not sequence:
osmhm.fetch.fetch_next(time_type='hour', reset=True)
sequence = osmhm.fetch.fetch_last_read()
if sequence['read_flag'] is False:
print "Processing sequence %s." % (sequence['sequencenumber'])
data_stream = osmdt.fetch(sequence['sequencenumber'], time='hour')
data_object = osmdt.process(data_stream)
users = osmdt.extract_users(data_object)
conn = osmhm.connect.connect()
for user in users:
userid = users[user]['uid']
username = user
a_time = users[user]['timestamps'][0]
a_time = convert_time(a_time)
cursor = conn.cursor()
cursor.execute("""DO $do$ BEGIN IF EXISTS (SELECT 1 FROM users WHERE username = %s) THEN
UPDATE users SET end_date = %s WHERE username = %s; ELSE
INSERT INTO users (userid, username, start_date, end_date)
VALUES (%s, %s, %s, %s); END IF; END $do$""",
(username, a_time, username, userid, username, a_time, a_time))
def run(time_type='hour', history=False, suspicious=False, monitor=True,
notification=False, notifier=send_notification.send_mail):
"""
"""
import osmhm
import osmdt
import datetime
import time
while True:
sequence = osmhm.fetch.fetch_last_read()
if not sequence:
osmhm.fetch.fetch_next(time_type=time_type, reset=True)
sequence = osmhm.fetch.fetch_last_read()
if sequence['read_flag'] is False:
print "Processing sequence %s." % (sequence['sequencenumber'])
count = 0
while True:
try:
count += 1
data_stream = osmdt.fetch(sequence['sequencenumber'], time=time_type)
break
except:
if count == 5:
msg = 'Current state file not retrievable after five times.'
raise Exception(msg)
print "File not reachable; waiting 60 more seconds..."
time.sleep(60)
data_object = osmdt.process(data_stream)
del data_stream
changesets = osmdt.extract_changesets(data_object)
objects = osmdt.extract_objects(data_object)
users = osmdt.extract_users(data_object)
del data_object
if history:
osmhm.inserts.insert_all_changesets(changesets)
if suspicious:
osmhm.filters.suspicious_filter(changesets)
if monitor:
osmhm.filters.object_filter(objects, notification=notification, notifier=notifier)
osmhm.filters.user_filter(changesets, notification=notification, notifier=notifier)
#osmhm.filters.user_object_filter(objects, notification=notification, notifier=notifier) # not implemented yet
osmhm.filters.key_filter(objects, notification=notification, notifier=notifier)
del changesets, objects, users
osmhm.inserts.insert_file_read()
print "Finished processing %s." % (sequence['sequencenumber'])
if sequence['timetype'] == 'minute':
delta_time = 1
extra_time = 10
elif sequence['timetype'] == 'hour':
delta_time = 60
extra_time = 120
elif sequence['timetype'] == 'day':
delta_time = 1440
extra_time = 300
next_time = datetime.datetime.strptime(sequence['timestamp'],
"%Y-%m-%dT%H:%M:%SZ") + datetime.timedelta(minutes=delta_time)
if datetime.datetime.utcnow() < next_time:
sleep_time = (next_time - datetime.datetime.utcnow()).seconds + delta_time
print "Waiting %2.1f seconds for the next file." % (sleep_time)
else:
sleep_time = 1
time.sleep(sleep_time)
count = 0
while True:
try:
count += 1
osmhm.fetch.fetch_next(sequence['sequencenumber'], time_type=time_type)
break
except:
if count == 5:
msg = 'New state file not retrievable after five times.'
raise Exception(msg)
print "Waiting %2.1f more seconds..." % (extra_time)
time.sleep(extra_time)
def convert_time(input_time):
return datetime.datetime.strptime(input_time, '%Y-%m-%dT%H:%M:%SZ')
conn = psycopg2.connect("dbname='osm_users' host='localhost' user='******' password='******' options='-c log_min_messages=PANIC'")
for i in range(99999999):
print 'Now starting file %i.' % (i)
try:
temp = osmdt.fetch(str(i), time='day')
except:
print 'Error with retrieving file %i.' % (i)
break
temp = osmdt.process(temp)
users = osmdt.extract_users(temp)
del temp
for user in users:
userid = users[user]['uid']
username = user
a_time = users[user]['timestamps'][-1]
a_time = convert_time(a_time)
cursor = conn.cursor()
cursor.execute("""DO $do$ BEGIN IF EXISTS (SELECT 1 FROM users WHERE username = %s) THEN
UPDATE users SET end_date = %s WHERE username = %s; ELSE
INSERT INTO users (userid, username, start_date, end_date)
VALUES (%s, %s, %s, %s); END IF; END $do$""",
(username, a_time, username, userid, username, a_time, a_time))
conn.commit()
