-
Notifications
You must be signed in to change notification settings - Fork 0
/
update.py
71 lines (53 loc) · 2.3 KB
/
update.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
import osmdt
import datetime
import osmhm
import time
osmhm.config.database_url = 'postgres://osm_users:osm@localhost/osm_users'
def convert_time(input_time):
return datetime.datetime.strptime(input_time, '%Y-%m-%dT%H:%M:%SZ')
while True:
sequence = osmhm.fetch.fetch_last_read()
if not sequence:
osmhm.fetch.fetch_next(time_type='hour', reset=True)
sequence = osmhm.fetch.fetch_last_read()
if sequence['read_flag'] is False:
print "Processing sequence %s." % (sequence['sequencenumber'])
data_stream = osmdt.fetch(sequence['sequencenumber'], time='hour')
data_object = osmdt.process(data_stream)
users = osmdt.extract_users(data_object)
conn = osmhm.connect.connect()
for user in users:
userid = users[user]['uid']
username = user
a_time = users[user]['timestamps'][0]
a_time = convert_time(a_time)
cursor = conn.cursor()
cursor.execute("""DO $do$ BEGIN IF EXISTS (SELECT 1 FROM users WHERE username = %s) THEN
UPDATE users SET end_date = %s WHERE username = %s; ELSE
INSERT INTO users (userid, username, start_date, end_date)
VALUES (%s, %s, %s, %s); END IF; END $do$""",
(username, a_time, username, userid, username, a_time, a_time))
conn.commit()
cursor.close()
del cursor
osmhm.inserts.insert_file_read()
print "Finished processing %s." % (sequence['sequencenumber'])
next_time = convert_time(sequence['timestamp']) + datetime.timedelta(minutes=60)
if datetime.datetime.utcnow() < next_time:
sleep_time = (next_time - datetime.datetime.utcnow()).seconds + 60
print "Waiting %2.1f seconds for the next file." % (sleep_time)
else:
sleep_time = 0
time.sleep(sleep_time)
count = 0
while True:
try:
count += 1
osmhm.fetch.fetch_next(sequence['sequencenumber'], time_type='hour')
break
except:
if count == 5:
msg = 'New state file not retrievable after five times.'
raise Exception(msg)
print "Waiting %2.1f more seconds..." % (extra_time)
time.sleep(extra_time)