forked from foxmask/django-th
-
Notifications
You must be signed in to change notification settings - Fork 0
/
fire_asyncio.py
204 lines (167 loc) · 7.02 KB
/
fire_asyncio.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import os
import datetime
import time
import arrow
import asyncio
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_th.settings")
from django.conf import settings
from django_th.services import default_provider
from django_th.models import TriggerService
from django.utils.log import getLogger
# create logger
logger = getLogger('django_th.trigger_happy')
q = asyncio.Queue(maxsize=0)
q2 = asyncio.Queue(maxsize=0)
def to_datetime(data):
"""
convert Datetime 9-tuple to the date and time format
feedparser provides this 9-tuple
"""
my_date_time = None
if 'published_parsed' in data:
my_date_time = datetime.datetime.fromtimestamp(
time.mktime(data.published_parsed))
elif 'updated_parsed' in data:
my_date_time = datetime.datetime.fromtimestamp(
time.mktime(data.updated_parsed))
return my_date_time
@asyncio.coroutine
def update_trigger(service):
"""
update the date when occurs the trigger
"""
my_count = yield from q2.get()
if my_count > 0:
logger.info("user: {} - provider: {} - consumer: {} - {} = {} new data".format(
service.user, service.provider.name.name, service.consumer.name.name, service.description, my_count))
trigger = TriggerService.objects.get(id=service.id)
if trigger:
# set the current datetime
now = arrow.utcnow().to(
settings.TIME_ZONE).format('YYYY-MM-DD HH:mm:ss')
trigger.date_triggered = now
trigger.save()
else:
logger.info("user: {} - provider: {} - consumer: {} - {} nothing new".format(
service.user, service.provider.name.name, service.consumer.name.name, service.description))
asyncio.get_event_loop().stop()
@asyncio.coroutine
def my_dummy_provider():
"""
just a dummy provider when its the first time the trigger is handling
"""
yield from q.put(1)
@asyncio.coroutine
def my_provider(service_provider, token, service_id, date_triggered):
"""
service_provider : the name of the class to trigger the service
token : is the token of the service provider from the database
service_id : is the service id from the database
date_triggered : date_triggered is the data from the database
"""
datas = getattr(service_provider, 'process_data')(
token, service_id, date_triggered)
for data in datas:
yield from q.put(data)
@asyncio.coroutine
def my_consumer(service_consumer, token, service_id, date_triggered):
"""
service_consumer : the name of the consumer 'service' class
token : is the token of the service consumer
service_id : is the service id from the database
date_triggered : date_triggered is the data from the database
"""
count_new_data = 0
proceed = False
while q.empty() is not True:
data = yield from q.get()
consumer = getattr(service_consumer, 'save_data')
published = ''
which_date = ''
# flag to know if we can push data to the consumer
# 2) for each one
# if in a pool of data once of them does not have
# a date, will take the previous date for this one
# if it's the first one, set it to 00:00:00
# let's try to determine the date contained in the data...
published = to_datetime(data)
if published is not None:
# get the published date of the provider
published = arrow.get(str(published), 'YYYY-MM-DD HH:mm:ss')
# store the date for the next loop
# if published became 'None'
which_date = published
#... otherwise set it to 00:00:00 of the current date
if which_date == '':
# current date
which_date = arrow.utcnow().replace(
hour=0, minute=0, second=0)
published = which_date
if published is None and which_date != '':
published = which_date
# 3) check if the previous trigger is older than the
# date of the data we retreived
# if yes , process the consumer
# add the TIME_ZONE settings
my_date_triggered = arrow.get(
str(date_triggered), 'YYYY-MM-DD HH:mm:ss').to(settings.TIME_ZONE)
# if the published date if greater or equal to the last
# triggered event ... :
if my_date_triggered is not None and published is not None and published.date() >= my_date_triggered.date():
# if date are the same ...
if published.date() == my_date_triggered.date():
# ... compare time and proceed if needed
if published.time() >= my_date_triggered.time():
proceed = True
# not same date so proceed !
else:
proceed = True
if proceed:
if 'title' in data:
logger.info("date {} >= date triggered {} title {}".format(
published, date_triggered, data['title']))
else:
logger.info(
"date {} >= date triggered {} ".format(published, my_date_triggered))
consumer(token, service_id, **data)
count_new_data += 1
# otherwise do nothing
else:
if 'title' in data:
logger.debug(
"data outdated skiped : [{}] {}".format(published, data['title']))
else:
logger.debug(
"data outdated skiped : [{}] ".format(published))
else:
# return the number of updates ( to be displayed in the log )
yield from q2.put(count_new_data)
default_provider.load_services()
trigger = TriggerService.objects.filter(status=True)
if trigger:
for service in trigger:
# provider - the service that offer datas
service_name = str(service.provider.name.name)
service_provider = default_provider.get_service(service_name)
# consumer - the service which uses the data
service_name = str(service.consumer.name.name)
service_consumer = default_provider.get_service(service_name)
# First run
if service.date_triggered is None:
logger.debug("first run for %s => %s " % (str(
service.provider.name), str(service.consumer.name.name)))
asyncio.get_event_loop().run_until_complete(my_dummy_provider)
# another run
else:
asyncio.get_event_loop().run_until_complete(
my_provider(service_provider, service.provider.token, service.id, service.date_triggered))
# process done in every case
asyncio.get_event_loop().run_until_complete(
my_consumer(service_consumer, service.consumer.token, service.id, service.date_triggered))
asyncio.get_event_loop().run_until_complete(update_trigger(service))
asyncio.get_event_loop().run_forever()
else:
print("No trigger set by any user")