def publishing(service):
"""
the purpose of this tasks is to get the data from the cache
then publish them
:param service: service object where we will publish
:type service: object
"""
# flag to know if we have to update
to_update = False
# flag to get the status of a service
status = False
# provider - the service that offer data
# check if the service has already been triggered
# if date_triggered is None, then it's the first run
if service.date_triggered is None:
logger.debug("first run {}".format(service))
to_update = True
status = True
# run run run
else:
service_provider = default_provider.get_service(
str(service.provider.name.name))
# 1) get the data from the provider service
kw = {'trigger_id': service.id}
data = getattr(service_provider, 'process_data')(**kw)
# counting the new data to store to display them in the log
count_new_data = len(data) if data else 0
if count_new_data > 0:
# consumer - the service which uses the data
service_consumer = default_provider.get_service(
str(service.consumer.name.name))
getattr(service_consumer, '__init__')(service.consumer.token)
consumer = getattr(service_consumer, 'save_data')
# 2) for each one
for d in data:
# the consumer will save the data and return if success or not
status = consumer(service.id, **d)
to_update = True
# let's log
log_update(service, to_update, status, count_new_data)
# let's update
if to_update and status:
update_trigger(service)
def reading(self, service):
"""
get the data from the service and put theme in cache
:param service: service object to read
:type service: object
"""
now = arrow.utcnow().to(settings.TIME_ZONE).format(
'YYYY-MM-DD HH:mm:ssZZ')
# flag to know if we have to update
to_update = False
# counting the new data to store to display them in the log
# provider - the service that offer data
provider_token = service.provider.token
default_provider.load_services()
service_provider = default_provider.get_service(
str(service.provider.name.name))
# check if the service has already been triggered
# if date_triggered is None, then it's the first run
# so it will be set to "now"
date_triggered = service.date_triggered if service.date_triggered \
else now
# 1) get the data from the provider service
# get a timestamp of the last triggered of the service
kwargs = {'token': provider_token,
'trigger_id': service.id,
'date_triggered': date_triggered}
data = self.provider(service_provider, **kwargs)
# counting the new data to store to display them in the log
count_new_data = len(data) if data else 0
if count_new_data > 0:
to_update = True
if to_update:
logger.info("{} - {} new data".format(service, count_new_data))
def reading(service):
"""
get the data from the service and put theme in cache
:param service: service object to read
"""
# flag to know if we have to update
to_update = False
# flag to get the status of a service
status = False
# counting the new data to store to display them in the log
# provider - the service that offer data
service_provider = default_provider.get_service(
str(service.provider.name.name))
# check if the service has already been triggered
# if date_triggered is None, then it's the first run
if service.date_triggered is None:
logger.debug("first time {}".format(service))
to_update = True
status = True
# run run run
else:
# 1) get the data from the provider service
# get a timestamp of the last triggered of the service
kw = {'token': service.provider.token,
'trigger_id': service.id,
'date_triggered': service.date_triggered}
datas = getattr(service_provider, '__init__')(service.provider.token)
datas = getattr(service_provider, 'read_data')(**kw)
if len(datas) > 0:
to_update = True
status = True
log_update(service, to_update, status, len(datas))
def consumer(self, service, data, to_update, status):
"""
call the consumer and handle the data
:param service:
:param data:
:param to_update:
:param status:
:return: status
"""
# consumer - the service which uses the data
service_consumer = default_provider.get_service(
str(service.consumer.name.name))
kwargs = {'user': service.user}
getattr(service_consumer, '__init__')(service.consumer.token,
**kwargs)
instance = getattr(service_consumer, 'save_data')
# 2) for each one
for d in data:
d['userservice_id'] = service.consumer.id
# the consumer will save the data and return if success or not
status = instance(service.id, **d)
to_update = True
return to_update, status
def form_valid(self, form):
name = form.cleaned_data.get('name').name
user = self.request.user
if UserService.objects.filter(name=name, user=user).exists():
from django.contrib import messages
messages.warning(
self.request,
'Service %s already activated' % name.split('Service')[1])
return HttpResponseRedirect(reverse('user_services'))
else:
form.save(user=user)
sa = ServicesActivated.objects.get(
name=form.cleaned_data.get('name').name)
# let's build the 'call' of the auth method
# which own to a Service Class
if sa.auth_required:
# use the default_provider to get the object from the ServiceXXX
default_provider.load_services()
service_object = default_provider.get_service(
str(form.cleaned_data.get('name').name))
# get the class object
lets_auth = getattr(service_object, 'auth')
# call the auth func from this class
# and redirect to the external service page
# to auth the application django-th to access to the user
# account details
return redirect(lets_auth(self.request))
return HttpResponseRedirect(
reverse('user_services', args=['added']))
def reading(self, service):
"""
get the data from the service and put theme in cache
:param service: service object to read
:type service: object
"""
# counting the new data to store to display them in the log provider - the service that offer data
provider_token = service.provider.token
default_provider.load_services()
service_provider = default_provider.get_service(
str(service.provider.name.name))
date_triggered = service.date_triggered if service.date_triggered else service.date_created
# get the data from the provider service
kwargs = {
'token': provider_token,
'trigger_id': service.id,
'date_triggered': date_triggered
}
data = self.provider(service_provider, **kwargs)
if len(data) > 0:
logger.info("{} - {} new data".format(service, len(data)))
elif data is False:
# if data is False, something went wrong
self.is_ceil_reached(service)
def consumer(self, service, data, to_update, status):
"""
call the consumer and handle the data
:param service:
:param data:
:param to_update:
:param status:
:return: status
"""
# consumer - the service which uses the data
service_consumer = default_provider.get_service(
str(service.consumer.name.name))
kwargs = {'user': service.user}
getattr(service_consumer, '__init__')(service.consumer.token, **kwargs)
instance = getattr(service_consumer, 'save_data')
# 2) for each one
for d in data:
d['userservice_id'] = service.consumer.id
# the consumer will save the data and return if success or not
status = instance(service.id, **d)
to_update = True
return to_update, status
def reading(self, service):
"""
get the data from the service and put theme in cache
:param service: service object to read
:type service: object
"""
now = arrow.utcnow().to(
settings.TIME_ZONE).format('YYYY-MM-DD HH:mm:ssZZ')
# counting the new data to store to display them in the log
# provider - the service that offer data
provider_token = service.provider.token
default_provider.load_services()
service_provider = default_provider.get_service(
str(service.provider.name.name))
# check if the service has already been triggered
# if date_triggered is None, then it's the first run
# so it will be set to "now"
date_triggered = service.date_triggered if service.date_triggered \
else now
# 1) get the data from the provider service
# get a timestamp of the last triggered of the service
kwargs = {
'token': provider_token,
'trigger_id': service.id,
'date_triggered': date_triggered
}
data = self.provider(service_provider, **kwargs)
if len(data) > 0:
logger.info("{} - {} new data".format(service, len(data)))
elif data is False:
# if data is False, something went wrong
self.is_ceil_reached(service)
def form_valid(self, form):
name = form.cleaned_data.get('name').name
user = self.request.user
if UserService.objects.filter(name=name, user=user).exists():
from django.contrib import messages
messages.warning(self.request, 'Service %s already activated' %
name.split('Service')[1])
return HttpResponseRedirect(reverse('user_services'))
else:
form.save(user=user)
sa = ServicesActivated.objects.get(
name=form.cleaned_data.get('name').name)
# let's build the 'call' of the auth method
# which own to a Service Class
if sa.auth_required:
# use the default_provider to get the object from the ServiceXXX
default_provider.load_services()
service_object = default_provider.get_service(
str(form.cleaned_data.get('name').name))
# get the class object
lets_auth = getattr(service_object, 'auth')
# call the auth func from this class
# and redirect to the external service page
# to auth the application django-th to access to the user
# account details
return redirect(lets_auth(self.request))
return HttpResponseRedirect(reverse('user_services',
args=['added']))
def form_valid(self, form):
"""
save the data
:param form:
:return:
"""
valid = True
# 'name' is injected in the clean() of the form line 56
name = form.cleaned_data.get('name').name
user = self.request.user
form.save(user=user, service_name=name)
sa = ServicesActivated.objects.get(name=name)
if sa.auth_required and sa.self_hosted:
# trigger the checking of the service
from django_th.services import default_provider
default_provider.load_services()
service_provider = default_provider.get_service(name)
result = service_provider.check(self.request, user)
if result is not True:
# the call of the API failed due to an error which is in the result string
# return by the call of the API
form.add_error('host', result)
messages.error(self.request, result)
return redirect('edit_service', pk=self.kwargs.get(self.pk_url_kwarg))
if valid:
messages.success(self.request, _('Service %s modified successfully') % name.split('Service')[1])
return HttpResponseRedirect(reverse('user_services'))
def go():
"""
run the main process
"""
trigger = TriggerService.objects.all()
if trigger:
for service in trigger:
logger.info("from %s to %s ", service.provider.name,
service.consummer.name)
# provider - the service that offer datas
service_name = service.provider.name
service_provider = default_provider.get_service(
'Service' + str(service_name).capitalize())
# consummer - the service which uses the datas
service_name = service.consummer.name
service_consummer = default_provider.get_service(
'Service' + str(service_name).capitalize())
# 1) get the datas from the provider service
datas = getattr(service_provider, 'process_data')(service.id)
consummer = getattr(service_consummer, 'save_data')
# 2) for each one
for data in datas:
title = data.title
content = data.content[0].value
logger.info("from the service %s", service.provider)
# 3) check if the previous trigger is older than the
# date of the data we retreived
# if yes , process the consummer
if service.date_triggered is None or to_datetime(
data.published) >= service.date_triggered:
logger.debug("date %s title %s", data.published,
data.title)
logger.info("to the service %s", service.consummer)
consummer(service.consummer.token, title, content,
service.id)
# otherwise do nothing
else:
logger.debug("DATA TOO OLD SKIPED : [%s] %s",
data.published, data.title)
# update the date of the trigger
update_trigger(service)
else:
print "No trigger set by any user"
def go():
"""
run the main process
"""
trigger = TriggerService.objects.all()
if trigger:
for service in trigger:
logger.info(
"from %s to %s ", service.provider.name, service.consummer.name)
# provider - the service that offer datas
service_name = service.provider.name
service_provider = default_provider.get_service(
'Service' + str(service_name).capitalize())
# consummer - the service which uses the datas
service_name = service.consummer.name
service_consummer = default_provider.get_service(
'Service' + str(service_name).capitalize())
# 1) get the datas from the provider service
datas = getattr(service_provider, 'process_data')(service.id)
consummer = getattr(service_consummer, 'save_data')
# 2) for each one
for data in datas:
title = data.title
content = data.content[0].value
logger.info("from the service %s", service.provider)
# 3) check if the previous trigger is older than the
# date of the data we retreived
# if yes , process the consummer
if service.date_triggered is None or to_datetime(data.published) >= service.date_triggered:
logger.debug(
"date %s title %s", data.published, data.title)
logger.info("to the service %s", service.consummer)
consummer(
service.consummer.token, title, content, service.id)
# otherwise do nothing
else:
logger.debug(
"DATA TOO OLD SKIPED : [%s] %s", data.published, data.title)
# update the date of the trigger
update_trigger(service)
else:
print "No trigger set by any user"
def handle(self, *args, **options):
"""
run the main process
"""
default_provider.load_services()
trigger = TriggerService.objects.filter(status=True).select_related(
'consumer__name', 'provider__name')
if trigger:
for service in trigger:
# provider - the service that offer data
service_name = str(service.provider.name.name)
service_provider = default_provider.get_service(service_name)
# consumer - the service which uses the data
service_name = str(service.consumer.name.name)
service_consumer = default_provider.get_service(service_name)
# First run
if service.date_triggered is None:
logger.debug("first run for %s => %s " % (str(
service.provider.name), str(service.consumer.name.name)))
asyncio.get_event_loop().run_until_complete(
self.my_dummy_provider())
# another run
else:
asyncio.get_event_loop().run_until_complete(
self.my_provider(service_provider,
service.provider.token,
service.id,
service.date_triggered))
asyncio.get_event_loop().run_until_complete(
self.my_consumer(service_consumer,
service.consumer.token,
service.id,
service.date_triggered))
asyncio.get_event_loop().run_until_complete(
self.update_trigger(service))
asyncio.get_event_loop().run_forever()
else:
print("No trigger set by any user")
def renew_service(request, pk):
"""
renew an existing service
"""
service = get_object_or_404(ServicesActivated, pk=pk)
service_name = str(service.name)
service_object = default_provider.get_service(service_name)
lets_auth = getattr(service_object, 'auth')
return redirect(lets_auth(request))
def handle(self, *args, **options):
"""
run the main process
"""
default_provider.load_services()
trigger = TriggerService.objects.filter(status=True).select_related(
'consummer__name', 'provider__name')
if trigger:
for service in trigger:
# provider - the service that offer datas
service_name = str(service.provider.name.name)
service_provider = default_provider.get_service(service_name)
# consumer - the service which uses the data
service_name = str(service.consumer.name.name)
service_consumer = default_provider.get_service(service_name)
# First run
if service.date_triggered is None:
logger.debug("first run for %s => %s " %
(str(service.provider.name),
str(service.consumer.name.name)))
asyncio.get_event_loop().run_until_complete(
self.my_dummy_provider())
# another run
else:
asyncio.get_event_loop().run_until_complete(
self.my_provider(service_provider,
service.provider.token, service.id,
service.date_triggered))
asyncio.get_event_loop().run_until_complete(
self.my_consumer(service_consumer,
service.consumer.token, service.id,
service.date_triggered))
asyncio.get_event_loop().run_until_complete(
self.update_trigger(service))
asyncio.get_event_loop().run_forever()
else:
print("No trigger set by any user")
def renew_service(request, pk):
"""
renew an existing service
:param pk: the primary key of the service to renew
:type pk: int
"""
service = get_object_or_404(ServicesActivated, pk=pk)
service_name = str(service.name)
service_object = default_provider.get_service(service_name)
lets_auth = getattr(service_object, 'auth')
return redirect(lets_auth(request))
def renew_service(request, pk):
"""
renew an existing service
:param pk: the primary key of the service to renew
:type pk: int
"""
service = get_object_or_404(ServicesActivated, pk=pk)
service_name = str(service.name)
service_object = default_provider.get_service(service_name)
lets_auth = getattr(service_object, 'auth')
return redirect(lets_auth(request))
def renew_service(request, pk):
"""
renew an existing service
"""
service = get_object_or_404(ServicesActivated, pk=pk)
service_name = str(service.name)
service_object = default_provider.get_service(service_name)
lets_auth = getattr(service_object, 'auth')
#lets_callback = getattr(service_object, 'auth')
# print lets_callback
return redirect(lets_auth(request))
def provider(self, service):
"""
get the data from (the cache of) the service provider
:param service:
:return: data
"""
service_provider = default_provider.get_service(str(service.provider.name.name))
# 1) get the data from the provider service
module_name = 'th_' + service.provider.name.name.split('Service')[1].lower()
kwargs = {'trigger_id': str(service.id), 'cache_stack': module_name}
return getattr(service_provider, 'process_data')(**kwargs)
def finalcallback(request, **kwargs):
"""
let's do the callback of the related service after
the auth request from UserServiceCreateView
"""
service_name = kwargs['service_name']
service_object = default_provider.get_service(service_name)
lets_callback = getattr(service_object, 'callback')
# call the auth func from this class
# and redirect to the external service page
# to auth the application django-th to access to the user
# account details
return render_to_response(lets_callback(request))
def finalcallback(request, **kwargs):
"""
let's do the callback of the related service after
the auth request from UserServiceCreateView
"""
service_name = kwargs['service_name']
service_object = default_provider.get_service(service_name)
lets_callback = getattr(service_object, 'callback')
# call the auth func from this class
# and redirect to the external service page
# to auth the application django-th to access to the user
# account details
return render_to_response(lets_callback(request))
def put_in_cache(service):
# flag to know if we have to update
to_update = False
# flag to get the status of a service
status = False
# counting the new data to store to display them in the log
# provider - the service that offer data
service_name = str(service.provider.name.name)
service_provider = default_provider.get_service(service_name)
service_name = str(service.consumer.name.name)
# check if the service has already been triggered
# if date_triggered is None, then it's the first run
if service.date_triggered is None:
logger.debug("first run for %s => %s " % (
str(service.provider.name), str(service.consumer.name.name)))
to_update = True
status = True
# run run run
else:
# 1) get the data from the provider service
# get a timestamp of the last triggered of the service
datas = getattr(service_provider, '__init__')(service.provider.token)
datas = getattr(service_provider, 'read_data')(
service.provider.token,
service.id,
service.date_triggered)
if len(datas) > 0:
to_update = True
status = True
# update the date of the trigger at the end of the loop
sentence = "user: {} - provider: {} - {}"
if to_update:
if status:
logger.info((sentence + " - {} data put in cache").format(
service.user,
service.provider.name.name,
service.description,
len(datas)))
else:
logger.info((sentence + " AN ERROR OCCURS ").format(
service.user,
service.provider.name.name,
service.description))
else:
logger.info((sentence + " nothing new").format(
service.user,
service.provider.name.name,
service.description))
def provider(self, service):
"""
get the data from (the cache of) the service provider
:param service:
:return: data
"""
service_provider = default_provider.get_service(
str(service.provider.name.name))
# 1) get the data from the provider service
module_name = 'th_' + \
service.provider.name.name.split('Service')[1].lower()
kwargs = {'trigger_id': str(service.id), 'cache_stack': module_name}
return getattr(service_provider, 'process_data')(**kwargs)
def test_reading(self):
from django_th.services import default_provider
service = self.create_triggerservice()
date_triggered = service.date_triggered if service.date_triggered else service.date_created
kwargs = {'token': service.provider.token,
'trigger_id': service.id,
'date_triggered': date_triggered}
with patch.object(ServiceProvider, 'get_service') as mock_it:
mock_it.return_value = default_provider.get_service(
str(service.provider.name.name))
with patch.object(Read, 'provider') as mock_read:
se = Read()
se.reading(service)
mock_read.assert_called_once_with(mock_it.return_value, **kwargs)
mock_it.assert_called_with(service.provider.name.name)
def form_valid(self, form):
self.object = form.save(user=self.request.user)
sa = ServicesActivated.objects.get(name=form.cleaned_data['name'].name)
# let's build the 'call' of the auth method which own to a Service Class
if sa.auth_required:
# use the default_provider to get the object from the ServiceXXX
service_object = default_provider.get_service(
str(form.cleaned_data['name'].name))
# get the class object
lets_auth = getattr(service_object, 'auth')
# call the auth func from this class
# and redirect to the external service page
# to auth the application django-th to access to the user
# account details
return redirect(lets_auth(self.request))
return HttpResponseRedirect(reverse('service_add_thanks'))
def form_valid(self, form):
self.object = form.save(user=self.request.user)
sa = ServicesActivated.objects.get(name=form.cleaned_data['name'].name)
# let's build the 'call' of the auth method which own to a Service Class
if sa.auth_required:
# use the default_provider to get the object from the ServiceXXX
service_object = default_provider.get_service(
str(form.cleaned_data['name'].name))
# get the class object
lets_auth = getattr(service_object, 'auth')
# call the auth func from this class
# and redirect to the external service page
# to auth the application django-th to access to the user
# account details
return redirect(lets_auth(self.request))
return HttpResponseRedirect(reverse('service_add_thanks'))
def test_reading(self):
from django_th.services import default_provider
now = arrow.utcnow().to(settings.TIME_ZONE).format(
'YYYY-MM-DD HH:mm:ssZZ')
service = self.create_triggerservice()
date_triggered = service.date_triggered if service.date_triggered \
else now
kwargs = {'token': service.provider.token,
'trigger_id': service.id,
'date_triggered': date_triggered}
with patch.object(ServiceProvider, 'get_service') as mock_it:
mock_it.return_value = default_provider.get_service(
str(service.provider.name.name))
with patch.object(Read, 'provider') as mock_read:
se = Read()
se.reading(service)
mock_read.assert_called_once_with(mock_it.return_value, **kwargs)
mock_it.assert_called_with(service.provider.name.name)
def save_data(trigger_id, data):
"""
call the consumer and handle the data
:param trigger_id:
:param data:
:return:
"""
status = True
# consumer - the service which uses the data
default_provider.load_services()
service = TriggerService.objects.get(id=trigger_id)
service_consumer = default_provider.get_service(str(service.consumer.name.name))
kwargs = {'user': service.user}
if len(data) > 0:
getattr(service_consumer, '__init__')(service.consumer.token, **kwargs)
status = getattr(service_consumer, 'save_data')(service.id, **data)
return status
def form_valid(self, form):
name = form.cleaned_data.get('name').name
user = self.request.user
form.save(user=user, service_name=self.kwargs.get('service_name'))
sa = ServicesActivated.objects.get(name=name)
# let's build the 'call' of the auth method
# which own to a Service Class
if sa.auth_required:
# use the default_provider to get the object from the ServiceXXX
default_provider.load_services()
service_object = default_provider.get_service(str(name))
# get the class object
lets_auth = getattr(service_object, 'auth')
# call the auth func from this class
# and redirect to the external service page
# to auth the application django-th to access to the user
# account details
return redirect(lets_auth(self.request))
messages.success(self.request, _('Service %s activated successfully') % name.split('Service')[1])
return HttpResponseRedirect(reverse('user_services'))
def save_data(trigger_id, data):
"""
call the consumer and handle the data
:param trigger_id:
:param data:
:return:
"""
status = True
# consumer - the service which uses the data
default_provider.load_services()
service = TriggerService.objects.get(id=trigger_id)
service_consumer = default_provider.get_service(
str(service.consumer.name.name))
kwargs = {'user': service.user}
if len(data) > 0:
getattr(service_consumer, '__init__')(service.consumer.token, **kwargs)
status = getattr(service_consumer, 'save_data')(service.id, **data)
return status
def reading(self, service):
"""
get the data from the service and put theme in cache
:param service: service object to read
:type service: object
"""
# counting the new data to store to display them in the log provider - the service that offer data
provider_token = service.provider.token
default_provider.load_services()
service_provider = default_provider.get_service(str(service.provider.name.name))
date_triggered = service.date_triggered if service.date_triggered else service.date_created
# get the data from the provider service
kwargs = {'token': provider_token, 'trigger_id': service.id, 'date_triggered': date_triggered}
data = self.provider(service_provider, **kwargs)
if len(data) > 0:
logger.info("{} - {} new data".format(service, len(data)))
elif data is False:
# if data is False, something went wrong
self.is_ceil_reached(service)
def reading(service):
"""
get the data from the service and put theme in cache
:param service: service object to read
:type service: object
"""
data = ()
# flag to know if we have to update
to_update = False
# flag to get the status of a service
status = False
count_new_data = 0
# counting the new data to store to display them in the log
# provider - the service that offer data
provider_token = service.provider.token
service_provider = default_provider.get_service(
str(service.provider.name.name))
# check if the service has already been triggered
# if date_triggered is None, then it's the first run
if service.date_triggered is None:
logger.debug("first time {}".format(service))
to_update = True
status = True
# run run run
else:
# 1) get the data from the provider service
# get a timestamp of the last triggered of the service
kw = {'token': provider_token,
'trigger_id': service.id,
'date_triggered': service.date_triggered}
getattr(service_provider, '__init__')(provider_token)
data = getattr(service_provider, 'read_data')(**kw)
# counting the new data to store to display them in the log
count_new_data = len(data) if data else 0
if count_new_data > 0:
to_update = True
status = True
log_update(service, to_update, status, count_new_data)
def test_reading(self):
from django_th.services import default_provider
now = arrow.utcnow().to(
settings.TIME_ZONE).format('YYYY-MM-DD HH:mm:ssZZ')
service = self.create_triggerservice()
date_triggered = service.date_triggered if service.date_triggered \
else now
kwargs = {
'token': service.provider.token,
'trigger_id': service.id,
'date_triggered': date_triggered
}
with patch.object(ServiceProvider, 'get_service') as mock_it:
mock_it.return_value = default_provider.get_service(
str(service.provider.name.name))
with patch.object(Read, 'provider') as mock_read:
se = Read()
se.reading(service)
mock_read.assert_called_once_with(mock_it.return_value, **kwargs)
mock_it.assert_called_with(service.provider.name.name)
def reading(service):
"""
get the data from the service and put theme in cache
:param service: service object to read
:type service: object
"""
# flag to know if we have to update
to_update = False
count_new_data = 0
# counting the new data to store to display them in the log
# provider - the service that offer data
provider_token = service.provider.token
service_provider = default_provider.get_service(
str(service.provider.name.name))
# check if the service has already been triggered
# if date_triggered is None, then it's the first run
if service.date_triggered is None:
logger.debug("first time {}".format(service))
to_update = True
# run run run
else:
# 1) get the data from the provider service
# get a timestamp of the last triggered of the service
kw = {'token': provider_token,
'trigger_id': service.id,
'date_triggered': service.date_triggered}
getattr(service_provider, '__init__')(provider_token)
data = getattr(service_provider, 'read_data')(**kw)
# counting the new data to store to display them in the log
count_new_data = len(data) if data else 0
if count_new_data > 0:
to_update = True
if to_update:
logger.info("{} - {} new data".format(service, count_new_data))
else:
logger.debug("data outdated skiped : [{}] ".format(published))
else:
# return the number of updates ( to be displayed in the log )
yield from q2.put(count_new_data)
default_provider.load_services()
trigger = TriggerService.objects.filter(status=True)
if trigger:
for service in trigger:
# provider - the service that offer datas
service_name = str(service.provider.name.name)
service_provider = default_provider.get_service(service_name)
# consumer - the service which uses the data
service_name = str(service.consumer.name.name)
service_consumer = default_provider.get_service(service_name)
# First run
if service.date_triggered is None:
logger.debug(
"first run for %s => %s " %
(str(service.provider.name), str(service.consumer.name.name)))
asyncio.get_event_loop().run_until_complete(my_dummy_provider)
# another run
else:
def go():
"""
run the main process
"""
trigger = TriggerService.objects.filter(status=True)
if trigger:
for service in trigger:
# flag to know if we have to update
to_update = False
# counting the new data to store to display them in the log
count_new_data = 0
# provider - the service that offer datas
service_name = str(service.provider.name.name)
service_provider = default_provider.get_service(service_name)
# consumer - the service which uses the data
service_name = str(service.consumer.name.name)
service_consumer = default_provider.get_service(service_name)
# check if the service has already been triggered
if service.date_triggered is None:
logger.debug("first run for %s => %s " % (str(
service.provider.name), str(service.consumer.name.name)))
to_update = True
# run run run
else:
# 1) get the datas from the provider service
# get a timestamp of the last triggered of the service
datas = getattr(service_provider, 'process_data')(
service.provider.token, service.id, service.date_triggered)
consumer = getattr(service_consumer, 'save_data')
published = ''
which_date = ''
# 2) for each one
for data in datas:
# if in a pool of data once of them does not have
# a date, will take the previous date for this one
# if it's the first one, set it to 00:00:00
# let's try to determine the date contained in the data...
published = to_datetime(data)
if published is not None:
# get the published date of the provider
published = arrow.get(
str(published), 'YYYY-MM-DD HH:mm:ss')
# store the date for the next loop
# if published became 'None'
which_date = published
#... otherwise set it to 00:00:00 of the current date
if which_date == '':
# current date
which_date = arrow.utcnow().replace(
hour=0, minute=0, second=0)
published = which_date
if published is None and which_date != '':
published = which_date
# 3) check if the previous trigger is older than the
# date of the data we retreived
# if yes , process the consumer
# add the TIME_ZONE settings
date_triggered = arrow.get(
str(service.date_triggered), 'YYYY-MM-DD HH:mm:ss').to(settings.TIME_ZONE)
# flag to know if we can push data to the consumer
proceed = False
# if the published date if greater or equal to the last
# triggered event ... :
if date_triggered is not None and published is not None and published.date() >= date_triggered.date():
# if date are the same ...
if published.date() == date_triggered.date():
# ... compare time and proceed if needed
if published.time() >= date_triggered.time():
proceed = True
# not same date so proceed !
else:
proceed = True
if proceed:
if 'title' in data:
logger.info("date {} >= date triggered {} title {}".format(
published, date_triggered, data['title']))
else:
logger.info(
"date {} >= date triggered {} ".format(published, date_triggered))
consumer(
service.consumer.token, service.id, **data)
to_update = True
count_new_data += 1
# otherwise do nothing
else:
if 'title' in data:
logger.debug(
"data outdated skiped : [{}] {}".format(published, data['title']))
else:
logger.debug(
"data outdated skiped : [{}] ".format(published))
# update the date of the trigger at the end of the loop
if to_update:
logger.info("user: {} - provider: {} - consumer: {} - {} = {} new data".format(
service.user, service.provider.name.name, service.consumer.name.name, service.description, count_new_data))
update_trigger(service)
else:
logger.info("user: {} - provider: {} - consumer: {} - {} nothing new".format(
service.user, service.provider.name.name, service.consumer.name.name, service.description))
else:
print("No trigger set by any user")
def publish_data():
"""
the purpose of this tasks is to get the data from the cache
then publish them
"""
now = arrow.utcnow().to(settings.TIME_ZONE)
trigger = TriggerService.objects.filter(status=True).select_related(
'consumer__name', 'provider__name')
if trigger:
for service in trigger:
# flag to know if we have to update
to_update = False
# flag to get the status of a service
status = False
# counting the new data to store to display them in the log
count_new_data = 0
# provider - the service that offer data
service_name = str(service.provider.name.name)
service_provider = default_provider.get_service(service_name)
# consumer - the service which uses the data
service_name = str(service.consumer.name.name)
service_consumer = default_provider.get_service(service_name)
# check if the service has already been triggered
# if date_triggered is None, then it's the first run
if service.date_triggered is None:
logger.debug("first run for %s => %s " % (str(
service.provider.name),
str(service.consumer.name.name)))
to_update = True
status = True
# run run run
else:
# 1) get the data from the provider service
# get a timestamp of the last triggered of the service
datas = getattr(service_provider, 'process_data')(service.id)
if datas is None or len(datas) == 0:
continue
consumer = getattr(service_consumer, '__init__')(
service.consumer.token)
consumer = getattr(service_consumer, 'save_data')
published = ''
which_date = ''
# 2) for each one
for data in datas:
# if in a pool of data once of them does not have
# a date, will take the previous date for this one
# if it's the first one, set it to 00:00:00
# let's try to determine the date contained in
# the data...
published = to_datetime(data)
if published is not None:
# get the published date of the provider
published = arrow.get(published).to(settings.TIME_ZONE)
# store the date for the next loop
# if published became 'None'
which_date = published
# ... otherwise set it to 00:00:00 of the current date
if which_date == '':
# current date
which_date = arrow.utcnow().replace(
hour=0, minute=0, second=0).to(
settings.TIME_ZONE)
published = which_date
if published is None and which_date != '':
published = which_date
# 3) check if the previous trigger is older than the
# date of the data we retrieved
# if yes , process the consumer
# add the TIME_ZONE settings
# to localize the current date
date_triggered = arrow.get(
str(service.date_triggered),
'YYYY-MM-DD HH:mm:ss').to(settings.TIME_ZONE)
# if the published date if greater or equal to the last
# triggered event ... :
if date_triggered is not None and \
published is not None and \
now >= published and \
published >= date_triggered:
if 'title' in data:
sentence = "date {} >= triggered {} title {}"
logger.info(sentence.format(published,
date_triggered,
data['title']))
else:
sentence = "date {} >= date triggered {}"
logger.info(sentence.format(published,
date_triggered))
status = consumer(
service.consumer.token, service.id, **data)
to_update = True
count_new_data += 1
# otherwise do nothing
else:
if 'title' in data:
sentence = "data outdated skipped : [{}] {}"
logger.debug(sentence.format(published,
data['title']))
else:
sentence = "data outdated skipped : [{}] "
logger.debug(sentence.format(published))
# update the date of the trigger at the end of the loop
sentence = "user: {} - provider: {} - consumer: {} - {}"
if to_update:
if status:
logger.info((sentence + " - {} new data").format(
service.user,
service.provider.name.name,
service.consumer.name.name,
service.description,
count_new_data))
update_trigger(service)
else:
logger.info((sentence + " AN ERROR OCCURS ").format(
service.user,
service.provider.name.name,
service.consumer.name.name,
service.description))
else:
logger.info((sentence + " nothing new").format(
service.user,
service.provider.name.name,
service.consumer.name.name,
service.description))
def publishing(service, now):
"""
the purpose of this tasks is to get the data from the cache
then publish them
:param service: service object where we will publish
:param now: it's the current date
:type service: obect
:type now: string date
"""
# flag to know if we have to update
to_update = False
# flag to get the status of a service
status = False
# counting the new data to store to display them in the log
count_new_data = 0
# provider - the service that offer data
service_provider = default_provider.get_service(
str(service.provider.name.name))
# consumer - the service which uses the data
service_consumer = default_provider.get_service(
str(service.consumer.name.name))
# check if the service has already been triggered
# if date_triggered is None, then it's the first run
if service.date_triggered is None:
logger.debug("first run {}".format(service))
to_update = True
status = True
# run run run
else:
# 1) get the data from the provider service
kw = {'trigger_id': service.id}
datas = getattr(service_provider, 'process_data')(**kw)
if datas is not None and len(datas) > 0:
consumer = getattr(service_consumer,
'__init__')(service.consumer.token)
consumer = getattr(service_consumer, 'save_data')
published = ''
which_date = ''
# 2) for each one
for data in datas:
if settings.DEBUG:
from django_th.tools import to_datetime
published = to_datetime(data)
published, which_date = get_published(
published, which_date)
date_triggered = arrow.get(str(service.date_triggered),
'YYYY-MM-DD HH:mm:ss').to(
settings.TIME_ZONE)
publish_log_data(published, date_triggered, data)
# the consummer will save the data and return if success or not
status = consumer(service.consumer.token, service.id, **data)
else:
count_new_data = len(datas)
to_update = True
# let's log
log_update(service, to_update, status, count_new_data)
# let's update
if to_update and status:
update_trigger(service)
logger.debug(
"data outdated skiped : [{}] ".format(published))
else:
# return the number of updates ( to be displayed in the log )
yield from q2.put(count_new_data)
default_provider.load_services()
trigger = TriggerService.objects.filter(status=True)
if trigger:
for service in trigger:
# provider - the service that offer datas
service_name = str(service.provider.name.name)
service_provider = default_provider.get_service(service_name)
# consumer - the service which uses the data
service_name = str(service.consumer.name.name)
service_consumer = default_provider.get_service(service_name)
# First run
if service.date_triggered is None:
logger.debug("first run for %s => %s " % (str(
service.provider.name), str(service.consumer.name.name)))
asyncio.get_event_loop().run_until_complete(my_dummy_provider)
# another run
else:
asyncio.get_event_loop().run_until_complete(
def publishing(service, now):
"""
the purpose of this tasks is to get the data from the cache
then publish them
:param service: service object where we will publish
:param now: it's the current date
:type service: obect
:type now: string date
"""
# flag to know if we have to update
to_update = False
# flag to get the status of a service
status = False
# counting the new data to store to display them in the log
count_new_data = 0
# provider - the service that offer data
service_provider = default_provider.get_service(
str(service.provider.name.name))
# consumer - the service which uses the data
service_consumer = default_provider.get_service(
str(service.consumer.name.name))
# check if the service has already been triggered
# if date_triggered is None, then it's the first run
if service.date_triggered is None:
logger.debug("first run {}".format(service))
to_update = True
status = True
# run run run
else:
# 1) get the data from the provider service
kw = {'trigger_id': service.id}
datas = getattr(service_provider, 'process_data')(**kw)
if datas is not None and len(datas) > 0:
consumer = getattr(service_consumer, '__init__')(
service.consumer.token)
consumer = getattr(service_consumer, 'save_data')
published = ''
which_date = ''
# 2) for each one
for data in datas:
if settings.DEBUG:
from django_th.tools import to_datetime
published = to_datetime(data)
published, which_date = get_published(published, which_date)
date_triggered = arrow.get(str(service.date_triggered), 'YYYY-MM-DD HH:mm:ss').to(settings.TIME_ZONE)
publish_log_data(published, date_triggered, data)
# the consummer will save the data and return if success or not
status = consumer(service.consumer.token, service.id, **data)
else:
count_new_data = len(datas)
to_update = True
# let's log
log_update(service, to_update, status, count_new_data)
# let's update
if to_update and status:
update_trigger(service)