def publishing(service): """ the purpose of this tasks is to get the data from the cache then publish them :param service: service object where we will publish :type service: object """ # flag to know if we have to update to_update = False # flag to get the status of a service status = False # provider - the service that offer data # check if the service has already been triggered # if date_triggered is None, then it's the first run if service.date_triggered is None: logger.debug("first run {}".format(service)) to_update = True status = True # run run run else: service_provider = default_provider.get_service( str(service.provider.name.name)) # 1) get the data from the provider service kw = {'trigger_id': service.id} data = getattr(service_provider, 'process_data')(**kw) # counting the new data to store to display them in the log count_new_data = len(data) if data else 0 if count_new_data > 0: # consumer - the service which uses the data service_consumer = default_provider.get_service( str(service.consumer.name.name)) getattr(service_consumer, '__init__')(service.consumer.token) consumer = getattr(service_consumer, 'save_data') # 2) for each one for d in data: # the consumer will save the data and return if success or not status = consumer(service.id, **d) to_update = True # let's log log_update(service, to_update, status, count_new_data) # let's update if to_update and status: update_trigger(service)
def reading(self, service): """ get the data from the service and put theme in cache :param service: service object to read :type service: object """ now = arrow.utcnow().to(settings.TIME_ZONE).format( 'YYYY-MM-DD HH:mm:ssZZ') # flag to know if we have to update to_update = False # counting the new data to store to display them in the log # provider - the service that offer data provider_token = service.provider.token default_provider.load_services() service_provider = default_provider.get_service( str(service.provider.name.name)) # check if the service has already been triggered # if date_triggered is None, then it's the first run # so it will be set to "now" date_triggered = service.date_triggered if service.date_triggered \ else now # 1) get the data from the provider service # get a timestamp of the last triggered of the service kwargs = {'token': provider_token, 'trigger_id': service.id, 'date_triggered': date_triggered} data = self.provider(service_provider, **kwargs) # counting the new data to store to display them in the log count_new_data = len(data) if data else 0 if count_new_data > 0: to_update = True if to_update: logger.info("{} - {} new data".format(service, count_new_data))
def reading(service): """ get the data from the service and put theme in cache :param service: service object to read """ # flag to know if we have to update to_update = False # flag to get the status of a service status = False # counting the new data to store to display them in the log # provider - the service that offer data service_provider = default_provider.get_service( str(service.provider.name.name)) # check if the service has already been triggered # if date_triggered is None, then it's the first run if service.date_triggered is None: logger.debug("first time {}".format(service)) to_update = True status = True # run run run else: # 1) get the data from the provider service # get a timestamp of the last triggered of the service kw = {'token': service.provider.token, 'trigger_id': service.id, 'date_triggered': service.date_triggered} datas = getattr(service_provider, '__init__')(service.provider.token) datas = getattr(service_provider, 'read_data')(**kw) if len(datas) > 0: to_update = True status = True log_update(service, to_update, status, len(datas))
def consumer(self, service, data, to_update, status): """ call the consumer and handle the data :param service: :param data: :param to_update: :param status: :return: status """ # consumer - the service which uses the data service_consumer = default_provider.get_service( str(service.consumer.name.name)) kwargs = {'user': service.user} getattr(service_consumer, '__init__')(service.consumer.token, **kwargs) instance = getattr(service_consumer, 'save_data') # 2) for each one for d in data: d['userservice_id'] = service.consumer.id # the consumer will save the data and return if success or not status = instance(service.id, **d) to_update = True return to_update, status
def form_valid(self, form): name = form.cleaned_data.get('name').name user = self.request.user if UserService.objects.filter(name=name, user=user).exists(): from django.contrib import messages messages.warning( self.request, 'Service %s already activated' % name.split('Service')[1]) return HttpResponseRedirect(reverse('user_services')) else: form.save(user=user) sa = ServicesActivated.objects.get( name=form.cleaned_data.get('name').name) # let's build the 'call' of the auth method # which own to a Service Class if sa.auth_required: # use the default_provider to get the object from the ServiceXXX default_provider.load_services() service_object = default_provider.get_service( str(form.cleaned_data.get('name').name)) # get the class object lets_auth = getattr(service_object, 'auth') # call the auth func from this class # and redirect to the external service page # to auth the application django-th to access to the user # account details return redirect(lets_auth(self.request)) return HttpResponseRedirect( reverse('user_services', args=['added']))
def reading(self, service): """ get the data from the service and put theme in cache :param service: service object to read :type service: object """ # counting the new data to store to display them in the log provider - the service that offer data provider_token = service.provider.token default_provider.load_services() service_provider = default_provider.get_service( str(service.provider.name.name)) date_triggered = service.date_triggered if service.date_triggered else service.date_created # get the data from the provider service kwargs = { 'token': provider_token, 'trigger_id': service.id, 'date_triggered': date_triggered } data = self.provider(service_provider, **kwargs) if len(data) > 0: logger.info("{} - {} new data".format(service, len(data))) elif data is False: # if data is False, something went wrong self.is_ceil_reached(service)
def consumer(self, service, data, to_update, status): """ call the consumer and handle the data :param service: :param data: :param to_update: :param status: :return: status """ # consumer - the service which uses the data service_consumer = default_provider.get_service( str(service.consumer.name.name)) kwargs = {'user': service.user} getattr(service_consumer, '__init__')(service.consumer.token, **kwargs) instance = getattr(service_consumer, 'save_data') # 2) for each one for d in data: d['userservice_id'] = service.consumer.id # the consumer will save the data and return if success or not status = instance(service.id, **d) to_update = True return to_update, status
def reading(self, service): """ get the data from the service and put theme in cache :param service: service object to read :type service: object """ now = arrow.utcnow().to( settings.TIME_ZONE).format('YYYY-MM-DD HH:mm:ssZZ') # counting the new data to store to display them in the log # provider - the service that offer data provider_token = service.provider.token default_provider.load_services() service_provider = default_provider.get_service( str(service.provider.name.name)) # check if the service has already been triggered # if date_triggered is None, then it's the first run # so it will be set to "now" date_triggered = service.date_triggered if service.date_triggered \ else now # 1) get the data from the provider service # get a timestamp of the last triggered of the service kwargs = { 'token': provider_token, 'trigger_id': service.id, 'date_triggered': date_triggered } data = self.provider(service_provider, **kwargs) if len(data) > 0: logger.info("{} - {} new data".format(service, len(data))) elif data is False: # if data is False, something went wrong self.is_ceil_reached(service)
def form_valid(self, form): name = form.cleaned_data.get('name').name user = self.request.user if UserService.objects.filter(name=name, user=user).exists(): from django.contrib import messages messages.warning(self.request, 'Service %s already activated' % name.split('Service')[1]) return HttpResponseRedirect(reverse('user_services')) else: form.save(user=user) sa = ServicesActivated.objects.get( name=form.cleaned_data.get('name').name) # let's build the 'call' of the auth method # which own to a Service Class if sa.auth_required: # use the default_provider to get the object from the ServiceXXX default_provider.load_services() service_object = default_provider.get_service( str(form.cleaned_data.get('name').name)) # get the class object lets_auth = getattr(service_object, 'auth') # call the auth func from this class # and redirect to the external service page # to auth the application django-th to access to the user # account details return redirect(lets_auth(self.request)) return HttpResponseRedirect(reverse('user_services', args=['added']))
def form_valid(self, form): """ save the data :param form: :return: """ valid = True # 'name' is injected in the clean() of the form line 56 name = form.cleaned_data.get('name').name user = self.request.user form.save(user=user, service_name=name) sa = ServicesActivated.objects.get(name=name) if sa.auth_required and sa.self_hosted: # trigger the checking of the service from django_th.services import default_provider default_provider.load_services() service_provider = default_provider.get_service(name) result = service_provider.check(self.request, user) if result is not True: # the call of the API failed due to an error which is in the result string # return by the call of the API form.add_error('host', result) messages.error(self.request, result) return redirect('edit_service', pk=self.kwargs.get(self.pk_url_kwarg)) if valid: messages.success(self.request, _('Service %s modified successfully') % name.split('Service')[1]) return HttpResponseRedirect(reverse('user_services'))
def go(): """ run the main process """ trigger = TriggerService.objects.all() if trigger: for service in trigger: logger.info("from %s to %s ", service.provider.name, service.consummer.name) # provider - the service that offer datas service_name = service.provider.name service_provider = default_provider.get_service( 'Service' + str(service_name).capitalize()) # consummer - the service which uses the datas service_name = service.consummer.name service_consummer = default_provider.get_service( 'Service' + str(service_name).capitalize()) # 1) get the datas from the provider service datas = getattr(service_provider, 'process_data')(service.id) consummer = getattr(service_consummer, 'save_data') # 2) for each one for data in datas: title = data.title content = data.content[0].value logger.info("from the service %s", service.provider) # 3) check if the previous trigger is older than the # date of the data we retreived # if yes , process the consummer if service.date_triggered is None or to_datetime( data.published) >= service.date_triggered: logger.debug("date %s title %s", data.published, data.title) logger.info("to the service %s", service.consummer) consummer(service.consummer.token, title, content, service.id) # otherwise do nothing else: logger.debug("DATA TOO OLD SKIPED : [%s] %s", data.published, data.title) # update the date of the trigger update_trigger(service) else: print "No trigger set by any user"
def go(): """ run the main process """ trigger = TriggerService.objects.all() if trigger: for service in trigger: logger.info( "from %s to %s ", service.provider.name, service.consummer.name) # provider - the service that offer datas service_name = service.provider.name service_provider = default_provider.get_service( 'Service' + str(service_name).capitalize()) # consummer - the service which uses the datas service_name = service.consummer.name service_consummer = default_provider.get_service( 'Service' + str(service_name).capitalize()) # 1) get the datas from the provider service datas = getattr(service_provider, 'process_data')(service.id) consummer = getattr(service_consummer, 'save_data') # 2) for each one for data in datas: title = data.title content = data.content[0].value logger.info("from the service %s", service.provider) # 3) check if the previous trigger is older than the # date of the data we retreived # if yes , process the consummer if service.date_triggered is None or to_datetime(data.published) >= service.date_triggered: logger.debug( "date %s title %s", data.published, data.title) logger.info("to the service %s", service.consummer) consummer( service.consummer.token, title, content, service.id) # otherwise do nothing else: logger.debug( "DATA TOO OLD SKIPED : [%s] %s", data.published, data.title) # update the date of the trigger update_trigger(service) else: print "No trigger set by any user"
def handle(self, *args, **options): """ run the main process """ default_provider.load_services() trigger = TriggerService.objects.filter(status=True).select_related( 'consumer__name', 'provider__name') if trigger: for service in trigger: # provider - the service that offer data service_name = str(service.provider.name.name) service_provider = default_provider.get_service(service_name) # consumer - the service which uses the data service_name = str(service.consumer.name.name) service_consumer = default_provider.get_service(service_name) # First run if service.date_triggered is None: logger.debug("first run for %s => %s " % (str( service.provider.name), str(service.consumer.name.name))) asyncio.get_event_loop().run_until_complete( self.my_dummy_provider()) # another run else: asyncio.get_event_loop().run_until_complete( self.my_provider(service_provider, service.provider.token, service.id, service.date_triggered)) asyncio.get_event_loop().run_until_complete( self.my_consumer(service_consumer, service.consumer.token, service.id, service.date_triggered)) asyncio.get_event_loop().run_until_complete( self.update_trigger(service)) asyncio.get_event_loop().run_forever() else: print("No trigger set by any user")
def renew_service(request, pk): """ renew an existing service """ service = get_object_or_404(ServicesActivated, pk=pk) service_name = str(service.name) service_object = default_provider.get_service(service_name) lets_auth = getattr(service_object, 'auth') return redirect(lets_auth(request))
def handle(self, *args, **options): """ run the main process """ default_provider.load_services() trigger = TriggerService.objects.filter(status=True).select_related( 'consummer__name', 'provider__name') if trigger: for service in trigger: # provider - the service that offer datas service_name = str(service.provider.name.name) service_provider = default_provider.get_service(service_name) # consumer - the service which uses the data service_name = str(service.consumer.name.name) service_consumer = default_provider.get_service(service_name) # First run if service.date_triggered is None: logger.debug("first run for %s => %s " % (str(service.provider.name), str(service.consumer.name.name))) asyncio.get_event_loop().run_until_complete( self.my_dummy_provider()) # another run else: asyncio.get_event_loop().run_until_complete( self.my_provider(service_provider, service.provider.token, service.id, service.date_triggered)) asyncio.get_event_loop().run_until_complete( self.my_consumer(service_consumer, service.consumer.token, service.id, service.date_triggered)) asyncio.get_event_loop().run_until_complete( self.update_trigger(service)) asyncio.get_event_loop().run_forever() else: print("No trigger set by any user")
def renew_service(request, pk): """ renew an existing service :param pk: the primary key of the service to renew :type pk: int """ service = get_object_or_404(ServicesActivated, pk=pk) service_name = str(service.name) service_object = default_provider.get_service(service_name) lets_auth = getattr(service_object, 'auth') return redirect(lets_auth(request))
def renew_service(request, pk): """ renew an existing service :param pk: the primary key of the service to renew :type pk: int """ service = get_object_or_404(ServicesActivated, pk=pk) service_name = str(service.name) service_object = default_provider.get_service(service_name) lets_auth = getattr(service_object, 'auth') return redirect(lets_auth(request))
def renew_service(request, pk): """ renew an existing service """ service = get_object_or_404(ServicesActivated, pk=pk) service_name = str(service.name) service_object = default_provider.get_service(service_name) lets_auth = getattr(service_object, 'auth') #lets_callback = getattr(service_object, 'auth') # print lets_callback return redirect(lets_auth(request))
def provider(self, service): """ get the data from (the cache of) the service provider :param service: :return: data """ service_provider = default_provider.get_service(str(service.provider.name.name)) # 1) get the data from the provider service module_name = 'th_' + service.provider.name.name.split('Service')[1].lower() kwargs = {'trigger_id': str(service.id), 'cache_stack': module_name} return getattr(service_provider, 'process_data')(**kwargs)
def finalcallback(request, **kwargs): """ let's do the callback of the related service after the auth request from UserServiceCreateView """ service_name = kwargs['service_name'] service_object = default_provider.get_service(service_name) lets_callback = getattr(service_object, 'callback') # call the auth func from this class # and redirect to the external service page # to auth the application django-th to access to the user # account details return render_to_response(lets_callback(request))
def finalcallback(request, **kwargs): """ let's do the callback of the related service after the auth request from UserServiceCreateView """ service_name = kwargs['service_name'] service_object = default_provider.get_service(service_name) lets_callback = getattr(service_object, 'callback') # call the auth func from this class # and redirect to the external service page # to auth the application django-th to access to the user # account details return render_to_response(lets_callback(request))
def put_in_cache(service): # flag to know if we have to update to_update = False # flag to get the status of a service status = False # counting the new data to store to display them in the log # provider - the service that offer data service_name = str(service.provider.name.name) service_provider = default_provider.get_service(service_name) service_name = str(service.consumer.name.name) # check if the service has already been triggered # if date_triggered is None, then it's the first run if service.date_triggered is None: logger.debug("first run for %s => %s " % ( str(service.provider.name), str(service.consumer.name.name))) to_update = True status = True # run run run else: # 1) get the data from the provider service # get a timestamp of the last triggered of the service datas = getattr(service_provider, '__init__')(service.provider.token) datas = getattr(service_provider, 'read_data')( service.provider.token, service.id, service.date_triggered) if len(datas) > 0: to_update = True status = True # update the date of the trigger at the end of the loop sentence = "user: {} - provider: {} - {}" if to_update: if status: logger.info((sentence + " - {} data put in cache").format( service.user, service.provider.name.name, service.description, len(datas))) else: logger.info((sentence + " AN ERROR OCCURS ").format( service.user, service.provider.name.name, service.description)) else: logger.info((sentence + " nothing new").format( service.user, service.provider.name.name, service.description))
def provider(self, service): """ get the data from (the cache of) the service provider :param service: :return: data """ service_provider = default_provider.get_service( str(service.provider.name.name)) # 1) get the data from the provider service module_name = 'th_' + \ service.provider.name.name.split('Service')[1].lower() kwargs = {'trigger_id': str(service.id), 'cache_stack': module_name} return getattr(service_provider, 'process_data')(**kwargs)
def test_reading(self): from django_th.services import default_provider service = self.create_triggerservice() date_triggered = service.date_triggered if service.date_triggered else service.date_created kwargs = {'token': service.provider.token, 'trigger_id': service.id, 'date_triggered': date_triggered} with patch.object(ServiceProvider, 'get_service') as mock_it: mock_it.return_value = default_provider.get_service( str(service.provider.name.name)) with patch.object(Read, 'provider') as mock_read: se = Read() se.reading(service) mock_read.assert_called_once_with(mock_it.return_value, **kwargs) mock_it.assert_called_with(service.provider.name.name)
def form_valid(self, form): self.object = form.save(user=self.request.user) sa = ServicesActivated.objects.get(name=form.cleaned_data['name'].name) # let's build the 'call' of the auth method which own to a Service Class if sa.auth_required: # use the default_provider to get the object from the ServiceXXX service_object = default_provider.get_service( str(form.cleaned_data['name'].name)) # get the class object lets_auth = getattr(service_object, 'auth') # call the auth func from this class # and redirect to the external service page # to auth the application django-th to access to the user # account details return redirect(lets_auth(self.request)) return HttpResponseRedirect(reverse('service_add_thanks'))
def form_valid(self, form): self.object = form.save(user=self.request.user) sa = ServicesActivated.objects.get(name=form.cleaned_data['name'].name) # let's build the 'call' of the auth method which own to a Service Class if sa.auth_required: # use the default_provider to get the object from the ServiceXXX service_object = default_provider.get_service( str(form.cleaned_data['name'].name)) # get the class object lets_auth = getattr(service_object, 'auth') # call the auth func from this class # and redirect to the external service page # to auth the application django-th to access to the user # account details return redirect(lets_auth(self.request)) return HttpResponseRedirect(reverse('service_add_thanks'))
def test_reading(self): from django_th.services import default_provider now = arrow.utcnow().to(settings.TIME_ZONE).format( 'YYYY-MM-DD HH:mm:ssZZ') service = self.create_triggerservice() date_triggered = service.date_triggered if service.date_triggered \ else now kwargs = {'token': service.provider.token, 'trigger_id': service.id, 'date_triggered': date_triggered} with patch.object(ServiceProvider, 'get_service') as mock_it: mock_it.return_value = default_provider.get_service( str(service.provider.name.name)) with patch.object(Read, 'provider') as mock_read: se = Read() se.reading(service) mock_read.assert_called_once_with(mock_it.return_value, **kwargs) mock_it.assert_called_with(service.provider.name.name)
def save_data(trigger_id, data): """ call the consumer and handle the data :param trigger_id: :param data: :return: """ status = True # consumer - the service which uses the data default_provider.load_services() service = TriggerService.objects.get(id=trigger_id) service_consumer = default_provider.get_service(str(service.consumer.name.name)) kwargs = {'user': service.user} if len(data) > 0: getattr(service_consumer, '__init__')(service.consumer.token, **kwargs) status = getattr(service_consumer, 'save_data')(service.id, **data) return status
def form_valid(self, form): name = form.cleaned_data.get('name').name user = self.request.user form.save(user=user, service_name=self.kwargs.get('service_name')) sa = ServicesActivated.objects.get(name=name) # let's build the 'call' of the auth method # which own to a Service Class if sa.auth_required: # use the default_provider to get the object from the ServiceXXX default_provider.load_services() service_object = default_provider.get_service(str(name)) # get the class object lets_auth = getattr(service_object, 'auth') # call the auth func from this class # and redirect to the external service page # to auth the application django-th to access to the user # account details return redirect(lets_auth(self.request)) messages.success(self.request, _('Service %s activated successfully') % name.split('Service')[1]) return HttpResponseRedirect(reverse('user_services'))
def save_data(trigger_id, data): """ call the consumer and handle the data :param trigger_id: :param data: :return: """ status = True # consumer - the service which uses the data default_provider.load_services() service = TriggerService.objects.get(id=trigger_id) service_consumer = default_provider.get_service( str(service.consumer.name.name)) kwargs = {'user': service.user} if len(data) > 0: getattr(service_consumer, '__init__')(service.consumer.token, **kwargs) status = getattr(service_consumer, 'save_data')(service.id, **data) return status
def reading(self, service): """ get the data from the service and put theme in cache :param service: service object to read :type service: object """ # counting the new data to store to display them in the log provider - the service that offer data provider_token = service.provider.token default_provider.load_services() service_provider = default_provider.get_service(str(service.provider.name.name)) date_triggered = service.date_triggered if service.date_triggered else service.date_created # get the data from the provider service kwargs = {'token': provider_token, 'trigger_id': service.id, 'date_triggered': date_triggered} data = self.provider(service_provider, **kwargs) if len(data) > 0: logger.info("{} - {} new data".format(service, len(data))) elif data is False: # if data is False, something went wrong self.is_ceil_reached(service)
def reading(service): """ get the data from the service and put theme in cache :param service: service object to read :type service: object """ data = () # flag to know if we have to update to_update = False # flag to get the status of a service status = False count_new_data = 0 # counting the new data to store to display them in the log # provider - the service that offer data provider_token = service.provider.token service_provider = default_provider.get_service( str(service.provider.name.name)) # check if the service has already been triggered # if date_triggered is None, then it's the first run if service.date_triggered is None: logger.debug("first time {}".format(service)) to_update = True status = True # run run run else: # 1) get the data from the provider service # get a timestamp of the last triggered of the service kw = {'token': provider_token, 'trigger_id': service.id, 'date_triggered': service.date_triggered} getattr(service_provider, '__init__')(provider_token) data = getattr(service_provider, 'read_data')(**kw) # counting the new data to store to display them in the log count_new_data = len(data) if data else 0 if count_new_data > 0: to_update = True status = True log_update(service, to_update, status, count_new_data)
def test_reading(self): from django_th.services import default_provider now = arrow.utcnow().to( settings.TIME_ZONE).format('YYYY-MM-DD HH:mm:ssZZ') service = self.create_triggerservice() date_triggered = service.date_triggered if service.date_triggered \ else now kwargs = { 'token': service.provider.token, 'trigger_id': service.id, 'date_triggered': date_triggered } with patch.object(ServiceProvider, 'get_service') as mock_it: mock_it.return_value = default_provider.get_service( str(service.provider.name.name)) with patch.object(Read, 'provider') as mock_read: se = Read() se.reading(service) mock_read.assert_called_once_with(mock_it.return_value, **kwargs) mock_it.assert_called_with(service.provider.name.name)
def reading(service): """ get the data from the service and put theme in cache :param service: service object to read :type service: object """ # flag to know if we have to update to_update = False count_new_data = 0 # counting the new data to store to display them in the log # provider - the service that offer data provider_token = service.provider.token service_provider = default_provider.get_service( str(service.provider.name.name)) # check if the service has already been triggered # if date_triggered is None, then it's the first run if service.date_triggered is None: logger.debug("first time {}".format(service)) to_update = True # run run run else: # 1) get the data from the provider service # get a timestamp of the last triggered of the service kw = {'token': provider_token, 'trigger_id': service.id, 'date_triggered': service.date_triggered} getattr(service_provider, '__init__')(provider_token) data = getattr(service_provider, 'read_data')(**kw) # counting the new data to store to display them in the log count_new_data = len(data) if data else 0 if count_new_data > 0: to_update = True if to_update: logger.info("{} - {} new data".format(service, count_new_data))
else: logger.debug("data outdated skiped : [{}] ".format(published)) else: # return the number of updates ( to be displayed in the log ) yield from q2.put(count_new_data) default_provider.load_services() trigger = TriggerService.objects.filter(status=True) if trigger: for service in trigger: # provider - the service that offer datas service_name = str(service.provider.name.name) service_provider = default_provider.get_service(service_name) # consumer - the service which uses the data service_name = str(service.consumer.name.name) service_consumer = default_provider.get_service(service_name) # First run if service.date_triggered is None: logger.debug( "first run for %s => %s " % (str(service.provider.name), str(service.consumer.name.name))) asyncio.get_event_loop().run_until_complete(my_dummy_provider) # another run else:
def go(): """ run the main process """ trigger = TriggerService.objects.filter(status=True) if trigger: for service in trigger: # flag to know if we have to update to_update = False # counting the new data to store to display them in the log count_new_data = 0 # provider - the service that offer datas service_name = str(service.provider.name.name) service_provider = default_provider.get_service(service_name) # consumer - the service which uses the data service_name = str(service.consumer.name.name) service_consumer = default_provider.get_service(service_name) # check if the service has already been triggered if service.date_triggered is None: logger.debug("first run for %s => %s " % (str( service.provider.name), str(service.consumer.name.name))) to_update = True # run run run else: # 1) get the datas from the provider service # get a timestamp of the last triggered of the service datas = getattr(service_provider, 'process_data')( service.provider.token, service.id, service.date_triggered) consumer = getattr(service_consumer, 'save_data') published = '' which_date = '' # 2) for each one for data in datas: # if in a pool of data once of them does not have # a date, will take the previous date for this one # if it's the first one, set it to 00:00:00 # let's try to determine the date contained in the data... published = to_datetime(data) if published is not None: # get the published date of the provider published = arrow.get( str(published), 'YYYY-MM-DD HH:mm:ss') # store the date for the next loop # if published became 'None' which_date = published #... otherwise set it to 00:00:00 of the current date if which_date == '': # current date which_date = arrow.utcnow().replace( hour=0, minute=0, second=0) published = which_date if published is None and which_date != '': published = which_date # 3) check if the previous trigger is older than the # date of the data we retreived # if yes , process the consumer # add the TIME_ZONE settings date_triggered = arrow.get( str(service.date_triggered), 'YYYY-MM-DD HH:mm:ss').to(settings.TIME_ZONE) # flag to know if we can push data to the consumer proceed = False # if the published date if greater or equal to the last # triggered event ... : if date_triggered is not None and published is not None and published.date() >= date_triggered.date(): # if date are the same ... if published.date() == date_triggered.date(): # ... compare time and proceed if needed if published.time() >= date_triggered.time(): proceed = True # not same date so proceed ! else: proceed = True if proceed: if 'title' in data: logger.info("date {} >= date triggered {} title {}".format( published, date_triggered, data['title'])) else: logger.info( "date {} >= date triggered {} ".format(published, date_triggered)) consumer( service.consumer.token, service.id, **data) to_update = True count_new_data += 1 # otherwise do nothing else: if 'title' in data: logger.debug( "data outdated skiped : [{}] {}".format(published, data['title'])) else: logger.debug( "data outdated skiped : [{}] ".format(published)) # update the date of the trigger at the end of the loop if to_update: logger.info("user: {} - provider: {} - consumer: {} - {} = {} new data".format( service.user, service.provider.name.name, service.consumer.name.name, service.description, count_new_data)) update_trigger(service) else: logger.info("user: {} - provider: {} - consumer: {} - {} nothing new".format( service.user, service.provider.name.name, service.consumer.name.name, service.description)) else: print("No trigger set by any user")
def publish_data(): """ the purpose of this tasks is to get the data from the cache then publish them """ now = arrow.utcnow().to(settings.TIME_ZONE) trigger = TriggerService.objects.filter(status=True).select_related( 'consumer__name', 'provider__name') if trigger: for service in trigger: # flag to know if we have to update to_update = False # flag to get the status of a service status = False # counting the new data to store to display them in the log count_new_data = 0 # provider - the service that offer data service_name = str(service.provider.name.name) service_provider = default_provider.get_service(service_name) # consumer - the service which uses the data service_name = str(service.consumer.name.name) service_consumer = default_provider.get_service(service_name) # check if the service has already been triggered # if date_triggered is None, then it's the first run if service.date_triggered is None: logger.debug("first run for %s => %s " % (str( service.provider.name), str(service.consumer.name.name))) to_update = True status = True # run run run else: # 1) get the data from the provider service # get a timestamp of the last triggered of the service datas = getattr(service_provider, 'process_data')(service.id) if datas is None or len(datas) == 0: continue consumer = getattr(service_consumer, '__init__')( service.consumer.token) consumer = getattr(service_consumer, 'save_data') published = '' which_date = '' # 2) for each one for data in datas: # if in a pool of data once of them does not have # a date, will take the previous date for this one # if it's the first one, set it to 00:00:00 # let's try to determine the date contained in # the data... published = to_datetime(data) if published is not None: # get the published date of the provider published = arrow.get(published).to(settings.TIME_ZONE) # store the date for the next loop # if published became 'None' which_date = published # ... otherwise set it to 00:00:00 of the current date if which_date == '': # current date which_date = arrow.utcnow().replace( hour=0, minute=0, second=0).to( settings.TIME_ZONE) published = which_date if published is None and which_date != '': published = which_date # 3) check if the previous trigger is older than the # date of the data we retrieved # if yes , process the consumer # add the TIME_ZONE settings # to localize the current date date_triggered = arrow.get( str(service.date_triggered), 'YYYY-MM-DD HH:mm:ss').to(settings.TIME_ZONE) # if the published date if greater or equal to the last # triggered event ... : if date_triggered is not None and \ published is not None and \ now >= published and \ published >= date_triggered: if 'title' in data: sentence = "date {} >= triggered {} title {}" logger.info(sentence.format(published, date_triggered, data['title'])) else: sentence = "date {} >= date triggered {}" logger.info(sentence.format(published, date_triggered)) status = consumer( service.consumer.token, service.id, **data) to_update = True count_new_data += 1 # otherwise do nothing else: if 'title' in data: sentence = "data outdated skipped : [{}] {}" logger.debug(sentence.format(published, data['title'])) else: sentence = "data outdated skipped : [{}] " logger.debug(sentence.format(published)) # update the date of the trigger at the end of the loop sentence = "user: {} - provider: {} - consumer: {} - {}" if to_update: if status: logger.info((sentence + " - {} new data").format( service.user, service.provider.name.name, service.consumer.name.name, service.description, count_new_data)) update_trigger(service) else: logger.info((sentence + " AN ERROR OCCURS ").format( service.user, service.provider.name.name, service.consumer.name.name, service.description)) else: logger.info((sentence + " nothing new").format( service.user, service.provider.name.name, service.consumer.name.name, service.description))
def publishing(service, now): """ the purpose of this tasks is to get the data from the cache then publish them :param service: service object where we will publish :param now: it's the current date :type service: obect :type now: string date """ # flag to know if we have to update to_update = False # flag to get the status of a service status = False # counting the new data to store to display them in the log count_new_data = 0 # provider - the service that offer data service_provider = default_provider.get_service( str(service.provider.name.name)) # consumer - the service which uses the data service_consumer = default_provider.get_service( str(service.consumer.name.name)) # check if the service has already been triggered # if date_triggered is None, then it's the first run if service.date_triggered is None: logger.debug("first run {}".format(service)) to_update = True status = True # run run run else: # 1) get the data from the provider service kw = {'trigger_id': service.id} datas = getattr(service_provider, 'process_data')(**kw) if datas is not None and len(datas) > 0: consumer = getattr(service_consumer, '__init__')(service.consumer.token) consumer = getattr(service_consumer, 'save_data') published = '' which_date = '' # 2) for each one for data in datas: if settings.DEBUG: from django_th.tools import to_datetime published = to_datetime(data) published, which_date = get_published( published, which_date) date_triggered = arrow.get(str(service.date_triggered), 'YYYY-MM-DD HH:mm:ss').to( settings.TIME_ZONE) publish_log_data(published, date_triggered, data) # the consummer will save the data and return if success or not status = consumer(service.consumer.token, service.id, **data) else: count_new_data = len(datas) to_update = True # let's log log_update(service, to_update, status, count_new_data) # let's update if to_update and status: update_trigger(service)
logger.debug( "data outdated skiped : [{}] ".format(published)) else: # return the number of updates ( to be displayed in the log ) yield from q2.put(count_new_data) default_provider.load_services() trigger = TriggerService.objects.filter(status=True) if trigger: for service in trigger: # provider - the service that offer datas service_name = str(service.provider.name.name) service_provider = default_provider.get_service(service_name) # consumer - the service which uses the data service_name = str(service.consumer.name.name) service_consumer = default_provider.get_service(service_name) # First run if service.date_triggered is None: logger.debug("first run for %s => %s " % (str( service.provider.name), str(service.consumer.name.name))) asyncio.get_event_loop().run_until_complete(my_dummy_provider) # another run else: asyncio.get_event_loop().run_until_complete(
def publishing(service, now): """ the purpose of this tasks is to get the data from the cache then publish them :param service: service object where we will publish :param now: it's the current date :type service: obect :type now: string date """ # flag to know if we have to update to_update = False # flag to get the status of a service status = False # counting the new data to store to display them in the log count_new_data = 0 # provider - the service that offer data service_provider = default_provider.get_service( str(service.provider.name.name)) # consumer - the service which uses the data service_consumer = default_provider.get_service( str(service.consumer.name.name)) # check if the service has already been triggered # if date_triggered is None, then it's the first run if service.date_triggered is None: logger.debug("first run {}".format(service)) to_update = True status = True # run run run else: # 1) get the data from the provider service kw = {'trigger_id': service.id} datas = getattr(service_provider, 'process_data')(**kw) if datas is not None and len(datas) > 0: consumer = getattr(service_consumer, '__init__')( service.consumer.token) consumer = getattr(service_consumer, 'save_data') published = '' which_date = '' # 2) for each one for data in datas: if settings.DEBUG: from django_th.tools import to_datetime published = to_datetime(data) published, which_date = get_published(published, which_date) date_triggered = arrow.get(str(service.date_triggered), 'YYYY-MM-DD HH:mm:ss').to(settings.TIME_ZONE) publish_log_data(published, date_triggered, data) # the consummer will save the data and return if success or not status = consumer(service.consumer.token, service.id, **data) else: count_new_data = len(datas) to_update = True # let's log log_update(service, to_update, status, count_new_data) # let's update if to_update and status: update_trigger(service)