Esempio n. 1
0
def publishing(service):
    """
        the purpose of this tasks is to get the data from the cache
        then publish them
        :param service: service object where we will publish
        :type service: object
    """
    # flag to know if we have to update
    to_update = False
    # flag to get the status of a service
    status = False

    # provider - the service that offer data
    # check if the service has already been triggered
    # if date_triggered is None, then it's the first run
    if service.date_triggered is None:
        logger.debug("first run {}".format(service))
        to_update = True
        status = True
    # run run run
    else:
        service_provider = default_provider.get_service(
                str(service.provider.name.name))
    
    # 1) get the data from the provider service
    kw = {'trigger_id': service.id}
    data = getattr(service_provider, 'process_data')(**kw)
    # counting the new data to store to display them in the log
    count_new_data = len(data) if data else 0
    if count_new_data > 0:
    
        # consumer - the service which uses the data
        service_consumer = default_provider.get_service(
                    str(service.consumer.name.name))
    
        getattr(service_consumer, '__init__')(service.consumer.token)
        consumer = getattr(service_consumer, 'save_data')
    
        # 2) for each one
        for d in data:
            # the consumer will save the data and return if success or not
            status = consumer(service.id, **d)
    
            to_update = True
        # let's log
    log_update(service, to_update, status, count_new_data)
    # let's update
    if to_update and status:
        update_trigger(service)
Esempio n. 2
0
    def reading(self, service):
        """
           get the data from the service and put theme in cache
           :param service: service object to read
           :type service: object
        """
        now = arrow.utcnow().to(settings.TIME_ZONE).format(
            'YYYY-MM-DD HH:mm:ssZZ')
        # flag to know if we have to update
        to_update = False

        # counting the new data to store to display them in the log
        # provider - the service that offer data
        provider_token = service.provider.token
        default_provider.load_services()
        service_provider = default_provider.get_service(
            str(service.provider.name.name))
        # check if the service has already been triggered
        # if date_triggered is None, then it's the first run
        # so it will be set to "now"
        date_triggered = service.date_triggered if service.date_triggered \
            else now
        # 1) get the data from the provider service
        # get a timestamp of the last triggered of the service
        kwargs = {'token': provider_token,
                  'trigger_id': service.id,
                  'date_triggered': date_triggered}
        data = self.provider(service_provider, **kwargs)
        # counting the new data to store to display them in the log
        count_new_data = len(data) if data else 0
        if count_new_data > 0:
            to_update = True

        if to_update:
            logger.info("{} - {} new data".format(service, count_new_data))
Esempio n. 3
0
def reading(service):
    """
       get the data from the service and put theme in cache
       :param service: service object to read
    """
    # flag to know if we have to update
    to_update = False
    # flag to get the status of a service
    status = False
    # counting the new data to store to display them in the log
    # provider - the service that offer data
    service_provider = default_provider.get_service(
        str(service.provider.name.name))

    # check if the service has already been triggered
    # if date_triggered is None, then it's the first run
    if service.date_triggered is None:
        logger.debug("first time {}".format(service))
        to_update = True
        status = True
        # run run run
    else:
        # 1) get the data from the provider service
        # get a timestamp of the last triggered of the service
        kw = {'token': service.provider.token,
              'trigger_id': service.id,
              'date_triggered': service.date_triggered}
        datas = getattr(service_provider, '__init__')(service.provider.token)
        datas = getattr(service_provider, 'read_data')(**kw)

        if len(datas) > 0:
            to_update = True
            status = True

        log_update(service, to_update, status, len(datas))
Esempio n. 4
0
    def consumer(self, service, data, to_update, status):
        """
            call the consumer and handle the data
            :param service:
            :param data:
            :param to_update:
            :param status:
            :return: status
        """
        # consumer - the service which uses the data
        service_consumer = default_provider.get_service(
            str(service.consumer.name.name))
        kwargs = {'user': service.user}
        getattr(service_consumer, '__init__')(service.consumer.token,
                                              **kwargs)
        instance = getattr(service_consumer, 'save_data')

        # 2) for each one
        for d in data:
            d['userservice_id'] = service.consumer.id
            # the consumer will save the data and return if success or not
            status = instance(service.id, **d)

            to_update = True

        return to_update, status
Esempio n. 5
0
    def form_valid(self, form):
        name = form.cleaned_data.get('name').name
        user = self.request.user
        if UserService.objects.filter(name=name, user=user).exists():
            from django.contrib import messages
            messages.warning(
                self.request,
                'Service %s already activated' % name.split('Service')[1])
            return HttpResponseRedirect(reverse('user_services'))
        else:
            form.save(user=user)

            sa = ServicesActivated.objects.get(
                name=form.cleaned_data.get('name').name)
            # let's build the 'call' of the auth method
            # which own to a Service Class
            if sa.auth_required:
                # use the default_provider to get the object from the ServiceXXX
                default_provider.load_services()
                service_object = default_provider.get_service(
                    str(form.cleaned_data.get('name').name))
                # get the class object
                lets_auth = getattr(service_object, 'auth')
                # call the auth func from this class
                # and redirect to the external service page
                # to auth the application django-th to access to the user
                # account details
                return redirect(lets_auth(self.request))

            return HttpResponseRedirect(
                reverse('user_services', args=['added']))
Esempio n. 6
0
    def reading(self, service):
        """
           get the data from the service and put theme in cache
           :param service: service object to read
           :type service: object
        """
        # counting the new data to store to display them in the log provider - the service that offer data
        provider_token = service.provider.token
        default_provider.load_services()
        service_provider = default_provider.get_service(
            str(service.provider.name.name))
        date_triggered = service.date_triggered if service.date_triggered else service.date_created

        # get the data from the provider service
        kwargs = {
            'token': provider_token,
            'trigger_id': service.id,
            'date_triggered': date_triggered
        }
        data = self.provider(service_provider, **kwargs)

        if len(data) > 0:
            logger.info("{} - {} new data".format(service, len(data)))
        elif data is False:
            # if data is False, something went wrong
            self.is_ceil_reached(service)
Esempio n. 7
0
    def consumer(self, service, data, to_update, status):
        """
            call the consumer and handle the data
            :param service:
            :param data:
            :param to_update:
            :param status:
            :return: status
        """
        # consumer - the service which uses the data
        service_consumer = default_provider.get_service(
            str(service.consumer.name.name))
        kwargs = {'user': service.user}
        getattr(service_consumer, '__init__')(service.consumer.token, **kwargs)
        instance = getattr(service_consumer, 'save_data')

        # 2) for each one
        for d in data:
            d['userservice_id'] = service.consumer.id
            # the consumer will save the data and return if success or not
            status = instance(service.id, **d)

            to_update = True

        return to_update, status
Esempio n. 8
0
    def reading(self, service):
        """
           get the data from the service and put theme in cache
           :param service: service object to read
           :type service: object
        """
        now = arrow.utcnow().to(
            settings.TIME_ZONE).format('YYYY-MM-DD HH:mm:ssZZ')
        # counting the new data to store to display them in the log
        # provider - the service that offer data
        provider_token = service.provider.token
        default_provider.load_services()
        service_provider = default_provider.get_service(
            str(service.provider.name.name))
        # check if the service has already been triggered
        # if date_triggered is None, then it's the first run
        # so it will be set to "now"
        date_triggered = service.date_triggered if service.date_triggered \
            else now
        # 1) get the data from the provider service
        # get a timestamp of the last triggered of the service
        kwargs = {
            'token': provider_token,
            'trigger_id': service.id,
            'date_triggered': date_triggered
        }
        data = self.provider(service_provider, **kwargs)

        if len(data) > 0:
            logger.info("{} - {} new data".format(service, len(data)))
        elif data is False:
            # if data is False, something went wrong
            self.is_ceil_reached(service)
Esempio n. 9
0
    def form_valid(self, form):
        name = form.cleaned_data.get('name').name
        user = self.request.user
        if UserService.objects.filter(name=name, user=user).exists():
            from django.contrib import messages
            messages.warning(self.request, 'Service %s already activated' %
                             name.split('Service')[1])
            return HttpResponseRedirect(reverse('user_services'))
        else:
            form.save(user=user)

            sa = ServicesActivated.objects.get(
                name=form.cleaned_data.get('name').name)
            # let's build the 'call' of the auth method
            # which own to a Service Class
            if sa.auth_required:
                # use the default_provider to get the object from the ServiceXXX
                default_provider.load_services()
                service_object = default_provider.get_service(
                    str(form.cleaned_data.get('name').name))
                # get the class object
                lets_auth = getattr(service_object, 'auth')
                # call the auth func from this class
                # and redirect to the external service page
                # to auth the application django-th to access to the user
                # account details
                return redirect(lets_auth(self.request))

            return HttpResponseRedirect(reverse('user_services',
                                                args=['added']))
Esempio n. 10
0
    def form_valid(self, form):
        """
        save the data
        :param form:
        :return:
        """
        valid = True
        # 'name' is injected in the clean() of the form line 56
        name = form.cleaned_data.get('name').name
        user = self.request.user
        form.save(user=user, service_name=name)

        sa = ServicesActivated.objects.get(name=name)
        if sa.auth_required and sa.self_hosted:
            # trigger the checking of the service
            from django_th.services import default_provider
            default_provider.load_services()
            service_provider = default_provider.get_service(name)
            result = service_provider.check(self.request, user)
            if result is not True:
                # the call of the API failed due to an error which is in the result string
                # return by the call of the API
                form.add_error('host', result)
                messages.error(self.request, result)
                return redirect('edit_service', pk=self.kwargs.get(self.pk_url_kwarg))

        if valid:
            messages.success(self.request, _('Service %s modified successfully') % name.split('Service')[1])
            return HttpResponseRedirect(reverse('user_services'))
Esempio n. 11
0
def go():
    """
        run the main process
    """
    trigger = TriggerService.objects.all()
    if trigger:
        for service in trigger:
            logger.info("from %s to %s ", service.provider.name,
                        service.consummer.name)

            # provider - the service that offer datas
            service_name = service.provider.name
            service_provider = default_provider.get_service(
                'Service' + str(service_name).capitalize())

            # consummer - the service which uses the datas
            service_name = service.consummer.name
            service_consummer = default_provider.get_service(
                'Service' + str(service_name).capitalize())

            # 1) get the datas from the provider service
            datas = getattr(service_provider, 'process_data')(service.id)
            consummer = getattr(service_consummer, 'save_data')
            # 2) for each one
            for data in datas:
                title = data.title
                content = data.content[0].value
                logger.info("from the service %s", service.provider)
                # 3) check if the previous trigger is older than the
                # date of the data we retreived
                # if yes , process the consummer
                if service.date_triggered is None or to_datetime(
                        data.published) >= service.date_triggered:
                    logger.debug("date %s title %s", data.published,
                                 data.title)
                    logger.info("to the service %s", service.consummer)
                    consummer(service.consummer.token, title, content,
                              service.id)
                # otherwise do nothing
                else:
                    logger.debug("DATA TOO OLD SKIPED : [%s] %s",
                                 data.published, data.title)
                # update the date of the trigger
                update_trigger(service)

    else:
        print "No trigger set by any user"
Esempio n. 12
0
def go():
    """
        run the main process
    """
    trigger = TriggerService.objects.all()
    if trigger:
        for service in trigger:
            logger.info(
                "from %s to %s ", service.provider.name, service.consummer.name)

            # provider - the service that offer datas
            service_name = service.provider.name
            service_provider = default_provider.get_service(
                'Service' + str(service_name).capitalize())

            # consummer - the service which uses the datas
            service_name = service.consummer.name
            service_consummer = default_provider.get_service(
                'Service' + str(service_name).capitalize())

            # 1) get the datas from the provider service
            datas = getattr(service_provider, 'process_data')(service.id)
            consummer = getattr(service_consummer, 'save_data')
            # 2) for each one
            for data in datas:
                title = data.title
                content = data.content[0].value
                logger.info("from the service %s", service.provider)
                # 3) check if the previous trigger is older than the
                # date of the data we retreived
                # if yes , process the consummer
                if service.date_triggered is None or to_datetime(data.published) >= service.date_triggered:
                    logger.debug(
                        "date %s title %s", data.published, data.title)
                    logger.info("to the service %s", service.consummer)
                    consummer(
                        service.consummer.token, title, content, service.id)
                # otherwise do nothing
                else:
                    logger.debug(
                        "DATA TOO OLD SKIPED : [%s] %s", data.published, data.title)
                # update the date of the trigger
                update_trigger(service)

    else:
        print "No trigger set by any user"
Esempio n. 13
0
    def handle(self, *args, **options):
        """
            run the main process
        """
        default_provider.load_services()
        trigger = TriggerService.objects.filter(status=True).select_related(
            'consumer__name', 'provider__name')
        if trigger:
            for service in trigger:

                # provider - the service that offer data
                service_name = str(service.provider.name.name)
                service_provider = default_provider.get_service(service_name)

                # consumer - the service which uses the data
                service_name = str(service.consumer.name.name)
                service_consumer = default_provider.get_service(service_name)

                # First run
                if service.date_triggered is None:
                    logger.debug("first run for %s => %s " % (str(
                        service.provider.name), str(service.consumer.name.name)))

                    asyncio.get_event_loop().run_until_complete(
                        self.my_dummy_provider())

                # another run
                else:
                    asyncio.get_event_loop().run_until_complete(
                        self.my_provider(service_provider,
                                         service.provider.token,
                                         service.id,
                                         service.date_triggered))
                    asyncio.get_event_loop().run_until_complete(
                        self.my_consumer(service_consumer,
                                         service.consumer.token,
                                         service.id,
                                         service.date_triggered))

                asyncio.get_event_loop().run_until_complete(
                    self.update_trigger(service))
                asyncio.get_event_loop().run_forever()

        else:
            print("No trigger set by any user")
Esempio n. 14
0
def renew_service(request, pk):
    """
        renew an existing service
    """
    service = get_object_or_404(ServicesActivated, pk=pk)
    service_name = str(service.name)
    service_object = default_provider.get_service(service_name)
    lets_auth = getattr(service_object, 'auth')
    return redirect(lets_auth(request))
Esempio n. 15
0
    def handle(self, *args, **options):
        """
            run the main process
        """
        default_provider.load_services()
        trigger = TriggerService.objects.filter(status=True).select_related(
            'consummer__name', 'provider__name')
        if trigger:
            for service in trigger:

                # provider - the service that offer datas
                service_name = str(service.provider.name.name)
                service_provider = default_provider.get_service(service_name)

                # consumer - the service which uses the data
                service_name = str(service.consumer.name.name)
                service_consumer = default_provider.get_service(service_name)

                # First run
                if service.date_triggered is None:
                    logger.debug("first run for %s => %s " %
                                 (str(service.provider.name),
                                  str(service.consumer.name.name)))

                    asyncio.get_event_loop().run_until_complete(
                        self.my_dummy_provider())

                # another run
                else:
                    asyncio.get_event_loop().run_until_complete(
                        self.my_provider(service_provider,
                                         service.provider.token, service.id,
                                         service.date_triggered))
                    asyncio.get_event_loop().run_until_complete(
                        self.my_consumer(service_consumer,
                                         service.consumer.token, service.id,
                                         service.date_triggered))

                asyncio.get_event_loop().run_until_complete(
                    self.update_trigger(service))
                asyncio.get_event_loop().run_forever()

        else:
            print("No trigger set by any user")
def renew_service(request, pk):
    """
        renew an existing service
        :param pk: the primary key of the service to renew
        :type pk: int
    """
    service = get_object_or_404(ServicesActivated, pk=pk)
    service_name = str(service.name)
    service_object = default_provider.get_service(service_name)
    lets_auth = getattr(service_object, 'auth')
    return redirect(lets_auth(request))
Esempio n. 17
0
def renew_service(request, pk):
    """
        renew an existing service
        :param pk: the primary key of the service to renew
        :type pk: int
    """
    service = get_object_or_404(ServicesActivated, pk=pk)
    service_name = str(service.name)
    service_object = default_provider.get_service(service_name)
    lets_auth = getattr(service_object, 'auth')
    return redirect(lets_auth(request))
Esempio n. 18
0
def renew_service(request, pk):
    """
        renew an existing service
    """
    service = get_object_or_404(ServicesActivated, pk=pk)
    service_name = str(service.name)
    service_object = default_provider.get_service(service_name)
    lets_auth = getattr(service_object, 'auth')
    #lets_callback = getattr(service_object, 'auth')
    # print lets_callback
    return redirect(lets_auth(request))
Esempio n. 19
0
    def provider(self, service):
        """
            get the data from (the cache of) the service provider
            :param service:
            :return: data
        """
        service_provider = default_provider.get_service(str(service.provider.name.name))

        # 1) get the data from the provider service
        module_name = 'th_' + service.provider.name.name.split('Service')[1].lower()
        kwargs = {'trigger_id': str(service.id), 'cache_stack': module_name}
        return getattr(service_provider, 'process_data')(**kwargs)
Esempio n. 20
0
def finalcallback(request, **kwargs):
    """
        let's do the callback of the related service after
        the auth request from UserServiceCreateView
    """
    service_name = kwargs['service_name']
    service_object = default_provider.get_service(service_name)
    lets_callback = getattr(service_object, 'callback')
    # call the auth func from this class
    # and redirect to the external service page
    # to auth the application django-th to access to the user
    # account details
    return render_to_response(lets_callback(request))
Esempio n. 21
0
def finalcallback(request, **kwargs):
    """
        let's do the callback of the related service after
        the auth request from UserServiceCreateView
    """
    service_name = kwargs['service_name']
    service_object = default_provider.get_service(service_name)
    lets_callback = getattr(service_object, 'callback')
    # call the auth func from this class
    # and redirect to the external service page
    # to auth the application django-th to access to the user
    # account details
    return render_to_response(lets_callback(request))
Esempio n. 22
0
def put_in_cache(service):
    # flag to know if we have to update
    to_update = False
    # flag to get the status of a service
    status = False
    # counting the new data to store to display them in the log
    # provider - the service that offer data
    service_name = str(service.provider.name.name)
    service_provider = default_provider.get_service(service_name)

    service_name = str(service.consumer.name.name)

    # check if the service has already been triggered
    # if date_triggered is None, then it's the first run
    if service.date_triggered is None:
        logger.debug("first run for %s => %s " % (
            str(service.provider.name), str(service.consumer.name.name)))
        to_update = True
        status = True
        # run run run
    else:
        # 1) get the data from the provider service
        # get a timestamp of the last triggered of the service
        datas = getattr(service_provider, '__init__')(service.provider.token)
        datas = getattr(service_provider, 'read_data')(
            service.provider.token,
            service.id,
            service.date_triggered)
        if len(datas) > 0:
            to_update = True
            status = True

        # update the date of the trigger at the end of the loop
        sentence = "user: {} - provider: {} - {}"
        if to_update:
            if status:
                logger.info((sentence + " - {} data put in cache").format(
                    service.user,
                    service.provider.name.name,
                    service.description,
                    len(datas)))
            else:
                logger.info((sentence + " AN ERROR OCCURS ").format(
                    service.user,
                    service.provider.name.name,
                    service.description))
        else:
            logger.info((sentence + " nothing new").format(
                        service.user,
                        service.provider.name.name,
                        service.description))
Esempio n. 23
0
    def provider(self, service):
        """
            get the data from (the cache of) the service provider
            :param service:
            :return: data
        """
        service_provider = default_provider.get_service(
            str(service.provider.name.name))

        # 1) get the data from the provider service
        module_name = 'th_' + \
                      service.provider.name.name.split('Service')[1].lower()
        kwargs = {'trigger_id': str(service.id), 'cache_stack': module_name}
        return getattr(service_provider, 'process_data')(**kwargs)
Esempio n. 24
0
    def test_reading(self):
        from django_th.services import default_provider
        service = self.create_triggerservice()
        date_triggered = service.date_triggered if service.date_triggered else service.date_created

        kwargs = {'token': service.provider.token,
                  'trigger_id': service.id,
                  'date_triggered': date_triggered}

        with patch.object(ServiceProvider, 'get_service') as mock_it:
            mock_it.return_value = default_provider.get_service(
                str(service.provider.name.name))
            with patch.object(Read, 'provider') as mock_read:
                se = Read()
                se.reading(service)
            mock_read.assert_called_once_with(mock_it.return_value, **kwargs)
        mock_it.assert_called_with(service.provider.name.name)
    def form_valid(self, form):
        self.object = form.save(user=self.request.user)

        sa = ServicesActivated.objects.get(name=form.cleaned_data['name'].name)
        # let's build the 'call' of the auth method which own to a Service Class
        if sa.auth_required:
            # use the default_provider to get the object from the ServiceXXX
            service_object = default_provider.get_service(
                str(form.cleaned_data['name'].name))
            # get the class object
            lets_auth = getattr(service_object, 'auth')
            # call the auth func from this class
            # and redirect to the external service page
            # to auth the application django-th to access to the user
            # account details
            return redirect(lets_auth(self.request))

        return HttpResponseRedirect(reverse('service_add_thanks'))
Esempio n. 26
0
    def form_valid(self, form):
        self.object = form.save(user=self.request.user)

        sa = ServicesActivated.objects.get(name=form.cleaned_data['name'].name)
        # let's build the 'call' of the auth method which own to a Service Class
        if sa.auth_required:
            # use the default_provider to get the object from the ServiceXXX
            service_object = default_provider.get_service(
                str(form.cleaned_data['name'].name))
            # get the class object
            lets_auth = getattr(service_object, 'auth')
            # call the auth func from this class
            # and redirect to the external service page
            # to auth the application django-th to access to the user
            # account details
            return redirect(lets_auth(self.request))

        return HttpResponseRedirect(reverse('service_add_thanks'))
Esempio n. 27
0
    def test_reading(self):
        from django_th.services import default_provider
        now = arrow.utcnow().to(settings.TIME_ZONE).format(
            'YYYY-MM-DD HH:mm:ssZZ')
        service = self.create_triggerservice()
        date_triggered = service.date_triggered if service.date_triggered \
            else now

        kwargs = {'token': service.provider.token,
                  'trigger_id': service.id,
                  'date_triggered': date_triggered}

        with patch.object(ServiceProvider, 'get_service') as mock_it:
            mock_it.return_value = default_provider.get_service(
                str(service.provider.name.name))
            with patch.object(Read, 'provider') as mock_read:
                se = Read()
                se.reading(service)
            mock_read.assert_called_once_with(mock_it.return_value, **kwargs)
        mock_it.assert_called_with(service.provider.name.name)
Esempio n. 28
0
def save_data(trigger_id, data):
    """
        call the consumer and handle the data
        :param trigger_id:
        :param data:
        :return:
    """
    status = True
    # consumer - the service which uses the data
    default_provider.load_services()
    service = TriggerService.objects.get(id=trigger_id)

    service_consumer = default_provider.get_service(str(service.consumer.name.name))
    kwargs = {'user': service.user}

    if len(data) > 0:
        getattr(service_consumer, '__init__')(service.consumer.token, **kwargs)
        status = getattr(service_consumer, 'save_data')(service.id, **data)

    return status
Esempio n. 29
0
    def form_valid(self, form):
        name = form.cleaned_data.get('name').name
        user = self.request.user
        form.save(user=user, service_name=self.kwargs.get('service_name'))

        sa = ServicesActivated.objects.get(name=name)
        # let's build the 'call' of the auth method
        # which own to a Service Class
        if sa.auth_required:
            # use the default_provider to get the object from the ServiceXXX
            default_provider.load_services()
            service_object = default_provider.get_service(str(name))
            # get the class object
            lets_auth = getattr(service_object, 'auth')
            # call the auth func from this class
            # and redirect to the external service page
            # to auth the application django-th to access to the user
            # account details
            return redirect(lets_auth(self.request))
        messages.success(self.request, _('Service %s activated successfully') % name.split('Service')[1])
        return HttpResponseRedirect(reverse('user_services'))
Esempio n. 30
0
def save_data(trigger_id, data):
    """
        call the consumer and handle the data
        :param trigger_id:
        :param data:
        :return:
    """
    status = True
    # consumer - the service which uses the data
    default_provider.load_services()
    service = TriggerService.objects.get(id=trigger_id)

    service_consumer = default_provider.get_service(
        str(service.consumer.name.name))
    kwargs = {'user': service.user}

    if len(data) > 0:
        getattr(service_consumer, '__init__')(service.consumer.token, **kwargs)
        status = getattr(service_consumer, 'save_data')(service.id, **data)

    return status
Esempio n. 31
0
    def reading(self, service):
        """
           get the data from the service and put theme in cache
           :param service: service object to read
           :type service: object
        """
        # counting the new data to store to display them in the log provider - the service that offer data
        provider_token = service.provider.token
        default_provider.load_services()
        service_provider = default_provider.get_service(str(service.provider.name.name))
        date_triggered = service.date_triggered if service.date_triggered else service.date_created

        # get the data from the provider service
        kwargs = {'token': provider_token, 'trigger_id': service.id, 'date_triggered': date_triggered}
        data = self.provider(service_provider, **kwargs)

        if len(data) > 0:
            logger.info("{} - {} new data".format(service, len(data)))
        elif data is False:
            # if data is False, something went wrong
            self.is_ceil_reached(service)
Esempio n. 32
0
def reading(service):
    """
       get the data from the service and put theme in cache
       :param service: service object to read
       :type service: object
    """
    data = ()
    # flag to know if we have to update
    to_update = False
    # flag to get the status of a service
    status = False
    count_new_data = 0
    # counting the new data to store to display them in the log
    # provider - the service that offer data
    provider_token = service.provider.token
    service_provider = default_provider.get_service(
        str(service.provider.name.name))

    # check if the service has already been triggered
    # if date_triggered is None, then it's the first run
    if service.date_triggered is None:
        logger.debug("first time {}".format(service))
        to_update = True
        status = True
        # run run run
    else:
        # 1) get the data from the provider service
        # get a timestamp of the last triggered of the service
        kw = {'token': provider_token,
              'trigger_id': service.id,
              'date_triggered': service.date_triggered}
        getattr(service_provider, '__init__')(provider_token)
        data = getattr(service_provider, 'read_data')(**kw)
        # counting the new data to store to display them in the log
        count_new_data = len(data) if data else 0
        if count_new_data > 0:
            to_update = True
            status = True

    log_update(service, to_update, status, count_new_data)
Esempio n. 33
0
    def test_reading(self):
        from django_th.services import default_provider
        now = arrow.utcnow().to(
            settings.TIME_ZONE).format('YYYY-MM-DD HH:mm:ssZZ')
        service = self.create_triggerservice()
        date_triggered = service.date_triggered if service.date_triggered \
            else now

        kwargs = {
            'token': service.provider.token,
            'trigger_id': service.id,
            'date_triggered': date_triggered
        }

        with patch.object(ServiceProvider, 'get_service') as mock_it:
            mock_it.return_value = default_provider.get_service(
                str(service.provider.name.name))
            with patch.object(Read, 'provider') as mock_read:
                se = Read()
                se.reading(service)
            mock_read.assert_called_once_with(mock_it.return_value, **kwargs)
        mock_it.assert_called_with(service.provider.name.name)
Esempio n. 34
0
def reading(service):
    """
       get the data from the service and put theme in cache
       :param service: service object to read
       :type service: object
    """
    # flag to know if we have to update
    to_update = False
    count_new_data = 0
    # counting the new data to store to display them in the log
    # provider - the service that offer data
    provider_token = service.provider.token
    service_provider = default_provider.get_service(
        str(service.provider.name.name))

    # check if the service has already been triggered
    # if date_triggered is None, then it's the first run
    if service.date_triggered is None:
        logger.debug("first time {}".format(service))
        to_update = True
        # run run run
    else:
        # 1) get the data from the provider service
        # get a timestamp of the last triggered of the service
        kw = {'token': provider_token,
              'trigger_id': service.id,
              'date_triggered': service.date_triggered}
        getattr(service_provider, '__init__')(provider_token)
        data = getattr(service_provider, 'read_data')(**kw)
        # counting the new data to store to display them in the log
        count_new_data = len(data) if data else 0
        if count_new_data > 0:
            to_update = True

    if to_update:
        logger.info("{} - {} new data".format(service, count_new_data))
Esempio n. 35
0
            else:
                logger.debug("data outdated skiped : [{}] ".format(published))

    else:
        # return the number of updates ( to be displayed in the log )
        yield from q2.put(count_new_data)


default_provider.load_services()
trigger = TriggerService.objects.filter(status=True)
if trigger:
    for service in trigger:

        # provider - the service that offer datas
        service_name = str(service.provider.name.name)
        service_provider = default_provider.get_service(service_name)

        # consumer - the service which uses the data
        service_name = str(service.consumer.name.name)
        service_consumer = default_provider.get_service(service_name)

        # First run
        if service.date_triggered is None:
            logger.debug(
                "first run for %s => %s " %
                (str(service.provider.name), str(service.consumer.name.name)))

            asyncio.get_event_loop().run_until_complete(my_dummy_provider)

        # another run
        else:
Esempio n. 36
0
def go():
    """
        run the main process
    """
    trigger = TriggerService.objects.filter(status=True)
    if trigger:
        for service in trigger:
            # flag to know if we have to update
            to_update = False
            # counting the new data to store to display them in the log
            count_new_data = 0
            # provider - the service that offer datas
            service_name = str(service.provider.name.name)
            service_provider = default_provider.get_service(service_name)

            # consumer - the service which uses the data
            service_name = str(service.consumer.name.name)
            service_consumer = default_provider.get_service(service_name)

            # check if the service has already been triggered
            if service.date_triggered is None:
                logger.debug("first run for %s => %s " % (str(
                    service.provider.name), str(service.consumer.name.name)))
                to_update = True
            # run run run
            else:
                # 1) get the datas from the provider service
                # get a timestamp of the last triggered of the service
                datas = getattr(service_provider, 'process_data')(
                    service.provider.token, service.id, service.date_triggered)
                consumer = getattr(service_consumer, 'save_data')

                published = ''
                which_date = ''
                # 2) for each one
                for data in datas:
                    # if in a pool of data once of them does not have
                    # a date, will take the previous date for this one
                    # if it's the first one, set it to 00:00:00

                    # let's try to determine the date contained in the data...
                    published = to_datetime(data)
                    if published is not None:
                        # get the published date of the provider
                        published = arrow.get(
                            str(published), 'YYYY-MM-DD HH:mm:ss')
                        # store the date for the next loop
                        # if published became 'None'
                        which_date = published
                    #... otherwise set it to 00:00:00 of the current date
                    if which_date == '':
                        # current date
                        which_date = arrow.utcnow().replace(
                            hour=0, minute=0, second=0)
                        published = which_date
                    if published is None and which_date != '':
                        published = which_date
                    # 3) check if the previous trigger is older than the
                    # date of the data we retreived
                    # if yes , process the consumer

                    # add the TIME_ZONE settings
                    date_triggered = arrow.get(
                        str(service.date_triggered), 'YYYY-MM-DD HH:mm:ss').to(settings.TIME_ZONE)

                    # flag to know if we can push data to the consumer
                    proceed = False

                    # if the published date if greater or equal to the last
                    # triggered event ... :
                    if date_triggered is not None and published is not None and published.date() >= date_triggered.date():
                        # if date are the same ...
                        if published.date() == date_triggered.date():
                            # ... compare time and proceed if needed
                            if published.time() >= date_triggered.time():
                                proceed = True
                        # not same date so proceed !
                        else:
                            proceed = True
                        if proceed:
                            if 'title' in data:
                                logger.info("date {} >= date triggered {} title {}".format(
                                    published, date_triggered, data['title']))
                            else:
                                logger.info(
                                    "date {} >= date triggered {} ".format(published, date_triggered))

                            consumer(
                                service.consumer.token, service.id, **data)

                            to_update = True
                            count_new_data += 1
                    # otherwise do nothing
                    else:
                        if 'title' in data:
                            logger.debug(
                                "data outdated skiped : [{}] {}".format(published, data['title']))
                        else:
                            logger.debug(
                                "data outdated skiped : [{}] ".format(published))

            # update the date of the trigger at the end of the loop
            if to_update:
                logger.info("user: {} - provider: {} - consumer: {} - {} = {} new data".format(
                    service.user, service.provider.name.name, service.consumer.name.name, service.description, count_new_data))
                update_trigger(service)
            else:
                logger.info("user: {} - provider: {} - consumer: {} - {} nothing new".format(
                    service.user, service.provider.name.name, service.consumer.name.name, service.description))
    else:
        print("No trigger set by any user")
Esempio n. 37
0
def publish_data():
    """
        the purpose of this tasks is to get the data from the cache
        then publish them
    """
    now = arrow.utcnow().to(settings.TIME_ZONE)

    trigger = TriggerService.objects.filter(status=True).select_related(
        'consumer__name', 'provider__name')
    if trigger:
        for service in trigger:

            # flag to know if we have to update
            to_update = False
            # flag to get the status of a service
            status = False
            # counting the new data to store to display them in the log
            count_new_data = 0
            # provider - the service that offer data
            service_name = str(service.provider.name.name)
            service_provider = default_provider.get_service(service_name)

            # consumer - the service which uses the data
            service_name = str(service.consumer.name.name)
            service_consumer = default_provider.get_service(service_name)

            # check if the service has already been triggered
            # if date_triggered is None, then it's the first run
            if service.date_triggered is None:
                logger.debug("first run for %s => %s " % (str(
                    service.provider.name),
                    str(service.consumer.name.name)))
                to_update = True
                status = True
            # run run run
            else:
                # 1) get the data from the provider service
                # get a timestamp of the last triggered of the service
                datas = getattr(service_provider, 'process_data')(service.id)
                if datas is None or len(datas) == 0:
                    continue
                consumer = getattr(service_consumer, '__init__')(
                    service.consumer.token)
                consumer = getattr(service_consumer, 'save_data')

                published = ''
                which_date = ''
                # 2) for each one
                for data in datas:
                    #  if in a pool of data once of them does not have
                    #  a date, will take the previous date for this one
                    #  if it's the first one, set it to 00:00:00

                    # let's try to determine the date contained in
                    # the data...
                    published = to_datetime(data)

                    if published is not None:
                        # get the published date of the provider
                        published = arrow.get(published).to(settings.TIME_ZONE)
                        # store the date for the next loop
                        #  if published became 'None'
                        which_date = published
                    # ... otherwise set it to 00:00:00 of the current date
                    if which_date == '':
                        # current date
                        which_date = arrow.utcnow().replace(
                            hour=0, minute=0, second=0).to(
                            settings.TIME_ZONE)
                        published = which_date
                    if published is None and which_date != '':
                        published = which_date
                    # 3) check if the previous trigger is older than the
                    #  date of the data we retrieved
                    #  if yes , process the consumer

                    # add the TIME_ZONE settings
                    # to localize the current date
                    date_triggered = arrow.get(
                        str(service.date_triggered),
                        'YYYY-MM-DD HH:mm:ss').to(settings.TIME_ZONE)

                    # if the published date if greater or equal to the last
                    # triggered event ... :
                    if date_triggered is not None and \
                       published is not None and \
                       now >= published and \
                       published >= date_triggered:

                        if 'title' in data:
                            sentence = "date {} >= triggered {} title {}"
                            logger.info(sentence.format(published,
                                                        date_triggered,
                                                        data['title']))
                        else:
                            sentence = "date {} >= date triggered {}"
                            logger.info(sentence.format(published,
                                                        date_triggered))

                        status = consumer(
                            service.consumer.token, service.id, **data)

                        to_update = True
                        count_new_data += 1
                    # otherwise do nothing
                    else:
                        if 'title' in data:
                            sentence = "data outdated skipped : [{}] {}"
                            logger.debug(sentence.format(published,
                                                         data['title']))
                        else:
                            sentence = "data outdated skipped : [{}] "
                            logger.debug(sentence.format(published))

            # update the date of the trigger at the end of the loop
            sentence = "user: {} - provider: {} - consumer: {} - {}"
            if to_update:
                if status:
                    logger.info((sentence + " - {} new data").format(
                        service.user,
                        service.provider.name.name,
                        service.consumer.name.name,
                        service.description,
                        count_new_data))
                    update_trigger(service)
                else:
                    logger.info((sentence + " AN ERROR OCCURS ").format(
                        service.user,
                        service.provider.name.name,
                        service.consumer.name.name,
                        service.description))
            else:
                logger.info((sentence + " nothing new").format(
                    service.user,
                    service.provider.name.name,
                    service.consumer.name.name,
                    service.description))
Esempio n. 38
0
def publishing(service, now):
    """
        the purpose of this tasks is to get the data from the cache
        then publish them
        :param service: service object where we will publish
        :param now: it's the current date
        :type service: obect
        :type now: string date
    """
    # flag to know if we have to update
    to_update = False
    # flag to get the status of a service
    status = False
    # counting the new data to store to display them in the log
    count_new_data = 0
    # provider - the service that offer data
    service_provider = default_provider.get_service(
        str(service.provider.name.name))

    # consumer - the service which uses the data
    service_consumer = default_provider.get_service(
        str(service.consumer.name.name))

    # check if the service has already been triggered
    # if date_triggered is None, then it's the first run
    if service.date_triggered is None:
        logger.debug("first run {}".format(service))
        to_update = True
        status = True
    # run run run
    else:
        # 1) get the data from the provider service
        kw = {'trigger_id': service.id}
        datas = getattr(service_provider, 'process_data')(**kw)
        if datas is not None and len(datas) > 0:
            consumer = getattr(service_consumer,
                               '__init__')(service.consumer.token)
            consumer = getattr(service_consumer, 'save_data')

            published = ''
            which_date = ''

            # 2) for each one
            for data in datas:

                if settings.DEBUG:
                    from django_th.tools import to_datetime
                    published = to_datetime(data)
                    published, which_date = get_published(
                        published, which_date)
                    date_triggered = arrow.get(str(service.date_triggered),
                                               'YYYY-MM-DD HH:mm:ss').to(
                                                   settings.TIME_ZONE)
                    publish_log_data(published, date_triggered, data)
                # the consummer will save the data and return if success or not
                status = consumer(service.consumer.token, service.id, **data)
            else:
                count_new_data = len(datas)
                to_update = True
    # let's log
    log_update(service, to_update, status, count_new_data)
    # let's update
    if to_update and status:
        update_trigger(service)
Esempio n. 39
0
                logger.debug(
                    "data outdated skiped : [{}] ".format(published))

    else:
        # return the number of updates ( to be displayed in the log )
        yield from q2.put(count_new_data)


default_provider.load_services()
trigger = TriggerService.objects.filter(status=True)
if trigger:
    for service in trigger:

        # provider - the service that offer datas
        service_name = str(service.provider.name.name)
        service_provider = default_provider.get_service(service_name)

        # consumer - the service which uses the data
        service_name = str(service.consumer.name.name)
        service_consumer = default_provider.get_service(service_name)

        # First run
        if service.date_triggered is None:
            logger.debug("first run for %s => %s " % (str(
                service.provider.name), str(service.consumer.name.name)))

            asyncio.get_event_loop().run_until_complete(my_dummy_provider)

        # another run
        else:
            asyncio.get_event_loop().run_until_complete(
Esempio n. 40
0
def publishing(service, now):
    """
        the purpose of this tasks is to get the data from the cache
        then publish them
        :param service: service object where we will publish
        :param now: it's the current date
        :type service: obect
        :type now: string date
    """
    # flag to know if we have to update
    to_update = False
    # flag to get the status of a service
    status = False
    # counting the new data to store to display them in the log
    count_new_data = 0
    # provider - the service that offer data
    service_provider = default_provider.get_service(
        str(service.provider.name.name))

    # consumer - the service which uses the data
    service_consumer = default_provider.get_service(
        str(service.consumer.name.name))

    # check if the service has already been triggered
    # if date_triggered is None, then it's the first run
    if service.date_triggered is None:
        logger.debug("first run {}".format(service))
        to_update = True
        status = True
    # run run run
    else:
        # 1) get the data from the provider service
        kw = {'trigger_id': service.id}
        datas = getattr(service_provider, 'process_data')(**kw)
        if datas is not None and len(datas) > 0:
            consumer = getattr(service_consumer, '__init__')(
                service.consumer.token)
            consumer = getattr(service_consumer, 'save_data')

            published = ''
            which_date = ''

            # 2) for each one
            for data in datas:

                if settings.DEBUG:
                    from django_th.tools import to_datetime
                    published = to_datetime(data)
                    published, which_date = get_published(published, which_date)
                    date_triggered = arrow.get(str(service.date_triggered), 'YYYY-MM-DD HH:mm:ss').to(settings.TIME_ZONE)
                    publish_log_data(published, date_triggered, data)
                # the consummer will save the data and return if success or not
                status = consumer(service.consumer.token, service.id, **data)
            else:
                count_new_data = len(datas)
                to_update = True
    # let's log
    log_update(service, to_update, status, count_new_data)
    # let's update
    if to_update and status:
        update_trigger(service)