def handle(self, *args, **options):
        # get some access_token
        try:
            fbc = FacebookConnection.objects.filter(
                expires__gt=datetime.datetime.now(), deleted=False)[0]
        except IndexError:
            self.stdout.write(u'Error: there is no FacebookConnection')
            fbc = None

        if fbc:

            # api client
            client = FacebookClient(
                client_id=settings.FB_CLIENT_ID,
                client_secret=settings.FB_CLIENT_SECRET,
                access_token=fbc.access_token,
            )

            if args:
                cfps = CampaignFacebookPage.objects.filter(
                    facebook_id__in=args, first_load=False, deleted=False)
            else:
                cfps = CampaignFacebookPage.objects.filter(first_load=False,
                                                           deleted=False)

            error = False
            # do first load
            for cfp in [c for c in cfps]:
                if error:
                    break

                # page info
                try:
                    cfp_info = client.obj_id(cfp.facebook_id)
                except FacebookGenericError:
                    if FacebookGenericError.code == u'17':
                        fbc, access_token = FacebookConnection.renew_client(
                            fbc)
                        client._access_token = access_token
                        cfp_info = client.obj_id(cfp.facebook_id)
                else:
                    if not cfp_info:
                        fbc, access_token = FacebookConnection.renew_client(
                            fbc)
                        client._access_token = access_token
                        cfp_info = client.obj_id(cfp.facebook_id)

                    if not cfp_info:
                        error = True
                        continue
                        #raise CommandError(u'\nThe limit of all robots was reached.')

                cfp_info = client.obj_id(cfp.facebook_id)
                cfp.name = cfp_info[u'name']
                cfp.link = cfp_info[u'link']
                cfp.talking_about = cfp_info[u'talking_about_count']
                cfp.likes = cfp_info[u'likes']

                cfp.save()

                evolution = EvolutionFacebookPageLike(
                    facebook_page=cfp,
                    likes=cfp_info[u'likes'],
                )
                evolution.save()

                self.stdout.write(
                    u'Successfully updated Campaign Facebook Page: %s \n\n' %
                    cfp.facebook_id)

                main_loop = True

                try:
                    until = APIPagination.objects.get(
                        app_name=u'metrics_social',
                        model_name='FacebookMessage',
                        path=u'/%s/feed' % cfp.facebook_id,
                    ).offset
                except APIPagination.DoesNotExist:
                    until = None

                while main_loop:
                    if error:
                        break

                    params = {
                        u'limit': 50,
                        u'format': u'json',
                        u'method': u'GET',
                    }

                    if until:
                        params.update({u'until': until})

                    try:
                        feed = client.feed(cfp.facebook_id, **params)
                    except FacebookGenericError:
                        if FacebookGenericError.code == u'17':
                            fbc, access_token = FacebookConnection.renew_client(
                                fbc)
                            client._access_token = access_token
                            feed = client.feed(cfp.facebook_id, **params)
                    else:
                        if not feed:
                            fbc, access_token = FacebookConnection.renew_client(
                                fbc)
                            client._access_token = access_token
                            feed = client.feed(cfp.facebook_id, **params)

                    if not feed:
                        error = True
                        continue
                        #raise CommandError(u'\nThe limit of all robots was reached.')

                    if not u'data' in feed or not len(feed[u'data']):
                        main_loop = False
                        cfp.first_load = True
                        cfp.save()
                        continue

                    page_save = PageSave('metrics_social', 'FacebookMessage')
                    page_save.start_save(feed[u'data'], fp=cfp)

                    self.stdout.write(u'Facebook message block was created\n')
                    self.stdout.write(u'\n'.join(page_save.responses))

                    if u'paging' in feed and u'next' in feed[u'paging']:
                        until = urlparse.parse_qs(
                            urlparse.urlparse(
                                feed[u'paging'][u'next']).query)[u'until'][0]
                        self.stdout.write(u'Going to next page \n\n\n')

                        try:
                            stopped_at = APIPagination.objects.get(
                                app_name=u'metrics_social',
                                model_name='FacebookMessage',
                                path=u'/%s/feed' % cfp.facebook_id,
                            )
                            stopped_at.offset = until
                            stopped_at.save()
                        except APIPagination.DoesNotExist:
                            stopped_at = APIPagination(
                                app_name=u'metrics_social',
                                model_name='FacebookMessage',
                                path=u'/%s/feed' % cfp.facebook_id,
                                offset=until,
                            )
                            stopped_at.save()
                    else:
                        main_loop = False
                        cfp.first_load = True
                        cfp.save()
    def handle(self, *args, **options):
        # get some access_token
        try:
            fbc = FacebookConnection.objects.filter(
                expires__gt=datetime.datetime.now(), deleted=False)[0]
        except IndexError:
            self.stdout.write(u'Error: there is no FacebookConnection')
            fbc = None

        if fbc:

            # api client
            client = FacebookClient(
                client_id=settings.FB_CLIENT_ID,
                client_secret=settings.FB_CLIENT_SECRET,
                access_token=fbc.access_token,
            )

            if args:
                cfps = CampaignFacebookPage.objects.filter(
                    facebook_id__in=args, deleted=False)
            else:
                cfps = CampaignFacebookPage.objects.filter(deleted=False)

            error = False
            # do daily load
            for cfp in [c for c in cfps]:
                if error:
                    break

                # page info
                try:
                    cfp_info = client.obj_id(cfp.facebook_id)
                except FacebookGenericError:
                    if FacebookGenericError.code == u'17':
                        fbc, access_token = FacebookConnection.renew_client(
                            fbc)
                        client._access_token = access_token
                        cfp_info = client.obj_id(cfp.facebook_id)
                else:
                    if not cfp_info:
                        fbc, access_token = FacebookConnection.renew_client(
                            fbc)
                        client._access_token = access_token
                        cfp_info = client.obj_id(cfp.facebook_id)

                    if not cfp_info:
                        error = True
                        continue
                        #raise CommandError(u'\nThe limit of all robots was reached.')

                cfp.name = cfp_info[u'name']
                cfp.link = cfp_info[u'link']
                cfp.talking_about = cfp_info[u'talking_about_count']
                cfp.likes = cfp_info[u'likes']

                cfp.save()

                evolution = EvolutionFacebookPageLike(
                    facebook_page=cfp,
                    likes=cfp_info[u'likes'],
                )
                evolution.save()

                self.stdout.write(
                    u'Successfully updated Campaign Facebook Page: %s \n\n' %
                    cfp.facebook_id)

                main_loop = True
                until = None
                while main_loop:
                    if error:
                        break

                    params = {u'limit': 50}
                    if until:
                        params.update({u'until': until})

                    # load feed
                    try:
                        feed = client.feed(cfp.facebook_id, **params)
                    except FacebookGenericError:
                        if FacebookGenericError.code == u'17':
                            fbc, access_token = FacebookConnection.renew_client(
                                fbc)
                            client._access_token = access_token
                            feed = client.feed(cfp.facebook_id, **params)
                    else:
                        if not feed:
                            fbc, access_token = FacebookConnection.renew_client(
                                fbc)
                            client._access_token = access_token
                            feed = client.feed(cfp.facebook_id, **params)

                    if not feed:
                        error = True
                        continue
                        #raise CommandError(u'\nThe limit of all robots was reached.')

                    if not u'data' in feed or not len(feed[u'data']):
                        main_loop = False
                        cfp.save()
                        continue

                    for row in feed[u'data']:
                        try:
                            fbm = FacebookMessage.objects.get(
                                facebook_id__exact=row[u'id'], deleted=False)
                        except FacebookMessage.DoesNotExist:
                            try:
                                if u'likes' in row and u'count' in row[
                                        u'likes']:
                                    likes = row[u'likes'][u'count']
                                else:
                                    likes = 0

                                if u'comments' in row and u'count' in row[
                                        u'comments']:
                                    comments = row[u'comments'][u'count']
                                else:
                                    comments = 0

                                content = u''
                                if u'message' in row:
                                    content = u'%s' % row[u'message']

                                if u'story' in row:
                                    content = u'%s %s' % (content,
                                                          row[u'story'])

                                if u'picture' in row:
                                    content = u'%s %s' % (content,
                                                          row[u'picture'])

                                if u'link' in row:
                                    content = u'%s %s' % (content,
                                                          row[u'link'])

                                fbm = FacebookMessage(
                                    facebook_page=cfp,
                                    facebook_id=row[u'id'],
                                    author_facebook_id=row[u'from'][u'id'],
                                    message=content,
                                    created_time=datetime.datetime.strptime(
                                        row[u'created_time'],
                                        '%Y-%m-%dT%H:%M:%S+0000'),
                                    message_type=row[u'type'],
                                    likes=0,
                                    comments=0,
                                    shares=row[u'shares'][u'count']
                                    if u'shares' in row else 0,
                                )
                            except Exception, e:
                                err = APIError(
                                    app_name=u'metrics_social',
                                    model_name='FacebookMessage',
                                    error=u'%s: %s' % (Exception, str(e)),
                                    response=row,
                                )
                                err.save()
                                self.stdout.write(
                                    u'Inserted error message: %s %s' %
                                    (Exception, str(e)))
                            else:
                                fbm.save()
                                self.stdout.write(u'Inserted message %s' %
                                                  fbm.facebook_id)
                        else:
                            self.stdout.write(u'Message already exists: %s' %
                                              fbm.facebook_id)
                            self.stdout.write(u'Stopping...')
                            main_loop = False
                            break

                    if u'paging' in feed and u'next' in feed[u'paging']:
                        until = urlparse.parse_qs(
                            urlparse.urlparse(
                                feed[u'paging'][u'next']).query)[u'until'][0]
                        self.stdout.write(u'Going to next page \n\n\n')
                    else:
                        main_loop = False
                        cfp.save()
    def handle(self, *args, **options):
        # get some access_token
        try:
            fbc = FacebookConnection.objects.filter(expires__gt=datetime.datetime.now(), deleted=False)[0]
        except IndexError:
            self.stdout.write(u'Error: there is no FacebookConnection')
            fbc = None

        if fbc:

            # api client
            client = FacebookClient(
                client_id=settings.FB_CLIENT_ID, 
                client_secret=settings.FB_CLIENT_SECRET,
                access_token=fbc.access_token,
                )

            if args:
                bfps = BrandFacebookPage.objects.filter(facebook_id__in=args, first_load=False, deleted=False)
            else:
                bfps = BrandFacebookPage.objects.filter(first_load=False, deleted=False)

            error = False
            # do first load
            for bfp in [b for b in bfps]:
                if error:
                    break

                # page info
                try:
                    bfp_info = client.obj_id(bfp.facebook_id)
                except FacebookGenericError:
                    if FacebookGenericError.code == u'17':
                        fbc, access_token = FacebookConnection.renew_client(fbc)
                        client._access_token = access_token
                        bfp_info = client.obj_id(bfp.facebook_id)
                else:
                    if not bfp_info:
                        fbc, access_token = FacebookConnection.renew_client(fbc)
                        client._access_token = access_token
                        bfp_info = client.obj_id(bfp.facebook_id)

                    if not bfp_info:
                        error = True
                        continue
                        #raise CommandError(u'\nThe limit of all robots was reached.')

                bfp_info = client.obj_id(bfp.facebook_id)
                bfp.name = bfp_info[u'name']
                bfp.link = bfp_info[u'link']
                bfp.talking_about = bfp_info[u'talking_about_count']
                bfp.likes = bfp_info[u'likes']

                bfp.save()

                evolution = EvolutionFacebookPageLike(
                    facebook_page=bfp,
                    likes=bfp_info[u'likes'],
                    )
                evolution.save()

                self.stdout.write(u'Successfully updated Brand Facebook Page: %s \n\n' % bfp.facebook_id)

                main_loop = True

                try:
                    until = APIPagination.objects.get(
                        app_name=u'metrics_social',
                        model_name='FacebookMessage',
                        path=u'/%s/feed' % bfp.facebook_id,
                        ).offset
                except APIPagination.DoesNotExist:
                    until = None

                while main_loop:
                    if error:
                        break

                    params = {
                        u'limit': 50,
                        u'format': u'json',
                        u'method': u'GET',
                        }

                    if until:
                        params.update({u'until': until})

                    try:
                        feed = client.feed(bfp.facebook_id, **params)
                    except FacebookGenericError:
                        if FacebookGenericError.code == u'17':
                            fbc, access_token = FacebookConnection.renew_client(fbc)
                            client._access_token = access_token
                            feed = client.feed(bfp.facebook_id, **params) 
                    else:
                        if not feed:
                            fbc, access_token = FacebookConnection.renew_client(fbc)
                            client._access_token = access_token
                            feed = client.feed(bfp.facebook_id, **params)

                    if not feed:
                        error = True
                        continue
                        #raise CommandError(u'\nThe limit of all robots was reached.')

                    if not u'data' in feed or not len(feed[u'data']):
                        main_loop = False
                        bfp.first_load = True
                        bfp.save()
                        continue

                    page_save = PageSave('metrics_social', 'FacebookMessage')
                    page_save.start_save(feed[u'data'], fp=bfp)

                    self.stdout.write(u'Facebook message block was created\n')
                    self.stdout.write(u'\n'.join(page_save.responses))

                    if u'paging' in feed and u'next' in feed[u'paging']:
                        until = urlparse.parse_qs(urlparse.urlparse(feed[u'paging'][u'next']).query)[u'until'][0]
                        self.stdout.write(u'Going to next page \n\n\n')

                        try:
                            stopped_at = APIPagination.objects.get(
                                app_name=u'metrics_social',
                                model_name='FacebookMessage',
                                path=u'/%s/feed' % bfp.facebook_id,
                                )
                            stopped_at.offset = until
                            stopped_at.save()
                        except APIPagination.DoesNotExist:
                            stopped_at = APIPagination(
                                app_name=u'metrics_social',
                                model_name='FacebookMessage',
                                path=u'/%s/feed' % bfp.facebook_id,
                                offset=until,
                                )
                            stopped_at.save()
                    else:
                        main_loop = False
                        bfp.first_load = True
                        bfp.save()
    def handle(self, *args, **options):
        # get some access_token
        try:
            fbc = FacebookConnection.objects.filter(expires__gt=datetime.datetime.now(), deleted=False)[0]
        except IndexError:
            self.stdout.write(u'Error: there is no FacebookConnection')
            fbc = None

        if fbc:
            metrics = [
                u'post_stories',
                u'post_storytellers',
                u'post_impressions',
                u'post_impressions_unique',
                u'post_impressions_paid',
                u'post_impressions_paid_unique',
                u'post_impressions_fan',
                u'post_impressions_fan_unique',
                u'post_impressions_fan_paid',
                u'post_impressions_fan_paid_unique',
                u'post_impressions_organic',
                u'post_impressions_organic_unique',
                u'post_impressions_viral',
                u'post_impressions_viral_unique',
                u'post_consumptions',
                u'post_consumptions_unique',
                u'post_engaged_users',
                u'post_negative_feedback',
                u'post_negative_feedback_unique',
                ]

            fields = OrderedDict([
                (u'facebook_message_id', u'INT'),
                (u'facebook_page_id', u'INT'),
                (u'metric', u'NVARCHAR(255)'),
                (u'period', u'NVARCHAR(255)'),
                (u'value', u'BIGINT'),
                (u'title', u'NVARCHAR(255)'),
                (u'description', u'NVARCHAR(MAX)'),
                ])

            # api client
            client = FacebookClient(
                client_id=settings.FB_CLIENT_ID, 
                client_secret=settings.FB_CLIENT_SECRET,
                access_token=fbc.access_token,
                )

            # do first load
            if args:
                ifps = [i for i in InsightsFacebookPage.objects.filter(
                                        facebook_page__facebook_id__in=args, 
                                        deleted=False)]
            else:
                ifps = [i for i in InsightsFacebookPage.objects.filter(deleted=False)]

            error = False
            for ifp in ifps:
                if error:
                    break

                # page info
                try:
                    ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
                except FacebookGenericError:
                    if FacebookGenericError.code == u'17':
                        fbc, access_token = FacebookConnection.renew_client(fbc)
                        client._access_token = access_token
                        ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
                else:
                    if not ifp_info:
                        fbc, access_token = FacebookConnection.renew_client(fbc)
                        client._access_token = access_token
                        ifp_info = client.obj_id(ifp.facebook_page.facebook_id)

                    if not ifp_info:
                        error = True
                        continue
                        #raise CommandError(u'\nThe limit of all robots was reached.')

                ifp.facebook_page.name = ifp_info[u'name']
                ifp.facebook_page.link = ifp_info[u'link']
                ifp.facebook_page.talking_about = ifp_info[u'talking_about_count']
                ifp.facebook_page.likes = ifp_info[u'likes']

                ifp.facebook_page.save()

                evolution = EvolutionFacebookPageLike(
                    facebook_page=ifp.facebook_page,
                    likes=ifp_info[u'likes'],
                    )
                evolution.save()

                self.stdout.write(u'Successfully updated Facebook Page: %s \n\n' % ifp.facebook_page.facebook_id)

                already_exists = [x for x in InsightsMetric.objects.values_list(
                        'facebook_message_id', 
                        flat=True,
                    ).filter(
                        facebook_page=ifp,
                        metric__in=metrics,
                        dimension=None,
                        deleted=False,
                    ) if x]


                end_date = datetime.date.today()
                end_date = datetime.date.fromordinal(end_date.toordinal() - 2)
                fbmsgs = [y for y in FacebookMessage.objects.filter(
                        facebook_page=ifp.facebook_page,
                        author_facebook_id=ifp.facebook_page.facebook_id,
                        created_time__lte=datetime.datetime.combine(end_date, datetime.time()),
                        deleted=False,
                    ) if y.id not in already_exists or \
                        y.created_time > datetime.datetime.combine(
                            datetime.date.fromordinal(end_date.toordinal() - 3), datetime.time()
                            )]

                for fbmsg in fbmsgs:
                    if error:
                        break

                    fbmsgs_tasks = []
                    for metric in metrics:
                        path = u'/%s/insights/%s/lifetime' % (fbmsg.facebook_id, metric)
                        fbmsgs_tasks.append({
                            u'id': path,
                            u'path': path,
                            u'args': [],
                            u'kwargs': {},
                            })

                    try:
                        me = client.me()
                    except FacebookGenericError:
                        if FacebookGenericError.code == u'17':
                            fbc, access_token = FacebookConnection.renew_client(fbc)
                            client._access_token = access_token
                    else:
                        if not me:
                            fbc, access_token = FacebookConnection.renew_client(fbc)
                            client._access_token = access_token

                    client.start_async_tasks(fbmsgs_tasks, fbc=fbc)

                    if not all([u'response' in task for task in client.tasks]):
                        error = True
                        continue
                        #raise CommandError(u'\nThe limit of all robots was reached.')

                    rows = []
                    for task in client.tasks:
                        if u'response' in task:
                            if not u'data' in task[u'response']:
                                continue
                            for data in task[u'response'][u'data']:
                                for row in data[u'values']:
                                    rows.append((
                                        fbmsg.id,
                                        ifp.id,
                                        str(data[u'name']),
                                        str(data[u'period']),
                                        row[u'value'],
                                        str(data[u'title']),
                                        str(data[u'description'])
                                        ))

                            if not len(task[u'response'][u'data']):
                                rows.append((
                                    fbmsg.id,
                                    ifp.id,
                                    str(task[u'path'].split('/')[3]),
                                    str(u'Not provided'),
                                    0,
                                    str(u'Not provided'),
                                    str(u'Not provided')
                                    ))


                    if rows:
                        bk = BulkInsertOrUpdate(fields=fields)
                        matches = [{
                            u'clause': None,
                            u'first_field': u'facebook_message_id',
                            u'comparison': u'=',
                            u'second_field': u'facebook_message_id'
                            },{
                            u'clause': u'AND',
                            u'first_field': u'facebook_page_id',
                            u'comparison': u'=',
                            u'second_field': u'facebook_page_id'
                            },{
                            u'clause': u'AND',
                            u'first_field': u'metric',
                            u'comparison': u'=',
                            u'second_field': u'metric'
                            },{
                            u'clause': u'AND',
                            u'first_field': u'period',
                            u'comparison': u'=',
                            u'second_field': u'period'
                            },]

                        insert_fields = [
                            u'facebook_message_id',
                            u'facebook_page_id',
                            u'metric',
                            u'period',
                            u'value',
                            u'title',
                            u'description'
                            ]

                        cursor = connection.cursor()
                        cursor.execute(
                            bk.bulk(
                                u'SET QUOTED_IDENTIFIER OFF;',
                                bk.command_bulk_create(),
                                bk.command_bulk_insert(rows),
                                bk.command_bulk_merge(
                                    u'metrics_social_insightsmetric',
                                    matches,
                                    [u'value'],
                                    insert_fields
                                    )
                                )
                            )
                        cursor.execute(bk.command_drop_action_table())
                        transaction.commit_unless_managed()


                    self.stdout.write(u'Facebook insights block was created\n')
    def handle(self, *args, **options):
        # get some access_token
        try:
            fbc = FacebookConnection.objects.filter(expires__gt=datetime.datetime.now(), deleted=False)[0]
        except IndexError:
            self.stdout.write(u'Error: there is no FacebookConnection')
            fbc = None

        if fbc:
            metrics = [
                u'page_stories',
                u'page_storytellers',
                u'page_impressions',
                u'page_impressions_unique',
                u'page_impressions_paid',
                u'page_impressions_paid_unique',
                u'page_impressions_organic',
                u'page_impressions_organic_unique',
                u'page_impressions_viral',
                u'page_impressions_viral_unique',
                u'page_engaged_users',
                u'page_consumptions',
                u'page_consumptions_unique',
                u'page_negative_feedback',
                u'page_negative_feedback_unique',
                u'page_views_unique',
                u'page_views_login',
                u'page_views_login_unique',
                u'page_posts_impressions',
                u'page_posts_impressions_unique',
                u'page_posts_impressions_paid',
                u'page_posts_impressions_paid_unique',
                u'page_posts_impressions_organic',
                u'page_posts_impressions_organic_unique',
                u'page_posts_impressions_viral',
                u'page_posts_impressions_viral_unique',
                ]


            # api client
            client = FacebookClient(
                client_id=settings.FB_CLIENT_ID, 
                client_secret=settings.FB_CLIENT_SECRET,
                access_token=fbc.access_token,
                )

            if args:
                ifps = [i for i in InsightsFacebookPage.objects.filter(
                                        facebook_page__facebook_id__in=args, 
                                        first_load_days_28=True, 
                                        deleted=False)]
            else:
                ifps = [i for i in InsightsFacebookPage.objects.filter(first_load_days_28=True, deleted=False)]

            error = False
            # do first load
            for ifp in ifps:
                if error:
                    break

                # page info
                try:
                    ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
                except FacebookGenericError:
                    if FacebookGenericError.code == u'17':
                        fbc, access_token = FacebookConnection.renew_client(fbc)
                        client._access_token = access_token
                        ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
                else:
                    if not ifp_info:
                        fbc, access_token = FacebookConnection.renew_client(fbc)
                        client._access_token = access_token
                        ifp_info = client.obj_id(ifp.facebook_page.facebook_id)

                    if not ifp_info:
                        error = True
                        continue
                        #raise CommandError(u'\nThe limit of all robots was reached.')

                ifp.facebook_page.name = ifp_info[u'name']
                ifp.facebook_page.link = ifp_info[u'link']
                ifp.facebook_page.talking_about = ifp_info[u'talking_about_count']
                ifp.facebook_page.likes = ifp_info[u'likes']

                ifp.facebook_page.save()

                evolution = EvolutionFacebookPageLike(
                    facebook_page=ifp.facebook_page,
                    likes=ifp_info[u'likes'],
                    )
                evolution.save()

                self.stdout.write(u'Successfully updated Facebook Page: %s \n\n' % ifp.facebook_page.facebook_id)

                main_loop = True
                lap = 0

                until = datetime.date.today()
                until = until + relativedelta(days=-2)
                since = until + relativedelta(days=-(until.day-1))
                
                since = int(time.mktime(since.timetuple()))
                until = int(time.mktime(until.timetuple()))

                while main_loop:
                    if error:
                        break

                    insights_tasks = []
                    for metric in metrics:

                        path = u'/%s/insights/%s/days_28' % (ifp.facebook_page.facebook_id, metric)

                        insights_tasks.append({
                            u'id': path,
                            u'path': path,
                            u'args': [],
                            u'kwargs': {'since': since, 'until': until},
                            })

                    try:
                        me = client.me()
                    except FacebookGenericError:
                        if FacebookGenericError.code == u'17':
                            fbc, access_token = FacebookConnection.renew_client(fbc)
                            client._access_token = access_token
                    else:
                        if not me:
                            fbc, access_token = FacebookConnection.renew_client(fbc)
                            client._access_token = access_token
                            me = client.me()

                        if not me:
                            error = True
                            continue
                            #raise CommandError(u'\nThe limit of all robots was reached.')

                    client.start_async_tasks(insights_tasks, fbc=fbc)

                    if not all([u'response' in task for task in client.tasks]):
                        error = True
                        continue
                        #raise CommandError(u'\nThe limit of all robots was reached.')

                    for task in client.tasks:
                        if u'response' in task:

                            if not u'data' in task[u'response']:
                                continue

                            for data in task[u'response'][u'data']:
                                max_days = 0
                                for row in reversed(data[u'values']):
                                    day = datetime.datetime.strptime(row[u'end_time'][:10], '%Y-%m-%d').date()
                                    day = day + relativedelta(days=-1)
                                   
                                    insights_metric, created = InsightsMetric.objects.get_or_create(
                                        facebook_page=ifp, 
                                        day=day, 
                                        title=data[u'title'],
                                        description=data[u'description'],
                                        metric=data[u'name'],
                                        period=data[u'period'],
                                        deleted=False,
                                        defaults={
                                            'value': row[u'value'],
                                            },
                                        )

                                    if not created:
                                        insights_metric.value = row[u'value']
                                        max_days += 1

                                    insights_metric.save()

                                    if not max_days < 10:
                                        main_loop = False
                                        break

                    self.stdout.write(u'Facebook insights block was created\n')
                    responses = [task[u'response'][u'data'][0] for task in client.tasks if len(task[u'response'][u'data'])]

                    if sum([v[u'value'] for r in responses for v in r[u'values']]):
                        until = datetime.datetime.fromtimestamp(since).date()
                        since = until + relativedelta(months=-1, day=1)
                        until = since + relativedelta(months=+1)

                        since = int(time.mktime(since.timetuple()))
                        until = int(time.mktime(until.timetuple()))

                        self.stdout.write(u'Going to next page \n\n\n')

                    else:
                        lap += 1

                        until = datetime.datetime.fromtimestamp(since).date()
                        since = until + relativedelta(months=-1, day=1)
                        until = since + relativedelta(months=+1)
                        
                        since = int(time.mktime(since.timetuple()))
                        until = int(time.mktime(until.timetuple()))

                    if lap > 2:
                        lap = 0
                        main_loop = False
    def handle(self, *args, **options):
        # get some access_token
        try:
            fbc = FacebookConnection.objects.filter(expires__gt=datetime.datetime.now(), deleted=False)[0]
        except IndexError:
            self.stdout.write(u'Error: there is no FacebookConnection')
            fbc = None

        if fbc:

            # api client
            client = FacebookClient(
                client_id=settings.FB_CLIENT_ID, 
                client_secret=settings.FB_CLIENT_SECRET,
                access_token=fbc.access_token,
                )

            if args:
                cfps = CampaignFacebookPage.objects.filter(facebook_id__in=args, deleted=False)
            else:
                cfps = CampaignFacebookPage.objects.filter(deleted=False)

            error = False
            # do daily load
            for cfp in [c for c in cfps]:
                if error:
                    break

                # page info
                try:
                    cfp_info = client.obj_id(cfp.facebook_id)
                except FacebookGenericError:
                    if FacebookGenericError.code == u'17':
                        fbc, access_token = FacebookConnection.renew_client(fbc)
                        client._access_token = access_token
                        cfp_info = client.obj_id(cfp.facebook_id)
                else:
                    if not cfp_info:
                        fbc, access_token = FacebookConnection.renew_client(fbc)
                        client._access_token = access_token
                        cfp_info = client.obj_id(cfp.facebook_id)

                    if not cfp_info:
                        error = True
                        continue
                        #raise CommandError(u'\nThe limit of all robots was reached.')

                cfp.name = cfp_info[u'name']
                cfp.link = cfp_info[u'link']
                cfp.talking_about = cfp_info[u'talking_about_count']
                cfp.likes = cfp_info[u'likes']

                cfp.save()

                evolution = EvolutionFacebookPageLike(
                    facebook_page=cfp,
                    likes=cfp_info[u'likes'],
                    )
                evolution.save()

                self.stdout.write(u'Successfully updated Campaign Facebook Page: %s \n\n' % cfp.facebook_id)

                main_loop = True; until = None
                while main_loop:
                    if error:
                        break

                    params = {u'limit': 50}
                    if until:
                        params.update({u'until': until})

                    # load feed
                    try:
                        feed = client.feed(cfp.facebook_id, **params)
                    except FacebookGenericError:
                        if FacebookGenericError.code == u'17':
                            fbc, access_token = FacebookConnection.renew_client(fbc)
                            client._access_token = access_token
                            feed = client.feed(cfp.facebook_id, **params)
                    else:
                        if not feed:
                            fbc, access_token = FacebookConnection.renew_client(fbc)
                            client._access_token = access_token
                            feed = client.feed(cfp.facebook_id, **params)

                    if not feed:
                        error = True
                        continue
                        #raise CommandError(u'\nThe limit of all robots was reached.')

                    if not u'data' in feed or not len(feed[u'data']):
                        main_loop = False
                        cfp.save()
                        continue

                    for row in feed[u'data']:
                        try:
                            fbm = FacebookMessage.objects.get(facebook_id__exact=row[u'id'], deleted=False)
                        except FacebookMessage.DoesNotExist:
                            try:
                                if u'likes' in row and u'count' in row[u'likes']:
                                    likes = row[u'likes'][u'count']
                                else:
                                    likes = 0

                                if u'comments' in row and u'count' in row[u'comments']:
                                    comments = row[u'comments'][u'count']
                                else:
                                    comments = 0

                                content = u''
                                if u'message' in row:
                                    content = u'%s' % row[u'message']

                                if u'story' in row:
                                    content = u'%s %s' % (content, row[u'story'])

                                if u'picture' in row:
                                    content = u'%s %s' % (content, row[u'picture'])

                                if u'link' in row:
                                    content = u'%s %s' % (content, row[u'link'])

                                fbm = FacebookMessage(
                                    facebook_page=cfp,
                                    facebook_id=row[u'id'],
                                    author_facebook_id=row[u'from'][u'id'],
                                    message=content,
                                    created_time=datetime.datetime.strptime(row[u'created_time'], '%Y-%m-%dT%H:%M:%S+0000'),
                                    message_type=row[u'type'],
                                    likes=0,
                                    comments=0,
                                    shares=row[u'shares'][u'count'] if u'shares' in row else 0,
                                    )
                            except Exception, e:
                                err = APIError(
                                    app_name=u'metrics_social',
                                    model_name='FacebookMessage',
                                    error=u'%s: %s' % (Exception, str(e)),
                                    response=row,
                                    )
                                err.save()
                                self.stdout.write(u'Inserted error message: %s %s' % (Exception, str(e)))
                            else:
                                fbm.save()
                                self.stdout.write(u'Inserted message %s' % fbm.facebook_id)
                        else:
                            self.stdout.write(u'Message already exists: %s' % fbm.facebook_id)
                            self.stdout.write(u'Stopping...')
                            main_loop = False
                            break

                    if u'paging' in feed and u'next' in feed[u'paging']:
                        until = urlparse.parse_qs(urlparse.urlparse(feed[u'paging'][u'next']).query)[u'until'][0]
                        self.stdout.write(u'Going to next page \n\n\n')
                    else:
                        main_loop = False
                        cfp.save()
예제 #7
0
    def handle(self, *args, **options):
        # get some access_token
        try:
            fbc = FacebookConnection.objects.filter(
                expires__gt=datetime.datetime.now(), deleted=False)[0]
        except IndexError:
            self.stdout.write(u'Error: there is no FacebookConnection')
            fbc = None

        if fbc:
            metrics = [
                u'post_stories',
                u'post_storytellers',
                u'post_impressions',
                u'post_impressions_unique',
                u'post_impressions_paid',
                u'post_impressions_paid_unique',
                u'post_impressions_fan',
                u'post_impressions_fan_unique',
                u'post_impressions_fan_paid',
                u'post_impressions_fan_paid_unique',
                u'post_impressions_organic',
                u'post_impressions_organic_unique',
                u'post_impressions_viral',
                u'post_impressions_viral_unique',
                u'post_consumptions',
                u'post_consumptions_unique',
                u'post_engaged_users',
                u'post_negative_feedback',
                u'post_negative_feedback_unique',
            ]

            fields = OrderedDict([
                (u'facebook_message_id', u'INT'),
                (u'facebook_page_id', u'INT'),
                (u'metric', u'NVARCHAR(255)'),
                (u'period', u'NVARCHAR(255)'),
                (u'value', u'BIGINT'),
                (u'title', u'NVARCHAR(255)'),
                (u'description', u'NVARCHAR(MAX)'),
            ])

            # api client
            client = FacebookClient(
                client_id=settings.FB_CLIENT_ID,
                client_secret=settings.FB_CLIENT_SECRET,
                access_token=fbc.access_token,
            )

            # do first load
            if args:
                ifps = [
                    i for i in FacebookInsightsReload.objects.filter(
                        facebook_page__facebook_id__in=args,
                        reload_post_level=False,
                        deleted=False)
                ]
            else:
                ifps = [
                    i for i in FacebookInsightsReload.objects.filter(
                        reload_post_level=False, deleted=False)
                ]

            error = False
            for ifp in ifps:
                if error:
                    break

                # page info
                try:
                    ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
                except FacebookGenericError:
                    if FacebookGenericError.code == u'17':
                        fbc, access_token = FacebookConnection.renew_client(
                            fbc)
                        client._access_token = access_token
                        ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
                else:
                    if not ifp_info:
                        fbc, access_token = FacebookConnection.renew_client(
                            fbc)
                        client._access_token = access_token
                        ifp_info = client.obj_id(ifp.facebook_page.facebook_id)

                    if not ifp_info:
                        error = True
                        continue
                        #raise CommandError(u'\nThe limit of all robots was reached.')

                ifp.facebook_page.name = ifp_info[u'name']
                ifp.facebook_page.link = ifp_info[u'link']
                ifp.facebook_page.talking_about = ifp_info[
                    u'talking_about_count']
                ifp.facebook_page.likes = ifp_info[u'likes']

                ifp.facebook_page.save()

                evolution = EvolutionFacebookPageLike(
                    facebook_page=ifp.facebook_page,
                    likes=ifp_info[u'likes'],
                )
                evolution.save()

                self.stdout.write(
                    u'Successfully updated Facebook Page: %s \n\n' %
                    ifp.facebook_page.facebook_id)

                fbmsgs = [
                    y for y in FacebookMessage.objects.filter(
                        facebook_page=ifp.facebook_page,
                        author_facebook_id=ifp.facebook_page.facebook_id,
                        created_time__lte=datetime.datetime.combine(
                            ifp.until, datetime.time()),
                        created_time__gte=datetime.datetime.combine(
                            ifp.since, datetime.time()),
                        deleted=False,
                    ) if y.created_time > datetime.datetime.combine(
                        datetime.date.fromordinal(
                            datetime.date.today().toordinal() -
                            3), datetime.time())
                ]

                for fbmsg in fbmsgs:
                    if error:
                        break

                    fbmsgs_tasks = []
                    for metric in metrics:
                        path = u'/%s/insights/%s/lifetime' % (
                            fbmsg.facebook_id, metric)
                        fbmsgs_tasks.append({
                            u'id': path,
                            u'path': path,
                            u'args': [],
                            u'kwargs': {},
                        })

                    try:
                        me = client.me()
                    except FacebookGenericError:
                        if FacebookGenericError.code == u'17':
                            fbc, access_token = FacebookConnection.renew_client(
                                fbc)
                            client._access_token = access_token
                    else:
                        if not me:
                            fbc, access_token = FacebookConnection.renew_client(
                                fbc)
                            client._access_token = access_token

                    client.start_async_tasks(fbmsgs_tasks, fbc=fbc)

                    if not all([u'response' in task for task in client.tasks]):
                        error = True
                        continue
                        #raise CommandError(u'\nThe limit of all robots was reached.')

                    rows = []
                    for task in client.tasks:
                        if u'response' in task:
                            if not u'data' in task[u'response']:
                                continue
                            for data in task[u'response'][u'data']:
                                for row in data[u'values']:
                                    rows.append(
                                        (fbmsg.id, ifp.id, str(data[u'name']),
                                         str(data[u'period']), row[u'value'],
                                         str(data[u'title']),
                                         str(data[u'description'])))

                            if not len(task[u'response'][u'data']):
                                rows.append((fbmsg.id, ifp.id,
                                             str(task[u'path'].split('/')[3]),
                                             str(u'Not provided'), 0,
                                             str(u'Not provided'),
                                             str(u'Not provided')))

                    if rows:
                        bk = BulkInsertOrUpdate(fields=fields)
                        matches = [
                            {
                                u'clause': None,
                                u'first_field': u'facebook_message_id',
                                u'comparison': u'=',
                                u'second_field': u'facebook_message_id'
                            },
                            {
                                u'clause': u'AND',
                                u'first_field': u'facebook_page_id',
                                u'comparison': u'=',
                                u'second_field': u'facebook_page_id'
                            },
                            {
                                u'clause': u'AND',
                                u'first_field': u'metric',
                                u'comparison': u'=',
                                u'second_field': u'metric'
                            },
                            {
                                u'clause': u'AND',
                                u'first_field': u'period',
                                u'comparison': u'=',
                                u'second_field': u'period'
                            },
                        ]

                        insert_fields = [
                            u'facebook_message_id', u'facebook_page_id',
                            u'metric', u'period', u'value', u'title',
                            u'description'
                        ]

                        cursor = connection.cursor()
                        cursor.execute(
                            bk.bulk(
                                u'SET QUOTED_IDENTIFIER OFF;',
                                bk.command_bulk_create(),
                                bk.command_bulk_insert(rows),
                                bk.command_bulk_merge(
                                    u'metrics_social_insightsmetric', matches,
                                    [u'value'], insert_fields)))
                        cursor.execute(bk.command_drop_action_table())
                        transaction.commit_unless_managed()

                    ifp.reload_post_level = True
                    ifp.save()

                    self.stdout.write(u'Facebook insights block was created\n')
예제 #8
0
    def handle(self, *args, **options):
        # get some access_token
        try:
            fbc = FacebookConnection.objects.filter(
                expires__gt=datetime.datetime.now(), deleted=False)[0]
        except IndexError:
            self.stdout.write(u'Error: there is no FacebookConnection')
            fbc = None

        if fbc:
            metrics = [
                u'page_stories',
                u'page_storytellers',
                u'page_impressions',
                u'page_impressions_unique',
                u'page_impressions_paid',
                u'page_impressions_paid_unique',
                u'page_impressions_organic',
                u'page_impressions_organic_unique',
                u'page_impressions_viral',
                u'page_impressions_viral_unique',
                u'page_engaged_users',
                u'page_negative_feedback',
                u'page_negative_feedback_unique',
                u'page_fans',
                u'page_fans_online_per_day',
                u'page_fan_adds',
                u'page_fan_removes',
                u'page_consumptions',
                u'page_consumptions_unique',
                u'page_views',
                u'page_views_unique',
                u'page_views_login',
                u'page_views_login_unique',
                u'page_posts_impressions',
                u'page_posts_impressions_unique',
                u'page_posts_impressions_paid',
                u'page_posts_impressions_paid_unique',
                u'page_posts_impressions_organic',
                u'page_posts_impressions_organic_unique',
                u'page_posts_impressions_viral',
                u'page_posts_impressions_viral_unique',
            ]

            # api client
            client = FacebookClient(
                client_id=settings.FB_CLIENT_ID,
                client_secret=settings.FB_CLIENT_SECRET,
                access_token=fbc.access_token,
            )

            if args:
                ifps = [
                    i for i in InsightsFacebookPage.objects.filter(
                        facebook_page__facebook_id__in=args,
                        first_load_daily=True,
                        deleted=False)
                ]
            else:
                ifps = [
                    i for i in InsightsFacebookPage.objects.filter(
                        first_load_daily=True, deleted=False)
                ]

            error = False
            # do first load
            for ifp in ifps:
                if error:
                    break

                # page info
                try:
                    ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
                except FacebookGenericError:
                    if FacebookGenericError.code == u'17':
                        fbc, access_token = FacebookConnection.renew_client(
                            fbc)
                        client._access_token = access_token
                        ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
                else:
                    if not ifp_info:
                        fbc, access_token = FacebookConnection.renew_client(
                            fbc)
                        client._access_token = access_token
                        ifp_info = client.obj_id(ifp.facebook_page.facebook_id)

                    if not ifp_info:
                        error = True
                        continue
                        #raise CommandError(u'\nThe limit of all robots was reached.')

                ifp.facebook_page.name = ifp_info[u'name']
                ifp.facebook_page.link = ifp_info[u'link']
                ifp.facebook_page.talking_about = ifp_info[
                    u'talking_about_count']
                ifp.facebook_page.likes = ifp_info[u'likes']

                ifp.facebook_page.save()

                evolution = EvolutionFacebookPageLike(
                    facebook_page=ifp.facebook_page,
                    likes=ifp_info[u'likes'],
                )
                evolution.save()

                self.stdout.write(
                    u'Successfully updated Facebook Page: %s \n\n' %
                    ifp.facebook_page.facebook_id)

                main_loop = True
                lap = 0

                until = datetime.date.today()
                until = until + relativedelta(days=-2)
                since = until + relativedelta(days=-(until.day - 1))

                since = int(time.mktime(since.timetuple()))
                until = int(time.mktime(until.timetuple()))

                while main_loop:
                    if error:
                        break

                    insights_tasks = []
                    for metric in metrics:

                        if metric == u'page_fans':
                            path = u'/%s/insights/%s' % (
                                ifp.facebook_page.facebook_id, metric)
                        else:
                            path = u'/%s/insights/%s/day' % (
                                ifp.facebook_page.facebook_id, metric)

                        insights_tasks.append({
                            u'id': path,
                            u'path': path,
                            u'args': [],
                            u'kwargs': {
                                'since': since,
                                'until': until
                            },
                        })

                    try:
                        me = client.me()
                    except FacebookGenericError:
                        if FacebookGenericError.code == u'17':
                            fbc, access_token = FacebookConnection.renew_client(
                                fbc)
                            client._access_token = access_token
                    else:
                        if not me:
                            fbc, access_token = FacebookConnection.renew_client(
                                fbc)
                            client._access_token = access_token
                            me = client.me()

                        if not me:
                            error = True
                            continue
                            #raise CommandError(u'\nThe limit of all robots was reached.')

                    client.start_async_tasks(insights_tasks, fbc=fbc)

                    if not all([u'response' in task for task in client.tasks]):
                        error = True
                        continue
                        #raise CommandError(u'\nThe limit of all robots was reached.')

                    for task in client.tasks:
                        if u'response' in task:

                            if not u'data' in task[u'response']:
                                continue

                            for data in task[u'response'][u'data']:
                                max_days = 0
                                for row in data[u'values']:
                                    day = datetime.datetime.strptime(
                                        row[u'end_time'][:10],
                                        '%Y-%m-%d').date()
                                    day = datetime.date.fromordinal(
                                        day.toordinal() - 1)

                                    insights_metric, created = InsightsMetric.objects.get_or_create(
                                        facebook_page=ifp,
                                        day=day,
                                        title=data[u'title'],
                                        description=data[u'description'],
                                        metric=data[u'name'],
                                        period=data[u'period'],
                                        deleted=False,
                                        defaults={
                                            'value': row[u'value'],
                                        },
                                    )

                                    if not created:
                                        insights_metric.value = row[u'value']
                                        max_days += 1

                                    insights_metric.save()

                                    if not max_days < 10:
                                        main_loop = False
                                        break

                    self.stdout.write(u'Facebook insights block was created\n')
                    responses = [
                        task[u'response'][u'data'][0] for task in client.tasks
                        if len(task[u'response'][u'data'])
                    ]

                    if sum(
                        [v[u'value'] for r in responses
                         for v in r[u'values']]):
                        until = datetime.datetime.fromtimestamp(since).date()
                        since = datetime.date.fromordinal(until.toordinal() -
                                                          until.day)

                        since = int(time.mktime(since.timetuple()))
                        until = int(time.mktime(until.timetuple()))

                        self.stdout.write(u'Going to next page \n\n\n')

                    else:
                        lap += 1

                        until = datetime.datetime.fromtimestamp(since).date()
                        since = datetime.date.fromordinal(until.toordinal() -
                                                          until.day)

                        since = int(time.mktime(since.timetuple()))
                        until = int(time.mktime(until.timetuple()))

                    if lap > 2:
                        lap = 0
                        main_loop = False
    def handle(self, *args, **options):
        # get some access_token
        try:
            fbc = FacebookConnection.objects.filter(expires__gt=datetime.datetime.now(), deleted=False)[0]
        except IndexError:
            self.stdout.write(u'Error: there is no FacebookConnection')
            fbc = None

        if fbc:
            metrics = [
                u'page_stories',
                u'page_storytellers',
                u'page_impressions',
                u'page_impressions_unique',
                u'page_impressions_paid',
                u'page_impressions_paid_unique',
                u'page_impressions_organic',
                u'page_impressions_organic_unique',
                u'page_impressions_viral',
                u'page_impressions_viral_unique',
                u'page_engaged_users',
                u'page_consumptions',
                u'page_consumptions_unique',
                u'page_negative_feedback',
                u'page_negative_feedback_unique',
                u'page_views_unique',
                u'page_views_login',
                u'page_views_login_unique',
                u'page_posts_impressions',
                u'page_posts_impressions_unique',
                u'page_posts_impressions_paid',
                u'page_posts_impressions_paid_unique',
                u'page_posts_impressions_organic',
                u'page_posts_impressions_organic_unique',
                u'page_posts_impressions_viral',
                u'page_posts_impressions_viral_unique',
                ]

            fields = OrderedDict([
                (u'facebook_page_id', u'INT'),
                (u'day', u'DATETIME'),
                (u'metric', u'NVARCHAR(255)'),
                (u'period', u'NVARCHAR(255)'),
                (u'value', u'BIGINT'),
                (u'title', u'NVARCHAR(255)'),
                (u'description', u'NVARCHAR(MAX)'),
                ])


            # api client
            client = FacebookClient(
                client_id=settings.FB_CLIENT_ID, 
                client_secret=settings.FB_CLIENT_SECRET,
                access_token=fbc.access_token,
                )

            if args:
                ifps = [i for i in FacebookInsightsReload.objects.filter(
                                        facebook_page__facebook_id__in=args, 
                                        reload_weekly=False, 
                                        deleted=False)]
            else:
                ifps = [i for i in FacebookInsightsReload.objects.filter(reload_weekly=False, deleted=False)]

            error = False
            # do first load
            for ifp in ifps:
                if error:
                    break

                # page info
                try:
                    ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
                except FacebookGenericError:
                    if FacebookGenericError.code == u'17':
                        fbc, access_token = FacebookConnection.renew_client(fbc)
                        client._access_token = access_token
                        ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
                else:
                    if not ifp_info:
                        fbc, access_token = FacebookConnection.renew_client(fbc)
                        client._access_token = access_token
                        ifp_info = client.obj_id(ifp.facebook_page.facebook_id)

                    if not ifp_info:
                        error = True
                        continue
                        #raise CommandError(u'\nThe limit of all robots was reached.')

                ifp.facebook_page.name = ifp_info[u'name']
                ifp.facebook_page.link = ifp_info[u'link']
                ifp.facebook_page.talking_about = ifp_info[u'talking_about_count']
                ifp.facebook_page.likes = ifp_info[u'likes']

                ifp.facebook_page.save()

                evolution = EvolutionFacebookPageLike(
                    facebook_page=ifp.facebook_page,
                    likes=ifp_info[u'likes'],
                    )
                evolution.save()

                self.stdout.write(u'Successfully updated Facebook Page: %s \n\n' % ifp.facebook_page.facebook_id)

                main_loop = True

                until = ifp.until.date()
                since = until + relativedelta(days=-(until.day-1))

                since = int(time.mktime(since.timetuple()))
                until = int(time.mktime(until.timetuple()))

                while main_loop:
                    if error:
                        break

                    insights_tasks = []
                    for metric in metrics:

                        path = u'/%s/insights/%s/week' % (ifp.facebook_page.facebook_id, metric)

                        insights_tasks.append({
                            u'id': path,
                            u'path': path,
                            u'args': [],
                            u'kwargs': {'since': since, 'until': until},
                            })

                    try:
                        me = client.me()
                    except FacebookGenericError:
                        if FacebookGenericError.code == u'17':
                            fbc, access_token = FacebookConnection.renew_client(fbc)
                            client._access_token = access_token
                    else:
                        if not me:
                            fbc, access_token = FacebookConnection.renew_client(fbc)
                            client._access_token = access_token
                            me = client.me()

                        if not me:
                            error = True
                            continue
                            #raise CommandError(u'\nThe limit of all robots was reached.')

                    client.start_async_tasks(insights_tasks, fbc=fbc)

                    if not all([u'response' in task for task in client.tasks]):
                        error = True
                        continue
                        #raise CommandError(u'\nThe limit of all robots was reached.')

                    rows = []
                    for task in client.tasks:
                        if u'response' in task:
                            if not u'data' in task[u'response']:
                                continue
                            for data in task[u'response'][u'data']:
                                for row in reversed(data[u'values']):
                                    day = datetime.datetime.strptime(row[u'end_time'][:10], '%Y-%m-%d').date()
                                    day = day + relativedelta(days=-1)

                                    rows.append((
                                        ifp.id,
                                        str(unicode(day)),
                                        str(data[u'name']),
                                        str(data[u'period']),
                                        row[u'value'],
                                        str(data[u'title']),
                                        str(data[u'description'])
                                        ))


                    if rows:
                        bk = BulkInsertOrUpdate(fields=fields)
                        matches = [{
                            u'clause': None,
                            u'first_field': u'facebook_page_id',
                            u'comparison': u'=',
                            u'second_field': u'facebook_page_id'
                            },{
                            u'clause': 'AND',
                            u'first_field': u'day',
                            u'comparison': u'=',
                            u'second_field': u'day'
                            },{
                            u'clause': 'AND',
                            u'first_field': u'metric',
                            u'comparison': u'=',
                            u'second_field': u'metric'
                            },{
                            u'clause': 'AND',
                            u'first_field': u'period',
                            u'comparison': u'=',
                            u'second_field': u'period'
                            },]

                        insert_fields = [
                            u'facebook_page_id',
                            u'day',
                            u'metric',
                            u'period',
                            u'value',
                            u'title',
                            u'description'
                            ]

                        cursor = connection.cursor()
                        cursor.execute(
                            bk.bulk(
                                u'SET QUOTED_IDENTIFIER OFF;',
                                bk.command_bulk_create(),
                                bk.command_bulk_insert(rows),
                                bk.command_bulk_merge(
                                    u'metrics_social_insightsmetric',
                                    matches,
                                    [u'value'],
                                    insert_fields
                                    )
                                )
                            )
                        cursor.execute(bk.command_drop_action_table())
                        transaction.commit_unless_managed()


                    self.stdout.write(u'Facebook insights block was created\n')
                    responses = [task[u'response'][u'data'][0] for task in client.tasks if len(task[u'response'][u'data'])]

                    if datetime.datetime.fromtimestamp(since).date() <= ifp.since.date():
                        main_loop = False
                        ifp.reload_weekly = True
                        ifp.save()
                    else:
                        until = datetime.datetime.fromtimestamp(since).date()
                        since = until + relativedelta(months=-1, day=1)
                        until = since + relativedelta(months=+1)
                        
                        since = int(time.mktime(since.timetuple()))
                        until = int(time.mktime(until.timetuple()))

                        self.stdout.write(u'Going to next page \n\n\n')
예제 #10
0
    def handle(self, *args, **options):
        # get some access_token
        try:
            fbc = FacebookConnection.objects.filter(expires__gt=datetime.datetime.now(), deleted=False)[0]
        except IndexError:
            self.stdout.write(u'Error: there is no FacebookConnection')
            fbc = None

        if fbc:
            metrics = [
                u'page_fans_by_like_source_unique',
                u'page_stories_by_story_type',
                u'page_fans_gender_age', # lifetime
                u'page_tab_views_login_top_unique',
                u'page_negative_feedback_by_type',
                u'page_negative_feedback_by_type_unique',
                u'page_fans_online',
                ]

            fields = OrderedDict([
                (u'facebook_page_id', u'INT'),
                (u'day', u'DATETIME'),
                (u'metric', u'NVARCHAR(255)'),
                (u'period', u'NVARCHAR(255)'),
                (u'value', u'BIGINT'),
                (u'dimension', u'NVARCHAR(255)'),
                (u'title', u'NVARCHAR(255)'),
                (u'description', u'NVARCHAR(MAX)'),
                ])

            # api client
            client = FacebookClient(
                client_id=settings.FB_CLIENT_ID, 
                client_secret=settings.FB_CLIENT_SECRET,
                access_token=fbc.access_token,
                )

            if args:
                ifps = [i for i in InsightsFacebookPage.objects.filter(
                                        facebook_page__facebook_id__in=args, 
                                        first_load_dimensions=False, 
                                        deleted=False)]
            else:
                ifps = [i for i in InsightsFacebookPage.objects.filter(first_load_dimensions=False, deleted=False)]

            error = False
            # do first load
            for ifp in ifps:
                if error:
                    break

                # page info
                try:
                    ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
                except FacebookGenericError:
                    if FacebookGenericError.code == u'17':
                        fbc, access_token = FacebookConnection.renew_client(fbc)
                        client._access_token = access_token
                        ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
                else:
                    if not ifp_info:
                        fbc, access_token = FacebookConnection.renew_client(fbc)
                        client._access_token = access_token
                        ifp_info = client.obj_id(ifp.facebook_page.facebook_id)

                    if not ifp_info:
                        error = True
                        continue
                        #raise CommandError(u'\nThe limit of all robots was reached.')

                ifp.facebook_page.name = ifp_info[u'name']
                ifp.facebook_page.link = ifp_info[u'link']
                ifp.facebook_page.talking_about = ifp_info[u'talking_about_count']
                ifp.facebook_page.likes = ifp_info[u'likes']

                ifp.facebook_page.save()

                evolution = EvolutionFacebookPageLike(
                    facebook_page=ifp.facebook_page,
                    likes=ifp_info[u'likes'],
                    )
                evolution.save()

                self.stdout.write(u'Successfully updated Facebook Page: %s \n\n' % ifp.facebook_page.facebook_id)

                main_loop = True
                lap = 0

                try:
                    since = APIPagination.objects.get(
                        app_name=u'metrics_social',
                        model_name='InsightsMetric',
                        path=u'/%s/insights/page_fans_by_like_source_unique' % ifp.facebook_page.facebook_id,
                        ).offset

                    since = int(since)

                    today = datetime.date.today()

                    until = datetime.datetime.fromtimestamp(since).date()

                    if until.month == today.month and until.year == today.year:
                        until = until + relativedelta(days=+(today.day-2))
                        until = int(time.mktime(until.timetuple()))
                    else:
                        until = until + relativedelta(months=+1)
                        until = until + relativedelta(days=-1)
                        until = int(time.mktime(until.timetuple()))

                except APIPagination.DoesNotExist:
                    until = datetime.date.today()
                    until = until + relativedelta(days=-2)
                    since = until + relativedelta(days=-(until.day-1))

                    since = int(time.mktime(since.timetuple()))
                    until = int(time.mktime(until.timetuple()))

                while main_loop:
                    if error:
                        break

                    insights_tasks = []
                    for metric in metrics:
                        path = u'/%s/insights/%s' % (ifp.facebook_page.facebook_id, metric)

                        insights_tasks.append({
                            u'id': path,
                            u'path': path,
                            u'args': [],
                            u'kwargs': {'since': since, 'until': until},
                            })

                    try:
                        me = client.me()
                    except FacebookGenericError:
                        if FacebookGenericError.code == u'17':
                            fbc, access_token = FacebookConnection.renew_client(fbc)
                            client._access_token = access_token
                    else:
                        if not me:
                            fbc, access_token = FacebookConnection.renew_client(fbc)
                            client._access_token = access_token
                            me = client.me()

                        if not me:
                            error = True
                            continue
                            #raise CommandError(u'\nThe limit of all robots was reached.')

                    client.start_async_tasks(insights_tasks, fbc=fbc)

                    if not all([u'response' in task for task in client.tasks]):
                        error = True
                        continue
                        #raise CommandError(u'\nThe limit of all robots was reached.')

                    rows = []
                    for task in client.tasks:
                        if u'response' in task:
                            if not u'data' in task[u'response']:
                                continue
                            for data in task[u'response'][u'data']:
                                for row in reversed(data[u'values']):
                                    day = datetime.datetime.strptime(row[u'end_time'][:10], '%Y-%m-%d').date()
                                    day = day + relativedelta(days=-1)

                                    if type(row[u'value']) is dict:
                                        for k, v in row[u'value'].iteritems():
                                            rows.append((
                                                ifp.id,
                                                str(unicode(day)),
                                                str(data[u'name']),
                                                str(data[u'period']),
                                                v,
                                                str(k),
                                                str(data[u'title']),
                                                str(data[u'description'])
                                                ))


                    if rows:
                        bk = BulkInsertOrUpdate(fields=fields)
                        matches = [{
                            u'clause': None,
                            u'first_field': u'facebook_page_id',
                            u'comparison': u'=',
                            u'second_field': u'facebook_page_id'
                            },{
                            u'clause': 'AND',
                            u'first_field': u'day',
                            u'comparison': u'=',
                            u'second_field': u'day'
                            },{
                            u'clause': 'AND',
                            u'first_field': u'metric',
                            u'comparison': u'=',
                            u'second_field': u'metric'
                            },{
                            u'clause': 'AND',
                            u'first_field': u'period',
                            u'comparison': u'=',
                            u'second_field': u'period'
                            },{
                            u'clause': 'AND',
                            u'first_field': u'dimension',
                            u'comparison': u'=',
                            u'second_field': u'dimension'
                            },]

                        insert_fields = [
                            u'facebook_page_id',
                            u'day',
                            u'metric',
                            u'period',
                            u'value',
                            u'dimension',
                            u'title',
                            u'description'
                            ]

                        cursor = connection.cursor()
                        cursor.execute(
                            bk.bulk(
                                u'SET QUOTED_IDENTIFIER OFF;',
                                bk.command_bulk_create(),
                                bk.command_bulk_insert(rows),
                                bk.command_bulk_merge(
                                    u'metrics_social_insightsmetric',
                                    matches,
                                    [u'value'],
                                    insert_fields
                                    )
                                )
                            )
                        cursor.execute(bk.command_drop_action_table())
                        transaction.commit_unless_managed()


                    self.stdout.write(u'Facebook insights block was created\n')
                    responses = [task[u'response'][u'data'][0] for task in client.tasks if len(task[u'response'][u'data'])]

                    if sum([len(v[u'value']) for r in responses for v in r[u'values']]):
                        until = datetime.datetime.fromtimestamp(since).date()
                        since = until + relativedelta(months=-1, day=1)
                        until = since + relativedelta(months=+1)
                        
                        since = int(time.mktime(since.timetuple()))
                        until = int(time.mktime(until.timetuple()))

                        self.stdout.write(u'Going to next page \n\n\n')

                        stopped_at, created = APIPagination.objects.get_or_create(
                            app_name=u'metrics_social',
                            model_name=u'InsightsMetric',
                            path=u'/%s/insights/page_fans_by_like_source_unique' % ifp.facebook_page.facebook_id,
                            defaults={
                                'offset': since,
                                }
                            )
 
                        stopped_at.offset = since
                        stopped_at.save()

                    else:
                        lap += 1

                        until = datetime.datetime.fromtimestamp(since).date()
                        since = until + relativedelta(months=-1, day=1)
                        until = since + relativedelta(months=+1)
                        
                        since = int(time.mktime(since.timetuple()))
                        until = int(time.mktime(until.timetuple()))

                    if lap > 2:
                        lap = 0
                        main_loop = False
                        ifp.first_load_dimensions = True
                        ifp.save()
예제 #11
0
    def handle(self, *args, **options):
        # get some access_token
        try:
            fbc = FacebookConnection.objects.filter(
                expires__gt=datetime.datetime.now(), deleted=False)[0]
        except IndexError:
            self.stdout.write(u'Error: there is no FacebookConnection')
            fbc = None

        if fbc:
            metrics = [
                u'page_stories',
                u'page_storytellers',
                u'page_impressions',
                u'page_impressions_unique',
                u'page_impressions_paid',
                u'page_impressions_paid_unique',
                u'page_impressions_organic',
                u'page_impressions_organic_unique',
                u'page_impressions_viral',
                u'page_impressions_viral_unique',
                u'page_engaged_users',
                u'page_negative_feedback',
                u'page_negative_feedback_unique',
                u'page_fans',
                u'page_fans_online_per_day',
                u'page_fan_adds',
                u'page_fan_removes',
                u'page_consumptions',
                u'page_consumptions_unique',
                u'page_views',
                u'page_views_unique',
                u'page_views_login',
                u'page_views_login_unique',
                u'page_posts_impressions',
                u'page_posts_impressions_unique',
                u'page_posts_impressions_paid',
                u'page_posts_impressions_paid_unique',
                u'page_posts_impressions_organic',
                u'page_posts_impressions_organic_unique',
                u'page_posts_impressions_viral',
                u'page_posts_impressions_viral_unique',
            ]

            fields = OrderedDict([
                (u'facebook_page_id', u'INT'),
                (u'day', u'DATETIME'),
                (u'metric', u'NVARCHAR(255)'),
                (u'period', u'NVARCHAR(255)'),
                (u'value', u'BIGINT'),
                (u'title', u'NVARCHAR(255)'),
                (u'description', u'NVARCHAR(MAX)'),
            ])

            # api client
            client = FacebookClient(
                client_id=settings.FB_CLIENT_ID,
                client_secret=settings.FB_CLIENT_SECRET,
                access_token=fbc.access_token,
            )

            if args:
                ifps = [
                    i for i in FacebookInsightsReload.objects.filter(
                        facebook_page__facebook_id__in=args,
                        reload_daily=False,
                        deleted=False)
                ]
            else:
                ifps = [
                    i for i in FacebookInsightsReload.objects.filter(
                        reload_daily=False, deleted=False)
                ]

            error = False
            # do first load
            for ifp in ifps:
                if error:
                    break

                # page info
                try:
                    ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
                except FacebookGenericError:
                    if FacebookGenericError.code == u'17':
                        fbc, access_token = FacebookConnection.renew_client(
                            fbc)
                        client._access_token = access_token
                        ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
                else:
                    if not ifp_info:
                        fbc, access_token = FacebookConnection.renew_client(
                            fbc)
                        client._access_token = access_token
                        ifp_info = client.obj_id(ifp.facebook_page.facebook_id)

                    if not ifp_info:
                        error = True
                        continue
                        #raise CommandError(u'\nThe limit of all robots was reached.')

                ifp.facebook_page.name = ifp_info[u'name']
                ifp.facebook_page.link = ifp_info[u'link']
                ifp.facebook_page.talking_about = ifp_info[
                    u'talking_about_count']
                ifp.facebook_page.likes = ifp_info[u'likes']

                ifp.facebook_page.save()

                evolution = EvolutionFacebookPageLike(
                    facebook_page=ifp.facebook_page,
                    likes=ifp_info[u'likes'],
                )
                evolution.save()

                self.stdout.write(
                    u'Successfully updated Facebook Page: %s \n\n' %
                    ifp.facebook_page.facebook_id)

                main_loop = True

                until = ifp.until.date()
                since = until + relativedelta(days=-(until.day - 1))

                since = int(time.mktime(since.timetuple()))
                until = int(time.mktime(until.timetuple()))

                while main_loop:
                    if error:
                        break

                    insights_tasks = []
                    for metric in metrics:

                        if metric == u'page_fans':
                            path = u'/%s/insights/%s' % (
                                ifp.facebook_page.facebook_id, metric)
                        else:
                            path = u'/%s/insights/%s/day' % (
                                ifp.facebook_page.facebook_id, metric)

                        insights_tasks.append({
                            u'id': path,
                            u'path': path,
                            u'args': [],
                            u'kwargs': {
                                'since': since,
                                'until': until
                            },
                        })

                    try:
                        me = client.me()
                    except FacebookGenericError:
                        if FacebookGenericError.code == u'17':
                            fbc, access_token = FacebookConnection.renew_client(
                                fbc)
                            client._access_token = access_token
                    else:
                        if not me:
                            fbc, access_token = FacebookConnection.renew_client(
                                fbc)
                            client._access_token = access_token
                            me = client.me()

                        if not me:
                            error = True
                            continue
                            #raise CommandError(u'\nThe limit of all robots was reached.')

                    client.start_async_tasks(insights_tasks, fbc=fbc)

                    if not all([u'response' in task for task in client.tasks]):
                        error = True
                        continue
                        #raise CommandError(u'\nThe limit of all robots was reached.')

                    rows = []
                    for task in client.tasks:
                        if u'response' in task:

                            if not u'data' in task[u'response']:
                                continue

                            for data in task[u'response'][u'data']:
                                for row in reversed(data[u'values']):
                                    day = datetime.datetime.strptime(
                                        row[u'end_time'][:10],
                                        '%Y-%m-%d').date()
                                    day = day + relativedelta(days=-1)

                                    rows.append(
                                        (ifp.id, str(unicode(day)),
                                         str(data[u'name']),
                                         str(data[u'period']), row[u'value'],
                                         str(data[u'title']),
                                         str(data[u'description'])))

                    if rows:
                        bk = BulkInsertOrUpdate(fields=fields)
                        matches = [
                            {
                                u'clause': None,
                                u'first_field': u'facebook_page_id',
                                u'comparison': u'=',
                                u'second_field': u'facebook_page_id'
                            },
                            {
                                u'clause': 'AND',
                                u'first_field': u'day',
                                u'comparison': u'=',
                                u'second_field': u'day'
                            },
                            {
                                u'clause': 'AND',
                                u'first_field': u'metric',
                                u'comparison': u'=',
                                u'second_field': u'metric'
                            },
                            {
                                u'clause': 'AND',
                                u'first_field': u'period',
                                u'comparison': u'=',
                                u'second_field': u'period'
                            },
                        ]

                        insert_fields = [
                            u'facebook_page_id', u'day', u'metric', u'period',
                            u'value', u'title', u'description'
                        ]

                        cursor = connection.cursor()
                        cursor.execute(
                            bk.bulk(
                                u'SET QUOTED_IDENTIFIER OFF;',
                                bk.command_bulk_create(),
                                bk.command_bulk_insert(rows),
                                bk.command_bulk_merge(
                                    u'metrics_social_insightsmetric', matches,
                                    [u'value'], insert_fields)))
                        cursor.execute(bk.command_drop_action_table())
                        transaction.commit_unless_managed()

                    self.stdout.write(u'Facebook insights block was created\n')
                    responses = [
                        task[u'response'][u'data'][0] for task in client.tasks
                        if len(task[u'response'][u'data'])
                    ]

                    if datetime.datetime.fromtimestamp(
                            since).date() <= ifp.since.date():
                        main_loop = False
                        ifp.reload_daily = True
                        ifp.save()
                    else:
                        until = datetime.datetime.fromtimestamp(since).date()
                        since = until + relativedelta(months=-1, day=1)
                        until = since + relativedelta(months=+1)

                        since = int(time.mktime(since.timetuple()))
                        until = int(time.mktime(until.timetuple()))

                        self.stdout.write(u'Going to next page \n\n\n')