def fetch_by_album(self, album, limit=100, offset=0, until=None, since=None, **kwargs): kwargs.update({ 'limit': int(limit), 'offset': int(offset), }) for field in ['until', 'since']: value = locals()[field] if isinstance(value, datetime): kwargs[field] = int(time.mktime(value.timetuple())) elif value is not None: try: kwargs[field] = int(value) except TypeError: raise ValueError('Wrong type of argument %s: %s' % (field, type(value))) ids = [] response = graph("%s/photos" % album.pk, **kwargs) #log.debug('response objects count - %s' % len(response.data)) extra_fields = {"album_id": album.pk} for resource in response.data: instance = self.get_or_create_from_resource(resource, extra_fields) ids += [instance.pk] return Photo.objects.filter(pk__in=ids), response
def fetch_likes(self, limit=1000, offset=0, delete_all=True): ''' Retrieve and save all likes of post ''' response = graph('%s/likes' % self.graph_id, limit=limit, offset=offset) if not response: return if delete_all: self.like_users.clear() response_count = len(response.data) for resource in response.data: user = get_or_create_from_small_resource(resource) self.like_users.add(user) log.debug('like users count - %s' % self.like_users.count()) log.debug('response objects count - %s, limit - %s, offset - %s' % (response_count, limit, offset)) if response_count != 0: return self.fetch_likes(limit=limit, offset=offset+response_count, delete_all=False) else: self.likes_real_count = self.like_users.count() self.save() return self.like_users.all()
def fetch_stats(self): ''' Retrieve and save statistic for group ''' response = graph('%s/stats' % self.adgroup_id) instance = AdStatistic.remote.get_or_create_from_resource(response) return instance
def fetch_page_wall(self, page, all=False, limit=1000, offset=0, until=None, since=None): kwargs = { 'limit': int(limit), 'offset': int(offset), } if until: kwargs['until'] = int(time.mktime(until.timetuple())) if since: kwargs['since'] = int(time.mktime(since.timetuple())) response = graph('%s/posts' % page.graph_id, **kwargs) # TODO: think about this condition more deeply if response is None: return page.wall_posts.all() # TODO: move this checking to level up if 'error_code' in response and response['error_code'] == 1: return self.fetch_page_wall(page, all, limit, offset, until) log.debug('response objects count - %s' % len(response.data)) instances = [] for resource in response.data: try: instance = Post.remote.get_or_create_from_resource(resource) except Exception, e: log.error("Impossible to save post with resource %s. Error is '%s'" % (resource, e)) continue if instance.owners.count() == 0: post_owner = PostOwner.objects.get_or_create(post=instance, owner_content_type=ContentType.objects.get_for_model(page), owner_id=page.id)[0] instance.owners.add(post_owner) instances += [instance]
def fetch_posts(self, all=False, limit=1000, offset=0, until=None, since=None): ''' Retrieve and save all posts of page ''' if 'facebook_posts' not in settings.INSTALLED_APPS: raise ImproperlyConfigured("Application 'facebook_posts' not in INSTALLED_APPS") from facebook_posts.models import Post, PostOwner kwargs = {} if until: kwargs['until'] = int(time.mktime(until.timetuple())) if since: kwargs['since'] = int(time.mktime(since.timetuple())) response = graph('%s/posts' % self.graph_id, limit=limit, offset=offset, **kwargs) # TODO: move this checking to level up if 'error_code' in response and response['error_code'] == 1: return self.fetch_posts(all=all, limit=limit, offset=offset, until=until) log.debug('response objects count - %s' % len(response.data)) instances = [] for resource in response.data: try: instance = Post.remote.get_or_create_from_resource(resource) except Exception, e: log.error("Impossible to save post with resource %s. Error is '%s'" % (resource, e)) continue if instance.owners.count() == 0: post_owner = PostOwner.objects.get_or_create(post=instance, owner_content_type=ContentType.objects.get_for_model(self), owner_id=self.id)[0] instance.owners.add(post_owner) instances += [instance]
def fetch_by_page(self, page, limit=1000, until=None, since=None, **kwargs): kwargs.update({ 'limit': int(limit), }) for field in ['until', 'since']: value = locals()[field] if isinstance(value, datetime): kwargs[field] = int(time.mktime(value.timetuple())) elif value is not None: try: kwargs[field] = int(value) except TypeError: raise ValueError('Wrong type of argument %s: %s' % (field, type(value))) ids = [] response = graph("%s/albums/" % page.graph_id, **kwargs) #log.debug('response objects count - %s' % len(response.data)) for resource in response.data: instance = self.get_or_create_from_resource(resource) ids += [instance.pk] return Album.objects.filter(pk__in=ids)
def test_request(self): response = graph('zuck') self.assertEqual(response.id, '4') self.assertEqual(response.last_name, 'Zuckerberg') self.assertEqual(response.first_name, 'Mark') self.assertEqual(response.gender, 'male')
def fetch_creatives(self): ''' Retrieve and save all creatives for account ''' response = graph('act_%s/adcreatives' % self.account_id) instances = [] for resource in response.data: instance = AdCreative.remote.get_or_create_from_resource(resource) instances += [instance] return instances
def fetch_groups(self): ''' Retrieve and save all groups for campaign ''' response = graph('%s/adgroups' % self.campaign_id) instances = [] for resource in response.data: instance = AdGroup.remote.get_or_create_from_resource(resource) instances += [instance] return instances
def fetch_images(self): ''' Retrieve and save images for account ''' response = graph('act_%s/adimages' % self.account_id) instances = [] for resource in response.data.values(): instance = AdImage.remote.get_or_create_from_resource(resource) instances += [instance] return instances
def fetch_comments(self, limit=1000, filter='stream', summary=True, **kwargs): ''' Retrieve and save all comments of post ''' ids = [] response = graph('%s/comments' % self.graph_id, limit=limit, filter=filter, summary=int(summary), **kwargs) if response: log.debug('response objects count=%s, limit=%s, after=%s' % (len(response.data), limit, kwargs.get('after'))) for resource in response.data: instance = Comment.remote.get_or_create_from_resource(resource, {'post_id': self.pk}) ids += [instance.pk] return Comment.objects.filter(pk__in=ids), response
def fetch_likes(self, limit=1000, **kwargs): ''' Retrieve and save all likes of post ''' ids = [] response = graph('%s/likes' % self.graph_id, limit=limit, **kwargs) if response: log.debug('response objects count=%s, limit=%s, after=%s' % (len(response.data), limit, kwargs.get('after'))) for resource in response.data: user = get_or_create_from_small_resource(resource) ids += [user.pk] return User.objects.filter(pk__in=ids), response
def fetch_stats_groups(self): ''' Retrieve and save statistic for groups of accout TODO: act_AccountID/adgroupstats?adgroup_ids=[JSON-encoded array of ad group IDs] ''' response = graph('act_%s/adgroupstats' % self.account_id) instances = [] for resource in response.data: instance = AdStatistic.remote.get_or_create_from_resource(resource) instances += [instance] return instances
def fetch_comments(self, limit=100, filter='stream', summary=True, **kwargs): ''' Retrieve and save all comments ''' extra_fields = {('%s_id' % self._meta.module_name): self.pk} # {"album_id": 1} ids = [] response = graph('%s/comments' % self.graph_id, limit=limit, filter=filter, summary=int(summary), **kwargs) if response: #log.debug('response objects count=%s, limit=%s, after=%s' % (len(response.data), limit, kwargs.get('after'))) for resource in response.data: instance = Comment.remote.get_or_create_from_resource(resource, extra_fields) ids += [instance.pk] return Comment.objects.filter(pk__in=ids), response
def fetch_campaigns(self): ''' Retrieve and save all campaigns for account ''' if not self.id: log.error('It is neccesary to save account before saving campaigns') response = graph('act_%s/adcampaigns' % self.account_id) instances = [] for resource in response.data: instance = AdCampaign.remote.get_or_create_from_resource(resource) instances += [instance] return instances
def fetch_comments(self, limit=100, offset=0): ''' Retrieve and save all comments of post ''' response = graph('%s/comments' % self.graph_id, limit=limit, offset=offset) if not response: return log.debug('response len - %s' % len(response.data)) instances = [] for resource in response.data: instance = Comment.remote.get_or_create_from_resource(resource, {'post_id': self.id}) log.debug('comments count - %s' % Comment.objects.count()) instances += [instance] return instances
def fetch_shares(self, limit=1000, **kwargs): ''' Retrieve and save all shares of post ''' ids = [] graph_id = self.graph_id.split('_').pop() response = graph('%s/sharedposts' % graph_id, **kwargs) if response: timestamps = dict([(int(post['from']['id']), dateutil.parser.parse(post['created_time'])) for post in response.data]) ids_new = timestamps.keys() # becouse we should use local pk, instead of remote, remove it after pk -> graph_id ids_current = map(int, User.objects.filter(pk__in=self.shares_users.get_query_set(only_pk=True).using(MASTER_DATABASE).exclude(time_from=None)).values_list('graph_id', flat=True)) ids_add = set(ids_new).difference(set(ids_current)) ids_add_pairs = [] ids_remove = set(ids_current).difference(set(ids_new)) log.debug('response objects count=%s, limit=%s, after=%s' % (len(response.data), limit, kwargs.get('after'))) for post in response.data: graph_id = int(post['from']['id']) if sorted(post['from'].keys()) == ['id', 'name']: try: user = get_or_create_from_small_resource(post['from']) ids += [user.pk] # this id in add list and still not in add_pairs (sometimes in response are duplicates) if graph_id in ids_add and graph_id not in map(lambda i:i[0], ids_add_pairs): ids_add_pairs += [(graph_id, user.pk)] # becouse we should use local pk, instead of remote except UnknownResourceType: continue m2m_model = self.shares_users.through # remove old shares without time_from self.shares_users.get_query_set_through().filter(time_from=None).delete() # add new shares get_share_date = lambda id: timestamps[id] if id in timestamps else self.created_time field_name = '%s_id' % self._meta.module_name # 'album_id' m2m_model.objects.bulk_create([m2m_model(**{'user_id': pk, field_name: self.pk, 'time_from': get_share_date(graph_id)}) for graph_id, pk in ids_add_pairs]) return User.objects.filter(pk__in=ids), response
def fetch_page_wall(self, page, limit=1000, offset=0, until=None, since=None, **kwargs): ''' Arguments: * until|since - timestamp or datetime ''' kwargs = { 'limit': int(limit), 'offset': int(offset), } for field in ['until', 'since']: value = locals()[field] if isinstance(value, datetime): kwargs[field] = int(time.mktime(value.timetuple())) elif value is not None: try: kwargs[field] = int(value) except TypeError: raise ValueError('Wrong type of argument %s: %s' % (field, type(value))) response = graph('%s/posts' % page.graph_id, **kwargs) # TODO: move this checking to level up if 'error_code' in response and response['error_code'] == 1: return self.fetch_page_wall(page, all, limit, offset, until, **kwargs) log.debug('response objects count - %s' % len(response.data)) ids = [] page_ct = ContentType.objects.get_for_model(page) if response: log.debug('response objects count=%s, limit=%s, after=%s' % (len(response.data), limit, kwargs.get('after'))) for resource in response.data: instance = Post.remote.get_or_create_from_resource(resource) if instance.owners.using(MASTER_DATABASE).count() == 0: post_owner = PostOwner.objects.get_or_create(post=instance, owner_content_type=page_ct, owner_id=page.pk)[0] instance.owners.add(post_owner) ids += [instance.pk] return Post.objects.filter(pk__in=ids), response
def test_empty_result(self): result = graph('8576093908/posts', **{'limit': 1000, 'until': 1345661805, 'offset': 0}) if result is not None: self.assertEqual(result.error_code, 1)