def put(self, source_name): helper = RequestHelper(self) source_name = urllib.unquote(source_name) success, values = read_json_fields(helper, "url", "active", logger=logging) if not success: return url, active = values # a Feed must be sole owner of an ArtifactSource; # fails if source already exists and is already linked to a feed source = ArtifactSourceAccessor.get_by_name(source_name, return_none=True) if source: source_feed_key = Feed.get_by_source(source, keys_only=True, return_none=True) if source_feed_key: msg = "source '%s' is referenced by feed %s" % (source_name, source_feed_key.name()) helper.error(409, msg) else: source = ArtifactSourceAccessor.create(source_name) # creates UrlResource if necessary resource = UrlResourceAccessor.get_by_url(url, return_none=True) if not resource: resource = UrlResourceAccessor.create(url) # create or update Feed feed = Feed.get_by_source_name(source_name, return_none=True) if feed: feed.artifact_source = source feed.url_resource = resource feed.put() else: Feed.create(source_name, artifact_source=source, url=url, url_resource=resource, active=bool(active)) helper.set_status(204)
def source_hash(source): c = Counters.source_counter(source.name) return { 'name': source.name, 'count': c.count(), 'content-count': ArtifactSourceAccessor.count_content(source), 'info-count': ArtifactSourceAccessor.count_infos(source) }
def test_get(self): request, response = new_mock_request_response(self.m) self.m.StubOutWithMock(ArtifactSourceAccessor, "find_artifact_counts") self.m.StubOutWithMock(ArtifactSourceAccessor, "find_artifact_counts_newer") self.m.StubOutWithMock(memcache, "get_stats") self.m.StubOutWithMock(ConfigurationAccessor, "get_or_create") ArtifactSourceAccessor.find_artifact_counts().AndReturn({}) ArtifactSourceAccessor.find_artifact_counts_newer(IgnoreArg()).AndReturn({}) memcache.get_stats().AndReturn({}) # token = "oauth_token_secret=hi&oauth_token=hi" # config = MockEntity(key_name="hi", twitter_oauth_enabled=True, twitter_access_token=token, twitter_read_only=False) # ConfigurationAccessor.get_or_create().AndReturn(config) api = new_mock_twitter_api(self.m) api.GetUserTimeline(count=IgnoreArg()).AndReturn(()) api.GetDirectMessages(since=IgnoreArg()).AndReturn(()) api.GetReplies(since=IgnoreArg()).AndReturn(()) handler = StatsHandler() handler.initialize(request, response) self.m.ReplayAll() handler.get() self.m.VerifyAll()
def delete(self, name, **kw): helper = RequestHelper(self) try: ArtifactSourceAccessor.delete_by_name(name) helper.set_status(204) except NotFoundException, ex: helper.error(404) return
def put(self, name, **kw): helper = RequestHelper(self) source = ArtifactSourceAccessor.get_by_name(name, return_none=True) if source: helper.set_status(409, "duplicate ArtifactSouce") return ArtifactSourceAccessor.create(name) helper.set_status(204)
def get(self, **kw): helper = RequestHelper(self) result = {} today = datetime.combine(datetime.now(), time(0, 0, 0)) result['timestamp'] = str(datetime.now()) result['today'] = str(today) warnings = [] result['warnings'] = warnings # source/artifact counts result[ 'source_artifact_counts'] = ArtifactSourceAccessor.find_artifact_counts( ) # new artifacts result[ 'new_artifacts'] = ArtifactSourceAccessor.find_artifact_counts_newer( today) # newer_arts = ArtifactAccessor.find_newer(today, refresh=True) # new_art_stats = [] # result['new_artifacts'] = new_art_stats # for art in newer_arts: # new_art_stats.append(dict(guid=art.guid, source_name=art.source_name)) # memcache stats result['memcache'] = memcache.get_stats() try: twactor = TwitterActor() # outgoing messages result['statuses_out'] = [ describe_status_with_timestamp(s) for s in twactor.latest_statuses(5) ] # incoming messages direct_stats = [] mention_stats = [] result['directs'] = direct_stats result['mentions'] = mention_stats directs, mentions = twactor.messages(today) directs.reverse() mentions.reverse() direct_stats.extend( [describe_status_with_timestamp(s) for s in directs]) mention_stats.extend( [describe_status_with_timestamp(s) for s in mentions]) except TwitterException, ex: warnings.append(str(ex))
def get(self, name, **kw): helper = RequestHelper(self) source = ArtifactSourceAccessor.get_by_name(name, return_none=True) if not source: helper.error(404) return helper.write_json(source_hash(source))
def get(self, **kw): helper = RequestHelper(self) result = {} today = datetime.combine(datetime.now(), time(0, 0, 0)) result['timestamp'] = str(datetime.now()) result['today'] = str(today) warnings = [] result['warnings'] = warnings # source/artifact counts result['source_artifact_counts'] = ArtifactSourceAccessor.find_artifact_counts() # new artifacts result['new_artifacts'] = ArtifactSourceAccessor.find_artifact_counts_newer(today) # newer_arts = ArtifactAccessor.find_newer(today, refresh=True) # new_art_stats = [] # result['new_artifacts'] = new_art_stats # for art in newer_arts: # new_art_stats.append(dict(guid=art.guid, source_name=art.source_name)) # memcache stats result['memcache'] = memcache.get_stats() try: twactor = TwitterActor() # outgoing messages result['statuses_out'] = [describe_status_with_timestamp(s) for s in twactor.latest_statuses(5)] # incoming messages direct_stats = [] mention_stats = [] result['directs'] = direct_stats result['mentions'] = mention_stats directs, mentions = twactor.messages(today) directs.reverse() mentions.reverse() direct_stats.extend([describe_status_with_timestamp(s) for s in directs]) mention_stats.extend([describe_status_with_timestamp(s) for s in mentions]) except TwitterException, ex: warnings.append(str(ex))
def test_delete_by_name_deletes_source_with_no_referencing_feed(self): self.m.StubOutWithMock(ArtifactSource, "get_by_name") self.m.StubOutWithMock(FeedAccessor, "get_by_source_name") self.m.StubOutWithMock(ArtifactInfo, "find_by_source") self.m.StubOutWithMock(ArtifactContent, "find_by_source") self.m.StubOutWithMock(db, "delete") self.m.StubOutWithMock(memcache, "delete") name = "mhawthorne" source = MockEntity(key_name=name) ArtifactSource.get_by_name(name).AndReturn(source) FeedAccessor.get_by_source_name(name, return_none=True) ArtifactInfo.find_by_source(source, keys_only=True).AndReturn(MockQuery(range(0,0))) ArtifactContent.find_by_source(source).AndReturn(MockQuery(range(0,0))) db.delete(source) memcache.delete(IsA(str)).AndReturn(1) self.m.ReplayAll() ArtifactSourceAccessor.delete_by_name(name) self.m.VerifyAll()
def test_delete_by_name_deletes_source_with_no_referencing_feed(self): self.m.StubOutWithMock(ArtifactSource, "get_by_name") self.m.StubOutWithMock(FeedAccessor, "get_by_source_name") self.m.StubOutWithMock(ArtifactInfo, "find_by_source") self.m.StubOutWithMock(ArtifactContent, "find_by_source") self.m.StubOutWithMock(db, "delete") self.m.StubOutWithMock(memcache, "delete") name = "mhawthorne" source = MockEntity(key_name=name) ArtifactSource.get_by_name(name).AndReturn(source) FeedAccessor.get_by_source_name(name, return_none=True) ArtifactInfo.find_by_source(source, keys_only=True).AndReturn( MockQuery(range(0, 0))) ArtifactContent.find_by_source(source).AndReturn(MockQuery(range(0, 0))) db.delete(source) memcache.delete(IsA(str)).AndReturn(1) self.m.ReplayAll() ArtifactSourceAccessor.delete_by_name(name) self.m.VerifyAll()