class TaskConfig(BaseModel):
source_config: Union[TwitterSourceConfig, PlayStoreConfig]
sink_config: Union[HttpSinkConfig, JiraSinkConfig, ElasticSearchSinkConfig,
DailyGetSinkConfig]
analyzer_config: AnalyzerConfig = AnalyzerConfig(use_sentiment_model=False)
time_in_seconds: int
def to_json(self):
return json.dumps(self,
default=lambda o: o.__dict__,
sort_keys=True,
indent=4)
class Config:
arbitrary_types_allowed = True
schema_extra = {
"example": {
"source_config":
TwitterSourceConfig(
keywords=["machine_learning"],
hashtags=["#ai"],
usernames=["@user1"],
operators=["-is:reply", "-is:retweet"],
since_id=1234,
until_id=9999,
lookup_period="1d",
tweet_fields=[
"author_id", "conversation_id", "created_at", "id",
"public_metrics", "text"
],
user_fields=[
"id", "name", "public_metrics", "username", "verified"
],
expansions=["author_id"],
place_fields=["country"],
max_tweets=10,
credential=TwitterCredentials(
bearer_token="<twitter_bearer_token>")).dict(),
"sink_config":
DailyGetSinkConfig(url="http://127.0.0.1:8080/endpoint",
partner_id="12345",
consumer_phone_number="1234567890",
source_information="Twitter",
base_payload={
"partnerId": "12345",
"consumerPhoneNumber": "1234567890",
}).dict(),
"analyzer_config":
AnalyzerConfig(labels=["service", "quality", "tracking"],
use_sentiment_model=True,
multi_class_classification=False).dict(),
"time_in_seconds":
300
}
}
def test_text_analyzer_with_vader(text_analyzer_with_vader):
source_responses = [AnalyzerRequest(text, "sample") for text in TEXTS]
analyzer_responses = text_analyzer_with_vader.analyze_input(
source_response_list=source_responses,
analyzer_config=AnalyzerConfig(use_sentiment_model=False))
assert len(analyzer_responses) == len(TEXTS)
for analyzer_response in analyzer_responses:
assert len(analyzer_response.classification) == 2
assert "positive" in analyzer_response.classification
assert "negative" in analyzer_response.classification
def test_text_analyzer_with_model(text_analyzer_with_model):
labels = ["facility", "food", "comfortable", "positive", "negative"]
source_responses = [AnalyzerRequest(text, "sample") for text in TEXTS]
analyzer_responses = text_analyzer_with_model.analyze_input(
source_response_list=source_responses,
analyzer_config=AnalyzerConfig(labels=labels,
use_sentiment_model=True))
assert len(analyzer_responses) == len(TEXTS)
for analyzer_response in analyzer_responses:
assert len(analyzer_response.classification) == len(labels)
assert "positive" in analyzer_response.classification
assert "negative" in analyzer_response.classification
class ClassifierRequest(BaseModel):
texts: List[str]
analyzer_config: AnalyzerConfig = AnalyzerConfig(use_sentiment_model=False)
class Config:
arbitrary_types_allowed = True
],
user_fields=["id", "name", "public_metrics", "username", "verified"],
expansions=["author_id"],
place_fields=None,
max_tweets=10,
)
source = TwitterSource()
sink = DailyGetSink()
text_analyzer = TextAnalyzer(
model_name_or_path="joeddav/bart-large-mnli-yahoo-answers",
# model_name_or_path="joeddav/xlm-roberta-large-xnli",
initialize_model=True,
analyzer_config=AnalyzerConfig(labels=[
"service", "delay", "tracking", "no response", "missing items",
"delivery", "mask"
],
use_sentiment_model=True))
source_response_list = source.lookup(source_config)
for idx, source_response in enumerate(source_response_list):
logger.info(f"source_response#'{idx}'='{source_response.__dict__}'")
analyzer_response_list = text_analyzer.analyze_input(
source_response_list=source_response_list)
for idx, an_response in enumerate(analyzer_response_list):
logger.info(f"analyzer_response#'{idx}'='{an_response.__dict__}'")
# HTTP Sink
sink_response_list = sink.send_data(analyzer_response_list, sink_config)
for sink_response in sink_response_list:
source = TwitterSource()
# To start jira server locally `atlas-run-standalone --product jira`
jira_sink_config = JiraSinkConfig(
url="http://localhost:2990/jira",
username=SecretStr("admin"),
password=SecretStr("admin"),
issue_type={"name": "Task"},
project={"key": "CUS"},
)
jira_sink = JiraSink()
text_analyzer = TextAnalyzer(
model_name_or_path="joeddav/bart-large-mnli-yahoo-answers",
initialize_model=True,
analyzer_config=AnalyzerConfig(labels=["service", "delay", "performance"],
use_sentiment_model=True))
source_response_list = source.lookup(source_config)
for idx, source_response in enumerate(source_response_list):
logger.info(f"source_response#'{idx}'='{source_response.__dict__}'")
analyzer_response_list = text_analyzer.analyze_input(
source_response_list=source_response_list)
for idx, an_response in enumerate(analyzer_response_list):
logger.info(f"analyzer_response#'{idx}'='{an_response.__dict__}'")
sink_response_list = jira_sink.send_data(analyzer_response_list,
jira_sink_config)
for sink_response in sink_response_list:
if sink_response is not None:
logger.info(f"sink_response='{sink_response}'")
def __init__(self, **data: Any):
super().__init__(**data)
self.analyzer_config = self.analyzer_config or AnalyzerConfig(
use_sentiment_model=False)