import logging
import sys
import time
from obsei.workflow.store import WorkflowStore
from obsei.source.twitter_source import TwitterSource, TwitterSourceConfig
from obsei.workflow.workflow import Workflow, WorkflowConfig
logger = logging.getLogger(__name__)
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
# Create workflow store instance, by default it will use SQLite to store state data
store = WorkflowStore()
# Pass store reference to observer, so it can use it to store state data
source = TwitterSource(store=store)
def print_state(id: str):
logger.info(f"Source State: {source.store.get_source_state(id)}")
source_config = TwitterSourceConfig(
keywords=["india"],
lookup_period="2m",
tweet_fields=[
"author_id",
"conversation_id",
"created_at",
"id",
"public_metrics",
import logging
import sys
import time
from obsei.workflow.store import WorkflowStore
from obsei.source.twitter_source import TwitterSource, TwitterSourceConfig
from obsei.workflow.workflow import Workflow, WorkflowConfig
logger = logging.getLogger(__name__)
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
source = TwitterSource(store=WorkflowStore())
def print_state(id: str):
logger.info(f"Source State: {source.store.get_source_state(id)}")
source_config = TwitterSourceConfig(
keywords=["india"],
lookup_period="2m",
tweet_fields=[
"author_id",
"conversation_id",
"created_at",
"id",
"public_metrics",
"text",
],
user_fields=["id", "name", "public_metrics", "username", "verified"],
expansions=["author_id"],
from obsei.analyzer.dummy_analyzer import DummyAnalyzer, DummyAnalyzerConfig
from obsei.processor import Processor
from obsei.sink.slack_sink import SlackSink, SlackSinkConfig
from obsei.source import PlayStoreScrapperConfig, PlayStoreScrapperSource
from obsei.workflow.store import WorkflowStore
from obsei.workflow.workflow import Workflow, WorkflowConfig
def print_state(identifier: str):
logger.info(f"Source State: {source.store.get_source_state(identifier)}")
logger = logging.getLogger(__name__)
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
workflow_store = WorkflowStore()
source_config = PlayStoreScrapperConfig(
app_url=
'https://play.google.com/store/apps/details?id=com.google.android.gm&hl=en_IN&gl=US',
max_count=3)
source = PlayStoreScrapperSource(store=workflow_store)
sink_config = SlackSinkConfig(slack_token=os.environ["SLACK_TOKEN"],
channel_id="C01TUPZ23NZ",
jinja_template="""
```
{%- for key, value in payload.items() recursive%}
{%- if value is mapping -%}
{{loop(value.items())}}
def print_state(id: str):
logger.info(f"Source State: {source.store.get_source_state(id)}")
logger = logging.getLogger(__name__)
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
since_time = datetime.utcnow().astimezone(pytz.utc) + timedelta(days=-5)
source_config = AppStoreScrapperConfig(
countries=["us", "de"],
app_id="497799835",
lookup_period=since_time.strftime(DATETIME_STRING_PATTERN),
)
source = AppStoreScrapperSource(store=WorkflowStore())
# This example also
workflow = Workflow(config=WorkflowConfig(source_config=source_config, ), )
source.store.add_workflow(workflow)
for i in range(1, 4):
print_state(workflow.id)
source_response_list = source.lookup(source_config, id=workflow.id)
if source_response_list is None or len(source_response_list) == 0:
break
for source_response in source_response_list:
logger.info(source_response.__dict__)
def print_state(id: str):
logger.info(f'Source State: {source.store.get_source_state(id)}')
logger = logging.getLogger(__name__)
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
since_time = datetime.utcnow().astimezone(pytz.utc) + timedelta(hours=-2)
# Credentials will be fetched from env variable named reddit_client_id and reddit_client_secret
source_config = RedditConfig(
subreddits=["wallstreetbets"],
lookup_period=since_time.strftime(DATETIME_STRING_PATTERN))
source = RedditSource(store=WorkflowStore())
workflow = Workflow(config=WorkflowConfig(source_config=source_config, ), )
source.store.add_workflow(workflow)
for i in range(1, 4):
print_state(workflow.id)
source_response_list = source.lookup(source_config, id=workflow.id)
if source_response_list is None or len(source_response_list) == 0:
break
for source_response in source_response_list:
logger.info(source_response.__dict__)
time.sleep(10)
def print_state(id: str):
logger.info(f"Source State: {source.store.get_source_state(id)}")
logger = logging.getLogger(__name__)
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
since_time = datetime.utcnow().astimezone(pytz.utc) + timedelta(days=-1)
source_config = RedditScrapperConfig(
url="https://www.reddit.com/r/wallstreetbets/comments/.rss?sort=new",
user_agent="testscript by u/FitStatistician7378",
lookup_period=since_time.strftime(DATETIME_STRING_PATTERN),
)
source = RedditScrapperSource(store=WorkflowStore())
workflow = Workflow(config=WorkflowConfig(source_config=source_config, ), )
source.store.add_workflow(workflow)
for i in range(1, 4):
print_state(workflow.id)
source_response_list = source.lookup(source_config, id=workflow.id)
if source_response_list is None or len(source_response_list) == 0:
break
for source_response in source_response_list:
logger.info(source_response.__dict__)
time.sleep(30)