import logging
import sys
import time
from obsei.workflow.store import WorkflowStore
from obsei.source.twitter_source import TwitterSource, TwitterSourceConfig
from obsei.workflow.workflow import Workflow, WorkflowConfig
logger = logging.getLogger(__name__)
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
# Create workflow store instance, by default it will use SQLite to store state data
store = WorkflowStore()
# Pass store reference to observer, so it can use it to store state data
source = TwitterSource(store=store)
def print_state(id: str):
logger.info(f"Source State: {source.store.get_source_state(id)}")
source_config = TwitterSourceConfig(
keywords=["india"],
lookup_period="2m",
tweet_fields=[
"author_id",
"conversation_id",
"created_at",
"id",
"public_metrics",
import logging
import sys
import time
from obsei.workflow.store import WorkflowStore
from obsei.source.twitter_source import TwitterSource, TwitterSourceConfig
from obsei.workflow.workflow import Workflow, WorkflowConfig
logger = logging.getLogger(__name__)
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
source = TwitterSource(store=WorkflowStore())
def print_state(id: str):
logger.info(f"Source State: {source.store.get_source_state(id)}")
source_config = TwitterSourceConfig(
keywords=["india"],
lookup_period="2m",
tweet_fields=[
"author_id",
"conversation_id",
"created_at",
"id",
"public_metrics",
"text",
],
user_fields=["id", "name", "public_metrics", "username", "verified"],
expansions=["author_id"],
def print_state(id: str):
logger.info(f'Source State: {source.store.get_source_state(id)}')
logger = logging.getLogger(__name__)
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
since_time = datetime.utcnow().astimezone(pytz.utc) + timedelta(hours=-2)
# Credentials will be fetched from env variable named reddit_client_id and reddit_client_secret
source_config = RedditConfig(
subreddits=["wallstreetbets"],
lookup_period=since_time.strftime(DATETIME_STRING_PATTERN))
source = RedditSource(store=WorkflowStore())
workflow = Workflow(config=WorkflowConfig(source_config=source_config, ), )
source.store.add_workflow(workflow)
for i in range(1, 4):
print_state(workflow.id)
source_response_list = source.lookup(source_config, id=workflow.id)
if source_response_list is None or len(source_response_list) == 0:
break
for source_response in source_response_list:
logger.info(source_response.__dict__)
time.sleep(10)
def print_state(id: str):
logger.info(f"Source State: {source.store.get_source_state(id)}")
logger = logging.getLogger(__name__)
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
since_time = datetime.utcnow().astimezone(pytz.utc) + timedelta(days=-5)
source_config = AppStoreScrapperConfig(
countries=["us", "de"],
app_id="497799835",
lookup_period=since_time.strftime(DATETIME_STRING_PATTERN),
)
source = AppStoreScrapperSource(store=WorkflowStore())
# This example also
workflow = Workflow(config=WorkflowConfig(source_config=source_config, ), )
source.store.add_workflow(workflow)
for i in range(1, 4):
print_state(workflow.id)
source_response_list = source.lookup(source_config, id=workflow.id)
if source_response_list is None or len(source_response_list) == 0:
break
for source_response in source_response_list:
logger.info(source_response.__dict__)
def print_state(id: str):
logger.info(f"Source State: {source.store.get_source_state(id)}")
logger = logging.getLogger(__name__)
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
since_time = datetime.utcnow().astimezone(pytz.utc) + timedelta(days=-1)
source_config = RedditScrapperConfig(
url="https://www.reddit.com/r/wallstreetbets/comments/.rss?sort=new",
user_agent="testscript by u/FitStatistician7378",
lookup_period=since_time.strftime(DATETIME_STRING_PATTERN),
)
source = RedditScrapperSource(store=WorkflowStore())
workflow = Workflow(config=WorkflowConfig(source_config=source_config, ), )
source.store.add_workflow(workflow)
for i in range(1, 4):
print_state(workflow.id)
source_response_list = source.lookup(source_config, id=workflow.id)
if source_response_list is None or len(source_response_list) == 0:
break
for source_response in source_response_list:
logger.info(source_response.__dict__)
time.sleep(30)