def toy_file_sensor(context):
if not directory_name:
yield SkipReason(
"No directory specified at environment variable `DAGSTER_TOY_SENSOR_DIRECTORY`"
)
return
if not os.path.isdir(directory_name):
yield SkipReason(f"Directory {directory_name} not found")
return
directory_files = get_directory_files(directory_name, context.cursor)
if not directory_files:
yield SkipReason(
f"No new files found in {directory_name} (after {context.cursor})"
)
return
for filename, mtime in directory_files:
yield RunRequest(
run_key="{}:{}".format(filename, str(mtime)),
run_config={
"solids": {
"read_file": {
"config": {
"directory": directory_name,
"filename": filename
}
}
}
},
)
def my_s3_sensor(context):
new_s3_keys = get_s3_keys("my_s3_bucket", since_key=context.last_run_key)
if not new_s3_keys:
yield SkipReason("No new s3 files found for bucket my_s3_bucket.")
return
for s3_key in new_s3_keys:
yield RunRequest(run_key=s3_key, run_config={})
def toy_s3_sensor(context):
if not bucket:
raise Exception(
"S3 bucket not specified at environment variable `DAGSTER_TOY_SENSOR_S3_BUCKET`."
)
new_s3_keys = get_s3_keys(bucket, since_key=context.last_run_key)
if not new_s3_keys:
yield SkipReason(f"No s3 updates found for bucket {bucket}.")
return
for s3_key in new_s3_keys:
yield RunRequest(
run_key=s3_key,
run_config={
"solids": {
"read_s3_key": {
"config": {
"bucket": bucket,
"s3_key": s3_key
}
}
}
},
)
def my_directory_sensor_with_skip_reasons():
has_files = False
for filename in os.listdir(MY_DIRECTORY):
filepath = os.path.join(MY_DIRECTORY, filename)
if os.path.isfile(filepath):
yield RunRequest(
run_key=filename,
run_config={"solids": {"process_file": {"config": {"filename": filename}}}},
)
has_files = True
if not has_files:
yield SkipReason(f"No files found in {MY_DIRECTORY}.")
def _validate_on_import_complete(_: SensorExecutionContext) -> Union[Iterator[RunRequest], SkipReason]:
sensor = ArgoHcaImportCompletionSensor(
argo_url=os.environ["HCA_ARGO_URL"],
access_token=default_google_access_token())
workflows = sensor.successful_hca_import_workflows()
if any(workflows):
for workflow in workflows:
yield sensor.generate_run_request(workflow)
else:
return SkipReason("No succeeded import-hca-total workflows returned by Argo.")
def slow_sensor(_):
time.sleep(5)
yield SkipReason("Oops fell asleep")