def get_streams_to_replicate(self):
streams = []
opportunity_child_catalogs = {}
if not self.catalog:
return streams, opportunity_child_catalogs
for stream_catalog in self.catalog.streams:
if not is_selected(stream_catalog):
LOGGER.info("'{}' is not marked selected, skipping."
.format(stream_catalog.stream))
continue
for available_stream in self.available_streams:
if available_stream.matches_catalog(stream_catalog):
if not available_stream.requirements_met(self.catalog):
raise RuntimeError(
"{} requires that that the following are "
"selected: {}"
.format(stream_catalog.stream,
','.join(available_stream.REQUIRES)))
if available_stream.TABLE in {'opportunity_applications',
'opportunity_offers',
'opportunity_referrals',
'opportunity_resumes',
'opportunity_feedback',
'opportunity_forms'}:
LOGGER.info('Will sync %s during the Opportunity stream sync', available_stream.TABLE)
opportunity_child_catalogs[available_stream.TABLE] = available_stream(self.config, self.state, stream_catalog, self.client)
else:
to_add = available_stream(self.config, self.state, stream_catalog, self.client)
streams.append(to_add)
return (streams, opportunity_child_catalogs)
def get_streams_to_replicate(config, state, catalog, client):
streams = []
campaign_substreams = []
list_substreams = []
for stream_catalog in catalog.streams:
if not is_selected(stream_catalog):
LOGGER.info("'{}' is not marked selected, skipping.".format(
stream_catalog.stream))
continue
for available_stream in AVAILABLE_STREAMS:
if available_stream.matches_catalog(stream_catalog):
if not available_stream.requirements_met(catalog):
raise RuntimeError(
"{} requires that that the following are selected: {}".
format(stream_catalog.stream,
','.join(available_stream.REQUIRES)))
to_add = available_stream(config, state, stream_catalog,
client)
if stream_catalog.stream in ['campaigns', 'lists']:
# the others will be triggered by these streams
streams.append(to_add)
elif stream_catalog.stream.startswith('campaign_'):
campaign_substreams.append(to_add)
to_add.write_schema()
elif stream_catalog.stream.startswith('list_'):
list_substreams.append(to_add)
to_add.write_schema()
return streams, campaign_substreams, list_substreams
def get_streams_to_replicate(self):
streams = []
if not self.catalog:
return streams
for stream_catalog in self.catalog.streams:
if not is_selected(stream_catalog):
LOGGER.info("'{}' is not marked selected, skipping.".format(
stream_catalog.stream))
continue
for available_stream in self.available_streams:
if available_stream.matches_catalog(stream_catalog):
if not available_stream.requirements_met(self.catalog):
raise RuntimeError(
"{} requires that that the following are "
"selected: {}".format(
stream_catalog.stream,
','.join(available_stream.REQUIRES)))
to_add = available_stream(self.config, self.state,
stream_catalog, self.client)
streams.append(to_add)
return streams