def get_schemas():
schemas = {}
schemas_metadata = {}
streams = flatten_streams(STREAMS, {})
for stream_name, stream_object in streams.items():
LOGGER.info('Getting schema for {}'.format(stream_name))
schema_path = get_abs_path('schemas/{}.json'.format(stream_name))
with open(schema_path) as file:
schema = json.load(file)
refs = schema.pop("definitions", {})
if refs:
singer.resolve_schema_references(schema, refs)
meta = metadata.get_standard_metadata(
schema=schema,
key_properties=stream_object.key_properties,
replication_method=stream_object.replication_method
)
meta = metadata.to_map(meta)
if stream_object.replication_key:
meta = metadata.write(
meta, ('properties', stream_object.replication_key), 'inclusion', 'automatic')
meta = metadata.to_list(meta)
schemas[stream_name] = schema
schemas_metadata[stream_name] = meta
return schemas, schemas_metadata
def load_schema(tap_stream_id):
path = "schemas/{}.json".format(tap_stream_id)
schema = utils.load_json(get_abs_path(path))
refs = schema.pop("definitions", {})
if refs:
singer.resolve_schema_references(schema, refs)
return schema
def load_schema(tap_stream_id):
path = "schemas/{}.json".format(tap_stream_id)
schema = utils.load_json(get_abs_path(path))
dependencies = schema.pop("tap_schema_dependencies", [])
refs = {}
for sub_stream_id in dependencies:
refs[sub_stream_id] = load_schema(sub_stream_id)
if refs:
singer.resolve_schema_references(schema, refs)
return schema
def discover_schemas():
# Load Facebook's shared schemas
refs = load_shared_schema_refs()
result = {'streams': []}
streams = initialize_streams_for_discovery()
for stream in streams:
LOGGER.info('Loading schema for %s', stream.name)
schema = singer.resolve_schema_references(load_schema(stream), refs)
mdata = metadata.to_map(
metadata.get_standard_metadata(
schema, key_properties=stream.key_properties))
bookmark_key = BOOKMARK_KEYS.get(stream.name)
if bookmark_key == UPDATED_TIME_KEY or bookmark_key == CREATED_TIME_KEY:
mdata = metadata.write(mdata, ('properties', bookmark_key),
'inclusion', 'automatic')
result['streams'].append({
'stream': stream.name,
'tap_stream_id': stream.name,
'schema': schema,
'metadata': metadata.to_list(mdata)
})
return result
def generate_catalog(self):
cls = self.__class__
# get the reference schemas
refs = load_schema_references()
# resolve the schema reference and make final schema
schema = singer.resolve_schema_references(load_schema(cls.TABLE), refs)
mdata = metadata.new()
# use 'get_standard_metadata' with primary key, replication key and replication method
mdata = metadata.get_standard_metadata(
schema=schema,
key_properties=self.KEY_PROPERTIES,
valid_replication_keys=self.REPLICATION_KEYS
if self.REPLICATION_KEYS else None,
replication_method=self.REPLICATION_METHOD)
mdata_map = metadata.to_map(mdata)
# make 'automatic' inclusion for replication keys
for replication_key in self.REPLICATION_KEYS:
mdata_map[('properties',
replication_key)]['inclusion'] = 'automatic'
return [{
'tap_stream_id': cls.TABLE,
'stream': cls.TABLE,
'key_properties': cls.KEY_PROPERTIES,
'schema': schema,
'metadata': metadata.to_list(mdata_map)
}]
def discover():
raw_schemas = load_schemas()
streams = []
refs = load_schema_references()
for schema_name, schema in raw_schemas.items():
# stream_objects are intialized in the stream object
if schema_name not in Context.stream_objects:
continue
stream = Context.stream_objects[schema_name]()
# create and add catalog entry
catalog_entry = {
'stream': stream.name,
'tap_stream_id': schema_name,
'schema': singer.resolve_schema_references(schema, refs),
'metadata': get_discovery_metadata(stream, schema),
'key_properties': stream.key_properties,
'replication_key': stream.replication_key,
'replication_method': stream.replication_method
}
streams.append(catalog_entry)
return {'streams': streams}
def discover():
raw_schemas = load_schemas()
streams = []
refs = load_schema_references()
for schema_name, schema in raw_schemas.items():
if schema_name not in Context.stream_objects:
continue
stream = Context.stream_objects[schema_name]()
schema = singer.resolve_schema_references(schema, refs)
# create and add catalog entry
catalog_entry = {
'stream': schema_name,
'tap_stream_id': schema_name,
'schema': schema,
'metadata' : get_discovery_metadata(stream, schema),
'key_properties': stream.key_properties,
'replication_key': stream.replication_key,
'replication_method': stream.replication_method,
'column_order': [str(column) for column in schema['properties']]
}
streams.append(catalog_entry)
return {'streams': streams}
def discover():
initialize_shopify_client() # Checking token in discover mode
raw_schemas = load_schemas()
streams = []
refs = load_schema_references()
for schema_name, schema in raw_schemas.items():
if schema_name not in Context.stream_objects:
continue
stream = Context.stream_objects[schema_name]()
# resolve_schema_references() is changing value of passed refs.
# Customer is a stream and it's a nested field of orders and abandoned_checkouts streams
# and those 3 _sdc fields are also added inside nested field customer for above 2 stream
# so create a copy of refs before passing it to resolve_schema_references().
refs_copy = copy.deepcopy(refs)
catalog_schema = add_synthetic_key_to_schema(
singer.resolve_schema_references(schema, refs_copy))
# create and add catalog entry
catalog_entry = {
'stream': schema_name,
'tap_stream_id': schema_name,
'schema': catalog_schema,
'metadata': get_discovery_metadata(stream, schema),
'key_properties': stream.key_properties,
'replication_key': stream.replication_key,
'replication_method': stream.replication_method,
'column_order': [str(column) for column in catalog_schema['properties']]
}
streams.append(catalog_entry)
return {'streams': streams}
def sync_batches(self, stream_objects):
refs = load_shared_schema_refs()
schema = singer.resolve_schema_references(
self.catalog_entry.schema.to_dict(), refs)
transformer = Transformer(pre_hook=transform_date_hook)
# Create the initial batch
api_batch = API.new_batch()
batch_count = 0
# This loop syncs minimal fb objects
for obj in stream_objects:
# Execute and create a new batch for every 50 added
if batch_count % 50 == 0:
api_batch.execute()
api_batch = API.new_batch()
# Add a call to the batch with the full object
obj.api_get(fields=self.fields(),
batch=api_batch,
success=partial(batch_record_success,
stream=self,
transformer=transformer,
schema=schema),
failure=batch_record_failure)
batch_count += 1
# Ensure the final batch is executed
api_batch.execute()
def discover():
raw_schemas = load_schemas()
streams = []
for stream_name, stream_map in STREAM_SDK_OBJECTS.items():
schema = raw_schemas[stream_name]['schema']
refs = load_shared_schema_refs()
# create and add catalog entry
catalog_entry = {
'stream':
stream_name,
'tap_stream_id':
stream_name,
'schema':
singer.resolve_schema_references(schema, refs),
'metadata':
get_discovery_metadata(schema, stream_map['key_properties'],
'INCREMENTAL',
STREAM_REPLICATION_KEY.get(stream_name)),
# Events may have a different key property than this. Change
# if it's appropriate.
'key_properties':
stream_map['key_properties']
}
streams.append(catalog_entry)
return {'streams': streams}
def do_sync(account, catalog, state):
streams_to_sync = get_streams_to_sync(account, catalog, state)
refs = load_shared_schema_refs()
for stream in streams_to_sync:
LOGGER.info('Syncing %s, fields %s', stream.name, stream.fields())
schema = singer.resolve_schema_references(load_schema(stream), refs)
metadata_map = metadata.to_map(stream.catalog_entry.metadata)
bookmark_key = BOOKMARK_KEYS.get(stream.name)
singer.write_schema(stream.name, schema, stream.key_properties,
bookmark_key, stream.stream_alias)
# NB: The AdCreative stream is not an iterator
if stream.name in {'adcreative', 'leads'}:
stream.sync()
continue
with Transformer(pre_hook=transform_date_hook) as transformer:
with metrics.record_counter(stream.name) as counter:
for message in stream:
if 'record' in message:
counter.increment()
time_extracted = utils.now()
record = transformer.transform(message['record'],
schema,
metadata=metadata_map)
singer.write_record(stream.name, record,
stream.stream_alias,
time_extracted)
elif 'state' in message:
singer.write_state(message['state'])
else:
raise TapFacebookException(
'Unrecognized message {}'.format(message))
def generate_catalog(self):
schema = self.get_schema()
mdata = singer.metadata.new()
metadata = {
"forced-replication-method": self.REPLICATION_METHOD,
"valid-replication-keys": self.VALID_REPLICATION_KEYS,
"inclusion": self.INCLUSION,
#"selected-by-default": self.SELECTED_BY_DEFAULT,
"table-key-properties": self.KEY_PROPERTIES
}
for k, v in metadata.items():
mdata = singer.metadata.write(mdata, (), k, v)
for field_name, field_schema in schema.get('properties').items():
inclusion = 'available'
if field_name in self.KEY_PROPERTIES or field_name in self.BOOKMARK_PROPERTIES:
inclusion = 'automatic'
mdata = singer.metadata.write(mdata, ('properties', field_name),
'inclusion', inclusion)
refs = self.load_shared_schema_refs()
return [{
'tap_stream_id': self.TABLE,
'stream': self.TABLE,
'schema': singer.resolve_schema_references(schema, refs),
'metadata': singer.metadata.to_list(mdata)
}]
def sync_batches(self, stream_objects):
refs = load_shared_schema_refs()
schema = singer.resolve_schema_references(self.catalog_entry.schema.to_dict(), refs)
transformer = Transformer(pre_hook=transform_date_hook)
# Create the initial batch
api_batch = API.new_batch()
batch_count = 0
# Keep track of most recent, lead for bookmarking
latest_lead = None
# This loop syncs minimal fb objects
for obj in stream_objects:
latest_lead = self.compare_lead_created_times(latest_lead, obj)
# Execute and create a new batch for every 50 added
if batch_count % 50 == 0:
api_batch.execute()
api_batch = API.new_batch()
# Add a call to the batch with the full object
obj.api_get(fields=self.fields(),
batch=api_batch,
success=partial(batch_record_success, stream=self, transformer=transformer, schema=schema),
failure=batch_record_failure)
batch_count += 1
# Ensure the final batch is executed
api_batch.execute()
return str(pendulum.parse(latest_lead[self.replication_key]))
def discover():
LOGGER.info("Starting discover")
raw_schemas = load_schemas()
streams = []
refs = {}
for schema_name, schema in raw_schemas.items():
if schema_name not in Context.stream_objects:
continue
stream = Context.stream_objects[schema_name]()
# create and add catalog entry
catalog_entry = {
'stream': schema_name,
'tap_stream_id': schema_name,
'schema': singer.resolve_schema_references(schema, refs),
'metadata' : get_discovery_metadata(stream, schema),
'key_properties': stream.key_properties,
'replication_key': stream.replication_key,
'replication_method': stream.replication_method
}
streams.append(catalog_entry)
LOGGER.info("Finished discover")
return {'streams': streams}
def discover_streams(client, config):
streams = []
error_list = []
refs = load_shared_schema_refs()
for stream in STREAMS.values():
# for each stream in the `STREAMS` check if the user has the permission to access the data of that stream
stream = stream(client, config)
schema = singer.resolve_schema_references(stream.load_schema(), refs)
try:
# Here it call the check_access method to check whether stream have read permission or not.
# If stream does not have read permission then append that stream name to list and at the end of all streams
# raise forbidden error with proper message containing stream names.
stream.check_access()
except ZendeskForbiddenError as e:
error_list.append(
stream.name) # Append stream name to the error_list
except zenpy.lib.exception.APIException as e:
args0 = json.loads(e.args[0])
err = args0.get('error')
# check if the error is of type dictionary and the message retrieved from the dictionary
# is the expected message. If so, only then print the logger message and return the schema
if isinstance(err, dict):
if err.get(
'message', None
) == "You do not have access to this page. Please contact the account owner of this help desk for further help.":
error_list.append(stream.name)
elif args0.get(
'description'
) == "You are missing the following required scopes: read":
error_list.append(stream.name)
else:
raise e from None # raise error if it is other than 403 forbidden error
streams.append({
'stream': stream.name,
'tap_stream_id': stream.name,
'schema': schema,
'metadata': stream.load_metadata()
})
if error_list:
total_stream = len(STREAMS.values()) # Total no of streams
streams_name = ", ".join(error_list)
if len(error_list) != total_stream:
message = "The account credentials supplied do not have 'read' access to the following stream(s): {}. "\
"The data for these streams would not be collected due to lack of required permission.".format(streams_name)
# If atleast one stream have read permission then just print warning message for all streams
# which does not have read permission
LOGGER.warning(message)
else:
message ="HTTP-error-code: 403, Error: The account credentials supplied do not have 'read' access to any "\
"of streams supported by the tap. Data collection cannot be initiated due to lack of permissions."
# If none of the streams are having the 'read' access, then the code will raise an error
raise ZendeskForbiddenError(message)
return streams
def discover_streams(client):
streams = []
for s in STREAMS.values():
s = s(client)
schema = singer.resolve_schema_references(s.load_schema())
streams.append({'stream': s.name, 'tap_stream_id': s.name, 'schema': schema, 'metadata': s.load_metadata()})
return streams
def ad_creative_success(response, stream=None):
'''A success callback for the FB Batch endpoint used when syncing AdCreatives. Needs the stream
to resolve schema refs and transform the successful response object.'''
refs = load_shared_schema_refs()
schema = singer.resolve_schema_references(stream.catalog_entry.schema.to_dict(), refs)
rec = response.json()
record = Transformer(pre_hook=transform_date_hook).transform(rec, schema)
singer.write_record(stream.name, record, stream.stream_alias, utils.now())
def _map_to_schema(self, swagger: JsonResult) -> Schema:
schema_with_refs = {
**swagger["definitions"]["RecordDTO"],
**{
"definitions": swagger["definitions"]
}
}
schema = resolve_schema_references(schema_with_refs)
return Schema.from_dict(schema)
def load_schema(self):
schema_path = self.get_abs_path('schemas')
schema = singer.utils.load_json('{}/{}.json'.format(
schema_path, self.name))
if 'definitions' in schema:
resolved_schema = singer.resolve_schema_references(
schema, schema['definitions'])
del resolved_schema['definitions']
else:
resolved_schema = schema
return resolved_schema
def get_schemas():
schemas = {}
schemas_metadata = {}
schemas_path = get_abs_path('schemas')
file_names = [
f for f in os.listdir(schemas_path)
if os.path.isfile(os.path.join(schemas_path, f))
]
for file_name in file_names:
stream_name = file_name[:-5]
with open(os.path.join(schemas_path, file_name)) as data_file:
schema = json.load(data_file)
refs = schema.pop("definitions", {})
if refs:
singer.resolve_schema_references(schema, refs)
replication = STREAM_CONFIGS[stream_name]['replication']
meta = metadata.get_standard_metadata(
schema=schema,
key_properties=['id'],
replication_method='FULL_TABLE'
if replication == 'full' else replication.upper())
meta = metadata.to_map(meta)
if replication == 'incremental':
meta = metadata.write(
meta,
('properties', STREAM_CONFIGS[stream_name]['filter_field']),
'inclusion', 'automatic')
meta = metadata.to_list(meta)
schemas[stream_name] = schema
schemas_metadata[stream_name] = meta
return schemas, schemas_metadata
def discover_schemas():
# Load Facebook's shared schemas
refs = load_shared_schema_refs()
result = {'streams': []}
streams = initialize_streams_for_discovery()
for stream in streams:
LOGGER.info('Loading schema for %s', stream.name)
schema = singer.resolve_schema_references(load_schema(stream), refs)
result['streams'].append({'stream': stream.name,
'tap_stream_id': stream.name,
'schema': schema})
return result
def load_schemas():
schemas = {}
# This schema represents many of the currency values as JSON schema
# 'number's, which may result in lost precision.
for filename in os.listdir(get_abs_path('schemas')):
path = get_abs_path('schemas') + '/' + filename
file_raw = filename.replace('.json', '')
with open(path) as file:
raw_dict = json.load(file)
schema = singer.resolve_schema_references(raw_dict, raw_dict)
schemas[file_raw] = schema
return schemas
def discover_streams(client):
streams = []
refs = load_shared_schema_refs()
for s in STREAMS.values():
s = s(client)
schema = singer.resolve_schema_references(s.load_schema(), refs)
streams.append({
"stream": s.name,
"tap_stream_id": s.name,
"schema": schema,
"metadata": s.load_metadata(),
})
return streams
def discover():
raw_schemas = load_schemas()
streams = []
refs = load_shared_schema_refs()
for schema_name, schema in raw_schemas.items():
catalog_entry = {
'stream': schema_name,
'tap_stream_id': schema_name,
'schema': singer.resolve_schema_references(schema, refs=refs),
'metadata' : generate_metadata(schema),
'key_properties': ['id']
}
streams.append(catalog_entry)
return {'streams': streams}
def do_sync(account, catalog, state):
streams_to_sync = get_streams_to_sync(account, catalog, state)
refs = load_shared_schema_refs()
for stream in streams_to_sync:
fields = stream.fields()
LOGGER.info('Syncing %s, fields %s', stream.name, fields)
schema = singer.resolve_schema_references(load_schema(stream), refs)
schema['properties'] = {
k: v
for k, v in schema['properties'].items() if k in fields
}
metadata_map = metadata.to_map(stream.catalog_entry.metadata)
bookmark_key = BOOKMARK_KEYS.get(stream.name)
singer.write_schema(stream.name, schema, stream.key_properties,
bookmark_key, stream.stream_alias)
# NB: The AdCreative stream is not an iterator
if stream.name in {'adcreative', 'leads', 'ads'}:
stream.sync()
continue
with Transformer(pre_hook=transform_date_hook) as transformer:
with metrics.record_counter(stream.name) as counter:
for message in stream:
if 'record' in message:
counter.increment()
time_extracted = utils.now()
record = transformer.transform(message['record'],
schema,
metadata=metadata_map)
singer.write_record(stream.name, record,
stream.stream_alias,
time_extracted)
elif 'state' in message:
if message['state'].get('bookmarks'):
# Keep the global state in a global variable
# and update it each time with each stream's state
if STATE.get('bookmarks'):
STATE['bookmarks'].update(
message['state']['bookmarks'])
else:
STATE.update(message['state'])
singer.write_state(STATE)
else:
raise TapFacebookException(
'Unrecognized message {}'.format(message))
def load(self, name):
"""
Load schema from JSON and resolve shared $ref
"""
refs = {}
shared_file_names = [
f for f in os.listdir(self.shared_schemas_path)
if os.path.isfile(os.path.join(self.shared_schemas_path, f))
]
for shared_file in shared_file_names:
with open(os.path.join(self.shared_schemas_path,
shared_file)) as data_file:
refs[shared_file] = json.load(data_file)
schema_file = self.schema_path + "/{}.json".format(name)
with open(schema_file) as f:
schema = json.load(f)
schema = resolve_schema_references(schema, refs)
return schema
def discover_streams(client):
streams = []
for s in STREAMS.values():
s = s(client)
schema = singer.resolve_schema_references(s.load_schema())
# If stream is `users`, then get dynamic fields via API.
if s.name == "users":
res = s.client.get_user_fields()
fields = res["fields"]
# Merge `fields` with schema.
field_schema = translate_to_schema(fields)
schema = merge(schema, field_schema)
streams.append({
'stream': s.name,
'tap_stream_id': s.name,
'schema': schema,
'metadata': s.load_metadata()
})
return streams
def generate_catalog(self):
schema = self.get_schema()
mdata = singer.metadata.new()
metadata = {
"forced-replication-method": self.REPLICATION_METHOD,
"valid-replication-keys": self.VALID_REPLICATION_KEYS,
"inclusion": self.INCLUSION,
# "selected-by-default": self.SELECTED_BY_DEFAULT,
"table-key-properties": self.KEY_PROPERTIES,
}
for k, v in metadata.items():
mdata = singer.metadata.write(mdata, (), k, v)
for field_name, field_schema in schema.get("properties").items():
inclusion = "available"
if (field_name in self.KEY_PROPERTIES
or field_name in self.BOOKMARK_PROPERTIES):
inclusion = "automatic"
mdata = singer.metadata.write(mdata, ("properties", field_name),
"inclusion", inclusion)
cards = singer.utils.load_json(
os.path.normpath(
os.path.join(self.get_class_path(),
"../schemas/{}.json".format("cards"))))
refs = {"cards.json": cards}
return [{
"tap_stream_id": self.TABLE,
"stream": self.TABLE,
"schema": singer.resolve_schema_references(schema, refs),
"metadata": singer.metadata.to_list(mdata),
}]
def get_schema(self):
schema = self.load_schema_by_name(self.TABLE)
return singer.resolve_schema_references(schema, None)
def sync(self):
url = ''.join(('https://graph.facebook.com/v6.0/act_',
auth[0],
'?fields=ads.limit(5000)%7Badcreatives%7B',
'url_tags%7D%7D&access_token=',
auth[1]))
session = Session()
@retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5)
@utils.ratelimit(1, 1500)
def do_request():
return session.request('get', url)
def ad_creatives_get(retry=0):
retry_count = 0
resp = do_request()
# parse json structure and continue to get next set of data until end
if 'ads' in resp.json() and 'data' in resp.json().get('ads'):
ads = resp.json().get('ads')
transformed = []
for adcreatives in ads.get('data'):
transformed.extend(adcreatives.get('adcreatives').get('data'))
next_page = ads.get('paging').get('next')
while next_page:
resp = session.request('get', next_page)
# bad responses - re-execute same next_page call again
if resp.status_code == 200 and resp:
next_page = resp.json().get('paging').get('next')
data = resp.json().get('data')
transformed = []
for adcreatives in data:
transformed.extend(adcreatives.get('adcreatives').get('data'))
retry_count = 0
elif retry_count <= 2:
LOGGER.info('Retrying creative paging request after sleeping. \
Status: {status_code} Retry Count: {retry_count}'
.format(status_code=resp.status_code, retry_count=retry_count))
time.sleep(5)
retry_count += 1
continue
else:
raise Exception('Facebook creative API paging didnt \
load after 3 attempts!')
return transformed
elif retry < 3:
LOGGER.info('Retrying creative paging request after sleeping. \
Status: {status_code} Retry Count: {retry_count}'
.format(status_code=resp.status_code, retry_count=retry_count))
time.sleep(5)
retry += 1
ad_creatives_get(retry)
else:
if not resp:
raise Exception(resp.content)
raise Exception('Facebook creative API json ads was empty \
after 3 attempts!')
response = ad_creatives_get()
refs = load_shared_schema_refs()
schema = singer.resolve_schema_references(self.catalog_entry.schema.to_dict(), refs)
for rec in response:
record = Transformer(pre_hook=transform_date_hook).transform(rec, schema)
singer.write_record(self.name, record, self.stream_alias, utils.now())