def do_sync(self):
max_product_date = self.bookmark_date
product_response = self.client.make_request("/products/mine")
product_ids = []
product_types = []
with singer.metrics.Counter('record_count',
{'endpoint': 'products'}) as counter:
for product in product_response.json().values():
record = ProductRecord(product, self.schema)
product_ids.append(record.clean_data['id'])
product_types.append(record.clean_data['type'])
# Only upsert messages which have changed
if record.product_date > self.bookmark_date:
singer.write_message(
singer.RecordMessage(
stream='products',
record=product,
))
max_product_date = max(max_product_date,
record.product_date)
counter.increment()
self.state = singer.write_bookmark(self.state, self.STREAM_NAME,
'last_record',
date_to_str(max_product_date))
self.product_ids = product_ids
self.product_types = product_types
def do_sync(self):
bookmark_date_as_date = str_to_date(self.bookmark_date)
max_product_date = bookmark_date_as_date
product_response = self.client.make_request("/products/mine")
product_ids = []
with singer.metrics.Counter('record_count',
{'endpoint': 'products'}) as counter:
for product in product_response.json().values():
product_ids.append(product['id'])
# Only upsert messages which have changed
product_date = product['updated_date'] if product['updated_date']\
else product['added_date']
product_date = str_to_date(product_date)
product = tidy_dates(product)
if product_date > bookmark_date_as_date:
singer.write_message(
singer.RecordMessage(
stream='products',
record=product,
))
max_product_date = max(max_product_date, product_date)
counter.increment()
self.state = singer.write_bookmark(self.state, self.STREAM_NAME,
'last_record',
date_to_str(max_product_date))
self.product_ids = product_ids
def do_sync(self):
start_date = self.bookmark_date
new_bookmark_date = self.bookmark_date
# Ranks cannot be fetched for inapp
product_ids = ','.join([
str(id) for i, id in enumerate(self.product_ids)
if self.product_types[i] != "inapp"
])
if any(
[product_type == "inapp" for product_type in self.product_types]):
LOGGER.info(
"Skipping id={} since ranks cannot be fetched for inapp purchases."
.format(','.join([
str(id) for i, id in enumerate(self.product_ids)
if self.product_types[i] == "inapp"
])))
while start_date.date() <= date.today():
end_date = start_date + timedelta(days=28)
uri = '/ranks/{}/daily/{}/{}'.format(
product_ids, start_date.strftime('%Y-%m-%d'),
end_date.strftime('%Y-%m-%d'))
data = self.client.make_request(uri).json()
rank_dates = data['dates']
rank_data = data['data']
with singer.metrics.Counter('record_count',
{'endpoint': 'ranks'}) as counter:
for rank_entry in rank_data:
for i, rank_date in enumerate(rank_dates):
record = RankRecord(
dict(
country=rank_entry['country'],
category=rank_entry['category'],
product_id=rank_entry['product_id'],
position=rank_entry['positions'][i],
delta=rank_entry['deltas'][i],
date=rank_date,
), self.schema)
new_bookmark_date = max(new_bookmark_date,
record.bookmark)
singer.write_message(
singer.RecordMessage(
stream=self.STREAM_NAME,
record=record.for_export,
))
counter.increment()
self.state = singer.write_bookmark(self.state, self.STREAM_NAME,
'last_record',
date_to_str(new_bookmark_date))
start_date = end_date
def do_sync(self):
start_date = str_to_date(self.bookmark_date)
new_bookmark_date = start_date
product_ids = ';'.join(str(id) for id in self.product_ids)
while start_date.date() <= date.today():
end_date = start_date + timedelta(days=28)
uri = '/ranks/{}/daily/{}/{}'.format(
product_ids, start_date.strftime('%Y-%m-%d'),
end_date.strftime('%Y-%m-%d'))
data = self.client.make_request(uri).json()
rank_dates = data['dates']
rank_data = data['data']
with singer.metrics.Counter('record_count',
{'endpoint': 'ranks'}) as counter:
for rank_entry in rank_data:
for i, rank_date in enumerate(rank_dates):
record_data = dict(
country=rank_entry['country'],
category=rank_entry['category'],
product_id=rank_entry['product_id'],
position=rank_entry['positions'][i],
delta=rank_entry['deltas'][i],
date=rank_date,
)
new_bookmark_date = max(
new_bookmark_date,
str_to_date(record_data['date']))
record_data = strings_to_floats(record_data)
singer.write_message(
singer.RecordMessage(
stream=self.STREAM_NAME,
record=record_data,
))
counter.increment()
self.state = singer.write_bookmark(self.state, self.STREAM_NAME,
'last_record',
date_to_str(new_bookmark_date))
start_date = end_date