def sync_entity_chunked(entity_name, key_properties, path):
schema = load_schema(entity_name)
singer.write_schema(entity_name, schema, key_properties)
start = get_start(entity_name)
now_ts = int(datetime.datetime.utcnow().timestamp() * 1000)
start_ts = int(utils.strptime(start).timestamp() * 1000)
url = get_url(entity_name)
while start_ts < now_ts:
end_ts = start_ts + CHUNK_SIZES[entity_name]
params = {
'startTimestamp': start_ts,
'endTimestamp': end_ts,
'limit': 1000,
}
for row in gen_request(url, params, path, "hasMore", "offset",
"offset"):
record = transform(row, schema)
singer.write_record(entity_name, record)
utils.update_state(STATE, entity_name,
datetime.datetime.utcfromtimestamp(end_ts / 1000))
singer.write_state(STATE)
start_ts = end_ts
def sync_deals():
last_sync = utils.strptime(get_start("deals"))
days_since_sync = (datetime.datetime.utcnow() - last_sync).days
if days_since_sync > 30:
endpoint = "deals_all"
else:
endpoint = "deals_recent"
schema = load_schema("deals")
singer.write_schema("deals", schema, ["portalId", "dealId"])
url = get_url(endpoint)
params = {'count': 250}
for i, row in enumerate(
gen_request(url, params, "deals", "hasMore", "offset", "offset")):
record = request(get_url("deals_detail", deal_id=row['dealId'])).json()
record = transform(record, schema)
modified_time = None
if 'hs_lastmodifieddate' in record:
modified_time = utils.strptime(
record['hs_lastmodifieddate']['value'])
elif 'createdate' in record:
modified_time = utils.strptime(record['createdate']['value'])
if not modified_time or modified_time >= last_sync:
singer.write_record("deals", record)
utils.update_state(STATE, "deals", modified_time)
if i % 250 == 0:
singer.write_state(STATE)
def sync_contact_lists():
schema = load_schema("contact_lists")
singer.write_schema("contact_lists", schema, ["internalListId"])
start = get_start("contact_lists")
url = get_url("contact_lists")
params = {'count': 250}
for i, row in enumerate(
gen_request(url, params, "lists", "has-more", "offset", "offset")):
record = transform(row, schema)
singer.write_record("contact_lists", record)
def sync_owners():
schema = load_schema("owners")
singer.write_schema("owners", schema, ["portalId", "ownerId"])
start = get_start("owners")
data = request(get_url("owners")).json()
for row in data:
record = transform(row, schema)
if record['updatedAt'] >= start:
singer.write_record("owners", record)
utils.update_state(STATE, "owners", record['updatedAt'])
singer.write_state(STATE)
def sync_keywords():
schema = load_schema("keywords")
singer.write_schema("keywords", schema, ["keyword_guid"])
start = get_start("keywords")
data = request(get_url("keywords")).json()
for row in data['keywords']:
record = transform(row, schema)
if record['created_at'] >= start:
singer.write_record("keywords", record)
utils.update_state(STATE, "keywords", record['created_at'])
singer.write_state(STATE)
def sync_workflows():
schema = load_schema("workflows")
singer.write_schema("workflows", schema, ["id"])
start = get_start("workflows")
data = request(get_url("workflows")).json()
for row in data['workflows']:
record = transform(row, schema)
if record['updatedAt'] >= start:
singer.write_record("workflows", record)
utils.update_state(STATE, "workflows", record['updatedAt'])
singer.write_state(STATE)
def sync_campaigns():
schema = load_schema("campaigns")
singer.write_schema("campaigns", schema, ["id"])
url = get_url("campaigns_all")
params = {'limit': 500}
for i, row in enumerate(
gen_request(url, params, "campaigns", "hasMore", "offset",
"offset")):
record = request(get_url("campaigns_detail",
campaign_id=row['id'])).json()
record = transform(record, schema)
singer.write_record("campaigns", record)
def sync_contacts():
last_sync = utils.strptime(get_start("contacts"))
days_since_sync = (datetime.datetime.utcnow() - last_sync).days
if days_since_sync > 30:
endpoint = "contacts_all"
offset_keys = ['vid-offset']
offset_targets = ['vidOffset']
else:
endpoint = "contacts_recent"
offset_keys = ['vid-offset', 'time-offset']
offset_targets = ['vidOffset', 'timeOffset']
schema = load_schema("contacts")
singer.write_schema("contacts", schema, ["canonical-vid"])
url = get_url(endpoint)
params = {
'showListMemberships': True,
'count': 100,
}
vids = []
for row in gen_request(url, params, 'contacts', 'has-more', offset_keys,
offset_targets):
modified_time = None
if 'lastmodifieddate' in row['properties']:
modified_time = utils.strptime(
_transform_datetime(
row['properties']['lastmodifieddate']['value']))
if not modified_time or modified_time >= last_sync:
vids.append(row['vid'])
if len(vids) == 100:
data = request(get_url("contacts_detail"), params={
'vid': vids
}).json()
for vid, record in data.items():
record = transform(record, schema)
singer.write_record("contacts", record)
modified_time = None
if 'lastmodifieddate' in record['properties']:
modified_time = record['properties']['lastmodifieddate'][
'value']
utils.update_state(STATE, "contacts", modified_time)
vids = []
singer.write_state(STATE)
def sync_companies():
last_sync = utils.strptime(get_start("companies"))
days_since_sync = (datetime.datetime.utcnow() - last_sync).days
if days_since_sync > 30:
endpoint = "companies_all"
path = "companies"
more_key = "has-more"
offset_keys = ["offset"]
offset_targets = ["offset"]
else:
endpoint = "companies_recent"
path = "results"
more_key = "hasMore"
offset_keys = ["offset"]
offset_targets = ["offset"]
schema = load_schema('companies')
singer.write_schema("companies", schema, ["companyId"])
url = get_url(endpoint)
params = {'count': 250}
for i, row in enumerate(
gen_request(url, params, path, more_key, offset_keys,
offset_targets)):
record = request(
get_url("companies_detail", company_id=row['companyId'])).json()
record = transform(record, schema)
modified_time = None
if 'hs_lastmodifieddate' in record:
modified_time = utils.strptime(
record['hs_lastmodifieddate']['value'])
elif 'createdate' in record:
modified_time = utils.strptime(record['createdate']['value'])
if not modified_time or modified_time >= last_sync:
singer.write_record("companies", record)
utils.update_state(STATE, "companies", modified_time)
if i % 250 == 0:
singer.write_state(STATE)