def gravity_user_apply(path, fetch_from, fetch_size, fetch_total):
if os.path.exists(path):
os.remove(path)
write_header_tags(path)
while True:
print ">>fetching from: {}, size: {}".format(fetch_from, fetch_size)
post = Post(index_url.format(fetch_from, fetch_size), credentials)
payload = {
"query": {
"range": {
"lastUpdateDate": {
"gte": "now-1M"
}
}
}
}
resp = post.make_request(payload)
total = resp["hits"]["total"]
print ">>es total: {}".format(total)
if fetch_total == -1:
fetch_total = total
fetch_total = min(total, fetch_total)
job_applications = []
for i in range(0, len(resp[u'hits'][u'hits'])):
source = resp[u'hits'][u'hits'][i][u'_source']
if u'applyHistories' in source:
for history in source[u'applyHistories']:
job_applications.append({
"userId": source["jobseekerId"],
"itemId": history["jobId"],
"applyDate": int(parser.parse(history["applyDate"]).strftime('%s'))
})
else:
print "No applied history"
write_job_application(job_applications, path)
fetch_from = fetch_from + fetch_size
if fetch_from > fetch_total:
print ">>Stopping"
break
write_footer_tags(path)
print ">>total: {}".format(fetch_from)
