def gravity_user_apply(path, fetch_from, fetch_size, fetch_total): if os.path.exists(path): os.remove(path) write_header_tags(path) while True: print ">>fetching from: {}, size: {}".format(fetch_from, fetch_size) post = Post(index_url.format(fetch_from, fetch_size), credentials) payload = { "query": { "range": { "lastUpdateDate": { "gte": "now-1M" } } } } resp = post.make_request(payload) total = resp["hits"]["total"] print ">>es total: {}".format(total) if fetch_total == -1: fetch_total = total fetch_total = min(total, fetch_total) job_applications = [] for i in range(0, len(resp[u'hits'][u'hits'])): source = resp[u'hits'][u'hits'][i][u'_source'] if u'applyHistories' in source: for history in source[u'applyHistories']: job_applications.append({ "userId": source["jobseekerId"], "itemId": history["jobId"], "applyDate": int(parser.parse(history["applyDate"]).strftime('%s')) }) else: print "No applied history" write_job_application(job_applications, path) fetch_from = fetch_from + fetch_size if fetch_from > fetch_total: print ">>Stopping" break write_footer_tags(path) print ">>total: {}".format(fetch_from)