-
Notifications
You must be signed in to change notification settings - Fork 0
/
data-import.py
48 lines (40 loc) · 1.64 KB
/
data-import.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
#!/usr/bin/python
import time
from novalabs.etl import jobProcessor
from novalabs import log
from novalabs import properties
import datetime
import logging
import AmazonHandler
import MeetupHandler
import SquareupHandler
def main():
logging.basicConfig(filename='/home/jleto/AmazonMeetupConnector/log/novalabs-data-import.log', level=logging.INFO)
projectProperties = properties.get('data-import.properties')
meetupProperties = properties.get('meetup.properties')
amazonProperties = properties.get('amazon.properties')
squareupProperties = properties.get('squareup.properties')
try:
log.writeInfo(' [GENERAL] Generating batches.')
result = jobProcessor.generate()
except Exception, e:
log.writeError(e)
job_cursor = jobProcessor.getReadyJobs()
log.writeInfo(' [GENERAL] Jobs to process: ('+str(job_cursor.rowcount)+').')
for job in job_cursor:
time.sleep(1)
jobDict = {}
jobDict['job_id'] = str(job[0])
jobDict['batch_id'] = str(job[1])
jobDict['batch_key'] = str(job[2])
jobDict['product_key'] = str(job[3])
if jobDict['product_key'] == 'meetup_payments':
MeetupHandler.process(jobDict, meetupProperties)
elif jobDict['product_key'] == 'amazon_payments':
AmazonHandler.process(jobDict, amazonProperties)
elif jobDict['product_key'] == 'squareup_payments':
SquareupHandler.process(jobDict, squareupProperties)
# logging.info('['+getTimeStamp()+'] [FINISH] Data import processing completed.')
log.writeInfo('[FINISH] Data import processing completed.')
if __name__ == "__main__":
main()