-
Notifications
You must be signed in to change notification settings - Fork 0
/
download_issue.py
87 lines (76 loc) · 2.53 KB
/
download_issue.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
#!/usr/bin/python
# -*- coding: utf-8 -*-
__author__ = "Wei Wang"
__email__ = "tskatom@vt.edu"
import sys
import os
import logging
import pytz
from datetime import datetime, timedelta
import time
import urllib2
import xlrd
import zmq
import calendar
def download(url, timeout=30):
excel = urllib2.urlopen(url, timeout=timeout).read()
wb = xlrd.open_workbook(file_contents=excel)
sb = wb.sheet_by_index(0)
issues = []
for i in range(1, sb.nrows):
issue = sb.row_values(i)
issues.append(issue)
return issues
def run():
now = datetime.now(tz).isoformat()
logging.info('Start Ingesting %s' % now)
url_format = "http://trend.caipiao.163.com/downloadTrendAwardNumber.html?gameEn=ssc&beginPeriod=%s&endPeriod=%s"
with open('./issues/issues_2.csv', 'w') as issue_file:
now = datetime.now(tz)
next_day = now + timedelta(days=1)
end_issue = "%s120" % next_day.strftime('%y%m%d')
url = url_format % (latest, end_issue)
download_issues = download(url)
for issue in download_issues:
issue_file.write("%s\n" % '|'.join(issue))
def main():
# get past 30 months data
tz = pytz.timezone(pytz.country_timezones('cn')[0])
now = datetime.now(tz).isoformat()
if len(sys.argv) > 1:
months = int(sys.argv[1])
else:
months = 30
current_day = datetime.now(tz)
url_format = "http://trend.caipiao.163.com/downloadTrendAwardNumber.html?gameEn=ssc&beginPeriod=%s&endPeriod=%s"
issues = []
for i in range(months):
end = "%s120" % current_day.strftime("%y%m%d")
begin_day = current_day - timedelta(days=30)
start = "%s001" % (begin_day.strftime("%y%m%d"))
print "%d : From %s to %s " % (i, start, end)
current_day = begin_day - timedelta(days=1)
url = url_format % (start, end)
max_try = 3
done = False
tried = 0
while tried < max_try and not done:
try:
batch = download(url)
done = True
except:
tried += 1
if not done:
print "Failed to download %d" % i
break
print 'Finish %d' % i
issues.insert(0, batch)
# write to file
with open('./issues/historical_issues.csv', 'a+') as f:
old_issues = {l.strip().split('|')[0]:1 for l in f}
for batch in issues:
for issue in batch:
if issue[0] not in old_issues:
f.write("%s\n" % '|'.join(issue))
if __name__ == "__main__":
main()