def doIt():
keyPath = os.path.join(os.getcwd(), 'src', 'key.properties')
key = mylib.getKey(keyPath)
_key = key['dataseoul']
_url = 'http://openAPI.seoul.go.kr:8088'
_type = 'json'
_service = 'CardSubwayStatisticsService'
_start_index = 1
_end_index = 5
_use_mon = '201306'
_maxIter = 20
_iter = 0
_jfname = 'src/ds_open_subwayPassengers.json'
while _iter < _maxIter:
_api = os.path.join(_url, _key, _type, _service, str(_start_index),
str(_end_index), _use_mon)
_api = _api.replace('\\', '/')
#print _api
r = requests.get(_api)
_json = r.json()
for node in _json[_service]['row']:
for key in node.keys():
print key, node[key]
saveJson(_jfname, _json)
saveDB(_json)
_start_index += 5
_end_index += 5
_iter += 1
def doIt():
#make raw url
_url = 'http://openAPI.seoul.go.kr:8088/'
keyPath = os.path.join(os.getcwd(), 'src', 'key.properties')
key = mylib.getKey(keyPath)
KEY = str(key['dataseoul'])
TYPE = 'xml'
SERVICE = 'CardSubwayStatisticsService'
START_INDEX = 1
END_INDEX = 5
USE_MON = '201306' #line number
#make real url for parmas
params = os.path.join(_url, KEY, TYPE, SERVICE, str(START_INDEX),
str(END_INDEX), USE_MON)
url = urlparse.urljoin(_url, params)
url_real = ''
for i in params:
if i == '\\':
url_real += '/'
else:
url_real += i
#open
data = requests.get(url_real).text
tree = lxml.etree.fromstring(data.encode('utf-8'))
nodes = tree.xpath('//STATION_NM')
#print
for node in nodes:
print node.text
def doIt():
keyPath = os.path.join(os.getcwd(), 'src', 'key.properties')
key = mylib.getKey(keyPath)
_key = key['dataseoul '] #KEY='73725.....'
_url = 'http://openAPI.seoul.go.kr:8088'
_type = 'json'
_service = 'CardSubwayStatisticsService'
_start_index = 1
_end_index = 5
_use_mon = 201300
_maxIter = 12
_iter = 0
_jfname = 'src/ds_open_subwayPassengers.json'
while _iter < _maxIter:
_use_mon += 1
_api = _url + "/" + _key + "/" + _type + "/" + _service + "/" + str(
_start_index) + "/" + str(_end_index) + "/" + str(_use_mon)
#print _api
r = requests.get(_api)
_json = r.json()
print _json
saveJson(_jfname, _json)
saveDB(_json)
#_start_index+=5
#_end_index+=5
_iter += 1
def doIt():
keyPath=os.path.join(os.getcwd(),'key.properties')
key=mylib.getKey(keyPath)
_key=str(key['gokr'])
_url='http://openAPI.seoul.go.kr:8088'
_type='json'
_service='CardSubwayStatisticsService'
_start_index=1
_end_index=5
_use_mon='201306'
_maxIter=20
_iter=0
_jfname='src/ds_open_subwayPassengers.json'
while _iter<_maxIter:
_api=os.path.join(_url,_key,_type,_service,str(_start_index),str(_end_index),_use_mon)
#print _api
_api = _api.replace("\\",'/')
r=requests.get(_api)
_json=r.json()
print _json
saveJson(_jfname,_json)
saveDB(_json)
_start_index+=5
_end_index+=5
_iter+=1
def doIt():
keyPath = os.path.join(os.getcwd(), 'src', 'key.properties')
key = mylib.getKey(keyPath)
_key = key['dataseoul']
_url = 'http://openAPI.seoul.go.kr:8088'
_type = 'json'
_service = 'CardSubwayStatisticsService'
_start_index = 1
_end_index = 5
_maxIter = 20
_iter = 0
_jfname = 'src/ds_open_subwayPassengers.json'
while _iter < _maxIter:
for index in range(12):
_use_mon = '201301'
_use_mon = str(int(_use_mon) + index)
_api = os.path.join(_url, _key, _type, _service, str(_start_index),
str(_end_index), _use_mon)
r = requests.get(_api)
_json = r.json()
print _json
saveJson(_jfname, _json)
saveDB(_json)
_start_index += 5
_end_index += 5
_iter += 1
def main():
basic_url = 'http://openapi.airkorea.or.kr/openapi/services/rest/ArpltnInforInqireSvc'
service = 'getMsrstnAcctoRltmMesureDnsty'
param_dict = dict()
pageNum = 1
param_dict['numOfRows'] = '5'
param_dict['stationName'] = '종로구'
param_dict['dataTerm'] = 'MONTH'
keyPath = os.path.join('src', 'key.properties')
key_dict = mylib.getKey(keyPath)
mykey = key_dict['gokr']
param_dict['ServiceKey'] = mykey
param_dict['_returnType'] = 'json'
while(pageNum < 20):
param_dict['pageNo'] = "%s"%pageNum
params = urllib.urlencode(param_dict)
url = basic_url + '/' + service + '?'+'&' + url_encode_problem_sol.convert_urllib_urlencode(params)
json_data = requests.get(url).json()
save_file(json_data)
save_mongo(json_data)
pageNum += 1
def doIt():
# (1) service + operation
SERVICE = 'ArpltnInforInqireSvc'
OPERATION_NAME = 'getMinuDustFrcstDspth'
params1 = SERVICE + "/" + OPERATION_NAME
# (2) query params encoding
_d = dict()
_d['dataTerm'] = 'month'
params2 = urllib.urlencode(_d)
# (3) add my service key
keyPath = os.path.join(os.getcwd(), 'src', 'key.properties')
key = mylib.getKey(keyPath)
keygokr = key['gokr'] # keygokr='8Bx4C1%2B...'
params = params1 + '?' + 'serviceKey=' + keygokr + '&' + params2
# (4) make a full url
_url = 'http://openapi.airkorea.or.kr/openapi/services/rest/ArpltnInforInqireSvc'
url = urlparse.urljoin(_url, params)
#print url
# (5) get data
data = requests.get(url).text
print data
def doIt():
keyPath = os.path.join(os.getcwd(), 'src', 'key.properties')
key = mylib.getKey(keyPath)
KEY = str(key['dataseoul'])
TYPE = 'xml'
SERVICE = 'SearchSTNBySubwayLineService'
START_INDEX = str(1)
END_INDEX = str(10)
LINE_NUM = str(2)
params = os.path.join(KEY, TYPE, SERVICE, START_INDEX, END_INDEX, LINE_NUM)
url_real = ''
for i in params:
if i == '\\':
url_real += '/'
else:
url_real += i
_url = 'http://openAPI.seoul.go.kr:8088/'
url = urlparse.urljoin(_url, url_real)
data = requests.get(url).text
tree = lxml.etree.fromstring(data.encode('utf-8'))
nodes = tree.xpath('//STATION_NM')
for node in nodes:
print node.text
def GetData():
todayyear = datetime.today().year
todaymon = datetime.today().month
_key = mylib.getKey(os.path.join('key.properties'))
_type='xml'
_service= 'CardSubwayPayFree'
_start = str(1)
_end = str(302)
for year in range(2015,todayyear):
for mon in range(1,13):
_mon = str.format("%d%02d") %(year,mon)
url = os.path.join('http://openapi.seoul.go.kr:8088/',_key['dataseoul'],_type,_service,_start,_end,_mon)
url = url.replace('\\','/')
response = requests.get(url).text
tree = lxml.etree.fromstring(response.encode('utf-8'))
nodes = tree.xpath('//row')
cnt=0;
for node in nodes:
NM = node.xpath('.//SUB_STA_NM')[0].text
LINE = node.xpath('.//LINE_NUM')[0].text
for cnt,Station in enumerate(d):
tNM = Station['SUB_STA_NM']
tLINE = Station['LINE_NUM']
if NM == tNM and tLINE == LINE:
cnt=-1
Station[_mon] = int(node.xpath('.//FREE_RIDE_NUM')[0].text) + int(node.xpath('.//FREE_ALIGHT_NUM')[0].text)
break;
if cnt!=-1:
d.append(dict())
d[len(d)-1]['SUB_STA_NM'] = NM
d[len(d)-1]['LINE_NUM'] = LINE
d[len(d)-1][_mon] = int(node.xpath('.//FREE_RIDE_NUM')[0].text) + int(node.xpath('.//FREE_ALIGHT_NUM')[0].text)
for mon in range(1,todaymon):
_mon = str.format("%d%02d") %(todayyear,mon)
url = os.path.join('http://openapi.seoul.go.kr:8088/',_key['dataseoul'],_type,_service,_start,_end,_mon)
url = url.replace('\\','/')
response = requests.get(url).text
tree = lxml.etree.fromstring(response.encode('utf-8'))
nodes = tree.xpath('//row')
cnt=0;
for node in nodes:
NM = node.xpath('.//SUB_STA_NM')[0].text
LINE = node.xpath('.//LINE_NUM')[0].text
for cnt,Station in enumerate(d):
tNM = Station['SUB_STA_NM']
tLINE = Station['LINE_NUM']
if NM == tNM and tLINE == LINE:
cnt=-1
Station[_mon] = int(node.xpath('.//FREE_RIDE_NUM')[0].text) + int(node.xpath('.//FREE_ALIGHT_NUM')[0].text)
break;
if cnt!=-1:
d.append(dict())
d[len(d)-1]['SUB_STA_NM'] = NM
d[len(d)-1]['LINE_NUM'] = LINE
d[len(d)-1][_mon] = int(node.xpath('.//FREE_RIDE_NUM')[0].text) + int(node.xpath('.//FREE_ALIGHT_NUM')[0].text)
def doIt():
keyPath=os.path.join(os.getcwd(), 'src', 'key.properties')
key=mylib.getKey(keyPath)
KEY=str(key['dataseoul'])
TYPE='xml'
SERVICE='SearchSTNBySubwayLineService'
START_INDEX=str(1)
END_INDEX=str(10)
for i in range(1,10):
LINE_NUM=str(i)
linenum(KEY,TYPE,SERVICE,START_INDEX,END_INDEX,LINE_NUM)
def doIt():
keyPath=os.path.join(os.getcwd(), 'src', 'key.properties')
key=mylib.getKey(keyPath)
# (1) make params with resource IDs
KEY=key['dataseoul1']
TYPE='xml'
SERVICE='MosquitoStatus'
START_INDEX= '1'
END_INDEX= '4'
size=len(list_of_date1)
for i in range(size):
MOSQUITO_DATE=list_of_date1[i]
params=os.path.join(KEY,TYPE,SERVICE,START_INDEX,END_INDEX,MOSQUITO_DATE)
# (2) make a full url
_url='http://openAPI.seoul.go.kr:8088/'
url=urlparse.urljoin(_url,params)
url_real = ''
for i in url:
if i=='\\':
url_real += '/'
else:
url_real += i
# (3) get url
data=requests.get(url_real).text
# (4) get MOSQUITO_VALUE URL을 text형식으로 바꾸고 regex로 파싱한다
regex="<MOSQUITO_VALUE>(.*)</MOSQUITO_VALUE>"
print ">MOSQUITO_DATE: ",MOSQUITO_DATE
values = re.findall(regex, data)
values = map(str,values)
strvalues=",".join(values)
global values__1
flovalues=float(strvalues)
values__1.append(flovalues)
print "-MOSQUITO VALUE: ", strvalues
print "--------------------------"
#(5) mongodb에 데이터 넣기
client = MongoClient()
db = client.myProject #databae name
db.mosqudata.insert({"Mosquito_date":MOSQUITO_DATE,"Mosquito_count":strvalues})
def doIt():
_url = 'http://openAPI.seoul.go.kr:8088'
keyPath = os.path.join(os.getcwd(), 'src', 'key.properties')
key = mylib.getKey(keyPath)
_key = str(key['dataseoul'])
_type = 'xml'
_service = 'CardSubwayStatisticsService'
_start_index = 1
_end_index = 5
_use_mon = '201306'
_api = _url + '/' + _key + '/' + _type + '/' + _service + '/' + str(
_start_index) + '/' + str(_end_index) + '/' + _use_mon
response = requests.get(_api).text
print response
def doIt():
SERVICE = 'ArpltnInforInqireSvc'
OPERATION_NAME = 'getMinuDustFrcstDspth'
params1 = os.path.join(SERVICE, OPERATION_NAME)
_d = dict()
_d['dataTerm'] = 'month'
params2 = urllib.urlencode(_d)
keyPath = os.path.join(os.getcwd(), 'src', 'key.properties')
key = mylib.getKey(keyPath)
keygokr = key['gokr']
params = params1 + '?' + 'serviceKey=' + keygokr + '&' + params2
_url = 'http://openapi.airkorea.or.kr/openapi/services/rest/ArpltnInforInqireSvc'
url = urlparse.urljoin(_url, params)
data = requests.get(url).text
print data
def make_url(START_INDEX, END_INDEX):
import mylib, url_encode_problem_sol
basic_url = 'http://openapi.seoul.go.kr:8088'
key_path = os.path.join(os.getcwd(), 'src', 'key.properties')
key_dict = mylib.getKey(key_path)
my_key = key_dict['subject7']
TYPE = 'json'
SERVICE = 'CardSubwayStatsNew'
USE_DT = '20170518'
url = os.path.join(basic_url, my_key, TYPE, SERVICE, str(START_INDEX),
str(END_INDEX), USE_DT)
url_c = url_encode_problem_sol.convert_url_os_path_join(url)
return url_c
def quiz5():
_url = 'http://openAPI.seoul.go.kr:8088'
keyPath = os.path.join(os.getcwd(), 'src', 'key.properties')
key = mylib.getKey(keyPath)
_key = str(key['dataseoul'])
_type = 'json'
_service = 'CardSubwayStatisticsService'
_start_index = 1
_end_index = 5
_use_mon = '201501'
for index in range(12):
_use_mon = '201501'
_use_mon = str(int(_use_mon) + index)
_api = os.path.join(_url, _key, _type, _service, str(_start_index),
str(_end_index), _use_mon)
response = requests.get(_api).text
print response
def doIt():
keyPath = os.path.join(os.getcwd(), 'src', 'key.properties')
key = mylib.getKey(keyPath)
# (1) make params with resource IDs
KEY = key['dataseoul']
TYPE = 'json'
SERVICE = 'SearchSTNBySubwayLineService'
START_INDEX = str(1)
END_INDEX = str(10)
LINE_NUM = str(5)
params = os.path.join(KEY, TYPE, SERVICE, START_INDEX, END_INDEX, LINE_NUM)
# (2) make a full url
_url = 'http://openAPI.seoul.go.kr:8088/'
url = urlparse.urljoin(_url, params)
# (3) get data
data = requests.get(url).text
print data[:300]
def FineDust():
keyPath=os.path.join(os.getcwd(), 'src', 'key.properties')
key=mylib.getKey(keyPath)
KEY=key['finedust']
TYPE='json'
SERVICE='ForecastWarningMinuteParticleOfDustService'
START_INDEX=str(1)
END_INDEX=str(10)
params='/'+KEY+'/'+TYPE+'/'+SERVICE+'/'+START_INDEX+'/'+END_INDEX
_url='http://openAPI.seoul.go.kr:8088/6a7a77597073323035357466574e77/xml/ForecastWarningMinuteParticleOfDustService/1/1/'
url=urlparse.urljoin(_url,params)
data=requests.get(url).text
jd = json.loads(data)
for item in jd['ForecastWarningMinuteParticleOfDustService']['row']:
for i in item.keys():
if i=='ALARM_CNDT':
print item.values()[4]
def dolt():
SERVICE = 'ArpltnInforInqireSvc'
OPERATION_NAME = 'getMinuDustFrcstDspth'
params1 = os.path.join(SERVICE, OPERATION_NAME)
_d = dict()
params2 = urllib.urlencode(_d)
keyPath = os.path.join(os.getcwd(), 'src', 'key.properties')
key = mylib.getKey(keyPath)
keygokr = key['gokr']
_d['serviceKey'] = keygokr
_d['dataTerm'] = 'month'
params = params1 + '?' + url_encode_problem_sol.convert_urllib_urlencode(
urllib.urlencode(_d))
_url = 'http://openapi.airkorea.or.kr/openapi/services/rest/ArpltnInforInqireSvc'
url = urlparse.urljoin(_url, params)
url_c = url_encode_problem_sol.convert_url_os_path_join(url)
data = requests.get(url_c).text
print data[:500]
def doIt():
keyPath = os.path.join(os.getcwd() + '/key.properties')
key = mylib.getKey(keyPath)
# (1) make params with resource IDs
KEY = key['dataseoul']
TYPE = 'json'
SERVICE = 'ForecastWarningMinuteParticleOfDustService'
START_INDEX = str(1)
END_INDEX = str(5)
#LINE_NUM=str(2)
params = os.path.join(KEY, TYPE, SERVICE, START_INDEX, END_INDEX)
# (2) make a full url
_url = 'http://openAPI.seoul.go.kr:8088/'
url = urlparse.urljoin(_url, params)
url = url.replace('\\', '/')
# (3) get data
data = requests.get(url).text
print data
def save_blog_data(keyword):
#블로그
key_path = os.path.join('src', 'key.properties')
keys = mylib.getKey(key_path)
info = keys['naver'].split(',')
client_id = info[0]
client_secret = info[1]
#https://openapi.naver.com/v1/search/news.json - 뉴스
#https://openapi.naver.com/v1/search/kin.json - 지식IN (날짜 없음)
#https://openapi.naver.com/v1/search/cafearticle.json - 카페글 (날짜 없음)
#https://openapi.naver.com/v1/search/webkr.json - 웹문서
start = 1
client = MongoClient('localhost:27017')
db = client.bigData
count = 0
big_str = ''
while(start < 1000):
# json 결과, start -> 시작위치(최대 1000), dispaly -> 출력갯수(최대 100)
blog_str = ''
url = "https://openapi.naver.com/v1/search/blog?query=" + keyword + '&display=50' + '&start=' + str(start)
start = start + 100
headers_info = {"X-Naver-Client-Id": client_id, "X-Naver-Client-Secret":client_secret}
response = requests.get(url, headers=headers_info)
result_json = response.json()
for ele in result_json['items']:
blog_str = blog_str + ele['postdate'] + ele['title'] + '\n'
big_str = big_str + blog_str + '\n'
count = count+1
big_list = []
big_list.append(big_str)
#db.blogData.insert_one({
# "entire": big_list
#})
f = open(os.path.join('data', 'blog.txt'), 'w')
f.write(big_str.encode('utf8'))
f.close()
def doIt():
KeyPath = os.path.join(os.getcwd(), 'src', 'key.properties')
key = mylib.getKey(KeyPath)
KEY = str(key['dataseoul'])
TYPE = 'xml'
SERVICE = 'SearchSTNBySubwayLineService'
START_INDEX = str(1)
END_INDEX = str(10)
LINE_NUM = str(2)
params = KEY + '/' + TYPE + '/' + SERVICE + '/' + START_INDEX + '/' + END_INDEX + '/' + LINE_NUM
_url = 'http://openAPI.seoul.go.kr:8088/'
url = urlparse.urljoin(_url, params)
data = requests.get(url).text
p = re.compile('<STATION_NM>(.+?)</STATION_NM>')
res = p.findall(data)
for item in res:
print item
import os import requests import mylib keyPath=os.path.join(os.getcwd(), 'src', 'key.properties') key=mylib.getKey(keyPath) _url='http://openAPI.seoul.go.kr:8088' _key=str(key['dataseoul']) _type='xml' _service='CardSubwayStatisticsService' _start_index=1 _end_index=5 _use_mon='201306' _api=os.path.join(_url,_key,_type,_service,str(_start_index),str(_end_index),_use_mon) response = requests.get(_api).text print response
import os
import mylib
import requests
import json
srcPath = os.path.join(os.getcwd(), 'src', 'key.properties')
key = mylib.getKey(srcPath)
_url = 'http://openAPI.seoul.go.kr:8088'
_key = str(key['dataseoul'])
_type = 'json'
_service = 'CardSubwayStatisticsService'
_start_index = 1
_end_index = 5
_use_mon = '201306'
_maxIter = 2
_iter = 0
while _iter < _maxIter:
_api = os.path.join(_url, _key, _type, _service, str(_start_index),
str(_end_index), _use_mon)
_api = _api.replace('\\', '/')
#print _api
response = requests.get(_api).text
js = json.loads(response)
for item in js[_service]['row']:
for i in item.keys():
print item[i],
print
_start_index += 5
_end_index += 5
_iter += 1
import os import mylib import urlparse keyPath = os.path.join(os.getcwd(), 'key.properties') key_dict = mylib.getKey(keyPath) KEY = str(key_dict['dataseoul']) TYPE = 'json' SERVICE = 'SearchSTNBySubwayLineService' START_INDEX = str(1) END_INDEX = str(1000) LINE_NUM = str(1) params = os.path.join(KEY, TYPE, SERVICE, START_INDEX, END_INDEX, LINE_NUM) _url = 'http://openAPI.seoul.go.kr:8088/' url = urlparse.urljoin(_url, params) print url
def doIt2():
keyPath=os.path.join(os.getcwd(), 'src', 'key.properties')
key=mylib.getKey(keyPath)
# (1) make params with resource IDs
KEY=key['dataseoul2']
TYPE='xml'
SERVICE='DailyAverageAirQuality'
START_INDEX= '1'
END_INDEX= '143'
MSRSTE_NM='송파구'
size=len(list_of_date2)
print " "
print " "
for i in range(size):
MSRDT_DE = list_of_date2[i]
params=os.path.join(KEY,TYPE,SERVICE,START_INDEX,END_INDEX,MSRDT_DE,MSRSTE_NM)
# (2) make a full url
_url='http://openAPI.seoul.go.kr:8088/'
url=urlparse.urljoin(_url,params)
url_real = ''
for i in url:
if i=='\\':
url_real += '/'
else:
url_real += i
# (3) get url
data=requests.get(url_real).text
#(4) get VALUE URL을 text형식으로 바꾸고 regex로 파싱한다
regex1="<NO2>(.*)</NO2>"
regex2="<O3>(.*)</O3>"
regex3="<CO>(.*)</CO>"
regex4="<SO2>(.*)</SO2>"
print ">MSRDT_DE:",MSRDT_DE
values1 = re.findall(regex1, data)
values1 = map(str,values1)
strvalues1 = ",".join(values1)
global values__2
flovalues1=float(strvalues1)
values__2.append(flovalues1)
print "-NO2:", strvalues1
values2 = re.findall(regex2, data)
values2 = map(str,values2)
strvalues2 = ",".join(values2)
global values__3
flovalues2=float(strvalues2)
values__3.append(flovalues2)
print "-O3: ", strvalues2
values3 = re.findall(regex3, data)
values3 = map(str,values3)
strvalues3 = ",".join(values3)
global values__4
flovalues3=float(strvalues3)
values__4.append(flovalues3)
print "-CO: ", strvalues3
values4 = re.findall(regex4, data)
values4 = map(str,values4)
strvalues4 = ",".join(values4)
global values__5
flovalues4=float(strvalues4)
values__5.append(flovalues4)
print "-SO2:",strvalues4
print "--------------------------------"
#(5) mongodb에 데이터 넣기
client = MongoClient()
db = client.myProject #databae name
db.atmosdata.insert({"Atmos_date":MSRDT_DE,"Atmos_NO2":strvalues1,"Atmos_O3":strvalues2,"Atmos_C0":strvalues3,"Atmos_SO2":strvalues4})