forked from openspending/dpkg-uk25k
/
common.py
62 lines (51 loc) · 1.72 KB
/
common.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
import os
import json
import logging
import datetime
import ConfigParser
import sqlaload as sl
import nomenklatura
from ckanclient import CkanClient
logging.basicConfig(level=logging.NOTSET)
logging.getLogger('sqlaload').setLevel(level=logging.WARN)
logging.getLogger('requests').setLevel(level=logging.WARN)
log = logging.getLogger('common')
def issue(engine, resource_id, resource_hash, stage, message,
data={}):
table = sl.get_table(engine, 'issue')
log = logging.getLogger('issue')
log.debug("R[%s]: %s", resource_id, message)
sl.add_row(engine, table, {
'resource_id': resource_id,
'resource_hash': resource_hash,
'timestamp': datetime.datetime.utcnow(),
'stage': stage,
'message': message,
'data': json.dumps(data)
})
def source_path(row):
source_dir = config_get('resource-cache.dir')
if not os.path.isdir(source_dir):
os.makedirs(source_dir)
return os.path.join(source_dir, row['resource_id'])
config = None
def config_get(option):
global config
if not config:
config = ConfigParser.ConfigParser()
config.read(['default.ini', 'config.ini'])
return config.get('uk25k', option)
def ckan_client():
ckan_api = config_get('ckan-api.url')
return CkanClient(base_location='http://data.gov.uk/api')
def db_connect():
sqlalchemy_url = config_get('sqlalchemy.url')
log.info('Using database: %s', sqlalchemy_url)
return sl.connect(sqlalchemy_url)
NK_DATASETS = {}
def nk_connect(dataset):
if not dataset in NK_DATASETS:
NK_DATASETS[dataset] = nomenklatura.Dataset(
dataset,
api_key='beaf2ff2-ea94-47c0-942f-1613a09056c2')
return NK_DATASETS[dataset]