forked from newslynx/newslynx-core
/
init.py
160 lines (130 loc) · 4.22 KB
/
init.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
"""
Utilities for initializing NewsLynx
"""
import os
from newslynx.lib.serialize import yaml_to_obj
from newslynx.exc import ConfigError
from newslynx import settings
from newslynx.models import sous_chef_schema
from newslynx.util import recursive_listdir, here
# directory of built-in sous chefs
SOUS_CHEF_DIR = here(__file__, 'sc')
# directory of built-in sql functions
SQL_DIR = here(__file__, 'sql')
def _load_config_file(fp):
"""
Attempt to load a file or raise a ConfigError
"""
try:
return yaml_to_obj(open(fp).read())
except Exception as e:
raise ConfigError(
"There was an error loding config '{}'.\n"
"Here is the error: \n{}"
.format(fp, e.message))
def _is_config_file(fp):
"""
Check if a file can be parsed as yaml.
"""
return (
fp.endswith('json') or
fp.endswith('yaml') or
fp.endswith('yml')
)
def load_sous_chef(fp):
"""
Load and validate a sous chef config file.
"""
sc = _load_config_file(fp)
return sous_chef_schema.validate(sc)
def load_sous_chefs():
"""
Get all internal and user-specified sous chef configurations.
"""
# internal sous chefs
for fp in recursive_listdir(SOUS_CHEF_DIR):
if _is_config_file(fp):
yield load_sous_chef(fp)
# user-generated sous-chefs.
if hasattr(settings, 'SOUS_CHEFS_DIR'):
sous_chef_dir = settings.SOUS_CHEFS_DIR
if sous_chef_dir.startswith('~'):
sous_chef_dir = \
os.path.expanduser(sous_chef_dir)
if not os.path.exists(sous_chef_dir):
raise ConfigError(
"'{}' was explicitly declared as "
"the sous_chef_dir but could "
"not be found."
.format(sous_chef_dir)
)
else:
sous_chef_dir = os.path.expanduser(
'~/.newslynx/sous-chefs/')
if os.path.exists(sous_chef_dir):
for fp in recursive_listdir(sous_chef_dir):
if _is_config_file(fp):
yield load_sous_chef(fp)
def load_default_tags():
"""
Get all default tags for specified in the conf
"""
# user-generated sous-chefs.
if hasattr(settings, 'DEFAULT_TAGS'):
default_tags = settings.DEFAULT_TAGS
if default_tags.startswith('~'):
default_tags = \
os.path.expanduser(default_tags)
if not os.path.exists(default_tags):
raise ConfigError(
"'{}' was explicitly declared as "
"the default_tags config but could "
"not be found."
.format(default_tags)
)
else:
default_tags = os.path.expanduser(
'~/.newslynx/defaults/tags.yaml')
if os.path.exists(default_tags):
tags = _load_config_file(default_tags)
if not isinstance(tags, list):
raise ConfigError(
'Default tags config must be a list of objects.'
)
for t in tags:
yield t
def load_default_recipes():
"""
Get all default tags for organizations specified in the conf.
"""
# user-generated sous-chefs.
if hasattr(settings, 'DEFAULT_RECIPES'):
default_recipes = settings.DEFAULT_RECIPES
if default_recipes.startswith('~'):
default_recipes = \
os.path.expanduser(default_recipes)
if not os.path.exists(default_recipes):
raise ConfigError(
"'{}' was explicitly declared as "
"the default_recipes config but could "
"not be found."
.format(default_recipes)
)
else:
default_recipes = os.path.expanduser(
'~/.newslynx/defaults/recipes.yaml')
if os.path.exists(default_recipes):
recipes = _load_config_file(default_recipes)
if not isinstance(recipes, list):
raise ConfigError(
'Default recipe config files must contain a list of objects.'
)
for r in recipes:
yield r
def load_sql():
"""
Get all sql files.
"""
for fp in sorted(list(recursive_listdir(SQL_DIR))):
if fp.endswith('sql'):
yield open(fp).read()