def create_setting(user, org, level):
if level not in ['me', 'orgs']:
raise NotFoundError(
'You cannot store settings for \'{}\''
.format(level))
# get the request data
req_data = request_data()
name = req_data.get('name')
value = req_data.get('value')
json_value = req_data.get('json_value', False)
if not name or not value:
raise RequestError(
"You must pass in a 'name' and 'value' to create a setting. "
"You only passed in: {}"
.format(", ".join(req_data.keys())))
# if it's a json_value check whether we can parse it as such
if json_value:
if isinstance(value, basestring):
try:
json_to_obj(value)
except:
raise RequestError(
"Setting '{}' with value '{}' was declared as a "
"'json_value' but could not be parsed as such."
.format(name, value))
s = Setting(
org_id=org.id,
user_id=user.id,
level=level,
name=name,
value=value,
json_value=json_value or False)
db.session.add(s)
# no duplicates.
try:
db.session.commit()
except Exception as e:
raise ConflictError(e.message)
# temporary hack for 'timezone' setting in the APP.
if 'name' == 'timezone' and level == 'orgs':
org.timezone = value
try:
db.session.add(org)
db.session.commit()
except Exception as e:
raise RequestError(
"An error occurred while updating the timezone. "
"Here's the error message: {}"
.format(org.name, e.message))
return jsonify(s)
def create_setting(user, org, level):
if level not in ['me', 'orgs']:
raise NotFoundError(
'You cannot store settings for \'{}\''.format(level))
# get the request data
req_data = request_data()
name = req_data.get('name')
value = req_data.get('value')
json_value = req_data.get('json_value', False)
if not name or not value:
raise RequestError(
"You must pass in a 'name' and 'value' to create a setting. "
"You only passed in: {}".format(", ".join(req_data.keys())))
# if it's a json_value check whether we can parse it as such
if json_value:
if isinstance(value, basestring):
try:
json_to_obj(value)
except:
raise RequestError(
"Setting '{}' with value '{}' was declared as a "
"'json_value' but could not be parsed as such.".format(
name, value))
s = Setting(org_id=org.id,
user_id=user.id,
level=level,
name=name,
value=value,
json_value=json_value or False)
db.session.add(s)
# no duplicates.
try:
db.session.commit()
except Exception as e:
raise ConflictError(e.message)
# temporary hack for 'timezone' setting in the APP.
if 'name' == 'timezone' and level == 'orgs':
org.timezone = value
try:
db.session.add(org)
db.session.commit()
except Exception as e:
raise RequestError(
"An error occurred while updating the timezone. "
"Here's the error message: {}".format(org.name, e.message))
return jsonify(s)
def load_data(path_or_string):
"""
Load data in from a filepath or a string
"""
if not path_or_string:
return {}
# treat it as a file first
if acceptable_file(path_or_string):
fp = os.path.expanduser(path_or_string)
try:
return serialize.yaml_to_obj(open(fp).read())
except Exception as e:
raise IOError(
"Could not read file '{}' \n{}"
.format(fp, e.message))
# otherwise assume it's a JSON blob
else:
try:
return serialize.json_to_obj(path_or_string)
except ValueError as e:
raise ValueError(
"Could not parse JSON string '{}' \n{}"
.format(path_or_string, e.message))
def load_data(path_or_string, opts):
"""
Load data in from a filepath or a string
"""
kwargs = {}
if not path_or_string:
return kwargs
try:
d = serialize.json_to_obj(path_or_string)
kwargs.update(d)
return kwargs
except ValueError as e:
# only deal with these formats.
if not ( path_or_string.endswith('.yaml') or
path_or_string.endswith('.yml') or
path_or_string.endswith('.json') ):
return kwargs
fp = os.path.expand_user(path_or_string)
try:
kwargs.update(serialize.yaml_to_obj(fp.read()))
return kwargs
except Exception as e:
pass
echo_error(RuntimeError("Could not parse input data:\n'{}'".format(e.message)),
no_color=opts.no_color)
sys.exit(1)
def update_setting(user, org, name_id):
s = fetch_by_id_or_field(Setting, 'name', name_id, org_id=org.id)
if not s:
raise NotFoundError(
'Setting "{}" does not yet exist for Org "{}"'
.format(name_id, org.name))
# get the request data
req_data = request_data()
name = req_data.get('name')
value = req_data.get('value')
json_value = req_data.get('json_value')
# if it's a json_value check whether we can parse it as such
if json_value:
if isinstance(value, basestring):
try:
json_to_obj(value)
except:
raise RequestError(
"Setting '{}' with value '{}' was declared as a "
"'json_value' but could not be parsed as such."
.format(name_id, value))
# upsert / patch values.
if name:
s.name = name
if value:
s.value = value
if json_value:
if not isinstance(json_value, bool):
if str(json_value).lower() in TRUE_VALUES:
json_value = True
else:
json_value = False
s.json_value = json_value
db.session.add(s)
db.session.commit()
return jsonify(s)
def _generate():
for line in r.iter_lines():
d = json_to_obj(line)
# catch errors
if d.get('error'):
err = ERRORS.get(d['error'])
if not err:
raise ClientError(d)
raise err(d['message'])
yield d
def settings_dict(self):
"""
An org's settings formatted as a dictionary.
"""
settings = {}
for s in self.settings:
if s.json_value:
v = json_to_obj(s.value)
else:
v = copy.copy(s.value)
settings[s.name] = v
return settings
def to_dict(self):
v = copy.copy(self.value)
if self.json_value:
v = json_to_obj(v)
return {
'id': self.id,
'org_id': self.org_id,
'name': self.name,
'value': v,
'json_value': self.json_value
}
def settings_dict(self):
"""
An org's settings formatted as a dictionary.
"""
settings = {}
for s in self.settings:
if s.json_value:
v = json_to_obj(s.value)
else:
v = copy.copy(s.value)
settings[s.name] = v
return settings
def request_bulk_data():
"""
Fetch request data from jsonlines.
"""
data = request.get_json(silent=True)
if data is None:
try:
data = json_to_obj(request.data)
except:
data = None
if data is None:
data = dict(request.form.items())
return data
def request_data():
"""
Fetch request data from json / form / raw json string.
"""
data = request.get_json(silent=True)
if data is None:
try:
data = json_to_obj(request.data)
except:
data = None
if data is None:
data = dict(request.form.items())
return data
def bulkworker(job_id, **qkw):
"""
Fetch a job and execute it.
"""
start = time.time()
try:
k = qkw['job_key_fmt'].format(job_id)
job = rds.get(k)
if not job:
raise InternalServerError(
'An unexpected error occurred while processing bulk upload.'
)
if qkw['serializer'] == 'json':
job = json_to_obj(job)
elif qkw['serializer'] == 'pickle':
job = pickle_to_obj(job)
data = job.pop('data', [])
job = job.pop('kw', {})
# delete them
rds.delete(k)
# chunk list
chunked_data = util.chunk_list(data, qkw.get('chunk_size'))
# partial funtion
load_fx = partial(ingest.source, **job)
# pooled execution
pool = Pool(qkw.get('max_workers', MAX_WORKERS))
for res in pool.imap_unordered(load_fx, chunked_data):
pass
return True
except Exception:
tb = format_exc()
raise RequestError('An Error Ocurred while running {}:\n{}'.format(job_id, tb))
except JobTimeoutException:
end = time.time()
raise InternalServerError(
'Bulk loading timed out after {} seconds'
.format(end-start))
def get_json(_u, **params):
"""
Fetches json from a url.
"""
session = requests.Session()
response = session.get(url=_u, params=params, **get_request_kwargs())
obj = None
if response.encoding != FAIL_ENCODING:
content = response.text
else:
content = response.content
try:
obj = json_to_obj(content)
except Exception as e:
log.warning('Unable to parse json from {}. Messsage: {}'.format(_u, e.message))
return obj
return obj
def bulkworker(job_id, **qkw):
"""
Fetch a job and execute it.
"""
start = time.time()
try:
k = qkw['job_key_fmt'].format(job_id)
job = rds.get(k)
if not job:
raise InternalServerError(
'An unexpected error occurred while processing bulk upload.')
if qkw['serializer'] == 'json':
job = json_to_obj(job)
elif qkw['serializer'] == 'pickle':
job = pickle_to_obj(job)
data = job.pop('data', [])
job = job.pop('kw', {})
# delete them
rds.delete(k)
# chunk list
chunked_data = util.chunk_list(data, qkw.get('chunk_size'))
# partial funtion
load_fx = partial(ingest.source, **job)
# pooled execution
pool = Pool(qkw.get('max_workers', MAX_WORKERS))
for res in pool.imap_unordered(load_fx, chunked_data):
pass
return True
except Exception:
tb = format_exc()
raise RequestError('An Error Ocurred while running {}:\n{}'.format(
job_id, tb))
except JobTimeoutException:
end = time.time()
raise InternalServerError(
'Bulk loading timed out after {} seconds'.format(end - start))
def get_json(_u, **params):
"""
Fetches json from a url.
"""
session = gen_session()
response = session.get(url=_u, params=params, **get_request_kwargs())
obj = None
if response.encoding != FAIL_ENCODING:
content = response.text
else:
content = response.content
try:
obj = json_to_obj(content)
except Exception as e:
log.warning('Unable to parse json from {}. Messsage: {}'.format(
_u, e.message))
return obj
return obj
def update(old_sous_chef, new_sous_chef):
"""
Given a partial or completely new sous-chef, update the souf-chef
and re-validate it.
"""
# if the old sous chef is a SousChef object, coerce it to and from json.
if isinstance(old_sous_chef, SousChef):
old_sous_chef = json_to_obj(obj_to_json(old_sous_chef))
# pop the id
old_sous_chef.pop('id', None)
# update the previous version.
new_sous_chef = update_nested_dict(
old_sous_chef, new_sous_chef, overwrite=True)
return validate(new_sous_chef, None)
def update(old_sous_chef, new_sous_chef):
"""
Given a partial or completely new sous-chef, update the souf-chef
and re-validate it.
"""
# if the old sous chef is a SousChef object, coerce it to and from json.
if isinstance(old_sous_chef, SousChef):
old_sous_chef = json_to_obj(obj_to_json(old_sous_chef))
# pop the id
old_sous_chef.pop('id', None)
# update the previous version.
new_sous_chef = update_nested_dict(old_sous_chef,
new_sous_chef,
overwrite=True)
return validate(new_sous_chef)
def _stream(self, r):
for line in r.iter_lines():
d = json_to_obj(line)
# catch bad responses:
if not isinstance(d, dict):
e = 'InternalServerError'
err_msg = 'Invalid Response: {}'.format(d)
if self._raise_errors:
raise ERRORS[e](err_msg)
yield {'status_code': 500, "error": e, "message": err_msg}
# catch errors
elif d.get('error'):
if self._raise_errors:
err = ERRORS.get(d['error'])
if not err:
raise ClientError(d)
raise err(d['message'])
yield d
def update(old_recipe, new_recipe, sous_chef):
"""
Given a partial or completely new recipe, update the old recipe
and re-validate it.
"""
# if the old recipe is a Recipe object, coerce it to and from json.
if isinstance(old_recipe, Recipe):
old_recipe = json_to_obj(obj_to_json(old_recipe))
# format it correctly first.
_rs = RecipeSchema(new_recipe, sous_chef)
_rs.format_recipe()
new_recipe = copy.copy(_rs.recipe)
# update the previous version.
new_recipe = update_nested_dict(old_recipe, new_recipe, overwrite=True)
# revalidate.
rs = RecipeSchema(new_recipe, sous_chef)
return rs.validate()
def update(old_recipe, new_recipe, sous_chef):
"""
Given a partial or completely new recipe, update the old recipe
and re-validate it.
"""
# if the old recipe is a Recipe object, coerce it to and from json.
if isinstance(old_recipe, Recipe):
old_recipe = json_to_obj(obj_to_json(old_recipe))
# format it correctly first.
_rs = RecipeSchema(new_recipe, sous_chef)
_rs.format_recipe()
new_recipe = copy.copy(_rs.recipe)
# update the previous version.
new_recipe = update_nested_dict(old_recipe, new_recipe, overwrite=True)
# revalidate.
rs = RecipeSchema(new_recipe, sous_chef)
return rs.validate()
def execute(self, query, **kw):
"""
Execute a sql command and stream resutls.
"""
# merge in api key
kw.update({'apikey': self.apikey})
url = self._format_url('sql')
# post request
r = requests.post(url, params=kw, data={'query': query})
# stream results.
for line in r.iter_lines():
d = json_to_obj(line)
# catch errors
if d.get('error'):
err = ERRORS.get(d['error'])
if not err:
raise ClientError(d)
raise err(d['message'])
yield d
def load_data(path_or_string):
"""
Load data in from a filepath or a string
"""
if not path_or_string:
return {}
# treat it as a file first
if acceptable_file(path_or_string):
fp = os.path.expanduser(path_or_string)
try:
return serialize.yaml_to_obj(open(fp).read())
except Exception as e:
raise IOError("Could not read file '{}' \n{}".format(
fp, e.message))
# otherwise assume it's a JSON blob
else:
try:
return serialize.json_to_obj(path_or_string)
except ValueError as e:
raise ValueError("Could not parse JSON string '{}' \n{}".format(
path_or_string, e.message))
def _parse_jsonp(self, text):
if not '(' in text:
return None
start = text.index('(') + 1
stop = text.rindex(')')
return json_to_obj(text[start:stop])