def check_and_replace_orchestrator_systems():
for old_filename in os.listdir(sesam_checkout_dir + "/unpacked/systems/"):
with open(
os.path.join(sesam_checkout_dir + "/unpacked/systems/",
old_filename), 'r') as f: # open in readonly mode
old_file = load_json(f.read())
try:
old_file["metadata"]["orchestrator"]["original_configuration"]
for new_filename in os.listdir(git_cloned_dir +
"/sesam-node/systems/"):
with open(
os.path.join(
git_cloned_dir + "/sesam-node/systems/",
new_filename),
'r') as g: # open in readonly mode
new_file = load_json(g.read())
if old_file["metadata"]["orchestrator"][
"original_configuration"] == new_file:
logging.info(
"The system %s is restored to orchestrator mode"
% new_file["_id"])
with open(
os.path.join(payload_dir + "/systems/",
new_filename), 'w') as h:
h.write(dump_json(old_file))
except KeyError:
None
def formdata(self, handler, instance):
request = handler.request
id = int(request.REQUEST.get('id', -1))
if request.method == "POST":
formdata = load_json(request.POST.get('data', '{}'))
extra = load_json(request.POST.get('extra', '{}'))
formname = extra.get('name', 'new form')
spokes = ",".join(extra.get('spokes', []))
if id == -1:
pf = PropertyForm(conf=instance, name=formname,
form=dump_json(formdata),
types=spokes)
pf.save()
id = pf.id
else:
pf = PropertyForm.objects.get(pk=id)
pf.name = formname
pf.form = dump_json(formdata)
pf.types = spokes
pf.save()
if id != -1:
pf = PropertyForm.objects.get(pk=id)
return dict(form=load_json(pf.form),
extra=dict(name=pf.name, id=pf.id,
spokes=pf.types.split(",")))
return dict(form=[], extra={'name':'', 'spokes':[]})
def properties_data_handler(handler, request, action):
""" return relevant form data """
## return first form that matches. TODO: combine forms into groups
## combine with data
# import pdb; pdb.set_trace()
spoke = handler.spoke()
if request.method == "POST":
data = load_json(request.POST.get('data'))
for (id, formdata) in data.iteritems():
form_obj_storage, _ = PropertyFormData.objects.get_or_create(form_id=id, content=spoke.instance)
form_obj_storage.properties = dump_json(formdata)
form_obj_storage.save()
spokename = handler.spoke().name()
forms = []
for pf in PropertyForm.objects.all():
types = pf.types.split(",")
if spokename in types:
forms.append(dict(id=pf.id, name=pf.name, form=load_json(pf.form)))
data = {}
for pfd in spoke.instance.properties.all():
data[pfd.form_id] = load_json(pfd.properties)
return dict(forms=forms, data=data)
def get_templates(self, path):
for f in listdir(path):
if f == 'node-metadata.conf.json':
self.node_metadata = load_json(open(f'{path}{f}', 'r').read())
# Extra pipes
if f == 'pipe_on_extra_from_extra_to_master.json':
self.pipe_on_extra_from_extra_to_master = load_json(open(f'{path}{f}', 'r').read())
if f == 'pipe_on_extra_from_master_to_extra.json':
self.pipe_on_extra_from_master_to_extra = load_json(open(f'{path}{f}', 'r').read())
# Extra systems
if f == 'system_on_extra_from_extra_to_master.json':
self.system_on_extra_from_extra_to_master = load_json(open(f'{path}{f}', 'r').read())
if f == 'system_on_extra_from_master_to_extra.json':
self.system_on_extra_from_master_to_extra = load_json(open(f'{path}{f}', 'r').read())
# Master pipes
if f == 'pipe_on_master_from_extra_to_master.json':
self.pipe_on_master_from_extra_to_master = load_json(open(f'{path}{f}', 'r').read())
if f == 'pipe_on_master_from_master_to_extra.json':
self.pipe_on_master_from_master_to_extra = load_json(open(f'{path}{f}', 'r').read())
# Master systems
if f == 'system_on_master_from_extra_to_master.json':
self.system_on_master_from_extra_to_master = load_json(open(f'{path}{f}', 'r').read())
if f == 'system_on_master_from_master_to_extra.json':
self.system_on_master_from_master_to_extra = load_json(open(f'{path}{f}', 'r').read())
def test_code():
with open("/tango/src/demo.R", "w") as r_code:
r_code.write(request.form["code"])
with open("/tango/testcases/demo.R", "w") as r_test:
r_test.write(request.form["tests"])
sp_run([
"docker", "run", "--rm", "--mount",
"type=bind,src=/tmp/tango/src,dst=/home/tango/src,readonly=true",
"--mount",
"type=bind,src=/tmp/tango/testcases,dst=/home/tango/testcases,readonly=true",
"--mount", "type=bind,src=/tmp/tango/out,dst=/home/tango/out",
"awoolley10/tango-demo"
])
with open("/tango/out/validation.json") as v_json:
validation_result = from_validation(load_json(v_json)["runners"][0])
if len(validation_result["successes"]) == 3:
with open("/tango/out/quality.json") as q_json:
q_res = load_json(q_json)["runners"][0]
q_e = from_quality(q_res["errors"]) # Quality errors
q_score = round(100 * q_res["score"])
with open("/tango/out/evaluation.json") as e_json:
e_res = load_json(e_json)["runners"][0]
try:
e_s = e_res["successes"]
except KeyError:
e_s = []
try:
e_f = e_res["failures"]
except KeyError:
e_f = []
try:
e_e = e_res["errors"]
except KeyError:
e_e = []
else:
q_e = None
q_score = None
e_s = None
e_f = None
e_e = None
return render_template("feedback.j2",
v_s=validation_result["successes"],
v_f=validation_result["failures"],
v_e=validation_result["errors"],
q_score=q_score,
q_e=q_e,
e_s=e_s,
e_f=e_f,
e_e=e_e)
def load_sesam_files_as_json(dir):
node_config = []
for name in os.listdir(dir):
path = os.path.join(dir, name)
if os.path.isfile(path) and fnmatch.fnmatch(name,
'node-metadata.conf.json'):
node_config.append(load_json(open(path).read()))
elif os.path.isdir(path):
if fnmatch.fnmatch(name, 'pipes') or fnmatch.fnmatch(
name, 'systems'):
pipes_or_systems = os.listdir(path)
for p_s in pipes_or_systems:
local_path = os.path.join(path, p_s)
node_config.append(load_json(open(local_path).read()))
return node_config
def __iter__(self):
"""Iterate over lines looking for resources."""
resource_class = self._resource_class
resource_args = self._resource_args
session = self._session
response = self._response
for chunk in self._read(response):
event_data = ""
for line in chunk.splitlines():
# Ignore empty lines
if not line.strip():
continue
data = line.split(self._FIELD_SEPARATOR, 1)
field = data[0]
data = data[1]
if field == 'data':
event_data += data
if not event_data:
# Don't report on events with no data
continue
event_data = load_json(event_data).get('data')
yield resource_class(event_data, session, **resource_args)
def runTest(self):
outfn = self.outfn
with open(self.fn, 'rb') as f:
compiler = Popen([COMPILER_EXE, outfn],
stdin=f,
stdout=PIPE,
stderr=STDOUT)
output = compiler.communicate()[0]
if compiler.returncode == -11: # SIGSEGV
self.fail('Optimizer segfaulted')
elif compiler.returncode != 0:
self.fail('Optimizer terminated with signal %d' %
(compiler.returncode))
f.seek(0)
origtm = TM(f.read().decode('latin-1'))
with open(outfn, 'rb') as outf:
optitm = TM(outf.read().decode('latin-1'))
f.seek(0)
for line in f:
line = line.decode('latin-1')
if not line.startswith('*:run'):
continue
inp, dm = load_json('[' + line[5:].strip() + ']')
origout, origtime = origtm.run(inp, dm)
optiout, optitime = optitm.run(inp, dm)
if origout != optiout:
print('FAIL: with input ' + str(inp) + ' and data ' +
str(dm) + ', original program outputs ' +
str(origout) + ' but optimized version outputs ' +
str(optiout))
if optitime > origtime:
slower.add((self.fn, tuple(inp), tuple(dm)))
elif optitime == origtime:
notfaster.add((self.fn, tuple(inp), tuple(dm)))
def get_error_message(error):
if "[errno 32] broken pipe" in error:
return format_error_message(32)
if "[errno 111] connection refused" in error or " operation timed out" in error or "reset by peer" in error:
return format_error_message(111)
res = re_twitter_error.search(error)
code = int(res.group(1)) if res else 0
if code != 500 and str(code).startswith('5') and '"code":' not in error:
return format_error_message(503)
message = ""
try:
jsonerr = load_json(res.group(2))["errors"]
if isinstance(jsonerr, list):
jsonerr = jsonerr[0]
if isinstance(jsonerr, unicode):
jsonerr = {"message": jsonerr}
message = jsonerr["message"][0].upper() + jsonerr["message"][1:] if jsonerr["message"] else ""
if "code" in jsonerr and jsonerr["code"] in [183,187]:
code = jsonerr["code"]
elif code == 403 and "statuses/retweet" in error:
code = 187
except:
if config.DEBUG:
loggerr("%s: %s" % (code, error))
if code == 404 and "direct_messages/new" in error or "friendships" in error:
code = 403
message = "No twitter account found with this name"
return format_error_message(code, message)
def runTest(self):
compiler = Popen(['python',COMPILER_EXE], stdin = open(self.fn, 'rb'), stdout = PIPE, stderr = STDOUT)
output = compiler.communicate()[0].decode('latin-1').strip()
if compiler.returncode == -11: # SIGSEGV
self.fail('Compiler segfaulted')
elif compiler.returncode < 0:
self.fail('Compiler terminated with signal %d'%(compiler.returncode))
seenDirective = False
for line in open(self.fn, 'r'):
if not line.startswith('#:'):
continue
seenDirective = True
if line.startswith('#:compile'):
if compiler.returncode:
self.fail('Compile failed: ' + repr(output))
elif line.startswith('#:error'):
if not compiler.returncode:
self.fail('Compile incorrectly succeeded')
words = load_json('['+line[7:].strip()+']')
if not CHECK_ERROR_CONTENTS:
if len(words) > 0 and type(words[0]) == int:
words = words[:1]
else:
words = []
for word in words:
if sys.version_info[0]+.1*sys.version_info[1] >= 2.7:
self.assertIn(str(word), output)
else:
self.assertTrue(str(word) in output)
else:
self.fail('Unrecognized test directive ' + repr(line))
if not seenDirective:
self.fail('No test directive')
def geojson_to_polygons(json, original_project, desired_projection):
"""Returns generator of GEOSGeometry polygons"""
def to_geos(json_dict_polygon):
"""Return GEOSGeometry polygon from polygon dictionary"""
json_dict_polygon = json_dict_polygon['geometry']
geo = GEOSGeometry(dump_json(json_dict_polygon))
geo.set_srid(original_project)
geo.transform(desired_projection)
return geo
if json == '':
raise ValueError('Json string is empty')
try:
json_dict = load_json(json)
except ValueError:
raise ValueError('Incorrect json format')
json_type = json_dict.get('type', 'fail')
if json_type == 'Feature':
json_dict_polygons = json_dict
elif json_type == 'FeatureCollection':
json_dict_polygons = json_dict.get('features')
else:
raise Exception('Incorrect json format')
return (to_geos(poly) for poly in json_dict_polygons)
def load_extension(extension_name):
json_file = open(join(dirname(__file__), "extensions", extension_name,
"ext.json"),
"r",
encoding="utf-8")
extension_info = load_json(json_file)
json_file.close()
globals_env = {
"run_in_sockethttp": True,
"sockethttp_version": __version__,
"request": request,
"get": get,
"post": post
}
for i in extension_info["extfiles"]:
globals_env[i] = open(
join(dirname(__file__), "extensions", extension_name,
extension_info['extfiles'][0][0]),
*extension_info['extfiles'][0][1:],
**extension_info['extfiles'][1])
locals_env = {}
extension_script = open(join(dirname(__file__), "extensions",
extension_name, "ext.py"),
"r",
encoding="utf-8")
exec(compile(
"\n".join([(i if "# DELETELINE" not in i else "")
for i in extension_script.readlines()]),
"<sockethttp extension>", "exec"),
locals=locals_env,
globals=globals_env)
return locals_env['ext'] or globals_env['ext']
def puzzle():
# Check user has started
if not request.cookies.get("data"):
return render_template("tamperer" + DEV + ".html")
# Get cookie data
cookie = get_data_from_cookie()
# Validate data
if not verify_data(cookie):
# Update tamper statistics
STATISTICS["Tamper Attempts"] += 1
# Remove data cookie
resp = make_response(render_template("tamperer" + DEV + ".html"))
resp.set_cookie("data", "", expires=0)
return resp
# Select & return current puzzle's html
player = UserData.query.filter_by(id=cookie[0]).first()
data = Puzzles.query.filter_by(id=player.current).first()
return render_template(
"puzzle" + DEV + ".html",
title=data.title,
prompt=data.prompt,
number=load_json(player.pages).index(player.current) + 1)
def finisher(player, hs=False):
# Get puzzles
puzzles = ""
for i, p in enumerate(load_json(player[1])):
puzzles += "\t\t" + str(i + 1) + ": " + p + "\n"
body = "New Finisher on " + format_time("%m-%d-%Y %H:%M:%S") + \
":\n\tName: " + STATISTICS["Finishers"][len(STATISTICS["Finishers"]) - 1]["name"] + ",\n" + \
"\tEmail: " + STATISTICS["Finishers"][len(STATISTICS["Finishers"]) - 1]["email"] + ",\n" + \
"\tTime: " + str(STATISTICS["Finishers"][len(STATISTICS["Finishers"]) - 1]["time"]) + " seconds\n" + \
"\tAssigned Puzzles: " + puzzles
if hs:
body += "New Highscore! Contact them & give them their reward."
a = ("api", MG_APIKEY)
d = {
"from": FROM,
"to": TO,
"subject": "Krantz's Challenge: New Finisher",
"text": body
}
return post("https://api.mailgun.net/v3/" + DOMAIN + "/messages",
auth=a,
data=d)
def _split_generators(self, dl_manager):
xml_urls = []
total_bytes = 0
# Re-use the generated status.json
with open(STATUS_FILE) as fh:
dump_info = load_json(fh)
multistream_dump_info = dump_info["jobs"]["articlesmultistreamdump"]
assert multistream_dump_info["status"] == "done"
for fname, info in multistream_dump_info["files"].items():
if ".xml" not in fname:
continue
total_bytes += info["size"]
xml_urls.append(DUMP_URL + fname)
downloaded_files = dl_manager.download_and_extract({"xml": xml_urls})
# Max 128MB
max_bytes = int(ceil(total_bytes / (128 * 2**20)))
return [
tfds.core.SplitGenerator(name=tfds.Split.TRAIN,
num_shards=max_bytes,
gen_kwargs={
"filepaths": downloaded_files["xml"],
"language": LANGUAGE
})
]
def put(self, request, *args, **kwargs):
# Отличие от POST. POST вызывается со страницы товара. А ПУТ со страницы корзины, когда лишь обновляем цифры.
raw_data = request.data.get('items')
if raw_data:
try:
data_dict = load_json(raw_data)
except:
return JsonResponse({
"Status": False,
"Error": "Wrong input format"
})
else:
for item in data_dict:
order_item = OrderInfo.objects.filter(id=item['id'])
if not order_item:
return JsonResponse({
"Status":
False,
"Error":
"Такого товара нет в корзине."
})
order_item.update(quantity=item['quantity'])
return JsonResponse({"Status": True})
return JsonResponse({
"Status": False,
"Error": "Не все аргументы указаны."
})
def get_train_data(data_config):
with open(data_config, 'r') as f:
sre_data = load_json(f.read())
data_root = sre_data['ROOT']
data_loc = sre_data['LOCATION']
speaker_key = sre_data['SPEAKER_KEY']
sre04 = make_old_sre_data(data_root, data_loc['SRE04'], 2004, speaker_key)
sre05_train = make_old_sre_data(data_root, data_loc['SRE05_TRAIN'], 2005, speaker_key)
sre05_test = make_old_sre_data(data_root, data_loc['SRE05_TEST'], 2005, speaker_key)
sre06 = make_old_sre_data(data_root, data_loc['SRE06'], 2006, speaker_key)
sre08 = make_sre08_data(data_root, data_loc['SRE08_TRAIN'], data_loc['SRE08_TEST'])
sre10 = make_sre10_data(data_root, data_loc['SRE10'])
sre16 = make_sre16_data(data_root, data_loc['SRE16_EVAL'])
swbd_c1 = make_swbd_cellular(data_root, data_loc['SWBD_C1'], 1)
swbd_c2 = make_swbd_cellular(data_root, data_loc['SWBD_C2'], 2)
swbd_p1 = make_swbd_phase(data_root, data_loc['SWBD_P1'], 1)
swbd_p2 = make_swbd_phase(data_root, data_loc['SWBD_P2'], 2)
swbd_p3 = make_swbd_phase(data_root, data_loc['SWBD_P3'], 3)
mx6_calls = make_mixer6_calls(data_root, data_loc['MX6'])
mx6_mic = make_mixer6_mic(data_root, data_loc['MX6'])
train_data = np.hstack([sre04, sre05_train, sre05_test, sre06, sre08, sre10, sre16, swbd_c1, swbd_c2, swbd_p1,
swbd_p2, swbd_p3, mx6_calls, mx6_mic]).T
print('Removing Duplicates...')
train_data, n_dup = remove_duplicates(train_data)
print('Removed {} duplicates.'.format(n_dup))
print('Sorting train data by index...')
return sort_by_index(train_data)
def read_capabilities(filename):
data_file = open(filename, 'r')
data = load_json(data_file)
addons = data['addons']
disclaimers = data['disclaimers']
ret = {}
for addon_name, info in addons.iteritems():
infos = []
for cap in CAPABILITY_SET:
status = info[cap].get('status') or ''
text = info[cap].get('text') or ''
if status == 'NA':
continue
infos.append({
'function': cap,
'status': status,
'detail': text,
'class': CLASS_MAP[status],
})
ret[addon_name] = {
'capabilities': infos,
'terms': disclaimers,
}
data_file.close()
return ret
def withdraw(self, cointype, address, volume, mark='AUTO', memo=None):
parms = {
'coinType': cointype,
'address': address,
'quantity': volume,
'mark': mark
}
action = 'withdraw'
if memo:
parms['extendParam'] = memo
ts = str(int(DateTime.now().timestamp() * 1000))
data = {
'apiKey': self.__secret.api_key,
'bizCode': action,
'msgNo': ts,
'timestamp': ts,
}
data.update(parms)
data['signature'] = self.__secret.sign(data)
response = self.session.post(url=URL_WITHDRAW, json=data, timeout=15)
response = load_json(response.text)
if response['code'] != '0':
raise BithumbGlobalError(response['code'], response['msg'])
return response['data']
def variable_verification(self):
if self.verify_vars is True:
if len(self.verify_vars_from_files) == 0:
self.LOGGER.critical(
'Verify vars is true but files to verify from is not specified!'
)
exit(-4)
else:
all_vars = {}
for f in self.verify_vars_from_files:
all_vars.update(load_json(open(f).read()))
missing_vars = []
for var in self.config_vars:
if var not in all_vars:
missing_vars.append(var)
if len(missing_vars) != 0:
self.LOGGER.critical(
f'Variables verification failed! Missing vars: "{missing_vars}"'
)
return False
else:
self.LOGGER.info(f'Variables verification succeeded :)')
return True
return True # If verify vars is false
def recursive_set_env_var(triple_tuple_env_var):
for var, t, child_required_vars in triple_tuple_env_var:
curvar = getenv(var, None)
if curvar is None:
if var not in OPTIONAL_ENV_VARS:
missing_vars.append(var)
else:
if t == bool:
curvar = curvar.lower() == 'true'
setattr(config, var, curvar)
if curvar and child_required_vars is not None:
recursive_set_env_var(child_required_vars)
elif t == list:
setattr(config, var, curvar.split(sep=';'))
elif t == dict:
jsoned_curvar = load_json(curvar.replace('`', ''))
if child_required_vars is not None:
for k in child_required_vars:
if k not in jsoned_curvar:
if k not in OPTIONAL_ENV_VARS:
missing_vars.append(f'{var}->{k}')
else:
curtype = child_required_vars[k]
if curtype == bool:
jsoned_curvar[k] = jsoned_curvar[k].lower(
) == 'true'
setattr(config, var, jsoned_curvar)
else:
setattr(config, var, curvar)
def get(self,
resource,
key='no_key',
environment='production',
parser='yaml'):
"""Query Puppet information using REST API and client SSL cert"""
url = '/'.join((self.endpoint, environment, resource, key))
request = urllib2.Request(url)
request.add_header('Accept', parser)
try:
try:
response = self.opener.open(request)
except urllib2.HTTPError as e:
raise APIError((str(e) + ": " + url))
if parser == 'yaml':
value = load_yaml(response)
elif parser == 'pson':
value = load_json(response)
else:
value = response.read()
finally:
try:
response.close()
except NameError:
# HTTPError would be raised, don't preempt it
pass
return value
def edit_docente(docForm):
form = DocenteForm(docForm)
form.is_valid()
form_values = form.getCleanedData()
old_values = load_json(docForm['olddoc'])
old_code = old_values['code']
try:
code = form_values['code']
except KeyError:
code = None
try:
name = form_values['name'] if form_values[
'name'] is not None else old_values['name']
except KeyError:
name = old_values['name']
try:
email = form_values['email'] if form_values[
'email'] is not None else old_values['email']
except KeyError:
email = old_values['email']
old_docente = DocenteData.getDocente(old_code)
if old_code != code and code is not None:
old_docente.delete()
DocenteData.createDocente(codigo=code, nombre=name, correo=email)
else:
old_docente.nombre = name
old_docente.correo = email
old_docente.save()
msg = {
'title': 'Información modificada',
'content': f'La información del docente ha sido modificada con éxito'
}
content = {'showModal': True, 'msg': msg, 'view': 0}
return content
def runTest(self):
outfn = self.outfn
with open(self.fn, 'rb') as f:
compiler = Popen([COMPILER_EXE, outfn], stdin = f, stdout = PIPE, stderr = STDOUT)
output = compiler.communicate()[0]
if compiler.returncode == -11: # SIGSEGV
self.fail('Optimizer segfaulted')
elif compiler.returncode != 0:
self.fail('Optimizer terminated with signal %d'%(compiler.returncode))
f.seek(0)
origtm = TM(f.read().decode('latin-1'))
with open(outfn, 'rb') as outf:
optitm = TM(outf.read().decode('latin-1'))
f.seek(0)
for line in f:
line = line.decode('latin-1')
if not line.startswith('*:run'):
continue
inp, dm = load_json('['+line[5:].strip()+']')
origout, origtime = origtm.run(inp, dm)
optiout, optitime = optitm.run(inp, dm)
if origout != optiout:
print('FAIL: with input ' + str(inp) + ' and data ' + str(dm) + ', original program outputs ' + str(origout) + ' but optimized version outputs ' + str(optiout))
if optitime > origtime:
slower.add((self.fn, tuple(inp), tuple(dm)))
elif optitime == origtime:
notfaster.add((self.fn, tuple(inp), tuple(dm)))
def verify_node(node):
node_string = dump_json(node)
variables = None
secrets = None
if upload_variables:
variables_in_conf = regex_findall(r'\$ENV\((\S*?)\)',
node_string) # Find env vars
variables: dict = load_json(
open(git_cloned_dir + sync_root + 'node/' + var_file_path).read())
for var in variables_in_conf: # Verify they exist in git repo
if var not in variables:
logging.error(
f'Missing env var {var} in variables file {var_file_path}')
if upload_secrets:
secrets_in_conf = regex_findall(r'\$SECRET\((\S*?)\)',
node_string) # Find secrets
vault = Vaulter(vault_url, vault_git_token,
vault_mounting_point) # Create keyvault object
secrets: dict = vault.get_secrets(
secrets_in_conf) # Get the secrets from keyvault
if vault.verify() is False: # Verify all secrets exist.
logging.error(
f'These secrets do not exist in the vault {vault.get_missing_secrets()}'
)
return variables, secrets
def __iter__(self):
"""Iterate over lines looking for resources."""
resource_class = self._resource_class
resource_args = self._resource_args
session = self._session
response = self._response
for chunk in self._read(response):
event_data = ""
for line in chunk.splitlines():
# Ignore empty lines
if not line.strip():
continue
data = line.split(self._FIELD_SEPARATOR, 1)
field = data[0]
data = data[1]
if field == 'data':
event_data += data
if not event_data:
# Don't report on events with no data
continue
event_data = load_json(event_data).get('data')
yield resource_class(event_data, session, **resource_args)
def owoify(text: str) -> str:
""" Converts your text to OwO """
smileys = [';;w;;', '^w^', '>w<', 'UwU', '(・`ω´・)', '(´・ω・`)']
with open(f"{module_dir}/assets/replacements.json") as fp:
replacements = load_json(fp)
for expression in replacements:
replacement = replacements[expression]
text = sub(expression, replacement, text, flags=IGNORECASE)
words = text.split()
first_letter = words[0][0]
letter_stutter = f"{first_letter}-{first_letter.lower()}-{first_letter.lower()}"
if len(words[0]) > 1:
words[0] = letter_stutter + words[0][1:]
else:
words[0] = letter_stutter
text = " ".join(words)
text = text.replace('L', 'W').replace('l', 'w')
text = text.replace('R', 'W').replace('r', 'w')
text = '! {}'.format(choice(smileys)).join(text.rsplit('!', 1))
text = '? OwO'.join(text.rsplit('?', 1))
text = '. {}'.format(choice(smileys)).join(text.rsplit('.', 1))
text = f"{text} desu"
for v in ['a', 'o', 'u', 'A', 'O', 'U']:
if 'n{}'.format(v) in text:
text = text.replace('n{}'.format(v), 'ny{}'.format(v))
if 'N{}'.format(v) in text:
text = text.replace('N{}'.format(v), 'N{}{}'.format('Y' if v.isupper() else 'y', v))
return text
def make_sre16_trials_file(sre_config, trials_file):
with open(sre_config, 'r') as f:
sre_data = load_json(f.read())
data_root = sre_data['ROOT']
data_loc = sre_data['LOCATION']['SRE16_EVAL']
sre_loc = join_path(data_root, data_loc)
segment_key = join_path(sre_loc, 'docs/sre16_eval_segment_key.tsv')
language_key = join_path(sre_loc, 'metadata/calls.tsv')
trial_key = join_path(sre_loc, 'docs/sre16_eval_trial_key.tsv')
utt_to_call = dict()
with open(segment_key, 'r') as f:
for line in f.readlines()[1:]:
tokens = re.split('[\s]+', line.strip())
utt_to_call[tokens[0]] = tokens[1]
call_to_language = dict()
with open(language_key, 'r') as f:
for line in f.readlines()[1:]:
tokens = re.split('[\s]+', line.strip())
call_to_language[tokens[0]] = tokens[1]
trials_list = []
with open(trial_key, 'r') as f:
for line in f.readlines()[1:]:
tokens = re.split('[\s]+', line.strip())
speaker_id = tokens[0]
file_name = tokens[1]
target_type = tokens[3]
trials_list.append('sre16_eval_enroll_{} sre16_eval_test_{} {}'.format(speaker_id, file_name, target_type))
with open(trials_file, 'w') as f:
for trial in trials_list:
f.write('{}\n'.format(trial))
def user_data(self, handler, instance):
if handler.request.method == "POST":
data = load_json(handler.request.POST.get('data', "{}"))
for user in data.get('existing', []):
state = user.get('state', '')
if state == 'deleted':
try:
User.objects.get(pk=user['id']).delete()
except (ValueError, User.DoesNotExist):
pass
elif state in ("added", "modified"):
try:
u = User.objects.get(pk=user['id'])
u.username = user.get('username').strip()
u.first_name = user.get('firstname', '')
u.last_name = user.get('lastname', '')
u.email = user.get('email', '')
## active?
except (ValueError, User.DoesNotExist):
u = User(username=user.get('username').strip(),
first_name=user.get('firstname', ''),
last_name=user.get('lastname', ''),
email=user.get('email', ''))
u.save()
password = user.get('password', '').strip()
if password:
u.set_password(password)
u.roles.all().delete()
for role, isset in user.get('roles', {}).iteritems():
if isset:
WheelRole(role=Role(role), user=u).save()
u.save()
## roles
users = User.objects.all()
data = {}
data['existing'] = [dict(id=u.id,
username=u.username,
firstname=u.first_name,
lastname=u.last_name,
email=u.email,
active=u.is_active,
superuser=u.is_superuser,
roles=dict((role.role.id, True)
for role in u.roles.all())
) for u in users]
data['roles'] = [dict(id=role.id,
name=role.name,
description=role.description)
for role in Role.all()]
return data
def submit(recaptcha_response_field, private_key, remoteip):
if not (recaptcha_response_field and len(recaptcha_response_field)):
return RecaptchaResponse(is_valid=False,
error_code='incorrect-captcha-sol')
def encode_if_necessary(s):
if isinstance(s, unicode):
return s.encode('utf-8')
return s
params = urllib.urlencode({
'secret':
encode_if_necessary(private_key),
'remoteip':
encode_if_necessary(remoteip),
'response':
encode_if_necessary(recaptcha_response_field),
})
request = urllib2.Request(
url="https://%s/recaptcha/api/siteverify" % VERIFY_SERVER,
data=params,
headers={
"Content-type": "application/x-www-form-urlencoded",
"User-agent": "reCAPTCHA Python"
})
json_values = load_json(urllib2.urlopen(request))
if json_values["success"]:
return RecaptchaResponse(is_valid=True)
else:
return RecaptchaResponse(is_valid=False,
error_code=json.dumps(json_values))
def post(self, request, *args, **kwargs):
if not request.user.is_authenticated:
return JsonResponse({'Status': False, 'Error': 'Log in required'}, status=403)
items_sting = request.data.get('items')
if items_sting:
try:
items_dict = load_json(items_sting)
except ValueError:
JsonResponse({'Status': False, 'Errors': 'Неверный формат запроса'})
else:
basket, _ = Order.objects.get_or_create(user_id=request.user.id, state='basket')
objects_created = 0
for order_item in items_dict:
order_item.update({'order': basket.id})
serializer = OrderItemSerializer(data=order_item)
if serializer.is_valid():
try:
serializer.save()
except IntegrityError as error:
return JsonResponse({'Status': False, 'Errors': str(error)})
else:
objects_created += 1
else:
JsonResponse({'Status': False, 'Errors': serializer.errors})
return JsonResponse({'Status': True, 'Создано объектов': objects_created})
return JsonResponse({'Status': False, 'Errors': 'Не указаны все необходимые аргументы'})
def data(request):
redis = get_redis_connection('cirrus')
data_id = request.GET.get('uuid', '')
df = _get_data(data_id)
key = KEY_DIGEST_PREFIX + data_id
header = load_json(redis.hget(key, 'header'))
new_limits = request.GET.get('new_limits', None)
if new_limits:
name, limits_lower, limits_upper = new_limits.split('|')
else:
name = None
for row in header:
if row['name'] == name:
row['limits'] = [limits_lower, limits_upper]
redis.hset(key, 'header', dump_json(header))
if row['limits'][0] != '':
if row['type'].startswith('float'):
lim = float(row['limits'][0])
elif row['type'].startswith('int'):
lim = int(row['limits'][0])
df = df[df[row['name']] >= lim]
elif row['limits'][1] != '':
if row['type'].startswith('float'):
lim = float(row['limits'][1])
elif row['type'].startswith('int'):
lim = int(row['limits'][1])
df = df[df[row['name']] <= lim]
print (df)
orders = request.GET.get('order', '')
orders = [row for row in orders.split(',') if row]
if orders:
sort_func = request.GET.get('sort', 'quicksort')
ascending = [
row[0] != '-'
for row in orders
]
by = [
row[1:] if row[0] == '-' else row
for row in orders
]
df.sort_values(by=by, ascending=ascending, inplace=True, kind=sort_func)
columns = {}
for row in df.columns:
if str(df[row].dtype).startswith('float'):
columns[row] = [(i if isfinite(i) else None) for i in df[row]]
else:
columns[row] = [i for i in df[row]]
return {
'index': [row for row in df.index],
'data': columns,
'order': orders
}
def get_oauth2_token(self):
res = self.conn.oauth2.token(grant_type="client_credentials")
obj = load_json(res)
if "token_type" not in obj or obj[
"token_type"] != "bearer" or "access_token" not in obj:
raise Exception("Wrong token type given by twitter, weird : %s" %
res)
return obj["access_token"]
def __call__(self, value):
if type(value) in (str, bytes):
try:
return load_json(value)
except Exception:
raise ValueError('Incorrectly formatted JSON provided')
else:
return value
def load_config():
global config
try:
with open("config.json", "r") as fh:
raw_data = fh.read()
config = load_json(raw_data)
except Exception as e:
print("Could not open config. Error: " + str(e))
def search_produto(request):
data = {'produto': False}
if request.body:
rget = load_json(request.body)
produto = models.Produto.objects.filter(
nomeproduto=rget.get('produto'))
data = serialize('json', produto)
return JsonResponse(data, safe=False)
def __call__(self, value):
if type(value) in (str, bytes):
try:
return load_json(value)
except Exception:
raise ValueError('Incorrectly formatted JSON provided')
else:
return value
def json(value):
'''Accepts a JSON formatted data structure'''
if type(value) in (str, bytes):
try:
return load_json(value)
except Exception:
raise ValueError('Incorrectly formatted JSON provided')
else:
return value
def json(value):
'''Accepts a JSON formatted data structure'''
if type(value) in (str, bytes):
try:
return load_json(value)
except Exception:
raise ValueError('Incorrectly formatted JSON provided')
else:
return value
def check_file(filename):
# test filename are structured as inXX.vtt,
# and result filenames are structured as outXX.json
resultname = "out%s.json" % (filename[2:-4])
with open(join(DIR, filename)) as infile:
got = parse(infile, True)
with open(join(DIR, resultname)) as outfile:
exp = load_json(outfile)
assert got == exp, dumps(got, indent=4)
def __init__(self, fname): data = load_json(open(fname).read()) topic = Comment(data[0]['data']['children'][0]) comments = [Comment(c) for c in data[1]['data']['children'] if 'body' in c['data']] topic.children = comments self.comment_tree = topic self.comment_list = flatten_comment_tree(self.comment_tree)
def get_pypi_hash(baseurl):
logger = logging.getLogger(__name__)
dist = find_distributions('.').next()
logger.debug('dist found: {0}'.format(dist))
targer_url = urljoin(baseurl, 'pypi/{0}/json'.format(dist.project_name))
logger.debug('opening url: {0}'.format(targer_url))
response = urlopen(targer_url)
pypi_json = load_json(response.read())
md5_digest = pypi_json['releases'][dist.version][0]['md5_digest']
logger.debug('pypi md5: {0}'.format(md5_digest))
return md5_digest
def diagnostics(self):
"""Return diagnostics string.
:return: Dictionary with disk parition information
:rtype: dict
"""
status = run(['sfdisk', '--json', self.path])
disk_info = load_json(status.stdout)
# TBD:
# - check status
# - log stderr
return disk_info
def handle(self, *args, **options):
dir_path = options['files_path']
with open(dir_path) as fp:
all_data = load_json(fp)
for i, data in enumerate(all_data):
self.load_event(data)
if i % 10 == 0:
sys.stdout.write('.')
sys.stdout.flush()
sys.stdout.write('\nDone\n')
def build_features(feature_type, file_in):
'''
Build a feature object
:param feature_type: type of the feature to be built
:param file_in: the input data base
:return: the feature object containing the input data base
'''
features_json = load_json(open(file_in))
features = Sidewalk(features_json)
return features
def content_json(response):
"""
:type response: HttpResponse
:rtype: dict
"""
content = response.content.decode("utf-8")
if not content:
return {}
try:
return load_json(content)
except ValueError:
raise ValueError('invalid json content in response')
def get_user_data(self, cookies):
API_KEY = str(settings.FB_API_KEY)
fbs_cookie = cookies['fbs_%s' % API_KEY]
parsed_fbs = parse_qs(smart_unicode(fbs_cookie))
# Communicate with the access token to the Facebook oauth interface.
json = load_json(urlopen('https://graph.facebook.com/me?access_token=%s' % parsed_fbs['access_token'][0]))
# Return the user data.
return {
'username': '******' % (smart_unicode(json['first_name']), smart_unicode(json['last_name'])),
'email': smart_unicode(json['email']),
}
def from_http_request(cls, http_request):
"""
:type http_request: HttpRequest
:rtype: JsonRequest
:raises ValueError: if the request body is not valid JSON
"""
try:
json_data = load_json(http_request.body.decode("utf-8"))
except ValueError:
raise ValueError('incorrect json request')
return cls(http_request, json_data)
def login(self, username, password):
params = {
'client_id': CLIENT_ID,
'client_secret': CLIENT_SECRET,
'grant_type': 'password',
'username': username,
'password': password
}
path = quote('/api/users/{}/login/'.format(username))
response = self.client.post(path, params)
self.assertEqual(response.status_code, status.HTTP_200_OK)
token = load_json(response.content.decode())['access_token']
self.set_token(token)
def get_user_data(cookies):
request_data = {
'method': 'Users.getInfo',
'api_key': settings.FB_API_KEY,
'call_id': time(),
'v': '1.0',
'uids': cookies[settings.FB_API_KEY + '_user'],
'fields': 'name,first_name,last_name',
'format': 'json',
}
request_data['sig'] = generate_sig(request_data)
fb_response = urlopen(REST_SERVER, urlencode(request_data))
#print(fb_response)
return load_json(fb_response)[0]
async def __anext__(self):
"""Iterate over lines looking for resources."""
resource_class = self._resource_class
resource_args = self._resource_args
session = self._session
response = self._response
async for line in response.content:
line = line.decode('utf-8')
if len(line) == 0 and response.at_eof:
raise StopAsyncIteration
if len(line.strip()) > 0:
field, data = line.split(self._FIELD_SEPARATOR, 1)
if field.strip() == 'data':
json = load_json(data).get('data')
return resource_class(json, session, **resource_args)
def go( input_file_name,
sampling_number,
eigenvalue_zero_threshold,
interesting_signature_parameter,
periodicity_selection_parameter,
caution ):
with open(input_file_name) as f:
data = load_json(f)
input_parsing_start_time = process_time()
indeterminates = symbols(data["indeterminates"])
e_mat = [[parse_expr(s) for s in row] for row in data["matrix"]]
# TODO: use `logging` module instead of printing to `stderr`:
#
# https://docs.python.org/3/library/logging.html
#
print( "Input parsing took {:.3g}s."
.format(process_time() - input_parsing_start_time),
file = stderr )
initialization_start_time = process_time()
matrix_sampler = make_matrix_sampler( e_mat,
indeterminates,
sampling_number )
sample_index_iterator_maker = make_sample_index_iterator_maker(
len(indeterminates),
sampling_number,
periodicity_selection_parameter
)
# TODO: use `logging` module instead of printing to `stderr`
print( "Initialization took {:.3g}s."
.format(process_time() - initialization_start_time),
file = stderr )
process_data( matrix_sampler,
sample_index_iterator_maker,
eigenvalue_zero_threshold,
interesting_signature_parameter,
caution = caution )
def build_features(feature_type, file_in):
'''
Build a feature object
:param feature_type: type of the feature to be built
:param file_in: the input data base
:return: the feature object containing the input data base
'''
features = None
features_json = load_json(open(file_in))
if feature_type == 'sidewalks':
features = Sidewalk(features_json)
if feature_type == 'curbramps':
features = CurbRamp(features_json)
if feature_type == 'crossings':
features = Crossing(features_json)
return features
def get_user_data(self, key):
request_data = {
'method': 'Users.getInfo',
'api_key': settings.FB_API_KEY,
'call_id': time(),
'v': '1.0',
'uids': key,
'fields': 'name,first_name,last_name,email',
'format': 'json',
}
request_data['sig'] = self.generate_signature(request_data)
fb_response = load_json(urlopen(REST_SERVER, urlencode(request_data)))[0]
return {
'username': fb_response['first_name'] + ' ' + fb_response['last_name'],
'email': fb_response['email']
}
def get_userIds(hostname, token):
'''Get the list of profiles to send the notification to.
:param str hostname: The name of the host to use.
:param str token: The token to use for authentication.
:raises NotOk: When the page does not return an HTTP 200 status code.
:returns: A list of strings
:rtype: list'''
fields = {'token': token, 'get': '', }
status, reason, data = post_multipart(hostname, PROFILE_URI,
fields)
if status != HTTP_OK:
m = '{reason} ({status} <{host}>)'
msg = m.format(reason=reason, status=status, host=hostname)
raise NotOk(msg)
retval = load_json(data)
return retval
def send_status(hostname, userId, token):
'''Send a profile-status notification for a particular person
:param str hostname: The name of the host to use.
:param str userId: The identifier for the person.
:param str token: The token to use for authentication.
:raises NotOk: When the page does not return an HTTP 200 status code.'''
fields = {
'profileId': userId,
'token': token,
'send': 'Send'}
status, reason, data = post_multipart(hostname, SEND_STATUS_URI,
fields)
if status != HTTP_OK:
m = '{reason} ({status} <{host}>)'
msg = m.format(reason=reason, status=status, host=hostname)
raise NotOk(msg)
retval = load_json(data)
return retval
def get_digest_groups(hostname, token):
'''Get the list of groups to send the digest to.
:param str hostname: The name of the host to use.
:param str token: The token to use for authentication.
:raises NotOk: When the page does not return an HTTP 200 status code.
:returns: A list of 2-tuples ``(siteId, groupId)``, sorted alphabetically
:rtype: tuple'''
fields = {'token': token, 'get': '', }
status, reason, data = post_multipart(hostname, DIGEST_GROUPS_URI,
fields)
if status != HTTP_OK:
m = '{reason} ({status} <{host}>)'
msg = m.format(reason=reason, status=status, host=hostname)
raise NotOk(msg)
retval = load_json(data)
retval.sort(key=itemgetter(0, 1)) # Nicer when sorted by site & group
return retval
def do_fetch_nodemap(fqdn):
"""
Fetches an existing nodemap from a remote esgf node
fqdn
Fully-qualified domain name of the server. Rest of url assumed for Node Manager service
"""
arr = []
try:
f = open ('/esg/config/esgf_supernodes_list.json')
arr = load_json(f.read())
except:
pass
data = None
for host in arr:
data_str = ""
if host != fqdn:
url = "https://" + host + "/esgf-nm/api?action=sync_node_map_file"
try:
resp = requests.get(url, verify=False)
data_str = resp.text
data = json.loads(data_str)
except Exception as e:
print "An Error occurred:", str(e)
if not data is None and len(data_str) > 10:
write_json_file(MAP_FN, str(data_str))
return True
print "Could not retrieve the node map from another site. Will generate a fresh one. Advised that you attempt to retrieve the map in order to have a complete view of member nodes in the federation."
return False
def get_user_data(self, access_token):
profile = load_json(urlopen("https://graph.facebook.com/me?" + urlencode(dict(access_token=access_token))))
name = smart_unicode(profile.get("name", ""));
# If the name is longer than 30 characters - leave it blank
if len(name) > 30:
name = ''
# Check whether the length if the email is greater than 75, if it is -- just replace the email
# with a blank string variable, otherwise we're going to have trouble with the Django model.
email = smart_unicode(profile.get('email', ''))
if len(email) > 75:
email = ''
# Return the user data.
return {
'id' : profile['id'],
'username': name,
'email': email,
}
