def data_bubble(request, id=None):
dashboard = get_object_or_404(Dashboard, id=id)
dbCon = ConnectedDatabase.objects.get(project=dashboard.project)
workboard = get_object_or_404(Workboard, id=request.POST.get('workboard', None))
variables = json.loads(request.POST.get('variables', None))
engine = create_engine(dbCon.connection_string, echo=True)
cnx = engine.raw_connection()
collection_name = list(variables.keys())[0]
column_names = []
yaxis = None
for item in variables[collection_name]:
column_names.append(item['name'])
if item['type'] == 'string':
xaxis = item['name']
elif yaxis == None and item['type'] == 'integer':
yaxis = item['name']
elif item['type'] == 'integer':
zaxis = item['name']
finalResult = []
try:
data = pd.read_sql(
'Select \"' + collection_name + "\".\"" + xaxis + '\" as x, \"' + collection_name + "\".\"" + yaxis \
+ '\" as y, \"' + collection_name + "\".\"" + zaxis + '\" as size FROM \"' + collection_name + "\"",
cnx)
datasum = data.groupby(['x']).sum()
datasum['index_col'] = range(0, len(datasum))
datamin = data.groupby(['x']).min()
datamin['index_col'] = range(0, len(datamin))
datamax = data.groupby(['x']).max()
datamax['index_col'] = range(0, len(datamax))
dataavg = data.groupby(['x']).mean()
dataavg['index_col'] = range(0, len(dataavg))
resultsumdump = json.JSONDecoder().decode(datasum.reset_index().to_json(orient="records"))
resultmindump = json.JSONDecoder().decode(datamin.reset_index().to_json(orient="records"))
resultmaxdump = json.JSONDecoder().decode(datamax.reset_index().to_json(orient="records"))
resultavgdump = json.JSONDecoder().decode(dataavg.reset_index().to_json(orient="records"))
finalResult.append(
{'sum': resultsumdump, 'avg': resultavgdump, 'min': resultmindump, 'max': resultmaxdump, 'var_name': xaxis})
json_data = {
'data': json.dumps(finalResult),
'analysis_type': request.POST.get('type', None),
'columns': column_names,
'workboard': workboard,
'dashboard': dashboard,
'yaxis': yaxis,
'zaxis': zaxis
}
except Exception as e:
json_data = {
'analysis_type': request.POST.get('type', None),
'error': e.args[0],
'dashboard': dashboard,
'workboard': workboard
}
template = render_to_string('dashboards/dashboard_workboard.html', json_data)
return HttpResponse(json.dumps(template), content_type='application/json')
def find_results(self):
resultlist = []
if self.type == 'book':
set_url = self.url + urllib.quote('intitle:' + '"' + self.name +
'"')
else:
set_url = self.url + urllib.quote('inauthor:' + '"' + self.name +
'"')
logger.info('Searching url: ' + set_url)
try:
startindex = 0
resultcount = 0
ignored = 0
while True:
self.params['startIndex'] = startindex
URL = set_url + '&' + urllib.urlencode(self.params)
try:
jsonresults = json.JSONDecoder().decode(
urllib2.urlopen(URL, timeout=30).read())
except HTTPError, err:
logger.Error(
'Google API returned HTTP Error - probably time/rate limiting - [%s]'
% err.msg)
startindex = startindex + 40
for item in jsonresults['items']:
# skip if no author, no author is no book.
try:
authorname = item['volumeInfo']['authors'][0]
except KeyError:
logger.debug('Skipped a result without authorfield.')
break
try:
#skip if language is in ignore list
booklang = item['volumeInfo']['language']
if not booklang in lazylibrarian.IMP_PREFLANG:
ignored = ignored + 1
break
except KeyError:
ignored = ignored + 1
logger.debug(
'Skipped a result where no language is found')
break
try:
bookpub = item['volumeInfo']['publisher']
except KeyError:
bookpub = None
try:
booksub = item['volumeInfo']['subtitle']
except KeyError:
booksub = None
try:
bookdate = item['volumeInfo']['publishedDate']
except KeyError:
bookdate = '0000/00/00'
try:
bookimg = item['volumeInfo']['imageLinks']['thumbnail']
except KeyError:
bookimg = 'images/nocover.png'
try:
bookrate = item['volumeInfo']['averageRating']
except KeyError:
bookrate = 0
try:
bookpages = item['volumeInfo']['pageCount']
except KeyError:
bookpages = '0'
try:
bookgenre = item['volumeInfo']['categories'][0]
except KeyError:
bookgenre = None
try:
bookdesc = item['volumeInfo']['description']
except KeyError:
bookdesc = 'Not available'
try:
if item['volumeInfo']['industryIdentifiers'][0][
'type'] == 'ISBN_10':
bookisbn = item['volumeInfo'][
'industryIdentifiers'][0]['identifier']
else:
bookisbn = 0
except KeyError:
bookisbn = 0
resultlist.append({
'authorname':
authorname,
'bookid':
item['id'],
'bookname':
item['volumeInfo']['title'],
'booksub':
booksub,
'bookisbn':
bookisbn,
'bookpub':
bookpub,
'bookdate':
bookdate,
'booklang':
booklang,
'booklink':
item['volumeInfo']['canonicalVolumeLink'],
'bookrate':
float(bookrate),
'bookimg':
bookimg,
'bookpages':
bookpages,
'bookgenre':
bookgenre,
'bookdesc':
bookdesc
})
resultcount = resultcount + 1
except KeyError:
logger.info('Found %s results for %s with name: %s' %
(resultcount, self.type, self.name))
if ignored > 0:
logger.info(
'Skipped %s results because it is not a preferred language.'
% ignored)
return resultlist
def get_asset(uuid):
url = "{LR_ORIGIN}/c/portal/truenth/asset/detailed?uuid={uuid}".format(
LR_ORIGIN=current_app.config["LR_ORIGIN"], uuid=uuid)
data = requests.get(url).content
return json.JSONDecoder().decode(data)['asset']
#!/usr/bin/env python
# encoding: utf-8
from __future__ import print_function
import json
import sys
shard = sys.argv[1]
with open(shard, "r") as f:
input_value = json.JSONDecoder().decode(f.read())
out = []
if "Error" in input_value:
with open("errors", "a") as errors:
print("Data error: user %s %s" %
(input_value["User"], input_value["Error"]),
file=errors)
else:
for doc in input_value["Docs"]:
out.append({
"Key": doc,
"Value": {
"User": input_value["User"],
"Team": input_value["Team"]
}
})
print(json.JSONEncoder().encode(out))
import scrapy
import json
decoder = json.JSONDecoder()
#> pnp_previews_href.json && scrapy runspider pnp_2.py -o pnp_previews_href.json
with open('pnp_sections.json', 'r') as fp:
list_input = json.load(fp)
print list_input
print len(list_input)
print list_input[0]['href']
list_urls = []
list_urls.append('https://www.pnp.ru')
for x in list_input:
print x['href']
string = x['href']
list_urls.append(string)
print list_urls
class QuotesSpider(scrapy.Spider):
name = 'quotes'
start_urls = list_urls
def parse(self, response):
for quote in response.css('div.item div.title'):
yield {
## @author zhouyuefeng([email protected]) ## @date 2014/06/19 13:06:33 ## @brief ## import sys import MySQLdb import json import time cur_time = "\"" + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime( time.time())) + "\"" with open('conf/default.conf', 'r') as confFile: confStr = confFile.read() conf = json.JSONDecoder().decode(confStr) dbStaticResult = conf['database']['db_bim_rap_result_db'] conn = MySQLdb.connect(host=dbStaticResult['host'],\ user=dbStaticResult['user'],\ passwd=dbStaticResult['password'],\ db=dbStaticResult['database'],\ port=dbStaticResult['port']) cur = conn.cursor() tagged_query_dict_url = sys.stdin.readline().strip() tagged_query_dict_detail_url = sys.stdin.readline().strip() insertion = 'insert into tagged_query_dict(task_id, dict_file_url, insert_datetime, update_datetime ) values ("' + sys.argv[ 1] + '","' + tagged_query_dict_url + '",' + cur_time + ' ,' + cur_time + ' )' cur.execute(insertion)
import asyncio
from collections import OrderedDict
from datetime import datetime
import hashlib
import json
import ldap_login
import queue
import socket
import time
from threading import Thread
from util import prt, uniq
import uuid
import websocket
json_decoder = json.JSONDecoder(object_pairs_hook=OrderedDict)
connected2hoarder = False
company = 'hm'
department = 'cf'
group = 'jmw'
token = '441234567890=='
agents = {}
ws = None
agent_job2queue = {}
def handle_agent_action(sock, agent, action, data):
if action == 'register':
agent = data['agent']
inputs = uniq(data['inputs'])
cleanses = uniq(data['cleanses'])
def read_storage(storage_path):
with open(storage_path, 'r') as file_read:
data_read = file_read.read()
data_read = json.JSONDecoder().decode(data_read)
return data_read
def BuildSTBLChanges () -> bool:
canBuildSTBL = STBL.CanBuildSTBL() # type: bool
if not canBuildSTBL:
return False
for package in Mod.GetCurrentMod().Packages: # type: Mod.Package
if not os.path.exists(package.STBLPath):
continue
for stblXMLFileName in os.listdir(package.STBLPath): # type: str
stblXMLFilePath = os.path.join(package.STBLPath, stblXMLFileName) # type: str
if os.path.isfile(stblXMLFilePath) and os.path.splitext(stblXMLFileName)[1].casefold() == ".xml":
manifestFilePath = os.path.splitext(stblXMLFilePath)[0] + "_Manifest.json" # type: str
modifiedTime = os.path.getmtime(stblXMLFilePath) # type: float
builtModifiedTime = None # type: typing.Optional[int]
builtFileNames = list() # type: typing.List[str]
try:
if os.path.exists(manifestFilePath):
with open(manifestFilePath) as manifestFile:
manifest = json.JSONDecoder().decode(manifestFile.read()) # type: typing.Dict[str, typing.Any]
if not isinstance(manifest, dict):
raise Exceptions.IncorrectTypeException(manifest, "Root", (dict,))
if ManifestBuiltModifiedTimeKey in manifest:
builtModifiedTime = manifest[ManifestBuiltModifiedTimeKey]
if not isinstance(builtModifiedTime, float) and not isinstance(builtModifiedTime, int):
incorrectValue = builtModifiedTime # type: typing.Any
builtModifiedTime = None
raise Exceptions.IncorrectTypeException(incorrectValue, "Root[%s]" % ManifestBuiltModifiedTimeKey, (dict,))
if ManifestBuiltFileNamesKey in manifest:
builtFileNames = manifest[ManifestBuiltFileNamesKey]
if not isinstance(builtFileNames, list):
incorrectValue = builtFileNames # type: typing.Any
builtFileNames = list()
raise Exceptions.IncorrectTypeException(incorrectValue, "Root[%s]" % ManifestBuiltFileNamesKey, (dict,))
for builtFileNameIndex in range(len(builtFileNames)): # type: int
builtFileName = builtFileNames[builtFileNameIndex] # type: str
if not isinstance(builtFileName, str):
builtFileNames = list()
raise Exceptions.IncorrectTypeException(builtFileName, "Root[%s][%s]" % (ManifestBuiltFileNamesKey, builtFileNameIndex), (dict,))
except Exception as e:
print("Failed to read STBL manifest file at '" + manifestFilePath + "'\n" + str(e), file = sys.stderr)
missingBuiltFile = False # type: bool
for builtFileName in builtFileNames:
builtFilePath = os.path.join(os.path.join(package.SourceLoosePath, "STBL"), builtFileName) # type: str
if not os.path.exists(builtFilePath):
missingBuiltFile = True
break
if missingBuiltFile or modifiedTime != builtModifiedTime:
buildTempDirectory = stblXMLFilePath + "_Temp_Build" # type: str
if not os.path.exists(buildTempDirectory):
os.makedirs(buildTempDirectory)
try:
STBL.BuildSTBL(buildTempDirectory, stblXMLFilePath)
manifest = dict() # type: typing.Dict[str, typing.Any]
manifest[ManifestBuiltModifiedTimeKey] = modifiedTime
builtFileNames = list()
for builtFileName in os.listdir(buildTempDirectory):
builtFilePath = os.path.join(buildTempDirectory, builtFileName)
if os.path.isfile(builtFilePath):
builtFileNames.append(builtFileName)
manifest[ManifestBuiltFileNamesKey] = builtFileNames
with open(manifestFilePath, "w+") as manifestFile:
manifestFile.write(json.JSONEncoder(indent = "\t").encode(manifest))
dir_util.copy_tree(buildTempDirectory, os.path.join(package.SourceLoosePath, "STBL"))
finally:
shutil.rmtree(buildTempDirectory)
return True
def parse_json(self):
self.result = [json.JSONDecoder(object_pairs_hook=OrderedDict).decode(self.page)]
class FileOper:
_lock = None
fileName = 'RunTimeConfig.dat'
dirName = 'RunTimeData'
log = logger.log
filePos = 0
jsonDecoder = json.JSONDecoder()
jsonEncoder = json.JSONEncoder()
@property
def lock(self):
return self._lock
def __init__(self, fileName='RunTimeConfig.dat', forWrite=True):
if not os.path.exists(self.dirName):
os.makedirs(self.dirName)
self._lock = threading.Lock()
self.fileName = self.dirName + '/' + fileName
self.forWrite = forWrite
def writeObj(self, obj):
with self.lock:
try:
with open(self.fileName, 'a+') as fh:
# pickle.dump(obj, fh)
objStr = self.jsonEncoder.encode(obj)
fh.write(objStr + '\n')
except:
self.log._log.error('写文件失败!%s', sys.exc_info()[1])
def readObj(self):
with self.lock:
try:
if not os.path.exists(self.fileName):
return None
if self.filePos == os.path.getsize(self.fileName):
return None
with open(self.fileName, 'r+') as fh:
fh.seek(self.filePos)
# obj = pickle.load(fh)
obj = self.jsonDecoder.decode(fh.readline())
self.filePos = fh.tell()
return obj
except:
print(sys.exc_info()[1])
self.log._log.error('读文件失败!%s', sys.exc_info()[1])
return None
def clean(self):
with self.lock:
try:
if not os.path.exists(self.fileName):
return
with open(self.fileName, 'wb+') as fh:
fh.truncate()
except:
self.log._log.error('清空文件失败!%s', sys.exc_info()[1])
def reachEnd(self):
with self.lock:
try:
if self.filePos == os.path.getsize(self.fileName):
return True
else:
return False
except:
self.log._log.error('获取文件状态失败!%s', sys.exc_info()[1])
return True
# condition in libghfu
server_log_file_path = os.path.join(path, "Server", "log")
server_log_file = open(server_log_file_path, "a")
def server_log(_log):
server_log_file.write("\n{}: {}".format(time.asctime(), _log))
app = Flask(__name__)
known_clients = "127.0.0.1"
jencode = json.JSONEncoder().encode
jdecode = json.JSONDecoder().decode
#mutex = threading.Lock()
LAST_PERFORMED_MONTHLY_OPERATIONS = [0, 0, 0]
TRANSACTION_CHECK_DELAY = .5 # delay for checking if pending transaction has been effected
JPESA_DEPOSIT_CHARGES = .03 # also applies when tranfering funds from jpesa to jpesa
JPESA_WITHDRAW_CHARGES = 500.0
CODES = {
} # every time a random code is generated, its stored here to hold transaction data
# so when the client wants to know about the state of a transaction, i dont querry the
# jpesa api but rather, i consult with this dictionary and return whatever the latest status
# of the transaction is kept here. the moment a code is nolonger needed, its deleted otherwise
# the server would end up consuming GBs of memory holding un-needed data!
def read_json_content(filename):
with open(filename, 'r') as f:
return json.JSONDecoder(object_pairs_hook=collections.OrderedDict).decode(f.read(os.stat(filename).st_size))
def __init__(self, socket_path, executor=None, logger=logging, encoder_cls=json.JSONEncoder, decoder=json.JSONDecoder()):
self.socket_path = socket_path
self.encoder_cls = encoder_cls
self.decoder = decoder
self.executor = executor
self.logger = logger
# Do we require the compatibility mode?
self._compat = True
self.next_id = 0
def get(self, request, format=None):
variables = json.loads(request.GET.get('variables', None))
for item in variables:
print item['variable_name']
if item['axis'] == 'x':
xaxis = item['variable_name']
if item['axis'] == 'y':
yaxis = item['variable_name']
#collection name check
if variables[0]['collection_name'] == variables[1]['collection_name']:
collectionname = variables[0]['collection_name']
engine = create_engine('mysql+pymysql://root:@localhost/test?charset=utf8', echo=True)
conn = engine.connect(close_with_result=True)
cnx = engine.raw_connection()
xx = pd.read_sql('Select '+ xaxis +' as label, '+ yaxis +' as value FROM '+ collectionname, cnx)
xxsum = xx.groupby(['label']).sum()
xxsum['index_col'] = range(1, len(xxsum) + 1)
xxmin = xx.groupby(['label']).min()
xxmin['index_col'] = range(1, len(xxmin) + 1)
xxmax = xx.groupby(['label']).max()
xxmax['index_col'] = range(1, len(xxmax) + 1)
xxavg = xx.groupby(['label']).mean()
xxavg['index_col'] = range(1, len(xxavg) + 1)
resultsumdump = json.JSONDecoder().decode(xxsum.reset_index().to_json(orient="records"))
resultmindump = json.JSONDecoder().decode(xxmin.reset_index().to_json(orient="records"))
resultmaxdump = json.JSONDecoder().decode(xxmax.reset_index().to_json(orient="records"))
resultavgdump = json.JSONDecoder().decode(xxavg.reset_index().to_json(orient="records"))
# resultsum = conn.execute('Select district as label, SUM(salary_basic) as value FROM teacher_final_data group by district;')
# resultAvg = conn.execute('SELECT district as label, AVG(salary_basic) as value FROM teacher_final_data group by district;')
# resultMin = conn.execute('SELECT district as label, MIN(salary_basic) as value FROM teacher_final_data group by district;')
# resultMax = conn.execute('SELECT district as label, SUM(salary_basic) as value FROM teacher_final_data group by district;')
#
# resultsumdump = json.JSONDecoder().decode(json.dumps([dict(r) for r in resultsum], cls=DjangoJSONEncoder))
# resultavgdump = json.JSONDecoder().decode(json.dumps([dict(r) for r in resultAvg], cls=DjangoJSONEncoder))
# resultmindump = json.JSONDecoder().decode(json.dumps([dict(r) for r in resultMin], cls=DjangoJSONEncoder))
# resultmaxdump = json.JSONDecoder().decode(json.dumps([dict(r) for r in resultMax], cls=DjangoJSONEncoder))
finalRes =[];
finalRes.append({"sum": resultsumdump,"avg": resultavgdump, "min": resultmindump, "max": resultmaxdump})
# agg = request.GET.get('agg', None)
# if agg == "sum":
# # queryset = ChartData.objects.values('label').annotate(value=Sum('value'))
# result = conn.execute('Select district as label, SUM(salary_basic) as value FROM teacher_final_data group by district;')
# elif agg == "avg":
# # queryset = ChartData.objects.values('label').annotate(value=Avg('value'))
# result = conn.execute('SELECT id, district as label, AVG(salary_basic) as value FROM teacher_final_data group by district;')
# elif agg == "min":
# # queryset = ChartData.objects.values('label').annotate(value=Min('value'))
# result = conn.execute(
# 'SELECT id, district as label, MIN(salary_basic) as value FROM teacher_final_data group by district;')
# elif agg == "max":
# # queryset = ChartData.objects.values('label').annotate(value=Max('value'))
# result = conn.execute(
# 'SELECT id, district as label, MAX(salary_basic) as value FROM teacher_final_data group by district;')
# else:
# result = conn.execute(
# 'SELECT id, district as label, SUM(salary_basic) as value FROM teacher_final_data group by district;')
# # result = conn.execute(
# # 'SELECT district as label, SUM(salary_basic) as value FROM teacher_final_data group by district;')
# a = json.dumps([dict(r) for r in result], cls=DjangoJSONEncoder)
# b = json.JSONDecoder().decode(a)
response = Response(finalRes, status=status.HTTP_200_OK)
return response
def _read_only_self(self):
"""
Parse request file and return a tuple of pvs, metadata and includes.
In case of problems returns (but does not raise) exceptions.
OSError
ReqParseError
ReqFileFormatError
ReqFileInfLoopError
:return: A tuple (pv_list, metadata, includes_list)
"""
pvs = list()
includes = list()
try:
with open(self._path) as f:
file_data = f.read()
except OSError as e:
return e
if file_data.lstrip().startswith('{'):
try:
md = file_data.lstrip()
metadata, end_of_metadata = \
json.JSONDecoder().raw_decode(md)
except json.JSONDecodeError:
msg = f"{self._path}: Could not parse JSON metadata header."
return ReqParseError(msg)
# Ensure line counts make sense for error reporting.
actual_data = md[end_of_metadata:].lstrip()
actual_data_index = file_data.find(actual_data)
self._curr_line_n = len(file_data[:actual_data_index].splitlines())
file_data = file_data[actual_data_index:]
else:
metadata = {}
self._curr_line_n = 0
for self._curr_line in file_data.splitlines():
self._curr_line_n += 1
self._curr_line = self._curr_line.strip()
# skip comments, empty lines and "data{}" stuff
if not self._curr_line.startswith(('#', "data{", "}", "!")) \
and self._curr_line.strip():
# First replace macros, then check if any unreplaced macros
# which are not "global"
pvname = SnapshotPv.macros_substitution(
(self._curr_line.rstrip().split(',', maxsplit=1)[0]),
self._macros)
try:
# Check if any unreplaced macros
self._validate_macros_in_txt(pvname)
except MacroError as e:
return ReqParseError(
self._format_err((self._curr_line_n, self._curr_line),
e))
pvs.append(pvname)
elif self._curr_line.startswith('!'):
# Calling another req file
split_line = self._curr_line[1:].split(',', maxsplit=1)
if len(split_line) > 1:
macro_txt = split_line[1].strip()
if not macro_txt.startswith(('\"', '\'')):
return ReqFileFormatError(
self._format_err(
(self._curr_line_n, self._curr_line),
'Syntax error. Macro argument must be quoted'))
else:
quote_type = macro_txt[0]
if not macro_txt.endswith(quote_type):
return ReqFileFormatError(
self._format_err(
(self._curr_line_n, self._curr_line),
'Syntax error. Macro argument must be quoted'))
macro_txt = SnapshotPv.macros_substitution(
macro_txt[1:-1], self._macros)
try:
# Check for any unreplaced macros
self._validate_macros_in_txt(macro_txt)
macros = parse_macros(macro_txt)
except MacroError as e:
return ReqParseError(
self._format_err(
(self._curr_line_n, self._curr_line), e))
else:
macros = dict()
path = os.path.join(os.path.dirname(self._path), split_line[0])
msg = self._check_looping(path)
if msg:
return ReqFileInfLoopError(
self._format_err((self._curr_line_n, self._curr_line),
msg))
try:
sub_f = SnapshotReqFile(path, parent=self, macros=macros)
includes.append(sub_f)
except OSError as e:
return OSError(
self._format_err((self._curr_line, self._curr_line_n),
e))
return (pvs, metadata, includes)
新浪微博：http://weibo.com/ziyuetk
'''
import urllib2, json
from city import city
yourcity = raw_input("你想查那个城市的天气？")
#测试yourcity变量是否可以返回你想要的值
#print yourcity
url = "http://www.weather.com.cn/data/cityinfo/" + city[yourcity] + ".html"
#print url #同上
response = urllib2.urlopen(url, timeout=10)
city_dict = response.read()
#此处打印出来是一个json格式的东西
#print city_dict
#用Python的json库来解析获取到的网页json内容
jsondata = json.JSONDecoder().decode(city_dict)
#定义几个变量用来储存解析出来的内容
temp_low = jsondata['weatherinfo']['temp1']
temp_high = jsondata['weatherinfo']['temp2']
weather = jsondata['weatherinfo']['weather']
print yourcity
print weather
print temp_low + "~" + temp_high
#!/usr/bin/env python
#coding:utf-8
#########################################################################################################################
## Version : 0.0.7-1
## Developer : Yannyann (https://github.com/a2d8a4v)
## Website : https://www.yannyann.com
## License : MIT License
#########################################################################################################################
import sys, urllib2, socket, json
try:
socket.setdefaulttimeout(5)
if len(sys.argv) == 1:
apiurl = "http://ip-api.com/json"
elif len(sys.argv) == 2:
apiurl = "http://ip-api.com/json/%s" % sys.argv[1]
content = urllib2.urlopen(apiurl).read()
content = json.JSONDecoder().decode(content)
if content['status'] == 'success':
print(content['isp'])
else:
print("NO")
except:
print("Usage:%s IP" % sys.argv[0])
def _load_layers(self):
model_path = self._get_model_path()
decoder = json.JSONDecoder()
with open(model_path, 'r') as hand:
data = hand.read().strip()
return decoder.decode(data)
def update_app(self):
contract = jc.Contract()
spec = {}
for name, value in self.initial_app_spec.items():
if name == 'name':
spec[name] = value
elif name == 'cloudProviders':
spec[name] = value + ',kubernetes'
elif name in ['updateTs', 'createTs']:
spec[name] = str(int(value) + 1)
elif isinstance(value, basestring):
spec[name] = 'NEW_' + value
payload = self.agent.make_json_payload_from_object(spec)
expectUpdate = dict(spec)
# The actual update is determined by front50.
# The createTs we gave ignored.
# As before, the name is upper-cased.
del expectUpdate['updateTs']
expectUpdate['createTs'] = self.initial_app_spec['createTs']
expectUpdate['name'] = self.initial_app_spec['name'].upper()
self.app_history.insert(0, expectUpdate)
# TODO(ewiseblatt) 20160524:
# Add clauses that observe Front50 to verify the history method works
# and that the get method is the current version.
num_versions = 2 if self.versioning_enabled else 1
gcs_builder = gcp.GcpStorageContractBuilder(self.gcs_observer)
(gcs_builder.new_clause_builder(
'Google Cloud Storage Contains File',
retryable_for_secs=5).list_bucket(
self.BUCKET,
'/'.join([self.BASE_PATH, 'applications', self.TEST_APP]),
with_versions=True).contains_path_value('name',
self.TEST_APP,
min=num_versions,
max=num_versions))
(gcs_builder.new_clause_builder(
'Updated File Content', retryable_for_secs=5).retrieve_content(
self.BUCKET,
'/'.join([
self.BASE_PATH, 'applications', self.TEST_APP,
'specification.json'
]),
transform=json.JSONDecoder().decode).contains_match({
key: jp.EQUIVALENT(value)
for key, value in expectUpdate.items()
}))
for clause in gcs_builder.build().clauses:
contract.add_clause(clause)
f50_builder = st.http_observer.HttpContractBuilder(self.agent)
(f50_builder.new_clause_builder(
'History Records Changes').get_url_path(
'/v2/applications/{app}/history'.format(
app=self.TEST_APP)).contains_path_match(
'[0]', {
key: jp.EQUIVALENT(value)
for key, value in self.app_history[0].items()
}).contains_path_match(
'[1]', {
key: jp.EQUIVALENT(value)
for key, value in self.app_history[1].items()
}))
for clause in f50_builder.build().clauses:
contract.add_clause(clause)
# TODO(ewiseblatt): 20160524
# Add a mechanism here to check the previous version
# so that we can verify version recovery as well.
path = '/'.join(['/v2/applications', self.TEST_APP])
return st.OperationContract(self.new_patch_operation(
title='update_app', data=payload, path=path),
contract=contract)
def __init__(self, **kwargs):
super().__init__()
import json
self.decoder = json.JSONDecoder(**self.get_opts(kwargs))
def main():
args = parse_args()
# Initialize
dbmng = DBManager(args.db)
# Only one session to access to database
_session = dbmng.get_session()
# Open the file
decorder = json.JSONDecoder(object_pairs_hook=collections.OrderedDict)
input_f = open(args.input_file, 'r')
input_j = decorder.decode(input_f.read())
# Load json keys and make it inserted
nowtime = datetime.datetime.now()
for key in input_j.keys():
if key == 'translations':
for dic in input_j[key]:
rec = Translation(id=dic['id'],
locale=dic['locale'],
t=dic['t'],
created_at=nowtime,
created_by='rightctl_initializer')
_session.add(rec)
# break しないときは、else は使わないほうが良いらしい。
# else:
_session.commit()
elif key == 'rights':
for dic in input_j[key]:
rec = Right(action=dic['action'],
enable_flag=True,
created_at=nowtime,
created_by='rightctl_initializer')
_session.add(rec)
# break しないときは、else は使わないほうが良いらしい。
# else:
_session.commit()
elif key == 'policies':
for dic in input_j[key]:
rec = Policy(id=dic['id'],
policy_tid=dic['policy_tid'],
created_at=nowtime,
created_by='rightctl_initializer')
_session.add(rec)
_session.commit()
for right_s in dic['rights']:
right = _session.query(Right).filter(
Right.action == right_s).one()
child = PolicyHasRight(policy_id=rec.id, right_id=right.id)
_session.add(child)
# break しないときは、else は使わないほうが良いらしい。
# else:
_session.commit()
# break しないときは、else は使わないほうが良いらしい。
# else:
_session.commit()
elif key == 'roles':
for dic in input_j[key]:
rec = Role(id=dic['id'],
role_tid=dic['role_tid'],
created_at=nowtime,
created_by='rightctl_initializer')
_session.add(rec)
_session.commit()
for policy_s in dic['policies']:
policy = _session.query(Policy).filter(
Policy.id == policy_s).one()
child = RoleHasPolicy(role_id=rec.id, policy_id=policy.id)
_session.add(child)
# break しないときは、else は使わないほうが良いらしい。
# else:
_session.commit()
# break しないときは、else は使わないほうが良いらしい。
# else:
_session.commit()
import json
from collections import Mapping
from types import GeneratorType
from mo_dots import split_field, startswith_field, relative_field, Data, join_field, Null, wrap
from mo_logs import Log
DEBUG = False
MIN_READ_SIZE = 8 * 1024
WHITESPACE = b" \n\r\t"
CLOSE = {b"{": b"}", b"[": b"]"}
NO_VARS = set()
json_decoder = json.JSONDecoder().decode
def parse(json, query_path, expected_vars=NO_VARS):
"""
INTENDED TO TREAT JSON AS A STREAM; USING MINIMAL MEMORY WHILE IT ITERATES
THROUGH THE STRUCTURE. ASSUMING THE JSON IS LARGE, AND HAS A HIGH LEVEL
ARRAY STRUCTURE, IT WILL yield EACH OBJECT IN THAT ARRAY. NESTED ARRAYS
ARE HANDLED BY REPEATING THE PARENT PROPERTIES FOR EACH MEMBER OF THE
NESTED ARRAY. DEEPER NESTED PROPERTIES ARE TREATED AS PRIMITIVE VALUES;
THE STANDARD JSON DECODER IS USED.
LARGE MANY-PROPERTY OBJECTS CAN BE HANDLED BY `items()`
:param json: SOME STRING-LIKE STRUCTURE THAT CAN ASSUME WE LOOK AT
ONE CHARACTER AT A TIME, IN ORDER
def process_wiki_events(wiki, wiki_schedule, workshop_schedule = None, timestamp_offset = None, options = None, fetch_wikitext=True):
global sessions_complete, warnings, time_stamp_offset
if not timestamp_offset == None:
time_stamp_offset = timestamp_offset
sessions_complete = OrderedDict()
events_total = 0
events_successful = 0
events_in_halls = 0 # aka workshops
used_guids = []
debug = options and options.debug
def warn(msg, force = False):
global warnings, events_with_warnings, events_in_halls_with_warnings
if not warnings:
warnings = True
events_with_warnings += 1
if is_workshop_room_session:
events_in_halls_with_warnings += 1
if is_workshop_room_session or options.show_assembly_warnings or force:
print('')
print(event_wiki_name)
try: print(' at ' + start_time.isoformat() )
except NameError: pass
try: print(' in ' + room )
except NameError: pass
print(' ' + wiki_edit_url)
#if not is_workshop_room_session:
# msg += ' – at assembly?'
if is_workshop_room_session or options.show_assembly_warnings or force:
print(msg)
#for event_wiki_name, event_r in wiki.events.iteritems(): #python2
for event_wiki_name, event_r in wiki.events.items(): #python3
warnings = False
sys.stdout.write('.')
try:
wiki_page_name = event_wiki_name.split('#')[0].replace(' ', '_') # or see fullurl property
wiki_edit_url = wiki.wiki_url + '/index.php?title=' + urllib.parse.quote_plus(wiki_page_name) + '&action=edit'
wiki_parsetree_url = wiki.wiki_url + '/api.php?action=parse&format=json&page=' + urllib.parse.quote_plus(wiki_page_name) + '&prop=parsetree'
session = wiki.parent_of_event(event_wiki_name)
event = event_r['printouts']
event_n = None
events_total += 1
# One Event take place in multiple rooms...
# WORKAROND If that is the case just pick the first one
room = ''
is_workshop_room_session = False
if len(event['Has session location']) == 1:
room = event['Has session location'][0]['fulltext']
if room.split(':', 1)[0] == 'Room':
is_workshop_room_session = True
room = Wiki.remove_prefix(room)
elif len(event['Has session location']) == 0:
warn(" has no room yet, skipping...")
continue
else:
warn(" WARNING: has multiple rooms ???, just picking the first one…")
event['Has session location'] = event['Has session location'][0]['fulltext']
# http://stackoverflow.com/questions/22698244/how-to-merge-two-json-string-in-python
# This will only work if there are unique keys in each json string.
#combined = dict(session.items() + event.items()) #python2
combined = session.copy() #python3 TOOD test if this really leads to the same result
combined.update(event)
sessions_complete[event_wiki_name] = combined
if len(event['Has start time']) < 1:
warn(" has no start time")
day = None
else:
date_time = datetime.fromtimestamp(int(event['Has start time'][0]['timestamp']) + time_stamp_offset)
start_time = tz.localize(date_time)
day = wiki_schedule.get_day_from_time(start_time)
#if is_workshop_room_session and day is not None and event['Has duration']:
if day is not None and event['Has duration']:
duration = 0
if event['Has duration']:
duration = event['Has duration'][0]
if not isinstance(duration, int) and 'value' in duration:
duration = duration['value']
if duration > 60*24:
warn(' event takes longer than 24h, skipping...')
continue
lang = ''
if session['Held in language'] and len(session['Held in language']) > 0:
lang = session['Held in language'][0].split(' - ', 1)[0]
if len(event['GUID']) > 0:
guid = event['GUID'][0]
if not isinstance(guid, str):
raise Exception('GUID is not string, but ' + guid)
else:
guid = voc.tools.gen_uuid(session['fullurl'] + str(event['Has start time'][0]))
warn(' GUID was empty, generated one for now. Not shure if its stable...')
#if debug:
# print_json(event['GUID'])
if guid in used_guids:
warn(' GUID {} was already used before, generated a random one for now. Please fix the session wiki page to ensure users can stay subscribed to event!'.format(guid), force=True)
guid = voc.tools.gen_uuid(session['fullurl'] + str(event['Has start time'][0]))
used_guids.append(guid)
local_id = voc.tools.get_id(guid)
description = ("\n".join(session['Has description'])).strip()
if fetch_wikitext:
if int(session['Modification date'][0]['timestamp']) > voc.tools.last_edited.get(guid, 0):
wiki_text = ''
# Retry up to three times
for _ in range(3):
content_r = requests.get(wiki_parsetree_url, timeout=5)
if content_r.ok is True:
print("Page {0} requested successfully!".format(wiki_parsetree_url))
break
print(".")
if content_r.ok is False:
print(" Requesting {1} failed, HTTP {0}.".format(content_r.status_code, wiki_parsetree_url))
else:
try:
wiki_text_tree = ET.fromstring(voc.tools.parse_json(content_r.text)['parse']['parsetree'].replace('\n',''))
except AttributeError:
wiki_text_tree = ET.fromstring(voc.tools.parse_json(content_r.text)['parse']['parsetree']['*'].replace('\n',''))
except KeyError:
print(wiki_parsetree_url)
for element in wiki_text_tree.iterfind('template'):
if element.tail != None:
wiki_text += html.unescape(element.tail)
description = ("\n".join(session['Has description'])).strip() + '\n' + wiki_text
else: # unmodified
if os.path.isfile("events/{guid}.json".format(guid=guid)):
with open("events/{guid}.json".format(guid=guid), "r") as fp:
# maintain order from file
temp = fp.read()
old_event = json.JSONDecoder().decode(temp)
description = old_event['description']
voc.tools.last_edited[guid] = int(session['Modification date'][0]['timestamp'])
event_n = Event([
('id', local_id),
('guid', guid),
('url', session['fullurl']), # TODO: add enshure_url() which adds "https:"+ prefix when neccessary
('logo', None),
('date', start_time.isoformat()),
('start', start_time.strftime('%H:%M')),
#('duration', str(timedelta(minutes=event['Has duration'][0])) ),
('duration', '%d:%02d' % divmod(duration, 60) ),
('room', room),
('slug', '{slug}-{id}-{name}'.format(
slug=wiki_schedule.conference()['acronym'].lower(),
id=local_id,
name=voc.tools.normalise_string(session['wiki_name'].lower())
)),
('title', session['Has title'][0]),
('subtitle', "\n".join(event['Has subtitle']) ),
('track', 'self organized sessions'),
('type', " ".join(session['Has session type']).lower()),
('language', lang ),
('abstract', ''),
('description', description),
('persons', [ OrderedDict([
('id', 0),
('url', p['fullurl']), # sometimes a https: is needed...
('public_name', Wiki.remove_prefix(p['fulltext'])), # must be last element so that transformation to xml works
]) for p in session['Is organized by'] ]),
('links', [ OrderedDict([
('url', url), # TODO sometimes a https:// is needed...
('title', url),
]) for url in session['Has website'] ])
], start_time)
# Break if conference day date and event date do not match
conference_day_start = wiki_schedule.day(day).start
conference_day_end = wiki_schedule.day(day).end
if not conference_day_start <= event_n.start < conference_day_end:
raise Exception("Current conference day from {0} to {1} does not match current event {2} with date {3}."
.format(conference_day_start, conference_day_end, event_n["id"], event_n.start))
# Events from day 0 (26. December) do not go into the full schdedule
if start_time.day != 26 and not only_workshops:
wiki_schedule.add_event(event_n)
if workshop_schedule and is_workshop_room_session:
events_in_halls +=1
workshop_schedule.add_event(event_n)
events_successful += 1
except Warning as w:
warn(w)
except:
if 'event_n' in locals(): print(event_n)
if 'event' in locals(): print(json.dumps(event, indent=2))
print(" unexpected error: " + str(sys.exc_info()[0]))
traceback.print_exc()
if options.exit_when_exception_occours:
exit()
store_sos_ids()
store_last_edited()
if debug:
with open("sessions_complete.json", "w") as fp:
json.dump(sessions_complete, fp, indent=2)
print("\nFrom %d total events (%d in halls) where %d successful, while %d (%d in halls) produced warnings" % (events_total, events_in_halls, events_successful, events_with_warnings, events_in_halls_with_warnings))
if not options.show_assembly_warnings:
print(" (use --show-assembly-warnings cli option to show all warnings)")
def find_book(self, bookid=None):
resultlist = []
URL = 'https://www.googleapis.com/books/v1/volumes/' + bookid
jsonresults = json.JSONDecoder().decode(
urllib2.urlopen(URL, timeout=30).read())
try:
bookdate = item['volumeInfo']['publishedDate']
except KeyError:
bookdate = 'Unknown'
try:
bookimg = item['volumeInfo']['imageLinks']['thumbnail']
except KeyError:
bookimg = 'images/nocover.png'
try:
bookrate = item['volumeInfo']['averageRating']
except KeyError:
bookrate = 0
try:
bookpages = item['volumeInfo']['pageCount']
except KeyError:
bookpages = 0
try:
bookgenre = item['volumeInfo']['categories']
except KeyError:
bookgenre = 'Unknown'
try:
bookdesc = item['volumeInfo']['description']
except KeyError:
bookdesc = 'Not available'
try:
if item['volumeInfo']['industryIdentifiers'][0][
'type'] == 'ISBN_10':
bookisbn = item['volumeInfo']['industryIdentifiers'][0][
'identifier']
else:
bookisbn = 0
except KeyError:
bookisbn = 0
resultlist.append({
'bookname': item['volumeInfo']['title'],
'bookisbn': bookisbn,
'bookdate': bookdate,
'booklang': item['volumeInfo']['language'],
'booklink': item['volumeInfo']['canonicalVolumeLink'],
'bookrate': float(bookrate),
'bookimg': bookimg,
'bookpages': bookpages,
'bookgenre': bookgenre,
'bookdesc': bookdesc
})
return resultlist
def load_last_edited():
if os.path.isfile("_last_edited.json"):
with open("_last_edited.json", "r") as fp:
# maintain order from file
temp = fp.read()
voc.tools.last_edited = json.JSONDecoder(object_pairs_hook=OrderedDict).decode(temp)
def yq(input_streams=None, output_stream=None, input_format="yaml", output_format="json",
program_name="yq", width=None, indentless_lists=False, xml_root=None, xml_dtd=False,
explicit_start=False, explicit_end=False, jq_args=frozenset(), exit_func=None):
if not input_streams:
input_streams = [sys.stdin]
if not output_stream:
output_stream = sys.stdout
if not exit_func:
exit_func = sys.exit
converting_output = True if output_format != "json" else False
try:
# Note: universal_newlines is just a way to induce subprocess to make stdin a text buffer and encode it for us
jq = subprocess.Popen(["jq"] + list(jq_args),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE if converting_output else None,
universal_newlines=True)
except OSError as e:
msg = "{}: Error starting jq: {}: {}. Is jq installed and available on PATH?"
exit_func(msg.format(program_name, type(e).__name__, e))
try:
if converting_output:
# TODO: enable true streaming in this branch (with asyncio, asyncproc, a multi-shot variant of
# subprocess.Popen._communicate, etc.)
# See https://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python
use_annotations = True if output_format == "annotated_yaml" else False
input_docs = []
for input_stream in input_streams:
if input_format == "yaml":
loader = get_loader(use_annotations=use_annotations)
input_docs.extend(yaml.load_all(input_stream, Loader=loader))
elif input_format == "xml":
import xmltodict
input_docs.append(xmltodict.parse(input_stream.read(), disable_entities=True))
elif input_format == "toml":
import toml
input_docs.append(toml.load(input_stream))
else:
raise Exception("Unknown input format")
input_payload = "\n".join(json.dumps(doc, cls=JSONDateTimeEncoder) for doc in input_docs)
jq_out, jq_err = jq.communicate(input_payload)
json_decoder = json.JSONDecoder(object_pairs_hook=OrderedDict)
if output_format == "yaml" or output_format == "annotated_yaml":
yaml.dump_all(decode_docs(jq_out, json_decoder), stream=output_stream,
Dumper=get_dumper(use_annotations=use_annotations, indentless=indentless_lists),
width=width, allow_unicode=True, default_flow_style=False,
explicit_start=explicit_start, explicit_end=explicit_end)
elif output_format == "xml":
import xmltodict
for doc in decode_docs(jq_out, json_decoder):
if xml_root:
doc = {xml_root: doc}
elif not isinstance(doc, OrderedDict):
msg = ("{}: Error converting JSON to XML: cannot represent non-object types at top level. "
"Use --xml-root=name to envelope your output with a root element.")
exit_func(msg.format(program_name))
full_document = True if xml_dtd else False
try:
xmltodict.unparse(doc, output=output_stream, full_document=full_document, pretty=True,
indent=" ")
except ValueError as e:
if "Document must have exactly one root" in str(e):
raise Exception(str(e) + " Use --xml-root=name to envelope your output with a root element")
else:
raise
output_stream.write(b"\n" if sys.version_info < (3, 0) else "\n")
elif output_format == "toml":
import toml
for doc in decode_docs(jq_out, json_decoder):
if not isinstance(doc, OrderedDict):
msg = "{}: Error converting JSON to TOML: cannot represent non-object types at top level."
exit_func(msg.format(program_name))
if USING_PYTHON2:
# For Python 2, dump the string and encode it into bytes.
output = toml.dumps(doc)
output_stream.write(output.encode("utf-8"))
else:
# For Python 3, write the unicode to the buffer directly.
toml.dump(doc, output_stream)
else:
if input_format == "yaml":
loader = get_loader(use_annotations=False)
for input_stream in input_streams:
for doc in yaml.load_all(input_stream, Loader=loader):
json.dump(doc, jq.stdin, cls=JSONDateTimeEncoder)
jq.stdin.write("\n")
elif input_format == "xml":
import xmltodict
for input_stream in input_streams:
json.dump(xmltodict.parse(input_stream.read(), disable_entities=True), jq.stdin)
jq.stdin.write("\n")
elif input_format == "toml":
import toml
for input_stream in input_streams:
json.dump(toml.load(input_stream), jq.stdin)
jq.stdin.write("\n")
else:
raise Exception("Unknown input format")
jq.stdin.close()
jq.wait()
for input_stream in input_streams:
input_stream.close()
exit_func(jq.returncode)
except Exception as e:
exit_func("{}: Error running jq: {}: {}.".format(program_name, type(e).__name__, e))
def test_cli(node_factory):
l1 = node_factory.get_node()
out = subprocess.check_output([
'cli/lightning-cli',
'--lightning-dir={}'.format(l1.daemon.lightning_dir), 'help'
]).decode('utf-8')
# Test some known output.
assert 'help [command]\n List available commands, or give verbose help on one {command}' in out
# Test JSON output.
out = subprocess.check_output([
'cli/lightning-cli',
'--lightning-dir={}'.format(l1.daemon.lightning_dir), '-J', 'help'
]).decode('utf-8')
j, _ = json.JSONDecoder().raw_decode(out)
assert j['help'][0]['command'] is not None
assert j['help'][0]['description'] is not None
# Test keyword input (autodetect)
out = subprocess.check_output([
'cli/lightning-cli',
'--lightning-dir={}'.format(l1.daemon.lightning_dir), '-J', 'help',
'command=help'
]).decode('utf-8')
j, _ = json.JSONDecoder().raw_decode(out)
assert 'help [command]' in j['help'][0]['verbose']
# Test keyword input (forced)
out = subprocess.check_output([
'cli/lightning-cli',
'--lightning-dir={}'.format(l1.daemon.lightning_dir), '-J', '-k',
'help', 'command=help'
]).decode('utf-8')
j, _ = json.JSONDecoder().raw_decode(out)
assert 'help [command]' in j['help'][0]['verbose']
# Test ordered input (autodetect)
out = subprocess.check_output([
'cli/lightning-cli',
'--lightning-dir={}'.format(l1.daemon.lightning_dir), '-J', 'help',
'help'
]).decode('utf-8')
j, _ = json.JSONDecoder().raw_decode(out)
assert 'help [command]' in j['help'][0]['verbose']
# Test ordered input (forced)
out = subprocess.check_output([
'cli/lightning-cli',
'--lightning-dir={}'.format(l1.daemon.lightning_dir), '-J', '-o',
'help', 'help'
]).decode('utf-8')
j, _ = json.JSONDecoder().raw_decode(out)
assert 'help [command]' in j['help'][0]['verbose']
# Test missing parameters.
try:
# This will error due to missing parameters.
# We want to check if lightningd will crash.
out = subprocess.check_output([
'cli/lightning-cli',
'--lightning-dir={}'.format(l1.daemon.lightning_dir), '-J', '-o',
'sendpay'
]).decode('utf-8')
except Exception:
pass
# Test it escapes JSON completely in both method and params.
# cli turns " into \", reply turns that into \\\".
out = subprocess.run([
'cli/lightning-cli', '--lightning-dir={}'.format(
l1.daemon.lightning_dir), 'x"[]{}'
],
stdout=subprocess.PIPE)
assert 'Unknown command \'x\\\\\\"[]{}\'' in out.stdout.decode('utf-8')
subprocess.check_output([
'cli/lightning-cli',
'--lightning-dir={}'.format(l1.daemon.lightning_dir), 'invoice',
'123000', 'l"[]{}', 'd"[]{}'
]).decode('utf-8')
# Check label is correct, and also that cli's keyword parsing works.
out = subprocess.check_output([
'cli/lightning-cli',
'--lightning-dir={}'.format(l1.daemon.lightning_dir), '-k',
'listinvoices', 'label=l"[]{}'
]).decode('utf-8')
j = json.loads(out)
assert only_one(j['invoices'])['label'] == 'l"[]{}'
def getPics(html):
match = re.search('var J_photo = (.*);', html)
if match is not None:
obj_string = match.group(1)
obj = json.JSONDecoder().decode(obj_string)
return obj
def __init__(self, **kwargs):
self.decoder = json.JSONDecoder(**kwargs)
if kwargs:
self.module = 'default'
else:
self.module = MODULE
