def updateSummary(self):
context = aq_inner(self.context)
# If the Summarizer Source is inactive, we're done
if not context.active:
raise SourceNotActive(context)
# Check if the Summarizer Source has an Summarizer Generator
if not context.generator:
raise NoGeneratorError(context)
generator = context.generator.to_object
generatorPath = '/'.join(generator.getPhysicalPath())
# Adapt the generator to a graph generator, and get the graph in XML form.
serialized = None
mimetype = None
if generator.datatype == 'json':
generator = IJsonGenerator(generator)
serialized = generator.generateJson()
json = jsonlib.read(serialized)
mimetype = SUMMARIZER_JSON_MIMETYPE
# Is there an active file?
if context.approvedFile:
# Is it identical to what we just generated?
print context.approvedFile.to_object.get_data()
current = jsonlib.read(context.approvedFile.to_object.get_data())
if sorted(json.items()) == sorted(current.items()):
raise NoUpdateRequired(context)
elif generator.datatype == 'rdf':
generator = IGraphGenerator(generator)
rdf = generator.generateGraph()
serialized = rdf.serialize()
mimetype = SUMMARIZER_XML_MIMETYPE
# Is there an active file?
if context.approvedFile:
# Is it identical to what we just generated?
current = Graph().parse(data=context.approvedFile.to_object.get_data())
if isomorphic(rdf, current):
raise NoUpdateRequired(context)
else:
raise UnknownGeneratorError(context)
# Create a new file and set it active
# TODO: Add validation steps here
timestamp = datetime.datetime.utcnow().isoformat()
newFile = context[context.invokeFactory(
'File',
context.generateUniqueId('File'),
title=u'Summary %s' % timestamp,
description=u'Generated at %s by %s' % (timestamp, generatorPath),
file=serialized,
)]
newFile.getFile().setContentType(mimetype)
newFile.reindexObject()
intIDs = getUtility(IIntIds)
newFileID = intIDs.getId(newFile)
context.approvedFile = RelationValue(newFileID)
notify(ObjectModifiedEvent(context))
def get_json_obj(self, key):
import jsonlib
print self.params[key][0]
if key in self.params:
print jsonlib.read(self.params[key][0])
return jsonlib.read(self.params[key][0])
return None
def re(self, string, line, column, position, expected_error_message):
full_expected = ("JSON parsing error at line %d, column %d"
" (position %d): %s" %
(line, column, position, expected_error_message))
try:
read(string)
except ReadError, error:
self.assertEqual(unicode(error), full_expected)
def test_fixed_truncation_error(self):
data = [{"a": 3}] * 270
fp = io.StringIO()
jsonlib.dump(data, fp, encoding=None)
self.assertEqual(len(read(fp.getvalue())), len(data))
self.assertEqual(len(read(write(data))), len(data))
def test_preserve_negative_decimal_zero_fp(self):
# Don't use self.r, because 0.0 == -0.0
value = read('[0.0]', use_float=True)
self.assertEqual(len(value), 1)
self.assertEqual(type(value[0]), float)
self.assertEqual(repr(value[0]), repr(0.0))
value = read('[-0.0]', use_float=True)
self.assertEqual(len(value), 1)
self.assertEqual(type(value[0]), float)
self.assertEqual(repr(value[0]), repr(-0.0))
def test_preserve_negative_decimal_zero(self):
# Don't use self.r, because Decimal ('0.0') == Decimal ('-0.0')
value = read('[0.0]')
self.assertEqual(len(value), 1)
self.assertEqual(type(value[0]), Decimal)
self.assertEqual(repr(value[0]), repr(Decimal("0.0")))
value = read('[-0.0]')
self.assertEqual(len(value), 1)
self.assertEqual(type(value[0]), Decimal)
self.assertEqual(repr(value[0]), repr(Decimal("-0.0")))
def load_json(json_str, as_utf8=True):
'''Build object from json string using jsonlib'''
try:
json_obj = jsonlib.read(json_str)
except jsonlib.ReadError as e:
json_obj = jsonlib.read(_remove_invalid_escape(json_str))
# if as_utf8 is true, convert unicode string in json_obj to utf8
if as_utf8: _encode_utf8(json_obj)
return json_obj
def verifyPurchase(self, userId, itemId, transactions):
returnObj = None
#json correctness hack - remove surrounding quotes on each dict in the array
transactions = urllib2.unquote(transactions)
p = re.compile('"{')
transactions = p.sub('{', transactions)
p = re.compile('}"')
transactions = p.sub('}', transactions)
if transactions is not None:
arrTransactions = json.read(transactions)
transactionId = None
transactions = json.read(transactions)
if transactions is not None:
for item in transactions:
if item['itemId'] == itemId:
transactionId = item['transactionId']
if transactionId is not None:
postDetails = {
'USER': common.UID,
'PWD': common.PASSWORD,
'SIGNATURE': common.SIG,
'METHOD': 'GetTransactionDetails',
'VERSION': common.VER,
'TRANSACTIONID': transactionId
}
response = dict(
cgi.parse_qsl(common.curl(common.URLBASE, postDetails)))
custom = string.split(response["CUSTOM"], ',')
if (identity.getUserId() == custom[0]) and (itemId == custom[1]):
returnObj = {
'success': 'true',
'error': '',
'transactionId': response["TRANSACTIONID"],
'orderTime': response["ORDERTIME"],
'paymentStatus': response["PAYMENTSTATUS"],
'itemId': itemId,
'userId': userId
}
else:
returnObj = {
'success': 'false',
'error': 'Item not found in transaction history'
}
print json.write(returnObj)
def ParseInterPage(page, params):
'''
'''
try:
data = jsonlib.read(page.decode("GBK", "ignore"))
except Exception, e:
return -1
def handle(self, *args, **options):
if len(args) == 1:
max_listings = int(args[0])
else:
max_listings = 10000
count = 0
previous_address = None
previous_lat = None
previous_lon = None
for home in Home.objects.filter(latitude__isnull=False).iterator():
if count > max_listings:
break
address = '%s, %s, %s %s' % (home.address, home.city, home.state, home.zip_code)
geo_home, created = GeocodedHome.objects.get_or_create(home=home)
if not address == previous_address:
if created:
url_string = "http://maps.google.com/maps/api/geocode/json?address=%s&sensor=false"
url = url_string % (urllib.quote_plus(address))
print 'Raw address: %s' % address
url_response = urlopen(url).read()
response = jsonlib.read(url_response)
if response['status'] == 'OK':
results = response['results']
# Use first result, but flag it if there is more than 1
flagged = len(results)>1
lat = results[0]['geometry']['location']['lat']
lng = results[0]['geometry']['location']['lng']
formatted_address = results[0]['formatted_address']
print 'Geocded address: %s' % formatted_address
print 'Lat/Lon: %s, %s' % (lat, lng)
geo_home.latitude=lat
geo_home.longitude=lng
geo_home.formatted_address=formatted_address
geo_home.source=url_response
geo_home.flagged=flagged
geo_home.save()
previous_lat = lat
previous_lon = lng
previous_address = address
print 'Home saved...%s' % home.address
count += 1
time.sleep(3)
else:
print 'Error: %s' % response['status']
print 'Exiting'
return
else:
print 'Already exists.'
else:
if previous_lat and previous_lon:
print 'Duplicate lat/lon for %s' % home.address
else:
print 'Skipping home %s' % home.address
def GET(self):
q = web.input()["q"]
elasticQuery = {
"filtered": {
"query":
# highlight seems to not work with fuzzy
{
"fuzzy": {
"_all": {
"value": q,
"prefix_length": 2
}
}
},
"filter": {
"or": [
{
"prefix": {
"value": q,
"boost": 2.0
}
},
]
}
}
}
response = elastic.request(method="get",
path="mpd/song/_search",
payload=jsonlib.write(elasticQuery))
result = jsonlib.read(response.body_string())
web.header("Content-Type", "application/json")
return jsonlib.write(result)
def ParseInterPage(page,params):
'''
'''
try:
data = jsonlib.read(page.decode("GBK", "ignore"))
except Exception,e:
return -1
def postIdenticaPassword(graph, openid, msg):
login = _getUserPass(graph, openid, MB.IdenticaAccount)
api = restkit.Resource("http://identi.ca/api",
filters=[restkit.BasicAuth(login['user'],
login['password'])])
resp = api.post("statuses/update.json", status=msg)
return jsonlib.read(resp.body_string())
def _parse_response(self, req):
"""Parses a response."""
if req.status_code not in [200, 201]:
return None
return jsonlib.read(req.content)
def ParseInterPage(page):
'''
'''
data = jsonlib.read(page.decode("GBK", "ignore"))
allinfo = []
for node in data["FlightList"]:
dept_time = datetime.datetime.strptime(node["DepartTime"],
'%Y-%m-%d %H:%M:%S')
dept_time = str(dept_time).replace(
' ',
'T',
)
dest_time = datetime.datetime.strptime(node["ArrivalTime"],
'%Y-%m-%d %H:%M:%S')
dest_time = str(dest_time).replace(
' ',
'T',
)
# 航班信息
flight = Flight()
flight.flight_no = ''
flight.plane_no = ''
flight.airline = ''
dept_id_list = []
for flightNode in node["FlightDetail"]:
flight.flight_no = flight.flight_no + flightNode["FlightNo"] + '_'
flight.airline = flight.airline + flightNode["AirlineName"] + '_'
flight.plane_no = flight.plane_no + flightNode["CraftType"] + '_'
dept_id_list.append(flightNode["DPort"])
flight.dest_id = flightNode["APort"]
flight.stop = len(dept_id_list)
flight.dept_id = dept_id_list[0]
flight.flight_no = flight.flight_no[:-1]
flight.airline = flight.airline[:-1]
flight.plane_no = flight.plane_no[:-1]
flight.dept_time = dept_time
flight.dest_time = dest_time
flight.dept_day = flight.dept_time.split('T')[0]
flight.price = int(node["Price"])
flight.surcharge = int(
GetPriceByClass(node["OilFeeImage"], TaxPriceClasses))
flight.tax = int((GetPriceByClass(node["TaxImage"], TaxPriceClasses)))
flight.dur = int(node["FlightTime"]) * 60 #飞行时长，s
flight.currency = "CNY"
flight.source = "ctrip::ctrip"
flight.seat_type = node["ClassName"]
allinfo.append((flight.flight_no,flight.plane_no,flight.airline,flight.dept_id,flight.dest_id,\
flight.dept_day,flight.dept_time,flight.dest_time,flight.dur,flight.price,flight.tax,\
flight.surcharge,flight.currency,flight.seat_type,flight.source,flight.return_rule,flight.stop))
return allinfo
def got_data(self, data, cli): print "got data from flash:", data data = str(data) db = self.check_for_debug(cli, data) if db != True: obj = jsonlib.read(data) self.check_ready(obj, cli) self.got_packet(obj)
def onMessage( self, payload, isBinary ):
global pubModeUpdate
global actuationEna
global linearVelocity
global angularVelocity
global logMessages
if isBinary:
print( "Binary message received: {0} bytes".format( len( payload ) ) )
else:
# print( "Text message received: {0}".format( payload.decode( 'utf8' ) ) )
messageInRaw = payload.decode('utf8')
messageIn = jsonlib.read( messageInRaw )
if messageIn["messageType"] == STATUS_REQUEST:
massageOutRaw = {
"messageType":"status_response",
"data":{
"log":[logMessages[:-1]]
}
}
massageOut = json.dumps(massageOutRaw)
self.sendMessage( massageOut, isBinary )
logMessages = ""
elif messageIn["messageType"] == REMOTE_UPDATE:
leftButton = messageIn["data"]["left"]
rightButton = messageIn["data"]["right"]
enaSignal = messageIn["data"]["ena"]
linearVelocity = messageIn["data"]["linV"]
angularVelocity = messageIn["data"]["angV"]
self.updateActuation( enaSignal )
if leftButton == u"u":
drive( linearVelocity, 0.0 )
elif leftButton == u"r":
drive( 0.0, -angularVelocity )
elif leftButton == u"d":
drive( -linearVelocity, 0.0 )
elif leftButton == u"l":
drive( 0.0, angularVelocity )
elif rightButton == u"x":
tip( True )
elif rightButton == u"y":
tip( False )
elif rightButton == u"a":
setManualMode( False )
publishCommand( pubModeUpdate, u"start" )
elif rightButton == u"b":
publishCommand( pubModeUpdate, u"stop" )
def verifyPurchase(self, userId, itemId, transactions):
returnObj = None
#json correctness hack - remove surrounding quotes on each dict in the array
transactions = urllib2.unquote(transactions)
p = re.compile('"{')
transactions = p.sub( '{', transactions)
p = re.compile('}"')
transactions = p.sub( '}', transactions)
if transactions is not None:
arrTransactions = json.read(transactions)
transactionId = None
transactions = json.read(transactions)
if transactions is not None:
for item in transactions:
if item['itemId'] == itemId:
transactionId = item['transactionId']
if transactionId is not None:
postDetails = { 'USER': common.UID,
'PWD': common.PASSWORD,
'SIGNATURE': common.SIG,
'METHOD': 'GetTransactionDetails',
'VERSION': common.VER,
'TRANSACTIONID': transactionId }
response = dict(cgi.parse_qsl(common.curl(common.URLBASE, postDetails)))
custom = string.split(response["CUSTOM"], ',')
if (identity.getUserId() == custom[0]) and (itemId == custom[1]):
returnObj = { 'success': 'true',
'error': '',
'transactionId': response["TRANSACTIONID"],
'orderTime': response["ORDERTIME"],
'paymentStatus': response["PAYMENTSTATUS"],
'itemId': itemId,
'userId': userId }
else:
returnObj = {'success': 'false',
'error': 'Item not found in transaction history'}
print json.write(returnObj)
def GetCurrency(data):
start = data.find(tagStr)
if start == -1:
return None
end = data.find("\r\n", start)
if end == -1:
return None
data = jsonlib.read(data[start + len(tagStr): end-1])
if "currency" in data:
return data["currency"]
return None
def load(self):
import jsonlib
inp = openCompressed(self.fileName,"rb")
buf = inp.read()
inp.close()
#nodes = json.loads(buf,object_hook=lambda o: TaxaNode(**o) if "idpar" in o else o)
#return dict( ((node.id,node) for node in nodes) )
data = jsonlib.read(buf,use_float=True)
data["nodes"] = dict( ((node["id"],TaxaNode(**node)) for node in data["nodes"]) )
data["merged"] = dict( (item for item in data["merged"]) )
return data
def GetCurrency(data):
start = data.find(tagStr)
if start == -1:
return None
end = data.find("\r\n", start)
if end == -1:
return None
data = jsonlib.read(data[start + len(tagStr): end-1])
if "currency" in data:
return data["currency"]
return None
def __init__(self, dropbox_uploader_script, dropbox_uploader_config_file, dropbox_folder, transmission_remote_binary,
transmission_config_file, transmission_auth, tmp_folder):
self.dropbox_uploader_script = dropbox_uploader_script
self.dropbox_uploader_config_file = dropbox_uploader_config_file
self.dropbox_folder = dropbox_folder
self.transmission_remote_binary = transmission_remote_binary
self.transmission_config_file = transmission_config_file
self.transmission_auth = transmission_auth
self.tmp_folder = tmp_folder
ok, std_out = self._sudo('cat %s' % self.transmission_config_file)
self.transmission_config = jsonlib.read(std_out)
def main():
with Store() as store:
while True:
failed = store.redis.lpop('q:failed')
if not failed:
break
failed = jsonlib.read(failed)
print 'retrying', `failed`
store.push_queue(failed['queue'],
failed['values'])
def load(self):
import jsonlib
inp = openCompressed(self.fileName,"rb")
#nodes = json.loads(buf,object_hook=lambda o: TaxaNode(**o) if "idpar" in o else o)
#return dict( ((node.id,node) for node in nodes) )
nodes = {}
for line in inp:
node = jsonlib.read(line.strip(),use_float=True)
nodes[node["id"]] = TaxaNode(**node)
inp.close()
return nodes
def test_strings_contain(self, desired_key = 'country.us', desired_value='United States'):
response = self.app.get(url_for(controller='language', action='strings',
domain='cc_org', id='en'))
value = jsonlib.read(response.body)
strings = value['strings']
found_what_we_like = False
for stringthing in strings:
if stringthing['id'] == desired_key:
assert stringthing['value'] == desired_value
found_what_we_like = True
break
assert found_what_we_like
def parse(s):
'''
从json字符串中解析初task
'''
if s == None or len(s.strip()) == 0:
return None
data = None
try:
data = jsonlib.read(s)
except jsonlib.ReadError, e:
print "Parse json error. task_str = " + s
return None
def parse(s):
'''
从json字符串中解析初task
'''
if s == None or len(s.strip()) == 0:
return None
data = None
try:
data = jsonlib.read(s)
except jsonlib.ReadError, e:
print "Parse json error"
return None
def ParseInterPage(page):
'''
'''
data = jsonlib.read(page.decode("GBK", "ignore"))
allinfo = []
for node in data["FlightList"]:
dept_time = datetime.datetime.strptime(node["DepartTime"], '%Y-%m-%d %H:%M:%S')
dept_time = str(dept_time).replace(' ','T',)
dest_time = datetime.datetime.strptime(node["ArrivalTime"], '%Y-%m-%d %H:%M:%S')
dest_time = str(dest_time).replace(' ','T',)
# 航班信息
flight = Flight()
flight.flight_no = ''
flight.plane_no = ''
flight.airline = ''
dept_id_list = []
for flightNode in node["FlightDetail"]:
flight.flight_no = flight.flight_no + flightNode["FlightNo"] + '_'
flight.airline = flight.airline + flightNode["AirlineName"] + '_'
flight.plane_no = flight.plane_no + flightNode["CraftType"] + '_'
dept_id_list.append(flightNode["DPort"])
flight.dest_id = flightNode["APort"]
flight.stop = len(dept_id_list)
flight.dept_id = dept_id_list[0]
flight.flight_no = flight.flight_no[:-1]
flight.airline = flight.airline[:-1]
flight.plane_no = flight.plane_no[:-1]
flight.dept_time = dept_time
flight.dest_time = dest_time
flight.dept_day = flight.dept_time.split('T')[0]
flight.price = int(node["Price"])
flight.surcharge = int(GetPriceByClass(node["OilFeeImage"], TaxPriceClasses))
flight.tax = int((GetPriceByClass(node["TaxImage"], TaxPriceClasses)))
flight.dur = int(node["FlightTime"]) * 60 #飞行时长，s
flight.currency = "CNY"
flight.source = "ctrip::ctrip"
flight.seat_type = node["ClassName"]
allinfo.append((flight.flight_no,flight.plane_no,flight.airline,flight.dept_id,flight.dest_id,\
flight.dept_day,flight.dept_time,flight.dest_time,flight.dur,flight.price,flight.tax,\
flight.surcharge,flight.currency,flight.seat_type,flight.source,flight.return_rule,flight.stop))
return allinfo
def do_POST_inner(self):
path = self.path[1:]
content = self.rfile.readline()
message = jsonlib.read(content, use_float=True)
ret = ''
if path == 'dm':
ret = self._dm_process(message)
elif path == 'distance':
ret = self._distance_process(message)
else:
pass
return ret
def edit_string(self, domain, id):
"""Edit an individual string."""
language = herder.model.DomainLanguage.by_domain_id(domain, id)
data = jsonlib.read(request.params['data'])
# XXX trap an exception here that would be raised if edit conflict
if 'translate' in self._get_roles(domain, id):
# store the translation
language.update(data['id'], data['new_value'], data['old_value'])
else:
# store the translation as a suggestion
language.suggest(request.environ.get("REMOTE_USER", False),
data['id'], data['new_value'])
def do_POST_inner(self):
path = self.path[1:]
content = self.rfile.readline()
content = jsonlib.read(content,use_float = True)
session = content['session']
message = content['message']
print session, message
ret = ''
if path == 'dm':
ret = self._dm_process(session, message)
else:
pass
return ret
def test_nan_values_are_ignored(self):
self.maxDiff = None
worksheet = Worksheet()
worksheet.A1.value = float('nan')
worksheet.A2.value = float('inf')
worksheet.A3.value = float('-inf')
worksheet_json = worksheet_to_json(worksheet)
roundtripped = jsonlib.read(worksheet_json)
self.assertEquals(roundtripped["1,1"]['formatted_value'], 'nan')
self.assertEquals(roundtripped["1,2"]['formatted_value'], 'inf')
self.assertEquals(roundtripped["1,3"]['formatted_value'], '-inf')
self.assertFalse('value' in roundtripped["1,1"])
self.assertFalse('value' in roundtripped["1,2"])
self.assertFalse('value' in roundtripped["1,3"])
def search(self):
self.url = urllib2.urlopen('http://api.4chan.org/mu/catalog.json')\
.read()
self.data = jsonlib.read(self.url)
pattern = '(.*)KPOP(.*)'
compiled = re.compile(pattern)
for threads in self.data:
for thread in threads['threads']:
if 'sub' in thread:
if 'KPOP General' in thread['sub'] or 'KPOP general' in thread['sub']:
print thread['sub']
if 'com' in thread:
print thread['com']
print 'https://boards.4chan.org/mu/res/%d' % thread['no']
def test_nan_values_are_ignored(self):
self.maxDiff = None
worksheet = Worksheet()
worksheet.A1.value = float('nan')
worksheet.A2.value = float('inf')
worksheet.A3.value = float('-inf')
worksheet_json = worksheet_to_json(worksheet)
roundtripped = jsonlib.read(worksheet_json)
self.assertEquals(roundtripped["1,1"]['formatted_value'], 'nan')
self.assertEquals(roundtripped["1,2"]['formatted_value'], 'inf')
self.assertEquals(roundtripped["1,3"]['formatted_value'], '-inf')
self.assertFalse('value' in roundtripped["1,1"])
self.assertFalse('value' in roundtripped["1,2"])
self.assertFalse('value' in roundtripped["1,3"])
def get_secret(sec_name):
secret_name = sec_name
region_name = "eu-west-2"
logger.info("Fetching secret {}".format(sec_name))
# Create a Secrets Manager client
session = boto3.session.Session()
client = session.client(service_name='secretsmanager',
region_name=region_name)
get_secret_value_response = client.get_secret_value(SecretId=secret_name)
secret_string = get_secret_value_response["SecretString"]
return jsonlib.read(secret_string)
def GET(self):
q = web.input()["q"]
elasticQuery = {
"filtered" : {
"query" :
# highlight seems to not work with fuzzy
{"fuzzy" : {"_all" : {"value" : q, "prefix_length" : 2}}},
"filter" : {
"or" : [
{"prefix" : {"value" : q, "boost" : 2.0}},
]
}
}
}
response = elastic.request(method="get", path="mpd/song/_search",
payload=jsonlib.write(elasticQuery))
result = jsonlib.read(response.body_string())
web.header("Content-Type", "application/json")
return jsonlib.write(result)
def start(self):
if len(self.__task_str.strip()) == 0:
return self.__taskstatus
#tasks为json格式
tasks = jsonlib.read(self.__task_str)
for k, v in tasks.items():
logger.error("RequestThread::start k = %s v = %s" % (k, v))
self.__taskstatus[k] = -1
t = threading.Thread(target=self.single_task, args=(k, v))
self.__threads.append(t)
for thread in self.__threads:
thread.start()
for thread in self.__threads:
thread.join(MULTI_REQUEST_TIMEOUT)
def Parse(trip_type, orig, dest, dept_date, ret_ate):
page = GetData(trip_type, orig, dest, dept_date, ret_ate)
data = ParsePage(page)
if data == None:
return False
currency = GetCurrency(page)
tasks = []
data = jsonlib.read(data)
for k, v in data.items():
for one_day_flights in v:
for one_day_flight in one_day_flights[1]:
ticket = Ticket()
ticket.dept_day = one_day_flights[0]
strs = one_day_flight[1].split("~")
if len(strs) != 9:
continue
# flight info
ticket.flight = Flight()
ticket.flight.flight_no = strs[0].strip() + strs[1].strip()
ticket.flight.orig_id = strs[4].strip()
ticket.flight.dest_id = strs[6].strip()
ticket.flight.airlines = "ryanair"
ticket.source = "ryanair"
ticket.dept_time = datetime.datetime.strptime(strs[5], '%m/%d/%Y %H:%M')
ticket.reac_time = datetime.datetime.strptime(strs[7], '%m/%d/%Y %H:%M')
ticket.remain_tickets = one_day_flight[5]
ticket.price = GetPrice(one_day_flight[4])
ticket.currency = currency
logger.info(ticket.flight.orig_id + " " + ticket.flight.dest_id + " " + ticket.dept_day + " " + str(ticket.price))
task = FlightPriceTask()
task.task_id = FlightPriceTask.FormatTaskID(ticket.flight.orig_id, ticket.flight.dest_id, ticket.dept_day)
tasks.append((task, -1))
# 更新机票信息
UpdateTickets([ticket])
return tasks
def import_json_dump(store, f):
for i, line in enumerate(f):
data = jsonlib.read(line)
stamp = mktime(strptime(data['updated'],
time_format))
user = clean_user_re.sub('', data['author'])
user_key = user
tags = [t['term']
for t in data.get('tags', [])]
values = {
'stamp': stamp,
'user': user_key,
'url': data['link'],
'title': data['title'],
'description': '',
'tags': tags,
}
store.do_save_link(values)
if i % 100 == 0:
print i
def worksheet_from_json(json_string):
#use jsonlib for read ops because of better performance
#keep simplejson for write ops as it's more robust
worksheet_dict = jsonlib.read(json_string)
worksheet = Worksheet()
for (key, value) in worksheet_dict.iteritems():
if key == "_console_text":
worksheet._console_text = value
elif key == "_usercode_error":
worksheet._usercode_error = value
else:
col_str, row_str = key.split(",")
cell = Cell()
cell._formula = value["formula"]
cell._python_formula = value.get("python_formula")
cell.dependencies = map(tuple, value.get("dependencies", []))
cell.error = value.get("error")
cell._value = value.get("value", undefined)
cell.formatted_value = value["formatted_value"]
worksheet[int(col_str), int(row_str)] = cell
return worksheet
def edit_string(self, domain, id):
"""Edit an individual string."""
language = herder.model.language.Language.by_domain_id(domain, id)
data = jsonlib.read(request.params['data'])
try:
if 'translator' in self._get_roles(request.environ, domain, id):
# store the translation
language.update(data['id'], data['old_value'], data['new_value'])
return {'result': 'success',
'message': 'Translation updated'}
else:
# store the translation as a suggestion
language.suggest(session['user'].user_id,
data['id'], data['new_value'])
return {'result': 'success',
'message': 'Suggestion filed'}
except herder.model.TransactionAbort, e:
return {'result': 'error',
'message': e.message}
dept_date = contents[2][:4] + '-' + contents[2][4:6] + '-' + contents[2][6:]
ret_date = str(datetime.datetime.strptime(dept_date[2:], '%y-%m-%d') + datetime.timedelta(10)).split(' ')[0].strip()#do not use this value
except Exception,e:
logger.error('ryanairFlight: wrong content format with %s'%content)
return None
trip_type = 'Oneway'
page = GetData(trip_type, dept_id, dest_id, dept_date, ret_date)
data = ParsePage(page)
if data == None:
return []
currency = GetCurrency(page)
allinfo = []
data = jsonlib.read(data)
for k, v in data.items():
for one_day_flights in v:
for one_day_flight in one_day_flights[1]:
flight = Flight()
flight.dept_day = one_day_flights[0]
strs = one_day_flight[1].split("~")
if len(strs) != 9:
continue
flight.flight_no = strs[0].strip() + strs[1].strip()
flight.dept_id = strs[4].strip()
flight.dest_id = strs[6].strip()
flight.airline = "ryanair"
flight.source = "ryanair::ryanair"
def r(self, string, expected, **kwargs):
value = read(string, **kwargs)
self.assertEqual(value, expected)
self.assertEqual(type(value), type(expected))
def onMessage( self, payload, isBinary ):
global pubModeUpdate
global actuationEna
global linearVelocity, angularVelocity
global logMessages
global STATUS_REQUEST, REMOTE_UPDATE, TEXT_FIELD_MSG
global CURRENT_NODE_MESSAGE, PERFORM_ACTION_MESSAGE
global IDLE, AUTO, MANUAL, robotState
#if isBinary:
#print( "Binary message received: {0} bytes".format( len( payload ) ) )
#else:
# print( "Text message received: {0}".format( payload.decode( 'utf8' ) ) )
messageInRaw = payload.decode('utf8')
messageIn = jsonlib.read( messageInRaw )
#print messageInRaw
if messageIn["messageType"] == STATUS_REQUEST:
# TODO Handle signal from the emergency switch
handleEmergencySituation( messageIn["data"]["resume"] )
messageOutRaw = {
"messageType":"status_response",
"data":{
"log":[logMessages[:-1]]
}
}
#print messageOutRaw
messageOutRaw = json.dumps(messageOutRaw)
self.sendMessage( messageOutRaw, isBinary )
logMessages = ""
elif messageIn["messageType"] == REMOTE_UPDATE:
leftButton = messageIn["data"]["left"]
rightButton = messageIn["data"]["right"]
enaSignal = messageIn["data"]["ena"]
linearVelocity = messageIn["data"]["linV"]
angularVelocity = messageIn["data"]["angV"]
self.updateActuation( enaSignal )
if leftButton == u"u":
drive( linearVelocity, 0.0 )
elif leftButton == u"r":
drive( 0.0, -angularVelocity )
elif leftButton == u"d":
drive( -linearVelocity, 0.0 )
elif leftButton == u"l":
drive( 0.0, angularVelocity )
elif rightButton == u"x":
stop_aThreads()
setManualMode( False )
tipper( True )
elif rightButton == u"y":
stop_aThreads()
setManualMode( False )
tipper( False )
elif rightButton == u"a":
stop_aThreads()
if isManual == True:
setManualMode( False )
if robotState != AUTO:
sendMode( u"auto" )
robotState = AUTO
elif rightButton == u"b":
stop_aThreads()
if isManual == True:
setManualMode( False )
if robotState != IDLE:
sendMode( u"idle" )
robotState = IDLE
# TODO Test it!
elif messageIn["messageType"] == TEXT_FIELD_MSG:
target = messageIn["data"]["target"]
text_msg = messageIn["data"]["msg"]
if text_msg != "" and target == CURRENT_NODE_MESSAGE:
sendCurrentNodeMessage( text_msg )
elif text_msg != "" and target == PERFORM_ACTION_MESSAGE:
sendPerformActionMessage( text_msg )
def get_json_obj(self, key):
import jsonlib
print self.params[key][0]
if key in self.params:
return jsonlib.read(self.params[key][0])
return None
datetime.timedelta(10)).split(
' ')[0].strip() #do not use this value
except Exception, e:
logger.error('ryanairFlight: wrong content format with %s' % content)
return None
trip_type = 'Oneway'
page = GetData(trip_type, dept_id, dest_id, dept_date, ret_date)
data = ParsePage(page)
if data == None:
return []
currency = GetCurrency(page)
allinfo = []
data = jsonlib.read(data)
for k, v in data.items():
for one_day_flights in v:
for one_day_flight in one_day_flights[1]:
flight = Flight()
flight.dept_day = one_day_flights[0]
strs = one_day_flight[1].split("~")
if len(strs) != 9:
continue
flight.flight_no = strs[0].strip() + strs[1].strip()
flight.dept_id = strs[4].strip()
flight.dest_id = strs[6].strip()
flight.airline = "ryanair"
flight.source = "ryanair::ryanair"
def fastloads(str):
return jsonlib.read(str, use_float=True)
def fastloads(str): return jsonlib.read(str, use_float=True) fastdumps = jsonlib.write
def _unjsonify(text):
import jsonlib
return jsonlib.read(text)
def do_POST_inner(self):
## global db, dm, prevSysAction, dmModuleName, uuid
# strip of leading '/'
path = self.path[1:]
print '!!!!!!!!Get path=', path
# read in body content
inputText = self.rfile.readline()
self.log_message("Got message: %s" % (inputText))
# vivify into json
try:
inputJSON = jsonlib.read(inputText, use_float=True)
except:
self.log_error("Could not parse JSON: %s\nError: %s" %
(inputText, sys.exc_info()))
return '{"status": "error","error": "input-JSON-decoding","error_msg": "Could not parse input JSON"}\n'
if (path == 'dm'): #clien yeu cau cai gi? dua vao path cua url
print '!!!Request DM'
pdb.set_trace()
## # dm request
if (
inputJSON['session'] == 'new'
): #phien moi thi tao dm moi, gui ve askaction va luu he thong cu
## sysAction = dm.Init()
## prevSysAction = sysAction
## asrProbs = None
pass
elif (
prevSysAction == None
): #yeu cau lai ma he thong chua tung khoi toa, loi vi dm chua cod
self.log_error("DM not ready; resetting connection")
return '{"status": "error","error": "dm", "error_msg": "DM not ready"}\n'
else: #dua nhan dang dong noi vao de DM tinh toan
pass
## asrResult = ASRResult.FromWatson(inputJSON['message'],prevSysAction.grammar)#grammar is firt/last... or all
## asrProbs = asrResult.GetProbs()
## sysAction = dm.TakeTurn(asrResult)
## prevSysAction = sysAction
## messageJSON = {
## 'sysAction': sysAction.GetJSON(),
## 'asrProbs': asrProbs,
## 'dialogResult': dm.GetDisplayJSON(),
## }
messageJSON = {
'sysAction': None,
'asrProbs': None,
'dialogResult': None,
}
elif (path == 'sampleName'):
# sample a name
nameString = 'Accept'
messageJSON = {
'nameText': nameString,
}
elif (path == 'info'):
# config info
messageJSON = {
'dmModuleName': 'THANH',
'uuid': uuid,
}
else:
self.log_error("Dont recognize path %s" % (path))
return '{"status": "error","error": "path", "error_msg": "Dont recognize path %s"}\n' % (
path)
# add session & status
outputJSON = {}
outputJSON['status'] = 'ok'
outputJSON['message'] = messageJSON
# textify result
try:
output_text = jsonlib.write(outputJSON)
except:
self.log_error("Could not parse JSON into text: %s\nError: %s" %
(outputJSON, sys.exc_info()))
return '{"status": "error","error": "output-JSON-encoding","msg": "Could not parse JSON: %s\nError: %s."}\n' % (
outputJSON, sys.exc_info()[0])
return output_text
Session = sessionmaker(bind=engine) """ URLからJSONデータの取得 """ sensordata = None try: """ 各BBSのURLを取得するためのhtmlを取得 """ json = urllib2.urlopen(url).read() """ JSONを解析しセンサデータを取得 """ sensordata = jsonlib.read (json) except: pass """ センサーデータが正しく入力されているかの確認 """ if sensordata: """ センサデータの要素 """ elements = ["Temperature", "Humidity", "WindSpeed", "WindDir", "Pressure", "Rainfall"] for element in elements: """ 正しくデータが入っているかの確認 """
