def __str__(self):
encoder = JSONEncoder()
return encoder.encode({
'title': self.get_title(),
'link': self.get_link(),
'snippet': self.get_snippet(),
})
def __input_loop(self, protocol, base):
"""
:type protocol server.WebSocketServerProtocol
:param base:
:return:
"""
json_encoder = JSONEncoder()
yield from protocol.send(json_encoder.encode({"message": _GET_NICK}))
nick = yield from protocol.recv()
self._chat_api.enter_chat(nick)
yield from protocol.send(
json_encoder.encode({"message": _ADD, "text": ",".join(self._chat_api.nicks_in_chat)}))
while self._chat_api.connection_opened:
message = yield from protocol.recv()
self._chat_api.say_to_chat(message)
def __output_loop(self, protocol, base):
"""
:type protocol server.WebSocketServerProtocol
:param base:
:return:
"""
room_queue = self._chat_api.subscribe_to_chat()
json_encoder = JSONEncoder()
current_people = self._chat_api.nicks_in_chat
while self._chat_api.connection_opened:
message = yield from room_queue.get()
yield from protocol.send(json_encoder.encode({"message": _TEXT, "text": message}))
if current_people != self._chat_api.nicks_in_chat:
current_people = self._chat_api.nicks_in_chat
yield from protocol.send(
json_encoder.encode({"message": _ADD, "text": ",".join(self._chat_api.nicks_in_chat)}))
def projects_json(request):
result = list()
projects = Project.objects.all()
for project in projects:
vm_project = VM_Project(None, False, project, False)
temp_dict = {}
temp_dict["Project"] = project.PBTitle
temp_dict["Platform"] = vm_project.platform_title()
temp_dict["Creator"] = vm_project.project_lead()
temp_dict["Product"] = vm_project.product_title()
result.append(temp_dict)
json_encoder = JSONEncoder()
return HttpResponse(json_encoder.encode(result))
class JSONSerializer(CustomizableSerializer):
"""
Serializes objects using JSON (JavaScript Object Notation).
See the :mod:`json` module documentation in the standard library for more information on
available options.
Certain options can resolve references to objects:
* ``encoder_options['default']``
* ``decoder_options['object_hook']``
* ``decoder_options['object_pairs_hook']``
:param encoder_options: keyword arguments passed to :class:`~json.JSONEncoder`
:param decoder_options: keyword arguments passed to :class:`~json.JSONDecoder`
:param encoding: the text encoding to use for converting to and from bytes
:param custom_type_codec: wrapper to use to wrap custom types after marshalling
"""
__slots__ = ('encoder_options', 'decoder_options', 'encoding', 'custom_type_codec',
'_encoder', '_decoder', '_marshallers', '_unmarshallers')
def __init__(self, encoder_options: Dict[str, Any] = None,
decoder_options: Dict[str, Any] = None, encoding: str = 'utf-8',
custom_type_codec: Union[JSONTypeCodec, str] = None) -> None:
assert check_argument_types()
super().__init__(resolve_reference(custom_type_codec) or JSONTypeCodec())
self.encoding = encoding
self.encoder_options = encoder_options or {}
self.encoder_options['default'] = resolve_reference(self.encoder_options.get('default'))
self._encoder = JSONEncoder(**self.encoder_options)
self.decoder_options = decoder_options or {}
self.decoder_options['object_hook'] = resolve_reference(
self.decoder_options.get('object_hook'))
self.decoder_options['object_pairs_hook'] = resolve_reference(
self.decoder_options.get('object_pairs_hook'))
self._decoder = JSONDecoder(**self.decoder_options)
def serialize(self, obj) -> bytes:
return self._encoder.encode(obj).encode(self.encoding)
def deserialize(self, payload: bytes):
text_payload = payload.decode(self.encoding)
return self._decoder.decode(text_payload)
@property
def mimetype(self):
return 'application/json'
def get_user(self):
result = dict()
if self.user:
result['code'] = 0
result['username'] = self.user.last_name + self.user.first_name
result['email'] = self.user.email
result['message'] = "successful"
else:
result['code'] = 1
result['username'] = ""
result['email'] = ""
result['message'] = "no user found"
json_encoder = JSONEncoder()
return json_encoder.encode(result)
class JSONSerializer(CustomizableSerializer):
"""
Serializes objects using JSON (JavaScript Object Notation).
See the :mod:`json` module documentation in the standard library for more information on
available options.
Certain options can resolve references to objects:
* ``encoder_options['default']``
* ``decoder_options['object_hook']``
* ``decoder_options['object_pairs_hook']``
:param encoder_options: keyword arguments passed to :class:`~json.JSONEncoder`
:param decoder_options: keyword arguments passed to :class:`~json.JSONDecoder`
:param encoding: the text encoding to use for converting to and from bytes
:param custom_type_codec: wrapper to use to wrap custom types after marshalling
"""
__slots__ = ('encoder_options', 'decoder_options', 'encoding', 'custom_type_codec',
'_encoder', '_decoder', '_marshallers', '_unmarshallers')
def __init__(self, encoder_options: Dict[str, Any] = None,
decoder_options: Dict[str, Any] = None, encoding: str = 'utf-8',
custom_type_codec: Union[JSONTypeCodec, str] = None):
assert check_argument_types()
super().__init__(resolve_reference(custom_type_codec) or JSONTypeCodec())
self.encoding = encoding
self.encoder_options = encoder_options or {}
self.encoder_options['default'] = resolve_reference(self.encoder_options.get('default'))
self._encoder = JSONEncoder(**self.encoder_options)
self.decoder_options = decoder_options or {}
self.decoder_options['object_hook'] = resolve_reference(
self.decoder_options.get('object_hook'))
self.decoder_options['object_pairs_hook'] = resolve_reference(
self.decoder_options.get('object_pairs_hook'))
self._decoder = JSONDecoder(**self.decoder_options)
def serialize(self, obj) -> bytes:
return self._encoder.encode(obj).encode(self.encoding)
def deserialize(self, payload: bytes):
payload = payload.decode(self.encoding)
return self._decoder.decode(payload)
@property
def mimetype(self):
return 'application/json'
def main():
decoder = JSONDecoder()
encoder = JSONEncoder(indent=4)
# Loop over every file in the course_data folder.
# These files are dictionaries of data for single courses.
for course_file in os.listdir('./course_data'):
with open(os.path.join('./course_data', course_file)) as c:
data = decoder.decode(c.read())
if data is None:
continue
print('Parsing', data['course_code'])
with open('./prerequisites/'+data['course_code']+'.json', 'w') as out:
out.write(encoder.encode(
parse_prereq(data['prerequisites'], data['course_code'])
))
def encode_json(obj, inline=False, **kwargs):
"""Encode the given object as json.
Supports objects that follow the `_asdict` protocol. See `parse_json` for more information.
:param obj: A serializable object.
:param bool inline: `True` to inline all resolvable objects as nested JSON objects, `False` to
serialize those objects' addresses instead; `False` by default.
:param **kwargs: Any kwargs accepted by :class:`json.JSONEncoder` besides `encoding` and
`default`.
:returns: A UTF-8 json encoded blob representing the object.
:rtype: string
:raises: :class:`ParseError` if there were any problems encoding the given `obj` in json.
"""
encoder = JSONEncoder(default=functools.partial(_object_encoder, inline=inline), **kwargs)
return encoder.encode(obj)
def changeCredentials(server,
port,
user,
password,
pwd,
file="secret_credentials.json"):
jsonenc = JSONEncoder()
data = {"SERVER": server, "PORT": port, "USER": user, "PASSWORD": password}
json = jsonenc.encode(data)
print("Opening credential file...")
encryption.encrypt(text=json, password=pwd)
print("Saving...")
return
def encode_json(obj, inline=False, **kwargs):
"""Encode the given object as json.
Supports objects that follow the `_asdict` protocol. See `parse_json` for more information.
:param obj: A serializable object.
:param bool inline: `True` to inline all resolvable objects as nested JSON objects, `False` to
serialize those objects' addresses instead; `False` by default.
:param **kwargs: Any kwargs accepted by :class:`json.JSONEncoder` besides `encoding` and
`default`.
:returns: A UTF-8 json encoded blob representing the object.
:rtype: string
:raises: :class:`ParseError` if there were any problems encoding the given `obj` in json.
"""
encoder = JSONEncoder(encoding='UTF-8',
default=functools.partial(_object_encoder, inline=inline),
**kwargs)
return encoder.encode(obj)
def loop_messages(self):
"""
Getting messages (data instances) from the stream.
:return:
"""
messages = self.stub.sendData(self.generate_predictions(),
metadata=self.metadata)
try:
for message in messages:
message = json.loads(json_format.MessageToJson(message))
encoder = JSONEncoder()
mes = encoder.encode(message)
mes = mes.replace(" ", "")
self.messages.put(mes)
except Exception as e:
print(str(e))
pass
def index():
if request.method == 'POST':
site_hash = sha1(request.form['url']).hexdigest()
zipf_profile = r.get(site_hash)
if zipf_profile:
return zipf_profile
else:
try:
corpus = WebCorpus(request.form['url'])
except:
abort(404)
e = JSONEncoder()
zipf_profile = e.encode(corpus.freq_list)
r.set(site_hash, zipf_profile)
r.expire(site_hash, 120)
resp = make_response(zipf_profile)
return resp
elif request.method == 'GET':
return render_template("index.html")
def index():
if request.method == "POST":
site_hash = sha1(request.form["url"]).hexdigest()
zipf_profile = r.get(site_hash)
if zipf_profile:
return zipf_profile
else:
try:
corpus = WebCorpus(request.form["url"])
except:
abort(404)
e = JSONEncoder()
zipf_profile = e.encode(corpus.freq_list)
r.set(site_hash, zipf_profile)
r.expire(site_hash, 120)
resp = make_response(zipf_profile)
return resp
elif request.method == "GET":
return render_template("index.html")
class BaseAsyncTasks(object):
def __init__(self, priority=DEFAULT_JOB_PRIORITY, delay=0, ttr=DEFAULT_JOB_TTR):
self._priority = priority
self._delay = delay
self._ttr = ttr
self._jsonEncoder = JSONEncoder()
'''
@function -> the reference to the annotated asynchronous business logic function
@args -> a json serializable list of parameters
@priority -> you can set a priority for your task, so the consumers can process it accordingly
@delay -> you can set a delay, the default is 0
@ttr -> ttr value. see: http://github.com/kr/beanstalkd/wiki/faq
'''
def run(self, function, args):
logger = logging.getLogger('BaseAsyncTasks.run')
if settings.BEANSTALK_ENABLED :
beanstalkClient = BeanstalkClient()
module = function.__module__.__self__.app
# Add more metadata to the args
args['__uuid__'] = str(uuid4()).replace('-', '')
func = ".".join([module, function.__name__])
body = self._jsonEncoder.encode(args)
pid = beanstalkClient.call(func, body,
priority=self._priority,
delay=self._delay,
ttr=self._ttr)
try :
if not pid >= 0:
logger.critical("Failed to execute task: " + str(function) + " " + str(args))
JobRecord.objects.create(jid=pid, tube=func, body=body, uuid=args['__uuid__'], status=JobStatus.FAILED_TO_REGISTER)
else:
JobRecord.objects.create(jid=pid, tube=func, body=body, uuid=args['__uuid__'], status=JobStatus.READY)
except Exception, ex:
logger.exception('Error while persisting Job object. %s' % str(ex))
else :
language_path = os.path.dirname(sys.argv[1])
language_file = xml.parse(sys.argv[1])
else:
language_path = os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])),
'site', 'data')
language_file = xml.parse(os.path.join(language_path, 'language.xml'))
# get all languages
output = {}
for language in language_file.getElementsByTagName('language'):
code = language.attributes['short'].value
output[code] = {}
for constant in language.getElementsByTagName('constant'):
constant_name = constant.attributes['name'].value
if constant.firstChild:
output[code][constant_name] = constant.firstChild.data
else:
output[code][constant_name] = ''
# save output files
encoder = JSONEncoder(**encoder_options)
for code, language in output.items():
with open(os.path.join(language_path, 'language_{0}.json'.format(code)),
'w') as raw_file:
raw_file.write(encoder.encode(language).encode('utf8'))
def get_task_config(request,task_id):
''' index page'''
task_config=CITaskConfigService.get_ci_task_config_by_taskid(task_id)
json_encoder=JSONEncoder()
return HttpResponse(json_encoder.encode(str(task_config)))
def encode(self, o):
pickle_bytes = pickle.dumps(o)
pickle_str = binascii.b2a_qp(pickle_bytes).decode(encoding="utf8")
o = {JSONCodec._obj: pickle_str, JSONCodec._ver_key: JSONCodec._ver}
return JSONEncoder.encode(self, o)
def encode(self, o):
pickle_bytes = pickle.dumps(o)
pickle_str = binascii.b2a_qp(pickle_bytes).decode(encoding='utf8')
o = {JSONCodec._obj: pickle_str,
JSONCodec._ver_key: JSONCodec._ver}
return JSONEncoder.encode(self, o)
def __shutdown(self):
beanstalkClient = BeanstalkClient()
jsonEncoder = JSONEncoder()
args = {'timestamp' : time.time()}
pid = beanstalkClient.call("shutdown", jsonEncoder.encode(args), priority=0) #high pri job
class Trainer(object):
categories = {}
def __init__(self):
self._jsonDecoder = JSONDecoder()
self._jsonEncoder = JSONEncoder()
self.__featureExtractor = FeatureExtractor()
def setFeatureExtractor(self, featureExtractor):
self.__featuredExtractor = featureExtractor
def __isNumeric(self, feature):
isNumeric = False
try:
float(feature)
isNumeric = True
except ValueError:
pass
return isNumeric
"""
Given a list of yes category names, retrieves the yes/no category hash that the trainer needs
"""
def __getCategoriesFromNames(self, yesTagNames, noTagNames):
finalCategories = []
# create the categories if they don't already exist
for tagName in yesTagNames:
if tagName:
categoryYes, _ = ClassifierCategory.objects.get_or_create(categoryName=tagName, yes=True)
categoryNo, _ = ClassifierCategory.objects.get_or_create(categoryName=tagName, yes=False)
finalCategories.append(categoryYes)
for tagName in noTagNames:
if tagName:
categoryYes, _ = ClassifierCategory.objects.get_or_create(categoryName=tagName, yes=True)
categoryNo, _ = ClassifierCategory.objects.get_or_create(categoryName=tagName, yes=False)
finalCategories.append(categoryNo)
return finalCategories
"""
Trains a corpus of data.
"""
@transaction.commit_manually
def train(self, corpus="", yesTagNames=None, noTagNames=None):
logger = logging.getLogger("Trainer.train")
success = False
categories = []
try:
document = Document.getDocumentByCorpus(corpus)
if not document:
features = self.__featuredExtractor.getFeatures(corpus)
categories = self.__getCategoriesFromNames(yesTagNames, noTagNames)
document = Document(corpus=corpus)
document.save()
documentCounts = {}
for category in categories:
self.__incrementCategoryCount(documentCounts, category)
DocumentCategoryCounts(document=document, countData=self._jsonEncoder.encode(documentCounts)).save()
for feature in features:
featureCount, _ = FeatureCounts.objects.get_or_create(featureName=feature)
counts = self._jsonDecoder.decode(featureCount.countData) if featureCount.countData else {}
for category in categories:
self.__incrementCategoryCount(counts, category)
featureCount.countData = self._jsonEncoder.encode(counts)
featureCount.save()
# We keep an index of category document counts for faster classification later on
catDocCountIndex = CategoryDocumentCountIndex.getCountIndex()
index = self._jsonDecoder.decode(catDocCountIndex.countData) if catDocCountIndex.countData else {}
for category in categories:
self.__incrementCategoryCount(index, category)
catDocCountIndex.countData = self._jsonEncoder.encode(index)
catDocCountIndex.save()
success = True
transaction.commit()
else:
logger.info("Document already exists: " + str(document.id) + " - " + document.corpusHash)
success = True
except Exception, ex:
logger.info("Bad data:%s" % corpus)
logger.exception("Failed to save the trained data: " + str(ex))
transaction.rollback()
return success
import json
from json.encoder import JSONEncoder
version_json = """
{
"date": "2020-10-29T07:57:53+0000",
"dirty": false,
"error": null,
"full-revisionid": "539478470b53bd3e96473b411e1322bb158fd444",
"version": "0.12.1"
}
"""
result = json.loads(version_json)
print(type(result))
for index, val in enumerate(result):
print("%s---%s" % (index, val))
print(result["version"])
print(JSONEncoder.encode(result))
def get_json(self):
json_encoder = JSONEncoder()
result = json_encoder.encode(str(self.__dict__))
return result
# load original language file
if len(sys.argv) > 1:
language_path = os.path.dirname(sys.argv[1])
language_file = xml.parse(sys.argv[1])
else:
language_path = os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])), 'site', 'data')
language_file = xml.parse(os.path.join(language_path, 'language.xml'))
# get all languages
output = {}
for language in language_file.getElementsByTagName('language'):
code = language.attributes['short'].value
output[code] = {}
for constant in language.getElementsByTagName('constant'):
constant_name = constant.attributes['name'].value
if constant.firstChild:
output[code][constant_name] = constant.firstChild.data
else:
output[code][constant_name] = ''
# save output files
encoder = JSONEncoder(**encoder_options)
for code, language in output.items():
with open(os.path.join(language_path, 'language_{0}.json'.format(code)), 'w') as raw_file:
raw_file.write(encoder.encode(language).encode('utf8'))
class JSONSerializer(serialize_abcs.CustomizableSerializer):
"""
Serializes objects using JSON (JavaScript Object Notation).
See the :mod:`json` module documentation in the standard library for more information on
available options.
Certain options can resolve references to objects:
* ``encoder_options['default']``
* ``decoder_options['object_hook']``
* ``decoder_options['object_pairs_hook']``
:param encoder_options: keyword arguments passed to :class:`~json.JSONEncoder`
:param decoder_options: keyword arguments passed to :class:`~json.JSONDecoder`
:param encoding: the text encoding to use for converting to and from bytes
:param custom_type_key: magic key that identifies custom types in a JSON object
"""
__slots__ = ('encoder_options', 'decoder_options', 'encoding', 'custom_type_key', '_encoder',
'_decoder', '_marshallers', '_unmarshallers')
def __init__(self, encoder_options: Dict[str, Any] = None,
decoder_options: Dict[str, Any] = None, encoding: str = 'utf-8',
custom_type_key: str = '__type__'):
self.encoding = encoding
self.custom_type_key = custom_type_key
self._marshallers = OrderedDict() # class -> (typename, marshaller function)
self._unmarshallers = OrderedDict() # typename -> (class, unmarshaller function)
self.encoder_options = encoder_options or {}
self.encoder_options['default'] = resolve_reference(self.encoder_options.get('default'))
self._encoder = JSONEncoder(**self.encoder_options)
self.decoder_options = decoder_options or {}
self.decoder_options['object_hook'] = resolve_reference(
self.decoder_options.get('object_hook'))
self.decoder_options['object_pairs_hook'] = resolve_reference(
self.decoder_options.get('object_pairs_hook'))
self._decoder = JSONDecoder(**self.decoder_options)
def serialize(self, obj) -> bytes:
return self._encoder.encode(obj).encode(self.encoding)
def deserialize(self, payload: bytes):
payload = payload.decode(self.encoding)
return self._decoder.decode(payload)
def register_custom_type(
self, cls: type, marshaller: Optional[Callable[[Any], Any]] = default_marshaller,
unmarshaller: Optional[Callable[[Any, Any], Any]] = default_unmarshaller, *,
typename: str = None) -> None:
typename = typename or qualified_name(cls)
if marshaller:
self._marshallers[cls] = typename, marshaller
self.encoder_options['default'] = self._default_encoder
self._encoder = JSONEncoder(**self.encoder_options)
if unmarshaller:
self._unmarshallers[typename] = cls, unmarshaller
self.decoder_options['object_hook'] = self._custom_object_hook
self._decoder = JSONDecoder(**self.decoder_options)
def _default_encoder(self, obj):
obj_type = obj.__class__
try:
typename, marshaller = self._marshallers[obj_type]
except KeyError:
raise LookupError('no marshaller found for type "{}"'
.format(obj_type.__class__.__name__)) from None
state = marshaller(obj)
return {self.custom_type_key: typename, 'state': state}
def _custom_object_hook(self, obj: Dict[str, Any]):
if len(obj) == 2 and self.custom_type_key in obj:
typename = obj[self.custom_type_key]
try:
cls, unmarshaller = self._unmarshallers[typename]
except KeyError:
raise LookupError('no unmarshaller found for type "{}"'.format(typename)) from None
instance = cls.__new__(cls)
unmarshaller(instance, obj['state'])
return instance
else:
return obj
@property
def mimetype(self):
return 'application/json'
class Connector:
"""description of class"""
def __init__(self, api_key):
self.key = api_key
self.host = "api.projectoxford.ai"
self.base_url = "/face/v1.0/{}"
self.encoder = JSONEncoder()
self.decoder = JSONDecoder()
def encode_json(self, dictionary):
"""
encodes dictionaries to json to send to API
"""
return self.encoder.encode(dictionary)
def decode_json(self, json):
"""
decodes json to a dictionary
"""
return self.decoder.decode(json)
def send_request(self, method, url, qs_args=None, headers=None, body=None):
"""
Sends a request to the API.
"""
# Because having a dictionary as default value is dangerous
if qs_args is None:
qs_args = {}
if headers is None:
headers = {}
# Check what content type header to include in the HTTP message
if hasattr(body, "read") or isinstance(body, bytes):
headers["Content-Type"] = "application/octet-stream"
else:
body = self.encode_json(body)
headers["Content-Type"] = "application/json"
connection = HTTPSConnection(self.host)
# Format the url
url = self.base_url.format(url)
if len(qs_args) > 0:
url += "?{}".format(urlencode(qs_args))
# Add api-key to the headers
headers["Ocp-Apim-Subscription-Key"] = self.key
# Send the request
connection.request(method, url, headers=headers, body=body)
# Read the response and try to decode JSON
response = connection.getresponse()
data_bytes = response.read()
data = data_bytes.decode()
# TODO: Except data that is not JSON
if len(data) > 0:
data = self.decode_json(data)
return data, response
