def SetUpStatsForTesting(self, pkgstat_module): for md5_sum, data in pkgstat_module.pkgstats[0]['elfdump_info'].iteritems(): json = cjson.encode(data) content_hash = hashlib.md5() content_hash.update(json) models.ElfdumpInfoBlob( md5_sum=md5_sum, json=json, content_md5_sum=content_hash.hexdigest(), mime_type='application/json') data = copy.deepcopy(pkgstat_module.pkgstats[0]) data['elf_callback'] = None json = cjson.encode(data) content_hash = hashlib.md5() content_hash.update(json) md5_sum = pkgstat_module.pkgstats[0]['basic_stats']['md5_sum'] models.Srv4FileStatsBlob( md5_sum=md5_sum, json=json, content_md5_sum=content_hash.hexdigest(), mime_type='application/json') sqo_pkgstats, pkgstats = relational_util.StatsStructToDatabaseLevelOne( md5_sum, False) return sqo_pkgstats, pkgstats
def save(filename, data): """Accepts filenamestring and a list of objects, probably dictionaries. Writes these to a file with each object pickled using json on each line. """ if isinstance(data, dict): # save dict values recursively in separate files filename,ext = (filename.rsplit('.',1) if '.' in filename else (filename,'')) for k,v in data.iteritems(): save('{0}-{1}.{2}'.format(filename, k, ext), v) else: if isinstance(data, numpy.ndarray): # save numpy using its own thing for speed; .gz auto compresses numpy.savetxt(filename + '.npy.gz', data) elif isinstance(data, list) and isinstance(data[0], numpy.ndarray): # is a big list of arrays, so save in one big file numpy.savez(filename + '.npy.list', *data) else: with open(filename, 'w') as f: if hasattr(data, '__iter__'): for i,d in enumerate(data): if i != 0: f.write('\n') f.write(cjson.encode(d)) else: f.write(cjson.encode(data))
def got_command(self, dispatcher, command, headers): if command == "summary": summary = {'ip' : RelayConfig.relay_ip, 'version' : __version__, 'status' : self.status, 'uptime' : int(time() - self.start_time), 'session_count' : len(self.session_manager.sessions), 'stream_count' : self.session_manager.stream_count, 'bps_relayed' : self.session_manager.bps_relayed} return cjson.encode(summary) elif command == "sessions": return cjson.encode(self.session_manager.statistics) elif command == "update": if self.graceful_shutdown or self.shutting_down: if not self.session_manager.has_session(**headers): log.debug("cannot add new session: media-relay is shutting down") return 'halting' try: local_media = self.session_manager.update_session(dispatcher, **headers) except RelayPortsExhaustedError: log.error("Could not reserve relay ports for session, all allocated ports are being used") return "error" if local_media: return " ".join([RelayConfig.advertised_ip or local_media[0][0]] + [str(media[1]) for media in local_media]) else: # remove session = self.session_manager.remove_session(**headers) if session is None: return "error" else: return cjson.encode(session.statistics)
def _marshaled_dispatch(self, data, dispatch_method = None): id = None try: req = cjson.decode(data) method = req['method'] params = req['params'] id = req['id'] if dispatch_method is not None: result = dispatch_method(method, params) else: result = self._dispatch(method, params) response = dict(id=id, result=result, error=None) except: extpe, exv, extrc = sys.exc_info() err = dict(type=str(extpe), message=str(exv), traceback=''.join(traceback.format_tb(extrc))) response = dict(id=id, result=None, error=err) try: return cjson.encode(response) except: extpe, exv, extrc = sys.exc_info() err = dict(type=str(extpe), message=str(exv), traceback=''.join(traceback.format_tb(extrc))) response = dict(id=id, result=None, error=err) return cjson.encode(response)
def save(self): self.storage.start_txn() try: for k in self._local.removed_edges: try: del self.storage.left[k] del self.storage.left[invert_edge_key(k)] except KeyError: pass for node_id in self._local.removed_edges: node_key = pack_node_key(node_id) if node_key in self.storage.node: for k in self.storage.left.match_prefix(node_key): del self.storage.left[k] for k in self.storage.right.match_prefix(node_key): del self.storage.right[k] del self.storage.node[node_key] for n in self._local.dirty_nodes: self.storage.node[pack_node_key(n.id)] = cjson.encode(n._attrs) for e in self._local.dirty_edges: k = pack_edge_key(e.left_id, e.rel, e.right_id) self.storage.left[k] = cjson.encode(e._attrs) self.storage.right[invert_edge_key(k)] = '' self._reset_change_buffers() num_new_nodes = self.next_node_id - self.last_node_id if num_new_nodes > 0: self.storage.node[struct.pack('i', 0)] = struct.pack('Q', self.next_node_id) self.last_node_id = self.next_node_id self.storage.commit_txn() except: self.storage.abort_txn() raise
def _AggregateResponse(names, blocks, accept): """Retrieve Aggregate results, and add them to the response. Args: names - List of Taba Names to retrieve Aggregates for. blocks - List of Taba Name Block IDs to retrieve Aggregates for. accept - MIME type in which to format the output. """ names = names_param.value if names_param.isGlob: all_names = [] names = fnmatch.filter(global_taba_server.GetNames(), names_param.value) if names != None and len(names) == 0: aggregates = [] else: # Retrieve the requested Aggregates. aggregates = global_taba_server.GetAggregates(names, blocks) # Format the response in accordance to the Accept header. if accept == 'text/json': juno.append(cjson.encode(dict([a for a in aggregates]))) else: for agg_name, aggregate in aggregates: juno.append(agg_name).append(": ").append(cjson.encode(aggregate)) juno.append("\n")
def rename_ingestion(request, ing_id): """ Rename ingestion (regarding permissions) """ # FIXME: make a post query, not passing ing_id as parameter! name = request.POST.get('value', None) if not name: # Fails silently return HttpResponse(str(ing.name), mimetype = 'text/plain') try: ing = Ingestion.objects.get(id = ing_id) except: return HttpResponse(json.encode({'error': "This ingestion has not been found"}), mimetype = 'text/plain') perms = get_entity_permissions(request, target = 'ingestion', key = int(ing.id)) if not perms['currentUser']['write']: return HttpResponse(json.encode({'error': "You don't have permission to delete this ingestion"}), mimetype = 'text/plain') orig = ing.label try: ing.label = name ing.save() except IntegrityError: # Name not available return HttpResponse(json.encode({'old': orig, 'error': "An ingestion named <b>%s</b> already exists.<br/>Cannot rename ingestion." % name}), mimetype = 'text/plain') return HttpResponse(str(name), mimetype = 'text/plain')
def indexDocs(self): for root, _, filenames in os.walk(self.directory): for filename in filenames: try: path = os.path.join(root, filename) print path f = open(path, 'r') # every line in the file is a tweet document to be indexed for line in f: tweet = cjson.decode(line) doc = Document() doc.add(Field("tx", tweet['tx'], Field.Store.YES, Field.Index.NOT_ANALYZED)) doc.add(Field("h", cjson.encode(tweet['h']), Field.Store.YES, Field.Index.NOT_ANALYZED)) doc.add(Field("t", cjson.encode(tweet['t']), Field.Store.YES, Field.Index.NOT_ANALYZED)) doc.add(Field("ats", cjson.encode(tweet['ats']), Field.Store.YES, Field.Index.NOT_ANALYZED)) doc.add(Field("w", " ".join(tweet['w']), Field.Store.NO, Field.Index.ANALYZED)) doc.add(Field("user", cjson.encode(tweet['user']), Field.Store.YES, Field.Index.NOT_ANALYZED)) self.writer.addDocument(doc) f.close() except Exception, e: print "Failed in indexDocs:", e
def prepare(self, id="auto"): # Complete all stuff here and set the bytes to send if self.prepared: return if id == "auto": id = self.getUnqueuedRandID() if id is None: idobj = "" else: idobj = "@" + id self.queuedAnswer = self.chatter.language.cmds.answer.queueAnswerFor(id, self.autoremove_id) trama_args = ["!%s%s" % (self.command, idobj)] for arg in self.local_args: tr1 = ":" + arg trama_args.append(tr1) for k, arg in self.local_kwargs.items(): tr1 = k + ":" + arg trama_args.append(tr1) for arg in self.args: val = cjson.encode(arg) tr1 = "=" + val trama_args.append(tr1) for k, arg in self.kwargs.items(): val = cjson.encode(arg) tr1 = k + "=" + val trama_args.append(tr1) self.trama = "\t".join(trama_args) + "\n" self.prepared = True
def newConfiguration(self, formdata): cherrypy.response.headers['Content-Type'] = 'application/json' formdata = cjson.decode(formdata) print formdata error = "" if self.configurations.has_key(formdata["configname"]) == False: if formdata["port"] != "": if self.checkFreePorts(formdata["protocol"], formdata["port"]): formdata["manport"] = self.getFreePorts(formdata["configname"])[1] else: result = False error = "port is occupied" return cjson.encode({"result": result, "error": error}) else: freeport = self.getFreePorts(formdata["configname"]) formdata["port"] = freeport[0] formdata["manport"] = freeport[1] newConf = Configuration(0, formdata["configname"], "") newConf.new(formdata) self.configurations[newConf.id] = copy(newConf) self.setConfiguration(newConf.id) result = True else: result = False error = "name exists" self.loadConfigurations() return cjson.encode({"result": result, "error": error})
def mobile_authenticate(request,**kwargs): username = request.REQUEST.get('username', 'empty') password = request.REQUEST.get('password','empty') user = authenticate(username=username, password=password) observer = None role = 0 try: observer = Observer.objects.get(user__username=username) role = 1 if user.is_superuser else 2 except: pass if user is not None and observer is not None: return HttpResponse(cjson.encode( { 'status':'SUCCESS', 'code':200, 'message': [ { "uuid": observer.uuid, "role": role, } ]})) else: message = unicode('UNAUTHORIZED:Invalid credentials!') logging.warn(message) logging.debug(u'User' + username) return HttpResponse(cjson.encode({ 'status':'FAIL', 'code':401, 'message': []}))
def _RawResponse(clients_param, names_param, blocks, accept): """Retrieve raw State results, and add them to the response. Args: clients_param - Param object containing either client_id or client glob to retrieve State objects for. names_param - Param object containing either List of Taba Names, or a names_glob to retrieve State objects for. blocks - List of Taba Name Block IDs to retrieve State objects for. accept - MIME type in which to format the output. """ clients, names = _GetClientsAndNames(clients_param, names_param) if (clients != None and len(clients) == 0) \ or (names != None and len(names) == 0): states = [] else: # Get the requested State objects. states = global_taba_server.GetStates(clients, names, blocks) # Format the response in accordance with the Accept header. if accept == 'text/json': juno.append(cjson.encode(dict([s for s in states]))) else: for (state_client_id, state_name), state in states: juno.append("(").append(state_client_id).append(", ") juno.append(state_name).append("): ") juno.append(cjson.encode(state)).append("\n")
def _test__http_error(self, json): error_data = dict(a=1, b=2) code = 415 # just some code... headers = FakeHeaders() if json: headers._headers['content-type'] = 'application/json' send_data = dict(c=2, b=3) self._expect_json_rest_request('POST', self.uri, send_data).and_raise( urllib2.HTTPError(self.uri, code, 'msg', headers, StringIO(cjson.encode(error_data))) ) with self.forge.verified_replay_context(): with self.assertRaises(JSONRestRequestException) as caught: self.sender.post(data=send_data) exception = caught.exception #make sure the exception is printable self.assertIsInstance(str(exception), basestring) self.assertIsInstance(repr(exception), basestring) self.assertEquals(exception.code, code) self.assertIs(exception.sent_data, send_data) self.assertEquals(exception.url, self.uri) if json: self.assertEquals(exception.received_data, error_data) else: self.assertIsInstance(exception.received_data, Raw) self.assertEquals(exception.received_data.data, cjson.encode(error_data))
def fetch_gene_sequences(self): print "Fetching family FASTA files..." try: self.geneSequences = cjson.decode(open('%s/gene_sequences.json'%run_name).read()) except: self.geneSequences = {} genes_to_fetch = list(set(self.geneToSpecies.iterkeys())-set(self.geneSequences.iterkeys())) if len(genes_to_fetch) == 0: print "All genes already fetched." return self.genes_fetched=0 self.total_genes_to_fetch = len(genes_to_fetch) while len(genes_to_fetch) > 0: # Chop off a chunk of 1000 genes, fetch them, write them to the output file current_chunk = genes_to_fetch[:1000] genes_to_fetch = genes_to_fetch[1000:] for g in downloader_pool.imap(fetch_gene, current_chunk): i += 1 total_genes_to_fetch = len(genes_to_fetch) + len(current_chunk) fname='%s/gene_sequences.json'%run_name if os.path.isfile(fname): os.rename(fname,fname+'_old') open(fname,'w').write(cjson.encode(self.geneSequences)) os.unlink(fname+'_old') else: open(fname,'w').write(cjson.encode(self.geneSequences)) print "Done."
def GET(self, catrel_name, arch_name, osrel_name, catalogname): """Get a srv4 reference by catalog ane catalogname.""" web.header('Access-Control-Allow-Origin', '*') try: sqo_osrel, sqo_arch, sqo_catrel = models.GetSqoTriad( osrel_name, arch_name, catrel_name) except sqlobject.main.SQLObjectNotFound: raise web.notfound( cjson.encode({'message': 'catalog %s %s %s not found' % (osrel_name, arch_name, catrel_name)})) join = [ sqlbuilder.INNERJOINOn(None, models.Srv4FileInCatalog, models.Srv4FileInCatalog.q.srv4file==models.Srv4FileStats.q.id), ] res = models.Srv4FileStats.select( sqlobject.AND( models.Srv4FileInCatalog.q.osrel==sqo_osrel, models.Srv4FileInCatalog.q.arch==sqo_arch, models.Srv4FileInCatalog.q.catrel==sqo_catrel, models.Srv4FileStats.q.catalogname==catalogname, models.Srv4FileStats.q.use_to_generate_catalogs==True), join=join, ) try: srv4 = res.getOne() mimetype, data = srv4.GetRestRepr(quick=True) web.header('Content-type', mimetype) return cjson.encode(data) except sqlobject.main.SQLObjectNotFound: data = {'message': 'no %s %s %s %s packages found' % (catrel_name, arch_name, osrel_name, catalogname)} raise web.notfound(cjson.encode(data)) except sqlobject.dberrors.OperationalError as exc: raise web.internalerror(exc)
def get_media(mediaid,field_list = []): if type(mediaid) == list: return_list = [] no_cache_list = [] temp_map = {} for i in mediaid: media = cache.get(MEDIA_HEAD+str(i).strip()) if media: temp_map[str(i).strip()] = get_field_data_form_json(media,field_list) else: no_cache_list.append(str(i).strip()) media_list = MediaDetailInfo.objects.filter(mediaid__in=no_cache_list) for i in media_list: temp_map[i.mediaid] = get_field_data_form_obj(i) cache.set(MEDIA_HEAD+str(i.mediaid),cjson.encode(get_field_data_form_obj(i)),MEDIA_CACHE_TIME) for i in mediaid: if temp_map.has_key(str(i).strip()): return_list.append(temp_map[str(i).strip()]) return return_list else: media = cache.get(MEDIA_HEAD+str(mediaid)) if media: return get_field_data_form_json(media,field_list) else: media_obj = MediaDetailInfo.objects.filter(mediaid=mediaid) if len(media_obj) > 0: cache.set(MEDIA_HEAD+str(mediaid),cjson.encode(get_field_data_form_obj(media_obj[0])),MEDIA_CACHE_TIME) return get_field_data_form_obj(media_obj[0],field_list) else: media_obj = get_detail_from_spider(mediaid) return get_field_data_form_obj(media_obj,field_list)
def GET(self, catrel_name, arch_name, osrel_name, pkgname): """Get a srv4 reference by catalog ane pkgname.""" configuration.SetUpSqlobjectConnection() sqo_osrel, sqo_arch, sqo_catrel = pkgdb.GetSqoTriad( osrel_name, arch_name, catrel_name) join = [ sqlbuilder.INNERJOINOn(None, models.Srv4FileInCatalog, models.Srv4FileInCatalog.q.srv4file==models.Srv4FileStats.q.id), sqlbuilder.INNERJOINOn(None, models.Pkginst, models.Pkginst.q.id==models.Srv4FileStats.q.pkginst), ] res = models.Srv4FileStats.select( sqlobject.AND( models.Srv4FileInCatalog.q.osrel==sqo_osrel, models.Srv4FileInCatalog.q.arch==sqo_arch, models.Srv4FileInCatalog.q.catrel==sqo_catrel, models.Pkginst.q.pkgname==pkgname, models.Srv4FileStats.q.use_to_generate_catalogs==True), join=join, ) try: srv4 = res.getOne() mimetype, data = srv4.GetRestRepr() web.header('Content-type', mimetype) web.header('Access-Control-Allow-Origin', '*') return cjson.encode(data) except sqlobject.main.SQLObjectNotFound: return cjson.encode(None) except sqlobject.dberrors.OperationalError, e: raise web.internalerror(e)
def _ProjectionResponse(clients_param, names_param, blocks, accept): """Retrieve Projection results, and add them to the response. Args: clients_param - Param object containing either client_id or client glob to retrieve Projections for. names_param - Param object containing either List of Taba Names, or a names_glob to retrieve Projections for. blocks - List of Taba Name Block IDs to retrieve Projections for. accept - MIME type in which to format the output. """ clients, names = _GetClientsAndNames(clients_param, names_param) if (clients != None and len(clients) == 0) \ or (names != None and len(names) == 0): projections = [] else: # Retrieve the requested Projections. projections = global_taba_server.GetProjections(clients, names, blocks) # Render the Projection objects according to the requested format. if accept == 'text/json': juno.append(cjson.encode(dict([p for p in projections]))) else: for (proj_client_id, proj_name), projection in projections: juno.append("(").append(proj_client_id).append(", ") juno.append(proj_name).append("): ") juno.append(cjson.encode(projection)).append("\n")
def downloadImages(): services = {'twitpic': HTMLParsers.parseTwitpic, 'yfrog': HTMLParsers.parseYfrog, 'twitrpix': HTMLParsers.parseTwitrpix} for f in ['2011_2_%s'%i for i in range(20, 26)]: for tweet in Utilities.iterateTweetsFromFile(Settings.new_zealand_pics_folder+'tweets/'+f): d = datetime.strptime(tweet['created_at'], Settings.twitter_api_time_format) print d print cjson.encode(tweet) service, url = 'twitpic', '' if len(tweet['entities']['urls'])>0: url = tweet['entities']['urls'][0]['url'] else: for service, parseMethod in services.iteritems(): if service in tweet['text']: for term in tweet['text'].split(): if service in term: url = term break url = url.replace('\\', '') if not url.startswith('http'): url = 'http://'+url for service, parseMethod in services.iteritems(): if service in url: id = tweet['id'] fileName = Settings.new_zealand_pics_folder+Utilities.getDataFile(d)+'/%s_%s.jpeg'%(str(d).replace(' ', '_'), id) print url, fileName folder = '/'.join(fileName.split('/')[:-1]) if not os.path.exists(folder): os.makedirs(folder, 0777) if not os.path.exists(fileName): retry, notParsed = 0, True while retry<3 and notParsed: try: parseMethod(url, fileName) notParsed = False time.sleep(3) except: retry+=1
def home(request): """Top level url Displays :: {"path": HttpRequest.path, "host": HttpRequest.get_host(), "version": sana.api.version, "service": "REST"} """ username = request.REQUEST.get('username', 'empty') password = request.REQUEST.get('password','empty') user = authenticate(username=username, password=password) if user is not None: return HttpResponse(cjson.encode( { 'status':'SUCCESS', 'code':200, 'message': version()})) else: message = unicode('UNAUTHORIZED:Invalid credentials!') logging.warn(message) logging.debug(u'User' + username) return HttpResponse(cjson.encode({ 'status':'FAIL', 'code':401, 'message': message}))
def mqCallback(self, channel, method_frame, header_frame, body): try: if not self.zk.is_proxy_master(): return # master's business data_dict = cjson.decode(body) # ** MUST ** ack channel.basic_ack(method_frame.delivery_tag) utils.log(utils.cur(), body, data_dict) if not isinstance(data_dict, dict): return for db, forbid in data_dict.iteritems(): if not forbid[Forbid.KEY_TYPE] in (Forbid.FORBID_WORKING, Forbid.FORBID_FOREVER): return forbid[Forbid.KEY_START] = time.time() path = os.path.join(ZKConf.ZK_PATH_FORBID, db) orig = self.get_path(path) if orig is False: self.zk.mknode(path, cjson.encode(forbid)) else: old = cjson.decode(orig) if ( old[Forbid.KEY_TYPE] == forbid[Forbid.KEY_TYPE] and old[Forbid.KEY_TYPE] == Forbid.FORBID_WORKING and old[Forbid.KEY_START] + old[Forbid.KEY_DURATION] > time.time() ): utils.log(utils.cur(), "still forbidding") else: utils.log(utils.cur(), "change forbid") # change /database/forbid/db self.forbidinfo[db] = forbid self.zk.set(path, cjson.encode(forbid)) except Exception, err: utils.err(utils.cur(), err)
def thrash_cjson(): for obj, tns, json, msh in TESTS: if TEST_DUMP_ONLY: cjson.encode(obj) elif TEST_LOAD_ONLY: assert cjson.decode(json) == obj else: assert cjson.decode(cjson.encode(obj)) == obj
def action_export(conn, tables, excluded_urls): items = [] for table in tables: items += get_saved_sessions(conn, table, False) items = remove_duplicates(filter_excluded(items)) print cjson.encode(items)
def toJSON(self, fields=None): if fields == None: return cjson.encode(self) else: dct = {} for field in fields: dct[field] = self.__getattribute__(field) return cjson.encode(dct)
def PUT(self, name): if not re.match(r'', name): raise web.conflict() with Transaction(models.Srv4FileStats) as trans: res = models.CatalogRelease.selectBy(name=name, connection=trans) if res.count(): return cjson.encode('%s already exists' % name) models.CatalogRelease(name=name, connection=trans) return cjson.encode('%s has been created' % name)
def on_message(self, message): try: data = cjson.decode(message) resp = game.notify(self, data) self.write_message(cjson.encode(resp)) except Exception, e: print e req = cjson.encode({"error": "undefined"}) self.write_message(req)
def deleteplayhistory(request): res_json = {} res_json[RES_STATUS_CODE] = STATUS_SUCCESS deviceid = request.GET.get("deviceid","") if deviceid: History.objects.filter(deviceid=deviceid).delete() return HttpResponse(cjson.encode(res_json)) res_json[RES_STATUS_CODE] = STATUS_ARGUMENT_ERROR return HttpResponse(cjson.encode(res_json))
def add_test(v): # These modules have a few round-tripping problems... try: assert cjson.decode(cjson.encode(v)) == v assert yajl.loads(yajl.dumps(v)) == v except Exception: pass else: TESTS.append((v,tnetstring.dumps(v),cjson.encode(v)))
def searchKeywords(): keywords = sys.argv[1:] japan_bb=[30, 42, 129, 145] for f in ['2011_3_10.gz', '2011_3_11.gz']: for tweet in Utilities.iterateTweetsFromGzip(Settings.geo_folder+f): if Utilities.tweetInBoundingBox(tweet, japan_bb): for k in keywords: if k in tweet['text']: print cjson.encode(tweet)
def output_transformation(line): if not line: return '' if 'seq' in line: line['seq'] = base64.urlsafe_b64encode( zlib.compress(cjson.encode(line['seq']), 1)) elif 'last_seq' in line: line['last_seq'] = base64.urlsafe_b64encode( zlib.compress(cjson.encode(line['last_seq']), 1)) return cjson.encode(line) + '\n'
def jdumps(obj, encoding='utf-8'): """ Serializes ``obj`` to a JSON formatted string, using cjson. """ return cjson.encode(obj)
new_creator_addr['s'] = creator_addr['state'] if 'county' in creator_addr.keys(): new_creator_addr['c'] = creator_addr['county'] line['c_addr'] = new_creator_addr user_url = web + user_lat + user_lon #+extra user_info = urllib2.urlopen(user_url) user_info = user_info.read() user_addr = cjson.decode(user_info)['address'] # print user_addr new_user_addr = {} new_user_addr['c_code'] = user_addr['country_code'] if 'state' in user_addr.keys(): new_user_addr['s'] = user_addr['state'] if 'county' in user_addr.keys(): new_user_addr['c'] = user_addr['county'] line['u_addr'] = new_user_addr line = cjson.encode(line) outfile.write(line + '\n') except: print "error reading web" #pass #sys.exit() ##else: # pass #from httplib2 import Http #h=Http() #resp,content=h.request(std,'GET') #content=cjson.decode(content) #print content['county']
#elif freq1 ==freq2 and common>=3: # if ' ' in key1: # removelist.append(key2) # elif ' ' in key2: # removelist.append(key1) #print removelist #print'_______1_______' rulelist = [] outfile2 = sys.argv[3] #'/spare/wei/local/rulelist_en_ch3-log' out2 = open(outfile2, 'w') for key, value in relation.iteritems(): if key in removelist: print key continue for key1, value1 in value.iteritems(): if key1 in removelist: continue if value1[0] > 0.1 and key != key1: if (key in key1 or key1 in key) and value1[0] == 1: continue else: rulelist.append([key1, key, value1[0]]) rules = sorted(rulelist, key=itemgetter(2), reverse=True) for rule in rules: out2.write(cjson.encode(rule) + '\n') #print dist1 #print dist2 removelistf = open(sys.argv[4], 'w') removelistf.write(cjson.encode(removelist))
import cjson sift = 'tag_user_dict_sift_1' f = open(sift, 'w') i = 0 for line in open('tag_user_dict_sift', 'r'): # if i<22414: # i+=1 # continue #line=cjson.decode(line) #line=line.split('\t')[1] line = cjson.decode(line) if line[0][-2:-1] != 'of' and line[0][-3:-1] != 'and': f.write(cjson.encode(line) + '\n') else: pass #print line #line=cjson.decode(line) #f.write(line+'\n')
#remove tag that are not in the tag_user_dict import sys import cjson argv = sys.argv #inputfilename unique_tag outputfilename infile1 = open(argv[1], 'r') infile2 = open(argv[2], 'r') tag_f = cjson.decode(infile2.readline()) tag_f = tag_f.keys() outfile = open(argv[3], 'w') distinct = open(sys.argv[4], 'w') distinctdict = {} distinctlc = {} for line in infile1: line = line.split('\t')[1] line = cjson.decode(line) dictt = {} for key, value in line[1].iteritems(): if key in tag_f and value > 0.01: dictt[key] = round(value, 2) distinctdict[key] = distinctdict.get(key, 0) if value > 1: distinctlc[key] = distinctlc.get(key, 0) if dictt != {}: outfile.write(cjson.encode([line[0], dictt]) + '\n') distinct.write(cjson.encode(distinctdict) + '\n') distinct.write(cjson.encode(distinctlc) + '\n')
max_can = '' for node in G.nodes(): key1, key2 = tag + '_' + node, node + '_' + tag b = cosine.get(key1, 0) c = cosine.get(key2, 0) similarity = max(b, c) if similarity > max_can_value: max_can_value = similarity max_can = node if max_can_value > THRESHOLD: G.add_node(tag) G.add_edge(max_can, tag) #print 'new tag',tag #print 'root',max_can #print graph_dict #graph_dict[str(max_can)][tag]={} graph_parent_dict[tag] = max_can else: G.add_node(tag) graph_parent_dict[tag] = 'ROOT' nx.write_gml(G, "noise_0.1_folksonomy_0.3.gml") print graph_parent_dict a = build_tree(graph_parent_dict) print a outfile2 = open('noise_0.1_folksonomy_0.3', 'w') print_dict(a, outfile2) outfile3 = 'noise_0.1_graph_parent_dict_0.3' outfile3 = open(outfile3, 'w') outfile3.write(cjson.encode(graph_parent_dict))
def dataSent(self, data): #print "Server sent : " + str(data) self.transport.write(cjson.encode(data) + '\n')
def generate(numberOfDocuments=2500, dimensions=52): def pickOneByProbability(objects, probabilities): initialValue, objectToRange = 0.0, {} for i in range(len(objects)): objectToRange[objects[i]] = (initialValue, initialValue + probabilities[i]) initialValue += probabilities[i] randomNumber = random.random() for object, rangeVal in objectToRange.iteritems(): if rangeVal[0] <= randomNumber <= rangeVal[1]: return object topics = { 'elections': { 'prob': 0.3, 'tags': { '#gop': 0.4, '#bachmann': 0.2, '#perry': 0.2, '#romney': 0.2 } }, 'soccer': { 'prob': 0.2, 'tags': { '#rooney': 0.15, '#chica': 0.1, '#manutd': 0.6, '#fergie': 0.15 } }, 'arab': { 'prob': 0.3, 'tags': { '#libya': 0.4, '#arab': 0.3, '#eqypt': 0.15, '#syria': 0.15 } }, 'page3': { 'prob': 0.2, 'tags': { '#paris': 0.2, '#kim': 0.4, '#britney': 0.2, '#khloe': 0.2 } }, } stopwords = 'abcdefghijklmnopqrstuvwxyz1234567890' print '#', cjson.encode({'dimensions': dimensions}) for i in range(numberOfDocuments): topic = pickOneByProbability( topics.keys(), [topics[k]['prob'] for k in topics.keys()]) print ' '.join([topic] + [ pickOneByProbability(topics[topic]['tags'].keys(), [ topics[topic]['tags'][k] for k in topics[topic]['tags'].keys() ]) for i in range(2) ] + [random.choice(stopwords) for i in range(5)])
def serialize(feature_gene, hits): doc = {} doc['y'], doc['x'], doc['gene'], doc['barcode'] = feature_gene doc['hits'] = hits return cjson.encode(doc)
def encode_json(obj): """Wrapper to re-encode JSON string in an implementation-independent way.""" # TODO: Verify correctness of cjson return cjson.encode(obj)
def main(self): twitterurl = "http://api.twitter.com/1/users/search.json" if self.proxy: proxyhandler = urllib2.ProxyHandler({"http": self.proxy}) twitopener = urllib2.build_opener(proxyhandler) urllib2.install_opener(twitopener) headers = {'User-Agent': "BBC R&D Grabber"} postdata = None if self.keypair == False: # Perform OAuth authentication - as we don't have the secret key pair we need to request it # This will require some user input request_token_url = 'http://api.twitter.com/oauth/request_token' access_token_url = 'http://api.twitter.com/oauth/access_token' authorize_url = 'http://api.twitter.com/oauth/authorize' token = None consumer = oauth.Consumer(key=self.consumerkeypair[0], secret=self.consumerkeypair[1]) params = { 'oauth_version': "1.0", 'oauth_nonce': oauth.generate_nonce(), 'oauth_timestamp': int(time.time()), } params['oauth_consumer_key'] = consumer.key req = oauth.Request(method="GET", url=request_token_url, parameters=params) signature_method = oauth.SignatureMethod_HMAC_SHA1() req.sign_request(signature_method, consumer, token) requestheaders = req.to_header() requestheaders['User-Agent'] = "BBC R&D Grabber" # Connect to Twitter try: req = urllib2.Request( request_token_url, None, requestheaders ) # Why won't this work?!? Is it trying to POST? conn1 = urllib2.urlopen(req) except httplib.BadStatusLine: e = sys.exc_info()[1] Print("PeopleSearch BadStatusLine error:", e) conn1 = False except urllib2.HTTPError: e = sys.exc_info()[1] Print("PeopleSearch HTTP error:", e.code) # sys.stderr.write('PeopleSearch HTTP error: ' + str(e.code) + '\n') conn1 = False except urllib2.URLError: e = sys.exc_info()[1] Print("PeopleSearch URL error: ", e.reason) # sys.stderr.write('PeopleSearch URL error: ' + str(e.reason) + '\n') conn1 = False if conn1: content = conn1.read() conn1.close() request_token = dict(urlparse.parse_qsl(content)) Print("Request Token:") Print(" - oauth_token = ", request_token['oauth_token']) Print(" - oauth_token_secret = ", request_token['oauth_token_secret']) Print("") # The user must confirm authorisation so a URL is Printed here Print("Go to the following link in your browser:") Print("%s?oauth_token=%s" % (authorize_url, request_token['oauth_token'])) Print("") accepted = 'n' # Wait until the user has confirmed authorisation while accepted.lower() == 'n': accepted = raw_input('Have you authorized me? (y/n) ') oauth_verifier = raw_input('What is the PIN? ') token = oauth.Token(request_token['oauth_token'], request_token['oauth_token_secret']) token.set_verifier(oauth_verifier) params = { 'oauth_version': "1.0", 'oauth_nonce': oauth.generate_nonce(), 'oauth_timestamp': int(time.time()), } params['oauth_token'] = token.key params['oauth_consumer_key'] = consumer.key req = oauth.Request(method="GET", url=access_token_url, parameters=params) signature_method = oauth.SignatureMethod_HMAC_SHA1() req.sign_request(signature_method, consumer, token) requestheaders = req.to_header() requestheaders['User-Agent'] = "BBC R&D Grabber" # Connect to Twitter try: req = urllib2.Request( access_token_url, "oauth_verifier=%s" % oauth_verifier, requestheaders ) # Why won't this work?!? Is it trying to POST? conn1 = urllib2.urlopen(req) except httplib.BadStatusLine: e = sys.exc_info()[1] # sys.stderr.write('PeopleSearch BadStatusLine error: ' + str(e) + '\n') Print('PeopleSearch BadStatusLine error: ', e) conn1 = False except urllib2.HTTPError: e = sys.exc_info()[1] Print('PeopleSearch HTTP error: ', e.code) conn1 = False except urllib2.URLError: e = sys.exc_info()[1] # sys.stderr.write('PeopleSearch URL error: ' + str(e.reason) + '\n') Print('PeopleSearch URL error: ', e.reason) conn1 = False if conn1: content = conn1.read() conn1.close() access_token = dict(urlparse.parse_qsl(content)) # Access tokens retrieved from Twitter Print("Access Token:") Print(" - oauth_token = ", access_token['oauth_token']) Print(" - oauth_token_secret = ", access_token['oauth_token_secret']) Print("") Print( "You may now access protected resources using the access tokens above." ) Print("") save = False # Load config to save OAuth keys try: homedir = os.path.expanduser("~") file = open(homedir + "/twitter-login.conf", 'r') save = True except IOError: e = sys.exc_info()[1] Print( "Failed to load config file - not saving oauth keys: ", e) if save: raw_config = file.read() file.close() # Read config and add new values config = cjson.decode(raw_config) config['key'] = access_token['oauth_token'] config['secret'] = access_token['oauth_token_secret'] raw_config = cjson.encode(config) # Write out the new config file try: file = open(homedir + "/twitter-login.conf", 'w') file.write(raw_config) file.close() except IOError: e = sys.exc_info()[1] Print("Failed to save oauth keys: ", e) self.keypair = [ access_token['oauth_token'], access_token['oauth_token_secret'] ] while not self.finished(): # TODO: Implement backoff algorithm in case of connection failures - watch out for the fact this could delay the requester component if self.dataReady("inbox"): # Retieve keywords to look up person = self.recv("inbox") # Ensure we're not rate limited during the first request - if so we'll wait for 15 mins before our next request if (datetime.today() - timedelta(minutes=15)) > self.ratelimited: requesturl = twitterurl + "?q=" + urllib.quote( person) + "&per_page=5" params = { 'oauth_version': "1.0", 'oauth_nonce': oauth.generate_nonce(), 'oauth_timestamp': int(time.time()), } token = oauth.Token(key=self.keypair[0], secret=self.keypair[1]) consumer = oauth.Consumer(key=self.consumerkeypair[0], secret=self.consumerkeypair[1]) params['oauth_token'] = token.key params['oauth_consumer_key'] = consumer.key req = oauth.Request(method="GET", url=requesturl, parameters=params) signature_method = oauth.SignatureMethod_HMAC_SHA1() req.sign_request(signature_method, consumer, token) requestheaders = req.to_header() requestheaders['User-Agent'] = "BBC R&D Grabber" # Connect to Twitter try: req = urllib2.Request( requesturl, None, requestheaders ) # Why won't this work?!? Is it trying to POST? conn1 = urllib2.urlopen(req) except httplib.BadStatusLine: e = sys.exc_info()[1] # sys.stderr.write('PeopleSearch BadStatusLine error: ' + str(e) + '\n') Print('PeopleSearch BadStatusLine error: ', e) conn1 = False except urllib2.HTTPError: e = sys.exc_info()[1] # sys.stderr.write('PeopleSearch HTTP error: ' + str(e.code) + '\n') Print('PeopleSearch HTTP error: ', e.code) conn1 = False except urllib2.URLError: e = sys.exc_info()[1] # sys.stderr.write('PeopleSearch URL error: ' + str(e.reason) + '\n') Print('PeopleSearch URL error: ', e.reason) conn1 = False if conn1: # Check rate limiting here and Print current limit headers = conn1.info() try: headerlist = string.split(str(headers), "\n") except UnicodeEncodeError: # str may fail... headerlist = [] for line in headerlist: if line != "": splitheader = line.split() if splitheader[ 0] == "X-FeatureRateLimit-Remaining:" or splitheader[ 0] == "X-RateLimit-Remaining:": Print(splitheader[0], " ", splitheader[1]) if int(splitheader[1]) < 5: self.ratelimited = datetime.today() # Grab json format result of people search here try: data = conn1.read() try: content = cjson.decode(data) self.send(content, "outbox") except cjson.DecodeError: self.send(dict(), "outbox") except IOError: e = sys.exc_info()[1] # sys.stderr.write('PeopleSearch IO error: ' + str(e) + '\n') Print('PeopleSearch IO error: ', e) self.send(dict(), "outbox") conn1.close() else: self.send(dict(), "outbox") else: Print("Twitter search paused - rate limited") self.send(dict(), "outbox") self.pause() yield 1
def error(exception): #errors.append(exception) #errors = exception if exception else errors = traceback.format_exception_only(*sys.exc_info()[:2]) #.splitlines()[1:]#[] #et, val, tb = sys.exc_info() #tb = sys.exc_info()[2] #trace = traceback.format_tb(tb) #for tbm in traceback.format_tb(tb): #for trace in traceback.format_exception_only(sys.last_type, sys.last_value): # errors.append(repr(trace)) response = { 'status': 'FAILURE', 'code': 500, 'message': None, 'errors': errors, } return response def unauthorized(message): return fail(message, Codes.UNAUTHORIZED) if __name__ == 'main': import cjson data = cjson.encode({'a': '1'}) js = JSONResponse(data) print js._headers
key1,key2=key[0],key[1] #freq1=tagdict[key1+'_'+key1] #freq2=tagdict[key2+'_'+key2] freq1=tag_freq_dict[key1] freq2=tag_freq_dict[key2] common=tagdict.get(key1+'_'+key2,tagdict.get(key2+'_'+key1)) if common>=3 and freq1>=freq2: weight=(1-(log(freq1)-log(freq2))/lMAXF) #print weight #weight=(1-(freq1-freq2)/MAXF) relation[key1][key2]=common/freq2*weight elif common>=3 and freq2>=freq1: weight=(1-(log(freq2)-log(freq1))/lMAXF) #weight=(1-(freq2-freq1)/MAXF) #print weight relation[key2][key1]=common/freq1*weight print'_______1_______' rulelist=[] outfile2='/spare/wei/local/rulelist_en_ch3-log' out2=open(outfile2,'w') for key,value in relation.iteritems(): for key1,value1 in value.iteritems(): if value1>0.0000001 and value1!=1 and key!=key1: rulelist.append([key1,key,value1]) rules=sorted(rulelist,key=itemgetter(2),reverse=True) for rule in rules: out2.write(cjson.encode(rule)+'\n')
def get(self): userid = self.get_argument("userid", strip=True) amount = self.get_argument("amount", strip=True) data = mysql.YuheSub(userid, amount) self.write(cjson.encode(data))
#generating parent child dictionary like{food:{parent:media,child:[......]}} from #graph_parent_dict import cjson local = '/spare/wei/local/%s' infile = local % 'graph_parent_dict_chi' infile = open(infile, 'r') graph_parent_dict = cjson.decode(infile.readline()) p_c_dict = {} for key, value in graph_parent_dict.iteritems(): if key not in p_c_dict.keys(): p_c_dict[key] = {'parent': value, 'child': []} else: p_c_dict[key]['parent'] = value if value not in p_c_dict.keys(): p_c_dict[value] = {'parent': 'a', 'child': [key]} else: p_c_dict[value]['child'].append(key) outfile = open(local % 'p_c_dict_chi', 'w') outfile.write(cjson.encode(p_c_dict))
def main(self): cursor = self.dbConnect() while not self.finished(): twitdata = list() # Collect all current received tweet JSON and their related PIDs into a twitdata list while self.dataReady("inbox"): pids = list() data = self.recv("inbox") for pid in data[1]: pids.append(pid) twitdata.append([data[0], pids]) if len(twitdata) > 0: # Process the received twitdata for tweet in twitdata: tweet[0] = tweet[0].replace( "\\/", "/" ) # Fix slashes in links: This may need moving further down the line - ideally it would be handled by cjson if tweet[0] != "\r\n": # If \r\n is received, this is just a keep alive signal from Twitter every 30 secs # At this point, each 'tweet' contains tweetdata, and a list of possible pids newdata = cjson.decode(tweet[0]) if newdata.has_key('delete') or newdata.has_key( 'scrub_geo') or newdata.has_key('limit'): # Keep a record of all requests from Twitter for deletions, location removal etc # As yet none of these have been received, but this code will store them if they are received to enable debugging filepath = "contentDebug.txt" if os.path.exists(filepath): file = open(filepath, 'r') filecontents = file.read() else: filecontents = "" file = open(filepath, 'w') file.write(filecontents + "\n" + str(datetime.utcnow()) + " " + cjson.encode(newdata)) file.close() else: # This is a real tweet tweetid = newdata['id'] print "New tweet! @" + newdata['user'][ 'screen_name'] + ": " + newdata['text'] for pid in tweet[1]: # Cycle through possible pids, grabbing that pid's keywords from the DB # Then, check this tweet against the keywords and save to DB where appropriate (there may be more than one location) cursor.execute( """SELECT keyword,type FROM keywords WHERE pid = %s""", (pid)) data = cursor.fetchall() for row in data: # Some keywords are stored with a ^. These must be split, and the tweet checked to see if it has both keywords, but not necessarily next to each other keywords = row[0].split("^") if len(keywords) == 2: if string.lower( keywords[0]) in string.lower( newdata['text'] ) and string.lower( keywords[1] ) in string.lower( newdata['text']): cursor.execute( """SELECT timestamp,timediff FROM programmes WHERE pid = %s ORDER BY timestamp DESC""", (pid)) progdata = cursor.fetchone() if progdata != None: # Ensure the user hasn't already tweeted the same text # Also ensure they haven't tweeted in the past 10 seconds timestamp = time2.mktime( parse(newdata['created_at'] ).timetuple()) cursor.execute( """SELECT * FROM rawdata WHERE (pid = %s AND text = %s AND user = %s) OR (pid = %s AND user = %s AND timestamp >= %s AND timestamp < %s)""", (pid, newdata['text'], newdata['user'] ['screen_name'], pid, newdata['user'] ['screen_name'], timestamp - 10, timestamp + 10)) if cursor.fetchone() == None: print( "Storing tweet for pid " + pid) # Work out where this tweet really occurred in the programme using timestamps and DVB bridge data progposition = timestamp - ( progdata[0] - progdata[1]) cursor.execute( """INSERT INTO rawdata (tweet_id,pid,timestamp,text,user,programme_position) VALUES (%s,%s,%s,%s,%s,%s)""", (tweetid, pid, timestamp, newdata['text'], newdata['user'] ['screen_name'], progposition)) break # Break out of this loop and back to check the same tweet against the next programme else: print( "Duplicate tweet from user - ignoring" ) if string.lower(row[0]) in string.lower( newdata['text']): cursor.execute( """SELECT timestamp,timediff FROM programmes WHERE pid = %s ORDER BY timestamp DESC""", (pid)) progdata = cursor.fetchone() if progdata != None: # Ensure the user hasn't already tweeted the same text for this programme # Also ensure they haven't tweeted in the past 10 seconds timestamp = time2.mktime( parse(newdata['created_at']). timetuple()) cursor.execute( """SELECT * FROM rawdata WHERE (pid = %s AND text = %s AND user = %s) OR (pid = %s AND user = %s AND timestamp >= %s AND timestamp < %s)""", (pid, newdata['text'], newdata['user'] ['screen_name'], pid, newdata['user'] ['screen_name'], timestamp - 10, timestamp + 10)) if cursor.fetchone() == None: print( "Storing tweet for pid " + pid) # Work out where this tweet really occurred in the programme using timestamps and DVB bridge data progposition = timestamp - ( progdata[0] - progdata[1]) cursor.execute( """INSERT INTO rawdata (tweet_id,pid,timestamp,text,user,programme_position) VALUES (%s,%s,%s,%s,%s,%s)""", (tweetid, pid, timestamp, newdata['text'], newdata['user'] ['screen_name'], progposition)) break # Break out of this loop and back to check the same tweet against the next programme else: print( "Duplicate tweet from user - ignoring" ) else: print "Blank line received from Twitter - no new data" print("Done!") # new line to break up display else: time2.sleep(0.1)
def MakeRevIndex(self, catrel, arch, osrel): key = (catrel, arch, osrel) if key in self.cached_catalogs_bins: return fn_bins = cache_file_bins% key fn_links = cache_file_links% key fn_needed_bins = cache_file_needed_bins % key if os.path.exists(fn_bins) and os.path.exists(fn_needed_bins) and os.path.exists(fn_links): with open(fn_bins, "r") as fd: self.cached_catalogs_bins[key] = cjson.decode(fd.read()) with open(fn_links, "r") as fd: self.cached_catalogs_links[key] = cjson.decode(fd.read()) with open(fn_needed_bins, "r") as fd: self.cached_catalogs_needed_bins[key] = cjson.decode(fd.read()) return catalog = self.rest_client.GetCatalog(*key) bins = {} links = {} needed_bins = {} i = 0 for pkg_simple in catalog: i = i+1 cb = [] cl = [] nb = [] # pprint.pprint(pkg_simple) md5 = pkg_simple["md5_sum"] pkg = self.cp.GetPkgstats(md5) if not pkg: logging.warning("MakeRevIndex: No package for %r", md5) continue try: pkg_name = pkg["basic_stats"]["pkgname"] for p in pkg['binaries_dump_info']: for b in p['needed sonames']: if b not in nb: nb.append(b) for b in pkg['binaries']: if b not in cb: cb.append(b) else: logging.debug("MakeRevIndex: %s already in cache") for pm in pkg['pkgmap']: if pm['type'] == 's': # symbolic link cl.append(pm['line'].split(' ')[3].split('=')[0]) # take the linkname except KeyError: logging.warning("MakeRevIndex: no pkg structure: ") # logging.warning(pkg) bins[pkg_name] = cb needed_bins[pkg_name] = nb links[pkg_name] = cl sys.stdout.write("\rMakeRevIndex:%4d %s" % (i,pkg_name)) sys.stdout.flush() sys.stdout.write("\n") self.cached_catalogs_bins[key] = bins self.cached_catalogs_links[key] = links self.cached_catalogs_needed_bins[key] = needed_bins with open(fn_bins, "w") as fd: fd.write(cjson.encode(self.cached_catalogs_bins[key])) fd.close() with open(fn_links, "w") as fd: fd.write(cjson.encode(self.cached_catalogs_links[key])) fd.close() with open(fn_needed_bins, "w") as fd: fd.write(cjson.encode(self.cached_catalogs_needed_bins[key])) fd.close()
def serialize(cls, data): return cjson.encode(data)
def dump_json(replay): try: from cjson import encode except ImportError: from simplejson import dumps as encode print encode(replay_to_dict(replay))
def setmessage(self, value): self.level = cjson.encode(value, True)
if key1 in removelist or key2 in removelist: continue cosine_dict[key1][key2] = value cosine_dict[key2][key1] = value centrality_list = [] for key, value in cosine_dict.iteritems(): centrality = 0 # print key,value for tag, freq in value.iteritems(): if freq > 0.48: centrality += freq centrality_list.append([key, centrality]) a = sorted(centrality_list, key=itemgetter(1), reverse=1) outfile1 = open(sys.argv[3], 'w') for tag, cen in a: outfile1.write(cjson.encode([tag, cen]) + '\n') #algorithm THRESHOLD = 0.48 graph_parent_dict = {} graph_parent_dict_s = {} G = nx.DiGraph() #G.add_node('root') #for tag,centrality in centrality_list: b = a rule = open(sys.argv[7], 'w') for tag, cen in b: max_can_value = 0 max_can = '' for node in G.nodes(): similarity = cosine_dict[node].get(tag, 0)
def populate_ipo_table(): ipo_urls = IPOInfoUrl.query.all() known_unwrs = set() for url in ipo_urls: comp = Company.query.filter_by(symbol=url.symbol).first() if not comp: # session.add(comp) # session.commit() continue if HistoricalIPO.query.filter_by(company_id=comp.id).first(): print "Data exists for:", url.symbol continue # comp = get_company_overview(url.symbol) # if not comp: # log.warning("Cannot get company info for %s" % url.symbol) # continue ipo_data = scrape_ipo(url.url) if ipo_data == {}: continue log.info("IPO data from NASDAQ.com:\n%s" % cjson.encode(ipo_data)) underwriters = ipo_data["underwriters"] lead_underwriters = ipo_data["lead_underwriters"] del ipo_data["underwriters"] del ipo_data["lead_underwriters"] ipo_date = ipo_data["ipo_date"] try: month, day, year = [int(i) for i in ipo_date.split("/")] ipo_date = datetime.date(year, month, day).strftime("%Y%m%d") #ipo_data["ipo_date"] = datetime.date(year, month, day).strftime("%Y-%m-%d") ipo_data["ipo_date"] = datetime.date(year, month, day) except: log.error("Error in IPO date:%s" % url.symbol) continue ipo_data_dir = os.path.join(tickdata_dir, ipo_date) ipo_data_path = os.path.join(ipo_data_dir, "%s_markethours.csv.gz" % url.symbol) exist = False if os.path.exists(ipo_data_dir) and os.path.exists(ipo_data_path): exist = True log.info("IPO data found") else: request = { "command": "get", "symbol": url.symbol, "date": ipo_date, "gettrade": "true", "getquote": "true" } try: fetcher_caller = fetcher.FetcherCaller() fetcher_caller.set_request(cjson.encode(request)) response = fetcher_caller.send_request() fetcher_caller.close() except: log.error("Unable to send fetch request") continue count_down = 60 fetched = False while count_down > 0: if os.path.exists(ipo_data_path): log.info("IPO data fetched: %s" % url.symbol) fetched = True time.sleep(5) break time.sleep(1) count_down -= 1 if not fetched: log.error("Unable to download data for %s" % url.symbol) if exist or fetched: itd = process_ipo_tick_data(symbol, ipo_date) ipo_data["open_vol"] = itd["open_vol"] ipo_data["first_opening_price"] = itd["first_opening_price"] ipo_data["first_closing_price"] = itd["first_closing_price"] ipo_data["first_trade_time"] = itd["first_trade_time"] ipo_data["first_day_high"] = itd["first_day_high"] ipo_data["first_day_low"] = itd["first_day_low"] ipo_data["first_day_high_percent_change"] = itd[ "first_day_high_percent_change"] ipo_data["first_day_low_percent_change"] = itd[ "first_day_low_percent_change"] ipo_data["first_day_volume"] = itd["first_day_volume"] else: ipo_data["open_vol"] = None ipo_data["first_opening_price"] = None ipo_data["first_closing_price"] = None ipo_data["first_trade_time"] = None ipo_data["first_day_high"] = None ipo_data["first_day_low"] = None ipo_data["first_day_high_percent_change"] = None ipo_data["first_day_low_percent_change"] = None ipo_data["first_day_volume"] = None ipo_data["scoop_rating"] = 0 ipo_data["company_id"] = comp.id log.info("Final IPO data for %s:\n%s" % (url.symbol, ipo_data)) """ for u in underwriters: if u in known_unwrs: unwr = Underwriter.query.filter_by(name=u).first() else: unwr = Underwriter(u) known_unwrs.add(u) session.add(unwr) session.commit() a = CompanyUnderwriterAssociation(company_id=comp.id, underwriter_id=unwr.id, lead=False) comp.underwriters.append(a) session.commit() for u in lead_underwriters: if u in known_unwrs: unwr = Underwriter.query.filter_by(name=u).first() else: unwr = Underwriter(u) known_unwrs.add(u) session.add(unwr) session.commit() a = CompanyUnderwriterAssociation(company_id=comp.id, underwriter_id=unwr.id, lead=True) comp.underwriters.append(a) session.commit() """ historical_ipo = HistoricalIPO(**ipo_data) session.add(historical_ipo) session.commit()
probs) - posterior_normalizer best_genotype_combo = genotype_combo_probs[0][0] best_genotype_combo_prob = genotype_combo_probs[0][1] #best_genotype_probability = math.exp(sum([prob for name, (genotype, prob) in best_genotype_combo]) \ # + allele_frequency_probabilityln(count_frequencies([genotype for name, (genotype, prob) in best_genotype_combo])) \ # - posterior_normalizer) best_genotype_probability = math.exp(best_genotype_combo_prob - posterior_normalizer) position['best_genotype_combo'] = [[ name, genotype_str(genotype), math.exp(marginals[name][genotype_str(genotype)]) ] for name, (genotype, prob) in best_genotype_combo] position['best_genotype_combo_prob'] = best_genotype_probability position['posterior_normalizer'] = math.exp(posterior_normalizer) position['combos_tested'] = combos_tested #position['genotype_combo_probs'] = genotype_combo_probs # TODO estimate marginal probabilities of genotypings # here we cast everything into float-space for samplename, sample in samples.items(): sample['genotypes'] = sorted( [[genotype_str(genotype), math.exp(prob)] for genotype, prob in sample['genotypes']], key=lambda c: c[1], reverse=True) print cjson.encode(position) #print position['position']
cnt = 0 for key, value in bb.iteritems(): xxx = sorted(value.items(), key=itemgetter(1), reverse=1) prb = sorted(bbb[key].items(), key=itemgetter(1), reverse=1) print >> out1, key, xxx, prb if len(xxx) >= 2 and xxx[0][1] == xxx[1][1] and xxx[0][1] >= 3: #print>>out1,key,xxx,prb eqlist = [] for item in xxx: if xxx[0][1] == item[1]: print bbb[key], item[0] eqlist.append([item[0], bbb[key][item[0]]]) master = sorted(eqlist, key=itemgetter(1), reverse=1)[0] master.append(xxx[0][1]) bb1[key] = master elif len(xxx) == 1 and xxx[0][1] >= 3: bb1[key] = xxx[0] elif len(xxx) >= 2 and xxx[0][1] != xxx[1][1] and xxx[0][1] >= 3: bb1[key] = xxx[0] else: pass if ' ' not in key: cnt += 1 else: print '--', key, xxx print len(bb1) #with phrase print cnt #without phrase outfile = open('backbone-3', 'w') outfile.write(cjson.encode(bb1))
def initconfig(agi, cursor, args): # pylint: disable-msg=W0613 """ Provisioning by code on keypad. """ sip_uri = args[0] code = args[1] ua = args[2] isinalan = 1 # Get Sip User, IPv4 and Mac Address user_ipv4 = user_ipv4_from_sip_uri(sip_uri) if not user_ipv4: send_error(agi, "Could not parse Sip URI \"%s\"" % sip_uri) return sip_user, ipv4 = user_ipv4 # pylint: disable-msg=W0612 macaddr = xivo_config.macaddr_from_ipv4(ipv4) # XXX, agi_session.verbose) if not macaddr: send_error(agi, "Could not find Mac Address from IPv4 \"%s\"" % ipv4) return # Get Phone description (if we are able to handle this vendor...) phone_desc = xivo_config.phone_desc_by_ua(ua) # XXX, agitb.handler) if not phone_desc: send_error(agi, "Unknown UA %r" % (ua, )) return phone_vendor = phone_desc[0] phone_model = phone_desc[1] if code == 'init': code = '0' if not xivo_config.well_formed_provcode(code): send_error(agi, "Badly formed provisioning code", "privacy-incorrect") return code = int(code) command = { 'mode': 'authoritative', 'vendor': phone_vendor, 'model': phone_model, 'macaddr': macaddr, 'ipv4': ipv4, 'provcode': code, 'actions': 'yes', 'proto': 'sip', 'isinalan': isinalan, } try: socket.setdefaulttimeout(float(Pgc['http_request_to_s'])) conn = httplib.HTTPConnection(Pgc['connect_ipv4'] + ':' + str(Pgc['connect_port'])) conn.request("POST", "/provisioning", cjson.encode(command), {"Content-Type": "application/json"}) response = conn.getresponse() response.read() # eat every data sent by the provisioning server conn.close() reason = response.reason status = response.status except Exception, xcept: reason = str(xcept) status = 500 agitb.handler() del xcept sys.exc_clear()
save = True except IOError, e: print ("Failed to load config file - not saving oauth keys: " + str(e)) if save: raw_config = file.read() file.close() # Read config and add new values config = cjson.decode(raw_config) config['key'] = access_token['oauth_token'] config['secret'] = access_token['oauth_token_secret'] raw_config = cjson.encode(config) # Write out the new config file try: file = open(homedir + "/twitter-login.conf",'w') file.write(raw_config) file.close() except IOError, e: print ("Failed to save oauth keys: " + str(e)) self.keypair = [access_token['oauth_token'], access_token['oauth_token_secret']] while not self.finished(): # TODO: Implement backoff algorithm in case of connection failures - watch out for the fact this could delay the requester component if self.dataReady("inbox"):
def get(self): data = mysql.Query() self.write(cjson.encode(data))
import cjson from operator import itemgetter infile = '/spare/wei/user_tag_dict_sift_1' userlist=[] for line in open(infile,'r'): #line=line.split('\t')[1] line=cjson.decode(line) userlist.append(line) aaa=sorted(userlist,key=itemgetter(2),reverse=1) user_tag_coverage=[] user_tag_cnt=[] tags=[] last_user_sum=0 for ls in aaa: user_tag_cnt.append([ls[0],len(ls[1])]) num_new=0 for key,value in ls[1].iteritems(): if key not in tags: num_new+=1 tags.append(key) user_tag_coverage.append([ls[0],num_new,num_new+last_user_sum]) last_user_sum=num_new+last_user_sum outfile=open('/spare/wei/user_tag_coverage','w') outfile.write(cjson.encode(user_tag_coverage)+'\n') outfile.write(cjson.encode(user_tag_cnt)+'\n')
def __str__(self): return cjson.encode(self.attributes)
def get(self): user = self.get_argument("userid", strip=True) data = mysql.Myquery(user) self.write(cjson.encode(data))