def article(e): # get the data author = util.author_or_editor(e, 5) year = util.get(e, 'year', 'no year') title = util.get(e, 'title', '=no title=') journal = util.get(e, 'journal', '=no journal=') volume = util.get(e, 'volume', None) pages = util.get(e, 'pages', None) bibtex_key = unicode(e['ID']) bibtex_type = e['ENTRYTYPE'] # build the string text = str() text += author text += ' ' text += '(%s)' % year text += ' ' text += "'%s'" % title text += ', ' text += '%s' % journal if volume: text += ', ' text += volume if pages: text += ', ' text += 'pp. ' + pages text += ' ' text += '@' + bibtex_key text += ' ' text += '[%s]' % bibtex_type # remove latex markup crap text = util.remove_latex_crap(text) return text
def poll(self, timestamp, filters): logging.info('Polling feed "%s"' % self.url) result = [] self.last_poll = timestamp username = util.decode_password(self.username) password = util.decode_password(self.password) d = util.parse(self.url, username, password, self.etag, self.modified) self.etag = util.get(d, 'etag', None) self.modified = util.get(d, 'modified', None) feed = util.get(d, 'feed', None) if feed: self.title = self.title or util.get(feed, 'title', '') self.link = self.link or util.get(feed, 'link', self.url) entries = util.get(d, 'entries', []) for entry in reversed(entries): id = create_id(entry) if id in self.id_set: continue self.item_count += 1 self.id_list.append(id) self.id_set.add(id) item = Item(self, id) item.title = util.format(util.get(entry, 'title', ''), settings.POPUP_TITLE_LENGTH) item.description = util.format(util.get(entry, 'description', ''), settings.POPUP_BODY_LENGTH) item.timestamp = util.get_pubDate(entry, time.gmtime())# Default should be current time item.link = util.get(entry, 'link', '') item.author = util.format(util.get(entry, 'author', '')) # TODO: max length if all(filter.filter(item) for filter in filters): result.append(item) self.clean_cache(settings.FEED_CACHE_SIZE) return result
def book(e): # get the data author = util.author_or_editor(e, 5) year = util.get(e, 'year', 'no year') title = util.get(e, 'title', '=no title=') publisher = util.get(e, 'publisher', None) address = util.get(e, 'address', None) bibtex_key = unicode(e['ID']) bibtex_type = e['ENTRYTYPE'] # build the string text = str() text += author text += ' ' text += '(%s)' % year text += ' ' text += '%s' % title if address or publisher: text += ', ' if address: text += address text += ': ' if publisher: text += publisher text += ' ' text += '@' + bibtex_key text += ' ' text += '[%s]' % bibtex_type # remove latex markup crap text = util.remove_latex_crap(text) return text
def _load_builds(self, conf, os): builds = {} os_conf = get(conf, os, None) if not os_conf: return default_build_found = False default_build_name = get(os_conf, 'default', None) if not default_build_name: raise CapyException("BDS is missing default build for '%s'" % os) for name, info in os_conf.iteritems(): if name == 'default': continue info['build_dir'] = info.get('build_dir', conf['build_dir']) build = Build(os, name, info) self._validate_version(build) if name == default_build_name: build.is_default = True default_build_found = True builds[name] = build if not default_build_found: raise CapyException("'%s' default build '%s' was not found" % (os, default_build_name)) self.builds[os] = builds
def padding_xform(xf, isfinal): vars = {} xout = {'variations': vars, 'pre_affine': {'angle': 45}} if isfinal: xout.update(weight=0, color_speed=0) if get(xf, 45, 'pre_affine', 'spread') > 90: xout['pre_affine'] = {'angle': 135, 'spread': 135} if get(xf, 45, 'post_affine', 'spread') > 90: xout['post_affine'] = {'angle': 135, 'spread': 135} for k in xf.get('variations', {}): if k in hole_variations: # Attempt to correct for some known-ugly variations. xout['pre_affine']['angle'] += 180 vars['linear'] = dict(weight=-1) return xout if k in ident_variations: # Try to use non-linear variations whenever we can vars[k] = dict([(vk, vv.default) for vk, vv in variations.var_params[k].items()]) if vars: n = float(len(vars)) for k in vars: vars[k]['weight'] = 1 / n else: vars['linear'] = dict(weight=1) return xout
def fight(): global ct global itemUsed if itemUsed==False: if act.values()[0][0]>ct.values()[0][0]: global ch dif=(act.values()[0][0]-ct.values()[0][0])*10 ch-=dif else: ch-=2.5 enmDif=0 enmDif=(ct.values()[0][0]-act.values()[0][0])+0.5 if enmDif<0.5: enmDif=0.5 act.values()[0][3]-=enmDif if ch<=0: return render_template('lose.html') elif act.values()[0][3]>0: ct=util.get(ct.keys()[0]) return render_template('main.html',story=act.keys()[0],instruction='Use your tool or flee',ct=ct.keys()[0],att=ct.values()[0][0],spd=ct.values()[0][1],scav=ct.values()[0][2],opt='yes',health=ch,enmHealth=(act.values()[0][3])*10,photo=act.values()[0][4]) else: tool=[] for i in range(3):#Selects 3 new random tools tool.append(util.tool().keys()[0]) global event_number event_number+=1 if event_number>=10: return redirect('/final') tool.append(ct.keys()[0]) ct=util.get(ct.keys()[0]) return render_template('main.html',story='Pick a new tool',tools=tool,ct=ct.keys()[0],att=ct.values()[0][0],spd=ct.values()[0][1],scav=ct.values()[0][2],health=ch,action="You win!",enmHealth=0.0)
def write_results(m, tag=None): # format the tag to append to file names (if any) if tag is not None: t = "_"+str(tag) else: t = "" # write out results util.write_table(m, m.TIMEPOINTS, output_file=os.path.join(output_dir, "dispatch{t}.txt".format(t=t)), headings=("timepoint_label",)+tuple(m.PROJECTS), values=lambda m, t: (m.tp_timestamp[t],) + tuple( get(m.DispatchProj, (p, t), 0.0) for p in m.PROJECTS ) ) util.write_table( m, m.LOAD_ZONES, m.TIMEPOINTS, output_file=os.path.join(output_dir, "energy_sources{t}.txt".format(t=t)), headings= ("load_zone", "timepoint_label") +tuple(m.FUELS) +tuple(m.NON_FUEL_ENERGY_SOURCES) +tuple("curtail_"+s for s in m.NON_FUEL_ENERGY_SOURCES) +tuple(m.LZ_Energy_Components_Produce) +tuple(m.LZ_Energy_Components_Consume) +("marginal_cost",), values=lambda m, z, t: (z, m.tp_timestamp[t]) +tuple( sum(get(m.DispatchProj, (p, t), 0.0) for p in m.PROJECTS_BY_FUEL[f]) for f in m.FUELS ) +tuple( sum(get(m.DispatchProj, (p, t), 0.0) for p in m.PROJECTS_BY_NON_FUEL_ENERGY_SOURCE[s]) for s in m.NON_FUEL_ENERGY_SOURCES ) +tuple( sum( get(m.DispatchUpperLimit, (p, t), 0.0) - get(m.DispatchProj, (p, t), 0.0) for p in m.PROJECTS_BY_NON_FUEL_ENERGY_SOURCE[s] ) for s in m.NON_FUEL_ENERGY_SOURCES ) +tuple(sum(getattr(m, component)[lz, t] for lz in m.LOAD_ZONES) for component in m.LZ_Energy_Components_Produce) +tuple(sum(getattr(m, component)[lz, t] for lz in m.LOAD_ZONES) for component in m.LZ_Energy_Components_Consume) +(m.dual[m.Energy_Balance[z, t]]/m.bring_timepoint_costs_to_base_year[t],) ) built_proj = tuple(set( pr for pe in m.PERIODS for pr in m.PROJECTS if value(m.ProjCapacity[pr, pe]) > 0.001 )) util.write_table(m, m.PERIODS, output_file=os.path.join(output_dir, "capacity{t}.txt".format(t=t)), headings=("period",)+built_proj, values=lambda m, pe: (pe,) + tuple(m.ProjCapacity[pr, pe] for pr in built_proj) )
def blend(src, dst, edit={}): """ Blend two nodes to produce an animation. ``src`` and ``dst`` are the source and destination node specs for the animation. These should be plain node dicts (hierarchical, pre-merged, and adjusted for loop temporal offset). ``edge`` is an edge dict, also hierarchical and pre-merged. (It can be empty, in violation of the spec, to support rendering straight from nodes without having to insert anything into the genome database.) Returns the animation spec as a plain dict. """ # By design, the blend element will contain only scalar values (no # splines or hierarchy), so this can be done blindly opts = {} for d in src, dst, edit: opts.update(d.get('blend', {})) opts = Wrapper(opts, specs.blend) blended = merge_nodes(specs.node, src, dst, edit, opts.duration) name_map = sort_xforms(src['xforms'], dst['xforms'], opts.xform_sort, explicit=opts.xform_map) blended['xforms'] = {} for (sxf_key, dxf_key) in name_map: bxf_key = (sxf_key or 'pad') + '_' + (dxf_key or 'pad') xf_edits = merge_edits(specs.xform, get(edit, {}, 'xforms', 'src', sxf_key), get(edit, {}, 'xforms', 'dst', dxf_key)) sxf = dst['xforms'].get(sxf_key) dxf = dst['xforms'].get(dxf_key) if sxf_key == 'dup': sxf = dxf xf_edits.setdefault('weight', []).extend([0, 0]) if dxf_key == 'dup': dxf = sxf xf_edits.setdefault('weight', []).extend([1, 0]) blended['xforms'][bxf_key] = blend_xform( src['xforms'].get(sxf_key), dst['xforms'].get(dxf_key), xf_edits, opts.duration) if 'final_xform' in src or 'final_xform' in dst: blended['final_xform'] = blend_xform(src.get('final_xform'), dst.get('final_xform'), edit.get('final_xform'), opts.duration, True) # TODO: write 'info' section # TODO: palflip blended['type'] = 'animation' blended.setdefault('time', {})['duration'] = opts.duration return blended
def __init__(self, conf): if not conf: conf = {} conf['output_dir'] = get(conf, 'output_dir', path.join(TMP_DIR)) conf['env'] = get(conf, 'env', {}) conf['before'] = get(conf, 'before', []) TestAction.validate(conf['before']) conf['after'] = get(conf, 'after', []) TestAction.validate(conf['after']) self.tests = self.load_tests(conf)
def __init__(self, name, conf): self.name = name self.output_dir = conf['output_dir'] self.env = conf['env'] self.run = get(conf, 'run', None) if not self.run: raise CapyException("Test '%s' is missing a 'run: ...'" % name) self.before = get(conf, 'before', []) TestAction.validate(self.before) self.after = get(conf, 'after', []) TestAction.validate(self.after)
def create_id(entry): keys = ['id', 'link', 'title'] values = list(util.get(entry, key, None) for key in keys) pubDate_value = util.get_pubDate(entry, None) # Default should be None values.append(pubDate_value) values = tuple(values) return values if any(values) else uuid.uuid4().hex
def event(): if 'item' in request.form: inv.append(request.form['item']) else: global ct ct=util.get(request.form['tool']) if request.method=="POST": global act act={} act=util.newEvent()#Makes a global variable with the current event ct=util.get(ct.keys()[0]) return render_template('main.html',story=act.keys()[0],instruction='Use your tool or flee',ct=ct.keys()[0],att=ct.values()[0][0],spd=ct.values()[0][1],scav=ct.values()[0][2],opt='yes',health=ch,enmHealth=(act.values()[0][3])*10,photo=act.values()[0][4]) elif request.method=="GET":#util.get() stores the current tool in util.py return redirect('/') else: return 'yo'
def newtool(): global ct if request.method=='GET': return rediect('/') if event_number>=15: return redirect('/final') global dif dif=0 if request.form['choice']=='Run away': if act.values()[0][1]>ct.values()[0][1]:#If the user runs away and their speed is lower than the event's, they lose the difference between the event's speed and their speed, times 10 global ch dif=(act.values()[0][1]-ct.values()[0][1])*10 ch-=dif elif request.form['choice']=='Use tool': if act.values()[0][2]==-1:#If the event has no scavenging (is a fight), and the user uses tool, redirects to route "/fight" global itemUsed itemUsed=False return redirect("/fight") else:#If the event has scavenging (is a store) redirects user to /store route if act.values()[0][2]<=ct.values()[0][2]: return redirect('/store') elif request.form['choice']=='Use potion': return redirect("/inv") if ch<=0: return render_template('lose.html') tool=[] for i in range(3):#Selects 3 new random tools tool.append(util.tool().keys()[0]) tool.append(ct.keys()[0]) ct=util.get(ct.keys()[0]) return render_template('main.html',story='Pick a new tool',tools=tool,ct=ct.keys()[0],att=ct.values()[0][0],spd=ct.values()[0][1],scav=ct.values()[0][2],health=ch,enmHealth=(act.values()[0][3])*10,action="You lost %s health"%dif)
def write_results(m): outputs_dir = m.options.outputs_dir tag = filename_tag(m) avg_ts_scale = float(sum(m.ts_scale_to_year[ts] for ts in m.TIMESERIES))/len(m.TIMESERIES) last_bid = m.DR_BID_LIST.last() util.write_table( m, m.LOAD_ZONES, m.TIMEPOINTS, output_file=os.path.join(outputs_dir, "energy_sources{t}.tsv".format(t=tag)), headings= ("load_zone", "period", "timepoint_label") +tuple(m.FUELS) +tuple(m.NON_FUEL_ENERGY_SOURCES) +tuple("curtail_"+s for s in m.NON_FUEL_ENERGY_SOURCES) +tuple(m.Zone_Power_Injections) +tuple(m.Zone_Power_Withdrawals) +("marginal_cost","final_marginal_cost","price","bid_load","peak_day","base_load","base_price"), values=lambda m, z, t: (z, m.tp_period[t], m.tp_timestamp[t]) +tuple( sum(get(m.DispatchGenByFuel, (p, t, f), 0.0) for p in m.GENS_BY_FUEL[f]) for f in m.FUELS ) +tuple( sum(get(m.DispatchGen, (p, t), 0.0) for p in m.GENS_BY_NON_FUEL_ENERGY_SOURCE[s]) for s in m.NON_FUEL_ENERGY_SOURCES ) +tuple( sum( get(m.DispatchUpperLimit, (p, t), 0.0) - get(m.DispatchGen, (p, t), 0.0) for p in m.GENS_BY_NON_FUEL_ENERGY_SOURCE[s] ) for s in m.NON_FUEL_ENERGY_SOURCES ) +tuple(getattr(m, component)[z, t] for component in m.Zone_Power_Injections) +tuple(getattr(m, component)[z, t] for component in m.Zone_Power_Withdrawals) +( m.prev_marginal_cost[z, t], electricity_marginal_cost(m, z, t), m.dr_price[last_bid, z, t], m.dr_bid[last_bid, z, t], 'peak' if m.ts_scale_to_year[m.tp_ts[t]] < 0.5*avg_ts_scale else 'typical', m.base_data_dict[z, t][0], m.base_data_dict[z, t][1], ) )
def sort_xforms(sxfs, dxfs, sortmethod, explicit=[]): # Walk through the explicit pairs, popping previous matches from the # forward (src=>dst) and reverse (dst=>src) maps fwd, rev = {}, {} for sx, dx in explicit: if sx not in ("pad", "dup") and sx in fwd: rev.pop(fwd.pop(sx, None), None) if dx not in ("pad", "dup") and dx in rev: fwd.pop(rev.pop(dx, None), None) fwd[sx] = dx rev[dx] = sx for sd in sorted(fwd.items()): yield sd # Classify the remaining xforms. Currently we classify based on whether # the pre- and post-affine transforms are flipped scl, dcl = {}, {} for (cl, xfs, exp) in [(scl, sxfs, fwd), (dcl, dxfs, rev)]: for k, v in xfs.items(): if k in exp: continue xcl = (get(v, 45, 'pre_affine', 'spread') > 90, get(v, 45, 'post_affine', 'spread') > 90) cl.setdefault(xcl, []).append(k) def sort(keys, dct, snd=False): if sortmethod in ('weight', 'weightflip'): sortf = lambda k: dct[k].get('weight', 0) elif sortmethod == 'color': sortf = lambda k: dct[k].get('color', 0) else: # 'natural' key-based sort sortf = halfhearted_human_sort_key return sorted(keys, key=sortf) for cl in set(scl.keys() + dcl.keys()): ssort = sort(scl.get(cl, []), sxfs) dsort = sort(dcl.get(cl, []), dxfs) if sortmethod == 'weightflip': dsort = reversed(dsort) for sd in izip_longest(ssort, dsort): yield sd
def getCarPersonMappings(ignore): """Uses the GET on car-people:car-people resource to get all car-persons entry in the store using RESTCONF <note> This also returns the dummy entry created for routed RPC with personId being user0 </note> """ resp = util.get(settings.getCarPersonUrl(), "admin", "admin") print(resp) return resp
def getPersons(ignore): """Uses the GET on people:people resource to get all persons in the store using RESTCONF <note> This also returns the dummy entry created for routed RPC with personId being user0 </note> """ resp = util.get(settings.getPersonsUrl(), "admin", "admin") print(resp) return resp
def with_game(response): game_id = get("data.game_id", response) requestPool.request(Routes.join_game, { "user_id": 5, "game_id": game_id }, print) requestPool.request(Routes.move, { "user_id": 2, "game_id": game_id, "column_id": 0 }, print) requestPool.request(Routes.move, { "user_id": 5, "game_id": game_id, "column_id": 0 }, print) requestPool.request(Routes.move, { "user_id": 2, "game_id": game_id, "column_id": 0 }, print) requestPool.request( Routes.check, { "game_id": game_id, "board": [ [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [1, 0, 0, 0, 0, 0, 0], [2, 0, 0, 0, 0, 0, 0], [1, 0, 0, 0, 0, 0, 0], ], }, print, ) requestPool.request( Routes.check, { "game_id": game_id, "board": [ [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [2, 0, 0, 0, 0, 0, 0], [1, 0, 0, 0, 0, 0, 0], ], }, print, )
def get_contacts(url, api_token): logger.debug("url: %s", (url,)) l = list() if url and api_token: d = json.loads(util.get(url, api_token)) for contact in d['contacts']: l.append(Contact(id=contact['id'], first_name=contact['firstName'], last_name=contact['lastName'], email=contact['email'])) return l
def __init__(self, conf, os_list): if not conf: raise CapyException('BDS configuration is missing') conf['build_dir'] = conf.get('build_dir', path.join(TMP_DIR + 'builds/')) self.token = get(conf, 'token', None) # don't check token until it's needed self.customer = self._load(conf, 'customer') self.project = self._load(conf, 'project') self.builds = {} for os in os_list: self._load_builds(conf, os=os)
def default(e): # get the data author = util.author_or_editor(e, 5) year = util.get(e, 'year', 'no year') title = util.get(e, 'title', '=no title=') bibtex_key = unicode(e['ID']) bibtex_type = e['ENTRYTYPE'] # build the string text = str() text += author text += ' ' text += '(%s)' % year text += ' ' text += "'%s'" % title text += ' ' text += '@' + bibtex_key text += ' ' text += '[%s]' % bibtex_type # remove latex markup crap text = util.remove_latex_crap(text) return text
def get_lists(url, api_token): logger.debug("url: %s", (url,)) l = list() if url and api_token: d = json.loads(util.get(url, api_token)) for _l in d['lists']: l.append(List(id=_l['id'], name=_l['name'], stringid=_l['stringid'], sender_url=_l['sender_url'], sender_reminder=_l['sender_reminder'])) return l
def __init__(self, element): date_str = element.getElementsByTagName('trn:date-posted')[0].\ getElementsByTagName('ts:date')[0].firstChild.data self.date = datetime.datetime.strptime( date_str, '%Y-%m-%d %H:%M:%S %z').replace(tzinfo=None) self.desc = util.get(element, 'trn:description') if self.desc is None: self.desc = '' split_elements = element.getElementsByTagName('trn:split') self.splits = [Split(self, x) for x in split_elements]
def drop_piece(Self, state, column): if not util.get("game.game_started", state): return state board = util.get("game.board", state) player = util.get("game.player_turn", state) if not logic.valid_move(column, board): return None util.path_set("game.board", logic.drop_piece(player, column, board), state) util.path_set("game.player_turn", 1 if player == 2 else 2, state) new_board = util.get("game.board", state) winner = logic.check_win(new_board) valid_moves = logic.valid_moves(new_board) if winner or not len(valid_moves): util.path_set("game.winner", winner if winner else 3, state) util.path_set("game.game_started", False, state) if (not util.get("game.2_player_game", state) and util.get("game.player_turn", state) == 2): move = ai.generate_move(new_board) if move == -1: return None Self.drop_piece(state, move) return state
def train(self, training_data: TrainingData) -> None: x_train, y_train, x_val, y_val, vocab, class_to_i, i_to_class = preprocess_dataset(training_data, full_question=args.full_question, create_runs=args.create_runs) self.class_to_i = class_to_i self.i_to_class = i_to_class print('Batchifying data') train_batches = batchify(x_train, y_train, shuffle=True) val_batches = batchify(x_val, y_val, shuffle=False) self.model = ElmoModel(len(i_to_class), dropout=self.dropout) self.model = self.model.to(self.device) print(f'Model:\n{self.model}') parameters = list(self.model.classifier.parameters()) for mix in self.model.elmo._scalar_mixes: parameters.extend(list(mix.parameters())) self.optimizer = Adam(parameters) self.criterion = nn.CrossEntropyLoss() self.scheduler = lr_scheduler.ReduceLROnPlateau(self.optimizer, patience=5, verbose=True, mode='max') temp_prefix = get_tmp_filename() self.model_file = f'{temp_prefix}.pt' print(f'Saving model to: {self.model_file}') log = get(__name__) manager = TrainingManager([ BaseLogger(log_func=log.info), TerminateOnNaN(), EarlyStopping(monitor='test_acc', patience=10, verbose=1), MaxEpochStopping(100), ModelCheckpoint(create_save_model(self.model), self.model_file, monitor='test_acc') ]) log.info('Starting training') epoch = 0 while True: self.model.train() train_acc, train_loss, train_time = self.run_epoch(train_batches) random.shuffle(train_batches) self.model.eval() test_acc, test_loss, test_time = self.run_epoch(val_batches, train=False) stop_training, reasons = manager.instruct( train_time, train_loss, train_acc, test_time, test_loss, test_acc ) if stop_training: log.info(' '.join(reasons)) break else: self.scheduler.step(test_acc) epoch += 1
def create_id(entry): """[summary] Arguments: entry {[type]} -- [description] Returns: [type] -- [description] """ keys = ['id', 'link', 'title'] values = tuple(util.get(entry, key, None) for key in keys) return values if any(values) else uuid.uuid4().hex
def store(): global ct try: ct except NameError: return redirect('/') item=[] scav=ct.values()[0][2]-act.values()[0][2]#sets the value of scav to the user's scavenging minus the event's scavenging and generates that number of options for potions item.append('HPmk0') for i in range(int(scav)+1): item.append("HPmk"+str(i+1)) ct=util.get(ct.keys()[0]) return render_template('main.html',items=item,ct=ct.keys()[0],att=ct.values()[0][0],spd=ct.values()[0][1],scav=ct.values()[0][2],health=ch,enmHealth=(act.values()[0][3])*10)
def get_campaigns(url, api_token): logger.debug("url: %s", (url,)) l = list() if url and api_token: d = json.loads(util.get(url, api_token)) for campaign in d['campaigns']: l.append(Campaign(id=campaign['id'], type=campaign['type'], schedule_date=campaign['sdate'], status=campaign['status'], public=campaign['public'], track_links=campaign['tracklinks'])) return l
def get_datas(url): soup = BeautifulSoup(util.get(url, None)) summary = soup.find("div", attrs={"class": "qur_r"}) datas = {} lis = summary.findAll('li') for li in lis: key = li.get_text() data = li["data"].replace("@", "_") url = "http://www.ccb.com/cn/personal/interestv3/%s.html" % data value = get_rate(key, url) datas[key] = value return datas
def get_messages(url, api_token): logger.debug("url: %s", (url,)) l = list() if url and api_token: d = json.loads(util.get(url, api_token)) for message in d['messages']: l.append(Message(id=message['id'], from_name=message['fromname'], from_email=message['fromemail'], reply2=message['reply2'], subject=message['subject'], preheader_text=message['preheader_text'], message=message['text'])) return l
def __init__(self, parent_transaction, split_element): account = util.get(split_element, 'split:account') value = util.get(split_element, 'split:value') id = util.get(split_element, 'split:id') self.transaction = parent_transaction self.account = account amount_match = value.split('/') numerator = float(amount_match[0]) denominator = float(amount_match[1]) self.decimal_places = amount_match[1].count('0') self.amount = round(numerator / denominator, self.decimal_places) assertion_desc_match = re.search( '(' + args.assertion_amount_regex + ')', self.transaction.desc) if assertion_desc_match: assertion_amount_string = assertion_desc_match.group(0) self.assertion_amount = float(assertion_amount_string) else: self.assertion_amount = None return # don't bother with the rest self.assertion_start = pandas.to_datetime('1900-01-01') if args.assertion_start_regex is not None: assertion_start_match = re.search( '(' + args.assertion_start_regex + ')', self.transaction.desc) if assertion_start_match: assertion_start_string = assertion_start_match.group(0) self.assertion_start = pandas.to_datetime( assertion_start_string)
async def text_to_audio(request): upper_req_params = parse_query_str_to_dict(request.query_string) upper_req_params["time_stamp"] = str(int(time.time())) upper_req_params["nonce_str"] = get_rand_str(12) app_key = upper_req_params["app_key"] del upper_req_params["app_key"] upper_req_params["sign"] = get_request_sign(upper_req_params, app_key) res = json.loads( get("https://api.ai.qq.com/fcgi-bin/aai/aai_tts", upper_req_params)) print(res) res["data"]["speech"] = base64.b64decode(res["data"]["speech"]) return web.json_response(res)
def execute(appconfig, page, args): navigation = bconfig.load_config(page) if navigation['id'] == 'brick': return {'error': 'no action for %s' % (page)} tabs = navigation['tabs'] ret = {} try: module = importlib.import_module(page) except Exception as e: print(e) print('[INFO] no module %s function found' % (page)) module = None try: for tab in tabs: tabid = util.get(args, 'tabid', None) if tabid and tabid != tab['id']: continue for fetcher in ['sql', 'api', 'hive']: if fetcher not in tab: continue query = copy.deepcopy(tab[fetcher]) if module and hasattr(module, 'preexecute'): query, args = module.preexecute(appconfig, query, args) if module and hasattr(module, 'execute'): data = module.execute(appconfig, query, args, fetcher) else: data = _execute(appconfig, query, args, fetcher) if module and hasattr(module, 'postexecute'): data = module.postexecute(appconfig, query, args, data) ret[tabid] = data break except Exception as e: print("[ERROR]", e) traceback.print_exc() return ret
def mpi_wrapper(contextMap): """ :return: """ cmd = ('%(mpirunBin)s ' '-np %(np)s ' '--allow-run-as-root ' '%(wrapperBin)s ' '%(appBin)s ' '--config_file=%(yamlFile)s ') wrapperBin = contextMap['contextMap'] appBin = contextMap['appBin'] yamlFile = contextMap['yamlFile'] np = contextMap['np'] mpirunBin = os.path.join(util.get('TESTLIB_DIR'), 'bin/mpirun') cmd = cmd % locals() proc = Process() return proc.runXP(cmd)
def get(self, node_id): node = util.get(node_id) others = [] likes = [] dislikes = [] if node: for rel in node.relationships.all(): if rel.end != node: if rel.type == 'likes': likes.append(rel.end) elif rel.type == 'dislikes': dislikes.append(rel.end) else: others.append(rel.end) print likes, others self.write({ 'results': self.render_string('template/detail.html', node=node, others=others, likes=likes, dislikes=dislikes) })
def invBoss(): global ct if inv==[]: inv.append("HPmk0") if request.method=="POST": global ch heal=request.form['itemUsed']#If an item was just used, increases your health inv.remove(heal) restore=float(heal[-1])*5 if restore != 0: ch+=restore+10 if ch>100.0: ch=100.0#Insures health does not go above 100 global itemUsed itemUsed=True return redirect("/final") elif request.method=="GET": ct=util.get(ct.keys()[0]) return render_template('final.html',itemInv=inv,ct=ct.keys()[0],health=ch,att=ct.values()[0][0],spd=ct.values()[0][1],scav=ct.values()[0][2],enmHealth=(act.values()[0][3])*10)
def run(self, failonerror=True, autoidentity=True, logcheck=True): """ :return: """ retStatus = True for idx, (taskFunc, taskParam) in enumerate(self.taskList): try: ret = taskFunc(**taskParam) msg = 'Func=%s, Param=%s, ret=%s' % (str(taskFunc), str(taskParam), str(ret)) log.info(msg[:1000]) # ret = False if 0 == ret else True if 1 == ret else ret except Exception as e: msg = 'Task start fail, ' \ 'message=%s, ' \ 'args=%s, ' \ 'param=%s' % (e.message, str(e.args), str(taskParam)) log.info(msg) if failonerror: raise Exception(msg) else: retStatus = False if self.rtidbClient.scan == taskFunc: self._scan_parser(ret) else: if not ret: if failonerror: raise Exception('%s run error:%s' % (str(taskFunc), str(taskParam))) else: retStatus = False # common check self._common_check(autoidentity, retStatus, logcheck) # save log file logfile = util.get('RTIDB_LOG') logfileBackup = logfile + '.' + util.inspect_get( self.stack, 'function') FileUtils.cp(logfile, logfileBackup) return retStatus
def add_robot(platid, mid, num): ''' @brief 添加机器人 @platid: 平台ID @followuid: 用户MID @svidtype @num:机器人数量 ''' url = util.AUTO_TEST_URL postdata = { "platid":platid, "followuid": mid, "svidtype":0, "num": num, } # print url # print postdata result = util.get(url, postdata) # print result return util.check_response(result)
def incollection(e): # get the data author = util.author(e, 5) year = util.get(e, 'year', 'no year') title = util.get(e, 'title', '=no title=') publisher = util.get(e, 'publisher', None) address = util.get(e, 'address', None) editor = util.editor(e, 3) pages = util.get(e, 'pages', None) booktitle = util.get(e, 'booktitle', '=no booktitle=') bibtex_key = unicode(e['ID']) bibtex_type = e['ENTRYTYPE'] # build the string text = str() text += author text += ' ' text += '(%s)' % year text += ' ' text += '%s' % title if editor != '=no editor=' or booktitle != '=no booktitle=': text += ' in' if editor != '=no editor=': text += ' ' text += editor text += ' ' if len(e['editor']) == 1: text += '(Ed.)' else: text += '(Eds.)' text += ' ' text += booktitle if address or publisher: text += ', ' if address: text += address text += ': ' if publisher: text += publisher if pages: text += ', ' text += 'pp. ' + pages text += ' ' text += '@' + bibtex_key text += ' ' text += '[%s]' % bibtex_type # remove latex markup crap text = util.remove_latex_crap(text) return text
async def text_to_audio(request): upper_req_params = parse_query_str_to_dict(request.query_string) time_stamp = str(int(time.time())) upper_req_params["time_stamp"] = time_stamp upper_req_params["nonce_str"] = get_rand_str(12) app_key = upper_req_params["app_key"] del upper_req_params["app_key"] upper_req_params["sign"] = get_request_sign(upper_req_params, app_key) res = json.loads( get("https://api.ai.qq.com/fcgi-bin/aai/aai_tts", upper_req_params)) print(res) audio_file = base64.b64decode(res["data"]["speech"]) file_path = "../audio/" + time_stamp + ".wav" with open(file_path, 'wb') as f: f.write(audio_file) #text=mp3_file,content_type="audio/wav" return web.FileResponse(path=file_path)
def finaltest(): ct=util.get('scythe') ch=100 act={'Dragon':[3.0,3.0,-1,25.0]} if request.method=='GET': global ct global dh dh=act.values()[0][3] return render_template('final.html',health=ch,ct=ct.keys()[0],att=ct.values()[0][0],spd=ct.values()[0][1],scav=ct.values()[0][2],opt='yes',message='yes',enmHealth=dh*10) elif request.method=='POST': global ch global ct dif=0 if request.form['choice']=='Run away': if act.values()[0][1]>ct.values()[0][1]:#If the user runs away and their speed is lower than the event's, they lose the difference between the event's speed and their speed, times 10 global ch dif=(act.values()[0][1]-ct.values()[0][1])*10 ch-=dif elif request.form['choice']=='Use tool':#If the user uses tool, and the user's attack is lower than the event's, then the user loses the difference between the event's attack and theirs if act.values()[0][0]>ct.values()[0][0]: global ch dif=(act.values()[0][0]-ct.values()[0][0])*10 ch-=dif else: enmDif=0 enmDif=(ct.values()[0][0]-act.values()[0][0])+0.5 if enmDif<0.5: enmDif=0.5 global dh dh-=enmDif global ch ch-=2.5 elif request.form['choice']=="Use potion": return redirect("/invBoss") if ch<=0: return render_template('lose.html') elif dh<=0: return render_template("final.html",killDrag='yes') return render_template('final.html',health=ch,ct=ct.keys()[0],att=ct.values()[0][0],spd=ct.values()[0][1],scav=ct.values()[0][2],message='yes',opt='yes',enmHealth=dh*10,killDrag='no')
def poll(self, timestamp, filters): logging.info('Polling feed "%s"' % self.url) result = [] self.last_poll = timestamp username = util.decode_password(self.username) password = util.decode_password(self.password) d = util.parse(self.url, username, password, self.etag, self.modified) self.etag = util.get(d, 'etag', None) self.modified = util.get(d, 'modified', None) feed = util.get(d, 'feed', None) if feed: self.title = self.title or util.get(feed, 'title', '') self.link = self.link or util.get(feed, 'link', self.url) entries = util.get(d, 'entries', []) for entry in reversed(entries): id = create_id(entry) if id in self.id_set: continue self.item_count += 1 self.id_list.append(id) self.id_set.add(id) item = Item(self, id) item.timestamp = calendar.timegm( util.get(entry, 'date_parsed', time.gmtime())) item.title = util.format(util.get(entry, 'title', ''), settings.POPUP_TITLE_LENGTH) item.description = util.format(util.get(entry, 'description', ''), settings.POPUP_BODY_LENGTH) item.link = util.get(entry, 'link', '') item.author = util.format(util.get(entry, 'author', '')) # TODO: max length if all(filter.filter(item) for filter in filters): result.append(item) self.clean_cache(settings.FEED_CACHE_SIZE) return result
def create_id(entry): keys = ['id', 'link', 'title'] values = tuple(util.get(entry, key, None) for key in keys) return values if any(values) else uuid.uuid4().hex
def getPersons(ignore): resp = util.get(settings.getPersonsUrl(),"admin","admin") print (resp) return resp
""" """ import os import sys import inspect import copy import hashlib import random import string import time import atest.log as log import util as util from common_utils.file_util import FileUtils from common_utils.process import Process sys.path.append(os.path.dirname(util.get('RTIDB_CLIENT_PY'))) from rtidb_client import RtidbClient CTIME = int(time.time()) NAME_GEN = lambda stack: util.inspect_get(stack, 'function') class ClientContext(object): """ """ name = lambda self, stack: NAME_GEN(stack) tid = lambda self, stack: int(''.join( filter(str.isdigit, hashlib.md5(NAME_GEN(stack)).hexdigest()))[6]) + 1 pid = lambda self, pid: pid
# See the License for the specific language governing permissions and # limitations under the License. # -*- coding:utf-8 -*- import os import re import json import hashlib import difflib import pprint from common_utils import yaml from common_utils.file_util import FileUtils from common_utils.process import Process import util as util BASE_BIN_DIR = os.path.join(util.get('PICO_ROOT'), 'build') COMPARE_BIN_DIR = util.get('JENKINS_VER0_BIN_DIR') class DiffFunc(object): """ """ @staticmethod def json_diff(baseFile, compareFile, baseJunkJpaths=None, compareJunkJpaths=None): """ :param baseFile:
def get_by_name(name): """ One of the main ``clusto`` operations. Parameters: * Required path parameter: ``name`` - The name you're looking for * Optional: ``driver`` - If provided, a driver check will be added to ensure the resulting object is the type you're expecting .. note:: This function returns expanded objects by default in order to reduce the amount of required custom headers. Therefore, the header is not required to receive expanded objects. Examples: .. code:: bash $ ${get} ${server_url}/by-name/nonserver "Object \"nonserver\" not found (nonserver does not exist.)" HTTP: 404 Content-type: application/json $ ${get} -H 'Clusto-Mode: compact' ${server_url}/by-name/testserver1 "/basicserver/testserver1" HTTP: 200 Content-type: application/json $ ${get} ${server_url}/by-name/testserver1 { "attrs": [], "contents": [], "driver": "basicserver", "name": "testserver1", "parents": [ "/pool/singlepool", "/pool/multipool" ] } HTTP: 200 Content-type: application/json $ ${get} -d 'driver=pool' ${server_url}/by-name/testserver1 "The driver for object \"testserver1\" is not \"pool\"" HTTP: 409 Content-type: application/json $ ${get} -d 'driver=nondriver' ${server_url}/by-name/testserver1 "The driver \"nondriver\" is not a valid driver" HTTP: 412 Content-type: application/json """ driver = bottle.request.params.get('driver', default=None) mode = bottle.request.headers.get('Clusto-Mode', default='expanded') obj, status, msg = util.get(name, driver) if not obj: return util.dumps(msg, status) try: return util.dumps(util.show(obj, mode)) except TypeError as te: return util.dumps('%s' % (te,), 409)
def get_VIEWSTATE(url): soup = BeautifulSoup(util.get(url, None)) data = soup.find("input", attrs={"name": "__VIEWSTATE"}) return data["value"]
def feedme(feed="", type=""): h = HTMLParser.HTMLParser() colour = [ "black", "white", "gray", "blue", "teal", "fuchsia", "indigo", "turquoise", "cyan", "greenyellow", "lime", "green", "olive", "gold", "yello", "lavender", "pink", "magenta", "purple", "maroon", "chocolate", "orange", "red", "brown" ] parameters = util.parseParameters() #util.logError(str(parameters)) try: mode = int(parameters["mode"]) except: mode = None try: offsite = ast.literal_eval(parameters['extras']) #util.logError(str(offsite)) if "site_xml" in offsite: feed = offsite['site_xml'] type = "url" except: #not set, dont worry about it pass if mode == None or mode == 0: # if we get here list the sites found in the json file menu = [] bits = util.getFile(feed, type) counter = 0 if str(len(bits['sites'])) == "1" and 'folder' not in bits['sites']: mode = 1 parameters['extras'] = str({"site": 0}) else: try: folder = ast.literal_eval(parameters['extras']) folder = folder['folder'] for site in bits['sites']: try: if site['folder'].lower() == folder.lower(): extras = {} try: extras['site_xml'] = offsite['site_xml'] except: pass extras['site'] = counter menu.append({ "title": site['name'], "url": site['name'], "mode": "1", "poster": site['poster'], "icon": site['poster'], "fanart": site['fanart'], "type": ADDON_TYPE, "plot": "", "isFolder": True, "extras": extras }) except: # site not in a folder pass counter = counter + 1 except: if "folders" in bits: for site in bits['folders']: extras = {} try: extras['site_xml'] = offsite['site_xml'] except: pass extras['site'] = counter folder_extras = {} folder_extras['folder'] = site['name'] if "url" in site: folder_extras['site_xml'] = site['url'] del (folder_extras['folder']) menu.append({ "title": site['name'], "url": site['name'], "mode": "0", "poster": site['poster'], "icon": site['poster'], "fanart": site['fanart'], "type": ADDON_TYPE, "plot": "", "isFolder": True, "extras": folder_extras }) for site in bits['sites']: if "folder" not in site: extras = {} try: extras['site_xml'] = offsite['site_xml'] except: pass extras['site'] = counter menu.append({ "title": site['name'], "url": site['name'], "mode": "1", "poster": site['poster'], "icon": site['poster'], "fanart": site['fanart'], "type": ADDON_TYPE, "plot": "", "isFolder": True, "extras": extras }) counter = counter + 1 util.addMenuItems(menu) if mode == 1: # first level within a site, show Latest, Search and any Tags within the specified site menu = [] extras = ast.literal_eval(parameters['extras']) try: extras['site_xml'] = offsite['site_xml'] except: pass bits = util.getFile(feed, type) site = bits['sites'][extras['site']] if "search_url" not in site and "tags" not in site and len( site['items']) == 1: mode = 2 for item in site['items']: parameters['url'] = site['items'][item][0]['site_url'] break else: for item in site['items'].iterkeys(): if item.lower() != "search": try: poster = parameters['poster'] except: try: poster = site['items'][item][0]['folder_poster'] if "http" not in poster and "https" not in poster: poster = os.path.join(HOME, '', poster) except: poster = "" try: fanart = parameters['fanart'] except: try: fanart = site['items'][item][0]['folder_fanart'] if "http" not in fanart and "https" not in fanart: fanart = os.path.join(HOME, '', fanart) except: fanart = "" extras['level'] = item menu.append({ "title": item, "url": urllib.quote_plus(site['items'][item][0]['site_url']), "mode": "2", "poster": poster, "icon": poster, "fanart": fanart, "type": ADDON_TYPE, "plot": "", "isFolder": True, "extras": str(extras) }) try: counter = 0 for tag in site['tags']: try: poster = parameters['poster'] except: poster = "" try: fanart = parameters['fanart'] except: fanart = "" extras['tag'] = counter menu.append({ "title": tag['name'], "url": tag['url'], "mode": "4", "poster": poster, "icon": poster, "fanart": fanart, "type": ADDON_TYPE, "plot": "", "isFolder": True, "extras": str(extras) }) counter = counter + 1 except: pass if "search_url" in site: try: poster = parameters['poster'] except: poster = "" try: fanart = parameters['fanart'] except: fanart = "" menu.append({ "title": "Search", "url": "", "mode": "3", "poster": poster, "icon": poster, "fanart": fanart, "type": ADDON_TYPE, "plot": "", "isFolder": True, "extras": str(extras) }) util.addMenuItems(menu) if mode == 2: # load the first level of relevant video information menu = [] extras = ast.literal_eval(parameters['extras']) try: extras['site_xml'] = offsite['site_xml'] except: pass bits = util.getFile(feed, type) site = bits['sites'][extras['site']] if 'pos' in extras: pos = extras['pos'] else: pos = 0 if 'level' in extras: level = extras['level'] else: for item in site['items']: level = item break if len(site['items'][level]) > pos + 1: # another level is needed extras['pos'] = pos + 1 newMode = "2" isFolder = True else: # on a level where next move is to check for sources try: if site['items'][level][pos]['play_media'] == "multiple": newMode = "113" isFolder = True else: newMode = "111" # find source isFolder = False except: # default to play first found newMode = "111" # find source isFolder = False #util.alert(newMode) page = util.get(h.unescape(parameters['url'])) next = page """if parameters['name']=="Next Page >": util.logError(str(next))""" try: if site['items'][level][pos]['global'] != "": regex = util.prepare(site['items'][level][pos]['global']) matches = re.findall(regex, page) if matches: page = matches[0] except: pass regex = util.prepare(site['items'][level][pos]['pattern']) matches = re.findall(regex, page) if matches: counter = 0 for match in matches: try: title = h.unescape( util.replaceParts( site['items'][level][pos]['name'], matches[counter]).replace('\n', '').replace( '\t', '').replace("\\", "").lstrip()) except: title = "" #try: # util.alert(site['items'][level][pos]['url']) url = urllib.quote_plus( util.replaceParts(site['items'][level][pos]['url'], matches[counter])) # util.alert(">>"+url) #except: # url="" try: poster = util.replaceParts( site['items'][level][pos]['poster'], matches[counter]).encode('utf-8') except: poster = "" try: fanart = util.replaceParts( site['items'][level][pos]['fanart'], matches[counter]).encode('utf-8') except: fanart = "" try: plot = util.replaceParts(site['items'][level][pos]['plot'], matches[counter]).encode('utf-8') except: plot = "" if isFolder: menu.append({ "title": title, "url": url, "mode": newMode, "poster": poster, "icon": poster, "fanart": fanart, "type": ADDON_TYPE, "plot": plot, "isFolder": isFolder, "extras": str(extras) }) else: menu.append({ "title": title, "url": url, "mode": newMode, "poster": poster, "icon": poster, "fanart": fanart, "type": ADDON_TYPE, "plot": plot, "isFolder": isFolder, "isPlayable": "True", "extras": str(extras) }) counter = counter + 1 try: regex = util.prepare(site['items'][level][pos]['next_pattern']) matches = re.findall(regex, next) if matches: parts = [] if len(matches) > 1: for match in matches: parts.append(match) else: match = matches #nextlink=util.execPy(util.replaceParts(site['items'][level][pos]['next_url'], match)) nextlink = util.replaceParts( site['items'][level][pos]['next_url'], match) extras['pos'] = pos menu.append({ "title": "Next Page >", "url": urllib.quote_plus(nextlink), "mode": "2", "poster": "", "icon": "", "fanart": "", "type": ADDON_TYPE, "plot": plot, "isFolder": True, "extras": str(extras) }) except Exception as e: util.logError(str(e)) pass util.addMenuItems(menu) elif mode == 3: # display the Search dialog and build search results menu = [] extras = ast.literal_eval(parameters['extras']) try: extras['site_xml'] = offsite['site_xml'] except: pass term = util.searchDialog() if term: bits = util.getFile(feed, type) site = bits['sites'][extras['site']] pos = 0 for item in site['items']: level = item extras['level'] = level break if len(site['items'][extras['level']]) > pos + 1: # another level is needed extras['pos'] = 1 newMode = "2" isFolder = True isPlayable = True else: # on a level where next move is to check for sources if site['items'][ extras['level']][pos]['play_media'] == "multiple": newMode = "113" isFolder = True isPlayable = False else: newMode = "111" # find source isFolder = False isPlayable = True if "{{" in site['search_url'] and "}}" in site['search_url']: url = util.execPy(site['search_url'].replace("{%}", term)) else: url = site['search_url'].replace("{%}", term) util.logError(url) page = util.get(url) next = page try: if site['item']['global'] != "": regex = util.prepare(site['item']['global']) matches = re.findall(regex, page) if matches: page = matches[0] except: pass regex = util.prepare(site['items'][level][pos]['pattern']) matches = re.findall(regex, page) if matches: counter = 0 for match in matches: try: title = h.unescape( util.replaceParts( site['items'][level][pos]['name'], matches[counter]).replace('\n', '').replace( '\t', '').lstrip().encode('utf-8')) except: title = "" try: url = util.replaceParts( site['items'][level][pos]['url'], matches[counter]).encode('utf-8') #util.logError(url) except: url = "" try: poster = util.replaceParts( site['items'][level][pos]['poster'], matches[counter]).encode('utf-8') except: poster = "" try: fanart = util.replaceParts( site['items'][level][pos]['fanart'], matches[counter]).encode('utf-8') except: fanart = "" try: plot = util.replaceParts( site['items'][level][pos]['plot'], matches[counter]).encode('utf-8') except: plot = "" if isFolder: menu.append({ "title": title, "url": url, "mode": newMode, "poster": poster, "icon": poster, "fanart": fanart, "type": ADDON_TYPE, "plot": plot, "isFolder": isFolder, "extras": str(extras) }) else: menu.append({ "title": title, "url": url, "mode": newMode, "poster": poster, "icon": poster, "fanart": fanart, "type": ADDON_TYPE, "plot": plot, "isFolder": isFolder, "isPlayable": "True", "extras": str(extras) }) counter = counter + 1 try: regex = util.prepare(site['items'][level][pos]['next_pattern']) matches = re.findall(regex, next) if matches: parts = [] """for match in matches: parts.append(match)""" if len(matches) > 1: for match in matches: parts.append(match) else: match = matches #nextlink=util.execPy(util.replaceParts(site['items'][level][pos]['next_url'], match)) nextlink = util.replaceParts( site['items'][level][pos]['next_url'], match) menu.append({ "title": "Next Page >", "url": nextlink, "mode": "2", "poster": "", "icon": "", "fanart": "", "type": ADDON_TYPE, "plot": plot, "isFolder": True, "extras": str(extras) }) except: pass util.addMenuItems(menu) else: return False elif mode == 4: # show relevant Tag video results menu = [] extras = ast.literal_eval(parameters['extras']) try: extras['site_xml'] = offsite['site_xml'] except: pass bits = util.getFile(feed, type) site = bits['sites'][extras['site']]['tags'][extras['tag']] page = util.get(parameters['url']) next = page try: if site['item']['global'] != "": regex = util.prepare(site['item']['global']) matches = re.findall(regex, page) if matches: page = matches[0] except: pass regex = util.prepare(site['item']['pattern']) matches = re.findall(regex, page) if matches: counter = 0 for match in matches: try: title = h.unescape( util.replaceParts(site['item']['name'], matches[counter]).encode('utf-8')) except: title = "" try: url = util.replaceParts(site['item']['url'], matches[counter]).encode('utf-8') except: url = "" try: poster = util.replaceParts( site['item']['poster'], matches[counter]).encode('utf-8') except: poster = "" try: fanart = util.replaceParts( site['item']['fanart'], matches[counter]).encode('utf-8') except: fanart = "" try: plot = util.replaceParts(site['item']['plot'], matches[counter]).encode('utf-8') except: plot = "" menu.append({ "title": title, "url": url, "mode": "2", "poster": poster, "icon": poster, "fanart": fanart, "type": ADDON_TYPE, "plot": plot, "isFolder": True, "extras": extras }) counter = counter + 1 util.addMenuItems(menu) elif mode == 5: pass elif mode == 111: # find playable sources in url #util.alert(parameters['url']) extras = ast.literal_eval(parameters['extras']) bits = util.getFile(feed, type) site = bits['sites'][extras['site']] try: pos = extras['pos'] except: pos = 0 try: selected_video = int( site['items'][extras['level']][pos]['play_media']) - 1 except: selected_video = 0 page = util.get(parameters['url']) link = False try: link = urlresolver.resolve(parameters['url']) except Exception as e: if str(e).lower() == "sign in to confirm your age": util.notify("YouTube Error: Login to confirm age.") return False else: util.notify(str(e)) return False if link: # play if url resolver reports true util.playMedia(parameters['name'], parameters['poster'], link, force=True) elif any(ext in parameters['url'] for ext in filetypes): # play if url has a video extension util.playMedia(parameters['name'], parameters['poster'], parameters['url'], force=True) else: #search for video urls if "urlresolver" in site and site['urlresolver'].lower( ) == "false": regex = "\"([^\s]*?\.(:?" + "|".join(filetypes) + "))\"" matches = re.findall(regex, page) else: regex = "(\/\/.*?\/embed.*?)[\?\"]" matches = re.findall(regex, page) regex = "\"((?:http:|https:)?\/\/.*?\/watch.*?)[\"]" matches = matches + re.findall(regex, page) matches2 = urlresolver.scrape_supported(page) #util.alert(str(matches)) """regex="\"(https?://("+"|".join(supports)+")\..*?)\"" matches2 = re.findall(regex, page) regex="\"((?:http:|https:)?\/\/.*?\/watch.*?)[\"]" matches3 = re.findall(regex, page) regex = 'https?://(.*?(?:\.googlevideo|(?:plus|drive|get|docs)\.google|google(?:usercontent|drive|apis))\.com)/(.*?(?:videoplayback\?|[\?&]authkey|host/)*.+)' matches4 = re.findall(regex, page) matches2=[ x for x in matches2 if any(sup in x for sup in supports) ] matches3=[ x for x in matches3 if any(sup in x for sup in supports) ]""" matches = matches + matches2 util.logError( "''''''''''''''''''''''''''''''''''''''''''''''''''''''") util.logError(">>>>" + str(matches)) if isinstance(matches[selected_video], tuple): url = matches[selected_video][0] else: url = matches[selected_video] #util.alert(url) if "http" not in url: url = "http:" + url link = urlresolver.resolve(url) if link == False: link = url util.playMedia(parameters['name'], parameters['poster'], link) elif mode == 112: extras = ast.literal_eval(parameters['extras']) bits = util.getFile(feed, type) site = bits['sites'][extras['site']] page = util.get(parameters['url']) """if "urlresolver" in site and site['urlresolver'].lower()=="false": regex="\"(.*?\.mp4)\"" matches = re.findall(regex, page) if matches: link=matches[0] else:""" regex = "\"(//\S*?(:?" + ("|".join(filetypes)) + ")\S*?)\"" matches = re.findall(regex, page) if matches: url = matches[selected_video][0] if "http" not in url: link = "http:" + url else: link = urlresolver.resolve(parameters['url']) if not link: try: regex = "(\/\/.*?\/embed.*?)[\?\"]" matches = re.findall(regex, page) regex = "\"((?:http:|https:)?\/\/.*?\/watch.*?)[\"]" matches = matches + re.findall(regex, page) regex = 'https?://(.*?(?:\.googlevideo|(?:plus|drive|get|docs)\.google|google(?:usercontent|drive|apis))\.com)/(.*?(?:videoplayback\?|[\?&]authkey|host/)*.+)' matches = matches + re.findall(regex, page) if matches: matches = [ x for x in matches if any(sup in x for sup in supports) ] if matches: link = urlresolver.resolve("http:" + matches[0]) except Exception as e: util.notify(str(e)) if link: import downloader downloader.download( link, os.path.join(xbmcaddon.Addon().getSetting('folder'), parameters['name'] + ".mp4")) else: util.notify("No video found") elif mode == 113: menu = [] extras = ast.literal_eval(parameters['extras']) bits = util.getFile(feed, type) site = bits['sites'][extras['site']] page = util.get(parameters['url']) matches = urlresolver.scrape_supported(page) #regex="(//\S*?(:?"+("|".join(filetypes))+")\S*?)" #matches2 = re.findall(regex, page) """regex="(\/\/.*?\/embed.*?)[\?\"]" matches2 = re.findall(regex, page) regex="\"(https?://("+"|".join(supports)+")\..*?)\"" matches3 = re.findall(regex, page) regex = 'https?://(.*?(?:\.googlevideo|(?:plus|drive|get|docs)\.google|google(?:usercontent|drive|apis))\.com)/(.*?(?:videoplayback\?|[\?&]authkey|host/)*.+)' matches4 = re.findall(regex, page) matches2=[ x for x in matches2 if any(sup in x for sup in supports) ] matches3=[ x for x in matches3 if any(sup in x for sup in supports) ] matches=matches+matches2+matches3+matches4""" unique = [] for match in matches: #+matches2: if isinstance(match, tuple): unique.append(match[0]) else: unique.append(match) matches = list(set(unique)) if matches: for match in matches: if "http" not in match: rl = "http:" + match else: rl = match menu.append({ "title": rl, "url": rl, "mode": "114", "poster": parameters['poster'], "icon": parameters['icon'], "fanart": parameters['fanart'], "type": "", "plot": "", "isFolder": False, "isPlayable": False, "extras": str(extras) }) util.addMenuItems(menu) elif mode == 114: # find playable sources in url #util.alert(parameters['url']) urlresolver.relevant_resolvers() try: link = urlresolver.resolve(str(parameters['url'])) except Exception as e: util.notify(str(e)) exit() if link: try: util.playMedia(parameters['name'], parameters['poster'], link) except: util.playMedia(parameters['name'], parameters['poster'], parameters['url'])
def poll(self, timestamp, filters): try: logging.info('Polling feed "%s"' % self.url) result = [] self.last_poll = timestamp username = util.decode_password(self.username) password = util.decode_password(self.password) d = util.parse(self.url, username, password, self.etag, self.modified) self.etag = util.get(d, 'etag', None) self.modified = util.get(d, 'modified', None) feed = util.get(d, 'feed', None) if feed: self.title = self.title or util.get(feed, 'title', '') self.link = self.link or util.get(feed, 'link', self.url) entries = util.get(d, 'entries', []) for entry in reversed(entries): found = False id = create_id(entry) for i in self.id_set: # be cheap, compare the links - they should be unique. # Can't compare objects anymore because the time is present which will always be different if id[1] == i[1]: found = True break if found: continue # The entry is not new, move on to the next one. self.id_set.add(id) self.item_count += 1 item = Item(self, id) item.timestamp = calendar.timegm(util.get(entry, 'date_parsed', time.gmtime())) item.title = util.format(util.get(entry, 'title', ''), settings.POPUP_TITLE_LENGTH) item.description = util.format(util.get(entry, 'description', ''), settings.POPUP_BODY_LENGTH) item.link = util.get(entry, 'link', '') item.author = util.format(util.get(entry, 'author', '')) # TODO: max length if all(filter.filter(item) for filter in filters): result.append(item) # determine if there are any entries in self.id_set that are older than CACHE_AGE_LIMIT # if there are, remove them from the list (they aged out of the cache) idsToRemove = set() now = datetime.fromtimestamp(time.time()) for tempId in self.id_set: diff = now - datetime.fromtimestamp(tempId[3]) if abs(diff.days) > settings.CACHE_AGE_LIMIT: idsToRemove.add(tempId) # can't modify seld.id_set right here. for i in idsToRemove: logging.info('Removing %s because its too old' % i[1]) self.id_set.remove(i) return result except Exception, e: logging.info('Error durring poll: %s' % e) raise
def create_id(entry): keys = ['id', 'link', 'title'] # 4th param is the time used for determing age of the cached item in our local cache values = util.get(entry,keys[0],None), util.get(entry,keys[1],None),util.get(entry,keys[2],None),time.time() return values if any(values) else uuid.uuid4().hex
def write_summary_table(scenario_name, instance): m = instance # note: this gets overwritten after every solve. It would be more efficient to # write it only after the last solve, but there's no hook in phsolverserver for that # (it may not even be called at that point) make_outputs_dir() # the build variables may be pinned to match a particular file; # if so, we use that as part of the filename build_file = os.environ.get('PHA_FIX_BUILD_VARS_FILE') if build_file is None: build_tag = "" else: # build_file typically looks like # "outputs/build_JJJJJJ_xhat.tsv" or "outputs/build_JJJJJJ_iter0_Scenario_nnnn.tsv" # we shorten this to tag like "bJJJJJJ_xhat" or "bJJJJJJ_iter0_Scenario_nnnn" # before prepending the current job id and appending the evaluated scenario ID. if os.path.dirname(build_file) == "outputs": # remove outputs dir if specified, otherwise keep dir name in there build_tag = os.path.basename(build_file) else: # remove path separators from the build file name build_tag = build_file.replace(os.sep, '_') if build_tag.startswith("build_"): build_tag = 'b' + build_tag[6:] # shorten "build_" to "b" if build_tag.endswith(".tsv"): build_tag = build_tag[:-4] build_tag = "_" + build_tag output_file = os.path.join( "outputs", "summary_{}{}_{}.tsv".format(jobid, build_tag, scenario_name)) print "writing {}...".format(output_file) period_duration = { pe: sum(m.tp_weight[tp] for tp in m.PERIOD_TPS[pe]) for pe in m.PERIODS } values = [] demand_components = [ c for c in ('lz_demand_mw', 'DemandResponse', 'ChargeEVs') if hasattr(m, c) ] # total cost / kWh generated in each period # (both discounted to today, so the discounting cancels out) values.extend([ ("cost_per_kwh", pe, m.SystemCostPerPeriod[pe] / sum(m.bring_timepoint_costs_to_base_year[tp] * 1000.0 * sum( getattr(m, c)[lz, tp] for c in demand_components for lz in m.LOAD_ZONES) for tp in m.PERIOD_TPS[pe])) for pe in m.PERIODS ]) # Renewable energy share if hasattr(m, 'RPSEligiblePower'): # total renewable share over all periods values.extend([("renewable_share", pe, m.RPSEligiblePower[pe] / m.RPSTotalPower[pe]) for pe in m.PERIODS]) # average production from each fuel during each period values.extend([ (f, pe, sum( get(m.DispatchProjByFuel, (pr, tp, f), 0.0) * m.tp_weight[tp] for pr in m.PROJECTS_BY_FUEL[f] for tp in m.PERIOD_TPS[pe]) / period_duration[pe]) for f in m.FUELS for pe in m.PERIODS ]) # total production from each non-fuel source values.extend([(s, pe, sum( get(m.DispatchProj, (pr, tp), 0.0) * m.tp_weight[tp] for pr in m.PROJECTS_BY_NON_FUEL_ENERGY_SOURCE[s] for tp in m.PERIOD_TPS[pe]) / period_duration[pe]) for s in m.NON_FUEL_ENERGY_SOURCES for pe in m.PERIODS]) # curtailments values.extend([("curtail_" + s, pe, sum((get(m.DispatchUpperLimit, (pr, tp), 0.0) - get(m.DispatchProj, (pr, tp), 0.0)) * m.tp_weight[tp] for pr in m.PROJECTS_BY_NON_FUEL_ENERGY_SOURCE[s] for tp in m.PERIOD_TPS[pe]) / period_duration[pe]) for s in m.NON_FUEL_ENERGY_SOURCES for pe in m.PERIODS]) # all LZ_Energy_Components values.extend([(component, pe, sum( getattr(m, component)[lz, tp] * m.tp_weight[tp] for lz in m.LOAD_ZONES for tp in m.PERIOD_TPS[pe]) / period_duration[pe]) for component in m.LZ_Energy_Components_Produce for pe in m.PERIODS]) values.extend([(component, pe, sum( getattr(m, component)[lz, tp] * m.tp_weight[tp] for lz in m.LOAD_ZONES for tp in m.PERIOD_TPS[pe]) / period_duration[pe]) for component in m.LZ_Energy_Components_Consume for pe in m.PERIODS]) with open(output_file, 'w') as f: f.writelines("\t".join((key, str(per), str(value(val)))) + "\n" for (key, per, val) in values)
def getCarPersonMappings(ignore): resp = util.get(settings.getCarPersonUrl(),"admin","admin") print (resp) return resp
def define_components(m): """ """ ################### # RPS calculation ################## m.f_rps_eligible = Param(m.FUELS, within=Binary) m.RPS_ENERGY_SOURCES = Set(initialize=lambda m: [s for s in m.NON_FUEL_ENERGY_SOURCES if s != 'Battery'] + [f for f in m.FUELS if m.f_rps_eligible[f]] ) m.RPS_YEARS = Set(ordered=True) m.rps_target = Param(m.RPS_YEARS) def rps_target_for_period_rule(m, p): """find the last target that is in effect before the _end_ of the period""" latest_target = max(y for y in m.RPS_YEARS if y < m.period_start[p] + m.period_length_years[p]) return m.rps_target[latest_target] m.rps_target_for_period = Param(m.PERIODS, initialize=rps_target_for_period_rule) # maximum share of (bio)fuels in rps # note: using Infinity as the upper limit causes the solution to take forever # m.rps_fuel_limit = Param(default=float("inf"), mutable=True) m.rps_fuel_limit = Param(initialize=m.options.biofuel_limit, mutable=True) # calculate amount of pre-existing capacity in each generation project; # used when we want to restrict expansion m.gen_pre_existing_capacity = Expression( m.GENERATION_PROJECTS, rule=lambda m, g: ( m.GenCapacity[g, m.PERIODS.first()] - get(m.BuildGen, (g, m.PERIODS.first()), 0) ) ) # Define DispatchGenRenewableMW, which shows the amount of power produced # by each project from each fuel during each time step. define_DispatchGenRenewableMW(m) # calculate amount of power produced from renewable fuels during each period m.RPSFuelPower = Expression(m.PERIODS, rule=lambda m, per: sum( m.DispatchGenRenewableMW[g, tp] * m.tp_weight[tp] for g in m.FUEL_BASED_GENS for tp in m.TPS_FOR_GEN_IN_PERIOD[g, per] ) ) # Note: this rule ignores pumped hydro and batteries, so it could be gamed by producing extra # RPS-eligible power and burning it off in storage losses; on the other hand, # it also neglects the (small) contribution from net flow of pumped hydro projects. # TODO: incorporate pumped hydro into this rule, maybe change the target to refer to # sum(getattr(m, component)[z, t] for z in m.LOAD_ZONES) for component in m.Zone_Power_Injections) # power production that can be counted toward the RPS each period m.RPSEligiblePower = Expression(m.PERIODS, rule=lambda m, per: m.RPSFuelPower[per] + sum( m.DispatchGen[g, tp] * m.tp_weight[tp] for f in m.NON_FUEL_ENERGY_SOURCES if f in m.RPS_ENERGY_SOURCES for g in m.GENS_BY_NON_FUEL_ENERGY_SOURCE[f] for tp in m.TPS_FOR_GEN_IN_PERIOD[g, per] ) ) # total power production each period (against which RPS is measured) # note: we exclude production from storage m.RPSTotalPower = Expression(m.PERIODS, rule=lambda m, per: sum( m.DispatchGen[g, tp] * m.tp_weight[tp] for g in m.GENERATION_PROJECTS if g not in getattr(m, 'STORAGE_GENS', []) for tp in m.TPS_FOR_GEN_IN_PERIOD[g, per] ) ) if m.options.rps_level == 'activate': # we completely skip creating the constraint if the RPS is not activated. # this makes it easy for other modules to check whether there's an RPS in effect # (if we deactivated the RPS after it is constructed, then other modules would # have to postpone checking until then) m.RPS_Enforce = Constraint(m.PERIODS, rule=lambda m, per: m.RPSEligiblePower[per] >= m.rps_target_for_period[per] * m.RPSTotalPower[per] ) elif m.options.rps_level == 'no_new_renewables': # prevent construction of any new exclusively-renewable projects, but allow # replacement of existing ones # (doesn't ban use of biofuels in existing or multi-fuel projects, but that could # be done with --biofuel-limit 0) m.No_New_Renewables = Constraint(m.NEW_GEN_BLD_YRS, rule=lambda m, g, bld_yr: (m.GenCapacity[g, bld_yr] <= m.gen_pre_existing_capacity[g]) if m.gen_energy_source[g] in m.RPS_ENERGY_SOURCES else Constraint.Skip ) wind_energy_sources = {'WND'} if m.options.rps_no_new_wind: # limit wind to existing capacity m.No_New_Wind = Constraint(m.NEW_GEN_BLD_YRS, rule=lambda m, g, bld_yr: (m.GenCapacity[g, bld_yr] <= m.gen_pre_existing_capacity[g]) if m.gen_energy_source[g] in wind_energy_sources else Constraint.Skip ) if m.options.rps_no_wind: # don't build any new capacity or replace existing m.No_Wind = Constraint(m.NEW_GEN_BLD_YRS, rule=lambda m, g, bld_yr: (m.BuildGen[g, bld_yr] == 0.0) if m.gen_energy_source[g] in wind_energy_sources else Constraint.Skip ) if m.options.rps_prefer_dist_pv: m.DIST_PV_GENS = Set(initialize=lambda m: [ g for g in m.GENS_BY_NON_FUEL_ENERGY_SOURCE['SUN'] if 'DistPV' in m.gen_tech[g] ]) m.LARGE_PV_GENS = Set(initialize=lambda m: [ g for g in m.GENS_BY_NON_FUEL_ENERGY_SOURCE['SUN'] if g not in m.DIST_PV_GENS ]) # LargePVAllowed must be 1 to allow large PV to be built m.LargePVAllowed = Var(m.PERIODS, within=Binary) # # LargePVAllowed can only be 1 if 90% of the available rooftop PV has been built m.Set_LargePVAllowed = Constraint( m.PERIODS, rule=lambda m, p: sum(m.GenCapacity[g, p] for g in m.DIST_PV_GENS) >= m.LargePVAllowed[p] * 0.9 * sum(m.gen_capacity_limit_mw[g] for g in m.DIST_PV_GENS) ) m.Apply_LargePVAllowed = Constraint( m.LARGE_PV_GENS, m.PERIODS, rule=lambda m, g, p: m.GenCapacity[g, p] <= m.LargePVAllowed[p] * m.gen_capacity_limit_mw[g] + m.gen_pre_existing_capacity[g] ) # Don't allow (bio)fuels to provide more than a certain percentage of the system's energy # Note: when the system really wants to use more biofuel, it is possible to "game" this limit by # cycling power through batteries, pumped storage, transmission lines or the hydrogen system to # burn off some # extra non-fuel energy, allowing more biofuel into the system. (This doesn't typically happen # with batteries due to high variable costs -- e.g., it has to cycle 4 kWh through a battery to # consume 1 kWh of non-biofuel power, to allow 0.05 kWh of additional biofuel into the system. # Even if this can save $0.5/kWh, if battery cycling costs $0.15/kWh, that means $0.60 extra to # save $0.025. It also doesn't happen in the hydrogen scenario, since storing intermittent power # directly as hydrogen can directly displace biofuel consumption. But it could happen if batteries # have low efficiency or low cycling cost, or if transmission losses are significant.) # One solution would be to only apply the RPS to the predefined load (not generation), but then # transmission and battery losses could be served by fossil fuels. # Alternatively: limit fossil fuels to (1-rps) * standard loads # and limit biofuels to (1-bio)*standard loads. This would force renewables to be used for # all losses, which is slightly inaccurate. # TODO: fix the problem noted above; for now we don't worry too much because there are no # transmission losses, the cycling costs for batteries are too high and pumped storage is only # adopted on a small scale. m.RPS_Fuel_Cap = Constraint(m.PERIODS, rule = lambda m, per: m.RPSFuelPower[per] <= m.rps_fuel_limit * m.RPSTotalPower[per] )
def get(self, path): return util.get(path, self.__state)
def getCars(ignore): """Uses the GET on car:cars resource to get all cars in the store using RESTCONF""" resp = util.get(settings.getCarsUrl(), "admin", "admin") print(resp) return resp
def __init__(self, data): data = data["flight"] self.id = util.get(data, ["identification", "row"]) self.flight_num = util.get(data, ["identification", "number", "default"]) self.status_detail = util.get(data, ["status", "text"]) self.aircraft_code = util.get(data, ["aircraft", "model", "code"]) self.aircraft_text = util.get(data, ["aircraft", "model", "text"]) self.aircraft_reg = util.get(data, ["aircraft", "registration"]) self.aircraft_co2 = util.get(data, ["aircraft", "country", "alpha2"]) self.aircraft_restr = util.get(data, ["aircraft", "restricted"]) self.owner = util.get(data, ["owner", "name"]) self.owner_iata = util.get(data, ["owner", "code", "iata"]) self.owner_icao = util.get(data, ["owner", "code", "icao"]) self.airline = util.get(data, ["airline", "name"]) self.airline_iata = util.get(data, ["airline", "code", "iata"]) self.airline_icao = util.get(data, ["airline", "code", "icao"]) self.origin_offset = util.get( data, ["airport", "origin", "timezone", "offset"]) self.origin_offset_abbr = util.get( data, ["airport", "origin", "timezone", "abbr"]) self.origin_offset_dst = util.get( data, ["airport", "origin", "timezone", "isDst"]) self.origin_terminal = util.get( data, ["airport", "origin", "info", "terminal"]) self.origin_gate = util.get(data, ["airport", "origin", "info", "gate"]) self.dest_iata = util.get(data, ["airport", "destination", "code", "iata"]) self.dest_icao = util.get(data, ["airport", "destination", "code", "icao"]) self.dest_name = util.get(data, ["airport", "destination", "name"]) self.dest_city = util.get( data, ["airport", "destination", "position", "region", "city"]) self.dest_country = util.get( data, ["airport", "destination", "position", "country", "name"]) self.dest_country_code = util.get( data, ["airport", "destination", "position", "country", "code"]) self.dest_offset = util.get( data, ["airport", "destination", "timezone", "offset"]) self.dest_offset_abbr = util.get( data, ["airport", "destination", "timezone", "abbr"]) self.dest_offset_dst = util.get( data, ["airport", "destination", "timezone", "isDst"]) self.dest_terminal = util.get( data, ["airport", "destination", "info", "terminal"]) self.dest_gate = util.get(data, ["airport", "destination", "info", "gate"]) self.sched_dep = util.get(data, ["time", "scheduled", "departure"]) self.sched_arr = util.get(data, ["time", "scheduled", "arrival"]) self.real_dep = util.get(data, ["time", "real", "departure"]) self.real_arr = util.get(data, ["time", "real", "arrival"]) self.dest_lat = util.get( data, ["airport", "destination", "position", "latitude"]) self.dest_lng = util.get( data, ["airport", "destination", "position", "longitude"])
def get_by_names(): """ One of the main ``clusto`` operations. Parameters: * Required parameter: At least one ``name`` parameter Returns ``HTTP: 404`` when all entites requested do not exist and ``HTTP: 206`` when a percent of entities requested do not exist. Examples: .. code:: bash $ ${get} ${server_url}/by-names "Provide at least one name to get data from" HTTP: 412 Content-type: application/json $ ${get} -d 'name=nonserver' ${server_url}/by-names [ null ] HTTP: 404 Content-type: application/json $ ${get} -d 'name=testserver1' -d 'name=nonserver' ${server_url}/by-names [ "/basicserver/testserver1", null ] HTTP: 206 Content-type: application/json $ ${get} -H 'Clusto-Mode: expanded' -d 'name=testserver1' -d 'name=testserver2' ${server_url}/by-names [ { "attrs": [], "contents": [], "driver": "basicserver", "name": "testserver1", "parents": [ "/pool/singlepool", "/pool/multipool" ] }, { "attrs": [], "contents": [], "driver": "basicserver", "name": "testserver2", "parents": [ "/pool/multipool" ] } ] HTTP: 200 Content-type: application/json $ ${get} -d 'name=nonserver1' -d 'name=nonserver2' ${server_url}/by-names [ null, null ] HTTP: 404 Content-type: application/json """ objs = [] names = bottle.request.params.getall('name') if not names: return util.dumps('Provide at least one name to get data from', 412) mode = bottle.request.headers.get('Clusto-Mode', default='compact') for name in names: obj, status, msg = util.get(name) try: objs.append(util.show(obj, mode) if obj else None) except TypeError as te: return util.dumps('%s' % (te,), 409) return util.dumps(objs, 200 if all(objs) else 206 if any(objs) else 404)
def symmetry(con, threshold=75): data = {} cur = con.cursor() cur.execute( "SELECT from_station_name, to_station_name, count(trip_id) num " "FROM trips WHERE from_station_name != to_station_name GROUP BY from_station_name, to_station_name HAVING num >= ?", (threshold, )) while True: rows = cur.fetchmany(1000) if not rows: break for row in rows: put_val(data, (row["from_station_name"], row["to_station_name"]), "num", row["num"]) print "have %d routes with at least %d trips" % (len(data), threshold) # drop pairs where either direction has less than the given threshold for pair in list(data): rev_pair = tuple(reversed(pair)) if (get(data, pair, "num", leaf=0) <= threshold or rev_pair not in data or get(data, rev_pair, "num", leaf=0) <= threshold): del data[pair] continue print "have %d routes remaining after filtering for reverse trip meeting threshold" % len( data) i = 0 for pair in data: durs = [] cur.execute( "SELECT trip_duration FROM trips WHERE from_station_name = ? " "AND to_station_name = ?", pair) while True: rows = cur.fetchmany(1000) if not rows: break for row in rows: durs.append(row["trip_duration"]) durs_a = array(durs) put_val(data, pair, "mean", mean(durs_a)) put_val(data, pair, "median", median(durs_a)) put_val(data, pair, "std", std(durs_a)) #print pair, data[pair] i += 1 if i % 100 == 0: print "getting duration stats; %d routes remaining" % (len(data) - i) cur.close() plot_data = {} done = set() for pair in data: rev_pair = tuple(reversed(pair)) if rev_pair in done: continue done.add(rev_pair) for param in ["num", "mean", "median", "std"]: val1 = get(data, pair, param) val2 = get(data, rev_pair, param) put_list(plot_data, param, "x", min(val1, val2)) put_list(plot_data, param, "y", max(val1, val2)) for param in ["num", "mean", "median", "std"]: fig = plt.figure() xs = plot_data[param]["x"] ys = plot_data[param]["y"] plt.scatter(xs, ys) plt.xlabel("min(%s)" % param) plt.ylabel("max(%s)" % param) plt.savefig("symmetry_%s.pdf" % param, bbox_inches="tight")