def _make_counterfactual_graph(account_names, sim_symbol, lines, min_date, max_date): if not account_names: return None all_traces = calculate_traces(account_names, min_date, max_date) line = _standardize_lines(lines)[0] plots = [] y = all_traces[line.split(':')[0]] plots.append( go.Scatter(x=y.index, y=y, mode='lines', name='Actual Portfolio', yaxis='y1')) sim_traces = calculate_sim_traces(account_names, sim_symbol, min_date, max_date) y_sim = sim_traces[line.split(':')[0]] plots.append( go.Scatter(x=y_sim.index, y=y_sim, mode='lines', name='Counterfactual Portfolio', yaxis='y1')) return { 'data': plots, 'layout': get_layout(conf('which_axis')[line], [conf('lines')[line]]), }
def _make_summed_graph(account_names, lines, min_date, max_date): if not account_names: return None all_traces = calculate_traces(account_names, min_date, max_date) lines = _standardize_lines(lines) plots = [] axes_used = {conf('which_axis')[l] for l in lines} lines_on_y2 = list( filter(lambda x: conf('which_axis')[x] == 'pct', conf('which_axis'))) for line in lines: if line in lines_on_y2 and len(axes_used) > 1: which_y = 'y2' else: which_y = 'y1' y = all_traces[line.split(':')[0]] plots.append( go.Scatter(x=y.index, y=y, mode='lines', name=conf('lines')[line], yaxis=which_y)) return { 'data': plots, 'layout': get_layout(axes_used, [conf('lines')[l] for l in lines]), }
def _make_counterfactual_graph(account_names, sim_symbol, lines, min_date, max_date): if not account_names: return None all_traces = calculate_traces(account_names, min_date, max_date) line = _standardize_lines(lines)[0] plots = [] y = all_traces[line.split(":")[0]] plots.append( go.Scatter(x=y.index, y=y, mode="lines", name="Actual Portfolio", yaxis="y1") ) sim_traces = calculate_sim_traces(account_names, sim_symbol, min_date, max_date) y_sim = sim_traces[line.split(":")[0]] plots.append( go.Scatter( x=y_sim.index, y=y_sim, mode="lines", name="Counterfactual Portfolio", yaxis="y1", ) ) return { "data": plots, "layout": get_layout(conf("which_axis")[line], [conf("lines")[line]]), }
def generate_aug(which): images, tags = load() images, tags = augment_and_shuffle(images, tags, conf(f'aug.{which}.factor')) dataset = {'images': images, 'tags': tags} with open(conf(f'aug.{which}.output'), 'wb') as f: f.write(json.dumps(dataset).encode('utf-8'))
def send(self, server=None, img=None, quality=conf("image.jpg_quality")): """ Envoyer une image sur le serveur UDP du client. Parameters ---------- server : tuple of str and int Adresse du serveur du client (IP et port) img : ndarray or None Si None, l'image enregistrée est self._image sinon l'image à enregistrer Returns ------- void Rien. """ if server is None: if conf("debug.network"): print("NO VIDEO SOCKET") return if img is None: img = self._image res, jpg = cv2.imencode('.jpg', img, [int(cv2.IMWRITE_JPEG_QUALITY), quality]) try: server.sendall( int(jpg.size).to_bytes(4, 'big', signed=True) + jpg.tobytes()) except OSError: print("Erreur: image::Image.send")
def isFin(self, seuil_fin=conf("image.seuil.fin"), prc_fin=conf("image.prc_fin")): """ Détecter la présence d'une ligne ou non sur une image Parameters ---------- seuil_fin : float Tout ce qui est supérieur à ce seuil est considéré comme étant la ligne. prc_fin : float pourcentage minimum de ligne requis pour que l'image soit considérée comme "ayant une ligne" et que la fonction retourne False Returns ------- bool Vrai si il n'y a pas de ligne, faux si il y a une ligne. Notes ----- Seuillage de la ligne noire (pix<50) Compare le nbr de non-zero retrouvé l'image seuillé Il doit y avoir plus de prc_fin pourcents de ligne visible sinon le seuillage OTSU ne fonctionnera pas. """ return self._image.size * prc_fin > cv2.countNonZero( cv2.threshold(self._image, seuil_fin, 255, cv2.THRESH_BINARY_INV)[1])
def _make_summed_graph(account_names, lines, min_date, max_date): if not account_names: return None all_traces = calculate_traces(account_names, min_date, max_date) lines = _standardize_lines(lines) plots = [] axes_used = {conf("which_axis")[l] for l in lines} lines_on_y2 = list( filter(lambda x: conf("which_axis")[x] == "pct", conf("which_axis")) ) for line in lines: if line in lines_on_y2 and len(axes_used) > 1: which_y = "y2" else: which_y = "y1" y = all_traces[line.split(":")[0]] plots.append( go.Scatter( x=y.index, y=y, mode="lines", name=conf("lines")[line], yaxis=which_y ) ) return { "data": plots, "layout": get_layout(axes_used, [conf("lines")[l] for l in lines]), }
def _get_accounts(refresh: bool = False, download_quotes: bool = False) -> Dict[str, pd.DataFrame]: """Return dictionary of all accounts, keyed by account name""" if refresh or not os.path.exists(conf('etl_accts')): accts = processing.refresh_portfolio(refresh_cache=download_quotes)[0] else: accts = pickle.load(open(conf('etl_accts'), 'rb'))[0] return accts
def build_thread(gitpath, ref, buildid, cburl=None, submodules=False): tmpdir = os.path.join(conf('buildbot.buildpath'), buildid) repo = GitRepository(tmpdir) output, retcode = repo.clone(gitpath) if retcode: buildlog(buildid, 'Unable to clone %s. %s\n' % (gitpath, '\n'.join(output))) return output, retcode = repo.checkout(ref) if retcode: buildlog(buildid, 'Unable to checkout %s. %s\n' % (ref, '\n'.join(output))) return if submodules: output, retcode = repo.submodule_init() buildlog(buildid, output[0]) buildlog(buildid, output[1]) output, retcode = repo.submodule_update() buildlog(buildid, output[0]) buildlog(buildid, output[1]) resultsdir = os.path.join(tmpdir, '.build_results') os.makedirs(resultsdir) output, retcode = repo.build(conf('buildbot.signkey'), conf('buildbot.pbuilderrc'), resultsdir) buildlog(buildid, output[0]) buildlog(buildid, output[1]) #logging.debug(output[0]) #logging.debug(output[1]) os.chdir(resultsdir) if not os.listdir(resultsdir) or retcode != 0: buildlog(buildid, 'Nothing in results directory. Giving up.') return tarpath = os.path.join(tmpdir, 'package.tar.gz') tar = tarfile.open(tarpath, 'w:gz') for name in os.listdir(resultsdir): tar.add(name) tar.close() buildlog(buildid, 'Build complete. Results in %s\n' % tarpath) data = file(tarpath, 'rb').read() buildlog(buildid, 'Built %i byte tarball' % len(data)) if cburl: buildlog(buildid, 'Performing callback: %s' % cburl) req = Curl() req.setopt(req.POST, 1) req.setopt(req.URL, str(cburl)) req.setopt(req.HTTPPOST, [('package', (req.FORM_FILE, str(tarpath)))]) req.setopt(req.WRITEDATA, file('%s/build.log' % tmpdir, 'a+')) req.perform() req.close()
def sign(self, dist): self._reprepro('export %s' % dist) gpg = GPG(gnupghome=conf('repository.gpghome')) filename = os.path.join(self.path, 'dists/%s/Release' % dist) detach_file = filename + '.gpg' try: os.unlink(detach_file) except: pass result = gpg.sign_file(file(filename, 'r'), keyid=conf('repository.signkey'), outputfile=detach_file)
def build_thread(gitpath, ref, buildid, cburl=None, submodules=False): tmpdir = os.path.join(conf('buildbot.buildpath'), buildid) repo = GitRepository(tmpdir) output, retcode = repo.clone(gitpath) if retcode: buildlog(buildid, 'Unable to clone %s. %s\n' % (gitpath, '\n'.join(output))) return output, retcode = repo.checkout(ref) if retcode: buildlog(buildid, 'Unable to checkout %s. %s\n' % (ref, '\n'.join(output))) return if submodules: output, retcode = repo.submodule_init() buildlog(buildid, output[0]) buildlog(buildid, output[1]) output, retcode = repo.submodule_update() buildlog(buildid, output[0]) buildlog(buildid, output[1]) resultsdir = os.path.join(tmpdir, '.build_results') os.makedirs(resultsdir) output, retcode = repo.build(conf('buildbot.signkey'), conf('buildbot.pbuilderrc'), resultsdir) buildlog(buildid, output[0]) buildlog(buildid, output[1]) #logging.debug(output[0]) #logging.debug(output[1]) os.chdir(resultsdir) if not os.listdir(resultsdir) or retcode != 0: buildlog(buildid, 'Nothing in results directory. Giving up.') return tarpath = os.path.join(tmpdir, 'package.tar.gz') tar = tarfile.open(tarpath, 'w:gz') for name in os.listdir(resultsdir): tar.add(name) tar.close() buildlog(buildid, 'Build complete. Results in %s\n' % tarpath) data = file(tarpath, 'rb').read() buildlog(buildid, 'Built %i byte tarball' % len(data)) if cburl: buildlog(buildid, 'Performing callback: %s' % cburl) req = Curl() req.setopt(req.POST, 1) req.setopt(req.URL, str(cburl)) req.setopt(req.HTTPPOST, [('package', (req.FORM_FILE, str(tarpath)))]) req.setopt(req.WRITEDATA, file('%s/build.log' % tmpdir, 'a+')) req.perform() req.close()
def lenet5(self, x): for p in conf('network.lenet5.cnn'): x = self.conv(x, p['conv']['filter'], p['conv']['bias']) x = tf.nn.relu(x) x = self.pool(x, p['pool']['ksize'], p['pool']['strides']) i = 0 x = tf.reshape(x, [-1, 13 * 13 * 120]) for p in conf('network.lenet5.fc'): x = tf.layers.dense(x, p, activation='sigmoid', name=f'fc_{i}') i += 1 return x
def __init__(self, initial, correction, angle_min, angle_max, canal): super().__init__(conf("controle.servo.periode_min"), conf("controle.servo.periode_max"), canal) if conf("debug.controle"): print("Servomoteur: créé : ", self) self._angle_min = angle_min self._angle_max = angle_max self._angle_correction = correction self._angle_initial = initial self._angle_courant = self._angle_initial self.calibrer()
def load(): result = [] tags = [] for triple in os.walk(conf('image.root')): for file in triple[2]: if file.endswith('.jpg') or file.endswith('.png'): path = triple[0] + file result.append( np.array(Image.open(path).resize(conf('image.size')))) tags.append(get_label_vector(file.split('.')[0])) return result, tags
def __init__(self): self.x1 = tf.placeholder(tf.float32, [None, *conf('image.size'), 3]) self.x2 = tf.placeholder(tf.float32, [None, *conf('image.size'), 3]) with tf.variable_scope("siamese") as scope: self.o1 = self.network(self.x1) scope.reuse_variables() self.o2 = self.network(self.x2) # Create loss self.y_ = tf.placeholder(tf.float32, [None]) self.loss = self.loss_with_spring()
def refresh_portfolio(refresh_cache: bool = False): """This is the "main" function; it runs everything.""" os.makedirs(conf("cache_dir"), exist_ok=True) inv = read_investment_transactions(conf("investment_transactions")) portfolio_transactions = read_portfolio_transactions( conf("account_transactions")) # Read all the quotes either from disk or from the web. if refresh_cache: quotes.refresh_quotes(inv.Symbol.unique()) max_date = quotes.get_max_date() log.info(f"The most recent quote is from {max_date}") min_port = min(p["Date"].min() for p in portfolio_transactions.values()) index = pd.date_range(start=min(inv["Date"].min(), min_port), end=max_date, freq="D") accounts = {} all_accounts = port.init_portfolio(index) log.info("Logging investment transactions for all portfolios.") for idx, row in inv.iterrows(): if row["Account"] not in accounts: accounts[row["Account"]] = port.init_portfolio(index) record_inv_action(all_accounts, row) record_inv_action(accounts[row["Account"]], row) log.info("Totaling portfolio transactions for all accounts.") port.total_portfolio(all_accounts) for acct in accounts.values(): port.total_portfolio(acct) max_trans_date = None for acct_name, trans in portfolio_transactions.items(): log.info("Logging portfolio transactions for %s.", acct_name) record_trans(accounts[acct_name], trans) record_trans(all_accounts, trans) accounts[acct_name]["_total"] += accounts[acct_name]["cash"] accounts[acct_name].loc[accounts[acct_name]["_total"].abs() < 0.01, "_total"] = 0 if not max_trans_date: max_trans_date = trans["Date"].max() else: max_trans_date = max((max_trans_date, trans["Date"].max())) all_accounts["_total"] += all_accounts["cash"] log.info("The most recent transaction was on %s", max_trans_date) _check_account_dict(accounts) with open(conf("etl_accts"), "wb") as _fout: pickle.dump((accounts, max_trans_date.date()), _fout) return accounts, max_trans_date.date()
def _make_comparison_graph(account_names, lines, min_date, max_date): if not account_names: return None traces = {n: calculate_traces([n], min_date, max_date) for n in account_names} line = _standardize_lines(lines)[0] plots = [] for name, trace in traces.items(): y = trace[line.split(":")[0]] plots.append(go.Scatter(x=y.index, y=y, mode="lines", name=name, yaxis="y1")) return { "data": plots, "layout": get_layout(conf("which_axis")[line], [conf("lines")[line]]), }
def write_sample_data( monthly_contribution: float, security_mix: Dict[str, float], start_date: Union[str, datetime], stop_date: Union[str, datetime] = None, account: str = 'My 401(k)', config_fname: str = 'sample_config.yaml', av_api_key: str = None, ) -> None: """Write sample CSVs for specified investment contributions Assume that there's one contribution at the end of each month, and that contributions are split between securities in a way described by the `security_mix` input. Example ------- # This assumes that there's an AlphaVantage API key in the # environment variable "AV_API_KEY". python -c "from portdash.sample import *; write_sample_data(100, {'VFAIX': 0.5, 'VDAIX': 0.3, 'VEIEX': 0.2}, '2010-01-01')" python portdash/acemoney.py -c sample_config.yaml python dash_app.py -c sample_config.yaml """ logging.basicConfig(level='INFO') # Use default config as much as possible and create file names for the # sample data we're going to write. config.update_config({ 'investment_transactions': 'sample_investment_transactions.csv', 'account_transactions': { account: 'sample_account.csv' }, 'av_api_key': av_api_key, }) invest_trans, acct_trans = create_transactions(monthly_contribution, security_mix, start_date, stop_date, account) os.makedirs(config.conf('data_dir'), exist_ok=True) invest_trans.to_csv(config.conf('investment_transactions'), index=False, quoting=csv.QUOTE_ALL) acct_trans.to_csv(config.conf('account_transactions')[account], index=False, quoting=csv.QUOTE_ALL) print(f'Writing a sample config file to {config_fname}.') with open(config_fname, 'wt') as _fout: yaml.dump(config.get_config(), _fout)
def mapping(map_para): fastq1, fastq2, outdir = map_para # merged for multiprocessing in py2 sample_name = fastq1.split('/')[-1].split('.')[0][:-3] conf = config.conf() cutadapt(fastq1, fastq2, outdir, conf.adapter1, conf.adapter2) hisat2(sample_name, outdir, conf.index) spicard(sample_name, outdir, conf.picard)
def fetch_from_web(symbol: str, start_time: datetime = None) -> pd.DataFrame: """Return a table of historical security valuations Parameters ---------- symbol : str The stock ticker symbol start_time : datetime, optional If supplied, start the output table at this date. Supplying a recent date will allow us to request fewer rows returned from the Alpha Vantage service. The default will return all available historical quotes. Returns ------- pd.DataFrame A table of historical quotes, indexed by the datetime of the quote """ log.info(f"Reading {symbol} data from Alpha Vantage.") client = AlphaVantageClient(conf("av_api_key")) new_quotes = client.historical_quotes( symbol, start_time=start_time, return_dividends=True ) new_quotes.index.name = "date" return new_quotes
def buildlog(buildid, message): filename = os.path.join(conf('buildbot.buildpath'), '%s/build.log' % buildid) fd = file(filename, 'a+') fd.write(message + '\n') fd.close() logging.debug(message)
def connectAndMoveToSetPoint(): c = conf() data = { "setpoint": c.get('ttm_setpoint') ,"setpointTime": c.get('ttm_setpointtime') } response = connectToCommand("moveToSetpoint", data) return connectAndGetStatus()
def remove(self,widget,data): if data == "album": for x in self.get_album_files(): if x in self.bilinmiyor_artist: continue del self.sozluk[x] remover.remove_iconview(self.iconView ) if data == "artist": for x in self.get_artist_files(self.dizelge): if x in self.bilinmiyor_artist: continue else: del self.sozluk[x] remover.remove_treeview(self.treeview,self.dizi) if data == "title": try: liste = self.dizelge.sozluk[self.bilgi] except KeyError: liste = [] for x in self.get_title_files(): if x in liste: liste.remove(x) self.dizelge.yaz() elif x in self.sevilen.sections(): self.sevilen._remove(x) del self.sevilen self.sevilen = conf(svcfg) else: try: del self.sozluk[x] except KeyError: print "Key Error:{0}".format(x) return False remover.remove_iconview(self.iconView1) self.pick.yaz()
def twitter_bot(rss_guid=None): """ Consumes a feed and checks if there are new entries in db. If so, gets a shortened url and tweets the new status. """ if rss_guid is None: # ancestor_key = ndb.Key("RSS_GUID", rss_guid or "*norss*") # consumer = FeedConsume.get_last_rss_guid(ancestor_key) # rss_guid = consumer[0].rss_guid query = FeedConsume.gql("WHERE entry = :1", "latest") result = query.get() rss_guid = result.rss_guid else: consumer = FeedConsume(parent=ndb.Key("RSS_GUID", rss_guid or "*norss*"), rss_guid=rss_guid, entry="latest") consumer.put() url = "{}erss.cgi?rss_guid={}".format(conf("pubmed_rss"), rss_guid) feeds = feedparser.parse(url) tweets = [] for feed in feeds["items"]: pmid = (feed["link"].split("/")[-1]).rstrip("?dopt=Abstract") if "entrez?" in pmid: continue query = FeedItem.gql("WHERE pmid = :1", pmid) # if pmid not in db if (query.count() == 0): title = feed["title"] otitle = title url = feed["link"] category = feed["category"] item = FeedItem() item.pmid = pmid # shorten the url with Bitly.com shorturl = shorten_url_bitly(url) # tweet the new entry max_length = (140 - len(category) - len(shorturl) - 7) print(max_length) if len(title) > max_length: title = title[0:max_length] status = "#{}: {}... {}".format("".join(category.split()), title.rstrip(". "), shorturl) try: status = unicode(status).encode("utf-8") except UnicodeEncodeError: pass # TODO: add logging # tweet new status # tweets.append({'title': "{}...".format(title.rstrip(". ")), 'url': shorturl}) ttitle = "#{}: {}...".format("".join(category.split()), otitle[0:100].rstrip(". ")) tweets.append({'title': ttitle, 'url': shorturl}) try: update_status_twitter(status) item.put() except: pass return tweets
def connectAndInitRamp(): c = conf() data = { "rampRate": c.get('ttm_ramprate') ,"rampAmplitude": c.get('ttm_rampamplitude') } connection = connectToCommand("initRamp", data) return connectAndGetStatus()
def pointInPolygon(walkshed): import config connection_string = config.conf() conn = psycopg2.connect(connection_string) cur = conn.cursor() polygon = getPolygon(walkshed) selectQuery = "SELECT id, ST_AsText(crime_location), crime_time, crime_type FROM cps_crime_data.crime_data WHERE ST_Within(crime_location, ST_GeomFromText(%s, 4326));" parameters = [polygon] cur.execute(selectQuery, parameters) rows = cur.fetchall() if len(rows) > 0: result_json = '{"type": "FeatureCollection", "features": [' for i in xrange(len(rows)): id = rows[i][0] location = rows[i][1] location = location[6:].split(' ') longitude = location[0] latitude = location[1][:-1] location = "[%s,%s]" % (longitude, latitude) time = rows[i][2] time = "%s-%s-%s %s:%s:00" % (time.year, time.month, time.day, time.hour, time.minute) type = rows[i][3] result_json += '{"type": "Feature","geometry": {"type": "Point", "coordinates":%s}, "properties": {"id": %s, "time": "%s","type": "%s"}},' % ( location, id, time, type) result_json = result_json[:-1] result_json += ']}' else: result_json = '"NULL"' conn.commit() cur.close() conn.close() return result_json
def do(self, packet): """ Execute l'action selon les données du paquet : Parameters ---------- packet : str données du paquet. Returns ------- void Rien. Notes ----- Paquets : * start : Mettre la boucle d'analyse en position True Lancer l'analyse dans un nouveau thread * stop : Arrêter le robot Arrêter la boucle d'analyse * direction_* : Ordres de contrôle manuel du robot """ if packet == b'start': self._robot.init_vars() self.lancer_video(True) if conf("debug.network"): print("Commande START reçue") elif packet == b'stop': self._robot._video_on = False self._pad = False if conf("debug.network"): print("Commande STOP reçue") moteur.do("stop") elif packet[:9] == b'direction': if conf("debug.network"): print("Commande de direction :\"", packet[10:], "\"reçue") self.pad_control(packet[10:]) if self._pad is False: self.lancer_video(False) self._pad = True else: print("Commande non reconnue : ", packet)
def connectToCommand(command, data=None): c = conf() basic_url = "http://%s:%s/lumikki/csm_lumikki_instron_" % (c.get('ttm_ip'),c.get('ttm_port')) url = basic_url + command if data != None: url += "?%s" % urlencode(data) connection = urlopen(url, timeout=timeout) return connection
def pre_start(): global parametrs, parametr1, parametr2, parametr3, parametr4 parametrs = {} parametrs = conf() parametr1 = parametrs["enable_default_dictionary"] parametr2 = parametrs["default_dictionary"] parametr3 = parametrs["maximal_random_in_encrypting_withowt_key"] parametr4 = parametrs["save_all_actions"]
def read_portfolio_transactions(acct_fnames: Dict[str, str]=None, ignore_future: bool=True) \ -> Dict[str, pd.DataFrame]: """Read a CSV of account transactions written by AceMoney""" if not acct_fnames: acct_fnames = conf('account_transactions') trans = { acct_name: pd.read_csv(fname, parse_dates=[0], thousands=',') for acct_name, fname in acct_fnames.items() if fname and acct_name not in conf('skip_accounts') } if ignore_future: # Filter out any transactions marked as being in the future. # This can happen with scheduled transactions. today = datetime.today() trans = {k: v[v['Date'] <= today] for k, v in trans.items()} return trans
def read_investment_transactions(fname: str = None) -> pd.DataFrame: """Read a CSV of investment transactions written by AceMoney""" if not fname: fname = conf('investment_transactions') inv = pd.read_csv(fname, dtype={ 'Dividend': float, 'Price': float, 'Total': float, 'Commission': float, 'Quantity': float }, parse_dates=[0], thousands=',') log.info(f'Ignore transactions in {conf("skip_accounts")} accounts.') inv = inv.drop(inv[inv.Account.isin(conf('skip_accounts'))].index) return inv
def __init__(self, robot, portNo=conf("network.port.main")): super().__init__() self.setDaemon(True) # Le thread s'arrête en même temps que le process self._robot = robot self._direction = True # True : avancer, False : reculer self._sock = socket() self._sock.bind(("", portNo)) self._sock.listen() self._pad = False
def connectAndStartSwitch(data=None): c = conf() basic_url = "http://%s:%s/lumikki/ttm_run2" % (c.get('ttm_ip'),c.get('ttm_port')) url = basic_url if data != None: url += "?%s" % urlencode(data) connection = urlopen(url, timeout=timeout) return connection
def get(self, gitpath): try: gitindex = conf('buildbot.gitindex.%s' % gitpath) except KeyError: return Response(status=404, body='Unknown git path') response = urllib.urlopen(gitindex) index = response.read() index = [x.strip('\r\n ').split(' ')[0].rsplit('.')[0] for x in index.split('\n') if x.strip('\r\n ')] return Response(status=200, body=dumps(index))
def _make_comparison_graph(account_names, lines, min_date, max_date): if not account_names: return None traces = { n: calculate_traces([n], min_date, max_date) for n in account_names } line = _standardize_lines(lines)[0] plots = [] for name, trace in traces.items(): y = trace[line.split(':')[0]] plots.append( go.Scatter(x=y.index, y=y, mode='lines', name=name, yaxis='y1')) return { 'data': plots, 'layout': get_layout(conf('which_axis')[line], [conf('lines')[line]]), }
def day(self, mode='daily'): path = conf('day') # 自定义输入保存路径 if mode == 'daily': yesterday = str(datetime.date.today() - datetime.timedelta(days=1)) else: yesterday = str(datetime.date.today() - datetime.timedelta(days=1)) + '_r18' path = os.path.join(path, yesterday) self._day_parser(path, mode)
def connectAndStartCam(data): c = conf() basic_url = "http://%s:%s/camera/manualCamera" % (c.get('cam_ip'),c.get('cam_port')) url = basic_url if data != None: url += "?%s" % urlencode(data) connection = urlopen(url, timeout=timeout) return connection
def post(self, dist=None, package=None, action=None): repo = Repository(conf('repository.path')) if not dist or not package or not action: return Response(status=405) if action == 'copy': if not 'dstdist' in self.request.params: return Response(status=400, body='A required parameter, dstdist is missing') repo.copy_package(dist, self.request.params['dstdist'], package) return Response(status=200)
def __init__(self, canal, pin_negatif, pin_positif): super().__init__(0, 4095, canal) if conf("debug.controle"): print("DCMoteur: créé : ", self) self._pin_negatif = pin_negatif self._pin_positif = pin_positif self._sens = True self._changement_sens = True GPIO.setup(self._pin_negatif, GPIO.OUT, initial=GPIO.LOW) GPIO.setup(self._pin_positif, GPIO.OUT, initial=GPIO.LOW)
def sign(self, dist): self._reprepro('export %s' % dist) gpg = GPG() filename = os.path.join(self.path, 'dists/%s/Release' % dist) detach_file = filename + '.gpg' try: os.unlink(detach_file) except: pass result = gpg.sign_file(file(filename, 'r'), keyid=conf('repository.signkey'), outputfile=detach_file)
def delete(self, dist=None, package=None, action=None): repo = Repository(conf('repository.path')) if action: return Response(status=405, body='You cannot delete an action') if not dist or not package: return Response(status=400, body='You must specify a dist and package to delete from it') result = repo.remove_package(dist, package) if result: return Response(status=404, body=result) return Response(status=200)
def distanceDecay(start_point, data, radius): import config connection_string = config.conf() # first turning point (.25 miles) x1 = 402.336 y1 = 1 #second turning point (1 mile) x2 = float(radius) * 1000 #1609.34 y2 = 0 #in order to get rid of division by zero when x1=x2 #the point is when x2<x1 (threshold) then all the weights will be 1 if x1 == x2: x2 -= 1 #linear equation m = (y2 - y1) / float(x2 - x1) b = y2 - m * x2 conn = psycopg2.connect(connection_string) cur = conn.cursor() start_point = start_point.split(',') longitude = start_point[1] latitude = start_point[0] start_point = 'POINT (%s %s)' % (longitude, latitude) data_json = json.loads(data) ref_poi_list = {} for item in data_json['features']: location = item['geometry']['coordinates'] location = 'POINT (%s %s)' % (location[0], location[1]) poi_type = str(item['properties']['type']) #check if the requirement is met (true/false) _isContinued = check_no_of_poi(ref_poi_list, poi_type) if _isContinued: selectQuery = "SELECT ST_Distance(ST_Transform(ST_GeomFromText(%s, 4326), 3776), " \ "ST_Transform(ST_GeomFromText(%s, 4326), 3776));" parameters = [start_point, location] cur.execute(selectQuery, parameters) rows = cur.fetchall() distance = rows[0][0] if distance < x1: weight = 1 if (distance > x1) and (distance < x2): weight = m * distance + b if distance > x2: weight = 0 if poi_type in ref_poi_list: ref_poi_list[poi_type].append(weight) ref_poi_list[poi_type].sort(reverse=True) else: ref_poi_list[poi_type] = ''.split() ref_poi_list[poi_type].append(weight) ref_poi_list[poi_type].sort(reverse=True) return ref_poi_list
def connectToCommand(data=None): c = conf() # TODO: no hardcoded values # move ae_service to config.py as variable # move vi_basename to config.py as variable url = "http://%s:%s/%s/%s" % (c.get('ae_ip'),c.get('ae_port'),c.get('ae_service'),c.get('ae_viname')) if data != None: # this converts dict to ?key1=value1&key2=value2&... url += "?%s" % urlencode(data) connection = urlopen(url, timeout=timeout) return connection
def generate(env): print '---- Loading Config: ' + repr(env) + ' -----' c = config.conf(env) # In real life the loaded config could be used to do template substitution in a weblogic config.xml.template # Now we just print some env specific values print 'c.ms_suffixes:', repr(c.ms_suffixes) print 'c.admin_server:', repr(c.admin_server) print 'c.managed_servers:', repr(c.managed_servers) print 'c.datasources:', repr(c.datasources) print '---- Loaded: ' + repr(c.selected_env) + ' -----' print
def load(app, execution=True, redis_session=False, mongo_database=None, name=None, models=None): global APP load_all() debug = config.conf("DEBUG", False) redis_url = config.conf("REDISTOGO_URL", None) mongo_url = config.conf("MONGOHQ_URL", None) smtp_host = config.conf("SMTP_HOST", None) smtp_user = config.conf("SMTP_USER", None) smtp_password = config.conf("SMTP_PASSWORD", None) if not debug and name: start_log(app, name) if redis_url: redisdb.url = redis_url if mongo_url: mongodb.url = mongo_url if smtp_host: mail.SMTP_HOST = smtp_host if smtp_user: mail.SMTP_USER = smtp_user if smtp_password: mail.SMTP_PASSWORD = smtp_password if execution: start_execution() if redis_session: app.session_interface = session.RedisSessionInterface(url=redis_url) if mongo_database: mongodb.database = mongo_database if models: setup_models(models) app.request_class = request.Request APP = app
def post(self, gitpath, gitrepo): if not 'ref' in self.request.params: return Response(status=400, body='Required parameter "ref" is missing. You must pass a git tag, branch, or commit ID to be built.\n') gitpath = os.path.join(conf('buildbot.gitpath.%s' % gitpath), gitrepo) ref = self.request.params['ref'] cburl = self.request.params.get('cburl', None) submodules = self.request.params.get('submodules', None) buildid = uuid.uuid4().hex build_worker(gitpath, ref, buildid, cburl, submodules) return Response(status=200, body=buildid + '\n')
def post(self): repo = Repository(conf('repository.path')) dist = { 'Version': '5.0', 'Architectures': 'amd64 source any', 'Components': 'main contrib non-free', 'Description': 'Default package repository', } dist.update(json.loads(self.request.body)) for field in ['Origin', 'Label', 'Suite', 'Codename']: if not field in dist: return Response(status=400, body='Required field %s is missing.' % field) repo.create_dist(dist)
def get(self, buildid): builddir = os.path.join(conf('buildbot.buildpath'), buildid) if not os.path.exists(builddir): return Response(status=404, body='The build ID does not exist.\n') try: log = file('%s/build.log' % builddir, 'r').read() except: log = '' if not os.path.exists(builddir + '/package.tar.gz'): return Response(status=400, body='The build is not done yet.\n' + log) else: return Response(status=200, body='Build complete.\n' + log)
def get(self, buildid): builddir = os.path.join(conf('buildbot.buildpath'), buildid) if not os.path.exists(builddir): return Response(status=404, body='The build ID does not exist.\n') tarpath = os.path.join(builddir, 'package.tar.gz') if not os.path.exists(tarpath): return Response(status=400, body='The build is not done yet.\n') else: fd = file(tarpath, 'rb') data = fd.read() fd.close() return Response(status=200, body=data, content_type='application/x-tar-gz')
def connectAndSendStart(): ''' TODO: this is the only function that is called from cgi. This includes if-else for constant load, speed, etc ''' c = conf() if c.get('ttm_creep_experiment') == True: data = {"load": c.get('ttm_load')} else: ret = ttm.connectAndInitRamp() ret = ttm.connectAndStartLogging() ret = ttm.connectAndStartRamp()
def post(self, dist=None, package=None, action=None): repo = Repository(conf('repository.path')) if not dist or not package or not action: return Response(status=405) if action == 'copy': if not 'dstdist' in self.request.params: return Response(status=400, body='A required parameter, dstdist is missing') if not repo.get_package(dist, package): return Response(statuss=404, body='Package %s not found') repo.copy_package(dist, self.request.params['dstdist'], package) return Response(status=200)
def connectAndStart(): c = conf() data = {"offsetY": c.get('cam_yoffset'), "offsetX": c.get('cam_xoffset'), "fps": c.get('cam_rawfps'), "width": c.get('cam_width'), "height": c.get('cam_height'), "testName": c.get('g_measurementid'), "packSize": c.get('cam_packetsize'), "index": c.get('cam_index'), "exposure": c.get('cam_exposure'), "parentFolder": c.get('cam_parentfolder') } return connectAndStartCam(data)
def post(self, dist): repo = Repository(conf('repository.path')) response = None basedir = '/tmp/repoman.upload/%s' % uuid.uuid4().hex os.makedirs(basedir) os.chdir(basedir) field = self.request.params['package'] name = os.path.basename(field.filename) if not name.endswith('tar.gz') and not name.endswith('tar.bz2'): return Response(status=400, body='Packages must be uploaded as .tar.gz or tar.bz2 files containing .changes, .dsc, and .deb files') fd = file(name, 'wb') fd.write(field.value) fd.close() tf = tarfile.open(name, 'r|*') tf.extractall() changesfile = [x for x in os.listdir(basedir) if x.endswith('.changes')] if not changesfile: return Response(status=400, body='Tarball does not contain a .changes file') packages = [] for changes in changesfile: changes = os.path.join(basedir, changes) stderr, stdout = repo.add_package(dist, changes)[0] if stdout: logging.debug('add_package: %s' % stdout) if stderr: logging.warning('add_package: %s' % stderr) for p in [x.split(': ', 1)[1].rstrip('\r\n').split(' ') for x in file(changes, 'r').readlines() if x.startswith('Binary: ')]: for bin in p: pkg = repo.get_package(dist, bin) packages.append(pkg) response = Response(status=200, body=dumps(packages)) for dirpath, dirnames, filenames in os.walk(basedir): for filename in filenames: filename = os.path.join(dirpath, filename) os.remove(filename) os.rmdir(basedir) if not response: response = Response(status=500) return response
def generate(env): print('---- Instantiating config for env: ' + repr(env) + ' -----') c = config.conf(env) print('\n---- Printing entire config as "compact" json: -----') # json will dump property method values as well as multiconf property values # compact give a more human readable output print(c.json(compact=True)) # In real life the loaded config could be used to do template substitution in a weblogic config.xml.template # Now we just print some env specific values print('\n---- Access config objects/properties: -----') print('c.admin_server:', c.admin_server.json(compact=True)) # Repeatable objects are inserted in an ordered dict print('c.managed_servers["ms1"]:', c.managed_servers['ms1'].json()) print('ms1.port:', c.managed_servers['ms1'].port) print('c.datasources:', c.datasources) print()
def generate_rss_from_pubmed(input_string, feeds=50): """Returns the url of a rss generated at Pubmed by the queried string.""" input_string = urllib.quote_plus(input_string) pubmed_url = conf("pubmed_search") url = "{}?term={}".format(pubmed_url, input_string) read = request_url(url, list_iter=False, verbose=False) if read: # parse info from the response html - quick and dirty hid = '' qk = '' line = read.read() # the field to parse is the HID and qk if 'data-hid="' in line: hid = line.split('data-hid="')[1] hid = hid.split('"')[0] if 'data-qk="' in line: qk = line.split('data-qk="')[1] qk = qk.split('"')[0] dataload = {'p$site': 'pubmed', 'p$rq': 'EntrezSystem2.PEntrez.PubMed.Pubmed_SearchBar.Entrez_SearchBar:CreateRssFeed', 'QueryKey': qk, 'Db': 'pubmed', 'RssFeedName': input_string, 'RssFeedLimit': str(feeds), 'HID': hid} dataload = urllib.urlencode(dataload) read = post_url(pubmed_url, dataload, verbose=False) if read: rss_guid = '' line = read.read() # the field to parse is rss_guid if "rss_guid=" in line: rss_guid = line.split("rss_guid=")[2] rss_guid = rss_guid.split("'")[0] # url = "{}erss.cgi?rss_guid={}".format(conf("pubmed_rss"), rss_guid) return rss_guid else: return 0 else: return 0
def get(self, dist, package): repo = Repository(conf('repository.path')) if dist and package: pkg = repo.get_package(dist, package) if not pkg: return Response(status=404, body=dumps([])) return Response(status=200, body=dumps(pkg)) if dist and not package: result = repo.get_packages(dist).keys() if not result: return Response(status=404, body=dumps([])) return Response(status=200, body=dumps(result)) if not dist: result = repo.get_dists() if not result: return Response(status=404, body=dumps([])) return Response(status=200, body=dumps(result))
def connectAndStart(): ''' TODO: this is the only function that is called from cgi. This includes if-else for constant load, speed, etc ''' c = conf() data = {"protocolName": c.get('ttm_protocol'), "channelInt": c.get('ttm_channel'), "rampRateAE": c.get('ttm_ramprateae'), "rampRateSilent": c.get('ttm_rampratesilent'), "timeWindow": c.get('ttm_window'), "cycleLength": c.get('ttm_cyclelength'), "holdTime": c.get('ttm_holdtime'), "rampRate": c.get('ttm_ramprate'), "load": c.get('ttm_load'), "rampAmplitude": c.get('ttm_rampamplitude'), "expId": c.get('g_measurementid') } ret = connectAndStartSwitch(data)
def pointInCircle(start_point, radius): import config connection_string = config.conf() conn = psycopg2.connect(connection_string) cur = conn.cursor() radius = float(radius) * 1000 #in meters start_point = start_point.split(',') start_point = "SRID=4326;POINT(%s %s)" % (start_point[1], start_point[0]) selectQuery = "SELECT id, ST_AsText(crime_location), crime_time, crime_type " \ "FROM cps_crime_data.crime_data " \ "WHERE ST_DWithin(ST_GeogFromText(%s), geography(crime_location), %s);" parameters = [start_point, radius] cur.execute(selectQuery, parameters) rows = cur.fetchall() if len(rows) > 0: result_json = '{"type": "FeatureCollection", "features": [' for i in xrange(len(rows)): id = rows[i][0] location = rows[i][1] location = location[6:].split(' ') longitude = location[0] latitude = location[1][:-1] location = "[%s,%s]" % (longitude, latitude) time = rows[i][2] time = "%s-%s-%s %s:%s:00" % (time.year, time.month, time.day, time.hour, time.minute) type = rows[i][3] result_json += '{"type": "Feature","geometry": {"type": "Point", "coordinates":%s}, "properties": {"id": %s, "time": "%s","type": "%s"}},' % ( location, id, time, type) result_json = result_json[:-1] result_json += ']}' else: result_json = '"NULL"' conn.commit() cur.close() conn.close() return result_json
def connectAndGetStatus(): c = conf() responce = connectAndSend(c, createStatusMessage(c)) # TODO: this is a hack that will be fixed on # device side during summer 2011 rDict = {} splittedResponce = responce.split('\n') for item in splittedResponce: splittedItem = item.split(':',1) rDict[splittedItem[0].strip()] = splittedItem[1].strip() if not rDict.has_key('Errors'): return 240 if not rDict['Errors'] == '': return 240 if not rDict.has_key('Satus'): return 240 if rDict['Satus'].find('CAMERA IS STOPPED') >= 0: return 210 if rDict['Satus'].find('CAMERA IS ON') >= 0: return 220 if len(responce) == 0: return 240 return responce
def calculateArea(polygon): polygonJSON = json.loads(polygon) finalPolygon = "POLYGON((" for point in polygonJSON['coordinates'][0]: longitude = point[0] latitude = point[1] vertex = "%s %s" % (longitude, latitude) finalPolygon += "%s," % (vertex,) finalPolygon = finalPolygon[:-1] finalPolygon += "))" import config connection_string = config.conf() conn = psycopg2.connect(connection_string) cur = conn.cursor() select_query = "SELECT ST_Area(ST_Transform(ST_GeomFromText(%s, 4326), 3776));" parameters = [finalPolygon] cur.execute(select_query, parameters) rows = cur.fetchone() conn.commit() cur.close() conn.close() return rows[0]