def get_inventory(self): if self.inventory: return self.inventory latest_inventory_id = db_conn.query_one( self.query_for_latest_inventory, {"set_num": self.num})[0] config.get_logger("sets").info("latest inventory id: %s" % latest_inventory_id) self.inventory = parts.for_inventory_id(latest_inventory_id) return self.inventory
def run_dmrg(config): np.random.seed(config['random_seed']) log_directory = config['log_directory'] data_path = config['data_path'] test_fraction = config['test_fraction'] max_sweeps = config['max_sweeps'] patience = config['patience'] num_sites = config['num_sites'] bond_dimension = config['bond_dimension'] ix_to_char = config.get('ix_to_char') logger, save_name = get_logger() logger.info(config) tf_logger = Logger('tensorboard/{}'.format(save_name)) text = process_text(data_path, lower=True, remove_punctuation=False) char_to_ix, ix_to_char = encodings(text, ix_to_char) site_dimension = len(char_to_ix) numeric = prepare_numeric(text, char_to_ix) logger.info("Data has {} characters, {} unique.".format( len(text), len(char_to_ix))) train_batch, cv_batch, test_batch = data_split(numeric, num_sites, test_fraction) mps = random_gauged_mps(num_sites, site_dimension, bond_dimension) context = {'config': config, 'step': 0} stats_history = [mps_stats(mps, train_batch, cv_batch, test_batch)] sweep, cv_bumps = 1, 0 while sweep <= max_sweeps and cv_bumps <= patience: dmrg_sweep(mps, train_batch, context) stats = mps_stats(mps, train_batch, cv_batch, test_batch) stats_history.append(stats) log_sweep(logger, tf_logger, stats, sweep) save_path = '{}/{}/mps-after-step-{}.pickle'.format( log_directory, save_name, sweep) data_to_save = { 'config': config, 'mps': mps, 'ix_to_char': ix_to_char, 'save_name': save_name, 'sweep': sweep } save_object(data_to_save, save_path) logger.info("saved mps to: {}".format(data_path)) if config['generate_samples']: samples_per_sweep = config['samples_per_sweep'] samples_txt = list( generate_samples(mps, ix_to_char, samples_per_sweep)) for phrase in samples_txt: logger.info("sample phrase: {}".format(phrase)) sweep += 1 cv_bumps = update_cv_bumps(cv_bumps, stats_history) return stats
def __init__(self, database, config: str = 'default', log=None): self.app = create_app(config) self.app_context = self.app.test_request_context() self.app_context.push() self.client = self.app.test_client() database.app = self.app self.database = database self.log = log or get_logger('errors')
def main(): # import ipdb; ipdb.set_trace() config = get_args() os.environ["CUDA_VISIBLE_DEVICES"] = "{}".format(config.gpu_id) os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' logger = get_logger(config) voc = load_vocab(config) config.n_voc = len(voc.voc) logger.info(config) config.embeddings = voc.embeddings config.use_product_info = False config.use_user_info = False if config.use_user_info and config.use_product_info: usrdict = UserTable('../data/' + config.dataname + '/usrlist.txt') prddict = ProductTable('../data/' + config.dataname + '/prdlist.txt') config.n_users = usrdict.size + 1 config.n_products = prddict.size + 1 else: usrdict = None prddict = None logger.info("build model...") with tf.device("/device:{}:{}".format(config.device_type, config.gpu_id)): model = Model(config) logger.info("creating session...") gpu_options = tf.GPUOptions( per_process_gpu_memory_fraction=config.gpu_allocate_rate) gpu_options.allow_growth = True session_config = tf.ConfigProto(allow_soft_placement=True, gpu_options=gpu_options) sess = tf.Session(config=session_config) model.init_variable(sess) trainset, devset, testset = load_dataset(config, voc, usrdict, prddict) if config.load_model: logger.info("restoring model...") model.restore(sess) if not config.test_only: logger.info("starting training...") model.train(sess, trainset, devset, testset) logger.info("training done.") logger.info("starting testing...") test_acc, test_mae, test_rmse = model.evaluate(sess, testset) logger.info( "final result of testset: acc = {:.4f}, mae = {:.4f}, rmse = {:.4f}". format(test_acc, test_mae, test_rmse)) logger.info("testing done.")
def https_post(url, payload, params={}, ignore_status=False): '''Posts a data payload to Bill.com. It can optionally check for failed status. Args: payload (dict): A JSON compatible dict with data to send to bill.com's api. ignore_status (bool): If True, don't check for failed status codes. Returns: Dict of the JSON response. Raises: ServerReponseError HTTPError ''' LOG = get_logger() api_url = API_URL + '/' + url headers = {'content-type': 'application/x-www-form-urlencoded'} try: response = requests.post(api_url, params=params, data=payload, headers=headers) except Exception as e: raise ServerResponseError('Could not post to {0}: {1}'.format( api_url, e)) if response.status_code not in OK_CODES: message = "received HTTP {0}: {1} when sending to {2}: {3}".format( response.status_code, response.text, API_URL, payload) LOG.error(message) raise HTTPError(message) try: data = json.loads(response.text) status, message = get_status_and_message(data) except: message = 'sent {0} got badly formatted reponse: {1}'.format( payload, response.text) LOG.error(message) LOG.error("SENT TO {}: {}".format(response.url, payload)) LOG.error("RECEIVED {}".format(data)) raise if not ignore_status and status and status != 'OK': LOG.error(message) LOG.error("SENT TO {}: {}".format(response.url, payload)) LOG.error("RECEIVED {}".format(data)) raise ServerResponseError(message) return data['response_data']
def __init__(self, name, image, shell_cmd, shell_args): if not os.path.exists("/var/run/docker.sock"): raise EnvironmentError("Need installed docker.") self.logger = config.get_logger().getChild(__name__) self.docker_cmd = "docker run --rm" self.add_volume("/etc/localtime", "/etc/localtime", "ro") self.set_tty() self.name = name self.image = image self.shell_cmd = shell_cmd self.shell_args = shell_args self.dockerfile_path = None
def __init__(self, args_as_dict): ''' :arg dict args_as_dict: a dictionary ''' # pylint:disable=no-member # no-member (pylint E1101): instance has no member. in this case it # does because we're messing with the __dict__ member self.__dict__ = dict(args_as_dict) if self.debug: self.log_level = logging.DEBUG self.logger = config.get_logger('pyro4_netem', log_path=self.log_directory, log_level=self.log_level)
def create_app(config_name: str): """Creates a Flask app instance using one of predefined configs.""" app = Flask(__name__) conf_obj = config[config_name] app.config.from_object(conf_obj) conf_obj.init_app(app) db.init_app(app) app.custom_logger = get_logger('main') from .main import main as main_blueprint app.register_blueprint(main_blueprint) return app
def https_post(url, payload, params={}, ignore_status=False): '''Posts a data payload to Bill.com. It can optionally check for failed status. Args: payload (dict): A JSON compatible dict with data to send to bill.com's api. ignore_status (bool): If True, don't check for failed status codes. Returns: Dict of the JSON response. Raises: ServerReponseError HTTPError ''' LOG = get_logger() api_url = API_URL + '/' + url headers = {'content-type': 'application/x-www-form-urlencoded'} try: response = requests.post(api_url, params=params, data=payload, headers=headers) except Exception as e: raise ServerResponseError('Could not post to {0}: {1}'.format(api_url, e)) if response.status_code not in OK_CODES: message = "received HTTP {0}: {1} when sending to {2}: {3}".format( response.status_code, response.text, API_URL, payload ) LOG.error(message) raise HTTPError(message) try: data = json.loads(response.text) status, message = get_status_and_message(data) except: message = 'sent {0} got badly formatted reponse: {1}'.format(payload, response.text) LOG.error(message) LOG.error("SENT TO {}: {}".format(response.url, payload)) LOG.error("RECEIVED {}".format(data)) raise if not ignore_status and status and status != 'OK': LOG.error(message) LOG.error("SENT TO {}: {}".format(response.url, payload)) LOG.error("RECEIVED {}".format(data)) raise ServerResponseError(message) return data['response_data']
def __init__(self, in_spec_str, out_spec_str, err_spec_str=[]): # spec string : [int, ...] or [[(str, int)], (str, str), ...] # dictionary must specify "key" value, but "value" gives type ex: {2:str, # "dsadasdas":int} self.logger = config.get_logger().getChild(__name__) self.logger.debug( "__init__" + repr((in_spec_str, out_spec_str, err_spec_str))) if not isinstance(in_spec_str, list) or not isinstance(in_spec_str, list) or not isinstance(in_spec_str, list): raise TypeError("Spec string should be list") self.input_specs = in_spec_str self.output_specs = out_spec_str self.err_specs = err_spec_str self.TYPE_GEN_HANDLERS[tuple] = self.gen_by_spec self.TYPE_GEN_HANDLERS[list] = self.gen_by_spec self.TYPE_GEN_HANDLERS[dict] = self.gen_dict_by_spec
def load_scenario( scenario_name ): """ * a global variable 'wallet' * a global variable 'consensus' * a callable method 'scenario' * a callable method 'check' """ log = get_logger("ZONEFILEMANAGE") log.debug("Load scenario %s " % sys.argv[1]) # strip .py from scenario name if scenario_name.endswith(".py"): scenario_name = scenario_name[:-3] try: scenario = importlib.import_module(scenario_name) except ImportError, ie: raise Exception("Failed to import %s" % scenario_name)
def __init__(self, input_file, output_file, moses_opts, session_id, target_feature="case"): """ :param input_file: The input file to run MOSES on :param output_file: The file to write MOSES program outputs to :param moses_opts: MOSES parameters """ self.input = input_file self.output = output_file self.moses_options = moses_opts if not "W1" in moses_opts: moses_opts += ' -W1' self.output_regex = re.compile(r"(-?\d+) (.+) \[(.+)\]") self.logger = get_logger(session_id) self.target_feature = target_feature
def start_analysis(**kwargs): """ A celery task that runs the MOSES analysis :param session: The session object :param cwd: Current working directory to store the results of the analysis :param db: A reference to the mongo database :return: """ db = pymongo.MongoClient(MONGODB_URI)[DB_NAME] swd, file_path = write_dataset(kwargs["dataset"], kwargs["mnemonic"]) session = Session(kwargs["id"], kwargs["moses_options"], kwargs["crossval_options"], file_path, kwargs["mnemonic"], kwargs["target_feature"]) logger = get_logger(session.mnemonic) session.save(db) session.status = 1 session.progress = 1 session.start_time = time.time() session.update_session(db) try: filter_type, value = kwargs["filter_opts"]["score"], kwargs[ "filter_opts"]["value"] filter_cls = loader.get_score_filters(filter_type) moses_cross_val = CrossValidation(session, db, filter_cls, value, swd) logger.info("Started cross-validation run") moses_cross_val.run_folds() logger.info("Cross-validation done successfully") session.status = 2 session.message = "Success" session.progress = 100 except Exception as e: session.status = -1 session.message = e.__str__() logger.error( f"Task failed with {traceback.format_exception(etype=type(e), value=e, tb=e.__traceback__)}" ) finally: session.end_time = time.time() session.update_session(db)
def __init__(self, params, is_predict=False): super().__init__() global logger logger = get_logger(__name__, params['model_config_name'] + '.log') if params.get('type') and params['type'] == "cnn": self.model = CNN_Net(params['layers']) else: self.model = Net(params['layers']) if not is_predict: if params.get('runAllIters'): self.allIters = params['runAllIters'] else: self.allIters = True self.log_step = log_config['pytorch'] self.criterion = nn.MSELoss() self.optimizer = optim.SGD(self.model.parameters(), lr=params['lr']) self.iteraions = params['iters'] self.trainDatsetPart = params['trainDatasetPart']
def StartAnalysis(self, request, context): session_id = uuid.uuid4() mnemonic = encode(session_id) logger = get_logger(mnemonic) crossval_opts = { "folds": request.crossValOpts.folds, "testSize": request.crossValOpts.testSize, "randomSeed": request.crossValOpts.randomSeed } moses_opts, dataset, target_feature = request.mosesOpts, request.dataset, request.targetFeature filter_opts = { "score": request.filter.score, "value": request.filter.value } logger.info( f"Received request with Moses Options: {moses_opts}\n Cross Validation Options: {crossval_opts}\n" ) if is_valid_dataset(dataset, target_feature): start_analysis.delay(id=session_id, moses_options=moses_opts, crossval_options=crossval_opts, filter_opts=filter_opts, dataset=dataset, mnemonic=mnemonic, target_feature=target_feature) url = f"{MOZI_URI}/?id={mnemonic}" logger.info(f"Session {session_id} analysis started.") return Result(resultUrl=url, description="Analysis started") else: context.set_code(grpc.StatusCode.INVALID_ARGUMENT) context.set_details( f"Invalid dataset.Dataset doesn't contain a column with named {target_feature} or has " f"invalid characters") logger.error("Error occurred while validating request") return Result( resultUrl="", description= f"Validation error occurred. Dataset doesn't contain a column with named {target_feature} or has invalid characters" )
def __init__(self, session, db, filter_type, value, cwd): """ :param session: The session object that has the moses, cross-validation options and other metadata :param db: The db connection object :param cwd: Current working directory to save files to """ self.session = session self.cwd = cwd self.db = db self.filter = filter_type self.filter_value = value self.logger = get_logger(session.mnemonic) self.fold_files = [] self._set_dir() self.total_runs = (self.session.crossval_options["folds"] * self.session.crossval_options["randomSeed"]) + 1 self.runs = 0 self.tree_transformer = ComboTreeTransform() self.result_models = {} self.logger.info(f"Current working dir: {os.getcwd()}")
class Configurator(): """ Class reader configs for datasets and models """ logger = get_logger(__name__ + '.Configurator') pytorch_train_eps = 5e-2 def get_dataset_config(name): return Configurator.__read_file(name, 'dataset') def get_model_config(name): return Configurator.__read_file(name, 'model') def get_predict_config(name): return Configurator.__read_file(name, 'predict') def __read_file(name, file_type): path = os.path.abspath('.') + f'/configs/{file_type}/{name}.json' if not os.path.exists(path): Configurator.logger.error(f'{file_type} config {name} not exist') raise FileNotFoundError(f'{file_type} config {name} not exist') with open(path, 'rb') as f: return json.load(f)
from config import CURRENT_EXP_DIR, config, get_logger, log_config logger = get_logger(exp_dir=CURRENT_EXP_DIR) log_config(logger) import numpy as np import sherpa import sherpa.algorithms.bayesian_optimization as bayesian_optimization from cade.metrics.comparative import intersection_nn, lncs2, moving_lncs2 from gensim.models.word2vec import Word2Vec from scipy.spatial.distance import cosine from sklearn.metrics import ( accuracy_score, f1_score, precision_score, recall_score, ) def fitness(threshold: float, topn: int, t: float): similarity = 3 # Task 1 - Binary Classification predictions = [] for word in binary_truth[:, 0]: if similarity == 0: # cosine similarity prediction = (0 if 1 - cosine(model1[word], model2[word]) >= threshold else 1) elif similarity == 1: # lncs2 similarity prediction = (0 if lncs2(word, model1, model2, topn) >= threshold
from __future__ import print_function import os from six import iteritems import config as bconfig from invenio.legacy.bibclassify import ontology_reader as reader import text_extractor as extractor import text_normalizer as normalizer import keyword_analyzer as keyworder import acronym_analyzer as acronymer from invenio.utils.url import make_user_agent_string from invenio.utils.text import encode_for_xml log = bconfig.get_logger("bibclassify.engine") # --------------------------------------------------------------------- # API # --------------------------------------------------------------------- def output_keywords_for_sources(input_sources, taxonomy_name, output_mode="text", output_limit=bconfig.CFG_BIBCLASSIFY_DEFAULT_OUTPUT_NUMBER, spires=False, match_mode="full", no_cache=False, with_author_keywords=False, rebuild_cache=False, only_core_tags=False, extract_acronyms=False, api=False, **kwargs): """Output the keywords for each source in sources.""" # Inner function which does the job and it would be too much work to # refactor the call (and it must be outside the loop, before it did
import json import logging.config from flask import Flask, render_template, request from config import get_logger from helper import get_activity app = Flask(__name__) LOGGING = get_logger() logging.config.dictConfig(LOGGING) logging.captureWarnings(True) logger = logging.getLogger(__name__) @app.route('/', methods=['GET', 'POST']) def index(): activity_type = request.form.get('type') response = get_activity(activity_type=activity_type) response_json = response.json() status_code = response.status_code return render_template('index.html', response=response_json), response.status_code
import sys import os import config from utils import req from itertools import combinations_with_replacement import simplejson import datetime log = config.get_logger("rfun.measure_qf") def run(solr_url, query, haystack, qf="title,author", min=0.0, max=2.0, increment=0.25, max_hits=100): min = float(min) max = float(max) increment = float(increment) max_hits = int(max_hits) if (os.path.exists(query)): queries = load_queries(query) else: queries = [query] if (os.path.exists(haystack)):
import os from protocoin.serializers import * from protocoin.clients import * from protocoin.fields import * import protocoin from keys import version_byte as VERSION_BYTE from config import get_logger log = get_logger("spv") GENESIS_BLOCK_HASH = None GENESIS_BLOCK_MERKLE_ROOT = None USE_MAINNET = False USE_TESTNET = False BLOCK_HEADER_SIZE = 80 + 1 GENESIS_BLOCK_HASH_MAINNET = "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f" GENESIS_BLOCK_MERKLE_ROOT_MAINNET = "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b" GENESIS_BLOCK_HASH_TESTNET = "0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206" GENESIS_BLOCK_MERKLE_ROOT_TESTNET = "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b" BLOCK_DIFFICULTY_CHUNK_SIZE = 2016 BLOCK_DIFFICULTY_INTERVAL = 14 * 24 * 60 * 60 # two weeks, in seconds if VERSION_BYTE == 0: log.debug("Using mainnet") USE_MAINNET = True GENESIS_BLOCK_HASH = GENESIS_BLOCK_HASH_MAINNET
import os import time import json import config as conf import torch BEGIN, logger, LOG_PATH = conf.get_logger() class Game: def __init__(self, env_cls, nets_dict, dqns_dict, reward_dict=None, train_dict=None, preload=None, seed=None, debug=False): if reward_dict is None: reward_dict = {'lord': 100, 'down': 50, 'up': 50} if train_dict is None: train_dict = {'lord': True, 'down': True, 'up': True} if preload is None: preload = {} assert not (nets_dict.keys() ^ dqns_dict.keys()), 'Net and DQN must match' self.lord_wins, self.down_wins, self.up_wins = [], [], [] self.lord_total_loss = self.down_total_loss = self.up_total_loss = 0 self.lord_loss_count = self.down_loss_count = self.up_loss_count = 0
from bs4 import BeautifulSoup import requests import re import os from subprocess import check_output from time import sleep import logging from datetime import datetime import config logger = config.get_logger('wishlist') BASE_URL = 'https://www.amazon.com/gp/registry/wishlist/' PETES_WISHLIST_ID = '1ZF0FXNHUY7IG' def get_items_from_local_file(filename=None): if filename is None: current_folder = os.path.dirname(os.path.realpath(__file__)) filename = os.path.join(current_folder,'data','wishlist_02_19_2018.html') soup = BeautifulSoup(open(filename), 'html.parser') all_asin_inputs_in_soup = soup.findAll('input',{'name':'itemId'}) asin_values = [] for asin_input in all_asin_inputs_in_soup:
import random from Crypto.Hash import SHA256 as HashAlg from Crypto.PublicKey import RSA as CryptoKey from Crypto import Random from Crypto.Signature import PKCS1_PSS as CryptoSigner import syndicate.util.objects as object_stub import syndicate.util.crypto as crypto import client import config as conf import syndicate.protobufs.ms_pb2 as ms_pb2 import syndicate.protobufs.sg_pb2 as sg_pb2 import syndicate.ms.msconfig as msconfig log = conf.get_logger("syndicate-certs") def syndicate_public_key_name( ms_url ): """ Get the name of a syndicate public key, given the MS url it was fetched from. """ host, port, no_tls = client.parse_url( ms_url ) return host + ":" + str(port) def syndicate_public_key_fetch( ms_url, downloader_path ): """ Use a helper program to go and fetch the Syndicate public key. Return the key itself on success. Return None on error
from db import User, Wish, Role, ModerationStatus, Token import config import social cache = SimpleCache() SECRET_KEY = str(config.SECRET_KEY) TRAP_HTTP_EXCEPTIONS = True PERMANENT_SESSION_LIFETIME = 60*60*24*31 app = Flask(__name__) app.config.from_object(__name__) handler = config.get_logger(__file__).handlers[0] app.logger.addHandler(handler) def render_template_my(*args, **kwargs): currentPage = "" if "s" in request.args: currentPage = "s%s" % request.args["s"] if "f" in request.args: currentPage = "f" kwargs["current_page"] = currentPage kwargs["debug"] = config.DEBUG
This module contains methods to extract keywords from texts. It provides 3 different methods for 3 different types of keywords: single keywords, composite keywords and author keywords. This module is STANDALONE safe """ from __future__ import print_function import re import time import config as bconfig log = bconfig.get_logger("bibclassify.keyword_analyzer") _MAXIMUM_SEPARATOR_LENGTH = max([len(_separator) for _separator in bconfig.CFG_BIBCLASSIFY_VALID_SEPARATORS]) # XXX - rebuild this whole thing def get_single_keywords(skw_db, fulltext): """Find single keywords in the fulltext @var skw_db: list of KeywordToken objects @var fulltext: string, which will be searched @return : dictionary of matches in a format { <keyword object>, [[position, position...], ], .. }
#!/usr/bin/env python # -*- coding: utf-8 -*- import re import commands as run import config as cfg logging = cfg.get_logger() config = cfg.get_config() def last_commit(repopath): """ Gets the last commit information. This can be used to verify the latest commit """ command = "cd %s; git log -1i --date=iso" % repopath out = run.command(command) if out: creg = re.compile(r"commit\s+(?P<remote_host>([a-f0-9]+))") # Commitid areg = re.compile(r"Author:\s+(?P<author>(.*$))") # author dreg = re.compile(r"Date:\s+(?P<date>(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}))") # Date last = {} for line in out.splitlines(): commit = creg.search(line) author = areg.search(line) date = dreg.search(line) if commit: last['id'] = commit.group(1) elif author: last['author'] = author.group(1) elif date: last['date'] = date.group(1)
def __init__(self): self.logger = config.get_logger().getChild( __name__ + "." + self.__class__.__name__) self.app_dict = {}
def __init__(self, cmd_type, url, bucket): self.logger = config.get_logger().getChild(__name__) self._cmd_type = cmd_type self._url = url self._bucket = bucket
import random from Crypto.Hash import SHA256 as HashAlg from Crypto.PublicKey import RSA as CryptoKey from Crypto import Random from Crypto.Signature import PKCS1_PSS as CryptoSigner import syndicate.util.objects as object_stub import syndicate.util.crypto as crypto import client import config as conf import syndicate.protobufs.ms_pb2 as ms_pb2 import syndicate.protobufs.sg_pb2 as sg_pb2 import syndicate.ms.msconfig as msconfig log = conf.get_logger("syndicate-certs") def syndicate_public_key_name(ms_url): """ Get the name of a syndicate public key, given the MS url it was fetched from. """ host, port, no_tls = client.parse_url(ms_url) return host + ":" + str(port) def syndicate_public_key_fetch(ms_url, downloader_path): """ Use a helper program to go and fetch the Syndicate public key. Return the key itself on success.
# Purpose: Utility wrappers for persistence of suppporting data for Tree View import hashlib import config # Config logger = config.get_logger() allowed_extensions = set(['csv']) def to_logging(df, floor_height_column, face_direction_column, window_vertical_position_column): da = to_dictarray( df, { floor_height_column: [], face_direction_column: [], window_vertical_position_column: [] }) dump_dictarray_head(da) dump_dictarray_tail(da) def to_dictarray(df, da): for df_row in df.itertuples(): dfdict_row = df_row._asdict() dr = {} for column in da.keys(): try: da[column].append(float(dfdict_row[column])) except Exception as e:
import os from os.path import isfile as fileExists from os.path import join as pathJoin from os.path import dirname from os.path import abspath import json import makeUtils import timeout import config from ..adb import adb from ..androidUtils import fileUtils from ..dockerBuildUtils import dockerBuildUtils logger = config.get_logger().getChild(__name__) THIS_DIR = abspath(dirname(__file__)) HCFS_LIB_PHONE_PATH = "/system/lib64/libHCFS_api.so" HCFS_LIB = THIS_DIR + "/libHCFS_api.so" ADAPTER_NAME = "adapter" ADAPTER_BIN_LOCAL = pathJoin(THIS_DIR, ADAPTER_NAME) ADAPTER_BIN = "/data/" + ADAPTER_NAME def setup(): logger.info("adapter setup") cleanup() check_build_env() get_hcfs_lib_from_phone()
import tornado.ioloop import tornado.options import tornado.web import tornado.escape import json import time from tornado.options import define, options sys.path = ["."] + sys.path import config import model import repository import rest_json logger = config.get_logger(logging.WARNING, __name__) application = None define("port", default=9081, help="run on the given port", type=int) define("debug", default=False, type=bool) class BaseHandler(tornado.web.RequestHandler): def prepare(self): self.registry = repository.RepositoryRegistry() def __init__(self, application, request, **kwargs): super(BaseHandler, self).__init__(application, request, **kwargs) def set_default_headers(self):
import os import tempfile import errno import virtualchain import pybitcoin from config import get_logger, get_bitcoin_regtest_opts import name_service from blockchain.session import get_bitcoind_connection log = get_logger("testlib") snapshots_dir = None state_engine = None class TestAPIProxy(object): def __init__(self): global utxo_opts client_path = os.environ.get("ZONEFILEMANAGE_CLIENT_CONFIG", None) assert client_path is not None class Wallet(object): def __init__(self, pk_wif, ignored): pk = virtualchain.BitcoinPrivateKey(pk_wif)
import os import config import datetime from state_machine.nameset import indexer from config import get_logger log = get_logger("virtual_") def setup_virtualchain(impl=None): if impl is not None: config.set_implementation(impl) def sync_virtualchain(bitcoind_opts, last_block, state_engine, expected_snapshots={}, tx_filter=None): """ Synchronize the virtualchain state up until a given block """ rc = False start = datetime.datetime.now() while True: # advance the state try: rc = indexer.StateEngine.build( bitcoind_opts,
from blockstack_client import storage from blockstack_client import user as user_db from storage import hash_zonefile import pybitcoin import bitcoin import binascii from utilitybelt import is_hex from config import get_logger, DEBUG, MAX_RPC_LEN, find_missing, BLOCKSTACKD_SERVER, \ BLOCKSTACKD_PORT, BLOCKSTACK_METADATA_DIR, BLOCKSTACK_DEFAULT_STORAGE_DRIVERS, \ FIRST_BLOCK_MAINNET, NAME_OPCODES, OPFIELDS, CONFIG_DIR, SPV_HEADERS_PATH, BLOCKCHAIN_ID_MAGIC, \ NAME_PREORDER, NAME_REGISTRATION, NAME_UPDATE, NAME_TRANSFER, NAMESPACE_PREORDER, NAME_IMPORT, \ USER_ZONEFILE_TTL, CONFIG_PATH, get_config log = get_logger() def set_profile_timestamp(profile, now=None): """ Set the profile's timestamp to now """ if now is None: now = time.time() profile['timestamp'] = now return profile def get_profile_timestamp(profile): """
#!/usr/bin/env python # -*- coding: utf-8 -*- import urllib2 import json import config import db from db import User, Token logger = config.get_logger(__file__) logger.info("Start updating user info") db_session = db.Session() def get_users_info(user_ids): url = "https://api.vk.com/method/users.get?user_ids=%s&fields=photo_50,sex" % (user_ids) f = urllib2.urlopen(url) data = f.read() data = json.loads(data) if "response" not in data: logger.warning("Error while trying get user info from url %s" % url) return None return data["response"] tokens = db_session.query(Token).limit(20) user_ids = [] for token in tokens: user_ids.append(str(token.user_social_id))
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Development configuration """ import os from config import BaseConfig, get_logger logger = get_logger(__name__) class MyConfig(BaseConfig): HOST = '0.0.0.0' WTF_CSRF_SECRET_KEY = 'a random string' BASE_DIR = os.path.abspath(os.path.dirname(__file__)) UPLOAD_FOLDER = '/uploads' ALLOWED_EXTENSIONS = set(['txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif']) logger.info("Current config: DEVEL. Keepeing sqllite db.")
from profile import * from proxy import * from storage import hash_zonefile import pybitcoin import bitcoin import binascii from utilitybelt import is_hex from config import get_logger, DEBUG, MAX_RPC_LEN, find_missing, BLOCKSTACKD_SERVER, \ BLOCKSTACKD_PORT, BLOCKSTACK_METADATA_DIR, BLOCKSTACK_DEFAULT_STORAGE_DRIVERS, \ FIRST_BLOCK_MAINNET, NAME_OPCODES, OPFIELDS, CONFIG_DIR, SPV_HEADERS_PATH, BLOCKCHAIN_ID_MAGIC, \ NAME_PREORDER, NAME_REGISTRATION, NAME_UPDATE, NAME_TRANSFER, NAMESPACE_PREORDER, NAME_IMPORT, \ USER_ZONEFILE_TTL, CONFIG_PATH, get_utxo_provider_client, get_tx_broadcaster log = get_logger() import virtualchain def serialize_mutable_data_id( data_id ): """ Turn a data ID into a suitable filesystem name """ return urllib.quote(data_id.replace("\0", "\\0")).replace("/", r"\x2f") def load_mutable_data_version(conf, name, fq_data_id): """ Get the version field of a piece of mutable data from local cache. """
import errno import time import atexit from defusedxml import xmlrpc # prevent the usual XML attacks xmlrpc.monkey_patch() import signal import json import config as blockstack_config import backend import proxy log = blockstack_config.get_logger() from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler running = False # need to wrap CLI methods to capture arguments def local_rpc_factory( method, config_path ): """ Factory for producing wrappers around CLI functions """ import blockstack_client.method_parser as method_parser import blockstack_client.cli as cli method_info = method_parser.parse_methods( [method] )[0] def argwrapper( *args, **kw ):
import db import config import cache log = config.get_logger('colors') db_conn = db.get_instance() class Color(object): def __init__(self, data): try: self.id = data['id'] self.rgb = data['rgb'] self.is_trans = data['is_trans'] self.name = data['name'] except KeyError as e: log.error('missing key in data %s' % data, e) query_by_id = 'SELECT * FROM colors WHERE id=:id' def from_id(id): def create(): return Color(db_conn.query_one(query_by_id, {'id': id})) return cache.remember('color', id, create)
import sys import config from utils import req import datetime log = config.get_logger("rfun.find_citation_queries") """ Finds queries that are raising the number of returned hits """ def run(solr_url, query_name='citations', fl="title,author,recid,bibcode", min=0, max=4000000, increment=9, max_hits=10, max_qtime=60000 ): results = [] # first start with simple queries i = 0 j = None empty = 0 while False: i = i + 1
import re import string import struct import types from geom3d import Triangle, BBox, Vector, BinaryTree import config logger = config.get_logger('STL') class Stl: def __init__(self, *args): if type(args[0]) in (types.StringType, types.UnicodeType): self.filename = args[0] self.triangles = [] inf = open(self.filename) triangleList = self.from_input_file(inf) else: self.filename = self.preamble = '' triangleList = args self.triangles = triangleList bb = BBox() for tri in triangleList: bb = bb.expand(tri._bbox) self._bbox = bb self._classify_triangles_by_z() def from_input_file(self, inf): triangleList = [] R = inf.read()
are tuned to work with the output of pdftotext and documents in the HEP field. Methods can be tuned to your needs through the configuration file. This modules uses the refextract module of BibEdit in order to find the references section and to replace unicode characters. """ import re import config as bconfig from six import iteritems from invenio.legacy.docextract.pdf import replace_undesirable_characters from invenio.legacy.refextract.find import find_reference_section, find_end_of_reference_section log = bconfig.get_logger("bibclassify.text_normalizer") _washing_regex = [] def get_washing_regex(): global _washing_regex if len(_washing_regex): return _washing_regex washing_regex = [ # Replace non and anti with non- and anti-. This allows a better # detection of keywords such as nonabelian. (re.compile(r"(\snon)[- ](\w+)"), r"\1\2"), (re.compile(r"(\santi)[- ](\w+)"), r"\1\2"), # Remove all leading numbers (e.g. 2-pion -> pion).
from gravi import reverse from gravi.reverse.builder import complex_build from numpy.core.records import array from gravi.reverse import net, min from .data import DataCreator, DataReader, Configurator from datetime import datetime from .models.pytorch import ModelPyTorch import numpy.linalg as l import numpy as np import os from copy import copy from config import get_logger logger = get_logger(__name__) def research(size, dataset_name): i, o = prepare_data(size, dataset_name) learn(i, o, dataset_name) def learn(len_i, len_o, dataset_name, model_config_name, model_params=None): shape = 'default' if not model_params: model_params = Configurator.get_model_config(model_config_name) if model_params.get('type') and model_params['type'] == "cnn": shape = [ 1, model_params['shape']['in']['w'], model_params['shape']['in']['h'] ] else:
import config logger = config.get_logger('GEOM3D') # Floating point madness WIGGLE_ROOM = 1.0e-6 def sign(x): return x > 0.0 class Vector(object): def __init__(self, x, y=None, z=None): if y is None: self.x, self.y, self.z = x else: self.x, self.y, self.z = x, y, z def serialize(self): import cPickle return cPickle.dumps((self.x, self.y, self.z)) @classmethod def unserialize(self, str): import cPickle x, y, z = cPickle.loads(str) return Vector(x, y, z) def __repr__(self): return "<{0},{1},{2}>".format(self.x, self.y, self.z)
from flask_mail import Message from datetime import datetime, date from app import db, mail, app from app.models import Item, ParentItem, Image, Offer, LastRefreshed from amazon_api import get_parent_ASIN, get_item_attributes, get_amazon_api, get_images, get_item_variations_from_parent, get_offers from wishlist import get_items_from_wishlist, get_items_from_local_file from config import get_logger from sqlalchemy import or_ import os import logging logger = get_logger('refresh_data') WISHLIST_ID = '1ZF0FXNHUY7IG' MAILTO = '*****@*****.**' DEBUG = True def get_buybox_price(item): ''' take an Item object, return the buybox price Returns None if not ''' buybox_price = None for offer in item.offers.all(): if offer.offer_source == 'Buybox': buybox_price = offer.offer_price_amount
from state_machine import nameset as state_engine from bin.zonefilemanage_client import * wallets = [ #prvate key wif Wallet( "5JesPiN68qt44Hc2nT8qmyZ1JDwHebfoh9KQ52Lazb1m1LaKNj9", 100000000000 ), Wallet( "5KHqsiU9qa77frZb6hQy9ocV7Sus9RWJcQGYYBJJBb2Efj1o77e", 100000000000 ), Wallet( "5Kg5kJbQHvk1B64rJniEmgbD83FpZpbw2RjdAZEzTefs9ihN3Bz", 100000000000 ), Wallet( "5JuVsoS9NauksSkqEjbUZxWwgGDQbMwPsEfoRBSpLpgDX1RtLX7", 100000000000 ), Wallet( "5KEpiSRr1BrT8vRD7LKGCEmudokTh1iMHbiThMQpLdwBwhDJB1T", 100000000000 ) ] db_inst = None log = get_logger("ZONEFILEMANAGE") nameset_cache = [] class Pinger(threading.Thread): def __init__(self): threading.Thread.__init__(self) self.running = False def run(self): self.running = True bitcoind = bitcoin_regtest_connect(bitcoin_regtest_opts()) while self.running: try: bitcoind.ping()
#!/usr/bin/env python3 import psycopg2 import spacy.en import attribute_utils from src.model import Question, Attributes import sys from os.path import dirname, abspath sys.path.insert(0, dirname(dirname(abspath(__file__)))) import config logger = config.get_logger() def parse(question: str) -> Question: # returns a list of question's attributes try: parse.nlp except AttributeError: parse.nlp = spacy.en.English() print("attributes logger name=" + logger.name) doc = parse.nlp(question) result = Attributes() try: result.location = attribute_utils.get_attribute_location_spacy(doc) result.named_entity = attribute_utils.get_attribute_named_entity(question) result.time = attribute_utils.get_attribute_time_spacy(doc, question) result.action = attribute_utils.get_attribute_action(doc) result.product = attribute_utils.get_attribute_product(question) except KeyError as e: logger.error("KeyError. Json config doesn't contain such key:", str(e)) raise e
# @Email: [email protected] # @Last modified by: lorenzocorneo # @Last modified time: 2016-10-28T15:04:05+02:00 #!/usr/bin/python import MySQLdb import itertools import config from collections import defaultdict from models.EndNode import EndNode from models.Measurement import Measurement from models.HopMeasurement import HopMeasurement from models.Path import Path dup_log = config.get_logger("duplicates", config.log_patch_with_duplicates) dbname = config.db_credentials["db"] GET_NODES_QUERY = "SELECT * FROM {}.EndNodes".format(dbname) GET_TRACEDIRECTIONS_QUERY = "SELECT * FROM {}.TraceDirections".format(dbname) GET_PATHS_QUERY = "SELECT * FROM {}.Paths".format(dbname) GET_HOPS_QUERY = "SELECT * FROM {}.Hops".format(dbname) GET_MEASUREMENTS_QUERY = "SELECT * FROM {}.Measurements".format(dbname) GET_HOPMEASUREMENTS_QUERY = "SELECT * FROM {}.HopMeasurements".format(dbname) INSERT_ENDNODE_QUERY = "INSERT INTO {}.EndNodes (ip, name) VALUES ('%s', '%s');".format( dbname) UPDATE_ENDNODE_QUERY = "UPDATE {}.EndNodes set ip='%s', name='%s' where idNode = %s;".format( dbname) INSERT_TRACEDIRECTION_QUERY = "INSERT INTO {}.TraceDirections (idSrc, idDst) VALUES (%s, %s);".format( dbname) INSERT_PATH_QUERY = "INSERT INTO {}.Paths (idTraceDirection) VALUES (%s);".format(
"""Template for the bibclassify - this modules is NOT standalone safe - it is not expected to be used in a stanalone mode ever. Some template variables are coming directly from the config module, those starting with CFG_BIBCLASSIFY_WEB.... """ import cgi from invenio import config from invenio.base.i18n import gettext_set_language from urllib import quote from invenio.utils.html import escape_html import config as bconfig log = bconfig.get_logger("bibclassify.template") class Template: def tmpl_page( self, keywords=None, top="", middle="", bottom="", navbar=None, req=None, ln=None, generate=None, sorting=None, type=None,
import asyncio from threading import Thread import requests import urllib3 from datetime import datetime from config import get_logger urllib3.disable_warnings() requests.packages.urllib3.disable_warnings() #异步访问网页 # sem = asyncio.Semaphore(10) logger = get_logger('asyncFetch') class FilePath: @classmethod def file_path_detail(cls): now = datetime.now() now_date = now.strftime('%Y-%m-%d %H:%M:%S') FirstLevelDir = now_date.split(' ')[0] SecondLevelDir = now_date.split(' ')[1].split(':')[0] ThirdLevelDir = str(int(now_date.split(' ')[1].split(':')[1]) / 10) return FirstLevelDir, SecondLevelDir, ThirdLevelDir class Fetcher(object): @classmethod
import config from selenium import webdriver import time logger = config.get_logger(__name__) def create_driver(): """ create selenium chrome driver :return: driver """ options = webdriver.ChromeOptions() options.add_argument('headless') driver = webdriver.Chrome(chrome_options=options, executable_path='./chromedriver') return driver def get_from_leaderboard(driver, row, column, column_name): """ get data from one cell in leader board :param driver: chromedriver :param row: row number :param column: column number :param column_name: column name :return: text from cell """ try: xpath = '//*[@id="site-content"]/div[2]/div/div[2]/div/div[2]/div/table/tbody/tr[' + str(row) + ' ]/td[' \ + str(column) + ']'
import urllib2 import random import unicodedata from time import sleep import logging import os from datetime import datetime from lxml import objectify import bottlenose from bs4 import BeautifulSoup from config import get_logger from amazonconfig import AMAZON_KEY_ID, AMAZON_SECRET_KEY, AMAZON_AFFILIATE_ID logger = get_logger('amazon_api') # allow us to print lxml.objectify objects in a nice way # can pull this out in prod # objectify.enable_recursive_str() def api_error_handler(err): ex = err['exception'] url = err['api_url'] logger.debug('%s error getting %s ', type(ex), url) if isinstance(ex, urllib2.HTTPError) and ex.code == 503: logger.info('hit rate limit on API... waiting...') sleep(random.expovariate(0.1)) return True