def get_inventory(self):
if self.inventory:
return self.inventory
latest_inventory_id = db_conn.query_one(
self.query_for_latest_inventory, {"set_num": self.num})[0]
config.get_logger("sets").info("latest inventory id: %s" %
latest_inventory_id)
self.inventory = parts.for_inventory_id(latest_inventory_id)
return self.inventory
def run_dmrg(config):
np.random.seed(config['random_seed'])
log_directory = config['log_directory']
data_path = config['data_path']
test_fraction = config['test_fraction']
max_sweeps = config['max_sweeps']
patience = config['patience']
num_sites = config['num_sites']
bond_dimension = config['bond_dimension']
ix_to_char = config.get('ix_to_char')
logger, save_name = get_logger()
logger.info(config)
tf_logger = Logger('tensorboard/{}'.format(save_name))
text = process_text(data_path, lower=True, remove_punctuation=False)
char_to_ix, ix_to_char = encodings(text, ix_to_char)
site_dimension = len(char_to_ix)
numeric = prepare_numeric(text, char_to_ix)
logger.info("Data has {} characters, {} unique.".format(
len(text), len(char_to_ix)))
train_batch, cv_batch, test_batch = data_split(numeric, num_sites,
test_fraction)
mps = random_gauged_mps(num_sites, site_dimension, bond_dimension)
context = {'config': config, 'step': 0}
stats_history = [mps_stats(mps, train_batch, cv_batch, test_batch)]
sweep, cv_bumps = 1, 0
while sweep <= max_sweeps and cv_bumps <= patience:
dmrg_sweep(mps, train_batch, context)
stats = mps_stats(mps, train_batch, cv_batch, test_batch)
stats_history.append(stats)
log_sweep(logger, tf_logger, stats, sweep)
save_path = '{}/{}/mps-after-step-{}.pickle'.format(
log_directory, save_name, sweep)
data_to_save = {
'config': config,
'mps': mps,
'ix_to_char': ix_to_char,
'save_name': save_name,
'sweep': sweep
}
save_object(data_to_save, save_path)
logger.info("saved mps to: {}".format(data_path))
if config['generate_samples']:
samples_per_sweep = config['samples_per_sweep']
samples_txt = list(
generate_samples(mps, ix_to_char, samples_per_sweep))
for phrase in samples_txt:
logger.info("sample phrase: {}".format(phrase))
sweep += 1
cv_bumps = update_cv_bumps(cv_bumps, stats_history)
return stats
def __init__(self, database, config: str = 'default', log=None):
self.app = create_app(config)
self.app_context = self.app.test_request_context()
self.app_context.push()
self.client = self.app.test_client()
database.app = self.app
self.database = database
self.log = log or get_logger('errors')
def main():
# import ipdb; ipdb.set_trace()
config = get_args()
os.environ["CUDA_VISIBLE_DEVICES"] = "{}".format(config.gpu_id)
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
logger = get_logger(config)
voc = load_vocab(config)
config.n_voc = len(voc.voc)
logger.info(config)
config.embeddings = voc.embeddings
config.use_product_info = False
config.use_user_info = False
if config.use_user_info and config.use_product_info:
usrdict = UserTable('../data/' + config.dataname + '/usrlist.txt')
prddict = ProductTable('../data/' + config.dataname + '/prdlist.txt')
config.n_users = usrdict.size + 1
config.n_products = prddict.size + 1
else:
usrdict = None
prddict = None
logger.info("build model...")
with tf.device("/device:{}:{}".format(config.device_type, config.gpu_id)):
model = Model(config)
logger.info("creating session...")
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=config.gpu_allocate_rate)
gpu_options.allow_growth = True
session_config = tf.ConfigProto(allow_soft_placement=True,
gpu_options=gpu_options)
sess = tf.Session(config=session_config)
model.init_variable(sess)
trainset, devset, testset = load_dataset(config, voc, usrdict, prddict)
if config.load_model:
logger.info("restoring model...")
model.restore(sess)
if not config.test_only:
logger.info("starting training...")
model.train(sess, trainset, devset, testset)
logger.info("training done.")
logger.info("starting testing...")
test_acc, test_mae, test_rmse = model.evaluate(sess, testset)
logger.info(
"final result of testset: acc = {:.4f}, mae = {:.4f}, rmse = {:.4f}".
format(test_acc, test_mae, test_rmse))
logger.info("testing done.")
def https_post(url, payload, params={}, ignore_status=False):
'''Posts a data payload to Bill.com. It can optionally check for failed status.
Args:
payload (dict): A JSON compatible dict with data to send to bill.com's api.
ignore_status (bool): If True, don't check for failed status codes.
Returns:
Dict of the JSON response.
Raises:
ServerReponseError
HTTPError
'''
LOG = get_logger()
api_url = API_URL + '/' + url
headers = {'content-type': 'application/x-www-form-urlencoded'}
try:
response = requests.post(api_url,
params=params,
data=payload,
headers=headers)
except Exception as e:
raise ServerResponseError('Could not post to {0}: {1}'.format(
api_url, e))
if response.status_code not in OK_CODES:
message = "received HTTP {0}: {1} when sending to {2}: {3}".format(
response.status_code, response.text, API_URL, payload)
LOG.error(message)
raise HTTPError(message)
try:
data = json.loads(response.text)
status, message = get_status_and_message(data)
except:
message = 'sent {0} got badly formatted reponse: {1}'.format(
payload, response.text)
LOG.error(message)
LOG.error("SENT TO {}: {}".format(response.url, payload))
LOG.error("RECEIVED {}".format(data))
raise
if not ignore_status and status and status != 'OK':
LOG.error(message)
LOG.error("SENT TO {}: {}".format(response.url, payload))
LOG.error("RECEIVED {}".format(data))
raise ServerResponseError(message)
return data['response_data']
def __init__(self, name, image, shell_cmd, shell_args):
if not os.path.exists("/var/run/docker.sock"):
raise EnvironmentError("Need installed docker.")
self.logger = config.get_logger().getChild(__name__)
self.docker_cmd = "docker run --rm"
self.add_volume("/etc/localtime", "/etc/localtime", "ro")
self.set_tty()
self.name = name
self.image = image
self.shell_cmd = shell_cmd
self.shell_args = shell_args
self.dockerfile_path = None
def __init__(self, args_as_dict):
'''
:arg dict args_as_dict: a dictionary
'''
# pylint:disable=no-member
# no-member (pylint E1101): instance has no member. in this case it
# does because we're messing with the __dict__ member
self.__dict__ = dict(args_as_dict)
if self.debug:
self.log_level = logging.DEBUG
self.logger = config.get_logger('pyro4_netem',
log_path=self.log_directory,
log_level=self.log_level)
def create_app(config_name: str):
"""Creates a Flask app instance using one of predefined configs."""
app = Flask(__name__)
conf_obj = config[config_name]
app.config.from_object(conf_obj)
conf_obj.init_app(app)
db.init_app(app)
app.custom_logger = get_logger('main')
from .main import main as main_blueprint
app.register_blueprint(main_blueprint)
return app
def https_post(url, payload, params={}, ignore_status=False):
'''Posts a data payload to Bill.com. It can optionally check for failed status.
Args:
payload (dict): A JSON compatible dict with data to send to bill.com's api.
ignore_status (bool): If True, don't check for failed status codes.
Returns:
Dict of the JSON response.
Raises:
ServerReponseError
HTTPError
'''
LOG = get_logger()
api_url = API_URL + '/' + url
headers = {'content-type': 'application/x-www-form-urlencoded'}
try:
response = requests.post(api_url, params=params, data=payload, headers=headers)
except Exception as e:
raise ServerResponseError('Could not post to {0}: {1}'.format(api_url, e))
if response.status_code not in OK_CODES:
message = "received HTTP {0}: {1} when sending to {2}: {3}".format(
response.status_code, response.text, API_URL, payload
)
LOG.error(message)
raise HTTPError(message)
try:
data = json.loads(response.text)
status, message = get_status_and_message(data)
except:
message = 'sent {0} got badly formatted reponse: {1}'.format(payload, response.text)
LOG.error(message)
LOG.error("SENT TO {}: {}".format(response.url, payload))
LOG.error("RECEIVED {}".format(data))
raise
if not ignore_status and status and status != 'OK':
LOG.error(message)
LOG.error("SENT TO {}: {}".format(response.url, payload))
LOG.error("RECEIVED {}".format(data))
raise ServerResponseError(message)
return data['response_data']
def __init__(self, in_spec_str, out_spec_str, err_spec_str=[]):
# spec string : [int, ...] or [[(str, int)], (str, str), ...]
# dictionary must specify "key" value, but "value" gives type ex: {2:str,
# "dsadasdas":int}
self.logger = config.get_logger().getChild(__name__)
self.logger.debug(
"__init__" + repr((in_spec_str, out_spec_str, err_spec_str)))
if not isinstance(in_spec_str, list) or not isinstance(in_spec_str, list) or not isinstance(in_spec_str, list):
raise TypeError("Spec string should be list")
self.input_specs = in_spec_str
self.output_specs = out_spec_str
self.err_specs = err_spec_str
self.TYPE_GEN_HANDLERS[tuple] = self.gen_by_spec
self.TYPE_GEN_HANDLERS[list] = self.gen_by_spec
self.TYPE_GEN_HANDLERS[dict] = self.gen_dict_by_spec
def load_scenario( scenario_name ):
"""
* a global variable 'wallet'
* a global variable 'consensus'
* a callable method 'scenario'
* a callable method 'check'
"""
log = get_logger("ZONEFILEMANAGE")
log.debug("Load scenario %s " % sys.argv[1])
# strip .py from scenario name
if scenario_name.endswith(".py"):
scenario_name = scenario_name[:-3]
try:
scenario = importlib.import_module(scenario_name)
except ImportError, ie:
raise Exception("Failed to import %s" % scenario_name)
def __init__(self,
input_file,
output_file,
moses_opts,
session_id,
target_feature="case"):
"""
:param input_file: The input file to run MOSES on
:param output_file: The file to write MOSES program outputs to
:param moses_opts: MOSES parameters
"""
self.input = input_file
self.output = output_file
self.moses_options = moses_opts
if not "W1" in moses_opts: moses_opts += ' -W1'
self.output_regex = re.compile(r"(-?\d+) (.+) \[(.+)\]")
self.logger = get_logger(session_id)
self.target_feature = target_feature
def start_analysis(**kwargs):
"""
A celery task that runs the MOSES analysis
:param session: The session object
:param cwd: Current working directory to store the results of the analysis
:param db: A reference to the mongo database
:return:
"""
db = pymongo.MongoClient(MONGODB_URI)[DB_NAME]
swd, file_path = write_dataset(kwargs["dataset"], kwargs["mnemonic"])
session = Session(kwargs["id"], kwargs["moses_options"],
kwargs["crossval_options"], file_path,
kwargs["mnemonic"], kwargs["target_feature"])
logger = get_logger(session.mnemonic)
session.save(db)
session.status = 1
session.progress = 1
session.start_time = time.time()
session.update_session(db)
try:
filter_type, value = kwargs["filter_opts"]["score"], kwargs[
"filter_opts"]["value"]
filter_cls = loader.get_score_filters(filter_type)
moses_cross_val = CrossValidation(session, db, filter_cls, value, swd)
logger.info("Started cross-validation run")
moses_cross_val.run_folds()
logger.info("Cross-validation done successfully")
session.status = 2
session.message = "Success"
session.progress = 100
except Exception as e:
session.status = -1
session.message = e.__str__()
logger.error(
f"Task failed with {traceback.format_exception(etype=type(e), value=e, tb=e.__traceback__)}"
)
finally:
session.end_time = time.time()
session.update_session(db)
def __init__(self, params, is_predict=False):
super().__init__()
global logger
logger = get_logger(__name__, params['model_config_name'] + '.log')
if params.get('type') and params['type'] == "cnn":
self.model = CNN_Net(params['layers'])
else:
self.model = Net(params['layers'])
if not is_predict:
if params.get('runAllIters'):
self.allIters = params['runAllIters']
else:
self.allIters = True
self.log_step = log_config['pytorch']
self.criterion = nn.MSELoss()
self.optimizer = optim.SGD(self.model.parameters(),
lr=params['lr'])
self.iteraions = params['iters']
self.trainDatsetPart = params['trainDatasetPart']
def StartAnalysis(self, request, context):
session_id = uuid.uuid4()
mnemonic = encode(session_id)
logger = get_logger(mnemonic)
crossval_opts = {
"folds": request.crossValOpts.folds,
"testSize": request.crossValOpts.testSize,
"randomSeed": request.crossValOpts.randomSeed
}
moses_opts, dataset, target_feature = request.mosesOpts, request.dataset, request.targetFeature
filter_opts = {
"score": request.filter.score,
"value": request.filter.value
}
logger.info(
f"Received request with Moses Options: {moses_opts}\n Cross Validation Options: {crossval_opts}\n"
)
if is_valid_dataset(dataset, target_feature):
start_analysis.delay(id=session_id,
moses_options=moses_opts,
crossval_options=crossval_opts,
filter_opts=filter_opts,
dataset=dataset,
mnemonic=mnemonic,
target_feature=target_feature)
url = f"{MOZI_URI}/?id={mnemonic}"
logger.info(f"Session {session_id} analysis started.")
return Result(resultUrl=url, description="Analysis started")
else:
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
context.set_details(
f"Invalid dataset.Dataset doesn't contain a column with named {target_feature} or has "
f"invalid characters")
logger.error("Error occurred while validating request")
return Result(
resultUrl="",
description=
f"Validation error occurred. Dataset doesn't contain a column with named {target_feature} or has invalid characters"
)
def __init__(self, session, db, filter_type, value, cwd):
"""
:param session: The session object that has the moses, cross-validation options and other metadata
:param db: The db connection object
:param cwd: Current working directory to save files to
"""
self.session = session
self.cwd = cwd
self.db = db
self.filter = filter_type
self.filter_value = value
self.logger = get_logger(session.mnemonic)
self.fold_files = []
self._set_dir()
self.total_runs = (self.session.crossval_options["folds"] *
self.session.crossval_options["randomSeed"]) + 1
self.runs = 0
self.tree_transformer = ComboTreeTransform()
self.result_models = {}
self.logger.info(f"Current working dir: {os.getcwd()}")
class Configurator():
"""
Class reader configs for datasets and models
"""
logger = get_logger(__name__ + '.Configurator')
pytorch_train_eps = 5e-2
def get_dataset_config(name):
return Configurator.__read_file(name, 'dataset')
def get_model_config(name):
return Configurator.__read_file(name, 'model')
def get_predict_config(name):
return Configurator.__read_file(name, 'predict')
def __read_file(name, file_type):
path = os.path.abspath('.') + f'/configs/{file_type}/{name}.json'
if not os.path.exists(path):
Configurator.logger.error(f'{file_type} config {name} not exist')
raise FileNotFoundError(f'{file_type} config {name} not exist')
with open(path, 'rb') as f:
return json.load(f)
from config import CURRENT_EXP_DIR, config, get_logger, log_config
logger = get_logger(exp_dir=CURRENT_EXP_DIR)
log_config(logger)
import numpy as np
import sherpa
import sherpa.algorithms.bayesian_optimization as bayesian_optimization
from cade.metrics.comparative import intersection_nn, lncs2, moving_lncs2
from gensim.models.word2vec import Word2Vec
from scipy.spatial.distance import cosine
from sklearn.metrics import (
accuracy_score,
f1_score,
precision_score,
recall_score,
)
def fitness(threshold: float, topn: int, t: float):
similarity = 3
# Task 1 - Binary Classification
predictions = []
for word in binary_truth[:, 0]:
if similarity == 0: # cosine similarity
prediction = (0 if
1 - cosine(model1[word], model2[word]) >= threshold
else 1)
elif similarity == 1: # lncs2 similarity
prediction = (0 if lncs2(word, model1, model2, topn) >= threshold
from __future__ import print_function
import os
from six import iteritems
import config as bconfig
from invenio.legacy.bibclassify import ontology_reader as reader
import text_extractor as extractor
import text_normalizer as normalizer
import keyword_analyzer as keyworder
import acronym_analyzer as acronymer
from invenio.utils.url import make_user_agent_string
from invenio.utils.text import encode_for_xml
log = bconfig.get_logger("bibclassify.engine")
# ---------------------------------------------------------------------
# API
# ---------------------------------------------------------------------
def output_keywords_for_sources(input_sources, taxonomy_name, output_mode="text",
output_limit=bconfig.CFG_BIBCLASSIFY_DEFAULT_OUTPUT_NUMBER, spires=False,
match_mode="full", no_cache=False, with_author_keywords=False,
rebuild_cache=False, only_core_tags=False, extract_acronyms=False,
api=False, **kwargs):
"""Output the keywords for each source in sources."""
# Inner function which does the job and it would be too much work to
# refactor the call (and it must be outside the loop, before it did
import json
import logging.config
from flask import Flask, render_template, request
from config import get_logger
from helper import get_activity
app = Flask(__name__)
LOGGING = get_logger()
logging.config.dictConfig(LOGGING)
logging.captureWarnings(True)
logger = logging.getLogger(__name__)
@app.route('/', methods=['GET', 'POST'])
def index():
activity_type = request.form.get('type')
response = get_activity(activity_type=activity_type)
response_json = response.json()
status_code = response.status_code
return render_template('index.html',
response=response_json), response.status_code
import sys
import os
import config
from utils import req
from itertools import combinations_with_replacement
import simplejson
import datetime
log = config.get_logger("rfun.measure_qf")
def run(solr_url, query, haystack,
qf="title,author",
min=0.0, max=2.0, increment=0.25,
max_hits=100):
min = float(min)
max = float(max)
increment = float(increment)
max_hits = int(max_hits)
if (os.path.exists(query)):
queries = load_queries(query)
else:
queries = [query]
if (os.path.exists(haystack)):
import os
from protocoin.serializers import *
from protocoin.clients import *
from protocoin.fields import *
import protocoin
from keys import version_byte as VERSION_BYTE
from config import get_logger
log = get_logger("spv")
GENESIS_BLOCK_HASH = None
GENESIS_BLOCK_MERKLE_ROOT = None
USE_MAINNET = False
USE_TESTNET = False
BLOCK_HEADER_SIZE = 80 + 1
GENESIS_BLOCK_HASH_MAINNET = "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f"
GENESIS_BLOCK_MERKLE_ROOT_MAINNET = "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b"
GENESIS_BLOCK_HASH_TESTNET = "0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206"
GENESIS_BLOCK_MERKLE_ROOT_TESTNET = "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b"
BLOCK_DIFFICULTY_CHUNK_SIZE = 2016
BLOCK_DIFFICULTY_INTERVAL = 14 * 24 * 60 * 60 # two weeks, in seconds
if VERSION_BYTE == 0:
log.debug("Using mainnet")
USE_MAINNET = True
GENESIS_BLOCK_HASH = GENESIS_BLOCK_HASH_MAINNET
import os
import time
import json
import config as conf
import torch
BEGIN, logger, LOG_PATH = conf.get_logger()
class Game:
def __init__(self,
env_cls,
nets_dict,
dqns_dict,
reward_dict=None,
train_dict=None,
preload=None,
seed=None,
debug=False):
if reward_dict is None:
reward_dict = {'lord': 100, 'down': 50, 'up': 50}
if train_dict is None:
train_dict = {'lord': True, 'down': True, 'up': True}
if preload is None:
preload = {}
assert not (nets_dict.keys()
^ dqns_dict.keys()), 'Net and DQN must match'
self.lord_wins, self.down_wins, self.up_wins = [], [], []
self.lord_total_loss = self.down_total_loss = self.up_total_loss = 0
self.lord_loss_count = self.down_loss_count = self.up_loss_count = 0
from bs4 import BeautifulSoup
import requests
import re
import os
from subprocess import check_output
from time import sleep
import logging
from datetime import datetime
import config
logger = config.get_logger('wishlist')
BASE_URL = 'https://www.amazon.com/gp/registry/wishlist/'
PETES_WISHLIST_ID = '1ZF0FXNHUY7IG'
def get_items_from_local_file(filename=None):
if filename is None:
current_folder = os.path.dirname(os.path.realpath(__file__))
filename = os.path.join(current_folder,'data','wishlist_02_19_2018.html')
soup = BeautifulSoup(open(filename), 'html.parser')
all_asin_inputs_in_soup = soup.findAll('input',{'name':'itemId'})
asin_values = []
for asin_input in all_asin_inputs_in_soup:
import random
from Crypto.Hash import SHA256 as HashAlg
from Crypto.PublicKey import RSA as CryptoKey
from Crypto import Random
from Crypto.Signature import PKCS1_PSS as CryptoSigner
import syndicate.util.objects as object_stub
import syndicate.util.crypto as crypto
import client
import config as conf
import syndicate.protobufs.ms_pb2 as ms_pb2
import syndicate.protobufs.sg_pb2 as sg_pb2
import syndicate.ms.msconfig as msconfig
log = conf.get_logger("syndicate-certs")
def syndicate_public_key_name( ms_url ):
"""
Get the name of a syndicate public key, given
the MS url it was fetched from.
"""
host, port, no_tls = client.parse_url( ms_url )
return host + ":" + str(port)
def syndicate_public_key_fetch( ms_url, downloader_path ):
"""
Use a helper program to go and fetch the Syndicate public key.
Return the key itself on success.
Return None on error
from db import User, Wish, Role, ModerationStatus, Token import config import social cache = SimpleCache() SECRET_KEY = str(config.SECRET_KEY) TRAP_HTTP_EXCEPTIONS = True PERMANENT_SESSION_LIFETIME = 60*60*24*31 app = Flask(__name__) app.config.from_object(__name__) handler = config.get_logger(__file__).handlers[0] app.logger.addHandler(handler) def render_template_my(*args, **kwargs): currentPage = "" if "s" in request.args: currentPage = "s%s" % request.args["s"] if "f" in request.args: currentPage = "f" kwargs["current_page"] = currentPage kwargs["debug"] = config.DEBUG
This module contains methods to extract keywords from texts. It provides 3
different methods for 3 different types of keywords: single keywords, composite
keywords and author keywords.
This module is STANDALONE safe
"""
from __future__ import print_function
import re
import time
import config as bconfig
log = bconfig.get_logger("bibclassify.keyword_analyzer")
_MAXIMUM_SEPARATOR_LENGTH = max([len(_separator)
for _separator in
bconfig.CFG_BIBCLASSIFY_VALID_SEPARATORS])
# XXX - rebuild this whole thing
def get_single_keywords(skw_db, fulltext):
"""Find single keywords in the fulltext
@var skw_db: list of KeywordToken objects
@var fulltext: string, which will be searched
@return : dictionary of matches in a format {
<keyword object>, [[position, position...], ],
..
}
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import re
import commands as run
import config as cfg
logging = cfg.get_logger()
config = cfg.get_config()
def last_commit(repopath):
""" Gets the last commit information.
This can be used to verify the latest commit
"""
command = "cd %s; git log -1i --date=iso" % repopath
out = run.command(command)
if out:
creg = re.compile(r"commit\s+(?P<remote_host>([a-f0-9]+))") # Commitid
areg = re.compile(r"Author:\s+(?P<author>(.*$))") # author
dreg = re.compile(r"Date:\s+(?P<date>(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}))") # Date
last = {}
for line in out.splitlines():
commit = creg.search(line)
author = areg.search(line)
date = dreg.search(line)
if commit:
last['id'] = commit.group(1)
elif author:
last['author'] = author.group(1)
elif date:
last['date'] = date.group(1)
def __init__(self):
self.logger = config.get_logger().getChild(
__name__ + "." + self.__class__.__name__)
self.app_dict = {}
def __init__(self, cmd_type, url, bucket):
self.logger = config.get_logger().getChild(__name__)
self._cmd_type = cmd_type
self._url = url
self._bucket = bucket
import random
from Crypto.Hash import SHA256 as HashAlg
from Crypto.PublicKey import RSA as CryptoKey
from Crypto import Random
from Crypto.Signature import PKCS1_PSS as CryptoSigner
import syndicate.util.objects as object_stub
import syndicate.util.crypto as crypto
import client
import config as conf
import syndicate.protobufs.ms_pb2 as ms_pb2
import syndicate.protobufs.sg_pb2 as sg_pb2
import syndicate.ms.msconfig as msconfig
log = conf.get_logger("syndicate-certs")
def syndicate_public_key_name(ms_url):
"""
Get the name of a syndicate public key, given
the MS url it was fetched from.
"""
host, port, no_tls = client.parse_url(ms_url)
return host + ":" + str(port)
def syndicate_public_key_fetch(ms_url, downloader_path):
"""
Use a helper program to go and fetch the Syndicate public key.
Return the key itself on success.
# Purpose: Utility wrappers for persistence of suppporting data for Tree View
import hashlib
import config
# Config
logger = config.get_logger()
allowed_extensions = set(['csv'])
def to_logging(df, floor_height_column, face_direction_column,
window_vertical_position_column):
da = to_dictarray(
df, {
floor_height_column: [],
face_direction_column: [],
window_vertical_position_column: []
})
dump_dictarray_head(da)
dump_dictarray_tail(da)
def to_dictarray(df, da):
for df_row in df.itertuples():
dfdict_row = df_row._asdict()
dr = {}
for column in da.keys():
try:
da[column].append(float(dfdict_row[column]))
except Exception as e:
import os
from os.path import isfile as fileExists
from os.path import join as pathJoin
from os.path import dirname
from os.path import abspath
import json
import makeUtils
import timeout
import config
from ..adb import adb
from ..androidUtils import fileUtils
from ..dockerBuildUtils import dockerBuildUtils
logger = config.get_logger().getChild(__name__)
THIS_DIR = abspath(dirname(__file__))
HCFS_LIB_PHONE_PATH = "/system/lib64/libHCFS_api.so"
HCFS_LIB = THIS_DIR + "/libHCFS_api.so"
ADAPTER_NAME = "adapter"
ADAPTER_BIN_LOCAL = pathJoin(THIS_DIR, ADAPTER_NAME)
ADAPTER_BIN = "/data/" + ADAPTER_NAME
def setup():
logger.info("adapter setup")
cleanup()
check_build_env()
get_hcfs_lib_from_phone()
import tornado.ioloop
import tornado.options
import tornado.web
import tornado.escape
import json
import time
from tornado.options import define, options
sys.path = ["."] + sys.path
import config
import model
import repository
import rest_json
logger = config.get_logger(logging.WARNING, __name__)
application = None
define("port", default=9081, help="run on the given port", type=int)
define("debug", default=False, type=bool)
class BaseHandler(tornado.web.RequestHandler):
def prepare(self):
self.registry = repository.RepositoryRegistry()
def __init__(self, application, request, **kwargs):
super(BaseHandler, self).__init__(application, request, **kwargs)
def set_default_headers(self):
import os
import tempfile
import errno
import virtualchain
import pybitcoin
from config import get_logger, get_bitcoin_regtest_opts
import name_service
from blockchain.session import get_bitcoind_connection
log = get_logger("testlib")
snapshots_dir = None
state_engine = None
class TestAPIProxy(object):
def __init__(self):
global utxo_opts
client_path = os.environ.get("ZONEFILEMANAGE_CLIENT_CONFIG", None)
assert client_path is not None
class Wallet(object):
def __init__(self, pk_wif, ignored):
pk = virtualchain.BitcoinPrivateKey(pk_wif)
import os
import config
import datetime
from state_machine.nameset import indexer
from config import get_logger
log = get_logger("virtual_")
def setup_virtualchain(impl=None):
if impl is not None:
config.set_implementation(impl)
def sync_virtualchain(bitcoind_opts,
last_block,
state_engine,
expected_snapshots={},
tx_filter=None):
"""
Synchronize the virtualchain state up until a given block
"""
rc = False
start = datetime.datetime.now()
while True:
# advance the state
try:
rc = indexer.StateEngine.build(
bitcoind_opts,
from blockstack_client import storage
from blockstack_client import user as user_db
from storage import hash_zonefile
import pybitcoin
import bitcoin
import binascii
from utilitybelt import is_hex
from config import get_logger, DEBUG, MAX_RPC_LEN, find_missing, BLOCKSTACKD_SERVER, \
BLOCKSTACKD_PORT, BLOCKSTACK_METADATA_DIR, BLOCKSTACK_DEFAULT_STORAGE_DRIVERS, \
FIRST_BLOCK_MAINNET, NAME_OPCODES, OPFIELDS, CONFIG_DIR, SPV_HEADERS_PATH, BLOCKCHAIN_ID_MAGIC, \
NAME_PREORDER, NAME_REGISTRATION, NAME_UPDATE, NAME_TRANSFER, NAMESPACE_PREORDER, NAME_IMPORT, \
USER_ZONEFILE_TTL, CONFIG_PATH, get_config
log = get_logger()
def set_profile_timestamp(profile, now=None):
"""
Set the profile's timestamp to now
"""
if now is None:
now = time.time()
profile['timestamp'] = now
return profile
def get_profile_timestamp(profile):
"""
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import urllib2
import json
import config
import db
from db import User, Token
logger = config.get_logger(__file__)
logger.info("Start updating user info")
db_session = db.Session()
def get_users_info(user_ids):
url = "https://api.vk.com/method/users.get?user_ids=%s&fields=photo_50,sex" % (user_ids)
f = urllib2.urlopen(url)
data = f.read()
data = json.loads(data)
if "response" not in data:
logger.warning("Error while trying get user info from url %s" % url)
return None
return data["response"]
tokens = db_session.query(Token).limit(20)
user_ids = []
for token in tokens:
user_ids.append(str(token.user_social_id))
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
""" Development configuration """
import os
from config import BaseConfig, get_logger
logger = get_logger(__name__)
class MyConfig(BaseConfig):
HOST = '0.0.0.0'
WTF_CSRF_SECRET_KEY = 'a random string'
BASE_DIR = os.path.abspath(os.path.dirname(__file__))
UPLOAD_FOLDER = '/uploads'
ALLOWED_EXTENSIONS = set(['txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif'])
logger.info("Current config: DEVEL. Keepeing sqllite db.")
from profile import *
from proxy import *
from storage import hash_zonefile
import pybitcoin
import bitcoin
import binascii
from utilitybelt import is_hex
from config import get_logger, DEBUG, MAX_RPC_LEN, find_missing, BLOCKSTACKD_SERVER, \
BLOCKSTACKD_PORT, BLOCKSTACK_METADATA_DIR, BLOCKSTACK_DEFAULT_STORAGE_DRIVERS, \
FIRST_BLOCK_MAINNET, NAME_OPCODES, OPFIELDS, CONFIG_DIR, SPV_HEADERS_PATH, BLOCKCHAIN_ID_MAGIC, \
NAME_PREORDER, NAME_REGISTRATION, NAME_UPDATE, NAME_TRANSFER, NAMESPACE_PREORDER, NAME_IMPORT, \
USER_ZONEFILE_TTL, CONFIG_PATH, get_utxo_provider_client, get_tx_broadcaster
log = get_logger()
import virtualchain
def serialize_mutable_data_id( data_id ):
"""
Turn a data ID into a suitable filesystem name
"""
return urllib.quote(data_id.replace("\0", "\\0")).replace("/", r"\x2f")
def load_mutable_data_version(conf, name, fq_data_id):
"""
Get the version field of a piece of mutable data from local cache.
"""
import errno
import time
import atexit
from defusedxml import xmlrpc
# prevent the usual XML attacks
xmlrpc.monkey_patch()
import signal
import json
import config as blockstack_config
import backend
import proxy
log = blockstack_config.get_logger()
from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
running = False
# need to wrap CLI methods to capture arguments
def local_rpc_factory( method, config_path ):
"""
Factory for producing wrappers around CLI functions
"""
import blockstack_client.method_parser as method_parser
import blockstack_client.cli as cli
method_info = method_parser.parse_methods( [method] )[0]
def argwrapper( *args, **kw ):
import db
import config
import cache
log = config.get_logger('colors')
db_conn = db.get_instance()
class Color(object):
def __init__(self, data):
try:
self.id = data['id']
self.rgb = data['rgb']
self.is_trans = data['is_trans']
self.name = data['name']
except KeyError as e:
log.error('missing key in data %s' % data, e)
query_by_id = 'SELECT * FROM colors WHERE id=:id'
def from_id(id):
def create():
return Color(db_conn.query_one(query_by_id, {'id': id}))
return cache.remember('color', id, create)
import sys
import config
from utils import req
import datetime
log = config.get_logger("rfun.find_citation_queries")
"""
Finds queries that are raising the number of returned
hits
"""
def run(solr_url,
query_name='citations',
fl="title,author,recid,bibcode",
min=0, max=4000000, increment=9,
max_hits=10, max_qtime=60000
):
results = []
# first start with simple queries
i = 0
j = None
empty = 0
while False:
i = i + 1
import re
import string
import struct
import types
from geom3d import Triangle, BBox, Vector, BinaryTree
import config
logger = config.get_logger('STL')
class Stl:
def __init__(self, *args):
if type(args[0]) in (types.StringType, types.UnicodeType):
self.filename = args[0]
self.triangles = []
inf = open(self.filename)
triangleList = self.from_input_file(inf)
else:
self.filename = self.preamble = ''
triangleList = args
self.triangles = triangleList
bb = BBox()
for tri in triangleList:
bb = bb.expand(tri._bbox)
self._bbox = bb
self._classify_triangles_by_z()
def from_input_file(self, inf):
triangleList = []
R = inf.read()
are tuned to work with the output of pdftotext and documents in the HEP field.
Methods can be tuned to your needs through the configuration file.
This modules uses the refextract module of BibEdit in order to find the
references section and to replace unicode characters.
"""
import re
import config as bconfig
from six import iteritems
from invenio.legacy.docextract.pdf import replace_undesirable_characters
from invenio.legacy.refextract.find import find_reference_section, find_end_of_reference_section
log = bconfig.get_logger("bibclassify.text_normalizer")
_washing_regex = []
def get_washing_regex():
global _washing_regex
if len(_washing_regex):
return _washing_regex
washing_regex = [
# Replace non and anti with non- and anti-. This allows a better
# detection of keywords such as nonabelian.
(re.compile(r"(\snon)[- ](\w+)"), r"\1\2"),
(re.compile(r"(\santi)[- ](\w+)"), r"\1\2"),
# Remove all leading numbers (e.g. 2-pion -> pion).
from gravi import reverse
from gravi.reverse.builder import complex_build
from numpy.core.records import array
from gravi.reverse import net, min
from .data import DataCreator, DataReader, Configurator
from datetime import datetime
from .models.pytorch import ModelPyTorch
import numpy.linalg as l
import numpy as np
import os
from copy import copy
from config import get_logger
logger = get_logger(__name__)
def research(size, dataset_name):
i, o = prepare_data(size, dataset_name)
learn(i, o, dataset_name)
def learn(len_i, len_o, dataset_name, model_config_name, model_params=None):
shape = 'default'
if not model_params:
model_params = Configurator.get_model_config(model_config_name)
if model_params.get('type') and model_params['type'] == "cnn":
shape = [
1, model_params['shape']['in']['w'],
model_params['shape']['in']['h']
]
else:
import config
logger = config.get_logger('GEOM3D')
# Floating point madness
WIGGLE_ROOM = 1.0e-6
def sign(x):
return x > 0.0
class Vector(object):
def __init__(self, x, y=None, z=None):
if y is None:
self.x, self.y, self.z = x
else:
self.x, self.y, self.z = x, y, z
def serialize(self):
import cPickle
return cPickle.dumps((self.x, self.y, self.z))
@classmethod
def unserialize(self, str):
import cPickle
x, y, z = cPickle.loads(str)
return Vector(x, y, z)
def __repr__(self):
return "<{0},{1},{2}>".format(self.x, self.y, self.z)
from flask_mail import Message
from datetime import datetime, date
from app import db, mail, app
from app.models import Item, ParentItem, Image, Offer, LastRefreshed
from amazon_api import get_parent_ASIN, get_item_attributes, get_amazon_api, get_images, get_item_variations_from_parent, get_offers
from wishlist import get_items_from_wishlist, get_items_from_local_file
from config import get_logger
from sqlalchemy import or_
import os
import logging
logger = get_logger('refresh_data')
WISHLIST_ID = '1ZF0FXNHUY7IG'
MAILTO = '*****@*****.**'
DEBUG = True
def get_buybox_price(item):
''' take an Item object, return the buybox price
Returns None if not
'''
buybox_price = None
for offer in item.offers.all():
if offer.offer_source == 'Buybox':
buybox_price = offer.offer_price_amount
from state_machine import nameset as state_engine
from bin.zonefilemanage_client import *
wallets = [
#prvate key wif
Wallet( "5JesPiN68qt44Hc2nT8qmyZ1JDwHebfoh9KQ52Lazb1m1LaKNj9", 100000000000 ),
Wallet( "5KHqsiU9qa77frZb6hQy9ocV7Sus9RWJcQGYYBJJBb2Efj1o77e", 100000000000 ),
Wallet( "5Kg5kJbQHvk1B64rJniEmgbD83FpZpbw2RjdAZEzTefs9ihN3Bz", 100000000000 ),
Wallet( "5JuVsoS9NauksSkqEjbUZxWwgGDQbMwPsEfoRBSpLpgDX1RtLX7", 100000000000 ),
Wallet( "5KEpiSRr1BrT8vRD7LKGCEmudokTh1iMHbiThMQpLdwBwhDJB1T", 100000000000 )
]
db_inst = None
log = get_logger("ZONEFILEMANAGE")
nameset_cache = []
class Pinger(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.running = False
def run(self):
self.running = True
bitcoind = bitcoin_regtest_connect(bitcoin_regtest_opts())
while self.running:
try:
bitcoind.ping()
#!/usr/bin/env python3
import psycopg2
import spacy.en
import attribute_utils
from src.model import Question, Attributes
import sys
from os.path import dirname, abspath
sys.path.insert(0, dirname(dirname(abspath(__file__))))
import config
logger = config.get_logger()
def parse(question: str) -> Question: # returns a list of question's attributes
try:
parse.nlp
except AttributeError:
parse.nlp = spacy.en.English()
print("attributes logger name=" + logger.name)
doc = parse.nlp(question)
result = Attributes()
try:
result.location = attribute_utils.get_attribute_location_spacy(doc)
result.named_entity = attribute_utils.get_attribute_named_entity(question)
result.time = attribute_utils.get_attribute_time_spacy(doc, question)
result.action = attribute_utils.get_attribute_action(doc)
result.product = attribute_utils.get_attribute_product(question)
except KeyError as e:
logger.error("KeyError. Json config doesn't contain such key:", str(e))
raise e
# @Email: [email protected] # @Last modified by: lorenzocorneo # @Last modified time: 2016-10-28T15:04:05+02:00 #!/usr/bin/python import MySQLdb import itertools import config from collections import defaultdict from models.EndNode import EndNode from models.Measurement import Measurement from models.HopMeasurement import HopMeasurement from models.Path import Path dup_log = config.get_logger("duplicates", config.log_patch_with_duplicates) dbname = config.db_credentials["db"] GET_NODES_QUERY = "SELECT * FROM {}.EndNodes".format(dbname) GET_TRACEDIRECTIONS_QUERY = "SELECT * FROM {}.TraceDirections".format(dbname) GET_PATHS_QUERY = "SELECT * FROM {}.Paths".format(dbname) GET_HOPS_QUERY = "SELECT * FROM {}.Hops".format(dbname) GET_MEASUREMENTS_QUERY = "SELECT * FROM {}.Measurements".format(dbname) GET_HOPMEASUREMENTS_QUERY = "SELECT * FROM {}.HopMeasurements".format(dbname) INSERT_ENDNODE_QUERY = "INSERT INTO {}.EndNodes (ip, name) VALUES ('%s', '%s');".format( dbname) UPDATE_ENDNODE_QUERY = "UPDATE {}.EndNodes set ip='%s', name='%s' where idNode = %s;".format( dbname) INSERT_TRACEDIRECTION_QUERY = "INSERT INTO {}.TraceDirections (idSrc, idDst) VALUES (%s, %s);".format( dbname) INSERT_PATH_QUERY = "INSERT INTO {}.Paths (idTraceDirection) VALUES (%s);".format(
"""Template for the bibclassify -
this modules is NOT standalone safe - it is not expected to be
used in a stanalone mode ever.
Some template variables are coming directly from the config
module, those starting with CFG_BIBCLASSIFY_WEB....
"""
import cgi
from invenio import config
from invenio.base.i18n import gettext_set_language
from urllib import quote
from invenio.utils.html import escape_html
import config as bconfig
log = bconfig.get_logger("bibclassify.template")
class Template:
def tmpl_page(
self,
keywords=None,
top="",
middle="",
bottom="",
navbar=None,
req=None,
ln=None,
generate=None,
sorting=None,
type=None,
import asyncio
from threading import Thread
import requests
import urllib3
from datetime import datetime
from config import get_logger
urllib3.disable_warnings()
requests.packages.urllib3.disable_warnings()
#异步访问网页
# sem = asyncio.Semaphore(10)
logger = get_logger('asyncFetch')
class FilePath:
@classmethod
def file_path_detail(cls):
now = datetime.now()
now_date = now.strftime('%Y-%m-%d %H:%M:%S')
FirstLevelDir = now_date.split(' ')[0]
SecondLevelDir = now_date.split(' ')[1].split(':')[0]
ThirdLevelDir = str(int(now_date.split(' ')[1].split(':')[1]) / 10)
return FirstLevelDir, SecondLevelDir, ThirdLevelDir
class Fetcher(object):
@classmethod
import config
from selenium import webdriver
import time
logger = config.get_logger(__name__)
def create_driver():
"""
create selenium chrome driver
:return: driver
"""
options = webdriver.ChromeOptions()
options.add_argument('headless')
driver = webdriver.Chrome(chrome_options=options,
executable_path='./chromedriver')
return driver
def get_from_leaderboard(driver, row, column, column_name):
"""
get data from one cell in leader board
:param driver: chromedriver
:param row: row number
:param column: column number
:param column_name: column name
:return: text from cell
"""
try:
xpath = '//*[@id="site-content"]/div[2]/div/div[2]/div/div[2]/div/table/tbody/tr[' + str(row) + ' ]/td[' \
+ str(column) + ']'
import urllib2
import random
import unicodedata
from time import sleep
import logging
import os
from datetime import datetime
from lxml import objectify
import bottlenose
from bs4 import BeautifulSoup
from config import get_logger
from amazonconfig import AMAZON_KEY_ID, AMAZON_SECRET_KEY, AMAZON_AFFILIATE_ID
logger = get_logger('amazon_api')
# allow us to print lxml.objectify objects in a nice way
# can pull this out in prod
# objectify.enable_recursive_str()
def api_error_handler(err):
ex = err['exception']
url = err['api_url']
logger.debug('%s error getting %s ', type(ex), url)
if isinstance(ex, urllib2.HTTPError) and ex.code == 503:
logger.info('hit rate limit on API... waiting...')
sleep(random.expovariate(0.1))
return True
