def main(): ''' Main entry. Here goes code for testing the methods and classes of this module. ''' logger_name = 'BayesianLogger' util.setup_logger(logger_name, os.path.join('logs', 'bayesian_inversion.log')) logger = util.get_logger(logger_name) logger.info('ID={0}'.format(time.time())) word2vec_model = make_word2vec(iterations=1, size=100, skipgram=1, alpha=0.025, workers=4, logger_name=logger_name) dataset = util.read_dataset_threaded(os.path.join('data', 'raw_texts.txt'), processes=10, logger_name=logger_name) evaluate(dataset, word2vec_model, priors=True, k_folds=6, logger_name=logger_name)
def main(): opts = parse_cli() setup_logger(stdout=not opts.quiet, network=opts.network_log) target_ = 'mainnet' if opts.testnet: target_ = 'testnet' server_ = opts.server or TARGET_SERVER[target_] ergo_update(server_, opts.api_key, opts.value)
async def main(): """ Main function """ env = Env() env.read_env() setup_logger(env.int("LOG_LEVEL", logging.INFO), env.path("LOG_FILE", None)) async with aiohttp.ClientSession() as session: bot = create_bot(env, session) LOGGER.debug("Starting bot") await bot.start(env.str("BOT_TOKEN"))
def setup_app(): app = Flask(__name__) # setting up the app logger with a rotating file handler, in addition to # the built-in flask logger which can be helpful in debug mode. additional_log_handlers = [ RotatingFileHandler( config.instance().rest_service_log_path, maxBytes=1024*1024*100, backupCount=20) ] app.logger_name = 'manager-rest' setup_logger(logger_name=app.logger.name, logger_level=logging.DEBUG, handlers=additional_log_handlers, remove_existing_handlers=False) app.before_request(log_request) app.after_request(log_response) # saving flask's original error handlers flask_handle_exception = app.handle_exception flask_handle_user_exception = app.handle_user_exception api = Api(app) # saving flask-restful's error handlers flask_restful_handle_exception = app.handle_exception flask_restful_handle_user_exception = app.handle_user_exception # setting it so that <500 codes use flask-restful's error handlers, # while 500+ codes use original flask's error handlers (for which we # register an error handler on somewhere else in this module) def handle_exception(flask_method, flask_restful_method, e): code = getattr(e, 'code', 500) if code >= 500: return flask_method(e) else: return flask_restful_method(e) app.handle_exception = functools.partial( handle_exception, flask_handle_exception, flask_restful_handle_exception) app.handle_user_exception = functools.partial( handle_exception, flask_handle_user_exception, flask_restful_handle_user_exception) resources.setup_resources(api) return app
def main(): opts = parse_cli() setup_logger(stdout=not opts.quiet, network=opts.network_log) target_ = 'mainnet' if opts.testnet: target_ = 'testnet' server_ = opts.server or TARGET_SERVER[target_] api = ErgoClient(server_, opts.api_key) with open('box.id') as inp: lines = inp.read().splitlines() if len(lines) == 1: logging.debug('One box ID in box.id') pass elif len(lines) != 2: logging.error('File box.id contains %d lines (expected 2)' % ( len(lines) )) sys.exit(1) else: logging.debug('Two box IDs in box.id') id1, id2 = lines if not is_box_spent(api, id1): logging.debug('First box ID is not spent') if opts.stop: with open('box.id', 'w') as out: out.write('%s\n' % id1) # remove 2nd line logging.debug('Removed second box ID') else: logging.debug('! First box ID is spent') if not is_box_spent(api, id2): logging.debug('Second box ID is not spent') res = api.request( '/wallet/boxes/unspent' '?minConfirmations=30&minInclusionHeight=0' ) found = any(x['box']['boxId'] == id2 for x in res) if found: with open('box.id', 'w') as out: out.write('%s\n' % id2) # remove 1st line logging.debug('Removed first box ID') else: logging.error('Not enough confirmations yet') sys.exit(1) else: logging.error( 'Both 1st and 2nd box IDs are spent' ) sys.exit(1)
def main(): opts = parse_cli() setup_logger(stdout=not opts.quiet, network=opts.network_log) target_ = 'mainnet' if opts.testnet: target_ = 'testnet' server_ = opts.server or TARGET_SERVER[target_] price = get_currency(opts.quandl_key) / OZT logging.info('AUG/USD price: %s' % price) ergo_update(server_, opts.api_key, int(price*1e9))
def main(): opts = parse_cli() setup_logger(stdout=not opts.quiet, network=opts.network_log) target_ = 'mainnet' if opts.testnet: target_ = 'testnet' server_ = opts.server or TARGET_SERVER[target_] cmc_api = CmcApi(opts.cmc_key) price = cmc_api.get_ergo_price('EUR') logging.info('EUR/ERG price: %s' % price) ergo_update(server_, opts.api_key, int(price * 1e7))
def generate(self): logger = util.setup_logger(INFO) env = self.env.lower() if self.env else 'production' conf = util.getstanza('getsnow', env) # Proxy not currently used in this version # proxy_conf = util.getstanza('getsnow', 'global') # proxies = util.setproxy(conf, proxy_conf) username = conf['user'] password = conf['password'] url = conf['url'] value_replacements = conf['value_replacements'] user_name = self.user_name.split(',') daysAgo = int(self.daysAgo) if self.daysAgo else 30 snowuser = snow(url, username, password) snowuser.replacementsdict(value_replacements) user_info = snowuser.getsysid('sys_user', 'user_name', user_name, mapto='user_name') for record in user_info[1]: record = snowuser.updaterecord(record, sourcetype='snow:user') record['_raw'] = util.tojson(record) yield record exuded = snowuser.filterbuilder('assigned_to', user_info[0]) url = snowuser.reqencode([exuded], table='alm_asset') for record in snowuser.getrecords(url): record = snowuser.updaterecord(record, sourcetype='snow:asset') record['_raw'] = util.tojson(record) yield record exuded = snowuser.filterbuilder('opened_by', user_info[0]) url = snowuser.reqencode([exuded], days=daysAgo) for record in snowuser.getrecords(url): record = snowuser.updaterecord(record, sourcetype='snow:incident') record['_raw'] = util.tojson(record) yield record
def main(): opts = parse_cli() setup_logger(stdout=not opts.quiet, network=opts.network_log) target_ = 'mainnet' if opts.testnet: target_ = 'testnet' server_ = opts.server or TARGET_SERVER[target_] api = ErgoClient(server_, opts.api_key) # Ger first wallet address res = api.request( '/wallet/addresses', ) addr = res[0] # Generate transaction tx_data = { 'requests': [{ 'address': addr, 'amount': 1, 'name': opts.name, 'description': opts.description, 'decimals': 0, }], 'fee': 1e6, 'inputsRaw': [], } res = api.request( '/wallet/transaction/generate', data=tx_data, ) out = res['outputs'][0] with open('address.id', 'w') as fobj: fobj.write(addr + '\n') with open('box.id', 'w') as fobj: fobj.write(out['boxId'] + '\n') with open('token.id', 'w') as fobj: fobj.write(out['assets'][0]['tokenId'] + '\n') api.request( '/wallet/transaction/send', data=tx_data, )
def main(): ''' Main entry. Here goes code for testing the methods and classes of this module. ''' logger_name = 'DistributedRepresentationLogger' util.setup_logger(logger_name, os.path.join('logs', 'distributed_representation.log')) logger = util.get_logger(logger_name) logger.info('ID={0}'.format(time.time())) dataset = util.read_dataset_threaded(os.path.join('data', 'raw_texts.txt'), processes=8,\ logger_name=logger_name) doc2vec_base_model_dbow = make_doc2vec(size=400, window=5, dm=0, hierarchical_softmax=0, negative=10, dm_mean=0, dm_concat=0, dbow_words=1, workers=8, min_count=2, sample=0, logger_name=logger_name) # doc2vec_base_model_dm = make_doc2vec(size=400, window=5, dm=1, hierarchical_softmax=1, # negative=0, dm_mean=0, dm_concat=0, dbow_words=0, workers=8, min_count=2, sample=0, # logger_name=logger_name) # doc2vec_base_model = wrappers.ConcatDoc2vec([doc2vec_base_model_dbow, doc2vec_base_model_dm]) # model = linear_model.LogisticRegression(solver='lbfgs') model = svm.SVC(kernel='linear') evaluate(dataset, model, doc2vec_base_model_dbow, k_folds=6, iterations=1, alpha=0.025, logger_name=logger_name)
def main( conf: str, log: str, ): # 設定を読み込み conf = load_config(conf) # 保存するファイル名を指定 log_file = log # ログの初期設定を行う logger = setup_logger(log_file) logger.debug("test")
def main(): opts = parse_cli() setup_logger(stdout=not opts.quiet, network=opts.network_log) target_ = 'mainnet' if opts.testnet: target_ = 'testnet' server_ = opts.server or TARGET_SERVER[target_] price1 = get_currency(opts.quandl_key) / OZT logging.info('AUG/USD price: %s' % price1) cmc_api = CmcApi(opts.cmc_key) price2 = cmc_api.get_ergo_price('USD') logging.info('USD/ERG price: %s' % price2) price = price1 / price2 logging.info('AUG/ERG price: %s' % price) ergo_update(server_, opts.api_key, int(price * 1e9))
def main(): ''' Main entry. Here goes code for testing the methods and classes of this module. ''' logger_name = 'NaiveLogger' util.setup_logger(logger_name, os.path.join('logs', 'naive.log')) logger = util.get_logger(logger_name) logger.info('ID={0}'.format(time.time())) dataset = util.read_dataset_threaded(os.path.join('data', 'raw_texts.txt'), processes=8, logger_name=logger_name) word2vec_base_model = make_word2vec(iterations=10, size=900, skipgram=1, hierarchical_softmax=0, negative=14, alpha=0.025, sample=1e-5, workers=8, logger_name=logger_name) # model = linear_model.SGDClassifier(loss='modified_huber') # model = linear_model.LogisticRegression(solver='lbfgs', n_jobs=1) # model = neighbors.NearestCentroid() model = svm.SVC(kernel='linear') evaluate(dataset, model, word2vec_base_model, k_folds=6, logger_name=logger_name)
def run(preprocessed_dir, invalid_thresh, invalid_user_thresh, relative_diff_thresh, data_split, no_interactions, negative, max_snps, model_id, cross_validation, output_dir): """ Builds a model to predict phenotype :param preprocessed_dir: The directory containing the preprocessed data :param invalid_thresh: The acceptable percentage of missing data before a SNP is discarded :param invalid_user_thresh: The acceptable percentage of missing data before a user is discarded :param relative_diff_thresh: The relative difference in mutation percent, calculated as a percent of the larger mutation percent value. :param data_split: The percent data used for testing. :param no_interactions: If True the model will not contain interactions :param negative: The negative phenotype label :param model_id: The id for the model to use :param cross_validation: number of folds for cross validation :param output_dir: The directory to write the model in """ # Expand file paths preprocessed_dir = expand_path(preprocessed_dir) # Make sure output directory exists before doing work clean_output(output_dir) setup_logger(output_dir, model_id + "_model") # Get model build_model = MODELS.get(model_id) if not build_model: raise ValueError('Model Id "{}" is not valid'.format(model_id)) phenotypes = timed_invoke('reading the preprocessed files', lambda: __read_phenotype_input(preprocessed_dir)) data_set = timed_invoke('creating model data set', lambda: mutation_difference.create_dataset( phenotypes, invalid_thresh, invalid_user_thresh, relative_diff_thresh) ) timed_invoke('building model', lambda: build_model(data_set, data_split, no_interactions, negative, max_snps, cross_validation, output_dir)) logger.info('Output written to "{}"'.format(output_dir))
def generate(self): logger = util.setup_logger(INFO) env = self.env.lower() if self.env else 'production' conf = util.getstanza('getsnow', env) #proxy_conf = util.getstanza('getsnow', 'global') username = conf['user'] password = conf['password'] active = self.active user_name = self.user_name.split(',') if self.user_name else [] assigment_group = self.assignment_group.split(',') if self.assignment_group else [] daysAgo = int(self.daysAgo) if self.daysAgo else None limit = self.limit daysBy = self.daysBy if self.daysBy else 'opened_at' filterBy = self.filterBy if self.filterBy else 'assigned_to' url = conf['url'] value_replacements = conf['value_replacements'] if active: try: active = active.strip() active = active[0].upper() + active[1:].lower() active = ast.literal_eval(active) except: active = True if limit: try: limit = int(limit) except: limit = 10000 snowincident = snow(url, username, password) snowincident.replacementsdict(value_replacements) user_info = snowincident.getsysid('sys_user', 'user_name', user_name, mapto='user_name')[0] group_info = snowincident.getsysid('sys_user_group', 'name', assigment_group, mapto='name')[0] exuded1 = snowincident.filterbuilder(filterBy, user_info) exuded2 = snowincident.filterbuilder('assignment_group', group_info) url = snowincident.reqencode([exuded1, exuded2], table='incident', active=active, timeby=daysBy, days=daysAgo) for record in snowincident.getrecords(url, limit=limit): record = snowincident.updaterecord(record, sourcetype='snow:incident') record['_raw'] = util.tojson(record) yield record
def __init__(self, q1, q2): self.q1 = q1 self.q2 = q2 self.save_dir = "/home/newuhe/Data9/" self.test_dir = "/home/newuhe/Data9/Test/" self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # self.rtmon = URRTMonitor("192.168.1.147") # som information is only available on rt interface # self.rtmon.start() # self.rtmon.set_csys(None) self.secmon = SecondaryMonitor("192.168.1.147") self.log = {} setup_logger('worker_log', r'worker_log') self.log['worker_log'] = logging.getLogger('worker_log') self.test_log = {} setup_logger('test_log', r'test_log') self.test_log['test_log'] = logging.getLogger('test_log') self.force = 18 self.d = -0.3 self.v = 0.0003 ## usb #self.primitive = [-0.61306625, -0.05149676 , 0.0065 ,-2.8868024, 1.23261833,0.00922825] ## round # self.primitive = [-0.30288 ,0.38096 , 0.019 , 2.3236 , -2.1558 , -0.0266] #(up pose)self.primitive = [-0.3026 ,0.37825 , 0.01041 , 2.3236 , -2.1558 , -0.0266] #(new round)self.primitive = [-4.96312008e-01 , 5.53456907e-01 ,1.12701285e-03, -1.16724081e+00, .87952391e+00 , 1.91615209e-02] self.primitive = [ -0.50454686, 0.56308032, 0.00392812, 1.19445411, -2.90431322, 0.05488621 ] self.half_count = 1 self.rcount = 3 self.count = 0 self.pcount = 0 ## for orientation self.sequence = [] self.rinit = -0.21 self.interval = 0.07 ## for translation self.xcount = 3 self.xhalf = (self.xcount - 1) / 2 self.ycount = 3 self.yhalf = (self.ycount - 1) / 2 ## round self.z = self.primitive[2] self.yinterval = 0.006 / self.xhalf self.xinterval = 0.006 / self.yhalf ## usb # self.z = 0.0065 # self.yinterval = 0.003 / self.yhalf # self.xinterval = 0.006 / self.xhalf # for i in range(9):b # self.xsequence.append(x) # self.ysequence.append(y) # x += self.xinterval # y += self.yinterval # self.ysequence_reverse = list(reversed(self.ysequence)) # self.sequence_reverse = list(reversed(self.sequence)) # print(self.xsequence) # print(self.ysequence) # print(self.sequence) #self.log['worker_log'].info('x- = %0.5f, x+ = %0.5f, y- = %0.5f, y+ = %0.5f, r- = %0.5f, r+ = %0.5f' % (self.xsequence[0],self.xsequence[8],self.ysequence[0],self.ysequence[8],self.sequence[0],self.sequence[8])) self.connect() ## reset to ground truth TCP:(0,0,0.225,0,0,0) self.movel(self.primitive, t=3) #self.movej([-1.20141346,-1.24530489,1.70199537,-2.01912338,-1.5878895,-0.38679821],t=3) ## take reference picture time.sleep(0.5) self.move_down(v=self.v) fz = 0 z = 0.1 # (up pose)while( abs(fz) < self.force and z>0.001): #(new round)while( abs(fz) < self.force and z>-0.005): while (abs(fz) < self.force and z > -0.004): pose = self.secmon.get_cartesian_info(False) z = pose["Z"] f.getForce() fx, fy, fz, px, py, pz = f.parseData() self.stop() ## take picture self.q1.put(1) pic, depth = self.q2.get(True) cv2.imwrite("/home/newuhe/Data9/" + 'reference.jpg', pic) cv2.imwrite("/home/newuhe/Data9/Test/" + 'reference.jpg', pic) ## pose init self.yi = int(self.count / self.ycount) self.xi = self.count - self.ycount * self.yi self.rxi = int(self.pcount / self.rcount) self.ryi = self.pcount - (self.half_count * 2 + 1) * self.rxi if self.yi % 2 == 0: if self.xi % 2 == 0: #orientation1 if self.rxi % 2 == 0: self.move(self.primitive, [ self.xinterval * (self.xhalf - self.xi), self.yinterval * (self.yhalf - self.yi), 0, self.interval * (self.half_count - self.rxi), self.interval * (self.half_count - self.ryi), 0 ]) else: self.move(self.primitive, [ self.xinterval * (self.xhalf - self.xi), self.yinterval * (self.yhalf - self.yi), 0, self.interval * (self.half_count - self.rxi), self.interval * (-self.half_count + self.ryi), 0 ]) else: #orientation2 if self.rxi % 2 == 0: self.move(self.primitive, [ self.xinterval * (self.xhalf - self.xi), self.yinterval * (self.yhalf - self.yi), 0, self.interval * (-self.half_count + self.rxi), self.interval * (-self.half_count + self.ryi), 0 ]) else: self.move(self.primitive, [ self.xinterval * (self.xhalf - self.xi), self.yinterval * (self.yhalf - self.yi), 0, self.interval * (-self.half_count + self.rxi), self.interval * (self.half_count - self.ryi), 0 ]) else: if self.xi % 2 == 0: #orientation2 if self.rxi % 2 == 0: self.move(self.primitive, [ self.xinterval * (-self.xhalf + self.xi), self.yinterval * (self.yhalf - self.yi), 0, self.interval * (-self.half_count + self.rxi), self.interval * (-self.half_count + self.ryi), 0 ]) else: self.move(self.primitive, [ self.xinterval * (-self.xhalf + self.xi), self.yinterval * (self.yhalf - self.yi), 0, self.interval * (-self.half_count + self.rxi), self.interval * (self.half_count - self.ryi), 0 ]) else: #orientation1 if self.rxi % 2 == 0: self.move(self.primitive, [ self.xinterval * (-self.xhalf + self.xi), self.yinterval * (self.yhalf - self.yi), 0, self.interval * (self.half_count - self.rxi), self.interval * (self.half_count - self.ryi), 0 ]) else: self.move(self.primitive, [ self.xinterval * (-self.xhalf + self.xi), self.yinterval * (self.yhalf - self.yi), 0, self.interval * (self.half_count - self.rxi), self.interval * (-self.half_count + self.ryi), 0 ])
burnin_str = '' if burnin: burnin_str = 'burn' + str(burnin).lower() reg_beta_str = '' if reg_beta > 0.0: reg_beta_str = 'reg' + str(reg_beta) + '_' additional_features_str = additional_features if additional_features != '': additional_features_str = additional_features + '_' tensorboard_name = str(args.logs_dir.absolute()) name_experiment = tensorboard_name logger = util.setup_logger(logs_dir=str(args.logs_dir.absolute()), also_stdout=True) logger.info('PARAMS : ' + name_experiment) logger.info('') logger.info(args) if dataset.startswith('PRFX'): if dataset == 'PRFX10': suffix = 'prefix_10_dataset' if dataset == 'PRFX30': suffix = 'prefix_30_dataset' if dataset == 'PRFX50': suffix = 'prefix_50_dataset' word_to_id_file_path = os.path.join(root_path, suffix, 'word_to_id') id_to_word_file_path = os.path.join(root_path, suffix, 'id_to_word') training_data_file_path = os.path.join(root_path, suffix, 'train')
__license__ = "ASF" __version__ = "2.0" __maintainer__ = "Bernardo Macias" __email__ = '*****@*****.**' __status__ = 'Production' import util import sys import ast from snowpy import snow from logging import INFO from splunklib.searchcommands import \ dispatch, GeneratingCommand, Configuration, Option logger = util.setup_logger(INFO) @Configuration(local=True) class snowNowCommand(GeneratingCommand): """ %(synopsis) ##Syntax .. code-block:: getsnow filters="<key1>=<value1> <key2>=<value2>" daysAgo=<int> env=<str> table=<str> ##Description Returns json events for Service Now API from tables. Limit 1000 events. ##Example
import os import logging import code_generator import util CODES_STR = "C6 D8 E3 G9 L3 M8 R3 T3" CODEWORD_FRAGMENT = "T" DICTIONARY_DIRECTORY = "dicts/" if __name__ == "__main__": util.setup_logger() logger = logging.getLogger() codes = util.parse_codecards(CODES_STR) codeword_fragment = util.parse_codeword_fragment(CODEWORD_FRAGMENT) dict_dir = os.path.abspath(DICTIONARY_DIRECTORY) if codes is None: logger.error( "Could not parse codes, code string was '{}'".format(CODES_STR)) elif codeword_fragment is None: logger.error( "Could not parse codeword fragment, fragment was '{}'".format( CODEWORD_FRAGMENT)) else: logger.info("Codes used: {}".format(CODES_STR)) logger.info("Dictionary dir: {}".format(dict_dir)) results = code_generator.create_codes_by_fragment( codes, CODEWORD_FRAGMENT, DICTIONARY_DIRECTORY) if len(results) == 0: logger.info("No valid codes found!")
# Plot generation for classification in classification_types: fig = plt.figure(figsize=(15, 30)) for i, score in enumerate(metrics_dict.keys()): ax = fig.add_subplot(len(set(metrics_dict))//2+1, 2, i + 1) ax.plot(number_of_classes, metrics_dict[score][classification]) plt.title("Classification %s for %s" % (score, classification)) plt.xlabel("Number of classes") plt.ylabel(score) plt.savefig("../images/LSHTC_" + classification) plt.close(fig) fig = plt.figure(figsize=(15, 30)) for i, score in enumerate(metrics_dict.keys()): ax = fig.add_subplot(len(set(metrics_dict))//2+1, 2, i + 1) for classification in classification_types: ax.plot(number_of_classes, metrics_dict[score][classification]) plt.title("Classification %s" % score) plt.legend(classification_types, loc="upper right") plt.xlabel("Number of classes") plt.ylabel(score) plt.savefig("../images/LSHTC_all") plt.close(fig) if __name__ == '__main__': force_locale_utf8() logger = setup_logger('lshtc') main()
from bot import BotApp import util config = util.load_config('config.json') logger = util.setup_logger(config['log_file']) bot = BotApp(oauth_token=config['auth_token'], channel=config['join_channel'], nick=config['nick']) boops = 0 @bot.command(name='test') def test(sender, *args): bot.send(f'Hello, @{sender}') @bot.command(name='addboop') def add_boop(sender, *args): bot.send(f'Incremented boop count') global boops boops += 1 @bot.command(name='boops') def print_boops(sender, *args): bot.send(f'Total boops: {boops}') def post_login(): bot.send('/me has arrived!')
import config # from datetime import datetime from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys from selenium.webdriver.support.ui import WebDriverWait from selenium.common.exceptions import TimeoutException from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.common.action_chains import ActionChains from xvfbwrapper import Xvfb # setup log wikilog = util.setup_logger("wikilog", "pyout/wikilog.log") #----------------------------------------------- # Startup browser (Firefox) # - returns browser object #----------------------------------------------- def start_browser(): ff_browser = webdriver.Firefox() ff_browser.implicitly_wait(10) ff_browser.maximize_window() return ff_browser
'INP' + inputs_geom + '_' + \ 'BIAS' + bias_geom + fix_biases_str + '_' + mat_str +\ 'FFNN' + ffnn_geom + str(before_mlr_dim) + ffnn_non_lin + '_' +\ additional_features_str + \ drp_str +\ 'MLR' + mlr_geom + '_' + \ reg_beta_str + \ hyp_opt_str + \ c_str +\ 'prje' + str(PROJ_EPS) + '_' + \ 'bs' + str(batch_size) + '_' +\ burnin_str + '__' + now.strftime("%H:%M:%S,%dM") name_experiment = tensorboard_name logger = util.setup_logger(name_experiment, logs_dir=os.path.join(root_path, 'logs/'), also_stdout=True) logger.info('PARAMS : ' + name_experiment) logger.info('') logger.info(args) if dataset.startswith('PRFX'): if dataset == 'PRFX10': suffix = 'prefix_10_dataset' if dataset == 'PRFX30': suffix = 'prefix_30_dataset' if dataset == 'PRFX50': suffix = 'prefix_50_dataset' word_to_id_file_path = os.path.join(root_path, suffix, 'word_to_id') id_to_word_file_path = os.path.join(root_path, suffix, 'id_to_word') training_data_file_path = os.path.join(root_path, suffix, 'train')
import argparse import logging from os import mkdir from os.path import dirname, join, isdir from loader import DateBasedOrganizer, get_record_filepaths from merge import move_files, copy_files from util import load_config, inject_config_if_missing, setup_logger, check_third_party # Setup logging logger = setup_logger(logging, __name__) # Main function def run(video_dir, audio_dir, output_dir=None, move=False): if output_dir is None: output_dir = join(dirname(__file__), 'results') if not isdir(output_dir): mkdir(output_dir) logger.info('Prehladavam zlozky ktore si zadal...') files = get_record_filepaths(video_dir) files.extend(get_record_filepaths(audio_dir)) logger.info('Nasiel som %d nahravok. (konkretne mas v logu)' % len(files)) logger.debug('Najdene nahravky:\n%s' % '\n'.join(files)) logger.info('Skupinkujem ti nahravky...') organizer = DateBasedOrganizer()
help='Location of the noise') parser.add_argument('--poison_rate', default=1.0, type=float) parser.add_argument('--perturb_tensor_filepath', default=None, type=str) args = parser.parse_args() # Set up Experiments if args.exp_name == '': args.exp_name = 'exp_' + datetime.datetime.now() exp_path = os.path.join(args.exp_name, args.version) log_file_path = os.path.join(exp_path, args.version) checkpoint_path = os.path.join(exp_path, 'checkpoints') checkpoint_path_file = os.path.join(checkpoint_path, args.version) util.build_dirs(exp_path) util.build_dirs(checkpoint_path) logger = util.setup_logger(name=args.version, log_file=log_file_path + ".log") # CUDA Options logger.info("PyTorch Version: %s" % (torch.__version__)) if torch.cuda.is_available(): torch.cuda.manual_seed(args.seed) torch.backends.cudnn.enabled = True torch.backends.cudnn.benchmark = True device = torch.device('cuda') device_list = [ torch.cuda.get_device_name(i) for i in range(0, torch.cuda.device_count()) ] logger.info("GPU List: %s" % (device_list)) else: device = torch.device('cpu')
def train(): args = parser.parse_args() print(args.gpu) log = {} setup_logger('train_log', r'train_log') log['train_log'] = logging.getLogger('train_log') save_model_dir = "/home/newuhe/UR5_control/trained_model/" n_epochs = 1000000 save_time = 500 gpu = False generator = dataGenerator() model = locNet() if args.load is True: saved_state = torch.load(args.load_model_dir + str(args.epoch) + '.dat') model.load_state_dict(saved_state) print("load succssfully") model.train() if args.gpu is True: print("gpu used") model = model.cuda() criterion = nn.MSELoss() optimizer = optim.Adam(model.parameters(), lr=0.001) losses = np.zeros(n_epochs) for epoch in range(args.epoch, n_epochs): ## one sequence p, f, a, t = generator.train_data() pic = Variable(torch.from_numpy(np.array(p)).float()) pic = pic.permute(0, 3, 1, 2) # (10,3,120,160) force = Variable(torch.from_numpy(np.array(f)).float()) # (10,1,6) action = Variable(torch.from_numpy(np.array(a)).float()) target = Variable(torch.from_numpy(np.array(t)).float()) cx = Variable(torch.zeros(1, 1, 72)) hx = Variable(torch.zeros(1, 1, 72)) if args.gpu is True: hx = hx.cuda() cx = cx.cuda() pic = pic.cuda() force = force.cuda() action = action.cuda() target = target.cuda() model.zero_grad() # for pic, force, action, target in train_data: # pic = Variable(torch.from_numpy(np.array([pic.tolist(),])).float()) # pic = pic.permute(0,3,1,2) # force = Variable(torch.from_numpy(np.array([force,])).float()) # action = Variable(torch.from_numpy(np.array([action,])).float()) # target = Variable(torch.from_numpy(np.array([target,])).float()) pos, (hx, cx) = model((pic, force, action, (hx, cx))) loss = criterion(pos, target) #loss = criterion(pos + action, target) # loss.backward(retain_graph=True) loss.backward() optimizer.step() if epoch % 500 == 0: log['train_log'].info('epoch = %d,train_loss = %0.5f' % (epoch, loss)) if epoch % save_time == 0: ## save model and eval state_to_save = model.state_dict() torch.save(state_to_save, '{0}{1}.dat'.format(save_model_dir, epoch)) print("get: ", pos) print("target: ", target) pt, ft, at, tt = generator.test_data() pict = Variable(torch.from_numpy(np.array(pt)).float()) pict = pict.permute(0, 3, 1, 2) # (10,3,120,160) forcet = Variable(torch.from_numpy( np.array(ft)).float()) # (10,1,6) actiont = Variable(torch.from_numpy(np.array(at)).float()) targett = Variable(torch.from_numpy(np.array(tt)).float()) cxt = Variable(torch.zeros(1, 1, 72)) hxt = Variable(torch.zeros(1, 1, 72)) if args.gpu is True: hxt = hxt.cuda() cxt = cxt.cuda() pict = pict.cuda() forcet = forcet.cuda() actiont = actiont.cuda() targett = targett.cuda() post, (hxt, cxt) = model((pict, forcet, actiont, (hxt, cxt))) losst = criterion(post, targett) # for pic, force, action, target in test_data: # pic = Variable(torch.from_numpy(np.array([pic.tolist(),])).float()) # pic = pic.permute(0,3,1,2) # force = Variable(torch.from_numpy(np.array([force,])).float()) # action = Variable(torch.from_numpy(np.array([action,])).float()) # target = Variable(torch.from_numpy(np.array([target,])).float()) # if args.gpu==True: # pic = pic.cuda() # force = force.cuda() # ation = action.cuda() # target = target.cuda() # pos, (hx, cx) = model( (pic, force, action,(hx, cx)) ) # loss = criterion(pos, target) # test_loss.append(loss.data[0]) log['train_log'].info('epoch = %d,test_loss = %0.5f' % (epoch, losst))
'nb_actions': nb_actions, 'args': args, } agent = WolpertingerAgent(**agent_args) if args.load: agent.load_weights(args.load_model_dir) if args.gpu_ids[0] >= 0 and args.gpu_nums > 0: agent.cuda_convert() # set logger, log args here log = {} if args.mode == 'train': setup_logger('RS_log', r'{}/RS_train_log'.format(args.save_model_dir)) elif args.mode == 'test': setup_logger('RS_log', r'{}/RS_test_log'.format(args.save_model_dir)) else: raise RuntimeError('undefined mode {}'.format(args.mode)) log['RS_log'] = logging.getLogger('RS_log') d_args = vars(args) d_args['max_actions'] = args.max_actions for key in agent_args.keys(): if key == 'args': continue d_args[key] = agent_args[key] for k in d_args.keys(): log['RS_log'].info('{0}: {1}'.format(k, d_args[k])) if args.mode == 'train':
# Imports import argparse import util import os, random import json import torch #Configure log logging = util.setup_logger(__name__, 'app.log') # Configure ArgumentParser parser = argparse.ArgumentParser(description = 'Predict the different species of flowers.') parser.add_argument('img_path', action = 'store', help = 'Directory with images for predict.') parser.add_argument('checkpoint_file', action = 'store', help = 'checkpoint file.') parser.add_argument('--gpu', action='store_true', help='use gpu to infer classes') parser.add_argument('--topk', action = 'store', dest = 'topk', type=int, default = 5, required = False, help = 'Return top K most likely classes') parser.add_argument('--category_names', action='store', help='Label mapping file') arguments = parser.parse_args() try: # Use GPU if it's available #device = util.choose_device(arguments.gpu) #loads a checkpoint and rebuilds the model model = util.load_checkpoint(arguments.checkpoint_file) model.eval() #Image Preprocessing img_file = random.choice(os.listdir(arguments.img_path))
import json import os from load import Load from cdf import CDF from util import setup_logger with open('config.json', 'r') as f: cfg = json.load(f) proxy = '{}:{}'.format(cfg['proxyIp'], str(cfg['proxyPort'])) PROXYDICT = {"http": proxy, "https": proxy} urls = cfg['urls'] setup_logger('', console=True) load = Load(cfg['numClients'], cfg['reqsPerClient'], cfg['iterations'], PROXYDICT, urls) load.run() cdf = CDF(PROXYDICT, urls) cdf.run(cfg['numRequests'])
parser.add_argument(action="store", dest='output_path', type=str, metavar='PATH') parser.add_argument('-c', '--config', action="store", dest='config_file', default='/ARD_Clip.conf', required=False, type=str, metavar='PATH') return vars(parser.parse_args()) if __name__ == '__main__': args = parse_cli() conf = config.read_config(args['config_file']) setup_logger(level='debug' if conf.debug else 'info') logger.info('******************Start************') logger.info(' DB connection: %s', conf.connstr) logger.info(" Version: %s", conf.version) logger.info(" Debug: %s", conf.debug) logger.info('segment: %s', args['segment']) logger.info('output path: %s', args['output_path']) process_segments(args['segment'], args['output_path'], conf) logger.info('..................Normal End............')
def generate(self): logger = util.setup_logger(INFO) try: default_conf = util.getstanza('hipchat', 'default') local_conf = util.getstanza('hipchat', 'hipchat') proxy_conf = util.setproxy(local_conf, default_conf) hipchat_url = local_conf[ 'url'] if 'url' in local_conf else default_conf['url'] auth_token = local_conf[ 'authToken'] if 'authToken' in local_conf else default_conf[ 'autToken'] timeout = local_conf[ 'timeout'] if 'timeout' in local_conf else default_conf[ 'timeout'] finally: logger.info( 'Unable to parse Config File. Check if hipchat.conf exists') raise Exception( "Unable to parse Config File. Check if hipchat.conf exists") exit() headers = dict(Authorization='Bearer {0}'.format(auth_token)) data = dict(message=None, message_format='text') hipchat_room_url = '{0}/v2/room/{1}/notification'.format( hipchat_url, self.room) hipchat_room_list_url = '{0}/v2/room'.format(hipchat_url) if not self.listrooms: for record in records: message = None for key, value in record: message = message.join('{0}={1} '.format(key, value)) response = util.request(hipchat_room_url, data=data, headers=headers, timeout=timeout, proxy=proxy_conf) logger.info( 'sending notification room={0} status_code={1} response={2}' .format(self.room, response['code'], response['msg'])) record['status_code'] = response['code'] record['response'] = response['msg'] record['_raw'] = util.tojson(response) yield record else: while hipchat_room_list_url: response = util.request(hipchat_room_list_url, headers=headers, timeout=timeout, proxy=proxy_conf) if response['code'] == 200: room_list = json.loads(response['msg']) hipchat_room_list_url = room_list['links'][ 'next'] if 'next' in room_list['links'] else None for room in room_list['items']: room_info = dict() room_info['room_id'] = room['id'] room_info['room_name'] = room['id'] room_info['_raw'] = util.tojson(room_info) yield room_info else: yield response exit()
# --------- # # Constants # # --------- # PRODUCTS = { "CMS": "Cisco Meeting Server", "CMA": "Cisco Meeting App", "CMM": "Cisco Meeting Manager" } SWDLFILE = r'data\SWDL_data.xlsx' SWDLSHEET = r'SWDownloads-123' # setup log swdllog = util.setup_logger("swdllog", "swdllog.log") #------------------------------------------------------------- # Import data from a defined sheet in a given Excel workbook # - returns DataFrame structure #------------------------------------------------------------- def import_from_excel(xlfile, xlsheet): import_df = None if not xlfile or not xlsheet: swdllog.error("Excel filename and sheetname required for import") return import_df try:
from flask import Flask from flask import request from multiprocessing import Process import util from conf import config from main import notify_rt_oneshot app = Flask(__name__) logger = util.setup_logger() @app.route("/") def hello(): logger.debug('123') return "Hello World!" @app.route("/bus_watcher", methods=['POST']) def bus_watcher(): data = request.json route = data['route'] dep_station = data['dep_station'] at = data['at'] t = Process(target=notify_rt_oneshot, args=(route, dep_station, at)) t.deamon = True t.start() return "Enabled notify for bus %s from %s when it's at %s" % ( route, dep_station, at)
import json import requests import util log = util.setup_logger(__name__) class CoinCap(object): """ A class which manages connections to the CoinMarketCap.com API """ def __init__(self): self.url = "https://api.coinmarketcap.com/v1/ticker/" def get_coin_values_usd(self, coin_name_array, portfolio): """ For a dictionary which maps coin to their amount returns a dictionary which maps those coints to the corresponding values in USD. """ price_data = self.get_coin_price_data(coin_name_array) d = {} for symb, amount in portfolio.items(): for api_coin, price_dict in price_data.items(): if price_dict["symbol"] == symb: d[symb] = amount * float(price_dict["price"]) break return d def get_coin_names(self, count):
def generate(self): # Parse and set arguments logger = util.setup_logger(INFO) # get config env = self.env.lower() if self.env else 'production' conf = util.getstanza('getsnow', env) proxy_conf = util.getstanza('getsnow', 'global') proxies = util.setproxy(conf, proxy_conf) username = conf['user'] password = conf['password'] url = conf['url'] user_query = '%s/api/now/table/%s?sysparm_query=user_name=%s' % (url, 'sys_user', self.user_name) response = util.request(user_query, username=username, password=password, headers={'Accept': 'application/json'} ) if response['code'] == 200: records = json.loads(response['msg']) # for each event creating dic object for yield for record in records['result']: record['_time'] = time.mktime(datetime.datetime.strptime(record['sys_created_on'], "%Y-%m-%d %H:%M:%S").timetuple()) record['url'] = url if record['manager']['link']: response = util.request(record['manager']['link'], username=username, password=password, headers={'Accept': 'application/json'} ) manager = json.loads(response['msg'])['result'] record['manager'] = manager['name'] record['manager_email'] = manager['email'] record['manager_phone'] = manager['phone'] if record['u_office']['link']: response = util.request(record['u_office']['link'], username=username, password=password, headers={'Accept': 'application/json'} ) office = json.loads(response['msg'])['result'] record['office_number'] = office['u_office_number'] if record['location']['link']: response = util.request(record['location']['link'], username=username, password=password, headers={'Accept': 'application/json'} ) location = json.loads(response['msg'])['result'] record['office_name'] = location['full_name'] if record['department']['link']: response = util.request(record['department']['link'], username=username, password=password, headers={'Accept': 'application/json'} ) department = json.loads(response['msg'])['result'] record['department'] = department['name'] # removing unnecessary keys record.pop('sys_domain', None) record.pop('u_office', None) record.pop('company', None) record.pop('u_organization_group', None) record.pop('u_title', None) record.pop('ldap_server', None) record.pop('cost_center', None) user_sysid = record['sys_id'] record['sourcetype'] = 'snow:user' record['source'] = user_query # adding _raw to record record['_raw'] = util.tojson(record) record['_time'] = time.mktime(datetime.datetime.strptime(record['sys_created_on'], "%Y-%m-%d %H:%M:%S").timetuple()) #yielding record yield record # building query string incidents time_range = 'opened_at>=javascript:gs.daysAgo(%s)^' % self.daysAgo if self.daysAgo else '' incident_query = '%s/api/now/table/%s?sysparm_query=%sopened_by=%s' % (url, 'incident', time_range, user_sysid) response = util.request(incident_query, username=username, password=password, headers={'Accept': 'application/json'} ) if response['code'] == 200: incidents = json.loads(response['msg'])['result'] # replacing all sys_id with user_names for incident in incidents: incident = keyreplace(incident, 'closed_by', 'user_name', username, password) incident = keyreplace(incident, 'opened_by', 'user_name', username, password) incident = keyreplace(incident, 'assigned_to', 'user_name', username, password) incident = keyreplace(incident, 'resolved_by', 'user_name', username, password) incident = keyreplace(incident, 'caller_id', 'user_name', username, password) incident = keyreplace(incident, 'u_opened_for', 'user_name', username, password) incident = keyreplace(incident, 'assignment_group', 'name', username, password) incident['source'] = incident_query incident['sourcetype'] = 'snow:incident' incident['_time'] = time.mktime(datetime.datetime.strptime(incident['sys_created_on'], "%Y-%m-%d %H:%M:%S").timetuple()) incident['_raw'] = util.tojson(incident) # removing unnecessary keys incident.pop('company', None) incident.pop('location', None) # yield incident record yield incident else: try: # If not 200 status_code showing error message in Splunk UI record = util.dictexpand(response) record['url'] = url record['_raw'] = util.tojson(response) except Exception as e: record = dict() record['url'] = url record['error'] = e record['_raw'] = util.tojson(response) yield record asset_query = '%s/api/now/table/%s?sysparm_query=assigned_to=%s' % (url, 'alm_asset', user_sysid) response = util.request(asset_query, username=username, password=password, headers={'Accept': 'application/json'} ) if response['code'] == 200: assets = json.loads(response['msg'])['result'] for asset in assets: asset.pop('support_group', None) asset.pop('department', None) asset.pop('model', None) asset.pop('ci', None) asset.pop('company', None) asset.pop('location', None) asset.pop('model_category', None) asset.pop('cost_center', None) asset.pop('sys_domain', None) asset['source'] = asset_query asset['_time'] = time.mktime(datetime.datetime.strptime(asset['sys_created_on'], "%Y-%m-%d %H:%M:%S").timetuple()) asset['sourcetype'] = 'snow:asset' asset['_raw'] = util.tojson(asset) yield asset else: try: # If not 200 status_code showing error message in Splunk UI record = util.dictexpand(response) record['url'] = url record['_raw'] = util.tojson(response) except Exception as e: record = dict() record['url'] = url record['error'] = e record['_raw'] = util.tojson(response) yield record else: try: # If not 200 status_code showing error message in Splunk UI record = util.dictexpand(response) record['url'] = url record['_raw'] = util.tojson(response) except Exception as e: record = dict() record['url'] = url record['error'] = e record['_raw'] = util.tojson(response) yield record