def run_bert_finetuning(datadir: str, filename: str, train_size: float, k: int = 0): util.init_logging(logging.DEBUG, "relationModel_bert_" + str(k)) if k > 0: k_fold_split, data_df = dataset.prepare_data_cross_validation( datadir + filename, True, k) for train_indices, eval_indices in k_fold_split: # split feature vectors into train and eval train_df = data_df.loc[train_indices, :] eval_df = data_df.loc[eval_indices, :] # run BERT fine-tuning (training and evaluation) classifier_bert.model_with_simpletransformers( train_df, eval_df, filename.rsplit('.')[0]) elif train_size == 1.0: # run BERT fine-tuning (100% training) train_df = dataset.edit_class_labels_df( dataset.read_in_data(datadir + filename, True)) classifier_bert.model_with_simpletransformers(train_df, None, filename.rsplit('.')[0]) else: # split data into train and eval train_df, eval_df = dataset.prepare_data_train_test( datadir + filename, train_size, True) # run BERT fine-tuning (training and evaluation) classifier_bert.model_with_simpletransformers(train_df, eval_df, filename.rsplit('.')[0])
def __init__(self, host=None, port=None, uri=None, **kwargs): init_logging() cnf = loadConfig() self.host = cnf.getHost() self.port = cnf.getPort() con = '' if kwargs: for key in kwargs: uriStr = '' uriStr = '&' + key + "=" + kwargs[key] con += uriStr self.uri = uri + con httpClient = None try: httpClient = httplib.HTTPConnection(self.host, self.port, timeout=30) httpClient.request('GET', self.uri) tmp = httpClient.getresponse().read() buf = json.loads(tmp) self.result = buf except: logging.debug('get data failed.....') finally: if httpClient: httpClient.close()
def __init__(self, host=None, port=None, uri=None, **kwargs): init_logging() cnf = loadConfig() self.host = cnf.getHost() print self.host self.port = cnf.getPort() con = '' if kwargs: for key in kwargs: uriStr = '' uriStr = '&' + key + "=" + kwargs[key] con += uriStr self.uri = uri + con httpClient = None try: httpClient = httplib.HTTPConnection( self.host, self.port, timeout=30) httpClient.request('GET', self.uri) tmp = httpClient.getresponse().read() buf = json.loads(tmp) self.result = buf except: logging.debug('get data failed.....') finally: if httpClient: httpClient.close()
def handle_args(): """handle cmdline arguments usual procedure: 1. -f collect facts 2. --ensure-all-change-types post-process so that jgrok does not throw exception 3. -s run grok 4. -v verify 5.1 --count-class results in TeX, number of test classes 5.2.1 --count-method-json results in json, number of methods 5.2.2 --count-method-tex results in TeX, number of methods, in comparison with existing tools optional. --resume skip existing optional. --debug run with a small subset of projects optional. -l specify log level :return: arguments """ parser = argparse.ArgumentParser(description="Test selection on Defects4j") parser.add_argument("-l", metavar="LOG_LEVEL", type=str) parser.add_argument("-f", action="store_true", help="Collect facts") parser.add_argument( "--resume", action="store_true", help= "Resume process, according to the existence of files/dirs. Works with -f, -s" ) parser.add_argument( "--ensure-all-change-types", action="store_true", help= "Ensure that all change types exist in diff facts by adding NONE facts" ) parser.add_argument("--debug", action="store_true", help="Use one bug pair for testing, default: True") parser.add_argument("-s", metavar="GROK_SCRIPT", help="Run grok with specified grok script") parser.add_argument("-v", action="store_true", help="Verify grok results") parser.add_argument("--count-class", metavar="OUTPUT_TEX_SRC", help="Output TeX source for a results table") parser.add_argument( "--count-method-json", metavar="OUTPUT_JSON", help="Count test methods of affected test classes and write OUTPUT_JSON" ) parser.add_argument("--count-method-tex", nargs=2, metavar=("JSON_FILE", "OUTPUT_TEX_SRC"), help="Read JSON and write TeX table") parser.add_argument("--percent", metavar="JSON_FILE", help="Calculate percentage") args = parser.parse_args() init_logging(args.l) logger.debug(args) return args
def setUp(self): self.test_dir = tempfile.TemporaryDirectory() self.conf = util.init_conf([]) self.conf.steam_dir = self.test_dir.name self.conf.ts_root = self.test_dir.name self.conf.aoe2_usr_dir = 'replays' self.conf.player = 'mr_monkey' os.mkdir(os.path.join(self.test_dir.name, self.conf.aoe2_usr_dir)) util.add_derived_fields(self.conf) util.init_logging(self.conf)
def __init__(self): init_logging() self.path = os.path.split(os.path.realpath(__file__))[0] + "/config.json" with open(self.path) as f: configInfor = json.load(f) if configInfor is not None: self.host = configInfor["host"] self.port = configInfor["port"] else: logging.debug("load config failed.....")
def main(): for i in range(len(Stats.n)): for c in range(Stats.count[i]): print("i: " + str(i) + ", c: " + str(c)) max_circle_matrix = create_max_circle_matrix(Stats.n[i]) timer = datetime.now() result_max_circle_1 = tmg.make_relation_matrix( np.copy(max_circle_matrix)) Stats.max_circle_tmg_time[i] += datetime.now() - timer print( str(datetime.now()) + " max 1 " + str(Stats.max_circle_tmg_time[i])) #timer = datetime.now() #result_max_circle_2 = bilp.make_relation_matrix(np.copy(max_circle_matrix)) #Stats.max_circle_bilp_time[i] += datetime.now() - timer #print(str(datetime.now()) + " max 2 " + str(Stats.max_circle_bilp_time[i])) no_circle_matrix = create_no_circle_matrix(result_max_circle_1) timer = datetime.now() result_no_circle_1 = tmg.make_relation_matrix( np.copy(no_circle_matrix)) Stats.no_circle_tmg_time[i] += datetime.now() - timer print( str(datetime.now()) + " no 1 " + str(Stats.no_circle_tmg_time[i])) #timer = datetime.now() #result_no_circle_2 = bilp.make_relation_matrix(np.copy(no_circle_matrix)) #Stats.no_circle_bilp_time[i] += datetime.now() - timer #print(str(datetime.now()) + " no 2 " + str(Stats.no_circle_bilp_time[i])) single_circle_matrix = create_single_circle_matrix(Stats.n[i]) timer = datetime.now() result_single_circle_1 = tmg.make_relation_matrix( np.copy(single_circle_matrix)) Stats.single_circle_tmg_time[i] += datetime.now() - timer print( str(datetime.now()) + " single 1 " + str(Stats.single_circle_tmg_time[i])) #timer = datetime.now() #result_single_circle_2 = bilp.make_relation_matrix(np.copy(single_circle_matrix)) #Stats.single_circle_bilp_time[i] += datetime.now() - timer #print(str(datetime.now()) + " single 2 " + str(Stats.single_circle_bilp_time[i])) util.init_logging(logging.DEBUG, "GenerateTreeTiming_" + time.strftime("%Y%m%d-%H%M%S")) Stats.log_results() exit(0)
def main(): # logging util.init_logging(logging.INFO, "PredictRelationTiming_" + time.strftime("%Y%m%d-%H%M%S")) # command line interface parser = argparse.ArgumentParser() parser.add_argument('MajorClaim', type=str, help='topic of the discussion') parser.add_argument('--search', type=int, help='number of sentences to further process from the search results (if not given all sentences retrieved are used)') parser.add_argument('--classify', nargs='+', type=str, help='multiple sentences (group sentences with ""), a text or an url to be used as a source to collect arguments from') parser.add_argument('-svm', action='store_true', help='change classifier for estimating relation probabilities from BERT to SVM') parser.add_argument('-bilp', action='store_true', help='change from generate tree approach tmg (Traversing and Modifying Graphs) to bilp (Binary Linear Integer Programming)') parser.add_argument('--cluster', nargs=2, type=float, help='cluster arguments before processing them (relation only possible within cluster) -> first arg: similarity threshold, second arg: min_cluster_size') args = parser.parse_args() logging.info(args) # search engine search_engine = ArgumenText("userID", "apiKey") if args.classify is None: sentences = search_engine.query_search_api(args.MajorClaim) if args.search is not None and args.search < len(sentences): stance_pro = [a for a in sentences if a["stanceLabel"] == 'pro'] stance_con = [a for a in sentences if a["stanceLabel"] == 'contra'] stance_pro.sort(key=lambda s: s["argumentConfidence"]*s["stanceConfidence"], reverse=True) stance_con.sort(key=lambda s: s["argumentConfidence"]*s["stanceConfidence"], reverse=True) pro_len = min(int(args.search/2), len(stance_pro)) con_len = min(args.search - pro_len, len(stance_con)) diff = args.search - pro_len - con_len pro_len += diff sentences = stance_pro[:pro_len] sentences.extend(stance_con[:con_len]) else: if len(args.classify) is 1: args.classify = args.classify[0] sentences = search_engine.query_classify_api(args.MajorClaim, args.classify) arguments = ArgumentList(args.MajorClaim, sentences) # clustering if args.cluster is not None: clusters = search_engine.query_cluster_api([s["sentenceOriginal"] for s in sentences], args.cluster[0], args.cluster[1]) logging.debug(clusters) arguments.apply_clusters(clusters) # relation processing relation_processing = RelationProcessor(args.svm, args.bilp) relation_processing.generate_relation_matrix(arguments) Stats.log_results() exit(0)
def handle_args(): parser = argparse.ArgumentParser(description="compute difference of two generated list of commits") parser.add_argument("-p", required=True, metavar="FACTS_DIR", help="Path to the directory of generated facts") parser.add_argument("-o", metavar="OUTPUT_FILE", type=str) parser.add_argument("-l", metavar="LOG_LEVEL", type=str) args = parser.parse_args() init_logging(args.l) logger.debug(args) return args
def handle_args(): parser = argparse.ArgumentParser( description="Read ground truth from DoSC dataset") parser.add_argument("-f", required=True, metavar="FUNCTIONALITY", help="List slice for a specific functionality") parser.add_argument("-l", metavar="LOG_LEVEL", type=str) args = parser.parse_args() init_logging(args.l) logger.debug(args) return args
def handle_args(): parser = argparse.ArgumentParser(description="compute difference of two generated list of commits") parser.add_argument("-s", metavar="SEL_FILE", type=str, required=True) parser.add_argument("-c", metavar="ORIG_CFG_DIR", type=str) parser.add_argument("--os", metavar="OUT_NAMES_JSON", type=str, help="output json file for store benchmark names") parser.add_argument("--og", metavar="OUT_GROUP_JSON", type=str, help="output json file for store benchmark groups") parser.add_argument("-l", metavar="LOG_LEVEL", type=str) parser.add_argument("--all-benchmark", action="store_true") parser.add_argument("--existing-config", action="store_true") args = parser.parse_args() init_logging(args.l) logger.debug(args) return args
def run_sklearn_classifiers(datadir: str, filename: str, train_size: float, k: int = 0, feature_range=range(0, 15), specific_model=None): # read in data data_df = dataset.read_in_data(datadir + filename, True) for counter in feature_range: util.init_logging( logging.DEBUG, "relationModel_sklearn_" + str(counter) + "_" + str(k)) # generate feature vectors from data data_feature, data_label = features.make_featurevector( data_df, 'features/' + filename + '/bert/sentence_pair.joblib', 'features/' + filename + '/bert/sentence.joblib', counter) if k > 0: k_fold_split, data_label = dataset.prepare_feature_cross_validation( data_feature, data_label, k) for train_indices, eval_indices in k_fold_split: # split feature vectors into train and eval train_feature, train_label = data_feature[ train_indices], data_label[train_indices] eval_feature, eval_label = data_feature[ eval_indices], data_label[eval_indices] # run sklearn classifier bench (training and evaluation) classifier_sklearn.models_with_sklearn(train_feature, train_label, eval_feature, eval_label, filename.rsplit('.')[0], specific_model) elif train_size == 1.0: # run sklearn classifier bench (100% training) data_label = dataset.edit_class_label_np(data_label) classifier_sklearn.models_with_sklearn(data_feature, data_label, None, None, filename.rsplit('.')[0], specific_model) else: # split feature vectors into train and eval train_feature, train_label, eval_feature, eval_label = dataset.prepare_feature_train_test( data_feature, data_label, train_size) # run sklearn classifier bench (training and evaluation) classifier_sklearn.models_with_sklearn(train_feature, train_label, eval_feature, eval_label, filename.rsplit('.')[0], specific_model)
def __init__(self): init_logging() self.path = os.path.split( os.path.realpath(__file__))[0] + '/config.json' with open(self.path) as f: configInfor = json.load(f) if configInfor is not None: self.host = configInfor['host'] self.port = configInfor['port'] self.user = configInfor['user'] self.passwd = configInfor['passwd'] self.db = configInfor['db'] self.url = configInfor['url'] else: logging.debug("load config failed......")
def main(): # values datadir = "dataset/" filename = "MyDataset_balanced" train_size = 0.8 logfile_name = "relationModel_MyDatasetBalanced" # init logging util.init_logging(logging.DEBUG, logfile_name) # sklearn classifiers run_sklearn_classifiers(datadir, filename, train_size, k=5) # bert fine-tuning run_bert_finetuning(datadir, filename, train_size)
def main(argv=sys.argv): """Main test""" global USAGE parser = OptionParser(USAGE, formatter=TitledHelpFormatter(), version="benchbase %s" % get_version()) parser.add_option("-v", "--verbose", action="store_true", help="Verbose output") parser.add_option("-l", "--logfile", type="string", default=os.path.expanduser(DEFAULT_LOG), help="Log file path") parser.add_option("-d", "--database", type="string", default=os.path.expanduser(DEFAULT_DB), help="SQLite db path") parser.add_option("-m", "--comment", type="string", help="Add a comment") parser.add_option("-j", "--jmeter", action="store_true", default=True, help="JMeter input file") parser.add_option("-f", "--funkload", action="store_true", default=False, help="FunkLoad input file") parser.add_option("--rmdatabase", action="store_true", default=False, help="Remove existing database") parser.add_option("-o", "--output", type="string", help="Report output directory") parser.add_option("-H", "--host", type="string", help="Host name when adding sar report") parser.add_option("-r", "--runningavg", type="int", default=5, help="Number of second to compute the running average.") parser.add_option("--chart-width", type="int", default=800, help="Width of charts in report.") parser.add_option("--chart-height", type="int", default=768, help="Heigth of charts in report.") parser.add_option("--period", type="int", help="Resolution in second") options, args = parser.parse_args(argv) init_logging(options) if len(args) == 1: parser.error("Missing command") cmd = args[1] fn = globals()['cmd_' + cmd] ret = fn(args[2:], options) return ret
def main(): conf = util.init_conf() util.init_logging(conf) if not conf.subcommand: logging.error("No subcommand specified\n%s", conf.usage) return subcmd_func = { 'find_new': find_new_replay_and_ts, 'list': list_replay_in_repo, 'plot_ts': plot_ts_file, 'plot_last_ts': plot_most_recent_ts_in_repo, 'plot_replay': plot_replay_by_idx, 'plot_last_replay': plot_last_replay_in_repo, } lazy_import_matplotlib() subcmd_func[conf.subcommand[0]](conf)
def handle_args(): parser = argparse.ArgumentParser( description="compute difference of two generated list of commits") parser.add_argument( "--cslicer", metavar="NAMES", help="Run cslicer (old way) on a list of project names") parser.add_argument("--prepare", metavar="NAMES", help="Preparing work for a list of project names") parser.add_argument("--fact", metavar="GROUPS", help="Run cslicer with facts collection") parser.add_argument("--verify", action="store_true", help="verify results") parser.add_argument("-l", metavar="LOG_LEVEL", type=str) args = parser.parse_args() init_logging(args.l) logger.debug(args) return args
def main(argv=sys.argv): """Main test""" global USAGE parser = OptionParser(USAGE, formatter=TitledHelpFormatter(), version="jenkviz %s" % get_version()) parser.add_option("-v", "--verbose", action="store_true", help="Verbose output") parser.add_option("-l", "--logfile", type="string", default=os.path.expanduser(DEFAULT_LOG), help="Log file path") parser.add_option("-d", "--database", type="string", default=os.path.expanduser(DEFAULT_DB), help="SQLite db path") parser.add_option("-o", "--output", type="string", help="SVG output file") parser.add_option("--from-file", type="string", help="Use html files in the the FROM_FILE directory instead of querying jenkins server.") parser.add_option("--to-file", type="string", help="Save jenkins page into the TO_FILE directory.") parser.add_option("-r", "--reverse", action="store_true", default=False, help="Reverse crawl") parser.add_option("--direct", action="store_true", default=False, help="Display only direct upstream dependencies") parser.add_option("--explore", action="store_true", default=False, help="Display downstream build with external upstream") parser.add_option("-u", "--update", action="store_true", default=False, help="Always fetch build from server (update local database)") options, args = parser.parse_args(argv) if options.explore: options.direct = False init_logging(options) if len(args) == 1: parser.error("Missing command") cmd = args[1] fn = globals()['cmd_' + cmd] ret = fn(args[2:], options) return ret
def main(): logger = init_logging() parser = argparse.ArgumentParser( description= "This script analyzes a Redshift cluster and outputs a summary report with statistics" "its performance.") parser.add_argument('-b', '--bucket', nargs=1, type=str, help='location of replay outputs') parser.add_argument('-r1', '--replay_id1', nargs='?', type=str, default='', help='replay id 1') parser.add_argument('-r2', '--replay_id2', nargs='?', type=str, default='', help='replay id 2, required for ' 'comparison') parser.add_argument('-s', '--sql', action='store_true', help='sql') args = parser.parse_args() if not (args.bucket or args.replay_id1 or args.replay_id2): print("Find work location") elif args.bucket and not (args.replay_id1 or args.replay_id2): list_replays(args.bucket[0]) elif args.bucket and args.replay_id1 and not args.replay_id2: if args.sql: list_sql(args.bucket[0], args.replay_id1) else: analysis_summary(args.bucket[0], args.replay_id1) elif args.bucket and args.replay_id1 and args.replay_id2: if args.replay_id1 == args.replay_id2: logger.error( "Cannot compare same replay, please choose two distinct replay ids." ) exit(-1) else: print(f"Compare replays {args.replay_id1} and {args.replay_id2}.") run_comparison_analysis(args.bucket[0], args.replay_id1, args.replay_id2) else: print("Please enter valid arguments.") exit(-1)
def __init__(self, config, cluster_name, clock, limits_timeout=300): self.config = config self.cluster_name = cluster_name self.clock = clock self.limits_timeout = limits_timeout # initialize in constructor so that cyclecloud_provider can initialize this # with the proper log_level. In tests, this will use the default. self.logger = init_logging() default_dir = os.getenv('HF_WORKDIR', '/var/tmp') self.db_dir = config.get('symphony.hostfactory.db_path', default_dir) self.requests_db = JsonStore('azurecc_requests.json', self.db_dir) self.capacity_db = JsonStore('azurecc_capacity.json', self.db_dir)
def handle_args(): parser = argparse.ArgumentParser( description="compute difference of two generated list of commits") parser.add_argument("-p", required=True, metavar="FACTS_DIR", help="Path to the directory of generated facts") parser.add_argument( "-g", required=True, metavar="GROUP_FILE", help="A JSON file listing all groups for easy processing") parser.add_argument("-s", required=True, metavar="GROK_SCRIPT", help="Path of grok script") parser.add_argument("-o", metavar="OUTPUT_FILE", type=str) parser.add_argument("-l", metavar="LOG_LEVEL", type=str) args = parser.parse_args() init_logging(args.l) logger.debug(args) return args
def main(): conf = util.init_conf() util.init_logging(conf) game_details = write_game_details_internal(conf, conf.replay_file)
def __init__(self): self.driver = webdriver.Chrome() self.driver.get(url='http://www.google.com?hl=en') self.log = init_logging(__name__)
import simplejson as json from werkzeug.contrib.atom import AtomFeed from datetime import date import mysql as Mysql import domino_twilio as Twilio import user as User import alert as Alert import team as Team import email as Email import rule as Rule import notification as Notification import util as Util conf = Util.load_conf() Util.init_logging("api") class Api(): ''' This API class is designed to return json via http request ''' def __init__(self, **data): ''' Initialize the api class ''' # set some default attribute values # default option vars self.limit = 25 self.offset = 0 self.search = ''
import json import requests as req from constants import ID, CODE, DEFAULT_PLACES_RADIUS, POI_CATEGORIES from sln import GScraper from util import init_logging LOG = init_logging(__name__) def _get(url): param_connector = "&" if "?" in url else "?" creds = "{}app_id={}&app_code={}".format(param_connector, ID, CODE) url = url + creds LOG.debug(url) r = req.get(url) r.raise_for_status() return r.content def _get_flow(params): url = 'https://traffic.api.here.com/traffic/6.2/flow{}'.format(params) return _get(url) def _get_places(params): url = 'https://places.cit.api.here.com/places/v1/browse?{}'.format(params) return _get(url)
get_name, add_user, hash_uid, init_groups, restricted, group_chat_only, selected_groups_only, selected_messages_only, ) from config import TELEGRAM_BOT_TOKEN, BOT_VERSION, PUB_IP, CERT, PRIV_KEY # Debug Mode default Off DEBUG = False # Init logging LOGGER = init_logging() @selected_groups_only @selected_messages_only def process_message(update, context): """Process every new update.""" # pp = pprint.PrettyPrinter(indent=4) # pp.pprint(update.to_dict()) group_id = update.effective_chat.id user_id = update.effective_user.id user_name = get_name(update) msg_type = effective_message_type(update) text = update.effective_message.text if text:
def why_inviable_instance_is_disabled(outcome, *_): node, value = outcome assert not value fmt = '{node} is disabled because of an error: {node.error}' return fmt.format(**locals()) def resolve(initial_module): return Context().resolve(initial_module) if __name__ == '__main__': import util util.init_logging('%s.log' % __name__) from pprint import pprint from mybuild.binding.pydsl import * @module def conf(self): self._constrain(m1(bar=17)) # self._constrain(m3) self.sources = 'test.c' @module def m1(self, bar=42): self._constrain(m2(foo=bar))
else: group_counts = calculate_cm_counts(group, target_col, binary=False) confusion_analysis(group_counts, albreakdown_output_path, experiment_tag, name + "_peralsource", None, loss_al, acc_al, params, source=group_name) with open(output_path + "/" + name + "_predictionsal.pkl", 'wb') as outfile: pickle.dump(al_selection_df, outfile) #if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--log", dest="log_path", default=None) parser.add_argument("--temp", dest="temp", default=True) parser.add_argument("--experiment", dest="experiment_path", default=None) parser.add_argument("--row", dest="experiment_row", type=int, default=None) args = parser.parse_args() util.init_logging(args.log_path) if args.experiment_path is not None: logging.info("=====================================================") if args.experiment_row is not None: logging.info("Experiment %s %i started...",args.experiment_path, args.experiment_row) else: logging.info("Experiment %s started...",args.experiment_path) logging.info("=====================================================") with open(args.experiment_path, 'r') as infile: paramset = json.load(infile) util.TMP_PATH = args.temp if args.experiment_row is not None:
__author__ = 'Arne Recknagel' import logging from math import sqrt import codecs import numpy as np from sklearn.pipeline import Pipeline from sklearn.cluster import DBSCAN from sklearn import metrics from feature_extractors import DataSeparator from preprocessing import parse, POS, NEG, NEU from util import svm_pipeline, k_means_pipeline, get_corpus, init_logging, root, get_feature_union, get_corpus init_logging() def get_train_data(): classes = POS | NEU | NEG train_loc = root+'Data/twitterData/train_alternative.tsv' dev_loc = root+'Data/twitterData/dev_alternative.tsv' test_loc = root+'Data/twitterData/test_alternative.tsv' train_labels, train_tweets, train_pos = parse( train_loc, classes ) dev_labels, dev_tweets, dev_pos = parse( dev_loc, classes ) test_labels, test_tweets, test_pos = parse( test_loc, classes
import signal from tornado import ioloop from tornado.web import Application import tornado.httpserver from tcp_server import CtrlServer, MyTCPServer from zmq_server import zmq_server import conf from conn import Conn import util if __name__ == "__main__": if conf.DAEMONIZE: util.daemonize() util.init_logging("log/gate_server.log", colorfy=True) handlers = [ (r'/', Conn), (r'/snappy/', Conn, dict(compress="snappy", binary=True, conn_tag='SP')), (r'/msgpack-snappy-skip/', Conn, dict(msgpack=True, compress="snappy", skip_size=512, binary=True, conn_tag='MP-SP-skip')), (r'/msgpack-lz4-skip/', Conn, dict(msgpack=True, compress="lz4", skip_size=512,
client.send(p.encode()) logging.debug("Waiting for ack") message = client.recv() logging.info("Recieved ack") time.sleep(self.config['client']['sleep']/1000) self.check_config() except zmq.error.ZMQError as e: logger.critical("ZMQError, Exiting: %s", e) exit() if __name__ == '__main__': if zmq.zmq_version_info() < (4,0): raise RuntimeError("Security is not supported in libzmq version < 4.0. libzmq version {0}".format(zmq.zmq_version())) config = util.load_yaml_file(util.config.CLIENT["config"]) util.init_logging(**config['logging']) daemon = Client(config['pid_file'], config_file=util.config.CLIENT["config"]) daemon.run() if len(sys.argv) == 2: if 'start' == sys.argv[1]: daemon.start() elif 'stop' == sys.argv[1]: daemon.stop() elif 'restart' == sys.argv[1]: daemon.restart() else: print "Unknown command" sys.exit(2) sys.exit(0) else: print "usage: %s start|stop|restart" % sys.argv[0]
import getpass import os from os.path import split, join, exists, isdir, expanduser import glob from shutil import copy2, copytree import traceback import time from subprocess import Popen, CalledProcessError import yaml import util # systemctl should log this print print('trying to start logging') l = util.init_logging(__file__) evar = 'DRIVE_SYSTEMCTL_UNIT' try: # This should be set in the invokation of the service by systemd service_unit = os.environ[evar] l.info(f'service_unit: {service_unit}') except KeyError as e: l.error(f'failed to lookup {evar}') raise root = '/'.join([''] + service_unit[:-len('.mount')].split('-')) l.info(f'mount_point: {root}') ''' evar = 'DRIVE_MOUNT_POINT' try:
kv_d = kv_d - mx.nd.array(u.getA()) # exchange with kvstore util.update_param(kvstore, kv_d, kv_x, pull_only=util.need_restart()) kv_d = mx.nd.zeros((dim, 1)) x = kv_x.asnumpy() if not util.need_restart(): start, end = end, min(end + args.batch_size, size) if start == end: break util.reset_cancel() # compute objective loss = size / dim * np.dot((lambda_ / 2 * x - b).T, x) for i in range(*interval): loss -= (A[i] * x)**2 / 2 logging.info('Epoch[{}] loss={}'.format(epoch, np.sum(loss) + 2)) if __name__ == '__main__': util.init_logging() logging.info('Start executing train_eigen.py') args = parse_args() logging.info('Start preparing data.') data = prepare_data() train(data, args)
required=True, help='Comma-separated list of paths to layer configuration json.') arg_parser.add_argument( '--attention_configs', help='Comma-separated list of paths to attention configuration json.') arg_parser.add_argument( '--combine_test_files', action='store_true', help='Whether to combine list of test files into a single score.') arg_parser.set_defaults(debug=False) arg_parser.set_defaults(combine_test_files=False) args, leftovers = arg_parser.parse_known_args() util.init_logging(tf.logging.INFO) if not os.path.isdir(args.save_dir): util.fatal_error("save_dir not found: %s" % args.save_dir) # Load all the various configurations # todo: validate json data_config = train_utils.load_json_configs(args.data_config) model_config = train_utils.load_json_configs(args.model_configs) task_config = train_utils.load_json_configs(args.task_configs, args) layer_config = train_utils.load_json_configs(args.layer_configs) attention_config = train_utils.load_json_configs(args.attention_configs) # attention_config = {} # if args.attention_configs and args.attention_configs != '': # attention_config =
return fmt.format(**locals()) def why_inviable_instance_is_disabled(outcome, *_): node, value = outcome assert not value fmt = '{node} is disabled because of an error: {node.error}' return fmt.format(**locals()) def resolve(initial_module): return Context().resolve(initial_module) if __name__ == '__main__': import util util.init_logging('%s.log' % __name__) from pprint import pprint from mybuild.binding.pydsl import * @module def conf(self): self._constrain(m1(bar=17)) # self._constrain(m3) self.sources = 'test.c' @module def m1(self, bar=42): self._constrain(m2(foo=bar))
from flask import Blueprint, make_response from jwcrypto import jwk from util import init_logging logger = init_logging(__name__) def create_blueprint(): jwk_bp = Blueprint('jwk_blueprint', __name__, template_folder='templates') @jwk_bp.route('/jwk') def get_jwk(): with open("public.pem", "rb") as f: key = jwk.JWK.from_pem(f.read()) response = make_response(key.export(private_key=False), 200) response.headers['Content-Type'] = 'application/json' return response return jwk_bp
def m2(self): self._constrain(m3(a=True)) self._constrain(m3(a=False)) @module def m3(self, a=False): pass modules = resolve(conf) self.assertIn(conf, modules) self.assertIn(m1, modules) self.assertNotIn(m2, modules) self.assertIn(m3, modules) def suite(wafctx_): class WafCtxBoundTestCase(SolverTestCase): wafctx = wafctx_ return unittest.TestLoader().loadTestsFromTestCase(WafCtxBoundTestCase) if __name__ == '__main__': import util, sys, logging # util.init_logging(filename='%s.log' % __name__) util.init_logging(sys.stderr, level=logging.DUMP) unittest.main()
"""Load a data file and generate and upload VolcView images for any defined VolcView sectors covered by the data.""" start = time.time() # Convert volcview sector definitions to our "native" format _gen_sector_bounds(config.VOLCVIEW_SECTORS) # "converts" in-place. logging.info("Generating images") file_processor = DataFile(data_file) file_processor.use_spawn = use_spawn file_processor.process_data() logging.info("Completed run in %d seconds", time.time() - start) return if __name__ == "__main__": init_logging() parser = argparse.ArgumentParser( description="SO2 data file interface to VolcView") parser.add_argument( "files", nargs="*", default=[], help="SO2 data files to generate and upload VolcView images for") parser.add_argument( "-c", "--check", dest="check", action='store_const', help= "Check VolcView servers for the required bands/types, creating if needed", const=True,
def main(): global logger logger = init_logging(logging.INFO) global g_config parser = argparse.ArgumentParser() parser.add_argument( "config_file", type=argparse.FileType("r"), help="Location of extraction config file.", ) args = parser.parse_args() g_config = {} with args.config_file as stream: try: g_config = yaml.safe_load(stream) except yaml.YAMLError as exception: logger.error(f"Failed to parse extraction config yaml file: {exception}") exit(-1) validate_config_file(g_config) level = logging.getLevelName(g_config.get('log_level', 'INFO').upper()) set_log_level(level) if g_config.get("logfile_level") != "none": level = logging.getLevelName(g_config.get('logfile_level', 'DEBUG').upper()) log_file = 'extract.log' add_logfile(log_file, level=level, preamble=yaml.dump(g_config), backup_count=g_config.get("backup_count", 2)) # print the version log_version() interface = load_driver() if not interface: logger.error("Failed to load driver.") exit(-1) if g_config["source_cluster_endpoint"]: extraction_name = f'Extraction_{g_config["source_cluster_endpoint"].split(".")[0]}_{datetime.datetime.now().replace(tzinfo=datetime.timezone.utc).isoformat()}' else: extraction_name = f"Extraction_{datetime.datetime.now().replace(tzinfo=datetime.timezone.utc).isoformat()}" if g_config.get("start_time"): start_time = dateutil.parser.parse(g_config["start_time"]).astimezone( dateutil.tz.tzutc() ) else: start_time = "" if g_config.get("end_time"): end_time = dateutil.parser.parse(g_config["end_time"]).astimezone( dateutil.tz.tzutc() ) else: end_time = "" # read the logs if g_config.get("log_location"): log_location = g_config["log_location"] elif g_config.get("source_cluster_endpoint"): log_location = get_cluster_log_location(g_config["source_cluster_endpoint"]) else: logger.error("Either log_location or source_cluster_endpoint must be specified.") exit(-1) (connections, audit_logs, databases, last_connections) = get_logs(log_location, start_time, end_time) logger.debug(f"Found {len(connections)} connection logs, {len(audit_logs)} audit logs") if(len(audit_logs) == 0 or len(connections) == 0): logger.warning("No audit logs or connections logs found. Please verify that the audit log location or cluster endpoint is correct. Note, audit logs can take several hours to start appearing in S3 after logging is first enabled.") exit(-1) if g_config["source_cluster_endpoint"]: logger.info(f'Retrieving info from {g_config["source_cluster_endpoint"]}') source_cluster_urls = get_connection_string( g_config["source_cluster_endpoint"], g_config["master_username"], g_config["odbc_driver"], ) source_cluster_statement_text_logs = retrieve_source_cluster_statement_text( source_cluster_urls, databases, start_time, end_time, interface, ) combine_logs(audit_logs, source_cluster_statement_text_logs) if ( g_config["source_cluster_system_table_unload_location"] and g_config["unload_system_table_queries"] and g_config["source_cluster_system_table_unload_iam_role"] ): logger.info( f'Exporting system tables to {g_config["source_cluster_system_table_unload_location"]}' ) unload_system_table( source_cluster_urls, g_config["odbc_driver"], g_config["unload_system_table_queries"], g_config["source_cluster_system_table_unload_location"] + "/" + extraction_name, g_config["source_cluster_system_table_unload_iam_role"], ) logger.info( f'Exported system tables to {g_config["source_cluster_system_table_unload_location"]}' ) save_logs( audit_logs, last_connections, g_config["workload_location"] + "/" + extraction_name, connections, start_time, end_time )