def main(): args = parse_args() configure_logging(args.log_config) config = load_config(args.config) stations_pf = load_stations(config) data_gridded, data_gridded_lt, timestamps = load_forecast( args.wrfout, args.wrfout_long_term, args.components, config.get('spinup', 0)) for location in stations_pf: try: pf = timeseries_for_location(location, args, data_gridded, data_gridded_lt) except TargetOutsideGridError: LOG.info( 'Location of station %s outside wrfout grid, ignoring the station.', location['ref']) continue filename = templated_filename(config, analysis_date=args.analysis, ref=location['ref'], create_dirs=True) metadata = OrderedDict([ ('ref', location['ref']), ('name', location['name']), ('analysis_date', args.analysis.strftime('%Y-%m-%dT%H:%M:%S')), ('longitude', '{:.4f}'.format(location['lon'])), ('latitude', '{:.4f}'.format(location['lat'])) ]) save_timeseries(timestamps, pf, filename, metadata)
def load_config(options): """Load data from the specified configuration file.""" listify_string = lambda x: [item.strip() for item in x.split(",")] config = utilities.load_config(options.config) mykey = utilities.safe_load(config, "pgpprocessor", "keyid", 0) protocols = listify_string( utilities.safe_load(config, "connectors", "protocols")) connectors = {} force_sender = utilities.safe_load(config, "connectors", "force_sender") if protocols == ['']: raise RuntimeError("No protocols detected. Have you run 'make'?") # loop through the protocols, finding connectors each protocol uses # load the settings for each connector. for protocol in protocols: protocol_connectors = listify_string( utilities.safe_load(config, protocol, "connectors")) if not protocol_connectors: continue for connector in protocol_connectors: connectors[connector] = dict( utilities.safe_load(config, connector, None, {})) return mykey, protocols, connectors, force_sender
def setUp(self): """Do a good bit of setup to make this a nicer test-class. Successful tests will call ``Santiago.outgoing_request``, so that's overridden to record that the method is called. """ self.gpg = gnupg.GPG(gnupghome='../data/test_gpg_home') self.keyid = utilities.load_config().get("pgpprocessor", "keyid") self.santiago = santiago.Santiago( hosting = {self.keyid: {santiago.Santiago.SERVICE_NAME: [1] }}, consuming = {self.keyid: {santiago.Santiago.SERVICE_NAME: [1] }}, me = self.keyid, gpg = self.gpg) self.santiago.requested = False self.santiago.outgoing_request = (lambda *args, **kwargs: self.record_success()) self.from_ = self.keyid self.to = self.keyid self.host = self.keyid self.client = self.keyid self.service = santiago.Santiago.SERVICE_NAME self.reply_to = [1] self.request_version = 1 self.reply_versions = [1]
def __init__(self, client, loop): config = utilities.load_config() config["client"] = client self.client = client self.db = database.Discord_Database() self.commands = {} for i in Bot.INSTRUCTIONS: try: cmd = i(config) for c in cmd.commands: self.commands[c] = cmd except AssertionError: utilities.log_message(f"{i} disabled.") self.patterns = [] for p in Bot.PATTERNS: try: self.patterns.append(p(config)) except AssertionError: utilities.log_message(f"{p} disabled.") self.reaction_handlers = [] for h in self.patterns + list(self.commands.values()): if h.monitors_reactions: self.reaction_handlers.append(h) self.token = config["token"] loop.run_until_complete(database.init_db(config["db_file"]))
def setUp(self): """Create an encryptable request.""" self.gpg = gnupg.GPG(gnupghome='../data/test_gpg_home') self.keyid = utilities.load_config().get("pgpprocessor", "keyid") self.santiago = santiago.Santiago( me = self.keyid, consuming = { self.keyid: { santiago.Santiago.SERVICE_NAME: ( "https://1", )}}, gpg = self.gpg) self.request_sender = OutgoingRequest.TestRequestSender() self.santiago.senders = { "https": self.request_sender } self.host = self.keyid self.client = self.keyid self.service = santiago.Santiago.SERVICE_NAME self.reply_to = [ "https://1" ] self.locations = [1] self.request_version = 1 self.reply_versions = [1] self.request = { "host": self.host, "client": self.client, "service": self.service, "reply_to": self.reply_to, "locations": self.locations, "request_version": self.request_version, "reply_versions": self.reply_versions }
def create_santiago(self): gpg_to_use = gnupg.GPG(gnupghome='../data/test_gpg_home') # get my key, if possible try: mykey = utilities.load_config("../data/test.cfg").get( "pgpprocessor", "keyid") except configparser.NoSectionError: mykey = 0 # set up monitors, listeners, and senders cert = "../data/freedombuddy.crt" connector = "https" service = "freedombuddy" location = "https://localhost:" serving_port = 8080 listeners = { connector: { "socket_port": serving_port, "ssl_certificate": cert, "ssl_private_key": cert }, } senders = { connector: { "proxy_host": "localhost", "proxy_port": 8118} } monitors = { connector: {} } # services to host and consume hosting = { mykey: { service: [location + str(serving_port)] } } consuming = { mykey: { service: [location + str(serving_port)] } } # go! return santiago.Santiago(listeners, senders, hosting, consuming, me=mykey, monitors=monitors, gpg = gpg_to_use)
def load_config(options): """Load data from the specified configuration file.""" config = utilities.load_config(options.config) mykey = safe_load(config, "pgpprocessor", "keyid", 0) lang = safe_load(config, "general", "locale", "en") protocols = [safe_load(config, "connectors", "protocols", {})] connectors = {} if protocols == [{}]: raise RuntimeError("No protocols detected. Have you run 'make'?") # loop through the protocols, finding connectors each protocol uses # load the settings for each connector. for protocol in protocols: protocol_connectors = safe_load(config, protocol, "connectors", [ protocol + "-listener", protocol + "-sender", protocol + "-monitor" ]) # when we do load data from the file, it's a comma-delimited string if hasattr(protocol_connectors, "split"): protocol_connectors = protocol_connectors.split(", ") for connector in protocol_connectors: connectors[connector] = dict(safe_load(config, connector, None, {})) return mykey, lang, protocols, connectors
def __init__(self, port, host, request_handler, parameters): # Registering configuration settings and request handler, logger self.config_dict = utilities.load_config(CONFIG_FILE) self.logger = logging.getLogger() utilities.init_logger(self.logger, self.config_dict) check_config(self.config_dict, self.logger) self.request_handler = request_handler self.parameters = parameters self.servSock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.servSock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.servSock.bind((host, port)) self.servSock.listen(self.config_dict['listen_connections']) self.servSock.setblocking(0) if (self.config_dict['tcp_nagle']): self.servSock.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) # Intializing client dicts self.connections = {} self.responses = {} self.epoll = select.epoll() # Creating Epoll for future read events self.epoll.register(self.servSock.fileno(), select.EPOLLIN | select.EPOLLET) self.logger.info('[%s:%d] started' % (host, port))
def setUp(self): self.gpg = gnupg.GPG(gnupghome='../data/test_gpg_home') config = utilities.load_config() self.key_id = utilities.safe_load(config, "pgpprocessor", "keyid", 0) self.recipient = "*****@*****.**" self.message = {'lol': 'cats'}
def setUp(self): self.keyid = utilities.load_config().get("pgpprocessor", "keyid") self.santiago = santiago.Santiago(me = self.keyid) self.host = 1 self.service = 2 self.location = 3
def setUp(self): self.gpg = gnupg.GPG(gnupghome='../data/test_gpg_home') self.keyid = utilities.load_config().get("pgpprocessor", "keyid") self.santiago = santiago.Santiago(me = self.keyid, gpg=self.gpg) self.host = 1 self.service = 2 self.location = 3
def setUp(self): self.iterations = 3 self.gpg = gnupg.GPG(gnupghome='../data/test_gpg_home') config = utilities.load_config() self.key_id = utilities.safe_load(config, "pgpprocessor", "keyid", 0) self.messages = utilities.multi_sign( gpg = self.gpg, iterations = self.iterations, keyid = self.key_id)
def setUp(self): """Create a request.""" self.gpg = gnupg.GPG(gnupghome='../data/test_gpg_home') self.keyid = utilities.load_config().get("pgpprocessor", "keyid") self.santiago = santiago.Santiago(me = self.keyid, gpg = self.gpg) self.request = { "host": self.keyid, "client": self.keyid, "service": santiago.Santiago.SERVICE_NAME, "reply_to": [1], "locations": [1], "request_version": 1, "reply_versions": [1], } self.ALL_KEYS = set(("host", "client", "service", "locations", "reply_to", "request_version", "reply_versions")) self.REQUIRED_KEYS = set(("client", "host", "service", "request_version", "reply_versions")) self.OPTIONAL_KEYS = set(("locations", "reply_to")) self.LIST_KEYS = set(("reply_to", "locations", "reply_versions"))
def spectra_paths(srcfile): """ Return a list of paths to the pickle files generated by run_config() from srcfile. :param srcfile: path to the pybatchdict configuration JSON. :return: (dict, BatchDict): Dictionary containing each configuration, keyed by the hyphenated configuration name, where each configuration also has a key 'path' that denotes its file path. The BatchDict is from the original JSON configuration. """ config = utilities.load_config(srcfile) dirname = os.path.dirname(srcfile) configurations = dict(zip(config.hyphenate_changes(), config.combos)) for cname in configurations: configurations[cname]['path'] = os.path.join( dirname, 'spectra-%s.pickle' % cname ) return configurations, config
def _prepare_collaborator(self): config = utilities.load_config() lastfm_config = config['lastfmConfig'] spotify_config = config['spotifyConfig'] # we shouldn't need to be authenticated. We just need their library, nothing else self.lastfmNetwork = pylast.LastFMNetwork( api_key=lastfm_config['lastfmAPIKey'], api_secret=lastfm_config['lastfmAPISecret']) self.lastfmUser = self.lastfmNetwork.get_user(self.lastfmUsername) self.lastfmUserLibrary = self.lastfmUser.get_library() if self.spotifyUsername is not None and self.spotify is None: try: # todo: work out if it's possible to automate this? I hate the spotify API authentication self.spotifyToken = spotipy.util.prompt_for_user_token( self.spotifyUsername, self.spotifyScopes, spotify_config['spotifyClientId'], spotify_config['spotifyClientSecret'], spotify_config['redirectURI']) self.spotify = spotipy.Spotify(auth=self.spotifyToken) self.spotifyUser = self.spotify.current_user() except SpotifyOauthError: print("Failed to set up spotify user")
def test_find_equivalent_variant_whole_seq(fetch_seq_mock_data): with patch.object(bioutils.seqfetcher, 'fetch_seq', side_effect=lambda ac, s, e: fetch_seq_mock_data[ (str(ac), str(s), str(e))]): gene_config_path = os.path.join(pwd, 'test_files', 'gene_config_test.txt') cfg = load_config(gene_config_path) regions = extract_gene_regions_dict(cfg, 'start_hg38_legacy_variants', 'end_hg38_legacy_variants').keys() seq_wrapper = seq_utils.SeqRepoWrapper(regions_preload=regions) # empty case assert [] == find_equivalent_variants_whole_seq({}, seq_wrapper) # a bunch of variants. If they appear in the same set, they are considered equivalent example_variants = [ frozenset({'chr13:g.32355030:A>AA'}), frozenset({'chr13:g.32339774:GAT>G', 'chr13:g.32339776:TAT>T'}), frozenset({'chr17:g.43090921:G>GCA', 'chr17:g.43090921:GCA>GCACA'}) ] # construct variant dict (flattening example_variants!) variant_dict = { v: VCFVariant(int(v.split(':')[0].lstrip('chr')), int(v.split(':')[1].lstrip('g.')), v.split(':')[2].split('>')[0], v.split(':')[2].split('>')[1]) for eq_variants in example_variants for v in eq_variants } whole_seq_provider = seq_utils.WholeSeqSeqProvider(seq_wrapper) assert frozenset(example_variants) == frozenset( find_equivalent_variants_whole_seq(variant_dict, whole_seq_provider))
import os import sys import datetime as dt import pandas as pd import numpy as np from pre_process_library import batting as bats import utilities as util CONFIG = util.load_config() def add_batting_stats(data): """ """ game_batter_stats = pd.groupby( by=['gameId', 'batterId'], as_index=False ).agg(({ def process_date_games(path): """ """ # Read in 4 standardized tables df_batting = pd.read_parquet(path+"batting.parquet") df_pitching = pd.read_parquet(path+"pitching.parquet") df_boxscore = pd.read_parquet(path+"boxscore.parquet") df_innings = pd.read_parquet(path+"innings.parquet")
if not (sport and fport and log and host): logger.error('%s is not correctly configured' % CONFIG_FILE) sys.exit(-1) def proxy_request_handler(epoll_context, parameters): startTime = time.time() request, host, port = epoll_context def server_request_handler(epoll_context, parameters): startTime = time.time() request, host, port = epoll_context def receive_signal(signum, stack): print 'Received:', signum if __name__ == '__main__': opts = argparse.ArgumentParser(description=__description__) parse_config(opts) args = opts.parse_args() config_dict = utilities.load_config(args.config) utilities.init_logger(logger, config_dict) check_config(config_dict, logger) signal.signal(signal.SIGUSR1, receive_signal) #server = epoll.Server(int(args.port), args.host, request_handler, []) #thisserver.run()
def setUp(self): self.columns = ['Source', 'Gene_symbol_ENIGMA', 'Genomic_Coordinate', 'Chr', 'Pos', 'Ref', 'Alt', 'Reference_sequence_ENIGMA', 'HGVS_cDNA_ENIGMA', 'BIC_Nomenclature_ENIGMA', 'Abbrev_AA_change_ENIGMA', 'URL_ENIGMA', 'Condition_ID_type_ENIGMA', 'Condition_ID_value_ENIGMA', 'Condition_category_ENIGMA', 'Clinical_significance_ENIGMA', 'Date_last_evaluated_ENIGMA', 'Assertion_method_ENIGMA', 'Assertion_method_citation_ENIGMA', 'Clinical_significance_citations_ENIGMA', 'Comment_on_clinical_significance_ENIGMA', 'Collection_method_ENIGMA', 'Allele_origin_ENIGMA', 'ClinVarAccession_ENIGMA', 'HGVS_protein_ENIGMA', 'BX_ID_ENIGMA', 'Clinical_Significance_ClinVar', 'Date_Last_Updated_ClinVar', 'BX_ID_ClinVar', 'HGVS_ClinVar', 'Submitter_ClinVar', 'Protein_ClinVar', 'SCV_ClinVar', 'Allele_Origin_ClinVar', 'Method_ClinVar', 'Description_ClinVar', 'Summary_Evidence_ClinVar', 'Review_Status_ClinVar', 'Condition_Type_ClinVar', 'Condition_Value_ClinVar', 'Condition_DB_ID_ClinVar', 'Individuals_LOVD', 'BX_ID_LOVD', 'Variant_effect_LOVD', 'Variant_frequency_LOVD', 'HGVS_cDNA_LOVD', 'HGVS_protein_LOVD', 'Genetic_origin_LOVD', 'RNA_LOVD', 'Submitters_LOVD', 'DBID_LOVD', 'Functional_analysis_technique_LOVD', 'Functional_analysis_result_LOVD', 'Created_date_LOVD', 'Edited_date_LOVD', 'Submission_ID_LOVD', 'BX_ID_ESP', 'Minor_allele_frequency_percent_ESP', 'EA_Allele_Frequency_ESP', 'AA_Allele_Frequency_ESP', 'Allele_Frequency_ESP', 'polyPhen2_result_ESP', 'EUR_Allele_frequency_1000_Genomes', 'AFR_Allele_frequency_1000_Genomes', 'AMR_Allele_frequency_1000_Genomes', 'EAS_Allele_frequency_1000_Genomes', 'BX_ID_1000_Genomes', 'Allele_frequency_1000_Genomes', 'SAS_Allele_frequency_1000_Genomes', 'Allele_frequency_ExAC', 'BX_ID_ExAC', 'BX_ID_BIC', 'Patient_nationality_BIC', 'Clinical_importance_BIC', 'Clinical_classification_BIC', 'BIC_Designation_BIC', 'Literature_citation_BIC', 'Number_of_family_member_carrying_mutation_BIC', 'Germline_or_Somatic_BIC', 'Ethnicity_BIC', 'Mutation_type_BIC', 'IARC_class_exLOVD', 'BIC_Nomenclature_exLOVD', 'Sum_family_LR_exLOVD', 'Combined_prior_probablility_exLOVD', 'BX_ID_exLOVD', 'HGVS_cDNA_exLOVD', 'Literature_source_exLOVD', 'Co_occurrence_LR_exLOVD', 'Posterior_probability_exLOVD', 'Missense_analysis_prior_probability_exLOVD', 'Segregation_LR_exLOVD', 'HGVS_protein_exLOVD', "Allele_count_AFR_ExAC", "Allele_number_AFR_ExAC", "Homozygous_count_AFR_ExAC", "Allele_count_AMR_ExAC", "Allele_number_AMR_ExAC", "Homozygous_count_AMR_ExAC", "Allele_count_EAS_ExAC", "Allele_number_EAS_ExAC", "Homozygous_count_EAS_ExAC", "Allele_count_FIN_ExAC", "Allele_number_FIN_ExAC", "Homozygous_count_FIN_ExAC", "Allele_count_NFE_ExAC", "Allele_number_NFE_ExAC", "Homozygous_count_NFE_ExAC", "Allele_count_OTH_ExAC", "Allele_number_OTH_ExAC", "Homozygous_count_OTH_ExAC", "Allele_count_SAS_ExAC", "Allele_number_SAS_ExAC", "Homozygous_count_SAS_ExAC", "Allele_frequency_AFR_ExAC", "Allele_frequency_AMR_ExAC", "Allele_frequency_EAS_ExAC", "Allele_frequency_FIN_ExAC", "Allele_frequency_NFE_ExAC", "Allele_frequency_OTH_ExAC", "Allele_frequency_SAS_ExAC", "DateSignificanceLastEvaluated_ClinVar", "SCV_Version_ClinVar", "Synonyms_ClinVar", "HGVS_Nucleotide_Findlay_BRCA1_Ring_Function_Scores", "Log_RNA_Depletion_Findlay_BRCA1_Ring_Function_Scores", "Functional_Enrichment_Score_Findlay_BRCA1_Ring_Function_Scores", "BX_ID_Findlay_BRCA1_Ring_Function_Scores", "HGVS_cDNA_GnomAD", "HGVS_protein_GnomAD", "Flags_GnomAD", "Consequence_GnomAD", "Variant_id_GnomAD", "Allele_count_genome_AFR_GnomAD", "Allele_count_hemi_genome_AFR_GnomAD", "Allele_count_hom_genome_AFR_GnomAD", "Allele_number_genome_AFR_GnomAD", "Allele_frequency_genome_AFR_GnomAD", "Allele_count_genome_AMR_GnomAD", "Allele_count_hemi_genome_AMR_GnomAD", "Allele_count_hom_genome_AMR_GnomAD", "Allele_number_genome_AMR_GnomAD", "Allele_frequency_genome_AMR_GnomAD", "Allele_count_genome_ASJ_GnomAD", "Allele_count_hemi_genome_ASJ_GnomAD", "Allele_count_hom_genome_ASJ_GnomAD", "Allele_number_genome_ASJ_GnomAD", "Allele_frequency_genome_ASJ_GnomAD", "Allele_count_genome_EAS_GnomAD", "Allele_count_hemi_genome_EAS_GnomAD", "Allele_count_hom_genome_EAS_GnomAD", "Allele_number_genome_EAS_GnomAD", "Allele_frequency_genome_EAS_GnomAD", "Allele_count_genome_FIN_GnomAD", "Allele_count_hemi_genome_FIN_GnomAD", "Allele_count_hom_genome_FIN_GnomAD", "Allele_number_genome_FIN_GnomAD", "Allele_frequency_genome_FIN_GnomAD", "Allele_count_genome_NFE_GnomAD", "Allele_count_hemi_genome_NFE_GnomAD", "Allele_count_hom_genome_NFE_GnomAD", "Allele_number_genome_NFE_GnomAD", "Allele_frequency_genome_NFE_GnomAD", "Allele_count_genome_OTH_GnomAD", "Allele_count_hemi_genome_OTH_GnomAD", "Allele_count_hom_genome_OTH_GnomAD", "Allele_number_genome_OTH_GnomAD", "Allele_frequency_genome_OTH_GnomAD", "Allele_count_genome_SAS_GnomAD", "Allele_count_hemi_genome_SAS_GnomAD", "Allele_count_hom_genome_SAS_GnomAD", "Allele_number_genome_SAS_GnomAD", "Allele_frequency_genome_SAS_GnomAD", "Allele_count_genome_GnomAD", "Allele_number_genome_GnomAD", "Allele_frequency_genome_GnomAD", "Allele_count_exome_AFR_GnomAD", "Allele_count_exome_AFR_GnomAD", "Allele_count_hom_exome_AFR_GnomAD", "Allele_number_exome_AFR_GnomAD", "Allele_frequency_exome_AFR_GnomAD", "Allele_count_exome_AMR_GnomAD", "Allele_count_hemi_exome_AMR_GnomAD", "Allele_count_hom_exome_AMR_GnomAD", "Allele_number_exome_AMR_GnomAD", "Allele_frequency_exome_AMR_GnomAD", "Allele_count_exome_ASJ_GnomAD", "Allele_count_hemi_exome_ASJ_GnomAD", "Allele_count_hom_exome_ASJ_GnomAD", "Allele_number_exome_ASJ_GnomAD", "Allele_frequency_exome_ASJ_GnomAD", "Allele_count_exome_EAS_GnomAD", "Allele_count_hemi_exome_EAS_GnomAD", "Allele_count_hom_exome_EAS_GnomAD", "Allele_number_exome_EAS_GnomAD", "Allele_frequency_exome_EAS_GnomAD", "Allele_count_exome_FIN_GnomAD", "Allele_count_hemi_exome_FIN_GnomAD", "Allele_count_hom_exome_FIN_GnomAD", "Allele_number_exome_FIN_GnomAD", "Allele_frequency_exome_FIN_GnomAD", "Allele_count_exome_NFE_GnomAD", "Allele_count_hemi_exome_NFE_GnomAD", "Allele_count_hom_exome_NFE_GnomAD", "Allele_number_exome_NFE_GnomAD", "Allele_frequency_exome_NFE_GnomAD", "Allele_count_exome_OTH_GnomAD", "Allele_count_hemi_exome_OTH_GnomAD", "Allele_count_hom_exome_OTH_GnomAD", "Allele_number_exome_OTH_GnomAD", "Allele_frequency_exome_OTH_GnomAD", "Allele_count_exome_SAS_GnomAD", "Allele_count_hemi_exome_SAS_GnomAD", "Allele_count_hom_exome_SAS_GnomAD", "Allele_number_exome_SAS_GnomAD", "Allele_frequency_exome_SAS_GnomAD", "Allele_number_exome_GnomAD", "Allele_count_exome_GnomAD", "Allele_frequency_exome_GnomAD", "BX_ID_GnomAD"] self.sources = aggregate_reports.FIELD_DICT.keys() + ["ENIGMA"] self.vcf_test_file = VCF_TESTDATA_FILENAME self.tsv_test_file = TSV_TESTDATA_FILENAME pwd = os.path.dirname(os.path.realpath(__file__)) gene_config_df = utilities.load_config(os.path.join(pwd, 'test_files', 'gene_config_test.txt')) self.genome_regions_symbol_dict = utilities.get_genome_regions_symbol_dict(gene_config_df, 'start_hg38_legacy_variants', 'end_hg38_legacy_variants')
async def main(): if getattr(sys, "frozen", False): # Check if program is compiled to exe main_folder_path = os.path.dirname(sys.executable) else: main_folder_path = os.path.dirname(os.path.abspath(__file__)) config = load_config(os.path.join(main_folder_path, "horizon_config.ini")) setup_logging(main_folder_path, level=config['logging_level']) print_timestamp( f"Horizon Trade Notifier {version} - https://discord.gg/Xu8pqDWmgE - https://github.com/JartanFTW" ) logger.log( 49, f"Horizon Trade Notifier {version} - https://discord.gg/Xu8pqDWmgE - https://github.com/JartanFTW" ) update = await check_for_update(version) if update: print_timestamp("A new update is available!") tasks = [] user = await User.create(config["cookie"]) if config['completed']['enabled']: worker = await TradeWorker.create( main_folder_path, user, config['completed']['webhook'], config['completed']['update_interval'], config['completed']['theme_name'], trade_type="Completed", add_unvalued_to_value=config['add_unvalued_to_value'], testing=config['testing'], webhook_content=config['completed']['webhook_content']) tasks.append(asyncio.create_task(worker.check_trade_loop())) if config['inbound']['enabled']: worker = await TradeWorker.create( main_folder_path, user, config['inbound']['webhook'], config['inbound']['update_interval'], config['inbound']['theme_name'], trade_type="Inbound", add_unvalued_to_value=config['add_unvalued_to_value'], testing=config['testing'], double_check=config['double_check'], webhook_content=config['inbound']['webhook_content']) tasks.append(asyncio.create_task(worker.check_trade_loop())) if config['outbound']['enabled']: worker = await TradeWorker.create( main_folder_path, user, config['outbound']['webhook'], config['outbound']['update_interval'], config['outbound']['theme_name'], trade_type="Outbound", add_unvalued_to_value=config['add_unvalued_to_value'], testing=config['testing'], webhook_content=config['outbound']['webhook_content']) tasks.append(asyncio.create_task(worker.check_trade_loop())) if tasks: await asyncio.wait(tasks) else: print_timestamp( "Looks like you don't have any trade types enabled in the config! There is nothing for me to do :(" ) await user.client.aclose() return
def main(): configs = load_config("game") map_data = generateFullMap(configs["map"], configs["tile"])
plt.title('Simulator Precision-Coverage Curves auc = {0:02f}'.format(auc)) plt.xlabel('Coverage') plt.ylabel('Precision') plt.xlim([0, 0.6]) plt.ylim([0,1.0]) # plt.show() return auc, {'precisions': precisions, 'recalls': recalls, 'auc': auc, 'cfg': None} # pr_data = glob2.glob(os.path.join(sys.argv[1],'*','*','outfile.npy')) + glob2.glob(os.path.join(sys.argv[1],'*','outfile.npy')) + glob2.glob(os.path.join(sys.argv[1],'outfile.npy')) pr_data = glob2.glob(os.path.join(sys.argv[1],'*','*','all_full_results.npz')) + glob2.glob(os.path.join(sys.argv[1],'*','all_full_results.npz')) + glob2.glob(os.path.join(sys.argv[1],'all_full_results.npz')) default_compare = True if default_compare: default_config = utilities.load_config('/home/msundermeyer/ngc_ws/6dof-graspnet/contact_graspnet') else: default_config = np.load(pr_data[0], allow_pickle=True).item()['cfg'] legends = [] all_diff_dicts = {} cfgs = {} aucs_01 = {} name_dict = {} gt_grasps = [] for p in range(100): y=np.load('/home/msundermeyer/datasets/visibility_filtered_gt_grasp/{}_filtered_gt_grasps.npz'.format(p), allow_pickle=True) gt_grasps.append(y['gt_grasp_scene_trafos'])
from http.server import HTTPServer, BaseHTTPRequestHandler import utilities from request_parser import NestAwayRequestParser from nest_manager import NestAwayManager ## Configs CONFIGS = utilities.load_config() REQUEST_PARSER = NestAwayRequestParser() NEST_MANAGER = NestAwayManager() class RequestHandler(BaseHTTPRequestHandler): def do_POST(self): content_len = int(self.headers.get('content-length', 0)) body = self.rfile.read(content_len) state = REQUEST_PARSER.parse(body) # Safe(ish), valid json if (state): status_code, response_json = NEST_MANAGER.update_state(state) self.send_response(status_code, response_json) self.send_header("Content-Type", "application/json") self.end_headers() else: self.send_response( '400', '{\"error\": \"Malformed request, didn\'t contain valid \'user\' and \'away_state\' properties\"}' ) self.send_header("Content-Type", "application/json") self.end_headers()
def main(): global DISCARDED_REPORTS_WRITER parser = argparse.ArgumentParser() options(parser) args = parser.parse_args() gene_config_df = utilities.load_config(args.config) gene_regions_dict = utilities.extract_gene_regions_dict(gene_config_df, 'start_hg38_legacy_variants', 'end_hg38_legacy_variants') gene_regions_trees = seq_utils.build_interval_trees_by_chr(gene_regions_dict.keys(), lambda c,s,e: None) genome_regions_symbol_dict = utilities.get_genome_regions_symbol_dict(gene_config_df, 'start_hg38_legacy_variants', 'end_hg38_legacy_variants') seq_provider = seq_utils.SeqRepoWrapper(regions_preload=gene_regions_dict.keys()) if args.verbose: logging_level = logging.DEBUG else: logging_level = logging.CRITICAL log_file_path = args.artifacts_dir + "variant_merging.log" logging.basicConfig(filename=log_file_path, filemode="w", level=logging_level, format=' %(asctime)s %(filename)-15s %(message)s') discarded_reports_file = open(args.artifacts_dir + "discarded_reports.tsv", "w") fieldnames = ['Report_id', 'Source', 'Reason', 'Variant'] DISCARDED_REPORTS_WRITER = csv.DictWriter(discarded_reports_file, delimiter="\t", fieldnames=fieldnames) DISCARDED_REPORTS_WRITER.writeheader() # merge repeats within data sources before merging between data sources source_dict, columns, variants = preprocessing(args.input, args.output, seq_provider, gene_regions_trees) # merges repeats from different data sources, adds necessary columns and data print "\n------------merging different datasets------------------------------" for source_name, file in source_dict.iteritems(): (columns, variants) = add_new_source(columns, variants, source_name, file, FIELD_DICT[source_name], genome_regions_symbol_dict) # standardizes genomic coordinates for variants print "\n------------standardizing genomic coordinates-------------" variants = variant_standardize(columns, seq_provider, gene_regions_trees, variants=variants) # compare dna sequence results of variants and merge if equivalent print "------------dna sequence comparison merge-------------------------------" variants = string_comparison_merge(variants, seq_provider) # write final output to file write_new_tsv(args.output + "merged.tsv", columns, variants) # copy enigma file to artifacts directory along with other ready files copy(args.input + ENIGMA_FILE, args.output) # write reports to reports file aggregate_reports.write_reports_tsv(args.output + "reports.tsv", columns, args.output, genome_regions_symbol_dict) discarded_reports_file.close() print "final number of variants: %d" % len(variants) print "Done"
fn_words_dict = root + model + 'tc_words_dict.p' fn_embedding_words_dict = root + model + 'embedding_word_dict.p' do_submission = True fn_train_out = 'l2_train_data.csv' fn_valid_out = "l2_valid_data.csv" fn_test_out = "l2_test_data.csv" class Config: pass cfg = Config() load_config(cfg, model_fp) # should be removed cfg.level = 'word' cfg.glove = 'False' cfg.tokenize_mode = 'twitter' cfg.do_preprocess = 'True' tc = ToxicComments(cfg) epochs = [fn.split('.ckpt')[0] for fn in os.listdir(logs) if fn.endswith('.meta')] results = pd.read_csv(root + model + 'results.csv') def _get_score(): rocs = np.zeros(len(epochs)) rocs_t = np.zeros(len(epochs))
def plot(configfile, separate=False): # Load the configurations. """ Plot the cospectra for each model variant. :param configfile: (str) path to configuration file. :param separate: (bool) whether or not to plot each variant in its own plot. """ configdir = os.path.dirname(configfile) configs = utilities.load_config(configfile) cnames = configs.hyphenate_changes() # Cache models if they are used more than once. modelcache = {} axpanes = None ax = None # Compute and plot each spectrum. for r, (config, cname) in enumerate(itertools.izip(configs.combos, cnames)): print 'Plotting %s.' % cname # Set up the figure. Outside axes are for the legend. if separate or r == 0: axpanes = None pp.figure(figsize=(15, 15)) ax = pp.subplot(111) ax.patch.set_visible(False) pp.axis('off') # Fetch the model. If it's already been created, retrieve it from the # cache. modelname = pbdict.getkeypath(config, '/model/name', 'nbd2') if modelname in modelcache: model = modelcache[modelname] else: model = models.parasitism.get_model(modelname) # Set up model parameters. sym_params = { models.parasitism.params[name]: value for name, value in pbdict.getkeypath(config, '/model/params').iteritems() } noise = np.array(pbdict.getkeypath(config, '/model/noise')) # Get the spectrum. freqs = np.linspace(0, 0.5, config.get('nfreqs', 100)) spectrum = np.array([ model.calculate_spectrum(sym_params, noise, v) for v in freqs ]).T # Plot the spectrum. plotargs = dict( varnames=None if r < len(cnames) - 1 else model.vars, plotfun=pp.scatter, s=2, alpha=2**-5, axpanes=axpanes ) if 'color' in config: plotargs['color'] = config['color'] elif separate: plotargs['color'] = 'k' else: plotargs['color'] = cm.rainbow(float(r) / (len(cnames) - 1)) axpanes = models.plotting.plot_cospectra(freqs, spectrum, **plotargs) if separate: figpath = os.path.join(configdir, 'spectra-%s.png' % cname) print 'Writing %s.' % figpath pp.savefig(figpath) if not separate: # Make a legend for each model using its hyphenated configuration name. ax.legend(axpanes[0][0]['top'].get_lines(), cnames) # Save the plot. figpath = os.path.join(configdir, 'spectra.png') print 'Writing %s.' % figpath pp.savefig(figpath)
def run_config(srcfile, forceseries=False, forcespectra=False): """ Simulate the model specified in srcfile (a JSON config) and compute its cross-spectra. Save results in pickle files in the same directory as srcfile. :param srcfile: (str) path to JSON configuration file. :param forceseries: (bool) whether or not to force [re]calculation of time series. :param forcespectra: (bool) whether or not to force [re]calculation of spectral matrices. """ # Load each configuration specified by the pybatchdict JSON config. configs = utilities.load_config(srcfile) seriescache = {} for config, cname in izip(configs.combos, configs.hyphenate_changes()): print 'Running %s.' % cname # Simulate time series. Cache time series to avoid re-simulating for # the same parameter values. sim = config['simulation'] model = models.parasitism.get_model(sim['model']) sym_params = models.parasitism.sym_params(sim['params']) nvars = len(model.vars) # If we don't actually change the simulation, just use the same series # each time. serieshash = cname if 'simulation' in cname else 1 seriescache.setdefault(serieshash, {}) series = seriescache[serieshash] noise = np.array([ [sim['noise']['Sh'], sim['noise']['Shh'], 0, 0], [sim['noise']['Shh'], sim['noise']['Sh'], 0, 0], [0, 0, sim['noise']['Sp'], sim['noise']['Spp']], [0, 0, sim['noise']['Spp'], sim['noise']['Sp']] ]) nsteps = pbdict.getkeypath(config, '/simulation/timesteps') # If we haven't already simulated for these parameters, do so. if 'linear' not in series: # Output file for simulation. outfile = os.path.join( os.path.dirname(srcfile), 'series-%s.pickle' % cname ) if forceseries or not os.path.exists(outfile): # Simulate time series for linearized model. if 'linear' not in series: print '\tSimulating linear model (%d steps).' % nsteps series['linear'] = model.simulate_linear( np.zeros(nvars), sym_params, noise, nsteps ) print '\tWriting %s.' % outfile cPickle.dump(series, open(outfile, 'w')) else: # If the series has already been simulated and saved, just # load it and use it for subsequent cross-spectra computations. print '\tLoading saved %s.' % outfile loadseries = cPickle.load(open(outfile)) series['linear'] = loadseries['linear'] # Output pickle file for cross-spectra (analytic and estimation from # simulated time series). outfile = os.path.join( os.path.dirname(srcfile), 'spectra-%s.pickle' % cname ) if forcespectra or not os.path.exists(outfile): # Calculate spectral matrices. spectra = {} csdargs = pbdict.getkeypath(config, '/analysis/csd') smoothingargs = pbdict.getkeypath(config, '/analysis/smoothing') linear = series['linear'] # If we have more time series data than required by our NFFT window # size, only use the last NFFT data points rather than using # Welch's method to average over periodograms. Done to simplify # spectral estimation using smoothing instead. The LAST NFFT data # points are used in order to ensure the data are sufficiently # random (which they won't be if NFFT=len(linear), since the # nonlinear time series always starts at the equilibrium and the # linear time series always starts at zero. if 'NFFT' in csdargs and csdargs['NFFT'] < linear.shape[1]: linear = linear[:, -csdargs['NFFT']:] # Compute cross-spectra. freqs, spectra['linear'] = models.utilities.spectrum( linear, **csdargs ) # Smooth spectral matrices for better estimate. spectra['linear_smoothed'] = np.empty_like(spectra['linear']) spectra['linear_median'] = np.empty_like(spectra['linear']) magargs = dict(smoothingargs.iteritems()) magargs['window'] = 'median' for i, j in combinations_with_replacement(range(nvars), 2): spectra['linear_smoothed'][i, j] = models.utilities.smooth( spectra['linear'][i, j], **smoothingargs ) spectra['linear_median'][i, j] = \ models.utilities.smooth_phasors( spectra['linear'][i, j], magargs=magargs, phasorargs=smoothingargs ) # Evaluate analytic spectral matrix. spectra['analytic'] = np.array([ model.calculate_spectrum(sym_params, noise, v) for v in freqs ]).T if len(spectra['analytic'].shape) == 1: spectra['analytic'] = spectra['analytic'].reshape( spectra['linear'].shape ) # Save everything. print '\tWriting %s.' % outfile cPickle.dump( dict( config=config, name=cname, spectra=spectra, freqs=freqs, model=model ), open(outfile, 'w') ) else: print '\tSkipping %s (already exists).' % outfile
import os import discord from discord.ext import commands from praw import Reddit import utilities ## Config CONFIG_OPTIONS = utilities.load_config() class StupidQuestions: REDDIT_USER_AGENT = "discord:hawking:{} (by /u/hawking-py)".format( CONFIG_OPTIONS.get("version", "0.0.1")) def __init__(self, hawking, bot, *args, **kwargs): self.hawking = hawking self.bot = bot ## Load module specific configs from 'stupid_questions.json' located in modules folder modules_folder_name = CONFIG_OPTIONS.get("modules_folder", "modules") config = utilities.load_json( os.path.sep.join([ utilities.get_root_path(), modules_folder_name, "stupid_questions.json" ])) reddit_client_id = config.get("reddit_client_id") reddit_secret = config.get("reddit_secret") subreddits = CONFIG_OPTIONS.get("stupid_question_subreddits",
import torch import numpy as np import torch.nn as nn import torch.nn.functional as F import pytorch_lightning as pl from utilities import load_config from networks.layers import conv_block, convT_block, linear_block, UPSoftmax config = load_config() n_z = config["model"]["n_z"] n_gf = config["model"]["n_gf"] n_df = config["model"]["n_df"] class gen(nn.Module): # conv1 (100, 1, 1) -> (256, 5, 5) # conv2 (256, 5, 5) -> (128, 10, 10) # conv3 (128, 10, 10 -> (64, 20, 20) # conv4 (32, 20, 20) -> (16, 38, 38) # conv5 (32, 38, 38) -> (16, 76, 76) # conv6 (16, 76, 76) -> (1, 150, 150) def __init__(self): super().__init__() self.up1 = convT_block(n_z, n_gf * 16, 5, 1, 0, activation="relu") self.up2 = convT_block(n_gf * 16, n_gf * 8, 4, 2, 1, activation="relu") self.up3 = convT_block(n_gf * 8, n_gf * 4, 4, 2, 1, activation="relu")