def create_app(): """An application factory.""" load_config() app = Flask(__name__) app.config.from_object(__name__) app.secret_key = b'_5#y2L"F4Q8z\n\xec]/' app.debug = True # register blue print register_blueprints(app) return app
def copy_csv_from_s3_to_db(bucket_name, filepath, destination_table, db_name): """Copies an existing csv file from s3 to a specified DB. Args: bucket_name (str): name of an existing bucket in the configured aws account. filepath (str): remote path in s3 bucket to the file intended to be unloaded. destination_table (str): structure.table to use to unload the csv content. db_name (str): name of the db where the table will be unloaded. """ from src.config import load_config if filepath not in list_files_in_s3_bucket(bucket_name): raise FileNotFoundError( f"The filepath specified '{filepath}' does not exist in the" f" bucket '{bucket_name}'") creds = boto3.Session().get_credentials() query = ( f"COPY {destination_table} from 's3://{bucket_name}/{filepath}' credentials " f"'aws_access_key_id={creds.access_key};" f"aws_secret_access_key={creds.secret_key}' " f"ignoreheader 1 removequotes delimiter ',' region 'us-east-1'") config = load_config("credentials") if db_name not in config.keys(): raise CredentialsError(f'Credentials "{db_name}" not found') credentials = {k: v for k, v in config.items() if type(v) is not dict} credentials = {**credentials, **config[db_name]} with connect(**credentials) as conn: conn.cursor().execute(query) conn.commit()
def from_config(cls, path: Path) -> 'Server': server_config, wallet_config, users = load_config(path) cls_ = cls(server_config.getint('port'), wallet_config) cls_.users.update(users) return cls_
def post(sourcetype: str, eventtype: str, fields: dict): try: cfg = config.load_config() headers = {"Authorization": f"Splunk {cfg['token']}"} data = { "index": cfg['index'], "host": str(socket.gethostname()), "sourcetype": sourcetype, "event": eventtype, "fields": fields } # print('URL: ' + cfg['url'] + '\nHeaders: ' + json.dumps(headers) + '\nData: ' + json.dumps(data, indent=4)) requests.packages.urllib3.disable_warnings( category=InsecureRequestWarning) response = requests.post(cfg['url'], headers=headers, json=data, verify=False) print( f"[{int(str(time.time()).split('.')[0])}] Uploaded {data['fields']['metric_name']} -> Response: {response.status_code}" ) return response except Exception as e: print(str(e))
def _parse_credentials(query): # Query files should specify associated credentials on first line with open(_get_query_path(query)) as f: header = f.readline().strip() config = load_config("credentials") keys = {k for k, v in config.items() if type(v) is dict} prefix = "-- Credentials: " if not header.startswith(prefix): msg = f'Query "{query}" missing header "{prefix}[{", ".join(keys)}]"' raise QueryError(msg) key = header[len(prefix):] if key not in keys: raise CredentialsError(f'Credentials "{key}" not found') credentials = {k: v for k, v in config.items() if type(v) is not dict} credentials = {**credentials, **config[key]} if not credentials["user"]: raise CredentialsError(f'Username for "{key}" not found') if not credentials["password"]: raise CredentialsError(f'Password for "{key}" not found') return credentials
def test_load_config(self): config = load_config(["default"]) self.assertDictEqual( { k: v for k, v in _INITIAL_CONFIG["groups"].items() if k is not "default_groups" }, config)
def download_cache_string(cache_id): """ Fetch cached data as a string. Returns none if the cache does not exist. """ config = load_config() with tempfile.NamedTemporaryFile() as fd: config['cache_client'].download_cache(cache_id, fd.name) contents = fd.read().decode() print(f'Downloaded contents from cache ID {cache_id[0:16]}..') return contents
def set_env(args): random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) torch.set_num_threads(args.threads) current_time = datetime.now().strftime('%b_%d_%H-%M-%S') model_args = load_config(args.model_name, args.dataset_str, args.optimizer) if args.model_name in ['DPGVAE', 'GVAE']: args.discriminator_ratio = 0 return args, model_args, current_time
def _add_item(pipeline_name): url = request.form.get("url") config = load_config() pipeline = config.pipeline[pipeline_name] for p in pipeline.pull: pull = pull_services[p.service] item_id = pull.parse_item_id(url) if item_id is None: continue ItemInfo.add_index(IndexItem(service=p.service, item_id=item_id), [pipeline_name]) ItemInfo.set_status(p.service, item_id, TaskStage.Fetching, TaskStatus.Queued) return redirect(f"/pipeline/{pipeline_name}") return "Unknown URL"
def show_config(json_data, auth_token=None): config = load_config() # Explicitly write in config entries we want to return. Do not show anything private. result = { 'homology_url': config['homology_url'], 'id_mapper_url': config['id_mapper_url'], 'caching_service_url': config['caching_service_url'], 'kbase_endpoint': config['kbase_endpoint'], 'cache_client': config['client'] } return flask.jsonify({ 'version': '1.1', 'id': json_data.get('id'), 'result': result })
def _failure_browser_root(): config = load_config() service_types = [ t for t in [ ServiceType.Twitter, ServiceType.Pixiv, ServiceType.Fanbox, ServiceType.Weibo ] if t in config.api ] stage_list = [ TaskStage.Fetching, TaskStage.Downloading, TaskStage.Posting, TaskStage.Cleaning ] return render_template('failures_index.jinja2', stage_list=stage_list, service_types=service_types)
def test_load_config_with_vmrc(self): # This is the case which the config is extended from .vmrc file in current directory. copyfile( os.path.join(dirname(dirname(__file__)), 'test_resources', _RC_FILE), os.path.join(os.getcwd(), _RC_FILE)) config = load_config(["test"]) expected_config = { "test": { "files": [{ "names": ["test_files"] }], "excludes": ["test_excludes"] } } self.assertEqual(expected_config, config) os.remove(os.path.join(os.getcwd(), '.vmrc'))
def _pipeline(pipeline_name): config = load_config() pipeline = config.pipeline[pipeline_name] subs = [] for s in pipeline.subscribe: l = [(n, subscribe_services[s.service].get_title(n), subscribe_services[s.service].get_url(n)) for n, channels in SubscribeSource.get_subs_by_channel( *s.service, pipeline_name)] subss = subscribe_services[s.service] options = subss.options() subs.append((s.service[0].value, s.service[1], l, len(l), options)) status = ItemInfo.count_status() return render_template('pipeline.jinja2', pipeline_name=pipeline_name, subs=subs, status=status)
def test_load_config_with_vmrc(self): # This is the case which the config is extended from .vmrc file in current directory. copyfile(os.path.join(dirname(dirname(__file__)), 'test_resources', _RC_FILE), os.path.join(os.getcwd(), _RC_FILE)) config = load_config(["test"]) expected_config = { "test": { "files": [ { "names": ["test_files"] } ], "excludes": [ "test_excludes" ] } } self.assertEqual(expected_config, config) os.remove(os.path.join(os.getcwd(), '.vmrc'))
def update_subs(): config = load_config() for (stype, sfunc), service_type in subscribe_services.items(): if stype not in config.api: continue service_conf = list(config.api[stype].values())[0] service = service_type(service_conf) for name, channels in SubscribeSource.get_subs(stype, sfunc): for item in service.subscribe_index(name): if not ItemInfo.exists(item.service, item.item_id): ItemInfo.add_index(item, channels) print(stype.value, sfunc, name, item) ItemInfo.set_status(item.service, item.item_id, TaskStage.Fetching, TaskStatus.Queued) for item in service.subscribe_full(name): if not ItemInfo.exists(item.service, item.item_id): ItemInfo.add_item(item, channels) print(stype.value, sfunc, name, item) ItemInfo.set_status(item.service, item.item_id, TaskStage.Downloading, TaskStatus.Queued)
def autodownload(ref, save_dir, auth_token): """ Autodownload the fasta/fastq file for a Genome, Reads, or Assembly. Args: ref is a workspace reference ID in the form 'workspace_id/object_id/version' save_dir is the path of a directory in which to save the downloaded file Returns a tuple of (file_path, paired_end) file_path is the string path of the saved file paired_end is a boolean indicating if these are paired-end reads The generate_sketch function needs to know if it's working with paired-end reads or not """ config = load_config() ws = WorkspaceClient(url=config["kbase_endpoint"], token=auth_token) ws_obj = ws.req("get_objects2", {'objects': [{"ref": ref}], 'no_data': 1}) ws_type = ws_obj['data'][0]['info'][2] if valid_types['reads_paired'] in ws_type: paths = ws.download_reads_fastq(ref, save_dir) output_path = paths[0].replace(".paired.fwd.fastq", ".fastq") concatenate_files(paths, output_path) print(f'Downloaded fastq file(s) to {output_path}') return (output_path, True) elif valid_types['reads_single'] in ws_type: paths = ws.download_reads_fastq(ref, save_dir) output_path = paths[0] print(f'Downloaded fastq file(s) to {output_path}') return (output_path, False) elif valid_types['assembly'] in ws_type or valid_types[ 'assembly_legacy'] in ws_type: path = ws.download_assembly_fasta(ref, save_dir) print(f'Downloaded fasta file(s) from Assembly to {path}') return (path, False) elif valid_types['genome'] in ws_type: ref = ws.get_assembly_from_genome(ref) path = ws.download_assembly_fasta(ref, save_dir) print(f'Downloaded fasta file(s) from Genome to {path}') return (path, False) else: raise UnrecognizedWSType(ws_type, valid_types)
def _prepare_query(query): with open(_get_query_path(query)) as f: _ = f.readline() # Skip credentials header line query_sql = f.read() # Extract query parameters from raw SQL params = [p[1:-1] for p in findall(r"{[a-zA-Z_][a-zA-Z_\d]*}", query_sql)] if not params: return query_sql args = load_config("queries")[query] missing = list(set(params).difference(args.keys())) if missing: raise QueryError(f'Query "{query}" missing argument(s): {missing}') unused = list(set(args.keys()).difference(params)) if unused: raise QueryError(f'Unused argument(s) for query "{query}": {unused}') return query_sql.format(**args)
def map_refseq_ids_to_kbase(distances): """ Given the results from a request to the AssemblyHomologyService, iterate over every Refseq ID in the results and fetch the corresponding KBase ID using the ID Mapping service. Args: distances - an array of search result objects from the AssemblyHomologyService AssemblyHomology API: https://github.com/jgi-kbase/AssemblyHomologyService#api """ # Create a list of Refseq IDs config = load_config() refseq_ids = [d['sourceid'] for d in distances] req_data = {"ids": refseq_ids} req_json = json.dumps(req_data) endpoint = config['id_mapper_url'] + '/mapping/RefSeq' print(f"Getting KBase IDs for {refseq_ids} using endpoint {endpoint}") resp = requests.get(endpoint, data=req_json, timeout=999) # Handle any error case from the ID Mapper by exiting and logging everything if not resp.ok: print('=' * 80) print(f'ID Mapping error with status code {resp.status_code}') print(resp.text) print('=' * 80) raise Exception(f"Error from the ID Mapping service: {resp.text}") resp_json = resp.json() print(' response', resp.text) # Create a dict of indexes where each key is the refseq ID so we can refer to it below indexes = {} for (idx, dist) in enumerate(distances): indexes[dist['sourceid']] = idx # Find all KBase ids in the given mappings for (refseq_id, result) in resp_json.items(): distance_idx = indexes[refseq_id] mappings = result['mappings'] for mapping in mappings: if 'KBase' == mapping['ns']: kbase_id = mapping['id'] distances[distance_idx]['kbase_id'] = kbase_id return distances
def generate_sketch(file_path, search_db, paired_end=False): """ Generate a sketch file from a given downloaded fasta/fastq file. Args: downloaded_file is a DownloadedFile namedtuple defined in ./download_file.py Returns the full path of the sketch file """ config = load_config() # Fetch the k-mer size url = f"{config['homology_url']}/namespace/{search_db}" resp = requests.get(url, timeout=999) json_resp = resp.json() sketch_size = str(json_resp.get('sketchsize', 10000)) kmer_size = json_resp.get('kmersize', 19) if isinstance(kmer_size, list): kmer_size = kmer_size[0] output_name = os.path.basename(file_path + '.msh') output_path = os.path.join(os.path.dirname(file_path), output_name) args = ['mash', 'sketch', file_path, '-o', output_path, '-k', str(kmer_size), '-s', sketch_size] print(f"Generating sketch with command: {' '.join(args)}") if paired_end: # For paired end reads, sketch the reads using -m 2 to improve results by ignoring # single-copy k-mers, which are more likely to be erroneous. # See docs: # http://mash.readthedocs.io/en/latest/tutorials.html#querying-read-sets-against-an-existing-refseq-sketch args += ['-m', '2'] proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # nosec (stdout, stderr) = proc.communicate() print('-' * 80) print('mash output:') print(stdout) print(stderr) print('-' * 80) if proc.returncode != 0: raise Exception(f"Error generating sketch data: {stderr}") return output_path
def test_create_folder(self): config = load_config() conf: MegaConfig = config.api[ServiceType.Mega]['default'] client = mega.Mega() client.login(conf.username, conf.password) client.create_folder(conf.root)
help='Verbose output. Changes log level from INFO to DEBUG.') parser.add_argument( '--config', help='Specify a configuration file (defaults to ./config.yml)') parser.add_argument('-l', '--logfile', help="Log file to append logs to.", default=None) args = parser.parse_args() logging.basicConfig(level=logging.DEBUG if args.v else logging.INFO, filename=args.logfile, format="%(asctime)-15s: %(message)s") enable_color_logging(debug_lvl=logging.DEBUG if args.v else logging.INFO) logger.info(intro()) CONFIG = load_config(args.config or "./config.yml") li = lichess.Lichess(CONFIG["token"], CONFIG["url"], __version__) user_profile = li.get_profile() username = user_profile["username"] is_bot = user_profile.get("title") == "BOT" logger.info("Welcome {}!".format(username)) if args.u is True and is_bot is False: is_bot = upgrade_account(li) if is_bot: engine_factory = partial(engine_wrapper.create_engine, CONFIG) start(li, user_profile, engine_factory, CONFIG) else: logger.error(
def run(self): # read configuration files config_root = self.ENV.get("CONFIG_ROOT", "../config") config_schemas = config.load_config(config_root + "/schemas.yml", env=self.ENV) config_services = config.load_config(config_root + "/services.yml", env=self.ENV) app_config["config_root"] = config_root app_config["config_services"] = config_services agent_admin_url = self.ENV.get("AGENT_ADMIN_URL") if not agent_admin_url: raise RuntimeError( "Error AGENT_ADMIN_URL is not specified, can't connect to Agent." ) app_config["AGENT_ADMIN_URL"] = agent_admin_url # get public DID from our agent response = requests.get( agent_admin_url + "/wallet/did/public", headers=ADMIN_REQUEST_HEADERS, ) result = response.json() did = result["result"] LOGGER.info("Fetched DID from agent: %s", did) app_config["DID"] = did["did"] # determine pre-registered schemas and cred defs existing_schemas = agent_schemas_cred_defs(agent_admin_url) # register schemas and credential definitions for schema in config_schemas: schema_name = schema["name"] schema_version = schema["version"] schema_key = schema_name + "::" + schema_version if schema_key not in existing_schemas: schema_attrs = [] schema_descs = {} if isinstance(schema["attributes"], dict): # each element is a dict for attr, desc in schema["attributes"].items(): schema_attrs.append(attr) schema_descs[attr] = desc else: # assume it's an array for attr in schema["attributes"]: schema_attrs.append(attr) # register our schema(s) and credential definition(s) schema_request = { "schema_name": schema_name, "schema_version": schema_version, "attributes": schema_attrs, } response = agent_post_with_retry( agent_admin_url + "/schemas", json.dumps(schema_request), headers=ADMIN_REQUEST_HEADERS, ) response.raise_for_status() schema_id = response.json() else: schema_id = { "schema_id": existing_schemas[schema_key]["schema"]["id"] } app_config["schemas"]["SCHEMA_" + schema_name] = schema app_config["schemas"]["SCHEMA_" + schema_name + "_" + schema_version] = schema_id["schema_id"] LOGGER.info("Registered schema: %s", schema_id) if (schema_key not in existing_schemas or "cred_def" not in existing_schemas[schema_key]): cred_def_request = {"schema_id": schema_id["schema_id"]} response = agent_post_with_retry( agent_admin_url + "/credential-definitions", json.dumps(cred_def_request), headers=ADMIN_REQUEST_HEADERS, ) response.raise_for_status() credential_definition_id = response.json() else: credential_definition_id = { "credential_definition_id": existing_schemas[schema_key]["cred_def"]["id"] } app_config["schemas"]["CRED_DEF_" + schema_name + "_" + schema_version] = credential_definition_id[ "credential_definition_id"] LOGGER.info("Registered credential definition: %s", credential_definition_id) # what is the TOB connection name? tob_connection_params = config_services["verifiers"]["bctob"] # check if we have a TOB connection response = requests.get( agent_admin_url + "/connections?alias=" + tob_connection_params["alias"], headers=ADMIN_REQUEST_HEADERS, ) response.raise_for_status() connections = response.json()["results"] tob_connection = None for connection in connections: # check for TOB connection if connection["alias"] == tob_connection_params["alias"]: tob_connection = connection if not tob_connection: # if no tob connection then establish one (if we can) # (agent_admin_url is provided if we can directly ask the TOB agent for an invitation, # ... otherwise the invitation has to be provided manually through the admin api # ... WITH THE CORRECT ALIAS) if ("agent_admin_url" in tob_connection_params["connection"] and tob_connection_params["connection"]["agent_admin_url"]): tob_agent_admin_url = tob_connection_params["connection"][ "agent_admin_url"] response = requests.post( tob_agent_admin_url + "/connections/create-invitation", headers=TOB_REQUEST_HEADERS, ) response.raise_for_status() invitation = response.json() response = requests.post( agent_admin_url + "/connections/receive-invitation?alias=" + tob_connection_params["alias"], json.dumps(invitation["invitation"]), headers=ADMIN_REQUEST_HEADERS, ) response.raise_for_status() tob_connection = response.json() LOGGER.info("Established tob connection: %s", json.dumps(tob_connection)) time.sleep(5) # if we have a connection to the TOB agent, we can register our issuer if tob_connection: register_issuer_with_orgbook(tob_connection["connection_id"]) else: print( "No TOB connection found or established, awaiting invitation to connect to TOB ..." )
def test_load_config(self): config = load_config(["default"]) self.assertDictEqual({k: v for k, v in _INITIAL_CONFIG["groups"].items() if k is not "default_groups"}, config)
import traceback from functools import lru_cache from io import BytesIO from src.config import load_config from src.enums import TaskStatus, TaskStage, ServiceType from src.models.connect import connect_db from src.models.item import ItemInfo, SecondaryTask from src.services import push_services, pull_services config = load_config() def post_images(): for item in ItemInfo.poll_status(TaskStage.Posting, TaskStatus.Queued): channels = ItemInfo.get_channels(item) for ch in channels: for pipe in config.pipeline[ch].push: SecondaryTask.add_task(item.service, item.item_id, pipe.service, pipe.config, ch) ItemInfo.set_status(item.service, item.item_id, TaskStage.Posting, TaskStatus.Pending) for stype, item_id, ptype, conf, ch, poll_counter in SecondaryTask.poll_tasks(20): SecondaryTask.acquire_task(stype, item_id, ptype, conf, ch) print((stype.value, item_id), '=>', (ptype.value, conf)) item = ItemInfo.get_item(stype, item_id) if not service_exists(ptype, conf): continue client = get_service(ptype, conf) if poll_counter >= client.push_limit(): print("Failed to push item.") SecondaryTask.close_task(stype, item_id, ptype, conf, ch) else:
def launch(): conf = load_config().server connect_db() app.run(host=conf.host, port=conf.port)
import os import sys from logging.config import fileConfig from sqlalchemy import create_engine from sqlalchemy import pool from alembic import context from src.config import load_config from src.infra.db.session import metadata CONFIG = load_config() parent_dir = os.path.abspath(os.path.join(os.getcwd(), "..")) sys.path.append(parent_dir) # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config # Interpret the config file for Python logging. # This line sets up loggers basically. fileConfig(config.config_file_name) from src.infra.db.mapper import start_mappers start_mappers() target_metadata = metadata sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) # other values from the config, defined by the needs of env.py, # can be acquired: # my_important_option = config.get_main_option("my_important_option")
def get_cache_id(data): # Generate the cache_id config = load_config() cache_id = config['cache_client'].generate_cacheid(data) print(f'Fetched cache ID {cache_id[0:16]}..') return cache_id
def _index(): pipelines = load_config().pipeline.items() status = ItemInfo.count_status() return render_template('index.jinja2', pipelines=pipelines, status=status)
def upload_to_cache(cache_id, string): """Save string content to a cache.""" config = load_config() print(f'Uploading contents to cache {cache_id[0:16]}..') config['cache_client'].upload_cache(cache_id, string=string)
def test_write(self): config = load_config() conf: MegaConfig = config.api[ServiceType.Mega]['default'] client = MegaService(conf) print(client.ensure_dir(Path("/xnh/b/c/d/e"))) print(client.client.find("/xnh/b/c/d/e", exclude_deleted=True))
def __init__(self, groups): self._loaded_files = [] self.config = load_config(groups)
# Arguments parser = argparse.ArgumentParser( description='Train a 3D reconstruction model.') parser.add_argument('config', type=str, help='Path to config file.') #parser.add_argument('model_name', type=str, default='model', help='Model output file, i.e. for foo.pt insert foo') parser.add_argument('--no-cuda', action='store_true', help='Do not use cuda.') parser.add_argument( '--exit-after', type=int, default=-1, help='Checkpoint and exit after specified number of seconds' 'with exit code 2.') args = parser.parse_args() cfg = config.load_config(args.config, 'configs/default.yaml') is_cuda = (torch.cuda.is_available() and not args.no_cuda) device = torch.device("cuda" if is_cuda else "cpu") #DEGREES = cfg['degrees'] DEGREES = 1 #model_name = args.model_name ## # Set t0 t0 = time.time() # Shorthands out_dir = cfg['training']['out_dir'] batch_size = cfg['training']['batch_size'] backup_every = cfg['training']['backup_every'] vis_n_outputs = cfg['generation']['vis_n_outputs']
def __init__(self, groups): self._loaded_files = [] self.config = load_config(groups)