def test_load_yaml():
data_test = ordereddict([('image', ordereddict([('repository', 'nextcloud'), ('tag', '15.0.2-apache'), ('pullPolicy', 'IfNotPresent')])), ('nameOverride', ''), ('fullnameOverride', ''), ('replicaCount', 1), ('ingress', ordereddict([('enabled', True), ('annotations', ordereddict())])), ('nextcloud', ordereddict([('host', 'nextcloud.corp.justin-tech.com'), ('username', 'admin'), ('password', 'changeme')])), ('internalDatabase', ordereddict([('enabled', True), ('name', 'nextcloud')])), ('externalDatabase', ordereddict([('enabled', False), ('host', None), ('user', 'VAULT:/secret/testdata/user'), ('password', 'VAULT:/secret/{environment}/testdata/password'), ('database', 'nextcloud')])), ('mariadb', ordereddict([('enabled', True), ('db', ordereddict([('name', 'nextcloud'), ('user', 'nextcloud'), ('password', 'changeme')])), ('persistence', ordereddict([('enabled', True), ('storageClass', 'nfs-client'), ('accessMode', 'ReadWriteOnce'), ('size', '8Gi')]))])), ('service', ordereddict([('type', 'ClusterIP'), ('port', 8080), ('loadBalancerIP', 'nil')])), ('persistence', ordereddict([('enabled', True), ('storageClass', 'nfs-client'), ('accessMode', 'ReadWriteOnce'), ('size', '8Gi')])), ('resources', ordereddict()), ('nodeSelector', ordereddict()), ('tolerations', []), ('affinity', ordereddict())])
yaml_file = "./tests/test.yaml"
data = vault.load_yaml(yaml_file)
print(data)
assert_equal(data, data_test)
def __init__(self, document, imageTypes=None):
self.config = document.config
self.ownerDocument = document
if imageTypes is None:
self.imageTypes = [self.fileExtension]
else:
self.imageTypes = imageTypes[:]
# Dictionary that makes sure each image is only generated once.
# The key is the LaTeX source and the value is the image instance.
self._cache = {}
usednames = {}
# JAM: FIXME: This writes into some arbitrary directory that may or
# may not be related to the document we are processing. It at least
# needs to take document name/path into consideration (see below on
# graphicspath)
self._filecache = os.path.abspath(
os.path.join('.cache', self.__class__.__name__ + '.images'))
if self.config['images']['cache']:
usednames = self._read_cache()
# List of images in the order that they appear in the LaTeX file
self.images = ordereddict()
# Images that are simply copied from the source directory
self.staticimages = ordereddict()
# Filename generator
self.newFilename = Filenames(
self.config['images'].get('filenames', raw=True),
vars={'jobname': document.userdata.get('jobname', '')},
extension=self.fileExtension,
invalid=usednames)
# Start the document with a preamble
self.source = StringIO()
self.source.write('\\batchmode\n')
self.writePreamble(document)
self.source.write('\\begin{document}\n')
#We inject \graphicspath here because this processing occurs in some temp space but the image urls
#are relative to the original working directory.
#Documentation suggests that we could just set the TEXINPUTS environment variable but it does not work
self.source.write('\\graphicspath{{%s/}}\n' %
(self.ownerDocument.userdata['working-dir']))
# Set up additional options
self._configOptions = self.formatConfigOptions(self.config['images'])
def __init__(self, document, imageTypes=None):
self.config = document.config
self.ownerDocument = document
if imageTypes is None:
self.imageTypes = [self.fileExtension]
else:
self.imageTypes = imageTypes[:]
# Dictionary that makes sure each image is only generated once.
# The key is the LaTeX source and the value is the image instance.
self._cache = {}
usednames = {}
# JAM: FIXME: This writes into some arbitrary directory that may or
# may not be related to the document we are processing. It at least
# needs to take document name/path into consideration (see below on
# graphicspath)
self._filecache = os.path.abspath(os.path.join(".cache", self.__class__.__name__ + ".images"))
if self.config["images"]["cache"]:
usednames = self._read_cache()
# List of images in the order that they appear in the LaTeX file
self.images = ordereddict()
# Images that are simply copied from the source directory
self.staticimages = ordereddict()
# Filename generator
self.newFilename = Filenames(
self.config["images"].get("filenames", raw=True),
vars={"jobname": document.userdata.get("jobname", "")},
extension=self.fileExtension,
invalid=usednames,
)
# Start the document with a preamble
self.source = StringIO()
self.source.write("\\batchmode\n")
self.writePreamble(document)
self.source.write("\\begin{document}\n")
# We inject \graphicspath here because this processing occurs in some temp space but the image urls
# are relative to the original working directory.
# Documentation suggests that we could just set the TEXINPUTS environment variable but it does not work
self.source.write("\\graphicspath{{%s/}}\n" % (self.ownerDocument.userdata["working-dir"]))
# Set up additional options
self._configOptions = self.formatConfigOptions(self.config["images"])
def read(cls, stream, index):
self = cls(index.get_string(stream.read_u30()))
item_count = stream.read_u30()
self.item_info = ordereddict((index.get_string(stream.read_u30()),
index.get_string(stream.read_u30()))
for i in range(item_count))
return self
def __init__(
self,
name: str,
package_name: str,
package_path: str,
list_paths: _ListPaths,
get_data: _GetData,
save_data: Optional[_SaveData] = None,
visible: Optional[bool] = True,
editable: Optional[bool] = True,
sort_value: Optional[int] = None,
):
try:
self.name = name
self.package_name = package_name
self.package_path = package_path
self.visible = visible
self.editable = False if settings.DISABLE_EDIT_DATA_SOURCES else editable
self.sort_value = sort_value
self.nodes = ordereddict()
self.nodes_fully_loaded = False
self.error = None
self._list_paths = list_paths
self._get_data = get_data
self._save_data = save_data
self._path_generator = None
self._validate()
except BaseException as e:
self.error = str(e)
raise e
def prepResult(title, array):
arq_base = open("base.txt","r")
text = arq_base.read().lower().split()
chrs = (78 - len(title)) / 2
cont = 0
enf = ""
enf2 = "_"
while cont < chrs:
enf += "*"
enf2 += "_"
cont += 1
result = ("\n//" + enf + " " + title + " " + enf + "\\\\\n\n"
"| Palavra | | Frequência |\n\n")
frequencia = FreqDist(text)
frequencia_ord = ordereddict(sorted(frequencia.items(), key = lambda e: (-e[1], e[0])))
for freq in frequencia_ord:
if(freq in array):
lim = 84 / 2
right = lim / 2 + len(freq)
chrs = (78 - (len(freq)) + len(str(frequencia_ord[freq]))) / 4
cont = 0
enf = ""
while cont < chrs:
enf += " "
cont += 1
result += "|" + enf + freq + enf + " | " + enf + str(frequencia_ord[freq]) + enf + "|\n"
result += "\n\\\\________________________________________________________________________________//\n\n"
arq_base.close()
return result
def __init__(self):
self.structure_dict = ordereddict([
('name',''),
('property',''),
('value',''),
('source',''),
])
self.list_of_structures = []
def __init__(self):
self.config = ordereddict([
('name',''),
('state',''),
('read',''),
('write',''),
('cksum',''),
])
self.structure_dict = ordereddict([
('pool',''),
('state',''),
('scan',''),
('action',''),
('see',''),
('scrub',''),
('status',''),
('config', []),
('errors', []),
])
def stoichiometry(self) -> Tuple['Formula', 'Formula', int]:
"""Reduce to unique stoichiomerty using "chemical symbols" A, B, C, ...
Examples
--------
>>> Formula('CO2').stoichiometry()
(Formula('AB2'), Formula('CO2'), 1)
>>> Formula('(H2O)4').stoichiometry()
(Formula('AB2'), Formula('OH2'), 4)
"""
count1, N = self._reduce()
c = ord('A')
count2 = ordereddict()
count3 = ordereddict()
for n, symb in sorted((n, symb) for symb, n in count1.items()):
count2[chr(c)] = n
count3[symb] = n
c += 1
return self.from_dict(count2), self.from_dict(count3), N
def __format__(self, fmt: str) -> str:
"""Format Formula as str.
Possible formats: ``'hill'``, ``'metal'``, ``'abc'``, ``'latex'``,
``'html'``, ``'rest'``.
Example
-------
>>> f = Formula('OH2')
>>> '{f}, {f:hill}, {f:latex}'.format(f=f)
'OH2, H2O, OH$_{2}$'
"""
if fmt == 'hill':
count = self.count()
count2 = ordereddict()
for symb in 'CH':
if symb in count:
count2[symb] = count.pop(symb)
for symb, n in sorted(count.items()):
count2[symb] = n
return dict2str(count2)
if fmt == 'metal':
count = self.count()
result2 = [(s, count.pop(s)) for s in non_metals if s in count]
result = [(s, count[s]) for s in sorted(count)]
result += sorted(result2)
return dict2str(ordereddict(result))
if fmt == 'abc':
_, f, N = self.stoichiometry()
return dict2str({symb: n * N for symb, n in f._count.items()})
if fmt == 'latex':
return self._tostr('$_{', '}$')
if fmt == 'html':
return self._tostr('<sub>', '</sub>')
if fmt == 'rest':
return self._tostr(r'\ :sub`', r'`\ ')
if fmt == '':
return self._formula
raise ValueError('Invalid format specifier')
def __init__(self):
self.structure_dict = ordereddict([
('name',''),
('full_size',''),
('m_size',''),
('full_size2',''),
('usage',''),
('speed',''),
('status',''),
('other_part',''),
])
self.list_of_structures = []
def prepResultCSV(title, array, spamwriter):
arq_base = open("base.txt","r")
text = arq_base.read().lower().split()
frequencia = FreqDist(text)
frequencia_ord = ordereddict(sorted(frequencia.items(), key = lambda e: (-e[1], e[0])))
spamwriter.writerow([title])
spamwriter.writerow(["Palavra", "Frequência"])
for freq in frequencia_ord:
if(freq in array):
spamwriter.writerow([freq, str(frequencia_ord[freq])])
arq_base.close()
def get_configuration():
""" Get the configuration from command line and config files """
# This is the dict we will return
configuration = {
'global': {},
'logging': {},
'tables': ordereddict()
}
# Read the command line options
cmd_line_options = command_line_parser.parse()
# If a configuration file is specified, read that as well
conf_file_options = None
if 'config' in cmd_line_options:
conf_file_options = config_file_parser.parse(
cmd_line_options['config'])
# Extract global config
configuration['global'] = __get_global_options(
cmd_line_options,
conf_file_options)
# Extract logging config
configuration['logging'] = __get_logging_options(
cmd_line_options,
conf_file_options)
# Extract table configuration
# If the --table cmd line option is set, it indicates that only table
# options from the command line should be used
if 'table_name' in cmd_line_options:
configuration['tables'] = __get_cmd_table_options(cmd_line_options)
elif 'dynamodb_table' in conf_file_options:
configuration['tables'] = __get_dynamodb_config_options(conf_file_options)
else:
configuration['tables'] = __get_config_table_options(conf_file_options)
# Ensure some basic rules
__check_gsi_rules(configuration)
__check_logging_rules(configuration)
__check_table_rules(configuration)
return configuration
def all_entries(self):
entry_and_victim = (
app.db.session
.query(Entry, EntryVictim)
.filter(Entry.event == self.id,
EntryVictim.entry == Entry.id)
.order_by(Entry.date.desc(), Entry.id.desc())
.all())
# Reshape (entry, victim) list into a list of (entry, victims).
entries = ordereddict()
for entry, victim in entry_and_victim:
if entry.id not in entries:
entry.victims = []
entries[entry.id] = entry
entries[entry.id].victims.append(victim)
return entries.values()
def get_configuration():
""" Get the configuration from command line and config files """
# This is the dict we will return
configuration = {
'global': {},
'logging': {},
'tables': ordereddict()
}
# Read the command line options
cmd_line_options = command_line_parser.parse()
# If a configuration file is specified, read that as well
conf_file_options = None
if 'config' in cmd_line_options:
conf_file_options = config_file_parser.parse(
cmd_line_options['config'])
# Extract global config
configuration['global'] = __get_global_options(
cmd_line_options,
conf_file_options)
# Extract logging config
configuration['logging'] = __get_logging_options(
cmd_line_options,
conf_file_options)
# Extract table configuration
# If the --table cmd line option is set, it indicates that only table
# options from the command line should be used
if 'table_name' in cmd_line_options:
configuration['tables'] = __get_cmd_table_options(cmd_line_options)
else:
configuration['tables'] = __get_config_table_options(conf_file_options)
# Ensure some basic rules
__check_gsi_rules(configuration)
__check_logging_rules(configuration)
__check_table_rules(configuration)
return configuration
def deserialize(self, overwrite_existing: bool = False) -> DataSource:
"""
Returns the corresponding DataSource.
The back end is treated as the universal source of truth UNLESS overwrite_existing is set to True.
"""
if self.id_ not in SOURCES:
self._register_as_new_custom_source()
elif overwrite_existing:
if not self.editable:
raise HTTPException(status_code=400,
detail="This source can not be edited")
existing = SOURCES[self.id_].serialize()
if (self.list_paths != existing.list_paths) or (self.get_data !=
existing.get_data):
SOURCES[self.id_].kill_all_nodes()
self.nodes = ordereddict()
self.nodes_fully_loaded = False
self._register_as_new_custom_source()
return SOURCES[self.id_]
def __init__(self, items: _SourceT) -> None:
r"Initializer."
super().__init__()
self._simple_init = None
tmp: Counter[_OutcomeT] = counter()
if isinstance(items, (int, Integral)):
if items != 0:
self._simple_init = items
outcome_type = type(items)
count_1 = type(items)(1)
outcome_range = range(
items,
0,
1 if items < 0 else -1 # count toward zero
)
tmp.update({outcome_type(i): count_1 for i in outcome_range})
elif isinstance(items, HAbleT):
tmp.update(items.h())
elif isinstance(items, ABCMapping):
tmp.update(items)
elif isinstance(items, ABCIterable):
# Items is either an Iterable[_OutcomeT] or an Iterable[Tuple[_OutcomeT,
# _CountT]] (although this technically supports Iterable[Union[_OutcomeT,
# Tuple[_OutcomeT, _CountT]]])
for item in items:
if isinstance(item, tuple):
outcome, count = item
tmp[outcome] += count
else:
tmp[item] += 1
else:
raise ValueError("unrecognized initializer {}".format(type(items)))
# Sort and omit zero counts. We use an OrderedDict instead of a Counter to
# support Python versions earlier than 3.7 which did not guarantee order
# preservation for the latter.
self._h: _MappingT = ordereddict(
{outcome: tmp[outcome]
for outcome in sorted(tmp) if tmp[outcome]})
def parse(config_path):
""" Parse the configuration file
:type config_path: str
:param config_path: Path to the configuration file
"""
config_path = os.path.expanduser(config_path)
# Read the configuration file
config_file = ConfigParser.RawConfigParser()
config_file.SECTCRE = re.compile(r"\[ *(?P<header>.*) *\]")
config_file.optionxform = lambda option: option
config_file.read(config_path)
#
# Handle [global]
#
if 'global' in config_file.sections():
global_config = __parse_options(
config_file,
'global',
[
{
'key': 'aws_access_key_id',
'option': 'aws-access-key-id',
'required': False,
'type': 'str'
},
{
'key': 'aws_secret_access_key',
'option': 'aws-secret-access-key-id',
'required': False,
'type': 'str'
},
{
'key': 'region',
'option': 'region',
'required': False,
'type': 'str'
},
{
'key': 'check_interval',
'option': 'check-interval',
'required': False,
'type': 'int'
},
{
'key': 'circuit_breaker_url',
'option': 'circuit-breaker-url',
'required': False,
'type': 'str'
},
{
'key': 'circuit_breaker_timeout',
'option': 'circuit-breaker-timeout',
'required': False,
'type': 'float'
},
])
#
# Handle [logging]
#
if 'logging' in config_file.sections():
logging_config = __parse_options(
config_file,
'logging',
[
{
'key': 'log_level',
'option': 'log-level',
'required': False,
'type': 'str'
},
{
'key': 'log_file',
'option': 'log-file',
'required': False,
'type': 'str'
},
{
'key': 'log_config_file',
'option': 'log-config-file',
'required': False,
'type': 'str'
}
])
if 'default_options' in config_file.sections():
# nothing is required in defaults, so we set required to False
default_config_options = deepcopy(TABLE_CONFIG_OPTIONS)
for item in default_config_options:
item['required'] = False
default_options = __parse_options(
config_file, 'default_options', default_config_options)
# if we've got a default set required to be false for table parsing
for item in TABLE_CONFIG_OPTIONS:
if item['key'] in default_options:
item['required'] = False
else:
default_options = {}
#
# Handle [table: ]
#
table_config = {'tables': ordereddict()}
# Find the first table definition
found_table = False
for current_section in config_file.sections():
if current_section.rsplit(':', 1)[0] != 'table':
continue
found_table = True
current_table_name = current_section.rsplit(':', 1)[1].strip()
table_config['tables'][current_table_name] = \
dict(default_options.items() + __parse_options(
config_file, current_section, TABLE_CONFIG_OPTIONS).items())
if not found_table:
print('Could not find a [table: <table_name>] section in {0}'.format(
config_path))
sys.exit(1)
# Find gsi definitions - this allows gsi's to be defined before the table
# definitions we don't worry about parsing everything twice here
for current_section in config_file.sections():
try:
header1, gsi_key, header2, table_key = current_section.split(' ')
except ValueError:
continue
if header1 != 'gsi:':
continue
if table_key not in table_config['tables']:
print('No table configuration matching {0} found.'.format(
table_key))
sys.exit(1)
if 'gsis' not in table_config['tables'][table_key]:
table_config['tables'][table_key]['gsis'] = {}
table_config['tables'][table_key]['gsis'][gsi_key] = \
ordereddict(default_options.items() + __parse_options(
config_file, current_section, TABLE_CONFIG_OPTIONS).items())
return ordereddict(
global_config.items() +
logging_config.items() +
table_config.items())
### AVALIANDO RETWEETS ###
arq_base = open("base.txt","r")
text = arq_base.readlines()
freq_rts = defaultdict(int)
for stmt in text:
words = stmt.lower().split()
i = 0
if(stmt[0:2].lower() == 'rt'):
for word in words:
if(word == 'rt'):
word = words[i] + " " + words[i +1]
if(word in rts):
freq_rts[word] += 1
i += 1
spamwriter.writerow(["RETWEETS"])
spamwriter.writerow(["Palavra", "Frequência"])
freq_rts_ord = ordereddict(sorted(freq_rts.items(), key = lambda e: (-e[1], e[0])))
for freq in freq_rts_ord:
spamwriter.writerow([freq, str(freq_rts_ord[freq])])
arq_base.close()
### FIM AVALIANDO RETWEETS ###
prepResultCSV("MENTIONS", mentions, spamwriter)
prepResultCSV("VERBOS", verbos, spamwriter)
prepResultCSV("STOP WORDS", stop_words, spamwriter)
prepResultCSV("DEMAIS EXPRESSÕES", exprs, spamwriter)
arq.close()
arq_results_csv.close()
print("Processamento Finalizado!")
def __init__(self, name, items=()):
self.name = name
self.item_info = ordereddict(items)
def make_class_timetable_array(klass, lessons):
# Load abbreviations for subjects: "Inglese" instead "Lingua e
# Cultura Straniera Inglese" as description for ING).
mat_names = get_mat_names()
# print(f"\n=== {klass} ====================")
rr = list()
# dd = ["Ora"] + [d[:3].upper() for d in DAYS_SHIFT]
# START_TIMES = "07h50 08h40 09h30 10h30 11h20 12h15 13h10 14h00".split()
END_TIMES = [ # BUG: stupid name!
"8:00 - 8:40",
"8:50 - 9:30",
"9:40 - 10:20",
"10:40 - 11:20",
"11:30 - 12:10",
"12:20 - 13:00",
"13:10 - 14:00",
"14:00 - 14:40",
]
END_TIMES = ordereddict(( # BUG: stupid name!
("07h50", "8:00\n 8:40"),
("08h40", "8:50\n 9:30"),
("09h30", "9:40\n10:20"),
("10h30", "10:40\n11:20"),
("11h20", "11:30\n12:10"),
("12h15", "12:20\n13:00"),
("13h10", "13:10\n14:00"),
("14h00", "14:00\n14:40"),
))
# BUG
# hh = [""] + [s.replace("h", ".").lstrip("0") for s in START_TIMES]
hh = [""] + list(END_TIMES.values())
rr.append(hh)
for day, lessons in sorted(lessons.items(), key=day_sorter):
r = [day.capitalize()]
# BUG:
# Here I'm looping on lessons but it is important to also
# produce lines (hour) where this class has no lesson (e.g.:
# this class start the day with the second hour on monday).
# for o in sorted(lessons, key=start_sorter):
# mat = mat_names[o.MAT_COD] # ING -> Inglese
# r.append(f"{mat.strip()}\n{o.DOC_COGN.strip()}")
foo = ordereddict()
# for st in START_TIMES: # prefill with all start times!
# foo[st] = None
# for less in lessons:
# foo[less.ORA_INIZIO] = less
for st in START_TIMES: # prefill with all start times!
foo[st] = list()
for less in lessons:
foo[less.ORA_INIZIO].append(less)
for index, (st, oo) in enumerate(foo.items()):
if not oo: # no lessons at this time
r.append("")
else:
mm = list()
try:
for o in oo:
mat = mat_names[o.MAT_COD] # ING -> Inglese
mat = o.MAT_COD # .capitalize()
mm.append(mat)
except KeyError as e:
foo = f"{index=},{o.MAT_COD=},{o.DOC_NOME=}"
debug(f"make_class_timetable_array ({foo}) {e}")
continue
mat = "/".join(mm).strip()
# if len(oo) > 1:
# debug(f"MULTIPLE {mat}")
pp = list()
for o in oo:
pp.append(o.DOC_COGN.strip())
prof = "/".join(pp)
# BUG: single MAT only ???
r.append(f"{mat}\n{prof}\nsincrona")
rr.append(r)
rr = list(zip(*rr, fillvalue=""))
kk = [""] * 3 + [klass] + [""] * 3
rr.insert(0, [""] * len(kk))
rr.insert(1, kk)
return rr
def __init__(self):
self.structure_dict = ordereddict([
('name',''),
('uniq', ''),
])
self.disk_map = []
def __get_dynamodb_config_options(conf_file_options):
""" Get all table options from DynamoDb table
In order to use this feature your IAM User or Role will need to have access to Read (Scan) the
table that contains the data. For security it is recommended that there is a separate statement
added to the existing policy that limits read to that table. See below
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": "dynamodb:Scan",
"Resource": "arn:aws:dynamodb:eu-west-1:1234567890:table/dynamic-dynamodb-config-table"
}
]
}
The table must have the following field defined:
table_name : This is the name of the table to be processed
:type conf_file_options: dict
:param conf_file_options: Dictionary with all config file options
:returns: ordereddict -- E.g. {'table_name': {}}
"""
options = ordereddict()
config_table_name = conf_file_options['dynamodb_table']
# todo :- this is a HACK : Sort it out !!!! This should be using the logic in aws.dynamodb
conn = __get_connection_dynamodb(conf_file_options=conf_file_options)
config_table = Table(table_name=config_table_name, connection=conn)
#
# This needs to be aware that the results of the scan may be paginated
#
for row in config_table.scan():
table_name = row['table_name']
options[table_name] = {}
# Regular table options
for option in DEFAULT_OPTIONS['table'].keys():
options[table_name][option] = DEFAULT_OPTIONS['table'][option]
if option not in row:
continue
if option == 'sns_message_types':
try:
raw_list = row[option]
options[table_name][option] = [i.strip() for i in raw_list.split(',')]
except:
print(
'Error parsing the "sns-message-types" option: {0}'.format(row[option]))
else:
options[table_name][option] = row[option]
return options
def main(argv=None):
"""
Run amplimap setup wizard.
"""
try:
basedir = os.path.dirname(os.path.realpath(__file__))
# parse the arguments, which will be available as properties of args (e.g. args.probe)
parser = argparse.ArgumentParser(
description="amplimap v{} setup wizard".format(__version__),
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# specify parameters
parser.add_argument("operation",
help="type of setup to perform: paths / indices",
action="store_true")
parser.add_argument("--debug", help="debug mode", action="store_true")
if argv is None:
args = parser.parse_args()
else:
args = parser.parse_args(argv)
raise Exception('Not implemented yet.')
if args.operation == 'paths':
# TODO:
# - ask aligner, caller
# - ask about modules:
# - check path for:
# aligner, caller, bedtools, samtools (error if not found)
# annovar, picard, bcftools (warning if not found)
answers = collections.ordereddict()
for key, q in questions.items():
answers[key] = q.ask()
# print('Your answers:')
# for key, answer in answers.items():
# print('{}: {}'.format(
# key, answer
# ))
tools_required = ['bedtools', 'samtools']
tools_for_variants = ['annovar', 'bcftools']
if answers['aligner'] == 'bwa':
tools_required.append('bwa')
elif answers['aligner'] == 'bowtie2':
tools_required.append('bwa')
elif answers['aligner'] == 'star':
tools_required.append('bwa')
elif args.operation == 'indices':
# TODO:
# - ask path to reference genome
# - build fasta index, default aligner index
# - ask to enter name of other index to create (bwa, bowtie2, star)
pass
else:
raise Exception('Please specify a valid operation!')
except KeyboardInterruptException:
sys.stderr.write('Interrupted.\n')
return 1
except EOFError:
sys.stderr.write('Aborted.\n')
return 1
except Exception as e:
sys.stderr.write('\nERROR: {}\n\n'.format(e))
sys.stderr.write('{} {} failed!\n'.format(__title__, __version__))
return 1
def __get_config_table_options(conf_file_options):
""" Get all table options from the config file
:type conf_file_options: ordereddict
:param conf_file_options: Dictionary with all config file options
:returns: ordereddict -- E.g. {'table_name': {}}
"""
options = ordereddict()
if not conf_file_options:
return options
for table_name in conf_file_options['tables']:
options[table_name] = {}
# Regular table options
for option in DEFAULT_OPTIONS['table'].keys():
options[table_name][option] = DEFAULT_OPTIONS['table'][option]
if option not in conf_file_options['tables'][table_name]:
continue
if option == 'sns_message_types':
try:
raw_list = conf_file_options['tables'][table_name][option]
options[table_name][option] = \
[i.strip() for i in raw_list.split(',')]
except:
print('Error parsing the "sns-message-types" '
'option: {0}'.format(
conf_file_options['tables'][table_name][option]))
else:
options[table_name][option] = \
conf_file_options['tables'][table_name][option]
# GSI specific options
if 'gsis' in conf_file_options['tables'][table_name]:
for gsi_name in conf_file_options['tables'][table_name]['gsis']:
for option in DEFAULT_OPTIONS['gsi'].keys():
opt = DEFAULT_OPTIONS['gsi'][option]
if 'gsis' not in options[table_name]:
options[table_name]['gsis'] = {}
if gsi_name not in options[table_name]['gsis']:
options[table_name]['gsis'][gsi_name] = {}
if (option not in conf_file_options['tables'][table_name]
['gsis'][gsi_name]):
options[table_name]['gsis'][gsi_name][option] = opt
continue
if option == 'sns_message_types':
try:
raw_list = conf_file_options['tables'][table_name][
'gsis'][gsi_name][option]
opt = [i.strip() for i in raw_list.split(',')]
except:
print('Error parsing the "sns-message-types" '
'option: {0}'.format(
conf_file_options['tables'][table_name]
['gsis'][gsi_name][option]))
else:
opt = conf_file_options['tables'][table_name]['gsis'][
gsi_name][option]
options[table_name]['gsis'][gsi_name][option] = opt
return options
def __get_config_table_options(conf_file_options):
""" Get all table options from the config file
:type conf_file_options: ordereddict
:param conf_file_options: Dictionary with all config file options
:returns: ordereddict -- E.g. {'table_name': {}}
"""
options = ordereddict()
if not conf_file_options:
return options
for table_name in conf_file_options['tables']:
options[table_name] = {}
# Regular table options
for option in DEFAULT_OPTIONS['table'].keys():
options[table_name][option] = DEFAULT_OPTIONS['table'][option]
if option not in conf_file_options['tables'][table_name]:
continue
if option == 'sns_message_types':
try:
raw_list = conf_file_options['tables'][table_name][option]
options[table_name][option] = \
[i.strip() for i in raw_list.split(',')]
except:
print(
'Error parsing the "sns-message-types" '
'option: {0}'.format(
conf_file_options['tables'][table_name][option]))
else:
options[table_name][option] = \
conf_file_options['tables'][table_name][option]
# GSI specific options
if 'gsis' in conf_file_options['tables'][table_name]:
for gsi_name in conf_file_options['tables'][table_name]['gsis']:
for option in DEFAULT_OPTIONS['gsi'].keys():
opt = DEFAULT_OPTIONS['gsi'][option]
if 'gsis' not in options[table_name]:
options[table_name]['gsis'] = {}
if gsi_name not in options[table_name]['gsis']:
options[table_name]['gsis'][gsi_name] = {}
if (option not in conf_file_options[
'tables'][table_name]['gsis'][gsi_name]):
options[table_name]['gsis'][gsi_name][option] = opt
continue
if option == 'sns_message_types':
try:
raw_list = conf_file_options[
'tables'][table_name]['gsis'][gsi_name][option]
opt = [i.strip() for i in raw_list.split(',')]
except:
print(
'Error parsing the "sns-message-types" '
'option: {0}'.format(
conf_file_options[
'tables'][table_name][
'gsis'][gsi_name][option]))
else:
opt = conf_file_options[
'tables'][table_name]['gsis'][gsi_name][option]
options[table_name]['gsis'][gsi_name][option] = opt
return options
from dtale_desktop.pydantic_utils import BaseApiModel
from dtale_desktop.settings import settings
from dtale_desktop.source_code_tools import (
get_source_file,
create_package_name,
create_data_source_package,
move_data_source_package,
load_data_source_package,
)
from dtale_desktop.subprocesses import execute_profile_report_builder
_ListPaths = Callable[..., Union[List[str], Awaitable[List[str]]]]
_GetData = Callable[[str], Union[pd.DataFrame, Awaitable[pd.DataFrame]]]
_SaveData = Callable[[str, pd.DataFrame], None]
SOURCES: Dict[str, "DataSource"] = ordereddict()
class DataSource:
name: str
package_name: str
package_path: str
visible: bool
editable: bool
sort_value: int
nodes: Dict[str, "Node"]
nodes_fully_loaded: bool
error: Optional[str]
_list_paths: _ListPaths
_get_data: _GetData
_save_data: Optional[_SaveData]
def parse(config_path):
""" Parse the configuration file
:type config_path: str
:param config_path: Path to the configuration file
"""
config_path = os.path.expanduser(config_path)
# Read the configuration file
config_file = ConfigParser.RawConfigParser()
config_file.optionxform = lambda option: option
config_file.read(config_path)
#
# Handle [global]
#
if 'global' in config_file.sections():
global_config = __parse_options(
config_file,
'global',
[
{
'key': 'aws_access_key_id',
'option': 'aws-access-key-id',
'required': False,
'type': 'str'
},
{
'key': 'aws_secret_access_key',
'option': 'aws-secret-access-key-id',
'required': False,
'type': 'str'
},
{
'key': 'region',
'option': 'region',
'required': False,
'type': 'str'
},
{
'key': 'check_interval',
'option': 'check-interval',
'required': False,
'type': 'int'
},
{
'key': 'circuit_breaker_url',
'option': 'circuit-breaker-url',
'required': False,
'type': 'str'
},
{
'key': 'circuit_breaker_timeout',
'option': 'circuit-breaker-timeout',
'required': False,
'type': 'float'
},
])
#
# Handle [logging]
#
if 'logging' in config_file.sections():
logging_config = __parse_options(
config_file,
'logging',
[
{
'key': 'log_level',
'option': 'log-level',
'required': False,
'type': 'str'
},
{
'key': 'log_file',
'option': 'log-file',
'required': False,
'type': 'str'
},
{
'key': 'log_config_file',
'option': 'log-config-file',
'required': False,
'type': 'str'
}
])
if 'default_options' in config_file.sections():
# nothing is required in defaults, so we set required to False
default_config_options = deepcopy(TABLE_CONFIG_OPTIONS)
for item in default_config_options:
item['required'] = False
default_options = __parse_options(
config_file, 'default_options', default_config_options)
# if we've got a default set required to be false for table parsing
for item in TABLE_CONFIG_OPTIONS:
if item['key'] in default_options:
item['required'] = False
else:
default_options = {}
#
# Handle [table: ]
#
table_config = {'tables': ordereddict()}
# Find the first table definition
found_table = False
for current_section in config_file.sections():
if current_section.rsplit(':', 1)[0] != 'table':
continue
found_table = True
current_table_name = current_section.rsplit(':', 1)[1].strip()
table_config['tables'][current_table_name] = \
dict(default_options.items() + __parse_options(
config_file, current_section, TABLE_CONFIG_OPTIONS).items())
if not found_table:
print('Could not find a [table: <table_name>] section in {0}'.format(
config_path))
sys.exit(1)
# Find gsi definitions - this allows gsi's to be defined before the table
# definitions we don't worry about parsing everything twice here
for current_section in config_file.sections():
try:
header1, gsi_key, header2, table_key = current_section.split(' ')
except ValueError:
continue
if header1 != 'gsi:':
continue
if table_key not in table_config['tables']:
print('No table configuration matching {0} found.'.format(
table_key))
sys.exit(1)
if 'gsis' not in table_config['tables'][table_key]:
table_config['tables'][table_key]['gsis'] = {}
table_config['tables'][table_key]['gsis'][gsi_key] = \
ordereddict(default_options.items() + __parse_options(
config_file, current_section, TABLE_CONFIG_OPTIONS).items())
return ordereddict(
global_config.items() +
logging_config.items() +
table_config.items())
def parse(config_path):
""" Parse the configuration file
:type config_path: str
:param config_path: Path to the configuration file
"""
config_path = os.path.expanduser(config_path)
# Read the configuration file
config_file = ConfigParser.RawConfigParser()
config_file.optionxform = lambda option: option
config_file.read(config_path)
#
# Handle [global]
#
if "global" in config_file.sections():
global_config = __parse_options(
config_file,
"global",
[
{"key": "aws_access_key_id", "option": "aws-access-key-id", "required": False, "type": "str"},
{
"key": "aws_secret_access_key",
"option": "aws-secret-access-key-id",
"required": False,
"type": "str",
},
{"key": "region", "option": "region", "required": False, "type": "str"},
{"key": "check_interval", "option": "check-interval", "required": False, "type": "int"},
{"key": "circuit_breaker_url", "option": "circuit-breaker-url", "required": False, "type": "str"},
{
"key": "circuit_breaker_timeout",
"option": "circuit-breaker-timeout",
"required": False,
"type": "float",
},
],
)
#
# Handle [logging]
#
if "logging" in config_file.sections():
logging_config = __parse_options(
config_file,
"logging",
[
{"key": "log_level", "option": "log-level", "required": False, "type": "str"},
{"key": "log_file", "option": "log-file", "required": False, "type": "str"},
{"key": "log_config_file", "option": "log-config-file", "required": False, "type": "str"},
],
)
if "default_options" in config_file.sections():
# nothing is required in defaults, so we set required to False
default_config_options = deepcopy(TABLE_CONFIG_OPTIONS)
for item in default_config_options:
item["required"] = False
default_options = __parse_options(config_file, "default_options", default_config_options)
# if we've got a default set required to be false for table parsing
for item in TABLE_CONFIG_OPTIONS:
if item["key"] in default_options:
item["required"] = False
else:
default_options = {}
#
# Handle [table: ]
#
table_config = {"tables": ordereddict()}
# Find the first table definition
found_table = False
for current_section in config_file.sections():
if current_section.rsplit(":", 1)[0] != "table":
continue
found_table = True
current_table_name = current_section.rsplit(":", 1)[1].strip()
table_config["tables"][current_table_name] = dict(
default_options.items() + __parse_options(config_file, current_section, TABLE_CONFIG_OPTIONS).items()
)
if not found_table:
print("Could not find a [table: <table_name>] section in {0}".format(config_path))
sys.exit(1)
# Find gsi definitions - this allows gsi's to be defined before the table
# definitions we don't worry about parsing everything twice here
for current_section in config_file.sections():
try:
header1, gsi_key, header2, table_key = current_section.split(" ")
except ValueError:
continue
if header1 != "gsi:":
continue
if table_key not in table_config["tables"]:
print("No table configuration matching {0} found.".format(table_key))
sys.exit(1)
if "gsis" not in table_config["tables"][table_key]:
table_config["tables"][table_key]["gsis"] = {}
table_config["tables"][table_key]["gsis"][gsi_key] = ordereddict(
default_options.items() + __parse_options(config_file, current_section, TABLE_CONFIG_OPTIONS).items()
)
return ordereddict(global_config.items() + logging_config.items() + table_config.items())
