"""
if isinstance(m, nn.Module):
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight,
mode='fan_out',
nonlinearity='relu')
else: # elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
nn.init.constant_(m.bias, 0)
################################################################################
# MAIN
if __name__ == '__main__':
parser = argparser({'desc': "Utils for PyTorch: pytorch_utils.py"})
parser.add_argument(
'--rate_by_epoch',
help='Test a learning rate against the rate_by_epoch function',
required=False,
type=float,
default=0)
args = parser.parse_args()
if args.rate_by_epoch:
orig = args.rate_by_epoch
for i in range(50):
epoch = i + 1
output = learning_rate_by_epoch(epoch, args.rate_by_epoch)
print("E %d Lr: %0.8f" % (epoch, output))
if make_series:
X = pd.Series(X)
elif isinstance(X, scipy.sparse.csr_matrix):
X = pd.DataFrame(X.toarray())
else:
X = pd.DataFrame(X)
return X
################################################################################
## MAIN ##
if __name__ == '__main__':
try:
parser = argparser({
'desc':
"Tools to model patterns in text using scikit-learn, keras, and TF: model.py"
})
# -- Tool-specific command-line args may be added here
args = parser.parse_args() # Get inputs and options
print(__doc__)
except Exception as e:
print(__doc__)
err([], {'exception': e, 'exit': True})
################################################################################
################################################################################
Used to get the lemma, but may also be used for future synset generators
"""
output = []
synsets = wordnet.synsets(lemma)
for synset in synsets:
for sl in synset.lemmas():
output.append(sl.name())
return output
################################################################################
# MAIN
if __name__ == '__main__':
parser = argparser({'desc': "Some general NLP tasks: nlp.py"})
parser.add_argument('--synset',
help='Get the WordNet synonym set for a word',
required=False,
type=str)
args = parser.parse_args()
text = ''
if args.file:
for file in args.file:
text += read_file(file)
elif args.str:
text = ' '.join(args.str)
self.first_shard_id,
"LATEST")["ShardIterator"]
while True:
t0 = time.time()
out = kinesis.get_records(shard_it, limit=limit)
shard_it = out["NextShardIterator"]
yield out
t1 = time.time()
if t1 - t0 < 0.2:
time.sleep(0.2 - (t1 - t0))
################################################################################
# MAIN
if __name__ == '__main__':
parser = argparser(
{'desc': "Helper functions for AWS Kinesis: kinesis_utils.py"})
parser.add_argument('--describe_stream',
help='Describe a specific Kinesis stream',
required=False,
type=str)
args = parser.parse_args() # Get inputs and options
if args.describe_stream:
describe_stream(args.describe_stream)
################################################################################
input_dir : str
Directory where each file will be read
Returns
-------
array of DataFrame (4 of them: x_train, x_test, y_train, y_test)
"""
files = read_dir(input_dir, {'fullpath': True})
return default_read_data_files(files)
################################################################################
## MAIN ##
if __name__ == '__main__':
try:
parser = argparser(
{'desc': "Tools to manage a training set: dataset.py"})
# -- Tool-specific command-line args may be added here
args = parser.parse_args() # Get inputs and options
print(__doc__)
except Exception as e:
print(__doc__)
err([], {'exception': e, 'exit': True})
################################################################################
################################################################################
def findall_offsets(s, text):
"""
Find all char offsets of s in text. Return list of pairs
"""
spans = []
for m in re.finditer(s, text, flags=re.I):
spans.append(m.span())
return spans
################################################################################
### MAIN
if __name__ == '__main__':
parser = argparser({'desc': "normalize.py"})
parser.add_argument('--ascii_fold',
help='Fold special characters to ASCII',
required=False,
type=str)
args = parser.parse_args() # Get inputs and options
if args.str:
print(
normalize(args.str[0],
options={
'verbose': args.verbose,
'remove_citations': True
}))
elif args.ascii_fold:
if not doc['id'] in seen:
seen.add(doc['id'])
doc['score'] = phrase_similarity(line, doc['name'])
docs.append(doc)
docs = sorted(docs, reverse=True, key=lambda x: x['score'])
assert (isinstance(docs, list))
return docs
################################################################################
# MAIN
if __name__ == '__main__':
parser = argparser(
{'desc': "Helper functions for Elasticsearch: elastic_utils.py"})
# -- Tool-specific command-line args may be added here
parser.add_argument('--list',
help='List all indices',
required=False,
action='store_true')
parser.add_argument('--get',
help='Get all docs from index',
required=False,
action='store_true')
parser.add_argument('--delete',
help='Get all docs from index',
required=False,
action='store_true')
parser.add_argument('--quiet',
"""
Run a given query against a cursor
"""
cur.execute(query)
for row in cur.fetchall():
print(row)
return "done"
################################################################################
# MAIN
if __name__ == '__main__':
parser = argparser({'desc': "Helper functions for MySQL: mysql_utils.py"})
parser.add_argument('--query', help='Query', required=False, type=str)
parser.add_argument('--user', help='Username', required=False, type=str)
parser.add_argument('--passwd', help='Password', required=False, type=str)
parser.add_argument('--db', help='Database name', required=False, type=str)
args = parser.parse_args() # Get inputs and options
if args.query:
cur = mysql_connect(host=args.host,
db=args.db,
user=args.user,
passwd=args.passwd)
print(mysql_query(cur, args.query))
################################################################################
for node in G.nodes:
print(node)
print("\nRELATIONSHIPS:")
for relationship in G.relationships:
rels = []
for n in relationship.nodes:
rels.append('(%s)' % list(n.labels)[0])
rel = '--'.join(rels)
print('\t', rel)
################################################################################
# MAIN
if __name__ == '__main__':
parser = argparser({'desc': "Utils for Neo4J: neo4j_utils.py"})
parser.add_argument('--query',
help='Query for Neo4J',
required=False,
type=str)
args = parser.parse_args()
if args.test:
test(args.query, args.host, args.port, args.user, args.passwd)
else:
print(__doc__)
################################################################################
################################################################################
"""
Load word vectors
"""
if file is None:
# file = os.environ['HOME'] + '/data/ConceptNet/numberbatch_en.pkl'
file = '..' + '/data/ConceptNet/numberbatch_en.pkl'
mult_vocab = deserialize(file)
vocab = mult_vocab['en']
return vocab
################################################################################
## MAIN ##
if __name__ == '__main__':
parser = argparser({'desc': "Document object: document.py"})
parser.add_argument('--vocab',
help='File with word embeddings',
required=False,
type=str)
parser.add_argument('--print_vocab',
help='Print out with word embeddings',
required=False,
action='store_true')
parser.add_argument('--trinary',
help='Build trinary tree',
required=False,
action='store_true')
args = parser.parse_args() # Get inputs and options
vocab = load_vocab(args.vocab)
"""
for feature in features_to_drop:
try:
parsed.pop(feature, None)
underlying.remove(feature)
except:
pass
return parsed, underlying
################################################################################
# MAIN
if __name__ == '__main__':
try:
parser = argparser({'desc': "Tools to vectorize text: vectorizer.py"})
# -- Tool-specific command-line args may be added here
args = parser.parse_args() # Get inputs and options
print(__doc__)
except Exception as e:
print(__doc__)
err([], {'exception': e, 'exit': True})
################################################################################
################################################################################
n += 1
if n % 1000 == 0:
print("n =", n)
except StopIteration:
pass
except Exception as e:
err([], {'exception': e})
return n
################################################################################
# MAIN
if __name__ == '__main__':
parser = argparser({'desc': "Helper functions for Mongo: mongo_utils.py"})
# -- Tool-specific command-line args may be added here
parser.add_argument('--list',
help='List all databases',
required=False,
action='store_true')
args = parser.parse_args() # Get inputs and options
if args.list:
test(db_name='db', collection_name='collection') # just dummy names
################################################################################
################################################################################
