""" if isinstance(m, nn.Module): if isinstance(m, nn.Conv2d): nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') else: # elif isinstance(m, nn.BatchNorm2d): nn.init.constant_(m.weight, 1) nn.init.constant_(m.bias, 0) ################################################################################ # MAIN if __name__ == '__main__': parser = argparser({'desc': "Utils for PyTorch: pytorch_utils.py"}) parser.add_argument( '--rate_by_epoch', help='Test a learning rate against the rate_by_epoch function', required=False, type=float, default=0) args = parser.parse_args() if args.rate_by_epoch: orig = args.rate_by_epoch for i in range(50): epoch = i + 1 output = learning_rate_by_epoch(epoch, args.rate_by_epoch) print("E %d Lr: %0.8f" % (epoch, output))
if make_series: X = pd.Series(X) elif isinstance(X, scipy.sparse.csr_matrix): X = pd.DataFrame(X.toarray()) else: X = pd.DataFrame(X) return X ################################################################################ ## MAIN ## if __name__ == '__main__': try: parser = argparser({ 'desc': "Tools to model patterns in text using scikit-learn, keras, and TF: model.py" }) # -- Tool-specific command-line args may be added here args = parser.parse_args() # Get inputs and options print(__doc__) except Exception as e: print(__doc__) err([], {'exception': e, 'exit': True}) ################################################################################ ################################################################################
Used to get the lemma, but may also be used for future synset generators """ output = [] synsets = wordnet.synsets(lemma) for synset in synsets: for sl in synset.lemmas(): output.append(sl.name()) return output ################################################################################ # MAIN if __name__ == '__main__': parser = argparser({'desc': "Some general NLP tasks: nlp.py"}) parser.add_argument('--synset', help='Get the WordNet synonym set for a word', required=False, type=str) args = parser.parse_args() text = '' if args.file: for file in args.file: text += read_file(file) elif args.str: text = ' '.join(args.str)
self.first_shard_id, "LATEST")["ShardIterator"] while True: t0 = time.time() out = kinesis.get_records(shard_it, limit=limit) shard_it = out["NextShardIterator"] yield out t1 = time.time() if t1 - t0 < 0.2: time.sleep(0.2 - (t1 - t0)) ################################################################################ # MAIN if __name__ == '__main__': parser = argparser( {'desc': "Helper functions for AWS Kinesis: kinesis_utils.py"}) parser.add_argument('--describe_stream', help='Describe a specific Kinesis stream', required=False, type=str) args = parser.parse_args() # Get inputs and options if args.describe_stream: describe_stream(args.describe_stream) ################################################################################
input_dir : str Directory where each file will be read Returns ------- array of DataFrame (4 of them: x_train, x_test, y_train, y_test) """ files = read_dir(input_dir, {'fullpath': True}) return default_read_data_files(files) ################################################################################ ## MAIN ## if __name__ == '__main__': try: parser = argparser( {'desc': "Tools to manage a training set: dataset.py"}) # -- Tool-specific command-line args may be added here args = parser.parse_args() # Get inputs and options print(__doc__) except Exception as e: print(__doc__) err([], {'exception': e, 'exit': True}) ################################################################################ ################################################################################
def findall_offsets(s, text): """ Find all char offsets of s in text. Return list of pairs """ spans = [] for m in re.finditer(s, text, flags=re.I): spans.append(m.span()) return spans ################################################################################ ### MAIN if __name__ == '__main__': parser = argparser({'desc': "normalize.py"}) parser.add_argument('--ascii_fold', help='Fold special characters to ASCII', required=False, type=str) args = parser.parse_args() # Get inputs and options if args.str: print( normalize(args.str[0], options={ 'verbose': args.verbose, 'remove_citations': True })) elif args.ascii_fold:
if not doc['id'] in seen: seen.add(doc['id']) doc['score'] = phrase_similarity(line, doc['name']) docs.append(doc) docs = sorted(docs, reverse=True, key=lambda x: x['score']) assert (isinstance(docs, list)) return docs ################################################################################ # MAIN if __name__ == '__main__': parser = argparser( {'desc': "Helper functions for Elasticsearch: elastic_utils.py"}) # -- Tool-specific command-line args may be added here parser.add_argument('--list', help='List all indices', required=False, action='store_true') parser.add_argument('--get', help='Get all docs from index', required=False, action='store_true') parser.add_argument('--delete', help='Get all docs from index', required=False, action='store_true') parser.add_argument('--quiet',
""" Run a given query against a cursor """ cur.execute(query) for row in cur.fetchall(): print(row) return "done" ################################################################################ # MAIN if __name__ == '__main__': parser = argparser({'desc': "Helper functions for MySQL: mysql_utils.py"}) parser.add_argument('--query', help='Query', required=False, type=str) parser.add_argument('--user', help='Username', required=False, type=str) parser.add_argument('--passwd', help='Password', required=False, type=str) parser.add_argument('--db', help='Database name', required=False, type=str) args = parser.parse_args() # Get inputs and options if args.query: cur = mysql_connect(host=args.host, db=args.db, user=args.user, passwd=args.passwd) print(mysql_query(cur, args.query)) ################################################################################
for node in G.nodes: print(node) print("\nRELATIONSHIPS:") for relationship in G.relationships: rels = [] for n in relationship.nodes: rels.append('(%s)' % list(n.labels)[0]) rel = '--'.join(rels) print('\t', rel) ################################################################################ # MAIN if __name__ == '__main__': parser = argparser({'desc': "Utils for Neo4J: neo4j_utils.py"}) parser.add_argument('--query', help='Query for Neo4J', required=False, type=str) args = parser.parse_args() if args.test: test(args.query, args.host, args.port, args.user, args.passwd) else: print(__doc__) ################################################################################ ################################################################################
""" Load word vectors """ if file is None: # file = os.environ['HOME'] + '/data/ConceptNet/numberbatch_en.pkl' file = '..' + '/data/ConceptNet/numberbatch_en.pkl' mult_vocab = deserialize(file) vocab = mult_vocab['en'] return vocab ################################################################################ ## MAIN ## if __name__ == '__main__': parser = argparser({'desc': "Document object: document.py"}) parser.add_argument('--vocab', help='File with word embeddings', required=False, type=str) parser.add_argument('--print_vocab', help='Print out with word embeddings', required=False, action='store_true') parser.add_argument('--trinary', help='Build trinary tree', required=False, action='store_true') args = parser.parse_args() # Get inputs and options vocab = load_vocab(args.vocab)
""" for feature in features_to_drop: try: parsed.pop(feature, None) underlying.remove(feature) except: pass return parsed, underlying ################################################################################ # MAIN if __name__ == '__main__': try: parser = argparser({'desc': "Tools to vectorize text: vectorizer.py"}) # -- Tool-specific command-line args may be added here args = parser.parse_args() # Get inputs and options print(__doc__) except Exception as e: print(__doc__) err([], {'exception': e, 'exit': True}) ################################################################################ ################################################################################
n += 1 if n % 1000 == 0: print("n =", n) except StopIteration: pass except Exception as e: err([], {'exception': e}) return n ################################################################################ # MAIN if __name__ == '__main__': parser = argparser({'desc': "Helper functions for Mongo: mongo_utils.py"}) # -- Tool-specific command-line args may be added here parser.add_argument('--list', help='List all databases', required=False, action='store_true') args = parser.parse_args() # Get inputs and options if args.list: test(db_name='db', collection_name='collection') # just dummy names ################################################################################ ################################################################################