def load_all(dataset_dir, doc_ids=None, filter_types=None, filter_senses=None):
"""Load whole CoNLL16st dataset by document id."""
# load all provided files untouched
parses = load_parses(dataset_dir, doc_ids=doc_ids)
doc_ids = sorted(parses.keys())
raws = load_raws(dataset_dir, doc_ids=doc_ids)
relations_gold = load_relations_gold(dataset_dir, doc_ids=doc_ids, with_senses=True, filter_types=filter_types, filter_senses=filter_senses)
if relations_gold:
relationsnos_gold = relations_gold
else:
relationsnos_gold = load_relations_gold(dataset_dir, doc_ids=doc_ids, with_senses=False, filter_types=filter_types, filter_senses=filter_senses)
# extract data by document id and token id
words = get_words(parses)
pos_tags = get_pos_tags(parses)
word_metas = get_word_metas(parses, raws)
# extract data by document id and token id pairs
dependencies = get_dependencies(parses)
# extract data by document id
parsetrees = get_parsetrees(parses)
# extract data by relation id
rel_parts = get_rel_parts(relationsnos_gold)
rel_ids = sorted(rel_parts.keys())
rel_types = get_rel_types(relations_gold)
rel_senses = get_rel_senses(relations_gold)
# add extra fields
add_relation_tags(word_metas, rel_types, rel_senses)
return doc_ids, words, word_metas, pos_tags, dependencies, parsetrees, rel_ids, rel_parts, rel_types, rel_senses, relations_gold
def populate():
print "Getting the list of all packages."
client = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
packages = client.list_packages()
print "Ridiculous! Found %i packages." % len(packages)
print "Ingesting %i packages." % len(packages)
with cm(shelve.open(fname, writeback=True)) as d:
d['packages'] = d.get('packages', {})
for package in packages:
if package not in d['packages']:
result = ingest_package(package)
try:
requires = dependencies.get_dependencies(result['name'])
print result['name'], "depends on %r" % requires
except Exception as e:
print str(e)
continue
d['packages'][result['name']] = result
d['packages'][result['name']]['dependencies'] = requires
d.sync()
else:
print "Skipping ", package
print "Complete!"
def load_all(dataset_dir, doc_ids=None, filter_types=None, filter_senses=None):
"""Load whole CoNLL16st dataset by document id."""
# load all provided files untouched
parses = load_parses(dataset_dir, doc_ids=doc_ids)
doc_ids = sorted(parses.keys())
raws = load_raws(dataset_dir, doc_ids=doc_ids)
relations_gold = load_relations_gold(dataset_dir,
doc_ids=doc_ids,
with_senses=True,
filter_types=filter_types,
filter_senses=filter_senses)
if relations_gold:
relationsnos_gold = relations_gold
else:
relationsnos_gold = load_relations_gold(dataset_dir,
doc_ids=doc_ids,
with_senses=False,
filter_types=filter_types,
filter_senses=filter_senses)
# extract data by document id and token id
words = get_words(parses)
pos_tags = get_pos_tags(parses)
word_metas = get_word_metas(parses, raws)
# extract data by document id and token id pairs
dependencies = get_dependencies(parses)
# extract data by document id
parsetrees = get_parsetrees(parses)
# extract data by relation id
rel_parts = get_rel_parts(relationsnos_gold)
rel_ids = sorted(rel_parts.keys())
rel_types = get_rel_types(relations_gold)
rel_senses = get_rel_senses(relations_gold)
# add extra fields
add_relation_tags(word_metas, rel_types, rel_senses)
return doc_ids, words, word_metas, pos_tags, dependencies, parsetrees, rel_ids, rel_parts, rel_types, rel_senses, relations_gold
import dependencies
map=dependencies.get_dependencies("packages")
print "graph deps {"
for pkg in map.keys():
for dep in map[pkg]:
print "\t%s -> %s;" % (pkg,dep)
print "}"
def install_deps():
# Pretty much just `yum -y install <dependencies>`
yum = ['/usr/bin/yum', '-e1', '-y', 'install']
__call(yum, dependencies.get_dependencies(),
fail_message="Unable to install required packages.")
