def dois_from_scholar_urls():
sh = SciHub()
no_doi_found_file = data_path + 'no_doi_found.txt'
doi_not_in_scihub_file = data_path + 'doi_not_in_scihub.txt'
no_doi_found = []
doi_not_in_scihub = []
shit_counter = 0
pattern = re.compile('10.\d{4,9}/[-._;()/:A-Z0-9]+$')
articles = get_artices_from_googlescholar(phrase='emotional geographies',
max_num_hits=100)
print('num hits %d' % len(articles))
for art in articles:
url = art.attrs['url'][0]
if any([p in url for p in ['https://books.google.com/books']]):
shit_counter += 1
continue
# url = 'https://journals.sagepub.com/doi/abs/10.1177/0891243287001002002'
dois = [doi for doi in pattern.findall(url)]
if len(dois) > 0:
print(dois)
for doi in dois:
try:
sh.download(doi, destination=data_path)
print('got: %s' % doi)
except:
doi_not_in_scihub.append(doi)
else:
no_doi_found.append(url)
data_io.write_to_file(no_doi_found_file, no_doi_found, 'ab')
data_io.write_to_file(doi_not_in_scihub_file, doi_not_in_scihub, 'ab')
print('shit: %d' % shit_counter)
def write_brat_annotation(doc, path):
doc_name, text, spans, relations, attributes, notes = build_brat_lines(doc)
data_io.write_to_file(path + '/' + doc_name + '.txt', [text])
lines = [span_to_ann_line(d, text) for d in spans]
lines += [to_rel_ann_line(d) for d in relations]
lines += [to_attr_ann_line(d) for d in attributes]
lines += [to_notes_ann_line(d) for d in notes]
ann_file = path + '/' + doc_name + '.ann'
data_io.write_to_file(ann_file, lines)
return ann_file
def convert_pdfs_to_text(path_pdfs, path_texts):
if not os.path.isdir(path_texts):
os.mkdir(path_texts)
for i, file in enumerate(os.listdir(path_pdfs)):
try:
with open(path_pdfs + '/' + file, "rb") as f:
pdf = pdftotext.PDF(f)
except:
continue
pages = [page for page in pdf]
data_io.write_to_file(path_texts + '/' + file.replace('.pdf', '.txt'),
pages)
if i % 100 == 0:
sys.stdout.write('\r%d' % i)
def process_file(file):
with open(file) as f:
xml = f.read()
soup = BeautifulSoup(xml)
raw_paragraphs = soup.find_all('p')
refs = soup.find_all('ref')
formula = soup.find_all('formula')
list = soup.find_all('list')
shit = [refs, formula, list]
paragraphs = [
' '.join(
[str(x) for x in paragraph if not any(x in sh for sh in shit)])
for paragraph in raw_paragraphs
]
data_io.write_to_file(
processed_path + '/' +
os.path.split(file)[1].replace('.xml', '_processed.txt'),
paragraphs)
nlp = spacy.load('en')
# Add neural coref to SpaCy's pipe
import neuralcoref
neuralcoref.add_to_pipe(nlp)
path = '/home/tilo/code/misc/clef2019-factchecking-task1/data/training'
dump_path = '/home/tilo/data/coref'
debate_lines = get_Clef2019_data(path)
for k, batch in enumerate(
iterable_to_batches([d['utterance'] for d in debate_lines],
batch_size=32)):
k += 1
text = ' '.join(batch)
doc = nlp(text)
data_io.write_to_file(dump_path + '/text_%d.txt' % k, [text])
m_id = [0]
def get_inc_m_id():
val = m_id[0]
m_id[0] += 1
return val
standoff_corefs = []
for cluster_id, cluster in enumerate(doc._.coref_clusters):
mentions = [(s.start_char, s.end_char, s.text) for s in cluster]
cluster_mention = (get_inc_m_id(), cluster.main.start_char,
cluster.main.end_char, cluster.main.text)
mentions = [
(get_inc_m_id(), start, end, text)
for start, end, text in mentions if text != cluster.main.text
retmax=1000000)
search_results = Entrez.read(handle)
ids = search_results["IdList"]
num_hits = len(ids)
print("Found %d citations" % num_hits)
num_to_dump = min(num_to_dump, num_hits)
webenv = search_results["WebEnv"]
query_key = search_results["QueryKey"]
for start in range(0, num_to_dump, batch_size):
print(start)
fetch_handle = Entrez.efetch(
db=database,
# rettype="xml",
# retmode="xml",
retstart=start,
retmax=batch_size,
webenv=webenv,
query_key=query_key)
sleep(0.3)
data = fetch_handle.read()
# text = html2text.html2text(data)
# print(text)
fetch_handle.close()
yield data
if __name__ == '__main__':
data_io.write_to_file('./from_pubmed.xmls', pubmed_artice_generator())
for query_keyword in ['Geschlecht', 'Eine', 'theorie',
'theory']: #'women','Frau'
for page_number in range(1, 200000):
while True:
try:
filenames = get_pdf_urls(query_keyword, page_number)
break
except Exception:
print('retrying query: %s; page: %d' %
(query_keyword, page_number))
time.sleep(5)
if len(filenames) == 0:
break
new_urls = [
p for p in filenames if p not in already_downloaded_urls
]
print('found %d new urls' % len(new_urls))
already_downloaded_urls.extend(new_urls)
for i, p in enumerate(new_urls):
os.system('wget -N -b -q -P %s %s' % (data_dir, p))
time.sleep(0.2)
sys.stdout.write('\r')
sys.stdout.flush()
sys.stdout.write('\rnum-current page: %d' % i)
sys.stdout.flush()
data_io.write_to_file(urls_file, new_urls, 'ab')
if len(already_downloaded_urls) % 100 == 0:
print('got %d files' % len(already_downloaded_urls))
return s
if __name__ == '__main__':
data_path = '/home/tilo/code/NLP/scisci_nlp/data/scierc_data/json/'
data = getting_scierc_tagged_data(data_path + 'dev.json')
# for text,spans in data:
# for start,end,label in spans:
# print('%s - %s'%(text[start:end],label))
path = './dingens'
if not os.path.isdir(path):
os.mkdir(path)
for doc_name, text, spans, relations in data:
data_io.write_to_file(path + '/' + doc_name + '.txt', [text])
startend2Id = {
'%d-%d' % (s, e): i
for i, (s, e, l) in enumerate(spans)
}
def get_mention_id(s, e):
return startend2Id['%d-%d' % (s, e)]
lines = [
span_to_ann_line(get_mention_id(s, e), s, e, l, text[s:e])
for s, e, l in spans
]
lines += [
to_rel_ann_line(i, get_mention_id(s1, e1), get_mention_id(s2, e2),
label)
optimizer.zero_grad()
loss = loss_fun(model,batch).mean() # mean() for Data Parallelism
loss.backward()
optimizer.step()
return loss.item()
pytorch_methods.train(train_on_batch,
data_iter,
cfg.n_epochs,
tol=cfg.tol,
patience=cfg.patience,
verbose=True)
if __name__ == '__main__':
path = '/home/tilo/data'
data_path = path+'/ml_nlp_parsed'
lines_g = data_io.read_lines_from_files(data_path,limit=100)
def line_generator():
line_len=100
c=0
for k,line in enumerate(lines_g):
for s in range(0,len(line)-line_len,line_len):
yield line[s:(s+line_len)].replace('\n','')
c+=1
if c%10==0:
yield '\n'
data_io.write_to_file('/tmp/text.txt',line_generator())
data_file = '/tmp/text.txt'
fire.Fire(main)
# lines_g = (d['pdf_full_text'].replace('\n', '') for d in data_io.read_jsons_from_file('%s/arxiv.jsonl.gz' % path)
# if isinstance(d['pdf_full_text'],str) and len(d['pdf_full_text']) > 10)
# lines_g = data_io.read_lines_from_files(data_path)
lines_g = build_opensubtitles_lines_generator(data_source)
corpus_path = '%s/de_corpus' % path
# corpus_path = '%s/corpus' % '/tmp'
build_train_valid_test_files = True
if build_train_valid_test_files:
if os.path.isdir(corpus_path):
shutil.rmtree(corpus_path)
if not os.path.exists(corpus_path): os.mkdir(corpus_path)
num_train_docs = 20_000
train_file = '%s/train.txt'% corpus_path
data_io.write_to_file(train_file, (next(lines_g) for k in range(num_train_docs)))
train_split_folder = corpus_path + '/train'
if not os.path.exists(train_split_folder): os.mkdir(train_split_folder)
train_lines_g = data_io.read_lines(train_file)
num_train_docs = sum((1 for line in data_io.read_lines(train_file)))
lines_per_split = 100
num_train_splits = int(numpy.ceil(num_train_docs / lines_per_split))
for k in range(num_train_splits):
split_file = train_split_folder + '/train_split_%d'%k
data_io.write_to_file(split_file, (next(train_lines_g) for k in range(lines_per_split)))
# shutil.copy(train_file, train_split_folder + '/train_split_%d'%k)
data_io.write_to_file('%s/valid.txt' % corpus_path, (next(lines_g) for k in range(lines_per_split)))
data_io.write_to_file('%s/test.txt' % corpus_path, (next(lines_g) for k in range(lines_per_split)))
is_forward_lm = True