Esempio n. 1
0
def write_parsed(input_dir, output_dir, lps):

    print("Copying parsed files to " + output_dir + ' ...')

    path_tgt = usr(output_dir + '/' + 'tgt' + '.parse')
    path_ref = usr(output_dir + '/' + 'ref' + '.parse')

    with codecs.open(path_tgt, 'w', 'utf8') as output_tgt:
        with codecs.open(path_ref, 'w', 'utf8') as output_ref:

            counter_tgt = 0
            counter_ref = 0

            for dataset in sorted(os.listdir(input_dir + '/' + 'references')):
                if dataset.startswith('.'):
                    continue

                for lp in sorted(os.listdir(input_dir + '/' + 'system-outputs' + '/' + dataset)):
                    if lp.startswith('.'):
                        continue
                    if lp not in lps:
                        continue

                    with codecs.open(wmt.reference_path(input_dir, dataset, lp) + '.out', 'r', 'utf8') as input_ref:
                        ref_lines = input_ref.readlines()

                    for sys_file_name in sorted(os.listdir(input_dir + '/' + 'system-outputs' + '/' + dataset + '/' + lp)):
                        if sys_file_name.startswith('.'):
                            continue

                        with codecs.open(input_dir + '/' + 'system-outputs' + '/' + dataset + '/' + lp + '/' + sys_file_name, 'r', 'utf8') as input_sys:

                            for line in input_sys.readlines():
                                if line.startswith('Sentence #'):
                                    counter_tgt += 1
                                    output_tgt.write(wmt.substitute_line_number(line, counter_tgt))
                                else:
                                    output_tgt.write(line)

                            for line in ref_lines:
                                if line.startswith('Sentence #'):
                                    counter_ref += 1
                                    output_ref.write(wmt.substitute_line_number(line, counter_ref))
                                else:
                                    output_ref.write(line)
Esempio n. 2
0
    def write_dataset(self, parsed=False, verbose=False):

        print("Copying dataset to " + self.cfg.get('Data', 'working_dir') + ' ...')

        path_tgt = usr(self.cfg.get('Data', 'working_dir') + '/' + 'tgt.txt')
        path_ref = usr(self.cfg.get('Data', 'working_dir') + '/' + 'ref.txt')

        counter_tgt = 0
        counter_ref = 0

        with codecs.open(path_tgt, 'w', 'utf8') as output_tgt:
            with codecs.open(path_ref, 'w', 'utf8') as output_ref:

                for dataset in self.datasets:
                    for lp in sorted(dataset.system_names.keys()):

                        with codecs.open(wmt.reference_path(self.dir, dataset.name, lp), 'r', 'utf8') as input_ref:
                            ref_lines = input_ref.readlines()

                        for sys_name in dataset.system_names[lp]:
                            counter_sys = 0
                            with codecs.open(wmt.system_path(self.dir, dataset.name, lp, sys_name), 'r', 'utf8') as input_sys:
                                for line in input_sys.readlines():
                                    counter_tgt += 1
                                    counter_sys += 1
                                    if parsed and line.startswith('Sentence #'):
                                        output_tgt.write(wmt.substitute_line_number(line, counter_tgt))
                                    else:
                                        if verbose:
                                            output_tgt.write('{}\t{}\t{}\t{}\t{}'.format(dataset.name,
                                                                                         lp,
                                                                                         sys_name,
                                                                                         counter_sys,
                                                                                         line))
                                        else:
                                            output_tgt.write(line)

                                for line in ref_lines:
                                    counter_ref += 1
                                    if parsed and line.startswith('Sentence #'):
                                        output_ref.write(wmt.substitute_line_number(line, counter_ref))
                                    else:
                                        output_ref.write(line)
Esempio n. 3
0
def read_buffer():
    global buffer_data
    with open(usr(f'~/augentbot-{VERSION}/data/buffer.txt'),
              encoding='utf-16') as file:
        return file.readlines()
Esempio n. 4
0
def read_corpus():
    global corpus_data
    with open(usr(f'~/augentbot-{VERSION}/data/corpus.txt'),
              'r',
              encoding='utf_16') as file:
        return file.read()
Esempio n. 5
0
def read_coll():
    global coll_data
    with open(usr(f'~/augentbot-{VERSION}/data/data.txt'),
              encoding='utf-16') as file:
        return file.read()
Esempio n. 6
0
#! python3.6

import _io
from os.path import expanduser as usr
from typing import List

import tweepy

with open('VERSION') as file:
    VERSION: str = file.read()

with open(usr(
        f'~/augentbot-{VERSION}/credentials/twitter_consumer_key')) as file:
    TWITTER_CONSUMER_KEY: str = file.read()
with open(usr(
        f'~/augentbot-{VERSION}/credentials/twitter_consumer_secret')) as file:
    TWITTER_CONSUMER_SECRET: str = file.read()
with open(usr(
        f'~/augentbot-{VERSION}/credentials/twitter_access_token')) as file:
    TWITTER_ACCESS_TOKEN: str = file.read()
with open(usr(f'~/augentbot-{VERSION}/credentials/twitter_access_token_secret')
          ) as file:
    TWITTER_ACCESS_SECRET: str = file.read()

HOST_NAME: str = '_jfde'

auth: tweepy.OAuthHandler = tweepy.OAuthHandler(TWITTER_CONSUMER_KEY,
                                                TWITTER_CONSUMER_SECRET)
auth.set_access_token(TWITTER_ACCESS_TOKEN, TWITTER_ACCESS_SECRET)

api: tweepy.API = tweepy.API(auth)
Esempio n. 7
0
from distutils.core import setup
from distutils.extension import Extension
from Cython.Build import cythonize
import numpy

import os
from os.path import expanduser as usr
from os.path import join as pjoin

import subprocess

HALIDE_PATH = usr('~/src/halide-binary')

os.environ['LD_LIBRARY_PATH'] = pjoin(HALIDE_PATH, 'bin')

cmd = 'g++ h_brightness.cpp -g -std=c++11 ' \
      '-I {} -L {} -lHalide -lpthread -ldl -o h_brightness'
cmd = cmd.format(pjoin(HALIDE_PATH, 'include'),
                 pjoin(HALIDE_PATH, 'bin'))

print(cmd)
#subprocess.call(cmd.split())
#subprocess.call('./h_brightness')

ext_modules = cythonize(
    ["brightness.pyx", "h_brightness.cpp"],
    language="c++",
    )

setup(ext_modules=ext_modules)
Esempio n. 8
0
from distutils.core import setup
from distutils.extension import Extension
from Cython.Build import cythonize
import numpy

import os
from os.path import expanduser as usr
from os.path import join as pjoin

import subprocess

HALIDE_PATH = usr('~/src/halide-binary')

os.environ['LD_LIBRARY_PATH'] = pjoin(HALIDE_PATH, 'bin')

cmd = 'g++ h_brightness.cpp -g -std=c++11 ' \
      '-I {} -L {} -lHalide -lpthread -ldl -o h_brightness'
cmd = cmd.format(pjoin(HALIDE_PATH, 'include'), pjoin(HALIDE_PATH, 'bin'))

print(cmd)
#subprocess.call(cmd.split())
#subprocess.call('./h_brightness')

ext_modules = cythonize(
    ["brightness.pyx", "h_brightness.cpp"],
    language="c++",
)

setup(ext_modules=ext_modules)

subprocess.call(