def test_5_harvest_multi(self):
'''test harvesting in multiprocessing'''
infos, _ = harvest.get_ds_infos('.*')
destdir = '/gridgroup/cms/cbernet/unittests/multi'
harvest.harvest(infos, destdir, ntgzs=2, nworkers=2, delete='y')
results = os.listdir(destdir)
self.assertEqual(len(results), 2)
def test_6_harvest_sequential(self):
'''test that harvesting can be done in steps'''
infos, _ = harvest.get_ds_infos('.*')
destdir = '/gridgroup/cms/cbernet/unittests/multi'
infos1 = infos[:1]
infos2 = infos[1:2]
harvest.harvest(infos1, destdir, ntgzs=2, nworkers=1, delete='y')
harvest.harvest(infos2, destdir, ntgzs=2, nworkers=1, delete='n')
results = os.listdir(destdir)
self.assertEqual(len(results), 2)
def get_f0(self,
fs: int,
x: np.ndarray,
f0_method: str = 'harvest',
f0_floor: int = 71,
f0_ceil: int = 800,
channels_in_octave: int = 2,
target_fs: int = 4000,
frame_period: int = 5) -> tuple:
"""
:param fs: sample frequency
:param x: signal
:param f0_method: f0 extraction method: dio, harvest
:param f0_floor: smallest f0
:param f0_ceil: largest f0
:param channels_in_octave:
:param target_fs: downsampled frequency for f0 extraction
:param frame_period: in ms
:return:
"""
if f0_method == 'dio':
source = dio(x, fs, f0_floor, f0_ceil, channels_in_octave,
target_fs, frame_period)
source['f0'] = stonemask(x, fs, source['temporal_positions'],
source['f0'])
elif f0_method == 'harvest':
source = harvest(x, fs, f0_floor, f0_ceil, frame_period)
# elif f0_method == 'swipe':
# source = swipe(fs, x, plim=[f0_floor, f0_ceil],sTHR=0.3)
else:
raise Exception
return source['temporal_positions'], source['f0'], source[
'vuv'] # or a dict
def test_sort_shallowfirst(self):
data = join(harvest('test_data', ['.txt'], glob_sort='shallowfirst'))
self.assertEqual(
"""contents of test_data/a.txt
contents of test_data/b.txt
contents of test_data/y.txt
contents of test_data/z.txt
contents of test_data/subdir1/a.txt
contents of test_data/subdir1/b.txt
contents of test_data/subdir1/c.txt
contents of test_data/subdir2/a.txt
contents of test_data/subdir2/b.txt
contents of test_data/subdir2/z.txt
contents of test_data/subdir2/subdir3/a.txt
""", data)
def test_paths_only(self):
data = join(harvest('test_data', ['.txt'], paths_only=True))
self.assertEqual(
"""test_data/a.txt
test_data/b.txt
test_data/subdir1/a.txt
test_data/subdir1/b.txt
test_data/subdir1/c.txt
test_data/subdir2/a.txt
test_data/subdir2/b.txt
test_data/subdir2/subdir3/a.txt
test_data/subdir2/z.txt
test_data/y.txt
test_data/z.txt
""", data)
def test_all_files(self):
data = join(harvest('test_data', ['.txt']))
self.assertEqual(
"""contents of test_data/a.txt
contents of test_data/b.txt
contents of test_data/subdir1/a.txt
contents of test_data/subdir1/b.txt
contents of test_data/subdir1/c.txt
contents of test_data/subdir2/a.txt
contents of test_data/subdir2/b.txt
contents of test_data/subdir2/subdir3/a.txt
contents of test_data/subdir2/z.txt
contents of test_data/y.txt
contents of test_data/z.txt
""", data)
def get_spectrum(self,
fs: int,
x: np.ndarray,
f0_method: str = 'harvest',
f0_floor: int = 71,
f0_ceil: int = 800,
channels_in_octave: int = 2,
target_fs: int = 4000,
frame_period: int = 5,
fft_size=None) -> dict:
'''
This function extract pitch-synchronous WORLD spectrogram
:param fs: sampling frequency
:param x: signal (in float)
:param f0_method: dio, harvest #, swipe
:param f0_floor: f0 min
:param f0_ceil: f0 max
:param frame_period: frame shift
:param fft_size: fourier transform length
:param: channels_in_octave: channels per octave
:return:
'''
if f0_method == 'dio':
source = dio(x, fs, f0_floor, f0_ceil, channels_in_octave,
target_fs, frame_period)
source['f0'] = stonemask(x, fs, source['temporal_positions'],
source['f0'])
elif f0_method == 'harvest':
source = harvest(x, fs, f0_floor, f0_ceil, frame_period)
# elif f0_method == 'swipe':
# source = swipe(fs, x, plim=[f0_floor, f0_ceil],sTHR=0.3)
else:
raise Exception
filter = cheaptrick(x, fs, source, fft_size=fft_size)
return {
'f0': source['f0'],
'temporal_positions': source['temporal_positions'],
'fs': fs,
'ps spectrogram': filter['ps spectrogram'],
'spectrogram': filter['spectrogram']
}
def encode(self,
fs: int,
x: np.ndarray,
f0_method: str = 'harvest',
f0_floor: int = 71,
f0_ceil: int = 800,
channels_in_octave: int = 2,
target_fs: int = 4000,
frame_period: int = 5,
allowed_range: float = 0.1,
fft_size=None,
is_requiem: bool = False) -> dict:
'''
encode speech to excitation signal, f0, spectrogram
:param fs: sample frequency
:param x: signal
:param f0_method: f0 extraction method: dio, harvest
:param f0_floor: smallest f0
:param f0_ceil: largest f0
:param channels_in_octave: number of channels per octave
:param target_fs: downsampled frequency for f0 extraction
:param frame_period: in ms
:param allowed_range:
:param fft_size: length of Fourier transform
:return: a dictionary contains WORLD components
'''
print('enter encode')
if fft_size != None:
f0_floor = 3.0 * fs / fft_size
if f0_method == 'dio':
print('f0_method dio')
source = dio(x,
fs,
f0_floor=f0_floor,
f0_ceil=f0_ceil,
channels_in_octave=channels_in_octave,
target_fs=target_fs,
frame_period=frame_period,
allowed_range=allowed_range)
source['f0'] = stonemask(x, fs, source['temporal_positions'],
source['f0'])
elif f0_method == 'harvest':
print('f0_method harvest')
source = harvest(x,
fs,
f0_floor=f0_floor,
f0_ceil=f0_ceil,
frame_period=frame_period)
#elif f0_method == 'swipe':
# print ('f0_method swipe')
# source = swipe(fs, x, plim=[f0_floor, f0_ceil], sTHR=0.3)
else:
raise Exception
filter = cheaptrick(x, fs, source, fft_size=fft_size)
if is_requiem:
print('call d4cRequiem')
source = d4cRequiem(x, fs, source, fft_size=fft_size)
else:
print('call d4c')
source = d4c(x, fs, source, fft_size_for_spectrum=fft_size)
return {
'temporal_positions': source['temporal_positions'],
'vuv': source['vuv'],
'fs': filter['fs'],
'f0': source['f0'],
'aperiodicity': source['aperiodicity'],
'ps spectrogram': filter['ps spectrogram'],
'spectrogram': filter['spectrogram'],
'is_requiem': is_requiem
}
def test_specific_file_path_only(self):
data = join(harvest('test_data/a.txt', paths_only=True))
self.assertEqual("test_data/a.txt\n", data)
def test_specific_file(self):
data = join(harvest('test_data/a.txt'))
self.assertEqual("contents of test_data/a.txt\n", data)
solr_conn.add(**args)
break
except Exception as ex:
logging.error(
"An error has occurred trying to access Solr. Arguments passed to Solr and the traceback are below:")
logging.error(args)
logging.exception(ex)
logging.error("Sleeping for 1 minute to try again...")
time.sleep(60)
if __name__ == '__main__':
FORMAT = '%(asctime)-15s %(message)s'
logging.basicConfig(level=logging.INFO, format=FORMAT)
config = ConfigParser.ConfigParser()
config.read('harvest.cfg')
solr_uri = config.get("harvest", "solr_uri")
solr_conn = solr.SolrConnection(solr_uri)
harvest(config.get("harvest", "article_meta_uri"),
"article",
config.get("harvest", "data_source_name"),
"article_data",
config.get("harvest", "pg_shared_folder_output"),
config.get("harvest", "pg_shared_folder_input"),
(add_update_article_entry, delete_article_entry)
)
solr_conn.commit()
def parse(text):
print "----------------------------------"
print text
#harvest.harvest(text, harvesters=HARVESTERS)
print harvest.harvest(text)
import robin_stocks as r
from sentiment import analyze
from pulldata import gethighest
from pulldata import getoverallnews
from harvest import harvest
debug = True
#connect to robinhood, access tickers
username = '******'
password = '******'
login = r.login(username, password)
temp = r.build_holdings()
tickersHeld = []
for key in temp:
tickersHeld.append(key)
if debug:
print(f'Appending {key} to tickers held')
print("starting overallnews")
#hey = getoverallnews(5,10,10)
#week = analyze(tickersHeld,2020,5,1)
print("finished")
#print(week["highestpop"])
harvest()