def add_string(file, txt):
global f
global cptFile
global cptNbCar,cptNbCarTotal
global listFilesToTranslate
txt = txt.replace('\\ ', '\\').replace('\\ n ', '\\n').replace('\\n ', '\\n').replace('/ ', '/').strip()
txt+= "\n" + SEPARATOR_COPYPASTE + "\n"
cptNbCar+= len(txt)
cptNbCarTotal+= cptNbCar
if(cptNbCar > MAX_CAR_COPY_PASTE):
#file too long, new file
cptFile+=1
cptNbCar = len(txt)
filename = get_filename(OUTFILE, cptFile)
filenameTranslated = get_filename(OUTFILE_TRANSLATED, cptFile)
#work with this new file
f.close()
f = open(WORK_DIRECTORY + "/" + filename, "w")
listFilesToTranslate.append(filename)
#empty translated file
fTranslated = open(WORK_DIRECTORY + "/" + filenameTranslated, "w").close()
listFilesTranslated.append(filenameTranslated)
f.write(txt)
def storeScoreInClient(scoreInformation):
filename = common.get_filename('scores_client.json')
file = open(filename, 'r')
data = json.load(file)
file.close()
for user in data:
if user['userName'] == scoreInformation['userName']:
firstTime = False
for dif in user['scores']:
if dif['difficulty'] == scoreInformation['difficulty']:
dif['scores'].append(scoreInformation['score']
) #dif['scores'] is a list of score
if firstTime:
dif0 = {'difficulty': 0, 'scores': []}
dif1 = {'difficulty': 1, 'scores': []}
dif2 = {'difficulty': 2, 'scores': []}
diflst = [dif0, dif1, dif2]
for dif in diflst:
if dif['difficulty'] == scoreInformation['difficulty']:
dif['scores'].append(scoreInformation['score'])
d = {}
d['userName'] = scoreInformation['userName']
d['scores'] = diflst
data.append(d)
file = open(filename, 'w')
json.dump(data, file, ensure_ascii=False)
file.close()
def main():
filename = common.get_filename()
workbook = common.get_workbook(filename)
print(workbook.get_sheet_names())
worksheets = workbook.worksheets
assert len(worksheets) == 1, 'support only one worksheet'
lines = read_lines_from_worksheet(worksheets[0])
app_data = group_by_line_key(lines)
for line in app_data.values()[5]:
pprint(line)
def main():
filename = common.get_filename()
workbook = common.get_workbook(filename)
names = normalize_sheet_names(common.get_sheet_names(workbook))
old_name2new = {}
for name in names:
old_name2new.setdefault(
name['old_name'],
'{} {} {} {}'.format(name['code_name'], name['real_name'],
name['module'], name['os']))
for worksheet in common.list_worksheets(workbook):
common.set_worksheet_name(worksheet, old_name2new[worksheet.title])
common.save_workbook(workbook, filename)
def generate_fold(output_dir, group, input_shape, normalize, output_shape,
fold_index, fold_files, language_set):
fold_files = sorted(fold_files)
fold_files = shuffle(fold_files, random_state=SEED)
metadata = []
# create a file array
filename = "{group}_data.fold_{index}.npy".format(group=group,
index=fold_index)
features = np.memmap(os.path.join(output_dir, filename),
dtype=DATA_TYPE,
mode='w+',
shape=(len(fold_files), ) + output_shape)
# append data to a file array
# append metadata to an array
for index, fold_file in enumerate(fold_files):
#print('Group {} index {} file: {} '.format(group, index, fold_file))
language_set.add(fold_file.split('_')[0].split('/')[-1])
filename = common.get_filename(fold_file)
language = filename.split('_')[0]
data = np.load(fold_file)[DATA_KEY]
assert data.shape == input_shape
assert data.dtype == DATA_TYPE
features[index] = normalize(data)
metadata.append((language, filename))
assert len(metadata) == len(fold_files)
filename = "{group}_metadata.fold_{index}.npy".format(group=group,
index=fold_index)
print("\tout filename: ", filename)
np.save(os.path.join(output_dir, filename), metadata)
# flush changes to a disk
features.flush()
del features
def setScoreListLocal():
global scoreListLocal
filename = common.get_filename('scores_client.json')
file = open(filename,'r')
data = json.load(file)
file.close()
scorelist = []
for user in data:
userName = user['userName']
for score in user['scores']:
if score['difficulty'] == difficulty:
for i in range(0,len(score['scores'])):
scorelist.append((userName,score['scores'][i]))
# print 'scorelist:',scorelist
def cmp(s):
return s[1]
sortedlist = sorted(scorelist,key = cmp,reverse = True)
toReturn = sortedlist[:10]
# print 'toReturn:',toReturn
scoreListLocal = toReturn
def aggregation(path):
path_files = common.get_path(path)
print('Start filtering and aggregation:')
for path in tqdm(path_files):
df = pd.read_csv(path)
filename = common.get_filename(path)
logging.debug(f"File {filename} start fitering and aggreagation")
df = filter_data(df)
# Create date-hour colomn
df['date_hour'] = [
datetime(item.year, item.month, item.day, item.hour)
for item in df.tpep_pickup_datetime
]
# Group by on date_hour and region
data_group = df.groupby(['date_hour',
'region']).size().reset_index(name='count')
data_group.to_csv('data_aggregation/' + filename + '.csv')
logging.debug(f"File {filename} is ready \n")
import time
# parameters
INFILE = input("Enter input xml strings filename: [default: strings.xml]\n")
if not INFILE:
INFILE = "strings.xml"
OUTFILE = input("Enter output filename base : [default: totranslate.txt]\n")
if not OUTFILE:
OUTFILE = "totranslate.txt"
OUTFILE_TRANSLATED = input("Enter empty translated filename: [default: translated.txt] (useful for translating several languages)\n")
if not OUTFILE_TRANSLATED:
OUTFILE_TRANSLATED = "translated.txt"
create_dir(WORK_DIRECTORY)
filename = f = open(WORK_DIRECTORY + "/" + get_filename(OUTFILE), "w")
#initialize empty translated file
filenameTranslated = get_filename(OUTFILE_TRANSLATED)
fTranslated = open(WORK_DIRECTORY + "/" + filenameTranslated, "w").close()
listFilesToTranslate.append(filenameTranslated)
print("==========================\n\n")
# read xml structure
tree = ET.parse(INFILE)
root = tree.getroot()
iElement = 0
for i in range(len(root)):
isTranslatable=root[i].get('translatable')
def generate_fold(
uids,
input_dir,
input_ext,
output_dir,
group,
fold_index,
input_shape,
normalize,
output_shape):
# pull uid for each a language, gender pair
fold_uids = []
for language in LANGUAGES:
for gender in GENDERS:
fold_uids.append(uids[language][gender].pop())
# find files for given uids
fold_files = []
for fold_uid in fold_uids:
filename = '*{uid}*{extension}'.format(
uid=fold_uid,
extension=input_ext)
fold_files.extend(glob(os.path.join(input_dir, filename)))
fold_files = sorted(fold_files)
fold_files = shuffle(fold_files, random_state=SEED)
metadata = []
# create a file array
filename = "{group}_data.fold{index}.npy".format(
group=group, index=fold_index)
features = np.memmap(
os.path.join(output_dir, filename),
dtype=DATA_TYPE,
mode='w+',
shape=(len(fold_files),) + output_shape)
# append data to a file array
# append metadata to an array
for index, fold_file in enumerate(fold_files):
print(fold_file)
filename = common.get_filename(fold_file)
language = filename.split('_')[0]
gender = filename.split('_')[1]
data = np.load(fold_file)[DATA_KEY]
assert data.shape == input_shape
assert data.dtype == DATA_TYPE
features[index] = normalize(data)
metadata.append((language, gender, filename))
assert len(metadata) == len(fold_files)
# store metadata in a file
filename = "{group}_metadata.fold{index}.npy".format(
group=group,
index=fold_index)
np.save(
os.path.join(output_dir, filename),
metadata)
# flush changes to a disk
features.flush()
del features
"Speech3": 13
}
return path % (speaker, speaker, smds[smd], repetition)
# Get the properties of the recorded excitation and response
length = 2**Signal
samplingrate = 48000
prp = sumpf.modules.ChannelDataProperties(signal_length=length,
samplingrate=samplingrate)
sweep_start_frequency, sweep_stop_frequency, sweep_duration = head_specific.get_sweep_properties(
sumpf.modules.SilenceGenerator(length=length,
samplingrate=samplingrate).GetSignal())
print "Input sweep prop: startfreq-%f, stopfreq-%f, duration-%f" % (
sweep_start_frequency, sweep_stop_frequency, sweep_duration)
load = sumpf.modules.SignalFile(filename=common.get_filename(
Speaker, "Sweep%i" % Signal, 1),
format=sumpf.modules.SignalFile.WAV_FLOAT)
split_excitation = sumpf.modules.SplitSignal(channels=[0])
sumpf.connect(load.GetSignal, split_excitation.SetInput)
split_response = sumpf.modules.SplitSignal(channels=[1])
# Model for extracting the harmonics of the recorded signal
sumpf.connect(load.GetSignal, split_response.SetInput)
fft_excitation = sumpf.modules.FourierTransform()
sumpf.connect(split_excitation.GetOutput, fft_excitation.SetSignal)
fft_response = sumpf.modules.FourierTransform()
sumpf.connect(split_response.GetOutput, fft_response.SetSignal)
inversion = sumpf.modules.RegularizedSpectrumInversion(
start_frequency=max(sweep_start_frequency * 4.0, 20.0),
stop_frequency=sweep_stop_frequency / 4.0,
transition_length=100,
OUTFILE = input(
"Enter output filename base : [default: strings-translated.xml]\n")
if not OUTFILE:
OUTFILE = "strings-translated.xml"
OUTFILE_TRANSLATED = input(
"Enter empty translated filename: [default: translated.txt] (useful for translating several languages)\n"
)
if not OUTFILE_TRANSLATED:
OUTFILE_TRANSLATED = "translated.txt"
print("==========================\n\n")
#parse all files file-1, file-2, ...
tabAllString = []
cptFile = 1
filenameTranslated = get_filename(OUTFILE_TRANSLATED, cptFile)
while (os.path.isfile(os.path.join(WORK_DIRECTORY, filenameTranslated))):
f = open(os.path.join(WORK_DIRECTORY, filenameTranslated), encoding="utf8")
txt = f.read()
tab = txt.split(SEPARATOR_COPYPASTE)
#if last line is empty (because multiple files)
if (tab[len(tab) - 1] == ""):
tab.pop()
tabAllString.extend(tab)
f.close()
#next file
cptFile += 1
filenameTranslated = get_filename(OUTFILE_TRANSLATED, cptFile)
magnitude = numpy.array(positive.GetMagnitude())
cropped = magnitude[:, int(round(50.0/positive.GetResolution())):int(round(1500.0/positive.GetResolution()))]
exp = numpy.exp(cropped)
errorexp = numpy.sum(exp)
error = numpy.sum(cropped)
print "\nerrorexp: ", errorexp
print "error : ", error
print
return errorexp
# Get the properties of the recorded excitation and response
length = 2**Signal
samplingrate = 48000
sweep_start_frequency, sweep_stop_frequency, sweep_duration = head_specific.get_sweep_properties(sumpf.modules.SilenceGenerator(length=length, samplingrate=samplingrate).GetSignal())
print "Input sweep prop: startfreq-%f, stopfreq-%f, duration-%f" %(sweep_start_frequency, sweep_stop_frequency, sweep_duration)
load = sumpf.modules.SignalFile(filename=common.get_filename(Speaker, "Sweep%i" % Signal, 1),format=sumpf.modules.SignalFile.WAV_FLOAT)
split_excitation = sumpf.modules.SplitSignal(channels=[0])
sumpf.connect(load.GetSignal, split_excitation.SetInput)
split_response = sumpf.modules.SplitSignal(channels=[1])
# Model for extracting the harmonics of the recorded signal
sumpf.connect(load.GetSignal, split_response.SetInput)
fft_excitation = sumpf.modules.FourierTransform()
sumpf.connect(split_excitation.GetOutput, fft_excitation.SetSignal)
fft_response = sumpf.modules.FourierTransform()
sumpf.connect(split_response.GetOutput, fft_response.SetSignal)
inversion = sumpf.modules.RegularizedSpectrumInversion(start_frequency=max(sweep_start_frequency*4.0, 20.0),stop_frequency=sweep_stop_frequency/4.0,transition_length=100,epsilon_max=0.1)
sumpf.connect(fft_excitation.GetSpectrum,inversion.SetSpectrum)
tf_measured = sumpf.modules.MultiplySpectrums()
sumpf.connect(inversion.GetOutput, tf_measured.SetInput1)
sumpf.connect(fft_response.GetSpectrum,tf_measured.SetInput2)
import common
filename = common.get_filename()
lines1 = common.read_lines_from_file(filename)
import requests
import re
import sys
import unicodedata
from bs4 import BeautifulSoup
import os
from yandex_translate import YandexTranslate
translate = YandexTranslate(
'trnsl.1.1.20170222T052338Z.74276f7925a61714.f88a6340cffbdf48e1600c4cfb2f9ba3a22c514f'
)
urls = []
values2 = []
keys_ads = [
'remove', 'ad', 'ads', 'virus', 'advertising', 'pop-up', 'pop-ups',
'Remove', 'Ad', 'Ads', 'Virus', 'Advertising', 'Pop-up', 'Pop-ups',
'remove.', 'ad.', 'ads.', 'virus.', 'advertising.', 'pop-up.', 'pop-ups.',
'Remove.', 'Ad.', 'Ads.', 'Virus.', 'Advertising.', 'Pop-up.', 'Pop-ups.'
'remove,', 'ad,', 'ads,', 'virus,', 'advertising,', 'pop-up,', 'pop-ups,',
'Remove,', 'Ad,', 'Ads,', 'Virus,', 'Advertising,', 'Pop-up,', 'Pop-ups,'
]
keys_commerce = [
'buy', 'shop', 'shopping', 'buyers', 'sellers', 'online-store',
'online-marketplace', 'commerce', 'e-commerce', 'import', 'imports',
'export', 'exports', 'Buy', 'Shop', 'Shopping', 'Buyers', 'Sellers',
import json
import common
filename = common.get_filename('scores_client.json')
file = open(filename, 'w')
dif0 = {'difficulty': 0, 'scores': [1]}
dif1 = {'difficulty': 1, 'scores': [1]}
dif2 = {'difficulty': 2, 'scores': [1]}
diflst = [dif0, dif1, dif2]
data = {}
data['userName'] = '******'
data['scores'] = diflst
datalst = []
datalst.append(data)
# print(data)
json.dump(datalst, file, ensure_ascii=False)
file.close()
file = open(filename, 'r')
s = json.load(file)
print s[0]['userName']