async def _execute_sql_scripts(self, reindex: bool) -> None:
"""Execute SQL included with project"""
sql_path = join(self._config.package_path, 'sql')
if not exists(sql_path):
return
if any(
map(lambda p: p not in ('on_reindex', 'on_restart'),
listdir(sql_path))):
raise ConfigurationError(
f'SQL scripts must be placed either to `{self._config.package}/sql/on_restart` or to `{self._config.package}/sql/on_reindex` directory'
)
if not isinstance(self._config.database, PostgresDatabaseConfig):
self._logger.warning(
'Execution of user SQL scripts is supported on PostgreSQL only, skipping'
)
return
sql_path = join(sql_path, 'on_reindex' if reindex else 'on_restart')
if not exists(sql_path):
return
self._logger.info('Executing SQL scripts from `%s`', sql_path)
for filename in sorted(listdir(sql_path)):
if not filename.endswith('.sql'):
continue
with open(join(sql_path, filename)) as file:
sql = file.read()
self._logger.info('Executing `%s`', filename)
await get_connection(None).execute_script(sql)
def Main():
localfile = open('/home/www/pals/html/dowser/html/cdf.txt','w')
comp = localfile.read()
files = posix.listdir('/home/www/pals/html/dowser/')
os.chdir('/home/www/pals/html/dowser/')
dirs = []
paths = []
for i in range(len(files)):
file = files[i]
if posixpath.isdir(file):
os.chdir('/home/www/pals/html/dowser/'+file+'/')
hello = posix.getcwd()
refs = posix.listdir(hello)
for it in range(len(refs)):
ref = refs[it]
paths.append(hello+"/"+ref)
os.chdir('/home/www/pals/html/dowser/')
print comp
for i in range(len(paths)):
path = paths[i]
localfile.write(path+"\n")
localfile.close()
print "Files in dowser updated"
def _autoload(package):
mypath = package.__path__[0]
onlyfiles = [f for f in listdir(mypath) if isfile(join(mypath, f))]
_importAll(onlyfiles, package, ".*Service.py$")
_importAll(onlyfiles, package, ".*TestData.py$")
_importAll(onlyfiles, package, ".*Stubs.py$")
_importAll(onlyfiles, package, ".*Factory.py$")
onlydirs = [f for f in listdir(mypath) if isdir(join(mypath, f))]
for loadedPackage in _importAll(onlydirs, package, ".*", lambda x: x):
_autoload(loadedPackage)
def 輸出HTS標仔問題音檔而且訓練(cls, 合成語料資料夾, 對齊聲韻結果資料夾, 合成模型資料夾路徑, 決策樹仔):
HTS訓練過程目錄 = cls._細項目錄(合成模型資料夾路徑, 'HTS訓練過程')
HTS資料目錄 = cls._細項目錄(HTS訓練過程目錄, 'data')
HTS標仔目錄 = cls._細項目錄(HTS資料目錄, 'labels')
HTS孤音標仔目錄 = join(HTS標仔目錄, 'mono')
HTS相依標仔目錄 = join(HTS標仔目錄, 'full')
for 來源, 目標 in [
(對齊聲韻結果資料夾, HTS孤音標仔目錄),
(join(合成語料資料夾, '相依標仔'), HTS相依標仔目錄),
]:
if exists(目標):
shutil.rmtree(目標)
shutil.copytree(來源, 目標)
HTS試驗標仔目錄 = cls._細項目錄(HTS標仔目錄, 'gen')
for 檔名 in listdir(HTS相依標仔目錄)[:10]:
來源完整標仔 = join(HTS相依標仔目錄, 檔名)
目標完整標仔 = join(HTS試驗標仔目錄, 檔名)
shutil.copy(來源完整標仔, 目標完整標仔)
print(' 2.HTS愛分類標仔，所以愛決策樹仔的問題')
問題 = 決策樹仔.生()
HTS問題目錄 = cls._細項目錄(HTS資料目錄, 'questions')
HTS問題檔案 = join(HTS問題目錄, 'questions_qst001.hed')
cls._陣列寫入檔案(HTS問題檔案, 問題)
print(' 3.HTS愛無檔頭的音標，所以愛提掉檔頭')
HTS原始檔目錄 = join(HTS資料目錄, 'raw')
if exists(HTS原始檔目錄):
shutil.rmtree(HTS原始檔目錄)
makedirs(HTS原始檔目錄, exist_ok=True)
全部頻率 = set()
音檔目錄 = join(合成語料資料夾, '音檔')
for 檔名 in listdir(音檔目錄):
if 檔名.endswith('wav'):
音檔檔名 = join(音檔目錄, 檔名)
音檔 = 聲音檔.對檔案讀(音檔檔名)
with open(join(HTS原始檔目錄, 檔名[:-4] + '.raw'), 'wb') as 原始檔:
原始檔.write(音檔.wav音值資料())
if 音檔.幾个聲道 != 1 or 音檔.一點幾位元組 != 2:
raise RuntimeError('音標愛單聲道，逐的點愛兩位元組的整數')
全部頻率.add(音檔.一秒幾點)
if len(全部頻率) > 1:
raise RuntimeError('音檔的取樣頻率愛仝款！！有{0}Hz'.format(
'、'.join([str(頻率) for 頻率 in sorted(全部頻率)])
))
頻率 = 全部頻率.pop()
return 訓練HTSEngine模型.訓練(HTS訓練過程目錄, 頻率)
def _訓練恬音模型(cls):
訓練目錄 = join(cls.專案目錄, '訓練音檔')
題目 = []
答案 = []
for 類型 in sorted(listdir(訓練目錄)):
類型目錄 = join(訓練目錄, 類型)
for 音檔 in sorted(listdir(類型目錄)):
音檔所在 = join(類型目錄, 音檔)
音檔 = 聲音檔.對檔案讀(音檔所在)
for 音框 in 音檔.全部音框():
特徵 = 恬音判斷.算特徵參數(音框)
題目.append(cls._特徵轉陣列(特徵))
答案.append(類型)
cls.恬音模型 = svm.SVC()
cls.恬音模型.fit(題目, 答案)
def process_dir(data_dir_path, output_file_path):
# get the list of files in the directory
onlyfiles = [ f for f in listdir(data_dir_path) if isfile(join(data_dir_path,f))]
# crate a new workbook and sheet
wb_new = Workbook()
sheet_names = wb_new.get_sheet_names()
ws_new = wb_new.get_sheet_by_name(sheet_names[0])
# keep track of number of files processed
global file_count
file_count = 0
# keep track of the row number in the new merged file
global row_count
row_count = 1
# save daily sales
global daily_sales
daily_sales = {}
# process each file in given path
for f in onlyfiles:
process_file(data_dir_path + f, ws_new, file_count)
file_count += 1
# finally save daily sales json
with open(output_file_path + 'daily_sale_figures.json', 'w', encoding = 'utf-8') as fp:
json.dump(daily_sales, fp)
# finally save the new excel file
wb_new.save(output_file_path + 'daily_product_sales.xlsx')
def _listdir(dir, cache=None):
# The cache is not used. It was found to cause problems
# with programs that dynamically add python modules to be
# reimported by that same program (i.e., plugins), because
# the cache is only built once at the beginning, and never
# updated. So, we must really list the directory again.
return listdir(dir)
def get_exe_locations(target_dir):
"""Generate possible locations from which to chainload in the target dir."""
# TODO: let this be a generator when not compiling with PyPy, so we can
# avoid a few stat() calls in the common case.
locs = []
appdir = appdir_from_executable(sys.executable)
# If we're in an appdir, first try to launch from within "<appname>.app"
# directory. We must also try the default scheme for backwards compat.
if sys.platform == "darwin":
if basename(dirname(sys.executable)) == "MacOS":
if __esky_name__:
locs.append(pathjoin(target_dir,
__esky_name__+".app",
sys.executable[len(appdir)+1:]))
else:
for nm in listdir(target_dir):
if nm.endswith(".app"):
locs.append(pathjoin(target_dir,
nm,
sys.executable[len(appdir)+1:]))
# This is the default scheme: the same path as the exe in the appdir.
locs.append(target_dir + sys.executable[len(appdir):])
# If sys.executable was a backup file, try using original filename.
orig_exe = get_original_filename(sys.executable)
if orig_exe is not None:
locs.append(target_dir + orig_exe[len(appdir):])
return locs
def decrypt_directory(indir, outdir, re_encrypt=False):
'''Decrypt the OmniFocus data in indir, writing the result to outdir. Prompts for a passphrase.'''
files = posix.listdir(indir)
indir = os.path.abspath(indir)
if outdir:
outdir = os.path.abspath(outdir)
if metadata_filename not in files:
raise EnvironmentError('Expected to find %r in %r' %
(metadata_filename, indir))
encryptionMetadata = DocumentKey.parse_metadata(
open(os.path.join(indir, metadata_filename), 'rb'))
metadataKey = DocumentKey.use_passphrase(
encryptionMetadata, getpass.getpass(prompt="Passphrase: "))
docKey = DocumentKey(encryptionMetadata.get('key').data,
unwrapping_key=metadataKey)
for secret in docKey.secrets:
secret.print()
workdir = outdir
if outdir is not None:
posix.mkdir(outdir)
if re_encrypt:
workdir = tempfile.mkdtemp()
basename = os.path.basename(indir)
for dirpath, dirnames, filenames in os.walk(indir):
for datafile in filenames:
inpath = os.path.join(dirpath, datafile)
if workdir is not None:
outpath = inpath.replace(indir, workdir)
if datafile == metadata_filename:
continue
display = "%s/%s" % (os.path.basename(dirpath),
datafile) if os.path.basename(
dirpath) != basename else datafile
if outdir is not None:
print('Decrypting %r' % (display, ))
if not os.path.exists(os.path.split(outpath)[0]):
os.makedirs(os.path.split(outpath)[0])
with open(os.path.join(indir, inpath), "rb") as infp, \
open(os.path.join(workdir, outpath), "wb") as outfp:
docKey.decrypt_file(datafile, infp, outfp)
else:
print('Reading %r' % (display, ))
with open(os.path.join(indir, inpath), "rb") as infp:
docKey.decrypt_file(datafile, infp, None)
if re_encrypt and outdir is not None:
print()
print("Re-Encrypt the database\n")
newKey = docKey.get_key_of_type(ActiveAES_CTR_HMAC, True)
encryptionMetadata['key'] = plistlib.Data(
docKey.wrapped_secrets(metadataKey))
encrypt_directory(encryptionMetadata, docKey, workdir, outdir)
# We've created a tempdirectory lets clean it up
import shutil
shutil.rmtree(workdir)
def process_dir(data_dir_path, output_file_path):
# crate a new workbook and sheets
wb_new = Workbook()
ws_deliveries = wb_new.get_active_sheet()
ws_deliveries.title = 'Deliveries'
ws_returns = wb_new.create_sheet(1)
ws_returns.title = 'Returns'
ws_wastage = wb_new.create_sheet(2)
ws_wastage.title = 'Wastage'
ws_staff_meals = wb_new.create_sheet(3)
ws_staff_meals.title = 'Staff Meals'
ws_transfers_in = wb_new.create_sheet(4)
ws_transfers_in.title = 'Transfers In'
ws_transfers_out = wb_new.create_sheet(5)
ws_transfers_out.title = 'Transfers Out'
# get the list of files in the directory
onlyfiles = [ f for f in listdir(data_dir_path) if isfile(join(data_dir_path,f))]
# process each file
for f in onlyfiles:
process_file(data_dir_path + f, wb_new)
# save the new workbook
wb_new.save(output_file_path)
def load_files(self, directory="dummy_files"):
dir_path = join(dirname(__file__), "..", directory)
if not isdir(dir_path):
print "Skipping non-existing dir", directory
return
file_names = listdir(dir_path)
for fn in file_names:
if fn.startswith("."):
continue
path = join(dir_path, fn)
name = unicode(fn, errors="replace")
data = open(path).read()
mime_type = mimetypes.guess_type(fn)[0]
f = File(name, data, mime_type)
f.creator_id = choice(self.users).uid
f.owner_id = choice(self.users).uid
n_tags = random.randint(0, len(TAGS))
tags = random.sample(TAGS, n_tags)
f.tags = u",".join(tags)
self.db.session.add(f)
def 接檔案(原本音檔, 目錄音檔):
makedirs(目錄音檔, exist_ok=True)
for 資料夾 in sorted(listdir(原本音檔)):
所在 = join(原本音檔, 資料夾)
if isdir(所在):
全部音檔合起來 = None
for 第幾個 in range(4):
for 檔案 in sorted(listdir(所在))[第幾個::4]:
這個音檔 = 聲音檔.對檔案讀(join(所在, 檔案))
try:
全部音檔合起來 = 全部音檔合起來.接(這個音檔)
except:
全部音檔合起來 = 這個音檔
結果檔案 = join(目錄音檔, "{}-{}.wav".format(資料夾, 第幾個))
with open(結果檔案, "wb") as 檔案:
檔案.write(全部音檔合起來.wav格式資料())
def test_壓縮檔案(self):
語料對齊 = Kaldi語料對齊.匯入音檔(
'台語',
'媠媠',
聲音檔.對參數轉(2, 16, 1, b'sui2khiau2' * 160),
'tsiang5 tsiang5\nkhiau2',
)
語料對齊.對齊成功([
{
'開始': 0.30,
'長度': 0.23,
'分詞': 'tsiang5',
},
{
'開始': 0.60,
'長度': 0.23,
'分詞': 'tsiang5',
},
{
'開始': 0.83,
'長度': 0.23,
'分詞': 'khiau2',
},
])
with TemporaryDirectory() as 資料夾路徑:
wav路徑 = join(資料夾路徑, 'wav')
語料對齊.產生音檔(wav路徑)
self.assertEqual(len(listdir(資料夾路徑)), 1)
tar路徑 = join(資料夾路徑, 'tar')
語料對齊.壓縮音檔(wav路徑, tar路徑)
self.assertTrue(isfile(tar路徑))
def _listdir(dir, cache={}):
# since this function is only used by caseOk, it's fine to cache the
# results and avoid reading the whole contents of a directory each time
# we just want to check the case of a filename.
if not dir in cache:
cache[dir] = listdir(dir)
return cache[dir]
def get_exe_locations(target_dir):
"""Generate possible locations from which to chainload in the target dir."""
# TODO: let this be a generator when not compiling with PyPy, so we can
# avoid a few stat() calls in the common case.
locs = []
appdir = appdir_from_executable(sys.executable)
# If we're in an appdir, first try to launch from within "<appname>.app"
# directory. We must also try the default scheme for backwards compat.
if sys.platform == "darwin":
if basename(dirname(sys.executable)) == "MacOS":
if __esky_name__:
locs.append(pathjoin(target_dir,
__esky_name__+".app",
sys.executable[len(appdir)+1:]))
else:
for nm in listdir(target_dir):
if nm.endswith(".app"):
locs.append(pathjoin(target_dir,
nm,
sys.executable[len(appdir)+1:]))
# This is the default scheme: the same path as the exe in the appdir.
locs.append(target_dir + sys.executable[len(appdir):])
# If sys.executable was a backup file, try using original filename.
orig_exe = get_original_filename(sys.executable)
if orig_exe is not None:
locs.append(target_dir + orig_exe[len(appdir):])
return locs
def _listdir(dir, cache={}):
# since this function is only used by caseOk, it's fine to cache the
# results and avoid reading the whole contents of a directory each time
# we just want to check the case of a filename.
if not dir in cache:
cache[dir] = listdir(dir)
return cache[dir]
def encrypt_directory(metadata, docKey, indir, outdir):
'''Encrypt all individual files in indir, writing them to outdir.'''
files = posix.listdir(indir)
assert metadata_filename not in files # A non-encrypted directory must not have an encryption metadata file
if not os.path.exists(outdir):
posix.mkdir(outdir)
print('Writing encryption metadata')
if isinstance(metadata, dict):
metadata = [ metadata ] # Current OmniFileStore expects a 1-element list of dictionaries.
with open(os.path.join(outdir, metadata_filename), "wb") as outfp:
if hasattr(plistlib, 'dump'):
plistlib.dump(metadata, outfp)
else:
plistlib.writePlist(metadata, outfp)
basename = os.path.basename(indir)
for dirpath, dirnames, filenames in os.walk(indir):
for datafile in filenames:
inpath = os.path.join(dirpath, datafile)
outpath = inpath.replace(indir, outdir)
display = "%s/%s" % (os.path.basename(dirpath), datafile) if os.path.basename(dirpath) != basename else datafile
print('Encrypting %r' % (display,))
if not os.path.exists(os.path.split(outpath)[0]):
os.makedirs(os.path.split(outpath)[0])
with open(os.path.join(indir, inpath), "rb") as infp, \
open(outpath, "wb") as outfp:
docKey.encrypt_file(datafile, infp, outfp)
def _listdir(dir, cache=None):
# The cache is not used. It was found to cause problems
# with programs that dynamically add python modules to be
# reimported by that same program (i.e., plugins), because
# the cache is only built once at the beginning, and never
# updated. So, we must really list the directory again.
return listdir(dir)
def 全部資料(self, *args, **參數):
'https://github.com/g0v/moedict-data-twblg/tree/master/uni'
詞目 = {}
with open(
join(dirname(abspath(__file__)), '..', '..', '教育部閩南語常用詞辭典',
'詞目總檔.csv')) as 檔案:
for 一筆 in DictReader(檔案):
編號 = 一筆['主編碼'].strip()
漢字 = 一筆['詞目'].strip()
羅馬字 = 一筆['音讀'].strip().split('/')[0]
try:
詞目[int(編號)] = 拆文分析器.對齊組物件(漢字, 羅馬字)
except 解析錯誤:
pass
音檔目錄 = join(settings.BASE_DIR, '語料', '教育部閩南語常用詞辭典wav')
匯入數量 = 0
for 路徑 in sorted(listdir(音檔目錄)):
音檔路徑 = join(音檔目錄, 路徑)
if 音檔路徑.endswith('.wav'):
try:
音檔編號 = int(basename(音檔路徑).split('.')[0])
except ValueError:
raise ValueError('有的音檔有重錄過')
try:
台語物件 = 詞目[音檔編號]
except KeyError: # 有的詞條尾仔提掉矣，親像編號5
continue
yield 訓練過渡格式(影音所在=音檔路徑, 文本=台語物件.看分詞(), **self.公家內容)
匯入數量 += 1
if 匯入數量 == 參數['匯入幾筆']:
break
def test_對齊檔案(self):
語料對齊 = Kaldi語料對齊.匯入音檔(
'台語',
'媠媠',
聲音檔.對參數轉(2, 16, 1, b'sui2khiau2' * 160),
'tsiang5 tsiang5\nkhiau2',
)
語料對齊.對齊成功([
{
'開始': 0.30,
'長度': 0.23,
'分詞': 'tsiang5',
},
{
'開始': 0.60,
'長度': 0.23,
'分詞': 'tsiang5',
},
{
'開始': 0.83,
'長度': 0.23,
'分詞': 'khiau2',
},
])
with TemporaryDirectory() as 資料夾路徑:
語料對齊.產生音檔(資料夾路徑)
self.assertEqual(len(listdir(資料夾路徑)), 2)
def _照性別合一組音檔(cls, 編號):
編號音檔目錄 = cls._編號分割音檔目錄(編號)
合做伙目錄 = join(cls.專案目錄, '音檔合做伙')
makedirs(合做伙目錄, exist_ok=True)
頂一個查甫查某 = None
號碼 = 0
合音檔 = 聲音檔.對檔案讀(join(編號音檔目錄, '0000.wav')).產生仝參數空聲音檔()
for 檔名 in sorted(listdir(編號音檔目錄)):
if re.match('\d+.wav\Z', 檔名):
這馬查甫查某 = cls._檢查查甫查某(join(編號音檔目錄, 檔名))
if 這馬查甫查某 is not None and 頂一個查甫查某 != 這馬查甫查某:
if 頂一個查甫查某 is not None:
with open(join(
合做伙目錄, '{:02}-{:04}-{}.wav'.format(編號, 號碼, 頂一個查甫查某)), 'wb'
) as 檔案:
檔案.write(合音檔.wav格式資料())
合音檔 = 合音檔.產生仝參數空聲音檔()
號碼 += 1
頂一個查甫查某 = 這馬查甫查某
音檔 = 聲音檔.對檔案讀(join(編號音檔目錄, 檔名))
合音檔 = 合音檔.接(音檔)
with open(join(
合做伙目錄, '{:02}-{:04}-{}.wav'.format(編號, 號碼, 頂一個查甫查某)), 'wb'
) as 檔案:
檔案.write(合音檔.wav格式資料())
def updateuploadinGDrive():
isfolderexists(
) # auto create docs folder excel file will have in this folder
folder_content = [
f for f in listdir('./docs') if isfile(join('./docs/', f))
]
os.system('cls' if os.name == 'nt' else 'clear')
print(len(folder_content), ' docs folder content')
if len(folder_content) <= 0:
print("There are no files in docs folder")
return
uploadupdate = input(
"\nPress 1 to upload file to G Drive,\nPress 3 to Update file in G Drive,\nPress 0 to exit: "
)
if uploadupdate == "1":
filename = input("Enter file name: ")
if filename:
uploadfile('./docs/', filename + '.xlsx')
else:
print("Must profie file name.\n")
return
elif uploadupdate == '3':
filename = input("Enter file name: ")
if filename:
updatefile(filename + '.xlsx')
else:
print("Must provide file.\n")
return
def process_dir(data_dir_path, output_file_path):
# crate a new workbook and sheets
wb_new = Workbook()
ws_deliveries = wb_new.get_active_sheet()
ws_deliveries.title = 'Deliveries'
ws_returns = wb_new.create_sheet(1)
ws_returns.title = 'Returns'
ws_wastage = wb_new.create_sheet(2)
ws_wastage.title = 'Wastage'
ws_staff_meals = wb_new.create_sheet(3)
ws_staff_meals.title = 'Staff Meals'
ws_transfers_in = wb_new.create_sheet(4)
ws_transfers_in.title = 'Transfers In'
ws_transfers_out = wb_new.create_sheet(5)
ws_transfers_out.title = 'Transfers Out'
# get the list of files in the directory
onlyfiles = [
f for f in listdir(data_dir_path) if isfile(join(data_dir_path, f))
]
# process each file
for f in onlyfiles:
process_file(data_dir_path + f, wb_new)
# save the new workbook
wb_new.save(output_file_path)
def tsanpootong():
boklok = join('pojbh.lib.ntnu.edu.tw', 'script')
buntsiunn = re.compile('artical-(\d+).htm')
for mia in sorted(listdir(boklok)):
tuiing = buntsiunn.match(mia)
if tuiing:
yield tuiing.group(1), join(boklok, mia)
def openlistwindow(dirname):
list = posix.listdir(dirname)
list.sort()
i = 0
while i < len(list):
if list[i] == '.' or list[i] == '..':
del list[i]
else:
i = i+1
for i in range(len(list)):
name = list[i]
if path.isdir(path.join(dirname, name)):
list[i] = list[i] + '/'
width = maxwidth(list)
width = width + stdwin.textwidth(' ') # XXX X11 stdwin bug workaround
height = len(list) * stdwin.lineheight()
stdwin.setdefwinsize(width, min(height, 500))
w = stdwin.open(dirname)
stdwin.setdefwinsize(0, 0)
w.setdocsize(width, height)
w.drawproc = drawlistwindow
w.mouse = mouselistwindow
w.close = closelistwindow
w.dirname = dirname
w.list = list
w.selected = -1
return w
def Matches(self, comp):
to_complete = comp.to_complete
i = to_complete.rfind('/')
if i == -1: # it looks like 'foo'
to_list = '.'
base = ''
elif i == 0: # it's an absolute path to_complete like / or /b
to_list = '/'
base = '/'
else:
to_list = to_complete[:i]
base = to_list
#log('to_list %r', to_list)
try:
names = posix.listdir(to_list)
except OSError as e:
return # nothing
for name in names:
path = os_path.join(base, name)
if path.startswith(to_complete):
if self.dirs_only:
if os_path.isdir(path):
yield path
else:
if os_path.isdir(path):
yield path + '/'
else:
yield path
def Matches(self, comp):
"""
TODO: Cache is never cleared.
- When we get a newer timestamp, we should clear the old one.
- When PATH is changed, we can remove old entries.
"""
val = self.mem.GetVar('PATH')
if val.tag != value_e.Str:
# No matches if not a string
return
path_dirs = val.s.split(':')
#print(path_dirs)
names = []
for d in path_dirs:
try:
st = posix.stat(d)
except OSError as e:
# There could be a directory that doesn't exist in the $PATH.
continue
key = (d, st.st_mtime)
listing = self.cache.get(key)
if listing is None:
listing = posix.listdir(d)
self.cache[key] = listing
names.extend(listing)
# TODO: Shouldn't do the prefix / space thing ourselves. readline does
# that at the END of the line.
for word in listing:
if word.startswith(comp.to_complete):
yield word + ' '
def _資料夾的短恬量(self, 資料夾):
self.檢查資料夾有辨識出來的檔案(資料夾)
短恬數量 = 0
for 檔名 in listdir(資料夾):
for 一逝 in 程式腳本._讀檔案(join(資料夾, 檔名)):
if 一逝.endswith('sp'):
短恬數量 += 1
return 短恬數量
def deconstruct_quick_test_dataset(dataset_path):
files = listdir(dataset_path)
for file in files:
filename = os.path.join(dataset_path, file)
if filename.endswith('.bin'):
os.remove(filename)
elif not filename.endswith('.tmp'):
shutil.move(filename + '.tmp', filename)
def _資料夾的短恬量(self, 資料夾):
self.檢查資料夾有辨識出來的檔案(資料夾)
短恬數量 = 0
for 檔名 in listdir(資料夾):
for 一逝 in 程式腳本._讀檔案(join(資料夾, 檔名)):
if 一逝.endswith('sp'):
短恬數量 += 1
return 短恬數量
def which_days():
print '<H3>on this day <spacer type="horizontal" size="50">'
print '<select name="day">'
files = posix.listdir('/home/httpd/html/archivewx/data_days')
files.sort()
for file in files:
print '<option value="'+file+'">'+file+'\n'
print '</select></H3>'
def openlistwindow(dirname):
list = posix.listdir(dirname)
list.sort()
i = 0
while i < len(list):
if list[i] = '.' or list[i] = '..':
del list[i]
else:
i = i+1
def avail(day):
files = posix.listdir('/home/www/pals/html/archivewx/data/'+day)
files.sort()
print '<H3>Available files to link in:</H3>'
print '<form name="two">'
print '<SELECT>'
for file in files:
print '<OPTION>'+file
print '</SELECT></Form><HR>'
def sortFiles(source_path, dir_name, *types):
for filename in listdir(source_path):
for type in types:
if filename.endswith(type):
if os.path.isdir(f'{dest_path}/{dir_name}') == False:
os.mkdir(f'{dest_path}/{dir_name}')
dest_dir = f'{dest_path}/{dir_name}'
src_file = f'{source_path}/{filename}'
shutil.move(src_file, dest_dir)
def openlistwindow(dirname): list = posix.listdir(dirname) list.sort() i = 0 while i < len(list): if list[i] = '.' or list[i] = '..': del list[i] else: i = i+1
def listdir(path, fil=None, rec=False):
names = [posixpath.join(path, name) for name in posix.listdir(path)]
for name in names:
if posixpath.isdir(name):
for name in listdir(name, fil, rec):
yield name
if (fil is not None and fnmatch(name, fil)) or fil is None:
yield name
def get_SMP_files():
if os.path.exists(folder_path):
print('Directory already exists')
else:
print('Creating Directory')
os.mkdir(folder_path)
#zip file of previous year from 2020-11-01
url = 'https://www.enexgroup.gr/el/c/document_library/get_file?uuid=23d04ca0-5850-415b-4c7b-d0a65ed293cf&groupId=20126'
zip_path = folder_path + '2020_EL-DAM_ResultsSummary.zip'
if os.path.exists(zip_path):
print("Processing Zip file")
with ZipFile(zip_path, 'r') as zip_file:
zip_file.extractall(folder_path)
else:
print('Downloading Zip file')
with open(zip_path, 'wb') as zip_file:
zip_file.write(requests.get(url).content)
print("Processing Zip file")
with ZipFile(zip_path, 'r') as zip_file:
zip_file.extractall(folder_path)
flag = False
index = 1
while (True):
url = f'https://www.enexgroup.gr/el/web/guest/markets-publications-el-day-ahead-market?p_p_id=com_liferay_asset_publisher_web_portlet_AssetPublisherPortlet_INSTANCE_9CZslwWTpeD2&p_p_lifecycle=0&p_p_state=normal&p_p_mode=view&_com_liferay_asset_publisher_web_portlet_AssetPublisherPortlet_INSTANCE_9CZslwWTpeD2_delta=7&p_r_p_resetCur=false&_com_liferay_asset_publisher_web_portlet_AssetPublisherPortlet_INSTANCE_9CZslwWTpeD2_cur={index}'
data = requests.get(url)
print(data)
x = re.findall('Αποτελέσματα Aγοράς Επόμενης Ημέρας(.* ?)', data.text)
names = re.findall('[0-9]{8}_EL-DAM_ResultsSummary_EN_v01\.xlsx',
''.join(x))
urls = re.findall('uuid(.*?)"', ''.join(x))
for i, j in zip(names, urls[:len(names)]):
file_path = folder_path + i
if os.path.exists(file_path):
print(f'File {file_path} found')
flag = True
else:
url = f'https://www.enexgroup.gr/el/c/document_library/get_file?uuid{j}'
x = requests.get(url).content
print(f'Downloading File ')
with open(file_path, 'wb') as xlsx:
xlsx.write(x)
if not names or flag:
break
index += 1
files = [f for f in listdir(folder_path)]
for name in files:
if name.endswith('Copy.xlsx'):
os.remove(folder_path + name)
# https://www.enexgroup.gr/el/c/document_library/get_file?uuid=08c1813a-100c-42cb-0124-d1f6a7eb3325&groupId=20126
def glob1(dirname, pattern):
if not dirname: dirname = '.'
try:
names = posix.listdir(dirname)
except posix.error:
return []
result = []
for name in names:
if name[0] <> '.' or pattern[0] = '.':
if fnmatch.fnmatch(name, pattern): result.append(name)
def load_objects(self):
for filename in listdir(dirname(__file__)):
name, ext = splitext(filename)
if ext == ".py" and name != "__init__":
module = __import__(
"mio.traits.{0:s}".format(name), fromlist=["mio.traits"])
predicate = lambda x: isclass(x) and issubclass(
x, Object) and getmodule(x) is module and x is not Traits
for name, object in getmembers(module, predicate):
yield name, object
def recursiveList(rootDir):
fileList = []
for x in posix.listdir(rootDir):
fullPath = rootDir + "/" + x
if (os.path.isdir(fullPath)):
fileList += recursiveList(fullPath)
else:
# its a file, add it to our list to bakup
fileList.append(fullPath)
return fileList
def recursiveList( rootDir ): fileList = [] for x in posix.listdir( rootDir ): fullPath = rootDir + "/" + x try: if( os.path.isdir( fullPath ) ): fileList += recursiveList( fullPath ) else: fileList.append( fullPath ) except OSError, ( errno, error ): sys.stderr.write( error );
def 全部資料(self, *args, **參數):
語料目錄 = join(settings.BASE_DIR, '語料', 'Suí-Siann')
暫時檔案 = join(語料目錄, 'SuiSiann-0.1.tar')
if not isfile(暫時檔案):
makedirs(語料目錄, exist_ok=True)
run(['wget', '-O', 暫時檔案, self.網址],
stdout=PIPE,
stderr=PIPE,
check=True)
TarFile(暫時檔案).extractall(語料目錄)
轉檔所在 = join(
語料目錄,
'hue7jip8',
)
音檔目錄 = join(
語料目錄,
'0.1',
'ImTong',
)
轉檔目錄 = join(
轉檔所在,
'ImTong',
)
makedirs(轉檔目錄, exist_ok=True)
音檔陣列 = []
for 檔名 in sorted(listdir(音檔目錄)):
來源 = join(音檔目錄, 檔名)
目標 = join(轉檔目錄, 檔名)
print(來源)
run([
'ffmpeg',
'-i',
來源,
'-acodec',
'pcm_s16le',
'-ar',
'{}'.format(參數['頻率']),
'-ac',
'1',
'-y',
目標,
],
stdout=PIPE,
stderr=PIPE,
check=True)
音檔陣列.append(目標)
with open(join(語料目錄, '0.1', 'sui-siann.csv')) as tong:
for 一逝 in DictReader(tong):
yield 訓練過渡格式(影音所在=join(轉檔所在, 一逝['音檔']),
文本=拆文分析器.建立句物件(一逝['漢字'], 一逝['羅馬字']).看分詞(),
**self.公家內容)
def walk(top, func, arg):
try:
names = posix.listdir(top)
except posix.error:
return
func(arg, top, names)
exceptions = ('.', '..')
for name in names:
if name not in exceptions:
name = cat(top, name)
if isdir(name):
walk(name, func, arg)
def parse_module(self, path, parent = None):
if parent is None:
parent = self.document
xmlNode = self.createElement(path, parent, "package")
if(isdir(path)):
for fileName in listdir(path):
fullPath = join(path, fileName)
if isfile(fullPath) and fullPath[-3:] == '.py':
self.astParse(fullPath, xmlNode)
elif isdir(fullPath):
self.parse_module(fullPath, xmlNode)
return self.document
def walk(top, func, arg):
try:
names = posix.listdir(top)
except posix.error:
return
func(arg, top, names)
exceptions = ('.', '..')
for name in names:
if name not in exceptions:
name = join(top, name)
if isdir(name) and not islink(name):
walk(name, func, arg)
def 轉規个資料夾(來源):
for 檔名 in sorted(listdir(來源), key=lambda 檔名: 檔名[2:]):
來源檔案 = join(來源, 檔名)
if 檔名.endswith('doc'):
print(檔名)
try:
for 一逝 in 轉換doc(
來源檔案
).doc轉html().html轉array().提array():
yield 一逝
except Exception as 錯誤:
print(來源檔案, 錯誤)
def selector():
print '<form method="post" action="/cgi-bin/archivewx/page_gen.py">'
print '<select name="date">'
dates = posix.listdir(dir)
dates.sort()
for date in dates:
if date == "index.html":
blah = "1"
else:
print '<option value="'+date+'">'+date
print '</select><HR><H2>And get started:</H2>'
print '<input type="submit" value="Start exercise"></form>'
def test_對齊檔案(self):
語料對齊 = Kaldi語料對齊.匯入音檔(
'台語', '媠媠',
聲音檔.對參數轉(2, 16, 1, b'sui2khiau2' * 160), 'tsiang5 tsiang5\nkhiau2',
)
語料對齊.對齊成功([
{'開始': 0.30, '長度': 0.23, '分詞': 'tsiang5', },
{'開始': 0.60, '長度': 0.23, '分詞': 'tsiang5', },
{'開始': 0.83, '長度': 0.23, '分詞': 'khiau2', },
])
with TemporaryDirectory() as 資料夾路徑:
語料對齊.產生音檔(資料夾路徑)
self.assertEqual(len(listdir(資料夾路徑)), 2)
def which_days():
print '<H3>on this day <spacer type="horizontal" size="50">'
print '<select name="day">'
files = posix.listdir('/home/www/pals/html/archivewx/data')
files.sort()
for file in files:
try:
tester = mydb1.query("SELECT * from t"+file+" ")
print '<option value="'+file+'">'+file+'\n'
except ValueError:
continue
print '</select></H3>'
def mkrealdir(name):
st = posix.stat(name) # Get the mode
mode = S_IMODE(st[ST_MODE])
linkto = posix.readlink(name)
files = posix.listdir(name)
posix.unlink(name)
posix.mkdir(name, mode)
posix.chmod(name, mode)
linkto = cat('..', linkto)
#
for file in files:
if file not in ('.', '..'):
posix.symlink(cat(linkto, file), cat(name, file))
def getargs():
args = sys.argv[1:]
if args:
return args
print 'No arguments, checking almost *'
for file in posix.listdir('.'):
if not skipfile(file):
args.append(file)
if not args:
print 'Nothing to do -- exit 1'
sys.exit(1)
args.sort()
return args
def mkrealdir(name):
st = posix.stat(name) # Get the mode
mode = S_IMODE(st[ST_MODE])
linkto = posix.readlink(name)
files = posix.listdir(name)
posix.unlink(name)
posix.mkdir(name, mode)
posix.chmod(name, mode)
linkto = cat('..', linkto)
#
for file in files:
if file not in ('.', '..'):
posix.symlink(cat(linkto, file), cat(name, file))
def lsOpen(msg=None, fh=sys.stderr, pid=None):
"""list open files, mostly for debugging"""
if msg is not None:
_prLine(fh, msg)
if pid is None:
pid = os.getpid()
fddir = "/proc/" + str(pid) + "/fd/"
fds = posix.listdir(fddir)
fds.sort()
for fd in fds:
# entry will be gone for fd dir when it was opened
fdp = fddir + fd
if os.path.exists(fdp):
_prLine(fh, " ", fd, " -> ", os.readlink(fdp))
def 資料抓出來(self):
歌詞目錄 = join(_專案目錄, '咱的字你敢捌', 'min-nan-yu-ge-ci-xiao-zheng')
for 資料夾 in listdir(歌詞目錄):
if isdir(join(歌詞目錄, 資料夾)):
with open(join(歌詞目錄, 資料夾, 'index.html')) as 檔案:
格式 = BeautifulSoup(檔案, 'lxml')
舊歌詞 = self._整理出內底歌詞(
格式, "sites-layout-tile sites-tile-name-content-1"
)
新歌詞 = self._整理出內底歌詞(
格式, "sites-layout-tile sites-tile-name-content-2")
for 舊, 新 in zip(舊歌詞, 新歌詞):
if '純學術用絕無商業用途' not in 新 and 新 != '':
yield (舊, 新)
def _扣demo的資料到訓練資料夾(cls, htsDemo目錄, 訓練資料夾):
for 檔案 in listdir(htsDemo目錄):
路徑 = join(htsDemo目錄, 檔案)
if not 檔案.startswith('.') and 檔案 != 'data':
目標所在 = join(訓練資料夾, 檔案)
if isfile(路徑):
copyfile(路徑, 目標所在)
else:
if exists(目標所在):
rmtree(目標所在)
copytree(路徑, 目標所在)
data目錄 = join(htsDemo目錄, 'data')
訓練data資料夾 = join(訓練資料夾, 'data')
for 檔案 in listdir(data目錄):
路徑 = join(data目錄, 檔案)
目標所在 = join(訓練data資料夾, 檔案)
if isfile(路徑):
copyfile(路徑, 目標所在)
else:
if exists(目標所在):
rmtree(目標所在)
copytree(路徑, 目標所在)
def Main():
form = FormContent()
day = form["day"][0]
passwd = form["pass"][0]
checker(passwd)
style.header('Data for '+day, 'white')
style.std_top('Data for '+day)
files = posix.listdir(data_src+day+'/data/')
print '<multicol cols="3">'
for file in files:
print '<a href="'+url_ref+day+'/data/'+file+'">'
print file+'</a><BR>'
print '</multicol>'
style.std_bot()
def get_original_filename(backname):
"""Given a backup filename, get the original name to which it refers.
This is only really possible if the original file actually exists and
is not guaranteed to be correct in all cases; but unless you do something
silly it should work out OK.
If no matching original file is found, None is returned.
"""
filtered = ".".join([n for n in backname.split(".") if n != "old"])
for nm in listdir(dirname(backname)):
if nm == backname:
continue
if filtered == ".".join([n for n in nm.split(".") if n != "old"]):
return pathjoin(dirname(backname), nm)
return None
def test_壓縮檔案(self):
語料對齊 = Kaldi語料對齊.匯入音檔(
'台語', '媠媠',
聲音檔.對參數轉(2, 16, 1, b'sui2khiau2' * 160), 'tsiang5 tsiang5\nkhiau2',
)
語料對齊.對齊成功([
{'開始': 0.30, '長度': 0.23, '分詞': 'tsiang5', },
{'開始': 0.60, '長度': 0.23, '分詞': 'tsiang5', },
{'開始': 0.83, '長度': 0.23, '分詞': 'khiau2', },
])
with TemporaryDirectory() as 資料夾路徑:
wav路徑 = join(資料夾路徑, 'wav')
語料對齊.產生音檔(wav路徑)
self.assertEqual(len(listdir(資料夾路徑)), 1)
tar路徑 = join(資料夾路徑, 'tar')
語料對齊.壓縮音檔(wav路徑, tar路徑)
self.assertTrue(isfile(tar路徑))
