def testlhafile_read(self):
for lzhname in self.lzhnames:
lha = lhafile.Lhafile(lzhname)
files = [info.filename for info in lha.infolist()]
datafiles = self.datasets.keys()
datafiles.sort()
for filename in datafiles:
try:
norm_filename = os.sep.join(filename.split('/'))
data = lha.read(norm_filename)
except Exception as e:
self.assert_(False,
"Decode error happened in %s" % (filename, ))
if data == self.datasets[filename]:
continue
if len(data) != len(self.datasets[filename]):
self.assert_(
False, "Data length is not matched %s" % (filename, ))
elif data != self.datasets[filename]:
i = 0
while True:
if data[i] != self.datasets[filename][i]:
print(data[max(0, i - 10):i + 10], )
print(
self.datasets[filename][max(0, i - 10):i +
10], )
self.assert_(
False,
"Data mismatched in %s, %d" % (filename, i))
i += 1
def testlhafile_infolist(self):
for lzhname in self.lzhnames:
lha = lhafile.Lhafile(lzhname)
files = [info.filename for info in lha.infolist()]
for filename in self.datasets.keys():
filename = os.sep.join(filename.split('/'))
self.assert_(filename in files)
del files[files.index(filename)]
def execute(self):
if self.debug:
print "execute: unlhaSingleFile %s from %s to %s" % (
self.filename, self.fromfile, self.savename)
# 1: check if file exists
if os.path.exists(self.fromfile):
if lhafile.is_lhafile(self.fromfile):
f = lhafile.Lhafile(self.fromfile)
for name in f.namelist():
# need to sanitize savename, as lhafile munshes name and comment into name, seperated by \x00
splitmark = name.find("\x00")
if splitmark != -1:
savename = name[:splitmark]
# print savename
else:
savename = name
# only unpack the single wanted file
if savename == self.filename:
destination = self.savename
# 2: create required dirs
head, tail = os.path.split(destination)
if not os.path.exists(head):
if self.debug:
print "unlhaSingleFile: need to create directory %s" % head
shutil.os.makedirs(head)
# 3: get binary data
# print name
head, tail = os.path.split(name)
if len(tail) > 0:
content = f.read(
name) # todo: fix crash on "ProSIAK\"
if self.debug:
print "unlhaSingleFile: writing %s (%d bytes)" % (
destination, len(content))
# 4: save file
# print destination
try:
savefile = open(destination, "wb")
savefile.write(content)
savefile.close()
except:
pass
else:
if self.debug:
print "error: archive %s does not exist" % (self.fromfile)
def open_lha(target_file_path, target_date):
# Create Lhafile instance from filename
f = lhafile.Lhafile(target_file_path)
# Print each file informaion in archive file.
file_name = ''
for info in f.infolist():
# print(info.filename)
file_name = info.filename
open(setting.get_target_download_csv_file_path(target_date),
"wb").write(f.read(file_name))
f = None
try:
os.remove(target_file_path)
except:
print('---ERROR---Cannot delete the target LHA file:' +
target_file_path)
def unlzh(lzh_name):
"""Extract files under current directory"""
print("Extract", lzh_name, "...")
if not isinstance(lzh_name, Path):
raise ValueError("{} is not supported.".format(type(lzh_name)))
if not lzh_name.suffix == ".lzh":
raise ValueError("{} is not supported extension.".format(
lzh_name.suffix))
# make directory to extract
if not lzh_name.parent.exists():
lzh_name.parent.mkdir()
# open lzh file and get file names in it.
lha = lhafile.Lhafile(str(lzh_name))
# extract all files
for lha_info in lha.infolist():
file_path = lha_info.filename
print("extract", file_path)
with open(lzh_name.parent / file_path, "wb") as f:
f.write(lha.read(file_path))
def download(self, start, end):
period = pd.date_range(start, end)
for date in period:
# Get file from the website
dirname = date.strftime("%Y%m")
lzhname = date.strftime("%y%m%d")
uri = self.baseuri % (dirname, lzhname)
savename = "./data/results/lzh/%s.lzh" % lzhname
if not os.path.exists(savename):
print("Send request to", uri)
urllib.request.urlretrieve(uri, savename)
time.sleep(3)
unpackedpath = "./data/results/K%s.TXT" % lzhname
unpackedname = os.path.basename(unpackedpath)
if not os.path.exists(unpackedpath):
print("Unpacking", savename)
f = lhafile.Lhafile(savename)
data = f.read(unpackedname)
datastr = data.decode(encoding='shift-jis')
fileobj = open(unpackedpath, "w")
fileobj.write(datastr)
fileobj.close()
import lhafile
import os
import re
#https://trac.neotitans.net/wiki/lhafile/
#ディレクトリ内のファイル名一覧
file_list = os.listdir('timetable_lzh')
m = ""
f = ""
out = ""
name = ""
for m in file_list:
#拡張子がlzhなら
if re.search(".lzh", m):
#解凍する
f = lhafile.Lhafile("timetable_lzh/" + m)
info = f.infolist()
name = info[0].filename
out = open("timetable_txt/" + name, "wb").write(f.read(name))
print(name + "を作成しました")
import lhafile
import os
import re
#https://trac.neotitans.net/wiki/lhafile/
#ディレクトリ内のファイル名一覧
file_list = os.listdir('result_lzh')
m = ""
f = ""
out = ""
name = ""
for m in file_list:
#拡張子がlzhなら
if re.search(".lzh", m):
#解凍する
f = lhafile.Lhafile("result_lzh/" + m)
#解凍されたファイルの名前取得
info = f.infolist()
name = info[0].filename
#書き込む
out = open("result_txt/" + name, "wb").write(f.read(name))
print(name + "を作成しました")
import lhafile
import os
import re
#https://trac.neotitans.net/wiki/lhafile/
file_list = os.listdir('players_lzh')
m = ""
f = ""
out = ""
name = ""
for m in file_list:
if re.search(".lzh", m):
#解凍する
f = lhafile.Lhafile("players_lzh/" + m)
info = f.infolist()
name = info[0].filename
out = open("players_txt/" + name, "wb").write(f.read(name))