def startimport(daystr2):
log.info("start startimport=======")
jzdisp = {}
jzdispdir = rootpath+"/data/jzdisphourmysql/"+daystr2
jzclickdir = rootpath+"/data/jzclickhourmysql/"+daystr2
jzdispfout = "jzdisphourmysql-"+daystr2
jzclickfout = "jzclickhourmysql-"+daystr2
merge(jzdispdir,jzdispfout)
merge(jzclickdir,jzclickfout)
jzdispfile = jzdispdir + "/" + jzdispfout
jzclickfile = jzclickdir + "/" + jzclickfout
if getsize(jzdispfile)==0:
return
if getsize(jzclickfile)==0:
return
jzdisp = util.readfiles2mapCpcNew(daystr2,jzdispfile,jzclickfile,{})
log.info("jzdisp=%d"%(len(jzdisp.keys())))
mysqlhelper = MySQLHelper()
mysqlhelper.setstatconn()
pagesize = 100
#mysqlhelper.query("delete from t_jzcpcdisp_"+yyyymm+" where dispday like '"+daystr2+"%'")
#bench_insertdb(mysqlhelper,pagesize,jzdisp,sql,handlerValues_jzdisp)
bench_insertdb(mysqlhelper,pagesize,jzdisp,daystr2)
#mysqlhelper.close()
if os.path.exists(jzdispfile):
os.remove(jzdispfile)
if os.path.exists(jzclickfile):
os.remove(jzclickfile)
print "over.time=%s"%(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())))
def prepare(self):
if fileExists(self.srcfile, 'r') and fileExists(self.destfile, 'r'):
fsize1 = path.getsize(self.srcfile)
fsize2 = path.getsize(self.destfile)
self.srcsize = fsize1 + fsize2
self.ProgressTimer.start(7500, True)
self.toolbox.ptsFrontpanelActions('start')
def test_002_checkwavcopy(self):
infile = g_in_file
outfile = "test_out.wav"
wf_in = blocks.wavfile_source(infile)
wf_out = blocks.wavfile_sink(outfile,
wf_in.channels(),
wf_in.sample_rate(),
wf_in.bits_per_sample())
self.tb.connect(wf_in, wf_out)
self.tb.run()
wf_out.close()
# we're loosing all extra header chunks
self.assertEqual(getsize(infile) - g_extra_header_len, getsize(outfile))
in_f = file(infile, 'rb')
out_f = file(outfile, 'rb')
in_data = in_f.read()
out_data = out_f.read()
out_f.close()
os.remove(outfile)
# cut extra header chunks input file
self.assertEqual(in_data[:g_extra_header_offset] + \
in_data[g_extra_header_offset + g_extra_header_len:], out_data)
def dealFile(path):
dict = {}
if not os.path.exists(path):
return None
for root, dirs, files in os.walk(path):
for file in files :
if file.startswith(".DS_Store"):
pass
elif file.endswith("_base.xml"):
list = countItem(path + "/"+file)
try:
dict.update({'baseSize':getsize(path + "/"+ file)})
except Exception, e:
dict.update({'baseSize': '0'})
dict.update({"base_count": list[0]})
dict.update({"base_tag_count": list[1]})
elif file.endswith("_ext.xml"):
list = countItem(path + "/"+file)
try:
dict.update({'extSize':getsize(path + "/"+ file)})
except Exception, e:
dict.update({'extSize':'0'})
dict.update({"ext_count": list[0]})
dict.update({"ext_tag_count": list[1]})
def compress(sources):
"""
Compress a file/directory using 7zip.
If the gain is not large enough, remove the archive, otherwise remove
the source.
"""
dest = "{0}.{1}".format(file_without_ext(sources[0]), extension)
seven_zip(sources, dest)
sources_size = 0
for s in sources:
sources_size += path.getsize(s)
dest_size = path.getsize(dest)
gain = sources_size - dest_size
if gain < sources_size * min_ratio: # Not enought gain
os.unlink(dest)
return 0
else:
for s in sources:
os.unlink(s)
return gain
def calcsize(file):
if not isdir(file):
return getsize(file)
total = 0L
for s in subfiles(abspath(file)):
total += getsize(s[1])
return total
def test_write_truncate(self):
knowngood = {}
filelist = list(self.pak2.listfiles())
for f in filelist:
self.pak2.open(f)
data = self.pak2.read()
knowngood[f] = [len(data), md5(data).hexdigest()]
size = getsize(self.filename2)
buf = "123456789"
bufmd5 = md5(buf).hexdigest()
for i in xrange(0, len(filelist), 2):
self.pak2.open(filelist[i], "r")
size -= len(self.pak2.read())
self.pak2.close()
self.pak2.open(filelist[i], "w")
self.pak2.write(buf)
self.pak2.close()
size += len(buf)
knowngood[filelist[i]][0] = len(buf)
knowngood[filelist[i]][1] = bufmd5
for f in filelist:
self.assertEqual(filelist, list(self.pak2.listfiles()))
self.pak2.open(f)
data = self.pak2.read()
self.assertEqual(len(data), knowngood[f][0])
self.assertEqual(md5(data).hexdigest(), knowngood[f][1])
del self.pak2
self.assertEqual(getsize(self.filename2), size)
def test_run_summarize_taxa_through_plots(self):
""" run_summarize_taxa_through_plots generates expected results
"""
run_summarize_taxa_through_plots(
self.test_data['biom'][0],
self.test_data['map'][0],
self.test_out,
mapping_cat=None,
sort=False,
command_handler=call_commands_serially,
params=self.params,
qiime_config=self.qiime_config,
status_update_callback=no_status_updates)
# Check that summarized taxonomy files have non-zero size
input_file_basename = splitext(split(self.test_data['biom'][0])[1])[0]
for i in [2, 3, 4, 5, 6]:
sum_taxa_file = join(self.test_out, input_file_basename + '_L%s.txt'
% (str(i)))
self.assertTrue(getsize(sum_taxa_file) > 0)
# Check the html files are generated
self.assertTrue(getsize(join(self.test_out, 'taxa_summary_plots',
'area_charts.html')) > 0)
self.assertTrue(getsize(join(self.test_out, 'taxa_summary_plots',
'area_charts.html')) > 0)
# Check that the log file is created and has size > 0
log_fp = glob(join(self.test_out, 'log*.txt'))[0]
self.assertTrue(getsize(log_fp) > 0)
def down_book(down_url,url,book_name,local_dir):
"""
下载书籍，下载过程中后缀.tmp,下载完成判断大小，如果满足文件大小70%则认为成功并修改文件后缀
"""
book_name = validatename(book_name)
# book_name = book_name.replace("/","")
down_dir_tmp = str(local_dir) + str(book_name)+ ".pdf"+".tmp"
down_dir = str(local_dir) + str(book_name)+ ".pdf"
if os.path.exists(down_dir) == True and abs(round(float(getsize(down_dir))/1024/1024,1) - round(float(size.replace(' MB',"")),1)) < 1: #判断书籍是否被下载过，如已存在文件则跳过
#sys.exit()
print ("....<"+book_name+"> already exists...")
logging.info("....Books already exists....")
elif os.path.exists(down_dir_tmp) == True or os.path.exists(down_dir) == False:
if os.path.exists(down_dir_tmp) == True:
print "...ReDownloading <"+book_name+">..."
print "Original Size: "+size
os.remove(down_dir_tmp)
else:
print "...Downloading <"+book_name+">..."
print "Original Size: "+size
rp = requests.get(down_url,headers = {'Referer':url},allow_redirects = False)
r = requests.get(rp.headers['location'])
with open(down_dir_tmp, "wb") as code:
code.write(r.content)
print "Actual Size: "+str(round(float(getsize(down_dir_tmp))/1024/1024,1))+" MB"
if abs(round(float(getsize(down_dir_tmp))/1024/1024,1)
- round(float(size.replace(' MB',"")),1))/round(float(size.replace(' MB',"")),1) < 0.3:#此处可调整，如果下载不到原有的70%认为没下载成功
os.rename(down_dir_tmp,down_dir)
def repackage(self, extracted_apk_dir, dex_dir, have_locators):
shutil.move(join(dex_dir, 'classes.dex'), extracted_apk_dir)
jar_meta_path = join(dex_dir, 'metadata.txt')
with open(jar_meta_path, 'w') as jar_meta:
if have_locators:
jar_meta.write('.locators\n')
for i in range(1, 100):
oldpath = join(dex_dir, 'classes%d.dex' % (i + 1))
dexpath = join(dex_dir, 'secondary-%d.dex' % i)
if not isfile(oldpath):
break
shutil.move(oldpath, dexpath)
jarpath = dexpath + '.jar'
create_dex_jar(jarpath, dexpath)
dex_meta_base = 'secondary-%d.dex.jar.meta' % i
dex_meta_path = join(dex_dir, dex_meta_base)
with open(dex_meta_path, 'w') as dex_meta:
dex_meta.write('jar:%d dex:%d\n' %
(getsize(jarpath), getsize(dexpath)))
with open(jarpath, 'rb') as jar:
sha1hash = hashlib.sha1(jar.read()).hexdigest()
shutil.move(dex_meta_path,
join(extracted_apk_dir, self._secondary_dir))
shutil.move(jarpath, join(extracted_apk_dir,
self._secondary_dir))
jar_meta.write(
'secondary-%d.dex.jar %s secondary.dex%02d.Canary\n'
% (i, sha1hash, i))
shutil.move(jar_meta_path, join(extracted_apk_dir, self._secondary_dir))
def comp (source_patch, target_patch): from os import path, walk from filecmp import cmp # выходное сообщение о найденных отличиях message = '' path_f = [] tree = walk(source_patch) for d, dirs, files in tree: for f in files: patch = path.join(d,f) # формирование адреса path_f.append(patch) # добавление адреса в список # перибираем адреса файлов из списка for patch in path_f: # выполняем сравнение файлов и в случае отличий получаем информацию о файлах # проверяем существование файла if not path.exists(patch.replace(source_patch, target_patch)): message = message + 'Отсутствует целевой файл: '+ patch.replace(source_patch, target_patch) # сверяем размеры исходного и целевого файла elif path.getsize(patch.replace(source_patch, target_patch)) <> path.getsize(patch): message = message + file_info(patch, patch.replace(source_patch, target_patch)) # дата последней модификации elif path.getmtime(patch.replace(source_patch, target_patch)) <> path.getmtime(patch): message = message + file_info(patch, patch.replace(source_patch, target_patch)) # сравниваем файлы elif not cmp(patch.replace(source_patch, target_patch), patch): message = message + file_info(patch, patch.replace(source_patch, target_patch)) return message
def getchunks(infile, n_cpus, scheduler = 'guided'):
# Divide input data based on scheduler type
if scheduler == 'static':
size = getsize(infile) / n_cpus
else:
size = getsize(infile) / (n_cpus * 20)
# Open input file
try:
ifile = open(infile)
except:
print >> stderr, 'Error: Not able to open ', infile, '. Exiting.'
exit(-1)
# Create chunk of data to be distributed to nodes
while 1:
start = ifile.tell()
ifile.seek(size, 1)
s = ifile.readline()
yield start, ifile.tell() - start
if not s:
break
# Close the input file
try:
ifile.close()
except:
print >> stderr, 'Warning: Error closing the file ', ifile
def can_overwrite_old_file(old, new):
"""returns true if the old file does not exist and the new file is not significantly smaller"""
if not PATH.exists(old):
return True
if not PATH.exists(new):
return False
return PATH.getsize(new) > PATH.getsize(old) * 0.7
def processfile(self, filename):
"""Renames the specified image to a backup path,
and writes out the image again with optimal settings."""
try:
# Skip read-only files
if not stat(filename)[0] & S_IWRITE:
print 'Ignoring read-only file "' + filename + '".'
return False
# Create a backup
backupname = filename + "." + self.backupextension
if isfile(backupname):
print 'Ignoring file "' + filename + '" for which existing backup file is present.'
return False
rename(filename, backupname)
except Exception as e:
stderr.write('Skipping file "' + filename + '" for which backup cannot be made: ' + str(e) + "\n")
return False
ok = False
try:
# Open the image
with open(backupname, "rb") as file:
img = Image.open(file)
# Check that it's a supported format
format = str(img.format)
if format != "PNG" and format != "JPEG":
print 'Ignoring file "' + filename + '" with unsupported format ' + format
return False
# This line avoids problems that can arise saving larger JPEG files with PIL
ImageFile.MAXBLOCK = img.size[0] * img.size[1]
# The 'quality' option is ignored for PNG files
img.save(filename, quality=90, optimize=True)
# Check that we've actually made it smaller
origsize = getsize(backupname)
newsize = getsize(filename)
if newsize >= origsize:
print 'Cannot further compress "' + filename + '".'
return False
# Successful compression
ok = True
except Exception as e:
stderr.write('Failure whilst processing "' + filename + '": ' + str(e) + "\n")
finally:
if not ok:
try:
move(backupname, filename)
except Exception as e:
stderr.write('ERROR: could not restore backup file for "' + filename + '": ' + str(e) + "\n")
return ok
def command_patch_list_data(self):
requested_version = (self.reported_version + 1)
while requested_version in self.patch_catalog.getCatalog()[self.reported_client]:
p_file = path.join(self.patch_catalog.getCatalog()['path'],
self.patch_catalog.getCatalog()[self.reported_client][requested_version] + '.pat')
p_head, p_tail = path.split(p_file)
r_file = path.join(self.patch_catalog.getCatalog()['path'],
self.patch_catalog.getCatalog()[self.reported_client][requested_version] + '.rtp')
r_head, r_tail = path.split(r_file)
file_listing = {'patname':p_tail.encode('ascii'),
'patnamelen':pack('>i',
len(p_tail.encode('ascii'))),
'patlen':pack('>i',
path.getsize(p_file)),
'rtpname':r_tail.encode('ascii'),
'rtpnamelen':pack('>i',
len(r_tail.encode('ascii'))),
'rtplen':pack('>i',
path.getsize(r_file))
}
self.connection.send(file_listing['patnamelen'])
self.connection.send(file_listing['patname'])
self.connection.send(file_listing['patlen'])
self.connection.send(file_listing['rtpnamelen'])
self.connection.send(file_listing['rtpname'])
self.connection.send(file_listing['rtplen'])
print("\nSERVER << sent %s PatchListData entry %s\n" % (self.address, file_listing))
requested_version = requested_version + 1
self.connection.send(b'\x00\x00\x00\x00')
return
def add(self, mydata):
self.name = mydata
if os.path.isfile(mydata):
self.file = open(mydata, 'rb').read()
self.size = str(int(os.path.getsize(mydata)))
self.type = 'blob'
elif os.path.isdir(mydata):
self.file = []
# Print a list of filenames and their sizes.
for root, dirs, files in os.walk(mydata):
self.size = sum(int(getsize(join(root, name))) for name in files)
for name in files:
self.file.append([root, name, getsize(join(root, name))])
if '.git' in dirs:
dirs.remove('.git') # don't visit .git directories
self.type = 'tree'
self.mode = '10' + oct(os.stat(mydata)[0] & 0777)
# Header - different to Git
self.header = self.type + "*42*" + str(self.size) + "moose" + "\0"
# Calculate the SHA1 hexdigest.
self.stuff = self.header + str(self.file)
self.sha1 = hashlib.sha1(self.stuff).hexdigest()
self.dirname = self.sha1[0:2]
self.blobname = self.sha1[2:41]
# ( To do - calculate the SHA1s for a "tree" and a "commit". )
# Save in our dict.
self.data.update({ self.blobname: [self.dirname, self.name,
self.size, self.mode ] })
def duplicate(original, new, other):
if not exists(new):
return False
if new == original:
del_file(original)
return True
try:
if DadVision.args.force_rename or (other and other.lower() in ["proper"]):
log.warn("Replacing Existing with repack or Force Rename Requested: {}".format(original))
del_file(new)
return False
if getsize(original) > getsize(new):
log.info("Replacing Existing with Larger Version: {}".format(new))
del_file(new)
return False
log.trace("Comparing existing file to new, may run for some time.")
if filecmp.cmp(original, new):
log.info("Deleting New File, Same File already at destination!: {}".format(new))
del_file(original)
return True
del_file(new)
return False
except (InvalidPath, OSError), e:
log.error("Unable to remove File: {}".format(e))
return True
def folderDigger(parNR, directory, name):
global BadOnes
BadOnes=""
size = 0
subNR = parNR+1
for item in os.listdir(directory):
if not os.path.isdir(directory+"/"+item):
if os.path.isfile(directory+"/"+item):
BadOnes= BadOnes+"* "+item+" ("+directory+")\n"
for folder in os.listdir(directory):
if os.path.isdir(directory+"/"+folder):
try:
size = size + folderDigger(subNR, (directory+"/"+folder), folder)
except:
pass
for fil in os.listdir(directory):
if os.path.isfile(directory+"/"+fil):
size = size+getsize(directory+"/"+fil)
if showFiles:
pf(subNR+1, getsize(directory+"/"+fil), fil, directory)
pf(subNR, size, name, directory)
return size
def _get_file_details(self, file_name):
label = None
details = {
'path': path.realpath(file_name),
'sha1': self._get_sha1(file_name)
}
if path.isfile(file_name):
label, ext = path.splitext(path.basename(file_name))
if ext.lower() == '.mrxs':
details.update({
'name': label,
'mimetype': self.INDEX_FILE_MT,
'size': path.getsize(file_name)
})
else:
label, details = None, None
elif path.isdir(file_name):
label = path.basename(file_name)
details.update({
'name': label,
'mimetype': self.DATA_FOLDER_MT,
'size': sum([path.getsize(path.join(file_name, f))
for f in listdir(file_name)]),
})
self.logger.debug('Details for file %s: %r', file_name, details)
return label, details
def dir_cache_data(path):
"""
Return the data to store in the cache for directory at *path*.
"""
path = force_utf8(path)
files = []
directories = []
for entry in os.listdir(path):
for pattern in settings.EXCLUDE_FILES:
if pattern.match(entry):
continue
entry_path = op.join(path, entry)
if not op.exists(entry_path):
# File was deleted during directory listing
continue
timestamp = op.getmtime(entry_path)
if op.isdir(entry_path):
size = 0
for dirpath, dirnames, filenames in os.walk(entry_path):
for f in filenames:
fp = op.join(dirpath, f)
if op.exists(fp):
size += op.getsize(fp)
directories.append((entry, size, timestamp))
else:
size = op.getsize(entry_path)
files.append((entry, size, timestamp))
return directories, files
def createDict(path, root={}):
pathList = listdir(path)
for i, item in enumerate(pathList):
file_path = path_join(path, item)
if item not in ignore_dir and exists(file_path):
if isdir(file_path):
if not root.get(item, False):
root[item] = {"type": "dir", "files": {}}
createDict(file_path, root[item]["files"])
else:
if not root.get(item, False):
log("new file " + file_path)
root[item] = {"type": "file",
"file_size": getsize(file_path),
"mtime": getmtime(file_path),
"ctime": getctime(file_path),
"md5": md5(file_path),
"sha256": sha256(file_path)}
else:
if root[item]["mtime"] != getmtime(file_path):
log("rehashing " + file_path)
root[item] = {"type": "file",
"file_size": getsize(file_path),
"mtime": getmtime(file_path),
"ctime": getctime(file_path),
"md5": md5(file_path),
"sha256": sha256(file_path)}
return root
def disabled_test_xml_quiet(self):
""" Tests the 'quiet' parameter of the MARC8ToUnicode class,
passed in via the pymarc.record_to_xml() method
"""
outfile = 'test/dummy_stderr.txt'
# truncate outfile in case someone's fiddled with it
open(outfile, 'wb').close()
# redirect stderr
sys.stderr = open(outfile, 'wb')
# reload pymarc so it picks up the new sys.stderr
reload(pymarc)
# get problematic record
record = next(pymarc.reader.MARCReader(open('test/utf8_errors.dat', 'rb')))
# record_to_xml() with quiet set to False should generate errors
# and write them to sys.stderr
xml = pymarc.record_to_xml(record, quiet=False)
# close dummy stderr so we can accurately get its size
sys.stderr.close()
# file size should be greater than 0
self.assertNotEqual(getsize(outfile), 0)
# truncate file again
open(outfile, 'wb').close()
# be sure its truncated
self.assertEqual(getsize(outfile), 0)
# redirect stderr again
sys.stderr = open(outfile, 'wb')
reload(pymarc)
# record_to_xml() with quiet set to True should not generate errors
xml = pymarc.record_to_xml(record, quiet=True)
# close dummy stderr
sys.stderr.close()
# no errors should have been written
self.assertEqual(getsize(outfile), 0)
def repackage(self, extracted_apk_dir, dex_dir, have_locators):
BaseDexMode.repackage(self, extracted_apk_dir, dex_dir, have_locators)
metadata = DexMetadata(have_locators=have_locators,
store=self._store_id,
dependencies=self._dependencies)
for i in range(1, 100):
oldpath = join(dex_dir, self._dex_prefix + '%d.dex' % (i + 1))
dexpath = join(dex_dir, self._store_name + '-%d.dex' % i)
if not isfile(oldpath):
break
shutil.move(oldpath, dexpath)
jarpath = dexpath + '.jar'
create_dex_jar(jarpath, dexpath)
metadata.add_dex(jarpath, BaseDexMode.get_canary(self, i))
dex_meta_base = jarpath + '.meta'
dex_meta_path = join(dex_dir, dex_meta_base)
with open(dex_meta_path, 'w') as dex_meta:
dex_meta.write('jar:%d dex:%d\n' %
(getsize(jarpath), getsize(dexpath)))
shutil.move(dex_meta_path,
join(extracted_apk_dir, self._secondary_dir))
shutil.move(jarpath, join(extracted_apk_dir,
self._secondary_dir))
jar_meta_path = join(dex_dir, 'metadata.txt')
metadata.write(jar_meta_path)
shutil.move(jar_meta_path, join(extracted_apk_dir, self._secondary_dir))
def build_project(project, _zip, _url, _alias, _replace, _create, _option):
"""Build project."""
if _option:
project.properties = flatten(project.properties)
# to make sure we properly override nested options, we flatten first
project.properties.update(_parse_option(_option))
if _zip:
if osp.isdir(_zip):
_zip = osp.join(_zip, '%s.zip' % (project.versioned_name, ))
project.build(_zip, overwrite=_replace)
sys.stdout.write(
'Project %s successfully built and saved as %r (size: %s).\n'
% (project, _zip, human_readable(osp.getsize(_zip)))
)
else:
with temppath() as _zip:
project.build(_zip)
archive_name = '%s.zip' % (project.versioned_name, )
session = _get_session(_url, _alias)
res = _upload_zip(session, project.name, _zip, _create, archive_name)
sys.stdout.write(
'Project %s successfully built and uploaded '
'(id: %s, size: %s, upload: %s).\n'
'Details at %s/manager?project=%s\n'
% (
project,
res['projectId'],
human_readable(osp.getsize(_zip)),
res['version'],
session.url,
project,
)
)
def build_project(project, zip, url, alias, replace, create):
"""Build project."""
if zip:
project.build(zip, overwrite=replace)
stdout.write(
'Project successfully built and saved as %r (size: %s).\n'
% (zip, human_readable(getsize(zip)))
)
else:
with temppath() as zip:
project.build(zip)
session = Session(url, alias)
while True:
try:
res = session.upload_project(project.name, zip)
except AzkabanError as err:
if create:
session.create_project(project.name, project.name)
else:
raise err
else:
break
stdout.write(
'Project %s successfully built and uploaded '
'(id: %s, size: %s, version: %s).\n'
'Details at %s/manager?project=%s\n'
% (
project,
res['projectId'],
human_readable(getsize(zip)),
res['version'],
session.url,
project,
)
)
def _calcsize(file_to_torrent):
if not isdir(file_to_torrent):
return getsize(file_to_torrent)
total = 0
for s in _subfiles(abspath(file_to_torrent)):
total += getsize(s[1])
return total
def compressFile(self,fName):
if self.verbose>1:
print_(" Compressing",fName)
zippedName=fName+".gz"
if path.exists(zippedName):
self.warning("Zipped file",zippedName,"already existing for",fName)
return
oldSize=path.getsize(fName)
if oldSize<self.bigSize:
if self.verbose>2:
print_(" Skipping because it is too small")
self.nrSkipped+=1
return
# use gzip because that way the responsibility of removing the old file is with a 'tried and testd' program
ret=subprocess.call(["gzip",fName])
if ret!=0 or not path.exists(zippedName) or path.exists(fName):
self.warning("Problem compressing file",fName)
self.nrProblems+=1
return
newSize=path.getsize(zippedName)
if newSize>oldSize:
self.warning("Compression of",fName,"increased the filesize. Old:",
humanReadableSize(oldSize),"New:",humanReadableSize(newSize))
if self.verbose>2:
print_(" Old size:",humanReadableSize(oldSize),"New size:",humanReadableSize(newSize))
self.nrFiles+=1
self.prevSize+=oldSize
self.nowSize+=newSize
def run(self, filename, pipedata):
if 'optimize' in pipedata and not pipedata['optimize']:
return
self.bakery.logging_raw('### Optimize TTF {}'.format(filename))
# copied from https://code.google.com/p/noto/source/browse/nototools/subset.py
from fontTools.subset import Options, Subsetter, load_font, save_font
options = Options()
options.layout_features = "*"
options.name_IDs = "*"
options.hinting = True
options.notdef_outline = True
font = load_font(op.join(self.builddir, filename), options)
subsetter = Subsetter(options=options)
subsetter.populate(glyphs=font.getGlyphOrder())
subsetter.subset(font)
save_font(font, op.join(self.builddir, filename + '.opt'), options)
newsize = op.getsize(op.join(self.builddir, filename + '.opt'))
origsize = op.getsize(op.join(self.builddir, filename))
# compare filesizes TODO print analysis of this :)
comment = "# look at the size savings of that subset process"
self.bakery.logging_cmd("ls -l '%s'* %s" % (filename, comment))
statusmessage = "{0}.opt: {1} bytes\n{0}: {2} bytes\n"
self.bakery.logging_raw(statusmessage.format(filename, newsize, origsize))
# move ttx files to src
shutil.move(op.join(self.builddir, filename + '.opt'),
op.join(self.builddir, filename),
log=self.bakery.logger)
def compare_dns_dirs():
print(' Checking if there is need for an update .... ')
#first check to see if .ip_tmp_path exists
if ( path.exists('.ip_tmp_path') and (path.isdir('.ip_tmp_path')) ):
print(' Give me just a few seconds more')
sleep(2)
if ( int(path.getsize('.ip_tmp')) <= int(path.getsize('.ip_tmp_path')) ):
print(' \n Looks like new content is available ')
# copying new content in .dns_tmp_path to .dns_tmp
try:
rmtree('.dns_tmp')
copytree('.dns_tmp_path','.dns_tmp')
except:
print(' Failed to copy new data ... ')
print(' Exiting ... ')
exit(0)
else:
print(' Successfully moved new data')
else:
print(' Nothing new was added ... ')
print(' Exiting ... ')
exit(0)
else:
print(' This is first run ... \n moving on ... ')
sleep(2)
def unpack_file_from_file(in_file, out_file):
""" Uncompress a file from a file.
Parameters
----------
in_file : str
the name of the input file
out_file : str
the name of the output file
Returns
-------
metadata : bytes
the metadata contained in the file if present
Raises
------
FormatVersionMismatch
if the file has an unmatching format version number
ChecksumMismatch
if any of the chunks fail to produce the correct checksum
"""
in_file_size = path.getsize(in_file)
log.verbose('input file size: %s' % pretty_size(in_file_size))
with open(in_file, 'rb') as input_fp, open(out_file, 'wb') as output_fp:
source = CompressedFPSource(input_fp)
sink = PlainFPSink(output_fp, source.nchunks)
unpack(source, sink)
out_file_size = path.getsize(out_file)
log.verbose('output file size: %s' % pretty_size(out_file_size))
log.verbose('decompression ratio: %f' % (out_file_size / in_file_size))
return source.metadata
def _get_size(self):
try:
return commatize(getsize(self.file_name)) + ' bytes'
except:
return ''
sha = hashlib.sha256()
with open(filepath, 'rb') as fp:
while True:
data = fp.read(blocksize)
if not data:
break
sha.update(data)
return sha.hexdigest()
# --- /helpers ---
write("""\
%%%% HASHDEEP-1.0
%%%% size,sha256,filename
##
## $ hashdeep.py
##""")
ROOT = '.'
for root, dirs, files in os.walk(ROOT):
for fpath in [osp.join(root, f) for f in files]:
size = osp.getsize(fpath)
sha = filehash(fpath)
name = osp.relpath(fpath, ROOT)
write('%s,%s,%s' % (size, sha, name))
#for ignored in ['.hg', '.svn', 'git']:
# if ignored in dirs:
# dirs.remove(ignored)
e.show(f).grid(column=0,row=r)
return f
def __repr__(self) :
s = "Agenda: " + self.name + '\n\t'
for e in self :
s += str(e) + '\n'
return s
def clear(container) :
for w in container.winfo_children() :
w.destroy()
#----------------------------------------------------MAIN----------------------------------------------------------------
if path.exists(pathfile) and path.getsize(pathfile) > 0 :
try :
with open(pathfile,"rb") as readfile:
calendar = [ a.depklize() for a in pickle.load(readfile) ]
#wtfisgoinon("read 'STRICKYlist.pk' in 'listaEventi'")
except Exception as e :
raise e
else :
#default
calendar=[Agenda("test")]
calendar[0].insertEvent(Agenda_Event())
#structure of the main window
row_messages = tkinter.Frame(rootwindow)
row_commands = tkinter.Frame(rootwindow)
row_display = tkinter.Frame(rootwindow)
async def nem(context):
proxies = {}
proxynum = 0
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063',
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,"
"application/signed-exchange;v=b3;q=0.9",
"X-Real-IP":
"223.252.199.66"
}
proxy = [{
'http': 'http://music.lolico.me:39000',
'https': 'http://music.lolico.me:39000'
}, {
'http': 'http://netease.unlock.feiwuis.me:6958',
'https': 'https://netease.unlock.feiwuis.me:6958'
}]
helptext = "**使用方法:** `-nem` `<指令>` `<关键词>`\n\n指令s为搜索，p为点歌，id为歌曲ID点歌，r为随机热歌(无关键词)\n搜索在s后添加数字如`-nem` `s8` " \
"`<关键词>`调整结果数量\n搜索灰色歌曲请尽量**指定歌手**\n可回复搜索结果消息`-nem` `p` `<歌曲数字序号>`点歌 "
apifailtext = "出错了呜呜呜 ~ 试了好多好多次都无法访问到 API 服务器 。"
if len(context.parameter) < 2:
if len(context.parameter) == 0:
await context.edit(helptext)
return
elif context.parameter[0] == "r": # 随机热歌
await context.edit("随机中 . . .")
for _ in range(20): # 最多重试20次
apinum = random.randint(0, 1)
apitype = random.randint(0, 1)
if apitype == 0:
apitype = "飙升榜"
else:
apitype = "热歌榜"
if apinum == 0:
url = "https://api.vvhan.com/api/rand.music?type=json&sort=" + apitype
else:
url = "https://api.uomg.com/api/rand.music?sort=" + apitype + "&format=json"
status = False
try:
randsong = requests.get(url, headers=headers)
if randsong.status_code == 200:
randsong = json.loads(randsong.content)
if apinum == 0 and randsong['success'] is True:
context.parameter[0] = "id"
context.parameter.append(
str(randsong['info']['id']))
status = True
break
elif apinum == 1 and randsong['code'] == 1:
context.parameter[0] = "id"
context.parameter.append(
str(randsong['data']['url'][45:]))
status = True
break
except:
continue
if status is False:
await context.edit(apifailtext)
sleep(3)
await context.delete()
return
else: # 错误输入
await context.edit(helptext)
return
# 整理关键词
keyword = ''
for i in range(1, len(context.parameter)):
keyword += context.parameter[i] + " "
keyword = keyword[:-1]
idplay = False
if context.parameter[0] == "id": # ID点歌功能
if len(context.parameter) > 2:
await context.edit(helptext)
return
idplay = keyword
context.parameter[0] = "p"
if context.parameter[0][0] == "s": # 搜索功能
await context.edit(f"【{keyword}】搜索中 . . .")
if len(context.parameter[0]) > 1:
limit = str(context.parameter[0][1:])
else:
limit = "5"
url = "http://music.163.com/api/search/pc?&s=" + \
keyword + "&offset=0&limit=" + limit + "&type=1"
for _ in range(20): # 最多尝试20次
status = False
req = requests.request("GET", url, headers=headers)
if req.status_code == 200:
req = json.loads(req.content)
if req['code'] == 200:
result = req['result']
if req['result']['songCount'] == 0:
result = False
else:
result = False
if result:
info = defaultdict()
for i in range(len(req['result']['songs'])):
info[i] = {
'id': '',
'title': '',
'alias': '',
'album': '',
'albumpic': '',
'artist': ''
}
info[i]['id'] = req['result']['songs'][i]['id']
info[i]['title'] = req['result']['songs'][i]['name']
info[i]['alias'] = req['result']['songs'][i]['alias']
info[i]['album'] = req['result']['songs'][i]['album'][
'name']
info[i]['albumpic'] = req['result']['songs'][i][
'album']['picUrl']
for j in range(
len(req['result']['songs'][i]['artists'])):
info[i]['artist'] += req['result']['songs'][i][
'artists'][j]['name'] + " "
text = f"<strong>关于【{keyword}】的结果如下</strong> \n"
for i in range(len(info)):
text += f"#{i+1}： \n<strong>歌名</strong>： {info[i]['title']}\n"
if info[i]['alias']:
text += f"<strong>别名</strong>： <i>{info[i]['alias'][0]}</i>\n"
if info[i]['album']:
res = '<a href="' + \
info[i]['albumpic'] + '">' + \
info[i]['album'] + '</a>'
text += f"<strong>专辑</strong>： {res} \n"
text += f"<strong>作者</strong>： {info[i]['artist']}\n<strong>歌曲ID</strong>： <code>{info[i]['id']}</code>\n————————\n"
text += "\n<strong>回复此消息</strong><code>-nem p <歌曲序号></code><strong>即可点歌</strong>"
await context.edit(text,
parse_mode='html',
link_preview=True)
status = True
break
else:
await context.edit("**未搜索到结果**")
sleep(3)
await context.delete()
status = True
break
else:
continue
if status is False:
await context.edit(apifailtext)
sleep(3)
await context.delete()
return
elif context.parameter[0] == "p": # 点歌功能
try:
reply = await context.get_reply_message()
except ValueError:
await context.edit("出错了呜呜呜 ~ 无效的参数。")
return
search = ""
title = ""
if reply:
msg = reply.message
search = re.findall(".*【(.*)】.*", msg)
if search:
try:
start = "#" + context.parameter[1] + "："
search = ".*" + start + "(.*?)" + '————————' + ".*"
msg = re.findall(search, msg, re.S)[0]
search = ".*歌曲ID： (.*)\n.*"
title = ".*歌名： (.*?)\n.*"
title = "【" + re.findall(title, msg, re.S)[0] + "】"
idplay = re.findall(search, msg, re.S)[0]
if reply.sender.is_self:
await reply.edit(f"{title}点歌完成")
except:
await context.edit("出错了呜呜呜 ~ 无效的歌曲序号。")
return
else:
await context.edit("出错了呜呜呜 ~ 无效的参数。")
return
await context.edit("获取中 . . .")
try:
import eyed3
imported = True
except ImportError:
imported = False
await bot.send_message(
context.chat_id,
'(`eyeD3`支持库未安装，歌曲文件信息将无法导入\n请使用 `-sh` `pip3` `install` `eyed3` '
'安装，或自行ssh安装)')
url = "http://music.163.com/api/search/pc?&s=" + \
keyword + "&offset=0&limit=1&type=1"
for _ in range(20): # 最多尝试20次
status = False
if proxynum > (len(proxy) - 1): # 代理自动切换至下一个
proxynum = 0
proxies = proxy[proxynum]
proxynum += 1
if idplay: # 指定ID播放
url = "http://music.163.com/api/song/detail?id=" + \
idplay + "&ids=[" + idplay + "]"
# 搜索后播放
req = requests.request("GET", url, headers=headers)
if req.status_code == 200:
req = json.loads(req.content)
if req['code'] == 200:
if idplay:
req['result'] = req
result = req['result']
if not idplay:
if req['result']['songCount'] == 0:
result = False
else:
result = False
if result:
info = {
'id': req['result']['songs'][0]['id'],
'title': req['result']['songs'][0]['name'],
'alias': req['result']['songs'][0]['alias'],
'album': req['result']['songs'][0]['album']['name'],
'albumpic':
req['result']['songs'][0]['album']['picUrl'],
'artist': '',
'br': ''
}
if req['result']['songs'][0]['hMusic']:
info['br'] = req['result']['songs'][0]['hMusic'][
'bitrate']
elif req['result']['songs'][0]['mMusic']:
info['br'] = req['result']['songs'][0]['mMusic'][
'bitrate']
elif req['result']['songs'][0]['lMusic']:
info['br'] = req['result']['songs'][0]['lMusic'][
'bitrate']
for j in range(len(req['result']['songs'][0]['artists'])):
info['artist'] += req['result']['songs'][0]['artists'][
j]['name'] + "; "
info['artist'] = info['artist'][:-2]
if title:
title = ""
else:
title = f"【{info['title']}】"
await context.edit(f"{title}下载中 . . .")
try:
from Crypto.Cipher import AES
AES.new("0CoJUm6Qyw8W8jud".encode('utf-8'),
AES.MODE_CBC,
"0102030405060708".encode('utf-8'))
ccimported = True
except ImportError:
ccimported = False
await bot.send_message(
context.chat_id,
'(`PyCryptodome`支持库未安装，音乐曲库/音质受限\n请使用 `-sh` `pip3` '
'`install` `pycryptodome` 安装，或自行ssh安装)')
name = info['title'].replace('/', " ") + ".mp3"
name = name.encode('utf-8').decode('utf-8')
if ccimported: # 尝试使用高清音质下载
songid = str(info['id'])
class WangyiyunDownload(object):
def __init__(self):
self.key = '0CoJUm6Qyw8W8jud'
self.public_key = "010001"
self.modulus = '00e0b509f6259df8642dbc35662901477df22677ec152b5ff68ace615bb7b72515' \
'2b3ab17a876aea8a5aa76d2e417629ec4ee341f56135fccf695280104e0312ecbda92' \
'557c93870114af6c9d05c4f7f0c3685b7a46bee255932575cce10b424d813cfe4875d' \
'3e82047b97ddef52741d546b8e289dc6935b3ece0462db0a22b8e7 '
# 偏移量
self.iv = "0102030405060708"
# 请求头
self.headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ('
'KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36',
# 传入登录cookie,
'Cookie':
'MUSIC_U=f52f220df171da480dbf33ce899479615'
'85a7fdf08c89a2a4bdd6efebd86544233a649814e309366;',
"X-Real-IP":
"223.252.199.66",
}
# 请求url
self.url = 'https://music.163.com/weapi/song/enhance/player/url/v1?csrf_token='
# 生成16位随机数字符串
def set_random_num(self):
random_num = ''
# 随机取16个字符
string = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
for ___ in range(16):
n = math.floor(
random.uniform(0, 1) * len(string))
random_num += string[n]
# 返回16位随机数字符串
return random_num
# 生成encSecKey
# 通过public_key和modulus对random_num进行RSA加密
def RSA_encrypt(self, random_num):
# 将16位随机数字符串倒序并以utf-8编码
random_num = random_num[::-1].encode('utf-8')
# 将其以hex(16进制)编码
random_num = codecs.encode(
random_num, 'hex_codec')
# 加密(三者均要从16进制转换为10进制)
# int(n, 16) --> 将16进制字符串n转换为10进制
encryption = int(random_num, 16)**int(
self.public_key, 16) % int(
self.modulus, 16)
# 将加密后的数据转换为16进制字符串
encryption = format(encryption, 'x')
# 返回加密后的字符串
return encryption
# 生成params
# 根据key和iv对msg进行AES加密,需调用两次
# key:
# 第一次: key
# 第二次: random_num
# iv: 偏移量iv
def AES_encrypt(self, msg, key, iv):
# 先将msg按需补全至16的倍数
# 需补全的位数
pad = (16 - len(msg) % 16)
# 补全
msg = msg + pad * chr(pad)
# 将key,iv和msg均以utf-8编码
key = key.encode('utf-8')
iv = iv.encode('utf-8')
msg = msg.encode('utf-8')
# 根据key和iv生成密钥,模式为CBC模式
encryptor = AES.new(key, AES.MODE_CBC, iv)
# 加密
encrypt_aes = encryptor.encrypt(msg)
# 先将加密后的值进行base64编码
encrypt_text = base64.encodebytes(encrypt_aes)
# 将其转换为utf-8字符串
encrypt_text = str(encrypt_text, 'utf-8')
# 返回加密后的字符串
return encrypt_text
# 根据歌曲song_id,生成需要传输的data
# 其中包括params和encSecKey
def construct_data(self, song_id):
# 生成16位随机数字符串
random_num = self.set_random_num()
# 生成encSecKey
encSecKey = self.RSA_encrypt(
random_num=random_num)
# 调用两次AES加密生成params
# 初始化歌曲song_info
song_info = '{"ids":"[%s]","level":"exhigh","encodeType":"mp3",' \
'"csrf_token":"477c1bd99fddedb3adc074f47fee2d35"}' % song_id
# 第一次加密,传入encText, key和iv
first_encryption = self.AES_encrypt(
msg=song_info, key=self.key, iv=self.iv)
# 第二次加密, 传入first_encryption, random_num和iv
encText = self.AES_encrypt(
msg=first_encryption,
key=random_num,
iv=self.iv)
# 生成data
data = {
'params': encText,
'encSecKey': encSecKey
}
# 返回data
return data
# 发送请求,获取下载链接
def get_real_url(self):
# 输入歌曲song_id
self.song_id = songid
# 获取data
data = self.construct_data(
song_id=self.song_id)
# 发送请求
request = requests.post(url=self.url,
headers=self.headers,
data=data,
proxies=proxies,
verify=False)
# 初始化real_url
real_url = ''
# 处理返回信息
try:
js_text = json.loads(request.text)
data = js_text['data']
if len(data) != 0:
code = data[0]['code']
# 获取成功
if code == 200:
# 歌曲真实地址
real_url = data[0]['url']
else:
raise RetryError
except:
print('生成的params和encSecKey有误!重试中!')
raise RetryError
# 返回real_url
return real_url
def download(self):
# 获取下载链接
real_url = self.get_real_url()
if real_url == '':
print('链接获取失败!')
raise RetryError
else:
file = name
# 开始下载
try:
content = requests.get(
url=real_url,
headers=self.headers).content
with open(file, 'wb') as fp:
fp.write(content)
except:
print('服务器连接出错')
raise RetryError
for __ in range(6): # 最多尝试6次
if proxynum > (len(proxy) - 1): # 代理自动切换至下一个
proxynum = 0
proxies = proxy[proxynum]
proxynum += 1
try:
WangyiyunDownload().download()
ccimported = True
break
except:
ccimported = False
if not exists(name):
ccimported = False
if ccimported is False: # 下载(普通音质)
music = requests.request(
"GET",
"http://music.163.com/api/song/enhance/download/url?&br="
+ str(info['br']) + "&id=" + str(info['id']),
headers=headers,
proxies=proxies,
verify=False)
if music.status_code == 200:
music = json.loads(music.content)
if not music['data']['url']:
music = requests.request(
"GET",
"https://music.163.com/song/media/outer/url?id="
+ str(info['id']) + ".mp3",
headers=headers,
verify=False)
if music.status_code != 200:
continue
else:
music = requests.request("GET",
music['data']['url'],
headers=headers)
else:
continue
performers = info['artist'].replace(';', ',')
cap = performers + " - " + "**" + info['title'] + "**"
if ccimported is False:
with open(name, 'wb') as f:
f.write(music.content)
if (path.getsize(name) / 1024) < 100:
remove(name)
try:
if reply.sender.is_self:
await reply.delete()
except:
pass
await context.delete()
res = '或者你可以点击<a href="https://music.163.com/#/song?id=' + \
str(info['id']) + '">' + \
' <strong>这里</strong> ' + '</a>' + '前往网页版收听'
await bot.send_message(
context.chat_id,
f"<strong>【{info['title']}】</strong>\n" +
"歌曲获取失败，资源获取可能受限，你可以再次尝试。\n" + res,
parse_mode='html',
link_preview=True)
return
duration = 0
imagedata = requests.get(info['albumpic'],
headers=headers).content
if imported is True:
await context.edit(f"{title}信息导入中 . . .")
tag = eyed3.load(name)
duration = int(tag.info.time_secs)
tag.initTag()
tag = tag.tag
tag.artist = info['artist']
tag.title = info['title']
tag.album = info['album']
tag.images.remove('')
tag.images.set(6, imagedata, "image/jpeg", u"Media")
tag.save(version=eyed3.id3.ID3_DEFAULT_VERSION,
encoding='utf-8')
br = ""
if imported is True:
br = "#" + \
str(eyed3.mp3.Mp3AudioFile(
name).info.bit_rate[1]) + "kbps "
alias = ""
if info['alias']:
alias = "\n\n__" + info['alias'][0] + "__"
cap += "\n#NeteaseMusic " + br + alias
await context.edit(f"{title}上传中 . . .")
if not exists("plugins/NeteaseMusicExtra/FastTelethon.py"):
if not exists("plugins/NeteaseMusicExtra"):
mkdir("plugins/NeteaseMusicExtra")
for ____ in range(6): # 最多尝试6次
faster = requests.request(
"GET",
"https://gist.githubusercontent.com/TNTcraftHIM"
"/ca2e6066ed5892f67947eb2289dd6439/raw"
"/86244b02c7824a3ca32ce01b2649f5d9badd2e49/FastTelethon.py"
)
if faster.status_code == 200:
with open(
"plugins/NeteaseMusicExtra/FastTelethon.py",
"wb") as f:
f.write(faster.content)
break
else:
if exists(
"plugins/NeteaseMusicExtra/NoFastTelethon.txt"
):
break
try:
from NeteaseMusicExtra.FastTelethon import upload_file
file = await upload_file(context.client,
open(name, 'rb'), name)
except:
file = name
if not exists(
"plugins/NeteaseMusicExtra/NoFastTelethon.txt"
):
with open(
"plugins/NeteaseMusicExtra/NoFastTelethon.txt",
"w") as f:
f.write(
"此文件出现表示FastTelethon支持文件在首次运行NeteaseMusic插件时导入失败\n这可能是因为Github"
"服务器暂时性的访问出错导致的\nFastTelethon可以提升低网络性能机型在上传文件时的效率，但是正常情况提升并不明显\n"
"如想要手动导入，可以手动下载：\nhttps://gist.githubusercontent.com/TNTcraftHIM"
"/ca2e6066ed5892f67947eb2289dd6439/raw"
"/86244b02c7824a3ca32ce01b2649f5d9badd2e49/FastTelethon.py\n并放入当前文件夹"
)
await bot.send_message(
context.chat_id,
'`FastTelethon`支持文件导入失败，上传速度可能受到影响\n'
'此提示仅出现**一次**，手动导入可参考：\n`' + getcwd() +
'/plugins/NeteaseMusicExtra/NoFastTelethon.txt`'
)
await context.client.send_file(
context.chat_id,
file,
caption=cap,
link_preview=False,
force_document=False,
thumb=imagedata,
attributes=(DocumentAttributeAudio(
duration, False, info['title'], performers), ))
try:
if reply.sender.is_self:
await reply.delete()
except:
pass
try:
remove(name)
except:
pass
await context.delete()
status = True
break
else:
await context.edit("**未搜索到结果**")
sleep(3)
await context.delete()
status = True
break
else:
continue
if status is False:
await context.edit(apifailtext)
sleep(3)
await context.delete()
else: # 错误输入
await context.edit(helptext)
return
while line:
tk = line.split()
if len(tk) > 1:
t = float(tk[0])
gid = int(tk[1])
if gid in m:
m[gid].append(t)
else:
m[gid] = [t]
line = fi.readline()
fi.close()
fo1 = open(filespk + '.sbgh', 'wb')
fo = open(filespk + '.sbg', 'wb')
for gid, tspks in m.items():
fo.write(pack('>LL', gid, len(tspks)))
for x in tspks:
fo1.write(pack('>f', x))
fo.close()
fo1.close()
from os import path, system
f = open(filespk + '.size', 'wb')
f.write(pack('>q', path.getsize(filespk + '.size')))
f.close()
system('cat %s.size %s.sbgh %s.sbg > %s.spk' %
(filespk, filespk, filespk, filespk))
system('rm %s.size %s.sbgh %s.sbg' % (filespk, filespk, filespk))
quit()
julian_day = []
j = 0
st = '_' + int2str(jn, 3)
txt_file = open(file_name + "_packet_info.txt", "w+")
txt_file.write("\t\t---Sampling_Frequency_Summary---")
txt_file.close()
txt_file = open(file_name + "_packet_info.txt", "a")
#stop.append(0)
while (jn < 999):
print "Reading file " + file_name + st + ".hdr"
size = getsize(file_name + st + ".hdr")
i = 26
q = 0
fpga = np.empty(2)
gps = np.empty(2)
p_num = np.empty(2)
g_blip = []
p_blip = []
f_g_blip = []
f_p_blip = []
file = open(file_name + st + ".hdr", "rb")
while (i < size):
file.read(24)
fpga[q % 2] = int(str(file.read(2)).encode('hex'), 16)
gps[q % 2] = int(str(file.read(2)).encode('hex'), 16)
client = FileDeliverHost(host_info["host"], pwd=host_info["pwd"], pkey=host_info["pkey"])
redis.hset("file_deliver", "%s_%s" % (host_info["host"], remote_path), value="upload")
client.sftp.put(local_path, remote_path, callback=callback)
print(host_info["host"] + " done")
if __name__ == '__main__':
from multiprocessing.pool import Pool
from multiprocessing import Process
hosts = ["172.16.0.13", "172.18.0.21"] # , "172.16.0.17", "172.16.0.16", "172.16.0.18"]
pwd = "h@@T1onZ02Onew"
path = "/root/test.png"
local_path = "../dev/test.gif"
file_size = getsize(local_path)
processes = []
start = time.time()
for host in hosts:
name = "%s_%s" % (host, path)
p = Process(
target=task,
args=({
"host": host,
"pwd": pwd,
"pkey": None
},
local_path, path, partial(file_callback, host, path)),
name=name
)
p.start()
def finalize_redex(state):
state.lib_manager.__exit__(*sys.exc_info())
repack_start_time = timer()
state.unpack_manager.__exit__(*sys.exc_info())
state.zip_manager.__exit__(*sys.exc_info())
align_and_sign_output_apk(
state.zip_manager.output_apk,
state.args.out,
# In dev mode, reset timestamps.
state.args.reset_zip_timestamps or state.args.dev,
state.args.sign,
state.args.keystore,
state.args.keyalias,
state.args.keypass,
state.args.ignore_zipalign,
state.args.page_align_libs,
)
logging.debug(
"Creating output APK finished in {:.2f} seconds".format(
timer() - repack_start_time
)
)
meta_file_dir = join(state.dex_dir, "meta/")
assert os.path.isdir(meta_file_dir), "meta dir %s does not exist" % meta_file_dir
copy_all_file_to_out_dir(
meta_file_dir, state.args.out, "*", "all redex generated artifacts"
)
if state.args.enable_instrument_pass:
logging.debug("Creating redex-instrument-metadata.zip")
zipfile_path = join(dirname(state.args.out), "redex-instrument-metadata.zip")
FILES = [
join(dirname(state.args.out), f)
for f in (
"redex-instrument-metadata.txt",
"redex-source-block-method-dictionary.csv",
"redex-source-blocks.csv",
)
]
# Write a checksum file.
hash = hashlib.md5()
for f in FILES:
hash.update(open(f, "rb").read())
checksum_path = join(dirname(state.args.out), "redex-instrument-checksum.txt")
with open(checksum_path, "w") as f:
f.write(f"{hash.hexdigest()}\n")
with zipfile.ZipFile(zipfile_path, "w", compression=zipfile.ZIP_DEFLATED) as z:
for f in [*FILES, checksum_path]:
z.write(f, os.path.basename(f))
for f in [*FILES, checksum_path]:
os.remove(f)
redex_stats_filename = state.config_dict.get("stats_output", "redex-stats.txt")
redex_stats_file = join(dirname(meta_file_dir), redex_stats_filename)
if isfile(redex_stats_file):
with open(redex_stats_file, "r") as fr:
apk_input_size = getsize(state.args.input_apk)
apk_output_size = getsize(state.args.out)
redex_stats_json = json.load(fr)
redex_stats_json["input_stats"]["total_stats"][
"num_compressed_apk_bytes"
] = apk_input_size
redex_stats_json["output_stats"]["total_stats"][
"num_compressed_apk_bytes"
] = apk_output_size
update_redex_stats_file = join(
dirname(state.args.out), redex_stats_filename
)
with open(update_redex_stats_file, "w") as fw:
json.dump(redex_stats_json, fw)
# Write invocation file
with open(join(dirname(state.args.out), "redex.py-invocation.txt"), "w") as f:
print("%s" % " ".join(map(shlex.quote, sys.argv)), file=f)
copy_all_file_to_out_dir(
state.dex_dir, state.args.out, "*.dot", "approximate shape graphs"
)
def _parseHeader(self):
"""Read the header information from a dcd file.
Input: fd - a file struct opened for binary reading.
Output: 0 on success, negative error code on failure.
Side effects: *natoms set to number of atoms per frame
*nsets set to number of frames in dcd file
*istart set to starting timestep of dcd file
*nsavc set to timesteps between dcd saves
*delta set to value of trajectory timestep
*nfixed set to number of fixed atoms
*freeind may be set to heap-allocated space
*reverse set to one if reverse-endian, zero if not.
*charmm set to internal code for handling charmm data.
"""
dcd = self._file
endian = b'' #'=' # native endian
rec_scale = RECSCALE32BIT
charmm = None
dcdcordmagic = unpack(endian + b'i', b'CORD')[0]
# Check magic number in file header and determine byte order
bits = dcd.read(calcsize('ii'))
temp = unpack(endian + b'ii', bits)
if temp[0] + temp[1] == 84:
LOGGER.info('Detected CHARMM -i8 64-bit DCD file of native '
'endianness.')
rec_scale = RECSCALE64BIT
elif temp[0] == 84 and temp[1] == dcdcordmagic:
pass
#LOGGER.info('Detected standard 32-bit DCD file of native '
# 'endianness.')
else:
if unpack(b'>ii', bits) == temp:
endian = '>'
else:
endian = '<'
temp = unpack(endian + b'ii', bits)
if temp[0] + temp[1] == 84:
rec_scale = RECSCALE64BIT
LOGGER.info('Detected CHARMM -i8 64-bit DCD file of opposite '
'endianness.')
else:
endian = ''
temp = unpack(endian + b'ii', bits)
if temp[0] == 84 and temp[1] == dcdcordmagic:
LOGGER.info('Detected standard 32-bit DCD file of '
'opposite endianness.')
else:
raise IOError('Unrecognized DCD header or unsupported '
'DCD format.')
# check for magic string, in case of long record markers
if rec_scale == RECSCALE64BIT:
raise IOError('CHARMM 64-bit DCD files are not yet supported.')
temp = unpack(b'I', dcd.read(calcsize('I')))
if temp[0] != dcdcordmagic:
raise IOError('Failed to find CORD magic in CHARMM -i8 64-bit '
'DCD file.')
# Buffer the entire header for random access
bits = dcd.read(80)
# CHARMm-genereate DCD files set the last integer in the
# header, which is unused by X-PLOR, to its version number.
# Checking if this is nonzero tells us this is a CHARMm file
# and to look for other CHARMm flags.
temp = unpack(endian + b'i' * 20, bits)
if temp[-1] != 0:
charmm = True
if charmm:
#LOGGER.info('CHARMM format DCD file (also NAMD 2.1 and later).')
temp = unpack(endian + b'i' * 9 + b'f' + b'i' * 10, bits)
else:
LOGGER.info('X-PLOR format DCD file (also NAMD 2.0 and earlier) '
'is not supported.')
return None
# Store the number of sets of coordinates (NSET)
self._n_csets = temp[0]
# Store ISTART, the starting timestep
self._first_ts = temp[1]
# Store NSAVC, the number of timesteps between dcd saves
self._framefreq = temp[2]
# Store NAMNF, the number of fixed atoms
self._n_fixed = temp[8]
if self._n_fixed > 0:
raise IOError('DCD files with fixed atoms is not yet supported.')
# Read in the timestep, DELTA
# Note: DELTA is stored as double with X-PLOR but as float with CHARMm
self._timestep = temp[9]
self._unitcell = temp[10] == 1
# Get the end size of the first block
if unpack(endian + b'i', dcd.read(rec_scale * calcsize('i')))[0] != 84:
raise IOError('Unrecognized DCD format.')
# Read in the size of the next block
temp = unpack(endian + b'i', dcd.read(rec_scale * calcsize('i')))
if (temp[0] - 4) % 80 != 0:
raise IOError('Unrecognized DCD format.')
noremarks = temp[0] == 84
# Read NTITLE, the number of 80 character title strings there are
temp = unpack(endian + b'i', dcd.read(rec_scale * calcsize('i')))
self._dcdtitle = dcd.read(80)
if not noremarks:
self._remarks = dcd.read(80)
# Get the ending size for this block
temp = unpack(endian + b'i', dcd.read(rec_scale * calcsize('i')))
if (temp[0] - 4) % 80 != 0:
raise IOError('Unrecognized DCD format.')
# Read in an integer '4'
if unpack(endian + b'i', dcd.read(rec_scale * calcsize('i')))[0] != 4:
raise IOError('Unrecognized DCD format.')
# Read in the number of atoms
self._n_atoms = unpack(endian + b'i',
dcd.read(rec_scale * calcsize('i')))[0]
# Read in an integer '4'
if unpack(endian + b'i', dcd.read(rec_scale * calcsize('i')))[0] != 4:
raise IOError('Bad DCD format.')
self._is64bit = rec_scale == RECSCALE64BIT
self._endian = endian
self._n_floats = (self._n_atoms + 2) * 3
if self._is64bit:
if self._unitcell:
self._bytes_per_frame = 56 + self._n_floats * 8
else:
self._bytes_per_frame = self._n_floats * 8
LOGGER.warning('Reading of 64 bit DCD files has not been tested. '
'Please report any problems that you may find.')
self._dtype = np.float64
self._itemsize = 8
else:
if self._unitcell:
self._bytes_per_frame = 56 + self._n_floats * 4
else:
self._bytes_per_frame = self._n_floats * 4
self._dtype = np.float32
self._itemsize = 4
self._first_byte = self._file.tell()
n_csets = (getsize(self._filename) -
self._first_byte) // self._bytes_per_frame
if n_csets != self._n_csets:
LOGGER.warning('DCD header claims {0} frames, file size '
'indicates there are actually {1} frames.'.format(
self._n_csets, n_csets))
self._n_csets = n_csets
self._coords = self.nextCoordset()
self._file.seek(self._first_byte)
self._nfi = 0
def get_files_info(directory):
yield from ((extract_extension(f), getsize(f))
for f in get_files(directory))
def main():
# python27 unrpyc.py [-c] [-d] [--python-screens|--ast-screens|--no-screens] file [file ...]
cc_num = cpu_count()
parser = argparse.ArgumentParser(
description="Decompile .rpyc/.rpymc files")
parser.add_argument('-c',
'--clobber',
dest='clobber',
action='store_true',
help="overwrites existing output files")
parser.add_argument(
'-d',
'--dump',
dest='dump',
action='store_true',
help="instead of decompiling, pretty print the ast to a file")
parser.add_argument(
'-p',
'--processes',
dest='processes',
action='store',
type=int,
choices=range(1, cc_num),
default=cc_num - 1 if cc_num > 2 else 1,
help="use the specified number or processes to decompile."
"Defaults to the amount of hw threads available minus one, disabled when muliprocessing is unavailable."
)
parser.add_argument(
'-t',
'--translation-file',
dest='translation_file',
action='store',
default=None,
help="use the specified file to translate during decompilation")
parser.add_argument(
'-T',
'--write-translation-file',
dest='write_translation_file',
action='store',
default=None,
help="store translations in the specified file instead of decompiling")
parser.add_argument(
'-l',
'--language',
dest='language',
action='store',
default='english',
help=
"if writing a translation file, the language of the translations to write"
)
parser.add_argument(
'--sl1-as-python',
dest='decompile_python',
action='store_true',
help="Only dumping and for decompiling screen language 1 screens. "
"Convert SL1 Python AST to Python code instead of dumping it or converting it to screenlang."
)
parser.add_argument(
'--comparable',
dest='comparable',
action='store_true',
help=
"Only for dumping, remove several false differences when comparing dumps. "
"This suppresses attributes that are different even when the code is identical, such as file modification times. "
)
parser.add_argument(
'--no-pyexpr',
dest='no_pyexpr',
action='store_true',
help=
"Only for dumping, disable special handling of PyExpr objects, instead printing them as strings. "
"This is useful when comparing dumps from different versions of Ren'Py. "
"It should only be used if necessary, since it will cause loss of information such as line numbers."
)
parser.add_argument(
'--tag-outside-block',
dest='tag_outside_block',
action='store_true',
help=
"Always put SL2 'tag's on the same line as 'screen' rather than inside the block. "
"This will break compiling with Ren'Py 7.3 and above, but is needed to get correct line numbers "
"from some files compiled with older Ren'Py versions.")
parser.add_argument(
'--init-offset',
dest='init_offset',
action='store_true',
help=
"Attempt to guess when init offset statements were used and insert them. "
"This is always safe to enable if the game's Ren'Py version supports init offset statements, "
"and the generated code is exactly equivalent, only less cluttered.")
parser.add_argument(
'file',
type=str,
nargs='+',
help="The filenames to decompile. "
"All .rpyc files in any directories passed or their subdirectories will also be decompiled."
)
parser.add_argument(
'--try-harder',
dest="try_harder",
action="store_true",
help=
"Tries some workarounds against common obfuscation methods. This is a lot slower."
)
args = parser.parse_args()
if args.write_translation_file and not args.clobber and path.exists(
args.write_translation_file):
# Fail early to avoid wasting time going through the files
print(
"Output translation file already exists. Pass --clobber to overwrite."
)
return
if args.translation_file:
with open(args.translation_file, 'rb') as in_file:
args.translations = in_file.read()
# Expand wildcards
def glob_or_complain(s):
retval = glob.glob(s)
if not retval:
print("File not found: " + s)
return retval
filesAndDirs = map(glob_or_complain, args.file)
# Concatenate lists
filesAndDirs = list(itertools.chain(*filesAndDirs))
# Recursively add .rpyc files from any directories passed
files = []
for i in filesAndDirs:
if path.isdir(i):
for dirpath, dirnames, filenames in walk(i):
files.extend(
path.join(dirpath, j) for j in filenames
if len(j) >= 5 and j.endswith(('.rpyc', '.rpymc')))
else:
files.append(i)
# Check if we actually have files. Don't worry about
# no parameters passed, since ArgumentParser catches that
if len(files) == 0:
print("No script files to decompile.")
return
files = map(lambda x: (args, x, path.getsize(x)), files)
processes = int(args.processes)
if processes > 1:
# If a big file starts near the end, there could be a long time with
# only one thread running, which is inefficient. Avoid this by starting
# big files first.
files.sort(key=itemgetter(2), reverse=True)
results = Pool(int(args.processes), sharelock,
[printlock]).map(worker, files, 1)
else:
# Decompile in the order Ren'Py loads in
files.sort(key=itemgetter(1))
results = map(worker, files)
if args.write_translation_file:
print("Writing translations to %s..." % args.write_translation_file)
translated_dialogue = {}
translated_strings = {}
good = 0
bad = 0
for result in results:
if not result:
bad += 1
continue
good += 1
translated_dialogue.update(magic.loads(result[0], class_factory))
translated_strings.update(result[1])
with open(args.write_translation_file, 'wb') as out_file:
magic.safe_dump(
(args.language, translated_dialogue, translated_strings),
out_file)
else:
# Check per file if everything went well and report back
good = results.count(True)
bad = results.count(False)
if bad == 0:
print("Decompilation of %d script file%s successful" %
(good, 's' if good > 1 else ''))
elif good == 0:
print("Decompilation of %d file%s failed" %
(bad, 's' if bad > 1 else ''))
else:
print(
"Decompilation of %d file%s successful, but decompilation of %d file%s failed"
% (good, 's' if good > 1 else '', bad, 's' if bad > 1 else ''))
if line[0] == ">":
seq_name = line[1:]
if seq_name not in locus_list:
locus_list.append(seq_name)
Allseq_dict[seq_name] = ''
m = re.search("\|(.+)$", seq_name)
sample = m.group(1)
if sample not in sample_list:
sample_list.append(sample)
sample_seq_dict[sample] = []
else:
Allseq_dict[seq_name] += line
for key in Allseq_dict.keys():
m = re.search("\|(.+)$", key)
sample_seq_dict[m.group(1)].append(Allseq_dict[key])
filesize = getsize(fas_file)
if filesize == 0 or locus_list == [] or len(locus_list) < 2:
print("输入文件不合要求，请检查！")
OUTPUT.write("#" + fas_file + ",-\n")
os.exit(0)
#开始计算Pi
name = ""
count = 0
sum_pi = 0
pair_num = 0
processing_num = 1
for i in range(len(sample_list)):
#print("Sample:", sample_list[i])
for j in range(i+1, len(sample_list)):
total_pair = len(sample_list) * (len(sample_list)-1) / 2
print("Process rate: %d / %d\r" % (processing_num, total_pair), end = '')
source_folder_name = "source"
build_folder_name = "build"
driver = webdriver.Chrome(
path_to_webdriver
) # Change "Chrome" to the right type of browser driver you are using
# endregion
source_path = join(script_path, source_folder_name)
source_path_len = len(source_path) + len(pathsep)
build_path = join(script_path, build_folder_name, "html")
rst_files = glob(join(script_path, '**/*.rst'), recursive=True)
size_dict = {}
for rst_file in rst_files:
size_dict[rst_file] = [getsize(rst_file), getmtime(rst_file), 0]
while True:
for rst_file in rst_files:
if isfile(rst_file):
record = size_dict[rst_file]
new_size = getsize(rst_file)
new_mtime = getmtime(rst_file)
if record[0] != new_size or record[
1] < new_mtime: # Checks if file size or latest modify time change
html_file = join(build_path,
rst_file[source_path_len:-4] + ".html")
# import ipdb; ipdb.set_trace()
os.system("sphinx-build -M html {} {}".format(
source_folder_name, build_folder_name))
def has_size(self, file_name, size):
file_size = getsize(file_name)
assert file_size >= size, f"{file_name} deve ter pelo menos {size} bytes"
def __init__(self,
input_file=None,
output_file=None,
dfxml_file=None,
report_file=None,
commit=False,
ignore_patterns=[],
key=None,
rules=[]):
# Validate configuration
from schema import Schema, Optional, Or, Use, And, SchemaError
schema = Schema({
'input_file':
Use(lambda f: open(f, 'r'), error='Cannot read the input file'),
Optional('output_file'):
Or(
None,
Use(lambda f: open(f, 'w'),
error='Cannot write to the output file')),
Optional('dfxml_file'):
Or(None, Use(lambda f: open(f, 'r'),
error='Cannot read DFXML file')),
Optional('report_file'):
Or(None,
lambda f: open(f, 'w'),
error='Cannot write to the report file'),
'commit':
Or(True, False),
'ignore_patterns':
Use(lambda f: re.compile(convert_fileglob_to_re('|'.join(f))),
error='Cannot compile unified ignore regex'),
'key':
Or(None, str),
'rules':
And([(redact_rule, redact_action)], lambda f: len(f) > 0)
})
try:
kwargs = {
'input_file': input_file,
'output_file': output_file,
'dfxml_file': dfxml_file,
'report_file': report_file,
'commit': commit,
'ignore_patterns': ignore_patterns,
'key': key,
'rules': rules
}
self.conf = schema.validate(kwargs)
except SchemaError as e:
logging.warning('The redact configuration did not validate:')
exit(e)
if self.conf['commit'] and 'output_file' not in self.conf.keys():
logging.error('An output file is required when COMMIT is on.')
exit(1)
# TODO Check input and output are not same file
logging.debug('Configuration:\n%s' % self.conf)
# Print rules
logging.debug(
json.dumps(map(
lambda x, y: (x.line, x.__class__.__name__, y.__class__.
__name__, x.lgpattern
if hasattr(x, 'lgpattern') else ''),
self.conf['rules']),
indent=4))
self.input_file = self.conf['input_file']
from os import path
self.image_size = path.getsize(self.input_file.name)
self.output_file = self.conf['output_file']
self.report_file = self.conf['report_file']
self.dfxml_file = self.conf['dfxml_file']
self.commit = self.conf['commit']
self.configure_report_logger()
def main():
"""
Starts the parser on the file given by the filename as the first
argument on the command line.
"""
from optparse import OptionParser, OptionGroup
from os import sep
from os.path import basename, getsize
from sys import argv, exit as sysExit
from chardet import detect
from codecs import BOM_UTF8, open as codecsOpen
def optParse():
"""
Parses command line options.
Generally we're supporting all the command line options that doxypy.py
supports in an analogous way to make it easy to switch back and forth.
We additionally support a top-level namespace argument that is used
to trim away excess path information.
"""
parser = OptionParser(prog=basename(argv[0]))
parser.set_usage("%prog [options] filename")
parser.add_option(
"-a",
"--autobrief",
action="store_true",
dest="autobrief",
help=
"parse the docstring for @brief description and other information")
parser.add_option("-c",
"--autocode",
action="store_true",
dest="autocode",
help="parse the docstring for code samples")
parser.add_option(
"-n",
"--ns",
action="store",
type="string",
dest="topLevelNamespace",
help="specify a top-level namespace that will be used to trim paths"
)
parser.add_option(
"-t",
"--tablength",
action="store",
type="int",
dest="tablength",
default=4,
help="specify a tab length in spaces; only needed if tabs are used"
)
parser.add_option("-s",
"--stripinit",
action="store_true",
dest="stripinit",
help="strip __init__ from namespace")
parser.add_option(
"-O",
"--object-respect",
action="store_true",
dest="object_respect",
help=
"By default, doxypypy hides object class from class dependencies even if class inherits explictilty from objects (new-style class), this option disable this."
)
group = OptionGroup(parser, "Debug Options")
group.add_option("-d",
"--debug",
action="store_true",
dest="debug",
help="enable debug output on stderr")
parser.add_option_group(group)
## Parse options based on our definition.
(options, filename) = parser.parse_args()
# Just abort immediately if we are don't have an input file.
if not filename:
stderr.write("No filename given." + linesep)
sysExit(-1)
# Turn the full path filename into a full path module location.
fullPathNamespace = filename[0].replace(sep, '.')[:-3]
# Use any provided top-level namespace argument to trim off excess.
realNamespace = fullPathNamespace
if options.topLevelNamespace:
namespaceStart = fullPathNamespace.find(options.topLevelNamespace)
if namespaceStart >= 0:
realNamespace = fullPathNamespace[namespaceStart:]
if options.stripinit:
realNamespace = realNamespace.replace('.__init__', '')
options.fullPathNamespace = realNamespace
return options, filename[0]
# Figure out what is being requested.
(options, inFilename) = optParse()
# Figure out encoding of input file.
numOfSampleBytes = min(getsize(inFilename), 32)
sampleBytes = open(inFilename, 'rb').read(numOfSampleBytes)
sampleByteAnalysis = detect(sampleBytes)
encoding = sampleByteAnalysis['encoding'] or 'ascii'
# Switch to generic versions to strip the BOM automatically.
if sampleBytes.startswith(BOM_UTF8):
encoding = 'UTF-8-SIG'
if encoding.startswith("UTF-16"):
encoding = "UTF-16"
elif encoding.startswith("UTF-32"):
encoding = "UTF-32"
# Read contents of input file.
if encoding == 'ascii':
inFile = open(inFilename)
else:
inFile = codecsOpen(inFilename, encoding=encoding)
lines = inFile.readlines()
inFile.close()
# Create the abstract syntax tree for the input file.
astWalker = AstWalker(lines, options, inFilename)
astWalker.parseLines()
# Output the modified source.
# There is a "feature" in print on Windows. If linesep is
# passed, it will generate 0x0D 0x0D 0x0A each line which
# screws up Doxygen since it's expected 0x0D 0x0A line endings.
for line in astWalker.getLines().split(linesep):
print(line.rstrip())
def test_run_jackknifed_beta_diversity_parallel(self):
""" run_jackknifed_beta_diversity generates expected results """
run_jackknifed_beta_diversity(self.test_data['biom'][0],
self.test_data['tree'][0],
20,
self.test_out,
call_commands_serially,
self.params,
self.qiime_config,
self.test_data['map'][0],
parallel=True,
status_update_callback=no_status_updates)
weighted_unifrac_upgma_tree_fp = join(self.test_out,
'weighted_unifrac', 'upgma_cmp',
'jackknife_named_nodes.tre')
unweighted_unifrac_upgma_tree_fp = join(self.test_out,
'unweighted_unifrac',
'upgma_cmp',
'jackknife_named_nodes.tre')
weighted_unifrac_emperor_index_fp = join(self.test_out,
'weighted_unifrac',
'emperor_pcoa_plots',
'index.html')
unweighted_unifrac_emperor_index_fp = join(self.test_out,
'unweighted_unifrac',
'emperor_pcoa_plots',
'index.html')
input_file_basename = splitext(split(self.test_data['biom'][0])[1])[0]
unweighted_unifrac_dm_fp = join(
self.test_out, 'unrarefied_bdiv',
'unweighted_unifrac_%s.txt' % input_file_basename)
weighted_unifrac_dm_fp = join(
self.test_out, 'unrarefied_bdiv',
'weighted_unifrac_%s.txt' % input_file_basename)
# check for expected relations between values in the unweighted unifrac
# distance matrix
dm = parse_distmat_to_dict(open(unweighted_unifrac_dm_fp))
self.assertTrue(
dm['f1']['f2'] < dm['f1']['p1'],
"Distance between pair of fecal samples is larger than distance"
" between fecal and palm sample (unweighted unifrac).")
self.assertEqual(dm['f1']['f1'], 0)
# check for expected relations between values in the weighted unifrac
# distance matrix
dm = parse_distmat_to_dict(open(weighted_unifrac_dm_fp))
self.assertTrue(
dm['f1']['f2'] < dm['f1']['p1'],
"Distance between pair of fecal samples is larger than distance"
" between fecal and palm sample (unweighted unifrac).")
self.assertEqual(dm['f1']['f1'], 0)
# check that final output files have non-zero size
self.assertTrue(getsize(weighted_unifrac_upgma_tree_fp) > 0)
self.assertTrue(getsize(unweighted_unifrac_upgma_tree_fp) > 0)
self.assertTrue(getsize(weighted_unifrac_emperor_index_fp) > 0)
self.assertTrue(getsize(unweighted_unifrac_emperor_index_fp) > 0)
# Check that the log file is created and has size > 0
log_fp = glob(join(self.test_out, 'log*.txt'))[0]
self.assertTrue(getsize(log_fp) > 0)
except subprocess.CalledProcessError:
logging.warning("CALLEDPROCESERROR in initial sort (sam)")
except OSError:
logging.warning("OSERROR in initial sort (sam)")
logging.info(
"Initial name sorting completed, starting SAM-file splitting at: {}"
.format(str(datetime.datetime.now())))
errchk()
# sam_split(temp, outfile_perfect, outfile_secondary)
sam_parse.sam_split_runner(temp, directory + "/", 2000000, threads)
errchk()
logging.info("Splitting of the SAM-file completed at: {}".format(
str(datetime.datetime.now())))
subprocess.call(["rm", temp])
perfsize = path.getsize(outfile_perfect)
secsize = path.getsize(outfile_secondary)
logging.info(
"After splitting: ByteSize of Perfect: {}. Bytesize of secondary: {}".
format(perfsize, secsize))
# Get original headers
original_headers = directory + "/original_headers.sam"
subprocess.call("{} view -H {} > {}".format(samtools_path, infile,
original_headers),
shell=True)
logging.info(
"Headers extracted from original input file: {} to : {}".format(
infile, original_headers))
errchk()
def ping(self):
ret = PING_SUCCESS
counter = 0
for i in [1]:
try:
# 默认使用ping次数
# 修改最长时间为30天 modify by shenlige 2013-6-26
number = 2592000
cmd_smg = " -l %s -n %s %s" % (self.size,number,self.node)
cmd = "\"" + EXE_PATH + "\"" + cmd_smg
with open(self.ping_tmp_file,"w") as self.file_handel:
self.popen = subprocess.Popen(cmd,shell=True,
stdin=subprocess.PIPE,
stdout=self.file_handel,
stderr=subprocess.PIPE)
except Exception, e:
log_data = u"执行并行ping命令发生异常,message:%s" % e
log.user_err(log_data)
self.err_info = log_data
ret = PING_FAIL
break
# 确认ping临时文件不为空，增加等待时间，最长等待20秒 add by shenlige 2013-4-26
while counter < self.wait_time:
content = getsize(self.ping_tmp_file)
if content > 0L:
break
else:
counter += 1
time.sleep(1)
time.sleep(self.total_time) # 执行指定时间长度的ping命令
try:
if self.popen.poll() == None:
# 修改kill进程的实现方式 modify by shenlige 2013-7-4
pro_ret,data = attcommonfun.get_process_children(self.popen.pid)
if pro_ret == attcommonfun.ATTCOMMONFUN_SUCCEED:
dict_process = data
for process_pid, process_name in dict_process.items():
# modify by shenlige 2013-11-2 修改kill进程
#if process_name not in ["ATT.exe","robot.exe","cmd.exe"]:
if process_name.lower() == 'ping.exe':
try:
os.kill(process_pid, 9)
except:
pass
try:
os.kill(self.popen.pid, 9)
except:
pass
else:
pass
time.sleep(0.5) # zsj add 解决报windows32错误问题 2013-3-25
except Exception, e:
log_data = u"停止并行ping发生异常,message:%s" % e
log.user_err(log_data)
self.err_info = log_data
ret = PING_FAIL
import os
from os.path import getsize, splitext
import codecs
import json
base_url = "img/"
R = os.path.join("..", "R.js")
obj = []
for f in os.listdir(os.getcwd()):
names = splitext(f)
if names[1] != ".py" and names[1] != "":
print(names[0])
obj.append({
"id": names[0],
"src": base_url + f + "?121",
"size": round(getsize(f) * .001, 2)
})
fp = codecs.open(R, "w", "utf-8")
fp.write("R={};R.images=" + json.dumps(obj, ensure_ascii=False))
fp.close()
if os.name == "nt":
os.system("pause")
# -*- coding: utf-8 -*-
'''
Tip_010101 文件和目录操作-01
1. 显示搜索根目录下，每个子目录中文件的个数和总Bytes数量
2. 忽略名称中带CVS的子目录
3. https://docs.python.org/3.6/library/os.html
'''
import os
from os.path import join, getsize
for root, dirs, files in os.walk('..'):
print(root, "consumes", end=" ")
print(sum(getsize(join(root, name)) for name in files), end=" ")
print("bytes in", len(files), "non-directory files")
if 'CVS' in dirs:
dirs.remove('CVS') # don't visit CVS directories
'''
Tip_010102 文件和目录操作-02
1. 递归遍历目录目录中所有的文件和子目录
'''
import os
rootDir = r'..'
for root, dirs, files in os.walk(rootDir):
print('{}{}'.format(root, os.sep))
for i in files:
print('{}{}{}'.format(root, os.sep, i))
'''
Tip_010103 文件和目录操作-03
1. 临时文件
def get_size(self):
return getsize(self.path_str)
start_dir = getcwd()
info = []
for root in argv[1:]:
chdir(join(start_dir, root))
for infof in glob('**/*.info', recursive=True):
changed = False
with open(infof, 'r') as f:
d = json.load(f, object_pairs_hook=OrderedDict)
filename = infof[:-5]
location = d.get('url', '')
if location and (not location.startswith(URL)
or basename(location) != basename(filename)):
try:
urlretrieve(location, filename)
d['url'] = '{}/{}/{}'.format(URL, normpath(root), filename)
d['size'] = getsize(filename)
changed = True
except:
print('failed to get file',
filename,
location,
file=stderr)
ref = d.get('references', None)
if isinstance(ref, str):
d['references'] = [ref]
changed = True
if changed:
with open(infof, 'w') as f:
json.dump(d, f, indent=4)
if root:
file_path = [root, filename]
cmd = "\"" + EXE_PATH + "\"" + cmd_smg
with open(PING_TMP_FILE,"w") as file_tmp:
popen = subprocess.Popen(cmd,shell=True,
stdin=subprocess.PIPE,
stdout=file_tmp,
stderr=subprocess.PIPE)
except Exception, e:
log_data = u"执行ping.exe发生异常,message:%s" % e
log.user_err(log_data)
ret = PING_FAIL
break
while counter < 20:
content = getsize(PING_TMP_FILE)
if content > 0L:
break
else:
counter += 1
time.sleep(1)
try:
# modify by shenlige 2013-7-10 修改域名解析失败时attcommonfun.get_process_children执行失败的问题
if total_time:
if popen.poll() == None:
time.sleep(total_time) # 等待指定的ping时间
pro_ret,data = attcommonfun.get_process_children(popen.pid)
if pro_ret == attcommonfun.ATTCOMMONFUN_SUCCEED:
dict_process = data
for process_pid, process_name in dict_process.items():
# modify by shenlige 2013-11-2 修改kill进程
def loadASCII_data_header(filename, miss=None, fill=np.nan, *args, **kwargs):
'''
Open an ascii data file and load it into a python numpy array.
Input:
filename = CINDI data file name
miss = string or list denoting missing value options (default=None)
fill = fill value (default = NaN)
Output:
header = a list containing the header strings without the '#'
out = a dict containing the data in np.arrays, the dict keys are
specified by the header data line
'''
func_name = string.join([module_name, "loadASCII_data_header"], " ")
#-----------------------------------------------------------------------
# Test to ensure the file is small enough to read in. Python can only
# allocate 2GB of data. If you load something larger, python will crash
fsize = path.getsize(filename)
header = list()
out = dict()
if(fsize > 2.0e9):
print func_name, "WARNING: File size [", (fsize * 1e-9), "GB > 2 GB]"
return(header, out)
elif(fsize == 0):
print func_name, "WARNING: empty file [", filename, "]"
return(header, out)
#----------------------------------------------
# Open the datafile and read the header rows
f = open(filename, "r")
if not f:
print func_name, "ERROR: unable to open input file [", filename, "]"
return header, out
line = f.readline()
check = 0
while line.find("#") >= 0:
hline = string.strip(line.replace("#", ""))
line = f.readline()
check += 1
if(len(hline) > 0):
header.append(hline)
if(check > 0):
hline = hline.split()
else:
print func_name, "ERROR: no header in this file [", filename, "]"
return(header, out)
#-------------------------------------------
# Open the datafile and read the data rows
temp = np.genfromtxt(filename, comments="#", missing_values=miss,
filling_values=fill)
if len(temp) > 0:
#------------------------------------------
# Create the output dictionary
for num,name in enumerate(hline):
out[name] = temp[:,num]
del temp
return(header, out)
print('')
print('------We Out!------')
# Quit the program aka gtfo
raise SystemExit
# ------------------------------------------------------------------------------
# --- Make a copy if I haven't already today
# Make the new folder
newFolder = join(lD, da)
os.mkdir(newFolder)
# The for windows spotlight photos
winLockF = (r'C:\Users\Max\AppData\Local\Packages\Microsoft.Windows.Content' +
r'DeliveryManager_cw5n1h2txyewy\LocalState\Assets')
# Select the files bigger than 230 mB which is hopfully both all and only
# the wall papers
tbc = [f for f in os.listdir(winLockF) if getsize(join(winLockF, f)) > 230000]
for i in range(len(tbc)):
copy2(join(winLockF, tbc[i]), join(newFolder, tbc[i] + '.jpg'))
# Give some user feed back
print('1.', 'There were ' + str(len(tbc)) + ' files copyed to:')
print('2.', '"' + newFolder + '"')
# ------------------------------------------------------------------------------
print('')
print('--------End--------')
def loadASCII_data_hline(filename, hlines, miss=None, fill=np.nan, *args,
**kwargs):
'''
Open an ascii data file and load it into a python numpy array. File
header may be any number of lines and does not have to be preceeded by a
particular character.
Input:
filename = CINDI data file name
hlines = number of lines in header
miss = string or list denoting missing value options (default=None)
fill = fill value (default = NaN)
Output:
header = a list containing all specified header lines
out = a dict containing the data in np.arrays, the dict keys are
specified by the header data line
'''
func_name = string.join([module_name, "loadASCII_data_hline"])
#-----------------------------------------------------------------------
# Test to ensure the file is small enough to read in. Python can only
# allocate 2GB of data. If you load something larger, python will crash
fsize = path.getsize(filename)
header = list()
out = dict()
if(fsize > 2.0e9):
print func_name, "WARNING: File size [", (fsize * 1e-9), "GB > 2 GB]"
return hearder, out
elif(fsize == 0):
print func_name, "WARNING: empty file [", filename, "]"
return(header, out)
#----------------------------------------------
# Open the datafile and read the header rows
f = open(filename, "r")
if not f:
print func_name, "ERROR: unable to open input file [", filename, "]"
return out
for h in range(hlines):
header.append(f.readline())
#-------------------------------------------
# Open the datafile and read the data rows
temp = np.genfromtxt(filename, skip_header=hlines, missing_values=miss,
filling_values=fill)
if len(temp) > 0:
#---------------------------------------------------------------------
# Create the output dictionary, removing the point sign from any keys
for num,name in enumerate(header[-1].split()):
name = name.replace("#", "")
if len(name) > 0:
out[name] = temp[:,num]
del temp
return header, out
def getsize(self, fn):
return getsize(fn)
def _get_source_filesize(self):
if not hasattr(self, '_source_filesize'):
self._source_filesize = getsize(self.source)
return self._source_filesize
def loadASCII_index_profile(filename, miss=None, fill=np.nan, *args, **kwargs):
'''
Open an ascii data file and load it into a python numpy array. Assumes
this file is seperated into index blocks, which should be maintained.
Input:
filename = CINDI data file name
miss = string or list denoting missing value options (default=None)
fill = fill value (default = NaN)
Output:
header = a list containing the header strings without the '#'
out = a dict containing the data in np.arrays, the dict keys are
specified by the header data line
nblocks = number of indexed data blocks
'''
func_name = string.join([module_name, "loadASCII_data_header"])
#-----------------------------------------------------------------------
# Test to ensure the file is small enough to read in. Python can only
# allocate 2GB of data. If you load something larger, python will crash
fsize = path.getsize(filename)
header = list()
out = dict()
nblocks = 0
if(fsize > 2.0e9):
print func_name, "WARNING: File size [", (fsize * 1e-9), "GB > 2 GB]"
return(header, out, nblocks)
elif(fsize == 0):
print func_name, "WARNING: empty file [", filename, "]"
return(header, out, nblocks)
#----------------------------------------------
# Open the datafile and read the header rows
f = open(filename, "r")
if not f:
print func_name, "ERROR: unable to open input file [", filename, "]"
return header, out, nblocks
line = f.readline()
check = 0
while line.find("#") >= 0:
hline = string.strip(line.replace("#", ""))
line = f.readline()
check += 1
if(len(hline) > 0):
header.append(hline)
if(check > 0):
hline = hline.split()
else:
print func_name, "ERROR: no header in this file [", filename, "]"
return(header, out, nblocks)
#-------------------------------------------------
# Cycle through the data rows, identifying blocks
while len(line) > 0:
if (len(line) == 1 and line.find("\n") == 0):
# A new block has been found. Only incriment if data has been read
if len(out) > 0:
nblocks += 1
print "TEST: at block", nblocks, len(line)
# Cycle to new dataline
while len(line) == 1 and line.find("\n") == 0:
line = f.readline()
# Load the dataline into the output structure
dline = line.split()
for num,name in enumerate(hline):
if out.has_key(name):
if len(out[name]) < nblocks:
out[name][nblocks].append(dline[num])
else:
out[name].append([dline[num]])
else:
out[name] = [[dline[num]]]
return(header, out, nblocks)
