def run(self):
self.prepMeds()
byband = {}
" Run injection here "
" Prepare data for injected images "
self.prepInData()
" Create coadd images for each band and each revision "
for realV in self.realizationslist:
outpathR = os.path.join(self.medsdir,self.medsconf+'/balrog_images/'+str(realV)+'/'+self.tilename+'/coadd')
realD = os.path.join(self.medsdir,self.medsconf+'/balrog_images/'+str(realV)+'/'+self.tilename)
print " outpathR = %s \n" % outpathR
logpathR = shutil.abspath(outpathR)+'/LOGS/'
listpathR = shutil.abspath(outpathR)+'/lists/'
for band in self.bands:
self.makeCoadd(band,outpathR,self.tiledir)
" Now make detection image "
self.makeDetectionImage(outpathR)
" Create s-extractor catalogs "
for band in self.bands:
self.makeCatalog(band,outpathR)
" Now create fake objectMaps "
self.makeObjMaps(realD)
" Now crete meds for all bands "
for band in self.bands:
self.makeMeds(band,realD)
def make_project(dest, project_name, scan_path):
dest = shutil.abspath(dest)
scan_path = shutil.abspath(scan_path)
project_path = dest + "/" + project_name
make_dir(project_path)
shutil.copy(scan_path, project_path)
scan_path = project_path + "/" + os.path.basename(scan_path)
process_pdf.convert_pdf_to_image(scan_path)
return project_path
def __init__(self, rootDir, mainDomain):
self.cmdHost = None
if runo('hostname')[0].rstrip() != self.IMAGE_STORAGE_HOST:
self.cmdHost = self.IMAGE_STORAGE_HOST
if shutil.abspath(rootDir) != rootDir:
Exception('Absolute base path required!')
self.rootDir = shutil.abspath(rootDir) # removes trailing slashes
self.baseDir = '%s/%s' % (self.rootDir, self.BASE_IMAGE_PREFIX)
self.mainDomain = mainDomain
self.mainImage = self.getImagePathForDomain(self.mainDomain)
def bubbles(bubble_path, net, debug_path=None, align=False):
i = 0
bubble_path = shutil.abspath(bubble_path)
bubble_files = sorted(os.listdir(bubble_path))
top_img = cv2.imread(bubble_path + "/" + bubble_files[i])
bubble_locations = contour.grab_bubbles(top_img)
# use a copy because we want the top img to remain unchanged
score = score_bubbles(top_img.copy() , net, bubble_locations)
result = score[0]
if not debug_path is None:
analyzed_image_path = "%s/%05d.jpg" % (debug_path, i)
#print "Input file %s to %s" % (bubble_files[0], analyzed_image_path)
cv2.imwrite(analyzed_image_path, score[1])
i += 1
for bubble in bubble_files[1:]:
img = cv2.imread(bubble_path + "/" + bubble)
if not align:
bubble_locations = contour.grab_bubbles(img)
else:
img = alignImages(img, top_img)
score = score_bubbles(img, net, bubble_locations)
result += score[0]
if not debug_path is None:
analyzed_image_path = "%s/%05d.jpg" % (debug_path, i)
#print "Input file %s to %s" % (bubble, analyzed_image_path)
cv2.imwrite(analyzed_image_path, score[1])
i += 1
return result
def longforms(longform_path, dest, decide=False):
longform_path = shutil.abspath(longform_path)
longforms_files = sorted(os.listdir(longform_path))
top_img = cv2.imread(longform_path + "/" + longforms_files[0])
questions_location = contour.grab_answers(top_img)
result = {}
for i, longform in enumerate(longforms_files):
img = cv2.imread(longform_path + "/" + longform)
# img = alignImages(img, top_img)
for j, question_contour in enumerate(questions_location):
(x, y, w, h) = question_contour
question_img = img[y : y + h , x : x + w ]
if decide:
question_img = cv2.cvtColor(question_img, cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(question_img, 240, 255, cv2.THRESH_BINARY_INV)
if cv2.countNonZero(thresh) > 5500:
print "GOOD %d %s/q%02d-%s" % (cv2.countNonZero(thresh), dest, j, longform)
cv2.imwrite("%s/q%02d-%s" % (dest, j, longform), question_img)
else:
print "BAD %d %s/bad-q%02d-%s" % (cv2.countNonZero(thresh), dest, j, longform)
cv2.imwrite("%s/bad-q%02d-%s" % (dest, j, longform), question_img)
else:
cv2.imwrite("%s/q%02d-%s" % (dest, j, longform), question_img)
try:
result[j].append("%s/q%02d-%s" % (dest, j, longform))
except KeyError:
result[i] = [ "%s/q%02d-%s" % (dest, j, longform) ]
return result
def makeObjMaps(self,outpath):
print "makeObjMaps outpath=%s \n" % outpath
for band in self.bands:
# fname = os.path.join(outpath,'/lists/'+self.tilename+'_'+band+'_objmap-'+self.medsconf+'.fits')
fname = shutil.abspath(outpath) + '/lists/'+self.tilename+'_'+band+'_objmap-'+self.medsconf+'.fits'
print "Obj map file %s \n" % fname
catname = outpath+'/coadd/'+self.tilename+'_'+band+'_cat.fits'
print "makeObjMaps catname=%s n" % catname
if os.path.exists(fname):
os.remove(fname)
self.make_fake_objmap(catname, fname)
def download_isos(self):
# download zip file
resp = download(self._html)
f = open('./pyiso/iso_tmp/geo.txt', 'w+')
f.write(resp)
f.close()
# remove previous corrupted file
try:
os.remove(abspath(self._iso_path + 'corrupt_metadata_ids.txt'))
except:
pass
for line in open('./pyiso/iso_tmp/geo.txt','r'):
line_spl = line.split('\t')
if line_spl[1].strip() == 'id':
continue
else:
iid = int(line_spl[1].strip())
html = self._meta + str(iid)
# download the iso
xmlstr = download(html)
try:
tree = etree.fromstring(xmlstr)
# need the file identifier
fi = tree.find('.//' + self._file_identifier + '/' + self._character_string)
fname = fi.text
path = self._iso_path + fname + self._meta_folder
path = abspath(path)
print '\nWriting to: ' + path
if not os.path.exists(path):
os.makedirs(path)
f = open(path + '/' + self._meta_file, 'w+')
f.write(xmlstr)
f.close()
except:
path = abspath(self._iso_path + 'corrupt_metadata_ids.txt')
f = open(path, 'a+')
f.write('\nCorrupted/Missing xml for id: ' + str(iid))
f.close()
def filePath():
f = "data.txt"
print(shutil.abspath(f))
print(os.path.exists(""))
print(os.path.isfile(f))
print(os.path.isdir(f))
print(os.path.islink(f))
print(os.path.isabs(f))
print(os.path.split("a/b/c"))
print(os.path.join("a", "b", "c"))
print(os.path.splitext("dir/file.ext"))
def filePath():
f = "data.txt"
print(shutil.abspath(f))
print(os.path.exists(""))
print(os.path.isfile(f))
print(os.path.isdir(f))
print(os.path.islink(f))
print(os.path.isabs(f))
print(os.path.split( "a/b/c" ))
print(os.path.join( "a", "b", "c" ))
print(os.path.splitext( "dir/file.ext" ))
def __init__(self, confile, tilename):
'''
Constructor
'''
urlbase = "https://desar2.cosmology.illinois.edu/DESFiles/"
self.confile = confile
self.workdir = os.getcwd()
self.tilename = tilename
self.conf = self.read_config(confile)
self.dbname = self.conf['dbname']
self.bands = self.conf['bands']
self.det = self.conf['det']
self.medsconf = self.conf['medsconf'] # 'y3v02'
print self.medsconf
self.medsdir = os.getenv('MEDS_DATA')
self.bbase = os.getenv('BALROG_BASE')
self.desdata = os.getenv('DESDATA')
self.simData = self.medsdir + '/' + self.medsconf + '/balrog_images/'
print "simData = %s \n" % self.simData
print "meds_dir=%s \n" % self.medsdir
self.mofconfile = ''
self.pixscale = 0.2636
self.magbase = 30.
self.curdir = os.getcwd()
self.autocommit = True
self.quiet = False
print " Make coadds for bands: \n"
print self.bands
print " detection image : \n"
print self.det
desfile = os.getenv("DES_SERVICES")
if not desfile:
desfile = os.path.join(os.getenv("HOME"), ".desservices.ini")
#
self.desconfig = easyaccess.config_ea.get_desconfig(
desfile, self.dbname)
self.connectDB()
self.tileinfo = self.get_tile_infor()
self.tiledir = self.medsdir + '/' + self.medsconf + '/' + self.tilename
self.mofdir = self.tiledir + '/mof'
if not os.path.exists(self.mofdir):
os.makedirs(self.mofdir)
self.realDir = ''
self.curdir = os.getcwd()
#
self.outpath = os.path.join(
self.medsdir, self.medsconf + '/' + self.tilename + '/coadd/')
self.logpath = shutil.abspath(self.outpath) + '/LOGS/'
def process_project(project_path, net):
project_path = shutil.abspath(project_path)
make_dir(project_path + "/analyzed_bubbles")
make_dir(project_path + "/analyzed_longforms")
longforms = analyze.longforms(project_path + "/long_forms",
project_path + "/analyzed_longforms")
matrix = analyze.bubbles(project_path + "/bubbles/", net, \
align=True, \
debug_path=project_path + "/analyzed_bubbles")
histograms = []
for i, row in enumerate(matrix):
histograms.append(
plots.histogram(
numpy.asarray(row).reshape(-1), i + 1, project_path))
return (histograms, longforms)
def __init__(self, filename=None):
"""
"""
self.stream = None
self.filename = shutil.abspath(filename)
if filename:
self.stream = io.open(self.filename,'rb')
self.entries = []
self.header = self.Header()
self.index_flags = 0
self.static_key = ResourceKey()
self.__name_map = None
self.__loaded_resources = {}
self.__pending_deletes = {}
self.cache = False
self.load()
def get_aboot_image(self):
d = self.get_device()
for path in Config.get_config().ota_prevalent_aboot_paths:
try:
return (os.path.basename(path), self.zip.open(path))
except KeyError:
pass
if 'flounder' in d:
data = self.zip.read('bootloader.img')[256:]
fp = io.BytesIO(data)
return ('hboot.img', zipfile.ZipFile(fp).open('hboot.img'))
if 'fugu' == d:
tmpdir = tempfile.mkdtemp()
tmpfile = tempfile.NamedTemporaryFile()
self.zip.extract('droidboot.img', tmpdir)
curdir = shutil.abspath(".")
os.chdir(tmpdir)
try:
subprocess.check_output(
[Config.get_config().ota_umkbootimg_path, "droidboot.img"],
stderr=subprocess.STDOUT)
subprocess.check_output([
Config.get_config().ota_unpack_ramdisk_path,
"initramfs.cpio.gz"
],
stderr=subprocess.STDOUT)
except OSError as e:
E("Cannot execute umkbootimg/unpack_ramdisk while handling %s. "
% self.path)
E("ota_umkbootimg_path = %s" %
Config.get_config().ota_umkbootimg_path)
E("ota_unpack_ramdisk_path = %s" %
Config.get_config().ota_unpack_ramdisk_path)
raise OTAParseException()
data = file("./ramdisk/system/bin/droidboot", "rb").read()
tmpfile.write(data)
tmpfile.seek(0)
os.chdir(curdir)
shutil.rmtree(tmpdir)
return ('droidboot', tmpfile)
def make_dir(path):
path = shutil.abspath(path)
if os.path.exists(path):
pprint.pprint("Path \"%s\" already exist\n EXITING" % (path))
sys.exit(-1)
os.mkdir(path)
#coding:utf-8
import os
import glob
import shutil
print shutil.abspath(__file__)
print __file__
curpath = os.path.dirname(__file__)
test_dir = os.path.join(curpath, 'test')
test_dir_1 = os.path.join(curpath, 'test_1')
test_file = os.path.join(test_dir, 'test.txt')
# 初始化测试目录和文件
if not os.path.exists(test_dir):
os.mkdir(test_dir)
if not os.path.exists(test_file):
with open(test_file, 'w') as f:
f.write('test')
#copyfile
shutil.copyfile(test_file, os.path.join(test_dir, 'test1.txt'))
#copy
shutil.copy(test_file, os.path.join(test_dir, 'test2.txt'))
#copy2 复制软连接等属性
shutil.copy2(test_file, os.path.join(test_dir, 'test3.txt'))
#copyfileobj
#coding:utf-8
import os
import glob
import shutil
print shutil.abspath(__file__)
print __file__
curpath = os.path.dirname(__file__)
test_dir = os.path.join(curpath, 'test')
test_dir_1 = os.path.join(curpath, 'test_1')
test_file = os.path.join(test_dir, 'test.txt')
# 初始化测试目录和文件
if not os.path.exists(test_dir):
os.mkdir(test_dir)
if not os.path.exists(test_file):
with open(test_file, 'w') as f:
f.write('test')
#copyfile
shutil.copyfile(test_file, os.path.join(test_dir, 'test1.txt'))
#copy
shutil.copy(test_file, os.path.join(test_dir, 'test2.txt'))
#copy2 复制软连接等属性
shutil.copy2(test_file, os.path.join(test_dir, 'test3.txt'))
#copyfileobj
f = open(os.path.join(test_dir, 'test.txt'), 'r')
#!/usr/bin/env python import os import shutil mpath=os.path.dirname(shutil.abspath(__file__)) print mpath
def Evtx_dumper(self):
self.seperator()
evtx_file_name = str(raw_input("Enter the file name or press enter to select \"Security.evtx : "))
if evtx_file_name == "":
evtx_file_name = "Security.evtx"
dumped_logs_path = "C:\\Windows_Event_logs\\Logs\\"+evtx_file_name
self.seperator()
print "Dumped log file path : "+dumped_logs_path
#creating directory in same folder
if not os.path.exists("Windows_Event_logs"):
os.mkdir("Windows_Event_logs")
log_path = shutil.abspath("Windows_Event_logs")
#copying security.evtx log file to current directory
if os.path.exists("C:\Windows_Event_logs\Logs"):
shutil.copy(dumped_logs_path , log_path)
self.seperator()
print "File Copied : "+evtx_file_name
else:
self.seperator()
print r"Please run dump_logs.bat first to dump Security.evtx in C:\Windows_Event_logs\Logs"
#parsing evtx
file = open("logs.xml", "w")
file_rec = open("rec.txt", "w")
i=0
def spinning_cursor():
while True:
for cursor in '|/-\\':
yield cursor
spinner = spinning_cursor()
event_id_in = str(raw_input("Enter the event id or press enter for all events : "))
self.seperator()
no_of_records = str(raw_input("Enter no of records or press enter for 10 records : "))
self.seperator()
print "Starting Conversion of Evtx to XML"
self.seperator()
print "\n\"#\" - for record found \n\"-\" for no record found \n\"_\" for all records \n"
with open("Windows_Event_logs\\"+evtx_file_name, 'r') as f:
with contextlib.closing(mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)) as buf:
fh = FileHeader(buf, 0x0)
file.write("<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?>")
file.write("<Events>")
self.seperator()
i = 0
if no_of_records == "":
no_of_records = 100
for xml, record in Evtx.Views.evtx_file_xml_view(fh):
#print xml
if i<int(no_of_records):
if event_id_in == "":
file.write(xml)
sys.stdout.write("_")
else:
xmldoc = minidom.parseString(xml)
event_id = xmldoc.getElementsByTagName('EventID')[0].childNodes[0].nodeValue
if int(event_id) == int(event_id_in):
file.write(xml)
sys.stdout.write("#")
else:
sys.stdout.write("-")
sys.stdout.write(spinner.next())
sys.stdout.flush()
sys.stdout.write('\b')
file_rec.write(str(record)+"\n")
i=i+1
else:
break
file.write("</Events>")
print "\n"
self.seperator()
print "\nFile Writing Finished"
self.module2()
def Evtx_dumper(self):
self.seperator()
evtx_file_name = str(
raw_input(
"Enter the file name or press enter to select \"Security.evtx : "
))
if evtx_file_name == "":
evtx_file_name = "Security.evtx"
dumped_logs_path = "C:\\Windows_Event_logs\\Logs\\" + evtx_file_name
self.seperator()
print "Dumped log file path : " + dumped_logs_path
#creating directory in same folder
if not os.path.exists("Windows_Event_logs"):
os.mkdir("Windows_Event_logs")
log_path = shutil.abspath("Windows_Event_logs")
#copying security.evtx log file to current directory
if os.path.exists("C:\Windows_Event_logs\Logs"):
shutil.copy(dumped_logs_path, log_path)
self.seperator()
print "File Copied : " + evtx_file_name
else:
self.seperator()
print r"Please run dump_logs.bat first to dump Security.evtx in C:\Windows_Event_logs\Logs"
#parsing evtx
file = open("logs.xml", "w")
file_rec = open("rec.txt", "w")
i = 0
def spinning_cursor():
while True:
for cursor in '|/-\\':
yield cursor
spinner = spinning_cursor()
event_id_in = str(
raw_input("Enter the event id or press enter for all events : "))
self.seperator()
no_of_records = str(
raw_input("Enter no of records or press enter for 10 records : "))
self.seperator()
print "Starting Conversion of Evtx to XML"
self.seperator()
print "\n\"#\" - for record found \n\"-\" for no record found \n\"_\" for all records \n"
with open("Windows_Event_logs\\" + evtx_file_name, 'r') as f:
with contextlib.closing(
mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)) as buf:
fh = FileHeader(buf, 0x0)
file.write(
"<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?>"
)
file.write("<Events>")
self.seperator()
i = 0
if no_of_records == "":
no_of_records = 100
for xml, record in Evtx.Views.evtx_file_xml_view(fh):
#print xml
if i < int(no_of_records):
if event_id_in == "":
file.write(xml)
sys.stdout.write("_")
else:
xmldoc = minidom.parseString(xml)
event_id = xmldoc.getElementsByTagName(
'EventID')[0].childNodes[0].nodeValue
if int(event_id) == int(event_id_in):
file.write(xml)
sys.stdout.write("#")
else:
sys.stdout.write("-")
sys.stdout.write(spinner.next())
sys.stdout.flush()
sys.stdout.write('\b')
file_rec.write(str(record) + "\n")
i = i + 1
else:
break
file.write("</Events>")
print "\n"
self.seperator()
print "\nFile Writing Finished"
self.module2()
# -*- coding: UTF-8 -*-
#但是复制文件的函数居然不再os模块中存在，原因是复制文件并非由操作系统提供的系统调用，理论上我们通过读写文件可以完成文件复制
#幸运的是shutil模块提供来copyfile()函数，还可以在shutil模块中找到很多实用的函数，它们可以看成是os模块的补充
import shutil
shutil.copyfile()
shutil.copy()
shutil.copy2()
print shutil.abspath('.')
def __init__(self):
self.version = subprocess.Popen(["git","log","-n","1"],stdout=subprocess.PIPE).communicate()[0].split("\n")[0].split(" ")[1]
self.regex=re.compile(r"\W")
self.encoder = json.JSONEncoder().encode
self.baseDir = shutil.abspath(os.curdir)
if len(sys.argv) == 2:
if os.path.isfile(sys.argv[1]):
args = open(sys.argv[1]).readline().strip("\n").split(" ")
self.downDir = shutil.abspath(args[1])
self.cookieFile = shutil.abspath(args[2])
if len(args) == 4:
self.password = args[3]
else:
self.password = ""
try:
port = int(args[0])
except ValueError:
print "First Value of configfile have to be a number"
sys.exit()
else:
print sys.argv[1],"does not exist or is a directory"
sys.exit()
else:
print "Usage: rsbot.py configfile\ncontents: port downloadDir cookieFile password"
sys.exit()
if not os.path.exists(self.downDir):
try:
os.mkdir(self.downDir)
except OSError:
print "can't create downloadDir"
sys.exit()
self.downFile = self.baseDir+os.sep+"downloads"
try:
dumpFile = open(self.downFile,"r")
except IOError:
self.listDownloads = {}
self.waitingDownloads = {}
else:
data = json.load(dumpFile)
if not type(data) == list:
self.listDownloads,self.waitingDownloads = data,{}
else:
self.listDownloads,self.waitingDownloads = data
dumpFile.close()
self.dumpDownloads()
self.runningDownloads={}
self.validateDownloads(True)
self.resumeDownload()
self.authed = False
self.rarExit={255:"aborted by user",
9:"could not create file",
8:"not enough memory",
7:"wrong parameters",
6:"could not open file",
5:"could not write to disk",
4:"archive is locked",
3:"an CRC error occured",
2:"an fatal error occured",
1:"warning"
}
self.extractQueue = [] # simple list of dicts with kwd args to Popen
self.extractManager = False # indicator wether a manager thread is running or not
self.threadcount = 1 # how many extract processes are spawned at a time TODO: make configurable
self.happy = False # indicates happyhours
if os.path.exists(self.baseDir+os.sep+"cputime"):
if os.path.isfile(self.baseDir+os.sep+"cputime"):
self.cputime = self.baseDir+os.sep+"cputime"
elif os.path.exists(self.baseDir+os.sep+"cputime"+os.sep+"cputime") and os.path.isfile(self.baseDir+os.sep+"cputime"+os.sep+"cputime"):
self.cputime = self.baseDir+os.sep+"cputime"+os.sep+"cputime"
else:
self.cputime = None
fd = open(self.cookieFile,"r")
cookie = None
for i in fd.readlines():
match = re.match(r".*rapidshare\.com.*\s(\w+)",i,re.IGNORECASE)
if match:
cookie = match.group(1)
break
if cookie:
self.account = accountManager(cookie=cookie)
else:
self.account = None
self.spaceFactor = 0.2 # if less than 20 percent (0.2) of the hdd is available no downloads are started
self.acceptor = acceptor(self)
self.acceptor.spaceFactor = 1 - self.spaceFactor # factor by that the total diskspace is reduced to show actually available space
self.acceptor.startListen(port)
