Пример #1
0
 def run(self):
     self.prepMeds()
     byband = {}
     "  Run injection here "
     " Prepare data for injected images "
     self.prepInData()
     " Create coadd images for each band and each revision "
     for realV in self.realizationslist:
         outpathR = os.path.join(self.medsdir,self.medsconf+'/balrog_images/'+str(realV)+'/'+self.tilename+'/coadd')
         realD =  os.path.join(self.medsdir,self.medsconf+'/balrog_images/'+str(realV)+'/'+self.tilename)
         print " outpathR = %s \n" % outpathR
         logpathR = shutil.abspath(outpathR)+'/LOGS/'
         listpathR = shutil.abspath(outpathR)+'/lists/'
         for band in self.bands:
            self.makeCoadd(band,outpathR,self.tiledir)
         " Now make detection image "
         self.makeDetectionImage(outpathR)
         " Create s-extractor catalogs "
         for band in self.bands:
             self.makeCatalog(band,outpathR)
         " Now create fake objectMaps "
         self.makeObjMaps(realD)
         " Now crete meds for all bands "
         for band in self.bands:
             self.makeMeds(band,realD)
Пример #2
0
def make_project(dest, project_name, scan_path):
    dest = shutil.abspath(dest)
    scan_path = shutil.abspath(scan_path)
    project_path = dest + "/" + project_name
    make_dir(project_path)
    shutil.copy(scan_path, project_path)
    scan_path = project_path + "/" + os.path.basename(scan_path)
    process_pdf.convert_pdf_to_image(scan_path)
    return project_path
Пример #3
0
    def __init__(self, rootDir, mainDomain):
        self.cmdHost = None
        if runo('hostname')[0].rstrip() != self.IMAGE_STORAGE_HOST:
            self.cmdHost = self.IMAGE_STORAGE_HOST

        if shutil.abspath(rootDir) != rootDir:
            Exception('Absolute base path required!')
        self.rootDir = shutil.abspath(rootDir)  # removes trailing slashes
        self.baseDir = '%s/%s' % (self.rootDir, self.BASE_IMAGE_PREFIX)
        self.mainDomain = mainDomain

        self.mainImage = self.getImagePathForDomain(self.mainDomain)
Пример #4
0
def bubbles(bubble_path, net, debug_path=None, align=False):
    i = 0
    bubble_path = shutil.abspath(bubble_path)
    bubble_files        = sorted(os.listdir(bubble_path))
    top_img             = cv2.imread(bubble_path + "/" + bubble_files[i])
    bubble_locations    = contour.grab_bubbles(top_img)
    # use a copy because we want the top img to remain unchanged
    score               = score_bubbles(top_img.copy() , net, bubble_locations)

    result = score[0]
    if not debug_path is None:
        analyzed_image_path = "%s/%05d.jpg" % (debug_path, i)
        #print "Input file %s to %s" % (bubble_files[0], analyzed_image_path)
        cv2.imwrite(analyzed_image_path, score[1])

    i += 1
    for bubble in bubble_files[1:]:
        img =  cv2.imread(bubble_path + "/" + bubble)
        if not align:
            bubble_locations = contour.grab_bubbles(img)
        else:
            img = alignImages(img, top_img)
        score   = score_bubbles(img, net, bubble_locations)
        result += score[0]
        if not debug_path is None:
            analyzed_image_path = "%s/%05d.jpg" % (debug_path, i)
            #print "Input file %s to %s" % (bubble, analyzed_image_path)
            cv2.imwrite(analyzed_image_path, score[1])
        i += 1
    return result
Пример #5
0
def longforms(longform_path, dest, decide=False):
    longform_path = shutil.abspath(longform_path)
    longforms_files     = sorted(os.listdir(longform_path))
    top_img             = cv2.imread(longform_path + "/" + longforms_files[0])
    questions_location  = contour.grab_answers(top_img)
    result = {}
    for i, longform in enumerate(longforms_files):
        img     = cv2.imread(longform_path  + "/" + longform)
#        img     = alignImages(img, top_img)
        for j, question_contour in enumerate(questions_location):
            (x, y, w, h) = question_contour
            question_img = img[y : y + h , x : x + w ]

            if decide:
                question_img = cv2.cvtColor(question_img, cv2.COLOR_BGR2GRAY)
                ret, thresh = cv2.threshold(question_img, 240, 255, cv2.THRESH_BINARY_INV)
                if cv2.countNonZero(thresh) > 5500:
                    print "GOOD %d  %s/q%02d-%s" % (cv2.countNonZero(thresh), dest, j, longform)
                    cv2.imwrite("%s/q%02d-%s" % (dest, j, longform), question_img)
                else:
                    print "BAD %d  %s/bad-q%02d-%s" % (cv2.countNonZero(thresh), dest, j, longform)
                    cv2.imwrite("%s/bad-q%02d-%s" % (dest, j, longform), question_img)
            else:
                cv2.imwrite("%s/q%02d-%s" % (dest, j, longform), question_img)
            try:
                result[j].append("%s/q%02d-%s" % (dest, j, longform))
            except KeyError:
                result[i] = [ "%s/q%02d-%s" % (dest, j, longform) ]
    return result
Пример #6
0
    def makeObjMaps(self,outpath):  
        print "makeObjMaps outpath=%s \n" % outpath     
        for band in self.bands:
#            fname = os.path.join(outpath,'/lists/'+self.tilename+'_'+band+'_objmap-'+self.medsconf+'.fits')
            fname = shutil.abspath(outpath) + '/lists/'+self.tilename+'_'+band+'_objmap-'+self.medsconf+'.fits'
            print "Obj map file %s \n" % fname
            catname = outpath+'/coadd/'+self.tilename+'_'+band+'_cat.fits'
            print "makeObjMaps catname=%s n" % catname
            if os.path.exists(fname):
                os.remove(fname)
            self.make_fake_objmap(catname, fname)
Пример #7
0
	def download_isos(self):
		# download zip file
		resp = download(self._html)
		f = open('./pyiso/iso_tmp/geo.txt', 'w+')
		f.write(resp)
		f.close()

		# remove previous corrupted file
		try:
			os.remove(abspath(self._iso_path + 'corrupt_metadata_ids.txt'))
		except:
			pass

		for line in open('./pyiso/iso_tmp/geo.txt','r'):
			line_spl = line.split('\t')
			if line_spl[1].strip() == 'id':
				continue
			else:
				iid = int(line_spl[1].strip())
				html = self._meta + str(iid)
				# download the iso
				xmlstr = download(html)
				try:
					tree = etree.fromstring(xmlstr)
					# need the file identifier
					fi = tree.find('.//' + self._file_identifier + '/' + self._character_string)
					fname = fi.text
					path = self._iso_path + fname + self._meta_folder
					path = abspath(path)
					print '\nWriting to: ' + path
					if not os.path.exists(path):
						os.makedirs(path)

					f = open(path + '/' + self._meta_file, 'w+')
					f.write(xmlstr)
					f.close()
				except:
					path = abspath(self._iso_path + 'corrupt_metadata_ids.txt')
					f = open(path, 'a+')
					f.write('\nCorrupted/Missing xml for id: ' + str(iid))
					f.close()
Пример #8
0
def filePath():
    f = "data.txt"
    print(shutil.abspath(f))

    print(os.path.exists(""))
    print(os.path.isfile(f))
    print(os.path.isdir(f))
    print(os.path.islink(f))
    print(os.path.isabs(f))

    print(os.path.split("a/b/c"))
    print(os.path.join("a", "b", "c"))
    print(os.path.splitext("dir/file.ext"))
Пример #9
0
def filePath():
    f = "data.txt"
    print(shutil.abspath(f))
    
    print(os.path.exists(""))
    print(os.path.isfile(f))
    print(os.path.isdir(f))
    print(os.path.islink(f)) 
    print(os.path.isabs(f))

    print(os.path.split( "a/b/c" ))
    print(os.path.join( "a", "b", "c" ))
    print(os.path.splitext( "dir/file.ext" ))
Пример #10
0
    def __init__(self, confile, tilename):
        '''
        Constructor
        '''
        urlbase = "https://desar2.cosmology.illinois.edu/DESFiles/"
        self.confile = confile
        self.workdir = os.getcwd()
        self.tilename = tilename
        self.conf = self.read_config(confile)
        self.dbname = self.conf['dbname']
        self.bands = self.conf['bands']
        self.det = self.conf['det']
        self.medsconf = self.conf['medsconf']  # 'y3v02'
        print self.medsconf
        self.medsdir = os.getenv('MEDS_DATA')
        self.bbase = os.getenv('BALROG_BASE')
        self.desdata = os.getenv('DESDATA')
        self.simData = self.medsdir + '/' + self.medsconf + '/balrog_images/'
        print "simData = %s \n" % self.simData
        print "meds_dir=%s \n" % self.medsdir
        self.mofconfile = ''
        self.pixscale = 0.2636
        self.magbase = 30.

        self.curdir = os.getcwd()
        self.autocommit = True
        self.quiet = False
        print " Make coadds for bands: \n"
        print self.bands
        print " detection image : \n"
        print self.det
        desfile = os.getenv("DES_SERVICES")
        if not desfile:
            desfile = os.path.join(os.getenv("HOME"), ".desservices.ini")
        #
        self.desconfig = easyaccess.config_ea.get_desconfig(
            desfile, self.dbname)
        self.connectDB()
        self.tileinfo = self.get_tile_infor()
        self.tiledir = self.medsdir + '/' + self.medsconf + '/' + self.tilename
        self.mofdir = self.tiledir + '/mof'
        if not os.path.exists(self.mofdir):
            os.makedirs(self.mofdir)
        self.realDir = ''
        self.curdir = os.getcwd()
        #
        self.outpath = os.path.join(
            self.medsdir, self.medsconf + '/' + self.tilename + '/coadd/')
        self.logpath = shutil.abspath(self.outpath) + '/LOGS/'
Пример #11
0
def process_project(project_path, net):
    project_path = shutil.abspath(project_path)
    make_dir(project_path + "/analyzed_bubbles")
    make_dir(project_path + "/analyzed_longforms")
    longforms = analyze.longforms(project_path + "/long_forms",
                                  project_path + "/analyzed_longforms")
    matrix = analyze.bubbles(project_path + "/bubbles/", net, \
                            align=True, \
                            debug_path=project_path + "/analyzed_bubbles")
    histograms = []
    for i, row in enumerate(matrix):
        histograms.append(
            plots.histogram(
                numpy.asarray(row).reshape(-1), i + 1, project_path))
    return (histograms, longforms)
Пример #12
0
 def __init__(self, filename=None):
     """
     """
     self.stream = None
     self.filename = shutil.abspath(filename)
     if filename:
         self.stream = io.open(self.filename,'rb')
     self.entries = []
     self.header = self.Header()
     self.index_flags = 0
     self.static_key = ResourceKey()
     self.__name_map = None
     self.__loaded_resources = {}
     self.__pending_deletes = {}
     self.cache = False
     self.load()
Пример #13
0
    def get_aboot_image(self):
        d = self.get_device()

        for path in Config.get_config().ota_prevalent_aboot_paths:
            try:
                return (os.path.basename(path), self.zip.open(path))
            except KeyError:
                pass

        if 'flounder' in d:
            data = self.zip.read('bootloader.img')[256:]
            fp = io.BytesIO(data)
            return ('hboot.img', zipfile.ZipFile(fp).open('hboot.img'))

        if 'fugu' == d:
            tmpdir = tempfile.mkdtemp()
            tmpfile = tempfile.NamedTemporaryFile()
            self.zip.extract('droidboot.img', tmpdir)
            curdir = shutil.abspath(".")
            os.chdir(tmpdir)
            try:
                subprocess.check_output(
                    [Config.get_config().ota_umkbootimg_path, "droidboot.img"],
                    stderr=subprocess.STDOUT)
                subprocess.check_output([
                    Config.get_config().ota_unpack_ramdisk_path,
                    "initramfs.cpio.gz"
                ],
                                        stderr=subprocess.STDOUT)
            except OSError as e:
                E("Cannot execute umkbootimg/unpack_ramdisk while handling %s. "
                  % self.path)
                E("ota_umkbootimg_path = %s" %
                  Config.get_config().ota_umkbootimg_path)
                E("ota_unpack_ramdisk_path = %s" %
                  Config.get_config().ota_unpack_ramdisk_path)
                raise OTAParseException()

            data = file("./ramdisk/system/bin/droidboot", "rb").read()
            tmpfile.write(data)
            tmpfile.seek(0)
            os.chdir(curdir)
            shutil.rmtree(tmpdir)
            return ('droidboot', tmpfile)
Пример #14
0
def make_dir(path):
    path = shutil.abspath(path)
    if os.path.exists(path):
        pprint.pprint("Path \"%s\" already exist\n EXITING" % (path))
        sys.exit(-1)
    os.mkdir(path)
Пример #15
0
#coding:utf-8
import os
import glob
import shutil

print shutil.abspath(__file__)
print __file__

curpath = os.path.dirname(__file__)

test_dir = os.path.join(curpath, 'test')
test_dir_1 = os.path.join(curpath, 'test_1')
test_file = os.path.join(test_dir, 'test.txt')

# 初始化测试目录和文件
if not os.path.exists(test_dir):
    os.mkdir(test_dir)

if not os.path.exists(test_file):
    with open(test_file, 'w') as f:
        f.write('test')

#copyfile
shutil.copyfile(test_file, os.path.join(test_dir, 'test1.txt'))
#copy
shutil.copy(test_file, os.path.join(test_dir, 'test2.txt'))
#copy2  复制软连接等属性
shutil.copy2(test_file, os.path.join(test_dir, 'test3.txt'))


#copyfileobj
Пример #16
0
#coding:utf-8
import os
import glob
import shutil

print shutil.abspath(__file__)
print __file__

curpath = os.path.dirname(__file__)

test_dir = os.path.join(curpath, 'test')
test_dir_1 = os.path.join(curpath, 'test_1')
test_file = os.path.join(test_dir, 'test.txt')

# 初始化测试目录和文件
if not os.path.exists(test_dir):
    os.mkdir(test_dir)

if not os.path.exists(test_file):
    with open(test_file, 'w') as f:
        f.write('test')

#copyfile
shutil.copyfile(test_file, os.path.join(test_dir, 'test1.txt'))
#copy
shutil.copy(test_file, os.path.join(test_dir, 'test2.txt'))
#copy2  复制软连接等属性
shutil.copy2(test_file, os.path.join(test_dir, 'test3.txt'))

#copyfileobj
f = open(os.path.join(test_dir, 'test.txt'), 'r')
Пример #17
0
#!/usr/bin/env python
import os
import shutil

mpath=os.path.dirname(shutil.abspath(__file__))
print mpath
Пример #18
0
    def Evtx_dumper(self):
        self.seperator()
        evtx_file_name = str(raw_input("Enter the file name or press enter to select \"Security.evtx : "))
        if evtx_file_name == "":
            evtx_file_name = "Security.evtx"
        dumped_logs_path = "C:\\Windows_Event_logs\\Logs\\"+evtx_file_name
        self.seperator()
        print "Dumped log file path : "+dumped_logs_path

        #creating directory in same folder
        if not os.path.exists("Windows_Event_logs"):
            os.mkdir("Windows_Event_logs")    
        log_path = shutil.abspath("Windows_Event_logs")
        #copying security.evtx log file to current directory

        if os.path.exists("C:\Windows_Event_logs\Logs"):
            shutil.copy(dumped_logs_path , log_path)
            self.seperator()
            print "File Copied : "+evtx_file_name
        else:    
            self.seperator()
            print r"Please run dump_logs.bat first to dump Security.evtx in C:\Windows_Event_logs\Logs"        
        #parsing evtx
        file = open("logs.xml", "w")
        file_rec = open("rec.txt", "w")
        i=0

        def spinning_cursor():
            while True:
                for cursor in '|/-\\':
                    yield cursor

        spinner = spinning_cursor()
        event_id_in = str(raw_input("Enter the event id or press enter for all events : "))
        self.seperator()
        no_of_records = str(raw_input("Enter no of records or press enter for 10 records : "))
        self.seperator()
        print "Starting Conversion of Evtx to XML"
        self.seperator()
        print "\n\"#\" - for record found \n\"-\" for no record found \n\"_\" for all records \n"
        with open("Windows_Event_logs\\"+evtx_file_name, 'r') as f:
            with contextlib.closing(mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)) as buf:
                fh = FileHeader(buf, 0x0)
                file.write("<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?>")
                file.write("<Events>")
                self.seperator()
    
                i = 0
                if no_of_records == "":
                    no_of_records = 100

                for xml, record in Evtx.Views.evtx_file_xml_view(fh):            
                    #print xml
                    if i<int(no_of_records):
                        if event_id_in == "":
                            file.write(xml)
                            sys.stdout.write("_")
                        else:
                            xmldoc = minidom.parseString(xml)
                            event_id = xmldoc.getElementsByTagName('EventID')[0].childNodes[0].nodeValue
                            if int(event_id) == int(event_id_in):                
                                file.write(xml)
                                sys.stdout.write("#")            
                            else:
                                sys.stdout.write("-")
                        sys.stdout.write(spinner.next())
                        sys.stdout.flush()
                        sys.stdout.write('\b')
                        file_rec.write(str(record)+"\n")         
                        i=i+1
                    else:
                        break
                file.write("</Events>")
                print "\n"
                self.seperator()
                print "\nFile Writing Finished"
        self.module2()
Пример #19
0
    def Evtx_dumper(self):
        self.seperator()
        evtx_file_name = str(
            raw_input(
                "Enter the file name or press enter to select \"Security.evtx : "
            ))
        if evtx_file_name == "":
            evtx_file_name = "Security.evtx"
        dumped_logs_path = "C:\\Windows_Event_logs\\Logs\\" + evtx_file_name
        self.seperator()
        print "Dumped log file path : " + dumped_logs_path

        #creating directory in same folder
        if not os.path.exists("Windows_Event_logs"):
            os.mkdir("Windows_Event_logs")
        log_path = shutil.abspath("Windows_Event_logs")
        #copying security.evtx log file to current directory

        if os.path.exists("C:\Windows_Event_logs\Logs"):
            shutil.copy(dumped_logs_path, log_path)
            self.seperator()
            print "File Copied : " + evtx_file_name
        else:
            self.seperator()
            print r"Please run dump_logs.bat first to dump Security.evtx in C:\Windows_Event_logs\Logs"
        #parsing evtx
        file = open("logs.xml", "w")
        file_rec = open("rec.txt", "w")
        i = 0

        def spinning_cursor():
            while True:
                for cursor in '|/-\\':
                    yield cursor

        spinner = spinning_cursor()
        event_id_in = str(
            raw_input("Enter the event id or press enter for all events : "))
        self.seperator()
        no_of_records = str(
            raw_input("Enter no of records or press enter for 10 records : "))
        self.seperator()
        print "Starting Conversion of Evtx to XML"
        self.seperator()
        print "\n\"#\" - for record found \n\"-\" for no record found \n\"_\" for all records \n"
        with open("Windows_Event_logs\\" + evtx_file_name, 'r') as f:
            with contextlib.closing(
                    mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)) as buf:
                fh = FileHeader(buf, 0x0)
                file.write(
                    "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?>"
                )
                file.write("<Events>")
                self.seperator()

                i = 0
                if no_of_records == "":
                    no_of_records = 100

                for xml, record in Evtx.Views.evtx_file_xml_view(fh):
                    #print xml
                    if i < int(no_of_records):
                        if event_id_in == "":
                            file.write(xml)
                            sys.stdout.write("_")
                        else:
                            xmldoc = minidom.parseString(xml)
                            event_id = xmldoc.getElementsByTagName(
                                'EventID')[0].childNodes[0].nodeValue
                            if int(event_id) == int(event_id_in):
                                file.write(xml)
                                sys.stdout.write("#")
                            else:
                                sys.stdout.write("-")
                        sys.stdout.write(spinner.next())
                        sys.stdout.flush()
                        sys.stdout.write('\b')
                        file_rec.write(str(record) + "\n")
                        i = i + 1
                    else:
                        break
                file.write("</Events>")
                print "\n"
                self.seperator()
                print "\nFile Writing Finished"
        self.module2()
Пример #20
0
# -*- coding: UTF-8 -*-
#但是复制文件的函数居然不再os模块中存在,原因是复制文件并非由操作系统提供的系统调用,理论上我们通过读写文件可以完成文件复制
#幸运的是shutil模块提供来copyfile()函数,还可以在shutil模块中找到很多实用的函数,它们可以看成是os模块的补充
import shutil
shutil.copyfile()
shutil.copy()
shutil.copy2()
print shutil.abspath('.')
Пример #21
0
 def __init__(self):
     self.version = subprocess.Popen(["git","log","-n","1"],stdout=subprocess.PIPE).communicate()[0].split("\n")[0].split(" ")[1]
     self.regex=re.compile(r"\W")
     self.encoder = json.JSONEncoder().encode
     self.baseDir = shutil.abspath(os.curdir)
     if len(sys.argv) == 2:
         if os.path.isfile(sys.argv[1]):
             args = open(sys.argv[1]).readline().strip("\n").split(" ")
             self.downDir = shutil.abspath(args[1])
             self.cookieFile = shutil.abspath(args[2])
             if len(args) == 4:
                 self.password = args[3]
             else:
                 self.password = ""
             try:
                 port = int(args[0])
             except ValueError:
                 print "First Value of configfile have to be a number"
                 sys.exit()
         else:
             print sys.argv[1],"does not exist or is a directory"
             sys.exit()
     else:
         print "Usage: rsbot.py configfile\ncontents: port downloadDir cookieFile password"
         sys.exit()
     if not os.path.exists(self.downDir):
         try:
             os.mkdir(self.downDir)
         except OSError:
             print "can't create downloadDir"
             sys.exit()
     self.downFile = self.baseDir+os.sep+"downloads"
     try:
         dumpFile = open(self.downFile,"r")
     except IOError:
         self.listDownloads = {}
         self.waitingDownloads = {}
     else:
         data = json.load(dumpFile)
         if not type(data) == list:
             self.listDownloads,self.waitingDownloads = data,{}
         else:
             self.listDownloads,self.waitingDownloads = data
         dumpFile.close()
     self.dumpDownloads()
     self.runningDownloads={}
     self.validateDownloads(True)
     self.resumeDownload()
     self.authed = False
     self.rarExit={255:"aborted by user",
              9:"could not create file",
              8:"not enough memory",
              7:"wrong parameters",
              6:"could not open file",
              5:"could not write to disk",
              4:"archive is locked",
              3:"an CRC error occured",
              2:"an fatal error occured",
              1:"warning"
              }
     self.extractQueue = [] # simple list of dicts with kwd args to Popen
     self.extractManager = False # indicator wether a manager thread is running or not
     self.threadcount = 1 # how many extract processes are spawned at a time TODO: make configurable
     self.happy = False # indicates happyhours
     if os.path.exists(self.baseDir+os.sep+"cputime"):
         if os.path.isfile(self.baseDir+os.sep+"cputime"):
             self.cputime = self.baseDir+os.sep+"cputime"
         elif os.path.exists(self.baseDir+os.sep+"cputime"+os.sep+"cputime") and os.path.isfile(self.baseDir+os.sep+"cputime"+os.sep+"cputime"):
             self.cputime = self.baseDir+os.sep+"cputime"+os.sep+"cputime"
     else:
         self.cputime = None
     fd = open(self.cookieFile,"r")
     cookie = None
     for i in fd.readlines():
         match = re.match(r".*rapidshare\.com.*\s(\w+)",i,re.IGNORECASE)
         if match:
             cookie = match.group(1)
             break
     if cookie:
         self.account = accountManager(cookie=cookie)
     else:
         self.account = None
     self.spaceFactor = 0.2 # if less than 20 percent (0.2) of the hdd is available no downloads are started
     self.acceptor = acceptor(self)
     self.acceptor.spaceFactor = 1 - self.spaceFactor # factor by that the total diskspace is reduced to show actually available space
     self.acceptor.startListen(port)