def setUpClass(cls): cls.empty_file = tempfile.mkstemp() cls.empty_dir = tempfile.mkdtemp() # Create one good textfile cls.good_file = tempfile.mkstemp() # Create a color image cls.color_image_file = tempfile.mkstemp(suffix='.png') cls.numpy_image_color = np.ones((8,10,3), dtype='uint8') cls.pil_image_color = PIL.Image.fromarray(cls.numpy_image_color) cls.pil_image_color.save(cls.color_image_file[1]) # Create a grayscale image cls.gray_image_file = tempfile.mkstemp(suffix='.png') cls.numpy_image_gray = np.ones((8,10), dtype='uint8') cls.pil_image_gray = PIL.Image.fromarray(cls.numpy_image_gray) cls.pil_image_gray.save(cls.gray_image_file[1]) cls.image_count = 0 for i in xrange(3): for j in xrange(3): os.write(cls.good_file[0], '%s %s\n' % (cls.color_image_file[1], i)) os.write(cls.good_file[0], '%s %s\n' % (cls.gray_image_file[1], i)) cls.image_count += 2
def PlotFit( g, data, cols=(0,1) ): fh1, fn1 = tempfile.mkstemp() a,b = cols os.close(fh1) outfile = open(fn1, "w") for d in data: outfile.write("%f\t%f\n" % (d[a], d[b])) outfile.close() parameters = {} fh2, fn2 = tempfile.mkstemp() fh3, fn3 = tempfile.mkstemp() os.close(fh2) os.close(fh3) open(fn2, 'w').write('m=0\nb=0\n') g("f%i(x) = m * x + y0" % b) g("fit f%i(x) '%s' using 1:2 via y0, m" % (b, fn1)) g("replot f%i(x)" % (b)) ## g('fit m*x+b "%s" via "%s"' % (fn1, fn2) ) ## g('update "%s" "%s"' % (fn2, fn3)) ## execfile( fn3, globals(), parameters ) ## g.replot( Gnuplot.Func( "%f*x + %f" % (parameters['m'], parameters['b']) ) ) return [fn1, fn2, fn3]
def tag_raw_data(raw) : (name, temp) = tempfile.mkstemp() (rname, rtemp) = tempfile.mkstemp() f = os.fdopen(name, 'w') #weird function because we are using a tempfile f.writelines([r + '\n' for r in raw]) f.close() #call external tagger print(TAG_COMMAND.format(temp)) f = os.fdopen(rname) result = subprocess.call(TAG_COMMAND.format(temp),stdout=f, cwd=TAGGER_PATH, shell=True) #read them back in f.seek(0) result = f.readlines() f.close() result = [x.strip() for x in result] final_result = [] buf = [] for i in range(len(result)) : if (result[i] == '') : final_result.append(tuple(buf)) #you have to cut of that extra ' ' buf = [] else : pieces = result[i].split('\t') buf.append( (pieces[0],pieces[1] ) ) if (buf != []) : final_result.append(buf) return final_result
def compile_and_load(src_files, obj_files=[], cc="g++", flags=["O3", "Wall"], includes=[], links=[], defs=[]): """ Compile and load a shared object from a source file \param src_files list of source fiels (eg. ['~/src/foo.cc']) \param cc the path to the c++ compiler \param obj_files name of files to compile (eg. ['~/obj/foo.o']) \param flags list of falgs for the compiler (eg. ['O3', 'Wall']) \param includes list of directories to include (eg. ['~/includes/']) \param links list of libraries to link with (eg. ['pthread', 'gtest']) \param defs list of names to define with -D (eg. ['ENABLE_FOO']) \return (lib, fin) link to the library and a function to call to close the library """ __, obj_name = tempfile.mkstemp(suffix=".o", dir=TEMP_DIR) os.close(__) __, lib_name = tempfile.mkstemp(suffix=".so", dir=TEMP_DIR) os.close(__) compile_bin(obj_name, src_files, cc=cc, flags=flags, includes=includes, links=links, defs=defs, lib=True) # add the newly compiled object file to the list of objects for the lib obj_files = list(obj_files) obj_files.append(obj_name) compile_so(lib_name, obj_files, cc=cc, links=links) def finalize(): if os.path.exists(obj_name): os.unlink(obj_name) if os.path.exists(lib_name): os.unlink(lib_name) try: lib = ctypes.CDLL(lib_name) return lib, finalize except OSError: print "Failed link with library, source files:" print ", ".join(src_files) raise
def print_xcf(self, filename_or_obj, *args, **kwargs): "Writes the figure to a GIMP XCF image file" # If filename_or_obj is a file-like object we need a temporary file for # GIMP's output too... if is_string(filename_or_obj): out_temp_handle, out_temp_name = None, filename_or_obj else: out_temp_handle, out_temp_name = tempfile.mkstemp(suffix='.xcf') try: # Create a temporary file and write the "layer" to it as a PNG in_temp_handle, in_temp_name = tempfile.mkstemp(suffix='.png') try: FigureCanvasAgg.print_png(self, in_temp_name, *args, **kwargs) run_gimp_script( SINGLE_LAYER_SCRIPT.format( input=quote_string(in_temp_name), output=quote_string(out_temp_name))) finally: os.close(in_temp_handle) os.unlink(in_temp_name) finally: if out_temp_handle: os.close(out_temp_handle) # If we wrote the XCF to a temporary file, write its content to # the file-like object we were given (the copy is chunked as # XCF files can get pretty big) with open(out_temp_name, 'rb') as source: for chunk in iter(lambda: source.read(131072), ''): filename_or_obj.write(chunk) os.unlink(out_temp_name)
def ndiff(scan_a, scan_b): """Run Ndiff on two scan results, which may be filenames or NmapParserSAX objects, and return a running NdiffCommand object.""" temporary_filenames = [] if isinstance(scan_a, NmapParserSAX): fd, filename_a = tempfile.mkstemp( prefix=APP_NAME + "-diff-", suffix=".xml" ) temporary_filenames.append(filename_a) f = os.fdopen(fd, "wb") scan_a.write_xml(f) f.close() else: filename_a = scan_a if isinstance(scan_b, NmapParserSAX): fd, filename_b = tempfile.mkstemp( prefix=APP_NAME + "-diff-", suffix=".xml" ) temporary_filenames.append(filename_b) f = os.fdopen(fd, "wb") scan_b.write_xml(f) f.close() else: filename_b = scan_b return NdiffCommand(filename_a, filename_b, temporary_filenames)
def _execute(cmds): import subprocess stderrPath = tempfile.mkstemp()[1] stdoutPath = tempfile.mkstemp()[1] stderrFile = open(stderrPath, "w") stdoutFile = open(stdoutPath, "w") # get the os.environ env = _makeEnviron() # make a string of escaped commands cmds = subprocess.list2cmdline(cmds) # go popen = subprocess.Popen(cmds, stderr=stderrFile, stdout=stdoutFile, env=env, shell=True) popen.wait() # get the output stderrFile.close() stdoutFile.close() stderrFile = open(stderrPath, "r") stdoutFile = open(stdoutPath, "r") stderr = stderrFile.read() stdout = stdoutFile.read() stderrFile.close() stdoutFile.close() # trash the temp files os.remove(stderrPath) os.remove(stdoutPath) # done return stderr, stdout
def test_H_get_put(self, sftp): """ verify that get/put work. """ warnings.filterwarnings('ignore', 'tempnam.*') fd, localname = mkstemp() os.close(fd) text = b'All I wanted was a plastic bunny rabbit.\n' with open(localname, 'wb') as f: f.write(text) saved_progress = [] def progress_callback(x, y): saved_progress.append((x, y)) sftp.put(localname, sftp.FOLDER + '/bunny.txt', progress_callback) with sftp.open(sftp.FOLDER + '/bunny.txt', 'rb') as f: assert text == f.read(128) assert [(41, 41)] == saved_progress os.unlink(localname) fd, localname = mkstemp() os.close(fd) saved_progress = [] sftp.get(sftp.FOLDER + '/bunny.txt', localname, progress_callback) with open(localname, 'rb') as f: assert text == f.read(128) assert [(41, 41)] == saved_progress os.unlink(localname) sftp.unlink(sftp.FOLDER + '/bunny.txt')
def take_screenshot(self, filename): """Take a screenshot and save it to 'filename'""" # make sure filename is unicode filename = ensure_unicode(filename, "utf-8") if get_platform() == "windows": # use win32api to take screenshot # create temp file f, imgfile = tempfile.mkstemp(u".bmp", filename) os.close(f) mswin.screenshot.take_screenshot(imgfile) else: # use external app for screen shot screenshot = self.get_external_app("screen_shot") if screenshot is None or screenshot.prog == "": raise Exception(_("You must specify a Screen Shot program in Application Options")) # create temp file f, imgfile = tempfile.mkstemp(".png", filename) os.close(f) proc = subprocess.Popen([screenshot.prog, imgfile]) if proc.wait() != 0: raise OSError("Exited with error") if not os.path.exists(imgfile): # catch error if image is not created raise Exception(_("The screenshot program did not create the necessary image file '%s'") % imgfile) return imgfile
def _test_batch(self, setsize, batchsize): # Create a set of testfiles dummyfiles = [] for i in range(setsize): fh, dummyfile = tempfile.mkstemp(suffix=".batcher_test", dir=self._test_folder) os.close(fh) dummyfiles.append(dummyfile) # Create a file that should not be returned fh, nomatchfile = tempfile.mkstemp(suffix=".batcher_nomatch_test", dir=self._test_folder) os.close(fh) # Pattern pattern = os.path.join(self._test_folder, "*.batcher_test") batcher = Batcher(pattern,batchsize) expected_bno = 0 for b in batcher: expected_size = min(batchsize, setsize - batchsize*expected_bno) expected_bno += 1 # Assert that each batch has the expected size self.assertEqual(len(b),expected_size, "The returned batchsize does not match the expected size ({} != {})".format(len(b),expected_size)) # Assert that the batch enumerator gives the correct batch number self.assertEqual(batcher.batchno(),expected_bno, "The returned batch number does not match the expected ({} != {})".format(batcher.batchno(),expected_bno)) for bf in b: # Assert that the returned files existed in the original array and has not already been returned self.assertIn(bf, dummyfiles, "The returned file is not present in the expected set ({})".format(bf)) self.assertTrue(os.path.exists(bf), "The returned file has already been seen ({})".format(bf)) # Remove the returned files os.unlink(bf) # Assert that all files have been returned by the iterator for dummyfile in dummyfiles: self.assertFalse(os.path.exists(dummyfile), "Some files were not iterated over ({})".format(dummyfile)) os.unlink(nomatchfile)
def test_pipeline_with_temp(self): input_f = tempfile.mkstemp(suffix='capsul_input.txt') os.close(input_f[0]) input_name = input_f[1] open(input_name, 'w').write('this is my input data\n') output_f = tempfile.mkstemp(suffix='capsul_output.txt') os.close(output_f[0]) output_name = output_f[1] #os.unlink(output_name) try: self.pipeline.input_image = input_name self.pipeline.output_image = output_name # run sequentially self.pipeline() # test self.assertTrue(os.path.exists(output_name)) self.assertEqual(open(input_name).read(), open(output_name).read()) finally: try: os.unlink(input_name) except: pass try: os.unlink(output_name) except: pass
def chunk_scaffolds(target, size): chromos = [] # split target file into `options.size` (~10 Mbp) chunks temp_fd, temp_out = tempfile.mkstemp(suffix='.fasta') os.close(temp_fd) temp_out_handle = open(temp_out, 'w') tb = bx.seq.twobit.TwoBitFile(file(target)) sequence_length = 0 tb_key_len = len(tb.keys()) - 1 print '\nRunning against {}'.format(os.path.basename(target)) print 'Running with the --huge option. Chunking files into {0} bp...'.format(size) for sequence_count, seq in enumerate(tb.keys()): sequence = tb[seq][0:] sequence_length += len(sequence) # write it to the outfile temp_out_handle.write('>{0}\n{1}\n'.format(seq, sequence)) if sequence_length > size: temp_out_handle.close() # put tempfile name on stack chromos.append(temp_out) # open a new temp file temp_fd, temp_out = tempfile.mkstemp(suffix='.fasta') os.close(temp_fd) temp_out_handle = open(temp_out, 'w') # reset sequence length sequence_length = 0 # if we hit the end of the twobit file elif sequence_count >= tb_key_len: temp_out_handle.close() # put tempfile name on stack chromos.append(temp_out) return chromos
def test_cache(self): """Test the caching mechanism in the reporter.""" length = random.randint(1, 30) exit_code = random.randint(0, 3) threshold = random.randint(0, 10) message = ''.join(random.choice(string.printable) for x in range(length)) message = message.rstrip() (handle, filename) = tempfile.mkstemp() os.unlink(filename) os.close(handle) reporter = NagiosReporter('test_cache', filename, threshold, self.nagios_user) nagios_exit = [NAGIOS_EXIT_OK, NAGIOS_EXIT_WARNING, NAGIOS_EXIT_CRITICAL, NAGIOS_EXIT_UNKNOWN][exit_code] reporter.cache(nagios_exit, message) (handle, output_filename) = tempfile.mkstemp() os.close(handle) try: old_stdout = sys.stdout buffer = StringIO.StringIO() sys.stdout = buffer reporter_test = NagiosReporter('test_cache', filename, threshold, self.nagios_user) reporter_test.report_and_exit() except SystemExit, err: line = buffer.getvalue().rstrip() sys.stdout = old_stdout buffer.close() self.assertTrue(err.code == nagios_exit[0]) self.assertTrue(line == "%s %s" % (nagios_exit[1], message))
def test_simple(self): outstream = BytesIO() errstream = BytesIO() # create a file for initial commit handle, fullpath = tempfile.mkstemp(dir=self.repo.path) filename = os.path.basename(fullpath) porcelain.add(repo=self.repo.path, paths=filename) porcelain.commit(repo=self.repo.path, message='test', author='test', committer='test') # Setup target repo target_path = tempfile.mkdtemp() porcelain.clone(self.repo.path, target=target_path, outstream=outstream) # create a second file to be pushed handle, fullpath = tempfile.mkstemp(dir=self.repo.path) filename = os.path.basename(fullpath) porcelain.add(repo=self.repo.path, paths=filename) porcelain.commit(repo=self.repo.path, message='test2', author='test2', committer='test2') # Pull changes into the cloned repo porcelain.pull(target_path, self.repo.path, 'refs/heads/master', outstream=outstream, errstream=errstream) # Check the target repo for pushed changes r = Repo(target_path) self.assertEqual(r['HEAD'].id, self.repo['HEAD'].id)
def run_shell_cmd(cmd, echo=True): """ Run a command in a sub-shell, capturing stdout and stderr to temporary files that are then read. """ _, stdout_f = tempfile.mkstemp() _, stderr_f = tempfile.mkstemp() print("Running command") print(cmd) p = subprocess.Popen( '{} >{} 2>{}'.format(cmd, stdout_f, stderr_f), shell=True) p.wait() with open(stdout_f) as f: stdout = f.read() os.remove(stdout_f) with open(stderr_f) as f: stderr = f.read() os.remove(stderr_f) if echo: print("stdout:") print(stdout) print("stderr:") print(stderr) return stdout, stderr
def setUp(self): # Changes CONFIGDEFAULT super(TestConfig, self).setUp() osfd, filename = tempfile.mkstemp(suffix='.ini', dir=self.config.CONFIGDEFAULT) os.close(osfd) self.config.DEFAULTSFILE = filename self.deffile = open(filename, 'wb') # ConfigSection will open again self.deffile.close() # ConfigDict forbids writing foo = self.config.ConfigSection(None, 'DEFAULTS') # Verify these all are converted to lower-case automatically foo.set('tEsTOPTioNi', 2) # non-string values should also convert foo.set('TesToPTIONf', 3.14) foo.set('testoptionS', "foobarbaz") foo.write(self.deffile) # Set up separate test config file osfd, filename = tempfile.mkstemp(suffix='.ini', dir=self.config.CONFIGDEFAULT) os.close(osfd) # in case a test needs it self.cfgfile = open(filename, 'wb') # ConfigSection will open again self.cfgfile.close() # ConfigDict forbids writing bar = self.config.ConfigSection(None, 'TestSection') bar.set('TestOptionB', False) bar.set('TesTopTIONs', "baz!") bar.set("testoptionx", "True") # should convert to boolean bar.write(self.cfgfile)
def testParseMaincfg(self): """ Test parsing of different broker_module declarations """ path = "/var/lib/nagios/rw/livestatus" # Path to the livestatus socket # Test plain setup with no weird arguments fd, filename = tempfile.mkstemp() os.write(fd, 'broker_module=./livestatus.o /var/lib/nagios/rw/livestatus') status = pynag.Parsers.mk_livestatus(nagios_cfg_file=filename) self.assertEqual(path, status.livestatus_socket_path) os.close(fd) # Test what happens if arguments are provided fd, filename = tempfile.mkstemp() os.write(fd, 'broker_module=./livestatus.o /var/lib/nagios/rw/livestatus hostgroups=t') status = pynag.Parsers.mk_livestatus(nagios_cfg_file=filename) self.assertEqual(path, status.livestatus_socket_path) os.close(fd) # Test what happens if arguments are provided before and after file socket path fd, filename = tempfile.mkstemp() os.write(fd, 'broker_module=./livestatus.o num_client_threads=20 /var/lib/nagios/rw/livestatus hostgroups=t') status = pynag.Parsers.mk_livestatus(nagios_cfg_file=filename) self.assertEqual(path, status.livestatus_socket_path) os.close(fd) # Test what happens if livestatus socket path cannot be found try: fd, filename = tempfile.mkstemp() os.write(fd, 'broker_module=./livestatus.o num_client_threads=20') status = pynag.Parsers.mk_livestatus(nagios_cfg_file=filename) self.assertEqual(path, status.livestatus_socket_path) os.close(fd) self.assertEqual(True, "Above could should have raised exception") except pynag.Parsers.ParserError: pass
def translate(args): if args.update_translation: #from website _,fname = mkstemp(suffix=".pot") os.system("pybabel extract -F website/babel.cfg -o %s website/" % fname) os.system("pybabel update -D messages -i %s -d translations/" % fname) os.remove(fname) #from documentation from website.utils import Documentation fhandle,fname = mkstemp(suffix=".pot") with os.fdopen(fhandle,"w") as fid: docs = Documentation(os.path.join(args.repo,'website','docs','sources')) docs.extract_messages(fid) os.system("pybabel update -D docs -d translations -i %s" % fname) os.remove(fname) #from parts data repo = Repository(args.repo) from backends.translations import TranslationBackend fhandle,fname = mkstemp(suffix=".pot") TranslationBackend(repo,[]).write_output(fname) os.system("pybabel update -D parts -d translations -i %s" % fname) os.remove(fname) if args.compile_translation: os.system("pybabel compile -D messages -d translations/") os.system("pybabel compile -D parts -d translations/") os.system("pybabel compile -D docs -d translations/")
def prepare_files(self): if urlparse.urlsplit(self.inurl)[0] == 'file': self.infname = urllib.url2pathname(urlparse.urlsplit(self.inurl)[2]) self.infd = open(self.infname) else: # not a file url. download it. source = urllib.urlopen(self.inurl) self.infd, self.infname = tempfile.mkstemp(prefix="transcode-in-", suffix="." + self.inext) self._files_to_clean_up_on_success.append((self.infd, self.infname)) self._files_to_clean_up_on_error.append((self.infd, self.infname)) while True: chunk = source.read(1024 * 64) if not chunk: break os.write(self.infd, chunk) os.lseek(self.infd, 0, 0) self.outfd, self.outfname = tempfile.mkstemp(prefix="transcode-out-", suffix="." + self.tofmt) self._files_to_clean_up_on_error.append((self.outfd, self.outfname)) self.errfh, self.errfname = tempfile.mkstemp(prefix="transcode-", suffix=".log") self.outurl = urlparse.urlunsplit( ["file", None, self.outfname, None, None]) self._files_to_clean_up_on_success.append((self.errfh, self.errfname)) log.debug("Reading from " + self.infname + " (" + self.inurl + ")") log.debug("Outputting to " + self.outfname + " (" + self.outurl + ")") log.debug("Errors to " + self.errfname)
def setUp(self): self.tmpdir = tempfile.mkdtemp() self.rest_service_log = tempfile.mkstemp()[1] self.securest_log_file = tempfile.mkstemp()[1] self.file_server = FileServer(self.tmpdir) self.addCleanup(self.cleanup) self.file_server.start() storage_manager.storage_manager_module_name = \ STORAGE_MANAGER_MODULE_NAME # workaround for setting the rest service log path, since it's # needed when 'server' module is imported. # right after the import the log path is set normally like the rest # of the variables (used in the reset_state) tmp_conf_file = tempfile.mkstemp()[1] json.dump({'rest_service_log_path': self.rest_service_log, 'rest_service_log_file_size_MB': 1, 'rest_service_log_files_backup_count': 1, 'rest_service_log_level': 'DEBUG'}, open(tmp_conf_file, 'w')) os.environ['MANAGER_REST_CONFIG_PATH'] = tmp_conf_file try: from manager_rest import server finally: del(os.environ['MANAGER_REST_CONFIG_PATH']) server.reset_state(self.create_configuration()) utils.copy_resources(config.instance().file_server_root) server.setup_app() server.app.config['Testing'] = True self.app = server.app.test_client() self.client = self.create_client() self.initialize_provider_context()
def add_lang(args): #from website _,fname = mkstemp(suffix=".pot") os.system("pybabel extract -F website/babel.cfg -o %s website/" % fname) os.system("pybabel init -D messages -d translations/ -i %s -l %s" % (fname,args.lang)) os.remove(fname) #from documentation from website.utils import Documentation fhandle,fname = mkstemp(suffix=".pot") with os.fdopen(fhandle,"w") as fid: docs = Documentation(os.path.join(args.repo,'website','docs','sources')) docs.extract_messages(fid) os.system("pybabel init -D docs -d translations/ -i %s -l %s" % (fname,args.lang)) os.remove(fname) #from parts data repo = Repository(args.repo) from backends.translations import TranslationBackend fhandle,fname = mkstemp(suffix=".pot") TranslationBackend(repo,[]).write_output(fname) os.system("pybabel init -D parts -d translations/ -i %s -l %s" % (fname,args.lang)) os.remove(fname) print("Don't forget to edit website/templates/base.html and add the language to the dropdown menu")
def process_delta(data, delta): if not delta.specific.delta: return data if delta.specific.delta == 'cat': datalines = data.split('\n') for line in delta.data.split('\n'): if not line: continue if line[0] == '+': datalines.append(line[1:]) elif line[0] == '-': if line[1:] in datalines: datalines.remove(line[1:]) return "\n".join(datalines) elif delta.specific.delta == 'diff': basehandle, basename = tempfile.mkstemp() basefile = open(basename, 'w') basefile.write(data) basefile.close() os.close(basehandle) dhandle, dname = tempfile.mkstemp() dfile = open(dname, 'w') dfile.write(delta.data) dfile.close() os.close(dhandle) ret = os.system("patch -uf %s < %s > /dev/null 2>&1" \ % (basefile.name, dfile.name)) output = open(basefile.name, 'r').read() [os.unlink(fname) for fname in [basefile.name, dfile.name]] if ret >> 8 != 0: raise Bcfg2.Server.Plugin.PluginExecutionError, ('delta', delta) return output
def cmpIgProfReport(outdir,file1,file2,IgProfMemOpt=""): (tfile1, tfile2) = ("", "") try: # don't make temp files in /tmp because it's never bloody big enough (th1, tfile1) = tmp.mkstemp(prefix=os.path.join(outdir,"igprofRegressRep.")) (th2, tfile2) = tmp.mkstemp(prefix=os.path.join(outdir,"igprofRegressRep.")) os.close(th1) os.close(th2) os.remove(tfile1) os.remove(tfile2) ungzip2(file1,tfile1) ungzip2(file2,tfile2) perfreport(1,tfile1,tfile2,outdir,IgProfMemOpt) os.remove(tfile1) os.remove(tfile2) except OSError as detail: raise PerfReportErr("WARNING: The OS returned the following error when comparing %s and %s\n%s" % (file1,file2,str(detail))) if os.path.exists(tfile1): os.remove(tfile1) if os.path.exists(tfile2): os.remove(tfile2) except IOError as detail: raise PerfReportErr("IOError: When comparing %s and %s using temporary files %s and %s. Error message:\n%s" % (file1,file2,tfile1,tfile2,str(detail))) if os.path.exists(tfile1): os.remove(tfile1) if os.path.exists(tfile2): os.remove(tfile2)
def tempfilter(s, cmd): '''filter string S through a pair of temporary files with CMD. CMD is used as a template to create the real command to be run, with the strings INFILE and OUTFILE replaced by the real names of the temporary files generated.''' inname, outname = None, None try: infd, inname = tempfile.mkstemp(prefix='hg-filter-in-') fp = os.fdopen(infd, 'wb') fp.write(s) fp.close() outfd, outname = tempfile.mkstemp(prefix='hg-filter-out-') os.close(outfd) cmd = cmd.replace('INFILE', inname) cmd = cmd.replace('OUTFILE', outname) code = os.system(cmd) if sys.platform == 'OpenVMS' and code & 1: code = 0 if code: raise Abort(_("command '%s' failed: %s") % (cmd, explain_exit(code))) return open(outname, 'rb').read() finally: try: if inname: os.unlink(inname) except: pass try: if outname: os.unlink(outname) except: pass
def svg_formatter(data, format) : """ Generate a logo in Scalable Vector Graphics (SVG) format. Requires the program 'pdf2svg' be installed. """ pdf = pdf_formatter(data, format) try: command = find_command('pdf2svg') except EnvironmentError: raise EnvironmentError("Scalable Vector Graphics (SVG) format requires the program 'pdf2svg'. " "Cannot find 'pdf2svg' on search path.") import tempfile fpdfi, fname_pdf = tempfile.mkstemp(suffix=".pdf") fsvgi, fname_svg = tempfile.mkstemp(suffix=".svg") try: fpdf2 = open(fname_pdf, 'w') if sys.version_info[0] >= 3: fpdf2.buffer.write(pdf) else: fpdf2.write(pdf) fpdf2.seek(0) args = [command, fname_pdf, fname_svg] p = Popen(args) (out,err) = p.communicate() fsvg = open(fname_svg) return fsvg.read().encode() finally: os.remove(fname_svg) os.remove(fname_pdf)
def _inject_admin_password_into_fs(admin_passwd, fs): """Set the root password to admin_passwd admin_password is a root password fs is the path to the base of the filesystem into which to inject the key. This method modifies the instance filesystem directly, and does not require a guest agent running in the instance. """ # The approach used here is to copy the password and shadow # files from the instance filesystem to local files, make any # necessary changes, and then copy them back. LOG.debug(_("Inject admin password fs=%(fs)s " "admin_passwd=ha-ha-not-telling-you") % locals()) admin_user = '******' fd, tmp_passwd = tempfile.mkstemp() os.close(fd) fd, tmp_shadow = tempfile.mkstemp() os.close(fd) passwd_path = os.path.join('etc', 'passwd') shadow_path = os.path.join('etc', 'shadow') passwd_data = fs.read_file(passwd_path) shadow_data = fs.read_file(shadow_path) new_shadow_data = _set_passwd(admin_user, admin_passwd, passwd_data, shadow_data) fs.replace_file(shadow_path, new_shadow_data)
def dump_to_file(blurb, exit_fpr): """ Dump the given blurb to a randomly generated file which contains exit_fpr. This function is useful to save data obtained from bad exit relays to file for later analysis. """ if analysis_dir is None: fd, file_name = tempfile.mkstemp(prefix="%s_" % exit_fpr) else: try: os.makedirs(analysis_dir) except OSError as err: if err.errno != errno.EEXIST: raise fd, file_name = tempfile.mkstemp(prefix="%s_" % exit_fpr, dir=analysis_dir) try: with open(file_name, "w") as fd: fd.write(blurb) except IOError as err: log.warning("Couldn't write to \"%s\": %s" % (file_name, err)) return None log.debug("Wrote %d-length blurb to file \"%s\"." % (len(blurb), file_name)) return file_name
def shellScriptInWindow(c,script): if sys.platform == 'darwin': #@ << write script to temporary MacOS file >> #@+node:ekr.20040915105758.22:<< write script to temporary MacOS file >> handle, path = tempfile.mkstemp(text=True) directory = c.frame.openDirectory script = ("cd %s\n" % directory) + script + '\n' + ("rm -f %s\n" % path) os.write(handle, script) os.close(handle) os.chmod(path, 0700) #@nonl #@-node:ekr.20040915105758.22:<< write script to temporary MacOS file >> #@nl os.system("open -a /Applications/Utilities/Terminal.app " + path) elif sys.platform == 'win32': g.es("shellScriptInWindow not ready for Windows",color='red') else: #@ << write script to temporary Unix file >> #@+node:ekr.20040915105758.25:<< write script to temporary Unix file >> handle, path = tempfile.mkstemp(text=True) directory = c.frame.openDirectory script = ("cd %s\n" % directory) + script + '\n' + ("rm -f %s\n" % path) os.write(handle, script) os.close(handle) os.chmod(path, 0700) #@nonl #@-node:ekr.20040915105758.25:<< write script to temporary Unix file >> #@nl os.system("xterm -e sh " + path)
def change_admin_password(self, password): root_logger.debug("Changing admin password") dirname = config_dirname(self.serverid) dmpwdfile = "" admpwdfile = "" try: (dmpwdfd, dmpwdfile) = tempfile.mkstemp(dir='/var/lib/ipa') os.write(dmpwdfd, self.dm_password) os.close(dmpwdfd) (admpwdfd, admpwdfile) = tempfile.mkstemp(dir='/var/lib/ipa') os.write(admpwdfd, password) os.close(admpwdfd) args = ["/usr/bin/ldappasswd", "-h", self.fqdn, "-ZZ", "-x", "-D", str(DN(('cn', 'Directory Manager'))), "-y", dmpwdfile, "-T", admpwdfile, str(DN(('uid', 'admin'), ('cn', 'users'), ('cn', 'accounts'), self.suffix))] try: env = { 'LDAPTLS_CACERTDIR':os.path.dirname(CACERT), 'LDAPTLS_CACERT':CACERT } ipautil.run(args, env=env) root_logger.debug("ldappasswd done") except ipautil.CalledProcessError, e: print "Unable to set admin password", e root_logger.debug("Unable to set admin password %s" % e) finally: if os.path.isfile(dmpwdfile): os.remove(dmpwdfile) if os.path.isfile(admpwdfile): os.remove(admpwdfile)
def wnominate(state, session, chamber, polarity, r_bin="R", out_file=None): (fd, filename) = tempfile.mkstemp('.csv') with os.fdopen(fd, 'w') as out: vote_csv(state, session, chamber, out) if not out_file: (result_fd, out_file) = tempfile.mkstemp('.csv') os.close(result_fd) r_src_path = os.path.join(os.path.dirname(__file__), 'calc_wnominate.R') with open('/dev/null', 'w') as devnull: subprocess.check_call([r_bin, "-f", r_src_path, "--args", filename, out_file, polarity], stdout=devnull, stderr=devnull) results = {} with open(out_file) as f: c = csv.DictReader(f) for row in c: try: res = float(row['coord1D']) except ValueError: res = None results[row['leg_id']] = res os.remove(filename) return results
def _persist(self, state, checkpoint: Checkpoint): dup = [ c for c in self._checkpoints if c.global_step == checkpoint.global_step ] assert len(dup) <= 1 dup = dup[0] if dup else None if dup is not None: if dup.score is not None and checkpoint.score is None: checkpoint.score = dup.score # save checkpoint target = Path(self._save_dir, checkpoint.filename) if not target.parent.exists(): os.makedirs(str(target.parent), exist_ok=True) fd, name = tempfile.mkstemp(dir=self._save_dir) torch.save(state, name) os.close(fd) shutil.move(name, target) self._checkpoints.append(checkpoint) # remove checkpoints if dup is not None: self._checkpoints.remove(dup) discard = [] evals = sorted([c for c in self._checkpoints if c.score is not None], key=lambda c: c.score) periods = sorted([ c for c in self._checkpoints if c.is_periodic and not c.end_of_epoch ], key=lambda c: c.global_step) epochs = sorted( [c for c in self._checkpoints if c.is_periodic and c.end_of_epoch], key=lambda c: c.global_step) retains = set(evals[-self._keep_best_checkpoint_max:] + periods[-self._keep_checkpoint_max:] + epochs[-self._keep_epoch_checkpoint_max:]) retains = sorted(retains, key=lambda c: c.global_step) discard += [c for c in self._checkpoints if c not in retains] discard = list(set(discard)) for c in discard: path = Path(self._save_dir, c.filename) if path.exists(): path.unlink() self._checkpoints = retains # save meta statistics with Path(self._save_dir, _META_CHECKPOINT_PATH).open('w') as w: w.write( json.dumps([c.__dict__ for c in retains], indent=4, sort_keys=True)) return True
def setUp(self): self.cnt += 1 h, path = tempfile.mkstemp('.ini') assert QgsSettings.setGlobalSettingsPath(path) self.settings = QgsSettings('testqgissettings', 'testqgissettings%s' % self.cnt) self.globalsettings = QSettings(self.settings.globalSettingsPath(), QSettings.IniFormat)
def __init__(self, logging_group, scratch_dir): super(FaultyParser, self).__init__(logging_group) _, self.fake_thumb = tempfile.mkstemp(suffix=".png", dir=scratch_dir)
def __init__(self, logging_group, scratch_dir, archive_path): super(DummyParser, self).__init__(logging_group) _, self.fake_thumb = tempfile.mkstemp(suffix=".png", dir=scratch_dir) self.archive_path = archive_path
# scatter work units to nodes unit = comm.scatter(work, root=0) # =================================== # This should be node-related work # =================================== # open the file on a node f = MPI.File.Open(comm, unit, mode) # create a buffer for the data of size f.Get_size() ba = bytearray(f.Get_size()) # read the contents into a byte array f.Iread(ba) # close the file f.Close() # write buffer to a tempfile descriptor, path = tempfile.mkstemp(suffix='mpi.txt') print(path) tf = os.fdopen(descriptor, 'w') tf.write(ba) tf.close() # get contents of tempfile contents = open(path, 'rU').read() + str(comm.Get_rank()) os.remove(path) # =================================== # End of node-related work # =================================== # gather results result = comm.gather(contents, root=0) # do something with result
it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. """ __author__ = 'Tim Sutton' __date__ = '20/08/2012' __copyright__ = 'Copyright 2012, The QGIS Project' # This will get replaced with a git SHA1 when you do a git archive __revision__ = 'bbcd332a78c7149814e99b5e6a93310f5c00f016' import qgis # NOQA import tempfile import os (myFileHandle, myFilename) = tempfile.mkstemp() os.environ['QGIS_DEBUG'] = '2' os.environ['QGIS_LOG_FILE'] = myFilename from qgis.core import QgsLogger from qgis.testing import unittest # Convenience instances in case you may need them # not used in this test # from qgis.testing import start_app # start_app() class TestQgsLogger(unittest.TestCase): def testLogger(self):
# You should have received a copy of the GNU Lesser General Public # License along with MDTraj. If not, see <http://www.gnu.org/licenses/>. ############################################################################## from __future__ import print_function, division import numpy as np import re, os, tempfile from mdtraj.formats.pdb import pdbstructure from mdtraj.formats.pdb.pdbstructure import PdbStructure from mdtraj.testing import get_fn, eq, raises, assert_warns from mdtraj import load, load_pdb from mdtraj.utils import ilen from mdtraj import Topology pdb = get_fn('native.pdb') fd, temp = tempfile.mkstemp(suffix='.pdb') os.close(fd) def teardown_module(module): """remove the temporary file created by tests in this file this gets automatically called by nose""" os.unlink(temp) def test_pdbread(): p = load(pdb) def test_pdbwrite(): p = load(pdb)
def temp_file(ext='tmp'): """Get a temporary filename with the given extension. This function will actually attempt to create the file.""" tf, fn = tempfile.mkstemp(suffix='.%s' % ext) os.close(tf) return fn
def tmpfile_object(): fdesc, path = tempfile.mkstemp() return os.fdopen(fdesc, 'wb'), path
""" Tests for the config module """ import os from tempfile import mkstemp from onedrive_client.utils.config import config_load from onedrive_client.utils.errors import OneDriveConfigException import pytest _BAD_FILE_PATH = mkstemp()[1] _FILE_PATH = mkstemp()[1] _BAD_CONTENT = ':' _CONTENT = 'parameter: test' # pylint: disable=no-self-use class TestConfigLoad(object): """ Test class for the load functionality """ def setup_class(self): """ Write two temporary files for the tests """ with open(_BAD_FILE_PATH, 'w') as tmpfile: tmpfile.write(_BAD_CONTENT)
os.chdir(execution_path('.')) def teardown(): mapnik.logger.set_severity(default_logging_severity) def compare_map(xml): m = mapnik.Map(256, 256) absolute_base = os.path.abspath(os.path.dirname(xml)) try: mapnik.load_map(m, xml, False, absolute_base) except RuntimeError, e: # only test datasources that we have installed if not 'Could not create datasource' in str(e): raise RuntimeError(str(e)) return (handle, test_map) = tempfile.mkstemp(suffix='.xml', prefix='mapnik-temp-map1-') os.close(handle) (handle, test_map2) = tempfile.mkstemp(suffix='.xml', prefix='mapnik-temp-map2-') os.close(handle) if os.path.exists(test_map): os.remove(test_map) mapnik.save_map(m, test_map) new_map = mapnik.Map(256, 256) mapnik.load_map(new_map, test_map,False,absolute_base) open(test_map2,'w').write(mapnik.save_map_to_string(new_map)) diff = ' diff -u %s %s' % (os.path.abspath(test_map),os.path.abspath(test_map2)) try: eq_(open(test_map).read(),open(test_map2).read()) except AssertionError, e: raise AssertionError('serialized map "%s" not the same after being reloaded, \ncompare with command:\n\n$%s' % (xml,diff))
def main(arguments = None): """ main() function, encapsulated in a method to allow for easy invokation. This method follows Guido van Rossum's suggestions on how to write Python main() functions in order to make them more flexible. By encapsulating the main code of the script in a function and making it take an optional argument the script can be called not only from other modules, but also from the interactive Python prompt. Guido van van Rossum - Python main() functions: http://www.artima.com/weblogs/viewpost.jsp?thread=4829 Keyword arguments: arguments - the list of command line arguments passed to the script. """ if arguments is None: arguments = sys.argv[1:] # ignore argv[0], the script name (options, args) = parser.parse_args(args = arguments) # Adjust the logger level to WARNING, INFO or DEBUG, depending on the # given number of -v options (none, one or two or more, respectively) logging_level = logging.WARNING if options.verbose == 1: logging_level = logging.INFO elif options.verbose >= 2: logging_level = logging.DEBUG logging.basicConfig(format = style.LOG_FORMAT, level = logging_level) # Print the help and abort the execution if there are not two positional # arguments left after parsing the options, as the user must specify at # least one (only one?) input FITS file and the output JSON file. if len(args) < 2: parser.print_help() return 2 # 2 is generally used for command line syntax errors else: sources_img_path = args[0] input_paths = list(set(args[1:-1])) output_json_path = args[-1] # The execution of this module, especially when doing long-term monitoring # of reasonably crowded fields, may easily take several *days*. The least # we can do, in order to spare the end-user from insufferable grief because # of the waste of billions of valuable CPU cycles, is to avoid to have the # output file accidentally overwritten. if os.path.exists(output_json_path): if not options.overwrite: msg = "%sError. The output file '%s' already exists." print msg % (style.prefix, output_json_path) print style.error_exit_message return 1 msg = "%sExamining the headers of the %s FITS files given as input..." print msg % (style.prefix, len(input_paths)) files = fitsimage.InputFITSFiles() for index, img_path in enumerate(input_paths): img = fitsimage.FITSImage(img_path) pfilter = img.pfilter(options.filterk) files[pfilter].append(img) percentage = (index + 1) / len(input_paths) * 100 methods.show_progress(percentage) print # progress bar doesn't include newline print style.prefix # To begin with, we need to identify the most constant stars, something for # which we have to do photometry on all the stars and for all the images of # the campaign. But fret not, as this has to be done only this time: once # we get the light curves of all the stars and for all the images, we will # be able to determine which are the most constant among them and work # always with this subset in order to determine which aperture and sky # annulus are the optimal. msg = "%sDoing initial photometry with FWHM-derived apertures..." print msg % style.prefix print style.prefix # mkstemp() returns a tuple containing an OS-level handle to an open file # and its absolute pathname. Thus, we need to close the file right after # creating it, and tell the photometry module to overwrite (-w) it. kwargs = dict(prefix = 'photometry_', suffix = '.LEMONdB') phot_db_handle, phot_db_path = tempfile.mkstemp(**kwargs) atexit.register(methods.clean_tmp_files, phot_db_path) os.close(phot_db_handle) basic_args = [sources_img_path] + input_paths + \ [phot_db_path, '--overwrite'] phot_args = ['--maximum', options.maximum, '--margin', options.margin, '--cores', options.ncores, '--min-sky', options.min, '--objectk', options.objectk, '--filterk', options.filterk, '--datek', options.datek, '--timek', options.timek, '--expk', options.exptimek, '--coaddk', options.coaddk, '--gaink', options.gaink, '--fwhmk', options.fwhmk, '--airmk', options.airmassk] # The --gain and --uik options default to None, so add them to the list of # arguments only if they were given. Otherwise, (a) --gaink would be given # a value of 'None', a string, that would result in an error when optparse # attempted to convert it to float, and (b) --uik would understood 'None' # as the name of the keyword storing the path to the uncalibrated image. if options.gain: phot_args += ['--gain', options.gain] if options.uncimgk: phot_args += ['--uncimgk', options.uncimgk] # Pass as many '-v' options as we have received here [phot_args.append('-v') for x in xrange(options.verbose)] extra_args = ['--aperture', options.aperture, '--annulus', options.annulus, '--dannulus', options.dannulus] # Non-zero return codes raise subprocess.CalledProcessError args = basic_args + phot_args + extra_args check_run(photometry.main, [str(a) for a in args]) # Now we need to compute the light curves and find those that are most # constant. This, of course, has to be done for each filter, as a star # identified as constant in Johnson I may be too faint in Johnson B, for # example. In other words: we need to calculate the light curve of each # star and for each filter, and then determine which are the # options.nconstant stars with the lowest standard deviation. print style.prefix msg = "%sGenerating light curves for initial photometry." print msg % style.prefix print style.prefix kwargs = dict(prefix = 'diffphot_', suffix = '.LEMONdB') diffphot_db_handle, diffphot_db_path = tempfile.mkstemp(**kwargs) atexit.register(methods.clean_tmp_files, diffphot_db_path) os.close(diffphot_db_handle) diff_args = [phot_db_path, '--output', diffphot_db_path, '--overwrite', '--cores', options.ncores, '--minimum-images', options.min_images, '--stars', options.nconstant, '--minimum-stars', options.min_cstars, '--pct', options.pct, '--weights-threshold', options.wminimum, '--max-iters', options.max_iters, '--worst-fraction', options.worst_fraction] [diff_args.append('-v') for x in xrange(options.verbose)] check_run(diffphot.main, [str(a) for a in diff_args]) print style.prefix # Map each photometric filter to the path of the temporary file where the # right ascension and declination of each constant star, one per line, will # be saved. This file is from now on passed, along with the --coordinates # option, to photometry.main(), so that photometry is not done on all the # astronomical objects, but instead exclusively on these ones. coordinates_files = {} miner = mining.LEMONdBMiner(diffphot_db_path) for pfilter in miner.pfilters: # LEMONdBMiner.sort_by_curve() returns a list of two-element tuples, # mapping the ID of each star to the standard deviation of its light # curve in this photometric filter. The list is sorted in increasing # order by the standard deviation. We are only interested in the first # 'options.nconstant', needing at least 'options.pminimum'. msg = "%sIdentifying the %d most constant stars for the %s filter..." args = style.prefix, options.nconstant, pfilter print msg % args , sys.stdout.flush() kwargs = dict(minimum = options.min_images) stars_stdevs = miner.sort_by_curve_stdev(pfilter, **kwargs) cstars = stars_stdevs[:options.nconstant] if len(cstars) < options.pminimum: msg = ("fewer than %d stars identified as constant in the " "initial photometry for the %s filter") args = options.pminimum, pfilter raise NotEnoughConstantStars(msg % args) else: print 'done.' if len(cstars) < options.nconstant: msg = "%sBut only %d stars were available. Using them all, anyway." print msg % (style.prefix, len(cstars)) # Replacing whitespaces with underscores is easier than having to quote # the path to the --coordinates file if the name of the filter contains # them (otherwise, optparse would only see up to the first whitespace). prefix = '%s_' % str(pfilter).replace(' ', '_') kwargs = dict(prefix = prefix, suffix = '.coordinates') coords_fd, coordinates_files[pfilter] = tempfile.mkstemp(**kwargs) atexit.register(methods.clean_tmp_files, coordinates_files[pfilter]) # LEMONdBMiner.get_star() returns a five-element tuple with the x and y # coordinates, right ascension, declination and instrumental magnitude # of the astronomical object in the sources image. for star_id, _ in cstars: ra, dec = miner.get_star(star_id)[2:4] os.write(coords_fd, "%.10f\t%.10f\n" % (ra, dec)) os.close(coords_fd) msg = "%sStar coordinates for %s temporarily saved to %s" print msg % (style.prefix, pfilter, coordinates_files[pfilter]) # The constant astronomical objects, the only ones to which we will pay # attention from now on, have been identified. So far, so good. Now we # generate the light curves of these objects for each candidate set of # photometric parameters. We store the evaluated values in a dictionary in # which each filter maps to a list of json_parse.CandidateAnnuli objects. evaluated_annuli = collections.defaultdict(list) for pfilter, coords_path in coordinates_files.iteritems(): print style.prefix msg = "%sFinding the optimal photometric parameters for the %s filter." print msg % (style.prefix, pfilter) if len(files[pfilter]) < options.min_images: msg = "fewer than %d images (--minimum-images option) for %s" args = options.min_images, pfilter raise NotEnoughConstantStars(msg % args) # The median FWHM of the images is needed in order to calculate the # range of apertures that we need to evaluate for this filter. msg = "%sCalculating the median FWHM for this filter..." print msg % style.prefix , pfilter_fwhms = [] for img in files[pfilter]: img_fwhm = photometry.get_fwhm(img, options) logging.debug("%s: FWHM = %.3f" % (img.path, img_fwhm)) pfilter_fwhms.append(img_fwhm) fwhm = numpy.median(pfilter_fwhms) print ' done.' # FWHM to range of pixels conversion min_aperture = fwhm * options.lower max_aperture = fwhm * options.upper annulus = fwhm * options.sky dannulus = fwhm * options.width # The dimensions of the sky annulus remain fixed, while the # aperture is in the range [lower * FWHM, upper FWHM], with # increments of options.step pixels. filter_apertures = numpy.arange(min_aperture, max_aperture, options.step) assert filter_apertures[0] == min_aperture msg = "%sFWHM (%s passband) = %.3f pixels, therefore:" print msg % (style.prefix, pfilter, fwhm) msg = "%sAperture radius, minimum = %.3f x %.2f = %.3f pixels " print msg % (style.prefix, fwhm, options.lower, min_aperture) msg = "%sAperture radius, maximum = %.3f x %.2f = %.3f pixels " print msg % (style.prefix, fwhm, options.upper, max_aperture) msg = "%sAperture radius, step = %.2f pixels, which means that:" print msg % (style.prefix, options.step) msg = "%sAperture radius, actual maximum = %.3f + %d x %.2f = %.3f pixels" args = (style.prefix, min_aperture, len(filter_apertures), options.step, max(filter_apertures)) print msg % args msg = "%sSky annulus, inner radius = %.3f x %.2f = %.3f pixels" print msg % (style.prefix, fwhm, options.sky, annulus) msg = "%sSky annulus, width = %.3f x %.2f = %.3f pixels" print msg % (style.prefix, fwhm, options.width, dannulus) msg = "%s%d different apertures in the range [%.2f, %.2f] to be evaluated:" args = (style.prefix, len(filter_apertures), filter_apertures[0], filter_apertures[-1]) print msg % args # For each candidate aperture, and only with the images taken in # this filter, do photometry on the constant stars and compute the # median of the standard deviation of their light curves as a means # of evaluating the suitability of this combination of parameters. for index, aperture in enumerate(filter_apertures): print style.prefix kwargs = dict(prefix = 'photometry_', suffix = '.LEMONdB') fd, aper_phot_db_path = tempfile.mkstemp(**kwargs) atexit.register(methods.clean_tmp_files, aper_phot_db_path) os.close(fd) paths = [img.path for img in files[pfilter]] basic_args = [sources_img_path] + paths + \ [aper_phot_db_path, '--overwrite'] extra_args = ['--filter', str(pfilter), '--coordinates', coords_path, '--aperture-pix', aperture, '--annulus-pix', annulus, '--dannulus-pix', dannulus] args = basic_args + phot_args + extra_args check_run(photometry.main, [str(a) for a in args]) kwargs = dict(prefix = 'diffphot_', suffix = '.LEMONdB') fd, aper_diff_db_path = tempfile.mkstemp(**kwargs) atexit.register(methods.clean_tmp_files, aper_diff_db_path) os.close(fd) # Reuse the arguments used earlier for diffphot.main(). We only # need to change the first argument (path to the input LEMONdB) # and the third one (path to the output LEMONdB) diff_args[0] = aper_phot_db_path diff_args[2] = aper_diff_db_path check_run(diffphot.main, [str(a) for a in diff_args]) miner = mining.LEMONdBMiner(aper_diff_db_path) try: kwargs = dict(minimum = options.min_images) cstars = miner.sort_by_curve_stdev(pfilter, **kwargs) except mining.NoStarsSelectedError: # There are no light curves with at least options.min_images points. # Therefore, much to our sorrow, we cannot evaluate this aperture. msg = "%sNo constant stars for this aperture. Ignoring it..." print msg % style.prefix continue # There must be at most 'nconstant' stars, but there may be fewer # if this aperture causes one or more of the constant stars to be # too faint (INDEF) in so many images as to prevent their lights # curve from being computed. assert len(cstars) <= options.nconstant if len(cstars) < options.pminimum: msg = ("%sJust %d constant stars, fewer than the allowed " "minimum of %d, had their light curves calculated " "for this aperture. Ignoring it...") args = style.prefix, len(cstars), options.pminimum print style.prefix continue # 'cstars' contains two-element tuples: (ID, stdev) stdevs_median = numpy.median([x[1] for x in cstars]) params = (aperture, annulus, dannulus, stdevs_median) # NumPy floating-point data types are not JSON serializable args = (float(x) for x in params) candidate = json_parse.CandidateAnnuli(*args) evaluated_annuli[pfilter].append(candidate) msg = "%sAperture = %.3f, median stdev (%d stars) = %.4f" args = style.prefix, aperture, len(cstars), stdevs_median print msg % args percentage = (index + 1) / len(filter_apertures) * 100 msg = "%s%s progress: %.2f %%" args = style.prefix, pfilter, percentage print msg % args # Let the user know of the best 'annuli', that is, the one for # which the standard deviation of the constant stars is minimal kwargs = dict(key = operator.attrgetter('stdev')) best_candidate = min(evaluated_annuli[pfilter], **kwargs) msg = "%sBest aperture found at %.3f pixels with stdev = %.4f" args = style.prefix, best_candidate.aperture, best_candidate.stdev print msg % args print style.prefix msg = "%sSaving the evaluated apertures to the '%s' JSON file ..." print msg % (style.prefix, output_json_path) , json_parse.CandidateAnnuli.dump(evaluated_annuli, output_json_path) print ' done.' print "%sYou're done ^_^" % style.prefix return 0
def test_default(self): sf = SolverFactory() solve(sf, sf.Default, "examples/normal.png", tempfile.mkstemp())
def handle_file_post(req, allowed_mimetypes=None): """ Handle the POST of a file. @return: the a tuple with th full path to the file saved on disk, and it's mimetype as provided by the request. @rtype: (string, string) """ from invenio.bibdocfile import decompose_file, md5 ## We retrieve the length clen = req.headers_in["Content-Length"] if clen is None: raise InvenioWebInterfaceWSGIContentLenghtError("Content-Length header is missing") try: clen = int(clen) assert (clen > 1) except (ValueError, AssertionError): raise InvenioWebInterfaceWSGIContentLenghtError("Content-Length header should contain a positive integer") ## Let's take the content type ctype = req.headers_in["Content-Type"] if allowed_mimetypes and ctype not in allowed_mimetypes: raise InvenioWebInterfaceWSGIContentTypeError("Content-Type not in allowed list of content types: %s" % allowed_mimetypes) ## Let's optionally accept a suggested filename suffix = prefix = '' g = RE_CDISPOSITION_FILENAME.search(req.headers_in.get("Content-Disposition", "")) if g: dummy, prefix, suffix = decompose_file(g.group("filename")) ## Let's optionally accept an MD5 hash (and use it later for comparison) cmd5 = req.headers_in["Content-MD5"] if cmd5: the_md5 = md5() ## Ok. We can initialize the file fd, path = tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=CFG_TMPDIR) the_file = os.fdopen(fd, 'w') ## Let's read the file while True: chunk = req.read(max(10240, clen)) if len(chunk) < clen: ## We expected to read at least clen (which is different than 0) ## but chunk was shorter! Gosh! Error! Panic! the_file.close() os.close(fd) os.remove(path) raise InvenioWebInterfaceWSGIContentLenghtError("File shorter than what specified in Content-Length") if cmd5: ## MD5 was in the header let's compute it the_md5.update(chunk) ## And let's definitively write the content to disk :-) the_file.write(chunk) clen -= len(chunk) if clen == 0: ## That's it. Everything was read. break if cmd5 and the_md5.hexdigest().lower() != cmd5.strip().lower(): ## Let's check the MD5 the_file.close() os.close(fd) os.remove(path) raise InvenioWebInterfaceWSGIContentMD5Error("MD5 checksum does not match") ## Let's clean everything up the_file.close() return (path, ctype)
def setUp(self): self.fd, self.path = tempfile.mkstemp()
def process_batch(self): """Save the HTML file.""" ok_count = 0 fail_count = 0 old_count = 0 remote_count = 0 try: my_host = socket.gethostname().split(".")[0] except Exception: my_host = socket.gethostname() try: temp_file = tempfile.mkstemp() file_handle = os.fdopen(temp_file[0], "w") file_name = temp_file[1] except Exception: sys.stderr.write( "Couldn't create temporary file for HTML output\n") return output_ok = StringIO() output_fail = StringIO() keys = list(self.batch_data.keys()) keys.sort() for entry in keys: if self.batch_data[entry]["age"] > 120: status = "OLD" old_count += 1 elif self.batch_data[entry]["status"]: status = "OK" ok_count += 1 else: status = "FAIL" fail_count += 1 if self.batch_data[entry]["host"] != my_host: remote_count += 1 try: monitor_name = entry.split("/")[1] except Exception: monitor_name = entry if status == "FAIL": output = output_fail else: output = output_ok output.write("<tr class=\"%srow\">" % status.lower()) output.write(""" <td class="monitor_name">%s</td> <td class="status %s">%s</td> <td>%s</td> <td>%s</td> """ % ( monitor_name, status.lower(), status, self.batch_data[entry]["host"], self.batch_data[entry]["fail_time"], )) if self.batch_data[entry]["fail_count"] == 0: output.write("<td class=\"vfc\"> </td>") else: output.write("<td class=\"vfc\">%s</td>" % self.batch_data[entry]["fail_count"]) try: output.write("<td>%d+%02d:%02d:%02d</td>" % (self.batch_data[entry]["downtime"][0], self.batch_data[entry]["downtime"][1], self.batch_data[entry]["downtime"][2], self.batch_data[entry]["downtime"][3])) except Exception: output.write("<td> </td>") output.write("<td>%s </td>" % (self.batch_data[entry]["fail_data"])) if self.batch_data[entry]["failures"] == 0: output.write("<td></td><td></td>") else: output.write("""<td>%s</td> <td>%s</td>""" % (self.batch_data[entry]["failures"], self.format_datetime( self.batch_data[entry]["last_failure"]))) if self.batch_data[entry]["host"] == my_host: output.write("<td></td>") else: output.write("<td>%d</td>" % self.batch_data[entry]["age"]) output.write("</tr>\n") count_data = "<div id=\"summary\"" if old_count > 0: cls = "old" elif fail_count > 0: cls = "fail" else: cls = "ok" count_data = count_data + " class=\"%s\">%s" % (cls, cls.upper()) self.count_data = count_data + "<div id=\"details\"><span class=\"ok\">%d OK</span> <span class=\"fail\">%d FAIL</span> <span class=\"old\">%d OLD</span> <span class=\"remote\">%d remote</span></div></div>" % ( ok_count, fail_count, old_count, remote_count) self.status = cls.upper() with open(os.path.join(self.folder, self.header), "r") as file_input: file_handle.writelines(self.parse_file(file_input)) file_handle.write(output_fail.getvalue()) file_handle.write(output_ok.getvalue()) with open(os.path.join(self.folder, self.footer), "r") as file_input: file_handle.writelines(self.parse_file(file_input)) try: file_handle.flush() file_handle.close() os.chmod( file_name, stat.S_IREAD | stat.S_IWRITE | stat.S_IRGRP | stat.S_IROTH) shutil.move(file_name, os.path.join(self.folder, self.filename)) except Exception as e: print("problem closing temporary file for HTML output %s" % e)
def _run_wkhtmltopdf(self, cr, uid, headers, footers, bodies, landscape, paperformat, spec_paperformat_args=None, save_in_attachment=None, set_viewport_size=False): """Execute wkhtmltopdf as a subprocess in order to convert html given in input into a pdf document. :param header: list of string containing the headers :param footer: list of string containing the footers :param bodies: list of string containing the reports :param landscape: boolean to force the pdf to be rendered under a landscape format :param paperformat: ir.actions.report.paperformat to generate the wkhtmltopf arguments :param specific_paperformat_args: dict of prioritized paperformat arguments :param save_in_attachment: dict of reports to save/load in/from the db :returns: Content of the pdf as a string """ if not save_in_attachment: save_in_attachment = {} command_args = [] if set_viewport_size: command_args.extend(['--viewport-size', landscape and '1024x1280' or '1280x1024']) # Passing the cookie to wkhtmltopdf in order to resolve internal links. try: if request: command_args.extend(['--cookie', 'session_id', request.session.sid]) except AttributeError: pass # Wkhtmltopdf arguments command_args.extend(['--quiet']) # Less verbose error messages if paperformat: # Convert the paperformat record into arguments command_args.extend(self._build_wkhtmltopdf_args(paperformat, spec_paperformat_args)) # Force the landscape orientation if necessary if landscape and '--orientation' in command_args: command_args_copy = list(command_args) for index, elem in enumerate(command_args_copy): if elem == '--orientation': del command_args[index] del command_args[index] command_args.extend(['--orientation', 'landscape']) elif landscape and '--orientation' not in command_args: command_args.extend(['--orientation', 'landscape']) # Execute WKhtmltopdf pdfdocuments = [] temporary_files = [] for index, reporthtml in enumerate(bodies): local_command_args = [] pdfreport_fd, pdfreport_path = tempfile.mkstemp(suffix='.pdf', prefix='report.tmp.') temporary_files.append(pdfreport_path) # Directly load the document if we already have it if save_in_attachment and save_in_attachment['loaded_documents'].get(reporthtml[0]): with closing(os.fdopen(pdfreport_fd, 'w')) as pdfreport: pdfreport.write(save_in_attachment['loaded_documents'][reporthtml[0]]) pdfdocuments.append(pdfreport_path) continue else: os.close(pdfreport_fd) # Wkhtmltopdf handles header/footer as separate pages. Create them if necessary. if headers: head_file_fd, head_file_path = tempfile.mkstemp(suffix='.html', prefix='report.header.tmp.') temporary_files.append(head_file_path) with closing(os.fdopen(head_file_fd, 'w')) as head_file: head_file.write(headers[index]) local_command_args.extend(['--header-html', head_file_path]) if footers: foot_file_fd, foot_file_path = tempfile.mkstemp(suffix='.html', prefix='report.footer.tmp.') temporary_files.append(foot_file_path) with closing(os.fdopen(foot_file_fd, 'w')) as foot_file: foot_file.write(footers[index]) local_command_args.extend(['--footer-html', foot_file_path]) # Body stuff content_file_fd, content_file_path = tempfile.mkstemp(suffix='.html', prefix='report.body.tmp.') temporary_files.append(content_file_path) with closing(os.fdopen(content_file_fd, 'w')) as content_file: content_file.write(reporthtml[1]) try: wkhtmltopdf = [_get_wkhtmltopdf_bin()] + command_args + local_command_args wkhtmltopdf += [content_file_path] + [pdfreport_path] process = subprocess.Popen(wkhtmltopdf, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = process.communicate() if process.returncode not in [0, 1]: raise UserError(_('Wkhtmltopdf failed (error code: %s). ' 'Message: %s') % (str(process.returncode), err)) # Save the pdf in attachment if marked if reporthtml[0] is not False and save_in_attachment.get(reporthtml[0]): with open(pdfreport_path, 'rb') as pdfreport: attachment = { 'name': save_in_attachment.get(reporthtml[0]), 'datas': base64.encodestring(pdfreport.read()), 'datas_fname': save_in_attachment.get(reporthtml[0]), 'res_model': save_in_attachment.get('model'), 'res_id': reporthtml[0], } try: self.pool['ir.attachment'].create(cr, uid, attachment) except AccessError: _logger.info("Cannot save PDF report %r as attachment", attachment['name']) else: _logger.info('The PDF document %s is now saved in the database', attachment['name']) pdfdocuments.append(pdfreport_path) except: raise # Return the entire document if len(pdfdocuments) == 1: entire_report_path = pdfdocuments[0] else: entire_report_path = self._merge_pdf(pdfdocuments) temporary_files.append(entire_report_path) with open(entire_report_path, 'rb') as pdfdocument: content = pdfdocument.read() # Manual cleanup of the temporary files for temporary_file in temporary_files: try: os.unlink(temporary_file) except (OSError, IOError): _logger.error('Error when trying to remove file %s' % temporary_file) return content
def _download_updates(self): updates_filename = 'jenkins-plugin-cache.json' updates_dir = os.path.expanduser('~/.ansible/tmp') updates_file = "%s/%s" % (updates_dir, updates_filename) download_updates = True # Check if we need to download new updates file if os.path.isfile(updates_file): # Get timestamp when the file was changed last time ts_file = os.stat(updates_file).st_mtime ts_now = time.time() if ts_now - ts_file < self.params['updates_expiration']: download_updates = False updates_file_orig = updates_file # Download the updates file if needed if download_updates: url = "%s/update-center.json" % self.params['updates_url'] # Get the data r = self._get_url_data( url, msg_status="Remote updates not found.", msg_exception="Updates download failed.") # Write the updates file update_fd, updates_file = tempfile.mkstemp() os.write(update_fd, r.read()) try: os.close(update_fd) except IOError: e = get_exception() self.module.fail_json( msg="Cannot close the tmp updates file %s." % updates_file, details=to_native(e)) # Open the updates file try: f = open(updates_file) except IOError: e = get_exception() self.module.fail_json( msg="Cannot open temporal updates file.", details=to_native(e)) i = 0 for line in f: # Read only the second line if i == 1: try: data = json.loads(line) except Exception: e = get_exception() self.module.fail_json( msg="Cannot load JSON data from the tmp updates file.", details=e.message) break i += 1 # Move the updates file to the right place if we could read it if download_updates: # Make sure the destination directory exists if not os.path.isdir(updates_dir): try: os.makedirs(updates_dir, int('0700', 8)) except OSError: e = get_exception() self.module.fail_json( msg="Cannot create temporal directory.", details=e.message) self.module.atomic_move(updates_file, updates_file_orig) # Check if we have the plugin data available if 'plugins' not in data or self.params['name'] not in data['plugins']: self.module.fail_json( msg="Cannot find plugin data in the updates file.") return data['plugins'][self.params['name']]
def format(self, edit, selection=None): """ Format selection (if None then formats the entire document). Returns region containing the reformatted text. """ # determine selection to format if not selection: selection = sublime.Region(0, self.view.size()) self.debug('Formatting selection %r', selection) # retrieve selected text & dedent text = self.view.substr(selection) text, indent, trailing_nl = dedent_text(text) self.debug('Detected indent %r', indent) # encode text try: encoded_text = text.encode(self.encoding) except UnicodeEncodeError as err: msg = ( "You may need to re-open this file with a different encoding." " Current encoding is %r." % self.encoding) self.error("UnicodeEncodeError: %s\n\n%s", err, msg) return # pass source code to be formatted on stdin? if self.get_setting("use_stdin"): # run yapf self.debug('Running %s in %s', self.popen_args, self.popen_cwd) try: popen = subprocess.Popen(self.popen_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, cwd=self.popen_cwd, env=self.popen_env, startupinfo=self.popen_startupinfo) except OSError as err: # always show error in popup msg = "You may need to install YAPF and/or configure 'yapf_command' in PyYapf's Settings." sublime.error_message("OSError: %s\n\n%s" % (err, msg)) return encoded_stdout, encoded_stderr = popen.communicate(encoded_text) text = encoded_stdout.decode(self.encoding) else: # do _not_ use stdin. this avoids a unicode defect in yapf, see # https://github.com/google/yapf/pull/145. the downside is that # .style.yapf / setup.cfg configuration is not picked up properly, # see https://github.com/jason-kane/PyYapf/issues/36. # we may permanently use stdin and remove the use_stdin option and # this code once the upstream bug is fixed. file_obj, temp_filename = tempfile.mkstemp(suffix=".py") try: temp_handle = os.fdopen(file_obj, 'wb' if SUBLIME_3 else 'w') temp_handle.write(encoded_text) temp_handle.close() self.popen_args += ["--in-place", temp_filename] self.debug('Running %s in %s', self.popen_args, self.popen_cwd) try: popen = subprocess.Popen( self.popen_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.popen_cwd, env=self.popen_env, startupinfo=self.popen_startupinfo) except OSError as err: # always show error in popup msg = "You may need to install YAPF and/or configure 'yapf_command' in PyYapf's Settings." sublime.error_message("OSError: %s\n\n%s" % (err, msg)) return encoded_stdout, encoded_stderr = popen.communicate() if SUBLIME_3: open_encoded = open else: import codecs open_encoded = codecs.open with open_encoded(temp_filename, encoding=self.encoding) as fp: text = fp.read() finally: os.unlink(temp_filename) self.debug('Exit code %d', popen.returncode) # handle errors (since yapf>=0.3, exit code 2 means changed, not error) if popen.returncode not in (0, 2): stderr = encoded_stderr.decode(self.encoding) stderr = stderr.replace(os.linesep, '\n') self.debug('Error:\n%s', stderr) # report error err_lines = stderr.splitlines() msg = err_lines[-1] self.error('%s', msg) # attempt to highlight line where error occurred rel_line = parse_error_line(err_lines) if rel_line: line = self.view.rowcol(selection.begin())[0] pt = self.view.text_point(line + rel_line - 1, 0) region = self.view.line(pt) self.view.add_regions(KEY, [region], KEY, 'cross', ERROR_FLAGS) return # adjust newlines (only necessary when use_stdin is True, since # [codecs.]open uses universal newlines by default) text = text.replace(os.linesep, '\n') # re-indent and replace text text = indent_text(text, indent, trailing_nl) self.view.replace(edit, selection, text) # return region containing modified text if selection.a <= selection.b: return sublime.Region(selection.a, selection.a + len(text)) else: return sublime.Region(selection.b + len(text), selection.b)
def backup_autosave_file(path, content): handle, temp_path = tempfile.mkstemp() with open(temp_path, 'w') as f: f.write(content) print('A backup of %s has been saved here: %s' % (path, temp_path))
def setUp(self): self.var_name = 'TRMM_3B42_daily_precipitation_V7' _, self.mfst_file_name = tempfile.mkstemp()
def mktemp(dir=None): """Create a temp file, known by name, in a semi-secure manner.""" handle, filename = mkstemp(dir=dir) os.close(handle) return filename
sys.stderr.write("fork #2 failed: (%d) %s\n" % (e.errorno, e.strerror)) sys.exit(1) def atexit_cb(): try: os.remove(pidfile) except OSError: pass atexit.register(atexit_cb) try: if not os.path.exists(os.path.dirname(pidfile)): os.mkdir(os.path.dirname(pidfile)) fd, nm = tempfile.mkstemp(dir=os.path.dirname(pidfile)) os.write(fd, '%d\n' % os.getpid()) os.close(fd) os.rename(nm, pidfile) except: raise for f in sys.stdout, sys.stderr: f.flush() si = file(stdin, 'r') so = file(stdout, 'a+') se = file(stderr, 'a+', 0) if hasattr(sys.stdin, "fileno"): os.dup2(si.fileno(), sys.stdin.fileno()) if hasattr(sys.stdout, "fileno"):
def configure(ctx, self, target=None, arch=None, plat=None, options=[], force_configure=False): """ Configure a Unikraft application. """ if not self.is_configured(): self.init() if target is not None and isinstance(target, Target): arch = target.architecture plat = target.platform archs = list() plats = list() def match_arch(arch, target): if isinstance(arch, six.string_types) and \ arch == target.architecture.name: return target.architecture if isinstance(arch, Architecture) and \ arch.name == target.architecture.name: return arch return None def match_plat(plat, target): if isinstance(plat, six.string_types) and \ plat == target.platform.name: return target.platform if isinstance(plat, Platform) and \ plat.name == target.platform.name: return plat return None if len(self.config.targets.all()) == 1 \ and target is None and arch is None and plat is None: target = self.config.targets.all()[0] archs.append(target.architecture) plats.append(target.platform) else: for t in self.config.targets.all(): if match_arch(arch, t) is not None \ and match_plat(plat, t) is not None: archs.append(t.architecture) plats.append(t.platform) # Generate a dynamic .config to populate defconfig with based on # configure's parameterization. dotconfig = list() dotconfig.extend(self.config.unikraft.kconfig or []) for arch in archs: if not arch.is_downloaded: raise MissingComponent(arch.name) dotconfig.extend(arch.kconfig) dotconfig.append(arch.kconfig_enabled_flag) for plat in plats: if not plat.is_downloaded: raise MissingComponent(plat.name) dotconfig.extend(plat.kconfig) dotconfig.append(plat.kconfig_enabled_flag) for lib in self.config.libraries.all(): if not lib.is_downloaded: raise MissingComponent(lib.name) dotconfig.extend(lib.kconfig) dotconfig.append(lib.kconfig_enabled_flag) # Add any additional confguration options, and overriding existing # configuraton options. for new_opt in options: o = new_opt.split('=') for exist_opt in dotconfig: e = exist_opt.split('=') if o[0] == e[0]: dotconfig.remove(exist_opt) break dotconfig.append(new_opt) # Create a temporary file with the kconfig written to it fd, path = tempfile.mkstemp() with os.fdopen(fd, 'w+') as tmp: logger.debug('Using the following defconfig:') for line in dotconfig: logger.debug(' > ' + line) tmp.write(line + '\n') return_code = 0 try: return_code = self.make([ ('UK_DEFCONFIG=%s' % path), 'defconfig' ]) finally: os.remove(path) return return_code
def _handler(self, request, response): init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' response.update_status('Start process', 0) try: LOGGER.info('reading the arguments') resources = archiveextract( resource=rename_complexinputs(request.inputs['resource'])) taxon_name = request.inputs['taxon_name'][0].data bbox = [-180, -90, 180, 90] # bbox_obj = self.BBox.getValue() # bbox = [bbox_obj.coords[0][0], # bbox_obj.coords[0][1], # bbox_obj.coords[1][0], # bbox_obj.coords[1][1]] period = request.inputs['period'] period = period[0].data indices = [inpt.data for inpt in request.inputs['indices']] archive_format = request.inputs['archive_format'][0].data LOGGER.exception("indices = {} for {}".format(indices, taxon_name)) LOGGER.info("bbox={}".format(bbox)) except Exception as ex: msg = 'failed to read in the arguments: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) LOGGER.info('indices {}'.format(indices)) try: response.update_status('Fetching GBIF Data', 10) gbifdic = sdm.get_gbif(taxon_name, bbox=bbox) LOGGER.info('Fetched GBIF data') except Exception as ex: msg = 'failed to search gbif: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: response.update_status('write csv file', 70) gbifcsv = sdm.gbifdic2csv(gbifdic) LOGGER.info('GBIF data written to file') except Exception as ex: msg = 'failed to write csv file: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: response.update_status('plot map', 80) latlon = sdm.latlon_gbifdic(gbifdic) occurence_map = map_gbifoccurrences(latlon) except Exception as ex: msg = 'failed to plot occurence map: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) ################################# # calculate the climate indices ################################# # get the indices ncs_indices = None try: response.update_status('start calculation of climate indices for {}'.format(indices), 30) ncs_indices = sdm.get_indices(resource=resources, indices=indices) LOGGER.info('indice calculation done') except Exception as ex: msg = 'failed to calculate indices: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: # sort indices indices_dic = sdm.sort_indices(ncs_indices) LOGGER.info('indice files sorted in dictionary') except Exception as ex: msg = 'failed to sort indices: {}'.format(str(ex)) LOGGER.exception(msg) indices_dic = {'dummy': []} ncs_references = [] species_files = [] stat_infos = [] PAmask_pngs = [] response.update_status('Start processing for {} Datasets'.format(len(indices_dic.keys()))) for count, key in enumerate(indices_dic.keys()): try: status_nr = 40 + count * 10 response.update_status('Start processing of {}'.format(key), status_nr) ncs = indices_dic[key] LOGGER.info('with {} files'.format(len(ncs))) try: response.update_status('generating the PA mask', 20) PAmask = sdm.get_PAmask(coordinates=latlon, nc=ncs[0]) LOGGER.info('PA mask successfully generated') except Exception as ex: msg = 'failed to generate the PA mask: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: response.update_status('Plotting PA mask', 25) PAmask_pngs.extend([map_PAmask(PAmask)]) except Exception as ex: msg = 'failed to plot the PA mask: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: ncs_reference = sdm.get_reference(ncs_indices=ncs, period=period) ncs_references.extend(ncs_reference) LOGGER.info('reference indice calculated %s ' % ncs_references) except Exception as ex: msg = 'failed to calculate the reference: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: gam_model, predict_gam, gam_info = sdm.get_gam(ncs_reference, PAmask) stat_infos.append(gam_info) response.update_status('GAM sucessfully trained', status_nr + 5) except Exception as ex: msg = 'failed to train GAM for {}: {}'.format(key, str(ex)) LOGGER.exception(msg) raise Exception(msg) try: prediction = sdm.get_prediction(gam_model, ncs) response.update_status('prediction done', status_nr + 7) except Exception as ex: msg = 'failed to predict tree occurrence: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) # try: # response.update_status('land sea mask for predicted data', status_nr + 8) # from numpy import invert, isnan, nan, broadcast_arrays # , array, zeros, linspace, meshgrid # mask = invert(isnan(PAmask)) # mask = broadcast_arrays(prediction, mask)[1] # prediction[mask is False] = nan # except: # LOGGER.exception('failed to mask predicted data') try: species_files.append(sdm.write_to_file(ncs[0], prediction)) LOGGER.info('Favourability written to file') except Exception as ex: msg = 'failed to write species file: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) except Exception as ex: msg = 'failed to calculate reference indices: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: archive_indices = archive(ncs_indices, format=archive_format) LOGGER.info('indices added to archive') except Exception as ex: msg = 'failed adding indices to archive: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) archive_indices = tempfile.mkstemp(suffix='.tar', prefix='foobar-', dir='.') try: archive_references = archive(ncs_references, format=archive_format) LOGGER.info('indices reference added to archive') except Exception as ex: msg = 'failed adding reference indices to archive: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) archive_references = tempfile.mkstemp(suffix='.tar', prefix='foobar-', dir='.') try: archive_prediction = archive(species_files, format=archive_format) LOGGER.info('species_files added to archive') except Exception as ex: msg = 'failed adding species_files indices to archive: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) try: stat_infosconcat = pdfmerge(stat_infos) LOGGER.debug('pngs {}'.format(PAmask_pngs)) PAmask_png = concat_images(PAmask_pngs, orientation='h') LOGGER.info('stat info pdfs and mask pngs merged') except Exception as ex: msg = 'failed to concat images: {}'.format(str(ex)) LOGGER.exception(msg) raise Exception(msg) _, stat_infosconcat = tempfile.mkstemp(suffix='.pdf', prefix='foobar-', dir='.') _, PAmask_png = tempfile.mkstemp(suffix='.png', prefix='foobar-', dir='.') response.outputs['output_gbif'].file = occurence_map response.outputs['output_PA'].file = PAmask_png response.outputs['output_indices'].file = archive_indices response.outputs['output_reference'].file = archive_references response.outputs['output_prediction'].file = archive_prediction response.outputs['output_info'].file = stat_infosconcat response.outputs['output_csv'].file = gbifcsv response.update_status('done', 100) return response
def setUp(self): _, self.mfst_file_name = tempfile.mkstemp()
def temp_file(extension='.tmp'): (fd, tmp_file) = tempfile.mkstemp(extension) return tmp_file
def run_code(code_to_run, path, module=None, cls=None, shows_plot=False, imports_not_required=False): """ Run the given code chunk and collect the output. """ skipped = False failed = False if cls is None: use_mpi = False else: try: import mpi4py except ImportError: use_mpi = False else: N_PROCS = getattr(cls, 'N_PROCS', 1) use_mpi = N_PROCS > 1 try: # use subprocess to run code to avoid any nasty interactions between codes # Move to the test directory in case there are files to read. save_dir = os.getcwd() if module is None: code_dir = os.path.dirname(os.path.abspath(path)) else: code_dir = os.path.dirname(os.path.abspath(module.__file__)) os.chdir(code_dir) if use_mpi: env = os.environ.copy() # output will be written to one file per process env['USE_PROC_FILES'] = '1' env['OPENMDAO_CURRENT_MODULE'] = module.__name__ env['OPENMDAO_CODE_TO_RUN'] = code_to_run p = subprocess.Popen(['mpirun', '-n', str(N_PROCS), sys.executable, _sub_runner], env=env) p.wait() # extract output blocks from all output files & merge them output = [] for i in range(N_PROCS): with open('%d.out' % i) as f: output.append(f.read()) os.remove('%d.out' % i) elif shows_plot: if module is None: # write code to a file so we can run it. fd, code_to_run_path = tempfile.mkstemp() with os.fdopen(fd, 'w') as tmp: tmp.write(code_to_run) try: p = subprocess.Popen([sys.executable, code_to_run_path], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=os.environ) output, _ = p.communicate() if p.returncode != 0: failed = True finally: os.remove(code_to_run_path) else: env = os.environ.copy() env['OPENMDAO_CURRENT_MODULE'] = module.__name__ env['OPENMDAO_CODE_TO_RUN'] = code_to_run p = subprocess.Popen([sys.executable, _sub_runner], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env) output, _ = p.communicate() if p.returncode != 0: failed = True output = output.decode('utf-8', 'ignore') else: # just exec() the code for serial tests. # capture all output stdout = sys.stdout stderr = sys.stderr strout = StringIO() sys.stdout = strout sys.stderr = strout # We need more precision from numpy with printoptions(precision=8): if module is None: globals_dict = { '__file__': path, '__name__': '__main__', '__package__': None, '__cached__': None, } else: if imports_not_required: # code does not need to include all imports # Get from module globals_dict = module.__dict__ else: globals_dict = {} try: exec(code_to_run, globals_dict) except Exception as err: # for actual errors, print code (with line numbers) to facilitate debugging if not isinstance(err, unittest.SkipTest): for n, line in enumerate(code_to_run.split('\n')): print('%4d: %s' % (n, line), file=stderr) raise finally: sys.stdout = stdout sys.stderr = stderr output = strout.getvalue() except subprocess.CalledProcessError as e: output = e.output.decode('utf-8', 'ignore') # Get a traceback. if 'raise unittest.SkipTest' in output: reason_for_skip = output.splitlines()[-1][len('unittest.case.SkipTest: '):] output = reason_for_skip skipped = True else: output = "Running of embedded code {} in docs failed due to: \n\n{}".format(path, output) failed = True except unittest.SkipTest as skip: output = str(skip) skipped = True except Exception as exc: output = "Running of embedded code {} in docs failed due to: \n\n{}".format(path, traceback.format_exc()) failed = True finally: os.chdir(save_dir) return skipped, failed, output
def setUp(self): self.uvc = UpdateVerifyConfig() fd, self.tmpfilename = mkstemp() self.tmpfile = os.fdopen(fd, "wb")
def main(): parser = argparse.ArgumentParser( description='Run clang-tidy against changed files, and ' 'output diagnostics only for modified ' 'lines.') parser.add_argument('-clang-tidy-binary', metavar='PATH', default='clang-tidy', help='path to clang-tidy binary') parser.add_argument('-p', metavar='NUM', default=0, help='strip the smallest prefix containing P slashes') parser.add_argument('-regex', metavar='PATTERN', default=None, help='custom pattern selecting file paths to check ' '(case sensitive, overrides -iregex)') parser.add_argument('-iregex', metavar='PATTERN', default=r'.*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)', help='custom pattern selecting file paths to check ' '(case insensitive, overridden by -regex)') parser.add_argument('-j', type=int, default=1, help='number of tidy instances to be run in parallel.') parser.add_argument('-timeout', type=int, default=None, help='timeout per each file in seconds.') parser.add_argument('-fix', action='store_true', default=False, help='apply suggested fixes') parser.add_argument( '-checks', help='checks filter, when not specified, use clang-tidy ' 'default', default='') parser.add_argument('-path', dest='build_path', help='Path used to read a compile command database.') if yaml: parser.add_argument( '-export-fixes', metavar='FILE', dest='export_fixes', help='Create a yaml file to store suggested fixes in, ' 'which can be applied with clang-apply-replacements.') parser.add_argument('-extra-arg', dest='extra_arg', action='append', default=[], help='Additional argument to append to the compiler ' 'command line.') parser.add_argument('-extra-arg-before', dest='extra_arg_before', action='append', default=[], help='Additional argument to prepend to the compiler ' 'command line.') parser.add_argument('-quiet', action='store_true', default=False, help='Run clang-tidy in quiet mode') clang_tidy_args = [] argv = sys.argv[1:] if '--' in argv: clang_tidy_args.extend(argv[argv.index('--'):]) argv = argv[:argv.index('--')] args = parser.parse_args(argv) # Extract changed lines for each file. filename = None lines_by_file = {} for line in sys.stdin: match = re.search('^\+\+\+\ \"?(.*?/){%s}([^ \t\n\"]*)' % args.p, line) if match: filename = match.group(2) if filename is None: continue if args.regex is not None: if not re.match('^%s$' % args.regex, filename): continue else: if not re.match('^%s$' % args.iregex, filename, re.IGNORECASE): continue match = re.search('^@@.*\+(\d+)(,(\d+))?', line) if match: start_line = int(match.group(1)) line_count = 1 if match.group(3): line_count = int(match.group(3)) if line_count == 0: continue end_line = start_line + line_count - 1 lines_by_file.setdefault(filename, []).append([start_line, end_line]) if not any(lines_by_file): print("No relevant changes found.") sys.exit(0) max_task_count = args.j if max_task_count == 0: max_task_count = multiprocessing.cpu_count() max_task_count = min(len(lines_by_file), max_task_count) tmpdir = None if yaml and args.export_fixes: tmpdir = tempfile.mkdtemp() # Tasks for clang-tidy. task_queue = queue.Queue(max_task_count) # A lock for console output. lock = threading.Lock() # Run a pool of clang-tidy workers. start_workers(max_task_count, run_tidy, task_queue, lock, args.timeout) # Form the common args list. common_clang_tidy_args = [] if args.fix: common_clang_tidy_args.append('-fix') if args.checks != '': common_clang_tidy_args.append('-checks=' + args.checks) if args.quiet: common_clang_tidy_args.append('-quiet') if args.build_path is not None: common_clang_tidy_args.append('-p=%s' % args.build_path) for arg in args.extra_arg: common_clang_tidy_args.append('-extra-arg=%s' % arg) for arg in args.extra_arg_before: common_clang_tidy_args.append('-extra-arg-before=%s' % arg) for name in lines_by_file: line_filter_json = json.dumps([{ "name": name, "lines": lines_by_file[name] }], separators=(',', ':')) # Run clang-tidy on files containing changes. command = [args.clang_tidy_binary] command.append('-line-filter=' + line_filter_json) if yaml and args.export_fixes: # Get a temporary file. We immediately close the handle so clang-tidy can # overwrite it. (handle, tmp_name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir) os.close(handle) command.append('-export-fixes=' + tmp_name) command.extend(common_clang_tidy_args) command.append(name) command.extend(clang_tidy_args) task_queue.put(command) # Wait for all threads to be done. task_queue.join() if yaml and args.export_fixes: print('Writing fixes to ' + args.export_fixes + ' ...') try: merge_replacement_files(tmpdir, args.export_fixes) except: sys.stderr.write('Error exporting fixes.\n') traceback.print_exc() if tmpdir: shutil.rmtree(tmpdir)