def test(): d = {} for i in range( 10000 ): d[ 'foo' + str( i ) ] = 'bar' + str( i ) # Open temporary file and get name file = NamedTemporaryFile() file_name = file.name # Write cdb to file FileCDBDict.to_file( d, file ) file.flush() # Open on disk file2 = open( file_name ) cdb = FileCDBDict( file2 ) for key, value in d.iteritems(): assert cdb[key] == value try: cdb['notin'] assert False, "KeyError was not raised" except KeyError, e: pass
def _get_tmp_file(self): ''' Creates a tmp file with the file data ''' data = self.request.files['xml_file'][0]['body'] tmp_file = NamedTemporaryFile(delete=False) tmp_file.write(data) tmp_file.close() return tmp_file.name
def main(): logging.basicConfig(format='%(message)s', level=logging.INFO) args = parse_args() from_date = args.from_date to_date = args.to_date s3_client = S3Client.connect_with_profile(args.aws_profile) s3_bucket = args.s3_bucket s3_key = "{}_{}.logs.json".format(from_date.isoformat(), to_date.isoformat()) sumo_access_id = args.sumo_access_id or os.environ["SUMOLOGIC_ACCESS_ID"] sumo_access_key = args.sumo_access_key or os.environ["SUMOLOGIC_ACCESS_KEY"] sumo_logic_client = SumoLogicClient(sumo_access_id, sumo_access_key) logging.info( "Searching for request logs in {} in range {} to {}".format( args.environment, from_date.isoformat(), to_date.isoformat())) request_logs = sumo_logic_client.search( sumo_logic_query(args.environment), start_dt=from_date, end_dt=to_date, fields=REQUEST_FIELDS) tmp_file = NamedTemporaryFile(mode='w', encoding='utf-8', delete=False) logging.debug("Writing out logs to temporary file {}".format(tmp_file.name)) try: for request in request_logs: tmp_file.write(json.dumps(request, default=json_serialize) + '\n') s3_client.upload_from_file(s3_bucket, s3_key, tmp_file.name) finally: logging.debug("Removing temporary file {}".format(tmp_file.name)) os.remove(tmp_file.name)
class AtomicFileWriter(object): def __init__(self, path, tmp_prefix=None, encoding='utf-8'): self.name = path output_dir, base = os.path.split(path) if tmp_prefix is None: tmp_prefix = base + '.' self.tmpf = NamedTemporaryFile(dir=output_dir, prefix=tmp_prefix, mode='w', encoding=encoding, delete=False) def __enter__(self): self.tmpf.__enter__() return self def __exit__(self, exc_type, exc_value, exc_traceback): tmp_name = self.tmpf.name result = self.tmpf.__exit__(exc_type, exc_value, exc_traceback) if result or exc_type is None: os.rename(tmp_name, self.name) else: os.unlink(tmp_name) return result def write(self, data): return self.tmpf.write(data)
def run(self): active_view = self.window.active_view() text = "\n\n".join(getSelectedText(active_view)).strip() tf = NamedTemporaryFile(mode="w", delete=False) try: tf.write(text) tf.close() res = subprocess.check_output(["m4", tf.name], stderr=subprocess.STDOUT, cwd=os.path.dirname(os.path.abspath(active_view.file_name()))) res = res.decode('utf-8').replace('\r', '').strip() panel_name = "m4expand.results" panel = self.window.create_output_panel(panel_name) self.window.run_command("show_panel", {"panel": "output." + panel_name}) panel.set_read_only(False) panel.set_syntax_file(active_view.settings().get("syntax")) panel.run_command("append", {"characters": res}) panel.set_read_only(True) except Exception as e: print("M4Expand - An error occurred: ", e) finally: os.unlink(tf.name)
def ipconnections(target, **kwargs): """Returns a list of ip connections made by the target. A connection is a named tuple with the following properties: host (string), host_port (int), remote_port (string), protocol (string), timestamp(int). """ if not target: raise Exception("Invalid target for ipconnections()") output_file = NamedTemporaryFile() cmd = ["sudo", "/usr/sbin/dtrace", "-C"] if "timeout" in kwargs: cmd += ["-DANALYSIS_TIMEOUT=%d" % kwargs["timeout"]] cmd += ["-s", path_for_script("ipconnections.d")] cmd += ["-o", output_file.name] if "args" in kwargs: line = "%s %s" % (sanitize_path(target), " ".join(kwargs["args"])) cmd += ["-c", line] else: cmd += ["-c", sanitize_path(target)] # The dtrace script will take care of timeout itself, so we just launch # it asynchronously with open(os.devnull, "w") as f: handler = Popen(cmd, stdout=f, stderr=f) for entry in filelines(output_file): if "## ipconnections.d done ##" in entry.strip(): break yield _parse_single_entry(entry.strip()) output_file.close()
def apicalls(target, **kwargs): """ """ if not target: raise Exception("Invalid target for apicalls()") output_file = NamedTemporaryFile() kwargs.update({"output_file" : output_file}) cmd = _dtrace_command_line(target, **kwargs) # Generate dtrace probes for analysis definitions = os.path.abspath(os.path.join(__file__, "../../core/data/signatures.yml")) probes_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "probes.d") generate_probes(definitions, probes_file, overwrite=True) # The dtrace script will take care of timeout itself, so we just launch # it asynchronously with open(os.devnull, "w") as null: _ = Popen(cmd, stdout=null, stderr=null, cwd=current_directory()) with open('/Users/cloudmark/yield.txt', 'w+') as f: for entry in filelines(output_file): value = entry.strip() if "## apicalls.d done ##" in value: break if len(value) == 0: continue f.write(str(_parse_entry(value))) f.flush() import time time.sleep(1) yield _parse_entry(value) output_file.close() os.remove(probes_file)
def test_safe_md5(self): """Make sure we have the expected md5 with varied input types This method is ported from PyCogent (http://www.pycogent.org). PyCogent is a GPL project, but we obtained permission from the authors of this method to port it to the BIOM Format project (and keep it under BIOM's BSD license). """ exp = 'd3b07384d113edec49eaa6238ad5ff00' tmp_f = NamedTemporaryFile( mode='w', prefix='test_safe_md5', suffix='txt') tmp_f.write('foo\n') tmp_f.flush() obs = safe_md5(open(tmp_f.name, 'U')) self.assertEqual(obs, exp) obs = safe_md5(['foo\n']) self.assertEqual(obs, exp) # unsupported type raises TypeError self.assertRaises(TypeError, safe_md5, 42)
def testLogfile(self): """Test logging into a logfile""" f = NamedTemporaryFile(delete=False) filename = f.name try: set_log_level("error") # avoid using the console logger f.write(":-P\n") f.close() start_logfile(f.name, "devinfo") log = getLogger("prosoda.test.integration.test_logger") log.debug("Should not be in logfile! :-( ") log.info("Should be in logfile :-) ") log.devinfo("Should be in logfile :-) ") log.warning("Should really be in logfile :-D ") stop_logfile(f.name) contents = file(f.name).read() self.assertNotIn(":-(", contents) self.assertNotIn(":-P", contents) self.assertIn(":-)", contents) self.assertIn(":-D", contents) # Make sure no colour codes are leaked into the logfile self.assertNotIn("\033", contents) finally: set_log_level("debug") unlink(filename)
def odt_subreport(name=None, obj=None): if not aeroo_ooo: return _("Error! Subreports not available!") report_xml_ids = ir_obj.search(cr, uid, [('report_name', '=', name)], context=context) if report_xml_ids: service = netsvc.Service._services['report.%s' % name] report_xml = ir_obj.browse(cr, uid, report_xml_ids[0], context=context) data = {'model': obj._table_name, 'id': obj.id, 'report_type': 'aeroo', 'in_format': 'oo-odt'} ### Get new printing object ### sub_aeroo_print = AerooPrint() service.active_prints[sub_aeroo_print.id] = sub_aeroo_print context['print_id'] = sub_aeroo_print.id ############################### sub_aeroo_print.start_time = time.time() report, output = service.create_aeroo_report(cr, uid, \ [obj.id], data, report_xml, context=context, output='odt') # change for OpenERP 6.0 - Service class usage ### Delete printing object ### AerooPrint.print_ids.remove(sub_aeroo_print.id) del service.active_prints[sub_aeroo_print.id] ############################## temp_file = NamedTemporaryFile(suffix='.odt', prefix='aeroo-report-', delete=False) try: temp_file.write(report) finally: temp_file.close() # self.oo_subreports[print_id].append(temp_file.name) # aeroo_print.subreports.append(temp_file.name) self.active_prints[aeroo_print.id].subreports.append(temp_file.name) return "<insert_doc('%s')>" % temp_file.name return None
def tmpfile(stream, mode=None): """Context manager that writes a :class:`Stream` object to a named temporary file and yield it's filename. Cleanup deletes from the temporary file from disk. Args: stream (Stream): Stream object to write to disk as temporary file. mode (int, optional): File mode to set on temporary file. Returns: str: Temporoary file name """ tmp = NamedTemporaryFile(delete=False) if mode is not None: oldmask = os.umask(0) try: os.chmod(tmp.name, mode) finally: os.umask(oldmask) for data in stream: tmp.write(to_bytes(data)) tmp.close() yield tmp.name os.remove(tmp.name)
def test_cmyk(): ref = imread(os.path.join(data_dir, 'color.png')) img = Image.open(os.path.join(data_dir, 'color.png')) img = img.convert('CMYK') f = NamedTemporaryFile(suffix='.jpg') fname = f.name f.close() img.save(fname) try: img.close() except AttributeError: # `close` not available on PIL pass new = imread(fname) ref_lab = rgb2lab(ref) new_lab = rgb2lab(new) for i in range(3): newi = np.ascontiguousarray(new_lab[:, :, i]) refi = np.ascontiguousarray(ref_lab[:, :, i]) sim = ssim(refi, newi, dynamic_range=refi.max() - refi.min()) assert sim > 0.99
def run_solver(self, conflicts, election, deletion_handler, outfile=None): if not conflicts: return [], 0 self.deletion_handler = deletion_handler instance = self.generate_instance(conflicts, election) f = NamedTemporaryFile(delete=False) f.write(instance.encode(code)) f.close() process = Popen([self.cmd, f.name], stdout=PIPE) out, err = process.communicate() conflict_variables, optimum = self.parse_instance(out) if outfile: candidates = election[0] votes = election[1] votecounts = election[2] votemap = self.delete_votes(votes, votecounts, conflict_variables) votesum = sum(votemap.values()) write_map(candidates, votesum, votemap, open(outfile, "w")) remove(f.name) return conflict_variables, optimum
def _write_local_schema_file(self, cursor): """ Takes a cursor, and writes the BigQuery schema for the results to a local file system. :return: A dictionary where key is a filename to be used as an object name in GCS, and values are file handles to local files that contains the BigQuery schema fields in .json format. """ schema = [] for field in cursor.description: # See PEP 249 for details about the description tuple. field_name = field[0] field_type = self.type_map(field[1]) field_mode = 'REPEATED' if field[1] in (1009, 1005, 1007, 1016) else 'NULLABLE' schema.append({ 'name': field_name, 'type': field_type, 'mode': field_mode, }) self.log.info('Using schema for %s: %s', self.schema_filename, schema) tmp_schema_file_handle = NamedTemporaryFile(delete=True) s = json.dumps(schema, sort_keys=True) if PY3: s = s.encode('utf-8') tmp_schema_file_handle.write(s) return {self.schema_filename: tmp_schema_file_handle}
def __enter__(self): # Ensure that we have not re-entered if self.temp_path != None or self.service != None: raise Exception('Cannot use multiple nested with blocks on same Youtube object!') flow = flow_from_clientsecrets( self.client_secrets_path, scope=YOUTUBE_UPLOAD_SCOPE, message=MISSING_CLIENT_SECRETS_MESSAGE) temp_file = NamedTemporaryFile(delete=False) self.temp_path = temp_file.name temp_file.close() storage = Storage(self.temp_path) credentials = storage.get() if credentials is None or credentials.invalid: credentials = run_flow( flow, storage, argparser.parse_args(list()) ) self.service = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION, http=credentials.authorize(httplib2.Http())) return self
def _create_empty_image(self, image_width, image_height): # Check pycairo capabilities if not (cairo.HAS_IMAGE_SURFACE and cairo.HAS_PNG_FUNCTIONS): raise HTTPBadRequest("cairo was not compiled with ImageSurface and PNG support") # Create a new cairo surface surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, int(image_width), int(image_height)) ctx = cairo.Context(surface) text = "No imagery available for requested coordinates." x_bearing, y_bearing, width, height, x_advance, y_advance = ctx.text_extents(text) ctx.move_to((image_width / 2) - (width / 2), (image_height / 2) + (height / 2)) ctx.set_source_rgba(0, 0, 0, 0.85) ctx.show_text(text) temp_datadir = self.config.get("main", "temp.datadir") temp_url = self.config.get("main", "temp.url") file = NamedTemporaryFile(suffix=".png", dir=temp_datadir, delete=False) surface.write_to_png(file) file.close() return {"file": "%s/%s" % (temp_url, file.name.split("/")[-1])}
def run(self, uid, aid, publish=True): aid = int(aid) audiobook = Audiobook.objects.get(id=aid) self.set_status(aid, status.ENCODING) user = User.objects.get(id=uid) try: os.makedirs(BUILD_PATH) except OSError as e: if e.errno == errno.EEXIST: pass else: raise out_file = NamedTemporaryFile(delete=False, prefix='%d-' % aid, suffix='.%s' % self.ext, dir=BUILD_PATH) out_file.close() self.encode(audiobook.source_file.path, out_file.name) self.set_status(aid, status.TAGGING) self.set_tags(audiobook, out_file.name) self.set_status(aid, status.SENDING) if publish: self.put(user, audiobook, out_file.name) self.published(aid) else: self.set_status(aid, None) self.save(audiobook, out_file.name)
def _generate_training_files(self): """Returns a tuple of file objects suitable for passing to the RdpTrainer application controller. """ tmp_dir = get_qiime_temp_dir() training_set = RdpTrainingSet() reference_seqs_file = open(self.Params['reference_sequences_fp'], 'U') id_to_taxonomy_file = open(self.Params['id_to_taxonomy_fp'], 'U') for seq_id, seq in MinimalFastaParser(reference_seqs_file): training_set.add_sequence(seq_id, seq) for line in id_to_taxonomy_file: seq_id, lineage_str = map(strip, line.split('\t')) training_set.add_lineage(seq_id, lineage_str) training_set.dereplicate_taxa() rdp_taxonomy_file = NamedTemporaryFile( prefix='RdpTaxonAssigner_taxonomy_', suffix='.txt', dir=tmp_dir) rdp_taxonomy_file.write(training_set.get_rdp_taxonomy()) rdp_taxonomy_file.seek(0) rdp_training_seqs_file = NamedTemporaryFile( prefix='RdpTaxonAssigner_training_seqs_', suffix='.fasta', dir=tmp_dir) for rdp_id, seq in training_set.get_training_seqs(): rdp_training_seqs_file.write('>%s\n%s\n' % (rdp_id, seq)) rdp_training_seqs_file.seek(0) self._training_set = training_set return rdp_taxonomy_file, rdp_training_seqs_file
def parallelize(self, c, numSlices=None): """ Distribute a local Python collection to form an RDD. >>> sc.parallelize(range(5), 5).glom().collect() [[0], [1], [2], [3], [4]] """ numSlices = numSlices or self.defaultParallelism # Calling the Java parallelize() method with an ArrayList is too slow, # because it sends O(n) Py4J commands. As an alternative, serialized # objects are written to a file and loaded through textFile(). tempFile = NamedTemporaryFile(delete=False, dir=self._temp_dir) # Make sure we distribute data evenly if it's smaller than self.batchSize if "__len__" not in dir(c): c = list(c) # Make it a list so we can compute its length batchSize = min(len(c) // numSlices, self._batchSize) if batchSize > 1: serializer = BatchedSerializer(self._unbatched_serializer, batchSize) else: serializer = self._unbatched_serializer serializer.dump_stream(c, tempFile) tempFile.close() readRDDFromFile = self._jvm.PythonRDD.readRDDFromFile jrdd = readRDDFromFile(self._jsc, tempFile.name, numSlices) return RDD(jrdd, self, serializer)
def test_seq_pipeline_parallel_run_with_fasta_qual(self): 'The pipeline runs in parallel with fasta and qual' pipeline = 'sanger_with_qual' fhand_adaptors = NamedTemporaryFile() fhand_adaptors.write(ADAPTORS) fhand_adaptors.flush() arabidopsis_genes = 'arabidopsis_genes+' univec = os.path.join(TEST_DATA_DIR, 'blast', arabidopsis_genes) configuration = {'remove_vectors': {'vectors': univec}, 'remove_adaptors': {'adaptors': fhand_adaptors.name}} seq1 = create_random_seqwithquality(500, qual_range=50) seq2 = create_random_seqwithquality(500, qual_range=51) seq3 = create_random_seqwithquality(500, qual_range=52) seqs = [seq1, seq2, seq3] inseq_fhand, inqual_fhand = create_temp_seq_file(seqs, format='qual') in_fhands = {} in_fhands['in_seq'] = open(inseq_fhand.name) in_fhands['in_qual'] = open(inqual_fhand.name) outseq_fhand = NamedTemporaryFile() outqual_fhand = NamedTemporaryFile() writer = SequenceWriter(outseq_fhand, qual_fhand=outqual_fhand, file_format='fasta') writers = {'seq': writer} seq_pipeline_runner(pipeline, configuration, in_fhands, processes=4, writers=writers) out_fhand = open(outseq_fhand.name, 'r') result_seq = out_fhand.read() assert result_seq.count('>') == 3
def test_pipeline_run(): 'It tests that the pipeline runs ok' pipeline = 'sanger_with_qual' fhand_adaptors = NamedTemporaryFile() fhand_adaptors.write(ADAPTORS) fhand_adaptors.flush() arabidopsis_genes = 'arabidopsis_genes+' univec = os.path.join(TEST_DATA_DIR, 'blast', arabidopsis_genes) configuration = {'remove_vectors_blastdb': {'vectors': univec}, 'remove_adaptors': {'adaptors': fhand_adaptors.name}} seq_fhand = open(os.path.join(TEST_DATA_DIR, 'seq.fasta'), 'r') qual_fhand = open(os.path.join(TEST_DATA_DIR, 'qual.fasta'), 'r') seq_iter = seqs_in_file(seq_fhand, qual_fhand) filtered_seq_iter = _pipeline_builder(pipeline, seq_iter, configuration) seq_list = list(filtered_seq_iter) assert 'CGAtcgggggg' in str(seq_list[0].seq) assert len(seq_list) == 6
def reg_code(): img, code =generate_code_image((80,30),5) session["code"] = code tp = NamedTemporaryFile() img.save(tp.name,format="png") tp.seek(0) return send_file(tp.name,mimetype='image/png')
def test_toy_corpus(): keats = ('She dwells with Beauty - Beauty that must die;\n\n' 'And Joy, whose hand is ever at his lips\n\n' 'Bidding adieu; and aching Pleasure nigh,\n\n' 'Turning to poison while the bee-mouth sips:\n\n' 'Ay, in the very temple of Delight\n\n' 'Veil\'d Melancholy has her sovran shrine,\n\n' 'Though seen of none save him whose strenuous tongue\n\n' 'Can burst Joy\'s grape against his palate fine;\n\n' 'His soul shall taste the sadness of her might,\n\n' 'And be among her cloudy trophies hung.') assert toy_corpus(keats) assert toy_corpus(keats, nltk_stop=True) assert toy_corpus(keats, stop_freq=1) assert toy_corpus(keats, add_stop=['and', 'with']) assert toy_corpus(keats, nltk_stop=True, stop_freq=1, add_stop=['ay']) import os from tempfile import NamedTemporaryFile as NFT tmp = NFT(delete=False) tmp.write(keats) tmp.close() c = toy_corpus(tmp.name, is_filename=True, nltk_stop=True, add_stop=['ay']) assert c os.remove(tmp.name) return c
def execute(self, context): hive = HiveServer2Hook(hiveserver2_conn_id=self.hiveserver2_conn_id) logging.info("Extracting data from Hive") logging.info(self.sql) if self.bulk_load: tmpfile = NamedTemporaryFile() hive.to_csv(self.sql, tmpfile.name, delimiter='\t', lineterminator='\n', output_header=False) else: results = hive.get_records(self.sql) mysql = MySqlHook(mysql_conn_id=self.mysql_conn_id) if self.mysql_preoperator: logging.info("Running MySQL preoperator") mysql.run(self.mysql_preoperator) logging.info("Inserting rows into MySQL") if self.bulk_load: mysql.bulk_load(table=self.mysql_table, tmp_file=tmpfile.name) tmpfile.close() else: mysql.insert_rows(table=self.mysql_table, rows=results) if self.mysql_postoperator: logging.info("Running MySQL postoperator") mysql.run(self.mysql_postoperator) logging.info("Done.")
def write_temp_file(data): # create a temp file for use as a config file. This should get cleaned # up magically at the end of the run. fid = NamedTemporaryFile(mode='w+b', suffix='.tmp') fid.write(data) fid.seek(0) return fid
def roundtrip(self, dtype, x, suffix): f = NamedTemporaryFile(suffix='.' + suffix) fname = f.name f.close() sio.imsave(fname, x) y = sio.imread(fname) assert_array_equal(y, x)
def _build_and_catch_errors(self, build_func, options_bytes, source=None): try: return build_func() except _cl.RuntimeError as e: msg = e.what if options_bytes: msg = msg + "\n(options: %s)" % options_bytes.decode("utf-8") if source is not None: from tempfile import NamedTemporaryFile srcfile = NamedTemporaryFile(mode="wt", delete=False, suffix=".cl") try: srcfile.write(source) finally: srcfile.close() msg = msg + "\n(source saved as %s)" % srcfile.name code = e.code routine = e.routine err = _cl.RuntimeError( _cl.Error._ErrorRecord( msg=msg, code=code, routine=routine)) # Python 3.2 outputs the whole list of currently active exceptions # This serves to remove one (redundant) level from that nesting. raise err
def command_update(args): def write_to(out): config.output(out) library = KeyLibrary(args.key_directory) with open(args.config_file) as fd: config = SedgeEngine(library, fd, not args.no_verify, url=args.config_file) if args.output_file == '-': write_to(ConfigOutput(sys.stdout)) return if not check_or_confirm_overwrite(args.output_file): print("Aborting.", file=sys.stderr) sys.exit(1) tmpf = NamedTemporaryFile(mode='w', dir=os.path.dirname(args.output_file), delete=False) try: tmpf.file.write('''\ # :sedge: # # this configuration generated from `sedge' file: # %s # # do not edit this file manually, edit the source file and re-run `sedge' # ''' % (args.config_file)) write_to(ConfigOutput(tmpf.file)) tmpf.close() if args.verbose: diff_config_changes(args.output_file, tmpf.name) os.rename(tmpf.name, args.output_file) except: os.unlink(tmpf.name) raise
def _write_local_schema_file(self, cursor): """ Takes a cursor, and writes the BigQuery schema for the results to a local file system. :return: A dictionary where key is a filename to be used as an object name in GCS, and values are file handles to local files that contains the BigQuery schema fields in .json format. """ schema = [] for field in cursor.description: # See PEP 249 for details about the description tuple. field_name = field[0] field_type = self.type_map(field[1]) # Always allow TIMESTAMP to be nullable. MySQLdb returns None types # for required fields because some MySQL timestamps can't be # represented by Python's datetime (e.g. 0000-00-00 00:00:00). field_mode = 'NULLABLE' if field[6] or field_type == 'TIMESTAMP' else 'REQUIRED' schema.append({ 'name': field_name, 'type': field_type, 'mode': field_mode, }) self.log.info('Using schema for %s: %s', self.schema_filename, schema) tmp_schema_file_handle = NamedTemporaryFile(delete=True) s = json.dumps(schema, tmp_schema_file_handle) if PY3: s = s.encode('utf-8') tmp_schema_file_handle.write(s) return {self.schema_filename: tmp_schema_file_handle}
def test_seq_pipeline_run(): 'It tests that the pipeline runs ok' pipeline = 'sanger_with_qual' fhand_adaptors = NamedTemporaryFile() fhand_adaptors.write(ADAPTORS) fhand_adaptors.flush() arabidopsis_genes = 'arabidopsis_genes+' univec = os.path.join(TEST_DATA_DIR, 'blast', arabidopsis_genes) configuration = {'remove_vectors_blastdb': {'vectors': univec}, 'remove_adaptors': {'adaptors': fhand_adaptors.name}} in_fhands = {} in_fhands['in_seq'] = open(os.path.join(TEST_DATA_DIR, 'seq.fasta'), 'r') in_fhands['in_qual'] = open(os.path.join(TEST_DATA_DIR, 'qual.fasta'), 'r') out_seq_fhand = NamedTemporaryFile() out_qual_fhand = NamedTemporaryFile() writer = SequenceWriter(out_seq_fhand, qual_fhand=out_qual_fhand, file_format='fasta') seq_pipeline_runner(pipeline, configuration, in_fhands, writers={'seq': writer}) result_seq = open(out_seq_fhand.name).read() assert result_seq.count('>') == 6 #are we keeping the description? assert 'mdust' in result_seq
def treetime(nwk, fasta, outdir, binpath='treetime', clock=None, verbosity=1): """ :param nwk: str, Newick tree string from fasttree() :param fasta: dict, header-sequence pairs :param outdir: path to write output files :param clock: float, clock rate to constrain analysis - defaults to None (no constraint) :param verbosity: verbose level, defaults to 1 :return: path to NEXUS output file """ # extract dates from sequence headers datefile = NamedTemporaryFile('w', prefix="cvz_tt_", delete=False) datefile.write('name,date\n') alnfile = NamedTemporaryFile('w', prefix="cvz_tt_", delete=False) for h, s in fasta.items(): # TreeTime seems to have trouble handling labels with spaces _, accn, coldate = h.split('|') datefile.write('{},{}\n'.format(accn, coldate)) alnfile.write('>{}\n{}\n'.format(accn, s)) datefile.close() alnfile.close() with NamedTemporaryFile('w', prefix="cvz_tt_", delete=False) as nwkfile: nwkfile.write(nwk.replace(' ', '')) call = [ binpath, '--tree', nwkfile.name, '--aln', alnfile.name, '--dates', datefile.name, '--outdir', outdir, '--verbose', str(verbosity), '--plot-rtt', 'none' ] # see issue #66 if clock: call.extend(['--clock-rate', str(clock)]) check_call(call) # clean up temporary files os.remove(datefile.name) os.remove(alnfile.name) os.remove(nwkfile.name) # return path to NEXUS file nexus_file = os.path.join(outdir, 'timetree.nexus') if not os.path.exists(nexus_file): print( "Error: missing expected NEXUS output file {}".format(nexus_file)) return None return nexus_file
def mms_load_data(trange=['2015-10-16', '2015-10-17'], probe='1', data_rate='srvy', level='l2', instrument='fgm', datatype='', varformat=None, prefix='', suffix='', get_support_data=False, time_clip=False, no_update=False, center_measurement=False, available=False, notplot=False, latest_version=False, major_version=False, min_version=None, cdf_version=None): """ This function loads MMS data into pyTplot variables """ if not isinstance(probe, list): probe = [probe] if not isinstance(data_rate, list): data_rate = [data_rate] if not isinstance(level, list): level = [level] if not isinstance(datatype, list): datatype = [datatype] probe = [str(p) for p in probe] # allows the user to pass in trange as list of datetime objects if type(trange[0]) == datetime and type(trange[1]) == datetime: trange = [ time_string(trange[0].timestamp()), time_string(trange[1].timestamp()) ] # allows the user to pass in trange as a list of floats (unix times) if isinstance(trange[0], float): trange[0] = time_string(trange[0]) if isinstance(trange[1], float): trange[1] = time_string(trange[1]) start_date = parse(trange[0]).strftime( '%Y-%m-%d') # need to request full day, then parse out later end_date = parse(time_string(time_double(trange[1]) - 0.1)).strftime( '%Y-%m-%d-%H-%M-%S' ) # -1 second to avoid getting data for the next day download_only = CONFIG['download_only'] no_download = False if no_update or CONFIG['no_download']: no_download = True user = None if not no_download: sdc_session, user = mms_login_lasp() out_files = [] available_files = [] for prb in probe: for drate in data_rate: for lvl in level: for dtype in datatype: if user is None: url = 'https://lasp.colorado.edu/mms/sdc/public/files/api/v1/file_info/science?start_date=' + start_date + '&end_date=' + end_date + '&sc_id=mms' + prb + '&instrument_id=' + instrument + '&data_rate_mode=' + drate + '&data_level=' + lvl else: url = 'https://lasp.colorado.edu/mms/sdc/sitl/files/api/v1/file_info/science?start_date=' + start_date + '&end_date=' + end_date + '&sc_id=mms' + prb + '&instrument_id=' + instrument + '&data_rate_mode=' + drate + '&data_level=' + lvl if dtype != '': url = url + '&descriptor=' + dtype if CONFIG['debug_mode']: logging.info('Fetching: ' + url) if no_download == False: # query list of available files try: with warnings.catch_warnings(): warnings.simplefilter("ignore", category=ResourceWarning) http_json = sdc_session.get( url, verify=True).json() if CONFIG['debug_mode']: logging.info( 'Filtering the results down to your trange' ) files_in_interval = mms_files_in_interval( http_json['files'], trange) if available: for file in files_in_interval: logging.info( file['file_name'] + ' (' + str( np.round(file['file_size'] / (1024. * 1024), decimals=1)) + ' MB)') available_files.append(file['file_name']) continue for file in files_in_interval: file_date = parse(file['timetag']) if dtype == '': out_dir = os.sep.join([ CONFIG['local_data_dir'], 'mms' + prb, instrument, drate, lvl, file_date.strftime('%Y'), file_date.strftime('%m') ]) else: out_dir = os.sep.join([ CONFIG['local_data_dir'], 'mms' + prb, instrument, drate, lvl, dtype, file_date.strftime('%Y'), file_date.strftime('%m') ]) if drate.lower() == 'brst': out_dir = os.sep.join( [out_dir, file_date.strftime('%d')]) out_file = os.sep.join( [out_dir, file['file_name']]) if CONFIG['debug_mode']: logging.info('File: ' + file['file_name'] + ' / ' + file['timetag']) if os.path.exists(out_file) and str( os.stat(out_file).st_size) == str( file['file_size']): if not download_only: logging.info('Loading ' + out_file) out_files.append(out_file) continue if user is None: download_url = 'https://lasp.colorado.edu/mms/sdc/public/files/api/v1/download/science?file=' + file[ 'file_name'] else: download_url = 'https://lasp.colorado.edu/mms/sdc/sitl/files/api/v1/download/science?file=' + file[ 'file_name'] logging.info('Downloading ' + file['file_name'] + ' to ' + out_dir) with warnings.catch_warnings(): warnings.simplefilter( "ignore", category=ResourceWarning) fsrc = sdc_session.get(download_url, stream=True, verify=True) ftmp = NamedTemporaryFile(delete=False) with open(ftmp.name, 'wb') as f: copyfileobj(fsrc.raw, f) if not os.path.exists(out_dir): os.makedirs(out_dir) # if the download was successful, copy to data directory copy(ftmp.name, out_file) out_files.append(out_file) fsrc.close() ftmp.close() except requests.exceptions.ConnectionError: # No/bad internet connection; try loading the files locally logging.error('No internet connection!') if out_files == []: if not download_only: logging.info('Searching for local files...') out_files = mms_get_local_files( prb, instrument, drate, lvl, dtype, trange) if out_files == [] and CONFIG[ 'mirror_data_dir'] != None: # check for network mirror; note: network mirrors are assumed to be read-only # and we always copy the files from the mirror to the local data directory # before trying to load into tplot variables logging.info( 'No local files found; checking network mirror...' ) out_files = mms_get_local_files(prb, instrument, drate, lvl, dtype, trange, mirror=True) if not no_download: sdc_session.close() if available: return available_files if not download_only: out_files = sorted(out_files) filtered_out_files = mms_file_filter(out_files, latest_version=latest_version, major_version=major_version, min_version=min_version, version=cdf_version) if filtered_out_files == []: logging.info('No matching CDF versions found.') return new_variables = cdf_to_tplot(filtered_out_files, varformat=varformat, merge=True, get_support_data=get_support_data, prefix=prefix, suffix=suffix, center_measurement=center_measurement, notplot=notplot) if notplot: return new_variables if new_variables == []: logging.warning('No data loaded.') return if time_clip: for new_var in new_variables: tclip(new_var, trange[0], trange[1], suffix='') return new_variables else: return out_files
def test_vs_scoring(): protein = next(oddt.toolkit.readfile('pdb', xiap_protein)) protein.protein = True data_dir = os.path.join(test_data_dir, 'data') home_dir = mkdtemp() pdbbind_versions = (2007, 2013, 2016) pdbbind_dir = os.path.join(data_dir, 'pdbbind') for pdbbind_v in pdbbind_versions: version_dir = os.path.join(data_dir, 'v%s' % pdbbind_v) if not os.path.isdir(version_dir): os.symlink(pdbbind_dir, version_dir) filenames = [] # train mocked SFs for model in [nnscore(n_jobs=1) ] + [rfscore(version=v, n_jobs=1) for v in [1, 2, 3]]: model.gen_training_data(data_dir, pdbbind_versions=pdbbind_versions, home_dir=home_dir) filenames.append(model.train(home_dir=home_dir)) vs = virtualscreening(n_cpu=-1, chunksize=10) vs.load_ligands('sdf', xiap_actives_docked) # error if no protein is fed with pytest.raises(ValueError): vs.score('nnscore') # bad sf name with pytest.raises(ValueError): vs.score('bad_sf', protein=protein) vs.score('nnscore', protein=xiap_protein) vs.score('nnscore_pdbbind2016', protein=protein) vs.score('rfscore_v1', protein=protein) vs.score('rfscore_v1_pdbbind2016', protein=protein) vs.score('rfscore_v2', protein=protein) vs.score('rfscore_v3', protein=protein) vs.score('pleclinear', protein=protein) vs.score('pleclinear_p5_l1_s65536_pdbbind2016', protein=protein) # use pickle directly vs.score(filenames[0], protein=protein) # pass SF object directly vs.score(scorer.load(filenames[0]), protein=protein) # pass wrong object (sum is not an instance of scorer) with pytest.raises(ValueError): vs.score(sum, protein=protein) mols = list(vs.fetch()) assert len(mols) == 100 mol_data = mols[0].data assert 'nnscore' in mol_data assert 'rfscore_v1' in mol_data assert 'rfscore_v2' in mol_data assert 'rfscore_v3' in mol_data assert 'PLEClinear_p5_l1_s65536' in mol_data vs = virtualscreening(n_cpu=-1, chunksize=10) vs.load_ligands('sdf', xiap_actives_docked) vs.score('nnscore', protein=protein) vs.score('rfscore_v1', protein=protein) vs.score('rfscore_v2', protein=protein) vs.score('rfscore_v3', protein=protein) with NamedTemporaryFile('w', suffix='.sdf') as molfile: with NamedTemporaryFile('w', suffix='.csv') as csvfile: vs.write('sdf', molfile.name, csv_filename=csvfile.name) data = pd.read_csv(csvfile.name) assert 'nnscore' in data.columns assert 'rfscore_v1' in data.columns assert 'rfscore_v2' in data.columns assert 'rfscore_v3' in data.columns mols = list(oddt.toolkit.readfile('sdf', molfile.name)) assert len(mols) == 100 vs.write_csv( csvfile.name, fields=['nnscore', 'rfscore_v1', 'rfscore_v2', 'rfscore_v3']) data = pd.read_csv(csvfile.name) assert len(data.columns) == 4 assert len(data) == len(mols) assert 'nnscore' in data.columns assert 'rfscore_v1' in data.columns assert 'rfscore_v2' in data.columns assert 'rfscore_v3' in data.columns # remove files for f in filenames: os.unlink(f) # remove symlinks for pdbbind_v in pdbbind_versions: version_dir = os.path.join(data_dir, 'v%s' % pdbbind_v) if os.path.islink(version_dir): os.unlink(version_dir)
def handle_submission(request, su_id): """ Saves the submission if it is allowed """ uuid = request.cookies.get("uuid") logging.debug("Cookie: {}".format(uuid)) request_form = request.form logging.debug(request_form) ass_code = request_form["ass_code"] module = request_form["module_code"] path = "modules/{}/{}/submissions".format(module, ass_code) expected_file = "submission_file_{}".format(ass_code) if expected_file in request.files: file = request.files[expected_file] if file and allowed_file(file.filename): if su_id not in os.listdir(path): path = "{}/{}/".format(path, su_id) os.mkdir(path) else: path = "{}/{}/".format(path, su_id) if allowed_submission(su_id, ass_code): # Remove previous submissions for sub in os.listdir(path): os.unlink("{}/{}".format(path, sub)) # Add resubmission filename = secure_filename(file.filename) file.save("{}/{}".format(path, filename)) # Log submission and date tempfile = NamedTemporaryFile(delete=False) try: locks[ass_code].acquire(timeout=10) with open( "modules/{}/{}/assignment_files/logs.csv".format( module, ass_code), "rb", ) as logs, tempfile: logs = csv.reader(logs, delimiter=",") writer = csv.writer(tempfile, delimiter=",") found = False for row in logs: if str(row[0]) == str(su_id): found = True subs = int(row[2]) row[2] = subs + 1 row[3] = datetime.datetime.now() row[5] = request.remote_addr row[6] = uuid writer.writerow(row) if not found: # Still log submissions if the user is not a registered user somehow writer.writerow([ su_id.strip(), 0, 1, datetime.datetime.now(), None, request.remote_addr, uuid, None, None, ]) shutil.move( tempfile.name, "modules/{}/{}/assignment_files/logs.csv".format( module, ass_code), ) # Finally send email to inform student of their submission send_email(su_id, ass_code) finally: locks[ass_code].release() else: return False else: return False else: return False return True
def _build_and_map_disk_partitions(self): self.disk.wipe() if self.firmware.legacy_bios_mode(): log.info('--> creating EFI CSM(legacy bios) partition') self.disk.create_efi_csm_partition( self.firmware.get_legacy_bios_partition_size() ) if self.firmware.efi_mode(): log.info('--> creating EFI partition') self.disk.create_efi_partition( self.firmware.get_efi_partition_size() ) if self.firmware.ofw_mode(): log.info('--> creating PReP partition') self.disk.create_prep_partition( self.firmware.get_prep_partition_size() ) if self.disk_setup.need_boot_partition(): log.info('--> creating boot partition') self.disk.create_boot_partition( self.disk_setup.boot_partition_size() ) if self.spare_part_mbsize: log.info('--> creating spare partition') self.disk.create_spare_partition( self.spare_part_mbsize ) if self.root_filesystem_is_overlay: log.info('--> creating readonly root partition') squashed_root_file = NamedTemporaryFile() squashed_root = FileSystemSquashFs( device_provider=None, root_dir=self.root_dir ) squashed_root.create_on_file( filename=squashed_root_file.name, exclude=[Defaults.get_shared_cache_location()] ) squashed_rootfs_mbsize = os.path.getsize( squashed_root_file.name ) / 1048576 self.disk.create_root_readonly_partition( int(squashed_rootfs_mbsize + 50) ) if self.volume_manager_name and self.volume_manager_name == 'lvm': log.info('--> creating LVM root partition') self.disk.create_root_lvm_partition('all_free') elif self.mdraid: log.info('--> creating mdraid root partition') self.disk.create_root_raid_partition('all_free') else: log.info('--> creating root partition') self.disk.create_root_partition('all_free') if self.firmware.bios_mode(): log.info('--> setting active flag to primary boot partition') self.disk.activate_boot_partition() if self.firmware.ofw_mode(): log.info('--> setting active flag to primary PReP partition') self.disk.activate_boot_partition() if self.firmware.efi_mode(): if self.force_mbr: log.info('--> converting partition table to MBR') self.disk.create_mbr() elif self.hybrid_mbr: log.info('--> converting partition table to hybrid GPT/MBR') self.disk.create_hybrid_mbr() self.disk.map_partitions() return self.disk.get_device()
def create_disk(self): """ Build a bootable raw disk image """ if self.install_media and self.build_type_name != 'oem': raise KiwiInstallMediaError( 'Install media requires oem type setup, got %s' % self.build_type_name ) if self.root_filesystem_is_overlay and self.volume_manager_name: raise KiwiVolumeManagerSetupError( 'Volume management together with root overlay is not supported' ) # setup recovery archive, cleanup and create archive if requested self.system_setup.create_recovery_archive() # prepare boot(initrd) root system log.info('Preparing boot system') self.boot_image.prepare() # precalculate needed disk size disksize_mbytes = self.disk_setup.get_disksize_mbytes() # create the disk log.info('Creating raw disk image %s', self.diskname) loop_provider = LoopDevice( self.diskname, disksize_mbytes, self.blocksize ) loop_provider.create() self.disk = Disk( self.firmware.get_partition_table_type(), loop_provider ) # create the bootloader instance self.bootloader_config = BootLoaderConfig( self.bootloader, self.xml_state, self.root_dir, { 'targetbase': loop_provider.get_device(), 'grub_directory_name': Defaults.get_grub_boot_directory_name(self.root_dir) } ) # create disk partitions and instance device map device_map = self._build_and_map_disk_partitions() # create raid on current root device if requested if self.mdraid: self.raid_root = RaidDevice(device_map['root']) self.raid_root.create_degraded_raid(raid_level=self.mdraid) device_map['root'] = self.raid_root.get_device() # create luks on current root device if requested if self.luks: self.luks_root = LuksDevice(device_map['root']) self.luks_root.create_crypto_luks( passphrase=self.luks, os=self.luks_os ) device_map['root'] = self.luks_root.get_device() # create filesystems on boot partition(s) if any self._build_boot_filesystems(device_map) # create volumes and filesystems for root system if self.volume_manager_name: volume_manager_custom_parameters = { 'fs_mount_options': self.custom_root_mount_args, 'root_label': self.disk_setup.get_root_label(), 'root_is_snapshot': self.xml_state.build_type.get_btrfs_root_is_snapshot(), 'root_is_readonly_snapshot': self.xml_state.build_type.get_btrfs_root_is_readonly_snapshot(), 'image_type': self.xml_state.get_build_type_name() } volume_manager = VolumeManager( self.volume_manager_name, device_map['root'], self.root_dir + '/', self.volumes, volume_manager_custom_parameters ) volume_manager.setup( self.volume_group_name ) volume_manager.create_volumes( self.requested_filesystem ) volume_manager.mount_volumes() self.generic_fstab_entries += volume_manager.get_fstab( self.persistency_type, self.requested_filesystem ) self.system = volume_manager device_map['root'] = volume_manager.get_device()['root'] else: log.info( 'Creating root(%s) filesystem on %s', self.requested_filesystem, device_map['root'].get_device() ) filesystem_custom_parameters = { 'mount_options': self.custom_root_mount_args } filesystem = FileSystem( self.requested_filesystem, device_map['root'], self.root_dir + '/', filesystem_custom_parameters ) filesystem.create_on_device( label=self.disk_setup.get_root_label() ) self.system = filesystem # create a random image identifier self.mbrid = SystemIdentifier() self.mbrid.calculate_id() # create first stage metadata to boot image self._write_partition_id_config_to_boot_image() self._write_recovery_metadata_to_boot_image() self._write_raid_config_to_boot_image() self._write_generic_fstab_to_boot_image(device_map) self.system_setup.export_modprobe_setup( self.boot_image.boot_root_directory ) # create first stage metadata to system image self._write_image_identifier_to_system_image() self._write_crypttab_to_system_image() self._write_generic_fstab_to_system_image(device_map) # create initrd cpio archive self.boot_image.create_initrd(self.mbrid) # create second stage metadata to system image self._copy_first_boot_files_to_system_image() self._write_bootloader_config_to_system_image(device_map) self.mbrid.write_to_disk( self.disk.storage_provider ) # set SELinux file security contexts if context exists self._setup_selinux_file_contexts() # syncing system data to disk image log.info('Syncing system to image') if self.system_efi: log.info('--> Syncing EFI boot data to EFI partition') self.system_efi.sync_data() if self.system_boot: log.info('--> Syncing boot data at extra partition') self.system_boot.sync_data( self._get_exclude_list_for_boot_data_sync() ) log.info('--> Syncing root filesystem data') if self.root_filesystem_is_overlay: squashed_root_file = NamedTemporaryFile() squashed_root = FileSystemSquashFs( device_provider=None, root_dir=self.root_dir ) squashed_root.create_on_file( filename=squashed_root_file.name, exclude=self._get_exclude_list_for_root_data_sync(device_map) ) Command.run( [ 'dd', 'if=%s' % squashed_root_file.name, 'of=%s' % device_map['readonly'].get_device() ] ) else: self.system.sync_data( self._get_exclude_list_for_root_data_sync(device_map) ) # install boot loader self._install_bootloader(device_map) # set root filesystem properties self._setup_property_root_is_readonly_snapshot() # prepare for install media if requested if self.install_media: if self.initrd_system and self.initrd_system == 'dracut': # for the installation process we need a kiwi initrd # Therefore an extra install boot root system needs to # be prepared if dracut was set as the initrd system # to boot the system image log.info('Preparing extra install boot system') self.xml_state.build_type.set_initrd_system('kiwi') self.initrd_system = \ self.xml_state.build_type.get_initrd_system() self.boot_image = BootImageKiwi( self.xml_state, self.target_dir, signing_keys=self.signing_keys ) self.boot_image.prepare() # apply disk builder metadata also needed in the install initrd self._write_partition_id_config_to_boot_image() self._write_recovery_metadata_to_boot_image() self._write_raid_config_to_boot_image() self.system_setup.export_modprobe_setup( self.boot_image.boot_root_directory ) log.info('Saving boot image instance to file') self.boot_image.dump( self.target_dir + '/boot_image.pickledump' ) # store image file name in result self.result.add( key='disk_image', filename=self.diskname, use_for_bundle=True if not self.image_format else False, compress=True, shasum=True ) # create image root metadata self.result.add( key='image_packages', filename=self.system_setup.export_rpm_package_list( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) self.result.add( key='image_verified', filename=self.system_setup.export_rpm_package_verification( self.target_dir ), use_for_bundle=True, compress=False, shasum=False ) return self.result
def write_mesos_cli_config(config): mesos_cli_config_file = NamedTemporaryFile(delete=False) mesos_cli_config_file.write(json.dumps(config)) mesos_cli_config_file.close() return mesos_cli_config_file.name
class TestPushToken(TestController): def setUp(self): self.delete_all_policies() self.delete_all_token() self.delete_all_realms() self.delete_all_resolvers() super(TestPushToken, self).setUp() self.create_common_resolvers() self.create_common_realms() self.create_dummy_cb_policies() # ------------------------------------------------------------------ -- self.gda = 'DEADBEEF' self.tokens = defaultdict(dict) # ------------------------------------------------------------------ -- pk, sk = gen_dsa_keypair() self.secret_key = sk self.public_key = pk # ----------------------------------------------------------------- -- # we need a dummy file to sneak past the file existence check # in the initial provider configuration self.dummy_temp_cert = NamedTemporaryFile() # ------------------------------------------------------------------ -- # make dummy provider config p_config = { "push_url": "https://pushproxy.keyidentity.com", "access_certificate": self.dummy_temp_cert.name, "server_certificate": "" } params = { 'name': 'dummy_provider', 'class': 'DefaultPushProvider', 'config': json.dumps(p_config), 'timeout': '120', 'type': 'push' } self.make_system_request('setProvider', params=params) # ------------------------------------------------------------------ -- params = { 'name': 'dummy_push_policy', 'scope': 'authentication', 'action': 'push_provider=dummy_provider', 'user': '******', 'realm': '*', 'client': '', 'time': '' } self.create_policy(params=params) self.uri = self.app.config.get('MOBILE_APP_PROTOCOLL_ID', 'lseqr') # -------------------------------------------------------------------------- -- def tearDown(self): self.delete_all_policies() self.delete_all_realms() self.delete_all_resolvers() self.delete_all_token() super(TestPushToken, self).tearDown() # ------------------------------------------------------------------ -- # delete dummy provider config params = {'name': 'dummy_provider_config', 'type': 'push'} self.make_system_request('delProvider', params=params) # ------------------------------------------------------------------ -- # delete temp file self.dummy_temp_cert.close() # -------------------------------------------------------------------------- -- def setPolicy(self, params): """ sets a system policy defined by param """ response = self.make_system_request('setPolicy', params) response_dict = json.loads(response.body) assert 'result' in response_dict result = response_dict.get('result') assert 'status' in result status = result.get('status') assert status response = self.make_system_request('getPolicy', params) # -------------------------------------------------------------------------- -- def create_dummy_cb_policies(self): """ sets some dummy callback policies. callback policies get ignored by the tests, but are nonetheless necessary for the backend """ # ------------------------------------------------------------------ -- # set pairing callback policies params = { 'name': 'dummy1', 'scope': 'authentication', 'realm': '*', 'action': 'pushtoken_pairing_callback_url=foo', 'user': '******' } self.setPolicy(params) # ------------------------------------------------------------------ -- # set challenge callback policies params = { 'name': 'dummy2', 'scope': 'authentication', 'realm': '*', 'action': 'pushtoken_challenge_callback_url=foo', 'user': '******' } self.setPolicy(params) # -------------------------------------------------------------------------- -- def enroll_pushtoken(self, user=None, pin='1234', serial=None): """ enrolls a pushtoken :param user: the user to which the token should be assigned :param pin: the pin the token should have after enrollment (default is 1234) :returns pairing url """ # initialize an unfinished token on the server params = {'type': 'push', 'pin': pin} if user: params['user'] = user if serial: params['serial'] = serial response = self.make_admin_request('init', params) # ------------------------------------------------------------------ -- # response should contain pairing url, check if it was # sent and validate response_dict = json.loads(response.body) assert 'pairing_url' in response_dict.get('detail', {}) pairing_url = response_dict.get('detail', {}).get('pairing_url') assert pairing_url is not None assert pairing_url.startswith(self.uri + '://pair/') return pairing_url # -------------------------------------------------------------------------- -- def activate_token(self, user_token_id, data='', retry_activation=1): """ activate the token - create the activation challenge by calling /validate/check - calculate the resonse in the test user token - send the signature to /validate/accept_transaction :param user_token_id: the id to the to be used token :param data: the data which is used during activation :return: the activation challenge and signature """ for i in range(0, retry_activation): # ------------------------------------------------------------- -- # request activation challenge challenge_url = self.trigger_challenge(user_token_id, data=data) # ------------------------------------------------------------- -- # parse, descrypt and verify the challenge url challenge, sig = self.decrypt_and_verify_challenge(challenge_url, action='ACCEPT') # ------------------------------------------------------------- -- # check if the content type is right (we are doing pairing # right now, so type must be CONTENT_TYPE_PAIRING) content_type = challenge['content_type'] assert content_type == CONTENT_TYPE_PAIRING # ----------------------------------------------------------------- -- # prepare params for validate params = { 'transactionid': challenge['transaction_id'], 'signature': sig } # again, we ignore the callback definitions response = self.make_validate_request('accept_transaction', params) response_dict = json.loads(response.body) status = response_dict.get('result', {}).get('status') assert status value = response_dict.get('result', {}).get('value') assert value, response def execute_correct_pairing(self, user=None, pin='1234', serial=None, retry_pairing=1, retry_activation=1): """ enroll token and pair it :param user: the user the token should be assigned to (default: None) :param pin: the pin the token should have (default '1234') :param retry_pairing: num of re-pairing :param retry_activation: num of re-activation :return user_token_id (index for self.tokens) """ # ------------------------------------------------------------------ -- # enroll token pairing_url = self.enroll_pushtoken(user=user, pin=pin, serial=serial) # ------------------------------------------------------------------ -- # pair the token for i in range(0, retry_pairing): user_token_id = self.pair_token(pairing_url, pin) # ------------------------------------------------------------------ -- # activate the token self.activate_token(user_token_id, data='', retry_activation=retry_activation) return user_token_id # -------------------------------------------------------------------------- -- def pair_token(self, pairing_url, pin='1234'): """ Pair an enrolled token - use the qr url to instantiate the test user token and - call the /validate/pair to bind this to the LinOTP token :param pairing_url: the pairing url provided by the token :param pin: the pin of the token (default: '1234') :return: handle of the test user token """ # save data extracted from pairing url to the 'user database' user_token_id = self.create_user_token_by_pairing_url(pairing_url, pin) # ------------------------------------------------------------------ -- # create the pairing response pairing_response = self.create_pairing_response_by_serial( user_token_id) # ------------------------------------------------------------------ -- # send pairing response response_dict = self.send_pairing_response(pairing_response) # ------------------------------------------------------------------ -- # check if returned json is correct assert 'result' in response_dict result = response_dict.get('result') assert 'value' in result value = result.get('value') assert not value assert 'status' in result status = result.get('status') assert status return user_token_id # -------------------------------------------------------------------------- -- def create_user_token_by_pairing_url(self, pairing_url, pin='1234'): """ parses the pairing url and saves the extracted data in the fake token database of this test class. :param pairing_url: the pairing url received from the server :param pin: the pin of the token (default: '1234') :returns: user_token_id of newly created token """ # extract metadata and the public key data_encoded = pairing_url[len(self.uri + '://pair/'):] data = decode_base64_urlsafe(data_encoded) version, token_type, flags = struct.unpack('<bbI', data[0:6]) partition = struct.unpack('<I', data[6:10])[0] server_public_key = data[10:10 + 32] # validate protocol versions and type id assert token_type == TYPE_PUSHTOKEN assert version == PAIRING_URL_VERSION # ------------------------------------------------------------------ -- # extract custom data that may or may not be present # (depending on flags) custom_data = data[10 + 32:] token_serial = None if flags & FLAG_PAIR_SERIAL: token_serial, __, custom_data = custom_data.partition(b'\x00') callback_url = None if flags & FLAG_PAIR_CBURL: callback_url, __, custom_data = custom_data.partition(b'\x00') else: raise NotImplementedError( 'Callback URL is mandatory for PushToken') # ------------------------------------------------------------------ -- # save token data for later use user_token_id = len(self.tokens) self.tokens[user_token_id] = { 'serial': token_serial.decode(), 'server_public_key': server_public_key, 'partition': partition, 'callback_url': callback_url.decode(), 'pin': pin } # ------------------------------------------------------------------ -- return user_token_id # -------------------------------------------------------------------------- -- def decrypt_and_verify_challenge(self, challenge_url, action): """ Decrypts the data packed in the challenge url, verifies its content, returns the parsed data as a dictionary, calculates and returns the signature. The calling method must then send the signature back to the server. (The reason for this control flow is that the challenge data must be checked in different scenarios, e.g. when we have a pairing the data must be checked by the method that simulates the pairing) :param challenge_url: the challenge url as sent by the server :param action: a string identifier for the verification action (at the moment 'ACCEPT' or 'DENY') :returns: (challenge, signature) challenge has the keys * content_type - one of the three values CONTENT_TYPE_SIGNREQ, CONTENT_TYPE_PAIRING or CONTENT_TYPE_LOGIN) (all defined in this module) * transaction_id - used to identify the challenge on the server * callback_url (optional) - the url to which the challenge response should be set * user_token_id - used to identify the token in the user database for which this challenge was created depending on the content type additional keys are present * for CONTENT_TYPE_PAIRING: serial * for CONTENT_TYPE_SIGNREQ: message * for CONTENT_TYPE_LOGIN: login, host signature is the generated user signature used to respond to the challenge """ challenge_data_encoded = challenge_url[len(self.uri + '://chal/'):] challenge_data = decode_base64_urlsafe(challenge_data_encoded) # ------------------------------------------------------------------ -- # parse and verify header information in the # encrypted challenge data header = challenge_data[0:5] version, user_token_id = struct.unpack('<bI', header) assert version == CHALLENGE_URL_VERSION # ------------------------------------------------------------------ -- # get token from client token database token = self.tokens[user_token_id] server_public_key = token['server_public_key'] # ------------------------------------------------------------------ -- # prepare decryption by seperating R from # ciphertext and server signature R = challenge_data[5:5 + 32] ciphertext = challenge_data[5 + 32:-64] server_signature = challenge_data[-64:] # check signature data = challenge_data[0:-64] crypto_sign_verify_detached(server_signature, data, server_public_key) # ------------------------------------------------------------------ -- # key derivation secret_key_dh = dsa_to_dh_secret(self.secret_key) ss = calc_dh(secret_key_dh, R) U = SHA256.new(ss).digest() sk = U[0:16] nonce = U[16:32] # ------------------------------------------------------------------ -- # decrypt and verify challenge nonce_as_int = int_from_bytes(nonce, byteorder='big') ctr = Counter.new(128, initial_value=nonce_as_int) cipher = AES.new(sk, AES.MODE_CTR, counter=ctr) plaintext = cipher.decrypt(ciphertext) # ------------------------------------------------------------------ -- # parse/check plaintext header # 1 - for content type # 8 - for transaction id # 8 - for time stamp offset = 1 + 8 + 8 pt_header = plaintext[0:offset] (content_type, transaction_id, _time_stamp) = struct.unpack('<bQQ', pt_header) transaction_id = u64_to_transaction_id(transaction_id) # ------------------------------------------------------------------ -- # prepare the parsed challenge data challenge = {} challenge['content_type'] = content_type # ------------------------------------------------------------------ -- # retrieve plaintext data depending on content_type if content_type == CONTENT_TYPE_PAIRING: serial, callback_url, __ = plaintext[offset:].split(b'\x00') challenge['serial'] = serial.decode() elif content_type == CONTENT_TYPE_SIGNREQ: message, callback_url, __ = plaintext[offset:].split(b'\x00') challenge['message'] = message.decode() elif content_type == CONTENT_TYPE_LOGIN: login, host, callback_url, __ = plaintext[offset:].split(b'\x00') challenge['login'] = login.decode() challenge['host'] = host.decode() # ------------------------------------------------------------------ -- # prepare the parsed challenge data challenge['callback_url'] = callback_url.decode() challenge['transaction_id'] = transaction_id challenge['user_token_id'] = user_token_id # calculate signature sig_base = (struct.pack('<b', CHALLENGE_URL_VERSION) + b'%s\0' % action.encode('utf-8') + server_signature + plaintext) sig = crypto_sign_detached(sig_base, self.secret_key) encoded_sig = encode_base64_urlsafe(sig) return challenge, encoded_sig # -------------------------------------------------------------------------- -- def test_correct_pairing(self): """ PushToken: Check if pairing works correctly """ self.execute_correct_pairing() def test_multiple_pairing_activations(self): """ PushToken: Check if pairing works multiple times correctly """ self.execute_correct_pairing(retry_pairing=3, retry_activation=3) # -------------------------------------------------------------------------- -- def create_pairing_response_by_serial(self, user_token_id): """ Creates a base64-encoded pairing response that identifies the token by its serial :param user_token_id: the token id (primary key for the user token db) :returns base64 encoded pairing response """ token_serial = self.tokens[user_token_id]['serial'] server_public_key = self.tokens[user_token_id]['server_public_key'] partition = self.tokens[user_token_id]['partition'] # ------------------------------------------------------------------ -- # assemble header and plaintext header = struct.pack('<bI', PAIR_RESPONSE_VERSION, partition) pairing_response = b'' pairing_response += struct.pack('<bI', TYPE_PUSHTOKEN, user_token_id) pairing_response += self.public_key pairing_response += token_serial.encode('utf8') + b'\x00\x00' pairing_response += self.gda.encode('utf-8') + b'\x00' signature = crypto_sign_detached(pairing_response, self.secret_key) pairing_response += signature # ------------------------------------------------------------------ -- # create public diffie hellman component # (used to decrypt and verify the reponse) r = os.urandom(32) R = calc_dh_base(r) # ------------------------------------------------------------------ -- # derive encryption key and nonce server_public_key_dh = dsa_to_dh_public(server_public_key) ss = calc_dh(r, server_public_key_dh) U = SHA256.new(ss).digest() encryption_key = U[0:16] nonce = U[16:32] # ------------------------------------------------------------------ -- # encrypt in EAX mode cipher = AES.new(encryption_key, AES.MODE_EAX, nonce) cipher.update(header) ciphertext, tag = cipher.encrypt_and_digest(pairing_response) return encode_base64_urlsafe(header + R + ciphertext + tag) # -------------------------------------------------------------------------- -- def send_pairing_response(self, pairing_response): """ sends a pairing response to /validate/pair """ params = {'pairing_response': pairing_response} # we use the standard calback url in here # in a real client we would use the callback # defined in the pairing url (and saved in # the 'token database' of the user) response = self.make_validate_request('pair', params) response_dict = json.loads(response.body) return response_dict # -------------------------------------------------------------------------- -- def trigger_challenge(self, user_token_id, content_type=None, data=None): serial = self.tokens[user_token_id]['serial'] pin = self.tokens[user_token_id]['pin'] params = {'serial': serial, 'pass': pin} if content_type is not None: params['content_type'] = content_type if data is not None: params['data'] = data # ------------------------------------------------------------------ -- # we mock the interface of the push provider (namely the method # push_notification) to get the generated challenge_url passed # to it (which would normaly be sent over the PNP) with mock.patch.object(default_provider.DefaultPushProvider, 'push_notification', autospec=True) as mock_push_notification: mock_push_notification.return_value = (True, None) response = self.make_validate_request('check_s', params) challenge_url = mock_push_notification.call_args[0][1] response_dict = json.loads(response.body) assert 'result' in response_dict result = response_dict.get('result') assert 'status' in result assert 'value' in result status = result.get('status') value = result.get('value') assert status assert not value # ------------------------------------------------------------------ -- return challenge_url # -------------------------------------------------------------------------- -- def test_signreq(self): """ PushToken: Check if signing transactions works correctly """ user_token_id = self.execute_correct_pairing(user='******') challenge_url = self.trigger_challenge( user_token_id, data=( 'Yes, I want to know why doctors hate this guy. Take these ' '6000 $ with all my sincere benevolence and send me the black ' 'magic diet pill they don\'t want me to know about'), content_type=CONTENT_TYPE_SIGNREQ) challenge, sig = self.decrypt_and_verify_challenge(challenge_url, action='ACCEPT') # ------------------------------------------------------------------ -- # check if the content type is right content_type = challenge['content_type'] assert content_type == CONTENT_TYPE_SIGNREQ # ------------------------------------------------------------------ -- # prepare params for validate params = { 'transactionid': challenge['transaction_id'], 'signature': sig } # again, we ignore the callback definitions response = self.make_validate_request('accept_transaction', params) response_dict = json.loads(response.body) status = response_dict.get('result', {}).get('status') assert status value = response_dict.get('result', {}).get('value') assert value, response # ------------------------------------------------------------------ -- # status check params = { 'transactionid': challenge['transaction_id'], 'user': '******', 'pass': '******' } response = self.make_validate_request('check_status', params) response_dict = json.loads(response.body) transactions = response_dict.get('detail', {}).get('transactions', {}) transaction = transactions[challenge['transaction_id']] assert transaction['status'] == 'closed', response assert transaction['accept'], response assert transaction['valid_tan'], response assert 'KIPT' in transaction['token']['serial'], response return # -------------------------------------------------------------------------- -- def test_multiple_signreq(self): """ PushToken: Check if signing multiple transactions works correctly """ user_token_id = self.execute_correct_pairing(user='******', serial='KIPuOne') # ------------------------------------------------------------------ -- created_challenges = [] for i in range(0, 10): challenge_url = self.trigger_challenge( user_token_id, data= ('Yes, I want to know why doctors hate this guy. Take these ' '%d000 $ with all my sincere benevolence and send me the black ' 'magic diet pill they don\'t want me to know about' % i), content_type=CONTENT_TYPE_SIGNREQ) challenge, sig = self.decrypt_and_verify_challenge(challenge_url, action='ACCEPT') # ------------------------------------------------------------------ -- # check if the content type is right content_type = challenge['content_type'] assert content_type == CONTENT_TYPE_SIGNREQ created_challenges.append((challenge_url, challenge, sig)) # ------------------------------------------------------------------ -- # verify that all challenges are kept params = {'serial': 'KIPuOne', 'open': True} response = self.make_admin_request('checkstatus', params) response_dict = json.loads(response.body) challenges = response_dict.get('result', {}).get('value', {}).get( 'values', {}).get('KIPuOne', {}).get('challenges', []) # remark: # we have here one additonal challenge, which was the inital # pairing challenge assert len(challenges) == (len(created_challenges) + 1) # ------------------------------------------------------------------ -- # validate the one of the eldest challenge: # from 10 challenges 5 are left open, so we take the 7th one (challenge_url, challenge, sig) = created_challenges[7] # prepare params for validate params = { 'transactionid': challenge['transaction_id'], 'signature': sig } # again, we ignore the callback definitions response = self.make_validate_request('accept_transaction', params) response_dict = json.loads(response.body) status = response_dict.get('result', {}).get('status') assert status value = response_dict.get('result', {}).get('value') assert value, response # ------------------------------------------------------------------ -- # status check params = { 'transactionid': challenge['transaction_id'], 'user': '******', 'pass': '******' } response = self.make_validate_request('check_status', params) response_dict = json.loads(response.body) transactions = response_dict.get('detail', {}).get('transactions', {}) transaction = transactions[challenge['transaction_id']] assert transaction['status'] == 'closed', response assert transaction['accept'], response assert transaction['valid_tan'], response # verify that all challenges are kept params = {'serial': 'KIPuOne', 'open': True} response = self.make_admin_request('checkstatus', params) response_dict = json.loads(response.body) challenges = response_dict.get('result', {}).get('value', {}).get( 'values', {}).get('KIPuOne', {}).get('challenges', []) open_challenges = 0 accept_challenges = 0 for challenge in list(challenges.values()): status = challenge['session']['status'] accept = challenge['session'].get('accept') if status == 'open': open_challenges += 1 if status == 'closed' and accept: accept_challenges += 1 assert open_challenges == 9 assert accept_challenges == 2 return # -------------------------------------------------------------------------- -- def test_signreq_reject(self): """ PushToken: Check if reject signing transactions works correctly """ user_token_id = self.execute_correct_pairing(user='******', pin='1234') challenge_url = self.trigger_challenge( user_token_id, data=( 'Yes, I want to know why doctors hate this guy. Take these ' '6000 $ with all my sincere benevolence and send me the black ' 'magic diet pill they don\'t want me to know about'), content_type=CONTENT_TYPE_SIGNREQ) challenge, sig = self.decrypt_and_verify_challenge(challenge_url, action='DENY') # ------------------------------------------------------------------ -- # check if the content type is right content_type = challenge['content_type'] assert content_type == CONTENT_TYPE_SIGNREQ # ------------------------------------------------------------------ -- # prepare params for validate params = { 'transactionid': challenge['transaction_id'], 'signature': sig } # again, we ignore the callback definitions response = self.make_validate_request('reject_transaction', params) response_dict = json.loads(response.body) status = response_dict.get('result', {}).get('status') assert status value = response_dict.get('result', {}).get('value') assert value, response # ------------------------------------------------------------------ -- # status check params = { 'transactionid': challenge['transaction_id'], 'user': '******', 'pass': '******' } response = self.make_validate_request('check_status', params) response_dict = json.loads(response.body) transactions = response_dict.get('detail', {}).get('transactions', {}) transaction = transactions[challenge['transaction_id']] assert transaction['status'] == 'closed', response assert transaction['reject'], response assert not transaction['valid_tan'], response return # -------------------------------------------------------------------------- -- def test_failed_signreq(self): """ PushToken: Check if signing transactions fails correctly """ user_token_id = self.execute_correct_pairing() challenge_url = self.trigger_challenge( user_token_id, data=( 'Yes, I want to know why doctors hate this guy. Take these ' '6000 $ with all my sincere benevolence and send me the black ' 'magic diet pill they don\'t want me to know about'), content_type=CONTENT_TYPE_SIGNREQ) challenge, __ = self.decrypt_and_verify_challenge(challenge_url, action='ACCEPT') wrong_sig = 'DEADBEEF' * 32 # ------------------------------------------------------------------ -- # check if the content type is right content_type = challenge['content_type'] assert content_type == CONTENT_TYPE_SIGNREQ # ------------------------------------------------------------------ -- # prepare params for validate params = { 'transactionid': challenge['transaction_id'], 'signature': wrong_sig } # again, we ignore the callback definitions response = self.make_validate_request('accept_transaction', params) response_dict = json.loads(response.body) status = response_dict.get('result', {}).get('status') assert status value = response_dict.get('result', {}).get('value') assert not value, response # -------------------------------------------------------------------------- -- def test_repairing(self): """ PushToken: Check if repairing works correctly """ user_token_id = self.execute_correct_pairing() # temporarily switch the gda tmp_gda = self.gda self.gda = '7777' # ------------------------------------------------------------------ -- # send repairing pairing response pairing_response = self.create_pairing_response_by_serial( user_token_id) response_dict = self.send_pairing_response(pairing_response) # ------------------------------------------------------------------ -- # check if returned json is correct assert 'result' in response_dict result = response_dict.get('result') assert 'value' in result value = result.get('value') assert not value assert 'status' in result status = result.get('status') assert status # ------------------------------------------------------------------ -- # reset the gda self.gda = tmp_gda # -------------------------------------------------------------------------- -- def test_repairing_fail_sig(self): """ PushToken: Check if repairing fails correctly (wrong sig) """ user_token_id = self.execute_correct_pairing() # temporarily switch the secret key (used for signature) tmp_secret_key = self.secret_key _public_key, self.secret_key = crypto_sign_keypair() # ------------------------------------------------------------------ -- # send repairing pairing response pairing_response = self.create_pairing_response_by_serial( user_token_id) response_dict = self.send_pairing_response(pairing_response) # ------------------------------------------------------------------ -- # check if returned json is correct assert 'result' in response_dict result = response_dict.get('result') assert 'value' in result value = result.get('value') assert not value assert 'status' in result status = result.get('status') assert not status # ------------------------------------------------------------------ -- # reset the secret key self.secret_key = tmp_secret_key # -------------------------------------------------------------------------- -- def test_repairing_fail_pubkey(self): """ PushToken: Check if repairing fails correctly (wrong pubkey) """ user_token_id = self.execute_correct_pairing() # temporarily switch the keypair (used for signature) tmp_secret_key = self.secret_key tmp_public_key = self.public_key pk, sk = gen_dsa_keypair() self.secret_key = sk self.public_key = pk # ------------------------------------------------------------------ -- # send repairing pairing response pairing_response = self.create_pairing_response_by_serial( user_token_id) response_dict = self.send_pairing_response(pairing_response) # ------------------------------------------------------------------ -- # check if returned json is correct assert 'result' in response_dict result = response_dict.get('result') assert 'value' in result value = result.get('value') assert not value assert 'status' in result status = result.get('status') assert not status # ------------------------------------------------------------------ -- # reset the secret key self.secret_key = tmp_secret_key self.public_key = tmp_public_key # -------------------------------------------------------------------------- -- def test_login(self): """ PushToken: Check if signing logins works correctly """ user_token_id = self.execute_correct_pairing() challenge_url = self.trigger_challenge(user_token_id, data='root@foo', content_type=CONTENT_TYPE_LOGIN) challenge, sig = self.decrypt_and_verify_challenge(challenge_url, action='ACCEPT') # ------------------------------------------------------------------ -- # check if the content type is right content_type = challenge['content_type'] assert content_type == CONTENT_TYPE_LOGIN # ------------------------------------------------------------------ -- # prepare params for validate params = { 'transactionid': challenge['transaction_id'], 'signature': sig } # again, we ignore the callback definitions response = self.make_validate_request('accept_transaction', params) response_dict = json.loads(response.body) status = response_dict.get('result', {}).get('status') assert status value = response_dict.get('result', {}).get('value') assert value, response # -------------------------------------------------------------------------- -- def test_unsupported_content_type(self): """ PushToken: Check for unsupported content types """ user_token_id = self.execute_correct_pairing() serial = self.tokens[user_token_id]['serial'] pin = self.tokens[user_token_id]['pin'] params = { 'serial': serial, 'pass': pin, 'data': 'wohoooo', 'content_type': 99999999999 } response = self.make_validate_request('check_s', params) response_dict = json.loads(response.body) assert 'result' in response_dict result = response_dict.get('result') assert 'status' in result assert 'value' in result status = result.get('status') value = result.get('value') assert not status assert not value
def __init__(self, filename=None, title=None, overwrite=False): """ Used to initialize the makefile. Will generate a main function for the makefile. Optionally, will display a title string at beginning of execution. filename: str, path to the makefile (Default is None, i.e. generate a temporary file) title: string to display at the beginning of the execution overwrite: if set to True, will overwrite an existing makefile. """ # Create and open temporary file if filename is None: tmpf = NamedTemporaryFile(mode='w+') filename = tmpf.name f = tmpf.file self.tmpf = tmpf # Keep otherwise file will be deleted self.filename = tmpf.name # Open chosen file else: # Just open the file in append mode. # Won't work for temporary files though as self.tmpf is deleted. if os.path.exists(filename) & (not overwrite): f = open(filename, 'a+') # append and writing f.seek(0) self.filename = filename self.f = f return else: f = open(filename, 'w+') # writing and reading self.filename = filename # Write header f.write(".PHONY: MAIN\n\n") # Write colors for commands highlighting f.write("CMDCOL := [32m\n") f.write("DEFCOL := [0m\n\n") # Write MAIN line, without title if title is None: f.write("MAIN: \n\n") # with title else: f.write("MAIN: pre-build \n\n") f.write("pre-build:\n\t@+printf '%s\\n'\n\n" % title) # Add a function to list missing outputs, call with 'make list' f.write("list:\n") f.write("\t@printf '** Missing outputs **\\n'\n") f.write("\t@$(MAKE) -n --debug -f $(lastword $(MAKEFILE_LIST)) | \ sed -n -e 's/^.*Must remake target //p' | \ sed -e '/MAIN/d' | sed -e '/pre-build/d'\n\n") # save f.flush() self.f = f # Check if make version is newer than 4.3 make_version = get_make_version() if make_version > version.parse('4.3'): self.new_version = True else: self.new_version = False
def setUp(self): self.delete_all_policies() self.delete_all_token() self.delete_all_realms() self.delete_all_resolvers() super(TestPushToken, self).setUp() self.create_common_resolvers() self.create_common_realms() self.create_dummy_cb_policies() # ------------------------------------------------------------------ -- self.gda = 'DEADBEEF' self.tokens = defaultdict(dict) # ------------------------------------------------------------------ -- pk, sk = gen_dsa_keypair() self.secret_key = sk self.public_key = pk # ----------------------------------------------------------------- -- # we need a dummy file to sneak past the file existence check # in the initial provider configuration self.dummy_temp_cert = NamedTemporaryFile() # ------------------------------------------------------------------ -- # make dummy provider config p_config = { "push_url": "https://pushproxy.keyidentity.com", "access_certificate": self.dummy_temp_cert.name, "server_certificate": "" } params = { 'name': 'dummy_provider', 'class': 'DefaultPushProvider', 'config': json.dumps(p_config), 'timeout': '120', 'type': 'push' } self.make_system_request('setProvider', params=params) # ------------------------------------------------------------------ -- params = { 'name': 'dummy_push_policy', 'scope': 'authentication', 'action': 'push_provider=dummy_provider', 'user': '******', 'realm': '*', 'client': '', 'time': '' } self.create_policy(params=params) self.uri = self.app.config.get('MOBILE_APP_PROTOCOLL_ID', 'lseqr')
def set_stdout(self): self.stdout = NamedTemporaryFile('w', delete=False, prefix='astevaltest') self.interp.writer = self.stdout
def create_recovery_archive(self): """ Create a compressed recovery archive from the root tree for use with kiwi's recvoery system. The method creates additional data into the image root filesystem which is deleted prior to the creation of a new recovery data set """ # cleanup bash_comand = ['rm', '-f', self.root_dir + '/recovery.*'] Command.run(['bash', '-c', ' '.join(bash_comand)]) if not self.oemconfig['recovery']: return # recovery.tar log.info('Creating recovery tar archive') metadata = { 'archive_name': self.root_dir + '/recovery.tar', 'archive_filecount': self.root_dir + '/recovery.tar.files', 'archive_size': self.root_dir + '/recovery.tar.size', 'partition_size': self.root_dir + '/recovery.partition.size', 'partition_filesystem': self.root_dir + '/recovery.tar.filesystem' } recovery_archive = NamedTemporaryFile(delete=False) archive = ArchiveTar(filename=recovery_archive.name, create_from_file_list=False) archive.create(source_dir=self.root_dir, exclude=['dev', 'proc', 'sys'], options=[ '--numeric-owner', '--hard-dereference', '--preserve-permissions' ]) Command.run(['mv', recovery_archive.name, metadata['archive_name']]) # recovery.tar.filesystem recovery_filesystem = self.xml_state.build_type.get_filesystem() with open(metadata['partition_filesystem'], 'w') as partfs: partfs.write('%s' % recovery_filesystem) log.info('--> Recovery partition filesystem: %s', recovery_filesystem) # recovery.tar.files bash_comand = ['tar', '-tf', metadata['archive_name'], '|', 'wc', '-l'] tar_files_call = Command.run(['bash', '-c', ' '.join(bash_comand)]) tar_files_count = int(tar_files_call.output.rstrip('\n')) with open(metadata['archive_filecount'], 'w') as files: files.write('%d\n' % tar_files_count) log.info('--> Recovery file count: %d files', tar_files_count) # recovery.tar.size recovery_archive_size_bytes = os.path.getsize(metadata['archive_name']) with open(metadata['archive_size'], 'w') as size: size.write('%d' % recovery_archive_size_bytes) log.info('--> Recovery uncompressed size: %d mbytes', int(recovery_archive_size_bytes / 1048576)) # recovery.tar.gz log.info('--> Compressing recovery archive') compress = Compress(self.root_dir + '/recovery.tar') compress.gzip() # recovery.partition.size recovery_archive_gz_size_mbytes = int( os.path.getsize(metadata['archive_name'] + '.gz') / 1048576) recovery_partition_mbytes = recovery_archive_gz_size_mbytes \ + Defaults.get_recovery_spare_mbytes() with open(metadata['partition_size'], 'w') as gzsize: gzsize.write('%d' % recovery_partition_mbytes) log.info('--> Recovery partition size: %d mbytes', recovery_partition_mbytes) # delete recovery archive if inplace recovery is requested # In this mode the recovery archive is created at install time # and not at image creation time. However the recovery metadata # is preserved in order to be able to check if enough space # is available on the disk to create the recovery archive. if self.oemconfig['recovery_inplace']: log.info('--> Inplace recovery requested, deleting archive') Path.wipe(metadata['archive_name'] + '.gz')
def create_yaml_config(**kw): tmp = NamedTemporaryFile(mode="wt", suffix=".yml", delete=False) yaml.dump(kw, tmp, default_flow_style=False) tmp.flush() return tmp.name
def set_stderr(self): self.stderr = NamedTemporaryFile('w', delete=False, prefix='astevaltest_stderr') self.interp.err_writer = self.stderr
def run_tests(): import json def extract_json(resp): return json.loads(''.join( [l for l in resp[0] if not l.startswith('//')])) def test(assertion, test): print('Test %s: %s' % ('passed' if assertion else 'failed', test)) if not assertion: raise AssertionError resp = extract_json( do_request([ '# comment', '# :a=barf', 'GET http://httpbin.org/headers', 'X-Hey: :a', '# comment' ], [])) test(resp['headers']['X-Hey'] == 'barf', 'Headers are passed with variable substitution.') resp = extract_json( do_request(['# :a = barf', 'GET http://httpbin.org/get?data=:a'], [])) test(resp['args']['data'] == 'barf', 'GET data is passed with variable substitution.') resp = extract_json( do_request(['POST http://httpbin.org/post', 'some data'], [])) test(resp['data'] == 'some data', 'POST data is passed with variable substitution.') resp = extract_json( do_request([ 'POST http://httpbin.org/post', 'forma=a', 'formb=b', ], [])) test(resp['form']['forma'] == 'a', 'POST form data is passed.') resp = extract_json( do_request([ 'POST http://$global/post', 'forma=a', 'formb=b', ], ['# $global = httpbin.org'])) test(resp['form']['forma'] == 'a', 'Global variables are substituted.') import os from tempfile import NamedTemporaryFile SAMPLE_FILE_CONTENT = 'sample file content' temp_file = NamedTemporaryFile(delete=False) temp_file.write(SAMPLE_FILE_CONTENT) temp_file.close() resp = extract_json( do_request([ 'POST http://httpbin.org/post', 'forma=a', 'formb=b', "formc=!file(%s)" % temp_file.name, ], [])) test(resp['files']['formc'] == SAMPLE_FILE_CONTENT, 'Files given as path are sent properly.') test(not 'formc' in resp['form'], 'File not included in form data.') os.unlink(temp_file.name) resp = extract_json( do_request([ 'POST http://httpbin.org/post', 'forma=a', 'formb=b', "formc=!content(%s)" % SAMPLE_FILE_CONTENT, ], [])) test(resp['files']['formc'] == SAMPLE_FILE_CONTENT, 'Files given as content are sent properly.') resp = extract_json( do_request([ 'POST http://httpbin.org/post', "c=!content(foo \\(bar\\))", ], [])) test(resp['files']['c'] == 'foo (bar)', 'Escaped parenthesis should be unescaped during request')
class TestCase(unittest.TestCase): """testing of asteval""" def setUp(self): self.interp = Interpreter() self.symtable = self.interp.symtable self.set_stdout() self.set_stderr() def set_stdout(self): self.stdout = NamedTemporaryFile('w', delete=False, prefix='astevaltest') self.interp.writer = self.stdout def read_stdout(self): self.stdout.close() time.sleep(0.1) fname = self.stdout.name with open(self.stdout.name) as inp: out = inp.read() self.set_stdout() os.unlink(fname) return out def set_stderr(self): self.stderr = NamedTemporaryFile('w', delete=False, prefix='astevaltest_stderr') self.interp.err_writer = self.stderr def read_stderr(self): self.stderr.close() time.sleep(0.1) fname = self.stderr.name with open(self.stderr.name) as inp: out = inp.read() self.set_stderr() os.unlink(fname) return out def tearDown(self): if not self.stdout.closed: self.stdout.close() if not self.stderr.closed: self.stderr.close() # noinspection PyBroadException try: os.unlink(self.stdout.name) except: pass try: os.unlink(self.stderr.name) except: pass # noinspection PyUnresolvedReferences def isvalue(self, sym, val): """assert that a symboltable symbol has a particular value""" tval = self.interp.symtable[sym] if isinstance(val, np.ndarray): assert_allclose(tval, val, rtol=0.01) else: assert (tval == val) def isnear(self, expr, val): tval = self.interp(expr) assert_allclose(tval, val, rtol=1.e-4, atol=1.e-4) # noinspection PyUnresolvedReferences def istrue(self, expr): """assert that an expression evaluates to True""" val = self.interp(expr) if isinstance(val, np.ndarray): val = np.all(val) return self.assertTrue(val) # noinspection PyUnresolvedReferences def isfalse(self, expr): """assert that an expression evaluates to False""" val = self.interp(expr) if isinstance(val, np.ndarray): val = np.all(val) return self.assertFalse(val) def check_output(self, chk_str, exact=False): self.interp.writer.flush() out = self.read_stdout().split('\n') if out: if exact: return chk_str == out[0] return chk_str in out[0] return False def check_error(self, chk_type='', chk_msg=''): try: errtype, errmsg = self.interp.error[0].get_error() self.assertEqual(errtype, chk_type) if chk_msg: self.assertTrue(chk_msg in errmsg) except IndexError: if chk_type: self.assertTrue(False)
def get_shelf(): with NamedTemporaryFile(suffix=".ldb", mode="r") as tmpfile: yield partial(Shelf, tmpfile.name)
async def kubectl_create(definition): with NamedTemporaryFile('w') as f: json.dump(definition, f) f.flush() await scp_to(f.name, vessel, '/tmp/test-registry/temp.yaml') await kubectl('create -f /tmp/test-registry/temp.yaml')
def addBackgroundOnPdfFile(orginal_pdf, background_pdf): '''This method apply the background on all pages of the original pdf file Pdftk uses only the first page from the background_pdf_file and applies it to every page of the orginal_pdf_file. This page is scaled and rotated as needed to fit the input page.''' tmp_pdf_file_name = NamedTemporaryFile().name # create two temporary files to give to pdftk command orginal_pdf_file = NamedTemporaryFile() background_pdf_file = NamedTemporaryFile() # saving content orginal_pdf_file.write(orginal_pdf) orginal_pdf_file.seek(0) # saving content background_pdf_file.write(_unpackData(background_pdf.data)) background_pdf_file.seek(0) try: result = commands.getstatusoutput('pdftk %s background %s output %s' % \ (orginal_pdf_file.name, background_pdf_file.name, tmp_pdf_file_name)) # check that the command has been done succeful if result[0] != 0: LOG('addBackgroundOnPdfFile :', ERROR, 'pdftk command'\ 'failed with the following error message : \n%s' % result[1]) # delete created pdf before raise an error #os.remove(tmp_pdf_file_name) orginal_pdf_file.close() background_pdf_file.close() raise ValueError, 'Error: pdftk command failed with the following'\ 'error message : \n%s' % result[1] finally: background_pdf_file.close() orginal_pdf_file.close() return tmp_pdf_file_name
class BaseTaskRunner(LoggingMixin): """ Runs Airflow task instances by invoking the `airflow tasks run` command with raw mode enabled in a subprocess. :param local_task_job: The local task job associated with running the associated task instance. :type local_task_job: airflow.jobs.local_task_job.LocalTaskJob """ def __init__(self, local_task_job): # Pass task instance context into log handlers to setup the logger. super().__init__(local_task_job.task_instance) self._task_instance = local_task_job.task_instance popen_prepend = [] if self._task_instance.run_as_user: self.run_as_user = self._task_instance.run_as_user else: try: self.run_as_user = conf.get('core', 'default_impersonation') except AirflowConfigException: self.run_as_user = None # Add sudo commands to change user if we need to. Needed to handle SubDagOperator # case using a SequentialExecutor. self.log.debug("Planning to run as the %s user", self.run_as_user) if self.run_as_user and (self.run_as_user != getpass.getuser()): # We want to include any environment variables now, as we won't # want to have to specify them in the sudo call - they would show # up in `ps` that way! And run commands now, as the other user # might not be able to run the cmds to get credentials cfg_path = tmp_configuration_copy(chmod=0o600) # Give ownership of file to user; only they can read and write subprocess.call(['sudo', 'chown', self.run_as_user, cfg_path], close_fds=True) # propagate PYTHONPATH environment variable pythonpath_value = os.environ.get(PYTHONPATH_VAR, '') popen_prepend = ['sudo', '-E', '-H', '-u', self.run_as_user] if pythonpath_value: popen_prepend.append(f'{PYTHONPATH_VAR}={pythonpath_value}') else: # Always provide a copy of the configuration file settings. Since # we are running as the same user, and can pass through environment # variables then we don't need to include those in the config copy # - the runner can read/execute those values as it needs cfg_path = tmp_configuration_copy(chmod=0o600) self._error_file = NamedTemporaryFile(delete=True) self._cfg_path = cfg_path self._command = ( popen_prepend + self._task_instance.command_as_list( raw=True, pickle_id=local_task_job.pickle_id, mark_success=local_task_job.mark_success, job_id=local_task_job.id, pool=local_task_job.pool, cfg_path=cfg_path, ) + ["--error-file", self._error_file.name] ) self.process = None def deserialize_run_error(self) -> Optional[Union[str, Exception]]: """Return task runtime error if its written to provided error file.""" return load_error_file(self._error_file) def _read_task_logs(self, stream): while True: line = stream.readline() if isinstance(line, bytes): line = line.decode('utf-8') if not line: break self.log.info( 'Job %s: Subtask %s %s', self._task_instance.job_id, self._task_instance.task_id, line.rstrip('\n'), ) def run_command(self, run_with=None): """ Run the task command. :param run_with: list of tokens to run the task command with e.g. ``['bash', '-c']`` :type run_with: list :return: the process that was run :rtype: subprocess.Popen """ run_with = run_with or [] full_cmd = run_with + self._command self.log.info("Running on host: %s", get_hostname()) self.log.info('Running: %s', full_cmd) # pylint: disable=subprocess-popen-preexec-fn proc = subprocess.Popen( full_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, close_fds=True, env=os.environ.copy(), preexec_fn=os.setsid, ) # Start daemon thread to read subprocess logging output log_reader = threading.Thread( target=self._read_task_logs, args=(proc.stdout,), ) log_reader.daemon = True log_reader.start() return proc def start(self): """Start running the task instance in a subprocess.""" raise NotImplementedError() def return_code(self) -> Optional[int]: """ :return: The return code associated with running the task instance or None if the task is not yet done. :rtype: int """ raise NotImplementedError() def terminate(self) -> None: """Force kill the running task instance.""" raise NotImplementedError() def on_finish(self) -> None: """A callback that should be called when this is done running.""" if self._cfg_path and os.path.isfile(self._cfg_path): if self.run_as_user: subprocess.call(['sudo', 'rm', self._cfg_path], close_fds=True) else: os.remove(self._cfg_path) self._error_file.close()
def test_two_files_are_different_as_string_is_true___result_is_concatenated_list_of_differences( self): first = NamedTemporaryFile(mode='w', delete=False) second = NamedTemporaryFile(mode='w', delete=False) try: first.writelines( ['HEADING\n', 'first\n', 'same\n', 'second\n', 'FOOTER\n']) first.close() second.writelines( ['HEADING\n', 'third\n', 'same\n', 'fourth\n', 'FOOTER\n']) second.close() diff = unified_diff(first.name, second.name, as_string=True) self.assertEqual( diff, ''.join([ '--- {}\n'.format(first.name), '+++ {}\n'.format(second.name), '@@ -1,5 +1,5 @@\n', ' HEADING\n', '-first\n', '+third\n', ' same\n', '-second\n', '+fourth\n', ' FOOTER\n', ])) finally: os.remove(first.name) os.remove(second.name)
def mergePDF(pdf_document_list): '''Merge all pdf in the pdf_document_list in one using pdftk and return it''' from warnings import warn warn( "mergePDF is deprecated, use erp5_pdf_merge business template instead") tmp_pdf_list = [] # create as tmp file as there is pdf_documents for pdf_document in pdf_document_list: tmp_pdf_file = NamedTemporaryFile() # saving content tmp_pdf_file.write(_unpackData(pdf_document.data)) tmp_pdf_file.seek(0) tmp_pdf_list.append(tmp_pdf_file) # create a tmp file to put the resulting pdf file result_file = NamedTemporaryFile() result = None try: name_list = [x.name for x in tmp_pdf_list] cmd = 'pdftk %s cat output %s' % (' '.join(name_list), result_file.name) result = commands.getstatusoutput(cmd) # check that the command has been done succeful if result[0] != 0: LOG('mergePDF :', ERROR, 'pdftk command'\ 'failed with the following error message : \n%s' % result[1]) # delete created pdf before raise an error for tmp_file in tmp_pdf_list: tmp_file.close() raise ValueError, 'Error: pdftk command failed with the following'\ 'error message : \n%s' % result[1] else: # going to the begining of the input file result_file.seek(0) # put content in variable result = result_file.read() finally: for tmp_file in tmp_pdf_list: tmp_file.close() # close result file result_file.close() return result
def export_range_matrix_table_col_p100(): with NamedTemporaryFile() as f: mt = hl.utils.range_matrix_table(n_rows=1_000_000, n_cols=10, n_partitions=100) mt.col.export(f.name)
def main(argv=["-h"]): global items args = arg_parser.parse_args(argv) items = args.items netblocks = [] if args.update: from tempfile import NamedTemporaryFile tmpfile = NamedTemporaryFile() try: update(tmpfile) except Exception as e: print str(e) tmpfile.close() else: params = {} if args.ipaddr: params['ipaddr'] = args.ipaddr if args.inetnum: params['inetnum'] = args.inetnum if args.inetnums: params['inetnums'] = args.inetnums if args.netname: params['netname'] = args.netname if args.descr: params['descr'] = args.descr if args.country: params['country'] = args.country if args.notify: params['notify'] = args.notify if args.address: params['address'] = args.address if args.phone: params['phone'] = args.phone if params: if check_db(): netblocks = ripe_search(params) else: print "update database first" return if netblocks: summary = get_stat(netblocks, items) try: margins = map( lambda i: max( map(lambda n: len(str(n.get(i) or '').decode('utf-8')), netblocks) + [len(i), len(summary[i])]), items) except: margins = map( lambda i: max( map(lambda n: len(str(n.get(i) or '')), netblocks) + [len(i), len(summary[i])]), items) if len(items) > 1: print_row(tuple(items), margins) print_row(tuple(map(lambda m: '-' * m, margins)), margins) for netblock in netblocks: print_row(map(lambda i: str(netblock.get(i) or ''), items), margins) if len(items) > 1: print_row(tuple(map(lambda m: '-' * m, margins)), margins) print_row(tuple(map(lambda i: str(summary.get(i) or ''), items)), margins) if db: db.close()
def DriveKernel(env: _env.OpenCLEnvironment, src: str, inputs: np.array, gsize: typing.Union[typing.Tuple[int, int, int], NDRange], lsize: typing.Union[typing.Tuple[int, int, int], NDRange], timeout: int = -1, optimizations: bool = True, profiling: bool = False, debug: bool = False) -> np.array: """Drive an OpenCL kernel. Executes an OpenCL kernel on the given environment, over the given inputs. Execution is performed in a subprocess. Args: env: The OpenCL environment to run the kernel in. src: The OpenCL kernel source. inputs: The input data to the kernel. optimizations: Whether to enable or disbale OpenCL compiler optimizations. profiling: If true, print OpenCLevent times for data transfers and kernel executions to stderr. timeout: Cancel execution if it has not completed after this many seconds. A value <= 0 means never time out. debug: If true, silence the OpenCL compiler. Returns: A numpy array of the same shape as the inputs, with the values after running the OpenCL kernel. Raises: ValueError: If input types are incorrect. TypeError: If an input is of an incorrect type. LogicError: If the input types do not match OpenCL kernel types. PorcelainError: If the OpenCL subprocess exits with non-zero return code. RuntimeError: If OpenCL program fails to build or run. Examples: A simple kernel which doubles its inputs: >>> src = "kernel void A(global int* a) { a[get_global_id(0)] *= 2; }" >>> inputs = [[1, 2, 3, 4, 5]] >>> DriveKernel(env, src, inputs, gsize=(5,1,1), lsize=(1,1,1)) # doctest: +SKIP array([[ 2, 4, 6, 8, 10]], dtype=int32) """ def Log(*args, **kwargs): """Log a message to stderr.""" if debug: print("[cldrive] ", end="", file=sys.stderr) print(*args, **kwargs, file=sys.stderr) # Assert input types. err.assert_or_raise(isinstance(env, _env.OpenCLEnvironment), ValueError, "env argument is of incorrect type") err.assert_or_raise(isinstance(src, str), ValueError, "source is not a string") # Validate global and local sizes. err.assert_or_raise(len(gsize) == 3, TypeError) err.assert_or_raise(len(lsize) == 3, TypeError) gsize, lsize = NDRange(*gsize), NDRange(*lsize) err.assert_or_raise(gsize.product >= 1, ValueError, f"Scalar global size {gsize.product} must be >= 1") err.assert_or_raise(lsize.product >= 1, ValueError, f"Scalar local size {lsize.product} must be >= 1") err.assert_or_raise( gsize >= lsize, ValueError, f"Global size {gsize} must be larger than local size {lsize}") # Parse args in this process since we want to preserve the sueful exception # type. args = _args.GetKernelArguments(src) # Check that the number of inputs is correct. args_with_inputs = [ i for i, arg in enumerate(args) if not arg.address_space == 'local' ] err.assert_or_raise( len(args_with_inputs) == len(inputs), ValueError, "Kernel expects {} inputs, but {} were provided".format( len(args_with_inputs), len(inputs))) # All inputs must have some length. for i, x in enumerate(inputs): err.assert_or_raise(len(x), ValueError, f"Input {i} has size zero") # Copy inputs into the expected data types. data = np.array( [np.array(d).astype(a.numpy_type) for d, a in zip(inputs, args)]) job = { "env": env, "src": src, "args": args, "data": data, "gsize": gsize, "lsize": lsize, "optimizations": optimizations, "profiling": profiling } with NamedTemporaryFile('rb+', prefix='cldrive-', suffix='.job') as tmp_file: porcelain_job_file = tmp_file.name # Write job file. pickle.dump(job, tmp_file) tmp_file.flush() # Enforce timeout using sigkill. if timeout > 0: cli = ["timeout", "--signal=9", str(int(timeout))] else: cli = [] cli += [sys.executable, __file__, porcelain_job_file] cli_str = " ".join(cli) Log("Porcelain invocation:", cli_str) # Fork and run. process = Popen(cli, stdout=PIPE, stderr=PIPE) stdout, stderr = process.communicate() status = process.returncode if debug: print(stdout.decode('utf-8').strip(), file=sys.stderr) print(stderr.decode('utf-8').strip(), file=sys.stderr) elif profiling: # Print profiling output when not in debug mode. for line in stderr.decode('utf-8').split('\n'): if re.match(r'\[cldrive\] .+ time: [0-9]+\.[0-9]+ ms', line): print(line, file=sys.stderr) Log(f"Porcelain return code: {status}") # Test for non-zero exit codes. The porcelain subprocess catches exceptions # and completes gracefully, so a non-zero return code is indicative of a # more serious problem. # # FIXME: I'm seeing a number of SIGABRT return codes which I can't explain. # However, ignoring them seems to not cause a problem ... if status != 0 and status != -Signals['SIGABRT'].value: # A negative return code means a signal. Try and convert the value into a # signal name. with suppress(ValueError): status = Signals(-status).name if status == "SIGKILL": raise TimeoutError(timeout) else: raise PorcelainError(status) # Read result. tmp_file.seek(0) return_value = pickle.load(tmp_file) outputs = return_value["outputs"] error = return_value["err"] if error: # Porcelain raised an exception, re-raise it. raise error else: return outputs
def _dump(path: Path, thing: str) -> None: path.parent.mkdir(parents=True, exist_ok=True) with NamedTemporaryFile(dir=path.parent, mode="w", delete=False) as fd: fd.write(thing) Path(fd.name).replace(path)
class AbstractWrapper(object): ''' abstract solver wrapper ''' def __init__(self): ''' Constructor ''' #program_name = os.path.basename(sys.argv[0]) program_version = "v%s" % __version__ program_build_date = str(__updated__) program_version_message = "%%(prog)s %s (%s)" % (program_version, program_build_date) program_shortdesc = __import__("__main__").__doc__.split("\n")[1] program_license = '''%s Created by %s on %s. Copyright 2014 - AClib. All rights reserved. Licensed under the GPLv2 http://www.gnu.org/licenses/gpl-2.0.html Distributed on an "AS IS" basis without warranties or conditions of any kind, either express or implied. USAGE ''' % (program_shortdesc, str(__authors__), str(__date__)) self.parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter, add_help=False) self.args = None self.RESULT_MAPPING = {'SUCCESS': "SAT"} self._watcher_file = None self._solver_file = None self._instance = "" self._specifics = "" self._cutoff = 0.0 self._runlength = 0 self._seed = 0 self._exit_code = None self._runsolver = None self._mem_limit = 2048 self._tmp_dir = None self._crashed_if_non_zero_status = True self._subprocesses = [] self._DEBUG = True self._DELAY2KILL = 2 self._ta_status = "EXTERNALKILL" self._ta_runtime = 999999999.0 self._ta_runlength = -1 self._ta_quality = -1 self._ta_exit_code = None self._ta_misc = "" def print_d(self, str_): if self._DEBUG: print(str_) def main(self, argv=None): ''' parse command line''' if argv is None: argv = sys.argv else: sys.argv.extend(argv) try: signal.signal(signal.SIGTERM, signalHandler) signal.signal(signal.SIGQUIT, signalHandler) signal.signal(signal.SIGINT, signalHandler) # Setup argument parser run_group = self.parser.add_argument_group("Run") run_group.add_argument("--runsolver-path", dest="runsolver", default=os.path.join(os.path.join(os.path.dirname(__file__),"runsolver"), "runsolver"), help="path to runsolver binary (if None, the runsolver is deactivated)") run_group.add_argument("--temp-file-dir", dest="tmp_dir", default=".", help="directory for temporary files (relative to -exec-dir in SMAC scenario)") run_group.add_argument("--mem-limit", dest="mem_limit", default=self._mem_limit, type=int, help="memory limit in MB") run_group.add_argument("--internal", dest="internal", default=False, action="store_true", help="skip calling an external target algorithm") run_group = self.parser.add_argument_group("External Callstring Generation and Output Parsing") run_group.add_argument("--ext-callstring", dest="ext_callstring", default=None, help="Command to get call string via external program;" + "your programm gets a file with"+ "first line: instance name,"+ "second line: seed"+ "further lines: paramter name, paramater value;"+ "output: one line with callstring for target algorithm") run_group.add_argument("--ext-parsing", dest="ext_parsing", default=None, help="Command to use an external program to parse the output of your target algorihm;" + "only paramter: name of output file;"+ "output of your progam:"+ "status: SAT|UNSAT|TIMEOUT|CRASHED\n"+ "quality: <integer>\n"+ "misc: <string>") help_group = self.parser.add_argument_group("Help") help_group.add_argument("--help", dest="show_help", action="store_true", help="show this help message") # Process arguments self.args, target_args = self.parser.parse_known_args() args = self.args if args.show_help: self.parser.print_help() self._ta_status = "ABORT" self._ta_misc = "help was requested..." self._exit_code = 1 sys.exit(1) if args.runsolver != "None" and not os.path.isfile(args.runsolver) and not args.internal: self._ta_status = "ABORT" self._ta_misc = "runsolver is missing - should have been at %s." % (args.runsolver) self._exit_code = 1 sys.exit(1) else: self._runsolver = args.runsolver self._mem_limit = args.mem_limit if not os.path.isdir(args.tmp_dir): self._ta_status = "ABORT" self._ta_misc = "temp directory is missing - should have been at %s." % (args.tmp_dir) self._exit_code = 1 sys.exit(1) else: self._tmp_dir = args.tmp_dir if len(target_args) < 5: self._ta_status = "ABORT" self._ta_misc = "some required TA parameters (instance, specifics, cutoff, runlength, seed) missing - was [%s]." % (" ".join(target_args)) self._exit_code = 1 sys.exit(1) config_dict = self.build_parameter_dict(target_args) runargs = { "instance": self._instance, "specifics" : self._specifics, "cutoff" : self._cutoff, "runlength" : self._runlength, "seed" : self._seed } if args.ext_callstring: target_cmd = self.get_command_line_args_ext(runargs=runargs, config=config_dict, ext_call=args.ext_callstring).split(" ") else: target_cmd = self.get_command_line_args(runargs=runargs, config=config_dict).split(" ") if not args.internal: self.call_target(target_cmd) self.read_runsolver_output() if args.ext_parsing: resultMap = self.process_results_ext(self._solver_file, {"exit_code" : self._ta_exit_code}, ext_call=args.ext_parsing) else: resultMap = self.process_results(self._solver_file, {"exit_code" : self._ta_exit_code}) if ('status' in resultMap): self._ta_status = self.RESULT_MAPPING.get(resultMap['status'],resultMap['status']) if ('runtime' in resultMap): self._ta_runtime = resultMap['runtime'] if ('quality' in resultMap): self._ta_quality = resultMap['quality'] if ('misc' in resultMap): self._ta_misc = resultMap['misc'] # if still no status was determined, something went wrong and output files should be kept if self._ta_status is "EXTERNALKILL": self._ta_status = "CRASHED" sys.exit() except (KeyboardInterrupt, SystemExit): self.cleanup() self.print_result_string() if self._ta_exit_code: sys.exit(self._ta_exit_code) elif self._exit_code: sys.exit(self._exit_code) else: sys.exit(0) def build_parameter_dict(self, arg_list): ''' Reads all arguments which were not parsed by ArgumentParser, extracts all meta information and builds a mapping: parameter name -> parameter value Format Assumption: <instance> <specifics> <runtime cutoff> <runlength> <seed> <solver parameters> Args: list of all options not parsed by ArgumentParser ''' self._instance = arg_list[1] self._specifics = arg_list[2] self._cutoff = int(float(arg_list[3]) + 1) # runsolver only rounds down to integer self._runlength = int(arg_list[4]) self._seed = int(arg_list[5]) params = arg_list[6:] if (len(params)/2)*2 != len(params): self._ta_status = "ABORT" self._ta_misc = "target algorithm parameter list MUST have even length - found %d arguments." % (len(params)) self.print_d(" ".join(params)) self._exit_code = 1 sys.exit(1) return dict((name, value) for name, value in zip(params[::2], params[1::2])) def call_target(self, target_cmd): ''' extends the target algorithm command line call with the runsolver and executes it Args: list of target cmd (from getCommandLineArgs) ''' random_id = random.randint(0,1000000) self._watcher_file = NamedTemporaryFile(suffix=".log", prefix="watcher-%d-" %(random_id), dir=self._tmp_dir, delete=False) self._solver_file = NamedTemporaryFile(suffix=".log", prefix="solver-%d-" %(random_id), dir=self._tmp_dir, delete=False) runsolver_cmd = [] if self._runsolver != "None": runsolver_cmd = [self._runsolver, "-M", self._mem_limit, "-C", self._cutoff, "-w", self._watcher_file.name, "-o", self._solver_file.name] runsolver_cmd.extend(target_cmd) #for debugging self.print_d("Calling runsolver. Command-line:") self.print_d(" ".join(map(str,runsolver_cmd))) # run try: if self._runsolver != "None": io = Popen(map(str, runsolver_cmd), shell=False, preexec_fn=os.setpgrp) else: io = Popen(map(str, runsolver_cmd), stdout=self._solver_file, shell=False, preexec_fn=os.setpgrp) self._subprocesses.append(io) io.wait() self._subprocesses.remove(io) if io.stdout: io.stdout.flush() except OSError: self._ta_status = "ABORT" self._ta_misc = "execution failed: %s" % (" ".join(map(str,runsolver_cmd))) self._exit_code = 1 sys.exit(1) self._solver_file.seek(0) def float_regex(self): return '[+-]?\d+(?:\.\d+)?(?:[eE][+-]\d+)?' def read_runsolver_output(self): ''' reads self._watcher_file, extracts runtime and returns if memout or timeout found ''' if self._runsolver == "None": self._ta_exit_code = 0 return self.print_d("Reading runsolver output from %s" % (self._watcher_file.name)) data = self._watcher_file.read() if (re.search('runsolver_max_cpu_time_exceeded', data) or re.search('Maximum CPU time exceeded', data)): self._ta_status = "TIMEOUT" if (re.search('runsolver_max_memory_limit_exceeded', data)): self._ta_status = "TIMEOUT" self._ta_misc = "memory limit was exceeded" cpu_pattern1 = re.compile('runsolver_cputime: (%s)' % (self.float_regex())) cpu_match1 = re.search(cpu_pattern1, data) cpu_pattern2 = re.compile('CPU time \\(s\\): (%s)' % (self.float_regex())) cpu_match2 = re.search(cpu_pattern2, data) if (cpu_match1): self._ta_runtime = float(cpu_match1.group(1)) if (cpu_match2): self._ta_runtime = float(cpu_match2.group(1)) exitcode_pattern = re.compile('Child status: ([0-9]+)') exitcode_match = re.search(exitcode_pattern, data) if (exitcode_match): self._ta_exit_code = int(exitcode_match.group(1)) def print_result_string(self): sys.stdout.write("Result for ParamILS: %s, %s, %s, %s, %s" % (self._ta_status, str(self._ta_runtime), str(self._ta_runlength), str(self._ta_quality), str(self._seed))) if (len(self._ta_misc) > 0): sys.stdout.write(", %s" % (self._ta_misc)) print('') def cleanup(self): ''' cleanup if error occurred or external signal handled ''' if (len(self._subprocesses) > 0): print("killing the target run!") try: for sub in self._subprocesses: #sub.terminate() Popen(["pkill","-TERM", "-P",str(sub.pid)]) self.print_d("Wait %d seconds ..." % (self._DELAY2KILL)) time.sleep(self._DELAY2KILL) if sub.returncode is None: # still running sub.kill() self.print_d("done... If anything in the subprocess tree fork'd a new process group, we may not have caught everything...") self._ta_misc = "forced to exit by signal or keyboard interrupt." self._ta_runtime = self._cutoff except (OSError, KeyboardInterrupt, SystemExit): self._ta_misc = "forced to exit by multiple signals/interrupts." self._ta_runtime = self._cutoff if (self._ta_status is "ABORT" or self._ta_status is "CRASHED"): if (len(self._ta_misc) == 0): self._ta_misc = 'Problem with run. Exit code was %d.' % (self._ta_exit_code) if (self._watcher_file and self._solver_file): self._ta_misc = self._ta_misc + '; Preserving runsolver output at %s - preserving target algorithm output at %s' % (self._watcher_file.name or "<none>", self._solver_file.name or "<none>") try: if (self._watcher_file): self._watcher_file.close() if (self._solver_file): self._solver_file.close() if (self._ta_status is not "ABORT" and self._ta_status is not "CRASHED"): os.remove(self._watcher_file.name) os.remove(self._solver_file.name) except (OSError, KeyboardInterrupt, SystemExit): self._ta_misc = "problems removing temporary files during cleanup." except AttributeError: pass #in internal mode, these files are not generated if self._ta_status is "EXTERNALKILL": self._ta_status = "CRASHED" self._exit_code = 3 def get_command_line_args(self, runargs, config): ''' Returns the command call list containing arguments to execute the implementing subclass' solver. The default implementation delegates to get_command_line_args_ext. If this is not implemented, a NotImplementedError will be raised. Args: runargs: a map of any non-configuration arguments required for the execution of the solver. config: a mapping from parameter name (with prefix) to parameter value. Returns: A command call list to execute a target algorithm. ''' raise NotImplementedError() def get_command_line_args_ext(self, runargs, config, ext_call): ''' When production of the target algorithm is done from a source other than python, override this method to return a command call list to execute whatever you need to produce the command line. Args: runargs: a map of any non-configuration arguments required for the execution of the solver. config: a mapping from parameter name (with prefix) to parameter value. ext_call: string to call external program to get callstring of target algorithm Returns: A command call list to execute the command producing a single line of output containing the solver command string ''' callstring_in = NamedTemporaryFile(suffix=".csv", prefix="callstring", dir=self._tmp_dir, delete=False) callstring_in.write("%s\n" %(runargs["instance"])) callstring_in.write("%d\n" %(runargs["seed"])) for name,value in config.items(): callstring_in.write("%s,%s\n" %(name,value)) callstring_in.flush() cmd = ext_call.split(" ") cmd.append(callstring_in.name) self.print_d(" ".join(cmd)) try: io = Popen(cmd, shell=False, preexec_fn=os.setpgrp, stdout=PIPE) self._subprocesses.append(io) out_, _ = io.communicate() self._subprocesses.remove(io) except OSError: self._ta_misc = "failed to run external program for output parsing : %s" %(" ".join(cmd)) self._ta_runtime = self._cutoff self._exit_code = 2 sys.exit(2) if not out_ : self._ta_misc = "external program for output parsing yielded empty output: %s" %(" ".join(cmd)) self._ta_runtime = self._cutoff self._exit_code = 2 sys.exit(2) callstring_in.close() os.remove(callstring_in.name) return out_.strip("\n") def process_results(self, filepointer, out_args): ''' Parse a results file to extract the run's status (SUCCESS/CRASHED/etc) and other optional results. Args: filepointer: a pointer to the file containing the solver execution standard out. exit_code : exit code of target algorithm Returns: A map containing the standard AClib run results. The current standard result map as of AClib 2.06 is: { "status" : <"SAT"/"UNSAT"/"TIMEOUT"/"CRASHED"/"ABORT">, "runtime" : <runtime of target algrithm>, "quality" : <a domain specific measure of the quality of the solution [optional]>, "misc" : <a (comma-less) string that will be associated with the run [optional]> } ATTENTION: The return values will overwrite the measured results of the runsolver (if runsolver was used). ''' raise NotImplementedError() def process_results_ext(self, filepointer, out_args, ext_call): ''' Args: filepointer: a pointer to the file containing the solver execution standard out. exit_code : exit code of target algorithm Returns: A map containing the standard AClib run results. The current standard result map as of AClib 2.06 is: { "status" : <"SAT"/"UNSAT"/"TIMEOUT"/"CRASHED"/"ABORT">, "quality" : <a domain specific measure of the quality of the solution [optional]>, "misc" : <a (comma-less) string that will be associated with the run [optional]> } ''' cmd = ext_call.split(" ") cmd.append(filepointer.name) self.print_d(" ".join(cmd)) try: io = Popen(cmd, shell=False, preexec_fn=os.setpgrp, stdout=PIPE) self._subprocesses.append(io) out_, _ = io.communicate() self._subprocesses.remove(io) except OSError: self._ta_misc = "failed to run external program for output parsing" self._ta_runtime = self._cutoff self._exit_code = 2 sys.exit(2) result_map = {} for line in out_.split("\n"): if line.startswith("status:"): result_map["status"] = line.split(":")[1].strip(" ") elif line.startswith("quality:"): result_map["quality"] = line.split(":")[1].strip(" ") elif line.startswith("misc:"): result_map["misc"] = line.split(":")[1] return result_map
def export_range_matrix_table_entry_field_p100(): with NamedTemporaryFile() as f: mt = hl.utils.range_matrix_table(n_rows=1_000_000, n_cols=10, n_partitions=100) mt = mt.annotate_entries(x=mt.col_idx + mt.row_idx) mt.x.export(f.name)
def get_command_line_args_ext(self, runargs, config, ext_call): ''' When production of the target algorithm is done from a source other than python, override this method to return a command call list to execute whatever you need to produce the command line. Args: runargs: a map of any non-configuration arguments required for the execution of the solver. config: a mapping from parameter name (with prefix) to parameter value. ext_call: string to call external program to get callstring of target algorithm Returns: A command call list to execute the command producing a single line of output containing the solver command string ''' callstring_in = NamedTemporaryFile(suffix=".csv", prefix="callstring", dir=self._tmp_dir, delete=False) callstring_in.write("%s\n" %(runargs["instance"])) callstring_in.write("%d\n" %(runargs["seed"])) for name,value in config.items(): callstring_in.write("%s,%s\n" %(name,value)) callstring_in.flush() cmd = ext_call.split(" ") cmd.append(callstring_in.name) self.print_d(" ".join(cmd)) try: io = Popen(cmd, shell=False, preexec_fn=os.setpgrp, stdout=PIPE) self._subprocesses.append(io) out_, _ = io.communicate() self._subprocesses.remove(io) except OSError: self._ta_misc = "failed to run external program for output parsing : %s" %(" ".join(cmd)) self._ta_runtime = self._cutoff self._exit_code = 2 sys.exit(2) if not out_ : self._ta_misc = "external program for output parsing yielded empty output: %s" %(" ".join(cmd)) self._ta_runtime = self._cutoff self._exit_code = 2 sys.exit(2) callstring_in.close() os.remove(callstring_in.name) return out_.strip("\n")
def applyRules(outputLocation,output,Juniperip,Juniperusr,Juniperpwd): numnewrules = 0 # Set up the SSH client to connect to the Juniper firewall con_pre=paramiko.SSHClient() con_pre.set_missing_host_key_policy(paramiko.AutoAddPolicy()) con_pre.connect(Juniperip, username=Juniperusr, password=Juniperpwd,look_for_keys=False, allow_agent=False) # Connect to the Juniper firewall via SSH con = con_pre.invoke_shell() # Enter cli con.send('cli\n') time.sleep(.2) # Enter config con.send('configure\n') time.sleep(.2) # Open the rule file f = open(output, 'rb') reader = csv.reader(f, delimiter=',', quotechar='"') # Open the rule file and create a temporary file to store the applied firewall rules tf = NamedTemporaryFile(delete=False) writer = csv.writer(tf, delimiter=',', quotechar='"') # Loop through each row of the rule file to get IP addresses for row in reader: # Check if the rule is new if row[3] == 'new': time.sleep(.2) # If the rule is for an IP address, format the filter appropriatly and apply it to the firewall's configuration if row[0] == "ip": # Apply filter con.send('set firewall family inet filter MSISAC_Anomali term ' + today2 + ' from destination-address ' + row[1] + '\n') time.sleep(.2) numnewrules += 1 # Mark the rule as applied within the rule file to avoid duplicate entries row[3] = 'applied' if row[0] == "domain": # Apply filter con.send('set firewall family inet filter MSISAC_Anomali term ' + today2 + ' from destination-address ' + row[1] + '\n') time.sleep(.2) numnewrules += 1 # Mark the rule as applied within the rule file to avoid duplicate entries row[3] = 'applied' #Apply Protocols con.send('set firewall family inet filter MSISAC_Anomali term ' + today2 + ' from protocol icmp' + '\n') time.sleep(.2) con.send('set firewall family inet filter MSISAC_Anomali term ' + today2 + ' from protocol udp' + '\n') time.sleep(.2) con.send('set firewall family inet filter MSISAC_Anomali term ' + today2 + ' from protocol tcp' + '\n') time.sleep(.2) #Apply Discard con.send('set firewall family inet filter MSISAC_Anomali term ' + today2 + ' then discard' + '\n') time.sleep(.2) #Apply Counter con.send('set firewall family inet filter MSISAC_Anomali term ' + today2 + ' then count MSISAC_Anomali-Counter' + '\n') time.sleep(.2) #Commit Filter con.send('commit' + '\n') time.sleep(.2) #Set Interfaces con.send('set interfaces ' + Juniperif + ' unit 0 family inet filter MSISAC_Anomali' + '\n') time.sleep(.2) #Commit Filter to Interfaces con.send('commit' + '\n') time.sleep(.2) # Write the rules to the temporary file writer.writerow(row) # Replace the old rule file with the temporary file that has the updated 'applied' rules shutil.move(tf.name, output) f.close() return numnewrules