def get_subtitle(url, path): in_data = urllib2.urlopen(url) temp_file = NamedTemporaryFile() temp_file.write(in_data.read()) in_data.close() temp_file.seek(0) if is_zipfile(temp_file.name): zip_file = ZipFile(temp_file) for name in zip_file.namelist(): # don't unzip stub __MACOSX folders if '.srt' in name and '__MACOSX' not in name: logger.info(' '.join(['Unpacking zipped subtitle', name, 'to', os.path.dirname(path)])) zip_file.extract(name, os.path.dirname(path)) zip_file.close() elif is_rarfile(temp_file.name): rar_path = path + '.rar' logger.info('Saving rared subtitle as %s' % rar_path) with open(rar_path, 'w') as out_file: out_file.write(temp_file.read()) try: import subprocess #extract all .srt in the rared file ret_code = subprocess.call(['unrar', 'e', '-n*srt', rar_path]) if ret_code == 0: logger.info('Unpacking rared subtitle to %s' % os.path.dirname(path)) os.remove(rar_path) except OSError: logger.info('Unpacking rared subtitle failed.' 'Please, install unrar to automate this step.') temp_file.close()
def _get_tmp_file(self): ''' Creates a tmp file with the file data ''' data = self.request.files['xml_file'][0]['body'] tmp_file = NamedTemporaryFile(delete=False) tmp_file.write(data) tmp_file.close() return tmp_file.name
def main(): logging.basicConfig(format='%(message)s', level=logging.INFO) args = parse_args() from_date = args.from_date to_date = args.to_date s3_client = S3Client.connect_with_profile(args.aws_profile) s3_bucket = args.s3_bucket s3_key = "{}_{}.logs.json".format(from_date.isoformat(), to_date.isoformat()) sumo_access_id = args.sumo_access_id or os.environ["SUMOLOGIC_ACCESS_ID"] sumo_access_key = args.sumo_access_key or os.environ["SUMOLOGIC_ACCESS_KEY"] sumo_logic_client = SumoLogicClient(sumo_access_id, sumo_access_key) logging.info( "Searching for request logs in {} in range {} to {}".format( args.environment, from_date.isoformat(), to_date.isoformat())) request_logs = sumo_logic_client.search( sumo_logic_query(args.environment), start_dt=from_date, end_dt=to_date, fields=REQUEST_FIELDS) tmp_file = NamedTemporaryFile(mode='w', encoding='utf-8', delete=False) logging.debug("Writing out logs to temporary file {}".format(tmp_file.name)) try: for request in request_logs: tmp_file.write(json.dumps(request, default=json_serialize) + '\n') s3_client.upload_from_file(s3_bucket, s3_key, tmp_file.name) finally: logging.debug("Removing temporary file {}".format(tmp_file.name)) os.remove(tmp_file.name)
def test_main(self): xml = """<record> <datafield tag="999" ind1="C" ind2="5"> <subfield code="s">Test Journal Name,100,10</subfield> </datafield> </record>""" xml_temp_file = NamedTemporaryFile(dir=CFG_TMPDIR) xml_temp_file.write(xml) xml_temp_file.flush() kb = "TEST JOURNAL NAME---Converted" kb_temp_file = NamedTemporaryFile(dir=CFG_TMPDIR) kb_temp_file.write(kb) kb_temp_file.flush() dest_temp_fd, dest_temp_path = mkstemp(dir=CFG_TMPDIR) try: os.close(dest_temp_fd) process = subprocess.Popen([self.bin_path, xml_temp_file.name, '--kb', kb_temp_file.name, '-o', dest_temp_path], stderr=subprocess.PIPE, stdout=subprocess.PIPE) process.wait() transformed_xml = open(dest_temp_path).read() self.assertXmlEqual(transformed_xml, """<?xml version="1.0" encoding="UTF-8"?> <collection xmlns="http://www.loc.gov/MARC21/slim"> <record><datafield ind1="C" ind2="5" tag="999"><subfield code="s">Converted,100,10</subfield></datafield></record> </collection>""") finally: os.unlink(dest_temp_path)
def run(self): active_view = self.window.active_view() text = "\n\n".join(getSelectedText(active_view)).strip() tf = NamedTemporaryFile(mode="w", delete=False) try: tf.write(text) tf.close() res = subprocess.check_output(["m4", tf.name], stderr=subprocess.STDOUT, cwd=os.path.dirname(os.path.abspath(active_view.file_name()))) res = res.decode('utf-8').replace('\r', '').strip() panel_name = "m4expand.results" panel = self.window.create_output_panel(panel_name) self.window.run_command("show_panel", {"panel": "output." + panel_name}) panel.set_read_only(False) panel.set_syntax_file(active_view.settings().get("syntax")) panel.run_command("append", {"characters": res}) panel.set_read_only(True) except Exception as e: print("M4Expand - An error occurred: ", e) finally: os.unlink(tf.name)
def testLogfile(self): """Test logging into a logfile""" f = NamedTemporaryFile(delete=False) filename = f.name try: set_log_level("error") # avoid using the console logger f.write(":-P\n") f.close() start_logfile(f.name, "devinfo") log = getLogger("prosoda.test.integration.test_logger") log.debug("Should not be in logfile! :-( ") log.info("Should be in logfile :-) ") log.devinfo("Should be in logfile :-) ") log.warning("Should really be in logfile :-D ") stop_logfile(f.name) contents = file(f.name).read() self.assertNotIn(":-(", contents) self.assertNotIn(":-P", contents) self.assertIn(":-)", contents) self.assertIn(":-D", contents) # Make sure no colour codes are leaked into the logfile self.assertNotIn("\033", contents) finally: set_log_level("debug") unlink(filename)
def compile_inline(self,data,ext): """ Compile inline css. Have to compile to a file, because some css compilers may not output to stdout, but we know they all output to a file. It's a little hackish, but you shouldn't be compiling in production anyway, right? """ compiler = settings.COMPILER_FORMATS[ext] try: bin = compiler['binary_path'] except: raise Exception("Path to CSS compiler must be included in COMPILER_FORMATS") tmp_file = NamedTemporaryFile(mode='w',suffix=ext) tmp_file.write(dedent(data)) tmp_file.flush() path, ext = os.path.splitext(tmp_file.name) tmp_css = ''.join((path,'.css')) self.compile(path,compiler) data = open(tmp_css,'r').read() # cleanup tmp_file.close() os.remove(tmp_css) return data
def tmpfile(stream, mode=None): """Context manager that writes a :class:`Stream` object to a named temporary file and yield it's filename. Cleanup deletes from the temporary file from disk. Args: stream (Stream): Stream object to write to disk as temporary file. mode (int, optional): File mode to set on temporary file. Returns: str: Temporoary file name """ tmp = NamedTemporaryFile(delete=False) if mode is not None: oldmask = os.umask(0) try: os.chmod(tmp.name, mode) finally: os.umask(oldmask) for data in stream: tmp.write(to_bytes(data)) tmp.close() yield tmp.name os.remove(tmp.name)
def test_safe_md5(self): """Make sure we have the expected md5 with varied input types This method is ported from PyCogent (http://www.pycogent.org). PyCogent is a GPL project, but we obtained permission from the authors of this method to port it to the BIOM Format project (and keep it under BIOM's BSD license). """ exp = 'd3b07384d113edec49eaa6238ad5ff00' tmp_f = NamedTemporaryFile( mode='w', prefix='test_safe_md5', suffix='txt') tmp_f.write('foo\n') tmp_f.flush() obs = safe_md5(open(tmp_f.name, 'U')) self.assertEqual(obs, exp) obs = safe_md5(['foo\n']) self.assertEqual(obs, exp) # unsupported type raises TypeError self.assertRaises(TypeError, safe_md5, 42)
def run_solver(self, conflicts, election, deletion_handler, outfile=None): if not conflicts: return [], 0 self.deletion_handler = deletion_handler instance = self.generate_instance(conflicts, election) f = NamedTemporaryFile(delete=False) f.write(instance.encode(code)) f.close() process = Popen([self.cmd, f.name], stdout=PIPE) out, err = process.communicate() conflict_variables, optimum = self.parse_instance(out) if outfile: candidates = election[0] votes = election[1] votecounts = election[2] votemap = self.delete_votes(votes, votecounts, conflict_variables) votesum = sum(votemap.values()) write_map(candidates, votesum, votemap, open(outfile, "w")) remove(f.name) return conflict_variables, optimum
def odt_subreport(name=None, obj=None): if not aeroo_ooo: return _("Error! Subreports not available!") report_xml_ids = ir_obj.search(cr, uid, [('report_name', '=', name)], context=context) if report_xml_ids: service = netsvc.Service._services['report.%s' % name] report_xml = ir_obj.browse(cr, uid, report_xml_ids[0], context=context) data = {'model': obj._table_name, 'id': obj.id, 'report_type': 'aeroo', 'in_format': 'oo-odt'} ### Get new printing object ### sub_aeroo_print = AerooPrint() service.active_prints[sub_aeroo_print.id] = sub_aeroo_print context['print_id'] = sub_aeroo_print.id ############################### sub_aeroo_print.start_time = time.time() report, output = service.create_aeroo_report(cr, uid, \ [obj.id], data, report_xml, context=context, output='odt') # change for OpenERP 6.0 - Service class usage ### Delete printing object ### AerooPrint.print_ids.remove(sub_aeroo_print.id) del service.active_prints[sub_aeroo_print.id] ############################## temp_file = NamedTemporaryFile(suffix='.odt', prefix='aeroo-report-', delete=False) try: temp_file.write(report) finally: temp_file.close() # self.oo_subreports[print_id].append(temp_file.name) # aeroo_print.subreports.append(temp_file.name) self.active_prints[aeroo_print.id].subreports.append(temp_file.name) return "<insert_doc('%s')>" % temp_file.name return None
def get_tempfile(self, **kwargs): kwargs.setdefault('suffix', '.vrt') tempfile = NamedTemporaryFile(**kwargs) tempfile.write(self.content) tempfile.flush() tempfile.seek(0) return tempfile
def render_to_temporary_file(self, template_name, mode='w+b', bufsize=-1, suffix='.html', prefix='tmp', dir=None, delete=True): template = self.resolve_template(template_name) context = self.resolve_context(self.context_data) content = smart_str(template.render(context)) content = make_absolute_paths(content) try: tempfile = NamedTemporaryFile(mode=mode, bufsize=bufsize, suffix=suffix, prefix=prefix, dir=dir, delete=delete) except TypeError: tempfile = NamedTemporaryFile(mode=mode, buffering=bufsize, suffix=suffix, prefix=prefix, dir=dir, delete=delete) try: tempfile.write(content) tempfile.flush() return tempfile except TypeError: tempfile.write(bytes(content, 'UTF-8')) tempfile.flush() return tempfile except: # Clean-up tempfile if an Exception is raised. tempfile.close() raise
def create_temp_file(self, edid_binary): edid_file = NamedTemporaryFile(delete=False) edid_file.write(edid_binary) edid_file.flush() edid_file.seek(0) return edid_file
def _write_local_schema_file(self, cursor): """ Takes a cursor, and writes the BigQuery schema for the results to a local file system. :return: A dictionary where key is a filename to be used as an object name in GCS, and values are file handles to local files that contains the BigQuery schema fields in .json format. """ schema = [] for field in cursor.description: # See PEP 249 for details about the description tuple. field_name = field[0] field_type = self.type_map(field[1]) field_mode = 'REPEATED' if field[1] in (1009, 1005, 1007, 1016) else 'NULLABLE' schema.append({ 'name': field_name, 'type': field_type, 'mode': field_mode, }) self.log.info('Using schema for %s: %s', self.schema_filename, schema) tmp_schema_file_handle = NamedTemporaryFile(delete=True) s = json.dumps(schema, sort_keys=True) if PY3: s = s.encode('utf-8') tmp_schema_file_handle.write(s) return {self.schema_filename: tmp_schema_file_handle}
def test_pipeline_run(): 'It tests that the pipeline runs ok' pipeline = 'sanger_with_qual' fhand_adaptors = NamedTemporaryFile() fhand_adaptors.write(ADAPTORS) fhand_adaptors.flush() arabidopsis_genes = 'arabidopsis_genes+' univec = os.path.join(TEST_DATA_DIR, 'blast', arabidopsis_genes) configuration = {'remove_vectors_blastdb': {'vectors': univec}, 'remove_adaptors': {'adaptors': fhand_adaptors.name}} seq_fhand = open(os.path.join(TEST_DATA_DIR, 'seq.fasta'), 'r') qual_fhand = open(os.path.join(TEST_DATA_DIR, 'qual.fasta'), 'r') seq_iter = seqs_in_file(seq_fhand, qual_fhand) filtered_seq_iter = _pipeline_builder(pipeline, seq_iter, configuration) seq_list = list(filtered_seq_iter) assert 'CGAtcgggggg' in str(seq_list[0].seq) assert len(seq_list) == 6
def _generate_training_files(self): """Returns a tuple of file objects suitable for passing to the RdpTrainer application controller. """ tmp_dir = get_qiime_temp_dir() training_set = RdpTrainingSet() reference_seqs_file = open(self.Params['reference_sequences_fp'], 'U') id_to_taxonomy_file = open(self.Params['id_to_taxonomy_fp'], 'U') for seq_id, seq in MinimalFastaParser(reference_seqs_file): training_set.add_sequence(seq_id, seq) for line in id_to_taxonomy_file: seq_id, lineage_str = map(strip, line.split('\t')) training_set.add_lineage(seq_id, lineage_str) training_set.dereplicate_taxa() rdp_taxonomy_file = NamedTemporaryFile( prefix='RdpTaxonAssigner_taxonomy_', suffix='.txt', dir=tmp_dir) rdp_taxonomy_file.write(training_set.get_rdp_taxonomy()) rdp_taxonomy_file.seek(0) rdp_training_seqs_file = NamedTemporaryFile( prefix='RdpTaxonAssigner_training_seqs_', suffix='.fasta', dir=tmp_dir) for rdp_id, seq in training_set.get_training_seqs(): rdp_training_seqs_file.write('>%s\n%s\n' % (rdp_id, seq)) rdp_training_seqs_file.seek(0) self._training_set = training_set return rdp_taxonomy_file, rdp_training_seqs_file
def test_seq_pipeline_parallel_run(self): 'It tests that the pipeline runs ok' pipeline = 'sanger_without_qual' fhand_adaptors = NamedTemporaryFile() fhand_adaptors.write(ADAPTORS) fhand_adaptors.flush() arabidopsis_genes = 'arabidopsis_genes+' univec = os.path.join(TEST_DATA_DIR, 'blast', arabidopsis_genes) configuration = {'remove_vectors': {'vectors': univec}, 'remove_adaptors': {'adaptors': fhand_adaptors.name}} in_fhands = {} in_fhands['in_seq'] = open(os.path.join(TEST_DATA_DIR, 'seq.fasta'), 'r') out_fhand = NamedTemporaryFile() writer = SequenceWriter(out_fhand, file_format='fasta') writers = {'seq': writer} seq_pipeline_runner(pipeline, configuration, in_fhands, processes=4, writers=writers) out_fhand = open(out_fhand.name, 'r') result_seq = out_fhand.read() assert result_seq.count('>') == 6 #are we keeping the description? assert 'mdust' in result_seq
def test_seq_pipeline_parallel_run_with_fasta_qual(self): 'The pipeline runs in parallel with fasta and qual' pipeline = 'sanger_with_qual' fhand_adaptors = NamedTemporaryFile() fhand_adaptors.write(ADAPTORS) fhand_adaptors.flush() arabidopsis_genes = 'arabidopsis_genes+' univec = os.path.join(TEST_DATA_DIR, 'blast', arabidopsis_genes) configuration = {'remove_vectors': {'vectors': univec}, 'remove_adaptors': {'adaptors': fhand_adaptors.name}} seq1 = create_random_seqwithquality(500, qual_range=50) seq2 = create_random_seqwithquality(500, qual_range=51) seq3 = create_random_seqwithquality(500, qual_range=52) seqs = [seq1, seq2, seq3] inseq_fhand, inqual_fhand = create_temp_seq_file(seqs, format='qual') in_fhands = {} in_fhands['in_seq'] = open(inseq_fhand.name) in_fhands['in_qual'] = open(inqual_fhand.name) outseq_fhand = NamedTemporaryFile() outqual_fhand = NamedTemporaryFile() writer = SequenceWriter(outseq_fhand, qual_fhand=outqual_fhand, file_format='fasta') writers = {'seq': writer} seq_pipeline_runner(pipeline, configuration, in_fhands, processes=4, writers=writers) out_fhand = open(outseq_fhand.name, 'r') result_seq = out_fhand.read() assert result_seq.count('>') == 3
def write_temp_file(data): # create a temp file for use as a config file. This should get cleaned # up magically at the end of the run. fid = NamedTemporaryFile(mode='w+b', suffix='.tmp') fid.write(data) fid.seek(0) return fid
def test_toy_corpus(): keats = ('She dwells with Beauty - Beauty that must die;\n\n' 'And Joy, whose hand is ever at his lips\n\n' 'Bidding adieu; and aching Pleasure nigh,\n\n' 'Turning to poison while the bee-mouth sips:\n\n' 'Ay, in the very temple of Delight\n\n' 'Veil\'d Melancholy has her sovran shrine,\n\n' 'Though seen of none save him whose strenuous tongue\n\n' 'Can burst Joy\'s grape against his palate fine;\n\n' 'His soul shall taste the sadness of her might,\n\n' 'And be among her cloudy trophies hung.') assert toy_corpus(keats) assert toy_corpus(keats, nltk_stop=True) assert toy_corpus(keats, stop_freq=1) assert toy_corpus(keats, add_stop=['and', 'with']) assert toy_corpus(keats, nltk_stop=True, stop_freq=1, add_stop=['ay']) import os from tempfile import NamedTemporaryFile as NFT tmp = NFT(delete=False) tmp.write(keats) tmp.close() c = toy_corpus(tmp.name, is_filename=True, nltk_stop=True, add_stop=['ay']) assert c os.remove(tmp.name) return c
def _write_local_schema_file(self, cursor): """ Takes a cursor, and writes the BigQuery schema for the results to a local file system. :return: A dictionary where key is a filename to be used as an object name in GCS, and values are file handles to local files that contains the BigQuery schema fields in .json format. """ schema = [] for field in cursor.description: # See PEP 249 for details about the description tuple. field_name = field[0] field_type = self.type_map(field[1]) # Always allow TIMESTAMP to be nullable. MySQLdb returns None types # for required fields because some MySQL timestamps can't be # represented by Python's datetime (e.g. 0000-00-00 00:00:00). field_mode = 'NULLABLE' if field[6] or field_type == 'TIMESTAMP' else 'REQUIRED' schema.append({ 'name': field_name, 'type': field_type, 'mode': field_mode, }) self.log.info('Using schema for %s: %s', self.schema_filename, schema) tmp_schema_file_handle = NamedTemporaryFile(delete=True) s = json.dumps(schema, tmp_schema_file_handle) if PY3: s = s.encode('utf-8') tmp_schema_file_handle.write(s) return {self.schema_filename: tmp_schema_file_handle}
def get_logs(self): """ Build the logs entry for the metadata 'output' section :return: list, Output instances """ # Collect logs from server kwargs = {} if self.namespace is not None: kwargs['namespace'] = self.namespace logs = self.osbs.get_build_logs(self.build_id, **kwargs) # Deleted once closed logfile = NamedTemporaryFile(prefix=self.build_id, suffix=".log", mode='w') logfile.write(logs) logfile.flush() docker_logs = NamedTemporaryFile(prefix="docker-%s" % self.build_id, suffix=".log", mode='w') docker_logs.write("\n".join(self.workflow.build_logs)) docker_logs.flush() return [Output(file=docker_logs, metadata=self.get_output_metadata(docker_logs.name, "build.log")), Output(file=logfile, metadata=self.get_output_metadata(logfile.name, "openshift-final.log"))]
def testFileAnnotationSpeed(author_testimg_generated, gatewaywrapper): """ Tests speed of loading file annotations. See PR: 4176 """ try: f = NamedTemporaryFile() f.write("testFileAnnotationSpeed text") ns = TESTANN_NS image = author_testimg_generated # use the same file to create many file annotations for i in range(20): fileAnn = gatewaywrapper.gateway.createFileAnnfromLocalFile(f.name, mimetype="text/plain", ns=ns) image.linkAnnotation(fileAnn) finally: f.close() now = time.time() for ann in image.listAnnotations(): if ann._obj.__class__ == omero.model.FileAnnotationI: # mimmic behaviour of templates which call multiple times print ann.getId() print ann.getFileName() print ann.getFileName() print ann.getFileSize() print ann.getFileSize() print time.time() - now
def SubmitPlay(request): import jinja2 from tempfile import NamedTemporaryFile import os html = '' inventory = """ [current] {{ public_ip_address }} """ # for (name,value) in request.GET: # if name=='Servers': # html=html+str(value+'\n') html=request.GET['Severs'] inventory_template = jinja2.Template(inventory) rendered_inventory = inventory_template.render({ 'public_ip_address': html # and the rest of our variables }) # Create a temporary file and write the template string to it hosts = NamedTemporaryFile(delete=False) hosts.write(rendered_inventory) hosts.close() print(hosts.name) import ansiblepythonapi as myPlay args=['/home/ec2-user/playss/AnsiblePlus/test.yml'] # args.append('-i') # args.append(hosts.name) message=myPlay.main(args) objects=[] for runner_results in myPlay.message: values=[] for (host, value) in runner_results.get('dark', {}).iteritems(): try: values.append(host) values.append(value['failed']) values.append(value['msg']) objects.append(values) except: pass for (host, value) in runner_results.get('contacted', {}).iteritems(): try: values.append(host) values.append(value['failed']) values.append(value['msg']) objects.append(values) except: pass # for msg in pb.stats.output(): context=Context({'Summary':objects}) return render(request, 'AnsibleResponce.html',context)
def run_via_pbs(args, pbs): assert(pbs in ('condor',)) # for now # TODO: RF to support multiple backends, parameters, etc, for now -- just condor, no options f = NamedTemporaryFile('w', prefix='datalad-%s-' % pbs, suffix='.submit', delete=False) try: pwd = getpwd() logs = f.name.replace('.submit', '.log') exe = args[0] # TODO: we might need better way to join them, escaping spaces etc. There must be a stock helper #exe_args = ' '.join(map(repr, args[1:])) if len(args) > 1 else '' exe_args = ' '.join(args[1:]) if len(args) > 1 else '' f.write("""\ Executable = %(exe)s Initialdir = %(pwd)s Output = %(logs)s Error = %(logs)s getenv = True arguments = %(exe_args)s queue """ % locals()) f.close() Runner().run(['condor_submit', f.name]) lgr.info("Scheduled execution via %s. Logs will be stored under %s" % (pbs, logs)) finally: os.unlink(f.name)
def hmmscan(fasta, database_path, ncpus=10): F = NamedTemporaryFile() F.write(fasta) F.flush() OUT = NamedTemporaryFile() cmd = '%s --cpu %s -o /dev/null -Z 190000 --tblout %s %s %s' %(HMMSCAN, ncpus, OUT.name, database_path, F.name) #print cmd sts = subprocess.call(cmd, shell=True) byquery = defaultdict(list) if sts == 0: for line in OUT: #['#', '---', 'full', 'sequence', '----', '---', 'best', '1', 'domain', '----', '---', 'domain', 'number', 'estimation', '----'] #['#', 'target', 'name', 'accession', 'query', 'name', 'accession', 'E-value', 'score', 'bias', 'E-value', 'score', 'bias', 'exp', 'reg', 'clu', 'ov', 'env', 'dom', 'rep', 'inc', 'description', 'of', 'target'] #['#-------------------', '----------', '--------------------', '----------', '---------', '------', '-----', '---------', '------', '-----', '---', '---', '---', '---', '---', '---', '---', '---', '---------------------'] #['delNOG20504', '-', '553220', '-', '1.3e-116', '382.9', '6.2', '3.4e-116', '381.6', '6.2', '1.6', '1', '1', '0', '1', '1', '1', '1', '-'] if line.startswith('#'): continue fields = line.split() # output is not tab delimited! Should I trust this split? hit, _, query, _ , evalue, score, bias, devalue, dscore, dbias = fields[0:10] evalue, score, bias, devalue, dscore, dbias = map(float, [evalue, score, bias, devalue, dscore, dbias]) byquery[query].append([hit, evalue, score]) OUT.close() F.close() return byquery
def _write_local_data_files(self, cursor): """ Takes a cursor, and writes results to a local file. :return: A dictionary where keys are filenames to be used as object names in GCS, and values are file handles to local files that contain the data for the GCS objects. """ schema = list(map(lambda schema_tuple: schema_tuple[0], cursor.description)) file_no = 0 tmp_file_handle = NamedTemporaryFile(delete=True) tmp_file_handles = {self.filename.format(file_no): tmp_file_handle} for row in cursor: # Convert datetime objects to utc seconds, and decimals to floats row = map(self.convert_types, row) row_dict = dict(zip(schema, row)) # TODO validate that row isn't > 2MB. BQ enforces a hard row size of 2MB. s = json.dumps(row_dict) if PY3: s = s.encode('utf-8') tmp_file_handle.write(s) # Append newline to make dumps BigQuery compatible. tmp_file_handle.write(b'\n') # Stop if the file exceeds the file size limit. if tmp_file_handle.tell() >= self.approx_max_file_size_bytes: file_no += 1 tmp_file_handle = NamedTemporaryFile(delete=True) tmp_file_handles[self.filename.format(file_no)] = tmp_file_handle return tmp_file_handles
def get_splice_score(a, s_type=5): if s_type not in [3,5]: raise Exception("Invalid splice type {}, should be 3 or 5".format(s_type)) maxent = config.maxentpath if not maxent: raise Exception("Please provide path to the score5.pl and score3.pl maxent scripts in config file") tmp = NamedTemporaryFile() for name,seq in a: tmp.write(">{}\n{}\n".format(name,seq)) tmp.flush() cmd = "perl score{}.pl {}".format(s_type, tmp.name) p = sp.Popen(cmd, shell=True, cwd=maxent, stdout=sp.PIPE) score = 0 for line in p.stdout.readlines(): vals = line.strip().split("\t") if len(vals) > 1: try: score += float(vals[-1]) except ValueError: logger.error("valueError, skipping: {}".format(vals)) except: logger.error("Something unexpected happened") return score
def _build_and_catch_errors(self, build_func, options_bytes, source=None): try: return build_func() except _cl.RuntimeError as e: msg = e.what if options_bytes: msg = msg + "\n(options: %s)" % options_bytes.decode("utf-8") if source is not None: from tempfile import NamedTemporaryFile srcfile = NamedTemporaryFile(mode="wt", delete=False, suffix=".cl") try: srcfile.write(source) finally: srcfile.close() msg = msg + "\n(source saved as %s)" % srcfile.name code = e.code routine = e.routine err = _cl.RuntimeError( _cl.Error._ErrorRecord( msg=msg, code=code, routine=routine)) # Python 3.2 outputs the whole list of currently active exceptions # This serves to remove one (redundant) level from that nesting. raise err
class GitIndexTests(unittest.TestCase): def setUp(self): self.dir = TemporaryDirectory() self.file = NamedTemporaryFile(dir=self.dir.name, delete=False) self.filename = path.basename(self.file.name) self.author = Signature('QuitStoreTest', '*****@*****.**') self.comitter = Signature('QuitStoreTest', '*****@*****.**') self.repo = quit.git.Repository(self.dir.name, create=True) def tearDown(self): self.file = None self.filename = None self.dir.cleanup() self.dir = None self.reop = None def addfile(self): """Create a repository and add a file to the git index.""" # Write to file self.file.write(b'First Line\n') self.file.read() # Add file to index repo = Repository(self.dir.name) index = repo.index index.read() index.add(self.filename) index.write() self.repo = quit.git.Repository(self.dir.name) def createcommit(self): """Prepare a git repository with one existing commit. Create a directory, initialize a git Repository, add and commit a file. Returns: A list containing the directory and file """ self.addfile() # Create commit repo = Repository(self.dir.name) index = repo.index index.read() tree = index.write_tree() message = "First commit of temporary test repo" repo.create_commit('HEAD', self.author, self.comitter, message, tree, []) self.repo = quit.git.Repository(self.dir.name) def testIndexSetRevision(self): self.createcommit() reviosions = self.repo.revisions() index = quit.git.Index(self.repo) index.set_revision(reviosions[0].id) with self.assertRaises(Exception) as context: index.set_revision('not.existing.revision') def testIndexAddFile(self): index = quit.git.Index(self.repo) self.assertEqual(len(index.stash), 0) index.add(self.filename, b'First Line\n') self.assertEqual(len(index.stash), 1) def testIndexCommit(self): index = quit.git.Index(self.repo) self.assertFalse(index.dirty) commit = index.commit("First commit from quit test", "QuitTest", "*****@*****.**") self.assertTrue(index.dirty) with self.assertRaises(Exception) as context: index.commit("Second commit from quit test", "QuitTest", "*****@*****.**")
def flushPartition(self): # Flush a Partition to secondary memory. # Choose a random victim from main memory, which size is not empty. victim_len = 0 while (victim_len == 0): victim = randint( 0, (self.left_table.size + self.right_table.size) - 1) if (victim < self.left_table.size): victim_len = len(self.left_table.partitions[victim].records) else: victim_len = len( self.right_table.partitions[victim % self.right_table.size].records) if (victim < self.left_table.size): file_descriptor = self.fileDescriptor_left partition_to_flush = self.left_table.partitions[victim] self.memory_left = self.memory_left - len( partition_to_flush.records) self.left_table.partitions[victim] = Partition() else: victim = victim % self.right_table.size file_descriptor = self.fileDescriptor_right partition_to_flush = self.right_table.partitions[victim] self.memory_right = self.memory_right - len( partition_to_flush.records) self.right_table.partitions[victim] = Partition() # Update file descriptor if (file_descriptor.has_key(victim)): lenfile = file_descriptor[victim].size file = open(file_descriptor[victim].file.name, 'a') file_descriptor.update({ victim: FileDescriptor(file, len(partition_to_flush.records) + lenfile, file_descriptor[victim].timestamps) }) else: file = NamedTemporaryFile(suffix=".ht", prefix="", delete=False) file_descriptor.update({ victim: FileDescriptor(file, len(partition_to_flush.records), set()) }) # Flush partition in file. for record in partition_to_flush.records: self.timestamp += 1 tuple = str(record.tuple) ats = str(record.ats) dts = str(self.timestamp) #ats = "%.40r" % (record.ats) #dts = "%.40r" % (time()) file.write(tuple + '|') file.write(ats + '|') file.write(dts + '\n') file.close()
class _TestPymo(unittest.TestCase): """Test class for pymo module Example usage: python pymo.py (to execute all tests) python pymo.py -c (for verbose console logging) python pymo.py -f mylog.log (for logging to file mylog.log) python pymo.py -c -f mylog.log (for both) python pymo.py _Test_pymo.test_fetch_smiles # (to execute only the specified test) coverage run pymo.py (to run test code coverage analysis) coverage report pymo.py (to view the result of the test code coverage analysis) """ def setUp(self): """setup test data location, unittest config and logger""" # location of test data self.test_data_location = root_dir + '/contrib/script/py/mmp/testdata/' # temp output file and dir self.temp_inp_file = NamedTemporaryFile(encoding='utf-8', mode='wt', suffix='.smi', delete=False) self.temp_out_file = NamedTemporaryFile(encoding='utf-8', mode='wt', delete=False) self.temp_out_dir = mkdtemp() test_smiles = { # basic test set - all the below id's and structures are CHEMBL '3105327': 'Cc1ccc2c(ccn2c3nc(cs3)c4cc(ccc4F)C(F)(F)F)c1', '1526778': 'CC(=O)c1c(C)n(c(C)c1C(=O)C)c2nc(c(C)s2)c3ccc(C)c(C)c3', '1494678': 'CC(=O)c1c(C)n(c(C)c1C(=O)C)c2nc(c(C)s2)c3ccccc3', '472166': 'OC(CCn1ccnc1)(c2ccccc2)c3ccccc3', '69798': 'Cc1nccn1CCC(O)(c2ccccc2)c3ccccc3', '367346': 'Cc1sc(N)nc1c2cccc(Cl)c2', '366881': 'Cc1sc(N)nc1c2ccc(Cl)c(Cl)c2', '1477460': 'COc1ccc(cc1)c2nc(sc2C)n3c(C)c(C(=O)C)c(C(=O)C)c3C', '1441050': 'COc1ccc(cc1OC)c2nc(sc2C)n3c(C)c(C(=O)C)c(C(=O)C)c3C' } # write test data to temp file 01 for smi_id, smi in test_smiles.items(): string = smi + ' ' + smi_id + '\n' self.temp_inp_file.write(string) self.temp_inp_file.close() def tearDown(self): """cleanup test data and settings""" # Clean up the directory # os.removedirs(self.temp_out_dir) shutil.rmtree(self.temp_out_dir) def test_fileconv(self): log.debug("Testing fileconv") exit_status = fileconv(os.path.join(self.test_data_location, self.temp_inp_file.name), params_dict={ '-v': '', '-c': '10', '-C': '20', '-S': os.path.join(self.temp_out_dir, 'atomcountfilter'), '-o': 'smi' }) log.debug("fileconv return code was: %s" % exit_status) self.assertEqual(exit_status, 0) def test_make_these_molecules(self): log.debug("Testing _make_these_molecules") # test data from mmp_enum_mols_from_pairs.py context = NamedTemporaryFile(encoding='utf-8', mode='wt', suffix='.smi', delete=False) examp_context = "[1CH3]CCCCCC partOne_1\n[1CH3]CCCCCCC partOne_2\n[1CH3]CCCCCCCC partOne_3\n[1CH3]CCCCCCCCC partOne_4\n[1CH3]CCCCCCCCCC partOne_5" context.write(examp_context) context.close() frags = NamedTemporaryFile(encoding='utf-8', mode='wt', suffix='.smi', delete=False) examp_frags = "[1OH]CCCCCC partTwo_1\n[1OH]CCCCCCC partTwo_2\n[1OH]CCCCCCCC partTwo_3\n[1OH]CCCCCCCCC partTwo_4\n[1OH]CCCCCCCCCC partTwo_5\n[1OH]CCCCCCCCCCC partTwo_6\n" frags.write(examp_frags) frags.close() make_instr = NamedTemporaryFile(encoding='utf-8', mode='wt', delete=False) examp_make_instr = "partOne_2 partTwo_3\npartOne_4 partTwo_5\n" make_instr.write(examp_make_instr) make_instr.close() rxn = NamedTemporaryFile(encoding='utf-8', mode='wt', suffix='.rxn', delete=False) # this is the reaction specification that trxn needs to combine isotopically labelled mmp fragmentation points single_rxn = "(0 Reaction\n (0 Scaffold\n (A C smarts \"[!0*]\")\n (A I isotope (0 0))\n )\n" single_rxn += " (1 Sidechain\n (A C smarts \"[!0*]\")\n (A I isotope (0 0))\n (A I join (0 0))\n )\n)" rxn.write(single_rxn) rxn.close() exit_status = make_these_molecules([context.name, frags.name], make_instr.name, [rxn.name]) log.debug("make_these_molecules return code was: %s" % exit_status) self.assertEqual(exit_status, 0) def test_dicer(self): log.debug("Testing dicer") log.debug("Testing %s" % dicer.__name__) exit_status = dicer( os.path.join(self.test_data_location, self.temp_inp_file.name), os.path.join(self.temp_out_dir, 'dicer.out'), params_dict={}, ) log.debug("dicer return code was: %s" % exit_status) self.assertEqual(exit_status, 0) def test_alogp(self): log.debug("Testing CMI") exit_status = alogp( os.path.join(self.test_data_location, self.temp_inp_file.name), os.path.join(self.temp_out_dir, 'alogp.out')) log.debug("CMI return code was: %s" % exit_status) self.assertEqual(exit_status, 0)
def mrjob(options): "Generates and executes MR job script" user = os.getenv('USER') tstamp = int(time.time()) hdir = hdfs_dir(options.hdir, options.hdfs_prefix) if PYDOOP: odir = hdfs.path.join(hdir, options.odir) idir = hdfs.path.join(hdir, options.idir) schema = hdfs.path.join(hdir, options.schema) for name in [ hdir, odir, idir, ]: if options.verbose: print("Checking %s" % name) if not hdfs.path.isdir(name): if name in [hdir, idir]: print("ERROR: %s does not exist" % name) sys.exit(1) # else: # print(" Creating output directory: %s" % name) # hdfs.mkdir(name) elif name == odir: # in case odir exists and is not empty, move it somewhere and re-create if hdfs.ls(odir): ocache = hdfs.path.normpath(odir) + '_%d' % tstamp if options.verbose: print( " Non-empty output directory exists, saving it in %s" % ocache) hdfs.move(odir, ocache) # hdfs.mkdir(odir) # if it's empty, remove it else: hdfs.rmr(odir) if options.verbose: print("Checking %s" % schema) if not hdfs.path.isfile(schema): print("ERROR: %s does not exist" % schema) sys.exit(1) else: idir = '%s%s' % (hdir, 'data') odir = '%s%s' % (hdir, 'mrout') schema = '%s%s' % (hdir, options.schema) if options.verbose: msg = 'pydoop module is not present on this system' msg += ', will use input as is without checking' print('WARNING:', msg) for name in [options.mrpy, options.pydoop, options.avro]: if options.verbose: print("Checking %s" % name) if not os.path.isfile(name): print("ERROR: %s does not exist" % name) sys.exit(1) # module = os.path.basename(os.path.splitext(options.mrpy)[0]) code = create_mrpy(options.mrpy, options.verbose) cmd = """#!/bin/bash input={input} output={output} schema={schema} ifile=/tmp/mr_{user}_{tstamp}.py cat << EOF > $ifile {code} EOF module=mr_{user}_{tstamp} arch_pydoop={pydoop} arch_avro={avro} echo "Input URI : $input" echo "Output URI: $output" echo "Schema: $schema" echo "MR script : $ifile" echo "Module name : $module" echo "Pydoop archive: $arch_pydoop" echo "Avro archive : $arch_avro" echo "-----------------" echo "Submitting MR job" pydoop submit \ --upload-archive-to-cache $arch_pydoop \ --upload-archive-to-cache $arch_avro \ -D avro.schema=$schema \ --do-not-use-java-record-reader \ --log-level {loglevel} \ --job-name WMArchive \ --num-reducers 1 \ --upload-file-to-cache $ifile \ --mrv2 $module $input $output """.format(input=idir, output=odir, user=user, tstamp=tstamp, code=code, schema=schema, loglevel=options.loglevel, pydoop=os.path.abspath(options.pydoop), avro=os.path.abspath(options.avro)) fobj = NamedTemporaryFile(delete=False) fobj.write(cmd) fobj.close() fstat = os.stat(fobj.name) os.chmod(fobj.name, fstat.st_mode | stat.S_IEXEC) if options.execute: run(fobj.name, options.verbose) else: if options.verbose: print("------- Generated script --------") print(open(fobj.name, 'r').read()) if options.verbose: print("---------------------------------") # clean up temporary file os.unlink(fobj.name)
def run_p2p_prediction(tx_lat, tx_lng, rx_lat, rx_lng, path_ssn, path_name="", tx_antenna="ISOTROPIC", tx_gos=0.0, rx_antenna="ISOTROPIC", rx_gos=0.0, path_month=None, path_year=None, path_hour=[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24], path_frequency=[10], path_bw=3000, path_SNRr=15, tx_power=100, path_sorl="SHORTPATH", path_manmade_noise="CITY", report_format=["RPT_BCR"], data_path="./data/", report_dict_keys=['BCR'], zeroMidnight=False, returnInputFile=False, returnOutputFile=False ): tx_power = 10 * (log10(tx_power/1000.0)) report_format_str = " | ".join(report_format) path_frequency_str = ", ".join([str(n) for n in path_frequency]) path_hour_str = ", ".join([str(n) for n in path_hour]) buf = [] if path_name: buf.append('PathName "{:s}"'.format(path_name)) buf.append('Path.L_tx.lat {:.6f}'.format(tx_lat)) buf.append('Path.L_tx.lng {:.6f}'.format(tx_lng)) buf.append('TXAntFilePath "{:s}"'.format(tx_antenna)) if tx_antenna == "ISOTROPIC": buf.append('TXGOS {:.2f}'.format(tx_gos)) #buf.append('AntennaOrientation TX2RX') buf.append('Path.L_rx.lat {:.6f}'.format(rx_lat)) buf.append('Path.L_rx.lng {:.6f}'.format(rx_lng)) buf.append('RXAntFilePath "{:s}"'.format(rx_antenna)) if rx_antenna == "ISOTROPIC": buf.append('RXGOS {:.2f}'.format(rx_gos)) #buf.append('AntennaOrientation RX2TX') if not path_year: now = datetime.datetime.utcnow() path_year = now.year buf.append('Path.year {:d}'.format(path_year)) if not path_month: now = datetime.datetime.utcnow() path_month = now.month buf.append('Path.month {:d}'.format(path_month)) buf.append('Path.hour {:s}'.format(path_hour_str)) buf.append('Path.SSN {:d}'.format(path_ssn)) buf.append('Path.frequency {:s}'.format(path_frequency_str)) buf.append('Path.txpower {:.2f}'.format(tx_power)) buf.append('Path.BW {:.2f}'.format(path_bw)) buf.append('Path.SNRr {:.2f}'.format(path_SNRr)) buf.append('Path.SNRXXp 90') buf.append('Path.ManMadeNoise "{:s}"'.format(path_manmade_noise)) buf.append('Path.SorL "{:s}"'.format(path_sorl)) buf.append('RptFileFormat "{:s}"'.format(report_format_str)) buf.append('LL.lat {:.6f}'.format(rx_lat)) buf.append('LL.lng {:.6f}'.format(rx_lng)) buf.append('LR.lat {:.6f}'.format(rx_lat)) buf.append('LR.lng {:.6f}'.format(rx_lng)) buf.append('UL.lat {:.6f}'.format(rx_lat)) buf.append('UL.lng {:.6f}'.format(rx_lng)) buf.append('UR.lat {:.6f}'.format(rx_lat)) buf.append('UR.lng {:.6f}'.format(rx_lng)) buf.append('DataFilePath "{:s}"'.format(data_path)) input_file = NamedTemporaryFile(mode='w+t', prefix="proppy_", suffix='.in', delete=False) text_in = "{:s}\n".format('\n'.join(buf)) #print(text_in) input_file.write(text_in) input_file.close() FNULL = open(os.devnull, 'w') output_file = NamedTemporaryFile(prefix="proppy_", suffix='.out', delete=False) return_code = subprocess.call(["ITURHFProp", '-c', input_file.name, output_file.name], stderr=subprocess.STDOUT) if return_code != 232: raise ITURHFPropError("Internal Server Error: Return Code {:d}".format(return_code)) try: prediction_dict = get_predictions_as_dict(output_file.name, report_dict_keys, zeroMidnight=zeroMidnight) except: print("Unexpected error:", sys.exc_info()[0]) raise ITURHFPropError("Internal Server Error: Error parsing file") print(input_file.name) print(output_file.name) #os.remove(input_file.name) #os.remove(output_file.name) return prediction_dict
def get_readable_fileobj(name_or_obj, encoding=None, cache=False, show_progress=True, remote_timeout=None): """ Given a filename or a readable file-like object, return a context manager that yields a readable file-like object. This supports passing filenames, URLs, and readable file-like objects, any of which can be compressed in gzip or bzip2. Notes ----- This function is a context manager, and should be used for example as:: with get_readable_fileobj('file.dat') as f: contents = f.read() Parameters ---------- name_or_obj : str or file-like object The filename of the file to access (if given as a string), or the file-like object to access. If a file-like object, it must be opened in binary mode. encoding : str, optional When `None` (default), returns a file-like object with a `read` method that on Python 2.x returns `bytes` objects and on Python 3.x returns `str` (`unicode`) objects, using `locale.getpreferredencoding()` as an encoding. This matches the default behavior of the built-in `open` when no `mode` argument is provided. When `'binary'`, returns a file-like object where its `read` method returns `bytes` objects. When another string, it is the name of an encoding, and the file-like object's `read` method will return `str` (`unicode`) objects, decoded from binary using the given encoding. cache : bool, optional Whether to cache the contents of remote URLs. show_progress : bool, optional Whether to display a progress bar if the file is downloaded from a remote server. Default is `True`. remote_timeout : float Timeout for remote requests in seconds (default is the configurable REMOTE_TIMEOUT, which is 3s by default) Returns ------- file : readable file-like object """ # close_fds is a list of file handles created by this function # that need to be closed. We don't want to always just close the # returned file handle, because it may simply be the file handle # passed in. In that case it is not the responsibility of this # function to close it: doing so could result in a "double close" # and an "invalid file descriptor" exception. close_fds = [] delete_fds = [] if remote_timeout is None: # use configfile default remote_timeout = REMOTE_TIMEOUT() # Get a file object to the content if isinstance(name_or_obj, six.string_types): if _is_url(name_or_obj): name_or_obj = download_file(name_or_obj, cache=cache, show_progress=show_progress, timeout=remote_timeout) if six.PY3: fileobj = io.FileIO(name_or_obj, 'r') elif six.PY2: fileobj = open(name_or_obj, 'rb') close_fds.append(fileobj) else: fileobj = name_or_obj # Check if the file object supports random access, and if not, # then wrap it in a BytesIO buffer. It would be nicer to use a # BufferedReader to avoid reading loading the whole file first, # but that is not compatible with streams or urllib2.urlopen # objects on Python 2.x. if not hasattr(fileobj, 'seek'): fileobj = io.BytesIO(fileobj.read()) # Now read enough bytes to look at signature signature = fileobj.read(4) fileobj.seek(0) if signature[:3] == b'\x1f\x8b\x08': # gzip import struct try: from .compat import gzip fileobj_new = gzip.GzipFile(fileobj=fileobj, mode='rb') fileobj_new.read(1) # need to check that the file is really gzip except (IOError, EOFError): # invalid gzip file fileobj.seek(0) fileobj_new.close() except struct.error: # invalid gzip file on Python 3 fileobj.seek(0) fileobj_new.close() else: fileobj_new.seek(0) fileobj = fileobj_new elif signature[:3] == b'BZh': # bzip2 try: # bz2.BZ2File does not support file objects, only filenames, so we # need to write the data to a temporary file tmp = NamedTemporaryFile("wb", delete=False) tmp.write(fileobj.read()) tmp.close() delete_fds.append(tmp) import bz2 fileobj_new = bz2.BZ2File(tmp.name, mode='rb') fileobj_new.read(1) # need to check that the file is really bzip2 except IOError: # invalid bzip2 file fileobj.seek(0) fileobj_new.close() else: fileobj_new.seek(0) close_fds.append(fileobj_new) fileobj = fileobj_new # By this point, we have a file, io.FileIO, gzip.GzipFile, or # bz2.BZ2File instance opened in binary mode (that is, read # returns bytes). Now we need to, if requested, wrap it in a # io.TextIOWrapper so read will return unicode based on the # encoding parameter. if six.PY3: needs_textio_wrapper = encoding != 'binary' elif six.PY2: needs_textio_wrapper = encoding != 'binary' and encoding is not None if needs_textio_wrapper: # A bz2.BZ2File can not be wrapped by a TextIOWrapper, # so we decompress it to a temporary file and then # return a handle to that. import bz2 if isinstance(fileobj, bz2.BZ2File): tmp = NamedTemporaryFile("wb", delete=False) data = fileobj.read() tmp.write(data) tmp.close() delete_fds.append(tmp) if six.PY3: fileobj = io.FileIO(tmp.name, 'r') elif six.PY2: fileobj = open(tmp.name, 'rb') close_fds.append(fileobj) # On Python 2.x, we need to first wrap the regular `file` # instance in a `io.FileIO` object before it can be # wrapped in a `TextIOWrapper`. We don't just create an # `io.FileIO` object in the first place, because we can't # get a raw file descriptor out of it on Python 2.x, which # is required for the XML iterparser. if six.PY2 and isinstance(fileobj, file): fileobj = io.FileIO(fileobj.fileno()) fileobj = io.BufferedReader(fileobj) fileobj = io.TextIOWrapper(fileobj, encoding=encoding) # Ensure that file is at the start - io.FileIO will for # example not always be at the start: # >>> import io # >>> f = open('test.fits', 'rb') # >>> f.read(4) # 'SIMP' # >>> f.seek(0) # >>> fileobj = io.FileIO(f.fileno()) # >>> fileobj.tell() # 4096L fileobj.seek(0) try: yield fileobj finally: for fd in close_fds: fd.close() for fd in delete_fds: os.remove(fd.name)
def ccopen(source, *args, **kwargs): """Guess the identity of a particular log file and return an instance of it. Inputs: source - a single logfile, a list of logfiles (for a single job), an input stream, or an URL pointing to a log file. *args, **kwargs - arguments and keyword arguments passed to filetype Returns: one of ADF, DALTON, GAMESS, GAMESS UK, Gaussian, Jaguar, Molpro, MOPAC, NWChem, ORCA, Psi3, Psi/Psi4, QChem, CJSON or None (if it cannot figure it out or the file does not exist). """ inputfile = None is_stream = False # Check if source is a link or contains links. Retrieve their content. # Try to open the logfile(s), using openlogfile, if the source is a string (filename) # or list of filenames. If it can be read, assume it is an open file object/stream. is_string = isinstance(source, str) is_url = True if is_string and URL_PATTERN.match(source) else False is_listofstrings = isinstance(source, list) and all( [isinstance(s, str) for s in source]) if is_string or is_listofstrings: # Process links from list (download contents into temporary location) if is_listofstrings: filelist = [] for filename in source: if not URL_PATTERN.match(filename): filelist.append(filename) else: try: response = urlopen(filename) tfile = NamedTemporaryFile(delete=False) tfile.write(response.read()) # Close the file because Windows won't let open it second time tfile.close() filelist.append(tfile.name) # Delete temporary file when the program finishes atexit.register(os.remove, tfile.name) except (ValueError, URLError) as error: if not kwargs.get('quiet', False): (errno, strerror) = error.args return None source = filelist if not is_url: try: inputfile = logfileparser.openlogfile(source) except IOError as error: if not kwargs.get('quiet', False): (errno, strerror) = error.args return None else: try: response = urlopen(source) is_stream = True # Retrieve filename from URL if possible filename = re.findall("\w+\.\w+", source.split('/')[-1]) filename = filename[0] if filename else "" inputfile = logfileparser.openlogfile(filename, object=response.read()) except (ValueError, URLError) as error: if not kwargs.get('quiet', False): (errno, strerror) = error.args return None elif hasattr(source, "read"): inputfile = source is_stream = True # Streams are tricky since they don't have seek methods or seek won't work # by design even if it is present. We solve this now by reading in the # entire stream and using a StringIO buffer for parsing. This might be # problematic for very large streams. Slow streams might also be an issue if # the parsing is not instantaneous, but we'll deal with such edge cases # as they arise. Ideally, in the future we'll create a class dedicated to # dealing with these issues, supporting both files and streams. if is_stream: try: inputfile.seek(0, 0) except (AttributeError, IOError): contents = inputfile.read() try: inputfile = io.StringIO(contents) except: inputfile = io.StringIO(unicode(contents)) inputfile.seek(0, 0) # Proceed to return an instance of the logfile parser only if the filetype # could be guessed. Need to make sure the input file is closed before creating # an instance, because parsers will handle opening/closing on their own. filetype = guess_filetype(inputfile) # If the input file isn't a standard compchem log file, try one of # the readers, falling back to Open Babel. if not filetype: if kwargs.get("cjson"): filetype = readerclasses['cjson'] elif source and not is_stream: ext = os.path.splitext(source)[1][1:].lower() for extension in readerclasses: if ext == extension: filetype = readerclasses[extension] # Proceed to return an instance of the logfile parser only if the filetype # could be guessed. Need to make sure the input file is closed before creating # an instance, because parsers will handle opening/closing on their own. if filetype: # We're going to close and reopen below anyway, so this is just to avoid # the missing seek method for fileinput.FileInput. In the long run # we need to refactor to support for various input types in a more # centralized fashion. if is_listofstrings: pass else: inputfile.seek(0, 0) if not is_stream: if is_listofstrings: if filetype == Turbomole: source = sort_turbomole_outputs(source) inputfile.close() return filetype(source, *args, **kwargs) return filetype(inputfile, *args, **kwargs)
def submit(self, **kwargs): """ dictionary of keywords to be parsed """ executable = kwargs.get("executable") env = dict(kwargs.get("env", {})) memory = kwargs.get("memory", 0.) cpu = kwargs.get("cpu", 0.) dry = bool(kwargs.get("dry", False)) verbose = bool(kwargs.get("verbose", True)) wd = str(kwargs.get("workdir", "$(pwd)")) part = str(kwargs.get("partition", "debug")) #log.error("ENV SETTINGS: %s",str(env)) if cpu == 0.: raise Exception("must provide cpu time") sscript = NamedTemporaryFile(dir=wd, suffix=".sh", mode="w", delete=False) sname = abspath(sscript.name) sscript.write("#!/bin/sh\n") sscript.write("#SBATCH --ntasks=1\n") sscript.write("#SBATCH --partition={part}\n".format(part=part)) sscript.write("#SBATCH -e {exe}.err\n".format(exe=sname)) sscript.write("#SBATCH -o {exe}.out\n".format(exe=sname)) sscript.write("#SBATCH --time={cpu}\n#SBATCH --mem={mem}\n\n".format( cpu=cpu, mem=memory)) for key, value in env.iteritems(): environ[key] = value sscript.write("sbatch --export={key} # {value}\n".format( key=key, value=value)) sscript.write( "\nsrun bash {executable}\n".format(executable=executable)) sscript.close() chmod(sname, 0o755) chdir(dirname(sname)) cmd = "{sub} ./{fn}".format(sub=self.executor, fn=basename(sname)) return self.__submit__(cmd, verbose=verbose, dry=dry)
class JoinPairedEndsTests(TestCase): """Tests for join_paired_ends.""" def setUp(self): """set up files to be used in all tests""" # set up temp directory that all temp files will # be written to: self.temp_dir_path = mkdtemp() # store files: # joined_pairs self.joined_pe = NamedTemporaryFile(prefix='joined_', suffix='.fastq', dir=self.temp_dir_path, delete=False) self.jpe_fp = self.joined_pe.name self.joined_pe.write(joined_reads) self.joined_pe.close() # all barcodes self.all_barcodes = NamedTemporaryFile(prefix='all_bc_', suffix='.fastq', dir=self.temp_dir_path, delete=False) self.all_bc_fp = self.all_barcodes.name self.all_barcodes.write(all_barcodes) self.all_barcodes.close() # out of order barcodes self.ooo_barcodes = NamedTemporaryFile(prefix='ooo_bc_', suffix='.fastq', dir=self.temp_dir_path, delete=False) self.ooo_bc_fp = self.ooo_barcodes.name self.ooo_barcodes.write(all_barcodes_out_of_order) self.ooo_barcodes.close() # missing barcodes self.missing_barcodes = NamedTemporaryFile(prefix='missing_bc_', suffix='.fastq', dir=self.temp_dir_path, delete=False) self.missing_bc_fp = self.missing_barcodes.name self.missing_barcodes.write(missing_barcodes) self.missing_barcodes.close() def tearDown(self): """Remove all temp files""" shutil.rmtree(self.temp_dir_path) def test_write_synced_barcodes_fastq(self): """write_synced_barcodes_fastq: should work properly. This function expects the barcodes.fastq and joined_pairs.fastq files to be in the same order except for those missing data that could not be joined. That is, not all paired-ends will assemble. """ filtered_bc_path = write_synced_barcodes_fastq(self.jpe_fp, self.all_bc_fp) observed_barcodes = open(filtered_bc_path, 'U').read() self.assertEqual(observed_barcodes, synced_barcodes) os.remove(filtered_bc_path) def test_out_of_order_barcodes(self): """write_synced_barcodes_fastq: should fail if barcodes out of order By out of order I mean that the data in the barcodes file are not in the same order as the data within the joined paired-ends file. """ self.assertRaises(StopIteration, write_synced_barcodes_fastq, self.jpe_fp, self.ooo_bc_fp) def test_missing_barcodes(self): """"write_synced_barcodes_fastq: should fail if barcodes are missing.""" self.assertRaises(StopIteration, write_synced_barcodes_fastq, self.jpe_fp, self.missing_bc_fp)
class GitRepositoryTests(unittest.TestCase): def setUp(self): self.dir = TemporaryDirectory() self.remotedir = TemporaryDirectory() self.file = NamedTemporaryFile(dir=self.dir.name, delete=False) self.filename = path.basename(self.file.name) self.author = Signature('QuitStoreTest', '*****@*****.**') self.comitter = Signature('QuitStoreTest', '*****@*****.**') # Initialize repository init_repository(self.dir.name, False) def tearDown(self): self.file = None self.filename = None self.dir.cleanup() self.dir = None self.remotedir.cleanup() self.remotedir = None def addfile(self): """Create a repository and add a file to the git index.""" # Write to file self.file.write(b'First Line\n') self.file.read() # Add file to index repo = Repository(self.dir.name) index = repo.index index.read() index.add(self.filename) index.write() def createcommit(self): """Prepare a git repository with one existing commit. Create a directory, initialize a git Repository, add and commit a file. Returns: A list containing the directory and file """ self.addfile() # Create commit repo = Repository(self.dir.name) index = repo.index index.read() tree = index.write_tree() message = "First commit of temporary test repo" repo.create_commit('HEAD', self.author, self.comitter, message, tree, []) def testInitNotExistingsRepo(self): dir = TemporaryDirectory() repo = quit.git.Repository(dir.name, create=True) self.assertFalse(repo.is_bare) self.assertEqual(len(repo.revisions()), 0) dir.cleanup() def testInitEmptyRepo(self): self.addfile() repo = quit.git.Repository(self.dir.name, create=True) self.assertFalse(repo.is_bare) self.assertEqual(len(repo.revisions()), 0) def testInitRepoWithExistingCommit(self): self.createcommit() repo = quit.git.Repository(self.dir.name) self.assertFalse(repo.is_bare) self.assertEqual(len(repo.revisions()), 1) def testCloneRepo(self): REMOTE_NAME = 'origin' REMOTE_URL = 'git://github.com/AKSW/QuitStore.example.git' dir = TemporaryDirectory() repo = quit.git.Repository(dir.name, create=True, origin=REMOTE_URL) self.assertTrue(path.exists(path.join(dir.name, 'example.nt'))) self.assertTrue(path.exists(path.join(dir.name, 'example.nt.graph'))) self.assertFalse(repo.is_bare) dir.cleanup() @unittest.skip("Currently fails on travis") def testCloneRepoViaSSH(self): environ["QUIT_SSH_KEY_HOME"] = "./tests/assets/sshkey/" REMOTE_URL = '[email protected]:AKSW/QuitStore.example.git' dir = TemporaryDirectory() repo = quit.git.Repository(dir.name, create=True, origin=REMOTE_URL) self.assertTrue(path.exists(path.join(dir.name, 'example.nt'))) self.assertFalse(repo.is_bare) dir.cleanup() def testCloneRepoViaSSHNoKeyFiles(self): environ["QUIT_SSH_KEY_HOME"] = "./tests/assets/nosshkey/" if "SSH_AUTH_SOCK" in environ: del environ["SSH_AUTH_SOCK"] REMOTE_URL = '[email protected]:AKSW/QuitStore.example.git' dir = TemporaryDirectory() with self.assertRaises(Exception) as context: quit.git.Repository(dir.name, create=True, origin=REMOTE_URL) dir.cleanup() def testCloneNotExistingRepo(self): environ["QUIT_SSH_KEY_HOME"] = "./tests/assets/sshkey/" REMOTE_URL = '[email protected]:AKSW/ThereIsNoQuitStoreRepo.git' dir = TemporaryDirectory() with self.assertRaises(Exception) as context: quit.git.Repository(dir.name, create=True, origin=REMOTE_URL) dir.cleanup() def testPushRepo(self): """Test if it is possible to push to an empty remote repository.""" with TemporaryRepository(True) as remote: graphContent = """ <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> .""" with TemporaryRepositoryFactory().withGraph( "http://example.org/", graphContent) as local: local.remotes.create("origin", remote.path) quitRepo = quit.git.Repository(local.workdir) self.assertTrue(remote.is_empty) self.assertFalse(local.is_empty) quitRepo.push("origin", "master") self.assertFalse(remote.is_empty) self.assertFalse(local.is_empty) def testPushRefspecs(self): """Test if it is possible to push to an empty remote repository.""" for refspec in [ 'master', 'refs/heads/master', 'refs/heads/master:master', 'master:master', 'master:refs/heads/master', 'refs/heads/master:refs/heads/master' ]: with TemporaryRepository(True) as remote: graphContent = """ <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> .""" with TemporaryRepositoryFactory().withGraph( "http://example.org/", graphContent) as local: local.remotes.create("origin", remote.path) quitRepo = quit.git.Repository(local.workdir) self.assertTrue(remote.is_empty) self.assertFalse(local.is_empty) quitRepo.push("origin", refspec) self.assertFalse(remote.is_empty) self.assertFalse(local.is_empty) def testPushRepoNotConfiguredRemote(self): """Test if the push failes if the origin remote was not defined.""" with TemporaryRepository(True) as remote: graphContent = """ <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> .""" with TemporaryRepositoryFactory().withGraph( "http://example.org/", graphContent) as local: local.remotes.create("upstream", remote.path) quitRepo = quit.git.Repository(local.workdir) self.assertTrue(remote.is_empty) self.assertFalse(local.is_empty) with self.assertRaises(RemoteNotFound): quitRepo.push("origin", "master") self.assertTrue(remote.is_empty) self.assertFalse(local.is_empty) def testPushRepoWithRemoteName(self): """Test if it is possible to push to a remote repository, which is not called orign.""" with TemporaryRepository(True) as remote: graphContent = "<http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ." with TemporaryRepositoryFactory().withGraph( "http://example.org/", graphContent) as local: local.remotes.create("upstream", remote.path) quitRepo = quit.git.Repository(local.workdir) self.assertTrue(remote.is_empty) self.assertFalse(local.is_empty) quitRepo.push("upstream", "master") self.assertFalse(remote.is_empty) self.assertFalse(local.is_empty) def testPushRepoNotConfiguredNamedRemote(self): """Test if the push failes if the specified remote was not defined.""" with TemporaryRepository(is_bare=True) as remote: graphContent = """ <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> .""" with TemporaryRepositoryFactory().withGraph( "http://example.org/", graphContent) as local: local.remotes.create("origin", remote.path) quitRepo = quit.git.Repository(local.workdir) self.assertTrue(remote.is_empty) self.assertFalse(local.is_empty) with self.assertRaises(RemoteNotFound): quitRepo.push("upstream", "master") self.assertTrue(remote.is_empty) self.assertFalse(local.is_empty) def testPushRepoWithDivergedRemote(self): """Test for an exception, if the local and remote repositories are diverged.""" with TemporaryRepositoryFactory().withEmptyGraph( "http://example.org/") as remote: graphContent = """ <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> .""" with TemporaryRepositoryFactory().withGraph( "http://example.org/", graphContent) as local: local.remotes.create("origin", remote.path) quitRepo = quit.git.Repository(local.workdir) self.assertFalse(remote.is_empty) self.assertFalse(local.is_empty) with self.assertRaises(pygit2.GitError): quitRepo.push("origin", "master") @unittest.skip("requires a remote with pre-receive hook") def testPushRepoWithRemoteReject(self): """Test for an exception, if the remote repositories rejects a push. CAUTION: This test is disabled, because it requires a remote with pre-receive hook. Unfortunately the libgit2 does not execute pre-receive hooks on local repositories. """ graphContent = """ <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> .""" with TemporaryRepositoryFactory().withGraph("http://example.org/", graphContent) as local: local.remotes.create("origin", "ssh://[email protected]/testing.git") quitRepo = quit.git.Repository(local.workdir) self.assertFalse(local.is_empty) with self.assertRaises(QuitGitPushError): quitRepo.push() def testFetchRepo(self): """Test if it is possible to fetch from a remote repository.""" graphContent = """ <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> .""" with TemporaryRepositoryFactory().withGraph("http://example.org/", graphContent) as remote: with TemporaryRepository(False) as local: local.remotes.create("origin", remote.path) quitRepo = quit.git.Repository(local.workdir) self.assertFalse(remote.is_empty) self.assertTrue(local.is_empty) self.assertTrue(quitRepo.is_empty) remoteHead = remote.revparse_single('HEAD').hex with self.assertRaises(RevisionNotFound): quitRepo.revision('HEAD') quitRepo.fetch() self.assertEqual( quitRepo.revision('origin/master').id, remoteHead) self.assertFalse(remote.is_empty) self.assertFalse(local.is_empty) self.assertFalse(quitRepo.is_empty) def testFetchUpstreamRepo(self): """Test if it is possible to from from a remote, which set as upstream.""" graphContent = """ <http://ex.org/x> <http://ex.org/x> <http://ex.org/x> .""" with TemporaryRepositoryFactory().withGraph("http://example.org/", graphContent) as remote: with TemporaryRepository(clone_from_repo=remote) as local: quitRepo = quit.git.Repository(local.workdir) self.assertFalse(remote.is_empty) self.assertFalse(local.is_empty) self.assertFalse(quitRepo.is_empty) with open(path.join(remote.workdir, "graph.nt"), "a") as graphFile: graphContent = """ <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> .""" graphFile.write(graphContent) createCommit(repository=remote) remoteHead = remote.revparse_single('HEAD').hex localHead = local.revparse_single('HEAD').hex self.assertNotEqual(localHead, remoteHead) quitRepo.fetch() self.assertEqual( quitRepo.revision('origin/master').id, remoteHead) self.assertFalse(remote.is_empty) self.assertFalse(local.is_empty) self.assertFalse(quitRepo.is_empty) def testPullRepo(self): """Test if it is possible to pull from a remote repository.""" graphContent = """ <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> .""" with TemporaryRepositoryFactory().withGraph("http://example.org/", graphContent) as remote: with TemporaryRepository(False) as local: local.remotes.create("origin", remote.path) quitRepo = quit.git.Repository(local.workdir) self.assertFalse(remote.is_empty) self.assertTrue(local.is_empty) self.assertTrue(quitRepo.is_empty) remoteHead = remote.revparse_single('HEAD').hex with self.assertRaises(RevisionNotFound): quitRepo.revision('HEAD') quitRepo.pull("origin", "master") self.assertEqual(quitRepo.revision('HEAD').id, remoteHead) self.assertFalse(remote.is_empty) self.assertFalse(local.is_empty) self.assertFalse(quitRepo.is_empty) def testPullRepoWithUnbornHead(self): """Test if it is possible to pull from a remote repository.""" graphContent = """ <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> .""" with TemporaryRepositoryFactory().withGraph("http://example.org/", graphContent) as remote: with TemporaryRepository(False) as local: local.remotes.create("origin", remote.path) quitRepo = quit.git.Repository(local.workdir) self.assertFalse(remote.is_empty) self.assertTrue(local.is_empty) self.assertTrue(quitRepo.is_empty) remoteHead = remote.revparse_single('HEAD').hex with self.assertRaises(RevisionNotFound): quitRepo.revision('HEAD') quitRepo.pull() self.assertEqual( quitRepo.revision('origin/master').id, remoteHead) self.assertFalse(remote.is_empty) self.assertFalse(local.is_empty) self.assertFalse(quitRepo.is_empty) def testPullRepoClonedNoChanges(self): """Test pull if both repos are at the same state.""" graphContent = """ <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> .""" with TemporaryRepositoryFactory().withGraph("http://example.org/", graphContent) as remote: with TemporaryDirectory() as localDirectory: quitRepo = quit.git.Repository(localDirectory, create=True, origin=remote.path) self.assertFalse(remote.is_empty) self.assertFalse(quitRepo.is_empty) remoteHead = remote.revparse_single('HEAD').hex self.assertEqual(quitRepo.revision('HEAD').id, remoteHead) quitRepo.pull("origin", "master") self.assertEqual(quitRepo.revision('HEAD').id, remoteHead) self.assertFalse(remote.is_empty) self.assertFalse(quitRepo.is_empty) def testPullRepoClonedAndPullWithChanges(self): """Test clone, commit on remote and pull.""" graphContent = """ <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> .""" with TemporaryRepositoryFactory().withGraph("http://example.org/", graphContent) as remote: with TemporaryDirectory() as localDirectory: quitRepo = quit.git.Repository(localDirectory, create=True, origin=remote.path) self.assertFalse(remote.is_empty) self.assertFalse(quitRepo.is_empty) remoteHead = remote.revparse_single('HEAD').hex self.assertEqual(quitRepo.revision('HEAD').id, remoteHead) quitRepo.pull() self.assertEqual(quitRepo.revision('HEAD').id, remoteHead) self.assertFalse(remote.is_empty) self.assertFalse(quitRepo.is_empty) remoteQuitRepo = quit.git.Repository(remote.workdir) index = remoteQuitRepo.index(remoteHead) graphContent += """ <http://ex.org/x> <http://ex.org/z> <http://ex.org/z> .""" index.add("graph.nt", graphContent) author = Signature('QuitStoreTest', '*****@*****.**') commitid = index.commit("from test", author.name, author.email) quitRepo.pull() self.assertEqual(quitRepo.revision('HEAD').id, str(commitid)) def testPullRepoClonedAndPullWithThreeWayGitMerge(self): """Test clone, commit on remote and pull with merge, which resolves without conflicts.""" graphContent = "<http://ex.org/a> <http://ex.org/b> <http://ex.org/c> .\n" with TemporaryRepositoryFactory().withGraph("http://example.org/", graphContent) as remote: with TemporaryDirectory() as localDirectory: quitRepo = quit.git.Repository(localDirectory, create=True, origin=remote.path) self.assertFalse(remote.is_empty) self.assertFalse(quitRepo.is_empty) remoteHead = remote.revparse_single('HEAD').hex self.assertEqual(quitRepo.revision('HEAD').id, remoteHead) index = quitRepo.index(remoteHead) graph2Content = "<http://ex.org/x> <http://ex.org/y> <http://ex.org/y> .\n" index.add("graph2.nt", graph2Content) index.add("graph2.nt.graph", "http://example2.org/") author = Signature('QuitStoreTest', '*****@*****.**') localCommitid = index.commit("from local", author.name, author.email) quitRepo._repository.checkout_tree( quitRepo._repository.get(localCommitid)) self.assertEqual( quitRepo.revision('HEAD').id, str(localCommitid)) self.assertFalse(remote.is_empty) self.assertFalse(quitRepo.is_empty) remoteQuitRepo = quit.git.Repository(remote.workdir) index = remoteQuitRepo.index(remoteHead) graphContent += "<http://ex.org/x> <http://ex.org/z> <http://ex.org/z> .\n" index.add("graph.nt", graphContent) remoteCommitid = index.commit("from remote", author.name, author.email) remoteQuitRepo._repository.checkout_tree( remoteQuitRepo._repository.get(remoteCommitid)) quitRepo.pull(method="three-way-git") self.assertNotEqual( quitRepo.revision('HEAD').id, str(localCommitid)) self.assertNotEqual( quitRepo.revision('HEAD').id, str(remoteCommitid)) # check if head has local and remote commit id as ancestor self.assertListEqual([ parent.id for parent in quitRepo.revision('HEAD').parents ], [str(localCommitid), str(remoteCommitid)]) # check if the merged commit contains all file contents with open(path.join(localDirectory, "graph.nt")) as f: self.assertEqual("".join(f.readlines()), graphContent) with open(path.join(localDirectory, "graph2.nt")) as f: self.assertEqual("".join(f.readlines()), graph2Content) def testPullRepoClonedAndPullWithThreeWayMerge(self): """Test clone, commit on remote and pull with merge, which resolves without conflicts.""" graphContent = "<http://ex.org/a> <http://ex.org/b> <http://ex.org/c> .\n" with TemporaryRepositoryFactory().withGraph("http://example.org/", graphContent) as remote: with TemporaryDirectory() as localDirectory: quitRepo = quit.git.Repository(localDirectory, create=True, origin=remote.path) self.assertFalse(remote.is_empty) self.assertFalse(quitRepo.is_empty) remoteHead = remote.revparse_single('HEAD').hex self.assertEqual(quitRepo.revision('HEAD').id, remoteHead) index = quitRepo.index(remoteHead) graph2Content = "<http://ex.org/x> <http://ex.org/y> <http://ex.org/y> .\n" index.add("graph2.nt", graph2Content) index.add("graph2.nt.graph", "http://example2.org/\n") author = Signature('QuitStoreTest', '*****@*****.**') localCommitid = index.commit("from local", author.name, author.email) quitRepo._repository.checkout_tree( quitRepo._repository.get(localCommitid)) self.assertEqual( quitRepo.revision('HEAD').id, str(localCommitid)) self.assertFalse(remote.is_empty) self.assertFalse(quitRepo.is_empty) remoteQuitRepo = quit.git.Repository(remote.workdir) index = remoteQuitRepo.index(remoteHead) graphContent += "<http://ex.org/x> <http://ex.org/z> <http://ex.org/z> .\n" index.add("graph.nt", graphContent) remoteCommitid = index.commit("from remote", author.name, author.email) remoteQuitRepo._repository.checkout_tree( remoteQuitRepo._repository.get(remoteCommitid)) quitRepo.pull(method="three-way") self.assertNotEqual( quitRepo.revision('HEAD').id, str(localCommitid)) self.assertNotEqual( quitRepo.revision('HEAD').id, str(remoteCommitid)) # check if head has local and remote commit id as ancestor self.assertListEqual([ parent.id for parent in quitRepo.revision('HEAD').parents ], [str(localCommitid), str(remoteCommitid)]) # check if the merged commit contains all file contents with open(path.join(localDirectory, "graph.nt")) as f: self.assertEqual("".join(f.readlines()), graphContent) with open(path.join(localDirectory, "graph2.nt")) as f: self.assertEqual("".join(f.readlines()), graph2Content) def testPullRepoClonedAndPullWithConflict(self): """Test clone, commit on remote and pull with conflict.""" graphContent = "<http://ex.org/a> <http://ex.org/b> <http://ex.org/c> .\n" with TemporaryRepositoryFactory().withGraph("http://example.org/", graphContent) as remote: with TemporaryDirectory() as localDirectory: quitRepo = quit.git.Repository(localDirectory, create=True, origin=remote.path) self.assertFalse(remote.is_empty) self.assertFalse(quitRepo.is_empty) remoteHead = remote.revparse_single('HEAD').hex self.assertEqual(quitRepo.revision('HEAD').id, remoteHead) index = quitRepo._repository.index with open(path.join(localDirectory, "graph.nt"), "a") as graphFile: graphFile.write( "<http://ex.org/x> <http://ex.org/y> <http://ex.org/y> .\n" ) index.add("graph.nt") index.write() tree = index.write_tree() author = Signature('QuitStoreTest', '*****@*****.**') localCommitid = quitRepo._repository.create_commit( 'HEAD', author, author, "from local", tree, [remoteHead]) self.assertFalse(remote.is_empty) self.assertFalse(quitRepo.is_empty) index = remote.index with open(path.join(remote.workdir, "graph.nt"), "a") as graphFile: graphFile.write( "<http://ex.org/x> <http://ex.org/z> <http://ex.org/z> .\n" ) index.add("graph.nt") index.write() tree = index.write_tree() remoteCommitid = remote.create_commit('HEAD', author, author, "from remote", tree, [remoteHead]) remoteQuitRepo = quit.git.Repository(remote.workdir) with self.assertRaises(QuitMergeConflict): quitRepo.pull(method="context") self.assertEqual( quitRepo.revision('HEAD').id, str(localCommitid)) self.assertEqual( remoteQuitRepo.revision('HEAD').id, str(remoteCommitid)) def testPullRepoFromNamedRemote(self): """Test if it is possible to pull from a remote repository, which is not called origin.""" graphContent = """ <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> .""" with TemporaryRepositoryFactory().withGraph("http://example.org/", graphContent) as remote: with TemporaryRepository(False) as local: local.remotes.create("upstream", remote.path) quitRepo = quit.git.Repository(local.workdir) self.assertFalse(remote.is_empty) self.assertTrue(local.is_empty) self.assertTrue(quitRepo.is_empty) remoteHead = remote.revparse_single('HEAD').hex with self.assertRaises(RevisionNotFound): quitRepo.revision('HEAD') quitRepo.pull(remote_name='upstream', refspec="master") self.assertEqual(quitRepo.revision('HEAD').id, remoteHead) self.assertFalse(remote.is_empty) self.assertFalse(local.is_empty) self.assertFalse(quitRepo.is_empty) def testPullRepoFromNotConfiguredRemote(self): """Test if it is possible to pull from a remote repository, which is not called origin.""" graphContent = """ <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> .""" with TemporaryRepositoryFactory().withGraph("http://example.org/", graphContent) as remote: with TemporaryRepository(False) as local: local.remotes.create("origin", remote.path) quitRepo = quit.git.Repository(local.workdir) self.assertFalse(remote.is_empty) self.assertTrue(local.is_empty) self.assertTrue(quitRepo.is_empty) with self.assertRaises(RemoteNotFound): quitRepo.pull(remote_name='upstream') self.assertFalse(remote.is_empty) self.assertTrue(local.is_empty) self.assertTrue(quitRepo.is_empty) def testPullRepoNoFFW(self): """TODO""" pass def testPullRepoFromMasterToDevelop(self): """TODO""" pass def testPullRepoFromDevelopToMaster(self): """TODO""" pass def testPullRepoFromRemoteTrackingBranch(self): """TODO""" pass def testRepositoryIsEmpty(self): """Test that adding data causes a new commit.""" self.addfile() repo = quit.git.Repository(self.dir.name) self.assertTrue(repo.is_empty) self.createcommit() repo = quit.git.Repository(self.dir.name) self.assertFalse(repo.is_empty) def testRepositoryIsBare(self): """Test if is_bare is currently done in init/clone tests.""" pass def testNoGCConfiguration(self): """Test Garbage Collection configuration.""" quit.git.Repository(self.dir.name, garbageCollection=False) with subprocess.Popen(["git", "config", "gc.auto"], stdout=subprocess.PIPE, cwd=self.dir.name) as getGCAuto: stdout, stderr = getGCAuto.communicate() response = stdout.decode("UTF-8").strip() self.assertEqual(response, '') def testGCConfiguration(self): """Test Garbage Collection configuration.""" quit.git.Repository(self.dir.name, garbageCollection=True) with subprocess.Popen(["git", "config", "gc.auto"], stdout=subprocess.PIPE, cwd=self.dir.name) as getGCAuto: stdout, stderr = getGCAuto.communicate() response = stdout.decode("UTF-8").strip() self.assertNotEqual(response, '') self.assertEqual(response, '256')
def main(): from argparse import ArgumentParser from os import path import numpy as np parser = ArgumentParser() parser.add_argument('prototxt') parser.add_argument('output_caffemodel') parser.add_argument( '-l', '--load', help= 'Load a pretrained model and rescale it [bias and type are not supported]' ) parser.add_argument('-d', '--data', default=None, help='Image list to use [default prototxt data]') parser.add_argument('-b', '--bias', type=float, default=0.1, help='Bias') parser.add_argument( '-t', '--type', default='elwise', help= 'Type: elwise, pca, zca, kmeans, rand (random input patches). Add fast_ to speed up the initialization, but you might lose in precision.' ) parser.add_argument('-z', action='store_true', help='Zero all weights and reinitialize') parser.add_argument('-cs', action='store_true', help='Correct for scaling') parser.add_argument('-q', action='store_true', help='Quiet execution') parser.add_argument('-s', type=float, default=1.0, help='Scale the input [only custom data "-d"]') parser.add_argument('-bs', type=int, default=16, help='Batch size [only custom data "-d"]') parser.add_argument('-nit', type=int, default=10, help='Number of iterations') parser.add_argument( '--mem-limit', type=int, default=500, help='How much memory should we use for the data buffer (MB)?') parser.add_argument('--gpu', type=int, default=0, help='What gpu to run it on?') args = parser.parse_args() if args.q: from os import environ environ['GLOG_minloglevel'] = '2' import caffe, load from caffe import NetSpec, layers as L caffe.set_mode_gpu() if args.gpu is not None: caffe.set_device(args.gpu) model = load.ProtoDesc(args.prototxt) net = NetSpec() if args.data is not None: fl = getFileList(args.data) if len(fl) == 0: print("Unknown data type for '%s'" % args.data) exit(1) from tempfile import NamedTemporaryFile f = NamedTemporaryFile('w') f.write('\n'.join([path.abspath(i) + ' 0' for i in fl])) f.flush() net.data, net.label = L.ImageData(source=f.name, batch_size=args.bs, new_width=model.input_dim[-1], new_height=model.input_dim[-1], transform_param=dict( mean_value=[104, 117, 123], scale=args.s), ntop=2) net.out = model(data=net.data, label=net.label) else: net.out = model() n = netFromString('force_backward:true\n' + str(net.to_proto()), caffe.TRAIN) if args.load is not None: n.copy_from(args.load) # Rescale existing layers? #if args.fix: #magicFix(n, args.nit) if args.z: # Zero out all layers for l in n.layers: for b in l.blobs: b.data[...] = 0 if any([ np.abs(l.blobs[0].data).sum() < 1e-10 for l in n.layers if len(l.blobs) > 0 ]): print([ m for l, m in zip(n.layers, n._layer_names) if len(l.blobs) > 0 and np.abs(l.blobs[0].data).sum() < 1e-10 ]) magicInitialize(n, args.bias, NIT=args.nit, type=args.type, max_data=args.mem_limit * 1024 * 1024 / 4) else: print("Network already initialized, skipping magic init") if args.cs: # A simply helper function that lets you figure out which layers are not # homogeneous #print( estimateHomogenety(n) ) calibrateGradientRatio(n) n.save(args.output_caffemodel)
def run_playbook(hosts, playbook, tags=[], private_key_file=private_key_file): # initialize needed objects loader = DataLoader() options = Options(connection='ssh', private_key_file=private_key_file, module_path='', forks=100, become=True, become_method='sudo', become_user='******', check=False, tags=tags) passwords = dict(vault_pass='') results_callback = ResultCallback() host_file = NamedTemporaryFile(delete=False) host_file.write(b'[servers]\n') for i, h in enumerate(hosts): print(i, " : ", h) host_file.write(bytes('{0} num={1}\n'.format(h, i), encoding='utf8')) host_file.close() # set inventory inventory = InventoryManager(loader=loader, sources=host_file.name) variable_manager = VariableManager(loader=loader, inventory=inventory) # setup playbook executor, before the run pbex = PlaybookExecutor(playbooks=[playbook], inventory=inventory, variable_manager=variable_manager, loader=loader, options=options, passwords=passwords) pbex._tqm._stdout_callback = results_callback # run playbook and get stats result = pbex.run() stats = pbex._tqm._stats outputs = { 0: 'deployment successful', 1: 'error occurred during deployment', 2: 'one or more hosts failed', 4: 'one or more hosts unreachable', 255: 'unknown error occurred during deployment' } run_success = True hosts = sorted(stats.processed.keys()) for h in hosts: t = stats.summarize(h) if t['unreachable'] > 0 or t['failures'] > 0: run_success = False os.remove(host_file.name) try: out = outputs[result] except KeyError: out = 'unrecognised error code' return result, out
def from_file_using_temporary_files(cls, file, format=None, codec=None, parameters=None, start_second=None, duration=None, **kwargs): orig_file = file file, close_file = _fd_or_path_or_tempfile(file, 'rb', tempfile=False) if format: format = format.lower() format = AUDIO_FILE_EXT_ALIASES.get(format, format) def is_format(f): f = f.lower() if format == f: return True if isinstance(orig_file, basestring): return orig_file.lower().endswith(".{0}".format(f)) if isinstance(orig_file, bytes): return orig_file.lower().endswith( (".{0}".format(f)).encode('utf8')) return False if is_format("wav"): try: obj = cls._from_safe_wav(file) if close_file: file.close() if start_second is None and duration is None: return obj elif start_second is not None and duration is None: return obj[start_second * 1000:] elif start_second is None and duration is not None: return obj[:duration * 1000] else: return obj[start_second * 1000:(start_second + duration) * 1000] except: file.seek(0) elif is_format("raw") or is_format("pcm"): sample_width = kwargs['sample_width'] frame_rate = kwargs['frame_rate'] channels = kwargs['channels'] metadata = { 'sample_width': sample_width, 'frame_rate': frame_rate, 'channels': channels, 'frame_width': channels * sample_width } obj = cls(data=file.read(), metadata=metadata) if close_file: file.close() if start_second is None and duration is None: return obj elif start_second is not None and duration is None: return obj[start_second * 1000:] elif start_second is None and duration is not None: return obj[:duration * 1000] else: return obj[start_second * 1000:(start_second + duration) * 1000] input_file = NamedTemporaryFile(mode='wb', delete=False) try: input_file.write(file.read()) except (OSError): input_file.flush() input_file.close() input_file = NamedTemporaryFile(mode='wb', delete=False, buffering=2**31 - 1) if close_file: file.close() close_file = True file = open(orig_file, buffering=2**13 - 1, mode='rb') reader = file.read(2**31 - 1) while reader: input_file.write(reader) reader = file.read(2**31 - 1) input_file.flush() if close_file: file.close() output = NamedTemporaryFile(mode="rb", delete=False) conversion_command = [ cls.converter, '-y', # always overwrite existing files ] # If format is not defined # ffmpeg/avconv will detect it automatically if format: conversion_command += ["-f", format] if codec: # force audio decoder conversion_command += ["-acodec", codec] conversion_command += [ "-i", input_file.name, # input_file options (filename last) "-vn", # Drop any video streams if there are any "-f", "wav" # output options (filename last) ] if start_second is not None: conversion_command += ["-ss", str(start_second)] if duration is not None: conversion_command += ["-t", str(duration)] conversion_command += [output.name] if parameters is not None: # extend arguments with arbitrary set conversion_command.extend(parameters) log_conversion(conversion_command) with open(os.devnull, 'rb') as devnull: p = subprocess.Popen(conversion_command, stdin=devnull, stdout=subprocess.PIPE, stderr=subprocess.PIPE) p_out, p_err = p.communicate() log_subprocess_output(p_out) log_subprocess_output(p_err) try: if p.returncode != 0: raise CouldntDecodeError( "Decoding failed. ffmpeg returned error code: {0}\n\nOutput from ffmpeg/avlib:\n\n{1}" .format(p.returncode, p_err.decode(errors='ignore'))) obj = cls._from_safe_wav(output) finally: input_file.close() output.close() os.unlink(input_file.name) os.unlink(output.name) if start_second is None and duration is None: return obj elif start_second is not None and duration is None: return obj[0:] elif start_second is None and duration is not None: return obj[:duration * 1000] else: return obj[0:duration * 1000]
class UcareCommonConfigFileTest(unittest.TestCase): def setUp(self): self.tmp_config_file = NamedTemporaryFile(mode='w+t', delete=False) def tearDown(self): conf.pub_key = None conf.secret = None conf.api_base = 'https://api.uploadcare.com/' conf.upload_base = 'https://upload.uploadcare.com/' conf.verify_api_ssl = True conf.verify_upload_ssl = True def test_use_pub_key_from_config_file(self, request): request.return_value = MockListResponse() self.tmp_config_file.write( '[ucare]\n' 'pub_key = demopublickey' ) self.tmp_config_file.close() main(arg_namespace('list_files'), [self.tmp_config_file.name]) self.assertEqual(conf.pub_key, 'demopublickey') def test_redefine_pub_key_by_second_config_file(self, request): request.return_value = MockListResponse() self.tmp_config_file.write( '[ucare]\n' 'pub_key = demopublickey' ) self.tmp_config_file.close() second_tmp_conf_file = NamedTemporaryFile(mode='w+t', delete=False) second_tmp_conf_file.write( '[ucare]\n' 'pub_key = demopublickey_modified' ) second_tmp_conf_file.close() main(arg_namespace('list_files'), [self.tmp_config_file.name, second_tmp_conf_file.name]) self.assertEqual(conf.pub_key, 'demopublickey_modified') def test_use_available_pub_key_from_config_files(self, request): request.return_value = MockListResponse() self.tmp_config_file.write( '[ucare]\n' 'pub_key = demopublickey' ) self.tmp_config_file.close() second_tmp_conf_file = NamedTemporaryFile(mode='w+t', delete=False) second_tmp_conf_file.write( '[ucare]\n' 'secret = demosecretkey' ) second_tmp_conf_file.close() main(arg_namespace('list_files'), [self.tmp_config_file.name, second_tmp_conf_file.name]) self.assertEqual(conf.pub_key, 'demopublickey') def test_redefine_config_pub_key_by_args(self, request): request.return_value = MockListResponse() self.tmp_config_file.write( '[ucare]\n' 'pub_key = demopublickey' ) self.tmp_config_file.close() main(arg_namespace('--pub_key pub list_files'), [self.tmp_config_file.name]) self.assertEqual(conf.pub_key, 'pub') def test_load_verify_api_ssl_false_value_from_config(self, request): request.return_value = MockListResponse() self.tmp_config_file.write( '[ucare]\n' 'verify_api_ssl = false' ) self.tmp_config_file.close() main(arg_namespace('list_files'), [self.tmp_config_file.name]) self.assertFalse(conf.verify_api_ssl) def test_load_verify_api_ssl_true_value_from_config(self, request): request.return_value = MockListResponse() self.tmp_config_file.write( '[ucare]\n' 'verify_api_ssl = true' ) self.tmp_config_file.close() main(arg_namespace('list_files'), [self.tmp_config_file.name]) self.assertTrue(conf.verify_api_ssl) def test_redefine_config_verify_api_ssl_by_args(self, request): request.return_value = MockListResponse() self.tmp_config_file.write( '[ucare]\n' 'verify_api_ssl = true' ) self.tmp_config_file.close() main(arg_namespace('--no_check_api_certificate list_files'), [self.tmp_config_file.name]) self.assertFalse(conf.verify_api_ssl)
def run_fio(remote, config, rbd_test_dir): """ create fio config file with options based on above config get the fio from github, generate binary, and use it to run on the generated fio config file """ fio_config=NamedTemporaryFile(prefix='fio_rbd_', dir='/tmp/', delete=False) fio_config.write('[global]\n') if config.get('io-engine'): ioengine=config['io-engine'] fio_config.write('ioengine={ioe}\n'.format(ioe=ioengine)) else: fio_config.write('ioengine=sync\n') if config.get('bs'): bs=config['bs'] fio_config.write('bs={bs}\n'.format(bs=bs)) else: fio_config.write('bs=4k\n') iodepth = config.get('io-depth', 2) fio_config.write('iodepth={iod}\n'.format(iod=iodepth)) if config.get('fio-io-size'): size=config['fio-io-size'] fio_config.write('size={size}\n'.format(size=size)) else: fio_config.write('size=100m\n') fio_config.write('time_based\n') if config.get('runtime'): runtime=config['runtime'] fio_config.write('runtime={runtime}\n'.format(runtime=runtime)) else: fio_config.write('runtime=1800\n') fio_config.write('allow_file_create=0\n') image_size=10240 if config.get('image_size'): image_size=config['image_size'] formats=[1,2] features=[['layering'],['striping'],['exclusive-lock','object-map']] fio_version='2.7' if config.get('formats'): formats=config['formats'] if config.get('features'): features=config['features'] if config.get('fio-version'): fio_version=config['fio-version'] # handle package required for ioengine, if any sn=remote.shortname ioengine_pkg = get_ioengine_package_name(ioengine, remote) if ioengine_pkg: install_package(ioengine_pkg, remote) fio_config.write('norandommap\n') if ioengine == 'rbd': fio_config.write('clientname=admin\n') fio_config.write('pool=rbd\n') fio_config.write('invalidate=0\n') elif ioengine == 'libaio': fio_config.write('direct=1\n') for frmt in formats: for feature in features: log.info("Creating rbd images on {sn}".format(sn=sn)) feature_name = '-'.join(feature) rbd_name = 'i{i}f{f}{sn}'.format(i=frmt,f=feature_name,sn=sn) rbd_snap_name = 'i{i}f{f}{sn}@i{i}f{f}{sn}Snap'.format(i=frmt,f=feature_name,sn=sn) rbd_clone_name = 'i{i}f{f}{sn}Clone'.format(i=frmt,f=feature_name,sn=sn) create_args=['rbd', 'create', '--size', '{size}'.format(size=image_size), '--image', rbd_name, '--image-format', '{f}'.format(f=frmt)] map(lambda x: create_args.extend(['--image-feature', x]), feature) remote.run(args=create_args) remote.run(args=['rbd', 'info', rbd_name]) if ioengine != 'rbd': rbd_dev = run_rbd_map(remote, rbd_name, iodepth) if config.get('test-clone-io'): log.info("Testing clones using fio") remote.run(args=['rbd', 'snap', 'create', rbd_snap_name]) remote.run(args=['rbd', 'snap', 'protect', rbd_snap_name]) remote.run(args=['rbd', 'clone', rbd_snap_name, rbd_clone_name]) rbd_clone_dev = run_rbd_map(remote, rbd_clone_name, iodepth) fio_config.write('[{rbd_dev}]\n'.format(rbd_dev=rbd_dev)) if config.get('rw'): rw=config['rw'] fio_config.write('rw={rw}\n'.format(rw=rw)) else: fio_config .write('rw=randrw\n') fio_config.write('filename={rbd_dev}\n'.format(rbd_dev=rbd_dev)) if config.get('test-clone-io'): fio_config.write('[{rbd_clone_dev}]\n'.format(rbd_clone_dev=rbd_clone_dev)) fio_config.write('rw={rw}\n'.format(rw=rw)) fio_config.write('filename={rbd_clone_dev}\n'.format(rbd_clone_dev=rbd_clone_dev)) else: if config.get('test-clone-io'): log.info("Testing clones using fio") remote.run(args=['rbd', 'snap', 'create', rbd_snap_name]) remote.run(args=['rbd', 'snap', 'protect', rbd_snap_name]) remote.run(args=['rbd', 'clone', rbd_snap_name, rbd_clone_name]) fio_config.write('[{img_name}]\n'.format(img_name=rbd_name)) if config.get('rw'): rw=config['rw'] fio_config.write('rw={rw}\n'.format(rw=rw)) else: fio_config.write('rw=randrw\n') fio_config.write('rbdname={img_name}\n'.format(img_name=rbd_name)) if config.get('test-clone-io'): fio_config.write('[{clone_img_name}]\n'.format(clone_img_name=rbd_clone_name)) fio_config.write('rw={rw}\n'.format(rw=rw)) fio_config.write('rbdname={clone_img_name}\n'.format(clone_img_name=rbd_clone_name)) fio_config.close() remote.put_file(fio_config.name,fio_config.name) try: log.info("Running rbd feature - fio test on {sn}".format(sn=sn)) fio = "https://github.com/axboe/fio/archive/fio-" + fio_version + ".tar.gz" remote.run(args=['mkdir', run.Raw(rbd_test_dir),]) remote.run(args=['cd' , run.Raw(rbd_test_dir), run.Raw(';'), 'wget' , fio , run.Raw(';'), run.Raw('tar -xvf fio*tar.gz'), run.Raw(';'), run.Raw('cd fio-fio*'), 'configure', run.Raw(';') ,'make']) remote.run(args=['ceph', '-s']) remote.run(args=[run.Raw('{tdir}/fio-fio-{v}/fio --showcmd {f}'.format(tdir=rbd_test_dir,v=fio_version,f=fio_config.name))]) remote.run(args=['sudo', run.Raw('{tdir}/fio-fio-{v}/fio {f}'.format(tdir=rbd_test_dir,v=fio_version,f=fio_config.name))]) remote.run(args=['ceph', '-s']) finally: out=StringIO.StringIO() remote.run(args=['rbd','showmapped', '--format=json'], stdout=out) mapped_images = json.loads(out.getvalue()) if mapped_images: log.info("Unmapping rbd images on {sn}".format(sn=sn)) for image in mapped_images.itervalues(): remote.run(args=['sudo', 'rbd', 'unmap', str(image['device'])]) log.info("Cleaning up fio install") remote.run(args=['rm','-rf', run.Raw(rbd_test_dir)]) if ioengine_pkg: remove_package(ioengine_pkg, remote)
def gen_keys(): # good account key account_key = NamedTemporaryFile() Popen(["openssl", "genrsa", "-out", account_key.name, "2048"]).wait() # weak 1024 bit key weak_key = NamedTemporaryFile() Popen(["openssl", "genrsa", "-out", weak_key.name, "1024"]).wait() # good domain key domain_key = NamedTemporaryFile() domain_csr = NamedTemporaryFile() Popen([ "openssl", "req", "-newkey", "rsa:2048", "-nodes", "-keyout", domain_key.name, "-subj", "/CN={0}".format(DOMAIN), "-out", domain_csr.name ]).wait() # subject alt-name domain san_csr = NamedTemporaryFile() san_conf = NamedTemporaryFile() san_conf.write(open("/etc/ssl/openssl.cnf").read().encode("utf8")) san_conf.write( "\n[SAN]\nsubjectAltName=DNS:{0}\n".format(DOMAIN).encode("utf8")) san_conf.seek(0) Popen([ "openssl", "req", "-new", "-sha256", "-key", domain_key.name, "-subj", "/", "-reqexts", "SAN", "-config", san_conf.name, "-out", san_csr.name ]).wait() # invalid domain csr invalid_csr = NamedTemporaryFile() Popen([ "openssl", "req", "-new", "-sha256", "-key", domain_key.name, "-subj", "/CN=\xC3\xA0\xC2\xB2\xC2\xA0_\xC3\xA0\xC2\xB2\xC2\xA0.com", "-out", invalid_csr.name ]).wait() # nonexistent domain csr nonexistent_csr = NamedTemporaryFile() Popen([ "openssl", "req", "-new", "-sha256", "-key", domain_key.name, "-subj", "/CN=404.{0}".format(DOMAIN), "-out", nonexistent_csr.name ]).wait() #account-signed domain csr account_csr = NamedTemporaryFile() Popen([ "openssl", "req", "-new", "-sha256", "-key", account_key.name, "-subj", "/CN={0}".format(DOMAIN), "-out", account_csr.name ]).wait() return { "account_key": account_key, "weak_key": weak_key, "domain_key": domain_key, "domain_csr": domain_csr, "san_csr": san_csr, "invalid_csr": invalid_csr, "nonexistent_csr": nonexistent_csr, "account_csr": account_csr, }
def write(data, path, html=None): with ZipFile(path, 'w', zipfile.ZIP_DEFLATED) as zip: if html is not None: zip.writestr('index.html', html) content = io.StringIO() content.write('Manifest-Version: 1.0\n') content.write('Created-By: jamovi\n') content.write('Data-Archive-Version: 1.0.2\n') content.write('jamovi-Archive-Version: 1.0\n') zip.writestr('META-INF/MANIFEST.MF', bytes(content.getvalue(), 'utf-8'), zipfile.ZIP_DEFLATED) content = None fields = [] for column in data.dataset: field = {} field['name'] = column.name field['columnType'] = ColumnType.stringify(column.column_type) field['measureType'] = MeasureType.stringify(column.measure_type) field['formula'] = column.formula field['formulaMessage'] = column.formula_message if column.measure_type == MeasureType.CONTINUOUS: field['type'] = 'number' else: field['type'] = 'integer' field['importName'] = column.import_name fields.append(field) metadata = {} metadataset = {} metadataset['rowCount'] = data.dataset.row_count metadataset['columnCount'] = data.dataset.column_count metadataset['fields'] = fields metadata['dataSet'] = metadataset zip.writestr('metadata.json', json.dumps(metadata), zipfile.ZIP_DEFLATED) metadata = None xdata = {} for column in data.dataset: if column.has_levels: xdata[column.name] = {'labels': column.levels} zip.writestr('xdata.json', json.dumps(xdata), zipfile.ZIP_DEFLATED) xdata = None row_count = data.dataset.row_count required_bytes = 0 for column in data.dataset: if column.measure_type == MeasureType.CONTINUOUS: required_bytes += (8 * row_count) else: required_bytes += (4 * row_count) temp_file = NamedTemporaryFile(delete=False) temp_file.truncate(required_bytes) for column in data.dataset: if column.measure_type == MeasureType.CONTINUOUS: for i in range(0, row_count): value = column.raw(i) byts = struct.pack('<d', value) temp_file.write(byts) else: for i in range(0, row_count): value = column.raw(i) byts = struct.pack('<i', value) temp_file.write(byts) temp_file.close() zip.write(temp_file.name, 'data.bin') os.remove(temp_file.name) resources = [] for analysis in data.analyses: if analysis.has_results is False: continue analysis_dir = '{:02} {}/analysis'.format(analysis.id, analysis.name) zip.writestr(analysis_dir, analysis.serialize(), zipfile.ZIP_DEFLATED) resources += analysis.resources for rel_path in resources: abs_path = os.path.join(data.instance_path, rel_path) zip.write(abs_path, rel_path)
CN_FILE = abspath(expanduser(CONFIG_DICT['contact_network_file'])) CONFIG_DICT['contact_network_file'] = '/FAVITES_MOUNT/%s' % CN_FILE.split('/')[-1] TN_FILE = None if 'transmission_network_file' in CONFIG_DICT: TN_FILE = abspath(expanduser(CONFIG_DICT['transmission_network_file'])) CONFIG_DICT['transmission_network_file'] = '/FAVITES_MOUNT/%s' % TN_FILE.split('/')[-1] TREE_FILE = None if 'tree_file' in CONFIG_DICT: TREE_FILE = abspath(expanduser(CONFIG_DICT['tree_file'])) CONFIG_DICT['tree_file'] = '/FAVITES_MOUNT/%s' % TREE_FILE.split('/')[-1] ERRORFREE_SEQ_FILE = None if 'errorfree_sequence_file' in CONFIG_DICT: ERRORFREE_SEQ_FILE = abspath(expanduser(CONFIG_DICT['errorfree_sequence_file'])) CONFIG_DICT['errorfree_sequence_file'] = '/FAVITES_MOUNT/%s' % ERRORFREE_SEQ_FILE.split('/')[-1] TMP_CONFIG = NamedTemporaryFile('w') TMP_CONFIG.write(str(CONFIG_DICT)) TMP_CONFIG.flush() # pull the newest versioned Docker image (if applicable) if args.update is None: version = None try: o = check_output(['docker','images']).decode().splitlines() for l in o: if l.startswith(DOCKER_IMAGE): version = '%s:%s' % (DOCKER_IMAGE,l.split()[1]); break except CalledProcessError as e: raise RuntimeError("docker images command failed\n%s"%e.output) if version is None: args.update = [] if args.update is not None:
class Git: def __init__(self, git_repo, repo_dir, pkey=None): self.git_repo = git_repo self.repo_dir = repo_dir self.repo = None self.pkey = pkey self.fd = None self.env = {} def archive(self, filepath, commit): with open(filepath, 'wb') as f: self.repo.archive(f, commit) def fetch_branches_tags(self): self.fetch() branches, tags = {}, {} for ref in self.repo.references: if isinstance(ref, RemoteReference): if ref.remote_head != 'HEAD': branches[ref.remote_head] = self._get_commits( f'origin/{ref.remote_head}') elif isinstance(ref, TagReference): tags[ref.name] = { 'id': ref.tag.hexsha, 'author': ref.tag.tagger.name, 'date': self._format_date(ref.tag.tagged_date), 'message': ref.tag.message.strip() } if ref.tag else { 'id': ref.commit.binsha.hex(), 'author': ref.commit.author.name, 'date': self._format_date(ref.commit.authored_date), 'message': ref.commit.message.strip() } tags = sorted(tags.items(), key=lambda x: x[1]['date'], reverse=True) return branches, dict(tags) def fetch(self): try: self.repo.remotes.origin.fetch(f=True, p=True, P=True) except GitCommandError as e: if self.env: self.repo.remotes.origin.fetch(env=self.env, f=True, p=True, P=True) else: raise e def _get_repo(self): if os.path.exists(self.repo_dir): try: return Repo(self.repo_dir) except InvalidGitRepositoryError: if os.path.isdir(self.repo_dir): shutil.rmtree(self.repo_dir) else: os.remove(self.repo_dir) try: repo = Repo.clone_from(self.git_repo, self.repo_dir) except GitCommandError as e: if self.env: repo = Repo.clone_from(self.git_repo, self.repo_dir, env=self.env) else: raise e return repo def _get_commits(self, branch, count=10): commits = [] for commit in self.repo.iter_commits(branch): if len(commits) == count: break commits.append({ 'id': commit.hexsha, 'author': commit.author.name, 'date': self._format_date(commit.committed_date), 'message': commit.message.strip() }) return commits def _format_date(self, timestamp): if isinstance(timestamp, int): date = datetime.fromtimestamp(timestamp) return date.strftime('%Y-%m-%d %H:%M') return timestamp def __enter__(self): if self.pkey: self.fd = NamedTemporaryFile() self.fd.write(self.pkey.encode()) self.fd.flush() self.env = {'GIT_SSH_COMMAND': f'ssh -i {self.fd.name}'} self.repo = self._get_repo() return self def __exit__(self, exc_type, exc_val, exc_tb): if self.fd: self.fd.close()
jobLog = open('patch_tested.txt', 'w+') if jobLogHistory: if len(jobLogHistory) > options.history: jobLogHistory = [jobLogHistory[0]] \ + jobLogHistory[len(jobLogHistory) - options.history:] for jobHistoryRecord in jobLogHistory: jobLog.write(jobHistoryRecord + '\n') else: jobLog.write('TESTED ISSUES\n') jobLog.flush() rssData = httpGet(options.jiraFilter, False, options.jiraUser, options.jiraPassword) tempFile.write(rssData) tempFile.flush() for (key, attachment) in parseJiraData(tempFile.name).items(): (project, issue) = key if jenkinsUrlOverrides.has_key(project): url = jenkinsUrlOverrides[project] else: url = options.jenkinsUrl jenkinsUrlTemplate = url + '/job/' \ + options.jenkinsJobTemplate \ + '/buildWithParameters?' + tokenFrag \ + '&ISSUE_NUM={issue}&ATTACHMENT_ID={attachment}' urlArgs = { 'project': project,
def main(apiurl, opts, argv): repo = argv[0] arch = argv[1] build_descr = argv[2] xp = [] build_root = None cache_dir = None build_uid = '' vm_type = config['build-type'] vm_telnet = None build_descr = os.path.abspath(build_descr) build_type = os.path.splitext(build_descr)[1][1:] if os.path.basename(build_descr) == 'PKGBUILD': build_type = 'arch' if os.path.basename(build_descr) == 'build.collax': build_type = 'collax' if build_type not in ['spec', 'dsc', 'kiwi', 'arch', 'collax', 'livebuild']: raise oscerr.WrongArgs( 'Unknown build type: \'%s\'. Build description should end in .spec, .dsc, .kiwi or .livebuild.' \ % build_type) if not os.path.isfile(build_descr): raise oscerr.WrongArgs('Error: build description file named \'%s\' does not exist.' % build_descr) buildargs = [] if not opts.userootforbuild: buildargs.append('--norootforbuild') if opts.clean: buildargs.append('--clean') if opts.noinit: buildargs.append('--noinit') if opts.nochecks: buildargs.append('--no-checks') if not opts.no_changelog: buildargs.append('--changelog') if opts.root: build_root = opts.root if opts.target: buildargs.append('--target=%s' % opts.target) if opts.threads: buildargs.append('--threads=%s' % opts.threads) if opts.jobs: buildargs.append('--jobs=%s' % opts.jobs) elif config['build-jobs'] > 1: buildargs.append('--jobs=%s' % config['build-jobs']) if opts.icecream or config['icecream'] != '0': if opts.icecream: num = opts.icecream else: num = config['icecream'] if int(num) > 0: buildargs.append('--icecream=%s' % num) xp.append('icecream') xp.append('gcc-c++') if opts.ccache: buildargs.append('--ccache') xp.append('ccache') if opts.linksources: buildargs.append('--linksources') if opts.baselibs: buildargs.append('--baselibs') if opts.debuginfo: buildargs.append('--debug') if opts._with: for o in opts._with: buildargs.append('--with=%s' % o) if opts.without: for o in opts.without: buildargs.append('--without=%s' % o) if opts.define: for o in opts.define: buildargs.append('--define=%s' % o) if config['build-uid']: build_uid = config['build-uid'] if opts.build_uid: build_uid = opts.build_uid if build_uid: buildidre = re.compile('^[0-9]{1,5}:[0-9]{1,5}$') if build_uid == 'caller': buildargs.append('--uid=%s:%s' % (os.getuid(), os.getgid())) elif buildidre.match(build_uid): buildargs.append('--uid=%s' % build_uid) else: print('Error: build-uid arg must be 2 colon separated numerics: "uid:gid" or "caller"', file=sys.stderr) return 1 if opts.vm_type: vm_type = opts.vm_type if opts.vm_telnet: vm_telnet = opts.vm_telnet if opts.alternative_project: prj = opts.alternative_project pac = '_repository' else: prj = store_read_project(os.curdir) if opts.local_package: pac = '_repository' else: pac = store_read_package(os.curdir) if opts.shell: buildargs.append("--shell") orig_build_root = config['build-root'] # make it possible to override configuration of the rc file for var in ['OSC_PACKAGECACHEDIR', 'OSC_SU_WRAPPER', 'OSC_BUILD_ROOT']: val = os.getenv(var) if val: if var.startswith('OSC_'): var = var[4:] var = var.lower().replace('_', '-') if var in config: print('Overriding config value for %s=\'%s\' with \'%s\'' % (var, config[var], val)) config[var] = val pacname = pac if pacname == '_repository': if not opts.local_package: try: pacname = store_read_package(os.curdir) except oscerr.NoWorkingCopy: opts.local_package = True if opts.local_package: pacname = os.path.splitext(build_descr)[0] apihost = urlsplit(apiurl)[1] if not build_root: build_root = config['build-root'] if build_root == orig_build_root: # ENV var was not set build_root = config['api_host_options'][apiurl].get('build-root', build_root) try: build_root = build_root % {'repo': repo, 'arch': arch, 'project': prj, 'package': pacname, 'apihost': apihost} except: pass cache_dir = config['packagecachedir'] % {'apihost': apihost} extra_pkgs = [] if not opts.extra_pkgs: extra_pkgs = config['extra-pkgs'] elif opts.extra_pkgs != ['']: extra_pkgs = opts.extra_pkgs if xp: extra_pkgs += xp prefer_pkgs = {} build_descr_data = open(build_descr).read() # XXX: dirty hack but there's no api to provide custom defines if opts.without: s = '' for i in opts.without: s += "%%define _without_%s 1\n" % i build_descr_data = s + build_descr_data if opts._with: s = '' for i in opts._with: s += "%%define _with_%s 1\n" % i build_descr_data = s + build_descr_data if opts.define: s = '' for i in opts.define: s += "%%define %s\n" % i build_descr_data = s + build_descr_data cpiodata = None servicefile = os.path.join(os.path.dirname(build_descr), "_service") if not os.path.isfile(servicefile): servicefile = os.path.join(os.path.dirname(build_descr), "_service") if not os.path.isfile(servicefile): servicefile = None else: print('Using local _service file') buildenvfile = os.path.join(os.path.dirname(build_descr), "_buildenv." + repo + "." + arch) if not os.path.isfile(buildenvfile): buildenvfile = os.path.join(os.path.dirname(build_descr), "_buildenv") if not os.path.isfile(buildenvfile): buildenvfile = None else: print('Using local buildenv file: %s' % os.path.basename(buildenvfile)) if buildenvfile or servicefile: from .util import cpio if not cpiodata: cpiodata = cpio.CpioWrite() if opts.prefer_pkgs: print('Scanning the following dirs for local packages: %s' % ', '.join(opts.prefer_pkgs)) from .util import cpio if not cpiodata: cpiodata = cpio.CpioWrite() prefer_pkgs = get_prefer_pkgs(opts.prefer_pkgs, arch, build_type, cpiodata) if cpiodata: cpiodata.add(os.path.basename(build_descr), build_descr_data) # buildenv must come last for compatibility reasons... if buildenvfile: cpiodata.add("buildenv", open(buildenvfile).read()) if servicefile: cpiodata.add("_service", open(servicefile).read()) build_descr_data = cpiodata.get() # special handling for overlay and rsync-src/dest specialcmdopts = [] if opts.rsyncsrc or opts.rsyncdest : if not opts.rsyncsrc or not opts.rsyncdest: raise oscerr.WrongOptions('When using --rsync-{src,dest} both parameters have to be specified.') myrsyncsrc = os.path.abspath(os.path.expanduser(os.path.expandvars(opts.rsyncsrc))) if not os.path.isdir(myrsyncsrc): raise oscerr.WrongOptions('--rsync-src %s is no valid directory!' % opts.rsyncsrc) # can't check destination - its in the target chroot ;) - but we can check for sanity myrsyncdest = os.path.expandvars(opts.rsyncdest) if not os.path.isabs(myrsyncdest): raise oscerr.WrongOptions('--rsync-dest %s is no absolute path (starting with \'/\')!' % opts.rsyncdest) specialcmdopts = ['--rsync-src='+myrsyncsrc, '--rsync-dest='+myrsyncdest] if opts.overlay: myoverlay = os.path.abspath(os.path.expanduser(os.path.expandvars(opts.overlay))) if not os.path.isdir(myoverlay): raise oscerr.WrongOptions('--overlay %s is no valid directory!' % opts.overlay) specialcmdopts += ['--overlay='+myoverlay] bi_file = None bc_file = None bi_filename = '_buildinfo-%s-%s.xml' % (repo, arch) bc_filename = '_buildconfig-%s-%s' % (repo, arch) if is_package_dir('.') and os.access(osc.core.store, os.W_OK): bi_filename = os.path.join(os.getcwd(), osc.core.store, bi_filename) bc_filename = os.path.join(os.getcwd(), osc.core.store, bc_filename) elif not os.access('.', os.W_OK): bi_file = NamedTemporaryFile(prefix=bi_filename) bi_filename = bi_file.name bc_file = NamedTemporaryFile(prefix=bc_filename) bc_filename = bc_file.name else: bi_filename = os.path.abspath(bi_filename) bc_filename = os.path.abspath(bc_filename) try: if opts.noinit: if not os.path.isfile(bi_filename): raise oscerr.WrongOptions('--noinit is not possible, no local buildinfo file') print('Use local \'%s\' file as buildinfo' % bi_filename) if not os.path.isfile(bc_filename): raise oscerr.WrongOptions('--noinit is not possible, no local buildconfig file') print('Use local \'%s\' file as buildconfig' % bc_filename) elif opts.offline: if not os.path.isfile(bi_filename): raise oscerr.WrongOptions('--offline is not possible, no local buildinfo file') print('Use local \'%s\' file as buildinfo' % bi_filename) if not os.path.isfile(bc_filename): raise oscerr.WrongOptions('--offline is not possible, no local buildconfig file') else: print('Getting buildinfo from server and store to %s' % bi_filename) bi_text = ''.join(get_buildinfo(apiurl, prj, pac, repo, arch, specfile=build_descr_data, addlist=extra_pkgs)) if not bi_file: bi_file = open(bi_filename, 'w') # maybe we should check for errors before saving the file bi_file.write(bi_text) bi_file.flush() print('Getting buildconfig from server and store to %s' % bc_filename) bc = get_buildconfig(apiurl, prj, repo) if not bc_file: bc_file = open(bc_filename, 'w') bc_file.write(bc) bc_file.flush() except HTTPError as e: if e.code == 404: # check what caused the 404 if meta_exists(metatype='prj', path_args=(quote_plus(prj), ), template_args=None, create_new=False, apiurl=apiurl): pkg_meta_e = None try: # take care, not to run into double trouble. pkg_meta_e = meta_exists(metatype='pkg', path_args=(quote_plus(prj), quote_plus(pac)), template_args=None, create_new=False, apiurl=apiurl) except: pass if pkg_meta_e: print('ERROR: Either wrong repo/arch as parameter or a parse error of .spec/.dsc/.kiwi file due to syntax error', file=sys.stderr) else: print('The package \'%s\' does not exist - please ' \ 'rerun with \'--local-package\'' % pac, file=sys.stderr) else: print('The project \'%s\' does not exist - please ' \ 'rerun with \'--alternative-project <alternative_project>\'' % prj, file=sys.stderr) sys.exit(1) else: raise bi = Buildinfo(bi_filename, apiurl, build_type, list(prefer_pkgs.keys())) if bi.debuginfo and not (opts.disable_debuginfo or '--debug' in buildargs): buildargs.append('--debug') if opts.release: bi.release = opts.release if bi.release: buildargs.append('--release=%s' % bi.release) # real arch of this machine # vs. # arch we are supposed to build for if bi.hostarch != None: if hostarch != bi.hostarch and not bi.hostarch in can_also_build.get(hostarch, []): print('Error: hostarch \'%s\' is required.' % (bi.hostarch), file=sys.stderr) return 1 elif hostarch != bi.buildarch: if not bi.buildarch in can_also_build.get(hostarch, []): # OBSOLETE: qemu_can_build should not be needed anymore since OBS 2.3 if vm_type != "emulator" and not bi.buildarch in qemu_can_build: print('Error: hostarch \'%s\' cannot build \'%s\'.' % (hostarch, bi.buildarch), file=sys.stderr) return 1 print('WARNING: It is guessed to build on hostarch \'%s\' for \'%s\' via QEMU.' % (hostarch, bi.buildarch), file=sys.stderr) rpmlist_prefers = [] if prefer_pkgs: print('Evaluating preferred packages') for name, path in prefer_pkgs.items(): if bi.has_dep(name): # We remove a preferred package from the buildinfo, so that the # fetcher doesn't take care about them. # Instead, we put it in a list which is appended to the rpmlist later. # At the same time, this will make sure that these packages are # not verified. bi.remove_dep(name) rpmlist_prefers.append((name, path)) print(' - %s (%s)' % (name, path)) print('Updating cache of required packages') urllist = [] if not opts.download_api_only: # transform 'url1, url2, url3' form into a list if 'urllist' in config: if isinstance(config['urllist'], str): re_clist = re.compile('[, ]+') urllist = [ i.strip() for i in re_clist.split(config['urllist'].strip()) ] else: urllist = config['urllist'] # OBS 1.5 and before has no downloadurl defined in buildinfo if bi.downloadurl: urllist.append(bi.downloadurl + '/%(extproject)s/%(extrepository)s/%(arch)s/%(filename)s') if opts.disable_cpio_bulk_download: urllist.append( '%(apiurl)s/build/%(project)s/%(repository)s/%(repoarch)s/%(repopackage)s/%(repofilename)s' ) fetcher = Fetcher(cache_dir, urllist = urllist, api_host_options = config['api_host_options'], offline = opts.noinit or opts.offline, http_debug = config['http_debug'], enable_cpio = not opts.disable_cpio_bulk_download, cookiejar=cookiejar) if not opts.trust_all_projects: # implicitly trust the project we are building for check_trusted_projects(apiurl, [ i for i in bi.projects.keys() if not i == prj ]) # now update the package cache fetcher.run(bi) old_pkg_dir = None if opts.oldpackages: old_pkg_dir = opts.oldpackages if not old_pkg_dir.startswith('/') and not opts.offline: data = [ prj, pacname, repo, arch] if old_pkg_dir == '_link': p = osc.core.findpacs(os.curdir)[0] if not p.islink(): raise oscerr.WrongOptions('package is not a link') data[0] = p.linkinfo.project data[1] = p.linkinfo.package repos = osc.core.get_repositories_of_project(apiurl, data[0]) # hack for links to e.g. Factory if not data[2] in repos and 'standard' in repos: data[2] = 'standard' elif old_pkg_dir != '' and old_pkg_dir != '_self': a = old_pkg_dir.split('/') for i in range(0, len(a)): data[i] = a[i] destdir = os.path.join(cache_dir, data[0], data[2], data[3]) old_pkg_dir = None try: print("Downloading previous build from %s ..." % '/'.join(data)) binaries = get_binarylist(apiurl, data[0], data[2], data[3], package=data[1], verbose=True) except Exception as e: print("Error: failed to get binaries: %s" % str(e)) binaries = [] if binaries: class mytmpdir: """ temporary directory that removes itself""" def __init__(self, *args, **kwargs): self.name = mkdtemp(*args, **kwargs) _rmtree = staticmethod(shutil.rmtree) def cleanup(self): self._rmtree(self.name) def __del__(self): self.cleanup() def __exit__(self): self.cleanup() def __str__(self): return self.name old_pkg_dir = mytmpdir(prefix='.build.oldpackages', dir=os.path.abspath(os.curdir)) if not os.path.exists(destdir): os.makedirs(destdir) for i in binaries: fname = os.path.join(destdir, i.name) os.symlink(fname, os.path.join(str(old_pkg_dir), i.name)) if os.path.exists(fname): st = os.stat(fname) if st.st_mtime == i.mtime and st.st_size == i.size: continue get_binary_file(apiurl, data[0], data[2], data[3], i.name, package = data[1], target_filename = fname, target_mtime = i.mtime, progress_meter = True) if old_pkg_dir != None: buildargs.append('--oldpackages=%s' % old_pkg_dir) # Make packages from buildinfo available as repos for kiwi if build_type == 'kiwi': if os.path.exists('repos'): shutil.rmtree('repos') os.mkdir('repos') for i in bi.deps: if not i.extproject: # remove bi.deps.remove(i) continue # project pdir = str(i.extproject).replace(':/', ':') # repo rdir = str(i.extrepository).replace(':/', ':') # arch adir = i.repoarch # project/repo prdir = "repos/"+pdir+"/"+rdir # project/repo/arch pradir = prdir+"/"+adir # source fullfilename sffn = i.fullfilename filename = sffn.split("/")[-1] # target fullfilename tffn = pradir+"/"+filename if not os.path.exists(os.path.join(pradir)): os.makedirs(os.path.join(pradir)) if not os.path.exists(tffn): print("Using package: "+sffn) if opts.linksources: os.link(sffn, tffn) else: os.symlink(sffn, tffn) if prefer_pkgs: for name, path in prefer_pkgs.items(): if name == filename: print("Using prefered package: " + path + "/" + filename) os.unlink(tffn) if opts.linksources: os.link(path + "/" + filename, tffn) else: os.symlink(path + "/" + filename, tffn) # Is a obsrepositories tag used? try: tree = ET.parse(build_descr) except: print('could not parse the kiwi file:', file=sys.stderr) print(open(build_descr).read(), file=sys.stderr) sys.exit(1) root = tree.getroot() # product for xml in root.findall('instsource'): if xml.find('instrepo').find('source').get('path') == 'obsrepositories:/': print("obsrepositories:/ for product builds is not yet supported in osc!") sys.exit(1) # appliance expand_obsrepos=None for xml in root.findall('repository'): if xml.find('source').get('path') == 'obsrepositories:/': expand_obsrepos=True if expand_obsrepos: buildargs.append('--kiwi-parameter') buildargs.append('--ignore-repos') for xml in root.findall('repository'): if xml.find('source').get('path') == 'obsrepositories:/': for path in bi.pathes: if not os.path.isdir("repos/"+path): continue buildargs.append('--kiwi-parameter') buildargs.append('--add-repo') buildargs.append('--kiwi-parameter') buildargs.append("repos/"+path) buildargs.append('--kiwi-parameter') buildargs.append('--add-repotype') buildargs.append('--kiwi-parameter') buildargs.append('rpm-md') if xml.get('priority'): buildargs.append('--kiwi-parameter') buildargs.append('--add-repoprio='+xml.get('priority')) else: m = re.match(r"obs://[^/]+/([^/]+)/(\S+)", xml.find('source').get('path')) if not m: # short path without obs instance name m = re.match(r"obs://([^/]+)/(.+)", xml.find('source').get('path')) project=m.group(1).replace(":", ":/") repo=m.group(2) buildargs.append('--kiwi-parameter') buildargs.append('--add-repo') buildargs.append('--kiwi-parameter') buildargs.append("repos/"+project+"/"+repo) buildargs.append('--kiwi-parameter') buildargs.append('--add-repotype') buildargs.append('--kiwi-parameter') buildargs.append('rpm-md') if xml.get('priority'): buildargs.append('--kiwi-parameter') buildargs.append('--add-repopriority='+xml.get('priority')) if vm_type == "xen" or vm_type == "kvm" or vm_type == "lxc": print('Skipping verification of package signatures due to secure VM build') elif bi.pacsuffix == 'rpm': if opts.no_verify: print('Skipping verification of package signatures') else: print('Verifying integrity of cached packages') verify_pacs(bi) elif bi.pacsuffix == 'deb': if opts.no_verify or opts.noinit: print('Skipping verification of package signatures') else: print('WARNING: deb packages get not verified, they can compromise your system !') else: print('WARNING: unknown packages get not verified, they can compromise your system !') for i in bi.deps: if i.hdrmd5: from .util import packagequery hdrmd5 = packagequery.PackageQuery.queryhdrmd5(i.fullfilename) if not hdrmd5: print("Error: cannot get hdrmd5 for %s" % i.fullfilename) sys.exit(1) if hdrmd5 != i.hdrmd5: print("Error: hdrmd5 mismatch for %s: %s != %s" % (i.fullfilename, hdrmd5, i.hdrmd5)) sys.exit(1) print('Writing build configuration') if build_type == 'kiwi': rpmlist = [ '%s %s\n' % (i.name, i.fullfilename) for i in bi.deps if not i.noinstall ] else: rpmlist = [ '%s %s\n' % (i.name, i.fullfilename) for i in bi.deps ] rpmlist += [ '%s %s\n' % (i[0], i[1]) for i in rpmlist_prefers ] rpmlist.append('preinstall: ' + ' '.join(bi.preinstall_list) + '\n') rpmlist.append('vminstall: ' + ' '.join(bi.vminstall_list) + '\n') rpmlist.append('runscripts: ' + ' '.join(bi.runscripts_list) + '\n') if build_type != 'kiwi' and bi.noinstall_list: rpmlist.append('noinstall: ' + ' '.join(bi.noinstall_list) + '\n') if build_type != 'kiwi' and bi.installonly_list: rpmlist.append('installonly: ' + ' '.join(bi.installonly_list) + '\n') rpmlist_file = NamedTemporaryFile(prefix='rpmlist.') rpmlist_filename = rpmlist_file.name rpmlist_file.writelines(rpmlist) rpmlist_file.flush() subst = { 'repo': repo, 'arch': arch, 'project' : prj, 'package' : pacname } vm_options = [] # XXX check if build-device present my_build_device = '' if config['build-device']: my_build_device = config['build-device'] % subst else: # obs worker uses /root here but that collides with the # /root directory if the build root was used without vm # before my_build_device = build_root + '/img' need_root = True if vm_type: if config['build-swap']: my_build_swap = config['build-swap'] % subst else: my_build_swap = build_root + '/swap' vm_options = [ '--vm-type=%s' % vm_type ] if vm_telnet: vm_options += [ '--vm-telnet=' + vm_telnet ] if config['build-memory']: vm_options += [ '--memory=' + config['build-memory'] ] if vm_type != 'lxc': vm_options += [ '--vm-disk=' + my_build_device ] vm_options += [ '--vm-swap=' + my_build_swap ] vm_options += [ '--logfile=%s/.build.log' % build_root ] if vm_type == 'kvm': if os.access(build_root, os.W_OK) and os.access('/dev/kvm', os.W_OK): # so let's hope there's also an fstab entry need_root = False if config['build-kernel']: vm_options += [ '--vm-kernel=' + config['build-kernel'] ] if config['build-initrd']: vm_options += [ '--vm-initrd=' + config['build-initrd'] ] build_root += '/.mount' if config['build-memory']: vm_options += [ '--memory=' + config['build-memory'] ] if config['build-vmdisk-rootsize']: vm_options += [ '--vmdisk-rootsize=' + config['build-vmdisk-rootsize'] ] if config['build-vmdisk-swapsize']: vm_options += [ '--vmdisk-swapsize=' + config['build-vmdisk-swapsize'] ] if config['build-vmdisk-filesystem']: vm_options += [ '--vmdisk-filesystem=' + config['build-vmdisk-filesystem'] ] if config['build-vm-user']: vm_options += [ '--vm-user='******'build-vm-user'] ] if opts.preload: print("Preload done for selected repo/arch.") sys.exit(0) print('Running build') cmd = [ config['build-cmd'], '--root='+build_root, '--rpmlist='+rpmlist_filename, '--dist='+bc_filename, '--arch='+bi.buildarch ] cmd += specialcmdopts + vm_options + buildargs cmd += [ build_descr ] if need_root: sucmd = config['su-wrapper'].split() if sucmd[0] == 'su': if sucmd[-1] == '-c': sucmd.pop() cmd = sucmd + ['-s', cmd[0], 'root', '--' ] + cmd[1:] else: cmd = sucmd + cmd # change personality, if needed if hostarch != bi.buildarch and bi.buildarch in change_personality: cmd = [ change_personality[bi.buildarch] ] + cmd try: rc = run_external(cmd[0], *cmd[1:]) if rc: print() print('The buildroot was:', build_root) sys.exit(rc) except KeyboardInterrupt as i: print("keyboard interrupt, killing build ...") cmd.append('--kill') run_external(cmd[0], *cmd[1:]) raise i pacdir = os.path.join(build_root, '.build.packages') if os.path.islink(pacdir): pacdir = os.readlink(pacdir) pacdir = os.path.join(build_root, pacdir) if os.path.exists(pacdir): (s_built, b_built) = get_built_files(pacdir, bi.buildtype) print() if s_built: print(s_built) print() print(b_built) if opts.keep_pkgs: for i in b_built.splitlines() + s_built.splitlines(): shutil.copy2(i, os.path.join(opts.keep_pkgs, os.path.basename(i))) if bi_file: bi_file.close() if bc_file: bc_file.close() rpmlist_file.close()
def create_router(transport): print("\033[1A\033[K" + "Creating shaper for {}".format(transport)) if transport == "bus": routing_type = { "weights": {"motorway": 1.5, "trunk": 1.5, "primary": 1.4, "secondary": 1.3, "tertiary": 1.3, "unclassified": 1, "residential": 0.6, "living_street": 0.6, "track": 0.3, "service": 0.5}, "access": ["access", "vehicle", "motor_vehicle", "psv", "bus", "routing:ztm"], "name": "bus" } elif transport == "train": routing_type = "train" elif transport == "tram": routing_type = "tram" else: raise ValueError("Invalid transport type {} for Shaper".format(transport)) if transport in {"train", "tram"}: request = requests.get("https://mkuran.pl/feed/ztm/ztm-km-rail-shapes.osm", stream=True) else: # Overpass Query: query = "\n".join([ "[bbox:51.9144,20.4438,52.5007,21.4844][out:xml];" "(" ' way["highway"="motorway"];' ' way["highway"="motorway_link"];' ' way["highway"="trunk"];' ' way["highway"="trunk_link"];' ' way["highway"="primary"];' ' way["highway"="primary_link"];' ' way["highway"="secondary"];' ' way["highway"="secondary_link"];' ' way["highway"="tertiary"];' ' way["highway"="tertiary_link"];' ' way["highway"="unclassified"];' ' way["highway"="minor"];' ' way["highway"="residential"];' ' way["highway"="living_street"];' ' way["highway"="service"];' ');' 'way._(poly:"52.4455 20.6858 52.376 20.6872 52.3533 20.7868 52.2929 20.726 52.2694 20.6724 52.2740 20.4465 52.2599 20.4438 52.2481 20.5832 52.2538 20.681 52.1865 20.6786 52.1859 20.7129 52.1465 20.7895 52.0966 20.783 52.0632 20.7222 52.0151 20.7617 51.9873 20.9351 51.9269 20.9509 51.9144 21.0226 51.9322 21.1987 51.9569 21.2472 52.0463 21.2368 52.1316 21.4844 52.1429 21.4404 52.2130 21.3814 52.2622 21.3141 52.2652 21.1977 52.3038 21.173 52.3063 21.2925 52.3659 21.3515 52.3829 21.3001 52.4221 21.1929 52.4898 21.1421");' '>->.n;' '<->.r;' '(._;.n;.r;);' 'out;' ]) request = requests.get( "https://overpass-api.de/api/interpreter/", params={"data": query}, stream=True, ) temp_xml = NamedTemporaryFile(delete=False) for chunk in request.iter_content(chunk_size=1024): temp_xml.write(chunk) request.close() temp_xml.seek(0) router = pyroutelib3.Router(routing_type, temp_xml.name) temp_xml.close() return router
class ParallelBlasterTests(TestCase): def setUp(self): """ """ self.files_to_remove = [] self.dirs_to_remove = [] tmp_dir = get_qiime_temp_dir() self.test_out = get_tmp_filename( tmp_dir=tmp_dir, prefix='qiime_parallel_blaster_tests_', suffix='', result_constructor=str) self.dirs_to_remove.append(self.test_out) create_dir(self.test_out) self.tmp_seq_filepath = get_tmp_filename( tmp_dir=self.test_out, prefix='qiime_parallel_blaster_tests_input', suffix='.fasta') seq_file = open(self.tmp_seq_filepath, 'w') seq_file.write(blast_test_seqs) seq_file.close() self.files_to_remove.append(self.tmp_seq_filepath) self.reference_seqs_file = NamedTemporaryFile( prefix='qiime_parallel_blaster_tests_ref_seqs', suffix='.fasta', dir=tmp_dir) self.reference_seqs_file.write(blast_ref_seqs) self.reference_seqs_file.seek(0) initiate_timeout(60) def tearDown(self): """ """ disable_timeout() remove_files(self.files_to_remove) # remove directories last, so we don't get errors # trying to remove files which may be in the directories for d in self.dirs_to_remove: if exists(d): rmtree(d) def test_parallel_blaster(self): """Test ParallelBlaster functions as expected.""" params = { 'refseqs_path': self.reference_seqs_file.name, 'disable_low_complexity_filter': False, 'e_value': 0.001, 'num_hits': 1, 'word_size': 30, 'suppress_format_blastdb': False, 'blastmat_dir': None } app = ParallelBlaster() r = app(self.tmp_seq_filepath, self.test_out, params, job_prefix='BLASTTEST', poll_directly=True, suppress_submit_jobs=False) # Basic sanity checks: we should get two blast hits (lines). We ignore # all of the comments in the file. Each line should have 12 fields # separated by tabs. results = [ line for line in open( glob(join(self.test_out, '*_blast_out.txt'))[0], 'U') if not line.startswith('#') ] self.assertEqual(len(results), 2) self.assertEqual(len(results[0].split('\t')), 12) self.assertEqual(len(results[1].split('\t')), 12)
regenerate = '--regenerate-test' in sys.argv[1:] for dvd in False,True: if regenerate: f = open(test_template(dvd=dvd), 'w') else: from tempfile import NamedTemporaryFile f = NamedTemporaryFile(mode='w') f.write( re.sub('(?<!\r)\n', '\r\n', generate( dvd=dvd, version=r'${NORMALIZED_VERSION}', human_version=r'%(human_version)s', sections='%(sections)s', architecture='32' ))) f.flush() if not regenerate: print subprocess.Popen( ['diff', '-wdu', test_template(dvd=dvd), f.name]).communicate()[0] if not dvd: print print '==============================' print
class FileIndex: SIGNATURE = b'BOB1' CACHE_ENTRY_FMT = '=QQLqLQ20sH' CACHE_ENTRY_SIZE = struct.calcsize(CACHE_ENTRY_FMT) class Stat: def __init__(self): self.name = b"" self.ctime = 0 self.mtime = 0 self.dev = 0 self.ino = 0 self.mode = 0 self.size = 0 self.digest = b'' def __init__(self, cachePath): self.__cachePath = cachePath self.__cacheDir = os.path.dirname(cachePath) def open(self): self.__inPos = 0 self.__inPosOld = 0 self.__outFile = None try: if os.path.exists(self.__cachePath): self.__inFile = open(self.__cachePath, "rb") sig = self.__inFile.read(4) if sig == DirHasher.FileIndex.SIGNATURE: self.__mismatch = False self.__inPos = self.__inPosOld = 4 else: logging.getLogger(__name__).info( "Wrong signature at '%s': %s", self.__cachePath, sig) self.__inFile.close() self.__inFile = None self.__mismatch = True else: self.__inFile = None self.__mismatch = True except OSError as e: raise BuildError("Error opening hash cache: " + str(e)) self.__current = DirHasher.FileIndex.Stat() def close(self): try: if self.__inFile: self.__inFile.close() if self.__outFile: self.__outFile.close() os.replace(self.__outFile.name, self.__cachePath) except OSError as e: raise BuildError("Error closing hash cache: " + str(e)) def __readEntry(self): if not self.__inFile: return False raw = self.__inFile.read(DirHasher.FileIndex.CACHE_ENTRY_SIZE) if len(raw) < DirHasher.FileIndex.CACHE_ENTRY_SIZE: return False e = self.__current (e.ctime, e.mtime, e.dev, e.ino, e.mode, e.size, e.digest, nameLen) = struct.unpack(DirHasher.FileIndex.CACHE_ENTRY_FMT, raw) e.name = self.__inFile.read(nameLen) self.__inPosOld = self.__inPos self.__inPos += DirHasher.FileIndex.CACHE_ENTRY_SIZE + nameLen return True def __writeEntry(self, name, st, digest): if not self.__outFile: self.__outFile = NamedTemporaryFile(mode="wb", dir=self.__cacheDir, delete=False) if self.__inFile: pos = self.__inFile.tell() self.__inFile.seek(0) self.__outFile.write(self.__inFile.read(self.__inPosOld)) self.__inFile.seek(pos) else: self.__outFile.write(DirHasher.FileIndex.SIGNATURE) self.__outFile.write( struct.pack(DirHasher.FileIndex.CACHE_ENTRY_FMT, float2ns(st.st_ctime), float2ns(st.st_mtime), st.st_dev, st.st_ino, st.st_mode, st.st_size, digest, len(name))) self.__outFile.write(name) def __match(self, name, st): while self.__current.name < name: if not self.__readEntry(): break e = self.__current res = ((e.name == name) and (e.ctime == float2ns(st.st_ctime)) and (e.mtime == float2ns(st.st_mtime)) and (e.dev == st.st_dev) and (e.ino == st.st_ino) and (e.mode == st.st_mode) and (e.size == st.st_size)) #if not res: print("Mismatch", e.name, name) return res def check(self, prefix, name, st, process): if self.__match(name, st): digest = self.__current.digest else: digest = process(os.path.join(prefix, name)) self.__mismatch = True if self.__mismatch: self.__writeEntry(name, st, digest) return digest
""" def sample_run(input_file_path): with cp2k.ForceEnvironment(input_file_path, "/dev/null") as fenv: print("potential energy: {:e}".format(fenv.potential_energy)) print("calculating energy..") fenv.calc_energy() print(".. done!") print("potential energy: {:e}".format(fenv.potential_energy)) print("positions:") pos = fenv.positions print(pos) print("zeroify positions..") fenv.positions = np.zeros(pos.shape) print("positions:") print(fenv.positions) if __name__ == "__main__": cp2k.init() tempfile = NamedTemporaryFile(mode="w+") tempfile.write(TEST_FILE_CONTENT) tempfile.flush() sample_run(tempfile.name) cp2k.finalize()
def _write_local_data_files(self, cursor): """ Takes a cursor, and writes results to a local file. :return: A dictionary where keys are filenames to be used as object names in GCS, and values are file handles to local files that contain the data for the GCS objects. """ schema = list( map(lambda schema_tuple: schema_tuple[0], cursor.description)) col_type_dict = self._get_col_type_dict() file_no = 0 tmp_file_handle = NamedTemporaryFile(delete=True) if self.export_format == 'csv': file_mime_type = 'text/csv' elif self.export_format == 'parquet': file_mime_type = 'application/octet-stream' else: file_mime_type = 'application/json' files_to_upload = [{ 'file_name': self.filename.format(file_no), 'file_handle': tmp_file_handle, 'file_mime_type': file_mime_type, }] self.log.info("Current file count: %d", len(files_to_upload)) if self.export_format == 'csv': csv_writer = self._configure_csv_file(tmp_file_handle, schema) if self.export_format == 'parquet': parquet_schema = self._convert_parquet_schema(cursor) parquet_writer = self._configure_parquet_file( tmp_file_handle, parquet_schema) for row in cursor: # Convert datetime objects to utc seconds, and decimals to floats. # Convert binary type object to string encoded with base64. row = self.convert_types(schema, col_type_dict, row) if self.export_format == 'csv': if self.null_marker is not None: row = [ value if value is not None else self.null_marker for value in row ] csv_writer.writerow(row) elif self.export_format == 'parquet': if self.null_marker is not None: row = [ value if value is not None else self.null_marker for value in row ] row_pydic = {col: [value] for col, value in zip(schema, row)} tbl = pa.Table.from_pydict(row_pydic, parquet_schema) parquet_writer.write_table(tbl) else: row_dict = dict(zip(schema, row)) tmp_file_handle.write( json.dumps(row_dict, sort_keys=True, ensure_ascii=False).encode("utf-8")) # Append newline to make dumps BigQuery compatible. tmp_file_handle.write(b'\n') # Stop if the file exceeds the file size limit. if tmp_file_handle.tell() >= self.approx_max_file_size_bytes: file_no += 1 tmp_file_handle = NamedTemporaryFile(delete=True) files_to_upload.append({ 'file_name': self.filename.format(file_no), 'file_handle': tmp_file_handle, 'file_mime_type': file_mime_type, }) self.log.info("Current file count: %d", len(files_to_upload)) if self.export_format == 'csv': csv_writer = self._configure_csv_file( tmp_file_handle, schema) if self.export_format == 'parquet': parquet_writer = self._configure_parquet_file( tmp_file_handle, parquet_schema) return files_to_upload
class K8SCopyToPod(K8SCopy): """ Copy files/directory from local filesystem into remote Pod """ def __init__(self, module, client): super(K8SCopyToPod, self).__init__(module, client) self.files_to_copy = list() def run_from_pod(self, command): response = stream(self.api_instance.connect_get_namespaced_pod_exec, self.name, self.namespace, command=command, stderr=True, stdin=False, stdout=True, tty=False, _preload_content=False, **self.container_arg) errors = [] while response.is_open(): response.update(timeout=1) if response.peek_stderr(): errors.append(response.read_stderr()) response.close() err = response.read_channel(ERROR_CHANNEL) err = yaml.safe_load(err) response.close() if err['status'] != 'Success': self.module.fail_json( msg="Failed to run {0} on Pod.".format(command), errors=errors) def is_remote_path_dir(self): pod_command = ['test', '-d', self.remote_path] response = stream(self.api_instance.connect_get_namespaced_pod_exec, self.name, self.namespace, command=pod_command, stdout=True, stderr=True, stdin=False, tty=False, _preload_content=False, **self.container_arg) while response.is_open(): response.update(timeout=1) err = response.read_channel(ERROR_CHANNEL) err = yaml.safe_load(err) response.close() if err['status'] == 'Success': return True return False def close_temp_file(self): if self.named_temp_file: self.named_temp_file.close() def run(self): try: # remove trailing slash from destination path dest_file = self.remote_path.rstrip("/") src_file = self.local_path self.named_temp_file = None if self.content: self.named_temp_file = NamedTemporaryFile(mode="w") self.named_temp_file.write(self.content) self.named_temp_file.flush() src_file = self.named_temp_file.name else: if not os.path.exists(self.local_path): self.module.fail_json( msg="{0} does not exist in local filesystem".format( self.local_path)) if not os.access(self.local_path, os.R_OK): self.module.fail_json( msg="{0} not readable".format(self.local_path)) if self.is_remote_path_dir(): if self.content: self.module.fail_json( msg= "When content is specified, remote path should not be an existing directory" ) else: dest_file = os.path.join(dest_file, os.path.basename(src_file)) if self.no_preserve: tar_command = [ 'tar', '--no-same-permissions', '--no-same-owner', '-xmf', '-' ] else: tar_command = ['tar', '-xmf', '-'] if dest_file.startswith("/"): tar_command.extend(['-C', '/']) response = stream( self.api_instance.connect_get_namespaced_pod_exec, self.name, self.namespace, command=tar_command, stderr=True, stdin=True, stdout=True, tty=False, _preload_content=False, **self.container_arg) with TemporaryFile() as tar_buffer: with tarfile.open(fileobj=tar_buffer, mode='w') as tar: tar.add(src_file, dest_file) tar_buffer.seek(0) commands = [] # push command in chunk mode size = 1024 * 1024 while True: data = tar_buffer.read(size) if not data: break commands.append(data) stderr, stdout = [], [] while response.is_open(): if response.peek_stdout(): stdout.append(response.read_stdout().rstrip("\n")) if response.peek_stderr(): stderr.append(response.read_stderr().rstrip("\n")) if commands: cmd = commands.pop(0) response.write_stdin(cmd) else: break response.close() if stderr: self.close_temp_file() self.module.fail_json( command=tar_command, msg= "Failed to copy local file/directory into Pod due to: {0}" .format(''.join(stderr))) self.close_temp_file() if self.content: self.module.exit_json( changed=True, result="Content successfully copied into {0} on remote Pod" .format(self.remote_path)) self.module.exit_json( changed=True, result="{0} successfully copied into remote Pod into {1}". format(self.local_path, self.remote_path)) except Exception as e: self.module.fail_json( msg="Failed to copy local file/directory into Pod due to: {0}". format(to_native(e)))
def run_long_running(params, is_test=False): """ Start the long running server :param params: Demisto params :param is_test: Indicates whether it's test-module run or regular run :return: None """ certificate: str = params.get('certificate', '') private_key: str = params.get('key', '') certificate_path = str() private_key_path = str() try: port = get_params_port(params) ssl_args = dict() if (certificate and not private_key) or (private_key and not certificate): raise DemistoException( 'If using HTTPS connection, both certificate and private key should be provided.' ) if certificate and private_key: certificate_file = NamedTemporaryFile(delete=False) certificate_path = certificate_file.name certificate_file.write(bytes(certificate, 'utf-8')) certificate_file.close() private_key_file = NamedTemporaryFile(delete=False) private_key_path = private_key_file.name private_key_file.write(bytes(private_key, 'utf-8')) private_key_file.close() context = SSLContext(PROTOCOL_TLSv1_2) context.load_cert_chain(certificate_path, private_key_path) ssl_args['ssl_context'] = context demisto.debug('Starting HTTPS Server') else: demisto.debug('Starting HTTP Server') server = WSGIServer(('0.0.0.0', port), APP, **ssl_args, log=DEMISTO_LOGGER) if is_test: server_process = Process(target=server.serve_forever) server_process.start() time.sleep(5) server_process.terminate() else: server.serve_forever() except SSLError as e: ssl_err_message = f'Failed to validate certificate and/or private key: {str(e)}' demisto.error(ssl_err_message) raise ValueError(ssl_err_message) except Exception as e: demisto.error(f'An error occurred in long running loop: {str(e)}') raise ValueError(str(e)) finally: if certificate_path: os.unlink(certificate_path) if private_key_path: os.unlink(private_key_path)
class FileSourceTest(unittest.TestCase): def setUp(self): dirlist = ["plugins/datagraph"] self.loader = PluginsLoader(Application) self.loader.load(dirlist) self._dataFile = NamedTemporaryFile(mode='w', delete=False, encoding='utf8') def tearDown(self): self.loader.clear() self._dataFile.close() os.remove(self._dataFile.name) self._dataFile = None def testFileSource_empty_01(self): from datagraph.datasources import FileSource data = '''''' self._dataFile.write(data) self._dataFile.flush() source = FileSource(self._dataFile.name) items = list(source.getRowsIterator()) self.assertEqual(len(items), 0, items) def testFileSource_empty_02(self): from datagraph.datasources import FileSource data = ''' ''' self._dataFile.write(data) self._dataFile.flush() source = FileSource(self._dataFile.name) items = list(source.getRowsIterator()) self.assertEqual(len(items), 0) def testFileSource_single_01(self): from datagraph.datasources import FileSource data = '''123''' self._dataFile.write(data) self._dataFile.flush() source = FileSource(self._dataFile.name) items = list(source.getRowsIterator()) self.assertEqual(len(items), 1) self.assertEqual(items[0], ['123']) def testFileSource_single_02(self): from datagraph.datasources import FileSource data = '''123 ''' self._dataFile.write(data) self._dataFile.flush() source = FileSource(self._dataFile.name) items = list(source.getRowsIterator()) self.assertEqual(len(items), 1) self.assertEqual(items[0], ['123']) def testFileSource_single_03(self): from datagraph.datasources import FileSource data = ''' 123 ''' self._dataFile.write(data) self._dataFile.flush() source = FileSource(self._dataFile.name) items = list(source.getRowsIterator()) self.assertEqual(len(items), 1) self.assertEqual(items[0], ['123']) def testFileSource_single_04(self): from datagraph.datasources import FileSource data = '''123 ''' self._dataFile.write(data) self._dataFile.flush() source = FileSource(self._dataFile.name) items = list(source.getRowsIterator()) self.assertEqual(len(items), 1) self.assertEqual(items[0], ['123']) def testFileSource_single_05(self): from datagraph.datasources import FileSource data = '''123 ''' self._dataFile.write(data) self._dataFile.flush() source = FileSource(self._dataFile.name) items = list(source.getRowsIterator()) self.assertEqual(len(items), 1) self.assertEqual(items[0], ['123']) def testFileSource_one_col_01(self): from datagraph.datasources import FileSource data = '''123 234 456''' self._dataFile.write(data) self._dataFile.flush() source = FileSource(self._dataFile.name) items = list(source.getRowsIterator()) self.assertEqual(len(items), 3) self.assertEqual(items[0], ['123']) self.assertEqual(items[1], ['234']) self.assertEqual(items[2], ['456']) def testFileSource_one_col_02(self): from datagraph.datasources import FileSource data = '''123 234 456 ''' self._dataFile.write(data) self._dataFile.flush() source = FileSource(self._dataFile.name) items = list(source.getRowsIterator()) self.assertEqual(len(items), 3) self.assertEqual(items[0], ['123']) self.assertEqual(items[1], ['234']) self.assertEqual(items[2], ['456']) def testFileSource_one_col_03(self): from datagraph.datasources import FileSource data = '''123 234 456 ''' self._dataFile.write(data) self._dataFile.flush() source = FileSource(self._dataFile.name) items = list(source.getRowsIterator()) self.assertEqual(len(items), 3) self.assertEqual(items[0], ['123']) self.assertEqual(items[1], ['234']) self.assertEqual(items[2], ['456']) def testFileSource_one_col_04(self): from datagraph.datasources import FileSource data = ''' 123 234 4560 1000 ''' self._dataFile.write(data) self._dataFile.flush() source = FileSource(self._dataFile.name) items = list(source.getRowsIterator()) self.assertEqual(len(items), 4) self.assertEqual(items[0], ['123']) self.assertEqual(items[1], ['234']) self.assertEqual(items[2], ['4560']) self.assertEqual(items[3], ['1000']) def testFileSource_col_01(self): from datagraph.datasources import FileSource data = '''123 456 789 234 100 111 456 101 99 -10\t55 66 20 30 40 ''' self._dataFile.write(data) self._dataFile.flush() source = FileSource(self._dataFile.name) items = list(source.getRowsIterator()) self.assertEqual(len(items), 5) self.assertEqual(items[0], ['123', '456', '789']) self.assertEqual(items[1], ['234', '100', '111']) self.assertEqual(items[2], ['456', '101', '99']) self.assertEqual(items[3], ['-10', '55', '66']) self.assertEqual(items[4], ['20', '30', '40']) def testFileSource_col_02(self): from datagraph.datasources import FileSource data = ''' 123 456 789 234 100 111 456 101 99 -10\t55 66 20 30 40 ''' self._dataFile.write(data) self._dataFile.flush() source = FileSource(self._dataFile.name) items = list(source.getRowsIterator()) self.assertEqual(len(items), 5) self.assertEqual(items[0], ['123', '456', '789']) self.assertEqual(items[1], ['234', '100', '111']) self.assertEqual(items[2], ['456', '101', '99']) self.assertEqual(items[3], ['-10', '55', '66']) self.assertEqual(items[4], ['20', '30', '40']) def testFileSource_col_03(self): from datagraph.datasources import FileSource data = '''123 456 789 234 100 111 456 101 ''' self._dataFile.write(data) self._dataFile.flush() source = FileSource(self._dataFile.name) items = list(source.getRowsIterator()) self.assertEqual(len(items), 2) self.assertEqual(items[0], ['123', '456', '789']) self.assertEqual(items[1], ['234', '100', '111']) def testFileSource_col_04(self): from datagraph.datasources import FileSource data = '''123 456 789 234 100 111 456 101 99 78 ''' self._dataFile.write(data) self._dataFile.flush() source = FileSource(self._dataFile.name) items = list(source.getRowsIterator()) self.assertEqual(len(items), 2) self.assertEqual(items[0], ['123', '456', '789']) self.assertEqual(items[1], ['234', '100', '111']) def testFileSource_skiprows_01(self): from datagraph.datasources import FileSource data = ''' 123 456 789 234 100 111 456 101 99 ''' self._dataFile.write(data) self._dataFile.flush() source = FileSource(self._dataFile.name) items = list(source.getRowsIterator()) self.assertEqual(len(items), 3) self.assertEqual(items[0], ['123', '456', '789']) self.assertEqual(items[1], ['234', '100', '111']) def testFileSource_skiprows_02(self): from datagraph.datasources import FileSource data = '''Абырвалг ---- 123 456 789 234 100 111 456 101 99 ''' self._dataFile.write(data) self._dataFile.flush() source = FileSource(self._dataFile.name, skiprows=2) items = list(source.getRowsIterator()) self.assertEqual(len(items), 3) self.assertEqual(items[0], ['123', '456', '789']) self.assertEqual(items[1], ['234', '100', '111']) def testFileSource_skiprows_03(self): from datagraph.datasources import FileSource data = '''Абырвалг Абыр ---- 123 456 789 234 100 111 456 101 99 ''' self._dataFile.write(data) self._dataFile.flush() source = FileSource(self._dataFile.name, skiprows=0) items = list(source.getRowsIterator()) self.assertEqual(len(items), 1) self.assertEqual(items[0], ['Абырвалг', 'Абыр']) def testFileSource_skiprows_04(self): from datagraph.datasources import FileSource data = '''Абырвалг Абыр ---- 123 456 789 234 100 111 456 101 99 ''' self._dataFile.write(data) self._dataFile.flush() source = FileSource(self._dataFile.name, skiprows=5) items = list(source.getRowsIterator()) self.assertEqual(len(items), 0) def testFileSource_invalid_01(self): from datagraph.datasources import FileSource source = FileSource('invalid_fname.txt') items = list(source.getRowsIterator()) self.assertEqual(len(items), 0) def testFileSource_invalid_02(self): from datagraph.datasources import FileSource source = FileSource('testdata/samplefiles/image.png') items = list(source.getRowsIterator()) self.assertEqual(len(items), 0) def testFileSource_invalid_03(self): from datagraph.datasources import FileSource source = FileSource('testdata/samplefiles/invalid.exe') items = list(source.getRowsIterator()) self.assertEqual(len(items), 0) def testFileSource_invalid_04(self): from datagraph.datasources import FileSource source = FileSource('testdata/samplefiles/text_1251.txt') items = list(source.getRowsIterator()) self.assertEqual(len(items), 0)