Ejemplo n.º 1
0
    def _store_output(self):
        log.debug("Storing output ...")

        tempfile = open(self._tempfile, 'r')

        log.debug("Current position: " + str(self._current_position))

        tempfile.seek(self._current_position)
        finished = False

        started = False
        while self.silent and not finished:
            lines = tempfile.readlines()
            self._current_position = tempfile.tell()
            for line in lines:
                line = line.strip()
                log.debug("Started: %s | Line: %s" %(str(started), line))
                log.debug("Condition: " +str(line.strip().endswith(self.STARTED)))
                if not started:
                    started = line.endswith(self.STARTED)
                    continue

                if line.endswith(self.FINISHED):
                    finished = True
                    break
                line = self._prepare_output(line)
                if line:
                    self.output.append(line)

        tempfile.close()
Ejemplo n.º 2
0
def unpickle(tempfile):
    """unpickle(tempfile) - unpickles object from open filed object tempfile oposite of pickle() returns original pickled object."""
    startTime = time.time()
    tempfile.seek(0)
    object = cPickle.load(tempfile)
    tempfile.seek(0)
    print "TIME: unpickle() " + str(time.time() - startTime) + " s"
    return object
Ejemplo n.º 3
0
def _load_db_to_memory(in_db_path):
    # Read database to tempfile
    conn = sqlite3.connect('file:%s?mode=ro' % in_db_path, uri=True)
    tempfile = io.StringIO()
    for line in conn.iterdump():
        tempfile.write('%s\n' % line)
    conn.close()
    tempfile.seek(0)

    # Create a database in memory and import from tempfile
    conn = sqlite3.connect(":memory:")
    conn.cursor().executescript(tempfile.read())
    return conn
Ejemplo n.º 4
0
    def __from_wave__(cls, filename, wave_filename, compression=None):
        if (str(compression) not in cls.COMPRESSION_MODES):
            compression = cls.DEFAULT_COMPRESSION

        #mppenc requires files to end with .mpc for some reason
        if (not filename.endswith(".mpc")):
            import tempfile
            actual_filename = filename
            tempfile = tempfile.NamedTemporaryFile(suffix=".mpc")
            filename = tempfile.name
        else:
            actual_filename = tempfile = None

        ###Musepack SV7###
        #sub = subprocess.Popen([BIN['mppenc'],
        #                        "--silent",
        #                        "--overwrite",
        #                        "--%s" % (compression),
        #                        wave_filename,
        #                        filename],
        #                       preexec_fn=ignore_sigint)

        ###Musepack SV8###
        sub = subprocess.Popen([BIN['mpcenc'],
                                "--silent",
                                "--overwrite",
                                "--%s" % (compression),
                                wave_filename,
                                filename])

        if (sub.wait() == 0):
            if (tempfile is not None):
                filename = actual_filename
                f = file(filename, 'wb')
                tempfile.seek(0, 0)
                transfer_data(tempfile.read, f.write)
                f.close()
                tempfile.close()

            return MusepackAudio(filename)
        else:
            if (tempfile is not None):
                tempfile.close()
            raise EncodingError(u"error encoding file with mpcenc")
Ejemplo n.º 5
0
    def __from_wave__(cls, filename, wave_filename, compression=None):
        if (str(compression) not in cls.COMPRESSION_MODES):
            compression = cls.DEFAULT_COMPRESSION

        #mppenc requires files to end with .mpc for some reason
        if (not filename.endswith(".mpc")):
            import tempfile
            actual_filename = filename
            tempfile = tempfile.NamedTemporaryFile(suffix=".mpc")
            filename = tempfile.name
        else:
            actual_filename = tempfile = None

        ###Musepack SV7###
        #sub = subprocess.Popen([BIN['mppenc'],
        #                        "--silent",
        #                        "--overwrite",
        #                        "--%s" % (compression),
        #                        wave_filename,
        #                        filename],
        #                       preexec_fn=ignore_sigint)

        ###Musepack SV8###
        sub = subprocess.Popen([BIN['mpcenc'],
                                "--silent",
                                "--overwrite",
                                "--%s" % (compression),
                                wave_filename,
                                filename])

        if (sub.wait() == 0):
            if (tempfile is not None):
                filename = actual_filename
                f = file(filename, 'wb')
                tempfile.seek(0, 0)
                transfer_data(tempfile.read, f.write)
                f.close()
                tempfile.close()

            return MusepackAudio(filename)
        else:
            if (tempfile is not None):
                tempfile.close()
            raise EncodingError(u"error encoding file with mpcenc")
Ejemplo n.º 6
0
def main():
    parser = argparse.ArgumentParser(description='Amazon S3 Backup.')
    parser.add_argument('config_file', help='JSON configuration file')
    args = parser.parse_args()

    try:
        config = read_config(args.config_file)
    except ConfigError as e:
        logger.exception("Error reading configuration file")
        sys.exit(1)

    logger.debug(config)

    logger.debug("Creating temporary tar file")
    tempfile = create_tempfile()

    try:
        create_tarfile(tempfile, config.get("sources"),
                       config.get("exclusions"))
    except TarFileCreationError as e:
        logger.error(f"Error creating tar file: {e}")
        sys.exit(1)

    tempfile.seek(0)  # Needed for S3 upload to work

    logger.debug("Creating S3 bucket object")
    s3_bucket = S3Bucket(
        config.get("s3").get("bucket_name"),
        config.get("s3").get("bucket_region"),
        config.get("s3").get("storage_class"))

    s3_metadata = {
        "sources": ",".join(config.get("sources")),
        "exclusions": ",".join(config.get("exclusions"))
    }
    s3_key = generate_s3_key(config.get("tarfile_name_prefix"))

    logger.info("Uploading file to S3")
    s3_bucket.upload_fileobj(tempfile, s3_key, s3_metadata)

    logger.debug("Closing temporary tar file")
    tempfile.close()

    logger.info("Done!")
Ejemplo n.º 7
0
    def testTrollitaireConverter(self):
        """Test that the Trollitaire converter works properly."""
        # First test case (no surprises)
        tempfile = cStringIO.StringIO()
        with open(old_troll_file) as infile:
            trollconvert.convert_draft_file(infile, tempfile)

        expected = open(converted_troll).read()
        tempfile.seek(0)
        result = tempfile.read()
        self.assertEqual(expected, result)

        # Second test case (has an UNDO)
        tempfile = cStringIO.StringIO()
        with open(old_troll_file2) as infile:
            trollconvert.convert_draft_file(infile, tempfile)

        expected = open(converted_troll2).read()
        tempfile.seek(0)
        result = tempfile.read()
        self.assertEqual(expected, result)
Ejemplo n.º 8
0
            sub.stdin.close()
            sub.wait()
            cls.__unlink__(filename)
            raise err

        try:
            pcmreader.close()
        except DecodingError, err:
            raise EncodingError(err.error_message)
        sub.stdin.close()

        if sub.wait() == 0:
            if tempfile is not None:
                filename = actual_filename
                f = file(filename, "wb")
                tempfile.seek(0, 0)
                transfer_data(tempfile.read, f.write)
                f.close()
                tempfile.close()

            return M4AAudio(filename)
        else:
            if tempfile is not None:
                tempfile.close()
            raise EncodingError(u"unable to write file with faac")

    @classmethod
    def can_add_replay_gain(cls):
        """Returns False."""

        return False
Ejemplo n.º 9
0
    def from_pcm(cls,
                 filename,
                 pcmreader,
                 compression=None,
                 total_pcm_frames=None):
        """encodes a new file from PCM data

        takes a filename string, PCMReader object,
        optional compression level string and optional
        total_pcm_frames integer
        encodes a new audio file from pcmreader's data
        at the given filename with the specified compression level
        and returns a new M4AAudio object"""

        import subprocess
        import os
        from audiotools import PCMConverter
        from audiotools import transfer_data
        from audiotools import transfer_framelist_data
        from audiotools import ignore_sigint
        from audiotools import EncodingError
        from audiotools import DecodingError
        from audiotools import ChannelMask
        from audiotools import __default_quality__

        if ((compression is None)
                or (compression not in cls.COMPRESSION_MODES)):
            compression = __default_quality__(cls.NAME)

        if pcmreader.channels > 2:
            pcmreader = PCMConverter(pcmreader,
                                     sample_rate=pcmreader.sample_rate,
                                     channels=2,
                                     channel_mask=ChannelMask.from_channels(2),
                                     bits_per_sample=pcmreader.bits_per_sample)

        # faac requires files to end with .m4a for some reason
        if not filename.endswith(".m4a"):
            import tempfile
            actual_filename = filename
            tempfile = tempfile.NamedTemporaryFile(suffix=".m4a")
            filename = tempfile.name
        else:
            actual_filename = tempfile = None

        sub = subprocess.Popen(
            [
                BIN['faac'], "-q", compression, "-P", "-R",
                str(pcmreader.sample_rate), "-B",
                str(pcmreader.bits_per_sample), "-C",
                str(pcmreader.channels), "-X", "-o", filename, "-"
            ],
            stdin=subprocess.PIPE,
            stderr=subprocess.DEVNULL
            if hasattr(subprocess, "DEVNULL") else open(os.devnull, "wb"),
            stdout=subprocess.DEVNULL
            if hasattr(subprocess, "DEVNULL") else open(os.devnull, "wb"),
            preexec_fn=ignore_sigint)
        # Note: faac handles SIGINT on its own,
        # so trying to ignore it doesn't work like on most other encoders.

        try:
            if total_pcm_frames is not None:
                from audiotools import CounterPCMReader
                pcmreader = CounterPCMReader(pcmreader)

            transfer_framelist_data(pcmreader, sub.stdin.write)

            if ((total_pcm_frames is not None)
                    and (total_pcm_frames != pcmreader.frames_written)):
                from audiotools.text import ERR_TOTAL_PCM_FRAMES_MISMATCH
                raise EncodingError(ERR_TOTAL_PCM_FRAMES_MISMATCH)

        except (ValueError, IOError) as err:
            sub.stdin.close()
            sub.wait()
            cls.__unlink__(filename)
            raise EncodingError(str(err))
        except Exception:
            sub.stdin.close()
            sub.wait()
            cls.__unlink__(filename)
            raise

        sub.stdin.close()

        if sub.wait() == 0:
            if tempfile is not None:
                filename = actual_filename
                f = open(filename, 'wb')
                tempfile.seek(0, 0)
                transfer_data(tempfile.read, f.write)
                f.close()
                tempfile.close()

            return M4AAudio(filename)
        else:
            if tempfile is not None:
                tempfile.close()
            raise EncodingError(u"unable to write file with faac")
Ejemplo n.º 10
0
    def from_pcm(cls, filename, pcmreader,
                 compression=None, total_pcm_frames=None):
        """encodes a new file from PCM data

        takes a filename string, PCMReader object,
        optional compression level string and optional
        total_pcm_frames integer
        encodes a new audio file from pcmreader's data
        at the given filename with the specified compression level
        and returns a new M4AAudio object"""

        import subprocess
        import os
        from audiotools import PCMConverter
        from audiotools import transfer_data
        from audiotools import transfer_framelist_data
        from audiotools import ignore_sigint
        from audiotools import EncodingError
        from audiotools import DecodingError
        from audiotools import ChannelMask
        from audiotools import __default_quality__

        if ((compression is None) or (compression not in
                                      cls.COMPRESSION_MODES)):
            compression = __default_quality__(cls.NAME)

        if pcmreader.bits_per_sample not in {8, 16, 24}:
            from audiotools import UnsupportedBitsPerSample
            pcmreader.close()
            raise UnsupportedBitsPerSample(filename, pcmreader.bits_per_sample)

        if pcmreader.channels > 2:
            pcmreader = PCMConverter(pcmreader,
                                     sample_rate=pcmreader.sample_rate,
                                     channels=2,
                                     channel_mask=ChannelMask.from_channels(2),
                                     bits_per_sample=pcmreader.bits_per_sample)

        # faac requires files to end with .m4a for some reason
        if not filename.endswith(".m4a"):
            import tempfile
            actual_filename = filename
            tempfile = tempfile.NamedTemporaryFile(suffix=".m4a")
            filename = tempfile.name
        else:
            actual_filename = tempfile = None

        sub = subprocess.Popen(
            [BIN['faac'],
             "-q", compression,
             "-P",
             "-R", str(pcmreader.sample_rate),
             "-B", str(pcmreader.bits_per_sample),
             "-C", str(pcmreader.channels),
             "-X",
             "-o", filename,
             "-"],
            stdin=subprocess.PIPE,
            stderr=subprocess.DEVNULL if hasattr(subprocess, "DEVNULL") else
            open(os.devnull, "wb"),
            stdout=subprocess.DEVNULL if hasattr(subprocess, "DEVNULL") else
            open(os.devnull, "wb"),
            preexec_fn=ignore_sigint)
        # Note: faac handles SIGINT on its own,
        # so trying to ignore it doesn't work like on most other encoders.

        try:
            if total_pcm_frames is not None:
                from audiotools import CounterPCMReader
                pcmreader = CounterPCMReader(pcmreader)

            transfer_framelist_data(pcmreader, sub.stdin.write)

            if ((total_pcm_frames is not None) and
                (total_pcm_frames != pcmreader.frames_written)):
                from audiotools.text import ERR_TOTAL_PCM_FRAMES_MISMATCH
                raise EncodingError(ERR_TOTAL_PCM_FRAMES_MISMATCH)

        except (ValueError, IOError) as err:
            sub.stdin.close()
            sub.wait()
            cls.__unlink__(filename)
            raise EncodingError(str(err))
        except Exception:
            sub.stdin.close()
            sub.wait()
            cls.__unlink__(filename)
            raise

        sub.stdin.close()

        if sub.wait() == 0:
            if tempfile is not None:
                filename = actual_filename
                f = open(filename, 'wb')
                tempfile.seek(0, 0)
                transfer_data(tempfile.read, f.write)
                f.close()
                tempfile.close()

            return M4AAudio(filename)
        else:
            if tempfile is not None:
                tempfile.close()
            raise EncodingError(u"unable to write file with faac")
Ejemplo n.º 11
0
    resource_link_dict = core.resource_link_dict
    core.remove_links(resource_link_dict.keys())
    core.ignored_resources = []
    remote_resources = core.remote_resources

    # Write out resources in Dropbox folder in temp file
    json_obj = dict.fromkeys(remote_resources, " ")
    tempfile = tempfile.NamedTemporaryFile(delete=False)
    json.dump(json_obj, tempfile, indent=4, separators=(",", "\t:\t"))
    tempfile.flush()

    # Open editor to let user set what should be installed
    editor = os.environ.get("EDITOR", "vim")
    return_code = subprocess.call([editor, tempfile.name])
    tempfile.seek(0)
    desired_links = json.load(tempfile)
    tempfile.close()

    # Process user's decisions
    resources_to_ignore = [
        resource for resource in desired_links.keys()
        if resource not in remote_resources
    ]
    core.ignored_resources.append(resources_to_ignore)
    core.forge_links(desired_links)
# Print help
elif command_arg == "help":
    printHelp()
else:
    usage()
Ejemplo n.º 12
0
            actions['forward'] = []
            print 'Removing forwards...'
            continue
        elif x[0] == '.forward':
            actions['forward'] += [x[1]]
            print 'Adding forward:', x[1]
            continue
        elif x[0] == '.edit':
            default = ['base'] if len(x) == 1 else x[[1]:]
            print 'Entering editor...'
            with tempfile.NamedTemporaryFile(suffix=".tmp") as tempfile:
                tempfile.write(comment + '\n' + '-' * 10 + '\n' + '\n'.join(map(lambda x: responses[x], default)))
                tempfile.flush()
                call([EDITOR, tempfile.name])
                tempfile.flush()
                tempfile.seek(0)
                contents = tempfile.readlines()
                print contents
                reply = ''.join(contents[contents.index('-'*10+'\r\n') + 1 :])
        elif x[0] in responses:
            reply = responses[x[0]]
            break
        else:
            try:
                choice = int(x[0]) - 1
            except:
                continue
            if choice >= 0 and choice < len(actions['reply']):
                reply = responses[actions['reply'][choice]]
                break
Ejemplo n.º 13
0
def _download_and_unpack_file(url):
    """Downloads the database files created with setup-exfor-db.py as
    a tarball and unpacks them to the correct folder."""

    from tqdm import tqdm
    from glob import glob
    import requests
    import math
    import tarfile
    import tempfile
    import shutil

    # cleanup
    for f in [
        fullIndexFileName, fullErrorFileName,
        fullCoupledFileName, fullMonitoredFileName,
        fullReactionCountFileName, fullDBPath, dbTagFile
    ]:
        try:
            shutil.rmtree(f)
        except NotADirectoryError:
            os.remove(f)
        except FileNotFoundError:
            pass
    # Tag files:
    tag_files = [
        f for tag in ['X4-*', 'EXFOR-*'] for f in glob(os.path.join(DATAPATH, tag))
        ]
    for tagfile in tag_files:
        try:
            os.remove(tagfile)
        except FileNotFoundError:
            pass
        

    # Streaming, so we can iterate over the response.
    r = requests.get(url, stream=True)
    tarname = os.path.basename(url)

    # Total size in bytes.
    total_size = int(r.headers.get('content-length', 0))
    block_size = 1024 * 1024
    wrote = 0
    tempfile = tempfile.TemporaryFile()
    
    print('Downloading data file', tarname)
    for data in tqdm(r.iter_content(block_size), total=math.ceil(total_size // block_size),
                    unit='MB', unit_scale=True):
        wrote = wrote + len(data)
        tempfile.write(data)
    if total_size != 0 and wrote != total_size:
        raise Exception("ERROR, something went wrong")
    tempfile.flush()
    tempfile.seek(0)
    print('Decompressing archive', tarname)
    wrote = 0
    with tarfile.open(fileobj=tempfile, mode='r') as _tar:
        total = len(_tar.getmembers())
        for member in tqdm(_tar.getmembers(), total=total):
            wrote = wrote + len(data)
            _tar.extract(member, DATAPATH)
    tempfile.close()

    with open(dbTagFile,'wb') as f:
        print('Installed database version', dbTagFile)
        pass
Ejemplo n.º 14
0
	def pack(self):
		"""
		Packs the fasta and sqlite databases to remove redundancy and merge sqlite columns

		Returns:
		    void 
		"""
# First we handle the fastq file 
		unique_hash = set()
		redundant_file = fasta.file(self.database_fasta_file)
		temp = gzip.open(self.database_temp_file,'wb')
		
		for record in redundant_file.read():
			if not record['header'] in unique_hash :
				unique_hash.add(record['header'])
				temp.write('>'+record['header']+os.linesep+record['sequence']+os.linesep)
		
		os.rename(self.database_temp_file , self.database_fasta_file)

# Now the sqlite (http://stackoverflow.com/a/10856450)
		from StringIO import StringIO

		tempfile = StringIO()
		for line in self.sq3_connection.iterdump():
			tempfile.write('%s\n' % line)
		tempfile.seek(0)

		sq3_temp_connection 	= sql.connect(self.database_temp_file)
		sq3_temp_cursor 	= sq3_temp_connection.cursor()
		sq3_temp_cursor.execute("CREATE TABLE IF NOT EXISTS genes ( sequence_hash TEXT, genus TEXT, species TEXT, NCBItaxID TEXT, kegg_ontology TEXT , kegg_reaction TEXT , go_term TEXT,  kegg_map TEXT , sequence TEXT)")
		sq3_temp_connection.commit()

		sq3_memory_connection = sql.connect(":memory:")
		sq3_memory_cursor = sq3_memory_connection.cursor()
		sq3_memory_cursor.executescript(tempfile.read())
		sq3_memory_connection.commit()
		sq3_memory_connection.row_factory = sql.Row


		for h in unique_hash:
			sq3_memory_cursor.execute('SELECT * FROM genes WHERE `sequence_hash` = \'%s\'' % h)
			rows = sq3_memory_cursor.fetchall()
			rows_selected = len(rows)

			columns = tuple ([c[0] for c in sq3_memory_cursor.description])
			merge_dict = dict.fromkeys(columns)
			
			for r in rows:
				r=[str(x) if x else None for x in r ]
				incoming = dict(zip(columns,r))
				merge_dict = merge_insert_dicts(merge_dict,incoming)
			merge_dict = {i:j for i,j in merge_dict.items() if j != []}
			insert = 'INSERT INTO genes({}) VALUES ({})'.format(', '.join(merge_dict.keys()),', '.join('?' * len(merge_dict)))
			try:
				sq3_temp_cursor.execute(insert,merge_dict.values())
			except sql.Error as e:
				print merge_dict
				logger.warn(e)
				raise
			sq3_temp_connection.commit()
		
		sq3_temp_connection.close()