Example #1
0
    def run(self, message_queue):

        from .movie import EXIT_SUCCESS, EXIT_ERROR, EXIT_CANCEL

        arg0 = self.arg_list[0]
        import os.path
        if not os.path.isfile(arg0):
            self.exit_status = (-1, EXIT_ERROR,
                                'Could not find %s executable at %s' %
                                (self.ffmpeg_cmd, arg0))
            return

        # all output is on stderr, but Windows needs all standard I/O to
        # be redirected if one is, so stdout is a pipe too
        from io import StringIO
        out = StringIO()
        from subprocess import Popen, PIPE, DEVNULL
        p = Popen(self.arg_list, stdin=DEVNULL, stdout=PIPE, stderr=PIPE)
        out, err = p.communicate()
        exit_code = p.returncode

        status = EXIT_SUCCESS if exit_code == 0 else EXIT_ERROR
        ffmpeg_output = 'stdout:\n%s\nstderr:\n%s' % (out.decode('utf-8'),
                                                      err.decode('utf-8'))
        error = '' if exit_code == 0 else ffmpeg_output
        self.exit_status = (exit_code, status, error)

        if exit_code or self.verbose:
            self.session.logger.info(' '.join(self.arg_list) + '\n' +
                                     ffmpeg_output)

        self.remove_backwards_frames()
Example #2
0
    def __init__(self, f=sys.stdin, filename=None):
        """Initialize a tokenizer instance.

        @param f: The file to tokenize.  The default is sys.stdin.
        This parameter may also be a string, in which case the tokenizer
        will take its input from the contents of the string.
        @type f: file or string
        @param filename: the name of the filename that the L{where} method
        will return.
        @type filename: string
        """

        if isinstance(f, text_type):
            f = StringIO(f)
            if filename is None:
                filename = '<string>'
        elif isinstance(f, binary_type):
            f = StringIO(f.decode())
            if filename is None:
                filename = '<string>'
        else:
            if filename is None:
                if f is sys.stdin:
                    filename = '<stdin>'
                else:
                    filename = '<file>'
        self.file = f
        self.ungotten_char = None
        self.ungotten_token = None
        self.multiline = 0
        self.quoting = False
        self.eof = False
        self.delimiters = _DELIMITERS
        self.line_number = 1
        self.filename = filename
Example #3
0
    def parse(self, input):
        """Parse the given file or file source string."""
        if hasattr(input, 'name'):
            self.filename = input.name
        elif not getattr(self, 'filename', ''):
            self.filename = ''
        if hasattr(input, "read"):
            inisrc = input.read()
            input.close()
            input = inisrc

        if isinstance(input, bytes):
            if sys.version_info[0] == 3:
                input = StringIO(input.decode('utf-8'))
            else:
                input = BytesIO(input)
            self._inifile = INIConfig(input, optionxformvalue=None)
        else:
            self._inifile = INIConfig(open(input), optionxformvalue=None)

        for section in self._inifile:
            for entry in self._inifile[section]:
                source = self._dialect.unescape(self._inifile[section][entry])
                newunit = self.addsourceunit(source)
                newunit.addlocation("[%s]%s" % (section, entry))
def open_csv_url(page_url):
    try:
        csv_page = urlopen(page_url)
    except (HTTPError, URLError) as e:
        print('\033[1;31m Error occurred when request the url %s' % page_url)
        print(e)
        return None

    # 使用库在线读取,返回的csv_reader对象是列表,可迭代的
    print('\033[1;31m Using csv_reader to get the list_data.')
    csv_data = csv_page.read()  # 获取csv文件的内容
    csv_data = csv_data.decode('ascii', 'ignore')   # 将csv文件中的内容按ASCII解码
    csv_data = StringIO(csv_data)   # 将编码后的内容封装成StringIO对象,可以被Python的csv库处理
    csv_reader = csv.reader(csv_data)
    for row in csv_reader:
        print('\033[1;33m The album \'' + row[0] + '\'was released in ' + str(row[1]))
        print(row)

    # 使用库在线读取,返回的csv_dic_reader对象是字典,同时字段保存在csv_dic_reader.filenames里,字段同时作为字典的键
    print('\033[1;31m Using csv_dic_reader to get the dict_data.')
    csv_data = csv_page.read()  # 获取csv文件的内容
    csv_data = csv_data.decode('ascii', 'ignore')  # 将csv文件中的内容按ASCII解码
    csv_data = StringIO(csv_data)  # 将编码后的内容封装成StringIO对象,可以被Python的csv库处理
    csv_dic_reader = csv.DictReader(csv_data)
    print(csv_dic_reader.fieldnames)
    for row in csv_dic_reader:
        print('\033[1;33m The album \'' + row['Name'] + '\' was released in ' + row['Year'])
        print(row)
    def intersect_with_state(self, bed_file):
        chromstate_dir = "/home/zyang/Project/CRC/step45_chromatin_state_in_cluster/state13"
        df_list = []
        columns = []
        for one_file in os.listdir(chromstate_dir):

            if "segments.bed" in one_file:
                df = pd.read_csv(os.path.join(chromstate_dir, one_file),
                                 sep="\t",
                                 names=["chr", "start", "end", "state"])
                df = df[df.state != "E2"]
                df_tmp = self.writeToTmp(df)
                #tmp_result = "/home/zyang/Project/CRC/step45_chromatin_state_in_cluster/test.txt"
                cmd = "bedtools intersect -wao -a %s -b %s" % (bed_file,
                                                               df_tmp)
                out_string = subprocess.check_output([cmd], shell=True)
                out_string = StringIO(out_string.decode('utf-8'))
                df_result = pd.read_csv(out_string, sep="\t", \
                            names=["chr", "start", "end", "strand", "ensembl", "chr_state", "start_state", "end_state", "state", "overlap"])
                df_result = df_result[["ensembl", "state", "overlap"]]

                df_result = df_result[df_result.state != "."].reset_index()
                df_result = df_result.groupby(
                    ['ensembl', 'state'])["overlap"].sum().reset_index()

                idx = df_result.groupby([
                    'ensembl'
                ])['overlap'].transform(max) == df_result['overlap']
                df_result = df_result[idx]
                counts = collections.Counter(df_result.ensembl)
                ##some gene has several equit max value
                del_ensembl = []
                for key in counts:
                    if counts[key] > 1:
                        del_ensembl.append(key)
                df_result = df_result[~df_result["ensembl"].isin(del_ensembl)]
                print(len(set(df_result.ensembl)))
                print(len(df_result.ensembl))
                #df_result.to_csv("test1.txt", sep="\t", index=True)
                #os.remove(tmp_result)
                os.remove(df_tmp)
                df_result = df_result[["ensembl", "state"]]
                df_list.append(df_result)
                columns.append(one_file.split("_")[0])
                #print(df_result[0:5])

        df_merged = reduce(
            lambda left, right: pd.merge(
                left, right, on=['ensembl'], how='outer'), df_list)
        df_merged.index = df_merged["ensembl"]
        df_merged.drop(['ensembl'], axis=1, inplace=True)
        df_merged.columns = columns
        df_merged = df_merged[[
            "ctrl", "2weeks", "4weeks", "7weeks", "10weeks"
        ]]
        #df_merged.to_csv("state_summary_cluster.txt", sep="\t", header=True)
        return df_merged
Example #6
0
def detect_linesep(data):
    if isinstance(data, (str, memoryview)):
        data = StringIO(data)

    offset = data.decode('utf-8').tell()
    line = data.readline()
    data.seek(offset)

    if line[-2:] == "\r\n":
        return "\r\n"
    else:
        return line[-1]
 def parse(self, data):
     """Dummy csv parse"""
     try:
         data = StringIO(data.decode())
         reader = csv.DictReader(data)
         payment_returns = []
         for row in reader:
             payment_return = self.parse_payment_return(row)
             if len(payment_return['transactions']):
                 payment_returns.append(payment_return)
     except Exception:
         raise UserError(_("Couldn't load file data"))
     return payment_returns
Example #8
0
def send_request(method, path, data=None):
    out = StringIO()
    args = [
        "-X",
        method,
        f"http://127.0.0.1:8081/{path.lstrip('/')}",
        "-H",
        f"X-API-Key: {pm.config['environment']['PDNS_API_KEY']}",
    ]
    if data:
        args += ["-d", json.dumps(data)]

    out = pm.get_exec("powerdns", ["curl", *args])
    try:
        return json.loads(out or '"success!"')
    except Exception as e:
        exit(out.decode())
Example #9
0
def list_contents(rar_file_path):
    """
    Returns a list of the archive's contents.
    """

    assert os.path.isfile(rar_file_path) and rar_file_path.endswith('.rar')

    contents = []
    count = 0
    command = '"{unrar}" v -- "{file}"'
    command = command.format(unrar=config.UNRAR_PATH, file=rar_file_path)

    try:

        output = subprocess.check_output(command,
                                         stderr=subprocess.STDOUT,
                                         shell=True)

    except subprocess.CalledProcessError as e:

        output = e.output.decode(encoding='utf-8')
        msg = 'Error while listing archive contents: "{error_string}"'
        raise FileUtilsError(msg.format(error_string=output.strip()))

    else:

        output = StringIO(output.decode(encoding='utf-8'))
        parse = False
        for line in output.readlines():
            line_list = line.strip().split()
            # If the line is not empty...
            if line_list:
                # This marks the start and end of the section we want to parse
                if line_list[
                        0] == '-------------------------------------------------------------------------------':
                    parse = not parse
                    count = 0
                # If we're in the section of the output we want to parse...
                elif parse:
                    # Parse every other line (only the file paths)
                    if count % 2 == 0:
                        contents.append(line_list[0])
                    count += 1

    return contents
Example #10
0
def list_contents(rar_file_path):
    """
    Returns a list of the archive's contents.
    """

    assert os.path.isfile(rar_file_path) and rar_file_path.endswith('.rar')

    contents = []
    count = 0
    command = '"{unrar}" v -- "{file}"'
    command = command.format(unrar=config.UNRAR_PATH, file=rar_file_path)

    try:

        output = subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True)

    except subprocess.CalledProcessError as e:

        output = e.output.decode(encoding='utf-8')
        msg = 'Error while listing archive contents: "{error_string}"'
        raise FileUtilsError(msg.format(error_string=output.strip()))

    else:

        output = StringIO(output.decode(encoding='utf-8'))
        parse = False
        for line in output.readlines():
            line_list = line.strip().split()
            # If the line is not empty...
            if line_list:
                # This marks the start and end of the section we want to parse
                if line_list[0] == '-------------------------------------------------------------------------------':
                    parse = not parse
                    count = 0
                # If we're in the section of the output we want to parse...
                elif parse:
                    # Parse every other line (only the file paths)
                    if count % 2 == 0:
                        contents.append(line_list[0])
                    count += 1

    return contents
Example #11
0
    def parse(self, source):
        """Parses an SVG document, and returns the root element of it.

        Arguments:
            source (str, file): An URL or a file-like object of an SVG
                document.
        Returns:
            Element: A root element of the document.
        """
        if isinstance(source, str):
            data, headers = load(source)
            content_type = get_content_type(headers)
            if content_type is None:
                charset = 'utf-8'
            else:
                charset = content_type.get('charset', 'utf-8')
            data = StringIO(data.decode(charset))
        else:
            data = source
        tree = self._parser.parse(data)
        return tree.getroot()
Example #12
0
def parse_titpub(text, buf):
    pp = PortugueseRulesParser2()
    text = StringIO(text.decode('ISO-8859-1'))
    writer = csv.writer(buf)
    _drop_first_2 = dropwhile(lambda x: x[0] < 2, enumerate(text))
    _drop_empy = filter(lambda x: x[1].strip() is not '', _drop_first_2)
    for c, line in _drop_empy:
        row = line.split('@')
        tit = dict(titulo=row[0],
                   data_referencia=row[1],
                   codigo_selic=row[2],
                   data_base=row[3],
                   data_vencimento=row[4],
                   taxa_max=pp.parse(row[5]),
                   taxa_min=pp.parse(row[6]),
                   taxa_ind=pp.parse(row[7]),
                   pu=pp.parse(row[8]),
                   desvio_padrao=pp.parse(row[9]))
        writer.writerow(tit.values())
    date_str = datetime.strptime(tit['data_referencia'], '%Y%m%d') \
        .strftime('%Y-%m-%d')
    return date_str
Example #13
0
    def parse(self, input):
        """Parse the given file or file source string."""
        if hasattr(input, "name"):
            self.filename = input.name
        elif not getattr(self, "filename", ""):
            self.filename = ""
        if hasattr(input, "read"):
            inisrc = input.read()
            input.close()
            input = inisrc

        if isinstance(input, bytes):
            input = StringIO(input.decode("utf-8"))
            self._inifile = INIConfig(input, optionxformvalue=None)
        else:
            self._inifile = INIConfig(open(input), optionxformvalue=None)

        for section in self._inifile:
            for entry in self._inifile[section]:
                source = self._dialect.unescape(self._inifile[section][entry])
                newunit = self.addsourceunit(source)
                newunit.addlocation(f"[{section}]{entry}")
Example #14
0



zippy = gzip.GzipFile(fileobj=memfile)
uncompressed = zippy.read()


respData = resp.read()

#### !!!!
mysock = urllib.request.urlopen(url)
memfile = BytesIO(mysock.read())
f = gzip.GzipFile(fileobj=memfile)
r = f.read()
t = r.decode('utf-8')

###

import gzip
from io import BytesIO
import shutil
from ftplib import FTP

ftp = FTP('ftp.ncbi.nlm.nih.gov')
ftp.login('anonymous', '')

flo = BytesIO()

ftp.retrbinary('RETR /pubmed/id_list.txt.gz', flo.write)
Example #15
0
            l = str(l, errors='ignore')
            f.write(l)
            l = conn.recv(1024)
        f.close()
        print("Done Receiving")
        conn.close()
    elif bufferdata == "No":
        print("Wrong prediction!")
        globvar = globvar + 1

        # Receiving data from client
        buff = StringIO()  # Create the buffer
        while True:
            data = conn.recv(1)
            print(data)
            buff.write(data.decode())  # Append that segment to the buffer
            if '\n' in data.decode(): break  # If that segment had '\n', break

        # Get the buffer data, split it over newlines, print the first line
        right_name = buff.getvalue().splitlines()[0]

        #right_name = conn.recv(1024)
        print(right_name)
        #right_name = right_name.decode()
        right_name = right_name[:1].upper() + right_name[1:]
        nametime = datetime.datetime.now().strftime("_%Y%m%d_%H%M%S_")
        name = right_name + "O1" + nametime + ".csv"
        # Set working directory
        os.chdir(os.path.join(parent_dir, 'temporary'))

        f = open(name, 'w')
instance = 'file-splitter'
zone = 'us-central1-a'
client = storage.Client()
source_bucket = client.get_bucket(source_bucket_id)
service = discovery.build('compute', 'v1', cache_discovery=False)
request = service.instances().get(project=project_name, zone=zone, instance=instance)
response = request.execute()
metadata_keyvalue = response["metadata"]["items"]
source_bucket_filename = metadata_keyvalue[0]["value"]
total_records = 30000
filename = source_bucket_id + "/" + source_bucket_filename
read_storage_client = storage.Client()
bucket = read_storage_client.get_bucket(source_bucket_id)
blob = bucket.get_blob(source_bucket_filename)
data_string = blob.download_as_string()
data_string = StringIO(data_string.decode('utf-8'))
df = pd.read_csv(data_string, sep='\n', header=None)
content = ""
#partial = []
res = []
count = 0
record = 0
for i in range(len(df)):
    content +=  df.iloc[i,0] + "\n"
    #partial.append(df.iloc[i, 0] + "\n")
    count += 1
    if count == 8:
        content += "\n"
        #partial.add("\n")
        count = 0
        record += 1