def run(self, message_queue):
from .movie import EXIT_SUCCESS, EXIT_ERROR, EXIT_CANCEL
arg0 = self.arg_list[0]
import os.path
if not os.path.isfile(arg0):
self.exit_status = (-1, EXIT_ERROR,
'Could not find %s executable at %s' %
(self.ffmpeg_cmd, arg0))
return
# all output is on stderr, but Windows needs all standard I/O to
# be redirected if one is, so stdout is a pipe too
from io import StringIO
out = StringIO()
from subprocess import Popen, PIPE, DEVNULL
p = Popen(self.arg_list, stdin=DEVNULL, stdout=PIPE, stderr=PIPE)
out, err = p.communicate()
exit_code = p.returncode
status = EXIT_SUCCESS if exit_code == 0 else EXIT_ERROR
ffmpeg_output = 'stdout:\n%s\nstderr:\n%s' % (out.decode('utf-8'),
err.decode('utf-8'))
error = '' if exit_code == 0 else ffmpeg_output
self.exit_status = (exit_code, status, error)
if exit_code or self.verbose:
self.session.logger.info(' '.join(self.arg_list) + '\n' +
ffmpeg_output)
self.remove_backwards_frames()
def __init__(self, f=sys.stdin, filename=None):
"""Initialize a tokenizer instance.
@param f: The file to tokenize. The default is sys.stdin.
This parameter may also be a string, in which case the tokenizer
will take its input from the contents of the string.
@type f: file or string
@param filename: the name of the filename that the L{where} method
will return.
@type filename: string
"""
if isinstance(f, text_type):
f = StringIO(f)
if filename is None:
filename = '<string>'
elif isinstance(f, binary_type):
f = StringIO(f.decode())
if filename is None:
filename = '<string>'
else:
if filename is None:
if f is sys.stdin:
filename = '<stdin>'
else:
filename = '<file>'
self.file = f
self.ungotten_char = None
self.ungotten_token = None
self.multiline = 0
self.quoting = False
self.eof = False
self.delimiters = _DELIMITERS
self.line_number = 1
self.filename = filename
def parse(self, input):
"""Parse the given file or file source string."""
if hasattr(input, 'name'):
self.filename = input.name
elif not getattr(self, 'filename', ''):
self.filename = ''
if hasattr(input, "read"):
inisrc = input.read()
input.close()
input = inisrc
if isinstance(input, bytes):
if sys.version_info[0] == 3:
input = StringIO(input.decode('utf-8'))
else:
input = BytesIO(input)
self._inifile = INIConfig(input, optionxformvalue=None)
else:
self._inifile = INIConfig(open(input), optionxformvalue=None)
for section in self._inifile:
for entry in self._inifile[section]:
source = self._dialect.unescape(self._inifile[section][entry])
newunit = self.addsourceunit(source)
newunit.addlocation("[%s]%s" % (section, entry))
def open_csv_url(page_url):
try:
csv_page = urlopen(page_url)
except (HTTPError, URLError) as e:
print('\033[1;31m Error occurred when request the url %s' % page_url)
print(e)
return None
# 使用库在线读取，返回的csv_reader对象是列表，可迭代的
print('\033[1;31m Using csv_reader to get the list_data.')
csv_data = csv_page.read() # 获取csv文件的内容
csv_data = csv_data.decode('ascii', 'ignore') # 将csv文件中的内容按ASCII解码
csv_data = StringIO(csv_data) # 将编码后的内容封装成StringIO对象，可以被Python的csv库处理
csv_reader = csv.reader(csv_data)
for row in csv_reader:
print('\033[1;33m The album \'' + row[0] + '\'was released in ' + str(row[1]))
print(row)
# 使用库在线读取，返回的csv_dic_reader对象是字典，同时字段保存在csv_dic_reader.filenames里，字段同时作为字典的键
print('\033[1;31m Using csv_dic_reader to get the dict_data.')
csv_data = csv_page.read() # 获取csv文件的内容
csv_data = csv_data.decode('ascii', 'ignore') # 将csv文件中的内容按ASCII解码
csv_data = StringIO(csv_data) # 将编码后的内容封装成StringIO对象，可以被Python的csv库处理
csv_dic_reader = csv.DictReader(csv_data)
print(csv_dic_reader.fieldnames)
for row in csv_dic_reader:
print('\033[1;33m The album \'' + row['Name'] + '\' was released in ' + row['Year'])
print(row)
def intersect_with_state(self, bed_file):
chromstate_dir = "/home/zyang/Project/CRC/step45_chromatin_state_in_cluster/state13"
df_list = []
columns = []
for one_file in os.listdir(chromstate_dir):
if "segments.bed" in one_file:
df = pd.read_csv(os.path.join(chromstate_dir, one_file),
sep="\t",
names=["chr", "start", "end", "state"])
df = df[df.state != "E2"]
df_tmp = self.writeToTmp(df)
#tmp_result = "/home/zyang/Project/CRC/step45_chromatin_state_in_cluster/test.txt"
cmd = "bedtools intersect -wao -a %s -b %s" % (bed_file,
df_tmp)
out_string = subprocess.check_output([cmd], shell=True)
out_string = StringIO(out_string.decode('utf-8'))
df_result = pd.read_csv(out_string, sep="\t", \
names=["chr", "start", "end", "strand", "ensembl", "chr_state", "start_state", "end_state", "state", "overlap"])
df_result = df_result[["ensembl", "state", "overlap"]]
df_result = df_result[df_result.state != "."].reset_index()
df_result = df_result.groupby(
['ensembl', 'state'])["overlap"].sum().reset_index()
idx = df_result.groupby([
'ensembl'
])['overlap'].transform(max) == df_result['overlap']
df_result = df_result[idx]
counts = collections.Counter(df_result.ensembl)
##some gene has several equit max value
del_ensembl = []
for key in counts:
if counts[key] > 1:
del_ensembl.append(key)
df_result = df_result[~df_result["ensembl"].isin(del_ensembl)]
print(len(set(df_result.ensembl)))
print(len(df_result.ensembl))
#df_result.to_csv("test1.txt", sep="\t", index=True)
#os.remove(tmp_result)
os.remove(df_tmp)
df_result = df_result[["ensembl", "state"]]
df_list.append(df_result)
columns.append(one_file.split("_")[0])
#print(df_result[0:5])
df_merged = reduce(
lambda left, right: pd.merge(
left, right, on=['ensembl'], how='outer'), df_list)
df_merged.index = df_merged["ensembl"]
df_merged.drop(['ensembl'], axis=1, inplace=True)
df_merged.columns = columns
df_merged = df_merged[[
"ctrl", "2weeks", "4weeks", "7weeks", "10weeks"
]]
#df_merged.to_csv("state_summary_cluster.txt", sep="\t", header=True)
return df_merged
def detect_linesep(data):
if isinstance(data, (str, memoryview)):
data = StringIO(data)
offset = data.decode('utf-8').tell()
line = data.readline()
data.seek(offset)
if line[-2:] == "\r\n":
return "\r\n"
else:
return line[-1]
def parse(self, data):
"""Dummy csv parse"""
try:
data = StringIO(data.decode())
reader = csv.DictReader(data)
payment_returns = []
for row in reader:
payment_return = self.parse_payment_return(row)
if len(payment_return['transactions']):
payment_returns.append(payment_return)
except Exception:
raise UserError(_("Couldn't load file data"))
return payment_returns
def send_request(method, path, data=None):
out = StringIO()
args = [
"-X",
method,
f"http://127.0.0.1:8081/{path.lstrip('/')}",
"-H",
f"X-API-Key: {pm.config['environment']['PDNS_API_KEY']}",
]
if data:
args += ["-d", json.dumps(data)]
out = pm.get_exec("powerdns", ["curl", *args])
try:
return json.loads(out or '"success!"')
except Exception as e:
exit(out.decode())
def list_contents(rar_file_path):
"""
Returns a list of the archive's contents.
"""
assert os.path.isfile(rar_file_path) and rar_file_path.endswith('.rar')
contents = []
count = 0
command = '"{unrar}" v -- "{file}"'
command = command.format(unrar=config.UNRAR_PATH, file=rar_file_path)
try:
output = subprocess.check_output(command,
stderr=subprocess.STDOUT,
shell=True)
except subprocess.CalledProcessError as e:
output = e.output.decode(encoding='utf-8')
msg = 'Error while listing archive contents: "{error_string}"'
raise FileUtilsError(msg.format(error_string=output.strip()))
else:
output = StringIO(output.decode(encoding='utf-8'))
parse = False
for line in output.readlines():
line_list = line.strip().split()
# If the line is not empty...
if line_list:
# This marks the start and end of the section we want to parse
if line_list[
0] == '-------------------------------------------------------------------------------':
parse = not parse
count = 0
# If we're in the section of the output we want to parse...
elif parse:
# Parse every other line (only the file paths)
if count % 2 == 0:
contents.append(line_list[0])
count += 1
return contents
def list_contents(rar_file_path):
"""
Returns a list of the archive's contents.
"""
assert os.path.isfile(rar_file_path) and rar_file_path.endswith('.rar')
contents = []
count = 0
command = '"{unrar}" v -- "{file}"'
command = command.format(unrar=config.UNRAR_PATH, file=rar_file_path)
try:
output = subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True)
except subprocess.CalledProcessError as e:
output = e.output.decode(encoding='utf-8')
msg = 'Error while listing archive contents: "{error_string}"'
raise FileUtilsError(msg.format(error_string=output.strip()))
else:
output = StringIO(output.decode(encoding='utf-8'))
parse = False
for line in output.readlines():
line_list = line.strip().split()
# If the line is not empty...
if line_list:
# This marks the start and end of the section we want to parse
if line_list[0] == '-------------------------------------------------------------------------------':
parse = not parse
count = 0
# If we're in the section of the output we want to parse...
elif parse:
# Parse every other line (only the file paths)
if count % 2 == 0:
contents.append(line_list[0])
count += 1
return contents
def parse(self, source):
"""Parses an SVG document, and returns the root element of it.
Arguments:
source (str, file): An URL or a file-like object of an SVG
document.
Returns:
Element: A root element of the document.
"""
if isinstance(source, str):
data, headers = load(source)
content_type = get_content_type(headers)
if content_type is None:
charset = 'utf-8'
else:
charset = content_type.get('charset', 'utf-8')
data = StringIO(data.decode(charset))
else:
data = source
tree = self._parser.parse(data)
return tree.getroot()
def parse_titpub(text, buf):
pp = PortugueseRulesParser2()
text = StringIO(text.decode('ISO-8859-1'))
writer = csv.writer(buf)
_drop_first_2 = dropwhile(lambda x: x[0] < 2, enumerate(text))
_drop_empy = filter(lambda x: x[1].strip() is not '', _drop_first_2)
for c, line in _drop_empy:
row = line.split('@')
tit = dict(titulo=row[0],
data_referencia=row[1],
codigo_selic=row[2],
data_base=row[3],
data_vencimento=row[4],
taxa_max=pp.parse(row[5]),
taxa_min=pp.parse(row[6]),
taxa_ind=pp.parse(row[7]),
pu=pp.parse(row[8]),
desvio_padrao=pp.parse(row[9]))
writer.writerow(tit.values())
date_str = datetime.strptime(tit['data_referencia'], '%Y%m%d') \
.strftime('%Y-%m-%d')
return date_str
def parse(self, input):
"""Parse the given file or file source string."""
if hasattr(input, "name"):
self.filename = input.name
elif not getattr(self, "filename", ""):
self.filename = ""
if hasattr(input, "read"):
inisrc = input.read()
input.close()
input = inisrc
if isinstance(input, bytes):
input = StringIO(input.decode("utf-8"))
self._inifile = INIConfig(input, optionxformvalue=None)
else:
self._inifile = INIConfig(open(input), optionxformvalue=None)
for section in self._inifile:
for entry in self._inifile[section]:
source = self._dialect.unescape(self._inifile[section][entry])
newunit = self.addsourceunit(source)
newunit.addlocation(f"[{section}]{entry}")
zippy = gzip.GzipFile(fileobj=memfile)
uncompressed = zippy.read()
respData = resp.read()
#### !!!!
mysock = urllib.request.urlopen(url)
memfile = BytesIO(mysock.read())
f = gzip.GzipFile(fileobj=memfile)
r = f.read()
t = r.decode('utf-8')
###
import gzip
from io import BytesIO
import shutil
from ftplib import FTP
ftp = FTP('ftp.ncbi.nlm.nih.gov')
ftp.login('anonymous', '')
flo = BytesIO()
ftp.retrbinary('RETR /pubmed/id_list.txt.gz', flo.write)
l = str(l, errors='ignore')
f.write(l)
l = conn.recv(1024)
f.close()
print("Done Receiving")
conn.close()
elif bufferdata == "No":
print("Wrong prediction!")
globvar = globvar + 1
# Receiving data from client
buff = StringIO() # Create the buffer
while True:
data = conn.recv(1)
print(data)
buff.write(data.decode()) # Append that segment to the buffer
if '\n' in data.decode(): break # If that segment had '\n', break
# Get the buffer data, split it over newlines, print the first line
right_name = buff.getvalue().splitlines()[0]
#right_name = conn.recv(1024)
print(right_name)
#right_name = right_name.decode()
right_name = right_name[:1].upper() + right_name[1:]
nametime = datetime.datetime.now().strftime("_%Y%m%d_%H%M%S_")
name = right_name + "O1" + nametime + ".csv"
# Set working directory
os.chdir(os.path.join(parent_dir, 'temporary'))
f = open(name, 'w')
instance = 'file-splitter'
zone = 'us-central1-a'
client = storage.Client()
source_bucket = client.get_bucket(source_bucket_id)
service = discovery.build('compute', 'v1', cache_discovery=False)
request = service.instances().get(project=project_name, zone=zone, instance=instance)
response = request.execute()
metadata_keyvalue = response["metadata"]["items"]
source_bucket_filename = metadata_keyvalue[0]["value"]
total_records = 30000
filename = source_bucket_id + "/" + source_bucket_filename
read_storage_client = storage.Client()
bucket = read_storage_client.get_bucket(source_bucket_id)
blob = bucket.get_blob(source_bucket_filename)
data_string = blob.download_as_string()
data_string = StringIO(data_string.decode('utf-8'))
df = pd.read_csv(data_string, sep='\n', header=None)
content = ""
#partial = []
res = []
count = 0
record = 0
for i in range(len(df)):
content += df.iloc[i,0] + "\n"
#partial.append(df.iloc[i, 0] + "\n")
count += 1
if count == 8:
content += "\n"
#partial.add("\n")
count = 0
record += 1