def TempFile(infile=None):
"Create a suitable temporary file"
outfile = TemporaryFile()
if infile and hasattr(infile, 'read'):
outfile.writelines(infile)
outfile.seek(0)
return outfile
def fix_img_gff_errors(gff_file):
''' GFF files in the IMG directory can contain errors. This fixes them and returns a file handle to the fixed file
transforms literal semicolon (;) characters in field 9 to their proper percent encoding (%3B)
fixes the CRISPR lines
'''
new_gff = TemporaryFile()
with open(gff_file) as fp:
for ln, line in enumerate(fp):
if line[0] == '#':
new_gff.write(line)
else:
fields = line.split('\t')
if fields[2] == 'CRISPR':
fields[5] = '.'
fields[6] = '?'
fields[7] = '.'
else:
attributes = fields[8]
attribute_kv = attributes.split(';')
new_attributes = []
for i in range(len(attribute_kv)):
if ',' in attribute_kv[i]:
attribute_kv[i] = re.sub(',', '%2C', attribute_kv[i])
if '=' not in attribute_kv[i]:
new_attributes[-1] += '%3B' + attribute_kv[i]
else:
new_attributes.append(attribute_kv[i])
fields[8] = ';'.join(new_attributes)
new_gff.write('\t'.join(fields))
new_gff.seek(0)
return new_gff
def test_read_several(self):
"""Read several stanzas from file"""
tmpf = TemporaryFile()
tmpf.write("""\
version_header: 1
name: foo
val: 123
name: quoted
address: "Willowglen"
\t 42 Wallaby Way
\t Sydney
name: bar
val: 129319
""")
tmpf.seek(0)
s = read_stanza(tmpf)
self.assertEquals(s, Stanza(version_header='1'))
s = read_stanza(tmpf)
self.assertEquals(s, Stanza(name="foo", val='123'))
s = read_stanza(tmpf)
self.assertEqualDiff(s.get('name'), 'quoted')
self.assertEqualDiff(s.get('address'), ' "Willowglen"\n 42 Wallaby Way\n Sydney')
s = read_stanza(tmpf)
self.assertEquals(s, Stanza(name="bar", val='129319'))
s = read_stanza(tmpf)
self.assertEquals(s, None)
self.check_rio_file(tmpf)
def getUpdateElements(self, valueMap):
'''
@param valueMap:
'''
elements = ""
for name in valueMap.keys():
fullname = name
if isinstance(name, types.StringType):
fullname = (self.defaultNameSpace, name)
if not fullname[0]:
tag = fullname[1]
else:
tag = self.shortcuts[fullname[0]] + ':' + fullname[1]
value = valueMap[name]
if value:
if isinstance(value, qp_xml._element):
tmpFile = TemporaryFile('w+')
value = qp_xml.dump(tmpFile, value)
tmpFile.flush()
tmpFile.seek(0)
tmpFile.readline()
value = tmpFile.read()
else:
value = "<![CDATA[%s]]>" % value
else:
value = ""
elements += "<%s>%s</%s>" % (tag, value, tag)
return elements
def run_reduce(self):
self.stopped_received = 0
self.merged_files = []
merged_iterator = None
while True:
# Iterate and merge files until all jobs are processed
get_next = self.get_next_file()
files = get_next
# itertools.islice(get_next, self.reduce_max_files)
all_files = [file for file in files]
iterables = [self.iter_on_file(file) for file in all_files]
merged_iterator = heapq.merge(*iterables)
if self.stopped_received < self.numprocs:
if self.debug:
debug_print("Performing intermediate merge on %u files" % len(iterables))
f = TemporaryFile()
self.merged_files.append(f)
for m in merged_iterator:
cPickle.dump(m, f, cPickle.HIGHEST_PROTOCOL)
f.seek(0)
f.flush()
else:
break
if len(self.merged_files) > 0:
if self.debug:
debug_print("Final merge")
# Final merge if required
merged_iterator = heapq.merge(
*([self.iter_on_file(stream) for stream in self.merged_files] + [merged_iterator])
)
if self.debug:
debug_print("Reduce loop")
result = self.reduce_loop(merged_iterator)
return result
def test_one_key_per_block_writer(self):
# 2 pointers and a 1 byte string null terminated string = 10 bytes
stream = TemporaryFile()
i = IndexWriter(stream, block_size=10, terminator='\0')
i.add(0, 'b')
eq_(len(i.indexes), 1)
i.add(0, 'c')
eq_(len(i.indexes), 2)
i.finish()
stream.seek(0)
packet = stream.read()
eq_(len(packet), 30)
root_block = packet[:10]
eq_(root_block, '\x01\x00\x00\x00c\x00\x02\x00\x00\x00')
block_1 = packet[10:20]
eq_(block_1, '\x03\x00\x00\x00b\x00\x04\x00\x00\x00')
block_2 = packet[20:]
eq_(block_2, '\x04\x00\x00\x00c\x00\x05\x00\x00\x00')
def image_register(request):
params = request.GET if request.method == "GET" else request.POST
if not ("url" in params):
content = {"message": u"パラメータ`url`が指定されていません"}
return Response(content, status=status.HTTP_400_BAD_REQUEST)
try:
image = Images.objects.get(url=params["url"])
except ObjectDoesNotExist:
image = Images(url=params["url"], adult_flag=False, grotesque_flag=False)
descriptor = factory.descriptor(filepath=image.local_path)
if descriptor == None:
content = {"message": u"ローカルに画像が見つかりません"}
return Response(content, status=status.HTTP_412_PRECONDITION_FAILED)
else:
tmp = TemporaryFile()
try:
np.save(tmp, descriptor)
tmp.seek(0)
image.description = tmp.read()
image.save()
finally:
tmp.close()
return Response(ImageMapper(image).as_dict())
def _create_temp_file(edid_binary):
edid_file = TemporaryFile()
edid_file.write(edid_binary)
edid_file.flush()
edid_file.seek(0)
return edid_file
def backup_dir(key, data_node, directory): temp = TemporaryFile() archiver = Popen(["ssh", data_node, "tar", "c", directory], stdout=PIPE) compressor = Popen(["lzma", "-z", "-9"], stdin=archiver.stdout, stdout=temp) compressor.wait() temp.seek(0) key.set_contents_from_file(temp)
def send_form(self,):
import csv
product = self[0]
#_logger.warning('data %s b64 %s ' % (account.data,base64.decodestring(account.data)))
if not product.data == None:
fileobj = TemporaryFile('w+')
fileobj.write(base64.decodestring(product.data))
fileobj.seek(0)
try:
for row in csv.DictReader(fileobj):
pass
finally:
fileobj.close()
return True
#product.write({'state': 'get', 'name': '%s.xml' % account.model.model.replace('.','_'),'data': base64.b64encode(account._export_xml()) })
return {
'type': 'ir.actions.act_window',
'res_model': 'account.export',
'view_mode': 'form',
'view_type': 'form',
'res_id': product.id,
'views': [(False, 'form')],
'target': 'new',
}
def _getPID():
"""Get PID from specified PIDFile.
Returns:
int: >0 -- RESTservice PID
-1 -- _pidfile contains invalid PID
-2 -- _pidfile not found
"""
pid = 0
try:
f = open(_pidfile, 'r')
pid = int(f.read())
f.close()
except IOError as e:
if e.errno == 2:
return -2
raise e
except ValueError:
return -1
# Double check PID from PIDFile:
outfile = TemporaryFile(mode='w+')
call(['ps', 'x'], stdout=outfile)
outfile.seek(0)
for line in outfile:
line = line.strip()
if line.startswith(str(pid)) and line.endswith(_script_name):
return pid
return -1
def get_upload_results(self, job_id, batch_id, callback = dump_results, batch_size=0, logger=None):
job_id = job_id or self.lookup_job_id(batch_id)
if not self.is_batch_done(job_id, batch_id):
return False
http = Http()
uri = self.endpoint + "/services/async/29.0/job/%s/batch/%s/result" % (job_id, batch_id)
resp, content = http.request(uri, method="GET", headers=self.headers())
tf = TemporaryFile()
tf.write(content)
total_remaining = self.count_file_lines(tf)
if logger:
logger("Total records: %d" % total_remaining)
tf.seek(0)
records = []
line_number = 0
col_names = []
reader = csv.reader(tf, delimiter=",", quotechar='"')
for row in reader:
line_number += 1
records.append(UploadResult(*row))
if len(records) == 1:
col_names = records[0]
if batch_size > 0 and len(records) >= (batch_size+1):
callback(records, total_remaining, line_number)
total_remaining -= (len(records)-1)
records = [col_names]
callback(records, total_remaining, line_number)
tf.close()
return True
class PackageZipBuilder(object):
def __init__(self, namespace, version=None):
self.namespace = namespace
self.version = version
def open_zip(self):
self.zip_file = TemporaryFile()
self.zip= ZipFile(self.zip_file, 'w')
def install_package(self):
self.open_zip()
if not self.version:
raise ValueError('You must provide a version to install a package')
package_xml = PACKAGE_XML % self.namespace
#package_xml = package_xml.encode('utf-8')
self.zip.writestr('package.xml', package_xml)
installed_package = INSTALLED_PACKAGE % self.version
#installed_package.encode('utf-8')
self.zip.writestr('installedPackages/%s.installedPackage' % self.namespace, installed_package)
return self.encode_zip()
def uninstall_package(self):
self.open_zip()
self.zip.writestr('package.xml', EMPTY_PACKAGE_XML)
self.zip.writestr('destructiveChanges.xml', PACKAGE_XML % self.namespace)
return self.encode_zip()
def encode_zip(self):
self.zip.close()
self.zip_file.seek(0)
return b64encode(self.zip_file.read())
def dsorted(iterable, buffer_size=1e6, tempdir="."):
from disco.compat import pickle_load, pickle_dump
from heapq import merge
from itertools import islice
from tempfile import TemporaryFile
def read(handle):
while True:
try:
yield pickle_load(handle)
except EOFError:
return
iterator = iter(iterable)
subiters = []
while True:
buffer = sorted(islice(iterator, buffer_size))
handle = TemporaryFile(dir=tempdir)
for item in buffer:
pickle_dump(item, handle, -1)
handle.seek(0)
subiters.append(read(handle))
if len(buffer) < buffer_size:
break
return merge(*subiters)
class CandidateUploadFile(BaseHandler):
def initialize(self):
self.tempfile = TemporaryFile()
@tornado.web.authenticated
@granted()
@tornado.web.asynchronous
def post(self):
fp_url = self.get_argument("url")
mime_type = self.get_argument("data[type]")
size = int(self.get_argument("data[size]"))
candidate_id = self.get_argument("id")
self.candidate = self.db.query(Candidate).get(int(candidate_id))
logging.info("type: %s, size: %r", mime_type, size)
if mime_type == "image/jpeg" and size < MAX_UPLOAD_SIZE:
http_client = tornado.httpclient.AsyncHTTPClient()
request = tornado.httpclient.HTTPRequest(url=fp_url, streaming_callback=self.streaming_callback)
http_client.fetch(request, self.on_download)
else:
self.finish(dict(status=0))
def streaming_callback(self, data):
self.tempfile.write(data)
logging.info("This is the streaming_callback file tell function: %r", self.tempfile.tell())
def on_download(self, response):
img_path = os.path.join(os.path.dirname(__file__), "static/profiles/img/" + str(self.candidate.backup_id) + '.jpg')
self.tempfile.seek(0)
ptr = open(img_path, 'wb')
ptr.write(self.tempfile.read())
ptr.close()
self.tempfile.close()
self.finish(dict(src="/static/profiles/img/" + str(self.candidate.backup_id) + '.jpg', status=1))
def backup(self):
"""
Create backup
"""
if self.dry_run:
return
if not os.path.exists(self.config["tar"]["directory"]) or not os.path.isdir(
self.config["tar"]["directory"]
):
raise BackupError("{0} is not a directory!".format(self.config["tar"]["directory"]))
out_name = "{0}.tar".format(self.config["tar"]["directory"].lstrip("/").replace("/", "_"))
outfile = os.path.join(self.target_directory, out_name)
args = ["tar", "c", self.config["tar"]["directory"]]
errlog = TemporaryFile()
stream = open_stream(outfile, "w", **self.config["compression"])
LOG.info("Executing: %s", list2cmdline(args))
pid = Popen(args, stdout=stream.fileno(), stderr=errlog.fileno(), close_fds=True)
status = pid.wait()
try:
errlog.flush()
errlog.seek(0)
for line in errlog:
LOG.error("%s[%d]: %s", list2cmdline(args), pid.pid, line.rstrip())
finally:
errlog.close()
if status != 0:
raise BackupError("tar failed (status={0})".format(status))
def generate_minion_keys(self):
# XXX TODO: Replace M2Crypto with PyCrypto
# see: https://github.com/saltstack/salt/pull/1112/files
# generate keys
keyname = self.get_keyname()
if not keyname:
LOG.error("Must specify salt_id or hostname")
return False
gen = RSA.gen_key(2048, 1, callback=lambda x, y, z: None)
pubpath = os.path.join(self.pki_dir, "{0}.pub".format(keyname))
gen.save_pub_key(pubpath)
LOG.info("public key {0}".format(pubpath))
if self.config.get("save_keys"):
cumask = os.umask(191)
gen.save_key(os.path.join(self.pki_dir, "{0}.pem".format(keyname)), None)
os.umask(cumask)
# public key
_pub = TemporaryFile()
bio_pub = BIO.File(_pub)
m2.rsa_write_pub_key(gen.rsa, bio_pub._ptr())
_pub.seek(0)
self.config["public_key"] = self.public_key = _pub.read()
self.config["formatted_public_key"] = "\n".join(" {0}".format(k) for k in self.public_key.split("\n"))
# private key
_pem = TemporaryFile()
bio_pem = BIO.File(_pem)
gen.save_key_bio(bio_pem, None)
_pem.seek(0)
self.config["private_key"] = self.private_key = _pem.read()
self.config["formatted_private_key"] = "\n".join(" {0}".format(k) for k in self.private_key.split("\n"))
return True
def importXML(jar, file, clue=""):
from OFS.XMLExportImport import save_record, save_zopedata, start_zopedata
from tempfile import TemporaryFile
import xml.parsers.expat
if type(file) is str:
file = open(file, "rb")
outfile = TemporaryFile()
data = file.read()
F = xmlPickler()
F.end_handlers["record"] = save_record
F.end_handlers["ZopeData"] = save_zopedata
F.start_handlers["ZopeData"] = start_zopedata
F.binary = 1
F.file = outfile
# Our BTs XML files don't declare encoding but have accented chars in them
# So we have to declare an encoding but not use unicode, so the unpickler
# can deal with the utf-8 strings directly
p = xml.parsers.expat.ParserCreate("utf-8")
p.returns_unicode = False
p.CharacterDataHandler = F.handle_data
p.StartElementHandler = F.unknown_starttag
p.EndElementHandler = F.unknown_endtag
r = p.Parse(data)
outfile.seek(0)
return jar.importFile(outfile, clue)
def _open(self):
tmp = TemporaryFile()
resp = requests.get(self.metadata['url'], stream=True)
for chunk in resp.iter_content(256*1024):
tmp.write(chunk)
tmp.seek(0)
return tmp
def append(self, seg, crossfade=100):
seg1, seg2 = AudioSegment._sync(self, seg)
if not crossfade:
return seg1._spawn(seg1._data + seg2._data)
elif crossfade > len(self):
raise ValueError("Crossfade is longer than the original AudioSegment ({}ms > {}ms)".format(
crossfade, len(self)
))
elif crossfade > len(seg):
raise ValueError("Crossfade is longer than the appended AudioSegment ({}ms > {}ms)".format(
crossfade, len(seg)
))
xf = seg1[-crossfade:].fade(to_gain=-120, start=0, end=float('inf'))
xf *= seg2[:crossfade].fade(from_gain=-120, start=0, end=float('inf'))
output = TemporaryFile()
output.write(seg1[:-crossfade]._data)
output.write(xf._data)
output.write(seg2[crossfade:]._data)
output.seek(0)
return seg1._spawn(data=output)
def write_lines(self, key, lines):
self._verify_key_format(key)
storage = self.bucket.new_key(key + ".json.gz")
buff = TemporaryFile()
archive = gzip.GzipFile(fileobj=buff, mode='w')
count = 0
for l in lines:
if hasattr(l, "__iter__"):
for ll in l:
archive.write(ll.encode("utf8"))
archive.write(b"\n")
count += 1
else:
archive.write(l.encode("utf8"))
archive.write(b"\n")
count += 1
archive.close()
file_length = buff.tell()
retry = 3
while retry:
try:
with Timer("Sending {{count}} lines in {{file_length|comma}} bytes", {"file_length": file_length, "count": count}, debug=self.settings.debug):
buff.seek(0)
storage.set_contents_from_file(buff)
break
except Exception, e:
Log.warning("could not push data to s3", cause=e)
retry -= 1
def set_sff_trimpoints_with_sfftools(
sff_dir, technical_lengths, sffinfo_path='sffinfo', sfffile_path='sfffile',
debug=False):
"""Set trimpoints to end of technical read for all SFF files in directory.
This function essentially provides the reference implementation.
It uses the official sfftools from Roche to process the SFF files.
"""
if not (exists(sffinfo_path) or which(sffinfo_path)):
raise ApplicationNotFoundError(
'sffinfo executable not found. Is it installed and in your $PATH?')
if not (exists(sfffile_path) or which(sfffile_path)):
raise ApplicationNotFoundError(
'sfffile executable not found. Is it installed and in your $PATH?')
for lib_id, sff_fp in get_per_lib_sff_fps(sff_dir):
try:
readlength = technical_lengths[lib_id]
except KeyError:
continue
sffinfo_args = [sffinfo_path, '-s', sff_fp]
if debug:
print "Running sffinfo command %s" % sffinfo_args
sffinfo_output_file = TemporaryFile()
check_call(sffinfo_args, stdout=sffinfo_output_file)
sffinfo_output_file.seek(0)
seqlengths = {}
for line in sffinfo_output_file:
if line.startswith('>'):
fields = line[1:].split()
seq_len = fields[1].split('=')[1]
seqlengths[fields[0]] = seq_len
trim_fp = sff_fp + '.trim'
trim_file = open(trim_fp, 'w')
for id_, length in seqlengths.items():
curr_length = int(seqlengths[id_])
# Sfftools use 1-based index
left_trim = readlength + 1
# Key sequence not included in FASTA length
right_trim = curr_length + 4
if curr_length > left_trim:
trim_file.write(
"%s\t%s\t%s\n" % (id_, left_trim, right_trim))
else:
stderr.write(
'Rejected read %s with trim points %s and %s (orig '
'length %s)' % (id_, left_trim, curr_length, length))
trim_file.close()
trimmed_sff_fp = sff_fp + '.trimmed'
sfffile_args = [
sfffile_path, '-t', trim_fp, '-o', trimmed_sff_fp, sff_fp]
if debug:
print "Running sfffile command:", sfffile_args
check_call(sfffile_args, stdout=open(devnull, 'w'))
remove(sff_fp)
rename(trimmed_sff_fp, sff_fp)
def numpy_to_string(array): """Convert numpy array into human-readable string. Good for passing to other programs. Notes: human-readable string example: 1 2 3 4 5 6 is a string for the following array: [[1,2,3] [4,5,6]] Args: array (numpy): array to convert to human-readable string Returns: human-readable string of array """ f = TemporaryFile() np.savetxt(f, array, fmt='%.8g') f.seek(0) string = f.read() return string
def string_to_numpy(string): """Convert human-readable string into numpy array. Note: loads as floats even if stored as ints. human-readable string example: 1 2 3 4 5 6 is a string for the following array: [[1,2,3] [4,5,6]] Args: string (string): human-readable string to convert to numpy array Returns: numpy array """ f = TemporaryFile() f.write(string) f.seek(0) array = np.loadtxt(f) return array
def merge_in_pdf(fa, fb):
for f in [fa, fb]:
if _is_img(f.name):
img = Image.open(f.file)
try:
r, g, b, a = img.split() #alpha
except Exception as e:
r, g, b = img.split()
img = Image.merge('RGB', (r, g, b))
temp_file = TemporaryFile()
img.save(temp_file, "PDF", resolution=100, transparency=0)
temp_file.seek(0)
f.file = temp_file
merger = PdfFileMerger()
for f in [fa, fb]:
merger.append(PdfFileReader(f.file))
temp_file = TemporaryFile()
merger.write(temp_file)
temp_file.seek(0)
pdf_file = File(temp_file)
pdf_file.name = 'id_card.pdf'
return pdf_file
def fft_file_and_process(input_filename,nchannel,size_adjust,unwrap_phases):
smp=read_wave_file(input_filename,nchannel)
smp.resize(size_adjust)
freqs=np.fft.rfft(smp)
#get amplitude spectrum and phase spectrum
freqs_amp=np.abs(freqs)
freqs_amp*=1.0/(max(freqs_amp)+1e-6)
freqs_phases=np.angle(freqs)
if unwrap_phases!=0:
freqs_phases=np.unwrap(freqs_phases)
del smp
del freqs
cleanup_memory()
freqs_amp_file = TemporaryFile()
freqs_phases_file = TemporaryFile()
np.save(freqs_amp_file,freqs_amp)
np.save(freqs_phases_file,freqs_phases)
freqs_amp_file.seek(0)
freqs_phases_file.seek(0)
files_freqs=Object()
files_freqs.amp=freqs_amp_file
files_freqs.phases=freqs_phases_file
del freqs_amp
del freqs_phases
cleanup_memory()
return files_freqs
def test_execute_commands(self):
"""Test executing arbitrary commands and logging their output."""
# All commands succeed.
exp = (True, [])
log_f = TemporaryFile(prefix=self.prefix, suffix='.txt')
obs = _execute_commands(['echo foo', 'echo bar'], log_f, 1)
self.assertEqual(obs, exp)
exp = ("Command:\n\necho foo\n\nStdout:\n\nfoo\n\nStderr:\n\n\n"
"Command:\n\necho bar\n\nStdout:\n\nbar\n\nStderr:\n\n\n")
log_f.seek(0, 0)
obs = log_f.read()
self.assertEqual(obs, exp)
# One command fails.
exp = (False, [])
log_f = TemporaryFile(prefix=self.prefix, suffix='.txt')
obs = _execute_commands(['echo foo', 'foobarbaz'], log_f, 1)
self.assertEqual(obs, exp)
exp = ("Command:\n\necho foo\n\nStdout:\n\nfoo\n\nStderr:\n\n\n"
"Command:\n\nfoobarbaz\n\nStdout:\n\n\nStderr:\n\n\n\n")
log_f.seek(0, 0)
obs = sub('Stderr:\n\n.*\n\n', 'Stderr:\n\n\n\n',
log_f.read())
self.assertEqual(obs, exp)
def proxy_stdf():
"""
Circulate stdout/stderr via a proper file object. Designed to work around a
problem where Python nose replaces sys.stdout/stderr with a custom 'Tee'
object that is not a file object (compatible) and thus causes a crash with
Popen.
"""
tmp_stdout = sys.stdout
try:
tmp_stdout.fileno()
except Exception:
tmp_stdout = TemporaryFile()
tmp_stderr = sys.stderr
try:
tmp_stderr.fileno()
except Exception:
tmp_stderr = TemporaryFile()
try:
yield tmp_stdout, tmp_stderr
finally:
if tmp_stdout != sys.stdout:
tmp_stdout.seek(0)
sys.stdout.write(tmp_stdout.read().decode())
if tmp_stderr != sys.stderr:
tmp_stderr.seek(0)
sys.stderr.write(tmp_stderr.read().decode())
def draw(self):
"""Erstellt Chart als GIF. Gibt das GIF als string zurück."""
from tempfile import TemporaryFile
f = TemporaryFile()
self.chart.draw(f)
f.seek(0)
return f.read()
def to_xml(self, f=None):
"""Get this domain as an XML DOM Document
:param f: Optional File to dump directly to
:type f: File or Stream
:return: File object where the XML has been dumped to
:rtype: file
"""
if not f:
from tempfile import TemporaryFile
f = TemporaryFile()
print('<?xml version="1.0" encoding="UTF-8"?>', file=f)
print('<Domain id="%s">' % self.name, file=f)
for item in self:
print('\t<Item id="%s">' % item.name, file=f)
for k in item:
print('\t\t<attribute id="%s">' % k, file=f)
values = item[k]
if not isinstance(values, list):
values = [values]
for value in values:
print('\t\t\t<value><![CDATA[', end=' ', file=f)
if isinstance(value, unicode):
value = value.encode('utf-8', 'replace')
else:
value = unicode(value, errors='replace').encode('utf-8', 'replace')
f.write(value)
print(']]></value>', file=f)
print('\t\t</attribute>', file=f)
print('\t</Item>', file=f)
print('</Domain>', file=f)
f.flush()
f.seek(0)
return f
class ContentReceiver(object):
"Write-only file object used to receive data from FTP"
def __init__(self, callback, *args):
from tempfile import TemporaryFile
self.data = TemporaryFile('w+b')
self.callback = callback
self.args = args
def write(self, data):
self.data.write(data)
def close(self):
size = self.data.tell()
self.data.seek(0)
args = self.args + (self.data, size)
c = self.callback
self.callback = None
self.args = None
c(*args)
def check_read_write(self, k):
"""Check the weave k can be written & re-read."""
from tempfile import TemporaryFile
tf = TemporaryFile()
write_weave(k, tf)
tf.seek(0)
k2 = read_weave(tf)
if k != k2:
tf.seek(0)
self.log('serialized weave:')
self.log(tf.read())
self.log('')
self.log('parents: %s' % (k._parents == k2._parents))
self.log(' %r' % k._parents)
self.log(' %r' % k2._parents)
self.log('')
self.fail('read/write check failed')
def main() -> int:
with open(Path(__file__).parent / "pipeline.template.yml") as f:
pipeline = yaml.safe_load(f)
if os.environ["BUILDKITE_BRANCH"] != "master" and not os.environ["BUILDKITE_TAG"]:
print("--- Trimming unchanged steps from pipeline")
trim_pipeline(pipeline)
# Remove the Materialize-specific keys from the configuration that are
# only used to inform how to trim the pipeline.
for step in pipeline["steps"]:
if "inputs" in step:
del step["inputs"]
f = TemporaryFile()
yaml.dump(pipeline, f, encoding="utf-8") # type: ignore
f.seek(0)
spawn.runv(["buildkite-agent", "pipeline", "upload"], stdin=f)
return 0
def fpost(
myHeaders={
'User-Agent': 'PcGroup Util Client',
'content-type': 'application/json; charset=UTF-8',
'Accept-Encoding': '*/*'
}):
url = 'http://192.168.12.81:9200/_bulk'
data = "{ \"index\" : { \"_index\" : \"test2\", \"_type\" : \"log\"} }\n"
data = data + "{\"ok\":\"a\"," + "\"ok2\":1}\n"
data = data + "{\"ok\":\"b\"," + "\"ok2\":2}"
temp = TemporaryFile()
temp.writelines('{"index" : { "_index" : "test2", "_type" : "log"}}')
temp.writelines('{"ok":"a","ok2":1}')
#temp.write('{"ok":"a","ok2":1}')
#temp.write(data)
#temp.flush()
temp.seek(0)
print temp.readline()
print temp.readline()
print temp.mode
files = {
'file':
open('/Users/sky/Documents/company/esManager/pcPython/logs/test.log',
"rb")
}
#files = {'file': temp}
r = requests.post(url,
base64.b64encode(data.encode("utf8")).decode("ascii"),
headers=myHeaders)
print r.request.headers
print r.headers
#r = requests.request("POST" , url , files = files, headers = myHeaders)
print r.text
class APRFile(object):
"""Wrap a Python file-like object as an APR File"""
def __init__(self, pyfile):
self.pyfile = pyfile
self.pool = Pool()
self._as_parameter_ = POINTER(apr_file_t)()
self.tempfile = None
if hasattr(pyfile, "fileno"):
# Looks like this is a real file. We can just write
# directly to said file
osfile = apr_os_file_t(get_osfhandle(pyfile.fileno()))
else:
# Looks like this is a StringIO buffer or a fake file.
# Write to a temporary file and copy the output to the
# buffer when we are closed or flushed
self.tempfile = TemporaryFile()
osfile = apr_os_file_t(get_osfhandle(self.tempfile.fileno()))
apr_os_file_put(byref(self._as_parameter_), byref(osfile),
APR_CREATE | APR_WRITE | APR_BINARY, self.pool)
def flush(self):
"""Flush output to the underlying Python object"""
if self.tempfile:
self.tempfile.seek(0)
copyfileobj(self.tempfile, self.pyfile)
self.tempfile.truncate(0)
def close(self):
"""Close the APR file wrapper, leaving the underlying Python object
untouched"""
self.flush()
if self.tempfile:
self.tempfile.close()
self.tempfile = None
self.pool.destroy()
self.pool = None
def __del__(self):
if self.pool:
self.close()
def _change_ck2st(self, filename, debug=False):
# STEP1: change CKs -----------------------------------------
input = open(filename, 'r')
temp = TemporaryFile('w+')
alim = re.compile('.*(vdd|vss)+.*', re.IGNORECASE)
changed = 0
reg_declare = re.compile('COMPONENT fd1qll')
reg_inst = re.compile(': fd1qll')
reg_aff = re.compile('ck =>')
for line in input:
if reg_declare.search(line):
temp.write(line)
temp.write(input.next())
temp.write(' cp : IN STD_LOGIC;\n')
input.next()
changed += 1
elif reg_inst.search(line):
temp.write(line)
temp.write(input.next())
line = input.next()
temp.write(reg_aff.sub('cp =>', line))
changed += 1
else:
temp.write(line)
if debug:
print ' - changed %d ck' % changed
# STEP2: write changes to input file ------------------------
temp.seek(0)
input.close()
input = open(filename, 'w')
for line in temp:
input.write(line)
# END -------------------------------------------------------
input.close()
temp.close()
def get_upload_results(self,
job_id,
batch_id,
callback=(lambda *args, **kwargs: None),
batch_size=0,
logger=None):
job_id = job_id or self.lookup_job_id(batch_id)
if not self.is_batch_done(job_id, batch_id):
return False
http = Http()
uri = self.endpoint + \
"/services/async/29.0/job/%s/batch/%s/result" % (job_id, batch_id)
resp, content = http.request(uri, method="GET", headers=self.headers())
tf = TemporaryFile()
tf.write(content)
total_remaining = self.count_file_lines(tf)
if logger:
logger("Total records: %d" % total_remaining)
tf.seek(0)
records = []
line_number = 0
col_names = []
reader = csv.reader(tf, delimiter=",", quotechar='"')
for row in reader:
line_number += 1
records.append(UploadResult(*row))
if len(records) == 1:
col_names = records[0]
if batch_size > 0 and len(records) >= (batch_size + 1):
callback(records, total_remaining, line_number)
total_remaining -= (len(records) - 1)
records = [col_names]
callback(records, total_remaining, line_number)
tf.close()
return True
def process_notes():
database.connect()
if not check_update():
return
response = urllib2.urlopen(NOTES_URI)
# Parsing bz2 through a temporary file
tmpfile = TemporaryFile()
while True:
chunk = response.read(512*1024)
if not chunk:
break
tmpfile.write(chunk)
tmpfile.seek(0)
with database.atomic():
with BZ2File(tmpfile) as f:
for event, element in etree.iterparse(f):
if element.tag == 'note':
if len(element) > 0 and element[0].text and '#mapsme' in element[0].text:
note_id = element.get('id')
try:
ch = Change.get(Change.changeset == note_id, Change.action == 'n')
if element[-1].get('action') == 'closed' and ch.processed is None:
print('Found closed note {0}'.format(note_id))
ch.processed = hour_difference(ch.timestamp, element[-1].get('timestamp'))
ch.save()
except Change.DoesNotExist:
ch = Change()
ch.action = 'n'
ch.version = ''
ch.changeset = note_id
ch.user = element[0].get('user') if element[0].get('uid') else 'Anonymous Note'
print('Found new note {0} by {1}'.format(note_id, ch.user.encode('utf-8')))
ch.timestamp = datetime.strptime(element[0].get('timestamp'), '%Y-%m-%dT%H:%M:%SZ')
if element[-1].get('action') == 'closed' and ch.processed is None:
ch.processed = hour_difference(ch.timestamp, element[-1].get('timestamp'))
changes = [(element.get('lon'), element.get('lat')), {'note': element[0].text}]
ch.changes = json.dumps(changes, ensure_ascii=False)
ch.save()
element.clear()
def process_s3_warc(self, uri):
try:
no_sign_request = botocore.client.Config(
signature_version=botocore.UNSIGNED)
s3client = boto3.client('s3', config=no_sign_request)
s3pattern = re.compile('^s3://([^/]+)/(.+)')
s3match = s3pattern.match(uri)
if s3match is None:
print("Invalid URI: {}".format(uri))
self.failed_segment.add(1)
return None
bucketname = s3match.group(1)
path = s3match.group(2)
warctemp = TemporaryFile(mode='w+b')
s3client.download_fileobj(bucketname, path, warctemp)
warctemp.seek(0)
return warctemp
except BaseException as e:
print("Failed fetching {}\nError: {}".format(uri, e))
self.failed_segment.add(1)
return None
def __init__(self, req):
size = req.get_header('Content-Length')
if size is None:
size = -1
else:
size = int(size)
tempfile = TemporaryFile()
input = req.environ['wsgi.input']
while True:
buf = input.read(min(4096, size))
if not buf:
break
tempfile.write(buf)
size -= len(buf)
tempfile.flush()
tempfile.seek(0)
self.file = tempfile
filename = req.get_header('X-TracDragDrop-Filename')
self.filename = unicode_unquote(filename or '').encode('utf-8')
def test_read_iter(self):
"""Read several stanzas from file"""
tmpf = TemporaryFile()
tmpf.write(b"""\
version_header: 1
name: foo
val: 123
name: bar
val: 129319
""")
tmpf.seek(0)
reader = read_stanzas(tmpf)
read_iter = iter(reader)
stuff = list(reader)
self.assertEqual(stuff, [
Stanza(version_header='1'),
Stanza(name="foo", val='123'),
Stanza(name="bar", val='129319'),
])
def decrypt_pdf(src):
decrypted = TemporaryFile()
popen = subprocess.Popen(['qpdf', '--decrypt', '/dev/stdin', '-'],
stdin=src,
stdout=decrypted,
stderr=subprocess.PIPE)
stdout, stderr = popen.communicate()
if popen.returncode in (0, 3): # 0 == ok, 3 == warning
if popen.returncode == 3:
logger.warn('qpdf warning:\n%s',
smart_bytes(stderr, errors='backslashreplace'))
else:
from ecs.users.utils import get_current_user
user = get_current_user()
logger.warn('qpdf error (returncode=%s):\nUser: %s (%s)\n%s',
popen.returncode, user,
user.email if user else 'anonymous',
smart_bytes(stderr, errors='backslashreplace'))
raise ValueError('pdf broken')
decrypted.seek(0)
return decrypted
def test_view_zipped_big_raw(self):
big_content = self.big_log_content
response = self.get_log('zipped_big.log', data={'format': 'raw'})
self.assertEqual(response.status_code, 200, response)
# Response should be streamed in chunks, and once decompressed,
# should be the expected content
self.assertTrue(response.streaming)
tempfile = TemporaryFile()
for chunk in response.streaming_content:
tempfile.write(chunk)
tempfile.seek(0)
gz_file = GzipFile(mode='r', fileobj=tempfile)
all_data = gz_file.read()
if six.PY3:
all_data = str(all_data, encoding=locale.getpreferredencoding())
self.assertEqual(all_data, big_content)
def addValues(self, values, experiment):
if len(self.variables) == len(values):
readyProtocol = self.info
newProtocol = []
for i in range(0, len(readyProtocol)):
mystring = readyProtocol[i]
newProtocol.append(mystring)
for v in range(0, len(values)):
newProtocol[i] = newProtocol[i].replace(self.variables[v], values[v])
from tempfile import TemporaryFile
protocolFile = TemporaryFile(mode='r+')
protocolFile.writelines(newProtocol)
protocolFile.seek(0)
line = protocolFile.readline()
experiment.addComment('------ BEGIN PROTOCOL ' + self.name + ', variables: ' + ' '.join(self.variables) + '; values: ' + ' '.join(values) + ' ------')
while line != '':
splitline = line.split()
LineToList(splitline, protocolFile, experiment)
line = protocolFile.readline()
experiment.addComment('------ END PROTOCOL ' + self.name + ' ------')
def rtf(url):
'''
gets the url of the rtf file, and (tries to) return an xhtml version of it.
returns False if couldn't convert.
'''
remote = urlopen(url)
data = remote.read()
remote.close()
temp = TemporaryFile()
temp.write(data)
temp.seek(0)
try:
doc = Rtf15Reader.read(temp)
xhtml = XHTMLWriter.write(doc, pretty=True).read()
except:
xhtml = False
exceptionType, exceptionValue, exceptionTraceback = sys.exc_info()
logger.warn(''.join(traceback.format_exception(exceptionType, exceptionValue, exceptionTraceback)))
temp.close()
return xhtml
def textToSpeech(text, language, isSlow):
# Passing the text and language to the engine,
# here we have marked slow=False. Which tells
# the module that the converted audio should
# have a high speed
textToSpeechEngine = gTTS(text=text, lang=language, slow=isSlow)
# Saving the converted audio in a mp3 file named
# welcome
sf = TemporaryFile()
textToSpeechEngine.write_to_fp(sf)
sf.seek(0)
mixer.music.load(sf)
##os.system("start welcome.mp3")
mixer.music.play()
while not pygame.mixer.music.get_pos() == -1:
pass
def textToSpeech(text, language, isSlow):
# Passing the text and language to the engine,
# here we have marked slow=False. Which tells
# the module that the converted audio should
# have a high speed
textToSpeechEngine = gTTS(text=text, lang=language, slow=isSlow)
# Saving the converted audio in a mp3 temporary file
sf = TemporaryFile()
textToSpeechEngine.write_to_fp(sf)
sf.seek(0)
#loads and plays temporary mp3
mixer.music.load(sf)
mixer.music.play()
#waits till the mp3 is done playing
while not pygame.mixer.music.get_pos() == -1:
pass
def manage_upload(self,
file='',
content_type='',
is_preview=0,
create_prev=NO_PREVIEW,
maxx='',
maxy='',
ratio=0,
REQUEST=None):
""" Upload image from file handle or string buffer """
if self.wl_isLocked():
raise ResourceLockedError("File is locked via WebDAV")
if isinstance(file, str):
temp_file = TemporaryFile()
temp_file.write(file)
temp_file.seek(0)
else:
temp_file = file
return self.manage_file_upload(temp_file, content_type, is_preview,
create_prev, maxx, maxy, ratio, REQUEST)
def _check_output(*popenargs, **kwargs):
if 'stdout' in kwargs:
raise ValueError('stdout argument not allowed, it will be overridden.')
if 'stdin' in kwargs:
curr = kwargs['stdin']
if isinstance(curr, basestring):
f = TemporaryFile()
f.write(curr)
f.seek(0)
kwargs['stdin'] = f
process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
output, unused_err = process.communicate()
retcode = process.poll()
if retcode:
cmd = kwargs.get("args")
if cmd is None:
cmd = popenargs[0]
raise subprocess.CalledProcessError(retcode, cmd)
return output
def test_output(self):
stdout = TemporaryFile()
stderr = TemporaryFile()
run_caqe(os.path.join(script_dir, 'unittests/bosy_sat.qcir'), stdout,
stderr)
stdout.seek(0)
for line in stdout:
line = line.decode().strip()
if not line:
continue
elif 'c caqe' in line:
continue
elif 'r SAT' in line:
continue
elif 'r UNSAT' in line:
continue
else:
self.fail('Found unnecessary output {}'.format(line.strip()))
stderr.seek(0)
self.assertFalse(stderr.read().strip())
def run_git(*args, **kwargs):
git_cmd = kwargs.get('git_cmd', 'git')
cwd = kwargs.get('cwd')
cmd = (git_cmd, ) + args
stderr = TemporaryFile()
try:
proc = Popen(cmd,
stdout=PIPE,
stderr=stderr,
cwd=cwd,
universal_newlines=True)
except OSError as ex:
if ex.errno == errno.ENOENT:
raise GitNotFound("%r not found in PATH" % git_cmd)
raise
output = proc.stdout.readlines()
if proc.wait() != 0:
stderr.seek(0)
raise GitFailed(cmd, proc.returncode, stderr.read().rstrip())
return output
def generate_report_charts(game_cards: List[GameCard]) -> Dict[str, IO]:
rarity = TemporaryFile()
hist = rarity_histogram(get_rarity_dist(game_cards))
hist.savefig(rarity)
rarity.seek(0)
natures = TemporaryFile()
heatmap = natures_heatmap(get_nature_matrix(game_cards))
heatmap.savefig(natures)
natures.seek(0)
return {
'rarity': rarity,
'natures': natures,
}
def rtf(url):
'''
gets the url of the rtf file, and (tries to) return an xhtml version of it.
returns False if couldn't convert.
'''
remote = urlopen(url)
data = remote.read()
remote.close()
temp = TemporaryFile()
temp.write(data)
temp.seek(0)
try:
doc = Rtf15Reader.read(temp, errors='ignore')
xhtml = XHTMLWriter.write(doc, pretty=True).read()
except:
xhtml = False
logger.exception('Failed reading rtf from {0}'.format(url))
temp.close()
return xhtml
def export(self):
'''
'''
outfile = TemporaryFile()
np.savez(
outfile,
shadows=self.shadows,
)
_ = outfile.seek(0)
name = f'{self.data[:-4]}_correction.npz'
return pn.widgets.FileDownload(file=outfile, filename=name)
def upload_file():
if request.method == 'POST' and session['legit']:
uploaded_file = request.files['archive']
if not uploaded_file:
return ('You did not upload a file.', 400, [])
mirror = mirrorselect(uploaded_file.filename)
if mirror is None:
return (
'Unrecognized file extension. Only zip, css and image files are accepted.',
400, [])
mirrored_file = TemporaryFile()
mirror(uploaded_file, mirrored_file)
mirrored_file.seek(0)
root, ext = os.path.splitext(uploaded_file.filename)
attachment_filename = root + '-rtl' + ext
mimetype = guess_type(uploaded_file.filename)[0]
headers = Headers()
headers.add('Content-Disposition',
'attachment',
filename=attachment_filename)
data = wrap_file(request.environ, mirrored_file)
rv = app.response_class(data,
mimetype=mimetype,
headers=headers,
direct_passthrough=True)
rv.cache_control.public = True
cache_timeout = app.get_send_file_max_age(attachment_filename)
if cache_timeout is not None:
rv.cache_control.max_age = cache_timeout
rv.expires = int(time() + cache_timeout)
return rv
else:
session['legit'] = True
return render_template('index.html')
def shell(cmd, stdin=None, stdout_as_debug=False):
"""
Runs `cmd` in the shell and returns a stream of the output.
Raises `errors.StepFailure` if the command fails.
"""
if isinstance(cmd, list):
cmd = " ".join(cmd)
if stdout_as_debug:
cmd += ">&2"
assert isinstance(cmd, str)
log.debug(cmd)
stdout = TemporaryFile()
stderr = None
# if we are not in debug mode, capture stderr
if not is_debug():
stderr = TemporaryFile()
proc = subprocess.Popen(cmd,
shell=True,
stdin=stdin,
stdout=stdout,
stderr=stderr,
env=os.environ)
proc.wait()
stdout.seek(0)
if proc.returncode != 0:
if stderr is not None:
stderr.seek(0)
raise errors.StepFailure(cmd,
stdout.read().decode("UTF-8"),
stderr.read().decode("UTF-8"))
else:
raise errors.StepFailure(cmd, "No stdout captured.",
"No stderr captured.")
return stdout
def subprocess_check_output(*popenargs, **kwargs):
"""
Function to call a subprocess and gather the output.
Addresses a lack of check_output() prior to Python 2.7
"""
if 'stdout' in kwargs:
raise ValueError('stdout argument not allowed, it will be overridden.')
if 'stderr' in kwargs:
raise ValueError('stderr argument not allowed, it will be overridden.')
executable_exists(popenargs[0][0])
# NOTE: it is very, very important that we use temporary files for
# collecting stdout and stderr here. There is a nasty bug in python
# subprocess; if your process produces more than 64k of data on an fd that
# is using subprocess.PIPE, the whole thing will hang. To avoid this, we
# use temporary fds to capture the data
stdouttmp = TemporaryFile()
stderrtmp = TemporaryFile()
process = subprocess.Popen(stdout=stdouttmp,
stderr=stderrtmp,
*popenargs,
**kwargs)
process.communicate()
retcode = process.poll()
stdouttmp.seek(0, 0)
stdout = stdouttmp.read()
stdouttmp.close()
stderrtmp.seek(0, 0)
stderr = stderrtmp.read()
stderrtmp.close()
if retcode:
cmd = ' '.join(*popenargs)
raise Exception("'%s' failed(%d): %s" % (cmd, retcode, stderr),
retcode)
return (stdout, stderr, retcode)
class io:
def __init__(self, data=""):
self.stream = TemporaryFile(mode="w+b")
self.stream.write(data)
def __getitem__(self, key):
self.stream.seek(key)
return self.stream.read(1)
def __setitem__(self, key, item):
self.stream.seek(key)
self.stream.write(item)
def __str__(self):
self.stream.seek(0)
return self.stream.read()
def __len__(self):
return len(self)
def save(self, path):
with open(
path,
"w+",
) as f:
f.write(str(self))
def productSoldDetail(productId, address):
temp = TemporaryFile('w+t', encoding='utf-8')
temp.write(
'PurchaseID-CopyrightID-Permission-Timestmap-Price(Wei)-Buyer-address-Update'
+ '\n')
all_sell = action.functions.getPurchaseIdByProduceId(int(productId)).call(
{'from': keyring.get_password('DRMDEMO', 'address')})
for sell in all_sell[1:]:
information = action.functions.getPurchaseStorageById(sell).call(
{'from': keyring.get_password('DRMDEMO', 'address')})
temp.write(str(sell) + '----')
temp.write('----'.join('%s' % id for id in information) + '\n')
content = '尊敬的版权发布者，截止到%s时，您的版权ID为%s所售出的详细信息，请查收。' % (datetime.now(),
productId)
textApart = MIMEText(content)
temp.seek(0)
zipApart = MIMEApplication(temp.read())
zipApart.add_header('Content-Disposition',
'attachment',
filename='copyrightID_%s.txt' % (productId))
m = MIMEMultipart()
m.attach(textApart)
m.attach(zipApart)
m['Subject'] = Header('您的版权售卖详情——区块链音乐版权平台', 'utf-8')
m['from'] = EMAIL_HOST_USER
m['to'] = address
try:
server = smtplib.SMTP()
server.connect(EMAIL_HOST, 25)
server.login(EMAIL_HOST_USER, EMAIL_HOST_PASSWORD)
server.sendmail(EMAIL_HOST_USER, address, m.as_string())
server.quit()
temp.close()
return 'email send success'
except smtplib.SMTPException as e:
temp.close()
return 'email send error%s' % e
def get_registry_proto(self):
from google.cloud import storage
from google.cloud.exceptions import NotFound
file_obj = TemporaryFile()
registry_proto = RegistryProto()
try:
bucket = self.gcs_client.get_bucket(self._bucket)
except NotFound:
raise Exception(
f"No bucket named {self._bucket} exists; please create it first."
)
if storage.Blob(bucket=bucket, name=self._blob).exists(self.gcs_client):
self.gcs_client.download_blob_to_file(
self._uri.geturl(), file_obj, timeout=30
)
file_obj.seek(0)
registry_proto.ParseFromString(file_obj.read())
return registry_proto
raise FileNotFoundError(
f'Registry not found at path "{self._uri.geturl()}". Have you run "feast apply"?'
)
