def request(self, url, params={}, headers={}): """ Retrieve SDMX messages. If needed, override in subclasses to support other data providers. :param url: The URL of the message. :type url: str :return: the xml data as file-like object """ # Generate current config. Merge in any given headers cur_config = self.config.copy() if 'headers' in cur_config: cur_config['headers'] = cur_config['headers'].copy() cur_config['headers'].update(headers) else: cur_config['headers'] = headers with closing(requests.get(url, params=params, **cur_config)) as response: if response.status_code == requests.codes.OK: source = STF(max_size=self.max_size) for c in response.iter_content(chunk_size=1000000): source.write(c) else: source = None code = int(response.status_code) if 400 <= code <= 499: raise response.raise_for_status() return source, response.url, response.headers, code
def run(self, opts): from lzma.xz import compress self.h = sha1() tdir = mkdtemp("calibre-mathjax-build") try: src = opts.path_to_mathjax or self.download_mathjax_release(tdir, opts.mathjax_url) self.info("Compressing MathJax...") t = SpooledTemporaryFile() with ZipFile(t, "w", ZIP_STORED) as zf: self.add_file(zf, self.j(src, "unpacked", "MathJax.js"), "MathJax.js") self.add_tree( zf, self.j(src, "fonts", "HTML-CSS", self.FONT_FAMILY, "woff"), "fonts/HTML-CSS/%s/woff" % self.FONT_FAMILY, ) for d in "extensions jax/element jax/input jax/output/CommonHTML".split(): self.add_tree(zf, self.j(src, "unpacked", *d.split("/")), d) zf.comment = self.h.hexdigest() t.seek(0) with open(self.j(self.RESOURCES, "content-server", "mathjax.zip.xz"), "wb") as f: compress(t, f, level=9) with open(self.j(self.RESOURCES, "content-server", "mathjax.version"), "wb") as f: f.write(zf.comment) finally: shutil.rmtree(tdir)
def send(self, request, stream=None, timeout=None, verify=None, cert=None, proxies=None): pathname = url_to_path(request.url) resp = Response() resp.status_code = 200 resp.url = request.url try: stats = lstat(pathname) except (IOError, OSError) as exc: resp.status_code = 404 message = { "error": "file does not exist", "path": pathname, "exception": repr(exc), } fh = SpooledTemporaryFile() fh.write(ensure_binary(json.dumps(message))) fh.seek(0) resp.raw = fh resp.close = resp.raw.close else: modified = formatdate(stats.st_mtime, usegmt=True) content_type = guess_type(pathname)[0] or "text/plain" resp.headers = CaseInsensitiveDict({ "Content-Type": content_type, "Content-Length": stats.st_size, "Last-Modified": modified, }) resp.raw = open(pathname, "rb") resp.close = resp.raw.close return resp
def gen_data(location=None, **kwargs): """Fetches realtime data and generates records""" url = '%s/%s' % (kwargs['BASE_URL'], location) r = requests.get(url) f = SpooledTemporaryFile() # wrap to access `fileno` f.write(r.content) return io.read_xls(r., sanitize=True, encoding=r.encoding)
def fetch_data(config): """Fetches realtime data and generates records""" ckan = CKAN(config['ENDPOINT'], apikey=config['API_KEY']) # r = ckan.fetch_resource(config['RID']) # if using ckanutils resource = ckan.action.resource_show(id=config['RID']) url = resource.get('perma_link') or resource.get('url') r = requests.get(url, stream=True) if any('403' in h.headers.get('x-ckan-error', '') for h in r.history): raise NotAuthorized( 'Access to fetch resource %s was denied.' % config['RID']) try: ext = splitext(url)[1].split('.')[1] except IndexError: ext = cv.ctype2ext(r.headers['Content-Type']) if ext == 'csv': records = io.read_csv(r.raw, sanitize=True, encoding=r.encoding) elif ext in {'xls', 'xlsx'}: r = requests.get(url) f = SpooledTemporaryFile() f.write(r.content) records = io.read_xls(f, sanitize=True, encoding=r.encoding) else: msg = 'Filetype `%s` unsupported.' msg += 'Please view tabutils.io documentation for assistance.' raise TypeError(msg) constraints = [('adm0_name', 'a'), ('mp_month', '3'), ('mp_year', '2015')] filterer = lambda x: all(x[k].lower().startswith(v) for k, v in constraints) return it.ifilter(filterer, records)
def string2spool(input_string): """Takes a string as an argument and returns an open file handle with the contents of the string""" file_object=SpooledTemporaryFile() file_object.write(input_string) file_object.seek(0) return file_object
def create_dump(self): if not self.connection.is_usable(): self.connection.connect() dump_file = SpooledTemporaryFile(max_size=10 * 1024 * 1024) self._write_dump(dump_file) dump_file.seek(0) return dump_file
def push_index(self): stream = SpooledTemporaryFile(max_size=20 * MB) pointers = 0 stream.write(struct.pack(OFFSET_FMT, pointers)) self.indexes.append([ stream, pointers, self.block_size - self.pointer_size ])
def __init__(self, data=None, fp=None, length=-1): assert bool(data is not None) ^ bool(fp) if length == -1: if data is not None: length = len(data) else: length = get_size(fp) # can be -1 # We allow writer reuse, but if we're working with a stream, we cannot # seek. Copy the data to a tempfile. if fp and not can_seek(fp): newfp = SpooledTemporaryFile(MAX_INMEMORY_SIZE) sendfile(newfp, fp) length = newfp.tell() newfp.seek(0) fp = newfp self.data = data self.fp = fp self.fpreads = 0 # keep track of fp usage self.length = length assert length >= 0 self.use_tempfile = length > MAX_INMEMORY_SIZE
def close(self): """Send the change to the DFS, and close the file.""" self.flush() if 'c' not in self.mode: SpooledTemporaryFile.close(self)
class GoogleCloudFile(File): def __init__(self, name, mode, storage): self.name = name self.mime_type = mimetypes.guess_type(name)[0] self._mode = mode self._storage = storage # NOTE(mattrobenolt): This is the same change in behavior as in # the s3 backend. We're opting now to load the file # or metadata at this step. This means we won't actually # know a file doesn't exist until we try to read it. self.blob = FancyBlob(storage.download_url, self.name, storage.bucket) self._file = None self._is_dirty = False @property def size(self): return self.blob.size def _get_file(self): if self._file is None: with metrics.timer('filestore.read', instance='gcs'): self._file = SpooledTemporaryFile( max_size=self._storage.max_memory_size, suffix=".GSStorageFile", dir=None, ) if 'r' in self._mode: self._is_dirty = False self.blob.download_to_file(self._file) self._file.seek(0) return self._file def _set_file(self, value): self._file = value file = property(_get_file, _set_file) def read(self, num_bytes=None): if 'r' not in self._mode: raise AttributeError("File was not opened in read mode.") if num_bytes is None: num_bytes = -1 return super(GoogleCloudFile, self).read(num_bytes) def write(self, content): if 'w' not in self._mode: raise AttributeError("File was not opened in write mode.") self._is_dirty = True return super(GoogleCloudFile, self).write(force_bytes(content)) def close(self): if self._file is not None: if self._is_dirty: self.file.seek(0) self.blob.upload_from_file(self.file, content_type=self.mime_type) self._file.close() self._file = None
def _open(self, name, mode = 'rb') -> File: name = self._transform_name(name) content = self.service.get_blob_content(self.container, name) file = SpooledTemporaryFile() file.write(content) file.seek(0) # explicitly reset to allow reading from the beginning afterwards as-is return File(file)
def start(self, args): self.outFile = SpooledTemporaryFile() self.errFile = SpooledTemporaryFile() self.cmdline = list2cmdline(args) print 'starting: ' + self.cmdline self.process = Popen(args, stderr=self.errFile, stdout=self.outFile, universal_newlines=False) self.process_start = time()
def generate(self): points = self.points() self.buffer = 2*self.pad count = np.zeros([x + 2*self.buffer for x in self.expanded_size]) density = np.zeros([x + 2*self.buffer for x in self.expanded_size]) # Render the B&W density version of the heatmap dot_size = self.dot.shape[0] for x, y, weight in points: x1 = x + self.buffer - (dot_size - 1)/2 y1 = y + self.buffer - (dot_size - 1)/2 count[y1:(y1 + dot_size), x1:(x1 + dot_size)] += self.dot density[y1:(y1 + dot_size), x1:(x1+ dot_size)] += self.dot*float(weight) # Pick the field to map if gheat_settings.GHEAT_MAP_MODE == gheat_settings.GHEAT_MAP_MODE_COUNT: img = count #opacity = np.zeros(img.shape()) + 255 elif gheat_settings.GHEAT_MAP_MODE == gheat_settings.GHEAT_MAP_MODE_SUM_DENSITY: img = density #opacity = np.clip(count, 0, gheat_settings.GHEAT_OPACITY_LIMIT) elif gheat_settings.GHEAT_MAP_MODE == gheat_settings.GHEAT_MAP_MODE_MEAN_DENSITY: img = density img[count > 0] /= count[count > 0] #opacity = np.clip(count, 0, gheat_settings.GHEAT_OPACITY_LIMIT) else: raise ValueError, 'Unknown map mode' # Crop resulting density image (which could have grown) into the # actual canvas size we want img = img[(self.pad + self.buffer):(SIZE + self.pad + self.buffer), (self.pad + self.buffer):(SIZE + self.pad + self.buffer)] #opacity = opacity[self.pad:(SIZE + self.pad), self.pad:(SIZE + self.pad)] # Maybe use a logarithm img = np.where(img>0, np.log(img)+1, img) # Convert to a 0 to 255 image img = np.clip(256.0*np.power(img/gheat_settings.GHEAT_MAX_VALUE, gheat_settings.GHEAT_SCALING_COEFFICIENT), 0, 255.999).astype('uint8') # Given the B&W density image, generate a color heatmap based on # this Tile's color scheme. colour_image = np.zeros((SIZE, SIZE, 4), 'uint8') + 255 for i in range(3): colour_image[:,:,i] = self.schemeobj.colors[:,i][255 - img] colour_image[:,:,3] = np.where(img > gheat_settings.GHEAT_MIN_DENSITY, 255, 0) tmpfile = SpooledTemporaryFile() writer = png.Writer(SIZE, SIZE, alpha=True, bitdepth=8) writer.write(tmpfile, np.reshape(colour_image, (SIZE, SIZE*4))) tmpfile.seek(0) return tmpfile
def file_from_content(content): f = content if isinstance(content, cgi.FieldStorage): f = content.file elif isinstance(content, byte_string): f = SpooledTemporaryFile(INMEMORY_FILESIZE) f.write(content) f.seek(0) return f
def test_run_command_stdin(self): connector = BaseCommandDBConnector() stdin = SpooledTemporaryFile() stdin.write(b'foo') stdin.seek(0) # Run stdout, stderr = connector.run_command('cat', stdin=stdin) self.assertEqual(stdout.read(), b'foo') self.assertFalse(stderr.read())
class GoogleCloudFile(File): def __init__(self, name, mode, storage): self.name = name self.mime_type = mimetypes.guess_type(name)[0] self._mode = mode self._storage = storage self.blob = storage.bucket.get_blob(name) if not self.blob and 'w' in mode: self.blob = Blob(self.name, storage.bucket) self._file = None self._is_dirty = False @property def size(self): return self.blob.size def _get_file(self): if self._file is None: self._file = SpooledTemporaryFile( max_size=self._storage.max_memory_size, suffix=".GSStorageFile", dir=setting("FILE_UPLOAD_TEMP_DIR", None) ) if 'r' in self._mode: self._is_dirty = False self.blob.download_to_file(self._file) self._file.seek(0) return self._file def _set_file(self, value): self._file = value file = property(_get_file, _set_file) def read(self, num_bytes=None): if 'r' not in self._mode: raise AttributeError("File was not opened in read mode.") if num_bytes is None: num_bytes = -1 return super(GoogleCloudFile, self).read(num_bytes) def write(self, content): if 'w' not in self._mode: raise AttributeError("File was not opened in write mode.") self._is_dirty = True return super(GoogleCloudFile, self).write(force_bytes(content)) def close(self): if self._file is not None: if self._is_dirty: self.file.seek(0) self.blob.upload_from_file(self.file, content_type=self.mime_type) self._file.close() self._file = None
def read_file_handle(self, filename): """Get the file data, put it in a SpooledTemporaryFile object for return and reading """ logger.debug("read_file_handle('%s')" % filename) read_buffer = SpooledTemporaryFile() response = self.rc.fs.read_file(read_buffer, filename) logger.debug(response) read_buffer.seek(0) return read_buffer
def close(self): """On close, seek to 0 and write the data via the API, then close() for realz """ logger.debug("close() called on WriteBuffer") self.seek(0) logger.debug("Attempting to create file at dir_path %s with name %s" % (self.path, self.filename)) self.fs.rc.fs.write_file(self, self.fullpath) SpooledTemporaryFile.close(self) # old-style class!
def filter_file(filter, filename, membuffer=10485760): tmp = SpooledTemporaryFile(max_size=membuffer) with open(filename) as input: for line in input: if filter(line): tmp.write(line) tmp.seek(0) with open(filename, "w") as output: for line in tmp: output.write(line)
def run_command(self, command, stdin=None, env=None): """ Launch a shell command line. :param command: Command line to launch :type command: str :param stdin: Standard input of command :type stdin: file :param env: Environment variable used in command :type env: dict :return: Standard output of command :rtype: file """ cmd = shlex.split(command) stdout = SpooledTemporaryFile(max_size=settings.TMP_FILE_MAX_SIZE, dir=settings.TMP_DIR) stderr = SpooledTemporaryFile(max_size=settings.TMP_FILE_MAX_SIZE, dir=settings.TMP_DIR) full_env = self.env.copy() full_env.update(env or {}) try: process = Popen(cmd, stdin=stdin, stdout=stdout, stderr=stderr, env=full_env) process.wait() if process.poll(): stderr.seek(0) raise exceptions.CommandConnectorError( "Error running: {}\n{}".format(command, stderr.read())) stdout.seek(0) stderr.seek(0) return stdout, stderr except OSError as err: raise exceptions.CommandConnectorError( "Error running: {}\n{}".format(command, str(err)))
def __init__(self, path, filename, fs, max_size=WRITE_BUFFER_SIZE): """We need the path so we can write the buffered file to the API""" SpooledTemporaryFile.__init__(self, max_size=max_size) # old-style! self.path = path self.filename = filename self.fs = fs self.fullpath = '' try: self.fullpath = self.create_file() except RequestError, e: SpooledTemporaryFile.close(self) raise FilesystemError(str(e))
def upload_file(self, user, stream, expected_size, filename, force_coll_name=''): """Upload WARC archive. :param User user: user :param stream: file object :param int expected_size: expected WARC archive size :param str filename: WARC archive filename :param str force_coll_name: name of collection to upload into :returns: upload information :rtype: dict """ temp_file = None logger.debug('Upload Begin') logger.debug('Expected Size: ' + str(expected_size)) #is_anon = False size_rem = user.get_size_remaining() logger.debug('User Size Rem: ' + str(size_rem)) if size_rem < expected_size: return {'error': 'out_of_space'} if force_coll_name and not user.has_collection(force_coll_name): #if is_anon: # user.create_collection(force_coll, 'Temporary Collection') #else: #status = 'Collection {0} not found'.format(force_coll_name) return {'error': 'no_such_collection'} temp_file = SpooledTemporaryFile(max_size=BLOCK_SIZE) stream = CacheingLimitReader(stream, expected_size, temp_file) if filename.endswith('.har'): stream, expected_size = self.har2warc(filename, stream) temp_file.close() temp_file = stream infos = self.parse_uploaded(stream, expected_size) total_size = temp_file.tell() if total_size != expected_size: return {'error': 'incomplete_upload', 'expected': expected_size, 'actual': total_size} upload_id, upload_key = self._init_upload_status(user, total_size, 1, filename=filename) return self.handle_upload(temp_file, upload_id, upload_key, infos, filename, user, force_coll_name, total_size)
def get_empty(self, opacity=OPAQUE): color = self.colors[255,:] #color[3] = int(color[3]*float(opacity)/255) color[3] = 0 empty = np.tile(color, SIZE*SIZE).reshape(SIZE, SIZE*4) tmpfile = SpooledTemporaryFile() writer = png.Writer(SIZE, SIZE, alpha=True, bitdepth=8) writer.write(tmpfile, empty) tmpfile.seek(0) return tmpfile
class DropBoxFile(File): def __init__(self, name, storage): self.name = name self._storage = storage @property def file(self): if not hasattr(self, '_file'): response = self._storage.client.get_file(self.name) self._file = SpooledTemporaryFile() copyfileobj(response, self._file) self._file.seek(0) return self._file
def generate(self): points = self.points() # Grab a new PIL image canvas img = Image.new('RGBA', self.expanded_size, 'white') # Render the B&W density version of the heatmap for x,y in points: dot_placed = Image.new('RGBA', self.expanded_size, 'white') dot_placed.paste(self.dot, (x, y)) img = ImageChops.multiply(img, dot_placed) # Crop resulting density image (which could have grown) into the # actual canvas size we want img = img.crop((self.pad, self.pad, SIZE+self.pad, SIZE+self.pad)) img = ImageChops.duplicate(img) # converts ImageCrop => Image # Given the B&W density image, generate a color heatmap based on # this Tile's color scheme. _computed_opacities = dict() pix = img.load() # Image => PixelAccess for x in range(SIZE): for y in range(SIZE): # Get color for this intensity # ============================ # is a value val = self.schemeobj.colors[0, pix[x,y][0]] try: pix_alpha = val[3] # the color image has transparency except IndexError: pix_alpha = OPAQUE # it doesn't # Blend the opacities # =================== conf, pixel = self.opacity, pix_alpha if (conf, pixel) not in _computed_opacities: opacity = int(( (conf/255.0) # from configuration * (pixel/255.0) # from per-pixel alpha ) * 255) _computed_opacities[(conf, pixel)] = opacity pix[x,y] = val[:3] + (_computed_opacities[(conf, pixel)],) tmpfile = SpooledTemporaryFile() img.save(tmpfile, 'PNG') tmpfile.seek(0) return tmpfile
def generate_thumbnail(self, content): content = file_from_content(content) uploaded_image = Image.open(content) if max(uploaded_image.size) >= self.max_size: uploaded_image.thumbnail((self.max_size, self.max_size), Image.BILINEAR) content = SpooledTemporaryFile(INMEMORY_FILESIZE) uploaded_image.save(content, uploaded_image.format) content.seek(0) thumbnail = uploaded_image.copy() thumbnail.thumbnail(self.thumbnail_size, Image.ANTIALIAS) thumbnail = thumbnail.convert('RGBA') thumbnail.format = self.thumbnail_format output = SpooledTemporaryFile(INMEMORY_FILESIZE) thumbnail.save(output, self.thumbnail_format) output.seek(0) thumb_path, thumb_id = self.store_content(output, 'thumb.%s' % self.thumbnail_format.lower()) self['thumb_id'] = thumb_id self['thumb_path'] = thumb_path thumbnail_file = self.thumb_file self['_thumb_public_url'] = thumbnail_file.public_url content.close()
def _buffer_response(status_headers, iterator): out = SpooledTemporaryFile(ProxyRouter.BUFF_RESPONSE_MEM_SIZE) size = 0 for buff in iterator: size += len(buff) out.write(buff) content_length_str = str(size) # remove existing content length status_headers.replace_header('Content-Length', content_length_str) out.seek(0) return RewriteContent.stream_to_gen(out)
def __init__(self, filepath, mode='rtc'): """filepath: the path of the distant file mode: take the same argument as mode argument of the global open() + optional flag c (which mean store in cache). """ self.mode = mode self.filepath = filepath host, port = utils.get_host_port(_config['nameserver']) self.srv = utils.get_server(filepath, host, port) if self.srv is None: raise DFSIOError('Impossible to find a server that serve %s.' % filepath) self.last_modified = None SpooledTemporaryFile.__init__(self, _config['max_size'], mode.replace('c', '')) host, port = utils.get_host_port(_config['lockserver']) if utils.is_locked(filepath, host, port): raise DFSIOError('The file %s is locked.' % filepath) if 'w' not in mode: host, port = utils.get_host_port(self.srv) with closing(HTTPConnection(host, port)) as con: con.request('GET', filepath) response = con.getresponse() self.last_modified = response.getheader('Last-Modified') status = response.status if status not in (200, 204): raise DFSIOError('Error (%d) while opening file.' % status) if status != 204: self.write(response.read()) if 'r' in mode: self.seek(0) self.lock_id = None if 'a' in mode or 'w' in mode: # automatically gets a lock if we're in write/append mode host, port = utils.get_host_port(_config['lockserver']) self.lock_id = int(utils.get_lock(filepath, host, port)) if 'c' in mode: File._cache[filepath] = self
def file_from_content(content): """Provides a real file object from file content Converts ``FileStorage``, ``FileIntent`` and ``bytes`` to an actual file. """ f = content if isinstance(content, cgi.FieldStorage): f = content.file elif isinstance(content, FileIntent): f = content._fileobj elif isinstance(content, byte_string): f = SpooledTemporaryFile(INMEMORY_FILESIZE) f.write(content) f.seek(0) return f
def _har2warc_temp_file(self): return SpooledTemporaryFile(max_size=BLOCK_SIZE)
def insert_extracts(self, user, payload): """Insert multiple extracts into the database, as a single request :param user: :param payload: :return: """ conn = g._db db_name = self._config.get("MySql", "db") env = self._config.get("api", "env") cor_admin = user.get("digs") # self._logger.error("oopies") user_info = payload.get("user") samples = payload.get("samples") facility = payload.get("facility") manifesto = payload.get("manifesto") institution = user.get("ldap", {}).get("actor_institution") request_uid = str(uuid4()) # create array of choices to insert into database # this is only for tracking the options presented purposes (possible debugging), # since only facility.get("choice") is critical information fse_choices = facility.get("digs") fse = [] for key in fse_choices: try: rank = int(key) except: pass else: fse.append(fse_choices[str(rank)]) request_id = None emailed = None # Create request query = """INSERT INTO `{}`.`Requests` (`uid`, `digs_id`, `fse`, `requester`, `institution`, `created` ) VALUES (\"{}\", (SELECT id from `{}`.`Digs` WHERE digs_core_number LIKE \"{}\"), \'{}\', "{}", \"{}\", NOW())""" \ .format(db_name, request_uid, db_name, facility.get("choice"), json.dumps(fse), user.get("ldap", {}).get("actor_username"), institution) with conn.cursor() as cursor: try: cursor.execute(query) conn.commit() except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() raise ApiSqlException(title=str(e), description=str(e), logger=self._logger, config=self._config, stacktrace=exc_value) else: query = """SELECT id from `{}`.`Requests` WHERE `uid` LIKE '{}'""".format(db_name, request_uid) try: cursor.execute(query) columns = [field[0] for field in cursor.description] res = cursor.fetchone() if res: request = dict(zip(columns, [res[0]])) except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() raise ApiSqlException(title=str(e), description=str(e), logger=self._logger, config=self._config, stacktrace=exc_value) else: request_id = request.get("id") selected_facility = None query = """SELECT * FROM `{}`.`Digs` WHERE `digs_core_number` LIKE '{}'""".format(db_name, facility.get("choice")) try: cursor.execute(query) columns = [field[0] for field in cursor.description] res = cursor.fetchone() if res: selected_facility = dict(zip(columns, res)) except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() raise ApiSqlException(title=str(e), description=str(e), logger=self._logger, config=self._config, stacktrace=exc_value) csv_manifest = [] res_manifesto = [] if request_id: csv_manifest.append('DIGS Sequencing Request Submission ID: R{}\n\n'.format(request_id)) # requester info block csv_manifest.append('Requester Contact Information\n') csv_manifest.append('Name,{}\n'.format(user_info.get("name"))) csv_manifest.append('Address,"{}, {} {} {}"\n'.format( user_info.get("street_address"), user_info.get("city"), user_info.get("state_province") if user_info.get("state_province") != "N/A" else "", selected_facility.get("zipcode"))) csv_manifest.append('Phone,{}\n'.format(user_info.get("daytime_phone"))) csv_manifest.append('Email,{}\n\n'.format(user_info.get("email"))) # facility info block csv_manifest.append('DIGS Facility Contact Information\n') csv_manifest.append('Name,{}\n'.format(selected_facility.get("contact_name"))) csv_manifest.append('Address,"{}, {} {} {}"\n'.format( selected_facility.get("shipping_address_street"), selected_facility.get("shipping_address_city"), selected_facility.get("shipping_address_state") if selected_facility.get("shipping_address_state") \ != "N/A" else "", selected_facility.get("shipping_address_zip"))) csv_manifest.append('Phone,{}\n'.format(selected_facility.get("contact_phone"))) csv_manifest.append('Email,{}\n\n'.format(selected_facility.get("contact_email"))) # sample table block csv_manifest.append("Sample Identifier,Extract Identifier,Sequencing Study Identifier,Submission ID," "Submission Type,Submitter Name,Submission Date,Project Identifier," "Contributing Institution,Virus Identifier,Strain Name,Influenza Subtype," "Host Species,Lab Host,Passage History,Pathogenicity," "Extract Material,Volume (µl),Concentration (ng/µl),Concentration Determined By," "Sequencing Technology,Analysis Type,Raw Sequences,Comments\n") request_contains_rna_sample = False for row, sample in enumerate(samples, 12): if sample.get("extract_material") == "Viral RNA": request_contains_rna_sample = True extract_id = "R{}_{}".format(request_id, sample["extract_id"]) query = """INSERT INTO `{}`.`Extracts` (`request_id`, `sample_id`, `extract_id`, `sequencing_study_identifier`, `submission_id`, `submission_type`, `submitter_name`, `submission_date`, `project_identifier`, `virus_identifier`, `influenza_subtype`, `host_species`, `lab_host`, `passage_history`, `pathogenicity`, `extract_material`, `volume`, `concentration`, `concentration_determined_by`, `sequencing_tecnhology`, `analysis_type`, `raw_sequences`, `comments`, `status`, `created`, `sample_identifier`) VALUES( {}, '{}', '{}', '{}', '{}', '{}', '{}', '{}', '{}', '{}', '{}', '{}', '{}', '{}', '{}', '{}', {}, {}, '{}', '{}', '{}','{}', '{}', 'Requested', CURRENT_TIMESTAMP, '{}' )""". \ format(db_name, request_id, sample.get("sample_id"), extract_id, sample.get("sequencing_study_identifier"), sample.get("submission_id"), sample.get("submission_type"), sample.get("submitter_name"), sample.get("submission_date"), sample.get("project_identifier"), sample.get("virus_identifier"), sample.get("influenza_subtype"), sample.get("host_species"), sample.get("lab_host"), sample.get("passage_history"), sample.get("pathogenicity"), sample.get("extract_material"), sample.get("volume"), sample.get("concentration"), sample.get("concentration_determined_by"), json.dumps(sample.get("sequencing_technology")), json.dumps(sample.get("analysis_type")), sample.get("raw_sequences", '0'), sample.get("comments"), sample.get("sample_identifier")) try: res = cursor.execute(query) if res: conn.commit() except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() raise ApiSqlException(title=str(e), description=str(e), logger=self._logger, config=self._config, stacktrace=exc_value) else: analysis_type = " / ".join( [analysis for analysis in sample["analysis_type"] if sample["analysis_type"][analysis]]) sequencing_technology = " or ".join( [tech for tech in sample["sequencing_technology"] if sample["sequencing_technology"][tech]]) csv_manifest.append("\"{}\",\"{}\",\"{}\",\"{}\t\",\"{}\",\"{}\",\"{}\",\"{}\",\"{}\",\"{}\"," "\"{}\",\"{}\",\"{}\",\"{}\",\"{}\",\"{}\",\"{}\",\"{}\",\"{}\",\"{}\"," "\"{}\",\"{}\",\"{}\",\"{}\"\n".format( sample.get("sample_identifier"), extract_id, sample.get("sequencing_study_identifier"), sample.get("submission_id", ""), sample.get("submission_type", ""), sample.get("submitter_name", ""), sample.get("submission_date", ""), sample.get("project_identifier"), sample.get("contributing_institution", ""), sample.get("virus_identifier"), sample.get("strain_name"), sample.get("influenza_subtype"), sample.get("host_species"), sample.get("lab_host"), sample.get("passage_history"), sample.get("pathogenicity"), sample.get("extract_material"), sample.get("volume"), sample.get("concentration"), sample.get("concentration_determined_by"), sequencing_technology, analysis_type, sample.get("raw_sequences", "N"), sample.get("comments", ""))) res_manifesto.append([sample.get("sample_identifier"), sample.get("extract_id"), sample.get("sequencing_study_identifier"), sample.get("submission_id", ""), sample.get("submission_type", ""), sample.get("submitter_name", ""), sample.get("submission_date", ""), sample.get("project_identifier"), sample.get("contributing_institution", ""), sample.get("virus_identifier"), sample.get("strain_name"), sample.get("influenza_subtype"), sample.get("host_species"), sample.get("lab_host"), sample.get("passage_history"), sample.get("pathogenicity"), sample.get("extract_material"), sample.get("volume"), sample.get("concentration"), sample.get("concentration_determined_by"), sequencing_technology, analysis_type, sample.get("raw_sequences", "N"), sample.get("comments", "")]) # wb.save("manifest.xls") with SpooledTemporaryFile() as fh: # writer = csv.writer(fh, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL) fh.writelines([x.encode('utf-8') for x in csv_manifest]) fh.seek(0) # with open("swt/scripts/request_{}.{}".format(request_id, "csv"), 'w') as file: # for line in csv_manifest: # file.write(line) aws_configuration = { 'aws_access_key_id': self._config.get('aws', 'aws_access_key_id'), 'aws_secret_access_key': self._config.get('aws', 'aws_secret_access_key'), 'region_name': self._config.get('aws', 'aws_region') } _s3 = S3(credentials=aws_configuration, logger=self._logger) filename = "request_{}.{}".format(request_id, "csv") bucket = self._config.get("aws", "s3_bucket") # bucket = "swt-prod" key = "manifest-files/" + filename pre_signed_url = _s3.upload(bucket, key, fh) if not pre_signed_url: return {"status": False, "statusText": "Error uploading manifest to aws"} rna_warning_msg = """<br><p><u>Notice</u>: This Sequencing Request includes RNA-based samples. Please handle accordingly when shipping your samples.</p>""" \ if request_contains_rna_sample else "" conn = boto.ses.connect_to_region(**aws_configuration) email_from = '*****@*****.**' user_email = user_info.get("email") facility_email = user_email if env == "prod": facility_email = facility.get("digs", {}).get("contact_info", {})\ .get(facility.get("choice")).get("email") email_body_facility = EXTRACT_EMAIL_FACILITY_BODY.format(request_id, user_info.get("name"), "{}, {} {} {}".format( user_info.get("street_address"), user_info.get("city"), user_info.get("state_province") \ if user_info.get("state_province") \ != "N/A" else "", user_info.get("zipcode")), user_info.get("daytime_phone"), user_info.get("email"), EXTRACT_EMAIL_SIGNATURE, rna_warning_msg) emailMsg = MIMEMultipart() emailMsg['Subject'] = EXTRACT_EMAIL_FACILITY_SUBJECT emailMsg['From'] = email_from emailMsg['To'] = facility_email emailMsg.preamble = 'Multipart message.\n' part = MIMEText(email_body_facility, 'html') emailMsg.attach(part) part = MIMEBase('application', 'octet-stream') fh.seek(0) part.set_payload(fh.read()) encode_base64(part) part.add_header('Content-Disposition', 'attachment; filename="shipping_manifest.csv"') emailMsg.attach(part) emailed_facility = conn.send_raw_email(emailMsg.as_string(), source=emailMsg['From'], destinations=[emailMsg['To']]) email_body_requester = EXTRACT_EMAIL_REQUESTER_BODY.format(user_info.get("name"), request_id, pre_signed_url, selected_facility.get( "digs_core_name"), selected_facility.get( "digs_core_name"), EXTRACT_EMAIL_SIGNATURE, rna_warning_msg) emailed_requester = conn.send_email(email_from, EXTRACT_EMAIL_REQUESTER_SUBJECT, None, to_addresses=user_info.get("email"), format="html", html_body=email_body_requester) email_failed = not emailed_facility or not emailed_requester if email_failed: raise ApiSqlException( title="Extracts have been saved, but there was a problem emailing the DIGS facility. Please contact the DPCC", description="Extracts have been saved, but there was a problem emailing the DIGS facility. Please contact the DPCC", logger=self._logger, config=self._config) return {"status": True, "request_id": request_id, "manifesto": res_manifesto}
def download_file(self, key, path='', temporary=False, on_memory=False, retries=2, return_path=False): """Download a file from S3, retrying if the download fails. Optionally, the file could be downloaded as a temporary file, returning a file handler. Input: key, the S3 key object to download path, folder to store the file temporary, indicates if the file should be downloaded as a temporary file. With this, the function returns a file handler on_memory, if the 'temporary' parameter is True, and this is True, the temporary file is created in memory, using an instance of tempfile.SpooledTemporaryFile class retries, number of retries if the download fails Output: The path where the file was downloaded, or a file handler if temporary is True """ k = self.get_key(key) # Creates the path if it not exists if not on_memory and path and not os.path.isdir(path): os.makedirs(path) fails = 0 for retry in xrange(retries): try: logger.info('Downloading {0}/{1} to {2}'.format(self.s3_bucket_url, key, path)) # When using regular files, it's not necessary to return a file handler tmp_file_handler = None if temporary: log_msg = "Using a temporary file" # This is a hack: gzip.GzipFile only accepts reading from files # with 'r' or 'rb' mode. This is checked recovering the first # letter from the mode, so, opening the file using the 'r+b' mode # allows reading and writting the temporary file without problems if re.search('\.gz$', key): file_mode = 'r+b' else: # In other cases, use the default mode for NamedTemporaryFile file_mode = 'w+b' if on_memory: log_msg += " [on memory]" tmp_file_handler = SpooledTemporaryFile(mode=file_mode, max_size=MAX_TEMP_FILE_SIZE) else: log_msg += " [on disk]" tmp_file_handler = NamedTemporaryFile(mode=file_mode, dir=path) log_msg += " [MODE: {0}]" logger.debug(log_msg.format(file_mode)) k.get_contents_to_file(tmp_file_handler) # This is necessary, Boto doesn't do it tmp_file_handler.seek(0) else: logger.debug("Using a regular file") _, filename = os.path.split(key) if path: final_path = os.path.join(path, filename) else: final_path = filename k.get_contents_to_filename(final_path) logger.info('File {0} was succesfully downloaded'.format(key)) if temporary: return tmp_file_handler elif return_path: return final_path except Exception: log_msg = 'Download failed for {0}/{1}...'.format(self.s3_bucket_url, key) fails += 1 if fails >= retries: log_msg += ' Not retrying (Failed retries: {0})'.format(retry) logger.error(log_msg) raise else: log_msg += ' Retrying' logger.info(log_msg)
def fetch_pack_from_origin( self, origin_url: str, base_repo: RepoRepresentation, do_activity: Callable[[bytes], None], ) -> FetchPackReturn: """Fetch a pack from the origin""" pack_buffer = SpooledTemporaryFile(max_size=self.temp_file_cutoff) transport_url = origin_url logger.debug("Transport url to communicate with server: %s", transport_url) client, path = dulwich.client.get_transport_and_path( transport_url, thin_packs=False ) logger.debug("Client %s to fetch pack at %s", client, path) size_limit = self.pack_size_bytes def do_pack(data: bytes) -> None: cur_size = pack_buffer.tell() would_write = len(data) if cur_size + would_write > size_limit: raise IOError( f"Pack file too big for repository {origin_url}, " f"limit is {size_limit} bytes, current size is {cur_size}, " f"would write {would_write}" ) pack_buffer.write(data) pack_result = client.fetch_pack( path, base_repo.determine_wants, base_repo.graph_walker(), do_pack, progress=do_activity, ) remote_refs = pack_result.refs or {} symbolic_refs = pack_result.symrefs or {} pack_buffer.flush() pack_size = pack_buffer.tell() pack_buffer.seek(0) logger.debug("fetched_pack_size=%s", pack_size) # check if repository only supports git dumb transfer protocol, # fetched pack file will be empty in that case as dulwich do # not support it and do not fetch any refs self.dumb = transport_url.startswith("http") and getattr(client, "dumb", False) return FetchPackReturn( remote_refs=utils.filter_refs(remote_refs), symbolic_refs=utils.filter_refs(symbolic_refs), pack_buffer=pack_buffer, pack_size=pack_size, )
def _write_tempfile(self, writer_callable): fh = SpooledTemporaryFile() writer_callable(fh) fh.seek(0) return fh
def create_zipfile_storage(self): spooled = SpooledTemporaryFile() spooled.write(self.zipfile_content) zip_file_storage = FileStorage(spooled) zip_file_storage.filename = DOWNLOAD_FILE.rpartition(os.path.sep)[-1] return zip_file_storage
def _temporary_file(): return SpooledTemporaryFile(max_size=1024 * 1024 * 10) # 10 MB.
def __init__(self, s3key, max_mem=0): self.key = s3key self.max_mem = max_mem self._stream = SpooledTemporaryFile(max_size=self.max_mem) self.key.get_contents_to_file(self._stream) self._stream.seek(0)
def run_command(self, command, stdin=None, env=None): """ Launch a shell command line. :param command: Command line to launch :type command: str :param stdin: Standard input of command :type stdin: file :param env: Environment variable used in command :type env: dict :return: Standard output of command :rtype: file """ cmd = shlex.split(command) stdout = SpooledTemporaryFile(max_size=settings.TMP_FILE_MAX_SIZE, dir=settings.TMP_DIR) stderr = SpooledTemporaryFile(max_size=settings.TMP_FILE_MAX_SIZE, dir=settings.TMP_DIR) full_env = os.environ.copy() if self.use_parent_env else {} full_env.update(self.env) full_env.update(env or {}) try: if isinstance(stdin, File): process = Popen( cmd, stdin=stdin.open("rb"), stdout=stdout, stderr=stderr, env=full_env ) else: process = Popen(cmd, stdin=stdin, stdout=stdout, stderr=stderr, env=full_env) process.wait() if process.poll(): stderr.seek(0) raise exceptions.CommandConnectorError( "Error running: {}\n{}".format(command, stderr.read().decode('utf-8'))) stdout.seek(0) stderr.seek(0) return stdout, stderr except OSError as err: raise exceptions.CommandConnectorError( "Error running: {}\n{}".format(command, str(err)))
def _create_buffer(self): return SpooledTemporaryFile(BUFF_SIZE)
class RecvWindow(object): """ Receiver windows receive data on a socket to fill their buffer. A window instance has the following attributes: name Name of the window. The name of all windows is shown by '> win show' in command line interface. It's usually the name of the plugin appended by the arguments passed for execution (e.g. ps.plg -A -t curesec.com) status indicates the state of the plugin (e.g. RUNNING or FINISHED) req_t request tuple that triggered plugin execution. It's used to verify incoming packets plg_id id of the plugin rid request id of the packet triggering execution wid the id of the window. Derived from a class variable which is increased after window creation. The window id is not fixed. So every new client session, the windows might have a new id. session the actual client session is_used flag, that indicates, whether the windows is used by user (via '> win show <id>') """ #: command to close window CMD_CLOSE = "close" #: window id, increased for every new window ID = 1 def __init__(self, name, req_t, session): logger.debug("creating window %s", name) self.name = name self.status = RUNNING self.req_t = req_t self.sid = req_t[1] self.rid = req_t[2] self.plg_id = req_t[4] self.wid = RecvWindow.ID self.session = session self.is_used = False self.closed = False self._buffer = SpooledTemporaryFile(max_size="8096", mode="wr+b") RecvWindow.ID += 1 def close(self): self.closed = True print("closing window %d" % self.wid) def set_state(self, input_code): state = None try: state = STATE[input_code + 1] except (IndexError, TypeError) as e: logger.warning('Could not get window state for code %d: %s', input_code, e) logger.exception(e) if not state: state = "Unknown" if self.is_used and state in (FINISHED, ERROR, KILLED): self.is_used = False code = "(%s)" % str(input_code) self.status = "{state} {code}".format(state=state, code=code) logger.debug("new window(id:%s) state: %s", self.wid, self.status) def _write_to_output(self, data): # if no more data is coming, set window state to # finished and break if we have a non-interactive # window (otherwise we need to wait for the user # to enter further input). #if not data: # self.status = FINISHED # TODO: that seems a bit heuristic to me ... # ok, we have data, so verify 'em and print it. this is # done by unpack try: actual_output = self._unpack(data) if actual_output: sys.stdout.write(actual_output) sys.stdout.flush() self._buffer.write(actual_output) self._buffer.flush() except errors.ClientError as e: if e.request: logger.warning('Request: %s', conn.ccdlib.pkt2str(e.request)) if e.response: logger.warning('Response: %s', conn.ccdlib.pkt2str(e.response)) logger.debug("received invalid packet:'%s'", e) logger.exception(e) except Exception as e: logger.debug("received invalid packet:'%s'", e) logger.exception(e) def _unpack(self, resp_t): #if not resp_t[0] == cnnm.ccdlib.OP_SUCCESS: # raise Exception("received packet that indicates a failure(%s)!", # hex(resp_t[0])) ## sid #if not resp_t[1] == self.req_t[1]: # raise Exception("received packet with invalid session id(%s)!", # resp_t[1]) ## rid #if not resp_t[2] == self.req_t[2]: # raise Exception("received packet with invalid rid(%s)!", # resp_t[2]) ## plg_id #if not resp_t[4] == self.req_t[4]: # raise Exception("received packet with invalid plugin id(%s)!", # resp_t[4]) # method indicates a plugin's state change. if resp_t[6] == conn.ccdlib.MTH_TERMINATE: state = 2 try: state = resp_t[-1]["code"] except (KeyError, IndexError): raise errors.InvalidServerResponse( message="No state in terminate response payload", request=self.req_t, response=resp_t) self.set_state(state) return elif resp_t[6] == conn.ccdlib.MTH_INPUT: return "" # plugin gives outpout elif not resp_t[6] == conn.ccdlib.MTH_OUTPUT: try: MTH_str = conn.ccdlib.rev_MTH_map[resp_t[6]] except: MTH_str = repr(resp_t[6]) raise errors.InvalidServerResponse( message= "Invalid method %s, expected MTH_OUTPUT, _TERMINATE or _INPUT" % MTH_str, request=self.req_t, response=resp_t) return resp_t[-1] def _pack(self, data): req_t = list(self.req_t) req_t[ 0] = conn.ccdlib.OP_PLUGIN # indicate operation concerning plugin req_t[6] = conn.ccdlib.MTH_INPUT # indicate plugin input is coming req_t[-1] = data # payload # if no input to commit, raise EmptyException if not req_t[-1]: raise conn.ccdlib.EmptyException return tuple(req_t) def use(self): # to get the plugin's output, we need to register rid = self.req_t[2] plg_id = self.req_t[4] pld = comm.sendpacket(self.session, op=conn.ccdlib.OP_REGISTER, plg=plg_id, pld=dict(rid=rid))[-1] interactive = pld["interactive"] logger.debug("successfully registered to plugin. have fun!") if interactive: print("Enter '%s' to leave interactive mode." % RecvWindow.CMD_CLOSE) sock = self.session.sock # write content that is already buffered for line in self._buffer: sys.stdout.write(line) # start window interaction self.is_used = True while self.is_used: try: r, _, _ = select.select([sys.stdin, sock], [], []) except KeyboardInterrupt: print("catched keyboard interrupt. closing window.") self.is_used = False break # if there is something to print to window if sock in r: # read data, that should be written to stdout. # the data is sent by the ccd resp_t = None try: resp_t = conn.ccdlib.recv(sock) #resp_t = comm.wait_for_response(self.session, self.req_t) except Exception as e: logger.error("Exception while receiving data: %s", e) logger.exception(e) self.is_used = False break if resp_t: self._write_to_output(resp_t) # there is something to read form window console if sys.stdin in r: if not interactive: # any keystroke sends active plugin to background self.is_used = False break # read the user's input to be sent to the ccd plugin #data = sys.stdin.readline() data = '' try: data = os.read(sys.stdin.fileno(), 1024) except Exception, e: logger.error("Error reading from stdin: '%s'!", e) logger.exception(e) continue logger.debug('all read on stdin \'%s\'', data) # catch command to close the interactive window if data.rstrip() == RecvWindow.CMD_CLOSE: self.is_used = False break # the actual write of the user's input actual_packet = self._pack(data) try: conn.ccdlib.send(sock, actual_packet) #comm.wait_for_response(self.session, actual_packet) except Exception as e: logger.error("Failed to send user input to plugin!:'%s'", e) logger.exception(e) try: comm.sendpacket(self.session, op=conn.ccdlib.OP_UNREGISTER, plg=plg_id, pld=dict(rid=rid)) except KeyboardInterrupt: print("catched keyboard interrupt. closing window.") except Exception as e: logger.error("Failed to unregister properly:'%s'!", e) logger.exception(e)
def __init__(self, threshold=10 * 1024 * 1024, **kw): # STF uses >, the old ATF used >= for the max_size check SpooledTemporaryFile.__init__(self, max_size=threshold - 1, **kw)
def compress(self, data: bytes): if self._buffer is None: self._buffer = SpooledTemporaryFile(mode='wb') self._buffer.write(data)
def __init__(self, sep=b','): self._buffer = SpooledTemporaryFile(mode='wb') self._result = SpooledTemporaryFile(mode='wb') self._sep = sep self.eof = False
def get(url, ignoreVersion=False): '''Given a Send url, download and return the encrypted data and metadata''' prefix, urlid, key = splitkeyurl(url) if checkServerVersion(prefix, ignoreVersion) == False: raise Exception( 'Potentially incompatible server version, use --ignore-version to disable version checks' ) data = SpooledTemporaryFile(max_size=SPOOL_SIZE, mode='w+b') r = requests.get(prefix + 'api/download/' + urlid, stream=True) r.raise_for_status() content_length = int(r.headers['Content-length']) meta = json.loads(r.headers['X-File-Metadata']) filename = unquote_plus(meta['filename']) iv = meta['id'] pbar = progbar(content_length) for chunk in r.iter_content(chunk_size=CHUNK_SIZE): data.write(chunk) pbar.update(len(chunk)) pbar.close() # The last 16 bytes / 128 bits of data is the GCM tag # https://www.w3.org/TR/WebCryptoAPI/#aes-gcm-operations :- # 7. Let ciphertext be equal to C | T, where '|' denotes concatenation. data.seek(-16, 2) tag = data.read() # now truncate the file to only contain encrypted data data.seek(-16, 2) data.truncate() data.seek(0) return data, filename, key, iv, tag
def reconstitute_vector(bytesblob): f = SpooledTemporaryFile(max_size=1000000000) f.write(bytesblob) f.seek(0) return scipyio.mmread(f)
def fetch_media(self, url, partial_fetch=False): """Retrieves a given media object from a remote (HTTP) location and returns the content-type and a file-like object containing the media content. The file-like object is a temporary file that - depending on the size - lives in memory or on disk. Once the file is closed, the contents are removed from storage. :param url: the URL of the media asset. :type url: str. :param partial_fetch: determines if the the complete file should be fetched, or if only the first 2 MB should be retrieved. This feature is used to prevent complete retrieval of large a/v material. :type partial_fetch: bool. :returns: a tuple with the ``content-type``, ``content-lenght`` and a file-like object containing the media content. The value of ``content-length`` will be ``None`` in case a partial fetch is requested and ``content-length`` is not returned by the remote server. """ http_resp = self.http_session.get(url, stream=True, timeout=(60, 120)) http_resp.raise_for_status() if not os.path.exists(TEMP_DIR_PATH): log.debug('Creating temp directory %s' % TEMP_DIR_PATH) os.makedirs(TEMP_DIR_PATH) # Create a temporary file to store the media item, write the file # to disk if it is larger than 1 MB. media_file = SpooledTemporaryFile(max_size=1024 * 1024, prefix='oad_m_', suffix='.tmp', dir=TEMP_DIR_PATH) # When a partial fetch is requested, request up to two MB partial_target_size = 1024 * 1024 * 2 content_length = http_resp.headers.get('content-length') if content_length and int(content_length) < partial_target_size: partial_target_size = int(content_length) retrieved_bytes = 0 for chunk in http_resp.iter_content(chunk_size=512 * 1024): if chunk: # filter out keep-alive chunks media_file.write(chunk) retrieved_bytes += len(chunk) if partial_fetch and retrieved_bytes >= partial_target_size: break media_file.flush() log.debug('Fetched media item %s [%s/%s]' % (url, retrieved_bytes, content_length)) # If the server doens't provide a content-length and this isn't # a partial fetch, determine the size by looking at the retrieved # content if not content_length and not partial_fetch: media_file.seek(0, 2) content_length = media_file.tell() return (http_resp.headers.get('content-type'), content_length, media_file)
def __enter__(self): self.tempfile = SpooledTemporaryFile() self.tempfile.write(self.inputstr) self.tempfile.seek(0) return self.tempfile
class VersionedFile(io.BufferedIOBase): def __init__(self, manager, filename, mode=Perm.read, requestor=Owner.ALL, meta=None, rev=None, file_info=None, **kwargs): io.BufferedIOBase.__init__(self) self.path = self.name = filename # manager.check_perm(self.path, owner=requestor, perm=mode) self.created = self.modified = None self.data = None self.meta = meta or {} self.mode = mode self._seekable = True self.length = 0 self.bs = 8192 self._cipher = None self.manager = manager self._file_info = file_info or manager.get_metadata_and_check_perm( filename, rev, mode=mode, owner=requestor) # self._file_info = manager.get_file_metadata(filename, rev, mode=mode) if self._file_info: self.update(self._file_info) if mode == Perm.read and not self._file_info: raise FileNotFoundError(self.path) elif mode == Perm.write: self.owner = requestor if kwargs: self.update(kwargs) self._pos = 0 if mode == Perm.read: if self.data: self._curr_chunk = self.data self._curr_chunk_num = 0 else: self._curr_chunk_num = None self._curr_chunk = None else: self._buf = SpooledTemporaryFile( max_size=getattr(self, 'buffer_threshold', 52428800)) self.hash = None @property def is_dir(self): return self.content_type == u'application/x-directory' def do_hash(self, algo='sha256'): self.hash = algo def __enter__(self): return self def __exit__(self, exc_type, exc, tb): if exc: if self.readable(): self.close() else: self._buf.close() self.mode = None import six six.reraise(exc_type, exc, tb) else: self.close() def close(self): if self.closed: return if self.writable(): self._buf.seek(0, 2) length = self.length = self._buf.tell() self._buf.seek(0) hist_data = { u'meta': self.meta, u'owner': getattr(self, 'owner', None), u'length': length, u'hash': self.hash, u'created': self.created, u'modified': self.modified, u'file_info': self._file_info, } content_type = getattr(self, 'content_type', None) if not content_type: content_type = mimetypes.guess_type(self.path)[0] hist_data[u'content_type'] = content_type if getattr(self, 'force_rev', None) is not None: hist_data[u'rev'] = rev = self.force_rev hist_data[u'modified'] = self.created self.update( self.manager.save_file_data(self.path, hist_data, self._buf, cipher=self._cipher)) self._buf.close() self._buf = None self.mode = None io.BufferedIOBase.close(self) # def __del__(self): # self.close() def readable(self): return self.mode == Perm.read def writable(self): return self.mode == Perm.write def seekable(self): return self._seekable def tell(self): if self.readable(): return self._pos else: return self._buf.tell() def seek(self, pos, whence=0): if self.mode == Perm.read: curpos = self._pos if whence == 0: abspos = pos elif whence == 1: abspos = curpos + pos elif whence == 2: abspos = self.length + pos self._pos = abspos return self._pos elif self.mode == Perm.write and self.seekable(): return self._buf.seek(pos, whence) def read(self, size=-1): if self.mode != Perm.read: return elif self._pos == self.length: return b'' buf = bytearray() if self._pos == 0 and size == -1: if self.data: self._pos = self.length return self.data else: # optimization for reading the whole file i = 0 for chunk in self.manager.get_file_chunks(self._file_info, cipher=self._cipher): i += 1 buf.extend(chunk) self._pos = len(buf) return bytes(buf) length = size if size > 0 else self.length where, pos = divmod(self._pos, self.bs) if self._curr_chunk_num != where: self._curr_chunk = self.manager.get_file_chunk(self._file_info, where, cipher=self._cipher) self._curr_chunk_num = where buf += self._curr_chunk[pos:] while len(buf) < length: where += 1 self._curr_chunk = self.manager.get_file_chunk(self._file_info, where, cipher=self._cipher) if self._curr_chunk is None: self._curr_chunk_num = None break buf.extend(self._curr_chunk) self._curr_chunk_num = where read = buf[:length] self._pos += len(read) return bytes(read) def readall(self): return self.read() def write(self, data): if not data: return if not self.writable(): raise FileError() if isinstance(data, six.text_type): data = data.encode('utf8') wrote = len(data) self._buf.write(data) return wrote def update(self, kwargs): if kwargs: for k, v in kwargs.items(): if k == 'modified' and self.mode == 'w': continue if v is not None: setattr(self, k, v) def set_encryption(self, password='', save_password=False): """ Set the encryption password, optionally saving the password in the metadata """ try: from nacl.secret import SecretBox except ImportError: SecretBox = None if SecretBox: password = hashlib.sha256(password.encode('utf8')).digest() else: password = hashlib.sha512(password.encode('utf8')).digest()[:56] if self.writable(): assert self._cipher is None if SecretBox: method = u'nacl' self.meta[u'_encryption'] = {u'method': method} else: method = u'cfb' self.meta[u'_encryption'] = { u'method': method, u'iv': os.urandom(8), } if save_password: self.meta[u'_encryption'][u'key'] = password else: assert u'_encryption' in self.meta method = self.meta[u'_encryption'][u'method'] password = self.meta[u'_encryption'].get(u'key', None) or password if method == u'nacl': c = SecretBox(password) self._cipher = {'encrypt': c.encrypt, 'decrypt': c.decrypt} else: import blowfish c = blowfish.Cipher(password) iv = self.meta[u'_encryption'][u'iv'] self._cipher = { 'encrypt': lambda chunk: b''.join(c.encrypt_cfb(chunk, iv)), 'decrypt': lambda chunk: b''.join(c.decrypt_cfb(chunk, iv)), } if self.data: self._curr_chunk = self._cipher['decrypt'](self.data)
def do_execute(self, code, silent, store_history=True, user_expressions=None, allow_stdin=False): """Execute user code.""" if len(code.strip()) == 0: return {'status': 'ok', 'execution_count': self.execution_count, 'payload': [], 'user_expressions': {}} env = builtins.__xonsh_env__ shell = builtins.__xonsh_shell__ hist = builtins.__xonsh_history__ enc = env.get('XONSH_ENCODING') out = SpooledTemporaryFile(max_size=MAX_SIZE, mode='w+t', encoding=enc, newline='\n') err = SpooledTemporaryFile(max_size=MAX_SIZE, mode='w+t', encoding=enc, newline='\n') try: with redirect_stdout(out), redirect_stderr(err), \ swap(builtins, '__xonsh_stdout_uncaptured__', out), \ swap(builtins, '__xonsh_stderr_uncaptured__', err), \ env.swap({'XONSH_STORE_STDOUT': False}): shell.default(code) interrupted = False except KeyboardInterrupt: interrupted = True if not silent: # stdout response if out.tell() > 0: out.seek(0) self._respond_in_chunks('stdout', out.read()) if err.tell() > 0: err.seek(0) self._respond_in_chunks('stderr', err.read()) if hasattr(builtins, '_') and builtins._ is not None: # rely on sys.displayhook functionality self._respond_in_chunks('stdout', pformat(builtins._)) builtins._ = None if hist is not None and len(hist) > 0 and out.tell() == 0 and err.tell() == 0: self._respond_in_chunks('stdout', hist.outs[-1]) out.close() err.close() if interrupted: return {'status': 'abort', 'execution_count': self.execution_count} rtn = 0 if (hist is None or len(hist) == 0) else hist.rtns[-1] if 0 < rtn: message = {'status': 'error', 'execution_count': self.execution_count, 'ename': '', 'evalue': str(rtn), 'traceback': []} else: message = {'status': 'ok', 'execution_count': self.execution_count, 'payload': [], 'user_expressions': {}} return message
def create_zipfile_storage(self): spooled = SpooledTemporaryFile() spooled.write(self.zipfile_file.read()) zip_file_storage = FileStorage(spooled) zip_file_storage.filename = self.ZIP_NAME return zip_file_storage
def runTest(test, testnum): # test is a tuple of ( filename , usedb<bool> ) # filename should be a js file to run # usedb is true if the test expects a mongod to be running (path, usedb) = test (ignore, ext) = os.path.splitext(path) # the dbtests know how to format themselves nicely, we'll detect if we're running them and if # so, we won't mess with the output is_test_binary = False if skipTest(path): if quiet: sys.stdout.write("skip %d %s\n" % (testnum, os.path.basename(path))) sys.stdout.flush() else: print "skipping " + path return if file_of_commands_mode: # smoke.py was invoked like "--mode files --from-file foo", # so don't try to interpret the test path too much if os.sys.platform == "win32": argv = [path] else: argv = shlex.split(path) path = argv[0] # if the command is a python script, use the script name if os.path.basename(path) in ('python', 'python.exe'): path = argv[1] elif ext == ".js": argv = [ shell_executable, "--port", mongod_port, '--authenticationMechanism', authMechanism ] if not usedb: argv += ["--nodb"] if small_oplog or small_oplog_rs: argv += ["--eval", 'testingReplication = true;'] if use_ssl: argv += [ "--ssl", "--sslPEMKeyFile", "jstests/libs/client.pem", "--sslCAFile", "jstests/libs/ca.pem" ] argv += [path] elif ext in ["", ".exe"]: # Blech. if os.path.basename(path) in [ "test", "test.exe", "perftest", "perftest.exe" ]: argv = [path] if os.path.basename(path) in ["test", "test.exe"]: is_test_binary = True # more blech elif os.path.basename(path) in ['mongos', 'mongos.exe']: argv = [path, "--test"] else: argv = [ test_path and os.path.abspath(os.path.join(test_path, path)) or path, "--port", mongod_port ] else: raise Bug("fell off in extension case: %s" % path) if keyFile: f = open(keyFile, 'r') keyFileData = re.sub(r'\s', '', f.read()) # Remove all whitespace f.close() os.chmod(keyFile, stat.S_IRUSR | stat.S_IWUSR) else: keyFileData = None mongo_test_filename = os.path.basename(path) if 'sharedclient' in path: mongo_test_filename += "-sharedclient" # sys.stdout.write() is more atomic than print, so using it prevents # lines being interrupted by, e.g., child processes if quiet and not is_test_binary: vlog = tests_log qlog = sys.stdout tlog = sys.stderr else: vlog = sys.stdout qlog = None tlog = None vlog.write(" *******************************************\n") vlog.write(" Test : %s ...\n" % mongo_test_filename) vlog.flush() # FIXME: we don't handle the case where the subprocess # hangs... that's bad. if (argv[0].endswith('mongo') or argv[0].endswith('mongo.exe')) and not '--eval' in argv: evalString = 'TestData = new Object();' + \ 'TestData.testPath = "' + path + '";' + \ 'TestData.testFile = "' + os.path.basename( path ) + '";' + \ 'TestData.testName = "' + re.sub( ".js$", "", os.path.basename( path ) ) + '";' + \ 'TestData.noJournal = ' + ternary( no_journal ) + ";" + \ 'TestData.noJournalPrealloc = ' + ternary( no_preallocj ) + ";" + \ 'TestData.auth = ' + ternary( auth ) + ";" + \ 'TestData.keyFile = ' + ternary( keyFile , '"' + str(keyFile) + '"' , 'null' ) + ";" + \ 'TestData.keyFileData = ' + ternary( keyFile , '"' + str(keyFileData) + '"' , 'null' ) + ";" if os.sys.platform == "win32": # double quotes in the evalString on windows; this # prevents the backslashes from being removed when # the shell (i.e. bash) evaluates this string. yuck. evalString = evalString.replace('\\', '\\\\') if auth and usedb: evalString += 'jsTest.authenticate(db.getMongo());' argv = argv + ['--eval', evalString] if argv[0].endswith('test') and no_preallocj: argv = argv + ['--nopreallocj'] vlog.write(" Command : %s\n" % ' '.join(argv)) vlog.write(" Date : %s\n" % datetime.now().ctime()) vlog.flush() tempfile = SpooledTemporaryFile(max_size=16 * 1024 * 1024) try: os.environ['MONGO_TEST_FILENAME'] = mongo_test_filename t1 = time.time() r = call( buildlogger(argv), cwd=test_path, # the dbtests know how to format their own output nicely stdout=ternary(is_test_binary, vlog, tempfile)) t2 = time.time() del os.environ['MONGO_TEST_FILENAME'] vlog.write(" %fms\n" % ((t2 - t1) * 1000)) vlog.flush() if not is_test_binary: tempfile.seek(0) for line in tempfile: vlog.write(line) vlog.flush() if quiet: if r == 0: qlog.write('ok %d %s\n' % (testnum, os.path.basename(path))) else: qlog.write('not ok %d %s # exit %d\n' % (testnum, os.path.basename(path), r)) qlog.flush() if r != 0: tempfile.seek(0) for line in tempfile: tlog.write(line) tlog.flush() if r != 0: raise TestExitFailure(path, r) finally: tempfile.close() if r != 0: raise TestExitFailure(path, r) if start_mongod: try: c = Connection(host="127.0.0.1", port=int(mongod_port), ssl=use_ssl) except Exception, e: print "Exception from pymongo: ", e raise TestServerFailure(path)
def __init__(self, file_obj, max_size=50 * 1024 * 1024): self._file_obj = file_obj SpooledTemporaryFile.__init__(self, max_size)
def flush(self): """Flush the data to the server.""" SpooledTemporaryFile.flush(self) self.commit()
def upload_file(self, user, stream, expected_size, filename, force_coll_name=''): """Upload WARC archive. :param User user: user :param stream: file object :param int expected_size: expected WARC archive size :param str filename: WARC archive filename :param str force_coll_name: name of collection to upload into :returns: upload information :rtype: dict """ temp_file = None logger.debug('Upload Begin') logger.debug('Expected Size: ' + str(expected_size)) #is_anon = False size_rem = user.get_size_remaining() logger.debug('User Size Rem: ' + str(size_rem)) if size_rem < expected_size: return {'error': 'out_of_space'} if force_coll_name and not user.has_collection(force_coll_name): #if is_anon: # user.create_collection(force_coll, 'Temporary Collection') #else: #status = 'Collection {0} not found'.format(force_coll_name) return {'error': 'no_such_collection'} temp_file = SpooledTemporaryFile(max_size=BLOCK_SIZE) stream = CacheingLimitReader(stream, expected_size, temp_file) if filename.endswith('.har'): stream, expected_size = self.har2warc(filename, stream) temp_file.close() temp_file = stream infos = self.parse_uploaded(stream, expected_size) total_size = temp_file.tell() if total_size != expected_size: return { 'error': 'incomplete_upload', 'expected': expected_size, 'actual': total_size } upload_id, upload_key = self._init_upload_status(user, total_size, 1, filename=filename) return self.handle_upload(temp_file, upload_id, upload_key, infos, filename, user, force_coll_name, total_size)
def upload_file(self): stream = None temp_file = None logger.debug('Upload Begin') expected_size = int(request.headers['Content-Length']) logger.debug('Expected Size: ' + str(expected_size)) if not expected_size: return {'error_message': 'No File Specified'} curr_user = self.manager.get_curr_user() if not curr_user: #user = self.manager.get_anon_user() #force_coll = 'temp' #is_anon = True return { 'error_message': 'Sorry, uploads only available for logged-in users' } user = curr_user force_coll = request.query.getunicode('force-coll', '') is_anon = False size_rem = self.manager.get_size_remaining(user) logger.debug('User Size Rem: ' + str(size_rem)) if size_rem < expected_size: return { 'error_message': 'Sorry, not enough space to upload this file' } if force_coll and not self.manager.has_collection(user, force_coll): if is_anon: self.manager.create_collection(user, force_coll, 'Temporary Collection') else: status = 'Collection {0} not found'.format(force_coll) return {'error_message': status} temp_file = SpooledTemporaryFile(max_size=BLOCK_SIZE) filename = request.query.getunicode('filename') stream = request.environ['wsgi.input'] stream = CacheingLimitReader(stream, expected_size, temp_file) if filename.endswith('.har'): stream, expected_size = self.har2warc(filename, stream) temp_file.close() temp_file = stream infos = self.parse_uploaded(stream, expected_size) total_size = temp_file.tell() if total_size != expected_size: return { 'error_message': 'size mismatch: expected {0}, got {1}'.format( expected_size, total_size) } upload_id = self._get_upload_id() upload_key = self.upload_key.format(user=user, upid=upload_id) with redis_pipeline(self.manager.redis) as pi: pi.hset(upload_key, 'size', 0) pi.hset(upload_key, 'total_size', total_size * 2) pi.hset(upload_key, 'filename', filename) pi.hset(upload_key, 'total_files', 1) pi.hset(upload_key, 'files', 1) return self.handle_upload(temp_file, upload_id, upload_key, infos, filename, user, force_coll, total_size)
class OcrSource(panoply.DataSource): """ One Click Retail data source. API Docs (require login) - https://api.oneclickretail.com/api-docs-login Implemented endpoints - GET /v5/clients/{client_uuid}/reports/export?format=csv """ def __init__(self, source, options): super(OcrSource, self).__init__(source, options) if not source.get('destination'): source['destination'] = DESTINATION if not source.get('idpattern'): source['idpattern'] = IDPATTERN if not source.get('resources'): raise Exception('No resources selected') self.resource = None self.data = None self.resources = source.get('resources') self.api_key = source.get('apiKey') self.weeks = source.get('weeks', DEFAULT_WEEKS_BACK) self.clientUUID = source.get('clientUUID') self.processed = 0 self.total = len(self.resources) def read(self, batch_size=None): try: if not self.resource: self.resource = self.resources.pop() self.processed += 1 except IndexError: # IndexError will occur when attempting to pop and element # off an empty list. This indicates that we have already # processed all resources. return None # Send progress message progress_msg = 'Fetching data for %s' % self.resource.get('name') self.progress(self.processed, self.total, progress_msg) # If data remains from the last resource, use it. # Otherwise fetch data from the current resource. self.data = self.data or self._fetch_resource() batch = self._extract_batch(self.data, batch_size) # If no data was returned for this batch we might have reached # the end of the source - attempt to read more. if not batch: # This was the last batch for this resource self.resource = None # Attempt to fetch the next resouce return self.read() return batch def _fetch_resource(self): """ Assemble the api call, execute it and parse the csv response as a list of dicts. Returns a dict generator """ qs = self._build_qs() # Build the query string url = self._build_url(qs) # Build the full url fp = self._api_call(url) # Fetch the data as a file pointer # Parse the list of dicts in to a dict generator return csv.DictReader(fp) def _build_qs(self): params = { 'format': 'csv', 'meta': FETCH_META, 'X-API-KEY': self.api_key, 'weeks_back': self.weeks } if self.source.get('filterId'): params['filter_id'] = self.source['filterId'] qs = urllib.urlencode(params) return qs def _build_url(self, qs): base = self.resource.get('value') % self.clientUUID url = '%(base)s/%(endpoint)s?%(qs)s' % { 'base': BASE_URL, 'endpoint': base, 'qs': qs } return url def _api_call(self, url): """ Returns a SpooledTemporaryFile - this is a file that is initially stored in memory but once its size exceedes max_size it will start writing to disk. It is used because there is no way of knowing how large of a file the api will return. We are expecting a csv file. """ self.log('Request URL', url) response = urllib2.urlopen(url) # the response MUST be a csv file content_type = response.info().get('content-type') if 'csv' not in content_type: raise Exception('ERROR - Non CSV response.') # Decode the returned data and replace any characters that generate # encoding errors with the default unicode replacement character. data = response.read().decode('utf-8', 'replace') self.tmp_file = SpooledTemporaryFile(max_size=MAX_SIZE) # Force writing the data encoded as utf-8. If we don't do this, # python will attempt to write the data as 'ascii' which does not # support special characters self.tmp_file.write(data.encode('utf-8')) # 'rewind' the file pointer in order to # read it back durring `_extract_batch()` self.tmp_file.seek(0) return self.tmp_file def _extract_batch(self, data, batch_size): """ Iterates over BATCH_SIZE of data returning a list or results """ batch_size = batch_size or BATCH_SIZE batch = [] try: for i in range(batch_size): batch.append(data.next()) except StopIteration: pass return batch
def push(): if not request.is_json: return status(400) data = request.json if not ('ref' in data and data['ref'].startswith('refs/tags/')): return status(200) base = data['compare_url'] + 'api/v1' headers = { 'Accept': 'application/json', 'Authorization': 'token ' + secret.token.gitea, } repo = data['repository'] # get tag information uri = uritemplate.expand(base + '/repos/{owner}/{repo}/git/tags/{sha}', owner=repo['owner']['username'], repo=repo['name'], sha=data['after']) r = requests.get(uri, headers=headers) if r.status_code != 200: return status(500, message=f'error fetching "{uri}"') tag = r.json() # create release uri = uritemplate.expand(base + '/repos/{owner}/{repo}/releases', owner=repo['owner']['username'], repo=repo['name']) payload = { 'body': tag['message'], 'draft': False, 'name': kebab2normal(repo['name']) + ' ' + tag['tag'], 'prerelease': False, 'tag_name': tag['tag'], 'target_commitish': repo['default_branch'], } r = requests.post(uri, headers=headers, json=payload) if r.status_code != 201: return status(500, message=f'error fetching "{uri}"') release = r.json() # create release zip with SpooledTemporaryFile() as f: with TemporaryDirectory() as d: p = run(['git', 'clone', repo['clone_url'], d]) if p.returncode != 0: return status(500, message='error cloning repository') cmd = ['sh', os.path.join(d, '.bin', 'release.sh'), tag['tag']] if not os.path.exists(cmd[1]): cmd = ['git', 'archive', '--format=zip', tag['tag']] p = run(cmd, stdout=PIPE, cwd=d) if p.returncode != 0: return status(500, message='error creating archive') b = p.stdout # upload release zip uri = uritemplate.expand(base + '/repos/{owner}/{repo}/releases/{id}/assets?name={name}', owner=repo['owner']['username'], repo=repo['name'], id=release['id'], name=repo['name'] + '.zip') payload = { 'attachment': (repo['name'] + '.zip', b, 'application/zip'), } r = requests.post(uri, headers=headers, files=payload) if r.status_code != 201: return status(500, message='error uploading archive') return status(200, message='release created')
class IncrementalWriter: """A streaming file writer for point clouds. Using the IncrementalWriter with spooled temporary files, which are only flushed to disk if they go above the given size, allows for streaming points to disk even when the header is unknown in advance. This allows some nice tricks, including splitting a point cloud into multiple files in a single pass, without memory issues. """ # pylint:disable=too-few-public-methods def __init__(self, filename: str, header: PlyHeader, utm: UTM_Coord = None, buffer=2**22) -> None: """ Args: filename: final place to save the file on disk. source_fname: source file for the pointcloud; used to detect file format for metadata etc. buffer (int): The number of bytes to hold in RAM before flushing the temporary file to disk. Default 1MB, which holds ~8300 points - enough for most objects but still practical to hold thousands in memory. Set a smaller buffer for large forests. """ self.filename = filename self.temp_storage = SpooledTemporaryFile(max_size=buffer, mode='w+b') self.count = 0 self.utm = utm logging.debug( 'At intitialisation, instance of IncrementalWriter.utm = {}'. format(self.utm)) self.header = header # Always write in little-endian mode; only store type information self.binary = struct.Struct('<' + header.form_str[1:]) def __call__(self, point) -> None: """Add a single point to this pointcloud, saving in binary format. Args: point (namedtuple): vertex attributes for the point, eg xyzrgba. """ self.temp_storage.write(self.binary.pack(*point)) self.count += 1 def __del__(self): """Flush data to disk and clean up.""" logging.debug('Flushing data to disk in IncrementalWriter.__del__()') to_ply_types = {v: k for k, v in PLY_TYPES.items()} properties = [ 'property {t} {n}'.format(t=t, n=n) for t, n in zip(( to_ply_types[p] for p in self.header.form_str[1:]), self.header.names) ] head = [ 'ply', 'format binary_little_endian 1.0', 'element vertex {}'.format(self.count), '\n'.join(properties), 'end_header' ] if self.utm is not None: head.insert( -1, 'comment UTM x y zone north ' + '{0.x} {0.y} {0.zone} {0.north}'.format(self.utm)) if not os.path.isdir(os.path.dirname(self.filename)): os.makedirs(os.path.dirname(self.filename)) with open(self.filename, 'wb') as f: f.write(('\n'.join(head) + '\n').encode('ascii')) self.temp_storage.seek(0) chunk = self.temp_storage.read(8192) while chunk: f.write(chunk) chunk = self.temp_storage.read(8192) self.temp_storage.close()
def run_command(self, command, stdin=None, env=None): """ Launch a shell command line. :param command: Command line to launch :type command: str :param stdin: Standard input of command :type stdin: file :param env: Environment variable used in command :type env: dict :return: Standard output of command :rtype: file """ cmd = shlex.split(command) stdout = SpooledTemporaryFile(max_size=settings.TMP_FILE_MAX_SIZE, dir=settings.TMP_DIR) stderr = SpooledTemporaryFile(max_size=settings.TMP_FILE_MAX_SIZE, dir=settings.TMP_DIR) full_env = self.env.copy() full_env.update(env or {}) try: if not getattr(stdin, 'fileno', None): if stdin: process = Popen(cmd, stdin=PIPE, stdout=stdout, stderr=stderr, env=full_env) grep_stdout = process.communicate( input=stdin.file.getvalue())[0] else: process = Popen(cmd, stdin=stdin, stdout=stdout, stderr=stderr, env=full_env) else: process = Popen(cmd, stdin=stdin, stdout=stdout, stderr=stderr, env=full_env) process.wait() if process.poll(): stderr.seek(0) raise exceptions.CommandConnectorError( "Error running: {}\n{}".format(command, stderr.read())) stdout.seek(0) stderr.seek(0) return stdout, stderr except OSError as err: raise exceptions.CommandConnectorError( "Error running: {}\n{}".format(command, str(err)))