def local_ds_files(ds): """ Gets the list of files corresponding to a published dataset stored on cms_dbs_ph_analysis_02. Args: ds: the path to the published dataset, ending in /USER Returns: A list of the LFN-s of the dataset. """ tf = TemporaryFile() stdout = sys.stdout stdout.flush() sys.stdout = tf print "Query" ret = call_das_cli('--query=file dataset=%s instance=cms_dbs_ph_analysis_02' % ds, '--limit=0') print ret tf.flush() tf.seek(0) sys.stdout = stdout fl = [] for li in tf.readlines(): if "/store/" in li: fl.append(li.strip()) tf.close() return fl
def _create_temp_file(edid_binary): edid_file = TemporaryFile() edid_file.write(edid_binary) edid_file.flush() edid_file.seek(0) return edid_file
def to_xml(self, f=None): """Get this domain as an XML DOM Document :param f: Optional File to dump directly to :type f: File or Stream :return: File object where the XML has been dumped to :rtype: file """ if not f: from tempfile import TemporaryFile f = TemporaryFile() print >> f, '<?xml version="1.0" encoding="UTF-8"?>' print >> f, '<Domain id="%s">' % self.name for item in self: print >> f, '\t<Item id="%s">' % item.name for k in item: print >> f, '\t\t<attribute id="%s">' % k values = item[k] if not isinstance(values, list): values = [values] for value in values: print >> f, '\t\t\t<value><![CDATA[', if isinstance(value, unicode): value = value.encode('utf-8', 'replace') else: value = unicode(value, errors='replace').encode( 'utf-8', 'replace') f.write(value) print >> f, ']]></value>' print >> f, '\t\t</attribute>' print >> f, '\t</Item>' print >> f, '</Domain>' f.flush() f.seek(0) return f
def latest_dataset(): """Retrive the latest CLDR dataset and provide a ZipFile interface, handling cleanup automatically. This streams the dataset into a temporary file before wrapping it in the ZipFile interface. """ spool = TemporaryFile(prefix='cldr', suffix='.zip') version, latest = get_latest_version_url() with Session() as http: response = http.get(latest, stream=True) for chunk in response.iter_content(chunk_size=4096): if chunk: spool.write(chunk) # Write out any uncommitted data, then return to the beginning. spool.flush() spool.seek(0) zipfile = ZipFile(spool, 'r') zipfile.version = version # Expose the version number through to consumers of the ZipFile. yield zipfile zipfile.close() spool.close()
def create_temp_file(self, edid_binary): edid_file = TemporaryFile() edid_file.write(edid_binary) edid_file.flush() edid_file.seek(0) return edid_file
def getUpdateElements(self, valueMap): ''' @param valueMap: ''' elements = "" for name in valueMap.keys(): fullname = name if isinstance(name, types.StringType): fullname = (self.defaultNameSpace, name) if not fullname[0]: tag = fullname[1] else: tag = self.shortcuts[fullname[0]] + ':' + fullname[1] value = valueMap[name] if value: if isinstance(value, qp_xml._element): tmpFile = TemporaryFile('w+') value = qp_xml.dump(tmpFile, value) tmpFile.flush() tmpFile.seek(0) tmpFile.readline() value = tmpFile.read() else: value = "<![CDATA[%s]]>" % value else: value = "" elements += "<%s>%s</%s>" % (tag, value, tag) return elements
def sandbox_helper(sandbox: Sandbox, command, privileged=False): stdout, stderr = TemporaryFile("wb+"), TemporaryFile("wb+") sandbox.execute(command=command, stdin_fd=None, stdout_fd=stdout, stderr_fd=stderr, privileged=privileged) stdout.flush() stdout.seek(0) stdout_text = stdout.read().decode().strip() stdout.close() stderr.flush() stderr.seek(0) stderr_text = stderr.read().decode().strip() stderr.close() # If running java or javac or jar the JVM prints an annoying message: # "Picked up JAVA_TOOL_OPTIONS: <actual options set by sandbox environment> # Remove it from the stderr if it is there if any(java in command for java in ["java", "javac", "jar"]): stdout_text = "\n".join([ line for line in stdout_text.splitlines() if not line.startswith("Picked up JAVA_TOOL_OPTIONS") ]) stderr_text = "\n".join([ line for line in stderr_text.splitlines() if not line.startswith("Picked up JAVA_TOOL_OPTIONS") ]) return stdout_text, stderr_text
def backup(self): if self.dry_run: return if not os.path.exists(self.config['tar']['directory']) \ or not os.path.isdir(self.config['tar']['directory']): raise BackupError('{0} is not a directory!'.format(self.config['tar']['directory'])) out_name = "{0}.tar".format( self.config['tar']['directory'].lstrip('/').replace('/', '_')) outfile = os.path.join(self.target_directory, out_name) args = ['tar', 'c', self.config['tar']['directory']] errlog = TemporaryFile() stream = self._open_stream(outfile, 'w') LOG.info("Executing: %s", list2cmdline(args)) pid = Popen( args, stdout=stream.fileno(), stderr=errlog.fileno(), close_fds=True) status = pid.wait() try: errlog.flush() errlog.seek(0) for line in errlog: LOG.error("%s[%d]: %s", list2cmdline(args), pid.pid, line.rstrip()) finally: errlog.close() if status != 0: raise BackupError('tar failed (status={0})'.format(status))
def backup(self, config, **flags): # check if this is a config file config_f = open(config, "rb") mode = ("w:%s" % flags['compress']) if flags.get('compress') else "w" buff = TemporaryFile() with config_f: cfg = ConfigParser() cfg.readfp(config_f) project = cfg.get("general", "project") databases = cfg.get("general", "databases").split() tarname = "%s.tar" % project tar = tarfile.open(fileobj=buff, mode=mode, name=tarname) to_close = self.__add_database_to_tar(tar, cfg, databases) tar.close() for f in to_close: f.close() buff.seek(0) name = project + ".tar" if flags.get('compress'): name = project + ".t%s" % flags['compress'] if flags.get("upload"): buff.flush() timestamp = datetime.now().isoformat() self.client.backup(project, name, buff, timestamp) buff.close()
def _push_pictures(self): """ Push the new picture into the NAS for GP. """ # Retrieve configuration smb_user = config.get('smb_user') smb_pass = config.get('smb_pwd') smb_ip = config.get('smb_ip') smb_port = int(config.get('smb_port', 0)) if not (smb_user and smb_pass and smb_ip and smb_port): raise Warning('Config Error', 'Missing Samba configuration in conf file.') child = self.child_id date_pic = self.date.replace('-', '') gp_pic_path = "{0}{1}/".format(config.get('gp_pictures'), child.unique_id[:2]) file_name = "{0}_{1}.jpg".format(child.unique_id, date_pic) picture_file = TemporaryFile() picture_file.write(base64.b64decode(self.fullshot)) picture_file.flush() picture_file.seek(0) # Upload file to shared folder smb_conn = SMBConnection(smb_user, smb_pass, 'openerp', 'nas') if smb_conn.connect(smb_ip, smb_port): try: smb_conn.storeFile('GP', gp_pic_path + file_name, picture_file) except OperationFailure: # Directory may not exist smb_conn.createDirectory('GP', gp_pic_path) smb_conn.storeFile('GP', gp_pic_path + file_name, picture_file)
def run_process(cmd, timeout=10): """ run process with timeout """ if type(cmd) == bytes: cmd = cmd.decode('utf-8') if type(cmd) == str: cmd = cmd.split() if not timeout: subprocess.Popen(cmd) return None, None, None try: out = TemporaryFile() err = TemporaryFile() prc = subprocess.Popen(cmd, stdout=out, stderr=err) except: LOG.exception('error in run_process %s' % cmd) return -1, None, None starttime = time.time() while 1: if time.time() - starttime > timeout: LOG.error('run command %s timeout' % ' '.join(cmd)) try: kill_prc(prc) except: pass return -1, None, None if not alive(prc): out.flush() err.flush() out.seek(0) err.seek(0) return prc.poll(), out.read().decode('utf-8'), err.read().decode( 'utf-8') time.sleep(0.1)
def post(self): # parse incoming POST data reqparse = RequestParser() reqparse.add_argument('blob', type=str, location='json') data = reqparse.parse_args() if not data.blob: return self.argument_required('blob') # encode content as bytestring tmp = base64.b64decode(data.blob) # calculate sha1 digest digest = hashlib.sha1(tmp).hexdigest() # write into temp file f = TemporaryFile() _ = f.write(tmp) f.flush() _ = f.seek(0) # upload blob created = self.blob_container.put(f, digest=digest) # response json and status code code = created and 201 or 409 response = dict( digest = digest, url = '/image/{0}'.format(digest) ) return response, code
def run_process(cmd, timeout=10): """ run process with timeout """ if type(cmd) == bytes: cmd = cmd.decode('utf-8') if type(cmd) == str: cmd = cmd.split() if not timeout: subprocess.Popen(cmd) return None, None, None try: out = TemporaryFile() err = TemporaryFile() prc = subprocess.Popen(cmd, stdout=out, stderr=err) except: LOG.exception('error in run_process %s' % cmd) return -1, None, None starttime = time.time() while 1: if time.time() - starttime > timeout: LOG.error('run command %s timeout' % ' '.join(cmd)) try: kill_prc(prc) except: pass return -1, None, None if not alive(prc): out.flush() err.flush() out.seek(0) err.seek(0) return prc.poll(), out.read().decode('utf-8'), err.read().decode('utf-8') time.sleep(0.1)
def run_reduce(self): self.stopped_received = 0 self.merged_files = [] merged_iterator = None while True: # Iterate and merge files until all jobs are processed get_next = self.get_next_file() files = get_next #itertools.islice(get_next, self.reduce_max_files) all_files = [file for file in files] iterables = [self.iter_on_file(file) for file in all_files] merged_iterator = heapq.merge(*iterables) if self.stopped_received < self.numprocs: if self.debug: debug_print("Performing intermediate merge on %u files" % len(iterables)) f = TemporaryFile() self.merged_files.append(f) for m in merged_iterator: pickle.dump(m, f, pickle.HIGHEST_PROTOCOL) f.seek(0) f.flush() else: break if len(self.merged_files) > 0: if self.debug: debug_print("Final merge") # Final merge if required merged_iterator = heapq.merge( *([self.iter_on_file(stream) for stream in self.merged_files] + [merged_iterator])) if self.debug: debug_print("Reduce loop") result = self.reduce_loop(merged_iterator) return result
def sort_diskbased(stream, field, nsize=100000): buf = [] files = [] count = 0 t = None def iter_on_file(f): try: while True: (key, v) = cPickle.load(f) yield (key, t._make(v)) except EOFError: f.close() for elt in stream: if isinstance(elt, StreamHeader): t = elt.t yield elt elif isinstance(elt, StreamFooter): buf.sort() iterables = [iter_on_file(f) for f in files] + [itertools.imap(lambda obj: (getattr(obj, field), obj), buf)] for (k, row) in heapq.merge(*iterables): yield row yield elt else: buf.append(elt) count = count + 1 if count % nsize == 0: buf.sort(key=lambda obj: getattr(obj, field)) f = TemporaryFile() for item in buf: cPickle.dump((getattr(item, field), list(item)), f, cPickle.HIGHEST_PROTOCOL) f.flush() files.append(f) del buf[:]
def run_reduce(self): self.stopped_received = 0 self.merged_files = [] merged_iterator = None while True: # Iterate and merge files until all jobs are processed get_next = self.get_next_file() files = get_next # itertools.islice(get_next, self.reduce_max_files) all_files = [file for file in files] iterables = [self.iter_on_file(file) for file in all_files] merged_iterator = heapq.merge(*iterables) if self.stopped_received < self.numprocs: if self.debug: debug_print("Performing intermediate merge on %u files" % len(iterables)) f = TemporaryFile() self.merged_files.append(f) for m in merged_iterator: cPickle.dump(m, f, cPickle.HIGHEST_PROTOCOL) f.seek(0) f.flush() else: break if len(self.merged_files) > 0: if self.debug: debug_print("Final merge") # Final merge if required merged_iterator = heapq.merge( *([self.iter_on_file(stream) for stream in self.merged_files] + [merged_iterator]) ) if self.debug: debug_print("Reduce loop") result = self.reduce_loop(merged_iterator) return result
def build_compose_file(repo: mzbuild.Repository, command: str, config_file: str) -> IO[bytes]: """Substitute known keys with mzbuild-provided values * Replace `mzimage` with fingerprinted image names """ images = [] default = os.getenv(f"MZBUILD_DOCKER_TAG", None) with open(config_file) as f: compose = yaml.safe_load(f) # strip mzconduct top-level key, if it exists compose.pop("mzconduct", None) for config in compose["services"].values(): if "mzbuild" in config: image_name = config["mzbuild"] if image_name not in repo.images: raise errors.BadSpec( f"mzcompose: unknown image {image_name}") image = repo.images[image_name] override_tag = os.getenv(f"MZBUILD_{image.env_var_name()}_TAG", default) if override_tag is not None: config["image"] = image.docker_name(override_tag) print( f"mzcompose: warning: overriding {image_name} image to tag {override_tag}", file=sys.stderr, ) del config["mzbuild"] else: images.append(image) if "propagate-uid-gid" in config: config["user"] = f"{os.getuid()}:{os.getgid()}" del config["propagate-uid-gid"] deps = repo.resolve_dependencies(images) for d in deps: say(d.spec()) for config in compose["services"].values(): if "mzbuild" in config: config["image"] = deps[config["mzbuild"]].spec() del config["mzbuild"] # Check if the command is going to create or start containers, and if so # build the dependencies. This can be slow, so we don't want to do it if we # can help it (e.g., for `down` or `ps`). if command in ["create", "run", "start", "up"]: deps.acquire() # Construct a configuration that will point Docker Compose at the correct # images. tempfile = TemporaryFile() os.set_inheritable(tempfile.fileno(), True) yaml.dump(compose, tempfile, encoding="utf-8") # type: ignore tempfile.flush() tempfile.seek(0) return tempfile
class stdIOCollector(collector.collectorBase.Collector): def __init__(self, stdout="", stderr=""): super().__init__(name='stdIO') def __del__(self): if self.stdoutLogger is not None: self.stdoutLogger.close() if self.stderrLogger is not None: self.stderrLogger.close() def prepare(self, conf: dict) -> dict: self.stdoutLogger = TemporaryFile('w+t') self.stderrLogger = TemporaryFile('w+t') logger = {'stdout': self.stdoutLogger, 'stderr': self.stderrLogger} return logger def start(self): super().start() def stop(self): super().stop() def getData(self): self.stdoutLogger.flush() self.stdoutLogger.seek(0) self.stderrLogger.flush() self.stderrLogger.seek(0) return self.stdoutLogger.readlines() + self.stderrLogger.readlines()
def to_xml(self, f=None): """Get this domain as an XML DOM Document :param f: Optional File to dump directly to :type f: File or Stream :return: File object where the XML has been dumped to :rtype: file """ if not f: from tempfile import TemporaryFile f = TemporaryFile() print('<?xml version="1.0" encoding="UTF-8"?>', file=f) print('<Domain id="%s">' % self.name, file=f) for item in self: print('\t<Item id="%s">' % item.name, file=f) for k in item: print('\t\t<attribute id="%s">' % k, file=f) values = item[k] if not isinstance(values, list): values = [values] for value in values: print('\t\t\t<value><![CDATA[', end=' ', file=f) if isinstance(value, unicode): value = value.encode('utf-8', 'replace') else: value = unicode(value, errors='replace').encode('utf-8', 'replace') f.write(value) print(']]></value>', file=f) print('\t\t</attribute>', file=f) print('\t</Item>', file=f) print('</Domain>', file=f) f.flush() f.seek(0) return f
def sort_diskbased(stream, field, nsize=100000): buf = [] files = [] count = 0 t = None def iter_on_file(f): try: while True: (key, v) = cPickle.load(f) yield (key, t._make(v)) except EOFError: f.close() for elt in stream: if isinstance(elt, StreamHeader): t = elt.t yield elt elif isinstance(elt, StreamFooter): buf.sort() iterables = [iter_on_file(f) for f in files] + [itertools.imap(lambda obj : (getattr(obj, field), obj), buf)] for (k, row) in heapq.merge(*iterables): yield row yield elt else: buf.append(elt) count = count + 1 if count % nsize == 0: buf.sort(key=lambda obj: getattr(obj, field)) f = TemporaryFile() for item in buf: cPickle.dump((getattr(item, field), list(item)), f, cPickle.HIGHEST_PROTOCOL) f.flush() files.append(f) del buf[:]
def send_email(send_target, send_port, send_sender, send_recipient, send_body, send_tls): logging.info("Sending email") t = TemporaryFile() available_fd = t.fileno() t.close() os.dup2(2, available_fd) t = TemporaryFile() os.dup2(t.fileno(), 2) try: server = SMTP(send_target, send_port) server.set_debuglevel(100) server.ehlo_or_helo_if_needed() if send_tls: try_tls(server) server.sendmail(send_sender, send_recipient, send_body) server.quit() sys.stderr.flush() t.flush() t.seek(0) stderr_output = t.read() t.close() os.dup2(available_fd, 2) os.close(available_fd) count = 0 for line in stderr_output.decode('utf-8').split("\n"): count += 1 logging.debug(line) print(line) except Exception, exc: logging.critical("Email failed: %s\r\nExiting." % str(exc)) # log error message sys.exit("Email failed: %s\r\nExiting." % str(exc)) # give an error message
def create_tarball(tar_paths): """ Context Manger that creates the tarball of the Docker Context to use for building the image Parameters ---------- tar_paths dict(str, str) Key representing a full path to the file or directory and the Value representing the path within the tarball Yields ------ The tarball file """ tarballfile = TemporaryFile() with tarfile.open(fileobj=tarballfile, mode="w") as archive: for path_on_system, path_in_tarball in tar_paths.items(): archive.add(path_on_system, arcname=path_in_tarball) # Flush are seek to the beginning of the file tarballfile.flush() tarballfile.seek(0) try: yield tarballfile finally: tarballfile.close()
def create_tarball(tar_paths): """ Context Manger that creates the tarball of the Docker Context to use for building the image Parameters ---------- tar_paths dict(str, str) Key representing a full path to the file or directory and the Value representing the path within the tarball Yields ------ The tarball file """ tarballfile = TemporaryFile() with tarfile.open(fileobj=tarballfile, mode='w') as archive: for path_on_system, path_in_tarball in tar_paths.items(): archive.add(path_on_system, arcname=path_in_tarball) # Flush are seek to the beginning of the file tarballfile.flush() tarballfile.seek(0) try: yield tarballfile finally: tarballfile.close()
def Popen(self, args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0): if isinstance(args, basestring): cmd = args else: cmd = ' '.join(args) if cmd not in self.commands: raise KeyError('Nothing specified for command %r' % cmd) self.stdout, self.stderr, self.returncode, pid, poll = self.commands[cmd] self.poll_count = poll for name in 'stdout', 'stderr': f = TemporaryFile() f.write(getattr(self, name)) f.flush() f.seek(0) setattr(self.mock.Popen_instance, name, f) self.mock.Popen_instance.pid = pid self.mock.Popen_instance.returncode = None return self.mock.Popen_instance
def backup(self): """ Create backup """ if self.dry_run: return if not os.path.exists( self.config["tar"]["directory"]) or not os.path.isdir( self.config["tar"]["directory"]): raise BackupError("{0} is not a directory!".format( self.config["tar"]["directory"])) out_name = "{0}.tar".format( self.config["tar"]["directory"].lstrip("/").replace("/", "_")) outfile = os.path.join(self.target_directory, out_name) args = ["tar", "c", self.config["tar"]["directory"]] errlog = TemporaryFile() stream = open_stream(outfile, "w", **self.config["compression"]) LOG.info("Executing: %s", list2cmdline(args)) pid = Popen(args, stdout=stream.fileno(), stderr=errlog.fileno(), close_fds=True) status = pid.wait() try: errlog.flush() errlog.seek(0) for line in errlog: LOG.error("%s[%d]: %s", list2cmdline(args), pid.pid, line.rstrip()) finally: errlog.close() if status != 0: raise BackupError("tar failed (status={0})".format(status))
def backup(self): """ Create backup """ if self.dry_run: return if not os.path.exists(self.config["tar"]["directory"]) or not os.path.isdir( self.config["tar"]["directory"] ): raise BackupError("{0} is not a directory!".format(self.config["tar"]["directory"])) out_name = "{0}.tar".format(self.config["tar"]["directory"].lstrip("/").replace("/", "_")) outfile = os.path.join(self.target_directory, out_name) args = ["tar", "c", self.config["tar"]["directory"]] errlog = TemporaryFile() stream = open_stream(outfile, "w", **self.config["compression"]) LOG.info("Executing: %s", list2cmdline(args)) pid = Popen(args, stdout=stream.fileno(), stderr=errlog.fileno(), close_fds=True) status = pid.wait() try: errlog.flush() errlog.seek(0) for line in errlog: LOG.error("%s[%d]: %s", list2cmdline(args), pid.pid, line.rstrip()) finally: errlog.close() if status != 0: raise BackupError("tar failed (status={0})".format(status))
def parse_images(instance): if instance._content is None or 'img' not in instance._content: return content = instance._content[:] soup = BeautifulSoup(content, "html.parser") for img in soup('img'): # Build the source image filename my_url2path_func = instance.settings['MY_IMG_URL2PATH_FUNC'] if not my_url2path_func: logger.error('Error: MY_IMG_URL2PATH_FUNC not defined in your pelican configuration.\n\ niux2_lazyload_helper cannot determine the image path from its url.\n') return imgPath, new_src = my_url2path_func(img['src']) if not new_src.startswith('http') and not (path.isfile(imgPath) and access(imgPath, R_OK)): logger.error('Error: image file not found: {}'.format(imgPath)) continue img['src'] = new_src # Open the source image and query dimensions if new_src.startswith('http'): img_data = urlopen(new_src).read() fid = TemporaryFile('wb+') fid.write(img_data) fid.flush() fid.seek(0) else: fid = open(imgPath, 'rb') im = Image.open(fid) imgWidth = im.size[0] imgHeight = im.size[1] imgResized = False if not img.get('width'): img['width'] = str(imgWidth) + 'px' else: imgResized = True # for lazyload.js if instance.settings.get('NIUX2_LAZY_LOAD', False): if img.get('class'): img['class'] += 'lazy' else: img['class'] = 'lazy' img['data-original'] = img['src'] del img['src'] if imgResized: newImgWidth = int(_width_attr_reg.sub('', img['width']).strip()) newImgHeight = imgHeight * newImgWidth / imgWidth img['data-width'] = str(newImgWidth) + 'px' img['data-height'] = str(newImgHeight) + 'px' else: img['data-width'] = str(imgWidth) + 'px' img['data-height'] = str(imgHeight) + 'px' instance._content = soup.decode()
def __init__(self, repo: mzbuild.Repository, name: str): self.name = name self.repo = repo self.images: List[mzbuild.Image] = [] default_tag = os.getenv(f"MZBUILD_TAG", None) if name in self.repo.compositions: self.path = self.repo.compositions[name] else: raise errors.UnknownComposition with open(self.path) as f: compose = yaml.safe_load(f) # Stash away sub workflows so that we can load them with the correct environment variables self.workflows = compose.pop("mzworkflows", None) # Resolve all services that reference an `mzbuild` image to a specific # `image` reference. for config in compose["services"].values(): if "mzbuild" in config: image_name = config["mzbuild"] if image_name not in self.repo.images: raise errors.BadSpec(f"mzcompose: unknown image {image_name}") image = self.repo.images[image_name] override_tag = os.getenv( f"MZBUILD_{image.env_var_name()}_TAG", default_tag ) if override_tag is not None: config["image"] = image.docker_name(override_tag) print( f"mzcompose: warning: overriding {image_name} image to tag {override_tag}", file=sys.stderr, ) del config["mzbuild"] else: self.images.append(image) if "propagate-uid-gid" in config: config["user"] = f"{os.getuid()}:{os.getgid()}" del config["propagate-uid-gid"] deps = self.repo.resolve_dependencies(self.images) for config in compose["services"].values(): if "mzbuild" in config: config["image"] = deps[config["mzbuild"]].spec() del config["mzbuild"] # Emit the munged configuration to a temporary file so that we can later # pass it to Docker Compose. tempfile = TemporaryFile() os.set_inheritable(tempfile.fileno(), True) yaml.dump(compose, tempfile, encoding="utf-8") # type: ignore tempfile.flush() self.file = tempfile
def savefile(fd,fname,bfirmid,bclientid): # Encrypt each chunk from fd as it is read into a # tmpfile which will be uploaded to Dropbox using # the given filename. r = requests.get("%s/keyserv/key/%s/%s" % (app.config['KEYSERVER_URI'],bfirmid,bclientid)) print "%s/keyserv/key/%s/%s" % (app.config['KEYSERVER_URI'],bfirmid,bclientid) keyobj = r.json() encrkey = keyobj['key'] print "Got key %s" % encrkey # Carve out a 32byte/256 bit key from the keyserver # but convert base64 back to binary first bkey = binascii.a2b_base64(encrkey) key = bkey[0:32] try: print "Starting encryption" # Setup our AES cipher iv = Random.new().read(AES.block_size) cipher = AES.new(key,AES.MODE_CFB,iv) #cipher = XORCipher.new(key) print "Cipher created using iv %s" % binascii.hexlify(iv) except: raise try: f = TemporaryFile() f.write(iv) for chunk in chunkfd(fd,blocksize=4194304): f.write(cipher.encrypt(chunk)) f.flush() f.seek(0,os.SEEK_END) fsize = f.tell() f.seek(0) except Exception as e: print e print "Getting ready for Dropbox upload" # Get a Dropbox uploader try: access_token = config.get('Credentials','access_token') dclient = DropboxClient(access_token) uploader = dclient.get_chunked_uploader(f,fsize) while uploader.offset < fsize: try: upload = uploader.upload_chunked() except Exception as e: print e except Exception as e: print e f.close() return uploader.finish(secure_filename("/%s_encr" % fname))
def Popen(self, args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0, restore_signals=True, start_new_session=False, pass_fds=(), encoding=None, errors=None): if isinstance(args, basestring): cmd = args else: cmd = ' '.join(args) behaviour = self.commands.get(cmd, self.default_command) if behaviour is None: raise KeyError('Nothing specified for command %r' % cmd) stdout_value, stderr_value, self.returncode, pid, poll = behaviour if stderr == STDOUT: line_iterator = chain.from_iterable( zip_longest(stdout_value.splitlines(True), stderr_value.splitlines(True))) stdout_value = b''.join(l for l in line_iterator if l) stderr_value = None self.poll_count = poll for name, option, mock_value in (('stdout', stdout, stdout_value), ('stderr', stderr, stderr_value)): value = None if option is PIPE: value = TemporaryFile() value.write(mock_value) value.flush() value.seek(0) setattr(self.mock.Popen_instance, name, value) if stdin == PIPE: self.mock.Popen_instance.stdin = Mock() self.mock.Popen_instance.pid = pid self.mock.Popen_instance.returncode = None return self.mock.Popen_instance
def _read(rej,res): try: f = TemporaryFile() f.write(str) f.flush() f.seek(0) res( f ) except IOError as e: rej(e)
def run(self, databases, stream, additional_options=None): """Run mysqldump with the options configured on this instance""" if self.mock_env: subprocess.Popen = self.mock_env.mocked_popen if not hasattr(stream, 'fileno'): raise MySQLDumpError("Invalid output stream") if not databases: raise MySQLDumpError("No databases specified to backup") args = [ self.cmd_path, ] if self.defaults_file: if self.extra_defaults: args.append('--defaults-extra-file=%s' % self.defaults_file) else: args.append('--defaults-file=%s' % self.defaults_file) args.extend([str(opt) for opt in self.options]) if additional_options: args.extend(additional_options) if databases is ALL_DATABASES: args.append('--all-databases') else: if len(databases) > 1: args.append('--databases') args.extend(databases) if self.mock_env: LOG.info("Dry Run: %s", subprocess.list2cmdline(args)) else: LOG.info("Executing: %s", subprocess.list2cmdline(args)) errlog = TemporaryFile() pid = subprocess.Popen(args, stdout=stream.fileno(), stderr=errlog.fileno(), close_fds=True) status = pid.wait() try: errlog.flush() errlog.seek(0) for line in errlog: LOG.error("%s[%d]: %s", self.cmd_path, pid.pid, line.rstrip()) finally: errlog.close() if status != 0: raise MySQLDumpError("mysqldump exited with non-zero status %d" % pid.returncode)
def buildvrt(src): """ built a vrt file from the input src files. Removes non existing srcs Returns None if no file (or no-valid file was given. :param src: list of Str or Path :return: file handler """ from subprocess import Popen, PIPE from tempfile import TemporaryFile vrt_exec = GdalFunctions.vrt_exec() if not isinstance(src, (list, tuple)): src = [src] # filter out non-existant src files src = [Path(x).as_posix() for x in src if Path(x).exists()] if len(src) < 1: return None cl = Popen( ' '.join([vrt_exec, "-q", "/vsistdout/"] + # virtual gdal output file, redirects the out to stdout src), shell=True, stdout=PIPE, stderr=PIPE) out, err = cl.communicate() # clean output from system newlines chars out = out.decode('utf8').replace(os.linesep, '') if err: print(' '.join( [vrt_exec, "-q", "/vsistdout/"] + # virtual gdal output file, redirects the out to stdout src, )) raise WpTillerError('GdalBuild VRT encountered an error:\n%s' % err) if not out: WpTillerWarning('GdalBuild VRT returned an empty xml') try: ElementTree.fromstring(out) except ElementTree.ParseError: raise WpTillerError('GdalBuild VRT return a corrupted XML\n%s' % out) vrt_handle = TemporaryFile() vrt_handle.write(out.encode()) vrt_handle.flush() vrt_handle.seek(0) return vrt_handle
def Popen(self, args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0, restore_signals=True, start_new_session=False, pass_fds=(), encoding=None, errors=None): if isinstance(args, basestring): cmd = args else: cmd = ' '.join(args) behaviour = self.commands.get(cmd, self.default_behaviour) if behaviour is None: raise KeyError('Nothing specified for command %r' % cmd) if callable(behaviour): behaviour = behaviour(command=cmd, stdin=stdin) self.returncode = behaviour.returncode stdout_value = behaviour.stdout stderr_value = behaviour.stderr if stderr == STDOUT: line_iterator = chain.from_iterable(zip_longest( stdout_value.splitlines(True), stderr_value.splitlines(True) )) stdout_value = b''.join(l for l in line_iterator if l) stderr_value = None self.poll_count = behaviour.poll_count for name, option, mock_value in ( ('stdout', stdout, stdout_value), ('stderr', stderr, stderr_value) ): value = None if option is PIPE: value = TemporaryFile() value.write(mock_value) value.flush() value.seek(0) setattr(self.mock.Popen_instance, name, value) if stdin == PIPE: self.mock.Popen_instance.stdin = Mock() self.mock.Popen_instance.pid = behaviour.pid self.mock.Popen_instance.returncode = None return self.mock.Popen_instance
def test_temporary_file(self): # GH13398 data1 = "0 0" from tempfile import TemporaryFile new_file = TemporaryFile("w+") new_file.write(data1) new_file.flush() new_file.seek(0) result = self.read_csv(new_file, sep=r"\s*", header=None) expected = DataFrame([[0, 0]]) tm.assert_frame_equal(result, expected)
def test_file(self): if version[0] == '3': f = TemporaryFile('w+', encoding='utf8') else: f = TemporaryFile('w+') print("hello world\n", file=f) f.flush() # f.seek(0) # print(f.readlines()) f.seek(0) w = Token('[a-z]+') s = Token(' +') v = w & s & w v.parse_iterable(f)
class T(threading.Thread): _shutdown_msg = "shutdown" def __init__(self): threading.Thread.__init__(self) self._fd = TemporaryFile() self._comm_fd = TemporaryFile() self._run = False def get_file_handle(self): return self._fd def run(self): self._run = True while self._run: t1 = time.time() r, _, _ = select.select([self._fd.fileno(), self._comm_fd.fileno()], [], []) print "select time:", time.time()-t1 for elem in r: if elem == self._fd.fileno(): s = self._fd.tell() self._fd.seek(0, os.SEEK_END) # to the end e = self._fd.tell() if s == e: # nothing new continue self._fd.seek(-(e-s), os.SEEK_END) diff = self._fd.read(e-s) if True: sys.stdout.write(diff) sys.stdout.flush() # exit elif elem == self._comm_fd.fileno(): self._comm_fd.seek(0, os.SEEK_END) if self._comm_fd.tell() == len(T._shutdown_msg): self._run = False self._comm_fd.write(T._shutdown_msg) self._comm_fd.flush() def stop(self): self._comm_fd.seek(0, os.SEEK_END) if self._comm_fd.tell() != 0: return self._comm_fd.write(T._shutdown_msg) self._comm_fd.flush() while self._comm_fd.tell() != 2*len(T._shutdown_msg): self._comm_fd.seek(0, os.SEEK_END) def __del__(self, ): self._fd.close()
class TSV: """ TSV Class Create files on the fly with tab-separated values """ _tsv = None _writer = None def __init__(self, path=None, mode='r', temporary=False): self.logger = logging.getLogger() self.logger.debug("Initiating TSV") if temporary: if 'r' in mode: raise ValueError("::Cannot read from temporary file") self._tsv = TemporaryFile(mode='w') self.logger.debug("Storing to temporary file") self._writer = csv.writer(self._tsv, delimiter="\t") return if path is None: raise ValueError("::No path supplied") self._tsv = open(path, mode) if 'r' in mode: self.logger.debug("Loading from file: {}".format(path)) return self.read() else: self.logger.debug("Storing to file: {}".format(path)) self._writer = csv.writer(self._tsv, delimiter="\t") def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): self._tsv.close() def read(self): return csv.reader(self._tsv, delimiter="\t") def writerow(self, row): self._writer.writerow(row) self._tsv.flush() def writerows(self, rows): self._writer.writerows(rows) self._tsv.flush()
def __call__(self): """Return File/Image Raw Data""" context = proxy.removeSecurityProxy(self.context) #response.setHeader('Content-Type', 'application/octect-stream') if len(self.traverse_subpath) != 1: return fname = self.traverse_subpath[0] tempfile = TemporaryFile() data = getattr(context, fname, None) if data is not None: tempfile.write(data) tempfile.flush() return tempfile else: raise NotFound(self.context, fname, self.request)
class tee: """ Inspired by: http://shallowsky.com/blog/programming/python-tee.html """ def __init__(self, stream): self.temp_file = TemporaryFile(mode="w+") self.stream = stream def write(self, text): self.temp_file.write(text) self.stream.write(text) def flush(self): self.temp_file.flush() self.stream.flush()
def __call__(self): """Return File/Image Raw Data""" context = removeSecurityProxy(self.context) #response.setHeader("Content-Type", "application/octect-stream") if len(self.traverse_subpath) != 1: return fname = self.traverse_subpath[0] tempfile = TemporaryFile() data = getattr(context, fname, None) if data is not None: tempfile.write(data) tempfile.flush() return tempfile else: raise NotFound(self.context, fname, self.request)
def _check(self): if self._is_file: return pos = self._stream.tell() if pos <= self._threshold: return stream = TemporaryFile('wb+') stream.write(self._stream.getvalue()) stream.flush() stream.seek(pos) self._stream.close() self._stream = stream self._is_file = True
class PassphraseFile(object): def __init__(self, passphrase): self.passphrase = passphrase.encode( 'utf-8') if type(passphrase) != bytes else passphrase self.file = TemporaryFile() def __enter__(self): self.file.write(self.passphrase) self.file.flush() return self.name() def __exit__(self, type, value, traceback): self.file.close() def name(self): return '/proc/%d/fd/%d' % (getpid(), self.file.fileno())
def run(self, databases, stream, additional_options=None): """Run mysqldump with the options configured on this instance""" if not hasattr(stream, "fileno"): raise MySQLDumpError("Invalid output stream") if not databases: raise MySQLDumpError("No databases specified to backup") args = [self.cmd_path] if self.defaults_file: if self.extra_defaults: args.append("--defaults-extra-file=%s" % self.defaults_file) else: args.append("--defaults-file=%s" % self.defaults_file) args.extend([str(opt) for opt in self.options]) if additional_options: args.extend(additional_options) if databases is ALL_DATABASES: args.append("--all-databases") else: if len(databases) > 1: args.append("--databases") args.extend(databases) if self.mock_env: LOG.info("Dry Run: %s", subprocess.list2cmdline(args)) popen = self.mock_env.mocked_popen else: LOG.info("Executing: %s", subprocess.list2cmdline(args)) popen = subprocess.Popen errlog = TemporaryFile() pid = popen(args, stdout=stream.fileno(), stderr=errlog.fileno(), close_fds=True) status = pid.wait() try: errlog.flush() errlog.seek(0) for line in errlog: LOG.error("%s[%d]: %s", self.cmd_path, pid.pid, line.rstrip()) finally: errlog.close() if status != 0: raise MySQLDumpError("mysqldump exited with non-zero status %d" % pid.returncode)
def run(self): """Run the command.""" if not self.to_exec: raise RxcCommandException('Command {0} has not been insuflated'.format(self.command_id)) stdout = TemporaryFile() stderr = TemporaryFile() return_code = call(self.to_exec, shell=True, stdout=stdout, stderr=stderr) stdout.flush() stderr.flush() stdout.seek(0) stderr.seek(0) return { 'command' : self.to_exec, 'return_code' : return_code, 'stdout' : stdout.read(), 'stderr' : stderr.read() }
def __call__(self): context = proxy.removeSecurityProxy(self.context) mimetype = getattr(context, 'file_mimetype', None) if mimetype == None: mimetype = 'application/octect-stream' filename = getattr(context, 'file_name', None) if filename == None: filename = getattr(context, 'file_title', None) tempfile = TemporaryFile() data = getattr(context, 'file_data', None) if data is not None: tempfile.write(data) tempfile.flush() self.request.response.setHeader('Content-type', mimetype) self.request.response.setHeader('Content-disposition', 'attachment;filename="%s"' % filename) return tempfile else: raise NotFound(context, "", self.request)
def open_thing(address, accept_types=None): """Try to open an URL or local file. Return a tuple (file, type, info), where: -- file: file object or None if an error occured -- type: MIME type (if known) -- info: httplib.HTTPMessage object (if present) """ f, t, i = None, None, None pa = urlparse(address) if pa.scheme in ['http', 'https', 'ftp']: try: tmp = urlopen(address, timeout=TIMEOUT) except URLError as ex: print_urlerror(address, ex) return (f, t, i) except: print(ERR, 'Bad URL:', address) return (f, t, i) i = tmp.info() i.url = tmp.url t = i.gettype() if accept_types is not None and t not in accept_types: return (f, t, i) try: f = TemporaryFile() f.write(tmp.read()) f.flush() f.seek(0) except Exception as ex: print(ERR, ex) return (None, None, None) else: # Unknown protocol, suppose it's local file path. fp = os.path.normpath(address) if os.path.isfile(fp): try: f = open(fp, 'rb') except IOError as ex: print(ERR, 'I/O error.', ex) else: print(ERR, 'Unknown object:', fp) return (f, t, i)
def __call__(self): context = proxy.removeSecurityProxy(self.context) mimetype = getattr(context, 'file_mimetype', None) if mimetype == None: mimetype = 'application/octect-stream' filename = getattr(context, 'file_name', None) if filename == None: filename = getattr(context, 'file_title', None) tempfile = TemporaryFile() data = getattr(context, 'file_data', None) if data is not None: tempfile.write(data) tempfile.flush() self.request.response.setHeader('Content-type', mimetype) self.request.response.setHeader( 'Content-disposition', 'attachment;filename="%s"' % filename) return tempfile else: raise NotFound(context, "", self.request)
def __init__(self, req): size = req.get_header('Content-Length') if size is None: size = -1 else: size = int(size) tempfile = TemporaryFile() input = req.environ['wsgi.input'] while True: buf = input.read(min(4096, size)) if not buf: break tempfile.write(buf) size -= len(buf) tempfile.flush() tempfile.seek(0) self.file = tempfile filename = req.get_header('X-TracDragDrop-Filename') self.filename = unicode_unquote(filename or '').encode('utf-8')
def _open_stream(self, path, mode, method=None): """Open a stream through the holland compression api, relative to this instance's target directory """ compression_method = method or self.config['compression']['method'] compression_level = self.config['compression']['level'] compression_options = self.config['compression']['options'] stream = open_stream(path, mode, compression_method, compression_level, extra_args=compression_options) return stream def backup(self): if self.dry_run: return if not os.path.exists(self.config['tar']['directory']) or not os.path.isdir(self.config['tar']['directory']): raise BackupError('{0} is not a directory!'.format(self.config['tar']['directory'])) out_name = "{0}.tar".format( self.config['tar']['directory'].lstrip('/').replace('/', '_')) outfile = os.path.join(self.target_directory, out_name) args = ['tar', 'c', self.config['tar']['directory']] errlog = TemporaryFile() stream = self._open_stream(outfile, 'w') LOG.info("Executing: %s", list2cmdline(args)) pid = Popen( args, stdout=stream.fileno(), stderr=errlog.fileno(), close_fds=True) status = pid.wait() try: errlog.flush() errlog.seek(0) for line in errlog: LOG.error("%s[%d]: %s", list2cmdline(args), pid.pid, line.rstrip()) finally: errlog.close()
def Popen( self, args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0, ): if isinstance(args, basestring): cmd = args else: cmd = " ".join(args) behaviour = self.commands.get(cmd, self.default_command) if behaviour is None: raise KeyError("Nothing specified for command %r" % cmd) self.stdout, self.stderr, self.returncode, pid, poll = behaviour self.poll_count = poll for name in "stdout", "stderr": f = TemporaryFile() f.write(getattr(self, name)) f.flush() f.seek(0) setattr(self.mock.Popen_instance, name, f) self.mock.Popen_instance.pid = pid self.mock.Popen_instance.returncode = None return self.mock.Popen_instance
def attach_pictures(self, cr, uid, ids, pictures_id, context=None): """ Push the new picture. """ res = super(child_property, self).attach_pictures( cr, uid, ids, pictures_id, context) pictures = self.pool.get('compassion.child.pictures').browse( cr, uid, pictures_id, context) # Retrieve configuration smb_user = config.get('smb_user') smb_pass = config.get('smb_pwd') smb_ip = config.get('smb_ip') smb_port = int(config.get('smb_port', 0)) if not (smb_user and smb_pass and smb_ip and smb_port): raise orm.except_orm( 'Config Error', 'Missing Samba configuration in conf file.') child = pictures.child_id date_pic = pictures.date.replace('-', '') gp_pic_path = "{0}{1}/".format(config.get('gp_pictures'), child.code[:2]) file_name = "{0}_{1}.jpg".format(child.code, date_pic) picture_file = TemporaryFile() picture_file.write(base64.b64decode(pictures.fullshot)) picture_file.flush() picture_file.seek(0) # Upload file to shared folder smb_conn = SMBConnection(smb_user, smb_pass, 'openerp', 'nas') if smb_conn.connect(smb_ip, smb_port): try: smb_conn.storeFile( 'GP', gp_pic_path + file_name, picture_file) except OperationFailure: # Directory may not exist smb_conn.createDirectory('GP', gp_pic_path) smb_conn.storeFile( 'GP', gp_pic_path + file_name, picture_file) return res
def doWork(self): tmp = TemporaryFile() tmp.write(HDDWorker.dataChunk) startTime = time.time() while time.time() - startTime < self.duration: flag = random.randrange(1,4) if flag == 1: tmp.seek(0, 2) tmp.write(HDDWorker.dataChunk) tmp.write(HDDWorker.dataChunk) tmp.write(HDDWorker.dataChunk) tmp.write(HDDWorker.dataChunk) tmp.flush() elif flag == 2: tmp.seek(0,0) tmp.read(len(HDDWorker.dataChunk)) tmp.read(len(HDDWorker.dataChunk)) tmp.read(len(HDDWorker.dataChunk)) tmp.read(len(HDDWorker.dataChunk)) else: time.sleep(random.random()) tmp.close()