def getUpdateElements(self, valueMap): ''' @param valueMap: ''' elements = "" for name in valueMap.keys(): fullname = name if isinstance(name, types.StringType): fullname = (self.defaultNameSpace, name) if not fullname[0]: tag = fullname[1] else: tag = self.shortcuts[fullname[0]] + ':' + fullname[1] value = valueMap[name] if value: if isinstance(value, qp_xml._element): tmpFile = TemporaryFile('w+') value = qp_xml.dump(tmpFile, value) tmpFile.flush() tmpFile.seek(0) tmpFile.readline() value = tmpFile.read() else: value = "<![CDATA[%s]]>" % value else: value = "" elements += "<%s>%s</%s>" % (tag, value, tag) return elements
def addValues(self, values, experiment): if len(self.variables) == len(values): readyProtocol = self.info newProtocol = [] for i in range(0, len(readyProtocol)): mystring = readyProtocol[i] newProtocol.append(mystring) for v in range(0, len(values)): newProtocol[i] = newProtocol[i].replace(self.variables[v], values[v]) from tempfile import TemporaryFile protocolFile = TemporaryFile(mode='r+') protocolFile.writelines(newProtocol) protocolFile.seek(0) line = protocolFile.readline() if experiment.platform != 'human': experiment.addComment('------ BEGIN PROTOCOL ' + self.name + ', variables: ' + ' '.join(self.variables) + '; values: ' + ' '.join(values) + ' ------') else: experiment.addComment('$') while line != '': splitline = line.split() LineToList(splitline, protocolFile, experiment) line = protocolFile.readline() if experiment.platform != 'human': experiment.addComment('------ END PROTOCOL ' + self.name + ' ------') else: experiment.addComment('$')
def dividing_file_and_sorting_by_lines( file): # dividing file on temporary smaller files and then # merge sorting them by lines file.seek(0) size = os.path.getsize(file.name) if size > 26214400: # temporary files weight is 25mb first_help_file, second_help_file = TemporaryFile( mode="w+t"), TemporaryFile(mode="w+t") for line in file: if os.path.getsize(first_help_file.name) <= size // 2: first_help_file.write(line) else: second_help_file.write(line) dividing_file_and_sorting_by_lines(first_help_file) dividing_file_and_sorting_by_lines(second_help_file) file.seek(0) first_help_file.seek(0) second_help_file.seek(0) first_file_line = first_help_file.readline() second_file_line = second_help_file.readline() while (first_file_line != "") and (second_file_line != ""): # merge sort of lines if first_file_line > second_file_line: file.write(second_file_line) second_file_line = second_help_file.readline( ) # step of your loop else: file.write(first_file_line) first_file_line = first_help_file.readline() while first_file_line != "": file.write(first_file_line) first_file_line = first_help_file.readline() # step of your loop while second_file_line != "": file.write(second_file_line) second_file_line = second_help_file.readline() # step of your loop else: # when our temporary file is less than 25mb finally sorting it by lines global current_progress info = file.readlines() merge_sort(info) file.seek(0) current_progress += len(info) info = "".join(info) file.write(info) print("\r", "Прогресс программы: ", 50 + round((current_progress / final_progress) * 50), "%", end="")
def import_lang(self, cr, uid, ids, context): """ Import Language @param cr: the current row, from the database cursor. @param uid: the current user’s ID for security checks. @param ids: the ID or list of IDs @param context: A standard dictionary """ import_data = self.browse(cr, uid, ids)[0] fileobj = TemporaryFile('w+') fileobj.write(base64.decodestring(import_data.data)) # now we determine the file format fileobj.seek(0) first_line = fileobj.readline().strip().replace('"', '').replace(' ', '') fileformat = first_line.endswith( "type,name,res_id,src,value") and 'csv' or 'po' fileobj.seek(0) tools.trans_load_data(cr, fileobj, fileformat, import_data.code, lang_name=import_data.name) tools.trans_update_res_ids(cr) fileobj.close() return {}
def import_lang(self, cr, uid, ids, context=None): """ Import Language @param cr: the current row, from the database cursor. @param uid: the current user’s ID for security checks. @param ids: the ID or list of IDs @param context: A standard dictionary """ if context is None: context = {} import_data = self.browse(cr, uid, ids)[0] if import_data.overwrite: context.update(overwrite=True) fileobj = TemporaryFile('w+') fileobj.write(base64.decodestring(import_data.data)) # now we determine the file format fileobj.seek(0) first_line = fileobj.readline().strip().replace('"', '').replace(' ', '') fileformat = first_line.endswith("type,name,res_id,src,value") and 'csv' or 'po' fileobj.seek(0) tools.trans_load_data(cr, fileobj, fileformat, import_data.code, lang_name=import_data.name, context=context) fileobj.close() return {}
def import_lang(self, cr, uid, ids, context=None): if context is None: context = {} this = self.browse(cr, uid, ids[0]) if this.overwrite: context = dict(context, overwrite=True) fileobj = TemporaryFile('w+') try: fileobj.write(base64.decodestring(this.data)) # now we determine the file format fileobj.seek(0) first_line = fileobj.readline().strip().replace('"', '').replace( ' ', '') fileformat = first_line.endswith( "type,name,res_id,src,value") and 'csv' or 'po' fileobj.seek(0) tools.trans_load_data(cr, fileobj, fileformat, this.code, lang_name=this.name, context=context) finally: fileobj.close() return True
def fpost( myHeaders={ 'User-Agent': 'PcGroup Util Client', 'content-type': 'application/json; charset=UTF-8', 'Accept-Encoding': '*/*' }): url = 'http://192.168.12.81:9200/_bulk' data = "{ \"index\" : { \"_index\" : \"test2\", \"_type\" : \"log\"} }\n" data = data + "{\"ok\":\"a\"," + "\"ok2\":1}\n" data = data + "{\"ok\":\"b\"," + "\"ok2\":2}" temp = TemporaryFile() temp.writelines('{"index" : { "_index" : "test2", "_type" : "log"}}') temp.writelines('{"ok":"a","ok2":1}') #temp.write('{"ok":"a","ok2":1}') #temp.write(data) #temp.flush() temp.seek(0) print temp.readline() print temp.readline() print temp.mode files = { 'file': open('/Users/sky/Documents/company/esManager/pcPython/logs/test.log', "rb") } #files = {'file': temp} r = requests.post(url, base64.b64encode(data.encode("utf8")).decode("ascii"), headers=myHeaders) print r.request.headers print r.headers #r = requests.request("POST" , url , files = files, headers = myHeaders) print r.text
def addValues(self, values, experiment): if len(self.variables) == len(values): readyProtocol = self.info newProtocol = [] for i in range(0, len(readyProtocol)): mystring = readyProtocol[i] newProtocol.append(mystring) for v in range(0, len(values)): newProtocol[i] = newProtocol[i].replace(self.variables[v], values[v]) from tempfile import TemporaryFile protocolFile = TemporaryFile(mode='r+') protocolFile.writelines(newProtocol) protocolFile.seek(0) line = protocolFile.readline() experiment.addComment('------ BEGIN PROTOCOL ' + self.name + ', variables: ' + ' '.join(self.variables) + '; values: ' + ' '.join(values) + ' ------') while line != '': splitline = line.split() LineToList(splitline, protocolFile, experiment) line = protocolFile.readline() experiment.addComment('------ END PROTOCOL ' + self.name + ' ------')
def disk(): """Write dummy data to a temporary file.""" f = TemporaryFile() buf = '0' * 1023 + '\n' for _ in xrange(10): for _ in xrange(1024 * 8): f.write(buf) f.seek(0) while f.readline(): pass f.close()
def _import_lang(self, cr, uid, data, context): form=data['form'] fileobj = TemporaryFile('w+') fileobj.write( base64.decodestring(form['data']) ) # now we determine the file format fileobj.seek(0) first_line = fileobj.readline().strip().replace('"', '').replace(' ', '') fileformat = first_line.endswith("type,name,res_id,src,value") and 'csv' or 'po' fileobj.seek(0) tools.trans_load_data(cr.dbname, fileobj, fileformat, form['code'], lang_name=form['name']) fileobj.close() return {}
def import_lang(self, cr, uid, ids, context=None): if context is None: context = {} this = self.browse(cr, uid, ids[0]) if this.overwrite: context = dict(context, overwrite=True) fileobj = TemporaryFile('w+') try: fileobj.write(base64.decodestring(this.data)) # now we determine the file format fileobj.seek(0) first_line = fileobj.readline().strip().replace('"', '').replace(' ', '') fileformat = first_line.endswith("type,name,res_id,src,value") and 'csv' or 'po' fileobj.seek(0) tools.trans_load_data(cr, fileobj, fileformat, this.code, lang_name=this.name, context=context) finally: fileobj.close() return True
class S3File(io.IOBase): """File like proxy for s3 files, manages upload and download of locally managed temporary file """ def __init__(self, bucket, key, mode='w+b', *args, **kwargs): super(S3File, self).__init__(*args, **kwargs) self.bucket = bucket self.key = key self.mode = mode self.path = self.bucket + '/' + self.key # converts mode to readable/writable to enable the temporary file to have S3 data # read or written to it even if the S3File is read/write/append # i.e. "r" => "r+", "ab" => "a+b" updatable_mode = re.sub(r'^([rwa]+)(b?)$', r'\1+\2', mode) self._tempfile = TemporaryFile(updatable_mode) try: with s3errors(self.path): if 'a' in mode: # File is in an appending mode, start with the content in file s3.Object(bucket, key).download_fileobj(self._tempfile) self.seek(0, os.SEEK_END) elif 'a' not in mode and 'w' not in mode and 'x' not in mode: # file is not in a create mode, so it is in read mode # start with the content in the file, and seek to the beginning s3.Object(bucket, key).download_fileobj(self._tempfile) self.seek(0, os.SEEK_SET) except Exception: self.close() raise def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() def close(self): try: if self.writable(): self.seek(0) with s3errors(self.path): s3.Object(self.bucket, self.key).upload_fileobj(self._tempfile) finally: self._tempfile.close() @property def closed(self): return self._tempfile.closed def fileno(self): return self._tempfile.fileno() def flush(self): return self._tempfile.flush() def isatty(self): return self._tempfile.isatty() def readable(self): return 'r' in self.mode or '+' in self.mode def read(self, n=-1): if not self.readable(): raise IOError('not open for reading') return self._tempfile.read(n) def readinto(self, b): return self._tempfile.readinto(b) def readline(self, limit=-1): if not self.readable(): raise IOError('not open for reading') return self._tempfile.readline(limit) def readlines(self, hint=-1): if not self.readable(): raise IOError('not open for reading') return self._tempfile.readlines(hint) def seek(self, offset, whence=os.SEEK_SET): self._tempfile.seek(offset, whence) return self.tell() def seekable(self): return True def tell(self): return self._tempfile.tell() def writable(self): return 'w' in self.mode or 'a' in self.mode or '+' in self.mode or 'x' in self.mode def write(self, b): if not self.writable(): raise IOError('not open for writing') self._tempfile.write(b) return len(b) def writelines(self, lines): if not self.writable(): raise IOError('not open for writing') return self._tempfile.writelines(lines) def truncate(self, size=None): if not self.writable(): raise IOError('not open for writing') if size is None: size = self.tell() self._tempfile.truncate(size) return size
if line: element = line.split()[5] if element.startswith('/') and 'cgroup' not in element: mountpoints.append(element) return mountpoints for path in get_all_mountpoint(): rstr = '@8qJnD&Y' value = 0 try: fd = TemporaryFile(dir=path) fd.write(rstr) fd.flush() fd.seek(0) content = fd.readline() fd.close() if rstr != content: value = 1 except OSError, IOError: value = 1 hostname = read_endpoint_value() record = {} record['metric'] = 'sys.disk.rw' record['endpoint'] = hostname record['timestamp'] = int(time.time()) record['step'] = 60 record['value'] = value
def configure(self, env): import params # generate sqconfig file cmd = "lscpu|grep -E '(^CPU\(s\)|^Socket\(s\))'|awk '{print $2}'" ofile = TemporaryFile() Execute(cmd,stdout=ofile) ofile.seek(0) # read from beginning core, processor = ofile.read().split('\n')[:2] ofile.close() core = int(core)-1 if int(core) <= 256 else 255 lines = ['begin node\n'] loc_node_list = [] for node_id, node in enumerate(params.traf_node_list): # find the local hostname for each node cmd = "ssh -q %s hostname" % node ofile = TemporaryFile() Execute(cmd,user=params.traf_user,stdout=ofile) ofile.seek(0) # read from beginning localhn = ofile.readline().rstrip() ofile.close() cmd = "ssh %s 'echo success'" % localhn Execute(cmd,user=params.traf_user) # verify we can use this hostname to communicate line = 'node-id=%s;node-name=%s;cores=0-%d;processors=%s;roles=connection,aggregation,storage\n' \ % (node_id, localhn, core, processor) lines.append(line) loc_node_list.append(localhn) lines.append('end node\n') lines.append('\n') lines.append('begin overflow\n') for scratch_loc in params.traf_scratch.split(','): line = 'hdd %s\n' % scratch_loc lines.append(line) lines.append('end overflow\n') # write sqconfig in trafodion home dir trafhome = os.path.expanduser("~" + params.traf_user) File(os.path.join(trafhome,"sqconfig"), owner = params.traf_user, group = params.traf_user, content=''.join(lines), mode=0644) # install sqconfig Execute('source ~/.bashrc ; mv -f ~/sqconfig $TRAF_HOME/sql/scripts/',user=params.traf_user) # write cluster-env in trafodion home dir traf_nodes = ' '.join(loc_node_list) traf_w_nodes = '-w ' + ' -w '.join(loc_node_list) traf_node_count = len(loc_node_list) if traf_node_count != len(params.traf_node_list): print "Error cannot determine local hostname for all Trafodion nodes" exit(1) cl_env_temp = os.path.join(trafhome,"traf-cluster-env.sh") File(cl_env_temp, owner = params.traf_user, group = params.traf_user, content=InlineTemplate(params.traf_clust_template, traf_nodes=traf_nodes, traf_w_nodes=traf_w_nodes, traf_node_count=traf_node_count, cluster_name=params.cluster_name), mode=0644) # install cluster-env on all nodes for node in params.traf_node_list: cmd = "scp %s %s:%s/" % (cl_env_temp, node, params.traf_conf_dir) Execute(cmd,user=params.traf_user) cmd = "rm -f %s" % (cl_env_temp) Execute(cmd,user=params.traf_user) # Execute SQ gen Execute('source ~/.bashrc ; rm -f $TRAF_HOME/sql/scripts/sqconfig.db; sqgen',user=params.traf_user)
def configure(self, env): import params # generate sqconfig file cmd = "lscpu|grep -E '(^CPU\(s\)|^Socket\(s\))'|awk '{print $2}'" ofile = TemporaryFile() Execute(cmd, stdout=ofile) ofile.seek(0) # read from beginning core, processor = ofile.read().split('\n')[:2] ofile.close() core = int(core) - 1 if int(core) <= 256 else 255 lines = ['begin node\n'] loc_node_list = [] for node_id, node in enumerate(params.traf_node_list): # find the local hostname for each node cmd = "ssh -q %s hostname" % node ofile = TemporaryFile() Execute(cmd, user=params.traf_user, stdout=ofile) ofile.seek(0) # read from beginning localhn = ofile.readline().rstrip() ofile.close() cmd = "ssh %s 'echo success'" % localhn Execute(cmd, user=params.traf_user ) # verify we can use this hostname to communicate line = 'node-id=%s;node-name=%s;cores=0-%d;processors=%s;roles=connection,aggregation,storage\n' \ % (node_id, localhn, core, processor) lines.append(line) loc_node_list.append(localhn) lines.append('end node\n') lines.append('\n') lines.append('begin overflow\n') for scratch_loc in params.traf_scratch.split(','): line = 'hdd %s\n' % scratch_loc lines.append(line) lines.append('end overflow\n') # write sqconfig in trafodion home dir trafhome = os.path.expanduser("~" + params.traf_user) File(os.path.join(trafhome, "sqconfig"), owner=params.traf_user, group=params.traf_user, content=''.join(lines), mode=0644) # install sqconfig Execute('source ~/.bashrc ; mv -f ~/sqconfig $TRAF_HOME/sql/scripts/', user=params.traf_user) # write cluster-env in trafodion home dir traf_nodes = ' '.join(loc_node_list) traf_w_nodes = '-w ' + ' -w '.join(loc_node_list) traf_node_count = len(loc_node_list) if traf_node_count != len(params.traf_node_list): print "Error cannot determine local hostname for all Trafodion nodes" exit(1) cl_env_temp = os.path.join(trafhome, "traf-cluster-env.sh") File(cl_env_temp, owner=params.traf_user, group=params.traf_user, content=InlineTemplate(params.traf_clust_template, traf_nodes=traf_nodes, traf_w_nodes=traf_w_nodes, traf_node_count=traf_node_count, cluster_name=params.cluster_name), mode=0644) # install cluster-env on all nodes for node in params.traf_node_list: cmd = "scp %s %s:%s/" % (cl_env_temp, node, params.traf_conf_dir) Execute(cmd, user=params.traf_user) cmd = "rm -f %s" % (cl_env_temp) Execute(cmd, user=params.traf_user) # Execute SQ gen Execute( 'source ~/.bashrc ; rm -f $TRAF_HOME/sql/scripts/sqconfig.db; sqgen', user=params.traf_user)
if line: element = line.split()[5] if element.startswith('/'): mountpoints.append(element) return mountpoints for path in get_all_mountpoint(): rstr = '@8qJnD&Y' value = 0 try: fd = TemporaryFile(dir=path) fd.write(rstr) fd.flush() fd.seek(0) content = fd.readline() fd.close() if rstr != content: value = 1 except OSError, IOError: value = 1 record = {} record['metric'] = 'sys.disk.rw' record['endpoint'] = os.uname()[1] record['timestamp'] = int(time.time()) record['step'] = 60 record['value'] = value record['counterType'] = 'GAUGE' record['tags'] = 'mount=%s' % path