def print_svgz(self, filename, *args, **kwargs): if is_string_like(filename): gzipwriter = gzip.GzipFile(filename, 'w') fh_to_close = svgwriter = codecs.EncodedFile(gzipwriter, 'utf-8') elif is_writable_file_like(filename): fh_to_close = gzipwriter = gzip.GzipFile(fileobj=filename, mode='w') svgwriter = codecs.EncodedFile(gzipwriter, 'utf-8') else: raise ValueError("filename must be a path or a file-like object") return self._print_svg(filename, svgwriter, fh_to_close)
def import_csv(request): logger = get_logger() if request.method == "POST" and request.FILES: csvfile = request.FILES['csv_file'] dialect = csv.Sniffer().sniff( codecs.EncodedFile(csvfile, "utf-8").read(1024)) csvfile.open() reader = csv.reader(codecs.EncodedFile(csvfile, "utf-8"), delimiter='|', dialect=dialect) i = 0 # skip the definition fields for row in reader: if i != 0: try: AssetList.objects.get(ip_address=row[0]) except ObjectDoesNotExist: a_list = AssetList() a_list.ip_address = row[0] a_list.tcp = row[1] a_list.udp = row[2] a_list.definition = row[3] if row[4] == '': pass else: try: group = Group.objects.get(definition=row[4]) except ObjectDoesNotExist: group = Group() group.definition = row[4] group.save() except Exception as e: error_message = "ERROR while getting whitelist details ::: {0} ::: line {1}"\ .format(str(e), sys.exc_info()[2].tb_lineno) logger.send_log("error", error_message) return render(request, "import_csv.html", {"message": error_message}) a_list.group = group a_list.created_by = request.user a_list.save() except Exception as e: error_message = "ERROR while getting whitelist details ::: {0} ::: line {1}"\ .format(str(e), sys.exc_info()[2].tb_lineno) logger.send_log("error", error_message) return render(request, "import_csv.html", {"message": error_message}) else: i = 1 context = {"message": "Başarıyla Yüklendi"} return render(request, "import_csv.html", context) else: return render(request, "import_csv.html")
def parse_sql(filename, pattern, func, lang): "Rips columns specified with the regex out of the sql" if filename.endswith('.gz'): fh = codecs.EncodedFile(gzip.open(filename), data_encoding="utf-8") else: fh = codecs.open(filename, encoding="utf-8") line = "" count = 0 while True: buff = fh.read(1024) if not buff: break line += buff rows = list(re.finditer(pattern, line)) for row in rows: try: func(row.groups(), lang) except Exception, e: print "uhoh: %s: %s w/ %s" % (func, e, row.groups()) # don't forget unconsumed bytes, need them for the next match if len(rows) > 0: line = line[rows[-1].end():]
def _get_packed_pkg_file_contents(self, override_file, as_list=True): try: with zipfile.ZipFile(self.package_file()) as zFile: info = find_zip_entry(zFile, override_file) file = codecs.EncodedFile(zFile.open(info, mode="rU"), "utf-8") if as_list: content = io.TextIOWrapper(file, encoding="utf-8").readlines() else: content = io.TextIOWrapper(file, encoding="utf-8").read() source = "Shipped Packages" if self.installed_path is not None: source = "Installed Packages" source = os.path.join(source, self.name, override_file) mtime = datetime(*info.date_time).strftime("%Y-%m-%d %H:%M:%S") return (content, _fixPath(source), mtime) except (KeyError, FileNotFoundError): print("Error loading %s:%s; cannot find file in sublime-package" % (self.package_file(), override_file)) return None except UnicodeDecodeError: print("Error loading %s:%s; unable to decode file contents" % (self.package_file(), override_file)) return None
def addreferences(refsdict, articlelink): print ' ... from %s%s' % (articlelink, 'references') driver.get(articlelink + 'references') WebDriverWait(driver, 30).until( EC.presence_of_element_located( (By.CLASS_NAME, 'stats-reference-link-googleScholar'))) refpage = BeautifulSoup(driver.page_source, features="lxml") arefs = [] refsdict[articlelink] = [] reffile = codecs.EncodedFile( codecs.open('/tmp/ieee.%s.refs' % re.sub('\D', '', articlelink), mode='wb'), 'utf8') for div in refpage.find_all('div', attrs={'class': 'reference-container'}): for span in div.find_all('span', attrs={'class': 'number'}): for b in span.find_all('b'): refnumber = re.sub('\.', '', span.text.strip()) span.replace_with('[%s] ' % (refnumber)) for a in div.find_all('a', attrs={'class': 'stats-reference-link-crossRef'}): rdoi = re.sub('.*doi.org\/(10.*)', r'\1', a['href']) a.replace_with(', DOI: %s' % (rdoi)) for a in div.find_all( 'a', attrs={'class': 'stats-reference-link-googleScholar'}): a.replace_with('') ref = re.sub('[\n\t]', ' ', div.text.strip()) ref = re.sub(' +', ' ', ref) if not ref in arefs: #print ' + ', ref refsdict[articlelink].append([('x', ref)]) arefs.append(ref) reffile.write(ref.encode('utf-8') + '\n') reffile.close() print ' found %i references' % (len(arefs)) time.sleep(22)
def open_transcoded(filename, mode, source_enc=None, target_enc="utf8", errors="strict", fallback_enc="utf8", detect_buffer_size=50 * int(10e6)): """Open a file and transcode the content on the fly.""" # ensure we open the file in a binary mode binary_mode = re.sub(r"([^b+]+)(?:b)?([+]?)", r"\1b\2", mode) if source_enc is None: # pylint: disable=broad-except try: with open(filename, "rb") as file_obj: data = file_obj.read(detect_buffer_size) source_enc = chardet.detect(data)["encoding"] del data except Exception as error: print("""Warning: an error occured while trying to guess the """ """encoding of the file '{filename}', will assume """ """{fallback} encoding: {message}.""".format( filename=filename, fallback=fallback_enc, message=str(error)), file=sys.stderr) source_enc = fallback_enc with open(filename, binary_mode) as stream: with codecs.EncodedFile(stream, target_enc, file_encoding=source_enc, errors=errors) as recorder: yield recorder
def get_file(self, fname, encoding=None, errors=None): # type: (str, Optional[str], Optional[str]) -> Union[IO[bytes], IO[str]] """Return a file object corresponding to a given file name. If encoding is given, then the file object will return Unicode data; otherwise, it will return binary data. """ fname = DebPart.__normalize_member(fname) try: fobj = self.tgz().extractfile('./' + fname) except KeyError: # XXX python << 2.5 TarFile compatibility fobj = self.tgz().extractfile(fname) if fobj is None: raise DebError("File not found inside package") if encoding is not None: if sys.version >= '3': import io # pylint: disable=import-outside-toplevel if not hasattr(fobj, 'flush'): # XXX http://bugs.python.org/issue13815 fobj.flush = lambda: None # type: ignore return io.TextIOWrapper(fobj, encoding=encoding, errors=errors) # CRUFT: Python 2 only import codecs # pylint: disable=import-outside-toplevel if errors is None: errors = 'strict' return codecs.EncodedFile(fobj, encoding, errors=errors) return fobj
def main(): '''Process a testfile of key sequences, one sequence per line, to give test results: comma separated for each keystroke, one sequence per line''' import argparse, codecs, sys parser = argparse.ArgumentParser() parser.add_argument('file', help='Input LDML keyboard file') parser.add_argument('-t', '--testfile', help='File of key sequences, one per line') parser.add_argument('-o', '--outfile', help='Where to send results') parser.add_argument('-L', '--limit', type=int, default=1000000, help='Maximum trie size') args = parser.parse_args() kbd = Keyboard(args.file, limit=args.limit) if args.outfile: outfile = codecs.open(args.outfile, "w", encoding="utf-8") else: outfile = codecs.EncodedFile(sys.stdout, "utf-8") with open(args.testfile) as inf: for l in inf.readlines(): res = list(kbd.process_string(l)) outfile.write(u", ".join(map(unicode, res)) + u"\n") outfile.close()
def __load(self): if self.__filename is None: return if self.__stat is None: self.__stat = self.__load_stat() if self.__mimetype is None: self.__mimetype = self.__load_mimetype() if self.__encoding is not None: # Loading was already found, we're done return # lines and string depend on encoding try: try: stream = open(self.__filename, "rb") self.__encoding = self.__detect_encoding( stream, self.__filename, self.__mimetype) stream.seek(0) stream = codecs.EncodedFile(stream, self.__encoding) self.__lines = list(stream) self.__string = "".join(self.__lines) finally: stream.close() except IOError: # When there's a problem set the encoding to None and the rest too self.__encoding = None self.__lines = None self.__string = None # Also warn the log about it self.log.warn('failed to open file %s', self.filename)
def fix_stdouterr(): import codecs import locale def null_decode(input, errors='strict'): return input, len(input) encoding = locale.getpreferredencoding() # happens for e. g. LC_MESSAGES=*.UTF-8 LANG=C (forgetting to set LC_CTYPE # as well); force UTF-8 in that case if encoding == 'ANSI_X3.4-1968': encoding = 'UTF-8' sys.stdout = codecs.EncodedFile(sys.stdout, encoding) sys.stdout.decode = null_decode sys.stderr = codecs.EncodedFile(sys.stderr, encoding) sys.stderr.decode = null_decode
def get_file(self, fname, encoding=None, errors=None): """Return a file object corresponding to a given file name. If encoding is given, then the file object will return Unicode data; otherwise, it will return binary data. """ fname = DebPart.__normalize_member(fname) try: fobj = self.tgz().extractfile('./' + fname) except KeyError: # XXX python << 2.5 TarFile compatibility fobj = self.tgz().extractfile(fname) if encoding is not None: if sys.version >= '3': import io if not hasattr(fobj, 'flush'): # XXX http://bugs.python.org/issue13815 fobj.flush = lambda: None return io.TextIOWrapper(fobj, encoding=encoding, errors=errors) else: import codecs if errors is None: errors = 'strict' return codecs.EncodedFile(fobj, encoding, errors=errors) else: return fobj
def mainWeb(): global userWeb global passwWeb global GoogleSpreadsheetKey user = userWeb passw = passwWeb printToLog('Initializing Web doc parsing...') # Create client and spreadsheet objects gs = Client(user, passw) ss = Spreadsheet(GoogleSpreadsheetKey) # Request a file-like object containing the spreadsheet's contents csv_file = gs.download(ss) encoded_csv_file = codecs.EncodedFile(csv_file, data_encoding='utf-8') # Parse as CSV and print the rows num = 1 for row in csv.reader(map(bytes.decode, csv_file)): if row[0] != "Timestamp": printToLog("parsing Web doc row " + str(num) + "...") appendToContact(row[0], row[1], "", row[2], row[3], row[4], row[5], "Web") num += 1 num -= 1 printToLog(str(num) + " Web doc entries found.") print(str(num) + " Web doc entries found.")
def readFile(self): total_bytes = os.stat(self.filename).st_size # read input file content and convert to unicode to_code = 'utf-8' s = None for from_code in self.encodings: try: s = codecs.EncodedFile(open(self.filename, 'rb'), to_code, from_code).read() break except UnicodeDecodeError: pass if not s: info('can\'t read content from file %s !', self.filename) return #-# info('got %d byte(s) from file %s', len(s), self.filename) self.s = s.decode(to_code) self.idx_start = 0 self.idx_end = len(self.s) self.idx_cur = 0 info('got %d byte(s), %d char(s) from file %s', total_bytes, self.idx_end, self.filename) return total_bytes
def run(self, edit): view = self.view encoding = view.settings().get('force_encoding') if not encoding: encoding = view.settings().get('origin_encoding') file_name = view.file_name() if not encoding or encoding == 'UTF-8': encoding_cache.pop(file_name) return fp = None try: fp = open(file_name, 'rb') contents = codecs.EncodedFile(fp, encoding, 'UTF-8').read() except (LookupError, UnicodeEncodeError) as e: sublime.error_message( u'Can not convert file encoding of {0} to {1}, it was saved as UTF-8 instead:\n\n{2}' .format(os.path.basename(file_name), encoding, e.message)) return finally: if fp: fp.close() fp = open(file_name, 'wb') fp.write(contents) fp.close() encoding_cache.set(file_name, encoding) view.settings().set('prevent_detect', True) sublime.status_message('UTF8 -> {0}'.format(encoding))
def new_test(request): if request.method=='POST': form = TestFileForm(request.POST, request.FILES) if form.is_valid(): test_file = form.save(commit=False) path = str(test_file.file) if check_metric(codecs.EncodedFile(request.FILES['file'],"utf-8")): form.save() questions, answers = load_data_from_txt(path) id = TestFileModel.objects.get(name = test_file.name) for line in questions: new_question = TestQuestionModel() new_question.test_id = int(id.id) new_question.question_id = line[0] new_question.content = line[1] new_question.save() for line in answers: new_answer = QuestionAnswerModel() new_answer.test_id = int(id.id) new_answer.question_id = line[0] new_answer.content = line[1] new_answer.letter = line[2] new_answer.is_right = line[3] new_answer.save() messages.success(request,'Dodano nowy test.') return redirect('tests_home') else: messages.warning(request,'Niepoprawna metryczka.') else: form = TestFileForm() return render(request, 'tests/new_test.html', {'form':form})
def parse_check(p): logger.info("processing {0}".format(p)) parser.reset() with open(p, 'rU') as f: iso_wrapper = codecs.EncodedFile(f, 'iso-8859-1') s = iso_wrapper.read() parser.parse(s)
def consultas_cargas(request): resultados_pag = [] usuario = request.user.get_username() page = request.GET.get('page') if not page: # Se obtiene los atributos a mostrar (SELECT), cuando la consulta es simple attos_select = request.POST.getlist('attos') # Para la lectura del archivo CSV if request.POST and request.FILES: csvfile = request.FILES['csv_file'] csvfile.open() reader = csv.reader(codecs.EncodedFile(csvfile, "utf-8")) cedulas = " persona.nac = 'V' AND persona.ci IN (" for index, row in enumerate(reader): if index == 1: cedulas = cedulas + row[0] elif index > 1: cedulas = cedulas + ',' + row[0] cedulas = cedulas + ');' #print cedulas consulta = crearConsulta(attos_select, [], [u''], [u'']) #print consulta if 'WHERE' not in consulta: consulta_final = consulta[:-1] + ' WHERE' + cedulas else: consulta_final = consulta[:-1] + ' AND' + cedulas #print consulta_final resultados_consulta = ejecutarConsulta(consulta_final, False, request.user.get_username()) # Para el manejo de la sesion y los resultados sin variables globales sesion_usuario = "sesion_" + usuario if sesion_usuario in request.session: del request.session[sesion_usuario] request.session[sesion_usuario] = [attos_select, resultados_consulta] sesion_usuario = request.session[sesion_usuario] else: sesion_usuario = "sesion_" + usuario sesion_usuario = request.session[sesion_usuario] resultados_consulta = sesion_usuario[1] attos_select = sesion_usuario[0] # Paginacion. paginator = Paginator(resultados_consulta, 10) # Muestra 10 elementos por pagina. try: resultados_pag = paginator.page(page) except PageNotAnInteger: resultados_pag = paginator.page(1) except EmptyPage: resultados_pag = paginator.page(paginator.num_pages) context = { 'atributos': attos_select, 'resultados_pag': resultados_pag, 'total_rows': len(resultados_consulta) } return render(request, 'asistente_de_consultas/consultas.html', context)
def upload_file(request): if request.method == 'POST': form = UploadFileForm(request.POST, request.FILES) result = {} if form.is_valid(): #handle_uploaded_file(request.FILES['file']) csvfile = request.FILES['file'] dialect = csv.Sniffer().sniff( codecs.EncodedFile(csvfile, "utf-8").read(1024)) csvfile.open() csvreader = csv.reader(codecs.EncodedFile(csvfile, "utf-8"), delimiter=',', dialect=dialect) for row in csvreader: info = ','.join(row[3:]).replace('\\', '').replace('\"', '').replace( '{', '').replace('}', '') info_obj = {} last_obj = '' for obj in info.split(','): if len(obj.split(':')) == 1: info_obj[last_obj] = info_obj[last_obj] + ',' + obj continue key, item = obj.split(':') info_obj[key] = item last_obj = key info_str = "" for key in info_obj: info_str += key info_str += ":" info_str += info_obj[key] info_str += "--" #info_str.strip("-") table_row, created = Table.objects.get_or_create( object_id=row[0], timestamp=row[2]) table_row.object_type = row[1] table_row.info = info_str.strip('-') table_row.save() result = table_row print >> sys.stderr, table_row #row[:3], info_str.strip("-").split("--") csvfile.close() return HttpResponse("success") else: form = UploadFileForm() return HttpResponse('response')
def load(self, file): """Load properties from an open file stream""" unicodeFile = codecs.EncodedFile(file, 'latin-1') try: lines = unicodeFile.readlines() unicodeFile.close() except IOError, e: raise
def main(): wiki_path = sys.argv[1] export = MWExport(MoinWiki(wiki_path)) export.add_pages() out = codecs.EncodedFile(sys.stdout, 'utf-8') out.write(u"<?xml version=\"1.0\" encoding=\"utf-8\"?>\n") export.write(out)
def __init__(self, csv_file, encoding='utf-8', **kwargs): if encoding == 'utf-8-sig': # convert from utf-8-sig (= UTF8 with BOM) to plain utf-8 (without BOM): self.csv_file = codecs.EncodedFile(csv_file, 'utf-8', 'utf-8-sig') encoding = 'utf-8' else: self.csv_file = csv_file self.csv_reader = csv.reader(self.csv_file, **kwargs) self.encoding = encoding
def index(request): user = UserProfileInfo.objects.get(id=request.user.id) usernamee = request.user email = user.email if user.position != "manager": print(user.position) raise Http404('You are not allowed to access this link') if request.POST and request.FILES: csvfile = request.FILES['csv_file'] dialect = csv.Sniffer().sniff( codecs.EncodedFile(csvfile, "utf-8").read(1024)) csvfile.open() reader = csv.reader(codecs.EncodedFile(csvfile, "utf-8"), delimiter=',', dialect=dialect) return render(request, "my_view.html", locals())
def csvImportView(request): if request.method == 'POST': form = UploadFileForm(request.POST, request.FILES) if form.is_valid(): # file upload instance = UploadFileModel(file=request.FILES['file']) instance.save() # make array csvfile = instance.file dialect = csv.Sniffer().sniff(codecs.EncodedFile(csvfile, "utf-8").read(1024)) csvfile.open() data = csv.reader(codecs.EncodedFile(csvfile, "utf-8"), delimiter=',', dialect=dialect) data = list(data) return render(request, 'polls/csvImport.html', {'data10': data[1][0],'data11': data[1][1],'data12': data[1][2],'data20': data[2][0],'data21': data[2][1],'data22': data[2][2],'data30': data[3][0],'data31': data[3][1],'data32': data[3][2],'data40': data[4][0],'data41': data[4][1],'data42': data[4][2]}) else: form = UploadFileForm() return render(request, 'polls/csvImport.html', {'form': form})
def print_svg(self, filename, *args, **kwargs): if is_string_like(filename): fh_to_close = svgwriter = codecs.open(filename, 'w', 'utf-8') elif is_writable_file_like(filename): svgwriter = codecs.EncodedFile(filename, 'utf-8') fh_to_close = None else: raise ValueError("filename must be a path or a file-like object") return self._print_svg(filename, svgwriter, fh_to_close)
def parseAuthorCSVFile(inputFile): csvFile = inputFile dialect = csv.Sniffer().sniff(codecs.EncodedFile(csvFile, "utf-8").read(1024)) csvFile.open() # reader = csv.reader(codecs.EncodedFile(csvFile, "utf-8"), delimiter=',', dialect=dialect) reader = csv.reader(codecs.EncodedFile(csvFile, "utf-8"), delimiter=',', dialect='excel') rowResults = [] for index, row in enumerate(reader): rowResults.append(row) print(row) print(type(row)) if index == 5: break parsedResult = {} return parsedResult
def get_content(self): args = ['ps2txt', self._file_path] output = codecs.EncodedFile(os.tmpfile(), 'utf8', errors='ignore') return_code = subprocess.call(args=args, stdout=output) content = '' if return_code == 0: output.seek(0) content = output.read() output.close() return content.strip()
def index(request): studentList = [] # Handle file upload if request.method == 'POST': form = DocumentForm(request.POST, request.FILES) if form.is_valid(): newdoc = Document(docfile=request.FILES['docfile']) newdoc.save() dialect = csv.Sniffer().sniff( codecs.EncodedFile(newdoc.docfile, "utf-8").read(1024)) newdoc.docfile.open() reader = csv.reader(codecs.EncodedFile(newdoc.docfile, "utf-8"), delimiter=',', dialect=dialect) for row in reader: if (row[0] != 'Name' and row[0] != ''): student = Student(name=row[0], role=row[8], leadership_pref=row[9], sex=row[1], schedule=row[3]) student.save() # Redirect to the document list after POST # return HttpResponseRedirect(reverse('list')) else: form = DocumentForm() # A empty, unbound form # Load documents for the list page documents = Document.objects.all() students = Student.objects.all() studentList = list(students) if (request.GET.get('mybtn')): teams = randomTeams.create_random_teams( int(request.GET.get('mytextbox')), studentList) return render(request, 'suggestedteams/test.html', locals()) # Render list page with the documents and the form # {'documents': documents, 'form': form} return render(request, 'suggestedteams/test.html', locals())
def writesyntax(labelsyntax, syntax, mkvl): if mkvl.unicodemode: inputencoding = "unicode_internal" outputencoding = "utf_8_sig" else: inputencoding = locale.getlocale()[1] outputencoding = inputencoding with codecs.EncodedFile(codecs.open(syntax, "wb"), inputencoding, outputencoding) as f: for line in labelsyntax: f.write(line + "\n")
def readSession(filename): # read in a cinelerra session file (XML) print "read session %s"%filename if not os.path.exists(filename): __err("can't find cinelerra session file \"%s\"." % filename) srcraw =open(filename, 'r') srcfile=codecs.EncodedFile(srcraw,'utf-8', ENCODING) #srcfile=srcraw dom = xml.dom.minidom.parse(srcfile) return dom
def parseCSVFile(inputFile): """ Parse the uploaded CSV file assuming that the uploaded file is of Excel CSV format: allowing multilines in one cell, use double quote to include such cells Inputs: Django uploaded file Returns: list of lists (inner list represent each row) """ csvFile = inputFile dialect = csv.Sniffer().sniff(codecs.EncodedFile(csvFile, "utf-8").read(1024)) csvFile.open() # reader = csv.reader(codecs.EncodedFile(csvFile, "utf-8"), delimiter=',', dialect=dialect) reader = csv.reader(codecs.EncodedFile(csvFile, "utf-8"), delimiter=',', dialect='excel') rowResults = [row for row in reader] return rowResults