def print_svgz(self, filename, *args, **kwargs):
if is_string_like(filename):
gzipwriter = gzip.GzipFile(filename, 'w')
fh_to_close = svgwriter = codecs.EncodedFile(gzipwriter, 'utf-8')
elif is_writable_file_like(filename):
fh_to_close = gzipwriter = gzip.GzipFile(fileobj=filename,
mode='w')
svgwriter = codecs.EncodedFile(gzipwriter, 'utf-8')
else:
raise ValueError("filename must be a path or a file-like object")
return self._print_svg(filename, svgwriter, fh_to_close)
def import_csv(request):
logger = get_logger()
if request.method == "POST" and request.FILES:
csvfile = request.FILES['csv_file']
dialect = csv.Sniffer().sniff(
codecs.EncodedFile(csvfile, "utf-8").read(1024))
csvfile.open()
reader = csv.reader(codecs.EncodedFile(csvfile, "utf-8"),
delimiter='|',
dialect=dialect)
i = 0 # skip the definition fields
for row in reader:
if i != 0:
try:
AssetList.objects.get(ip_address=row[0])
except ObjectDoesNotExist:
a_list = AssetList()
a_list.ip_address = row[0]
a_list.tcp = row[1]
a_list.udp = row[2]
a_list.definition = row[3]
if row[4] == '':
pass
else:
try:
group = Group.objects.get(definition=row[4])
except ObjectDoesNotExist:
group = Group()
group.definition = row[4]
group.save()
except Exception as e:
error_message = "ERROR while getting whitelist details ::: {0} ::: line {1}"\
.format(str(e), sys.exc_info()[2].tb_lineno)
logger.send_log("error", error_message)
return render(request, "import_csv.html",
{"message": error_message})
a_list.group = group
a_list.created_by = request.user
a_list.save()
except Exception as e:
error_message = "ERROR while getting whitelist details ::: {0} ::: line {1}"\
.format(str(e), sys.exc_info()[2].tb_lineno)
logger.send_log("error", error_message)
return render(request, "import_csv.html",
{"message": error_message})
else:
i = 1
context = {"message": "Başarıyla Yüklendi"}
return render(request, "import_csv.html", context)
else:
return render(request, "import_csv.html")
def parse_sql(filename, pattern, func, lang):
"Rips columns specified with the regex out of the sql"
if filename.endswith('.gz'):
fh = codecs.EncodedFile(gzip.open(filename), data_encoding="utf-8")
else:
fh = codecs.open(filename, encoding="utf-8")
line = ""
count = 0
while True:
buff = fh.read(1024)
if not buff:
break
line += buff
rows = list(re.finditer(pattern, line))
for row in rows:
try:
func(row.groups(), lang)
except Exception, e:
print "uhoh: %s: %s w/ %s" % (func, e, row.groups())
# don't forget unconsumed bytes, need them for the next match
if len(rows) > 0:
line = line[rows[-1].end():]
def _get_packed_pkg_file_contents(self, override_file, as_list=True):
try:
with zipfile.ZipFile(self.package_file()) as zFile:
info = find_zip_entry(zFile, override_file)
file = codecs.EncodedFile(zFile.open(info, mode="rU"), "utf-8")
if as_list:
content = io.TextIOWrapper(file,
encoding="utf-8").readlines()
else:
content = io.TextIOWrapper(file, encoding="utf-8").read()
source = "Shipped Packages"
if self.installed_path is not None:
source = "Installed Packages"
source = os.path.join(source, self.name, override_file)
mtime = datetime(*info.date_time).strftime("%Y-%m-%d %H:%M:%S")
return (content, _fixPath(source), mtime)
except (KeyError, FileNotFoundError):
print("Error loading %s:%s; cannot find file in sublime-package" %
(self.package_file(), override_file))
return None
except UnicodeDecodeError:
print("Error loading %s:%s; unable to decode file contents" %
(self.package_file(), override_file))
return None
def addreferences(refsdict, articlelink):
print ' ... from %s%s' % (articlelink, 'references')
driver.get(articlelink + 'references')
WebDriverWait(driver, 30).until(
EC.presence_of_element_located(
(By.CLASS_NAME, 'stats-reference-link-googleScholar')))
refpage = BeautifulSoup(driver.page_source, features="lxml")
arefs = []
refsdict[articlelink] = []
reffile = codecs.EncodedFile(
codecs.open('/tmp/ieee.%s.refs' % re.sub('\D', '', articlelink),
mode='wb'), 'utf8')
for div in refpage.find_all('div', attrs={'class': 'reference-container'}):
for span in div.find_all('span', attrs={'class': 'number'}):
for b in span.find_all('b'):
refnumber = re.sub('\.', '', span.text.strip())
span.replace_with('[%s] ' % (refnumber))
for a in div.find_all('a',
attrs={'class':
'stats-reference-link-crossRef'}):
rdoi = re.sub('.*doi.org\/(10.*)', r'\1', a['href'])
a.replace_with(', DOI: %s' % (rdoi))
for a in div.find_all(
'a', attrs={'class': 'stats-reference-link-googleScholar'}):
a.replace_with('')
ref = re.sub('[\n\t]', ' ', div.text.strip())
ref = re.sub(' +', ' ', ref)
if not ref in arefs:
#print ' + ', ref
refsdict[articlelink].append([('x', ref)])
arefs.append(ref)
reffile.write(ref.encode('utf-8') + '\n')
reffile.close()
print ' found %i references' % (len(arefs))
time.sleep(22)
def open_transcoded(filename,
mode,
source_enc=None,
target_enc="utf8",
errors="strict",
fallback_enc="utf8",
detect_buffer_size=50 * int(10e6)):
"""Open a file and transcode the content on the fly."""
# ensure we open the file in a binary mode
binary_mode = re.sub(r"([^b+]+)(?:b)?([+]?)", r"\1b\2", mode)
if source_enc is None:
# pylint: disable=broad-except
try:
with open(filename, "rb") as file_obj:
data = file_obj.read(detect_buffer_size)
source_enc = chardet.detect(data)["encoding"]
del data
except Exception as error:
print("""Warning: an error occured while trying to guess the """
"""encoding of the file '{filename}', will assume """
"""{fallback} encoding: {message}.""".format(
filename=filename,
fallback=fallback_enc,
message=str(error)),
file=sys.stderr)
source_enc = fallback_enc
with open(filename, binary_mode) as stream:
with codecs.EncodedFile(stream,
target_enc,
file_encoding=source_enc,
errors=errors) as recorder:
yield recorder
def get_file(self, fname, encoding=None, errors=None):
# type: (str, Optional[str], Optional[str]) -> Union[IO[bytes], IO[str]]
"""Return a file object corresponding to a given file name.
If encoding is given, then the file object will return Unicode data;
otherwise, it will return binary data.
"""
fname = DebPart.__normalize_member(fname)
try:
fobj = self.tgz().extractfile('./' + fname)
except KeyError: # XXX python << 2.5 TarFile compatibility
fobj = self.tgz().extractfile(fname)
if fobj is None:
raise DebError("File not found inside package")
if encoding is not None:
if sys.version >= '3':
import io # pylint: disable=import-outside-toplevel
if not hasattr(fobj, 'flush'):
# XXX http://bugs.python.org/issue13815
fobj.flush = lambda: None # type: ignore
return io.TextIOWrapper(fobj, encoding=encoding, errors=errors)
# CRUFT: Python 2 only
import codecs # pylint: disable=import-outside-toplevel
if errors is None:
errors = 'strict'
return codecs.EncodedFile(fobj, encoding, errors=errors)
return fobj
def main():
'''Process a testfile of key sequences, one sequence per line,
to give test results: comma separated for each keystroke,
one sequence per line'''
import argparse, codecs, sys
parser = argparse.ArgumentParser()
parser.add_argument('file', help='Input LDML keyboard file')
parser.add_argument('-t',
'--testfile',
help='File of key sequences, one per line')
parser.add_argument('-o', '--outfile', help='Where to send results')
parser.add_argument('-L',
'--limit',
type=int,
default=1000000,
help='Maximum trie size')
args = parser.parse_args()
kbd = Keyboard(args.file, limit=args.limit)
if args.outfile:
outfile = codecs.open(args.outfile, "w", encoding="utf-8")
else:
outfile = codecs.EncodedFile(sys.stdout, "utf-8")
with open(args.testfile) as inf:
for l in inf.readlines():
res = list(kbd.process_string(l))
outfile.write(u", ".join(map(unicode, res)) + u"\n")
outfile.close()
def __load(self):
if self.__filename is None:
return
if self.__stat is None:
self.__stat = self.__load_stat()
if self.__mimetype is None:
self.__mimetype = self.__load_mimetype()
if self.__encoding is not None:
# Loading was already found, we're done
return
# lines and string depend on encoding
try:
try:
stream = open(self.__filename, "rb")
self.__encoding = self.__detect_encoding(
stream, self.__filename, self.__mimetype)
stream.seek(0)
stream = codecs.EncodedFile(stream, self.__encoding)
self.__lines = list(stream)
self.__string = "".join(self.__lines)
finally:
stream.close()
except IOError:
# When there's a problem set the encoding to None and the rest too
self.__encoding = None
self.__lines = None
self.__string = None
# Also warn the log about it
self.log.warn('failed to open file %s', self.filename)
def fix_stdouterr():
import codecs
import locale
def null_decode(input, errors='strict'):
return input, len(input)
encoding = locale.getpreferredencoding()
# happens for e. g. LC_MESSAGES=*.UTF-8 LANG=C (forgetting to set LC_CTYPE
# as well); force UTF-8 in that case
if encoding == 'ANSI_X3.4-1968':
encoding = 'UTF-8'
sys.stdout = codecs.EncodedFile(sys.stdout, encoding)
sys.stdout.decode = null_decode
sys.stderr = codecs.EncodedFile(sys.stderr, encoding)
sys.stderr.decode = null_decode
def get_file(self, fname, encoding=None, errors=None):
"""Return a file object corresponding to a given file name.
If encoding is given, then the file object will return Unicode data;
otherwise, it will return binary data.
"""
fname = DebPart.__normalize_member(fname)
try:
fobj = self.tgz().extractfile('./' + fname)
except KeyError: # XXX python << 2.5 TarFile compatibility
fobj = self.tgz().extractfile(fname)
if encoding is not None:
if sys.version >= '3':
import io
if not hasattr(fobj, 'flush'):
# XXX http://bugs.python.org/issue13815
fobj.flush = lambda: None
return io.TextIOWrapper(fobj, encoding=encoding, errors=errors)
else:
import codecs
if errors is None:
errors = 'strict'
return codecs.EncodedFile(fobj, encoding, errors=errors)
else:
return fobj
def mainWeb():
global userWeb
global passwWeb
global GoogleSpreadsheetKey
user = userWeb
passw = passwWeb
printToLog('Initializing Web doc parsing...')
# Create client and spreadsheet objects
gs = Client(user, passw)
ss = Spreadsheet(GoogleSpreadsheetKey)
# Request a file-like object containing the spreadsheet's contents
csv_file = gs.download(ss)
encoded_csv_file = codecs.EncodedFile(csv_file, data_encoding='utf-8')
# Parse as CSV and print the rows
num = 1
for row in csv.reader(map(bytes.decode, csv_file)):
if row[0] != "Timestamp":
printToLog("parsing Web doc row " + str(num) + "...")
appendToContact(row[0], row[1], "", row[2], row[3], row[4], row[5],
"Web")
num += 1
num -= 1
printToLog(str(num) + " Web doc entries found.")
print(str(num) + " Web doc entries found.")
def readFile(self):
total_bytes = os.stat(self.filename).st_size
# read input file content and convert to unicode
to_code = 'utf-8'
s = None
for from_code in self.encodings:
try:
s = codecs.EncodedFile(open(self.filename, 'rb'), to_code,
from_code).read()
break
except UnicodeDecodeError:
pass
if not s:
info('can\'t read content from file %s !', self.filename)
return
#-# info('got %d byte(s) from file %s', len(s), self.filename)
self.s = s.decode(to_code)
self.idx_start = 0
self.idx_end = len(self.s)
self.idx_cur = 0
info('got %d byte(s), %d char(s) from file %s', total_bytes,
self.idx_end, self.filename)
return total_bytes
def run(self, edit):
view = self.view
encoding = view.settings().get('force_encoding')
if not encoding:
encoding = view.settings().get('origin_encoding')
file_name = view.file_name()
if not encoding or encoding == 'UTF-8':
encoding_cache.pop(file_name)
return
fp = None
try:
fp = open(file_name, 'rb')
contents = codecs.EncodedFile(fp, encoding, 'UTF-8').read()
except (LookupError, UnicodeEncodeError) as e:
sublime.error_message(
u'Can not convert file encoding of {0} to {1}, it was saved as UTF-8 instead:\n\n{2}'
.format(os.path.basename(file_name), encoding, e.message))
return
finally:
if fp:
fp.close()
fp = open(file_name, 'wb')
fp.write(contents)
fp.close()
encoding_cache.set(file_name, encoding)
view.settings().set('prevent_detect', True)
sublime.status_message('UTF8 -> {0}'.format(encoding))
def new_test(request):
if request.method=='POST':
form = TestFileForm(request.POST, request.FILES)
if form.is_valid():
test_file = form.save(commit=False)
path = str(test_file.file)
if check_metric(codecs.EncodedFile(request.FILES['file'],"utf-8")):
form.save()
questions, answers = load_data_from_txt(path)
id = TestFileModel.objects.get(name = test_file.name)
for line in questions:
new_question = TestQuestionModel()
new_question.test_id = int(id.id)
new_question.question_id = line[0]
new_question.content = line[1]
new_question.save()
for line in answers:
new_answer = QuestionAnswerModel()
new_answer.test_id = int(id.id)
new_answer.question_id = line[0]
new_answer.content = line[1]
new_answer.letter = line[2]
new_answer.is_right = line[3]
new_answer.save()
messages.success(request,'Dodano nowy test.')
return redirect('tests_home')
else:
messages.warning(request,'Niepoprawna metryczka.')
else:
form = TestFileForm()
return render(request, 'tests/new_test.html', {'form':form})
def parse_check(p):
logger.info("processing {0}".format(p))
parser.reset()
with open(p, 'rU') as f:
iso_wrapper = codecs.EncodedFile(f, 'iso-8859-1')
s = iso_wrapper.read()
parser.parse(s)
def consultas_cargas(request):
resultados_pag = []
usuario = request.user.get_username()
page = request.GET.get('page')
if not page:
# Se obtiene los atributos a mostrar (SELECT), cuando la consulta es simple
attos_select = request.POST.getlist('attos')
# Para la lectura del archivo CSV
if request.POST and request.FILES:
csvfile = request.FILES['csv_file']
csvfile.open()
reader = csv.reader(codecs.EncodedFile(csvfile, "utf-8"))
cedulas = " persona.nac = 'V' AND persona.ci IN ("
for index, row in enumerate(reader):
if index == 1:
cedulas = cedulas + row[0]
elif index > 1:
cedulas = cedulas + ',' + row[0]
cedulas = cedulas + ');'
#print cedulas
consulta = crearConsulta(attos_select, [], [u''], [u''])
#print consulta
if 'WHERE' not in consulta:
consulta_final = consulta[:-1] + ' WHERE' + cedulas
else:
consulta_final = consulta[:-1] + ' AND' + cedulas
#print consulta_final
resultados_consulta = ejecutarConsulta(consulta_final, False,
request.user.get_username())
# Para el manejo de la sesion y los resultados sin variables globales
sesion_usuario = "sesion_" + usuario
if sesion_usuario in request.session:
del request.session[sesion_usuario]
request.session[sesion_usuario] = [attos_select, resultados_consulta]
sesion_usuario = request.session[sesion_usuario]
else:
sesion_usuario = "sesion_" + usuario
sesion_usuario = request.session[sesion_usuario]
resultados_consulta = sesion_usuario[1]
attos_select = sesion_usuario[0]
# Paginacion.
paginator = Paginator(resultados_consulta,
10) # Muestra 10 elementos por pagina.
try:
resultados_pag = paginator.page(page)
except PageNotAnInteger:
resultados_pag = paginator.page(1)
except EmptyPage:
resultados_pag = paginator.page(paginator.num_pages)
context = {
'atributos': attos_select,
'resultados_pag': resultados_pag,
'total_rows': len(resultados_consulta)
}
return render(request, 'asistente_de_consultas/consultas.html', context)
def upload_file(request):
if request.method == 'POST':
form = UploadFileForm(request.POST, request.FILES)
result = {}
if form.is_valid():
#handle_uploaded_file(request.FILES['file'])
csvfile = request.FILES['file']
dialect = csv.Sniffer().sniff(
codecs.EncodedFile(csvfile, "utf-8").read(1024))
csvfile.open()
csvreader = csv.reader(codecs.EncodedFile(csvfile, "utf-8"),
delimiter=',',
dialect=dialect)
for row in csvreader:
info = ','.join(row[3:]).replace('\\',
'').replace('\"', '').replace(
'{', '').replace('}', '')
info_obj = {}
last_obj = ''
for obj in info.split(','):
if len(obj.split(':')) == 1:
info_obj[last_obj] = info_obj[last_obj] + ',' + obj
continue
key, item = obj.split(':')
info_obj[key] = item
last_obj = key
info_str = ""
for key in info_obj:
info_str += key
info_str += ":"
info_str += info_obj[key]
info_str += "--"
#info_str.strip("-")
table_row, created = Table.objects.get_or_create(
object_id=row[0], timestamp=row[2])
table_row.object_type = row[1]
table_row.info = info_str.strip('-')
table_row.save()
result = table_row
print >> sys.stderr, table_row #row[:3], info_str.strip("-").split("--")
csvfile.close()
return HttpResponse("success")
else:
form = UploadFileForm()
return HttpResponse('response')
def load(self, file):
"""Load properties from an open file stream"""
unicodeFile = codecs.EncodedFile(file, 'latin-1')
try:
lines = unicodeFile.readlines()
unicodeFile.close()
except IOError, e:
raise
def main():
wiki_path = sys.argv[1]
export = MWExport(MoinWiki(wiki_path))
export.add_pages()
out = codecs.EncodedFile(sys.stdout, 'utf-8')
out.write(u"<?xml version=\"1.0\" encoding=\"utf-8\"?>\n")
export.write(out)
def __init__(self, csv_file, encoding='utf-8', **kwargs):
if encoding == 'utf-8-sig':
# convert from utf-8-sig (= UTF8 with BOM) to plain utf-8 (without BOM):
self.csv_file = codecs.EncodedFile(csv_file, 'utf-8', 'utf-8-sig')
encoding = 'utf-8'
else:
self.csv_file = csv_file
self.csv_reader = csv.reader(self.csv_file, **kwargs)
self.encoding = encoding
def index(request):
user = UserProfileInfo.objects.get(id=request.user.id)
usernamee = request.user
email = user.email
if user.position != "manager":
print(user.position)
raise Http404('You are not allowed to access this link')
if request.POST and request.FILES:
csvfile = request.FILES['csv_file']
dialect = csv.Sniffer().sniff(
codecs.EncodedFile(csvfile, "utf-8").read(1024))
csvfile.open()
reader = csv.reader(codecs.EncodedFile(csvfile, "utf-8"),
delimiter=',',
dialect=dialect)
return render(request, "my_view.html", locals())
def csvImportView(request):
if request.method == 'POST':
form = UploadFileForm(request.POST, request.FILES)
if form.is_valid():
# file upload
instance = UploadFileModel(file=request.FILES['file'])
instance.save()
# make array
csvfile = instance.file
dialect = csv.Sniffer().sniff(codecs.EncodedFile(csvfile, "utf-8").read(1024))
csvfile.open()
data = csv.reader(codecs.EncodedFile(csvfile, "utf-8"), delimiter=',', dialect=dialect)
data = list(data)
return render(request, 'polls/csvImport.html', {'data10': data[1][0],'data11': data[1][1],'data12': data[1][2],'data20': data[2][0],'data21': data[2][1],'data22': data[2][2],'data30': data[3][0],'data31': data[3][1],'data32': data[3][2],'data40': data[4][0],'data41': data[4][1],'data42': data[4][2]})
else:
form = UploadFileForm()
return render(request, 'polls/csvImport.html', {'form': form})
def print_svg(self, filename, *args, **kwargs):
if is_string_like(filename):
fh_to_close = svgwriter = codecs.open(filename, 'w', 'utf-8')
elif is_writable_file_like(filename):
svgwriter = codecs.EncodedFile(filename, 'utf-8')
fh_to_close = None
else:
raise ValueError("filename must be a path or a file-like object")
return self._print_svg(filename, svgwriter, fh_to_close)
def parseAuthorCSVFile(inputFile):
csvFile = inputFile
dialect = csv.Sniffer().sniff(codecs.EncodedFile(csvFile, "utf-8").read(1024))
csvFile.open()
# reader = csv.reader(codecs.EncodedFile(csvFile, "utf-8"), delimiter=',', dialect=dialect)
reader = csv.reader(codecs.EncodedFile(csvFile, "utf-8"), delimiter=',', dialect='excel')
rowResults = []
for index, row in enumerate(reader):
rowResults.append(row)
print(row)
print(type(row))
if index == 5:
break
parsedResult = {}
return parsedResult
def get_content(self):
args = ['ps2txt', self._file_path]
output = codecs.EncodedFile(os.tmpfile(), 'utf8', errors='ignore')
return_code = subprocess.call(args=args, stdout=output)
content = ''
if return_code == 0:
output.seek(0)
content = output.read()
output.close()
return content.strip()
def index(request):
studentList = []
# Handle file upload
if request.method == 'POST':
form = DocumentForm(request.POST, request.FILES)
if form.is_valid():
newdoc = Document(docfile=request.FILES['docfile'])
newdoc.save()
dialect = csv.Sniffer().sniff(
codecs.EncodedFile(newdoc.docfile, "utf-8").read(1024))
newdoc.docfile.open()
reader = csv.reader(codecs.EncodedFile(newdoc.docfile, "utf-8"),
delimiter=',',
dialect=dialect)
for row in reader:
if (row[0] != 'Name' and row[0] != ''):
student = Student(name=row[0],
role=row[8],
leadership_pref=row[9],
sex=row[1],
schedule=row[3])
student.save()
# Redirect to the document list after POST
# return HttpResponseRedirect(reverse('list'))
else:
form = DocumentForm() # A empty, unbound form
# Load documents for the list page
documents = Document.objects.all()
students = Student.objects.all()
studentList = list(students)
if (request.GET.get('mybtn')):
teams = randomTeams.create_random_teams(
int(request.GET.get('mytextbox')), studentList)
return render(request, 'suggestedteams/test.html', locals())
# Render list page with the documents and the form
# {'documents': documents, 'form': form}
return render(request, 'suggestedteams/test.html', locals())
def writesyntax(labelsyntax, syntax, mkvl):
if mkvl.unicodemode:
inputencoding = "unicode_internal"
outputencoding = "utf_8_sig"
else:
inputencoding = locale.getlocale()[1]
outputencoding = inputencoding
with codecs.EncodedFile(codecs.open(syntax, "wb"), inputencoding, outputencoding) as f:
for line in labelsyntax:
f.write(line + "\n")
def readSession(filename):
# read in a cinelerra session file (XML)
print "read session %s"%filename
if not os.path.exists(filename):
__err("can't find cinelerra session file \"%s\"." % filename)
srcraw =open(filename, 'r')
srcfile=codecs.EncodedFile(srcraw,'utf-8', ENCODING)
#srcfile=srcraw
dom = xml.dom.minidom.parse(srcfile)
return dom
def parseCSVFile(inputFile): """ Parse the uploaded CSV file assuming that the uploaded file is of Excel CSV format: allowing multilines in one cell, use double quote to include such cells Inputs: Django uploaded file Returns: list of lists (inner list represent each row) """ csvFile = inputFile dialect = csv.Sniffer().sniff(codecs.EncodedFile(csvFile, "utf-8").read(1024)) csvFile.open() # reader = csv.reader(codecs.EncodedFile(csvFile, "utf-8"), delimiter=',', dialect=dialect) reader = csv.reader(codecs.EncodedFile(csvFile, "utf-8"), delimiter=',', dialect='excel') rowResults = [row for row in reader] return rowResults
