def getOCR(): if request.method == 'GET': # get ImageUrl and langcode from Form imageUrl = request.args.get('imageurl', '') # Hey why get langcode?, It is not using anyywhere # Well, It will use in future. This is important variable langcode = request.args.get('langcode', '') # Get api parameter value isAPI = request.args.get('api', '') # Create a unique file name based on time currentTime = str(datetime.datetime.now()) getfileName = currentTime.replace(':', '_') getfileName = getfileName.replace(' ', '_') # Download the Image File r = requests.get(imageUrl, allow_redirects=True) fileName = getfileName + "." + r.headers.get('content-type').replace( 'image/', '') open("ocr/" + fileName, 'wb').write(r.content) # Google Drive API Setup SCOPES = 'https://www.googleapis.com/auth/drive.file' store = file.Storage('token.json') creds = store.get() if not creds or creds.invalid: flow = client.flow_from_clientsecrets('client_secret.json', SCOPES) creds = tools.run_flow(flow, store) service = build('drive', 'v3', http=creds.authorize(Http())) # Upload the file on Goggle Drive folder_id = '1bUOQUn-ZYTpUYcMxD8myU9nKj2Vywtqo' mime = 'application/vnd.google-apps.document' file_metadata = { 'name': fileName, 'mimeType': mime, 'parents': [folder_id] } media = MediaFileUpload("ocr/" + fileName, mimetype=mime) Imgfile = service.files().create(body=file_metadata, media_body=media, fields='id').execute() # Delete Img file locally. os.remove("ocr/" + fileName) # Download the file in txt format from Google Drive getTxt = service.files().export_media(fileId=Imgfile.get('id'), mimeType='text/plain') fh = io.FileIO("ocr/" + getfileName + ".txt", 'wb') downloader = MediaIoBaseDownload(fh, getTxt) downloader.next_chunk() # Read the file OCRtext = io.open("ocr/" + getfileName + ".txt", mode="r", encoding="utf-8").read() # Check if it is api request if "True" in isAPI: return jsonify({"text": OCRtext}) # Return the html page with OCR data return render_template('getOCR.html', imageUrl=imageUrl, OCRtext=OCRtext)
def writeToFile(fileData): with io.FileIO('./test.jpg', 'w') as file: file.write(fileData)
def get_readable_fileobj(name_or_obj, encoding=None, cache=False, show_progress=True, remote_timeout=None): """ Given a filename, pathlib.Path object or a readable file-like object, return a context manager that yields a readable file-like object. This supports passing filenames, URLs, and readable file-like objects, any of which can be compressed in gzip, bzip2 or lzma (xz) if the appropriate compression libraries are provided by the Python installation. Notes ----- This function is a context manager, and should be used for example as:: with get_readable_fileobj('file.dat') as f: contents = f.read() Parameters ---------- name_or_obj : str or file-like object The filename of the file to access (if given as a string), or the file-like object to access. If a file-like object, it must be opened in binary mode. encoding : str, optional When `None` (default), returns a file-like object with a ``read`` method that returns `str` (``unicode``) objects, using `locale.getpreferredencoding` as an encoding. This matches the default behavior of the built-in `open` when no ``mode`` argument is provided. When ``'binary'``, returns a file-like object where its ``read`` method returns `bytes` objects. When another string, it is the name of an encoding, and the file-like object's ``read`` method will return `str` (``unicode``) objects, decoded from binary using the given encoding. cache : bool, optional Whether to cache the contents of remote URLs. show_progress : bool, optional Whether to display a progress bar if the file is downloaded from a remote server. Default is `True`. remote_timeout : float Timeout for remote requests in seconds (default is the configurable `astropy.utils.data.Conf.remote_timeout`, which is 3s by default) Returns ------- file : readable file-like object """ # close_fds is a list of file handles created by this function # that need to be closed. We don't want to always just close the # returned file handle, because it may simply be the file handle # passed in. In that case it is not the responsibility of this # function to close it: doing so could result in a "double close" # and an "invalid file descriptor" exception. PATH_TYPES = (str, pathlib.Path) close_fds = [] delete_fds = [] if remote_timeout is None: # use configfile default remote_timeout = conf.remote_timeout # Get a file object to the content if isinstance(name_or_obj, PATH_TYPES): # name_or_obj could be a Path object if pathlib is available name_or_obj = str(name_or_obj) is_url = _is_url(name_or_obj) if is_url: name_or_obj = download_file( name_or_obj, cache=cache, show_progress=show_progress, timeout=remote_timeout) fileobj = io.FileIO(name_or_obj, 'r') if is_url and not cache: delete_fds.append(fileobj) close_fds.append(fileobj) else: fileobj = name_or_obj # Check if the file object supports random access, and if not, # then wrap it in a BytesIO buffer. It would be nicer to use a # BufferedReader to avoid reading loading the whole file first, # but that is not compatible with streams or urllib2.urlopen # objects on Python 2.x. if not hasattr(fileobj, 'seek'): fileobj = io.BytesIO(fileobj.read()) # Now read enough bytes to look at signature signature = fileobj.read(4) fileobj.seek(0) if signature[:3] == b'\x1f\x8b\x08': # gzip import struct try: import gzip fileobj_new = gzip.GzipFile(fileobj=fileobj, mode='rb') fileobj_new.read(1) # need to check that the file is really gzip except (OSError, EOFError, struct.error): # invalid gzip file fileobj.seek(0) fileobj_new.close() else: fileobj_new.seek(0) fileobj = fileobj_new elif signature[:3] == b'BZh': # bzip2 try: import bz2 except ImportError: for fd in close_fds: fd.close() raise ValueError( ".bz2 format files are not supported since the Python " "interpreter does not include the bz2 module") try: # bz2.BZ2File does not support file objects, only filenames, so we # need to write the data to a temporary file with NamedTemporaryFile("wb", delete=False) as tmp: tmp.write(fileobj.read()) tmp.close() fileobj_new = bz2.BZ2File(tmp.name, mode='rb') fileobj_new.read(1) # need to check that the file is really bzip2 except OSError: # invalid bzip2 file fileobj.seek(0) fileobj_new.close() # raise else: fileobj_new.seek(0) close_fds.append(fileobj_new) fileobj = fileobj_new elif signature[:3] == b'\xfd7z': # xz try: import lzma fileobj_new = lzma.LZMAFile(fileobj, mode='rb') fileobj_new.read(1) # need to check that the file is really xz except ImportError: for fd in close_fds: fd.close() raise ValueError( ".xz format files are not supported since the Python " "interpreter does not include the lzma module.") except (OSError, EOFError) as e: # invalid xz file fileobj.seek(0) fileobj_new.close() # should we propagate this to the caller to signal bad content? # raise ValueError(e) else: fileobj_new.seek(0) fileobj = fileobj_new # By this point, we have a file, io.FileIO, gzip.GzipFile, bz2.BZ2File # or lzma.LZMAFile instance opened in binary mode (that is, read # returns bytes). Now we need to, if requested, wrap it in a # io.TextIOWrapper so read will return unicode based on the # encoding parameter. needs_textio_wrapper = encoding != 'binary' if needs_textio_wrapper: # A bz2.BZ2File can not be wrapped by a TextIOWrapper, # so we decompress it to a temporary file and then # return a handle to that. try: import bz2 except ImportError: pass else: if isinstance(fileobj, bz2.BZ2File): tmp = NamedTemporaryFile("wb", delete=False) data = fileobj.read() tmp.write(data) tmp.close() delete_fds.append(tmp) fileobj = io.FileIO(tmp.name, 'r') close_fds.append(fileobj) fileobj = io.BufferedReader(fileobj) fileobj = io.TextIOWrapper(fileobj, encoding=encoding) # Ensure that file is at the start - io.FileIO will for # example not always be at the start: # >>> import io # >>> f = open('test.fits', 'rb') # >>> f.read(4) # 'SIMP' # >>> f.seek(0) # >>> fileobj = io.FileIO(f.fileno()) # >>> fileobj.tell() # 4096L fileobj.seek(0) try: yield fileobj finally: for fd in close_fds: fd.close() for fd in delete_fds: os.remove(fd.name)
def create_malpdf(filename, host): print("[*] Starting Process.. [*]") with io.FileIO(filename, "w") as file: file.write(''' %PDF-1.7 1 0 obj <</Type/Catalog/Pages 2 0 R>> endobj 2 0 obj <</Type/Pages/Kids[3 0 R]/Count 1>> endobj 3 0 obj <</Type/Page/Parent 2 0 R/MediaBox[0 0 612 792]/Resources<<>>>> endobj xref 0 4 0000000000 65535 f 0000000015 00000 n 0000000060 00000 n 0000000111 00000 n trailer <</Size 4/Root 1 0 R>> startxref 190 3 0 obj << /Type /Page /Contents 4 0 R /AA << /O << /F (''' + host + '''test) /D [ 0 /Fit] /S /GoToE >> >> /Parent 2 0 R /Resources << /Font << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >> >> endobj 4 0 obj<< /Length 100>> stream BT /TI_0 1 Tf 14 0 0 14 10.000 753.976 Tm 0.0 0.0 0.0 rg (PDF Document) Tj ET endstream endobj trailer << /Root 1 0 R >> %%EOF ''')
def download(self, fileID, path, skipHash=False ): #Downloads a file. If a folder, pass to downloadFolder() #if '-nohash' in args: # skipHash = True if path[-1] != '/': path += '/' if not os.path.exists(path): os.makedirs(path) metaData = self.service.files().get(fileId=fileID, fields='name, mimeType, size', supportsAllDrives=True).execute() mimetype = metaData['mimeType'] name = metaData['name'] if 'google-apps' in mimetype: if mimetype == 'application/vnd.google-apps.folder': #If file is folder self.downloadFolder(fileID, path, skipHash=skipHash) return True else: #Skip Google files, as they require their own download process print( 'ERROR: Unable to download Google files (docs, sheets, etc)' ) return False size = metaData['size'] print('Downloading file \"{}\"'.format(name)) if size == '0': with open(path + name, 'w') as f: pass else: request = self.service.files().get_media(fileId=fileID) fh = io.FileIO(path + name, 'wb') self.downloader = MediaIoBaseDownload(fh, request) self.done = False errorCounter = 1 maxErrors = 5 done = False while done is False: while True: try: status, done = self.downloader.next_chunk() break except HttpError as e: #Wait 2 seconds then retry download request. Need to implement exponential backoff decorator status_code = e.resp['status'] print('HttpError {}: Retrying request ({}/{}).'.format( status_code, errorCounter, maxErrors)) errorCounter += 1 sleep(2) if errorCounter > maxErrors: print('Skipping file \"{}\"...\n'.format(name)) done = True return False if not skipHash: print('checking hash...') if self.getMd5(fileID) != self.calculateMd5(path + name): print('ERROR: Checksums do not match') print('File \"{}\" downloaded\n'.format(path + name))
'Prunes any samples shorter than the min duration (given in seconds, default 1)' ) parser.add_argument( '--max_duration', default=15, type=int, help= 'Prunes any samples longer than the max duration (given in seconds, default 15)' ) parser.add_argument('--output_path', default='merged_manifest.csv', help='Output path to merged manifest') args = parser.parse_args() file_paths = [] for file in os.listdir(args.merge_dir): if file.endswith(".csv"): with open(os.path.join(args.merge_dir, file), 'r') as fh: file_paths += fh.readlines() file_paths = [file_path.split(',')[0] for file_path in file_paths] file_paths = order_and_prune_files(file_paths, args.min_duration, args.max_duration) with io.FileIO(args.output_path, "w") as file: for wav_path in tqdm(file_paths, total=len(file_paths)): transcript_path = wav_path.replace('/wav/', '/txt/').replace('.wav', '.txt') sample = os.path.abspath(wav_path) + ',' + os.path.abspath( transcript_path) + '\n' file.write(sample.encode('utf-8'))
flow = google_auth_oauthlib.flow.InstalledAppFlow.from_client_secrets_file( client_secrets_file, scopes) credentials = flow.run_console() youtube = googleapiclient.discovery.build(api_service_name, api_version, credentials=credentials) # Grab caption ID from video request = youtube.captions().list(videoId=youtubevideo, part="snippet") captionid = '' response = request.execute() for i in response['items']: if i['snippet']['language'] == 'en': captionid = i['id'] # Download caption file. request = youtube.captions().download(id=captionid) fh = io.FileIO('captions_EN.svb', 'wb') download = MediaIoBaseDownload(fh, request) complete = False while not complete: status, complete = download.next_chunk() # Read caption file with open('captions_EN.svb', "r", encoding='utf-8') as myfile: input = myfile.readlines() # Translate fucntion def translate(langout): deepl_url = 'https://api.deepl.com/v2/translate?' data = { 'auth_key': deeplkey, 'text': input,
def makeChunksEXT4FS(self, name): """ Generate one or more .chunks files assuming an EXT4 FS """ os.chdir(os.path.dirname(name)) name = os.path.basename(name) baseName = name.rpartition(".")[0] + "_" sliceName = name.rpartition(".")[0].encode("utf8") # Check length before closing eof = self.file.seek(0, io.SEEK_END) # Alas, ext2simg can't take the image as stdin self.file.close() sparse = EXT4SparseFile(name, 1 << self.blockShift) # sigh, Python 2 hack for some variables class nl: yuck = "truly" nl.current = 0 nl.targetAddr = self.startLBA nl.trimCount = 0 # local function not used by anyone else def complete(): # nonlocal zlen, md5, current, targetAddr, trimCount zdata = zobj.flush(zlib.Z_FINISH) nl.zlen += len(zdata) out.write(zdata) nl.md5 = nl.md5.digest() out.seek(0, io.SEEK_SET) values = { 'sliceName': sliceName, 'chunkName': chunkName, 'targetSize': dataBlocks << self.blockShift, 'dataSize': nl.zlen, 'md5': nl.md5, 'targetAddr': nl.targetAddr, 'trimCount': nl.trimCount, 'crc32': crc & 0xFFFFFFFF, 'dev': self.dev, } nl.header = self.packdict(values) out.write(nl.header) out.close() nl.current += nl.trimCount << self.blockShift nl.targetAddr = self.startLBA + (nl.current >> self.blockShift) print("({:d} empty blocks)".format(nl.trimCount - dataBlocks)) nl.trimCount = 0 for chunk in sparse: if chunk.type == EXT4SparseChunk.typeRaw or chunk.type == EXT4SparseChunk.typeFill: if nl.trimCount: complete() dataBlocks = chunk.remaining >> self.blockShift nl.trimCount += dataBlocks nl.md5 = hashlib.md5() crc = crc32(b"") zobj = zlib.compressobj(1) chunkName = baseName + str(nl.targetAddr) + ".bin" out = io.FileIO(chunkName + ".chunk", "wb") sys.stdout.write("[+] Compressing {:s} to {:s} ".format( name, chunkName)) chunkName = chunkName.encode("utf8") out.seek(self._dz_length, io.SEEK_SET) nl.zlen = 0 for buf in chunk: nl.md5.update(buf) crc = crc32(buf, crc) zdata = zobj.compress(buf) nl.zlen += len(zdata) out.write(zdata) elif chunk.type == EXT4SparseChunk.typeDontCare: # check for EOF, lastWipe overrides if sparse.chunkCount == 0: nl.trimCount = self.lastWipe - nl.targetAddr else: nl.trimCount += chunk.remaining >> self.blockShift complete() elif chunk.type == EXT4SparseChunk.typeCrc32: pass else: print("[!] Error: unknown chunk, type=0x{:04X}".format( chunk.type), file=sys.stderr) sys.exit(64) if nl.trimCount: nl.trimCount = self.lastWipe - nl.targetAddr complete() print("[+] done\n")
def makeChunksProbe(self, name): """ Generate one or more .chunks files for the named file """ os.chdir(os.path.dirname(name)) name = os.path.basename(name) baseName = name.rpartition(".")[0] + "_" sliceName = name.rpartition(".")[0].encode("utf8") current = 0 targetAddr = self.startLBA eof = self.file.seek(0, io.SEEK_END) self.file.seek(0, io.SEEK_SET) # emulate characteristics of LG's tool, always find a block at start readSize = self.blockSize << 10 md5 = hashlib.md5() crc = crc32(b"") zobj = zlib.compressobj(1) self.file.seek(current, io.SEEK_SET) chunkName = baseName + str(targetAddr) + ".bin" out = io.FileIO(chunkName + ".chunk", "wb") sys.stdout.write("[+] Compressing {:s} to {:s} ".format( name, chunkName)) chunkName = chunkName.encode("utf8") out.seek(self._dz_length, io.SEEK_SET) zlen = 0 wipeData = readSize dataCount = readSize buf = self.file.read(readSize) while len(buf.lstrip(b'\x00')) == 0 and current < eof: md5.update(buf) crc = crc32(buf, crc) zdata = zobj.compress(buf) zlen += len(zdata) out.write(zdata) wipeData += readSize dataCount += readSize current += readSize buf = self.file.read(readSize) while current < eof: while len(buf.lstrip(b'\x00')) != 0 and current < eof: md5.update(buf) crc = crc32(buf, crc) zdata = zobj.compress(buf) zlen += len(zdata) out.write(zdata) wipeData += readSize dataCount += readSize current += readSize buf = self.file.read(readSize) zdata = zobj.flush(zlib.Z_FINISH) zlen += len(zdata) out.write(zdata) md5 = md5.digest() while len(buf.lstrip(b'\x00')) == 0 and current < eof: wipeData += readSize current += readSize buf = self.file.read(readSize) print("({:d} empty blocks)".format((wipeData - dataCount) >> self.blockShift)) out.seek(0, io.SEEK_SET) values = { 'sliceName': sliceName, 'chunkName': chunkName, 'targetSize': dataCount, 'dataSize': zlen, 'md5': md5, 'targetAddr': targetAddr, 'trimCount': wipeData >> self.blockShift, 'crc32': crc & 0xFFFFFFFF, 'dev': self.dev, } header = self.packdict(values) out.write(header) out.close() targetAddr = self.startLBA + (current >> self.blockShift) if current < eof: md5 = hashlib.md5() crc = crc32(b"") zobj = zlib.compressobj(1) chunkName = baseName + str(targetAddr) + ".bin" out = io.FileIO(chunkName + ".chunk", "wb") sys.stdout.write("[+] Compressing {:s} to {:s} ".format( name, chunkName)) chunkName = chunkName.encode("utf8") out.seek(self._dz_length, io.SEEK_SET) zlen = 0 wipeData = readSize dataCount = readSize print("[+] done\n")
def pack_lin(self, dir): # Collect our files. file_list = sorted(list_all_files(dir)) txt = [ filename for filename in file_list if os.path.splitext(filename)[1].lower() == ".txt" ] wrd = [ filename for filename in file_list if os.path.splitext(filename)[1].lower() == ".wrd" ] py = [ filename for filename in file_list if os.path.splitext(filename)[1].lower() == ".py" ] # If there are more than one for whatever reason, just take the first. # We only have use for a single wrd or python file. wrd = wrd[0] if wrd else None py = py[0] if py else None # Prepare our temporary output directory. temp_dir = tempfile.mkdtemp(prefix="sdse-") # Where we're outputting our wrd file, regardless of whether it's a python # file or a raw binary data file. wrd_dst = os.path.join(temp_dir, "0.scp.wrd") if py: # _LOGGER.info("Compiling %s to binary." % py) try: wrd_file = WrdFile(py) except: _LOGGER.warning( "%s failed to compile. Parsing wrd file instead. Exception info:\n%s" % (py, traceback.format_exc())) shutil.copy(wrd, wrd_dst) else: # If we succeeded in loading the python file, compile it to binary. # wrd_file.save_bin(wrd) wrd_file.save_bin(wrd_dst) else: shutil.copy(wrd, wrd_dst) # Pack the text files in-place to save us a bunch of copying # and then move it to the tmp directory with the wrd file. if txt: with io.FileIO(os.path.join(temp_dir, "1.dat"), "w") as h: self.pack_dir(dir, h, file_list=txt) # Then pack it like normal. data = io.BytesIO() with io.BufferedWriter(data) as h: self.pack_dir(temp_dir, h) h.flush() data = data.getvalue() shutil.rmtree(temp_dir) return data
def makeChunksHoles(self, name): """ Generate one or more .chunks files for the named file """ os.chdir(os.path.dirname(name)) name = os.path.basename(name) baseName = name.rpartition(".")[0] + "_" sliceName = name.rpartition(".")[0].encode("utf8") current = 0 targetAddr = self.startLBA eof = self.file.seek(0, io.SEEK_END) self.file.seek(0, io.SEEK_SET) while current < eof: hole = (self.file.seek(current, SEEK_HOLE) + self.blockSize - 1) & ~(self.blockSize - 1) # Python's handling of this condition is suboptimal try: next = self.file.seek(hole, SEEK_DATA) & ~(self.blockSize - 1) trimCount = (next - current) >> self.blockShift except IOError: next = eof trimCount = self.lastWipe - targetAddr # Watch out for chunks >4GB (too big!) # Also, try not to test the limits of LG's tools... if (hole - current) >= 1 << 27: hole = current + (1 << 27) next = hole trimCount = (next - current) >> self.blockShift md5 = hashlib.md5() crc = crc32(b"") zobj = zlib.compressobj(1) self.file.seek(current, io.SEEK_SET) chunkName = baseName + str(targetAddr) + ".bin" out = io.FileIO(chunkName + ".chunk", "wb") print("[+] Compressing {:s} to {:s} ({:d} empty blocks)".format( name, chunkName, (next - hole) >> self.blockShift)) chunkName = chunkName.encode("utf8") out.seek(self._dz_length, io.SEEK_SET) zlen = 0 for b in range((hole - current) >> self.blockShift): buf = self.file.read(self.blockSize) md5.update(buf) crc = crc32(buf, crc) zdata = zobj.compress(buf) zlen += len(zdata) out.write(zdata) zdata = zobj.flush(zlib.Z_FINISH) zlen += len(zdata) out.write(zdata) md5 = md5.digest() out.seek(0, io.SEEK_SET) values = { 'sliceName': sliceName, 'chunkName': chunkName, 'targetSize': hole - current, 'dataSize': zlen, 'md5': md5, 'targetAddr': targetAddr, 'trimCount': trimCount, 'crc32': crc & 0xFFFFFFFF, 'dev': self.dev, } header = self.packdict(values) out.write(header) out.close() current = next targetAddr = self.startLBA + (current >> self.blockShift) print("[+] done\n")
def create_archives(self): try: self.width = self.parent.width() self.height = self.parent.height() self.x = self.parent.x() self.y = self.parent.y() except: self.width = 1920 self.height = 1080 self.x = 0 self.y = 0 self.file_count = 0 self.progress = QProgressDialog("Reading...", QtCore.QString(), 0, 72000, self.parent) self.progress.setWindowModality(Qt.Qt.WindowModal) self.progress.setValue(0) self.progress.setAutoClose(False) self.progress.setMinimumDuration(0) # with open(common.editor_config.eboot_orig, "rb") as f: with open( os.path.join(common.editor_config.iso_dir, "PSP_GAME", "SYSDIR", "EBOOT.BIN"), "rb") as f: eboot = BitStream(bytes=f.read()) eboot, eboot_offset = eboot_patch.apply_eboot_patches(eboot) USRDIR = os.path.join(common.editor_config.iso_dir, "PSP_GAME", "USRDIR") # So we can loop. :) ARCHIVE_INFO = [ { "toc": UMDIMAGES.umdimage, "dir": common.editor_config.umdimage_dir, "dat": os.path.join(USRDIR, "umdimage.dat"), "name": "umdimage.dat", "pack": common.editor_config.pack_umdimage, "eof": False, }, { "toc": UMDIMAGES.umdimage2, "dir": common.editor_config.umdimage2_dir, "dat": os.path.join(USRDIR, "umdimage2.dat"), "name": "umdimage2.dat", "pack": common.editor_config.pack_umdimage2, "eof": False, }, { "toc": None, "dir": common.editor_config.voice_dir, "dat": os.path.join(USRDIR, "voice.pak"), "name": "voice.pak", "pack": common.editor_config.pack_voice, "eof": True, }, { "toc": None, "dir": common.editor_config.bgm_dir, "dat": os.path.join(USRDIR, "bgm.pak"), "name": "bgm.pak", "pack": common.editor_config.pack_bgm, "eof": True, }, ] for archive in ARCHIVE_INFO: if not archive["pack"]: continue self.progress.setWindowTitle("Building " + archive["name"]) toc_info = {} file_list = None if archive["toc"]: file_list = [] toc = get_toc(eboot, archive["toc"]) for entry in toc: filename = entry["filename"] pos_pos = entry["file_pos_pos"] len_pos = entry["file_len_pos"] toc_info[filename] = [pos_pos, len_pos] file_list.append(filename) # Causes memory issues if I use the original order, for whatever reason. file_list = None with io.FileIO(archive["dat"], "w") as handler: table_of_contents = self.pack_dir(archive["dir"], handler, file_list=file_list, eof=archive["eof"]) # We're playing fast and loose with the file count anyway, so why not? self.file_count += 1 self.progress.setValue(self.file_count) self.progress.setLabelText("Saving " + archive["name"] + "...") if archive["toc"]: for entry in table_of_contents: if not entry in toc_info: _LOGGER.warning( "%s missing from %s table of contents." % (entry, archive["name"])) continue file_pos = table_of_contents[entry]["pos"] file_size = table_of_contents[entry]["size"] eboot.overwrite(BitStream(uintle=file_pos, length=32), toc_info[entry][0] * 8) eboot.overwrite(BitStream(uintle=file_size, length=32), toc_info[entry][1] * 8) del table_of_contents self.progress.setLabelText("Saving EBOOT.BIN...") self.progress.setValue(self.progress.maximum()) # Text replacement to_replace = eboot_text.get_eboot_text() for replacement in to_replace: orig = bytearray(replacement.orig, encoding=replacement.enc) # If they left something blank, write the original text back. if len(replacement.text) == 0: data = orig else: data = bytearray(replacement.text, encoding=replacement.enc) pos = replacement.pos.int + eboot_offset padding = len(orig) - len(data) if padding > 0: # Null bytes to fill the rest of the space the original took. data.extend(bytearray(padding)) data = ConstBitStream(bytes=data) eboot.overwrite(data, pos * 8) eboot_out = os.path.join(common.editor_config.iso_dir, "PSP_GAME", "SYSDIR", "EBOOT.BIN") with open(eboot_out, "wb") as f: eboot.tofile(f) self.progress.close()
def save_file_in_path(file_path, file_content, file_name): with io.FileIO(file_path, "wb") as fn: fn.write(file_content) return True
def main_func(user_screen_name, tweets_count): from apiclient import discovery from httplib2 import Http import oauth2client from oauth2client import file, client, tools import io from googleapiclient.http import MediaIoBaseDownload import tweepy import csv import pandas as pd from bs4 import BeautifulSoup from nltk.tokenize import WordPunctTokenizer from nltk.stem.wordnet import WordNetLemmatizer from nltk.stem.lancaster import LancasterStemmer from gensim.models import Word2Vec import multiprocessing from nltk.corpus import stopwords import re import pickle import numpy as np from sklearn.model_selection import train_test_split import pandas as pd import tensorflow as tf from tensorflow.contrib.tensorboard.plugins import projector import numpy as np import gensim import os import warnings import nltk warnings.filterwarnings(action='ignore', category=UserWarning, module='gensim') pd.options.mode.chained_assignment = None pd.set_option('display.max_columns', 7) nltk.download('stopwords') nltk.download('wordnet') obj = lambda: None lmao = { "auth_host_name": 'localhost', 'noauth_local_webserver': 'store_true', 'auth_host_port': [8080, 8090], 'logging_level': 'ERROR' } for k, v in lmao.items(): setattr(obj, k, v) SCOPES = 'https://www.googleapis.com/auth/drive.readonly' store = file.Storage('token.json') creds = store.get() if not creds or creds.invalid: flow = client.flow_from_clientsecrets('client_id.json', SCOPES) creds = tools.run_flow(flow, store, obj) DRIVE = discovery.build('drive', 'v3', http=creds.authorize(Http())) file_id = '1gN9u4zFWfwR5n-LmBwrcwmNGIUKj4Y0F' request = DRIVE.files().get_media(fileId=file_id) fh = io.FileIO('lemmatization_nolim_all.sav', mode='w') downloader = MediaIoBaseDownload(fh, request) done = False while done is False: status, done = downloader.next_chunk() print("Download %d%%." % int(status.progress() * 100)) predict_model = 'lemmatization_nolim_all.sav' # parent_patch = os.getcwd() path = "test" stopWords = set(stopwords.words('english')) consumer_key = 'NgbszsMy18esxzBRpnS6YJSg5' consumer_secret = 'fUlGwElm7B7Q5UUl99TdnMewBA3xW9Cw5xmzBAq1xU9j5O6wUa' access_key = '3847979172-1TNy6qbn1DvF2lHuUMpM86hAyRSxN8Uc9WpZzET' # access_token access_secret = 'ZCooGbFqAqxCyFtZGqMPczAhD6IkZW1TfT1hocKVPm8pV' tok = WordPunctTokenizer() pat1 = r'@[A-Za-z0-9_]+' pat2 = r'https?://[^ ]+' combined_pat = r'|'.join((pat1, pat2)) www_pat = r'www.[^ ]+' negations_dic = { "isn't": "is not", "aren't": "are not", "wasn't": "was not", "weren't": "were not", "haven't": "have not", "hasn't": "has not", "hadn't": "had not", "won't": "will not", "wouldn't": "would not", "don't": "do not", "doesn't": "does not", "didn't": "did not", "can't": "can not", "couldn't": "could not", "shouldn't": "should not", "mightn't": "might not", "mustn't": "must not" } neg_pattern = re.compile(r'\b(' + '|'.join(negations_dic.keys()) + r')\b') def tweet_cleaner_updated(text, tweet_len=100): soup = BeautifulSoup(text, 'lxml') souped = soup.get_text() try: bom_removed = souped.decode("utf-8-sig").replace(u"\ufffd", "?") except: bom_removed = souped stripped = re.sub(combined_pat, '', bom_removed) stripped = re.sub(www_pat, '', stripped) lower_case = stripped.lower() neg_handled = neg_pattern.sub(lambda x: negations_dic[x.group()], lower_case) letters_only = re.sub("[^a-zA-Z]", " ", neg_handled) lema = WordNetLemmatizer() # lancaster_stemmer = LancasterStemmer() words = list() for word in tok.tokenize(letters_only): if len(word) > 1 and word not in stopWords: # print('raw', word) lema_word = lema.lemmatize(word) # lema_word = lancaster_stemmer.stem(word) if len(lema_word) == 1: lema_word = word # print('lem', lema_word) words.append(lema_word) if len(words) <= tweet_len: return words, (" ".join(words)).strip() def get_api_clean_tweets_df(screen_name, tweet_num=5, predict_model=''): auth = tweepy.OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_key, access_secret) api = tweepy.API(auth) columns = ['Screen_Name', 'Time_Stamp', 'raw_tweet', 'text', 'tokens'] tweet_df = pd.DataFrame(columns=columns) tweet_tokeenized = list() tweet_tokens = list() positive_tokens = list() tokens_positiveti_dict = dict() index = 0 for status in tweepy.Cursor(api.user_timeline, screen_name=screen_name, tweet_mode="extended").items(): clean_tweet = tweet_cleaner_updated(status.full_text) tweet_tokens += clean_tweet[0] tweet_tokeenized.append(clean_tweet[0]) tweet_df.loc[index] = [ status.user.screen_name, status.created_at, status.full_text, clean_tweet[1], clean_tweet[0] ] index += 1 if index == tweet_num: break tweet_df['target'] = None loaded_model = pickle.load(open(predict_model, 'rb')) for index in range(len(tweet_df)): if tweet_df['text'][index] is np.nan: pass else: if int(loaded_model.predict(list([tweet_df['text'][index] ]))) == 1: tweet_df['target'][index] = 'Postive' else: tweet_df['target'][index] = 'Negative' positive_tweet_df = tweet_df.loc[tweet_df['target'] == 'Postive'] for row in positive_tweet_df.index: # range(len(positive_tweet_df)): positive_tokens += positive_tweet_df['tokens'][row] tweet_tokens_set = set(tweet_tokens) for token in tweet_tokens_set: token_all_counter = tweet_tokens.count(token) token_pos_counter = positive_tokens.count(token) tokens_positiveti_dict[token] = int(token_pos_counter / token_all_counter * 100) tweet_df.drop(['Screen_Name', 'Time_Stamp', 'text', 'tokens'], axis=1, inplace=True) print(tweet_df) return tokens_positiveti_dict, tweet_tokens, tweet_tokeenized, tweet_df pos_tokens_dict, donalds_tokens_list, donalds_tokenized_tweets, donalds_df = \ get_api_clean_tweets_df(user_screen_name, tweets_count, predict_model) cores = multiprocessing.cpu_count() user_model = Word2Vec( donalds_tokenized_tweets, min_count=1, size=200, workers=cores, ) user_model.save("user_model") model = gensim.models.keyedvectors.KeyedVectors.load("user_model") max_size = len(model.wv.vocab) - 1 w2v = np.zeros((max_size, model.layer1_size)) with open("test/metadata.tsv", 'w+') as file_metadata: meta_word = ('word' + '\t' + 'Sentiment') file_metadata.write(meta_word + '\n') with open('tensors.tsv', 'w+') as tensors: with open("test/metadata.tsv", 'a') as file_metadata: for i, word in enumerate(model.wv.index2word[:max_size]): w2v[i] = model.wv[word] if pos_tokens_dict[word] < 50: meta_word = word + '(' + str( pos_tokens_dict[word]) + ')' + '\t' + str(0) file_metadata.write(meta_word + '\n') else: meta_word = word + '(' + str( pos_tokens_dict[word]) + ')' + '\t' + str(100) file_metadata.write(meta_word + '\n') vector_row = '\t'.join(map(str, model[word])) tensors.write(vector_row + '\n') sess = tf.InteractiveSession() with tf.device("/cpu:0"): embedding = tf.Variable(w2v, trainable=False, name='embedding') print(embedding) tf.global_variables_initializer().run() saver = tf.train.Saver() writer = tf.summary.FileWriter(path, sess.graph) config = projector.ProjectorConfig() embed = config.embeddings.add() embed.tensor_name = 'embedding' embed.metadata_path = 'metadata.tsv' projector.visualize_embeddings(writer, config) saver.save(sess, path + '/model.ckpt', global_step=max_size)
def get_readable_fileobj(name_or_obj, encoding=None, cache=False, show_progress=True, remote_timeout=None): """ Given a filename or a readable file-like object, return a context manager that yields a readable file-like object. This supports passing filenames, URLs, and readable file-like objects, any of which can be compressed in gzip or bzip2. Notes ----- This function is a context manager, and should be used for example as:: with get_readable_fileobj('file.dat') as f: contents = f.read() Parameters ---------- name_or_obj : str or file-like object The filename of the file to access (if given as a string), or the file-like object to access. If a file-like object, it must be opened in binary mode. encoding : str, optional When `None` (default), returns a file-like object with a `read` method that on Python 2.x returns `bytes` objects and on Python 3.x returns `str` (`unicode`) objects, using `locale.getpreferredencoding()` as an encoding. This matches the default behavior of the built-in `open` when no `mode` argument is provided. When `'binary'`, returns a file-like object where its `read` method returns `bytes` objects. When another string, it is the name of an encoding, and the file-like object's `read` method will return `str` (`unicode`) objects, decoded from binary using the given encoding. cache : bool, optional Whether to cache the contents of remote URLs. show_progress : bool, optional Whether to display a progress bar if the file is downloaded from a remote server. Default is `True`. remote_timeout : float Timeout for remote requests in seconds (default is the configurable REMOTE_TIMEOUT, which is 3s by default) Returns ------- file : readable file-like object """ # close_fds is a list of file handles created by this function # that need to be closed. We don't want to always just close the # returned file handle, because it may simply be the file handle # passed in. In that case it is not the responsibility of this # function to close it: doing so could result in a "double close" # and an "invalid file descriptor" exception. close_fds = [] delete_fds = [] if remote_timeout is None: # use configfile default remote_timeout = REMOTE_TIMEOUT() # Get a file object to the content if isinstance(name_or_obj, six.string_types): if _is_url(name_or_obj): name_or_obj = download_file(name_or_obj, cache=cache, show_progress=show_progress, timeout=remote_timeout) if PY3K: fileobj = io.FileIO(name_or_obj, 'r') else: fileobj = open(name_or_obj, 'rb') close_fds.append(fileobj) else: fileobj = name_or_obj # Check if the file object supports random access, and if not, # then wrap it in a BytesIO buffer. It would be nicer to use a # BufferedReader to avoid reading loading the whole file first, # but that is not compatible with streams or urllib2.urlopen # objects on Python 2.x. if not hasattr(fileobj, 'seek'): fileobj = io.BytesIO(fileobj.read()) # Now read enough bytes to look at signature signature = fileobj.read(4) fileobj.seek(0) if signature[:3] == b'\x1f\x8b\x08': # gzip import struct try: from .compat import gzip fileobj_new = gzip.GzipFile(fileobj=fileobj, mode='rb') fileobj_new.read(1) # need to check that the file is really gzip except (IOError, EOFError): # invalid gzip file fileobj.seek(0) fileobj_new.close() except struct.error: # invalid gzip file on Python 3 fileobj.seek(0) fileobj_new.close() else: fileobj_new.seek(0) fileobj = fileobj_new elif signature[:3] == b'BZh': # bzip2 try: # bz2.BZ2File does not support file objects, only filenames, so we # need to write the data to a temporary file tmp = NamedTemporaryFile("wb", delete=False) tmp.write(fileobj.read()) tmp.close() delete_fds.append(tmp) import bz2 fileobj_new = bz2.BZ2File(tmp.name, mode='rb') fileobj_new.read(1) # need to check that the file is really bzip2 except IOError: # invalid bzip2 file fileobj.seek(0) fileobj_new.close() else: fileobj_new.seek(0) close_fds.append(fileobj_new) fileobj = fileobj_new # By this point, we have a file, io.FileIO, gzip.GzipFile, or # bz2.BZ2File instance opened in binary mode (that is, read # returns bytes). Now we need to, if requested, wrap it in a # io.TextIOWrapper so read will return unicode based on the # encoding parameter. if PY3K: needs_textio_wrapper = encoding != 'binary' else: needs_textio_wrapper = encoding != 'binary' and encoding is not None if needs_textio_wrapper: # A bz2.BZ2File can not be wrapped by a TextIOWrapper, # so we decompress it to a temporary file and then # return a handle to that. import bz2 if isinstance(fileobj, bz2.BZ2File): tmp = NamedTemporaryFile("wb", delete=False) data = fileobj.read() tmp.write(data) tmp.close() delete_fds.append(tmp) if PY3K: fileobj = io.FileIO(tmp.name, 'r') else: fileobj = open(tmp.name, 'rb') close_fds.append(fileobj) # On Python 2.x, we need to first wrap the regular `file` # instance in a `io.FileIO` object before it can be # wrapped in a `TextIOWrapper`. We don't just create an # `io.FileIO` object in the first place, because we can't # get a raw file descriptor out of it on Python 2.x, which # is required for the XML iterparser. if not PY3K and isinstance(fileobj, file): fileobj = io.FileIO(fileobj.fileno()) fileobj = io.BufferedReader(fileobj) fileobj = io.TextIOWrapper(fileobj, encoding=encoding) # Ensure that file is at the start - io.FileIO will for # example not always be at the start: # >>> import io # >>> f = open('test.fits', 'rb') # >>> f.read(4) # 'SIMP' # >>> f.seek(0) # >>> fileobj = io.FileIO(f.fileno()) # >>> fileobj.tell() # 4096L fileobj.seek(0) try: yield fileobj finally: for fd in close_fds: fd.close() for fd in delete_fds: os.remove(fd.name)
def download_file(self, file_id, download_directory_path, username): """ Downloads the file for file_id to the given download_path. :param file_id: :param download_directory_path: :return: the full path to the downloaded file """ googledrive_file = self.googledrive_api.files().get( fileId=file_id, fields="name, mimeType").execute() # convert utf-8 chars safe_filename = googledrive_file['name'].encode( sys.getfilesystemencoding(), 'ignore') file_download_path = os.path.join(download_directory_path, safe_filename) logger.debug('Download file %s <= googledrive://file/%s', file_download_path, file_id) if 'vnd.google-apps' in googledrive_file['mimeType']: # if googledrive_file['mimeType'] == 'application/vnd.google-apps.spreadsheet': # mimeType = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' # elif googledrive_file['mimeType'] == 'application/vnd.google-apps.document': # mimeType = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' # elif googledrive_file['mimeType'] == 'application/vnd.google-apps.presentation': # mimeType = 'application/vnd.openxmlformats-officedocument.presentationml.presentation' # else: n = Notification( event_type='data', status=Notification.ERROR, operation='googledrive_download_error', message= 'Copying Google-type files is currently unsupported. Export the file to' ' a standard format and try again.', user=username, extra={ 'path': "'{}' of type {}".format(googledrive_file['name'], googledrive_file['mimeType']) }) n.save() return None request = self.googledrive_api.files().get_media(fileId=file_id) # Incremental Partial Download fh = io.FileIO(file_download_path, 'wb') downloader = MediaIoBaseDownload(fh, request) done = False backoff_attempts = 0 while done is False: try: status, done = downloader.next_chunk() # logger.debug('status: {} percent'.format(status.progress())) except HttpError as e: # Incremental backoff for exceeding google api rate limit if "Rate Limit Exceeded" in str(e): logger.debug('RATE LIMIT EXCEEDED') backoff_attempts += 1 time.sleep(backoff_attempts) if backoff_attempts > 10: n = Notification( event_type='data', status=Notification.ERROR, operation='googledrive_download_error', message= 'Rate Limit Exceeded. Try again after a few minutes for this file.', user=username, extra={ 'path': "{}".format(googledrive_file['name']) }) n.save() return None elif "Only files with binary content can be downloaded" in str( e): n = Notification( event_type='data', status=Notification.ERROR, operation='googledrive_download_error', message= 'Only files with binary content can be downloaded. Convert the file to' ' a standard format and try again.', user=username, extra={ 'path': "'{}' of type {}".format( googledrive_file['name'], googledrive_file['mimeType']) }) n.save() return None else: raise fh.close() return file_download_path
def download(service, file, destination, skip=False, noiter=False): # file is a dictionary with file id as well as name if skip and os.path.exists(os.path.join(destination, file['name'])): return -1 mimeType = file['mimeType'] if "application/vnd.google-apps" in mimeType: if "form" in mimeType: return -1 elif "document" in mimeType: dlfile = service.files().export_media( fileId=file['id'], mimeType= 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' ) elif "spreadsheet" in mimeType: dlfile = service.files().export_media( fileId=file['id'], mimeType= 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' ) elif "presentation" in mimeType: dlfile = service.files().export_media( fileId=file['id'], mimeType= 'application/vnd.openxmlformats-officedocument.presentationml.presentation' ) else: dlfile = service.files().export_media(fileId=file['id'], mimeType='application/pdf') else: dlfile = service.files().get_media(fileId=file['id'], supportsAllDrives=True, acknowledgeAbuse=True) rand_id = str(uuid.uuid4()) os.makedirs('buffer', exist_ok=True) fh = io.FileIO(os.path.join('buffer', rand_id), 'wb') downloader = MediaIoBaseDownload(fh, dlfile, chunksize=CHUNK_SIZE) if noiter: print(f"{Fore.GREEN}Downloading{Style.RESET_ALL} {file['name']} ...") done = False rate_limit_count = 0 while done is False and rate_limit_count < 20: try: status, done = downloader.next_chunk() except Exception as ex: DEBUG_STATEMENTS.append( f'File Name: {file["name"]}, File ID: {file["id"]}, Exception: {ex}' ) rate_limit_count += 1 fh.close() if noiter and rate_limit_count == 20: print(f"{Fore.RED}Error {Style.RESET_ALL} {file['name']} ...") os.makedirs(destination, exist_ok=True) while True: try: shutil.move(os.path.join('buffer', rand_id), os.path.join(destination, file['name'])) break except PermissionError: # wait out the file write before attempting to move pass return rate_limit_count
print("-----------------------------------------------------") for filepath in files: data = open(filepath, "r", encoding="utf-8") s = s + 1 start = True Fussnoten = False ## process lines for line in data: ## get title done = False if filepath in firstsite and start == True: if not os.path.exists("out/" + sys.argv[1]): os.makedirs("out/" + sys.argv[1]) with io.FileIO("./out/" + sys.argv[1] + "/" + titleN + ".txt", "w") as file: file.write(article.encode("utf-8")) if not os.path.exists("out_notes/" + sys.argv[1]): os.makedirs("out_notes/" + sys.argv[1]) with io.FileIO( "./out_notes/" + sys.argv[1] + "/" + titleN + ".txt", "w") as file: file.write(notes.encode("utf-8")) titleN = title[a] titleN = titleN[:-1] article = "" notes = "" article += title[a]
from google.colab import auth auth.authenticate_user() drive_service = build('drive', 'v3') #File Download !rm -f *.zip *.csv #!kaggle competitions download -c jigsaw-toxic-comment-classification-challenge #train.csv download results = drive_service.files().list(q="name = 'train.csv.zip'", fields="files(id)").execute() kaggle_api_key = results.get('files', []) filename = "/content/train.csv.zip" os.makedirs(os.path.dirname(filename), exist_ok=True) request = drive_service.files().get_media(fileId=kaggle_api_key[0]['id']) fh = io.FileIO(filename, 'wb') downloader = MediaIoBaseDownload(fh, request) done = False while done is False: status, done = downloader.next_chunk() print("train.csv.zip - Download %d%%." % int(status.progress() * 100)) os.chmod(filename, 600) #test.csv download results = drive_service.files().list(q="name = 'test.csv.zip'", fields="files(id)").execute() kaggle_api_key = results.get('files', []) filename = "/content/test.csv.zip" os.makedirs(os.path.dirname(filename), exist_ok=True) request = drive_service.files().get_media(fileId=kaggle_api_key[0]['id']) fh = io.FileIO(filename, 'wb') downloader = MediaIoBaseDownload(fh, request)
def open_read(self, path, mode="r"): return FileWrapper(io.BufferedReader(io.FileIO(path, mode)))
async def download_gdrive(gdrive, service, uri): reply = "" global is_cancelled if not isdir(TEMP_DOWNLOAD_DIRECTORY): os.mkdir(TEMP_DOWNLOAD_DIRECTORY) if "&export=download" in uri: uri = uri.split("&export=download")[0] elif "file/d/" in uri and "/view" in uri: uri = uri.split("?usp=drivesdk")[0] try: file_Id = uri.split("uc?id=")[1] except IndexError: try: file_Id = uri.split("open?id=")[1] except IndexError: if "/view" in uri: file_Id = uri.split("/")[-2] else: try: file_Id = uri.split("uc?export=download&confirm=")[1].split("id=")[ 1 ] except IndexError: file_Id = uri try: file = await get_information(service, file_Id) except HttpError as e: if "404" in str(e): drive = "https://drive.google.com" url = f"{drive}/uc?export=download&id={file_Id}" session = requests.session() download = session.get(url, stream=True) try: download.headers["Content-Disposition"] except KeyError: page = BeautifulSoup(download.content, "lxml") try: export = drive + page.find("a", {"id": "uc-download-link"}).get( "href" ) except AttributeError: try: error = ( page.find("p", {"class": "uc-error-caption"}).text + "\n" + page.find("p", {"class": "uc-error-subcaption"}).text ) except Exception: reply += ( "`[FILE - ERROR]`\n\n" "`Status` : **BAD** - failed to download.\n" "`Reason` : uncaught err." ) else: reply += ( "`[FILE - ERROR]`\n\n" "`Status` : **BAD** - failed to download.\n" f"`Reason` : {error}" ) return reply download = session.get(export, stream=True) file_size = human_to_bytes( page.find("span", {"class": "uc-name-size"}) .text.split()[-1] .strip("()") ) else: file_size = int(download.headers["Content-Length"]) file_name = re.search( 'filename="(.*)"', download.headers["Content-Disposition"] ).group(1) file_path = TEMP_DOWNLOAD_DIRECTORY + file_name with io.FileIO(file_path, "wb") as files: CHUNK_SIZE = None current_time = time.time() display_message = None first = True is_cancelled = False for chunk in download.iter_content(CHUNK_SIZE): if is_cancelled: raise CancelProcess if not chunk: break diff = time.time() - current_time if first: downloaded = len(chunk) first = False else: downloaded += len(chunk) percentage = downloaded / file_size * 100 speed = round(downloaded / diff, 2) eta = round((file_size - downloaded) / speed) prog_str = "[{0}{1}] `{2}%`".format( "".join("█" for i in range(math.floor(percentage / 10))), "".join("░" for i in range(10 - math.floor(percentage / 10))), round(percentage, 2), ) current_message = ( f"{file_name} - Downloading\n" f"{prog_str}\n" f"`Size:` {humanbytes(downloaded)} of {humanbytes(file_size)}" f"`Speed:` {humanbytes(speed)}`\n" f"`ETA:` {time_formatter(eta)}" ) if ( round(diff % 15.00) == 0 and (display_message != current_message) or (downloaded == file_size) ): await gdrive.edit(current_message) display_message = current_message files.write(chunk) else: file_name = file.get("name") mimeType = file.get("mimeType") if mimeType == "application/vnd.google-apps.folder": await gdrive.edit("`Aborting, folder download not support...`") return False file_path = TEMP_DOWNLOAD_DIRECTORY + file_name request = service.files().get_media(fileId=file_Id, supportsAllDrives=True) with io.FileIO(file_path, "wb") as df: downloader = MediaIoBaseDownload(df, request) complete = False is_cancelled = False current_time = time.time() display_message = None while not complete: if is_cancelled: raise CancelProcess status, complete = downloader.next_chunk() if status: file_size = status.total_size diff = time.time() - current_time downloaded = status.resumable_progress percentage = downloaded / file_size * 100 speed = round(downloaded / diff, 2) eta = round((file_size - downloaded) / speed) prog_str = " [{0}{1}] `{2}%`".format( "".join("█" for i in range(math.floor(percentage / 10))), "".join("░" for i in range(10 - math.floor(percentage / 10))), round(percentage, 2), ) current_message = ( f"{file_name} - Downloading\n" f"{prog_str}\n" f"`Size:` {humanbytes(downloaded)} of {humanbytes(file_size)}" f"`Speed:` {humanbytes(speed)}`\n" f"`ETA:` {time_formatter(eta)}" ) if ( round(diff % 15.00) == 0 and (display_message != current_message) or (downloaded == file_size) ): await gdrive.edit(current_message) display_message = current_message await gdrive.edit( "`[FILE - DOWNLOAD]`\n\n" f"`Name :` `{file_name}`\n" f"`Size :` `{humanbytes(file_size)}`\n" f"`Path :` `{file_path}`\n" "`Status :` **OK** - Successfully downloaded." ) msg = await gdrive.respond("`Answer the question in your BOTLOG group`") async with gdrive.client.conversation(BOTLOG_CHATID) as conv: ask = await conv.send_message("`Proceed with mirroring? [y/N]`") try: r = conv.wait_event(events.NewMessage(outgoing=True, chats=BOTLOG_CHATID)) r = await r except Exception: ans = "N" else: ans = r.message.message.strip() await gdrive.client.delete_messages(BOTLOG_CHATID, r.id) await gdrive.client.delete_messages(gdrive.chat_id, msg.id) await gdrive.client.delete_messages(BOTLOG_CHATID, ask.id) if ans.capitalize() == "N": return reply elif ans.capitalize() == "Y": try: result = await upload(gdrive, service, file_path, file_name, mimeType) except CancelProcess: reply += ( "`[FILE - CANCELLED]`\n\n" "`Status` : **OK** - received signal cancelled." ) else: reply += ( "`[FILE - UPLOADED]`\n\n" f"`Name :` `{file_name}`\n" f"`Size :` `{humanbytes(result[0])}`\n" f"`Link :` [{file_name}]({result[1]})\n" "`Status :` **OK**\n\n" ) return reply else: await gdrive.client.send_message( BOTLOG_CHATID, "`Invalid answer type [Y/N] only...`" ) return reply
''' Created on May 21, 2018 @author: venkateshwara.d ''' import os from selenium import webdriver import io chrome_driver_path = os.path.abspath('..') + "\\Drivers\\chromedriver.exe" driver = webdriver.Chrome(chrome_driver_path) driver.get('https://python.org') html = driver.page_source if (os.path.isfile("pageSource.txt")): os.remove("pageSource.txt") with io.FileIO("pageSource.txt", "w") as file: file.write(html.encode("utf-8")) driver.close()
def unix_getpass(prompt='Password: '******'Password: '******'/dev/tty', os.O_RDWR | os.O_NOCTTY) tty = io.FileIO(fd, 'w+') stack.enter_context(tty) input = io.TextIOWrapper(tty) stack.enter_context(input) if not stream: stream = input except OSError as e: # If that fails, see if stdin can be controlled. stack.close() try: fd = sys.stdin.fileno() except (AttributeError, ValueError): fd = None passwd = fallback_getpass(prompt, stream) input = sys.stdin if not stream: stream = sys.stderr if fd is not None: try: old = termios.tcgetattr(fd) # a copy to save new = old[:] new[3] &= ~termios.ECHO # 3 == 'lflags' tcsetattr_flags = termios.TCSAFLUSH if hasattr(termios, 'TCSASOFT'): tcsetattr_flags |= termios.TCSASOFT try: termios.tcsetattr(fd, tcsetattr_flags, new) passwd = _raw_input(prompt, stream, input=input) finally: termios.tcsetattr(fd, tcsetattr_flags, old) stream.flush() # issue7208 except termios.error: if passwd is not None: # _raw_input succeeded. The final tcsetattr failed. Reraise # instead of leaving the terminal in an unknown state. raise # We can't control the tty or stdin. Give up and use normal IO. # fallback_getpass() raises an appropriate warning. if stream is not input: # clean up unused file objects before blocking stack.close() passwd = fallback_getpass(prompt, stream) stream.write('\n') return passwd
def _readline(fd): """Read one line. May block forever if no newline is read.""" reader = io.FileIO(fd, mode='rb', closefd=False) return reader.readline()
def __init__(self, path): self._path = path fd = os.open(path, os.O_RDWR | os.O_DIRECT) self._file = io.FileIO(fd, "r+", closefd=True)
import pyaudio import io, sys p = pyaudio.PyAudio() try: filename = sys.argv[1] except: print( 'ERROR: Usage - python playfile.py <filename> (e.g. python playfile.py voiceC' ) data_file = io.FileIO('./sound/{}.pcm'.format(filename), mode='r') FORMAT = p.get_format_from_width(width=2) CHANNELS = 1 RATE = 16000 CHUNK = 16000 stream = p.open(format=FORMAT, channels=CHANNELS, rate=RATE, output=True, frames_per_buffer=CHUNK) print('playing...') data = data_file.readall() stream.write(data) stream.close()
def unix_getpass(prompt='Password: '******'Password: '******'/dev/tty', os.O_RDWR | os.O_NOCTTY) tty = io.FileIO(fd, 'w+') stack.enter_context(tty) input = io.TextIOWrapper(tty) stack.enter_context(input) if not stream: stream = input except OSError as e: try: stack.close() try: fd = sys.stdin.fileno() except (AttributeError, ValueError): fd = None passwd = fallback_getpass(prompt, stream) input = sys.stdin if not stream: stream = sys.stderr finally: e = None del e if fd is not None: try: old = termios.tcgetattr(fd) new = old[:] new[3] &= ~termios.ECHO tcsetattr_flags = termios.TCSAFLUSH if hasattr(termios, 'TCSASOFT'): tcsetattr_flags |= termios.TCSASOFT try: termios.tcsetattr(fd, tcsetattr_flags, new) passwd = _raw_input(prompt, stream, input=input) finally: termios.tcsetattr(fd, tcsetattr_flags, old) stream.flush() except termios.error: if passwd is not None: raise if stream is not input: stack.close() passwd = fallback_getpass(prompt, stream) stream.write('\n') return passwd
def __init__(self, **kwargs): other = kwargs.get("other", None) if other: self.fromOther(other) else: self.bufferPool = kwargs.get("bufferPool", None) if self.bufferPool is None: raise ValueError( "No buffer pool found when initializing a storage file") fileId = kwargs.get("fileId", None) filePath = kwargs.get("filePath", None) mode = kwargs.get("mode", None) existing = os.path.exists(filePath) if fileId and filePath: initHeader = False initFreePages = False if not existing and mode.lower() == "create": ioMode = "w+b" pageSize = kwargs.get("pageSize", io.DEFAULT_BUFFER_SIZE) pageClass = kwargs.get("pageClass", StorageFile.defaultPageClass) schema = kwargs.get("schema", None) if pageSize and pageClass and schema: self.header = FileHeader(pageSize=pageSize, pageClass=pageClass, schema=schema) initHeader = True initFreePages = False else: raise ValueError( "No page size, class or schema specified when creating a new storage file" ) elif existing and mode.lower() in ["update", "truncate"]: ioMode = "r+b" if mode.lower() == "update" else "w+b" f = io.BufferedReader(io.FileIO(filePath)) self.header = FileHeader.fromFile(f) pageSize = self.pageSize() initFreePages = True f.close() else: raise ValueError( "Incompatible storage file mode and on-disk file status" ) if self.header: self.fileId = fileId self.path = filePath self.file = io.BufferedRandom(io.FileIO(self.path, ioMode), buffer_size=pageSize) self.binrepr = Struct("H" + str(FileId.binrepr.size) + "s" + str(len(self.path)) + "s") self.freePages = set() page = self.pageClass()(pageId=self.pageId(0), buffer=bytes(self.pageSize()), schema=self.schema()) self.pageHdrSize = page.header.headerSize() if initFreePages: self.initializeFreePages() if initHeader: self.file.seek(0) self.header.toFile(self.file) self.file.flush() else: raise ValueError( "No valid header available for storage file") else: raise ValueError( "No file id or path specified in storage file constructor")
for name in shaders: if "_f" in name: profile = "sce_fp_psp2" else: profile = "sce_vp_psp2" cmd = "psp2cgc -profile %s ..\external\src\inline_gles2\shaders\%s.cg -o %s.bin" % ( profile, name, name) print(cmd) os.system(cmd) out = open("../external/src/cg_shaders.h", "wb") out.write("// shaders compiled by build_shaders.py\n\n") for name in shaders: f = io.FileIO("%s.bin" % (name), "rb") z = f.read() out.write("static const uint8_t cg_shaderbin_%s[] = {" % (name)) for byte in z: out.write("0x%02x," % (ord(byte))) out.write("};\n") out.write("\n") """ out.write("static const char * const cg_shader_names[] = {") for name in shaders: out.write('"%s", ' % (name)) out.write("NULL};\n") out.write("\n") out.write("\n")
def write(self, fileName): try: with io.FileIO(fileName, "wb") as out_file: formatName = "SMA" buffer1 = formatName.encode(encoding='UTF-8', errors='strict') if not buffer1: buffer1 = bytes() out_file.write(pack('<{}s'.format(8), buffer1)) version = 1 out_file.write(pack('<H', version)) buffer = self.name.encode(encoding='UTF-8', errors='strict') if not buffer: buffer = bytes() out_file.write(pack('<{}s'.format(32), buffer)) out_file.write(pack('<H', len(self.vertices))) for v in self.vertices: out_file.write(pack('<f', v)) out_file.write(pack('<H', len(self.normals))) for n in self.normals: out_file.write(pack('<f', n)) out_file.write(pack('<H', len(self.texcoords))) for uv in self.texcoords: out_file.write(pack('<f', uv)) self.textures.write(out_file) print('Saving bones') out_file.write(pack('<H', len(self.bones))) for bone in self.bones: out_file.write(pack('<h', bone.parent_idx)) out_file.write(pack('<f', bone.rot_x)) out_file.write(pack('<f', bone.rot_y)) out_file.write(pack('<f', bone.rot_z)) out_file.write(pack('<f', bone.pos_x)) out_file.write(pack('<f', bone.pos_y)) out_file.write(pack('<f', bone.pos_z)) #bone.print() out_file.write(pack('<H', len(self.vertex_weights))) for vert_w in self.vertex_weights: vert_w.write(out_file) out_file.write(pack('<H', len(self.animations))) for a in self.animations: abuffer = a.name.encode(encoding='UTF-8', errors='strict') if not abuffer: abuffer = bytes() out_file.write(pack('<{}s'.format(64), abuffer)) out_file.write(pack('<H', a.numKeyframes)) #print("Num keyframes", a.numKeyframes) # print(a.name) for kf in a.keyframe: #print("kf index", kf.index) out_file.write(pack('<H', kf.index)) idx = 0 for rot in kf.rotations: out_file.write(pack('<f', kf.rotations[idx].x)) out_file.write(pack('<f', kf.rotations[idx].y)) out_file.write(pack('<f', kf.rotations[idx].z)) out_file.write(pack('<f', kf.positions[idx].x)) out_file.write(pack('<f', kf.positions[idx].y)) out_file.write(pack('<f', kf.positions[idx].z)) #print(idx) idx += 1 out_file.flush() out_file.close() finally: pass