def checkMemberillust(memberillusturl, pid, soup=None): #读取标签 tags = getTags(soup) imgName = getName(soup) tmp = [val for val in tags if val in mytags] if len(tmp) != 0: img = soup.find('img', attrs={'alt': imgName}) if img: imgFile = FileUtils.joinPath(portraitDir, FileUtils.getfile(img['src'])) if FileUtils.exists(imgFile): print("文件已存在[%s]" % imgFile) else: print("开始下载图片[%s][%s]" % (imgFile, img['src'])) downLoad(img['src'], imgFile) print("下载图片[%s]结束" % imgFile) else: img = soup.find('div', attrs={'class': 'works_display'}) if img.find('a', attrs={'class': r' _work multiple '}): img1 = img.find('a', attrs={'class': r' _work multiple '}) if img.find('a', attrs={'class': r' _work manga multiple '}): img1 = img.find('a', attrs={'class': r' _work manga multiple '}) if img1 is not None: checkMemberillust_medium( "%s%s%s" % (indexurl, '/', img1['href']), pid, img1['href'].split('=', 2)[2])
def decodeGameAPK(): global backupPath if (isNeedDecodeGameAPK): print("start decode gameAPK......") else: print("no need to decode gameAPK.") return try: #delete backup folder FileUtils.delDir(backupPath) except Exception as ex: print("exception :{0}".format(str(ex))) gameApk = "games/" + gameApkName + ".apk" cmd = "".join([ "java -classpath ./libs/apktool.jar:. brut.apktool.Main d -f ", gameApk, " -o out/temp/" + str(timestamp) + "/" + gameApkName ]) print("cmd is ", cmd) (status, output) = commands.getstatusoutput(cmd) print output print("end decode gameAPK.") return
def logger(logger_name=None): # Create the root logger. logger = logging.getLogger(logger_name) logger.setLevel(Log.log_level) # Validate the given directory. Log.log_dir = os.path.normpath(Log.log_dir) # Create a folder for the logfile. FileUtils.make_dir(Log.log_dir) # Build the logfile name filename = Log.file_name_format + ".log" filename = os.path.join(Log.log_dir, filename) # Set up logging to the logfile file_handler = RotatingFileHandler( filename=filename # ,maxBytes=Log.max_bytes, backupCount=Log.backup_count ) file_handler.setLevel(Log.log_level) file_formatter = logging.Formatter(Log.file_msg_format) file_handler.setFormatter(file_formatter) logger.addHandler(file_handler) # Set up logging to console stream_handler = logging.StreamHandler() stream_handler.setLevel(Log.log_level) stream_formatter = logging.Formatter(Log.console_msg_format) stream_handler.setFormatter(stream_formatter) logger.addHandler(stream_handler) return logger
def setup(self, argv): """ Setup the parameters for the simulation. Create result folder and logger file. :param argv: Dictionary list of the parameters """ self.config_name = argv[ "config_name"] if "config_name" in argv else self.root + "/configs/default_dog_vert_simulation_config.json" log_file = argv[ "logfile"] if "logfile" in argv else os.path.expanduser("~").replace("\\", "/") + "/.log/locomotionSim.log" if "filename" in argv: self.save_file = argv["filename"] else: dirname = self.root + "/save" filename = "sim_" + datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f") + ".qsm" if not os.path.exists(dirname): os.makedirs(dirname) self.save_file = dirname + "/" + filename self.genome = eval(argv["genome"]) if "genome" in argv else False FileUtils.create_file(log_file) logging.config.fileConfig(self.root + "/etc/logging.conf", defaults={'logfilename': log_file, 'simLevel': "DEBUG"})
def rebuild(): if (isNeedRebuildGameAPK): print("start rebuild gameAPK......") else: print("no need to rebuild gameAPK.") return apkpath = "out/temp/" + str(timestamp) + "/real_" + gameApkName + ".apk" finalApkPath = "out/temp/" + str(timestamp) + "/" + gameApkName + ".apk" cmd = "".join([ "java -classpath ./libs/apktool.jar:. brut.apktool.Main b -o " + apkpath, " out/temp/", str(timestamp) + "/" + gameApkName ]) print("cmd is ", cmd) (status, output) = commands.getstatusoutput(cmd) print output # sign cmd = "jarsigner -verbose -keystore /Users/lala/Documents/workspace/keystore/android/android/pa_crack.keystore -storepass 123456 -signedjar " + finalApkPath + " -digestalg SHA1 -sigalg MD5withRSA " + apkpath + " pa_crack.keystore" print("cmd is ", cmd) (status, output) = commands.getstatusoutput(cmd) print output FileUtils.delFile(apkpath) print("end rebuild gameAPK.") return
def augment_faces(face_folder_path="face", augmented_folder_path="face_augmented", aug_per_image = 8): print("augment_faces", "start") FileUtils.remove_dirs(augmented_folder_path) FileUtils.make_dirs(augmented_folder_path) ImageAugmenter.augments(face_folder_path, augmented_folder_path, aug_per_image = aug_per_image) print("augment_faces", "completed")
def test_file_read(self): """ Test file read capability """ data="{key:value}" file1 = "filew.json" futils.write_to_file(file1,data) ret = futils.read_from_file(file1) self.assertTrue(data,ret) os.remove(file1)
def to_local_cache(self): if self.mode == "monkey": return for record in list(self.flow.exe_records.keys()): if self.flow.exe_records[record]["status"] == RecordsStatus.DONE: FileUtils.add_to_pickle(self.cache_file_name, self.flow.exe_records[record]) for k in list(self.flow.exe_records[record].keys()): if k != "status": del self.flow.exe_records[record][k]
def fit(cls, X, y): cls.X = X cls.classifier = BASE_CLASSIFIER() cls.classifier.fit(X, y) FileUtils.make_dirs(cls.model_folder) model_file = open(cls.classifier_path, "wb") pickle.dump(cls.classifier, model_file) model_file.close() X_file = open(cls.X_path, "wb") pickle.dump(cls.X, X_file) X_file.close()
def verifyCaptchaByClound(self, type=TYPE_LOGIN_NORMAL_WAY): captchaContent = self.getCaptcha(type) if captchaContent: FileUtils.saveBinary(Captcha.__CAPTCHA_PATH, captchaContent) else: Log.e('failed to save captcha') return None results = damatuWeb.verify(Captcha.__CAPTCHA_PATH) results = self.__cloundTransCaptchaResults(results) Log.v('captchaResult: %s' % results) return results, self.check(results)
def copyAssets(): if (isNeedCopyAssets): print("start copy assets......") else: print("no need to copy assets.") return FileUtils.doCopy(sdkApkPath + "/assets", gameApkPath + "/assets") print("end copy assets.") return
def copyUnkown(): if (isNeedCopyUnknown): print("start copy unknown......") else: print("no need to copy unknown.") return FileUtils.doCopy(sdkApkPath + "/unknown", gameApkPath + "/unknown") print("end copy unknown.") return
def copyResCommon(): if (isNeedCopyResCommon): print("start copy resource common......") else: print("no need to copy resource common.") return FileUtils.doCopy(sdkApkPath + "/res", gameApkPath + "/res") print("end copy res common") return
def copySmali(): if (isNeedCopySmali): print("start copy smali......") else: print("no need to copy smali.") return FileUtils.doCopy(sdkApkPath + "/smali", gameApkPath + "/smali") print("end copy smali.") return
def copyLibSO(): if (isNeedCopyLibSo): print("start copy lib so......") else: print("no need to copy lib so.") return FileUtils.doCopy(sdkApkPath + "/lib", gameApkPath + "/lib") print("end copy lib so.") return
def test_file_write(self): """ Test that file is written successfully """ data = "ABCDEFG" file1 = "file.json" file2 = "f.json" futils.write_to_file(file1,data) futils.write_to_file(file2,data) dir = os.path.dirname(os.path.abspath(__file__)) self.assertTrue(filecmp.cmp(os.path.join(dir,file2), os.path.join(dir,file1))) os.remove(file2) os.remove(file1)
def commit(self, f_name): """Adds new object to memory :type f_name: str :param f_name: name of file to insert object in return: id of commited object """ #prevent duplicate entries, clear list and start afresh self.objects = dict() #generate 4-character id id = str(uuid.uuid4())[:8] self.objects[id] = self.obj utils.write_to_file(f_name, json.dumps(self.objects)) return id
def __generate_training_and_test_series(self): Logger.info("Generating training and test datasets ...") self.__preprocess_bios() concatenated = PandasUtils.concat_series( [self.appropriate_bios, self.inappropriate_bios]) shuffled = PandasUtils.shuffle_series(concatenated) self.training_series = shuffled.head(self.number_of_training_records) self.test_series = shuffled.tail(self.number_of_test_records) FileUtils.write_series2file(self.bio_train_path, self.training_series) FileUtils.write_series2file(self.bio_test_path, self.test_series) self.test_list = self.test_series.to_list() self.cleaned_test_list = self.__remove_labels( self.test_series, ['__label__{}'.format(l.value) for l in self.Label]).tolist()
async def getFriendList(self): _user = Users('http://{}:{}'.format(Config._host, Config._restPort), Config._restPort_secret) try: ''' friendList的数据格式: {'rosterItem': [{'jid': '[email protected]', 'nickname': 'chinsing00', 'subscriptionType': 3, 'groups': ['Friends']},{...}]} ''' friend_List = _user.get_user_roster( self.core.jid.localpart) #只需要用户名就行,不需要加域名 del _user #清除user,避免服务器生成过多的连接导致服务器连接池溢出 metadata = await self.avatar_server.get_avatar_metadata( self.core.jid) if metadata: picture = await asyncio.ensure_future( metadata[0].get_image_bytes()) picPath = os.path.join( app.getAvatarRootPath(self.core.jid.localpart), '{}.jpg'.format(str(self.core.jid))) if not os.path.exists(picPath): FileUtils.savaToPng(picPath, picture) usericon = QPixmap.fromImage(QImage(picPath)) if os.path.exists( picPath) else QPixmap(":src\images\CustomerService.png") self.mWin.avatar.setBaseSize(QSize(40, 40)) temp = utils.PixmapToRound(self.mWin.avatar, usericon) self.mWin.avatar.setScaledContents(True) self.mWin.avatar.setPixmap(temp) for friend in friend_List['rosterItem']: # task执行后返回 <AbstractAvatarDescriptor> 对象 task = await get_event_loop().create_task( self.avatar_server.get_avatar_metadata(JID.fromstr( friend['jid']), require_fresh=True, disable_pep=False)) #得到<AbstractAvatarDescriptor> 对象后进行头像保存本地及加载好友列表 avatar_path = None if task: bin_data = await ensure_future( task[0].get_image_bytes() ) # get_image_bytes()为协程,result[0]为最新的图片 avatar_path = os.path.join( app.getAvatarRootPath(self.core.jid.localpart), '{}.jpg'.format(friend['jid'])) FileUtils.savaToPng(avatar_path, bin_data) friend['avatar_path'] = avatar_path self.mWin.loadData(friend) except InvalidResponseException: Log.info("RestAPi", "获取好友列表失败")
def checkMemberillust_medium(memberillusturl, pid, medium_id, soup=None): sections = soup.find('section', attrs={'class': 'manga'}) for section in sections.findAll('div', attrs={'class': 'item-container'}): img = section.find('a', attrs={'class': "full-size-container _ui-tooltip"}) HTMLData = HttpUtils.GET("%s%s" % (indexurl, img['href'])) soup = HTMLUtils.get_soup(HTMLData.text) img = soup.find('img') imgFile = FileUtils.joinPath(portraitDir, FileUtils.getfile(img['src'])) if FileUtils.exists(imgFile): print("文件已存在[%s]" % imgFile) else: print("开始下载图片[%s]" % imgFile) downLoad(img['src'], imgFile) print("下载图片[%s]结束" % imgFile)
async def getMembersAvatar(self, memberList): rootPath = os.path.join(app.getAvatarRootPath(self.core.jid.localpart), self.room_jid) if not os.path.exists(rootPath): os.mkdir(rootPath) self.room_path = rootPath for member in memberList: metadata = await asyncio.ensure_future( self._muc_avatarService.get_avatar_metadata( JID.fromstr(member))) if metadata: picture = await asyncio.ensure_future( metadata[0].get_image_bytes()) picPath = os.path.join(rootPath, '{}.jpg'.format(member)) FileUtils.savaToPng(picPath, picture) self.loadMember(memberList)
def load_from(file_path): """loads data from a defined file path :type file_path: str :param file_path: path from which to read data :return list """ return utils.read_from_file(file_path)
def main(): args = parse_options() print(args) device_type = args.type.lower() device_udid = args.udid execution_mode = args.mode.lower() config_file = args.config if device_type is None or device_type not in ["android", "ios"]: print("device type 参数指定错误; 使用 --help 命令查看详情") sys.exit(1) if device_udid is None: print("device udid 参数指定错误; 使用 --help 命令查看详情") sys.exit(1) if execution_mode is None or execution_mode not in EXECUTION_MODE: print("execution mode 参数指定错误; 使用 --help 命令查看详情") sys.exit(1) if config_file is None or FileUtils.file_available(config_file) is False: print("config file 参数指定错误; 使用 --help 命令查看详情") sys.exit(1) logger.info("config: device type is {0}".format(device_type)) logger.info("config: device udid is {0}".format(device_udid)) logger.info("config: execution mode is {0}".format(execution_mode)) logger.info("config: yaml config file is {0}".format(config_file)) builder = Builder(config_file=config_file, udid=device_udid, mode=execution_mode, device_type=device_type) builder.exe()
def main(): config_file = './data/configs/network.txt' initial_weights_file = './data/configs/initial_weights.txt' dataset_file = './data/datasets/wine.txt' fileUtils = FileUtils(dataset_file=dataset_file, config_file=config_file) dataset = fileUtils.getDataset() #normalized_dataset = normalizeDataset(dataset) neurons_per_layer = [1, 2, 1] network = NeuralNetwork(config_file=config_file, dataset=dataset, initial_weights_file=initial_weights_file, neurons_per_layer=neurons_per_layer) network.backpropagation()
def train_classifier(data_folder_path="face_augmented"): print("train_classifier", "start") X = [] y = [] print("train_classifier", "embed_faces", "start") for label in tqdm(FileUtils.list_top_folders_names(data_folder_path)): faces = [] for file_name in FileUtils.list_top_files_names(FileUtils.join(data_folder_path, label)): face = Image.open(FileUtils.join(data_folder_path, label, file_name)) faces.append(face) embeds = ImageEmbedder.embeds(faces) X += [embed for embed in embeds] y += [label] * len(faces) print("train_classifier", "embed_faces", "completed") print("train_classifier", "fit_classifier", "start") FaceClassifier.fit(numpy.array(X), numpy.array(y)) print("train_classifier", "fit_classifier", "completed")
def work_workStep1(url, pid, imgPid, soup=None): imgName = getName(soup) imgTags = getTags(soup) img = soup.find('img', attrs={'alt': imgName, 'class': 'original-image'}) imgSrc = img['data-src'] headers = copy.copy(HttpUtils.downLoadHead) headers[':path'] = imgSrc[imgSrc.find('.pixiv.net/')+10:] headers['referer'] = headers['referer'].replace("{id}", pid).replace('{imgid}', imgPid) downLoad(imgSrc, portraitDir + FileUtils.getfile(imgSrc), headers)
async def get_quote(ctx, id, user_id): quote = QuoteEnum.get_by_id(id) if quote == QuoteEnum.RANDOM: return await QuoteEnum.get_random_quote_from_history( ctx.guild.text_channels, user_id) elif quote.value.file: return FileUtils.get_file(quote.value.quote) else: return quote.value
def downLoad(src, imgFile, headers): try: HTMLUtils.downLoad_HTMLImg(src, imgFile, headers=headers) im = Image.open(imgFile) x, y = im.size print({'imagename': FileUtils.getfile(imgFile), 'image_x': x, 'image_y': y}) except Exception as error: print("downLoad function error:" + error) raise
def generate_map(data_file, to_file, precision=PRECISION_ONE_METER): # title = 'GPS rows (as DBs) - 2014-06-25 - 22-05-34' title = FileUtils.get_file_name(data_file) map_data = pandas.read_csv(data_file) map_data.columns = ['gps', 'count'] generate_google_maps_html(map_data[:], to_file, title, precision=precision) # plot_map(map_data[:]) print 'Done.'
def test_get_empty_file(self): returned_f = FileUtils.get_empty_file() self.assertEqual(returned_f, { 'version': FileUtils.LAST_FILE_VERSION, 'content': '', 'currentVideo': None, 'videos': {}, 'syncNotesVideo': True })
def get(self, file_id): client = self.get_authorized_client() if not client: return self.redirect('/auth-evernote?next={0}'.format(self.request.path)) try: file = self.get_file(file_id) except webapp2.HTTPException as http_ex: if http_ex.code == 401: return self.redirect('/auth?next={0}'.format(self.request.path)) base_url = self.request.host_url + '/edit/{0}'.format(file['id']) extension_loaded = bool(int(self.request.get('extensionLoaded', 1))) # Look for the VideoNot.es Notebook notestore = client.get_note_store() notesbooks = notestore.listNotebooks() notebook = None for a_notesbook in notesbooks: if a_notesbook.name == 'VideoNot.es': notebook = a_notesbook break if not notebook: notebook = Notebook() notebook.name = 'VideoNot.es' notebook = notestore.createNotebook(notebook) # Formatting the note in ENML content_enml = FileUtils.to_ENML(file, base_url) content_enml.append('<br></br><br></br>') content_enml.append('<a href="{0}">View in VideoNot.es</a>'.format(base_url)) if not extension_loaded: content_enml.append('<br/>') content_enml.append('(Tip: you can add snapshots of the video to your export by installing our <a href="https://chrome.google.com/webstore/detail/kmbcnighpdagelfjmlbakfnciogcelgi">Chrome Extension</a>)') # Saving the note in Evernote note = Note() note.title = file['title'] note_content = ''.join(content_enml).encode('utf-8') note.content = '<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE en-note SYSTEM "http://xml.evernote.com/pub/enml2.dtd"><en-note>{0}</en-note>'.format(note_content) note.content = note.content.replace('&', '&') if notebook: note.notebookGuid = notebook.guid note = notestore.createNote(note) logging.debug('VideoNot.es %s exported to Evernote: %s', file_id, note_content) logging.info('VideoNot.es %s exported to Evernote with id: %s', file_id, note.guid) # Returning to the new note in Evernote user_store = client.get_user_store() notestore_url = '/'.join(user_store.getNoteStoreUrl().split('/')[0:5]) return self.redirect(notestore_url + '/view/notebook/{0}'.format(note.guid))
class PlotAlgoritmos: ALGORITMOS = ['Bubble Sort', 'Binary Insertion Sort', 'Quick Sort'] def __init__(self): self.file_utils = FileUtils() self.tempo_algoritmos = TempoAlgoritmos() self._definir_o_estilo_de_plot() self._gerar_sub_plot_de_cada_quantidade_de_numeros() def _definir_o_estilo_de_plot(self): pyplot.style.use("classic") def _gerar_sub_plot_de_cada_quantidade_de_numeros(self): sub_plots = matplotlib.pyplot.subplots(3) self.sub_plot_mil_numeros = sub_plots[1][0] self.sub_plot_cinco_mil_numeros = sub_plots[1][1] self.sub_plot_dez_mil_numeros = sub_plots[1][2] def mostrar_plot_de_todas_as_quantias_de_numeros(self): self._definir_nomes_dos_labels_de_cada_sub_plot() pyplot.subplots_adjust(hspace=0.5) pyplot.show() def _definir_nomes_dos_labels_de_cada_sub_plot(self): self.sub_plot_mil_numeros.set( title="Grafico do Arquivo de 1000 Numeros", ylabel="Tempo") self.sub_plot_cinco_mil_numeros.set( title="Grafico do Arquivo de 5000 Numeros", ylabel="Tempo") self.sub_plot_dez_mil_numeros.set( title="Grafico do Arquivo de 10000 Numeros", ylabel="Tempo") def adicionar_tempo_de_mil_numeros_ao_plot(self): tempos = self._obter_tempo_de_execucao_dos_algoritmos() self.sub_plot_mil_numeros.bar(PlotAlgoritmos.ALGORITMOS, tempos) for index, value in enumerate(tempos): self.sub_plot_mil_numeros.text(index, value, str(value)) def adicionar_tempo_de_cinco_mil_numeros_ao_plot(self): tempos = self._obter_tempo_de_execucao_dos_algoritmos() self.sub_plot_cinco_mil_numeros.bar(PlotAlgoritmos.ALGORITMOS, tempos) for index, value in enumerate(tempos): self.sub_plot_cinco_mil_numeros.text(index, value, str(value)) def adicionar_tempo_de_dez_mil_numeros_ao_plot(self): tempos = self._obter_tempo_de_execucao_dos_algoritmos() self.sub_plot_dez_mil_numeros.bar(PlotAlgoritmos.ALGORITMOS, tempos) for index, value in enumerate(tempos): self.sub_plot_dez_mil_numeros.text(index, value, str(value)) def _obter_tempo_de_execucao_dos_algoritmos(self) -> List[float]: numeros = self.file_utils.getNumerosFromFile() tempos = self.tempo_algoritmos.obter_tempo_de_execucao_de_cada_algoritmo( numeros) return tempos
def get_file(self, file_id): service = self.CreateDrive() if service is None: return # Requests are expected to pass the file_id query parameter. logging.info('Get file %s', file_id) if file_id: # Fetch the file metadata by making the service.files().get method of # the Drive API. f = service.files().get(fileId=file_id).execute() downloadUrl = f.get('downloadUrl') # If a download URL is provided in the file metadata, use it to make an # authorized request to fetch the file ontent. Set this content in the # data to return as the 'content' field. If there is no downloadUrl, # just set empty content. if downloadUrl: logging.debug('Downloading the file from %s', downloadUrl) resp, raw_content = service._http.request(downloadUrl) logging.debug('Response status : %s', resp.status) logging.debug('Raw content : %s', raw_content) if resp and resp.status == int(200) and raw_content: try: json_content = json.loads(raw_content) f.update(json_content) except ValueError: logging.info("ValueError when decoding raw content in JSON") f.update(FileUtils.get_empty_file()) else: logging.debug("No content or error response") f.update(FileUtils.get_empty_file()) else: logging.debug('No download URL') f.update(FileUtils.get_empty_file()) else: f = None # Generate a JSON response with the file data and return to the client. return f
def _extract_values_set(self): ''' Loads the node data from the ground truth network file and fetches the values ''' bayesian_network_filename = FileUtils.get_network_filename(self._network_name) nodes_data = None with open(bayesian_network_filename) as file: data = file.read() nodes_data = loads(data) for node in nodes_data['Vdata']: self._variables_values_set[node] = nodes_data['Vdata'][node]['vals']
def test_transformations_v1_to_v2(self): v1_f = { 'content': '', 'video': 'test', 'syncNotesVideo': { 'enabled': True, '1': 10.00 } } v2_f = FileUtils.transformation_v1_to_v2(v1_f) self.assertEqual(v2_f['version'], 2) self.assertNotIn('video', v2_f) self.assertIsNotNone(v2_f['videos']['test']) self.assertEqual(v2_f['videos']['test']['1'], {'time':10.00})
def _extract_data(self, format): ''' Parses the generated data file and populates the data into a dictionary. ''' data_filename = FileUtils.get_observed_data_filename(self._network_name) with open(data_filename) as file: self.get_csv_data(file) if format == 'json': # Now we convert the data to json format keys = self._data_vectors.keys() data_length = len(self._data_vectors[keys[0]]) data_dict_list = [] for i in xrange(data_length): data_dict = {} for j in xrange(len(keys)): data_dict[keys[j]] = self._data_vectors[keys[j]][i] data_dict_list.append(data_dict) self._data_vectors = data_dict_list
def test_flatten_sync(self): input_sync = { 'video1': { 2: { 'time': 0, }, 1: { 'time': 0, } }, 'video2': { 0: { 'time': 0, }, 3: { 'time': 0, } } } expected_output = { 0: { 'url': 'video2', 'time': 0, }, 1: { 'url': 'video1', 'time': 0, }, 2: { 'url': 'video1', 'time': 0, }, 3: { 'url': 'video2', 'time': 0, } } expected_output = collections.OrderedDict(sorted(expected_output.items())) effective_output = FileUtils.flatten_sync(input_sync) self.assertDictEqual(expected_output, effective_output)
def test_to_enml_with_screenshots(self): file = { 'id': 'test', 'content': 'test' + '\n' + '<{0}>'.format(FileUtils.SNAPSHOT_KEY), 'videos': { 'video1': { 2: { 'time': 0, 'snapshot': None }, 1: { 'time': 0, 'snapshot': 'snapshot' } }, 'video2': { 0: { 'time': 0, 'snapshot': None }, 3: { 'time': 0, 'snapshot': None } } } } base_url = 'http://test.videonot.es/edit/' + file['id'] expected_enml = [ '<a href="{0}?l=1">+</a> test'.format(base_url), '<br></br>', '<br></br>', '<img src="{0}"></img>'.format('snapshot'), '<br></br>', '<a href="{0}">{0}</a>'.format('video1'), '<br></br><br></br>' ] content_enml = FileUtils.to_ENML(file, base_url) self.assertEqual(expected_enml, content_enml)
def test_to_enml_without_screenshots(self): file = { 'id': 'test', 'content': 'test' + '\n' + 'test2', 'videos': { 'video1': { 2: { 'time': 0, 'snapshot': None }, 1: { 'time': 0, 'snapshot': None } }, 'video2': { 0: { 'time': 0, 'snapshot': None }, 3: { 'time': 0, 'snapshot': None } } } } base_url = 'http://test.videonot.es/edit/' + file['id'] expected_enml = [ '<a href="{0}?l=1">+</a> test'.format(base_url), '<br></br>', '<a href="{0}?l=2">+</a> test2'.format(base_url), '<br></br>', ] content_enml = FileUtils.to_ENML(file, base_url) self.assertEqual(expected_enml, content_enml)
def get(self): """Called when HTTP GET requests are received by the web application. Use the query parameter file_id to fetch the required file's metadata then content and return it as a JSON object. Since DrEdit deals with text files, it is safe to dump the content directly into JSON, but this is not the case with binary files, where something like Base64 encoding is more appropriate. """ try: f = self.get_file(self.request.get('file_id')) f = FileUtils.transformations(f) self.RespondJSON(f) except AccessTokenRefreshError: # Catch AccessTokenRefreshError which occurs when the API client library # fails to refresh a token. This occurs, for example, when a refresh token # is revoked. When this happens the user is redirected to the # Authorization URL. logging.info('AccessTokenRefreshError') return self.abort(401)
def post(self): """ Called when HTTP POST requests are received by the web application. The POST body is JSON which is deserialized and used as values to create a new file in Drive. The authorization access token for this action is retrieved from the data store. """ # Create a Drive service service = self.CreateDrive() if service is None: return # Load the data that has been posted as JSON logging.debug('Get JSON data') data = self.RequestJSON() logging.debug('JSON data retrieved %s', json.dumps(data)) content = FileUtils.get_content_from_data(data) max_try = 5 for n in range(0, max_try): try: if 'templateId' in data: body = {'title': 'Your notes'} resource = service.files().copy(fileId=data['templateId'], body=body).execute() else: # Create a new file data structure. resource = { 'title': data['title'], 'description': data['description'], 'mimeType': data['mimeType'], } if 'parent' in data and data['parent']: logging.debug('Creating from a parent folder %s', data['parent']) default_folder_id = data['parent'] else: if 'defaultFolderId' in self.session and self.session['defaultFolderId']: default_folder_id = self.session['defaultFolderId'] else: default_folder_list = service.files().list(q='title="VideoNot.es"').execute() if default_folder_list and 'items' in default_folder_list and len(default_folder_list['items']): default_folder_id = default_folder_list['items'][0]['id'] self.session['defaultFolderId'] = default_folder_id else: folder_ressource = { 'title': 'VideoNot.es', 'mimeType': 'application/vnd.google-apps.folder' } default_folder = service.files().insert(body=folder_ressource).execute() default_folder_id = default_folder['id'] self.session['defaultFolderId'] = default_folder_id resource['parents'] = [{'id':default_folder_id}] # Make an insert request to create a new file. A MediaInMemoryUpload # instance is used to upload the file body. logging.debug('Calling Drive API with content %s', str(content)) resource = service.files().insert( body=resource, media_body=MediaInMemoryUpload( content, data['mimeType'], resumable=True) ).execute() if BaseHandler.is_production(): # clement_permission = { # 'value': '*****@*****.**', # 'type': 'user', # 'role': 'reader' # } anyone_permission = { 'type': 'anyone', 'role': 'reader', 'withLink': True } # try: # logging.info('Add Clement as a reader') # service.permissions().insert(fileId=resource['id'], body=clement_permission).execute() # except HttpError: # logging.info('Error when adding Clement as a reader') try: logging.info('Add anyone as a reader') service.permissions().insert(fileId=resource['id'], body=anyone_permission).execute() except HttpError: logging.info('Error when adding anyone as a reader') # Respond with the new file id as JSON. logging.debug('New ID created %s', resource['id']) return self.RespondJSON({'id': resource['id']}) except AccessTokenRefreshError: # In cases where the access token has expired and cannot be refreshed # (e.g. manual token revoking) redirect the user to the authorization page # to authorize. logging.info('AccessTokenRefreshError') return self.abort(401) except HttpError, http_error: logging.getLogger("error").exception("Try #%d: Exception occurred when creating file", n) # HTTP status code 403 indicates that the app is not authorized to save the file (third-party app disabled, user without access, etc.) # Don't need to try several times if http_error.resp.status == 403: return self.abort(403) else: time.sleep((2 ** n) + (random.randint(0, 1000) / 1000)) except HTTPException: logging.getLogger("error").exception("Try #%d: Exception occurred when creating file", n) time.sleep((2 ** n) + (random.randint(0, 1000) / 1000))
def put(self): """ Called when HTTP PUT requests are received by the web application. The PUT body is JSON which is deserialized and used as values to update a file in Drive. The authorization access token for this action is retreived from the data store. """ # Create a Drive service service = self.CreateDrive() if service is None: return # Load the data that has been posted as JSON logging.debug('Get JSON data') data = self.RequestJSON() logging.debug('JSON data retrieved %s', json.dumps(data)) logging.info('Updating file %s', data['id']) # Create a new file data structure. content = FileUtils.get_content_from_data(data) #data['indexableText'] = {'text': content['content']} max_try = 5 for n in range(0, max_try): try: if content is not None: # Make an update request to update the file. A MediaInMemoryUpload # instance is used to upload the file body. Because of a limitation, this # request must be made in two parts, the first to update the metadata, and # the second to update the body. resource = service.files().update( fileId=data['id'], newRevision=self.request.get('newRevision', False), body=data, media_body=MediaInMemoryUpload( content, data['mimeType'], resumable=True) ).execute() else: # Only update the metadata, a patch request is prefered but not yet # supported on Google App Engine; see # http://code.google.com/p/googleappengine/issues/detail?id=6316. resource = service.files().update( fileId=data['id'], newRevision=self.request.get('newRevision', False), body=data).execute() # Respond with the new file id as JSON. return self.RespondJSON({'id': resource['id']}) except HttpError, http_error: logging.getLogger("error").exception("Try #%d: Exception occurred when updating file", n) # HTTP status code 403 indicates that the app is not authorized to save the file (third-party app disabled, user without access, etc.) # Don't need to try several times if http_error.resp.status == 403: return self.abort(403) else: time.sleep((2 ** n) + (random.randint(0, 1000) / 1000)) except HTTPException: logging.getLogger("error").exception("Try #%d: Exception occurred when updating file", n) time.sleep((2 ** n) + (random.randint(0, 1000) / 1000))
def analyze(filename, known_words_filepath, not_known_words_filepath, print_example): known_words = set(FileUtils.read(known_words_filepath).split()) not_known_words = set(FileUtils.read(not_known_words_filepath).split()) tmp_filepath = FileUtils.random_path() output_filepath = tmp_filepath + '.xml' FileUtils.copy(filename, tmp_filepath) croncob_word_list = os.path.join('data', 'corncob_lowercase.txt') word_filter = WordFilterFactory.create_word_filter(croncob_word_list) cmd = ['java', '-cp', 'stanford-corenlp-full/stanford-corenlp-3.3.1.jar:stanford-corenlp-full/stanford-corenlp-3.3.1-models.jar:stanford-corenlp-full/xom.jar:stanford-corenlp-full/joda-time.jar:stanford-corenlp-full/jollyday.jar:stanford-corenlp-full/ejml-0.23.jar', '-Xmx2g', 'edu.stanford.nlp.pipeline.StanfordCoreNLP', '-annotators', 'tokenize,ssplit,pos,lemma', '-file', tmp_filepath, '-outputDirectory', '/tmp/' ] subprocess.call(cmd) raw_output = FileUtils.read(output_filepath) d = xmltodict.parse(raw_output) sentences = d['root']['document']['sentences']['sentence'] candidate_words = defaultdict(dict) def word_filter_fun(word, lemma, tag): del word del tag return word_filter.isok(lemma) def adjective_filter_fun(word, lemma, tag): del word del lemma if tag in ['JJR', 'JJS']: return False else: return True filters = [ word_filter_fun, adjective_filter_fun ] for sentence_dict in sentences: tokens = sentence_dict['tokens']['token'] if not isinstance(tokens, list): continue last_offset = int(tokens[0]['CharacterOffsetBegin']) sentence_raw = '' for token in tokens: word = token['word'] begin_offset = int(token['CharacterOffsetBegin']) sentence_raw += (begin_offset - last_offset) * ' ' sentence_raw += word last_offset = int(token['CharacterOffsetEnd']) for token in tokens: word = token['word'] lemma = token['lemma'] tag = token['POS'] if tag in TAG_TO_PART_OF_SPEECH: ok = True for filter_fun in filters: if not filter_fun(word, lemma, tag): ok = False break if ok: candidate_words[(lemma, TAG_TO_PART_OF_SPEECH[tag])] = { 'example_sentence': sentence_raw, 'word': word } not_known = [] for ((lemma, part_of_speech), d) in candidate_words.iteritems(): if lemma not in known_words and lemma not in not_known_words: not_known.append((lemma, part_of_speech, d)) for (lemma, part_of_speech, d) in not_known: word = d['word'] example_sentence = d['example_sentence'] out = '(%s.) %s' % ( convert_part_of_speech(part_of_speech), lemma ) if print_example: line = utils.fill_suffix(out, 22, ' ') + ' # ' + example_sentence match_pos = re.search(word, example_sentence).start() print line.encode('utf-8') print ((match_pos + 25) * ' ') + (len(word) * '^') else: print out.encode('utf-8')