Ejemplo n.º 1
0
def checkMemberillust(memberillusturl, pid, soup=None):
    #读取标签
    tags = getTags(soup)
    imgName = getName(soup)
    tmp = [val for val in tags if val in mytags]
    if len(tmp) != 0:
        img = soup.find('img', attrs={'alt': imgName})
        if img:
            imgFile = FileUtils.joinPath(portraitDir,
                                         FileUtils.getfile(img['src']))
            if FileUtils.exists(imgFile):
                print("文件已存在[%s]" % imgFile)
            else:
                print("开始下载图片[%s][%s]" % (imgFile, img['src']))
                downLoad(img['src'], imgFile)
                print("下载图片[%s]结束" % imgFile)
        else:
            img = soup.find('div', attrs={'class': 'works_display'})
            if img.find('a', attrs={'class': r' _work multiple '}):
                img1 = img.find('a', attrs={'class': r' _work multiple '})
            if img.find('a', attrs={'class': r' _work manga multiple '}):
                img1 = img.find('a',
                                attrs={'class': r' _work manga multiple '})
            if img1 is not None:
                checkMemberillust_medium(
                    "%s%s%s" % (indexurl, '/', img1['href']), pid,
                    img1['href'].split('=', 2)[2])
Ejemplo n.º 2
0
def decodeGameAPK():
    global backupPath

    if (isNeedDecodeGameAPK):
        print("start decode gameAPK......")
    else:
        print("no need to decode gameAPK.")
        return

    try:
        #delete backup folder
        FileUtils.delDir(backupPath)
    except Exception as ex:
        print("exception :{0}".format(str(ex)))

    gameApk = "games/" + gameApkName + ".apk"
    cmd = "".join([
        "java -classpath ./libs/apktool.jar:. brut.apktool.Main d -f ",
        gameApk, " -o out/temp/" + str(timestamp) + "/" + gameApkName
    ])
    print("cmd is ", cmd)
    (status, output) = commands.getstatusoutput(cmd)
    print output
    print("end decode gameAPK.")
    return
Ejemplo n.º 3
0
    def logger(logger_name=None):
        # Create the root logger.
        logger = logging.getLogger(logger_name)
        logger.setLevel(Log.log_level)

        # Validate the given directory.
        Log.log_dir = os.path.normpath(Log.log_dir)

        # Create a folder for the logfile.
        FileUtils.make_dir(Log.log_dir)

        # Build the logfile name
        filename = Log.file_name_format + ".log"
        filename = os.path.join(Log.log_dir, filename)

        # Set up logging to the logfile
        file_handler = RotatingFileHandler(
            filename=filename
            # ,maxBytes=Log.max_bytes, backupCount=Log.backup_count
        )
        file_handler.setLevel(Log.log_level)
        file_formatter = logging.Formatter(Log.file_msg_format)
        file_handler.setFormatter(file_formatter)
        logger.addHandler(file_handler)

        # Set up logging to console
        stream_handler = logging.StreamHandler()
        stream_handler.setLevel(Log.log_level)
        stream_formatter = logging.Formatter(Log.console_msg_format)
        stream_handler.setFormatter(stream_formatter)
        logger.addHandler(stream_handler)

        return logger
Ejemplo n.º 4
0
    def setup(self, argv):
        """
        Setup the parameters for the simulation. Create result folder and logger file.
        :param argv: Dictionary list of the parameters
        """

        self.config_name = argv[
            "config_name"] if "config_name" in argv else self.root + "/configs/default_dog_vert_simulation_config.json"
        log_file = argv[
            "logfile"] if "logfile" in argv else os.path.expanduser("~").replace("\\", "/") + "/.log/locomotionSim.log"

        if "filename" in argv:
            self.save_file = argv["filename"]
        else:
            dirname = self.root + "/save"
            filename = "sim_" + datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f") + ".qsm"
            if not os.path.exists(dirname):
                os.makedirs(dirname)
            self.save_file = dirname + "/" + filename

        self.genome = eval(argv["genome"]) if "genome" in argv else False

        FileUtils.create_file(log_file)

        logging.config.fileConfig(self.root + "/etc/logging.conf",
                                  defaults={'logfilename': log_file, 'simLevel': "DEBUG"})
Ejemplo n.º 5
0
def rebuild():
    if (isNeedRebuildGameAPK):
        print("start rebuild gameAPK......")
    else:
        print("no need to rebuild gameAPK.")
        return

    apkpath = "out/temp/" + str(timestamp) + "/real_" + gameApkName + ".apk"
    finalApkPath = "out/temp/" + str(timestamp) + "/" + gameApkName + ".apk"

    cmd = "".join([
        "java -classpath ./libs/apktool.jar:. brut.apktool.Main b -o " +
        apkpath, " out/temp/",
        str(timestamp) + "/" + gameApkName
    ])
    print("cmd is ", cmd)
    (status, output) = commands.getstatusoutput(cmd)
    print output

    # sign
    cmd = "jarsigner -verbose  -keystore /Users/lala/Documents/workspace/keystore/android/android/pa_crack.keystore -storepass 123456 -signedjar " + finalApkPath + " -digestalg SHA1 -sigalg MD5withRSA " + apkpath + " pa_crack.keystore"
    print("cmd is ", cmd)
    (status, output) = commands.getstatusoutput(cmd)
    print output

    FileUtils.delFile(apkpath)

    print("end rebuild gameAPK.")
    return
Ejemplo n.º 6
0
def augment_faces(face_folder_path="face", augmented_folder_path="face_augmented", aug_per_image = 8):
    print("augment_faces", "start")

    FileUtils.remove_dirs(augmented_folder_path)
    FileUtils.make_dirs(augmented_folder_path)

    ImageAugmenter.augments(face_folder_path, augmented_folder_path, aug_per_image = aug_per_image)

    print("augment_faces", "completed")
Ejemplo n.º 7
0
 def test_file_read(self):
     """
     Test file read capability
     """
     data="{key:value}"
     file1 = "filew.json"
     futils.write_to_file(file1,data)
     ret = futils.read_from_file(file1)
     self.assertTrue(data,ret)
     os.remove(file1)
Ejemplo n.º 8
0
 def to_local_cache(self):
     if self.mode == "monkey":
         return
     for record in list(self.flow.exe_records.keys()):
         if self.flow.exe_records[record]["status"] == RecordsStatus.DONE:
             FileUtils.add_to_pickle(self.cache_file_name,
                                     self.flow.exe_records[record])
             for k in list(self.flow.exe_records[record].keys()):
                 if k != "status":
                     del self.flow.exe_records[record][k]
Ejemplo n.º 9
0
 def fit(cls, X, y):
     cls.X = X
     cls.classifier = BASE_CLASSIFIER()
     cls.classifier.fit(X, y)
     FileUtils.make_dirs(cls.model_folder)
     model_file = open(cls.classifier_path, "wb")
     pickle.dump(cls.classifier, model_file)
     model_file.close()
     X_file = open(cls.X_path, "wb")
     pickle.dump(cls.X, X_file)
     X_file.close()
Ejemplo n.º 10
0
 def verifyCaptchaByClound(self, type=TYPE_LOGIN_NORMAL_WAY):
     captchaContent = self.getCaptcha(type)
     if captchaContent:
         FileUtils.saveBinary(Captcha.__CAPTCHA_PATH, captchaContent)
     else:
         Log.e('failed to save captcha')
         return None
     results = damatuWeb.verify(Captcha.__CAPTCHA_PATH)
     results = self.__cloundTransCaptchaResults(results)
     Log.v('captchaResult: %s' % results)
     return results, self.check(results)
Ejemplo n.º 11
0
def copyAssets():

    if (isNeedCopyAssets):
        print("start copy assets......")
    else:
        print("no need to copy assets.")
        return

    FileUtils.doCopy(sdkApkPath + "/assets", gameApkPath + "/assets")
    print("end copy assets.")
    return
Ejemplo n.º 12
0
def copyUnkown():

    if (isNeedCopyUnknown):
        print("start copy unknown......")
    else:
        print("no need to copy unknown.")
        return

    FileUtils.doCopy(sdkApkPath + "/unknown", gameApkPath + "/unknown")

    print("end copy unknown.")
    return
Ejemplo n.º 13
0
def copyResCommon():

    if (isNeedCopyResCommon):
        print("start copy resource common......")
    else:
        print("no need to copy resource common.")
        return

    FileUtils.doCopy(sdkApkPath + "/res", gameApkPath + "/res")

    print("end copy res common")
    return
Ejemplo n.º 14
0
def copySmali():

    if (isNeedCopySmali):
        print("start copy smali......")
    else:
        print("no need to copy smali.")
        return

    FileUtils.doCopy(sdkApkPath + "/smali", gameApkPath + "/smali")

    print("end copy smali.")
    return
Ejemplo n.º 15
0
def copyLibSO():

    if (isNeedCopyLibSo):
        print("start copy lib so......")
    else:
        print("no need to copy lib so.")
        return

    FileUtils.doCopy(sdkApkPath + "/lib", gameApkPath + "/lib")

    print("end copy lib so.")
    return
Ejemplo n.º 16
0
 def test_file_write(self):
     """
     Test that file is written successfully
     """
     data = "ABCDEFG"
     file1 = "file.json"
     file2 = "f.json"
     futils.write_to_file(file1,data)
     futils.write_to_file(file2,data)
     dir = os.path.dirname(os.path.abspath(__file__))
     self.assertTrue(filecmp.cmp(os.path.join(dir,file2), os.path.join(dir,file1)))
     os.remove(file2)
     os.remove(file1)
Ejemplo n.º 17
0
 def commit(self, f_name):
     """Adds new object to memory
     :type f_name: str
     :param f_name: name of file to insert object in
     return: id of commited object
     """
     #prevent duplicate entries, clear list and start afresh
     self.objects = dict()
     #generate 4-character id
     id = str(uuid.uuid4())[:8]
     self.objects[id] = self.obj
     utils.write_to_file(f_name, json.dumps(self.objects))
     return id
Ejemplo n.º 18
0
 def __generate_training_and_test_series(self):
     Logger.info("Generating training and test datasets ...")
     self.__preprocess_bios()
     concatenated = PandasUtils.concat_series(
         [self.appropriate_bios, self.inappropriate_bios])
     shuffled = PandasUtils.shuffle_series(concatenated)
     self.training_series = shuffled.head(self.number_of_training_records)
     self.test_series = shuffled.tail(self.number_of_test_records)
     FileUtils.write_series2file(self.bio_train_path, self.training_series)
     FileUtils.write_series2file(self.bio_test_path, self.test_series)
     self.test_list = self.test_series.to_list()
     self.cleaned_test_list = self.__remove_labels(
         self.test_series,
         ['__label__{}'.format(l.value) for l in self.Label]).tolist()
Ejemplo n.º 19
0
    async def getFriendList(self):
        _user = Users('http://{}:{}'.format(Config._host, Config._restPort),
                      Config._restPort_secret)
        try:
            '''
            friendList的数据格式:
            {'rosterItem': [{'jid': '[email protected]', 'nickname': 'chinsing00', 'subscriptionType': 3, 'groups': ['Friends']},{...}]}
            '''
            friend_List = _user.get_user_roster(
                self.core.jid.localpart)  #只需要用户名就行,不需要加域名
            del _user  #清除user,避免服务器生成过多的连接导致服务器连接池溢出
            metadata = await self.avatar_server.get_avatar_metadata(
                self.core.jid)
            if metadata:
                picture = await asyncio.ensure_future(
                    metadata[0].get_image_bytes())
                picPath = os.path.join(
                    app.getAvatarRootPath(self.core.jid.localpart),
                    '{}.jpg'.format(str(self.core.jid)))
                if not os.path.exists(picPath):
                    FileUtils.savaToPng(picPath, picture)
            usericon = QPixmap.fromImage(QImage(picPath)) if os.path.exists(
                picPath) else QPixmap(":src\images\CustomerService.png")
            self.mWin.avatar.setBaseSize(QSize(40, 40))
            temp = utils.PixmapToRound(self.mWin.avatar, usericon)
            self.mWin.avatar.setScaledContents(True)
            self.mWin.avatar.setPixmap(temp)

            for friend in friend_List['rosterItem']:
                # task执行后返回 <AbstractAvatarDescriptor> 对象
                task = await get_event_loop().create_task(
                    self.avatar_server.get_avatar_metadata(JID.fromstr(
                        friend['jid']),
                                                           require_fresh=True,
                                                           disable_pep=False))
                #得到<AbstractAvatarDescriptor> 对象后进行头像保存本地及加载好友列表
                avatar_path = None
                if task:
                    bin_data = await ensure_future(
                        task[0].get_image_bytes()
                    )  # get_image_bytes()为协程,result[0]为最新的图片
                    avatar_path = os.path.join(
                        app.getAvatarRootPath(self.core.jid.localpart),
                        '{}.jpg'.format(friend['jid']))
                    FileUtils.savaToPng(avatar_path, bin_data)
                friend['avatar_path'] = avatar_path
                self.mWin.loadData(friend)
        except InvalidResponseException:
            Log.info("RestAPi", "获取好友列表失败")
Ejemplo n.º 20
0
def checkMemberillust_medium(memberillusturl, pid, medium_id, soup=None):
    sections = soup.find('section', attrs={'class': 'manga'})
    for section in sections.findAll('div', attrs={'class': 'item-container'}):
        img = section.find('a',
                           attrs={'class': "full-size-container _ui-tooltip"})
        HTMLData = HttpUtils.GET("%s%s" % (indexurl, img['href']))
        soup = HTMLUtils.get_soup(HTMLData.text)
        img = soup.find('img')
        imgFile = FileUtils.joinPath(portraitDir,
                                     FileUtils.getfile(img['src']))
        if FileUtils.exists(imgFile):
            print("文件已存在[%s]" % imgFile)
        else:
            print("开始下载图片[%s]" % imgFile)
            downLoad(img['src'], imgFile)
            print("下载图片[%s]结束" % imgFile)
Ejemplo n.º 21
0
 async def getMembersAvatar(self, memberList):
     rootPath = os.path.join(app.getAvatarRootPath(self.core.jid.localpart),
                             self.room_jid)
     if not os.path.exists(rootPath):
         os.mkdir(rootPath)
     self.room_path = rootPath
     for member in memberList:
         metadata = await asyncio.ensure_future(
             self._muc_avatarService.get_avatar_metadata(
                 JID.fromstr(member)))
         if metadata:
             picture = await asyncio.ensure_future(
                 metadata[0].get_image_bytes())
             picPath = os.path.join(rootPath, '{}.jpg'.format(member))
             FileUtils.savaToPng(picPath, picture)
     self.loadMember(memberList)
Ejemplo n.º 22
0
 def load_from(file_path):
     """loads data from a defined file path
     :type file_path: str
     :param file_path: path from which to read data
     :return list
     """
     return utils.read_from_file(file_path)
Ejemplo n.º 23
0
def main():
    args = parse_options()
    print(args)
    device_type = args.type.lower()
    device_udid = args.udid
    execution_mode = args.mode.lower()
    config_file = args.config

    if device_type is None or device_type not in ["android", "ios"]:
        print("device type 参数指定错误; 使用 --help 命令查看详情")
        sys.exit(1)

    if device_udid is None:
        print("device udid 参数指定错误; 使用 --help 命令查看详情")
        sys.exit(1)

    if execution_mode is None or execution_mode not in EXECUTION_MODE:
        print("execution mode 参数指定错误; 使用 --help 命令查看详情")
        sys.exit(1)

    if config_file is None or FileUtils.file_available(config_file) is False:
        print("config file 参数指定错误; 使用 --help 命令查看详情")
        sys.exit(1)

    logger.info("config: device type is {0}".format(device_type))
    logger.info("config: device udid is {0}".format(device_udid))
    logger.info("config: execution mode is {0}".format(execution_mode))
    logger.info("config: yaml config file is {0}".format(config_file))

    builder = Builder(config_file=config_file,
                      udid=device_udid,
                      mode=execution_mode,
                      device_type=device_type)
    builder.exe()
Ejemplo n.º 24
0
def main():
    config_file = './data/configs/network.txt'
    initial_weights_file = './data/configs/initial_weights.txt'
    dataset_file = './data/datasets/wine.txt'

    fileUtils = FileUtils(dataset_file=dataset_file, config_file=config_file)
    dataset = fileUtils.getDataset()

    #normalized_dataset = normalizeDataset(dataset)

    neurons_per_layer = [1, 2, 1]
    network = NeuralNetwork(config_file=config_file,
                            dataset=dataset,
                            initial_weights_file=initial_weights_file,
                            neurons_per_layer=neurons_per_layer)

    network.backpropagation()
Ejemplo n.º 25
0
def train_classifier(data_folder_path="face_augmented"):
    print("train_classifier", "start")
    X = []
    y = []
    print("train_classifier", "embed_faces", "start")
    for label in tqdm(FileUtils.list_top_folders_names(data_folder_path)):
        faces = []
        for file_name in FileUtils.list_top_files_names(FileUtils.join(data_folder_path, label)):
            face = Image.open(FileUtils.join(data_folder_path, label, file_name))
            faces.append(face)
        embeds = ImageEmbedder.embeds(faces)
        X += [embed for embed in embeds]
        y += [label] * len(faces)
    print("train_classifier", "embed_faces", "completed")

    print("train_classifier", "fit_classifier", "start")
    FaceClassifier.fit(numpy.array(X), numpy.array(y))
    print("train_classifier", "fit_classifier", "completed")
Ejemplo n.º 26
0
def work_workStep1(url, pid, imgPid, soup=None):
    imgName = getName(soup)
    imgTags = getTags(soup)
    img = soup.find('img', attrs={'alt': imgName, 'class': 'original-image'})
    imgSrc = img['data-src']
    headers = copy.copy(HttpUtils.downLoadHead)
    headers[':path'] = imgSrc[imgSrc.find('.pixiv.net/')+10:]
    headers['referer'] = headers['referer'].replace("{id}", pid).replace('{imgid}', imgPid)
    downLoad(imgSrc, portraitDir + FileUtils.getfile(imgSrc), headers)
Ejemplo n.º 27
0
 async def get_quote(ctx, id, user_id):
     quote = QuoteEnum.get_by_id(id)
     if quote == QuoteEnum.RANDOM:
         return await QuoteEnum.get_random_quote_from_history(
             ctx.guild.text_channels, user_id)
     elif quote.value.file:
         return FileUtils.get_file(quote.value.quote)
     else:
         return quote.value
Ejemplo n.º 28
0
def downLoad(src, imgFile, headers):
    try:
        HTMLUtils.downLoad_HTMLImg(src, imgFile, headers=headers)
        im = Image.open(imgFile)
        x, y = im.size
        print({'imagename': FileUtils.getfile(imgFile), 'image_x': x, 'image_y': y})
    except Exception as error:
        print("downLoad function error:" + error)
        raise
Ejemplo n.º 29
0
def generate_map(data_file, to_file, precision=PRECISION_ONE_METER):
    #    title = 'GPS rows (as DBs) - 2014-06-25 - 22-05-34'
    title = FileUtils.get_file_name(data_file)

    map_data = pandas.read_csv(data_file)
    map_data.columns = ['gps', 'count']
    generate_google_maps_html(map_data[:], to_file, title, precision=precision)
    #    plot_map(map_data[:])

    print 'Done.'
Ejemplo n.º 30
0
    def test_get_empty_file(self):
        returned_f = FileUtils.get_empty_file()

        self.assertEqual(returned_f, {
            'version': FileUtils.LAST_FILE_VERSION,
            'content': '',
            'currentVideo': None,
            'videos': {},
            'syncNotesVideo': True
        })
Ejemplo n.º 31
0
    def get(self, file_id):
        client = self.get_authorized_client()
        if not client:
            return self.redirect('/auth-evernote?next={0}'.format(self.request.path))

        try:
            file = self.get_file(file_id)
        except webapp2.HTTPException as http_ex:
            if http_ex.code == 401:
                return self.redirect('/auth?next={0}'.format(self.request.path))

        base_url = self.request.host_url + '/edit/{0}'.format(file['id'])
        extension_loaded = bool(int(self.request.get('extensionLoaded', 1)))

        # Look for the VideoNot.es Notebook
        notestore = client.get_note_store()
        notesbooks = notestore.listNotebooks()
        notebook = None

        for a_notesbook in notesbooks:
            if a_notesbook.name == 'VideoNot.es':
                notebook = a_notesbook
                break

        if not notebook:
            notebook = Notebook()
            notebook.name = 'VideoNot.es'
            notebook = notestore.createNotebook(notebook)

        # Formatting the note in ENML
        content_enml = FileUtils.to_ENML(file, base_url)
        content_enml.append('<br></br><br></br>')

        content_enml.append('<a href="{0}">View in VideoNot.es</a>'.format(base_url))
        if not extension_loaded:
            content_enml.append('<br/>')
            content_enml.append('(Tip: you can add snapshots of the video to your export by installing our <a href="https://chrome.google.com/webstore/detail/kmbcnighpdagelfjmlbakfnciogcelgi">Chrome Extension</a>)')

        # Saving the note in Evernote
        note = Note()
        note.title = file['title']
        note_content = ''.join(content_enml).encode('utf-8')
        note.content = '<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE en-note SYSTEM "http://xml.evernote.com/pub/enml2.dtd"><en-note>{0}</en-note>'.format(note_content)
        note.content = note.content.replace('&', '&amp;')
        if notebook:
            note.notebookGuid = notebook.guid
        note = notestore.createNote(note)

        logging.debug('VideoNot.es %s exported to Evernote: %s', file_id, note_content)
        logging.info('VideoNot.es %s exported to Evernote with id: %s', file_id, note.guid)

        # Returning to the new note in Evernote
        user_store = client.get_user_store()
        notestore_url = '/'.join(user_store.getNoteStoreUrl().split('/')[0:5])
        return self.redirect(notestore_url + '/view/notebook/{0}'.format(note.guid))
Ejemplo n.º 32
0
class PlotAlgoritmos:
    ALGORITMOS = ['Bubble Sort', 'Binary Insertion Sort', 'Quick Sort']

    def __init__(self):
        self.file_utils = FileUtils()
        self.tempo_algoritmos = TempoAlgoritmos()
        self._definir_o_estilo_de_plot()
        self._gerar_sub_plot_de_cada_quantidade_de_numeros()

    def _definir_o_estilo_de_plot(self):
        pyplot.style.use("classic")

    def _gerar_sub_plot_de_cada_quantidade_de_numeros(self):
        sub_plots = matplotlib.pyplot.subplots(3)
        self.sub_plot_mil_numeros = sub_plots[1][0]
        self.sub_plot_cinco_mil_numeros = sub_plots[1][1]
        self.sub_plot_dez_mil_numeros = sub_plots[1][2]

    def mostrar_plot_de_todas_as_quantias_de_numeros(self):
        self._definir_nomes_dos_labels_de_cada_sub_plot()
        pyplot.subplots_adjust(hspace=0.5)
        pyplot.show()

    def _definir_nomes_dos_labels_de_cada_sub_plot(self):
        self.sub_plot_mil_numeros.set(
            title="Grafico do Arquivo de 1000 Numeros", ylabel="Tempo")
        self.sub_plot_cinco_mil_numeros.set(
            title="Grafico do Arquivo de 5000 Numeros", ylabel="Tempo")
        self.sub_plot_dez_mil_numeros.set(
            title="Grafico do Arquivo de 10000 Numeros", ylabel="Tempo")

    def adicionar_tempo_de_mil_numeros_ao_plot(self):
        tempos = self._obter_tempo_de_execucao_dos_algoritmos()
        self.sub_plot_mil_numeros.bar(PlotAlgoritmos.ALGORITMOS, tempos)
        for index, value in enumerate(tempos):
            self.sub_plot_mil_numeros.text(index, value, str(value))

    def adicionar_tempo_de_cinco_mil_numeros_ao_plot(self):
        tempos = self._obter_tempo_de_execucao_dos_algoritmos()
        self.sub_plot_cinco_mil_numeros.bar(PlotAlgoritmos.ALGORITMOS, tempos)
        for index, value in enumerate(tempos):
            self.sub_plot_cinco_mil_numeros.text(index, value, str(value))

    def adicionar_tempo_de_dez_mil_numeros_ao_plot(self):
        tempos = self._obter_tempo_de_execucao_dos_algoritmos()
        self.sub_plot_dez_mil_numeros.bar(PlotAlgoritmos.ALGORITMOS, tempos)
        for index, value in enumerate(tempos):
            self.sub_plot_dez_mil_numeros.text(index, value, str(value))

    def _obter_tempo_de_execucao_dos_algoritmos(self) -> List[float]:
        numeros = self.file_utils.getNumerosFromFile()
        tempos = self.tempo_algoritmos.obter_tempo_de_execucao_de_cada_algoritmo(
            numeros)

        return tempos
Ejemplo n.º 33
0
    def get_file(self, file_id):
        service = self.CreateDrive()
        if service is None:
            return

        # Requests are expected to pass the file_id query parameter.
        logging.info('Get file %s', file_id)

        if file_id:
            # Fetch the file metadata by making the service.files().get method of
            # the Drive API.
            f = service.files().get(fileId=file_id).execute()
            downloadUrl = f.get('downloadUrl')
            # If a download URL is provided in the file metadata, use it to make an
            # authorized request to fetch the file ontent. Set this content in the
            # data to return as the 'content' field. If there is no downloadUrl,
            # just set empty content.
            if downloadUrl:
                logging.debug('Downloading the file from %s', downloadUrl)
                resp, raw_content = service._http.request(downloadUrl)
                logging.debug('Response status : %s', resp.status)
                logging.debug('Raw content : %s', raw_content)
                if resp and resp.status == int(200) and raw_content:
                    try:
                        json_content = json.loads(raw_content)
                        f.update(json_content)
                    except ValueError:
                        logging.info("ValueError when decoding raw content in JSON")
                        f.update(FileUtils.get_empty_file())
                else:
                    logging.debug("No content or error response")
                    f.update(FileUtils.get_empty_file())
            else:
                logging.debug('No download URL')
                f.update(FileUtils.get_empty_file())
        else:
            f = None
            # Generate a JSON response with the file data and return to the client.

        return f
Ejemplo n.º 34
0
    def _extract_values_set(self):
        '''
        Loads the node data from the ground truth network file and
        fetches the values
        '''
        bayesian_network_filename = FileUtils.get_network_filename(self._network_name)
        nodes_data = None
        with open(bayesian_network_filename) as file:
            data = file.read()
            nodes_data = loads(data)

        for node in nodes_data['Vdata']:
            self._variables_values_set[node] = nodes_data['Vdata'][node]['vals']
Ejemplo n.º 35
0
    def test_transformations_v1_to_v2(self):
        v1_f = {
            'content': '',
            'video': 'test',
            'syncNotesVideo': {
                'enabled': True,
                '1': 10.00
            }
        }

        v2_f = FileUtils.transformation_v1_to_v2(v1_f)

        self.assertEqual(v2_f['version'], 2)
        self.assertNotIn('video', v2_f)
        self.assertIsNotNone(v2_f['videos']['test'])
        self.assertEqual(v2_f['videos']['test']['1'], {'time':10.00})
Ejemplo n.º 36
0
 def _extract_data(self, format):
     '''
     Parses the generated data file and populates the data into a dictionary.
     '''
     data_filename = FileUtils.get_observed_data_filename(self._network_name)
     with open(data_filename) as file:
         self.get_csv_data(file)
         if format == 'json':
             #  Now we convert the data to json format
             keys = self._data_vectors.keys()
             data_length = len(self._data_vectors[keys[0]])
             data_dict_list = []
             for i in xrange(data_length):
                 data_dict = {}
                 for j in xrange(len(keys)):
                     data_dict[keys[j]] = self._data_vectors[keys[j]][i]
                 data_dict_list.append(data_dict)
             self._data_vectors = data_dict_list
Ejemplo n.º 37
0
    def test_flatten_sync(self):
        input_sync = {
            'video1': {
                2: {
                    'time': 0,
                },
                1: {
                    'time': 0,
                }
            },
            'video2': {
                0: {
                    'time': 0,
                },
                3: {
                    'time': 0,
                }
            }
        }

        expected_output = {
            0: {
                'url': 'video2',
                'time': 0,
            },
            1: {
                'url': 'video1',
                'time': 0,
            },
            2: {
                'url': 'video1',
                'time': 0,
            },
            3: {
                'url': 'video2',
                'time': 0,
            }
        }

        expected_output = collections.OrderedDict(sorted(expected_output.items()))
        effective_output = FileUtils.flatten_sync(input_sync)

        self.assertDictEqual(expected_output, effective_output)
Ejemplo n.º 38
0
    def test_to_enml_with_screenshots(self):
        file = {
            'id': 'test',
            'content': 'test' + '\n' + '<{0}>'.format(FileUtils.SNAPSHOT_KEY),
            'videos': {
                'video1': {
                    2: {
                        'time': 0,
                        'snapshot': None
                    },
                    1: {
                        'time': 0,
                        'snapshot': 'snapshot'
                    }
                },
                'video2': {
                    0: {
                        'time': 0,
                        'snapshot': None
                    },
                    3: {
                        'time': 0,
                        'snapshot': None
                    }
                }
            }
        }

        base_url = 'http://test.videonot.es/edit/' + file['id']

        expected_enml = [
            '<a href="{0}?l=1">+</a> test'.format(base_url),
            '<br></br>',
            '<br></br>',
            '<img src="{0}"></img>'.format('snapshot'),
            '<br></br>',
            '<a href="{0}">{0}</a>'.format('video1'),
            '<br></br><br></br>'
        ]

        content_enml = FileUtils.to_ENML(file, base_url)
        self.assertEqual(expected_enml, content_enml)
Ejemplo n.º 39
0
    def test_to_enml_without_screenshots(self):
        file = {
            'id': 'test',
            'content': 'test' + '\n' + 'test2',
            'videos': {
                'video1': {
                    2: {
                        'time': 0,
                        'snapshot': None
                    },
                    1: {
                        'time': 0,
                        'snapshot': None
                    }
                },
                'video2': {
                    0: {
                        'time': 0,
                        'snapshot': None
                    },
                    3: {
                        'time': 0,
                        'snapshot': None
                    }
                }
            }
        }

        base_url = 'http://test.videonot.es/edit/' + file['id']

        expected_enml = [
            '<a href="{0}?l=1">+</a> test'.format(base_url),
            '<br></br>',
            '<a href="{0}?l=2">+</a> test2'.format(base_url),
            '<br></br>',
        ]

        content_enml = FileUtils.to_ENML(file, base_url)
        self.assertEqual(expected_enml, content_enml)
Ejemplo n.º 40
0
    def get(self):
        """Called when HTTP GET requests are received by the web application.

        Use the query parameter file_id to fetch the required file's metadata then
        content and return it as a JSON object.

        Since DrEdit deals with text files, it is safe to dump the content directly
        into JSON, but this is not the case with binary files, where something like
        Base64 encoding is more appropriate.
        """
        try:
            f = self.get_file(self.request.get('file_id'))
            f = FileUtils.transformations(f)

            self.RespondJSON(f)
        except AccessTokenRefreshError:
            # Catch AccessTokenRefreshError which occurs when the API client library
            # fails to refresh a token. This occurs, for example, when a refresh token
            # is revoked. When this happens the user is redirected to the
            # Authorization URL.
            logging.info('AccessTokenRefreshError')
            return self.abort(401)
Ejemplo n.º 41
0
    def post(self):
        """
        Called when HTTP POST requests are received by the web application.

        The POST body is JSON which is deserialized and used as values to create a
        new file in Drive. The authorization access token for this action is
        retrieved from the data store.
        """

        # Create a Drive service
        service = self.CreateDrive()
        if service is None:
            return

        # Load the data that has been posted as JSON
        logging.debug('Get JSON data')
        data = self.RequestJSON()
        logging.debug('JSON data retrieved %s', json.dumps(data))

        content = FileUtils.get_content_from_data(data)

        max_try = 5
        for n in range(0, max_try):
            try:
                if 'templateId' in data:
                    body = {'title': 'Your notes'}
                    resource = service.files().copy(fileId=data['templateId'], body=body).execute()
                else:
                    # Create a new file data structure.
                    resource = {
                        'title': data['title'],
                        'description': data['description'],
                        'mimeType': data['mimeType'],
                    }

                    if 'parent' in data and data['parent']:
                        logging.debug('Creating from a parent folder %s', data['parent'])
                        default_folder_id = data['parent']
                    else:
                        if 'defaultFolderId' in self.session and self.session['defaultFolderId']:
                            default_folder_id = self.session['defaultFolderId']
                        else:
                            default_folder_list = service.files().list(q='title="VideoNot.es"').execute()
                            if default_folder_list and 'items' in default_folder_list and len(default_folder_list['items']):
                                default_folder_id = default_folder_list['items'][0]['id']
                                self.session['defaultFolderId'] = default_folder_id
                            else:
                                folder_ressource = {
                                    'title': 'VideoNot.es',
                                    'mimeType': 'application/vnd.google-apps.folder'
                                }
                                default_folder = service.files().insert(body=folder_ressource).execute()
                                default_folder_id = default_folder['id']
                                self.session['defaultFolderId'] = default_folder_id
                    resource['parents'] = [{'id':default_folder_id}]

                    # Make an insert request to create a new file. A MediaInMemoryUpload
                    # instance is used to upload the file body.
                    logging.debug('Calling Drive API with content %s', str(content))
                    resource = service.files().insert(
                        body=resource,
                        media_body=MediaInMemoryUpload(
                            content,
                            data['mimeType'],
                            resumable=True)
                    ).execute()

                    if BaseHandler.is_production():
                        # clement_permission = {
                        #     'value': '*****@*****.**',
                        #     'type': 'user',
                        #     'role': 'reader'
                        # }

                        anyone_permission = {
                            'type': 'anyone',
                            'role': 'reader',
                            'withLink': True
                        }

                        # try:
                        #     logging.info('Add Clement as a reader')
                        #     service.permissions().insert(fileId=resource['id'], body=clement_permission).execute()
                        # except HttpError:
                        #     logging.info('Error when adding Clement as a reader')

                        try:
                            logging.info('Add anyone as a reader')
                            service.permissions().insert(fileId=resource['id'], body=anyone_permission).execute()
                        except HttpError:
                            logging.info('Error when adding anyone as a reader')

                # Respond with the new file id as JSON.
                logging.debug('New ID created %s', resource['id'])
                return self.RespondJSON({'id': resource['id']})
            except AccessTokenRefreshError:
                # In cases where the access token has expired and cannot be refreshed
                # (e.g. manual token revoking) redirect the user to the authorization page
                # to authorize.
                logging.info('AccessTokenRefreshError')
                return self.abort(401)
            except HttpError, http_error:
                logging.getLogger("error").exception("Try #%d: Exception occurred when creating file", n)
                # HTTP status code 403 indicates that the app is not authorized to save the file (third-party app disabled, user without access, etc.)
                # Don't need to try several times
                if http_error.resp.status == 403:
                    return self.abort(403)
                else:
                    time.sleep((2 ** n) + (random.randint(0, 1000) / 1000))
            except HTTPException:
                logging.getLogger("error").exception("Try #%d: Exception occurred when creating file", n)
                time.sleep((2 ** n) + (random.randint(0, 1000) / 1000))
Ejemplo n.º 42
0
    def put(self):
        """
        Called when HTTP PUT requests are received by the web application.

        The PUT body is JSON which is deserialized and used as values to update
        a file in Drive. The authorization access token for this action is
        retreived from the data store.
        """

        # Create a Drive service
        service = self.CreateDrive()
        if service is None:
            return

        # Load the data that has been posted as JSON
        logging.debug('Get JSON data')
        data = self.RequestJSON()
        logging.debug('JSON data retrieved %s', json.dumps(data))

        logging.info('Updating file %s', data['id'])

        # Create a new file data structure.
        content = FileUtils.get_content_from_data(data)
        #data['indexableText'] = {'text': content['content']}

        max_try = 5
        for n in range(0, max_try):
            try:
                if content is not None:
                    # Make an update request to update the file. A MediaInMemoryUpload
                    # instance is used to upload the file body. Because of a limitation, this
                    # request must be made in two parts, the first to update the metadata, and
                    # the second to update the body.
                    resource = service.files().update(
                        fileId=data['id'],
                        newRevision=self.request.get('newRevision', False),
                        body=data,
                        media_body=MediaInMemoryUpload(
                            content, data['mimeType'], resumable=True)
                    ).execute()
                else:
                    # Only update the metadata, a patch request is prefered but not yet
                    # supported on Google App Engine; see
                    # http://code.google.com/p/googleappengine/issues/detail?id=6316.
                    resource = service.files().update(
                        fileId=data['id'],
                        newRevision=self.request.get('newRevision', False),
                        body=data).execute()
                    # Respond with the new file id as JSON.
                return self.RespondJSON({'id': resource['id']})
            except HttpError, http_error:
                logging.getLogger("error").exception("Try #%d: Exception occurred when updating file", n)
                # HTTP status code 403 indicates that the app is not authorized to save the file (third-party app disabled, user without access, etc.)
                # Don't need to try several times
                if http_error.resp.status == 403:
                    return self.abort(403)
                else:
                    time.sleep((2 ** n) + (random.randint(0, 1000) / 1000))
            except HTTPException:
                logging.getLogger("error").exception("Try #%d: Exception occurred when updating file", n)
                time.sleep((2 ** n) + (random.randint(0, 1000) / 1000))
Ejemplo n.º 43
0
def analyze(filename, known_words_filepath, not_known_words_filepath, print_example):
    known_words = set(FileUtils.read(known_words_filepath).split())
    not_known_words = set(FileUtils.read(not_known_words_filepath).split())
    tmp_filepath = FileUtils.random_path()
    output_filepath = tmp_filepath + '.xml'
    FileUtils.copy(filename, tmp_filepath)

    croncob_word_list = os.path.join('data', 'corncob_lowercase.txt')
    word_filter = WordFilterFactory.create_word_filter(croncob_word_list)

    cmd = ['java',
           '-cp',
           'stanford-corenlp-full/stanford-corenlp-3.3.1.jar:stanford-corenlp-full/stanford-corenlp-3.3.1-models.jar:stanford-corenlp-full/xom.jar:stanford-corenlp-full/joda-time.jar:stanford-corenlp-full/jollyday.jar:stanford-corenlp-full/ejml-0.23.jar',
           '-Xmx2g',
           'edu.stanford.nlp.pipeline.StanfordCoreNLP',
           '-annotators',
           'tokenize,ssplit,pos,lemma',
           '-file',
           tmp_filepath,
           '-outputDirectory',
           '/tmp/'
    ]
    subprocess.call(cmd)
    raw_output = FileUtils.read(output_filepath)
    d = xmltodict.parse(raw_output)
    sentences = d['root']['document']['sentences']['sentence']

    candidate_words = defaultdict(dict)

    def word_filter_fun(word, lemma, tag):
        del word
        del tag
        return word_filter.isok(lemma)

    def adjective_filter_fun(word, lemma, tag):
        del word
        del lemma
        if tag in ['JJR', 'JJS']:
            return False
        else:
            return True

    filters = [
            word_filter_fun,
            adjective_filter_fun
    ]

    for sentence_dict in sentences:
        tokens = sentence_dict['tokens']['token']
        if not isinstance(tokens, list):
            continue

        last_offset = int(tokens[0]['CharacterOffsetBegin'])
        sentence_raw = ''
        for token in tokens:
            word = token['word']
            begin_offset = int(token['CharacterOffsetBegin'])
            sentence_raw += (begin_offset - last_offset) * ' '
            sentence_raw += word
            last_offset = int(token['CharacterOffsetEnd'])

        for token in tokens:
            word = token['word']
            lemma = token['lemma']
            tag = token['POS']

            if tag in TAG_TO_PART_OF_SPEECH:
                ok = True
                for filter_fun in filters:
                    if not filter_fun(word, lemma, tag):
                        ok = False
                        break
                if ok:
                    candidate_words[(lemma, TAG_TO_PART_OF_SPEECH[tag])] = {
                        'example_sentence': sentence_raw,
                        'word': word
                    }

    not_known = []
    for ((lemma, part_of_speech), d) in candidate_words.iteritems():
        if lemma not in known_words and lemma not in not_known_words:
            not_known.append((lemma, part_of_speech, d))

    for (lemma, part_of_speech, d) in not_known:
        word = d['word']
        example_sentence = d['example_sentence']
        out = '(%s.) %s' % (
            convert_part_of_speech(part_of_speech),
            lemma
            )

        if print_example:
            line = utils.fill_suffix(out, 22, ' ') + ' # ' + example_sentence
            match_pos = re.search(word, example_sentence).start()
            print line.encode('utf-8')
            print ((match_pos + 25) * ' ') + (len(word) * '^')
        else:
            print out.encode('utf-8')