Exemplo n.º 1
0
def load_revs(title, revid):
    url = 'https://ru.wikipedia.org/w/api.php?action=query&prop=revisions&rvstartid=' + revid \
          + '&rvprop=flags|timestamp|user|comment|ids|flagged|userid&titles=' + title \
          + '&rvlimit=50&format=json'
    res = requests.get(url)
    response = next(iter(res.json()["query"]["pages"].values()))  # type: dict

    raw_revs = response['revisions']
    if len(raw_revs) == 1:
        raise SkippedException()

    revs = [
        Revision(
            id=x['revid'],
            timestamp=parse_mw_date(
                x['timestamp']
            ),  #datetime.strptime(x['timestamp']+" UTC", '%Y-%m-%dT%H:%M:%SZ %z'),
            comment=x['comment'] if 'comment' in x else '',
            minor='minor' in x,
            flagged='flagged' in x,
            text='',
            user=User(id=None if 'userid' not in x or x['userid'] == 0 else
                      x['userid'],
                      name=None if 'user' not in x else x['user'],
                      flags=(users.get_flags(x['userid']) or [])
                      if 'userid' in x else [])) for x in raw_revs
    ]  # type: [Revision]
    revs.reverse()

    last_revision = revs[-1]  # type: Revision
    if 'bot' in last_revision.user.flags \
            or PageProcessor.is_trusted(last_revision.user.flags):
        raise TrustedUserException()

    last_trusted = None
    last_flagged = None
    for rev in reversed(revs):
        if last_trusted is None:
            if PageProcessor.is_trusted(rev.user.flags) or rev.flagged:
                last_trusted = rev

        if last_flagged is None:
            if rev.flagged:
                last_flagged = rev

    session_start = RevTools.session_start(revs)

    return {
        "page": {
            "id": response['pageid'],
            "title": title,
            "ns": response['ns']
        },
        'revs': [x.to_object() for x in revs],
        'last_trusted': None if last_trusted is None else last_trusted.id,
        'last_flagged':
        None if last_flagged is None else last_flagged.to_object(),
        'session_start': session_start.id
    }
Exemplo n.º 2
0
    def test__get_document_id(self):
        revision = Revision(
            id=1,
            content='hello',
            timestamp=datetime(2020, 1, 1, 1, 1, 1),
            document_id=5
        )

        assert revision.get_document_id() == 5
    def build_document(self, data_dict: dict) -> None:
        """
        Build a Document from the data passed in.

        :param data_dict: dict
        """
        self._validate_data(data_dict)

        document = self.document_repository.get_by_title(data_dict['title'])

        if document:
            new_revision = Revision(content=data_dict['content'])
            document.revisions.append(new_revision)
            self.document_repository.save(document)
        else:
            revision = Revision(content=data_dict['content'])
            document = Document(title=data_dict['title'], revisions=[revision])
            self.document_repository.save(document)
Exemplo n.º 4
0
    def test__get_latest_revision(self):
        timestamp = datetime(2020, 1, 1, 1, 1, 1)
        timestamp_latest = datetime(2020, 2, 1, 1, 1, 1)

        revision = Revision(id=1,
                            content='hello',
                            timestamp=timestamp,
                            document_id=5)

        revision_latest = Revision(id=2,
                                   content='hello again',
                                   timestamp=timestamp_latest,
                                   document_id=5)

        document = Document(id=5,
                            title='blah',
                            revisions=[revision, revision_latest])

        assert document.get_latest_revision() == revision_latest
Exemplo n.º 5
0
    def test__gt_lt(self):
        timestamp_lesser = datetime(2020, 1, 1, 1, 1, 1)
        timestamp_greater = datetime(2020, 1, 1, 1, 1, 2)

        revision_lesser = Revision(
            id=1,
            content='hello',
            timestamp=timestamp_lesser,
            document_id=5
        )

        revision_greater = Revision(
            id=2,
            content='hello',
            timestamp=timestamp_greater,
            document_id=5
        )

        assert revision_greater > revision_lesser
        assert revision_lesser < revision_greater
Exemplo n.º 6
0
    def test__get_revision_by_timestamp(self):
        timestamp = datetime(2020, 1, 1, 1, 1, 1)

        revision = Revision(id=1,
                            content='hello',
                            timestamp=timestamp,
                            document_id=5)

        document = Document(id=5, title='blah', revisions=[revision])

        assert document.get_revision_by_timestamp(timestamp) == revision
Exemplo n.º 7
0
    def test__get_revision_by_timestamp_expect_most_recent(self):
        timestamp = datetime(2020, 1, 1, 1, 1, 1)
        timestamp_latest = datetime(2020, 2, 1, 1, 1, 1)
        timestamp_in_between = datetime(2020, 1, 29, 1, 1, 1)

        revision = Revision(id=1,
                            content='hello',
                            timestamp=timestamp,
                            document_id=5)

        revision_latest = Revision(id=2,
                                   content='hello again',
                                   timestamp=timestamp_latest,
                                   document_id=5)

        document = Document(id=5,
                            title='blah',
                            revisions=[revision, revision_latest])

        assert document.get_revision_by_timestamp(
            timestamp_in_between) == revision
Exemplo n.º 8
0
    def test__ge_le(self):
        timestamp = datetime(2020, 1, 1, 1, 1, 1)
        timestamp_equal = datetime(2020, 1, 1, 1, 1, 1)

        revision = Revision(
            id=1,
            content='hello',
            timestamp=timestamp,
            document_id=5
        )

        revision_equal = Revision(
            id=2,
            content='hello',
            timestamp=timestamp_equal,
            document_id=5
        )

        assert revision >= revision_equal
        assert revision_equal >= revision

        assert revision <= revision_equal
        assert revision_equal <= revision
Exemplo n.º 9
0
    def mutate(self, info, **kwargs):
        sessionId = kwargs.get('session_id')
        uId = kwargs.get('user').id
        content = kwargs.get('content')
        defaultOverlapDuration = 2
        segment = json.loads(content)

        # check file exist
        myFile = Filedrive.query.filter_by(id=segment['fdId']).first()
        if not myFile:
            raise Exception("File not found.")

        # get last revision
        myLastRevision = Revision.query.filter(
            Revision.u_id == uId, Revision.session_id == sessionId).order_by(
                Revision.version.desc()).first()

        if segment['action'] == 'crossfade':
            if not myLastRevision:
                # first revision
                myRevision = Revision(u_id=uId,
                                      session_id=sessionId,
                                      version=1,
                                      content=json.dumps([{
                                          "fdId":
                                          myFile.id,
                                          "start":
                                          0,
                                          "end":
                                          myFile.duration,
                                          "color":
                                          'rgba(%s, %s, %s, 0.1)' %
                                          (r(), g(), b()),
                                          "label":
                                          myFile.name,
                                          "durationOverlap":
                                          segment['crossfadeDuration'],
                                          "type":
                                          "crossfade"
                                      }]),
                                      mixed_id=myFile.id,
                                      files_used=myFile.id)
                save(myRevision)

                return CreateRevision(revision=myRevision)
            else:
                myLastRevisionFile = Filedrive.query.filter_by(
                    id=myLastRevision.mixed_id).first()
                if not myLastRevisionFile:
                    raise Exception("Last revision file not found.")

                # run ffmpeg command then save to revision table.
                currentEpDir = os.path.join(upload_dir,
                                            ("audio/%s/%s" % (uId, sessionId)))
                os.makedirs(currentEpDir, exist_ok=True)
                mixedFilePath = os.path.join(
                    currentEpDir,
                    ("%sx%s.mp3" % (myLastRevisionFile.id, myFile.id)))

                out = subprocess.Popen([
                    'ffmpeg', '-y', '-i',
                    filedrive.getRelativePath('audio',
                                              myLastRevisionFile.path), '-i',
                    filedrive.getRelativePath('audio',
                                              myFile.path), '-filter_complex',
                    '[0][1]acrossfade=d=' + str(segment['crossfadeDuration']) +
                    ':o=1:c1=tri:c2=tri', '-ar', '44100', '-ac', '2', '-b:a',
                    '128k', '-acodec', 'libmp3lame', '-f', 'mp3', mixedFilePath
                ],
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.STDOUT)

                # waiting for command run complete
                stdout, stderr = out.communicate()
                print(stdout)

                # success command
                if os.path.isfile(mixedFilePath):
                    size = os.stat(mixedFilePath).st_size
                    duration = librosa.get_duration(filename=mixedFilePath)

                    # create mixed file
                    new_filedrive = Filedrive(
                        u_id=uId,
                        name=("%sx%s.mp3" %
                              (myLastRevisionFile.id, myFile.id)),
                        path=("%s/%s/%sx%s.mp3" %
                              (uId, sessionId, myLastRevisionFile.id,
                               myFile.id)),
                        size=size,
                        duration=duration,
                        type=Filedrive.TYPE_CROSSFADED,
                        is_tmp=Filedrive.IS_NOT_TMP,
                        is_common=Filedrive.IS_NOT_COMMON)
                    save(new_filedrive)

                    # create revision
                    revisionContent = json.loads(myLastRevision.content)

                    revisionContent.append({
                        "fdId":
                        myFile.id,
                        "start":
                        myLastRevisionFile.duration,
                        "end":
                        duration,
                        "color":
                        'rgba(%s, %s, %s, 0.1)' % (r(), g(), b()),
                        "label":
                        myFile.name,
                        "durationOverlap":
                        segment['crossfadeDuration'],
                        "type":
                        "crossfade"
                    })

                    myRevision = Revision(
                        session_id=sessionId,
                        u_id=uId,
                        version=myLastRevision.version + 1,
                        content=json.dumps(revisionContent),
                        mixed_id=new_filedrive.id,
                        files_used=("%s,%s" %
                                    (myLastRevision.mixed_id, myFile.id)))
                    save(myRevision)

                return CreateRevision(revision=myRevision)
        elif segment['action'] == 'mix':
            myLastRevisionFile = Filedrive.query.filter_by(
                id=myLastRevision.mixed_id).first()
            if not myLastRevisionFile:
                raise Exception("Last revision file not found.")

            # run ffmpeg command then save to revision table.
            currentEpDir = os.path.join(upload_dir,
                                        ("audio/%s/%s" % (uId, sessionId)))
            os.makedirs(currentEpDir, exist_ok=True)
            mixedFilePath = os.path.join(currentEpDir,
                                         ("%sx%s.mp3" %
                                          (myLastRevisionFile.id, myFile.id)))

            out = subprocess.Popen([
                'ffmpeg', '-y', '-i',
                filedrive.getRelativePath('audio',
                                          myLastRevisionFile.path), '-i',
                filedrive.getRelativePath('audio',
                                          myFile.path), '-filter_complex',
                ("[0:a]aformat=sample_fmts=fltp:sample_rates=44100:channel_layouts=stereo[a0];[1:a]aformat=sample_fmts=fltp:sample_rates=44100:channel_layouts=stereo[a1];[a1]atrim=%s:%s[a1trim];[a1trim]adelay=%s|%s[aud1];[aud1]amix=1,apad[a];[a0][a]amerge[a]"
                 %
                 (segment['duration'][0], segment['duration'][1],
                  int(segment['start']) * 1000, int(segment['start']) * 1000)),
                '-map', '[a]', '-ar', '44100', '-ac', '2', '-b:a', '128k',
                '-acodec', 'libmp3lame', '-f', 'mp3', mixedFilePath
            ],
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.STDOUT)

            # waiting for command run complete
            stdout, stderr = out.communicate()
            # print(stdout)

            # success command
            if os.path.isfile(mixedFilePath):
                size = os.stat(mixedFilePath).st_size
                duration = librosa.get_duration(filename=mixedFilePath)

                if segment['type'] == 'try':
                    is_tmp = Filedrive.IS_TMP
                else:
                    is_tmp = Filedrive.IS_NOT_TMP

                # create mixed file
                new_filedrive = Filedrive(
                    u_id=uId,
                    name=("%sx%s.mp3" % (myLastRevisionFile.id, myFile.id)),
                    path=("%s/%s/%sx%s.mp3" %
                          (uId, sessionId, myLastRevisionFile.id, myFile.id)),
                    size=size,
                    duration=duration,
                    type=Filedrive.TYPE_MIXED,
                    is_tmp=is_tmp,
                    is_common=Filedrive.IS_NOT_COMMON)
                save(new_filedrive)

                # fake return old revision if type = try
                if segment['type'] == 'try':
                    return CreateRevision(tmp_file=new_filedrive.path)

                # create revision
                revisionContent = json.loads(myLastRevision.content)

                revisionContent.append({
                    "fdId":
                    myFile.id,
                    "start":
                    segment['start'],
                    "end":
                    int(segment['start']) + (int(segment['duration'][1]) -
                                             int(segment['duration'][0])),
                    "color":
                    'rgba(%s, %s, %s, 0.1)' % (r(), g(), b()),
                    "label":
                    myFile.name,
                    "durationRange":
                    segment['duration'],
                    "type":
                    "mix"
                })

                myRevision = Revision(
                    session_id=sessionId,
                    u_id=uId,
                    version=myLastRevision.version + 1,
                    content=json.dumps(revisionContent),
                    mixed_id=new_filedrive.id,
                    files_used=("%s,%s" %
                                (myLastRevision.mixed_id, myFile.id)))
                save(myRevision)

            return CreateRevision(revision=myRevision)
Exemplo n.º 10
0
    def mutate(self, info, **kwargs):
        uId = kwargs.get('user').id
        sessionId = kwargs.get('session_id')
        version = kwargs.get('version')
        newTracksOrder = kwargs.get('new_tracks_order')

        myRevision = Revision.query.filter(
            Revision.session_id == sessionId,
            Revision.version == version,
        ).order_by(Revision.version.desc()).first()

        revisionContent = json.loads(myRevision.content)

        newArrOrder = []
        newRevisionContent = []
        for index in newTracksOrder.split(','):
            newArrOrder.append(revisionContent[int(index)])

        cmd, fileName = buildCmd(newArrOrder, uId, sessionId)

        # print(cmd.split(" "))
        out = subprocess.Popen(cmd.split(" "),
                               stdout=subprocess.PIPE,
                               stderr=subprocess.STDOUT)

        # waiting for command run complete
        stdout, stderr = out.communicate()
        # print(stdout)

        currentEpDir = os.path.join(upload_dir,
                                    ("audio/%s/%s" % (uId, sessionId)))
        os.makedirs(currentEpDir, exist_ok=True)
        mixedFilePath = os.path.join(currentEpDir, ("%s.mp3" % fileName))

        # success command
        if os.path.isfile(mixedFilePath):
            size = os.stat(mixedFilePath).st_size
            duration = librosa.get_duration(filename=mixedFilePath)

            # create mixed file
            new_filedrive = Filedrive(u_id=uId,
                                      name=("%s.mp3" % fileName),
                                      path=("%s/%s/%s.mp3" %
                                            (uId, sessionId, fileName)),
                                      size=size,
                                      duration=duration,
                                      type=Filedrive.TYPE_MIXED,
                                      is_tmp=Filedrive.IS_NOT_TMP,
                                      is_common=Filedrive.IS_NOT_COMMON)
            save(new_filedrive)

            for idx, track in enumerate(newArrOrder):
                print(idx)
                # print(track)
                myFile = Filedrive.query.filter_by(id=track['fdId']).first()
                # # calculate region start/end to display in visual
                if idx == 0:
                    track['start'] = 0
                    track['end'] = myFile.duration
                else:
                    myPreviousFile = Filedrive.query.filter_by(
                        id=newArrOrder[idx - 1]['fdId']).first()
                    track['start'] = myPreviousFile.duration
                    track['end'] = myFile.duration
                    if idx == len(newArrOrder) - 1:
                        track['end'] = duration

                print(track)
                print('----------------------')

            # print(newArrOrder)
            myRevision = Revision(session_id=sessionId,
                                  u_id=uId,
                                  version=myRevision.version + 1,
                                  content=json.dumps(newArrOrder),
                                  mixed_id=new_filedrive.id,
                                  files_used=("%s" % myRevision.mixed_id))
            save(myRevision)

        return CreateRevision(revision=myRevision)