Пример #1
0
def main():
    usage(sys.argv)

    config = None
    try:
        fd = open('config.json', 'r')
    except:
        print("config file [config.json] not found.")
        sys.exit(1)

    try:
        config = json.loads(fd.read())
    except:
        print("invalid json in config file.")
        sys.exit(1)

    factory = Factory(config)

    action = sys.argv[1]

    authorized_commands = [
        'upload-auth', 'comment', 'vote-gallery', 'vote-comment'
    ]

    oauth_commands = ['credits', 'refresh', 'authorize']

    if action in authorized_commands:
        handle_authorized_commands(factory, action)
    else:
        if action in oauth_commands:
            handle_oauth_commands(factory, config, action)
        else:
            handle_unauthorized_commands(factory, action)
Пример #2
0
    imgur = factory.buildAPI()
    req = factory.buildRequestOAuthTokenSwap('pin', pin)
    try:
        res = imgur.retrieveRaw(req)
    except HTTPError as e:
        print("Error %d\n%s" % (e.code, e.read().decode('utf8')))
        raise e

    print(
        "Access Token: %s\nRefresh Token: %s\nExpires: %d seconds from now." %
        (res[1]['access_token'], res[1]['refresh_token'],
         res[1]['expires_in']))
    config['refresh_token'] = res[1]['refresh_token']
    json.dump(config, open(configPath, 'w+'))
    return True


if __name__ == '__main__':
    config = None
    try:
        fd = open('../config.json', 'r')
    except:
        print("config file [config.json] not found.")
        sys.exit(1)
    try:
        config = json.loads(fd.read())
    except:
        print("invalid json in config file.")
        sys.exit(1)
    factory = Factory(config)
    getAndSaveNewToken(factory, config, '../config.json')
Пример #3
0
def main():
    config = None
    try:
        fd = open('config.json', 'r')
    except:
        print("config file [config.json] not found.")
        sys.exit(1)

    try:
        config = json.loads(fd.read())
    except:
        print("invalid json in config file.")
        sys.exit(1)

    factory = Factory(config)

    if len(sys.argv) <= 1:
        sep = '---------------------------------------------------------------------------------------------------------'
        print("Usage: python3 main.py (action) [options...]")
        print("\n" + sep + "\nUnauthorized Actions\n" + sep)
        print("upload [file]                    Anonymously upload a file")
        print(
            "album [id]                       View information about an album")
        print(
            "comments [hash]                  Get the comments (raw json) for a gallery item"
        )
        print(
            "comment-id [hash] [id]           Get a particular comment (raw json) for a gallery item"
        )
        print(
            "credits                          Inspect the rate limits for this client"
        )
        print("authorize                        Get the authorization URL")
        print("authorize [pin]                  Get an access token")
        print("\n" + sep + "\nAuthorized Actions\n" + sep)
        print("upload-auth [token] [file]       Upload a file to your account")
        print(
            "refresh [refresh-token]          Return a new OAuth access token after it's expired"
        )
        print("comment [token] [hash] [text]    Comment on a gallery image")
        sys.exit(1)

    time = int(dt.time())
    action = sys.argv[1]

    if action == 'upload':
        imgur = factory.buildAPI()
        req = factory.buildRequestUploadFromPath(sys.argv[2])
        res = imgur.retrieve(req)
        print(res['link'])

    if action == 'credits':
        imgur = factory.buildAPI()
        req = factory.buildRequest(('credits', ))
        res = imgur.retrieve(req)
        print(res)

    if action == 'authorize':
        if len(sys.argv) == 2:
            print("Visit this URL to get a PIN to authorize: " +
                  factory.getAPIUrl() + "oauth2/authorize?client_id=" +
                  config['client_id'] + "&response_type=pin")
        if len(sys.argv) == 3:
            pin = sys.argv[2]
            imgur = factory.buildAPI()
            req = factory.buildRequestOAuthTokenSwap('pin', pin)
            try:
                res = imgur.retrieveRaw(req)
            except urllib.request.HTTPError as e:
                print("Error %d\n%s" % (e.code, e.read().decode('utf8')))
                raise e

            print(
                "Access Token: %s\nRefresh Token: %s\nExpires: %d seconds from now."
                % (res[1]['access_token'], res[1]['refresh_token'],
                   res[1]['expires_in']))

    if action == 'refresh':
        token = sys.argv[2]
        imgur = factory.buildAPI()
        req = factory.buildRequestOAuthRefresh(token)
        res = imgur.retrieveRaw(req)
        print(
            'Access Token: %s\nRefresh Token: %s\nExpires: %d seconds from now.'
            % (res[1]['access_token'], res[1]['refresh_token'],
               res[1]['expires_in']))

    if action == 'upload-auth':
        (token, path) = sys.argv[2:]
        auth = factory.buildOAuth(token, None, time + 3600)
        imgur = factory.buildAPI(auth)
        req = factory.buildRequestUploadFromPath(path)
        try:
            res = imgur.retrieve(req)
            print(res['link'])
        except Expired:
            print("Expired access token")

    if action == 'comment':
        (token, thash) = sys.argv[2:4]
        text = ' '.join(sys.argv[4:])

        if len(text) > 140:
            print("Comment too long (trim by %d characters)." %
                  (len(text) - 140))
            sys.exit(1)

        auth = factory.buildOAuth(token, None, time + 3600)
        imgur = factory.buildAPI(auth)
        req = factory.buildRequest(('gallery', thash, 'comment'),
                                   {'comment': text})
        res = imgur.retrieve(req)
        print("Success! https://www.imgur.com/gallery/%s/comment/%s" %
              (thash, res['id']))

    if action == 'comments':
        thash = sys.argv[2]

        imgur = factory.buildAPI()
        req = factory.buildRequest(('gallery', thash, 'comments'))
        res = imgur.retrieve(req)
        print(res)

    if action == 'album':
        id = sys.argv[2]

        imgur = factory.buildAPI()
        req = factory.buildRequest(('album', id))
        res = imgur.retrieve(req)
        print(res)

    if action == 'comment-id':
        (thash, cid) = sys.argv[2:4]

        imgur = factory.buildAPI()
        req = factory.buildRequest(('gallery', thash, 'comments', cid))
        res = imgur.retrieve(req)
        print(res)
Пример #4
0
def main():
    if isfile('.lock'):
        lognprint("Already running")
        exit(0)
    lock = open(".lock", "wb")
    lock.write(str(time()))
    lock.close();
    
    lognprint_base("#"*80+"\n")
    lognprint("START", strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()))
    lognprint_base("#"*80+"\n")  
    
    
    
    # open our db and stuff
    connection = DBConnector("test", re_analyze=False)
    dupcheck = SimilarImagesSql(None, connection)
    config = None
    try:
        fd = open('config.json', 'r')
    except:
        lognprint("config file [config.json] not found.")
        sys.exit(1)
    try:
        config = json.loads(fd.read())
    except:
        lognprint("invalid json in config file.")
        sys.exit(1)
    factory = Factory(config)
    imgurApi = factory.buildAPI()
    commenter = CommentSender(factory, config, 40)
    commentTask = commenter.startSendLoop()


    
    # Adding user ids from our blocked ids file.
    # This isn't really secure but works so far.
    # The file is cleared after adding.
    lognprint("Adding user from blocked.txt")
    fd = open("blocked.txt", "rb+")
    for line in fd:
        line = line.strip()
        try:
            lognprint("Add user with id %s to blocklist" % line)
            connection.add_blocked_user(int(line)) 
        except Exception as e:
            lognprint("Problem adding user to blocklist:", e)
    fd.truncate(0)
    fd.close()
    # Same for user who wishe to be unblocked.
    lognprint("Removing user from unblocked.txt")
    fd = open("unblocked.txt", "rb+")
    for line in fd:
        line = line.strip()
        try:
            lognprint("Unblock user with id %s" % line)
            lognprint("Status: ", connection.remove_blocked_user(int(line))) 
        except Exception as e:
            lognprint("Problem unblocking user:"******"user", timeD="day"): commentTask.next()
    #for i in updateGallery(factory, imgurApi, dupcheck, commenter, 3, start=0, gType="user", timeD="day"): commentTask.next()
    connection.execute("ANALYZE;")
    #for i in updateGallery(factory, imgurApi, dupcheck, commenter, 2, start=4, gType="user", timeD="day"): commentTask.next()
    for i in updateGallery(factory, imgurApi, dupcheck, commenter, pages=4, start=0, gType="hot", timeD="day"): commentTask.next()
    #for i in updateGallery(factory, imgurApi, dupcheck, commenter, 3, start=0, gType="hot", timeD="day"): commentTask.next()
    connection.execute("ANALYZE;")
    for i in updateGallery(factory, imgurApi, dupcheck, commenter, pages=4, start=0, gType="top", timeD="day"): commentTask.next()
    #for i in updateGallery(factory, imgurApi, dupcheck, commenter, 3, start=0, gType="top", timeD="day"): commentTask.next()
    connection.execute("ANALYZE;")

    
    # Add old entries. Pretty dirty but works (kind of)
    tstep, hstep, nstep = (4, 4, 4)
    #tstep, hstep, nstep = (5, 5, 5)
    tstart, hstart, nstart = json.load(open('borders', 'rb')) 
    #map(int, bfile.read().split(';'))
    for i in updateGallery(factory, imgurApi, dupcheck, commenter, tstep, start=tstart, gType="top", timeD="all", post=True): commentTask.next()
    connection.execute("ANALYZE;")
    for i in updateGallery(factory, imgurApi, dupcheck, commenter, hstep, start=hstart, gType="hot", timeD="all", post=True): commentTask.next()
    connection.execute("ANALYZE;")
    for i in updateGallery(factory, imgurApi, dupcheck, commenter, nstep, start=nstart, gType="user", timeD="all", post=True): commentTask.next()
    connection.execute("ANALYZE;")
    # save the sites from which to crawl at the next run         
    json.dump([(tstart+tstep)%2000, (hstart+hstep)%800, (nstart+nstep)%2000], open('borders', 'wb') )
    
    w, m, y = json.load(open('borders_wmy', 'rb'))
    for i in updateGallery(factory, imgurApi, dupcheck, commenter, 4, start=w, gType="top", timeD="week", post=True): commentTask.next()
    connection.execute("ANALYZE;")
    for i in updateGallery(factory, imgurApi, dupcheck, commenter, 4, start=m, gType="top", timeD="month", post=True): commentTask.next()
    connection.execute("ANALYZE;")
    for i in updateGallery(factory, imgurApi, dupcheck, commenter, 4, start=y, gType="top", timeD="year", post=True): commentTask.next()
    connection.execute("ANALYZE;")
    json.dump([(w+4)%1000, (m+4)%2000, (y+4)%3000], open('borders_wmy', 'wb') )


    while len(commenter.queue) > 0:
        sleep(1)
        commentTask.next()


    # check notifications and scan posts where they occured.
    # TODO: this is ugly as hell and should also move somewhere else.
    dcommenter = commenter.commenter #DirtyCommenter(factory, config)
    notifies = dcommenter.sendAuthReqMessage('3/notification.json?new=true', retries=3)
    n = 0
    img_con_id = -1 
    notis = list()
    if notifies:
        for noti in notifies['messages']:
            nid = noti['id']
            noti = noti['content']
            if noti['with_account'] == u"48" and u"mentioned you in a comment" in noti["last_message"]:
                notis.append(nid)
                n += 1
                img_con_id = noti["id"]
    for i in xrange(1, 100):
        if n <= 0: break
        img_con = dcommenter.sendAuthReqMessage(('conversations', str(img_con_id), str(i)), retries=3)
        for message in img_con["messages"][::-1]:
            if n <= 0: break          
            if not u"mentioned you in a comment" in message["body"]:
                continue
            n -= 1
            image_id = re.search("glory at http://imgur\.com/gallery/([a-zA-Z0-9]+)/comment/[0-9]+/", message["body"])
            if not image_id:
                lognprint("Unparsable imgur notification '%s'" % str(message))
                continue
            image_id = image_id.group(1)
            if connection.already_commented(image_id):
                lognprint("Got notify for %s but already commented." % str(image_id))
                continue
            if connection.galerieExists(image_id):
                ori, dups = connection.get_reposts_by_image_hash(image_id)
                dups = list(dups)
                print "dup len", len(dups), dups
                if len(dups) <= 0:
                    lognprint("Got notify for %s but is no repost." % str(image_id))
                    continue
                #g.datetime, g.userurl, g.userid, g.link, g.title
                newE = {'datetime':ori[0], "account_id":ori[2], "id":image_id, "title":ori[4] }
                sendDupMessage(newE, dups, commenter, connection, connection.is_animated_by_hash(image_id))
                connection.commit()
                continue
            res = request(factory, imgurApi, ('gallery', 'image', str(image_id)), times=2, delay=20)
            sleep(20)
            if not res: continue
            if update_elem(res, factory, imgurApi, dupcheck, commenter, [[0]*5, [0]*6], 60*60*24*365*5, alwaysPost=True):
                lognprint("Successfuly summoned at %s." % image_id)
            else:
                lognprint("Failed summoning at %s." % image_id)
            connection.commit()
                
    if len(notis) > 0: img_con = dcommenter.sendAuthReqMessage(('notification',), {"ids":",".join(map(str, notis))}, retries=3)

    # Write out all remaining comments from the queue
    while len(commenter.queue) > 0:
        sleep(1)
        commentTask.next()


    


    # print some debuggin infos    
    lognprint("Current images indexed:", connection.getImageCount())
    lognprint('Oldest image', (time()-connection.getOldestImage())/31536000.0, 'years')
    lognprint('client_remaining', imgurApi.ratelimit.client_remaining)
    lognprint('user_remaining', imgurApi.ratelimit.user_remaining)
    lognprint('user_reset', imgurApi.ratelimit.user_reset - time(), 'seconds')
    
    # Get the current points, because i like statistics :)
    imgur = factory.buildAPI()  
    req = factory.buildRequest(('account', 'RepostStatistics'))
    ret = imgur.retrieve(req)['reputation']
    d = json.load(open('points', 'r'))
    d.append((time(), ret))
    json.dump(d, open('points', 'w'))
    lognprint('Points', ret)
    
    lognprint('Reposts: %i' % connection.getReprostCount())
    lognprint('blocked user:'******'SELECT count() from blocked_user;').fetchone())
    
    # I updated the database format some times, so we need to get the additional data for the old posts.
    # This shows how much entries still need updates. TODO:
    comments = connection.execute('SELECT count() from comments;').fetchone()[0]
    lognprint('comments written:', comments, "-", comments*2)
    lognprint('noUserId:', connection.execute('SELECT count() from galeries where userid = 0;').fetchone())
    lognprint('noSize:', connection.execute('SELECT count() from images where size = 0;').fetchone())
    lognprint('noWidth:', connection.execute('SELECT count() from images where width = -1;').fetchone())
    lognprint('noHeight:', connection.execute('SELECT count() from images where height = -1;').fetchone())