def main():
config = None
try:
fd = open('config.json', 'r')
except:
print("config file [config.json] not found.")
sys.exit(1)
try:
config = json.loads(fd.read())
except:
print("invalid json in config file.")
sys.exit(1)
factory = Factory(config)
if len(sys.argv) <= 1:
sep = '---------------------------------------------------------------------------------------------------------'
print("Usage: python3 main.py (action) [options...]")
print("\n" + sep + "\nUnauthorized Actions\n" + sep)
print("upload [file] Anonymously upload a file")
print(
"album [id] View information about an album")
print(
"comments [hash] Get the comments (raw json) for a gallery item"
)
print(
"comment-id [hash] [id] Get a particular comment (raw json) for a gallery item"
)
print(
"credits Inspect the rate limits for this client"
)
print("authorize Get the authorization URL")
print("authorize [pin] Get an access token")
print("\n" + sep + "\nAuthorized Actions\n" + sep)
print("upload-auth [token] [file] Upload a file to your account")
print(
"refresh [refresh-token] Return a new OAuth access token after it's expired"
)
print("comment [token] [hash] [text] Comment on a gallery image")
sys.exit(1)
time = int(dt.time())
action = sys.argv[1]
if action == 'upload':
imgur = factory.buildAPI()
req = factory.buildRequestUploadFromPath(sys.argv[2])
res = imgur.retrieve(req)
print(res['link'])
if action == 'credits':
imgur = factory.buildAPI()
req = factory.buildRequest(('credits', ))
res = imgur.retrieve(req)
print(res)
if action == 'authorize':
if len(sys.argv) == 2:
print("Visit this URL to get a PIN to authorize: " +
factory.getAPIUrl() + "oauth2/authorize?client_id=" +
config['client_id'] + "&response_type=pin")
if len(sys.argv) == 3:
pin = sys.argv[2]
imgur = factory.buildAPI()
req = factory.buildRequestOAuthTokenSwap('pin', pin)
try:
res = imgur.retrieveRaw(req)
except urllib.request.HTTPError as e:
print("Error %d\n%s" % (e.code, e.read().decode('utf8')))
raise e
print(
"Access Token: %s\nRefresh Token: %s\nExpires: %d seconds from now."
% (res[1]['access_token'], res[1]['refresh_token'],
res[1]['expires_in']))
if action == 'refresh':
token = sys.argv[2]
imgur = factory.buildAPI()
req = factory.buildRequestOAuthRefresh(token)
res = imgur.retrieveRaw(req)
print(
'Access Token: %s\nRefresh Token: %s\nExpires: %d seconds from now.'
% (res[1]['access_token'], res[1]['refresh_token'],
res[1]['expires_in']))
if action == 'upload-auth':
(token, path) = sys.argv[2:]
auth = factory.buildOAuth(token, None, time + 3600)
imgur = factory.buildAPI(auth)
req = factory.buildRequestUploadFromPath(path)
try:
res = imgur.retrieve(req)
print(res['link'])
except Expired:
print("Expired access token")
if action == 'comment':
(token, thash) = sys.argv[2:4]
text = ' '.join(sys.argv[4:])
if len(text) > 140:
print("Comment too long (trim by %d characters)." %
(len(text) - 140))
sys.exit(1)
auth = factory.buildOAuth(token, None, time + 3600)
imgur = factory.buildAPI(auth)
req = factory.buildRequest(('gallery', thash, 'comment'),
{'comment': text})
res = imgur.retrieve(req)
print("Success! https://www.imgur.com/gallery/%s/comment/%s" %
(thash, res['id']))
if action == 'comments':
thash = sys.argv[2]
imgur = factory.buildAPI()
req = factory.buildRequest(('gallery', thash, 'comments'))
res = imgur.retrieve(req)
print(res)
if action == 'album':
id = sys.argv[2]
imgur = factory.buildAPI()
req = factory.buildRequest(('album', id))
res = imgur.retrieve(req)
print(res)
if action == 'comment-id':
(thash, cid) = sys.argv[2:4]
imgur = factory.buildAPI()
req = factory.buildRequest(('gallery', thash, 'comments', cid))
res = imgur.retrieve(req)
print(res)
def main():
config = None
try:
fd = open('config.json', 'r')
except:
print("config file [config.json] not found.")
sys.exit(1)
try:
config = json.loads(fd.read())
except:
print("invalid json in config file.")
sys.exit(1)
factory = Factory(config)
if len(sys.argv) <= 1:
sep = '---------------------------------------------------------------------------------------------------------'
print("Usage: python3 main.py (action) [options...]")
print("\n" + sep + "\nUnauthorized Actions\n" + sep)
print("upload [file] Anonymously upload a file")
print("album [id] View information about an album")
print("comments [hash] Get the comments (raw json) for a gallery item")
print("comment-id [hash] [id] Get a particular comment (raw json) for a gallery item")
print("credits Inspect the rate limits for this client")
print("authorize Get the authorization URL")
print("authorize [pin] Get an access token")
print("\n" + sep + "\nAuthorized Actions\n" + sep)
print("upload-auth [token] [file] Upload a file to your account")
print("refresh [refresh-token] Return a new OAuth access token after it's expired")
print("comment [token] [hash] [text] Comment on a gallery image")
sys.exit(1)
time = int(dt.time())
action = sys.argv[1]
if action == 'upload':
imgur = factory.buildAPI()
req = factory.buildRequestUploadFromPath(sys.argv[2])
res = imgur.retrieve(req)
print(res['link'])
if action == 'credits':
imgur = factory.buildAPI()
req = factory.buildRequest(('credits',))
res = imgur.retrieve(req)
print(res)
if action == 'authorize':
if len(sys.argv) == 2:
print("Visit this URL to get a PIN to authorize: " + factory.getAPIUrl() + "oauth2/authorize?client_id=" + config['client_id'] + "&response_type=pin")
if len(sys.argv) == 3:
pin = sys.argv[2]
imgur = factory.buildAPI()
req = factory.buildRequestOAuthTokenSwap('pin', pin)
try:
res = imgur.retrieveRaw(req)
except urllib.request.HTTPError as e:
print("Error %d\n%s" % (e.code, e.read().decode('utf8')))
raise e
print("Access Token: %s\nRefresh Token: %s\nExpires: %d seconds from now." % (
res[1]['access_token'],
res[1]['refresh_token'],
res[1]['expires_in']
))
if action == 'refresh':
token = sys.argv[2]
imgur = factory.buildAPI()
req = factory.buildRequestOAuthRefresh(token)
res = imgur.retrieveRaw(req)
print('Access Token: %s\nRefresh Token: %s\nExpires: %d seconds from now.' % (
res[1]['access_token'],
res[1]['refresh_token'],
res[1]['expires_in']
))
if action == 'upload-auth':
(token, path) = sys.argv[2:]
auth = factory.buildOAuth(token, None, time+3600)
imgur = factory.buildAPI(auth)
req = factory.buildRequestUploadFromPath(path)
try:
res = imgur.retrieve(req)
print(res['link'])
except Expired:
print("Expired access token")
if action == 'comment':
(token, thash) = sys.argv[2:4]
text = ' '.join(sys.argv[4:])
if len(text) > 140:
print("Comment too long (trim by %d characters)." % (len(text) - 140))
sys.exit(1)
auth = factory.buildOAuth(token, None, time+3600)
imgur = factory.buildAPI(auth)
req = factory.buildRequest(('gallery', thash, 'comment'), {
'comment': text
})
res = imgur.retrieve(req)
print("Success! https://www.imgur.com/gallery/%s/comment/%s" % (thash, res['id']))
if action == 'comments':
thash = sys.argv[2]
imgur = factory.buildAPI()
req = factory.buildRequest(('gallery', thash, 'comments'))
res = imgur.retrieve(req)
print(res)
if action == 'album':
id = sys.argv[2]
imgur = factory.buildAPI()
req = factory.buildRequest(('album', id))
res = imgur.retrieve(req)
print(res)
if action == 'comment-id':
(thash, cid) = sys.argv[2:4]
imgur = factory.buildAPI()
req = factory.buildRequest(('gallery', thash, 'comments', cid))
res = imgur.retrieve(req)
print(res)
def main():
if isfile('.lock'):
lognprint("Already running")
exit(0)
lock = open(".lock", "wb")
lock.write(str(time()))
lock.close();
lognprint_base("#"*80+"\n")
lognprint("START", strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()))
lognprint_base("#"*80+"\n")
# open our db and stuff
connection = DBConnector("test", re_analyze=False)
dupcheck = SimilarImagesSql(None, connection)
config = None
try:
fd = open('config.json', 'r')
except:
lognprint("config file [config.json] not found.")
sys.exit(1)
try:
config = json.loads(fd.read())
except:
lognprint("invalid json in config file.")
sys.exit(1)
factory = Factory(config)
imgurApi = factory.buildAPI()
commenter = CommentSender(factory, config, 40)
commentTask = commenter.startSendLoop()
# Adding user ids from our blocked ids file.
# This isn't really secure but works so far.
# The file is cleared after adding.
lognprint("Adding user from blocked.txt")
fd = open("blocked.txt", "rb+")
for line in fd:
line = line.strip()
try:
lognprint("Add user with id %s to blocklist" % line)
connection.add_blocked_user(int(line))
except Exception as e:
lognprint("Problem adding user to blocklist:", e)
fd.truncate(0)
fd.close()
# Same for user who wishe to be unblocked.
lognprint("Removing user from unblocked.txt")
fd = open("unblocked.txt", "rb+")
for line in fd:
line = line.strip()
try:
lognprint("Unblock user with id %s" % line)
lognprint("Status: ", connection.remove_blocked_user(int(line)))
except Exception as e:
lognprint("Problem unblocking user:"******"user", timeD="day"): commentTask.next()
#for i in updateGallery(factory, imgurApi, dupcheck, commenter, 3, start=0, gType="user", timeD="day"): commentTask.next()
connection.execute("ANALYZE;")
#for i in updateGallery(factory, imgurApi, dupcheck, commenter, 2, start=4, gType="user", timeD="day"): commentTask.next()
for i in updateGallery(factory, imgurApi, dupcheck, commenter, pages=4, start=0, gType="hot", timeD="day"): commentTask.next()
#for i in updateGallery(factory, imgurApi, dupcheck, commenter, 3, start=0, gType="hot", timeD="day"): commentTask.next()
connection.execute("ANALYZE;")
for i in updateGallery(factory, imgurApi, dupcheck, commenter, pages=4, start=0, gType="top", timeD="day"): commentTask.next()
#for i in updateGallery(factory, imgurApi, dupcheck, commenter, 3, start=0, gType="top", timeD="day"): commentTask.next()
connection.execute("ANALYZE;")
# Add old entries. Pretty dirty but works (kind of)
tstep, hstep, nstep = (4, 4, 4)
#tstep, hstep, nstep = (5, 5, 5)
tstart, hstart, nstart = json.load(open('borders', 'rb'))
#map(int, bfile.read().split(';'))
for i in updateGallery(factory, imgurApi, dupcheck, commenter, tstep, start=tstart, gType="top", timeD="all", post=True): commentTask.next()
connection.execute("ANALYZE;")
for i in updateGallery(factory, imgurApi, dupcheck, commenter, hstep, start=hstart, gType="hot", timeD="all", post=True): commentTask.next()
connection.execute("ANALYZE;")
for i in updateGallery(factory, imgurApi, dupcheck, commenter, nstep, start=nstart, gType="user", timeD="all", post=True): commentTask.next()
connection.execute("ANALYZE;")
# save the sites from which to crawl at the next run
json.dump([(tstart+tstep)%2000, (hstart+hstep)%800, (nstart+nstep)%2000], open('borders', 'wb') )
w, m, y = json.load(open('borders_wmy', 'rb'))
for i in updateGallery(factory, imgurApi, dupcheck, commenter, 4, start=w, gType="top", timeD="week", post=True): commentTask.next()
connection.execute("ANALYZE;")
for i in updateGallery(factory, imgurApi, dupcheck, commenter, 4, start=m, gType="top", timeD="month", post=True): commentTask.next()
connection.execute("ANALYZE;")
for i in updateGallery(factory, imgurApi, dupcheck, commenter, 4, start=y, gType="top", timeD="year", post=True): commentTask.next()
connection.execute("ANALYZE;")
json.dump([(w+4)%1000, (m+4)%2000, (y+4)%3000], open('borders_wmy', 'wb') )
while len(commenter.queue) > 0:
sleep(1)
commentTask.next()
# check notifications and scan posts where they occured.
# TODO: this is ugly as hell and should also move somewhere else.
dcommenter = commenter.commenter #DirtyCommenter(factory, config)
notifies = dcommenter.sendAuthReqMessage('3/notification.json?new=true', retries=3)
n = 0
img_con_id = -1
notis = list()
if notifies:
for noti in notifies['messages']:
nid = noti['id']
noti = noti['content']
if noti['with_account'] == u"48" and u"mentioned you in a comment" in noti["last_message"]:
notis.append(nid)
n += 1
img_con_id = noti["id"]
for i in xrange(1, 100):
if n <= 0: break
img_con = dcommenter.sendAuthReqMessage(('conversations', str(img_con_id), str(i)), retries=3)
for message in img_con["messages"][::-1]:
if n <= 0: break
if not u"mentioned you in a comment" in message["body"]:
continue
n -= 1
image_id = re.search("glory at http://imgur\.com/gallery/([a-zA-Z0-9]+)/comment/[0-9]+/", message["body"])
if not image_id:
lognprint("Unparsable imgur notification '%s'" % str(message))
continue
image_id = image_id.group(1)
if connection.already_commented(image_id):
lognprint("Got notify for %s but already commented." % str(image_id))
continue
if connection.galerieExists(image_id):
ori, dups = connection.get_reposts_by_image_hash(image_id)
dups = list(dups)
print "dup len", len(dups), dups
if len(dups) <= 0:
lognprint("Got notify for %s but is no repost." % str(image_id))
continue
#g.datetime, g.userurl, g.userid, g.link, g.title
newE = {'datetime':ori[0], "account_id":ori[2], "id":image_id, "title":ori[4] }
sendDupMessage(newE, dups, commenter, connection, connection.is_animated_by_hash(image_id))
connection.commit()
continue
res = request(factory, imgurApi, ('gallery', 'image', str(image_id)), times=2, delay=20)
sleep(20)
if not res: continue
if update_elem(res, factory, imgurApi, dupcheck, commenter, [[0]*5, [0]*6], 60*60*24*365*5, alwaysPost=True):
lognprint("Successfuly summoned at %s." % image_id)
else:
lognprint("Failed summoning at %s." % image_id)
connection.commit()
if len(notis) > 0: img_con = dcommenter.sendAuthReqMessage(('notification',), {"ids":",".join(map(str, notis))}, retries=3)
# Write out all remaining comments from the queue
while len(commenter.queue) > 0:
sleep(1)
commentTask.next()
# print some debuggin infos
lognprint("Current images indexed:", connection.getImageCount())
lognprint('Oldest image', (time()-connection.getOldestImage())/31536000.0, 'years')
lognprint('client_remaining', imgurApi.ratelimit.client_remaining)
lognprint('user_remaining', imgurApi.ratelimit.user_remaining)
lognprint('user_reset', imgurApi.ratelimit.user_reset - time(), 'seconds')
# Get the current points, because i like statistics :)
imgur = factory.buildAPI()
req = factory.buildRequest(('account', 'RepostStatistics'))
ret = imgur.retrieve(req)['reputation']
d = json.load(open('points', 'r'))
d.append((time(), ret))
json.dump(d, open('points', 'w'))
lognprint('Points', ret)
lognprint('Reposts: %i' % connection.getReprostCount())
lognprint('blocked user:'******'SELECT count() from blocked_user;').fetchone())
# I updated the database format some times, so we need to get the additional data for the old posts.
# This shows how much entries still need updates. TODO:
comments = connection.execute('SELECT count() from comments;').fetchone()[0]
lognprint('comments written:', comments, "-", comments*2)
lognprint('noUserId:', connection.execute('SELECT count() from galeries where userid = 0;').fetchone())
lognprint('noSize:', connection.execute('SELECT count() from images where size = 0;').fetchone())
lognprint('noWidth:', connection.execute('SELECT count() from images where width = -1;').fetchone())
lognprint('noHeight:', connection.execute('SELECT count() from images where height = -1;').fetchone())