def main():
q = 'фото.png'
titleZip = (q.replace('.', '_')) + ".zip"
path = './files'
vk = VkParse(TOKEN, q, 2, path)
pathArchive = vk.zip()
# pathArchive = "./files/фото_png.zip"
drive = Gdrive() # auth
drive.upload(pathArchive, titleZip)
def handle_text(message):
lst = message.text.split()
# q = 'фото.png'
path = './files'
titleZip = (lst[0].replace('.', '_')) + ".zip"
vk = VkParse(TOKEN, lst[0], lst[1], path)
pathArchive = vk.zip()
# pathArchive = "./files/фото_png.zip"
drive = Gdrive() # auth
drive.upload(pathArchive, titleZip)
def main():
# Get identifiers for astro-ph preprints
global m
m = Metadata(update=True)
print('Identifiers collected: {}'.format(len(m.identifiers)))
# Connect to Google Drive
global g
global gdrive_tarfiles
g = Gdrive()
gdrive_tarfiles = g.get_tarfiles()
print('Tarfiles on Google Drive: {}'.format(len(gdrive_tarfiles)))
# Connect to Amazon S3 and get page iterator
global s3
s3 = Amazon_S3()
# Set up the parallel task pool to use all available processors
pool = mp.Pool(processes=mp.cpu_count())
# Iterate through each page
try:
for page in s3.get_page_iterator():
# Collect all tars on that page if they haven't already been downloaded
tasks = [
file['Key'] for file in page['Contents']
if file['Key'].endswith('.tar') and not os.path.isdir(''.join([
'latex/',
os.path.splitext(os.path.basename(file['Key']))[0]
]))
]
# Run the jobs
pool.map_async(work, tasks)
except KeyboardInterrupt:
print('\nYou interrupted the script!')
except Exception as e:
print('\nSomething went wrong: {}'.format(e))
traceback.print_exc()
finally: # IMPORTANT TO HAVE FINALLY, NOT ELSE, TO ALLOW ALL WORKERS FINISH HANDLING KEYBOARDINTERRUPT BEFORE POOL ENDS
pool.close() # necessary for zombies
pool.join() # wait for all processes to finish
print('The end.')
def thread_google_monitor():
global google
logger.info("Starting Google Drive changes monitor in 30 seconds...")
time.sleep(30)
# load access tokens
google = Gdrive(conf.configs, conf.settings['tokenfile'], conf.settings['cachefile'])
if not google.first_run():
logger.error("Failed to retrieve Google Drive access tokens...")
exit(1)
else:
logger.info("Google Drive access tokens were successfully loaded")
try:
logger.info("Google Drive changes monitor started")
while True:
if not google.token['page_token']:
# we have no page_token, likely this is first run, lets retrieve a starting page token
if not google.get_changes_first_page_token():
logger.error("Failed to retrieve starting Google Drive changes page token...")
return
else:
logger.info("Retrieved starting Google Drive changes page token: %s", google.token['page_token'])
time.sleep(conf.configs['GDRIVE']['POLL_INTERVAL'])
# get page changes
changes = []
changes_attempts = 0
while True:
try:
success, page = google.get_changes()
changes_attempts = 0
except Exception:
changes_attempts += 1
logger.exception(
"Exception occurred while polling Google Drive for changes on page %s on attempt %d/12: ",
str(google.token['page_token']), changes_attempts)
if changes_attempts < 12:
logger.warning("Sleeping for 5 minutes before trying to poll Google Drive for changes again...")
time.sleep(60 * 5)
continue
else:
logger.error("Failed to poll Google Drive changes after 12 consecutive attempts, "
"aborting...")
return
if not success:
logger.error("Failed to retrieve Google Drive changes for page: %s, aborting...",
str(google.token['page_token']))
return
else:
# successfully retrieved some changes
if 'changes' in page:
changes.extend(page['changes'])
# page logic
if page is not None and 'nextPageToken' in page:
# there are more pages to retrieve
logger.debug("There are more Google Drive changes pages to retrieve, retrieving next page...")
continue
elif page is not None and 'newStartPageToken' in page:
# there are no more pages to retrieve
break
else:
logger.error("There was an unexpected outcome when polling Google Drive for changes, "
"aborting future polls...")
return
# process changes
if len(changes):
logger.info("There's %d Google Drive change(s) to process", len(changes))
process_google_changes(changes)
# sleep before polling for changes again
time.sleep(conf.configs['GDRIVE']['POLL_INTERVAL'])
except Exception:
logger.exception("Fatal Exception occurred while monitoring Google Drive for changes, page = %s: ",
google.token['page_token'])
if __name__ == "__main__":
logger.info("""
PG Scan Started!
""")
if conf.args['cmd'] == 'sections':
plex.show_sections(conf.configs)
exit(0)
elif conf.args['cmd'] == 'update_sections':
plex.updateSectionMappings(conf)
elif conf.args['cmd'] == 'authorize':
if not conf.configs['GDRIVE']['ENABLED']:
logger.error("You must enable the ENABLED setting in the GDRIVE config section...")
exit(1)
else:
google = Gdrive(conf.configs, conf.settings['tokenfile'], conf.settings['cachefile'])
if not google.first_run():
logger.error("Failed to retrieve access tokens...")
exit(1)
else:
logger.info("Access tokens were successfully retrieved!")
exit(0)
elif conf.args['cmd'] == 'server':
if conf.configs['SERVER_USE_SQLITE']:
start_queue_reloader()
if conf.configs['GDRIVE']['ENABLED']:
if not os.path.exists(conf.settings['tokenfile']):
logger.error("You must authorize your Google Drive account with the authorize option...")
exit(1)
