def __init__(self, auth):
self.drive = GoogleDrive(auth)
def auth():
gauth = GoogleAuth()
gauth.LocalWebserverAuth()
return GoogleDrive(gauth)
async def gdrive_stuff(client, message):
gauth.LoadCredentialsFile("nana/session/drive")
if gauth.credentials is None:
if HEROKU_API:
if gdrive_credentials:
file = open("client_secrets.json", "w")
file.write(gdrive_credentials)
file.close()
await message.edit(
"You are not logged in to your google drive account!\nYour assistant bot may help you to login google "
"drive, check your assistant bot for more information!")
gdriveclient = os.path.isfile("client_secrets.json")
if not gdriveclient:
await setbot.send_message(
message.from_user.id,
"Hello, look like you're not logged in to google drive 🙂\nI can help you to "
"login.\n\nFirst of all, you need to activate your google drive API\n1. [Go "
"here](https://developers.google.com/drive/api/v3/quickstart/python), "
"click **Enable the drive API**\n2. Login to your google account (skip this if "
"you're already logged in)\n3. After logged in, click **Enable the drive API** "
"again, and click **Download Client Configuration** button, download that.\n4. "
"After downloaded that file, open that file then copy all of that content, "
"back to telegram then do .credentials (copy the content of that file) do "
"without bracket\n\nAfter that, you can go next guide by type /gdrive"
)
else:
try:
gauth.GetAuthUrl()
except:
await setbot.send_message(
message.from_user.id,
"Wrong Credentials! Check var ENV gdrive_credentials on heroku or do "
".credentials (your credentials) for change your Credentials"
)
return
await setbot.send_message(
message.from_user.id,
"Hello, look like you're not logged in to google drive :)\nI can help you to "
"login.\n\n**To login Google Drive**\n1. `/gdrive` to get login URL\n2. After "
"you're logged in, copy your Token.\n3. `/gdrive (token)` without `(` or `)` to "
"login, and your session will saved to `nana/session/drive`.\n\nDon't share your "
"session to someone, else they will hack your google drive account!"
)
return
elif gauth.access_token_expired:
# Refresh them if expired
gauth.Refresh()
else:
# Initialize the saved creds
gauth.Authorize()
drive = GoogleDrive(gauth)
drive_dir = await get_drivedir(drive)
if len(message.text.split()) == 3 and message.text.split(
)[1] == "download":
await message.edit("Downloading...")
driveid = await get_driveid(message.text.split()[2])
if not driveid:
await message.edit(
"Invaild URL!\nIf you think this is bug, please go to your Assistant bot and type `/reportbug`"
)
return
filename = await get_driveinfo(driveid)
if not filename:
await message.edit(
"Invaild URL!\nIf you think this is bug, please go to your Assistant bot and type `/reportbug`"
)
return
await message.edit("Downloading for `{}`\nPlease wait...".format(
filename.replace(' ', '_')))
download = drive.CreateFile({'id': driveid})
download.GetContentFile(filename)
try:
os.rename(filename, "nana/downloads/" + filename.replace(' ', '_'))
except FileExistsError:
os.rename(filename,
"nana/downloads/" + filename.replace(' ', '_') + ".2")
await message.edit("Downloaded!\nFile saved to `{}`".format(
"nana/downloads/" + filename.replace(' ', '_')))
elif len(
message.text.split()) == 3 and message.text.split()[1] == "upload":
filerealname = message.text.split()[2].split(None, 1)[0]
filename = "nana/downloads/{}".format(filerealname.replace(' ', '_'))
checkfile = os.path.isfile(filename)
if not checkfile:
await message.edit("File `{}` was not found!".format(filerealname))
return
await message.edit("Uploading `{}`...".format(filerealname))
upload = drive.CreateFile({
"parents": [{
"kind": "drive#fileLink",
"id": drive_dir
}],
'title':
filerealname
})
upload.SetContentFile(filename)
upload.Upload()
upload.InsertPermission({
'type': 'anyone',
'value': 'anyone',
'role': 'reader'
})
await message.edit(
"Uploaded!\nDownload link: [{}]({})\nDirect download link: [{}]({})"
.format(filerealname, upload['alternateLink'], filerealname,
upload['downloadUrl']))
elif len(
message.text.split()) == 3 and message.text.split()[1] == "mirror":
message.edit("Mirroring...")
driveid = await get_driveid(message.text.split()[2])
if not driveid:
await message.edit(
"Invaild URL!\nIf you think this is bug, please go to your Assistant bot and type `/reportbug`"
)
return
filename = await get_driveinfo(driveid)
if not filename:
await message.edit(
"Invaild URL!\nIf you think this is bug, please go to your Assistant bot and type `/reportbug`"
)
return
mirror = drive.auth.service.files().copy(fileId=driveid,
body={
"parents": [{
"kind":
"drive#fileLink",
"id":
drive_dir
}],
'title':
filename
}).execute()
new_permission = {
'type': 'anyone',
'value': 'anyone',
'role': 'reader'
}
drive.auth.service.permissions().insert(fileId=mirror['id'],
body=new_permission).execute()
await message.edit(
"Done!\nDownload link: [{}]({})\nDirect download link: [{}]({})".
format(filename, mirror['alternateLink'], filename,
mirror['downloadUrl']))
elif len(message.text.split()) == 2 and message.text.split(
)[1] == "tgmirror":
if message.reply_to_message:
await message.edit("__Downloading...__")
c_time = time.time()
if message.reply_to_message.photo:
nama = "photo_{}.png".format(
message.reply_to_message.photo.date)
await client.download_media(
message.reply_to_message.photo,
file_name="nana/downloads/" + nama,
progress=lambda d, t: asyncio.get_event_loop().create_task(
progressdl(d, t, message, c_time, "Downloading...")))
elif message.reply_to_message.animation:
nama = "giphy_{}-{}.gif".format(
message.reply_to_message.animation.date,
message.reply_to_message.animation.file_size)
await client.download_media(
message.reply_to_message.animation,
file_name="nana/downloads/" + nama,
progress=lambda d, t: asyncio.get_event_loop().create_task(
progressdl(d, t, message, c_time, "Downloading...")))
elif message.reply_to_message.video:
nama = "video_{}-{}.mp4".format(
message.reply_to_message.video.date,
message.reply_to_message.video.file_size)
await client.download_media(
message.reply_to_message.video,
file_name="nana/downloads/" + nama,
progress=lambda d, t: asyncio.get_event_loop().create_task(
progressdl(d, t, message, c_time, "Downloading...")))
elif message.reply_to_message.sticker:
nama = "sticker_{}_{}.webp".format(
message.reply_to_message.sticker.date,
message.reply_to_message.sticker.set_name)
await client.download_media(
message.reply_to_message.sticker,
file_name="nana/downloads/" + nama,
progress=lambda d, t: asyncio.get_event_loop().create_task(
progressdl(d, t, message, c_time, "Downloading...")))
elif message.reply_to_message.audio:
nama = "audio_{}.mp3".format(
message.reply_to_message.audio.date)
await client.download_media(
message.reply_to_message.audio,
file_name="nana/downloads/" + nama,
progress=lambda d, t: asyncio.get_event_loop().create_task(
progressdl(d, t, message, c_time, "Downloading...")))
elif message.reply_to_message.voice:
nama = "audio_{}.ogg".format(
message.reply_to_message.voice.date)
await client.download_media(
message.reply_to_message.voice,
file_name="nana/downloads/" + nama,
progress=lambda d, t: asyncio.get_event_loop().create_task(
progressdl(d, t, message, c_time, "Downloading...")))
elif message.reply_to_message.document:
nama = "{}".format(message.reply_to_message.document.file_name)
await client.download_media(
message.reply_to_message.document,
file_name="nana/downloads/" + nama,
progress=lambda d, t: asyncio.get_event_loop().create_task(
progressdl(d, t, message, c_time, "Downloading...")))
else:
await message.edit("Unknown file!")
return
upload = drive.CreateFile({
"parents": [{
"kind": "drive#fileLink",
"id": drive_dir
}],
'title':
nama
})
upload.SetContentFile("nana/downloads/" + nama)
upload.Upload()
upload.InsertPermission({
'type': 'anyone',
'value': 'anyone',
'role': 'reader'
})
await message.edit(
"Done!\nDownload link: [{}]({})\nDirect download link: [{}]({})"
.format(nama, upload['alternateLink'], nama,
upload['downloadUrl']))
os.remove("nana/downloads/" + nama)
else:
await message.edit("Reply document to mirror it to gdrive")
elif len(message.text.split()) == 3 and message.text.split(
)[1] == "urlmirror":
await message.edit("Downloading...")
URL = message.text.split()[2]
nama = URL.split("/")[-1]
time_dl = await download_url(URL, nama)
if "Downloaded" not in time_dl:
await message.edit("Failed to download file, invaild url!")
return
await message.edit(f"Downloaded with {time_dl}.\nNow uploading...")
upload = drive.CreateFile({
"parents": [{
"kind": "drive#fileLink",
"id": drive_dir
}],
'title':
nama
})
upload.SetContentFile("nana/downloads/" + nama)
upload.Upload()
upload.InsertPermission({
'type': 'anyone',
'value': 'anyone',
'role': 'reader'
})
await message.edit(
"Done!\nDownload link: [{}]({})\nDirect download link: [{}]({})".
format(nama, upload['alternateLink'], nama, upload['downloadUrl']))
os.remove("nana/downloads/" + nama)
else:
await message.edit(
"Usage:\n-> `gdrive download <url/gid>`\n-> `gdrive upload <file>`\n-> `gdrive mirror <url/gid>`\n\nFor "
"more information about this, go to your assistant.")
def __init__(self):
# 1. Authenticate and create the PyDrive client.
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
self.drive = GoogleDrive(gauth)
def load_data_path(folder_id, colab_path='/root/data/', local_path='../data/',
mime_types=['csv', 'zip']):
"""Boilerplate to download data from Google Drive into Colab
notebook or to point to local data folder
Behavior:
---------
1. Identify if Notebook is running in Colab
2. If Yes, then
a. do Google OAuth login (requires user interaction)
b. create a data folder in Colab (colab_path)
c. Search for all CSV files in Google Drive folder
d. Copy all CSV files from G Drive into colab_path folder
e. Return the colab_path variable
3. If No, then
a. Return the local_path variable
Example 1:
----------
!pip install colabtweak
from colabtweak import load_data_path
folder_id = "kasdhkfhjkashfjadskjfjsalk"
data_path = load_data_path(folder_id)
import pandas as pd
df = pd.read_csv(data_path + "train.csv")
df.head()
Example 2:
----------
!pip install colabtweak
from colabtweak import load_data_path
folder_id = "kasdhkfhjkashfjadskjfjsalk"
colab_path = "/root/somecustomfolderincolab/"
local_path = "../localsiblingprojectfolder/
data_path = load_data_path(
folder_id, colab_path=colab_path, local_path=local_path)
"""
if 'google.colab' in sys.modules:
print("Notebook is running in Colab")
if folder_id is None:
print((
"Folder ID is missing.\n"
"Click on the Google Drive folder and check your URL\n"
"'https://drive.google.com/drive/u/0/folders/<folder_id>'"))
# Login
from google.colab import auth
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)
# create "~/data" folder within the Colab image
download_path = os.path.expanduser(colab_path)
try:
os.makedirs(download_path)
except FileExistsError:
pass
# Extract the FileIDs from the Google Drive directory
tmp = ' or '.join(["title contains '." + m + "'" for m in mime_types])
querystr = "(" + tmp + ") and '" + folder_id + "' in parents"
listed = drive.ListFile({'q': querystr}).GetList()
# Copy all files
for file in listed:
try:
print('{} {}'.format(file['id'], file['title']))
output_file = os.path.join(download_path, file['title'])
temp_file = drive.CreateFile({'id': file['id']})
temp_file.GetContentFile(output_file)
except Exception as e:
print(e)
# Set directory path
return colab_path
else:
print("Notebook is running in Jupyter")
return local_path
def pydrive_load(args):
gauth = GoogleAuth()
code = gauth.CommandLineAuth()
if code != None:
gauth.Auth(code)
drive = GoogleDrive(gauth)
files = GoogleDriveFile(gauth)
# remove temp file for this share id
pro_temp = get_project_temp(drive, files, args.driveid)
# about = drive.GetAbout()
# print(about)
# get_root_info(files, DRIVE_ID)
root_node = Node('root', data=path_info(id=DRIVE_ID, title='', parent=''))
# drive_id = DRIVE_ID
drive_id = args.driveid
l = []
get_file_list(root_node, l, drive, drive_id)
# list path tree
if args.showtree:
print('path tree is:')
for pre, fill, node in RenderTree(root_node):
print('{}{}'.format(pre, node.name))
# make dir
base_dir = os.path.join(args.downdir, drive_id)
mkdir_in_tree(base_dir, root_node)
# list file
if args.showlist:
print('file list is:')
current = 0
total = len(l)
for i in l:
if args.showlist:
print(
'id: {}, is_folder: {}, title: {}, desc: {}, ext: {}, size: {}'
.format(i.id, i.is_folder, i.title, i.desc, i.ext, i.size))
if len(i.parents) > 0:
index = 0
for parent in i.parents:
if args.showlist:
print(' parents:{}={}, isRoot:{}'.format(
index, parent['id'], parent['isRoot']))
index += 1
if args.showlist:
print(' parent path={}'.format(i.parent_node.data.path))
retry = 0
if not i.is_folder:
while retry < args.retry_count:
try:
print('# {}/{} begin!'.format(current, total))
try:
file_path = i.parent_node.data.path
file_title = i.title
file_size = i.size
file_id = i.id
download_file(file_path, args.override, drive,
file_id, file_title, file_size)
except HttpError as http_error:
if http_error.resp.status == 403 and str(
http_error.content
).find('The download quota for this file has been exceeded'
) != -1:
make_copy_and_download(file_path,
drive.auth.service,
args.override, drive,
file_id, pro_temp,
file_title, file_size)
print_with_carriage_return('# {}/{} done!'.format(
current, total))
break
except Exception as e:
retry += 1
print('unexpeted error={}, retry={}'.format(e, retry))
current += 1
# remove temp
print('job done! remove project temp folder...')
get_project_temp(drive, files, args.driveid, False)
def login():
global gauth, drive
gauth = GoogleAuth()
gauth.LocalWebserverAuth() # Creates local webserver and auto handles authentication
drive = GoogleDrive(gauth) # Create GoogleDrive instance with authenticated GoogleAuth instance
def main(argv):
if len(argv) < 5:
print 'Usage: python main.py {Nuero de produtores} {Numero de consumidores} {Numero de arquivos por produtor} {Tamanho de cada arquivo}'
sys.exit(0)
n_produtores = int(argv[1])
n_consumidores = int(argv[2])
n_files = int(argv[3])
file_size = int(argv[4])
fila_locker = threading.Lock()
contador_locker = threading.Lock()
fila_conc_locker = threading.Lock()
cont_conc_locker = threading.Lock()
gauth = GoogleAuth()
gauth.LocalWebserverAuth() # Creates local webserver and auto handles authentication.
drive = GoogleDrive(gauth)
conc_file = drive.CreateFile({'title': 'Arquivo Final.txt'})
conc_file.Upload()
contador = drive.CreateFile({'title': 'Contador.txt'})
contador.SetContentString(str(n_files*n_produtores))
contador.Upload()
fila = drive.CreateFile({'title': 'Fila.txt'})
fila.SetContentString(' ')
fila.Upload()
cont_conc = drive.CreateFile({'title': 'Contador_Concatenador.txt'})
cont_conc.SetContentString(str(n_files*n_produtores))
cont_conc.Upload()
fila_conc = drive.CreateFile({'title': 'Fila_Concatenador.txt'})
fila_conc.SetContentString(' ')
fila_conc.Upload()
produtores = []
consumidores = []
concatenador = None
try:
for x in range(n_produtores):
produtores.append(Produtor(x, n_files, file_size, drive, contador, fila, fila_locker))
produtores[x].start()
for x in xrange(n_consumidores):
consumidores.append(Consumidor(x, drive, contador, fila, fila_locker, contador_locker, fila_conc, fila_conc_locker))
consumidores[x].start()
concatenador = Concatenador(drive, conc_file, fila_conc, fila_conc_locker, cont_conc, cont_conc_locker)
concatenador.start()
except:
print 'Unable to start Thread'
for x in range(n_produtores):
produtores[x].join()
for x in range(n_consumidores):
consumidores[x].join()
concatenador.join()
fila.Delete()
contador.Delete()
fila_conc.Delete()
cont_conc.Delete()
print 'FIM'
def auth(self):
gauth = GoogleAuth()
gauth.LocalWebserverAuth() # client_secrets.json need to be in the same directory as the script
drive = GoogleDrive(gauth)
return drive
def __init__(self):
self.gauth = GoogleAuth()
self.gauth.LocalWebserverAuth()
self.drive = GoogleDrive(self.gauth)
class GoogleDriveAPIHelper:
"""This class is supposed to help handling the GoogleDrive API connections.
It is assumed that during runtime the relevant folder structure of Google Drive does not change due synchronous nature of the script.
Thus queries only have to be done once and are saved to increase performance.
"""
# Static variables
gauth = GoogleAuth()
gauth.LocalWebserverAuth(
) # Creates local webserver and auto handles authentication.
drive = GoogleDrive(gauth)
def __init__(self, name=None, threads=4):
# Variable that keeps track of previous queries to avoid doing the same query over and over again
self.queries = {}
# Files that are already created for that run
self.complete_file_list = None
self.active_threads = {}
self.allowed_threads = threads
# Create logger
logging_config.create_logger(f'download_dataset_{name}',
config.LOGGING_PATH, True)
def query_api_get_list(self, query):
if query in self.queries:
res = self.queries[query]
else:
res = GoogleDriveAPIHelper.drive.ListFile({"q": query}).GetList()
self.queries[query] = res # save query for next use
return res
def get_sub_dirs(self, folder_id):
query = f"'{folder_id}' in parents and mimeType='application/vnd.google-apps.folder' and trashed=false"
drive_folders = self.query_api_get_list(query)
return drive_folders
def download_sub_folder(self, folder_id, output_dir):
sub_dirs = self.get_sub_dirs(folder_id)
total = len(sub_dirs)
self.progress_bar = tqdm(total=total)
for sub_dir in sub_dirs:
while len(self.active_threads
) > self.allowed_threads: # max allowed threads
time.sleep(2)
# Create thread
thr = threading.Thread(target=self.process_sub_dir,
args=(sub_dir, output_dir))
# Start thread
thr.start()
self.active_threads[sub_dir['id']] = thr
def process_sub_dir(self, sub_dir, output_dir):
sub_dir_id = sub_dir['id']
sub_dir_title = sub_dir['title']
# Only get files
query = f"'{sub_dir_id}' in parents and mimeType!='application/vnd.google-apps.folder' and trashed=false"
sub_dir_files = self.query_api_get_list(query)
if not sub_dir_files:
logging_config.logger.info(f'No files found for {sub_dir_title}.')
# Loop over files
for sub_dir_file in sub_dir_files:
sub_dir_file_title = sub_dir_file['title']
file_dir = f'{output_dir}/{sub_dir_title}'
logging_config.logger.info(
f'Process file {sub_dir_file_title} finished.')
# Check if path exists
if not os.path.exists(file_dir):
os.makedirs(file_dir)
file_path = f'{file_dir}/{sub_dir_file_title}'
# Check if file already exists
if not os.path.exists(file_path):
drive_file = GoogleDriveAPIHelper.drive.CreateFile(
{'id': sub_dir_file['id']})
drive_file.GetContentFile(file_path)
# Del task
del self.active_threads[sub_dir_id]
# Update progress bar
self.progress_bar.update(1)
logging_config.logger.info(f'{sub_dir_title} finished.')
def uploadData(oe):
filename = oe.finaldataPath + '\\PWVout_%s.fits' % oe.night
def ListFolder(parent):
filelist = []
file_list = drive.ListFile({
'q':
"'%s' in parents and trashed=false" % parent
}).GetList()
for f in file_list:
if f['mimeType'] == 'application/vnd.google-apps.folder': # if folder
filelist.append({
"id": f['id'],
"title": f['title'],
"list": ListFolder(f['id'])
})
else:
filelist.append({
"title": f['title'],
"title1": f['alternateLink']
})
return filelist
if filename: # whi was there an indent here?
gauth = GoogleAuth()
auth_url = gauth.GetAuthUrl(
) # Create authentication url user needs to visit
gauth.LocalWebserverAuth(
) #will this require me to click a button every time?????
#http://pythonhosted.org/PyDrive/oauth.html#customizing-authentication-with-settings-yaml
drive = GoogleDrive(gauth)
# Check if folder in CAMAL_data for night
out_root = ListFolder('root')
for ifold in range(len(out_root)):
if out_root[ifold]['title'] == u'Data':
camalid = out_root[ifold]['id']
#camalid = '0B18tnyqgrwlpbFV0aXA4ckxXUlE' # id of CAMAL data folder
#file_list = drive.ListFile({'q': "'%s' in parents and trashed=false" % camalid}).GetList()
file_list = ListFolder(camalid)
folders = {}
for f in file_list:
folders[str(f['title'])] = str(f['id'])
if oe.night in folders.keys():
nightid = folders[oe.night] # store night id
else:
# Create folder
nightfolder = drive.CreateFile({
'title':
oe.night,
"parents": [{
"id": camalid
}],
"mimeType":
"application/vnd.google-apps.folder"
})
nightfolder.Upload()
file_list = ListFolder(camalid)
for f in file_list:
folders[str(f['title'])] = str(f['id'])
nightid = folders[oe.night] # store night id
files = glob.glob(oe.finaldataPath + '\\*')
# Upload Files to night's folder
for filepath in files:
fname = filepath.split('\\')[-1]
f = drive.CreateFile({
'title':
fname,
"parents": [{
"kind": "drive#fileLink",
"id": nightid
}]
})
f.SetContentFile(filepath)
f.Upload()
def train(hyp, opt, device, tb_writer=None):
logger.info('')
logger.info(colorstr('hyperparameters: ') + ', '.join(f'{k}={v}' for k, v in hyp.items()))
save_dir, epochs, batch_size, total_batch_size, weights, rank, task_name, timestamp = \
Path(opt.save_dir), opt.epochs, opt.batch_size, opt.total_batch_size, opt.weights, opt.global_rank, opt.task,\
opt.timestamp
# Directories
from datetime import datetime
dt_str = timestamp
model_name = weights.split('.')[0]
# cropping_yolov5l_29May21-035456_exp2
full_model_name = f'{task_name}_{model_name}_{dt_str}'
wdir = save_dir / 'weights'
wdir.mkdir(parents=True, exist_ok=True) # make dir
last = wdir / f'{full_model_name}_last.pt'
best = wdir / f'{full_model_name}_best.pt'
results_file = save_dir / 'results.txt'
# Save run settings
with open(save_dir / 'hyp.yaml', 'w') as f:
yaml.safe_dump(hyp, f, sort_keys=False)
with open(save_dir / 'opt.yaml', 'w') as f:
yaml.safe_dump(vars(opt), f, sort_keys=False)
# Configure
plots = not opt.evolve # create plots
cuda = device.type != 'cpu'
init_seeds(2 + rank)
with open(opt.data) as f:
data_dict = yaml.safe_load(f) # data dict
is_coco = opt.data.endswith('coco.yaml')
# Logging- Doing this before checking the dataset. Might update data_dict
loggers = {'wandb': None} # loggers dict
if rank in [-1, 0]:
opt.hyp = hyp # add hyperparameters
run_id = torch.load(weights).get('wandb_id') if weights.endswith('.pt') and os.path.isfile(weights) else None
wandb_logger = WandbLogger(opt, save_dir.stem, run_id, data_dict)
loggers['wandb'] = wandb_logger.wandb
data_dict = wandb_logger.data_dict
if wandb_logger.wandb:
weights, epochs, hyp = opt.weights, opt.epochs, opt.hyp # WandbLogger might update weights, epochs if resuming
nc = 1 if opt.single_cls else int(data_dict['nc']) # number of classes
names = ['item'] if opt.single_cls and len(data_dict['names']) != 1 else data_dict['names'] # class names
assert len(names) == nc, '%g names found for nc=%g dataset in %s' % (len(names), nc, opt.data) # check
# Model
pretrained = weights.endswith('.pt')
if pretrained:
with torch_distributed_zero_first(rank):
attempt_download(weights) # download if not found locally
ckpt = torch.load(weights, map_location=device) # load checkpoint
model = Model(opt.cfg or ckpt['model'].yaml, ch=3, nc=nc, anchors=hyp.get('anchors')).to(device) # create
exclude = ['anchor'] if (opt.cfg or hyp.get('anchors')) and not opt.resume else [] # exclude keys
state_dict = ckpt['model'].float().state_dict() # to FP32
state_dict = intersect_dicts(state_dict, model.state_dict(), exclude=exclude) # intersect
model.load_state_dict(state_dict, strict=False) # load
logger.info('Transferred %g/%g items from %s' % (len(state_dict), len(model.state_dict()), weights)) # report
else:
model = Model(opt.cfg, ch=3, nc=nc, anchors=hyp.get('anchors')).to(device) # create
with torch_distributed_zero_first(rank):
check_dataset(data_dict) # check
train_path = data_dict['train']
test_path = data_dict['val']
# Freeze
freeze = [] # parameter names to freeze (full or partial)
for k, v in model.named_parameters():
v.requires_grad = True # train all layers
if any(x in k for x in freeze):
print('freezing %s' % k)
v.requires_grad = False
# Optimizer
nbs = 64 # nominal batch size
accumulate = max(round(nbs / total_batch_size), 1) # accumulate loss before optimizing
hyp['weight_decay'] *= total_batch_size * accumulate / nbs # scale weight_decay
logger.info(f"Scaled weight_decay = {hyp['weight_decay']}")
pg0, pg1, pg2 = [], [], [] # optimizer parameter groups
for k, v in model.named_modules():
if hasattr(v, 'bias') and isinstance(v.bias, nn.Parameter):
pg2.append(v.bias) # biases
if isinstance(v, nn.BatchNorm2d):
pg0.append(v.weight) # no decay
elif hasattr(v, 'weight') and isinstance(v.weight, nn.Parameter):
pg1.append(v.weight) # apply decay
if opt.adam:
optimizer = optim.Adam(pg0, lr=hyp['lr0'], betas=(hyp['momentum'], 0.999)) # adjust beta1 to momentum
else:
optimizer = optim.SGD(pg0, lr=hyp['lr0'], momentum=hyp['momentum'], nesterov=True)
optimizer.add_param_group({'params': pg1, 'weight_decay': hyp['weight_decay']}) # add pg1 with weight_decay
optimizer.add_param_group({'params': pg2}) # add pg2 (biases)
logger.info('Optimizer groups: %g .bias, %g conv.weight, %g other' % (len(pg2), len(pg1), len(pg0)))
del pg0, pg1, pg2
# Scheduler https://arxiv.org/pdf/1812.01187.pdf
# https://pytorch.org/docs/stable/_modules/torch/optim/lr_scheduler.html#OneCycleLR
if opt.linear_lr:
lf = lambda x: (1 - x / (epochs - 1)) * (1.0 - hyp['lrf']) + hyp['lrf'] # linear
else:
lf = one_cycle(1, hyp['lrf'], epochs) # cosine 1->hyp['lrf']
scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lf)
# plot_lr_scheduler(optimizer, scheduler, epochs)
# EMA
ema = ModelEMA(model) if rank in [-1, 0] else None
# Resume
start_epoch, best_fitness = 0, 0.0
if pretrained:
# Optimizer
if ckpt['optimizer'] is not None:
optimizer.load_state_dict(ckpt['optimizer'])
best_fitness = ckpt['best_fitness']
# EMA
if ema and ckpt.get('ema'):
ema.ema.load_state_dict(ckpt['ema'].float().state_dict())
ema.updates = ckpt['updates']
# Results
if ckpt.get('training_results') is not None:
results_file.write_text(ckpt['training_results']) # write results.txt
# Epochs
start_epoch = ckpt['epoch'] + 1
if opt.resume:
assert start_epoch > 0, '%s training to %g epochs is finished, nothing to resume.' % (weights, epochs)
if epochs < start_epoch:
logger.info('%s has been trained for %g epochs. Fine-tuning for %g additional epochs.' %
(weights, ckpt['epoch'], epochs))
epochs += ckpt['epoch'] # finetune additional epochs
del ckpt, state_dict
# Image sizes
gs = max(int(model.stride.max()), 32) # grid size (max stride)
nl = model.model[-1].nl # number of detection layers (used for scaling hyp['obj'])
imgsz, imgsz_test = [check_img_size(x, gs) for x in opt.img_size] # verify imgsz are gs-multiples
# DP mode
if cuda and rank == -1 and torch.cuda.device_count() > 1:
model = torch.nn.DataParallel(model)
# SyncBatchNorm
if opt.sync_bn and cuda and rank != -1:
model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model).to(device)
logger.info('Using SyncBatchNorm()')
# Trainloader
dataloader, dataset = create_dataloader(train_path, imgsz, batch_size, gs, opt,
hyp=hyp, augment=True, cache=opt.cache_images, rect=opt.rect, rank=rank,
world_size=opt.world_size, workers=opt.workers,
image_weights=opt.image_weights, quad=opt.quad, prefix=colorstr('train: '))
mlc = np.concatenate(dataset.labels, 0)[:, 0].max() # max label class
nb = len(dataloader) # number of batches
assert mlc < nc, 'Label class %g exceeds nc=%g in %s. Possible class labels are 0-%g' % (mlc, nc, opt.data, nc - 1)
# Process 0
if rank in [-1, 0]:
testloader = create_dataloader(test_path, imgsz_test, batch_size * 2, gs, opt, # testloader
hyp=hyp, cache=opt.cache_images and not opt.notest, rect=True, rank=-1,
world_size=opt.world_size, workers=opt.workers,
pad=0.5, prefix=colorstr('val: '))[0]
if not opt.resume:
labels = np.concatenate(dataset.labels, 0)
c = torch.tensor(labels[:, 0]) # classes
# cf = torch.bincount(c.long(), minlength=nc) + 1. # frequency
# model._initialize_biases(cf.to(device))
if plots:
plot_labels(labels, names, save_dir, loggers)
if tb_writer:
tb_writer.add_histogram('classes', c, 0)
# Anchors
if not opt.noautoanchor:
check_anchors(dataset, model=model, thr=hyp['anchor_t'], imgsz=imgsz)
model.half().float() # pre-reduce anchor precision
# DDP mode
if cuda and rank != -1:
model = DDP(model, device_ids=[opt.local_rank], output_device=opt.local_rank,
# nn.MultiheadAttention incompatibility with DDP https://github.com/pytorch/pytorch/issues/26698
find_unused_parameters=any(isinstance(layer, nn.MultiheadAttention) for layer in model.modules()))
# Model parameters
hyp['box'] *= 3. / nl # scale to layers
hyp['cls'] *= nc / 80. * 3. / nl # scale to classes and layers
hyp['obj'] *= (imgsz / 640) ** 2 * 3. / nl # scale to image size and layers
hyp['label_smoothing'] = opt.label_smoothing
model.nc = nc # attach number of classes to model
model.hyp = hyp # attach hyperparameters to model
model.gr = 1.0 # iou loss ratio (obj_loss = 1.0 or iou)
model.class_weights = labels_to_class_weights(dataset.labels, nc).to(device) * nc # attach class weights
model.names = names
# Start training
t0 = time.time()
nw = max(round(hyp['warmup_epochs'] * nb), 1000) # number of warmup iterations, max(3 epochs, 1k iterations)
# nw = min(nw, (epochs - start_epoch) / 2 * nb) # limit warmup to < 1/2 of training
maps = np.zeros(nc) # mAP per class
results = (0, 0, 0, 0, 0, 0, 0) # P, R, [email protected], [email protected], val_loss(box, obj, cls)
scheduler.last_epoch = start_epoch - 1 # do not move
scaler = amp.GradScaler(enabled=cuda)
compute_loss = ComputeLoss(model) # init loss class
logger.info(f'Image sizes {imgsz} train, {imgsz_test} test\n'
f'Using {dataloader.num_workers} dataloader workers\n'
f'Logging results to {save_dir}\n'
f'Starting training for {epochs} epochs...')
for epoch in range(start_epoch, epochs): # epoch ------------------------------------------------------------------
model.train()
# Update image weights (optional)
if opt.image_weights:
# Generate indices
if rank in [-1, 0]:
cw = model.class_weights.cpu().numpy() * (1 - maps) ** 2 / nc # class weights
iw = labels_to_image_weights(dataset.labels, nc=nc, class_weights=cw) # image weights
dataset.indices = random.choices(range(dataset.n), weights=iw, k=dataset.n) # rand weighted idx
# Broadcast if DDP
if rank != -1:
indices = (torch.tensor(dataset.indices) if rank == 0 else torch.zeros(dataset.n)).int()
dist.broadcast(indices, 0)
if rank != 0:
dataset.indices = indices.cpu().numpy()
# Update mosaic border
# b = int(random.uniform(0.25 * imgsz, 0.75 * imgsz + gs) // gs * gs)
# dataset.mosaic_border = [b - imgsz, -b] # height, width borders
mloss = torch.zeros(4, device=device) # mean losses
if rank != -1:
dataloader.sampler.set_epoch(epoch)
pbar = enumerate(dataloader)
logger.info(('\n' + '%10s' * 8) % ('Epoch', 'gpu_mem', 'box', 'obj', 'cls', 'total', 'labels', 'img_size'))
if rank in [-1, 0]:
pbar = tqdm(pbar, total=nb) # progress bar
optimizer.zero_grad()
for i, (imgs, targets, paths, _) in pbar: # batch -------------------------------------------------------------
ni = i + nb * epoch # number integrated batches (since train start)
imgs = imgs.to(device, non_blocking=True).float() / 255.0 # uint8 to float32, 0-255 to 0.0-1.0
# Warmup
if ni <= nw:
xi = [0, nw] # x interp
# model.gr = np.interp(ni, xi, [0.0, 1.0]) # iou loss ratio (obj_loss = 1.0 or iou)
accumulate = max(1, np.interp(ni, xi, [1, nbs / total_batch_size]).round())
for j, x in enumerate(optimizer.param_groups):
# bias lr falls from 0.1 to lr0, all other lrs rise from 0.0 to lr0
x['lr'] = np.interp(ni, xi, [hyp['warmup_bias_lr'] if j == 2 else 0.0, x['initial_lr'] * lf(epoch)])
if 'momentum' in x:
x['momentum'] = np.interp(ni, xi, [hyp['warmup_momentum'], hyp['momentum']])
# Multi-scale
if opt.multi_scale:
sz = random.randrange(imgsz * 0.5, imgsz * 1.5 + gs) // gs * gs # size
sf = sz / max(imgs.shape[2:]) # scale factor
if sf != 1:
ns = [math.ceil(x * sf / gs) * gs for x in imgs.shape[2:]] # new shape (stretched to gs-multiple)
imgs = F.interpolate(imgs, size=ns, mode='bilinear', align_corners=False)
# Forward
with amp.autocast(enabled=cuda):
pred = model(imgs) # forward
loss, loss_items = compute_loss(pred, targets.to(device)) # loss scaled by batch_size
if rank != -1:
loss *= opt.world_size # gradient averaged between devices in DDP mode
if opt.quad:
loss *= 4.
# Backward
scaler.scale(loss).backward()
# Optimize
if ni % accumulate == 0:
scaler.step(optimizer) # optimizer.step
scaler.update()
optimizer.zero_grad()
if ema:
ema.update(model)
# Print
if rank in [-1, 0]:
mloss = (mloss * i + loss_items) / (i + 1) # update mean losses
mem = '%.3gG' % (torch.cuda.memory_reserved() / 1E9 if torch.cuda.is_available() else 0) # (GB)
s = ('%10s' * 2 + '%10.4g' * 6) % (
'%g/%g' % (epoch, epochs - 1), mem, *mloss, targets.shape[0], imgs.shape[-1])
pbar.set_description(s)
# Plot
if plots and ni < 3:
f = save_dir / f'train_batch{ni}.jpg' # filename
Thread(target=plot_images, args=(imgs, targets, paths, f), daemon=True).start()
if tb_writer:
tb_writer.add_graph(torch.jit.trace(model, imgs, strict=False), []) # add model graph
# tb_writer.add_image(f, result, dataformats='HWC', global_step=epoch)
elif plots and ni == 10 and wandb_logger.wandb:
wandb_logger.log({"Mosaics": [wandb_logger.wandb.Image(str(x), caption=x.name) for x in
save_dir.glob('train*.jpg') if x.exists()]})
# end batch ------------------------------------------------------------------------------------------------
# end epoch ----------------------------------------------------------------------------------------------------
# Scheduler
lr = [x['lr'] for x in optimizer.param_groups] # for tensorboard
scheduler.step()
# DDP process 0 or single-GPU
if rank in [-1, 0]:
# mAP
ema.update_attr(model, include=['yaml', 'nc', 'hyp', 'gr', 'names', 'stride', 'class_weights'])
final_epoch = epoch + 1 == epochs
if not opt.notest or final_epoch: # Calculate mAP
wandb_logger.current_epoch = epoch + 1
results, maps, times = test.test(data_dict,
batch_size=batch_size * 2,
imgsz=imgsz_test,
model=ema.ema,
single_cls=opt.single_cls,
dataloader=testloader,
save_dir=save_dir,
verbose=nc < 50 and final_epoch,
plots=plots and final_epoch,
wandb_logger=wandb_logger,
compute_loss=compute_loss,
is_coco=is_coco)
# Write
with open(results_file, 'a') as f:
f.write(s + '%10.4g' * 7 % results + '\n') # append metrics, val_loss
# Log
tags = ['train/box_loss', 'train/obj_loss', 'train/cls_loss', # train loss
'metrics/precision', 'metrics/recall', 'metrics/mAP_0.5', 'metrics/mAP_0.5:0.95',
'val/box_loss', 'val/obj_loss', 'val/cls_loss', # val loss
'x/lr0', 'x/lr1', 'x/lr2'] # params
for x, tag in zip(list(mloss[:-1]) + list(results) + lr, tags):
if tb_writer:
tb_writer.add_scalar(tag, x, epoch) # tensorboard
if wandb_logger.wandb:
wandb_logger.log({tag: x}) # W&B
# Update best mAP
fi = fitness(np.array(results).reshape(1, -1)) # weighted combination of [P, R, [email protected], [email protected]]
if fi > best_fitness:
best_fitness = fi
wandb_logger.end_epoch(best_result=best_fitness == fi)
# Save model
if (not opt.nosave) or (final_epoch and not opt.evolve): # if save
ckpt = {'epoch': epoch,
'best_fitness': best_fitness,
'training_results': results_file.read_text(),
'model': deepcopy(model.module if is_parallel(model) else model).half(),
'ema': deepcopy(ema.ema).half(),
'updates': ema.updates,
'optimizer': optimizer.state_dict(),
'wandb_id': wandb_logger.wandb_run.id if wandb_logger.wandb else None}
# Save last, best and delete
torch.save(ckpt, last)
if best_fitness == fi:
torch.save(ckpt, best)
if wandb_logger.wandb:
if ((epoch + 1) % opt.save_period == 0 and not final_epoch) and opt.save_period != -1:
wandb_logger.log_model(
last.parent, opt, epoch, fi, best_model=best_fitness == fi)
del ckpt
# end epoch ----------------------------------------------------------------------------------------------------
# end training
if rank in [-1, 0]:
# Plots
if plots:
plot_results(save_dir=save_dir) # save as results.png
if wandb_logger.wandb:
files = ['results.png', 'confusion_matrix.png', *[f'{x}_curve.png' for x in ('F1', 'PR', 'P', 'R')]]
wandb_logger.log({"Results": [wandb_logger.wandb.Image(str(save_dir / f), caption=f) for f in files
if (save_dir / f).exists()]})
# Test best.pt
logger.info('%g epochs completed in %.3f hours.\n' % (epoch - start_epoch + 1, (time.time() - t0) / 3600))
if opt.data.endswith('coco.yaml') and nc == 80: # if COCO
for m in (last, best) if best.exists() else (last): # speed, mAP tests
results, _, _ = test.test(opt.data,
batch_size=batch_size * 2,
imgsz=imgsz_test,
conf_thres=0.001,
iou_thres=0.7,
model=attempt_load(m, device).half(),
single_cls=opt.single_cls,
dataloader=testloader,
save_dir=save_dir,
save_json=True,
plots=False,
is_coco=is_coco)
# Strip optimizers
final = best if best.exists() else last # final model
for f in last, best:
if f.exists():
strip_optimizer(f) # strip optimizers
if opt.bucket:
os.system(f'gsutil cp {final} gs://{opt.bucket}/weights') # upload
if wandb_logger.wandb and not opt.evolve: # Log the stripped model
wandb_logger.wandb.log_artifact(str(final), type='model',
name='run_' + wandb_logger.wandb_run.id + '_model',
aliases=['last', 'best', 'stripped'])
wandb_logger.finish_run()
else:
dist.destroy_process_group()
torch.cuda.empty_cache()
### archive and copy results to gdrive ###
source_dir = save_dir
import shutil
shutil.make_archive(source_dir, 'zip', source_dir)
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
gauth = GoogleAuth()
drive = GoogleDrive(gauth)
return results
context[dateindex] = datert
context[eventindex] = linkrt
doc.save(filename)
time.sleep(.25)
for x in os.listdir(index):
doc = docxtpl.DocxTemplate(index+x)
doc.render(context)
doc.save(index+x)
##### Variables
drive = GoogleDrive(GoogleAuth())
filesystem = "C:/Users/tyler/Documents/GitHub/The-Trialists-Toolkit/Archives/Filesystem/"
rootfilesystem = "C:/Users/tyler/Desktop/Workspace/TTT Admin Suite/"
index = filesystem + r"index/"
os.makedirs(index)
folder = "application/vnd.google-apps.folder"
filesystemIndex = parse_filesystem(filesystem)
filesystemIDs = verify_root(filesystem)
context = {}
##### Main
generate_filesystem(filesystemIndex)
print("\n[ FILESYSTEM SUCCESSFULLY GENERATED ]\n")
cleandata = generate_summary_files(clean_data(filesystemIndex))
print("\n[ SUMMARY FILES SUCCESSFULLY GENERATED ]\n")
def startDownloads(songsFolder):
gauth = GoogleAuth() # Google Drive authentication
gauth.LocalWebserverAuth() # Needed only for initial auth
drive = GoogleDrive(gauth)
connection = sqlite3.connect('../ChartBase.db')
cursor = connection.cursor()
cursor.execute('SELECT * FROM links WHERE downloaded=0')
links = cursor.fetchall()
for link in links:
url = link[0]
source = link[1]
urlDecoded = urllib.parse.unquote(url)
domain = re.search(r'.*?://(.*?)/', urlDecoded).group(1)
tmpFolder = os.path.join(songsFolder, 'tmp/')
if not os.path.exists(tmpFolder):
os.mkdir(tmpFolder)
if 'drive.google' in domain:
try:
print(f'downloading from gDrive: {url}')
gDriveDownload(drive, urlDecoded, tmpFolder)
except (KeyboardInterrupt, SystemExit):
if os.path.exists(tmpFolder):
print(f'removing tmpFolder due to sysexit: {tmpFolder}')
shutil.rmtree(tmpFolder)
raise
except:
cursor.execute(
f'UPDATE links SET downloaded=-1 WHERE url="{url}"')
connection.commit()
if os.path.exists(tmpFolder):
print(f'removing tmpFolder due to except: {tmpFolder}')
shutil.rmtree(tmpFolder)
if os.path.exists(tmpFolder):
print(f'importing: {url}')
importDownloaded(songsFolder, url, source, connection)
print(f'updating in db: {url}')
cursor.execute(
f'UPDATE links SET downloaded=1 WHERE url="{url}"')
connection.commit()
else:
try:
print(f'downloading: {url}')
_ = wget.download(urlDecoded, tmpFolder)
except (KeyboardInterrupt, SystemExit):
if os.path.exists(tmpFolder):
print(f'removing tmpFolder due to sysexit: {tmpFolder}')
shutil.rmtree(tmpFolder)
raise
except:
cursor.execute(
f'UPDATE links SET downloaded=-1 WHERE url="{url}"')
connection.commit()
if os.path.exists(tmpFolder):
print(f'removing tmpFolder due to except: {tmpFolder}')
shutil.rmtree(tmpFolder)
if os.path.exists(tmpFolder):
print(f'importing: {url}')
importDownloaded(songsFolder, url, source, connection)
print(f'updating in db: {url}')
cursor.execute(
f'UPDATE links SET downloaded=1 WHERE url="{url}"')
connection.commit()
def get_link(bot, update):
TRChatBase(update.from_user.id, update.text, "gofile")
if str(update.from_user.id) in Config.BANNED_USERS:
bot.send_message(chat_id=update.chat.id,
text=Translation.ABUSIVE_USERS,
reply_to_message_id=update.message_id,
disable_web_page_preview=True,
parse_mode=pyrogram.ParseMode.HTML)
return
logger.info(update.from_user)
if update.reply_to_message is not None:
reply_message = update.reply_to_message
download_location = Config.DOWNLOAD_LOCATION + "/"
start = datetime.now()
a = bot.send_message(chat_id=update.chat.id,
text=Translation.DOWNLOAD_START,
reply_to_message_id=update.message_id)
c_time = time.time()
after_download_file_name = bot.download_media(
message=reply_message,
file_name=download_location,
progress=progress_for_pyrogram,
progress_args=(Translation.DOWNLOAD_START, a.message_id,
update.chat.id, c_time))
download_extension = after_download_file_name.rsplit(".", 1)[-1]
upload_name = after_download_file_name.rsplit("/", 1)[-1]
upload_name = upload_name.replace(" ", "_")
bot.edit_message_text(text=Translation.SAVED_RECVD_DOC_FILE,
chat_id=update.chat.id,
message_id=a.message_id)
end_one = datetime.now()
if str(update.from_user.id) in Config.G_DRIVE_AUTH_DRQ:
gauth = Config.G_DRIVE_AUTH_DRQ[str(update.from_user.id)]
# Create GoogleDrive instance with authenticated GoogleAuth instance.
drive = GoogleDrive(gauth)
file_inance = drive.CreateFile()
# Read file and set it as a content of this instance.
file_inance.SetContentFile(after_download_file_name)
file_inance.Upload() # Upload the file.
end_two = datetime.now()
time_taken_for_upload = (end_two - end_one).seconds
logger.info(file_inance)
adfulurl = file_inance.webContentLink
max_days = 0
else:
url = "https://srv-file5.gofile.io/upload"
max_days = 5
timeseconds = int(time.time())
timesecondsplusexpiry = int(
time.time()) + (max_days * 24 * 60 * 60)
command_to_exec = [
"curl", "-F", "filesUploaded=@" + after_download_file_name,
"-F", "expire=" + str(timesecondsplusexpiry), "-F",
"category=file", "-F", "comments=0", url
]
bot.edit_message_text(text=Translation.UPLOAD_START,
chat_id=update.chat.id,
message_id=a.message_id)
try:
logger.info(command_to_exec)
t_response = subprocess.check_output(command_to_exec,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as exc:
logger.info("Status : FAIL", exc.returncode, exc.output)
bot.edit_message_text(chat_id=update.chat.id,
text=exc.output.decode("UTF-8"),
message_id=a.message_id)
return False
else:
logger.info(t_response)
print(t_response)
t_response_arry = "https://gofile.io/?c=" + json.loads(
t_response.decode("UTF-8").split("\n")
[-1].strip())['data']['code']
#shorten_api_url = "http://ouo.io/api/{}?s={}".format(Config.OUO_IO_API_KEY, t_response_arry)
#adfulurl = requests.get(shorten_api_url).text
bot.edit_message_text(chat_id=update.chat.id,
text=Translation.AFTER_GET_DL_LINK.format(
t_response_arry, max_days),
parse_mode=pyrogram.ParseMode.HTML,
message_id=a.message_id,
disable_web_page_preview=True)
try:
os.remove(after_download_file_name)
except:
pass
else:
bot.send_message(chat_id=update.chat.id,
text=Translation.REPLY_TO_DOC_GET_LINK,
reply_to_message_id=update.message_id)
def handle_action_tag(self, ttype, data):
logging.debug("Open : %s", data)
gauth = GoogleAuth()
drive = GoogleDrive(gauth)
gauth.LoadCredentialsFile("mycreds.txt")
check = os.stat("mod/hashtags").st_size
######---find FOLDER ID
#file_list = drive.ListFile({'q': "'root' in parents and trashed=false"}).GetList()
#for file1 in file_list:
# print 'title: %s, id: %s' % (file1['title'], file1['id'])
######---authentication
if gauth.credentials is None:
gauth.LocalWebserverAuth()
if gauth.access_token_expired:
gauth.Refresh()
else:
gauth.Authorize()
gauth.SaveCredentialsFile("mycreds.txt")
if (data == 'DOCS'):
### upload achat and depense
file1 = drive.CreateFile({
'parents': [{
"id":
'0B8mDDuHeuNHDfmM0OXlWTndpdkczNHBBY3VJaXJ2ZlNqVVBoWWk3UDZnc0NvMS1Gd1JtWU0'
}]
})
file1.SetContentFile('mod/achat.txt')
file1.Upload()
file2 = drive.CreateFile({
'parents': [{
"id":
'0B8mDDuHeuNHDfmM0OXlWTndpdkczNHBBY3VJaXJ2ZlNqVVBoWWk3UDZnc0NvMS1Gd1JtWU0'
}]
})
file2.SetContentFile('mod/depense.txt')
file2.Upload()
logging.debug("Upload done.")
if (data == 'PICS'):
file = open('mod/filepath', 'r')
path = file.readlines()
pic = path[0]
if (check == 0):
### list every photo
onlyfiles = [
f for f in listdir('/home/pi/images/')
if isfile(join('/home/pi/images/', f))
]
### upload pic
file1 = drive.CreateFile({
'parents': [{
"id":
'0B8mDDuHeuNHDfmM0OXlWTndpdkczNHBBY3VJaXJ2ZlNqVVBoWWk3UDZnc0NvMS1Gd1JtWU0'
}]
})
file1.SetContentFile(pic)
file1.Upload()
logging.debug("Upload done.")
file = open('mod/filepath', 'w').close()
else:
file = open('mod/hashtags', 'r')
hash = file.readlines()
hashtag = hash[0]
hashtag = hashtag[1:-1]
file_list = drive.ListFile({
'q':
"'root' in parents and trashed=false"
}).GetList()
for file1 in file_list:
print 'title: %s, id: %s' % (file1['title'], file1['id'])
print hashtag
print file1['title']
if (file1['title'] == hashtag):
file1 = drive.CreateFile(
{'parents': [{
"id": file1['id']
}]})
file1.SetContentFile(pic)
file1.Upload()
logging.debug("Upload done.")
file = open('mod/filepath', 'w').close()
file = open('mod/hashtags', 'w').close()
def callback(bot, update):
conn = sqlite3.connect('data/DMI_DB.db')
keyboard2 = [[]]
icona = ""
number_row = 0
number_array = 0
update.callback_query.data = update.callback_query.data.replace("Drive_", "")
# print('Callback query data: ' + str(update.callback_query.data))
if len(update.callback_query.data) < 13:
#conn.execute("DELETE FROM 'Chat_id_List'")
array_value = update['callback_query']['message']['text'].split(" ")
try:
if len(array_value) == 4:
array_value.insert(0, "None")
if len(array_value) == 5:
conn.execute("INSERT INTO 'Chat_id_List' VALUES ("+update.callback_query.data+",'" + array_value[4] + "','" + array_value[1] + "','" + array_value[2] + "','" + array_value[3] + "') ")
bot.sendMessage(chat_id=update.callback_query.data, text="🔓 La tua richiesta è stata accettata. Leggi il file README")
bot.sendDocument(chat_id=update.callback_query.data, document=open('data/README.pdf', 'rb'))
request_elimination_text = "Richiesta di " + str(array_value[1]) + " " + str(array_value[2]) + " estinta"
bot.editMessageText(text=request_elimination_text, chat_id=config_map['dev_group_chatid'], message_id=update.callback_query.message.message_id)
bot.sendMessage(chat_id=config_map['dev_group_chatid'], text=str(array_value[1]) + " " + str(array_value[2] + str(" è stato inserito nel database")))
elif len(array_value) == 4:
conn.execute("INSERT INTO 'Chat_id_List'('Chat_id','Nome','Cognome','Email') VALUES (" + update.callback_query.data + ",'" + array_value[1] + "','" + array_value[2] + "','" + array_value[3] + "')")
bot.sendMessage(chat_id=update.callback_query.data, text="🔓 La tua richiesta è stata accettata. Leggi il file README")
bot.sendDocument(chat_id=update.callback_query.data, document=open('data/README.pdf', 'rb'))
else:
bot.sendMessage(chat_id=config_map['dev_group_chatid'], text=str("ERRORE INSERIMENTO: ") + str(update['callback_query']['message']['text']) + " " + str(update['callback_query']['data']))
conn.commit()
except Exception as error:
print(error)
bot.sendMessage(chat_id=config_map['dev_group_chatid'], text=str("ERRORE INSERIMENTO: ") + str(update['callback_query']['message']['text']) + " " + str(update['callback_query']['data']))
text = ""
else:
pid = os.fork()
if (pid == 0):
settings_file = "config/settings.yaml"
gauth2 = GoogleAuth(settings_file=settings_file)
gauth2.CommandLineAuth()
# gauth2.LocalWebserverAuth()
drive2 = GoogleDrive(gauth2)
bot2 = telegram.Bot(TOKEN)
file1 = drive2.CreateFile({'id': update.callback_query.data})
if file1['mimeType'] == "application/vnd.google-apps.folder":
file_list2 = None
try:
istance_file = drive2.ListFile({'q': "'"+file1['id']+"' in parents and trashed=false", 'orderBy': 'folder,title'})
file_list2 = istance_file.GetList()
with open("./logs/debugDrive.txt", "a") as debugfile:
debugfile.write("- Log time:\n {}".format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
debugfile.write("- File:\n {}".format(str(json.dumps(file1))))
debugfile.write("- IstanceFile:\n {}".format(str(json.dumps(istance_file))))
debugfile.write("- FileList:\n {}".format(str(json.dumps(file_list2))))
debugfile.write("\n------------\n")
except Exception as e:
with open("./logs/debugDrive.txt", "a") as debugfile:
debugfile.write("- Log time:\n {}".format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
debugfile.write("- Error:\n {}".format(e))
debugfile.write("\n------------\n")
print("- Drive error: {}".format(e))
bot2.sendMessage(chat_id=update['callback_query']['from_user']['id'], text="Si è verificato un errore, ci scusiamo per il disagio. Contatta i devs. /help")
sys.exit(0)
formats = {
** { "pdf" : "📕 " },
** dict.fromkeys([' a', 'b', 'c'], 10),
** dict.fromkeys(["doc", "docx", "txt"], "📘 "),
** dict.fromkeys(["jpg", "png", "gif"], "📷 "),
** dict.fromkeys(["rar", "zip"], "🗄 "),
** dict.fromkeys(["out", "exe"], "⚙ "),
** dict.fromkeys(["c", "cpp", "h", "py", "java", "js", "html", "php"], "💻 ")
}
for file2 in file_list2:
if file2['mimeType'] == "application/vnd.google-apps.folder":
if number_row >= 1:
keyboard2.append([InlineKeyboardButton("🗂 "+file2['title'], callback_data="Drive_" + file2['id'])])
number_row = 0
number_array += 1
else:
keyboard2[number_array].append(InlineKeyboardButton("🗂 "+file2['title'], callback_data="Drive_" + file2['id']))
number_row += 1
else:
file_format = file2['title'][-5:] # get last 5 characters of strings
file_format = file_format.split(".") # split file_format per "."
file_format = file_format[len(file_format)-1] # get last element of file_format
icona = "📄 "
if file_format in formats.keys():
icona = formats[file_format]
if number_row >= 1:
keyboard2.append([InlineKeyboardButton(icona+file2['title'], callback_data="Drive_" + file2['id'])])
number_row = 0
number_array += 1
else:
keyboard2[number_array].append(InlineKeyboardButton(icona+file2['title'], callback_data="Drive_" + file2['id']))
number_row += 1
if len(file1['parents']) > 0 and file1['parents'][0]['id'] != '0ADXK_Yx5406vUk9PVA':
keyboard2.append([InlineKeyboardButton("🔙", callback_data="Drive_" + file1['parents'][0]['id'])])
reply_markup3 = InlineKeyboardMarkup(keyboard2)
bot2.sendMessage(chat_id=update['callback_query']['from_user']['id'], text=file1['title']+":", reply_markup=reply_markup3)
elif file1['mimeType'] == "application/vnd.google-apps.document":
bot2.sendMessage(chat_id=update['callback_query']['from_user']['id'], text="Impossibile scaricare questo file poichè esso è un google document, Andare sul seguente link")
bot2.sendMessage(chat_id=update['callback_query']['from_user']['id'], text=file1['exportLinks']['application/pdf'])
else:
try:
file_d = drive2.CreateFile({'id': file1['id']})
if int(file_d['fileSize']) < 5e+7:
file_d.GetContentFile('file/'+file1['title'])
file_s = file1['title']
filex = open(str("file/" + file_s), "rb")
bot2.sendChatAction(chat_id=update['callback_query']['from_user']['id'], action="UPLOAD_DOCUMENT")
bot2.sendDocument(chat_id=update['callback_query']['from_user']['id'], document=filex)
os.remove(str("file/" + file_s))
else:
bot2.sendMessage(chat_id=update['callback_query']['from_user']['id'], text="File troppo grande per il download diretto, scarica dal seguente link")
# file_d['downloadUrl']
bot2.sendMessage(chat_id=update['callback_query']['from_user']['id'], text=file_d['alternateLink'])
except Exception as e:
print("- Drive error: {}".format(e))
bot2.sendMessage(chat_id=update['callback_query']['from_user']['id'], text="Impossibile scaricare questo file, contattare gli sviluppatori del bot")
open("logs/errors.txt", "a+").write(str(e) + str(file_d['title'])+"\n")
sys.exit(0)
os.waitpid(pid, 0)
conn.close()
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
import os
import time
import stat
gAuth = GoogleAuth()
drive = GoogleDrive(gAuth)
time_to_sleep = 5 * 60
file_list = []
def getNewPhotos(folder_name=None):
authenticateUser()
walktree('./pictures', addtolist)
toDownload = getFileList(folder_name)
downloadFiles(toDownload)
time.sleep(time_to_sleep)
def walktree(top, callback):
"""recursively descend the directory tree rooted at top, calling the
callback function for each regular file. Taken from the module-stat
example at: http://docs.python.org/lib/module-stat.html
"""
for f in os.listdir(top):
pathname = os.path.join(top, f)
mode = os.stat(pathname)[stat.ST_MODE]
if stat.S_ISDIR(mode):
# It's a directory, recurse into it
walktree(pathname, callback)
def __init__(self, credsfile='mycreds.txt'):
from pydrive.drive import GoogleDrive
self.gdrive = GoogleDrive(self._get_auth(credsfile))
def auth_drive():
gauth = GoogleAuth()
gauth.LocalWebserverAuth()
drive = GoogleDrive(gauth)
return drive
def addtoGDrive(pdfloc, pdfname):
"""
Uploads water quality PDF file to Google drive account under My Drive\Projects\Water_Quality\pdf location,
id:1GRunRWB7SKmH3I0wWbtyJ_UOCDiHGAxO
Parameters
----------
pdfloc: String. Location of PDF to uploaded with filename
pdfname: String. Filename of PDF to upload, uploaded this with name.
Returns
-------
Print statement
"""
try:
application.logger.debug(
"Attempting to authenticate with Google Drive API through saved OAuth credentials"
)
# Change location for client_secrets.json, application.py sits one level above this file and the file isn't
# called properly when this function is called, but the file needs to stay in that location such that
# quickstart.py can be called directly if needed
GoogleAuth.DEFAULT_SETTINGS['client_config_file'] = os.path.join(
app.root_path, 'WebAppProjects', 'WaterQualityViewer',
'credentials.json')
# Use command line to auth, must connect to host, visit URL, login/auth with Google, then paste provided text
# into terminal
# gauth.CommandLineAuth()
# Create authenticated GoogleDrive instance using settings from the setting.yaml file to auto-authenticate with
# saved credentials
gauth = GoogleAuth(
settings_file=os.path.join(app.root_path, 'WebAppProjects',
'WaterQualityViewer', 'settings.yaml'))
# Establish connection with Google Drive API
drive = GoogleDrive(gauth)
application.logger.debug("Connected to Google Drive account")
# Create a new GoogleDriveFile instance with a PDF mimetype in the water quality PDF folder using the parent folder's ID
newfile = drive.CreateFile({
"title":
pdfname,
'mimeType':
'application/pdf',
'parents': [{
'id': "1GRunRWB7SKmH3I0wWbtyJ_UOCDiHGAxO"
}]
})
# Read file and set the content of the new file instance
newfile.SetContentFile(pdfloc)
# Upload the file to Google Drive
newfile.Upload()
# print("File uploaded to Google Drive!")
application.logger.debug(f"File {pdfname} uploaded to Google Drive!")
except Exception as e:
print("GoogleDrive upload threw an error, emailing exception")
application.logger.error("Failed to upload to Google Drive account")
application.logger.error(e)
errorEmail.sendErrorEmail(script="GoogleDrive",
exceptiontype=e.__class__.__name__,
body=e)
# testing
# pdfloc = r"G:\My Drive\Projects\test_documents\Ocean_Water_Quality_Report_testing_20201002.pdf"
# pdfname = r"Ocean_Water_Quality_Report_testing_20201002.pdf"
#
# addtoGDrive(pdfloc, pdfname)
def googledrive_login():
gauth = GoogleAuth()
gauth.LocalWebserverAuth()
return GoogleDrive(gauth)
error_found = False
counter = 0
occurance = 0
current_workdir = os.getcwd() # Getting Current working directory Information
xmlFile = current_workdir + '/' + 'apple_health_export' + '/' + 'export.xml'
beatFile = current_workdir + '/' + 'heartbeat.csv
hBFile = current_workdir + '/' + 'heartbeat_List.csv'
print("+++ Message : Working Directory = " + current_workdir)
# Connecting to Google Drive
gLogin = GoogleAuth()
gLogin.LocalWebserverAuth() # This opens a google login page and select account to access program
drive = GoogleDrive(gLogin)
heartBeat = '1aoURMlMlb0DYPeI2vcM7nGKI44uMnu_I' # Shared Folder ID - Folder Name "HearBeat"
data_files = drive.ListFile({'q':"'"+heartBeat+"' in parents and trashed=false"}).GetList()
for file1 in data_files:
print('+++ Message : Downloading File = %s' % (file1['title']))
file1.GetContentFile(file1['title'])
if not os.path.isfile(current_workdir + '/' + 'export.zip'):
print('!!! Error : Unable to locate data folder - export.zip ')
error_found = True
else:
# Extract zip folder
with zipfile.ZipFile(current_workdir + '/' + 'export.zip',"r") as zip_ref:
zip_ref.extractall()
def drive(self):
from pydrive.auth import RefreshError
if not hasattr(self, "_gdrive"):
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
with open(
self.gdrive_user_credentials_path, "w"
) as credentials_file:
credentials_file.write(
os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA)
)
GoogleAuth.DEFAULT_SETTINGS["client_config_backend"] = "settings"
GoogleAuth.DEFAULT_SETTINGS["client_config"] = {
"client_id": self.client_id,
"client_secret": self.client_secret,
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://oauth2.googleapis.com/token",
"revoke_uri": "https://oauth2.googleapis.com/revoke",
"redirect_uri": "",
}
GoogleAuth.DEFAULT_SETTINGS["save_credentials"] = True
GoogleAuth.DEFAULT_SETTINGS["save_credentials_backend"] = "file"
GoogleAuth.DEFAULT_SETTINGS[
"save_credentials_file"
] = self.gdrive_user_credentials_path
GoogleAuth.DEFAULT_SETTINGS["get_refresh_token"] = True
GoogleAuth.DEFAULT_SETTINGS["oauth_scope"] = [
"https://www.googleapis.com/auth/drive",
"https://www.googleapis.com/auth/drive.appdata",
]
# Pass non existent settings path to force DEFAULT_SETTINGS loading
gauth = GoogleAuth(settings_file="")
try:
gauth.CommandLineAuth()
except RefreshError as exc:
raise GDriveAccessTokenRefreshError(
"Google Drive's access token refreshment is failed"
) from exc
except KeyError as exc:
raise GDriveMissedCredentialKeyError(
"Google Drive's user credentials file '{}' "
"misses value for key '{}'".format(
self.gdrive_user_credentials_path, str(exc)
)
)
# Handle pydrive.auth.AuthenticationError and others auth failures
except Exception as exc:
raise DvcException(
"Google Drive authentication failed"
) from exc
finally:
if os.getenv(RemoteGDrive.GDRIVE_USER_CREDENTIALS_DATA):
os.remove(self.gdrive_user_credentials_path)
self._gdrive = GoogleDrive(gauth)
self.remote_root_id = self.get_remote_id(
self.path_info, create=True
)
self._cached_dirs, self._cached_ids = self.cache_root_dirs()
return self._gdrive
def __init__(self, settings_file="settings.yaml"):
self.gauth = GoogleAuth(settings_file)
self.gauth.ServiceAuth()
self.drive = GoogleDrive(self.gauth)
def __init__(self):
"""Create an instance of UploadDrive."""
self.gauth = GoogleAuth()
self.drive = GoogleDrive(self.gauth)
def main():
print("\n--------Exporting {} Applications--------".format(year))
# Import Qualtrics Data for use here- this is only the data, no file uploads
print("Importing Data...")
regs = ImportRegistrationData()
apps = ImportApplicationData()
recs = ImportRecommendationData()
dems = ImportDemographicData()
# Join registration data with application data
print("Combining data...")
for app in apps:
for reg in regs:
if reg["Email"] == app["Email"]:
app.update(reg)
break
# Join recommendation data with application data
for app in apps:
# Save number of recommenders
recCount = 0
for rec in recs:
# Link recommenders with applications
for num in range(1, 5):
if app["Rec{}Email".format(num)] is not "":
if app["Email".format(num)] == rec["AppEmail"] and app[
"Rec{}Email".format(num)] == rec["Email"]:
app["Rec{}ID".format(num)] = rec["recID"]
recCount += 1
app["RecCount"] = recCount
# Join demographic info with applications
for dem in dems:
for app in apps:
if dem["AppID"] == app["AppID"]:
app.update(dem)
# Create and/or clean up workspace for files
print("Creating folder for applications...")
appFolder = "../{}_Applications".format(year)
if not os.path.exists(appFolder):
# Create workspace (e.g. folder to hold applications)
os.makedirs(appFolder)
else:
# Clean up workspace (e.g. delete all files in folder)
for file in os.listdir(appFolder):
file_path = os.path.join(appFolder, file)
if os.path.isfile(file_path):
os.remove(file_path)
elif os.path.isdir(file_path):
shutil.rmtree(file_path)
# Make section template PDFs
templates = [
"Cover Page", "Statement of Interest", "CV or Resume",
"(Unofficial) Transcript", "Recommendation Letter #1",
"Recommendation Letter #2", "Recommendation Letter #3",
"Recommendation Letter #4"
]
for template in templates:
MakeSectionPdf(template)
# Make application PDFs
print("\n--------Making PDFs--------")
appCount = 1
for app in apps:
print("Starting Application {} of {}...".format(appCount, len(apps)))
#Create dictionary to hold PDF pages
docs = collections.OrderedDict()
# Make SOI first (basic info last, to check if all parts submitted)
MakeSoiPdf(app)
soi = GetPdf("{}_SOI.pdf".format(app["AppID"]))
docs["Statement of Interest"] = soi
# Get CV
cvExists = False
cv = GetPdf("../Summer_Course_{}_Application/Q12/{}*.pdf".format(
year, app["AppID"]))
if cv:
docs["CV or Resume"] = cv
cvExists = True
# Get transcript
transcriptExists = False
transcript = GetPdf(
"../Summer_Course_{}_Application/Q11/{}*.pdf".format(
year, app["AppID"]))
if transcript:
docs["(Unofficial) Transcript"] = transcript
transcriptExists = True
# Get recommendation letters and add it to WIP PDF
letterExists = [None]
for num in range(1, 5):
letterExists.append(False)
if "Rec{}ID".format(num) in app.keys():
letter = GetPdf("../Q1/{}*.pdf".format(
app["Rec{}ID".format(num)]))
if letter:
docs["Recommendation Letter #" + str(num)] = letter
letterExists[num] = True
# Dictionary of Existence
fileExists = {
"CV": cvExists,
"Transcript": transcriptExists,
"Letters": letterExists
}
# Make Cover Page
completed = MakeCoverPage(app, fileExists)
# Get Cover Page
cover = GetPdf("{}_cover.pdf".format(app["AppID"]))
# Add pages to PDF (with header and watermark, if appropriate)
appPdf = PdfFileWriter()
pages = AddHeader(cover.pages, app)
pages = AddSection(pages, "Cover Page")
if not completed:
pages = AddWatermark(pages)
for page in pages:
appPdf.addPage(page)
for section, doc in docs.items():
pages = AddHeader(doc.pages, app)
pages = AddSection(pages, section)
if not completed:
pages = AddWatermark(pages)
for page in pages:
appPdf.addPage(page)
# Write PDF
appStream = open(
"../{}_Applications/{}_{}.pdf".format(year, app["Last"],
app["AppID"]), "wb")
appPdf.write(appStream)
# Increase count for display
appCount += 1
print("\n--------Post-Processing PDFs--------")
# Delete temporary files
print("Deleting Temporary Files...")
filesToDelete = ["SOI", "cover", "WIP", "Header"]
for ext in filesToDelete:
for file in glob("*_{}.pdf".format(ext)):
os.remove(file)
os.remove("Cover Page.pdf")
os.remove("Statement of Interest.pdf")
os.remove("CV or Resume.pdf")
os.remove("(Unofficial) Transcript.pdf")
os.remove("Recommendation Letter #1.pdf")
os.remove("Recommendation Letter #2.pdf")
os.remove("Recommendation Letter #3.pdf")
os.remove("Recommendation Letter #4.pdf")
# Create applicant CSV file
print("Creating Applicant CSV File...")
with open("../{}_Applications/{} Applicants.csv".format(year, year),
"w") as appCsv:
csvHeader = [
"AppID", "First", "Last", "Email", "Gender", "Hispanic", "Race",
"Education"
]
writer = csv.DictWriter(appCsv,
fieldnames=csvHeader,
restval="ERROR",
extrasaction="ignore")
writer.writeheader()
for app in apps:
writer.writerow(app)
print("\n--------Uploading files to Google Drive--------")
# Authenticate Google Drive
gauth = GoogleAuth()
# Create local webserver and auto-handle authentication.
gauth.LocalWebserverAuth()
# Create GoogleDrive instance with authenticated GoogleAuth instance.
drive = GoogleDrive(gauth)
# Delete all old application files
file_list = drive.ListFile({
'q':
"'{}' in parents and trashed=false".format(GDriveDestID)
}).GetList()
for file in file_list:
if ("R_" in file["title"]
and ".pdf" in file["title"]) or ".csv" in file["title"]:
file.Delete()
# Upload files to Google Drive
appCount = 1
print("\n\n--------Starting Drive Upload--------")
for app in apps:
print("Uploading {} of {}...".format(appCount, len(apps)))
file = drive.CreateFile({
"parents": [{
"kind": "drive#fileLink",
"id": "{}".format(GDriveDestID)
}],
"title":
"{}: {}.pdf".format(app["Last"], app["AppID"])
})
# Read file and set it as a content of this instance.
file.SetContentFile("../{}_Applications/{}_{}.pdf".format(
year, app["Last"], app["AppID"]))
file.Upload() # Upload the file.
appCount += 1
file = drive.CreateFile({
"parents": [{
"kind": "drive#fileLink",
"id": "{}".format(GDriveDestID)
}],
"title":
"{} Applicants.csv".format(year)
})
file.SetContentFile("../{}_Applications/{} Applicants.csv".format(
year, year))
file.Upload()
print("\n--------Distributing Applications--------")
print("Getting application links...")
app_list = drive.ListFile({
'q':
"'{}' in parents and trashed=false".format(GDriveDestID)
}).GetList()
appPdfs = []
for file in app_list:
if "R_" in file["title"] and ".pdf" in file["title"]:
appPdfs.append(file)
print("Apportioning applications...")
# Divide up applications
numApps = len(appPdfs)
numReviewers = len(appReviewers)
reviewBurden = (numApps * 2) // numReviewers
# Generate list to sample from
appsToReview = appPdfs + appPdfs
random.shuffle(appsToReview)
# Make list of reviewers
reviewers = []
for reviewer, email in appReviewers.items():
reviewers.append({"Name": reviewer, "Email": email, "Apps": []})
# Distribute applications
while appsToReview:
for reviewer in reviewers:
if appsToReview:
selection = random.choice(appsToReview)
if selection in reviewer["Apps"]:
selection = random.choice(appsToReview)
else:
reviewer["Apps"].append(selection)
appsToReview.remove(selection)
print("Emailing applications...")
#Create string of application links
for reviewer in reviewers:
reviewer["AppLinks"] = ""
reviewer["HtmlLinks"] = ""
for app in reviewer["Apps"]:
reviewer["AppLinks"] += "{}: {}\n\t\t\t".format(
app["title"].replace(".pdf", ""), app["webContentLink"])
reviewer["HtmlLinks"] += '<p><a href="{}">{}</a></p>'.format(
app["webContentLink"], app["title"].replace(".pdf", ""))
# Create the base text message.
for reviewer in reviewers:
msg = EmailMessage()
msg['Subject'] = "{} NCAN Summer Course Application Evaluations".format(
year)
msg['From'] = Address("William Schmitt",
addr_spec="*****@*****.**")
msg['To'] = Address(reviewer["Name"], addr_spec=reviewer["Email"])
msg.set_content("""\
Dear {},
The application window for the {} NCAN Summer Course is now closed!
As such, it is now time for you to begin reviewing applictions to
determine who should be admitted to the Course. There were {}
applications this year, so we need you to review {} applications. In
an attempt to streamline this process, we have created an evaluation
form that you can quickly and easily fill out for each application.
This form is located here (https://goo.gl/forms/AHxAvtZDglX54DWd2).
NOTE: this form and all links below require the use of your
@neurotechcenter.org account. Please make sure you are logged into
your account (if you are not, you will be prompted to do so when
you click on the link).
The applications you have been assigned are:
(listed as lastName: Applicant ID)
{}
The links above should automatically download each application to
your computer for easy viewing, but in case they do not work, you
should be able to access all applications here
(https://drive.google.com/drive/folders/0B67b4FFl6pYlVnY2cVpFbjlGdmM?usp=sharing).
Thank you for the anticipated time and attention you will spend reviewing
these applications. If you have any questions about the process, please
feel free to contact Billy or Dr. Carp.
Thank you,
The NCAN Summer Course Bot""".format(reviewer["Name"], year,
numApps, reviewBurden,
reviewer["AppLinks"]))
# Add the html version. This converts the message into a multipart/alternative
# container, with the original text message as the first part and the new html
# message as the second part.
msg.add_alternative("""\
<html>
<head></head>
<body>
<p>Dear {},</p>
<p>
The application window for the {} NCAN Summer Course is now
closed! As such, it is now time for you to begin reviewing applictions
to determine who should be admitted to the Course. There were {}
applications this year, so we need you to review {} applications.
In an attempt to streamline this process, we have created an
evaluation form that you can quickly and easily fill out for each
application. This form is located
<a href="https://goo.gl/forms/PftRKWtL6SnG1Ozp1">here</a>.
NOTE: this form and all links below require the use of your
@neurotechcenter.org account. Please make sure you are logged
into your account (if you are not, you will be prompted to do so
when you click on the link).
</p>
<p>The applications you have been assigned are:</p>
(listed as lastName: Applicant ID)
{}
<p>
The links above should automatically download each application to
your computer for easy viewing, but in case they do not work, you
should be able to access all applications
<a href="https://drive.google.com/drive/folders/0B67b4FFl6pYlVnY2cVpFbjlGdmM?usp=sharing">here</a>.
</p>
<p>
Thank you for the anticipated time and attention you will spend
reviewing these applications. If you have any questions about the
process, please feel free to contact Billy or Dr. Carp.</p>
<p>Thank you,</p>
<p>The NCAN Summer Course Bot</p>
</body>
</html>
""".format(reviewer["Name"], year, numApps, reviewBurden,
reviewer["HtmlLinks"]),
subtype='html')
# Send the message via local SMTP server.
with smtplib.SMTP('smtp.gmail.com', 587) as server: #port 465 or 587
server.ehlo()
server.starttls()
server.ehlo()
server.login('*****@*****.**', password)
server.send_message(msg)
server.close()
print("\n--------Success! All Done.--------")
Requires correct client_secrets, credentials, and settings files.
"""
gauth = GoogleAuth(os.getcwd() + "/settings.yaml")
gauth.LoadCredentialsFile("mycreds.txt")
if gauth.credentials is None:
# Authenticate if they're not there
gauth.LocalWebserverAuth()
elif gauth.access_token_expired:
# Refresh them if expired
gauth.Refresh()
else:
# Initialize the saved creds
gauth.Authorize()
# Save the current credentials to a file
gauth.SaveCredentialsFile("mycreds.txt")
drive = GoogleDrive(gauth)
def remove_file(filename):
"""
Determines if a file exists before trying to delete it
:param filename: str, name of file to delete
:return: boolean if the filename doesn't exist
"""
if os.path.exists(filename):
os.remove(filename)
else:
return False
return
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
try:
global pydrive
import httplib2
from apiclient.discovery import build
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from pydrive.files import ApiRequestError, FileNotUploadedError
except ImportError as e:
raise BackendException(u"""\
PyDrive backend requires PyDrive installation. Please read the manpage for setup details.
Exception: %s""" % str(e))
# let user get by with old client while he can
try:
from oauth2client.client import SignedJwtAssertionCredentials
self.oldClient = True
except:
from oauth2client.service_account import ServiceAccountCredentials
from oauth2client import crypt
self.oldClient = False
if u'GOOGLE_DRIVE_ACCOUNT_KEY' in os.environ:
account_key = os.environ[u'GOOGLE_DRIVE_ACCOUNT_KEY']
if self.oldClient:
credentials = SignedJwtAssertionCredentials(
parsed_url.username + u'@' + parsed_url.hostname,
account_key,
scopes=u'https://www.googleapis.com/auth/drive')
else:
signer = crypt.Signer.from_string(account_key)
credentials = ServiceAccountCredentials(
parsed_url.username + u'@' + parsed_url.hostname,
signer,
scopes=u'https://www.googleapis.com/auth/drive')
credentials.authorize(httplib2.Http())
gauth = GoogleAuth()
gauth.credentials = credentials
elif u'GOOGLE_DRIVE_SETTINGS' in os.environ:
gauth = GoogleAuth(
settings_file=os.environ[u'GOOGLE_DRIVE_SETTINGS'])
gauth.CommandLineAuth()
elif (u'GOOGLE_SECRETS_FILE' in os.environ
and u'GOOGLE_CREDENTIALS_FILE' in os.environ):
gauth = GoogleAuth()
gauth.LoadClientConfigFile(os.environ[u'GOOGLE_SECRETS_FILE'])
gauth.LoadCredentialsFile(os.environ[u'GOOGLE_CREDENTIALS_FILE'])
if gauth.credentials is None:
gauth.CommandLineAuth()
elif gauth.access_token_expired:
gauth.Refresh()
else:
gauth.Authorize()
gauth.SaveCredentialsFile(os.environ[u'GOOGLE_CREDENTIALS_FILE'])
else:
raise BackendException(
u'GOOGLE_DRIVE_ACCOUNT_KEY or GOOGLE_DRIVE_SETTINGS environment '
u'variable not set. Please read the manpage to fix.')
self.drive = GoogleDrive(gauth)
# Dirty way to find root folder id
file_list = self.drive.ListFile({
u'q':
u"'Root' in parents and trashed=false"
}).GetList()
if file_list:
parent_folder_id = file_list[0][u'parents'][0][u'id']
else:
file_in_root = self.drive.CreateFile({u'title': u'i_am_in_root'})
file_in_root.Upload()
parent_folder_id = file_in_root[u'parents'][0][u'id']
# Fetch destination folder entry and create hierarchy if required.
folder_names = string.split(parsed_url.path, u'/')
for folder_name in folder_names:
if not folder_name:
continue
file_list = self.drive.ListFile({
u'q':
u"'" + parent_folder_id + u"' in parents and trashed=false"
}).GetList()
folder = next(
(item
for item in file_list if item[u'title'] == folder_name and
item[u'mimeType'] == u'application/vnd.google-apps.folder'),
None)
if folder is None:
folder = self.drive.CreateFile({
u'title':
folder_name,
u'mimeType':
u"application/vnd.google-apps.folder",
u'parents': [{
u'id': parent_folder_id
}]
})
folder.Upload()
parent_folder_id = folder[u'id']
self.folder = parent_folder_id
self.id_cache = {}
