class DriveMainThread(Thread):
def __init__(self, folders_to_track, start_sync_at, pyfile, **kwargs):
super(DriveMainThread, self).__init__(**kwargs)
self.drive_data = DBInterface(
os.path.dirname(os.path.abspath(pyfile)) + '/drive_data/',
'drive_files')
base_path = os.path.dirname(os.path.abspath(pyfile)) + '/'
self.updated_folders = set()
self.start_sync_at = start_sync_at
self.folders_to_track = folders_to_track
self.setup_drive()
self.update_queue = Queue.Queue()
self.get_tracked_folder_ids(
folders_to_track,
os.path.dirname(os.path.abspath(pyfile)) + '/')
self.tracked_widgets = {}
self.folders_to_track = [
base_path + path + '/' for path in folders_to_track
]
def add_drive_widget(self, widget, folder_to_track):
assert (folder_to_track in self.folders_to_track)
tracked_widgets = self.tracked_widgets
if folder_to_track not in tracked_widgets:
tracked_widgets[folder_to_track] = [widget]
else:
tracked_widgets[folder_to_track].append(widget)
def setup_drive(self):
credentials = storage.get()
self.queue = queue = Queue.Queue()
if credentials is None:
flow = OAuth2WebServerFlow(CLIENT_ID,
CLIENT_SECRET,
OAUTH_SCOPE,
redirect_uri=REDIRECT_URI)
authorize_url = flow.step1_get_authorize_url()
print 'Go to the following link in your browser: ' + authorize_url
code = raw_input('Enter verification code: ').strip()
credentials = flow.step2_exchange(code)
storage.put(credentials)
http = httplib2.Http()
http = credentials.authorize(http)
self.drive_service = drive_service = build('drive', 'v2', http=http)
self.thread_pool = thread_pool = DriveThreadPool(1, queue, credentials)
def get_tracked_folder_ids(self, folders_to_track, address):
drive_data = self.drive_data
for folder_name in self.folders_to_track:
q = "title = '{}'".format(folder_name)
data = self.drive_service.children().list(folderId='root',
q=q).execute()
folder_add = address + folder_name + '/'
self.ensure_dir(folder_add)
#self.add_folder_observer(self.get_callback, unicode(folder_add))
for dat in data[u'items']:
item_id = dat[u'id']
drive_data.set_entry('folders', item_id, 'file_add',
folder_add)
drive_data.set_entry('tracked_items', item_id, 'file_add',
folder_add)
def check_file_in_tracked_folders(self, file_data):
parents = file_data[u'parents']
folders = self.drive_data.get_table('folders')
for parent in parents:
parent_id = parent[u'id']
if parent_id in folders:
return True
return False
def run(self):
while True:
query_download = self.update_queue.get()
if query_download:
self.retrieve_all_changes()
self.update_queue.task_done()
def download_folder(self, file_id, file_data):
parents = file_data[u'parents']
drive_data = self.drive_data
#save_data = self.save_data
folders = drive_data.get_table('folders')
get_entry = drive_data.get_entry
append_entry = drive_data.append_entry
for parent in parents:
parent_id = parent[u'id']
if parent_id in folders:
address = get_entry('folders', parent_id, 'file_add')
folder_name = file_data[u'title']
folder_add = address + folder_name + '/'
self.ensure_dir(folder_add)
append_entry('folders', file_id, 'file_add', folder_add)
append_entry('tracked_items', file_id, 'file_add', folder_add)
def place_address_in_queue(self, address, file_name, file_id,
time_since_epoch, download_url):
drive_data = self.drive_data
self.ensure_dir(address)
file_add = address + file_name
drive_data.append_entry('tracked_items', file_id, 'file_add', file_add)
do_download = False
try:
mod_time = os.stat(file_add).st_mtime
if time_since_epoch > mod_time:
do_download = True
else:
do_download = False
except:
do_download = True
if do_download:
self.queue.put(
(self._download, [download_url, file_add,
time_since_epoch], {}))
def download_file(self, file_id, file_data):
mod_date = file_data[u'modifiedDate']
parents = file_data[u'parents']
time_since_epoch = self.calculate_time_since_epoch(mod_date)
drive_data = self.drive_data
folders = drive_data.get_table('folders')
get_entry = drive_data.get_entry
download_url = file_data[u'downloadUrl']
file_name = file_data[u'title']
place_address_in_queue = self.place_address_in_queue
for parent in parents:
parent_id = parent[u'id']
if parent_id in folders:
address = get_entry('folders', parent_id, 'file_add')
if isinstance(address, list):
for add in address:
place_address_in_queue(add, file_name, file_id,
time_since_epoch, download_url)
else:
place_address_in_queue(address, file_name, file_id,
time_since_epoch, download_url)
def retrieve_all_changes(self):
result = []
page_token = None
drive_data = self.drive_data
largest_change = int(self.get_largest_change_id())
try:
start_change_id = str(
int(drive_data.get_entry('changes', 'last_change', 'id')) + 1)
except:
start_change_id = str(int(self.start_sync_at))
# if self.progress_tracker.start_change_id is None:
# self.progress_tracker.start_change_id = int(start_change_id)
# self.progress_tracker.last_change_id = int(start_change_id)
if largest_change >= int(start_change_id):
print('we should parse')
# Clock.schedule_once(partial(self.retrieve_page_of_changes, None,
# start_change_id))
while True:
param = {}
print('grabbing changes')
if start_change_id is not None:
param['startChangeId'] = start_change_id
if page_token is not None:
param['pageToken'] = page_token
changes = self.drive_service.changes().list(**param).execute()
self.parse_changes(changes)
#self.sync()
page_token = changes.get('nextPageToken')
if not page_token:
print('done breaking')
break
def parse_changes(self, changes):
drive_data = self.drive_data
download_folder = self.download_folder
download_file = self.download_file
remove_file = self.remove_file
tracked_items = drive_data.get_table('tracked_items')
check_file_in_tracked_folders = self.check_file_in_tracked_folders
get_is_folder = self.get_is_folder
set_entry = drive_data.set_entry
get_file = self.get_file
if len(changes['items']) == 0:
return None
for change in changes['items']:
file_id = change[u'fileId']
change_id = change[u'id']
file_data = get_file(file_id)
is_deleted = change[u'deleted']
if is_deleted and file_id in tracked_items:
self.handle_remove_callback(file_id)
remove_file(file_id)
if file_data != None:
if check_file_in_tracked_folders(file_data):
if get_is_folder(file_data):
download_folder(file_id, file_data)
else:
download_file(file_id, file_data)
set_entry('changes', 'last_change', 'id', change_id)
return change_id
def handle_remove_callback(self, file_id):
file_adds = self.drive_data.get_entry('tracked_items', file_id,
'file_add')
tracked_widgets = self.tracked_widgets
if file_adds is not None:
if isinstance(file_adds, list):
for file_add in file_adds:
widgets_for_removal = []
widgets_a = widgets_for_removal.append
dir_name = str(dirname(file_add) + '/')
for wid in tracked_widgets[dir_name]:
r_wid = wid()
if r_wid is not None:
file_callback = r_wid.file_callback
Clock.schedule_once(
partial(file_callback, file_add, True))
else:
widgets_a(wid)
for wid_ref in widgets_for_removal:
tracked_widgets[file_adds].remove(wid_ref)
def remove_file(self, file_id):
drive_data = self.drive_data
to_remove = []
to_remove_a = to_remove.append
remove_entry = drive_data.remove_entry
file_adds = drive_data.get_entry('tracked_items', file_id, 'file_add')
if file_adds is not None:
for file_add in file_adds:
try:
os.remove(file_add)
to_remove_a(file_add)
except:
continue
for add in to_remove:
remove_entry('tracked_items', file_id, 'file_add', add)
def ensure_dir(self, f):
d = os.path.dirname(f)
if not os.path.exists(d):
os.makedirs(d)
def calculate_time_since_epoch(self, drive_time):
date, current_time = drive_time.split('T')
year, month, day = date.split('-')
hours, minutes, seconds = current_time.split(':')
seconds = seconds.split('.')[0]
py_date = datetime.datetime(int(year),
int(month),
int(day),
hour=int(hours),
minute=int(minutes),
second=int(seconds))
return time.mktime(py_date.timetuple())
def get_is_folder(self, file_data):
return file_data[u'mimeType'] == u'application/vnd.google-apps.folder'
def get_file(self, fid):
try:
return self.drive_service.files().get(fileId=fid).execute()
except:
return None
def list_folder_content(self, fid):
return self.drive_service.children().list(folderId=fid).execute()
def get_about_data(self):
return self.drive_service.about().get().execute()
def get_largest_change_id(self):
return self.get_about_data()[u'largestChangeId']
def get_content_title(self, fid):
return self.drive_service.files().get(fileId=fid).execute()[u'title']
def _download(self, queue, drive_service, download_url, name, time):
if download_url:
try:
resp, content = drive_service._http.request(download_url)
except:
print 'An error occurred: '
return
if resp.status == 200:
with open(name, 'w') as save_file:
save_file.write(content)
os.utime(name, (os.stat(name).st_atime, time))
folder_name = os.path.dirname(name) + '/'
tracked_widgets = self.tracked_widgets
widgets_for_removal = []
widget_a = widgets_for_removal.append
if folder_name in tracked_widgets:
for wid in tracked_widgets[folder_name]:
r_wid = wid()
if r_wid is not None:
Clock.schedule_once(
partial(r_wid.file_callback, name, False), 15.)
else:
widget_a(wid)
for wid_ref in widgets_for_removal:
tracked_widgets[folder_name].remove(wid_ref)
queue.task_done()
else:
print 'An error occurred: %s' % resp
class DriveMainThread(Thread):
def __init__(self, folders_to_track, start_sync_at, pyfile, **kwargs):
super(DriveMainThread, self).__init__(**kwargs)
self.drive_data = DBInterface(os.path.dirname(
os.path.abspath(pyfile))+'/drive_data/', 'drive_files')
base_path = os.path.dirname(os.path.abspath(pyfile))+'/'
self.updated_folders = set()
self.start_sync_at = start_sync_at
self.folders_to_track = folders_to_track
self.setup_drive()
self.update_queue = Queue.Queue()
self.get_tracked_folder_ids(
folders_to_track, os.path.dirname(
os.path.abspath(pyfile))+'/')
self.tracked_widgets = {}
self.folders_to_track = [
base_path+path+'/' for path in folders_to_track]
def add_drive_widget(self, widget, folder_to_track):
assert(folder_to_track in self.folders_to_track)
tracked_widgets = self.tracked_widgets
if folder_to_track not in tracked_widgets:
tracked_widgets[folder_to_track] = [widget]
else:
tracked_widgets[folder_to_track].append(widget)
def setup_drive(self):
credentials = storage.get()
self.queue = queue = Queue.Queue()
if credentials is None:
flow = OAuth2WebServerFlow(CLIENT_ID, CLIENT_SECRET, OAUTH_SCOPE,
redirect_uri=REDIRECT_URI)
authorize_url = flow.step1_get_authorize_url()
print 'Go to the following link in your browser: ' + authorize_url
code = raw_input('Enter verification code: ').strip()
credentials = flow.step2_exchange(code)
storage.put(credentials)
http = httplib2.Http()
http = credentials.authorize(http)
self.drive_service = drive_service = build('drive', 'v2', http=http)
self.thread_pool = thread_pool = DriveThreadPool(1, queue,
credentials)
def get_tracked_folder_ids(self, folders_to_track, address):
drive_data = self.drive_data
for folder_name in self.folders_to_track:
q="title = '{}'".format(folder_name)
data = self.drive_service.children().list(
folderId='root', q=q).execute()
folder_add = address+folder_name+'/'
self.ensure_dir(folder_add)
#self.add_folder_observer(self.get_callback, unicode(folder_add))
for dat in data[u'items']:
item_id = dat[u'id']
drive_data.set_entry(
'folders', item_id, 'file_add', folder_add)
drive_data.set_entry(
'tracked_items', item_id, 'file_add', folder_add)
def check_file_in_tracked_folders(self, file_data):
parents = file_data[u'parents']
folders = self.drive_data.get_table('folders')
for parent in parents:
parent_id = parent[u'id']
if parent_id in folders:
return True
return False
def run(self):
while True:
query_download = self.update_queue.get()
if query_download:
self.retrieve_all_changes()
self.update_queue.task_done()
def download_folder(self, file_id, file_data):
parents = file_data[u'parents']
drive_data = self.drive_data
#save_data = self.save_data
folders = drive_data.get_table('folders')
get_entry = drive_data.get_entry
append_entry = drive_data.append_entry
for parent in parents:
parent_id = parent[u'id']
if parent_id in folders:
address = get_entry('folders', parent_id, 'file_add')
folder_name = file_data[u'title']
folder_add = address+folder_name+'/'
self.ensure_dir(folder_add)
append_entry('folders', file_id, 'file_add', folder_add)
append_entry('tracked_items', file_id, 'file_add', folder_add)
def place_address_in_queue(self, address, file_name, file_id,
time_since_epoch, download_url):
drive_data = self.drive_data
self.ensure_dir(address)
file_add = address+file_name
drive_data.append_entry('tracked_items', file_id, 'file_add',
file_add)
do_download = False
try:
mod_time = os.stat(file_add).st_mtime
if time_since_epoch > mod_time:
do_download = True
else:
do_download = False
except:
do_download = True
if do_download:
self.queue.put((self._download, [download_url, file_add,
time_since_epoch], {}))
def download_file(self, file_id, file_data):
mod_date = file_data[u'modifiedDate']
parents = file_data[u'parents']
time_since_epoch = self.calculate_time_since_epoch(mod_date)
drive_data = self.drive_data
folders = drive_data.get_table('folders')
get_entry = drive_data.get_entry
download_url = file_data[u'downloadUrl']
file_name = file_data[u'title']
place_address_in_queue = self.place_address_in_queue
for parent in parents:
parent_id = parent[u'id']
if parent_id in folders:
address = get_entry('folders', parent_id, 'file_add')
if isinstance(address, list):
for add in address:
place_address_in_queue(add, file_name, file_id,
time_since_epoch, download_url)
else:
place_address_in_queue(address, file_name, file_id,
time_since_epoch, download_url)
def retrieve_all_changes(self):
result = []
page_token = None
drive_data = self.drive_data
largest_change = int(self.get_largest_change_id())
try:
start_change_id = str(int(
drive_data.get_entry('changes', 'last_change', 'id')) + 1)
except:
start_change_id = str(int(self.start_sync_at))
# if self.progress_tracker.start_change_id is None:
# self.progress_tracker.start_change_id = int(start_change_id)
# self.progress_tracker.last_change_id = int(start_change_id)
if largest_change >= int(start_change_id):
print('we should parse')
# Clock.schedule_once(partial(self.retrieve_page_of_changes, None,
# start_change_id))
while True:
param = {}
print('grabbing changes')
if start_change_id is not None:
param['startChangeId'] = start_change_id
if page_token is not None:
param['pageToken'] = page_token
changes = self.drive_service.changes().list(**param).execute()
self.parse_changes(changes)
#self.sync()
page_token = changes.get('nextPageToken')
if not page_token:
print('done breaking')
break
def parse_changes(self, changes):
drive_data = self.drive_data
download_folder = self.download_folder
download_file = self.download_file
remove_file = self.remove_file
tracked_items = drive_data.get_table('tracked_items')
check_file_in_tracked_folders = self.check_file_in_tracked_folders
get_is_folder = self.get_is_folder
set_entry = drive_data.set_entry
get_file = self.get_file
if len(changes['items']) == 0:
return None
for change in changes['items']:
file_id = change[u'fileId']
change_id = change[u'id']
file_data = get_file(file_id)
is_deleted = change[u'deleted']
if is_deleted and file_id in tracked_items:
self.handle_remove_callback(file_id)
remove_file(file_id)
if file_data != None:
if check_file_in_tracked_folders(file_data):
if get_is_folder(file_data):
download_folder(file_id, file_data)
else:
download_file(file_id, file_data)
set_entry('changes', 'last_change', 'id', change_id)
return change_id
def handle_remove_callback(self, file_id):
file_adds = self.drive_data.get_entry(
'tracked_items', file_id, 'file_add')
tracked_widgets = self.tracked_widgets
if file_adds is not None:
if isinstance(file_adds, list):
for file_add in file_adds:
widgets_for_removal = []
widgets_a = widgets_for_removal.append
dir_name = str(dirname(file_add) + '/')
for wid in tracked_widgets[dir_name]:
r_wid = wid()
if r_wid is not None:
file_callback = r_wid.file_callback
Clock.schedule_once(partial(file_callback,
file_add, True))
else:
widgets_a(wid)
for wid_ref in widgets_for_removal:
tracked_widgets[file_adds].remove(wid_ref)
def remove_file(self, file_id):
drive_data = self.drive_data
to_remove = []
to_remove_a = to_remove.append
remove_entry = drive_data.remove_entry
file_adds = drive_data.get_entry('tracked_items', file_id, 'file_add')
if file_adds is not None:
for file_add in file_adds:
try:
os.remove(file_add)
to_remove_a(file_add)
except:
continue
for add in to_remove:
remove_entry('tracked_items', file_id, 'file_add', add)
def ensure_dir(self, f):
d = os.path.dirname(f)
if not os.path.exists(d):
os.makedirs(d)
def calculate_time_since_epoch(self, drive_time):
date, current_time = drive_time.split('T')
year, month, day = date.split('-')
hours, minutes, seconds = current_time.split(':')
seconds = seconds.split('.')[0]
py_date = datetime.datetime(int(year), int(month), int(day),
hour=int(hours), minute=int(minutes),
second=int(seconds))
return time.mktime(py_date.timetuple())
def get_is_folder(self, file_data):
return file_data[u'mimeType'] == u'application/vnd.google-apps.folder'
def get_file(self, fid):
try:
return self.drive_service.files().get(fileId=fid).execute()
except:
return None
def list_folder_content(self, fid):
return self.drive_service.children().list(folderId=fid).execute()
def get_about_data(self):
return self.drive_service.about().get().execute()
def get_largest_change_id(self):
return self.get_about_data()[u'largestChangeId']
def get_content_title(self, fid):
return self.drive_service.files().get(fileId=fid).execute()[u'title']
def _download(self, queue, drive_service, download_url, name, time):
if download_url:
try:
resp, content = drive_service._http.request(download_url)
except:
print 'An error occurred: '
return
if resp.status == 200:
with open(name, 'w') as save_file:
save_file.write(content)
os.utime(name, (os.stat(name).st_atime,
time))
folder_name = os.path.dirname(name) + '/'
tracked_widgets = self.tracked_widgets
widgets_for_removal = []
widget_a = widgets_for_removal.append
if folder_name in tracked_widgets:
for wid in tracked_widgets[folder_name]:
r_wid = wid()
if r_wid is not None:
Clock.schedule_once(partial(r_wid.file_callback,
name, False), 15.)
else:
widget_a(wid)
for wid_ref in widgets_for_removal:
tracked_widgets[folder_name].remove(wid_ref)
queue.task_done()
else:
print 'An error occurred: %s' % resp
