def save_attachment(uid, yaml_filename, filename, email_id, index, content_type, extension, content): att_file_number = get_new_file_number(uid, filename, yaml_file_name=yaml_filename) attachment_record = EmailAttachment(email_id=email_id, index=0, content_type=content_type, extension=extension, upload=att_file_number) db.session.add(attachment_record) db.session.commit() saved_file_attachment = SavedFile(att_file_number, extension=extension) saved_file_attachment.write_content(content) saved_file_attachment.finalize()
def save_attachment(uid, yaml_filename, filename, email_id, index, content_type, extension, content): att_file_number = get_new_file_number(uid, filename, yaml_file_name=yaml_filename) attachment_record = EmailAttachment(email_id=email_id, index=0, content_type=content_type, extension=extension, upload=att_file_number) db.session.add(attachment_record) db.session.commit() saved_file_attachment = SavedFile(att_file_number, extension=extension) saved_file_attachment.write_content(content) saved_file_attachment.finalize()
def fix_ml_files(playground_number): playground = SavedFile(playground_number, section='playgroundsources', fix=False) changed = False for filename in playground.list_of_files(): if re.match(r'^ml-.*\.json', filename): playground.fix() try: if write_ml_source(playground, playground_number, filename, finalize=False): changed = True except: logmessage("Error writing machine learning source file " + str(filename)) if changed: playground.finalize()
def fix_ml_files(playground_number): playground = SavedFile(playground_number, section='playgroundsources', fix=False) changed = False for filename in playground.list_of_files(): if re.match(r'^ml-.*\.json', filename): playground.fix() try: if write_ml_source(playground, playground_number, filename, finalize=False): changed = True except: logmessage("Error writing machine learning source file " + str(filename)) if changed: playground.finalize()
def sync_with_google_drive(user_id): sys.stderr.write("sync_with_google_drive: starting\n") if worker_controller is None: initialize_db() sys.stderr.write("sync_with_google_drive: continuing\n") storage = RedisCredStorage(worker_controller.r, user_id, app='googledrive') credentials = storage.get() if not credentials or credentials.invalid: sys.stderr.write("sync_with_google_drive: credentials failed\n") return worker_controller.functions.ReturnValue( ok=False, error="credentials expired", restart=False) try: with worker_controller.flaskapp.app_context(): http = credentials.authorize(httplib2.Http()) service = worker_controller.apiclient.discovery.build('drive', 'v3', http=http) key = 'da:googledrive:mapping:userid:' + str(user_id) the_folder = worker_controller.r.get(key) response = service.files().get( fileId=the_folder, fields="mimeType, id, name, trashed").execute() the_mime_type = response.get('mimeType', None) trashed = response.get('trashed', False) if trashed is True or the_mime_type != "application/vnd.google-apps.folder": return worker_controller.functions.ReturnValue( ok=False, error="error accessing Google Drive", restart=False) local_files = dict() local_modtimes = dict() gd_files = dict() gd_ids = dict() gd_modtimes = dict() gd_deleted = dict() sections_modified = set() commentary = '' for section in [ 'static', 'templates', 'questions', 'modules', 'sources' ]: local_files[section] = set() local_modtimes[section] = dict() if section == 'questions': the_section = 'playground' elif section == 'templates': the_section = 'playgroundtemplate' else: the_section = 'playground' + section area = SavedFile(user_id, fix=True, section=the_section) for f in os.listdir(area.directory): local_files[section].add(f) local_modtimes[section][f] = os.path.getmtime( os.path.join(area.directory, f)) subdirs = list() page_token = None while True: response = service.files().list( spaces="drive", fields="nextPageToken, files(id, name)", q="mimeType='application/vnd.google-apps.folder' and trashed=false and name='" + section + "' and '" + str(the_folder) + "' in parents").execute() for the_file in response.get('files', []): if 'id' in the_file: subdirs.append(the_file['id']) page_token = response.get('nextPageToken', None) if page_token is None: break if len(subdirs) == 0: return worker_controller.functions.ReturnValue( ok=False, error="error accessing " + section + " in Google Drive", restart=False) subdir = subdirs[0] gd_files[section] = set() gd_ids[section] = dict() gd_modtimes[section] = dict() gd_deleted[section] = set() page_token = None while True: response = service.files().list( spaces="drive", fields= "nextPageToken, files(id, name, modifiedTime, trashed)", q="mimeType!='application/vnd.google-apps.folder' and '" + str(subdir) + "' in parents").execute() for the_file in response.get('files', []): if re.search(r'(\.tmp|\.gdoc)$', the_file['name']): continue if re.search(r'^\~', the_file['name']): continue gd_ids[section][the_file['name']] = the_file['id'] gd_modtimes[section][the_file[ 'name']] = strict_rfc3339.rfc3339_to_timestamp( the_file['modifiedTime']) sys.stderr.write("Google says modtime on " + unicode(the_file) + " is " + the_file['modifiedTime'] + "\n") if the_file['trashed']: gd_deleted[section].add(the_file['name']) continue gd_files[section].add(the_file['name']) page_token = response.get('nextPageToken', None) if page_token is None: break gd_deleted[section] = gd_deleted[section] - gd_files[section] for f in gd_files[section]: sys.stderr.write("Considering " + f + " on GD\n") if f not in local_files[section] or gd_modtimes[section][ f] - local_modtimes[section][f] > 3: sys.stderr.write("Considering " + f + " to copy to local\n") sections_modified.add(section) commentary += "Copied " + f + " from Google Drive.\n" the_path = os.path.join(area.directory, f) with open(the_path, 'wb') as fh: response = service.files().get_media( fileId=gd_ids[section][f]) downloader = worker_controller.apiclient.http.MediaIoBaseDownload( fh, response) done = False while done is False: status, done = downloader.next_chunk() #sys.stderr.write("Download %d%%." % int(status.progress() * 100) + "\n") os.utime( the_path, (gd_modtimes[section][f], gd_modtimes[section][f])) for f in local_files[section]: sys.stderr.write("Considering " + f + ", which is a local file\n") if f not in gd_deleted[section]: sys.stderr.write("Considering " + f + " is not in Google Drive deleted\n") if f not in gd_files[section]: sys.stderr.write("Considering " + f + " is not in Google Drive\n") the_path = os.path.join(area.directory, f) if os.path.getsize(the_path) == 0: sys.stderr.write("Found zero byte file: " + the_path + "\n") continue sys.stderr.write("Copying " + f + " to Google Drive.\n") commentary += "Copied " + f + " to Google Drive.\n" extension, mimetype = worker_controller.get_ext_and_mimetype( the_path) the_modtime = strict_rfc3339.timestamp_to_rfc3339_utcoffset( local_modtimes[section][f]) sys.stderr.write( "Setting GD modtime on new file " + unicode(f) + " to " + unicode(the_modtime) + "\n") file_metadata = { 'name': f, 'parents': [subdir], 'modifiedTime': the_modtime, 'createdTime': the_modtime } media = worker_controller.apiclient.http.MediaFileUpload( the_path, mimetype=mimetype) the_new_file = service.files().create( body=file_metadata, media_body=media, fields='id').execute() new_id = the_new_file.get('id') elif local_modtimes[section][f] - gd_modtimes[section][ f] > 3: sys.stderr.write( "Considering " + f + " is in Google Drive but local is more recent\n" ) the_path = os.path.join(area.directory, f) if os.path.getsize(the_path) == 0: sys.stderr.write( "Found zero byte file during update: " + the_path + "\n") continue commentary += "Updated " + f + " on Google Drive.\n" extension, mimetype = worker_controller.get_ext_and_mimetype( the_path) the_modtime = strict_rfc3339.timestamp_to_rfc3339_utcoffset( local_modtimes[section][f]) sys.stderr.write( "Setting GD modtime on modified " + unicode(f) + " to " + unicode(the_modtime) + "\n") file_metadata = {'modifiedTime': the_modtime} media = worker_controller.apiclient.http.MediaFileUpload( the_path, mimetype=mimetype) service.files().update(fileId=gd_ids[section][f], body=file_metadata, media_body=media).execute() for f in gd_deleted[section]: sys.stderr.write("Considering " + f + " is deleted on Google Drive\n") if f in local_files[section]: sys.stderr.write( "Considering " + f + " is deleted on Google Drive but exists locally\n") if local_modtimes[section][f] - gd_modtimes[section][ f] > 3: sys.stderr.write( "Considering " + f + " is deleted on Google Drive but exists locally and needs to be undeleted on GD\n" ) commentary += "Undeleted and updated " + f + " on Google Drive.\n" the_path = os.path.join(area.directory, f) extension, mimetype = worker_controller.get_ext_and_mimetype( the_path) the_modtime = strict_rfc3339.timestamp_to_rfc3339_utcoffset( local_modtimes[section][f]) sys.stderr.write( "Setting GD modtime on undeleted file " + unicode(f) + " to " + unicode(the_modtime) + "\n") file_metadata = { 'modifiedTime': the_modtime, 'trashed': False } media = worker_controller.apiclient.http.MediaFileUpload( the_path, mimetype=mimetype) service.files().update(fileId=gd_ids[section][f], body=file_metadata, media_body=media).execute() else: sys.stderr.write( "Considering " + f + " is deleted on Google Drive but exists locally and needs to deleted locally\n" ) sections_modified.add(section) commentary += "Deleted " + f + " from Playground.\n" the_path = os.path.join(area.directory, f) if os.path.isfile(the_path): area.delete_file(f) area.finalize() for key in worker_controller.r.keys( 'da:interviewsource:docassemble.playground' + str(user_id) + ':*'): worker_controller.r.incr(key) if commentary != '': sys.stderr.write(commentary + "\n") if 'modules' in sections_modified: do_restart = True else: do_restart = False return worker_controller.functions.ReturnValue(ok=True, summary=commentary, restart=do_restart) except Exception as e: return worker_controller.functions.ReturnValue( ok=False, error="Error syncing with Google Drive: " + str(e), restart=False)
class PlaygroundSection(object): def __init__(self, section=''): if docassemble.base.functions.this_thread.current_info['user'][ 'is_anonymous']: raise DAError( "Users must be logged in to create Playground objects") self.user_id = docassemble.base.functions.this_thread.current_info[ 'user']['theid'] self.current_info = docassemble.base.functions.this_thread.current_info self.section = section self.area = SavedFile(self.user_id, fix=True, section='playground' + self.section) self._update_file_list() def _update_file_list(self): self.file_list = sorted([ f for f in os.listdir(self.area.directory) if os.path.isfile(os.path.join(self.area.directory, f)) ]) def image_file_list(self): out_list = list() for the_file in self.file_list: extension, mimetype = get_ext_and_mimetype(the_file) if re.search(r'^image', mimetype): out_list.append(the_file) return out_list def reduced_file_list(self): lower_list = [f.lower() for f in self.file_list] out_list = [ f for f in self.file_list if os.path.splitext(f)[1].lower() == '.md' or os.path.splitext(f)[0].lower() + '.md' not in lower_list ] return out_list def get_file(self, filename): return os.path.join(self.area.directory, filename) def file_exists(self, filename): path = self.get_file(filename) if os.path.isfile(path): return True return False def read_file(self, filename): path = self.get_file(filename) if path is None: return None with open(path, 'rU') as fp: content = fp.read().decode('utf8') return content return None def write_file(self, filename, content): path = os.path.join(self.area.directory, filename) with open(path, 'w') as fp: fp.write(content.encode('utf8')) self.area.finalize() def commit(self): self.area.finalize() def copy_from(self, from_file, filename=None): if filename is None: filename = os.path.basename(from_file) to_path = self.get_file(filename) shutil.copyfile(from_file, to_path) self.area.finalize() return filename def is_markdown(self, filename): extension, mimetype = get_ext_and_mimetype(filename) if extension == "md": return True return False def is_pdf(self, filename): extension, mimetype = get_ext_and_mimetype(filename) if extension == "pdf": return True return False def get_fields(self, filename): return docassemble.base.pdftk.read_fields(self.get_file(filename)) def convert_file_to_md(self, filename, convert_variables=True): extension, mimetype = get_ext_and_mimetype(filename) if (mimetype and mimetype in convertible_mimetypes): the_format = convertible_mimetypes[mimetype] elif extension and extension in convertible_extensions: the_format = convertible_extensions[extension] else: return None if not self.file_exists(filename): return None path = self.get_file(filename) temp_file = word_to_markdown(path, the_format) if temp_file is None: return None out_filename = os.path.splitext(filename)[0] + '.md' if convert_variables: with open(temp_file.name, 'rU') as fp: self.write_file( out_filename, replace_square_brackets.sub(fix_variable_name, fp.read().decode('utf8'))) else: shutil.copyfile(temp_file.name, self.get_file(out_filename)) return out_filename def variables_from_file(self, filename): content = self.read_file(filename) if content is None: return None return Playground().variables_from(content)