def save_attachment(uid, yaml_filename, filename, email_id, index, content_type, extension, content):
att_file_number = get_new_file_number(uid, filename, yaml_file_name=yaml_filename)
attachment_record = EmailAttachment(email_id=email_id, index=0, content_type=content_type, extension=extension, upload=att_file_number)
db.session.add(attachment_record)
db.session.commit()
saved_file_attachment = SavedFile(att_file_number, extension=extension)
saved_file_attachment.write_content(content)
saved_file_attachment.finalize()
def save_attachment(uid, yaml_filename, filename, email_id, index, content_type, extension, content):
att_file_number = get_new_file_number(uid, filename, yaml_file_name=yaml_filename)
attachment_record = EmailAttachment(email_id=email_id, index=0, content_type=content_type, extension=extension, upload=att_file_number)
db.session.add(attachment_record)
db.session.commit()
saved_file_attachment = SavedFile(att_file_number, extension=extension)
saved_file_attachment.write_content(content)
saved_file_attachment.finalize()
def fix_ml_files(playground_number):
playground = SavedFile(playground_number, section='playgroundsources', fix=False)
changed = False
for filename in playground.list_of_files():
if re.match(r'^ml-.*\.json', filename):
playground.fix()
try:
if write_ml_source(playground, playground_number, filename, finalize=False):
changed = True
except:
logmessage("Error writing machine learning source file " + str(filename))
if changed:
playground.finalize()
def fix_ml_files(playground_number):
playground = SavedFile(playground_number, section='playgroundsources', fix=False)
changed = False
for filename in playground.list_of_files():
if re.match(r'^ml-.*\.json', filename):
playground.fix()
try:
if write_ml_source(playground, playground_number, filename, finalize=False):
changed = True
except:
logmessage("Error writing machine learning source file " + str(filename))
if changed:
playground.finalize()
def sync_with_google_drive(user_id):
sys.stderr.write("sync_with_google_drive: starting\n")
if worker_controller is None:
initialize_db()
sys.stderr.write("sync_with_google_drive: continuing\n")
storage = RedisCredStorage(worker_controller.r, user_id, app='googledrive')
credentials = storage.get()
if not credentials or credentials.invalid:
sys.stderr.write("sync_with_google_drive: credentials failed\n")
return worker_controller.functions.ReturnValue(
ok=False, error="credentials expired", restart=False)
try:
with worker_controller.flaskapp.app_context():
http = credentials.authorize(httplib2.Http())
service = worker_controller.apiclient.discovery.build('drive',
'v3',
http=http)
key = 'da:googledrive:mapping:userid:' + str(user_id)
the_folder = worker_controller.r.get(key)
response = service.files().get(
fileId=the_folder,
fields="mimeType, id, name, trashed").execute()
the_mime_type = response.get('mimeType', None)
trashed = response.get('trashed', False)
if trashed is True or the_mime_type != "application/vnd.google-apps.folder":
return worker_controller.functions.ReturnValue(
ok=False,
error="error accessing Google Drive",
restart=False)
local_files = dict()
local_modtimes = dict()
gd_files = dict()
gd_ids = dict()
gd_modtimes = dict()
gd_deleted = dict()
sections_modified = set()
commentary = ''
for section in [
'static', 'templates', 'questions', 'modules', 'sources'
]:
local_files[section] = set()
local_modtimes[section] = dict()
if section == 'questions':
the_section = 'playground'
elif section == 'templates':
the_section = 'playgroundtemplate'
else:
the_section = 'playground' + section
area = SavedFile(user_id, fix=True, section=the_section)
for f in os.listdir(area.directory):
local_files[section].add(f)
local_modtimes[section][f] = os.path.getmtime(
os.path.join(area.directory, f))
subdirs = list()
page_token = None
while True:
response = service.files().list(
spaces="drive",
fields="nextPageToken, files(id, name)",
q="mimeType='application/vnd.google-apps.folder' and trashed=false and name='"
+ section + "' and '" + str(the_folder) +
"' in parents").execute()
for the_file in response.get('files', []):
if 'id' in the_file:
subdirs.append(the_file['id'])
page_token = response.get('nextPageToken', None)
if page_token is None:
break
if len(subdirs) == 0:
return worker_controller.functions.ReturnValue(
ok=False,
error="error accessing " + section +
" in Google Drive",
restart=False)
subdir = subdirs[0]
gd_files[section] = set()
gd_ids[section] = dict()
gd_modtimes[section] = dict()
gd_deleted[section] = set()
page_token = None
while True:
response = service.files().list(
spaces="drive",
fields=
"nextPageToken, files(id, name, modifiedTime, trashed)",
q="mimeType!='application/vnd.google-apps.folder' and '"
+ str(subdir) + "' in parents").execute()
for the_file in response.get('files', []):
if re.search(r'(\.tmp|\.gdoc)$', the_file['name']):
continue
if re.search(r'^\~', the_file['name']):
continue
gd_ids[section][the_file['name']] = the_file['id']
gd_modtimes[section][the_file[
'name']] = strict_rfc3339.rfc3339_to_timestamp(
the_file['modifiedTime'])
sys.stderr.write("Google says modtime on " +
unicode(the_file) + " is " +
the_file['modifiedTime'] + "\n")
if the_file['trashed']:
gd_deleted[section].add(the_file['name'])
continue
gd_files[section].add(the_file['name'])
page_token = response.get('nextPageToken', None)
if page_token is None:
break
gd_deleted[section] = gd_deleted[section] - gd_files[section]
for f in gd_files[section]:
sys.stderr.write("Considering " + f + " on GD\n")
if f not in local_files[section] or gd_modtimes[section][
f] - local_modtimes[section][f] > 3:
sys.stderr.write("Considering " + f +
" to copy to local\n")
sections_modified.add(section)
commentary += "Copied " + f + " from Google Drive.\n"
the_path = os.path.join(area.directory, f)
with open(the_path, 'wb') as fh:
response = service.files().get_media(
fileId=gd_ids[section][f])
downloader = worker_controller.apiclient.http.MediaIoBaseDownload(
fh, response)
done = False
while done is False:
status, done = downloader.next_chunk()
#sys.stderr.write("Download %d%%." % int(status.progress() * 100) + "\n")
os.utime(
the_path,
(gd_modtimes[section][f], gd_modtimes[section][f]))
for f in local_files[section]:
sys.stderr.write("Considering " + f +
", which is a local file\n")
if f not in gd_deleted[section]:
sys.stderr.write("Considering " + f +
" is not in Google Drive deleted\n")
if f not in gd_files[section]:
sys.stderr.write("Considering " + f +
" is not in Google Drive\n")
the_path = os.path.join(area.directory, f)
if os.path.getsize(the_path) == 0:
sys.stderr.write("Found zero byte file: " +
the_path + "\n")
continue
sys.stderr.write("Copying " + f +
" to Google Drive.\n")
commentary += "Copied " + f + " to Google Drive.\n"
extension, mimetype = worker_controller.get_ext_and_mimetype(
the_path)
the_modtime = strict_rfc3339.timestamp_to_rfc3339_utcoffset(
local_modtimes[section][f])
sys.stderr.write(
"Setting GD modtime on new file " +
unicode(f) + " to " + unicode(the_modtime) +
"\n")
file_metadata = {
'name': f,
'parents': [subdir],
'modifiedTime': the_modtime,
'createdTime': the_modtime
}
media = worker_controller.apiclient.http.MediaFileUpload(
the_path, mimetype=mimetype)
the_new_file = service.files().create(
body=file_metadata,
media_body=media,
fields='id').execute()
new_id = the_new_file.get('id')
elif local_modtimes[section][f] - gd_modtimes[section][
f] > 3:
sys.stderr.write(
"Considering " + f +
" is in Google Drive but local is more recent\n"
)
the_path = os.path.join(area.directory, f)
if os.path.getsize(the_path) == 0:
sys.stderr.write(
"Found zero byte file during update: " +
the_path + "\n")
continue
commentary += "Updated " + f + " on Google Drive.\n"
extension, mimetype = worker_controller.get_ext_and_mimetype(
the_path)
the_modtime = strict_rfc3339.timestamp_to_rfc3339_utcoffset(
local_modtimes[section][f])
sys.stderr.write(
"Setting GD modtime on modified " +
unicode(f) + " to " + unicode(the_modtime) +
"\n")
file_metadata = {'modifiedTime': the_modtime}
media = worker_controller.apiclient.http.MediaFileUpload(
the_path, mimetype=mimetype)
service.files().update(fileId=gd_ids[section][f],
body=file_metadata,
media_body=media).execute()
for f in gd_deleted[section]:
sys.stderr.write("Considering " + f +
" is deleted on Google Drive\n")
if f in local_files[section]:
sys.stderr.write(
"Considering " + f +
" is deleted on Google Drive but exists locally\n")
if local_modtimes[section][f] - gd_modtimes[section][
f] > 3:
sys.stderr.write(
"Considering " + f +
" is deleted on Google Drive but exists locally and needs to be undeleted on GD\n"
)
commentary += "Undeleted and updated " + f + " on Google Drive.\n"
the_path = os.path.join(area.directory, f)
extension, mimetype = worker_controller.get_ext_and_mimetype(
the_path)
the_modtime = strict_rfc3339.timestamp_to_rfc3339_utcoffset(
local_modtimes[section][f])
sys.stderr.write(
"Setting GD modtime on undeleted file " +
unicode(f) + " to " + unicode(the_modtime) +
"\n")
file_metadata = {
'modifiedTime': the_modtime,
'trashed': False
}
media = worker_controller.apiclient.http.MediaFileUpload(
the_path, mimetype=mimetype)
service.files().update(fileId=gd_ids[section][f],
body=file_metadata,
media_body=media).execute()
else:
sys.stderr.write(
"Considering " + f +
" is deleted on Google Drive but exists locally and needs to deleted locally\n"
)
sections_modified.add(section)
commentary += "Deleted " + f + " from Playground.\n"
the_path = os.path.join(area.directory, f)
if os.path.isfile(the_path):
area.delete_file(f)
area.finalize()
for key in worker_controller.r.keys(
'da:interviewsource:docassemble.playground' +
str(user_id) + ':*'):
worker_controller.r.incr(key)
if commentary != '':
sys.stderr.write(commentary + "\n")
if 'modules' in sections_modified:
do_restart = True
else:
do_restart = False
return worker_controller.functions.ReturnValue(ok=True,
summary=commentary,
restart=do_restart)
except Exception as e:
return worker_controller.functions.ReturnValue(
ok=False,
error="Error syncing with Google Drive: " + str(e),
restart=False)
class PlaygroundSection(object):
def __init__(self, section=''):
if docassemble.base.functions.this_thread.current_info['user'][
'is_anonymous']:
raise DAError(
"Users must be logged in to create Playground objects")
self.user_id = docassemble.base.functions.this_thread.current_info[
'user']['theid']
self.current_info = docassemble.base.functions.this_thread.current_info
self.section = section
self.area = SavedFile(self.user_id,
fix=True,
section='playground' + self.section)
self._update_file_list()
def _update_file_list(self):
self.file_list = sorted([
f for f in os.listdir(self.area.directory)
if os.path.isfile(os.path.join(self.area.directory, f))
])
def image_file_list(self):
out_list = list()
for the_file in self.file_list:
extension, mimetype = get_ext_and_mimetype(the_file)
if re.search(r'^image', mimetype):
out_list.append(the_file)
return out_list
def reduced_file_list(self):
lower_list = [f.lower() for f in self.file_list]
out_list = [
f for f in self.file_list
if os.path.splitext(f)[1].lower() == '.md'
or os.path.splitext(f)[0].lower() + '.md' not in lower_list
]
return out_list
def get_file(self, filename):
return os.path.join(self.area.directory, filename)
def file_exists(self, filename):
path = self.get_file(filename)
if os.path.isfile(path):
return True
return False
def read_file(self, filename):
path = self.get_file(filename)
if path is None:
return None
with open(path, 'rU') as fp:
content = fp.read().decode('utf8')
return content
return None
def write_file(self, filename, content):
path = os.path.join(self.area.directory, filename)
with open(path, 'w') as fp:
fp.write(content.encode('utf8'))
self.area.finalize()
def commit(self):
self.area.finalize()
def copy_from(self, from_file, filename=None):
if filename is None:
filename = os.path.basename(from_file)
to_path = self.get_file(filename)
shutil.copyfile(from_file, to_path)
self.area.finalize()
return filename
def is_markdown(self, filename):
extension, mimetype = get_ext_and_mimetype(filename)
if extension == "md":
return True
return False
def is_pdf(self, filename):
extension, mimetype = get_ext_and_mimetype(filename)
if extension == "pdf":
return True
return False
def get_fields(self, filename):
return docassemble.base.pdftk.read_fields(self.get_file(filename))
def convert_file_to_md(self, filename, convert_variables=True):
extension, mimetype = get_ext_and_mimetype(filename)
if (mimetype and mimetype in convertible_mimetypes):
the_format = convertible_mimetypes[mimetype]
elif extension and extension in convertible_extensions:
the_format = convertible_extensions[extension]
else:
return None
if not self.file_exists(filename):
return None
path = self.get_file(filename)
temp_file = word_to_markdown(path, the_format)
if temp_file is None:
return None
out_filename = os.path.splitext(filename)[0] + '.md'
if convert_variables:
with open(temp_file.name, 'rU') as fp:
self.write_file(
out_filename,
replace_square_brackets.sub(fix_variable_name,
fp.read().decode('utf8')))
else:
shutil.copyfile(temp_file.name, self.get_file(out_filename))
return out_filename
def variables_from_file(self, filename):
content = self.read_file(filename)
if content is None:
return None
return Playground().variables_from(content)
