def upload(self, trans, **kwd):
params = util.Params(kwd)
message = util.restore_text(params.get("message", ""))
status = params.get("status", "done")
commit_message = util.restore_text(params.get("commit_message", "Uploaded"))
category_ids = util.listify(params.get("category_id", ""))
categories = get_categories(trans)
repository_id = params.get("repository_id", "")
repository = get_repository(trans, repository_id)
repo_dir = repository.repo_path
repo = hg.repository(get_configured_ui(), repo_dir)
uncompress_file = util.string_as_bool(params.get("uncompress_file", "true"))
remove_repo_files_not_in_tar = util.string_as_bool(params.get("remove_repo_files_not_in_tar", "true"))
uploaded_file = None
upload_point = self.__get_upload_point(repository, **kwd)
tip = repository.tip
file_data = params.get("file_data", "")
url = params.get("url", "")
# Part of the upload process is sending email notification to those that have registered to
# receive them. One scenario occurs when the first change set is produced for the repository.
# See the handle_email_alerts() method for the definition of the scenarios.
new_repo_alert = repository.is_new
if params.get("upload_button", False):
current_working_dir = os.getcwd()
if file_data == "" and url == "":
message = "No files were entered on the upload form."
status = "error"
uploaded_file = None
elif url:
valid_url = True
try:
stream = urllib.urlopen(url)
except Exception, e:
valid_url = False
message = "Error uploading file via http: %s" % str(e)
status = "error"
uploaded_file = None
if valid_url:
fd, uploaded_file_name = tempfile.mkstemp()
uploaded_file = open(uploaded_file_name, "wb")
while 1:
chunk = stream.read(CHUNK_SIZE)
if not chunk:
break
uploaded_file.write(chunk)
uploaded_file.flush()
uploaded_file_filename = url.split("/")[-1]
isempty = os.path.getsize(os.path.abspath(uploaded_file_name)) == 0
elif file_data not in ("", None):
uploaded_file = file_data.file
uploaded_file_name = uploaded_file.name
uploaded_file_filename = file_data.filename
isempty = os.path.getsize(os.path.abspath(uploaded_file_name)) == 0
if uploaded_file:
isgzip = False
isbz2 = False
if uncompress_file:
isgzip = is_gzip(uploaded_file_name)
if not isgzip:
isbz2 = is_bz2(uploaded_file_name)
ok = True
if isempty:
tar = None
istar = False
else:
# Determine what we have - a single file or an archive
try:
if (isgzip or isbz2) and uncompress_file:
# Open for reading with transparent compression.
tar = tarfile.open(uploaded_file_name, "r:*")
else:
tar = tarfile.open(uploaded_file_name)
istar = True
except tarfile.ReadError, e:
tar = None
istar = False
if istar:
ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = self.upload_tar(
trans,
repository,
tar,
uploaded_file,
upload_point,
remove_repo_files_not_in_tar,
commit_message,
new_repo_alert,
)
else:
if (isgzip or isbz2) and uncompress_file:
uploaded_file_filename = self.uncompress(
repository, uploaded_file_name, uploaded_file_filename, isgzip, isbz2
)
if upload_point is not None:
full_path = os.path.abspath(os.path.join(repo_dir, upload_point, uploaded_file_filename))
else:
full_path = os.path.abspath(os.path.join(repo_dir, uploaded_file_filename))
# Move the uploaded file to the load_point within the repository hierarchy.
shutil.move(uploaded_file_name, full_path)
# See if any admin users have chosen to receive email alerts when a repository is
# updated. If so, check every uploaded file to ensure content is appropriate.
check_contents = check_file_contents(trans)
if check_contents and os.path.isfile(full_path):
content_alert_str = self.__check_file_content(full_path)
else:
content_alert_str = ""
commands.add(repo.ui, repo, full_path)
# Convert from unicode to prevent "TypeError: array item must be char"
full_path = full_path.encode("ascii", "replace")
commands.commit(repo.ui, repo, full_path, user=trans.user.username, message=commit_message)
if full_path.endswith("tool_data_table_conf.xml.sample"):
# Handle the special case where a tool_data_table_conf.xml.sample
# file is being uploaded by parsing the file and adding new entries
# to the in-memory trans.app.tool_data_tables dictionary as well as
# appending them to the shed's tool_data_table_conf.xml file on disk.
error, error_message = handle_sample_tool_data_table_conf_file(trans.app, full_path)
if error:
message = "%s<br/>%s" % (message, error_message)
if full_path.endswith(".loc.sample"):
# Handle the special case where a xxx.loc.sample file is being uploaded by copying it to ~/tool-data/xxx.loc.
copy_sample_file(trans.app, full_path)
# See if the content of the change set was valid.
admin_only = len(repository.downloadable_revisions) != 1
handle_email_alerts(
trans,
repository,
content_alert_str=content_alert_str,
new_repo_alert=new_repo_alert,
admin_only=admin_only,
)
if ok:
# Update the repository files for browsing.
update_repository(repo)
# Get the new repository tip.
if tip != repository.tip:
if (isgzip or isbz2) and uncompress_file:
uncompress_str = " uncompressed and "
else:
uncompress_str = " "
message = "The file '%s' has been successfully%suploaded to the repository." % (
uploaded_file_filename,
uncompress_str,
)
if istar and (undesirable_dirs_removed or undesirable_files_removed):
items_removed = undesirable_dirs_removed + undesirable_files_removed
message += (
" %d undesirable items (.hg .svn .git directories, .DS_Store, hgrc files, etc) were removed from the archive."
% items_removed
)
if istar and remove_repo_files_not_in_tar and files_to_remove:
if upload_point is not None:
message += (
" %d files were removed from the repository relative to the selected upload point '%s'."
% (len(files_to_remove), upload_point)
)
else:
message += " %d files were removed from the repository root." % len(files_to_remove)
else:
message = "No changes to repository."
# Set metadata on the repository tip.
error_message, status = set_repository_metadata(
trans, repository_id, repository.tip, content_alert_str=content_alert_str, **kwd
)
if error_message:
# If there is an error, display it.
message = "%s<br/>%s" % (message, error_message)
return trans.response.send_redirect(
web.url_for(
controller="repository",
action="manage_repository",
id=repository_id,
message=message,
status=status,
)
)
else:
# If no error occurred in setting metadata on the repository tip, reset metadata on all
# changeset revisions for the repository. This will result in a more standardized set of
# valid repository revisions that can be installed.
reset_all_repository_metadata(trans, repository_id, **kwd)
trans.response.send_redirect(
web.url_for(
controller="repository",
action="browse_repository",
id=repository_id,
commit_message="Deleted selected files",
webapp="community",
message=message,
status=status,
)
)
else:
status = "error"
# The directory is not empty
pass
# See if any admin users have chosen to receive email alerts when a repository is
# updated. If so, check every uploaded file to ensure content is appropriate.
check_contents = check_file_contents(trans)
for filename_in_archive in filenames_in_archive:
# Check file content to ensure it is appropriate.
if check_contents and os.path.isfile(filename_in_archive):
content_alert_str += self.__check_file_content(filename_in_archive)
commands.add(repo.ui, repo, filename_in_archive)
if filename_in_archive.endswith("tool_data_table_conf.xml.sample"):
# Handle the special case where a tool_data_table_conf.xml.sample
# file is being uploaded by parsing the file and adding new entries
# to the in-memory trans.app.tool_data_tables dictionary as well as
# appending them to the shed's tool_data_table_conf.xml file on disk.
error, message = handle_sample_tool_data_table_conf_file(trans.app, filename_in_archive)
if error:
return (
False,
message,
files_to_remove,
content_alert_str,
undesirable_dirs_removed,
undesirable_files_removed,
)
if filename_in_archive.endswith(".loc.sample"):
# Handle the special case where a xxx.loc.sample file is being uploaded by copying it to ~/tool-data/xxx.loc.
copy_sample_file(trans.app, filename_in_archive)
commands.commit(repo.ui, repo, full_path, user=trans.user.username, message=commit_message)
# See if the content of the change set was valid.
admin_only = len(repository.downloadable_revisions) != 1
def upload(self, trans, **kwd):
params = util.Params(kwd)
message = util.restore_text(params.get('message', ''))
status = params.get('status', 'done')
commit_message = util.restore_text(
params.get('commit_message', 'Uploaded'))
category_ids = util.listify(params.get('category_id', ''))
categories = get_categories(trans)
repository_id = params.get('repository_id', '')
repository = get_repository(trans, repository_id)
repo_dir = repository.repo_path
repo = hg.repository(get_configured_ui(), repo_dir)
uncompress_file = util.string_as_bool(
params.get('uncompress_file', 'true'))
remove_repo_files_not_in_tar = util.string_as_bool(
params.get('remove_repo_files_not_in_tar', 'true'))
uploaded_file = None
upload_point = self.__get_upload_point(repository, **kwd)
tip = repository.tip
file_data = params.get('file_data', '')
url = params.get('url', '')
# Part of the upload process is sending email notification to those that have registered to
# receive them. One scenario occurs when the first change set is produced for the repository.
# See the handle_email_alerts() method for the definition of the scenarios.
new_repo_alert = repository.is_new
if params.get('upload_button', False):
current_working_dir = os.getcwd()
if file_data == '' and url == '':
message = 'No files were entered on the upload form.'
status = 'error'
uploaded_file = None
elif url:
valid_url = True
try:
stream = urllib.urlopen(url)
except Exception, e:
valid_url = False
message = 'Error uploading file via http: %s' % str(e)
status = 'error'
uploaded_file = None
if valid_url:
fd, uploaded_file_name = tempfile.mkstemp()
uploaded_file = open(uploaded_file_name, 'wb')
while 1:
chunk = stream.read(CHUNK_SIZE)
if not chunk:
break
uploaded_file.write(chunk)
uploaded_file.flush()
uploaded_file_filename = url.split('/')[-1]
isempty = os.path.getsize(
os.path.abspath(uploaded_file_name)) == 0
elif file_data not in ('', None):
uploaded_file = file_data.file
uploaded_file_name = uploaded_file.name
uploaded_file_filename = file_data.filename
isempty = os.path.getsize(
os.path.abspath(uploaded_file_name)) == 0
if uploaded_file:
isgzip = False
isbz2 = False
if uncompress_file:
isgzip = is_gzip(uploaded_file_name)
if not isgzip:
isbz2 = is_bz2(uploaded_file_name)
ok = True
if isempty:
tar = None
istar = False
else:
# Determine what we have - a single file or an archive
try:
if (isgzip or isbz2) and uncompress_file:
# Open for reading with transparent compression.
tar = tarfile.open(uploaded_file_name, 'r:*')
else:
tar = tarfile.open(uploaded_file_name)
istar = True
except tarfile.ReadError, e:
tar = None
istar = False
if istar:
ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = self.upload_tar(
trans, repository, tar, uploaded_file, upload_point,
remove_repo_files_not_in_tar, commit_message,
new_repo_alert)
else:
if (isgzip or isbz2) and uncompress_file:
uploaded_file_filename = self.uncompress(
repository, uploaded_file_name,
uploaded_file_filename, isgzip, isbz2)
if upload_point is not None:
full_path = os.path.abspath(
os.path.join(repo_dir, upload_point,
uploaded_file_filename))
else:
full_path = os.path.abspath(
os.path.join(repo_dir, uploaded_file_filename))
# Move the uploaded file to the load_point within the repository hierarchy.
shutil.move(uploaded_file_name, full_path)
# See if any admin users have chosen to receive email alerts when a repository is
# updated. If so, check every uploaded file to ensure content is appropriate.
check_contents = check_file_contents(trans)
if check_contents and os.path.isfile(full_path):
content_alert_str = self.__check_file_content(
full_path)
else:
content_alert_str = ''
commands.add(repo.ui, repo, full_path)
# Convert from unicode to prevent "TypeError: array item must be char"
full_path = full_path.encode('ascii', 'replace')
commands.commit(repo.ui,
repo,
full_path,
user=trans.user.username,
message=commit_message)
if full_path.endswith('tool_data_table_conf.xml.sample'):
# Handle the special case where a tool_data_table_conf.xml.sample
# file is being uploaded by parsing the file and adding new entries
# to the in-memory trans.app.tool_data_tables dictionary as well as
# appending them to the shed's tool_data_table_conf.xml file on disk.
error, error_message = handle_sample_tool_data_table_conf_file(
trans.app, full_path)
if error:
message = '%s<br/>%s' % (message, error_message)
if full_path.endswith('.loc.sample'):
# Handle the special case where a xxx.loc.sample file is being uploaded by copying it to ~/tool-data/xxx.loc.
copy_sample_file(trans.app, full_path)
# See if the content of the change set was valid.
admin_only = len(repository.downloadable_revisions) != 1
handle_email_alerts(trans,
repository,
content_alert_str=content_alert_str,
new_repo_alert=new_repo_alert,
admin_only=admin_only)
if ok:
# Update the repository files for browsing.
update_repository(repo)
# Get the new repository tip.
if tip != repository.tip:
if (isgzip or isbz2) and uncompress_file:
uncompress_str = ' uncompressed and '
else:
uncompress_str = ' '
message = "The file '%s' has been successfully%suploaded to the repository." % (
uploaded_file_filename, uncompress_str)
if istar and (undesirable_dirs_removed
or undesirable_files_removed):
items_removed = undesirable_dirs_removed + undesirable_files_removed
message += " %d undesirable items (.hg .svn .git directories, .DS_Store, hgrc files, etc) were removed from the archive." % items_removed
if istar and remove_repo_files_not_in_tar and files_to_remove:
if upload_point is not None:
message += " %d files were removed from the repository relative to the selected upload point '%s'." % (
len(files_to_remove), upload_point)
else:
message += " %d files were removed from the repository root." % len(
files_to_remove)
else:
message = 'No changes to repository.'
# Set metadata on the repository tip.
error_message, status = set_repository_metadata(
trans,
repository_id,
repository.tip,
content_alert_str=content_alert_str,
**kwd)
if error_message:
# If there is an error, display it.
message = '%s<br/>%s' % (message, error_message)
return trans.response.send_redirect(
web.url_for(controller='repository',
action='manage_repository',
id=repository_id,
message=message,
status=status))
else:
# If no error occurred in setting metadata on the repository tip, reset metadata on all
# changeset revisions for the repository. This will result in a more standardized set of
# valid repository revisions that can be installed.
reset_all_repository_metadata(trans, repository_id,
**kwd)
trans.response.send_redirect(
web.url_for(controller='repository',
action='browse_repository',
id=repository_id,
commit_message='Deleted selected files',
webapp='community',
message=message,
status=status))
else:
status = 'error'
def upload( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
commit_message = util.restore_text( params.get( 'commit_message', 'Uploaded' ) )
category_ids = util.listify( params.get( 'category_id', '' ) )
categories = get_categories( trans )
repository_id = params.get( 'repository_id', '' )
repository = get_repository( trans, repository_id )
repo_dir = repository.repo_path
repo = hg.repository( get_configured_ui(), repo_dir )
uncompress_file = util.string_as_bool( params.get( 'uncompress_file', 'true' ) )
remove_repo_files_not_in_tar = util.string_as_bool( params.get( 'remove_repo_files_not_in_tar', 'true' ) )
uploaded_file = None
upload_point = self.__get_upload_point( repository, **kwd )
tip = repository.tip
file_data = params.get( 'file_data', '' )
url = params.get( 'url', '' )
# Part of the upload process is sending email notification to those that have registered to
# receive them. One scenario occurs when the first change set is produced for the repository.
# See the handle_email_alerts() method for the definition of the scenarios.
new_repo_alert = repository.is_new
if params.get( 'upload_button', False ):
if file_data == '' and url == '':
message = 'No files were entered on the upload form.'
status = 'error'
uploaded_file = None
elif url:
valid_url = True
try:
stream = urllib.urlopen( url )
except Exception, e:
valid_url = False
message = 'Error uploading file via http: %s' % str( e )
status = 'error'
uploaded_file = None
if valid_url:
fd, uploaded_file_name = tempfile.mkstemp()
uploaded_file = open( uploaded_file_name, 'wb' )
while 1:
chunk = stream.read( CHUNK_SIZE )
if not chunk:
break
uploaded_file.write( chunk )
uploaded_file.flush()
uploaded_file_filename = url.split( '/' )[ -1 ]
isempty = os.path.getsize( os.path.abspath( uploaded_file_name ) ) == 0
elif file_data not in ( '', None ):
uploaded_file = file_data.file
uploaded_file_name = uploaded_file.name
uploaded_file_filename = file_data.filename
isempty = os.path.getsize( os.path.abspath( uploaded_file_name ) ) == 0
if uploaded_file:
isgzip = False
isbz2 = False
if uncompress_file:
isgzip = is_gzip( uploaded_file_name )
if not isgzip:
isbz2 = is_bz2( uploaded_file_name )
ok = True
if isempty:
tar = None
istar = False
else:
# Determine what we have - a single file or an archive
try:
if ( isgzip or isbz2 ) and uncompress_file:
# Open for reading with transparent compression.
tar = tarfile.open( uploaded_file_name, 'r:*' )
else:
tar = tarfile.open( uploaded_file_name )
istar = True
except tarfile.ReadError, e:
tar = None
istar = False
if istar:
ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = self.upload_tar( trans,
repository,
tar,
uploaded_file,
upload_point,
remove_repo_files_not_in_tar,
commit_message,
new_repo_alert )
else:
if ( isgzip or isbz2 ) and uncompress_file:
uploaded_file_filename = self.uncompress( repository, uploaded_file_name, uploaded_file_filename, isgzip, isbz2 )
if upload_point is not None:
full_path = os.path.abspath( os.path.join( repo_dir, upload_point, uploaded_file_filename ) )
else:
full_path = os.path.abspath( os.path.join( repo_dir, uploaded_file_filename ) )
# Move the uploaded file to the load_point within the repository hierarchy.
shutil.move( uploaded_file_name, full_path )
# See if any admin users have chosen to receive email alerts when a repository is
# updated. If so, check every uploaded file to ensure content is appropriate.
check_contents = check_file_contents( trans )
if check_contents and os.path.isfile( full_path ):
content_alert_str = self.__check_file_content( full_path )
else:
content_alert_str = ''
commands.add( repo.ui, repo, full_path )
# Convert from unicode to prevent "TypeError: array item must be char"
full_path = full_path.encode( 'ascii', 'replace' )
commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message )
if full_path.endswith( 'tool_data_table_conf.xml.sample' ):
# Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded by parsing the file and adding new entries
# to the in-memory trans.app.tool_data_tables dictionary.
error, error_message = handle_sample_tool_data_table_conf_file( trans.app, full_path )
if error:
message = '%s<br/>%s' % ( message, error_message )
# See if the content of the change set was valid.
admin_only = len( repository.downloadable_revisions ) != 1
handle_email_alerts( trans, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only )
if ok:
# Update the repository files for browsing.
update_repository( repo )
# Get the new repository tip.
if tip == repository.tip:
message = 'No changes to repository. '
else:
if ( isgzip or isbz2 ) and uncompress_file:
uncompress_str = ' uncompressed and '
else:
uncompress_str = ' '
message = "The file '%s' has been successfully%suploaded to the repository. " % ( uploaded_file_filename, uncompress_str )
if istar and ( undesirable_dirs_removed or undesirable_files_removed ):
items_removed = undesirable_dirs_removed + undesirable_files_removed
message += " %d undesirable items (.hg .svn .git directories, .DS_Store, hgrc files, etc) were removed from the archive. " % items_removed
if istar and remove_repo_files_not_in_tar and files_to_remove:
if upload_point is not None:
message += " %d files were removed from the repository relative to the selected upload point '%s'. " % ( len( files_to_remove ), upload_point )
else:
message += " %d files were removed from the repository root. " % len( files_to_remove )
kwd[ 'message' ] = message
set_repository_metadata_due_to_new_tip( trans, repository_id, repository, content_alert_str=content_alert_str, **kwd )
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
reset_tool_data_tables( trans.app )
trans.response.send_redirect( web.url_for( controller='repository',
action='browse_repository',
id=repository_id,
commit_message='Deleted selected files',
webapp='community',
message=message,
status=status ) )
else:
status = 'error'
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
reset_tool_data_tables( trans.app )
# See if any admin users have chosen to receive email alerts when a repository is
# updated. If so, check every uploaded file to ensure content is appropriate.
check_contents = check_file_contents(trans)
for filename_in_archive in filenames_in_archive:
# Check file content to ensure it is appropriate.
if check_contents and os.path.isfile(filename_in_archive):
content_alert_str += self.__check_file_content(
filename_in_archive)
commands.add(repo.ui, repo, filename_in_archive)
if filename_in_archive.endswith(
'tool_data_table_conf.xml.sample'):
# Handle the special case where a tool_data_table_conf.xml.sample
# file is being uploaded by parsing the file and adding new entries
# to the in-memory trans.app.tool_data_tables dictionary as well as
# appending them to the shed's tool_data_table_conf.xml file on disk.
error, message = handle_sample_tool_data_table_conf_file(
trans.app, filename_in_archive)
if error:
return False, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed
if filename_in_archive.endswith('.loc.sample'):
# Handle the special case where a xxx.loc.sample file is being uploaded by copying it to ~/tool-data/xxx.loc.
copy_sample_file(trans.app, filename_in_archive)
commands.commit(repo.ui,
repo,
full_path,
user=trans.user.username,
message=commit_message)
# See if the content of the change set was valid.
admin_only = len(repository.downloadable_revisions) != 1
handle_email_alerts(trans,
repository,
content_alert_str=content_alert_str,
