def init(my):
web = WebContainer.get_web()
my.is_refresh = my.kwargs.get('is_refresh')
my.search_type = my.kwargs.get('search_type')
if not my.search_type:
my.search_type = web.get_form_value('search_type_filter')
my.close_cbfn = my.kwargs.get('close_cbfn')
my.web_url = web.get_form_value("web_url")
my.file_path = None
if my.web_url:
import urllib2
response = urllib2.urlopen(my.web_url)
csv = response.read()
my.file_path = "/tmp/test.csv"
f = open(my.file_path, 'w')
f.write(csv)
f.close()
if not my.file_path:
my.file_path = web.get_form_value('file_path')
if not my.file_path:
file_name = web.get_form_value('file_name')
ticket = web.get_form_value('html5_ticket')
if not ticket:
ticket = web.get_form_value('csv_import|ticket')
if file_name:
# this is treated the same in FileUplaod class
file_name = File.get_filesystem_name(str(file_name))
my.file_path = '%s/%s' %(web.get_upload_dir(ticket=ticket), file_name)
def _read_ref_file(my):
'''read the reference file containing extra node information'''
dir = my.get_upload_dir()
xml = Xml()
key = my.sobject.get_code()
# make this filename good for the file system
filename = File.get_filesystem_name(key)
xml.read_file( "%s/%s-ref.xml" % (dir,filename) )
return xml
def _read_ref_file(my):
'''read the reference file containing extra node information'''
dir = my.get_upload_dir()
xml = Xml()
key = my.sobject.get_code()
# make this filename good for the file system
filename = File.get_filesystem_name(key)
xml.read_file("%s/%s-ref.xml" % (dir, filename))
return xml
def _get_unique_filename(my):
filename = my.file_object.get_full_file_name()
# find if this filename has been used for this project
file = File.get_by_filename(filename, skip_id=my.file_object.get_id())
if file:
root, ext = os.path.splitext(filename)
parts = [root]
filename = my.add_ending(parts, auto_version=True)
return filename
else:
return None
def _get_unique_filename(my):
filename = my.file_object.get_full_file_name()
# find if this filename has been used for this project
file = File.get_by_filename(filename, skip_id=my.file_object.get_id())
if file:
root, ext = os.path.splitext(filename)
parts = [root]
filename = my.add_ending(parts, auto_version=True)
return filename
else:
return None
def _init_file_object(self):
'''initialize the file object. Some fields are still empty before checkin postprocess'''
# if set externally already, skip and return
if self._file_object:
return
file_type = self.get_file_type()
if file_type and self.snapshot:
# get the file_object
file_code = self.snapshot.get_file_code_by_type(file_type)
from pyasm.biz import File
self._file_object = File.get_by_code(file_code)
def _get_file_obj(my, snapshot, type='main'):
if type:
xpath = "snapshot/file[@type='%s']" %type
else:
xpath = "snapshot/file[@type]"
xml = snapshot.get_xml_value('snapshot')
node = xml.get_node(xpath)
file = None
if node is not None:
file_code = Xml.get_attribute(node, "file_code")
file = File.get_by_code(file_code)
return file
def _init_file_object(my):
'''initialize the file object. Some fields are still empty before checkin postprocess'''
# if set externally already, skip and return
if my._file_object:
return
file_type = my.get_file_type()
if file_type and my.snapshot:
# get the file_object
file_code = my.snapshot.get_file_code_by_type(file_type)
from pyasm.biz import File
my._file_object = File.get_by_code(file_code)
def _get_file_obj(my, snapshot, type='main'):
if type:
xpath = "snapshot/file[@type='%s']" % type
else:
xpath = "snapshot/file[@type]"
xml = snapshot.get_xml_value('snapshot')
node = xml.get_node(xpath)
file = None
if node is not None:
file_code = Xml.get_attribute(node, "file_code")
file = File.get_by_code(file_code)
return file
def add_ref_by_file_path(my, file_path, type='ref', node_name='', tag='main'):
'''add a reference based on the file name. If the file is unique, then
a reference can be found based on the file name'''
from pyasm.biz import File
filename = os.path.basename(file_path)
file = File.get_by_filename(filename, padding=4)
if not file:
Environment.add_warning("Unknown File Reference", "File reference [%s] does not exist in database" % file_path)
my.add_unknown_ref(file_path)
return
else:
snapshot_code = file.get_value("snapshot_code")
return my.add_ref_by_snapshot_code(snapshot_code, type=type, node_name=node_name, tag=tag)
def get_parser_by_path(cls, path):
ext = File.get_extension(path)
parser_str = None
if ext in File.VIDEO_EXT:
if HAS_FFMPEG:
parser_str = "FFMPEG"
else:
parser_str = "PIL"
else:
if HAS_IMAGEMAGICK:
parser_str = "ImageMagick"
elif HAS_PIL:
parser_str = "PIL"
elif HAS_EXIF:
parser_str = "EXIF"
elif HAS_FFMPEG:
parser_str = "FFMPEG"
return cls.get_parser(parser_str, path)
def get_parser_by_path(cls, path):
ext = File.get_extension(path)
parser_str = None
if ext in File.VIDEO_EXT:
if HAS_FFMPEG:
parser_str = "FFMPEG"
else:
parser_str = "PIL"
else:
if HAS_IMAGEMAGICK:
parser_str = "ImageMagick"
elif HAS_PIL:
parser_str = "PIL"
elif HAS_EXIF:
parser_str = "EXIF"
elif HAS_FFMPEG:
parser_str = "FFMPEG"
return cls.get_parser(parser_str, path)
def add_ref_by_file_path(my,
file_path,
type='ref',
node_name='',
tag='main'):
'''add a reference based on the file name. If the file is unique, then
a reference can be found based on the file name'''
from pyasm.biz import File
filename = os.path.basename(file_path)
file = File.get_by_filename(filename, padding=4)
if not file:
Environment.add_warning(
"Unknown File Reference",
"File reference [%s] does not exist in database" % file_path)
my.add_unknown_ref(file_path)
return
else:
snapshot_code = file.get_value("snapshot_code")
return my.add_ref_by_snapshot_code(snapshot_code,
type=type,
node_name=node_name,
tag=tag)
def execute(my):
filenames = my.kwargs.get("filenames")
upload_dir = Environment.get_upload_dir()
base_dir = upload_dir
search_type = my.kwargs.get("search_type")
key = my.kwargs.get("key")
relative_dir = my.kwargs.get("relative_dir")
if not relative_dir:
project_code = Project.get_project_code()
search_type_obj = SearchType.get(search_type)
table = search_type_obj.get_table()
relative_dir = "%s/%s" % (project_code, table)
server = TacticServerStub.get()
parent_key = my.kwargs.get("parent_key")
category = my.kwargs.get("category")
keywords = my.kwargs.get("keywords")
extra_data = my.kwargs.get("extra_data")
if extra_data:
extra_data = jsonloads(extra_data)
else:
extra_data = {}
# TODO: use this to generate a category
category_script_path = my.kwargs.get("category_script_path")
"""
ie:
from pyasm.checkin import ExifMetadataParser
parser = ExifMetadataParser(path=file_path)
tags = parser.get_metadata()
date = tags.get("EXIF DateTimeOriginal")
return date.split(" ")[0]
"""
if not SearchType.column_exists(search_type, "name"):
raise TacticException('The Ingestion puts the file name into the name column which is the minimal requirement. Please first create a "name" column for this sType.')
for count, filename in enumerate(filenames):
# first see if this sobjects still exists
search = Search(search_type)
search.add_filter("name", filename)
if relative_dir and search.column_exists("relative_dir"):
search.add_filter("relative_dir", relative_dir)
sobject = search.get_sobject()
# else create a new one
if not sobject:
sobject = SearchType.create(search_type)
sobject.set_value("name", filename)
if relative_dir and sobject.column_exists("relative_dir"):
sobject.set_value("relative_dir", relative_dir)
# extract metadata
file_path = "%s/%s" % (base_dir, File.get_filesystem_name(filename))
# TEST: convert on upload
try:
convert = my.kwargs.get("convert")
if convert:
message_key = "IngestConvert001"
cmd = ConvertCbk(**convert)
cmd.execute()
except Exception, e:
print "WARNING: ", e
if not os.path.exists(file_path):
raise Exception("Path [%s] does not exist" % file_path)
# get the metadata from this image
if SearchType.column_exists(search_type, "relative_dir"):
if category and category not in ['none', None]:
from pyasm.checkin import ExifMetadataParser
parser = ExifMetadataParser(path=file_path)
tags = parser.get_metadata()
date = tags.get("EXIF DateTimeOriginal")
if not date:
date_str = "No-Date"
else:
date_str = str(date)
# this can't be parsed correctly by dateutils
parts = date_str.split(" ")
date_str = parts[0].replace(":", "-")
date_str = "%s %s" % (date_str, parts[1])
from dateutil import parser
orig_date = parser.parse(date_str)
if category == "by_day":
date_str = orig_date.strftime("%Y/%Y-%m-%d")
elif category == "by_month":
date_str = orig_date.strftime("%Y-%m")
elif category == "by_week":
date_str = orig_date.strftime("%Y/Week-%U")
full_relative_dir = "%s/%s" % (relative_dir, date_str)
sobject.set_value("relative_dir", full_relative_dir)
if parent_key:
parent = Search.get_by_search_key(parent_key)
if parent:
sobject.set_sobject_value(sobject)
if SearchType.column_exists(search_type, "keywords"):
if keywords:
sobject.set_value("keywords", keywords)
for key, value in extra_data.items():
if SearchType.column_exists(search_type, key):
sobject.set_value(key, value)
"""
if category:
if SearchType.column_exists(search_type, "category"):
sobject.set_value("category", category)
if SearchType.column_exists(search_type, "relative_dir"):
full_relative_dir = "%s/%s" % (relative_dir, category)
sobject.set_value("relative_dir", category)
"""
sobject.commit()
search_key = sobject.get_search_key()
# use API to check in file
process = my.kwargs.get("process")
if not process:
process = "publish"
if process == "icon":
context = "icon"
else:
context = "%s/%s" % (process, filename)
server.simple_checkin(search_key, context, filename, mode='uploaded')
percent = int((float(count)+1) / len(filenames)*100)
print "checking in: ", filename, percent
msg = {
'progress': percent,
'description': 'Checking in file [%s]' % filename,
}
server.log_message(key, msg, status="in progress")
def init(my):
WebContainer.register_cmd("pyasm.widget.AnnotateCbk")
sobject = my.get_current_sobject()
if not sobject:
if not my.__dict__.has_key("search_type"):
web = WebContainer.get_web()
my.search_type = web.get_form_value("search_type")
my.search_id = web.get_form_value("search_id")
if not my.search_type:
my.add("No search type")
return
search = Search(my.search_type)
search.add_id_filter(my.search_id)
sobject = search.get_sobject()
snapshot = Snapshot.get_latest_by_sobject(sobject)
# TODO:
# this is a bit klunky
snapshot_xml = snapshot.get_xml_value("snapshot")
file_code = snapshot_xml.get_value("snapshot/file[@type='web']/@file_code")
file = File.get_by_code(file_code)
web_dir = snapshot.get_web_dir()
path = "%s/%s" % (web_dir, file.get_full_file_name() )
# add the annotate js object
script = HtmlElement.script("annotate = new Annotate()")
my.add(script)
# add the image
my.add("<h3>Image Annotations</h3>")
width = 600
img = HtmlElement.img(path)
img.add_style("position: absolute")
img.set_id("annotate_image")
img.add_style("left: 0px")
img.add_style("top: 0px")
img.add_style("opacity: 1.0")
img.add_style("z-index: 1")
img.add_style("width", width)
img.add_event("onmouseup", "annotate.add_new(event)")
my.add(img)
# test
version = snapshot.get_value("version")
if version != 1:
last_version = version - 1
snapshot = Snapshot.get_by_version( \
my.search_type, my.search_id, version=last_version )
snapshot_xml = snapshot.get_xml_value("snapshot")
file_code = snapshot_xml.get_value("snapshot/file[@type='web']/@file_code")
file = File.get_by_code(file_code)
web_dir = snapshot.get_web_dir()
path = "%s/%s" % (web_dir, file.get_full_file_name() )
img = HtmlElement.img(path)
img.set_id("annotate_image_alt")
img.add_style("position: absolute")
img.add_style("left: 0px")
img.add_style("top: 0px")
img.add_style("opacity: 1.0")
img.add_style("z-index: 0")
img.add_style("width", width)
img.add_event("onmouseup", "annotate.add_new(event)")
my.add(img)
#script = HtmlElement.script("align_element('%s','%s')" % \
# ("annotate_image", "annotate_image_alt") )
#my.add(script)
div = DivWdg()
div.add_style("position: absolute")
div.add_style("left: 620")
div.add_style("top: 300")
my.add(div)
button = IconButtonWdg("Switch", IconWdg.REFRESH, True)
button.add_event("onclick", "annotate.switch_alt()")
div.add(button)
button = IconButtonWdg("Opacity", IconWdg.LOAD, True)
button.add_event("onclick", "annotate.set_opacity()")
div.add(button)
# add the new annotation div
new_annotation_div = DivWdg()
new_annotation_div.set_id("annotate_msg")
new_annotation_div.set_class("annotate_new")
new_annotation_div.add_style("top: 0")
new_annotation_div.add_style("left: 0")
title = DivWdg("Enter Annotation:")
title.add_style("background-color: #000000")
title.add_style("color: #ffffff")
new_annotation_div.add(title)
text = TextAreaWdg("annotate_msg")
text.set_attr("cols", "30")
text.set_attr("rows", "3")
new_annotation_div.add(text)
new_annotation_div.add("<br/>")
cancel = ButtonWdg("Cancel")
cancel.add_style("float: right")
cancel.add_event("onclick", "toggle_display('annotate_msg')")
new_annotation_div.add( cancel )
submit = SubmitWdg("Add Annotation")
submit.add_style("float: right")
new_annotation_div.add( submit )
new_annotation_div.add_style("display: none")
new_annotation_div.add_style("position: absolute")
my.add(new_annotation_div)
# get all of the stored annotations for this image
search = Search("sthpw/annotation")
search.add_order_by("login")
search.add_filter("file_code", file_code)
annotations = search.get_sobjects()
# sort by user
sorted_annotations = {}
for annotation in annotations:
user = annotation.get_value("login")
if not sorted_annotations.has_key(user):
sorted_annotations[user] = []
sorted_annotations[user].append(annotation)
buttons = []
for user, annotations in sorted_annotations.items():
button = IconButtonWdg(user, IconWdg.INSERT, True)
button.add_event("onclick", "annotate.show_marks('%s','%s')" % (user, len(annotations)-1) )
buttons.append(button)
count = 0
for annotation in annotations:
my.add( my.get_annotate_wdg(annotation,count) )
count += 1
# add the user buttons
table = Table()
table.set_class("table")
table.add_style("width: 0px")
table.add_style("position: absolute")
table.add_style("left: 620")
table.add_style("top: 0")
table.add_row()
table.add_header("Annotations")
for button in buttons:
table.add_row()
legend_wdg = DivWdg()
legend_wdg.set_class("annotate_mark")
legend_wdg.add_style("position: relative")
legend_wdg.add(" ")
table.add_cell(legend_wdg)
table.add_cell(button)
my.add(table)
# add form elements
hidden = HiddenWdg("mouse_xpos")
my.add(hidden)
hidden = HiddenWdg("mouse_ypos")
my.add(hidden)
hidden = HiddenWdg("file_code", file_code)
my.add(hidden)
# move the rest below
my.add("<div style='height:300'> </div>")
def execute(my):
# extra data from the file check-in
commit = my.kwargs.get("commit")
if commit in ['false', False]:
commit = False
else:
commit = True
# metadata can only be associated with a single file on the
# snapshot.
files = my.kwargs.get("files")
file_objects = my.kwargs.get("file_objects")
snapshot_metadata = None
parser_type = my.kwargs.get("parser")
for i, file in enumerate(files):
path = file
ext = File.get_extension(path)
file_object = file_objects[i]
if not os.path.exists(path):
continue
elif parser_type != "auto" and parser_type != "true":
pass
elif ext in File.VIDEO_EXT:
parser_type = "FFMPEG"
elif ext in File.NORMAL_EXT:
continue
else:
if HAS_IMAGEMAGICK:
parser_type = "ImageMagick"
else:
parser_type = "PIL"
metadata = {}
if parser_type == "FFMPEG":
parser = FFProbeMetadataParser(path=path)
metadata = parser.get_metadata()
elif parser_type == "ImageMagick":
parser = ImageMagickMetadataParser(path=path)
metadata = parser.get_metadata()
else:
parser = PILMetadataParser(path=path)
metadata = parser.get_metadata()
metadata['__parser__'] = parser_type
"""
if os.path.exists(path):
try:
parser = ImageMagickMetadataParser(path=path)
metadata = parser.get_metadata()
except:
try:
#parser = PILMetadataParser(path=path)
parser = FFProbeMetadataParser(path=path)
metadata = parser.get_metadata()
except:
metadata = {}
else:
metadata = {}
"""
# add some default metadata
basename = os.path.basename(path)
path = file_object.get_lib_path()
dirname = os.path.dirname(path)
parts = basename.split(".")
ext = parts[-1]
try:
# occasionally, people will put frame numbers as the last
# part
ext = int(ext)
ext = parts[-2]
except:
pass
metadata['Ext'] = ext
metadata['Basename'] = basename
metadata['Dirname'] = dirname
if metadata and not file_object.get_value("metadata"):
file_object.add_metadata(metadata, replace=True)
searchable = my.get_searchable(metadata)
file_object.set_value("metadata_search", searchable)
file_object.commit()
if i == 0:
snapshot = my.kwargs.get("snapshot")
snapshot.add_metadata(metadata, replace=True)
if commit:
snapshot.commit()
def init(self):
WebContainer.register_cmd("pyasm.widget.AnnotateCbk")
sobject = self.get_current_sobject()
if not sobject:
if not self.__dict__.has_key("search_type"):
web = WebContainer.get_web()
self.search_type = web.get_form_value("search_type")
self.search_id = web.get_form_value("search_id")
if not self.search_type:
self.add("No search type")
return
search = Search(self.search_type)
search.add_id_filter(self.search_id)
sobject = search.get_sobject()
snapshot = Snapshot.get_latest_by_sobject(sobject)
# TODO:
# this is a bit klunky
snapshot_xml = snapshot.get_xml_value("snapshot")
file_code = snapshot_xml.get_value(
"snapshot/file[@type='web']/@file_code")
file = File.get_by_code(file_code)
web_dir = snapshot.get_web_dir()
path = "%s/%s" % (web_dir, file.get_full_file_name())
# add the annotate js object
script = HtmlElement.script("annotate = new Annotate()")
self.add(script)
# add the image
self.add("<h3>Image Annotations</h3>")
width = 600
img = HtmlElement.img(path)
img.add_style("position: absolute")
img.set_id("annotate_image")
img.add_style("left: 0px")
img.add_style("top: 0px")
img.add_style("opacity: 1.0")
img.add_style("z-index: 1")
img.add_style("width", width)
img.add_event("onmouseup", "annotate.add_new(event)")
self.add(img)
# test
version = snapshot.get_value("version")
if version != 1:
last_version = version - 1
snapshot = Snapshot.get_by_version( \
self.search_type, self.search_id, version=last_version )
snapshot_xml = snapshot.get_xml_value("snapshot")
file_code = snapshot_xml.get_value(
"snapshot/file[@type='web']/@file_code")
file = File.get_by_code(file_code)
web_dir = snapshot.get_web_dir()
path = "%s/%s" % (web_dir, file.get_full_file_name())
img = HtmlElement.img(path)
img.set_id("annotate_image_alt")
img.add_style("position: absolute")
img.add_style("left: 0px")
img.add_style("top: 0px")
img.add_style("opacity: 1.0")
img.add_style("z-index: 0")
img.add_style("width", width)
img.add_event("onmouseup", "annotate.add_new(event)")
self.add(img)
#script = HtmlElement.script("align_element('%s','%s')" % \
# ("annotate_image", "annotate_image_alt") )
#self.add(script)
div = DivWdg()
div.add_style("position: absolute")
div.add_style("left: 620")
div.add_style("top: 300")
self.add(div)
button = IconButtonWdg("Switch", IconWdg.REFRESH, True)
button.add_event("onclick", "annotate.switch_alt()")
div.add(button)
button = IconButtonWdg("Opacity", IconWdg.LOAD, True)
button.add_event("onclick", "annotate.set_opacity()")
div.add(button)
# add the new annotation div
new_annotation_div = DivWdg()
new_annotation_div.set_id("annotate_msg")
new_annotation_div.set_class("annotate_new")
new_annotation_div.add_style("top: 0")
new_annotation_div.add_style("left: 0")
title = DivWdg("Enter Annotation:")
title.add_style("background-color: #000000")
title.add_style("color: #ffffff")
new_annotation_div.add(title)
text = TextAreaWdg("annotate_msg")
text.set_attr("cols", "30")
text.set_attr("rows", "3")
new_annotation_div.add(text)
new_annotation_div.add("<br/>")
cancel = ButtonWdg("Cancel")
cancel.add_style("float: right")
cancel.add_event("onclick", "toggle_display('annotate_msg')")
new_annotation_div.add(cancel)
submit = SubmitWdg("Add Annotation")
submit.add_style("float: right")
new_annotation_div.add(submit)
new_annotation_div.add_style("display: none")
new_annotation_div.add_style("position: absolute")
self.add(new_annotation_div)
# get all of the stored annotations for this image
search = Search("sthpw/annotation")
search.add_order_by("login")
search.add_filter("file_code", file_code)
annotations = search.get_sobjects()
# sort by user
sorted_annotations = {}
for annotation in annotations:
user = annotation.get_value("login")
if not sorted_annotations.has_key(user):
sorted_annotations[user] = []
sorted_annotations[user].append(annotation)
buttons = []
for user, annotations in sorted_annotations.items():
button = IconButtonWdg(user, IconWdg.INSERT, True)
button.add_event(
"onclick", "annotate.show_marks('%s','%s')" %
(user, len(annotations) - 1))
buttons.append(button)
count = 0
for annotation in annotations:
self.add(self.get_annotate_wdg(annotation, count))
count += 1
# add the user buttons
table = Table()
table.set_class("table")
table.add_style("width: 0px")
table.add_style("position: absolute")
table.add_style("left: 620")
table.add_style("top: 0")
table.add_row()
table.add_header("Annotations")
for button in buttons:
table.add_row()
legend_wdg = DivWdg()
legend_wdg.set_class("annotate_mark")
legend_wdg.add_style("position: relative")
legend_wdg.add(" ")
table.add_cell(legend_wdg)
table.add_cell(button)
self.add(table)
# add form elements
hidden = HiddenWdg("mouse_xpos")
self.add(hidden)
hidden = HiddenWdg("mouse_ypos")
self.add(hidden)
hidden = HiddenWdg("file_code", file_code)
self.add(hidden)
# move the rest below
self.add("<div style='height:300'> </div>")
def _test_inplace_checkin(self):
# create a new test.txt file
tmp_dir = Environment.get_tmp_dir()
dir = "%s/temp" % tmp_dir
if not os.path.exists(dir):
os.makedirs(dir)
file_path = "%s/test_inplace.txt" % dir
if os.path.exists(file_path):
os.unlink(file_path)
file = open(file_path, 'w')
file.write("whatever")
file.close()
# inplace checkin: tell tactic that this is the correct path
mode = "inplace"
base_dir = tmp_dir
context = "inplace"
checkin = FileCheckin(self.person, file_path, context=context, mode=mode)
checkin.execute()
snapshot = checkin.get_snapshot()
file_code = snapshot.get_file_code_by_type("main")
file_object = File.get_by_code(file_code)
relative_dir = file_object.get_value("relative_dir")
# The relative dir is empty if the file is outside the repository
self.assertEquals("", relative_dir)
lib_dir = snapshot.get_lib_dir(file_type="main")
file_name = snapshot.get_file_name_by_type("main")
lib_path = "%s/%s" % (lib_dir, file_name)
self.assertEquals( True, os.path.exists(lib_path) )
self.assertEquals( file_path, lib_path)
# check in a file alredy in the repository
asset_dir = Config.get_value("checkin", "asset_base_dir", sub_key="default")
file_path2 = "%s/unittest/text.txt" % asset_dir
file = open(file_path2, 'w')
file.write("whatever")
file.close()
checkin = FileCheckin(self.person, file_path2, context=context, mode=mode)
checkin.execute()
snapshot = checkin.get_snapshot()
file_code = snapshot.get_file_code_by_type("main")
file_object = File.get_by_code(file_code)
# check that the relative dir is as expected
relative_dir = file_object.get_value("relative_dir")
self.assertEquals( relative_dir, "unittest" )
# check that the path returned correctly
lib_path = snapshot.get_path_by_type("main")
self.assertEquals( file_path2, lib_path )
if os.path.exists(file_path):
os.unlink(file_path)
if os.path.exists(file_path2):
os.unlink(file_path2)
def handle_system_commands(my, snapshot, files, file_objects, mode, md5s, source_paths=[], file_sizes=[]):
'''move the tmp files in the appropriate directory'''
# if mode is local then nothing happens here
if mode == 'local':
return
sobject = snapshot.get_sobject()
# inplace mode does not move the file. It just registers the file
# object
if mode == 'inplace':
for i, file in enumerate(files):
file_object = file_objects[i]
to_name = file_object.get_full_file_name()
to_path = file
# This is handled in create_file_types
#file_type = snapshot.get_type_by_file_name(to_name)
#file_object.set_value('type', file_type)
if not os.path.isdir(to_path):
md5_checksum = None
if md5s:
md5_checksum = md5s[i]
if not md5_checksum:
md5_checksum = File.get_md5(to_path)
if md5_checksum:
file_object.set_value("md5", md5_checksum)
file_object.commit(triggers=False)
return
for i, file in enumerate(files):
file_object = file_objects[i]
to_name = file_object.get_full_file_name()
file_type = snapshot.get_type_by_file_name(to_name)
lib_dir = snapshot.get_lib_dir(file_type=file_type, file_object=file_object)
# it should have been created in postprocess_snapshot
System().makedirs(lib_dir)
to_path = "%s/%s" % (lib_dir, to_name )
#print "path: ", i, files[i]
#print to_path, os.path.exists(to_path)
# first make sure that the to path does not exist, if so, just skip
if os.path.exists(to_path) and mode not in ['inplace','preallocate']:
raise CheckinException('This path [%s] already exists'%to_path)
# add the file
try:
# inplace undo used to not touch the file,
# now it will be moved to cache on undo
io_action = True
if mode in ['preallocate']:
io_action = False
if mode == 'move':
FileUndo.move( source_paths[i], to_path )
#elif mode == 'copy': # was free_copy
#FileUndo.create( source_paths[i], to_path, io_action=io_action )
# make it look like the files was created in the repository
else: # mode ='create'
md5 = file_object.get_value("md5")
st_size = file_object.get_value("st_size")
rel_dir = file_object.get_value("relative_dir")
if mode == 'copy':
io_action = 'copy'
src_path = source_paths[i]
else:
src_path = files[i]
file_name = to_name
rel_path = "%s/%s" % (rel_dir, file_name)
FileUndo.create( src_path, to_path, io_action=io_action, extra={ "md5": md5, "st_size": st_size, "rel_path": rel_path } )
except IOError, e:
raise CheckinException('IO Error occurred. %s' %e.__str__())
# check to see that the file exists.
if not os.path.exists( to_path ):
if mode in ["inplace", "preallocate"]:
raise CheckinException("File not found in repo at [%s]" % to_path )
else:
raise CheckinException("Failed move [%s] to [%s]" % \
(files[i], to_path) )
file_object.set_value('type', file_type)
if not os.path.isdir(to_path):
md5_checksum = None
if md5s:
md5_checksum = md5s[i]
if not md5_checksum:
md5_checksum = File.get_md5(to_path)
if md5_checksum:
file_object.set_value("md5", md5_checksum)
file_object.commit(triggers=False)
def execute(my):
# extra data from the file check-in
commit = my.kwargs.get("commit")
if commit in ['false', False]:
commit = False
else:
commit = True
# metadata can only be associated with a single file on the
# snapshot.
files = my.kwargs.get("files")
file_objects = my.kwargs.get("file_objects")
snapshot_metadata = None
parser_type = my.kwargs.get("parser")
for i, file in enumerate(files):
path = file
ext = File.get_extension(path)
file_object = file_objects[i]
if not os.path.exists(path):
continue
elif parser_type != "auto" and parser_type != "true":
pass
elif ext in File.VIDEO_EXT:
parser_type = "FFMPEG"
elif ext in File.NORMAL_EXT:
continue
else:
if HAS_IMAGEMAGICK:
parser_type = "ImageMagick"
else:
parser_type = "PIL"
metadata = {}
if parser_type == "FFMPEG":
parser = FFProbeMetadataParser(path=path)
metadata = parser.get_metadata()
elif parser_type == "ImageMagick":
parser = ImageMagickMetadataParser(path=path)
metadata = parser.get_metadata()
else:
parser = PILMetadataParser(path=path)
metadata = parser.get_metadata()
metadata['__parser__'] = parser_type
"""
if os.path.exists(path):
try:
parser = ImageMagickMetadataParser(path=path)
metadata = parser.get_metadata()
except:
try:
#parser = PILMetadataParser(path=path)
parser = FFProbeMetadataParser(path=path)
metadata = parser.get_metadata()
except:
metadata = {}
else:
metadata = {}
"""
# add some default metadata
basename = os.path.basename(path)
path = file_object.get_lib_path()
dirname = os.path.dirname(path)
parts = basename.split(".")
ext = parts[-1]
try:
# occasionally, people will put frame numbers as the last
# part
ext = int(ext)
ext = parts[-2]
except:
pass
metadata['Ext'] = ext
metadata['Basename'] = basename
metadata['Dirname'] = dirname
if metadata and not file_object.get_value("metadata"):
file_object.add_metadata(metadata, replace=True)
searchable = my.get_searchable(metadata)
file_object.set_value("metadata_search", searchable)
file_object.commit()
if i == 0:
snapshot = my.kwargs.get("snapshot")
snapshot.add_metadata(metadata, replace=True)
if commit:
snapshot.commit()
def handle_system_commands(my,
snapshot,
files,
file_objects,
mode,
md5s,
source_paths=[],
file_sizes=[]):
'''move the tmp files in the appropriate directory'''
# if mode is local then nothing happens here
if mode == 'local':
return
sobject = snapshot.get_sobject()
# inplace mode does not move the file. It just registers the file
# object
if mode == 'inplace':
for i, file in enumerate(files):
file_object = file_objects[i]
to_name = file_object.get_full_file_name()
to_path = file
# This is handled in create_file_types
#file_type = snapshot.get_type_by_file_name(to_name)
#file_object.set_value('type', file_type)
if not os.path.isdir(to_path):
md5_checksum = None
if md5s:
md5_checksum = md5s[i]
if not md5_checksum:
md5_checksum = File.get_md5(to_path)
if md5_checksum:
file_object.set_value("md5", md5_checksum)
file_object.commit(triggers=False)
return
for i, file in enumerate(files):
file_object = file_objects[i]
to_name = file_object.get_full_file_name()
file_type = snapshot.get_type_by_file_name(to_name)
lib_dir = snapshot.get_lib_dir(file_type=file_type,
file_object=file_object)
# it should have been created in postprocess_snapshot
System().makedirs(lib_dir)
to_path = "%s/%s" % (lib_dir, to_name)
#print "path: ", i, files[i]
#print to_path, os.path.exists(to_path)
# first make sure that the to path does not exist, if so, just skip
if os.path.exists(to_path) and mode not in [
'inplace', 'preallocate'
]:
raise CheckinException('This path [%s] already exists' %
to_path)
# add the file
try:
# inplace undo used to not touch the file,
# now it will be moved to cache on undo
io_action = True
if mode in ['preallocate']:
io_action = False
if mode == 'move':
FileUndo.move(source_paths[i], to_path)
#elif mode == 'copy': # was free_copy
#FileUndo.create( source_paths[i], to_path, io_action=io_action )
# make it look like the files was created in the repository
else: # mode ='create'
md5 = file_object.get_value("md5")
st_size = file_object.get_value("st_size")
rel_dir = file_object.get_value("relative_dir")
if mode == 'copy':
io_action = 'copy'
src_path = source_paths[i]
else:
src_path = files[i]
file_name = to_name
rel_path = "%s/%s" % (rel_dir, file_name)
FileUndo.create(src_path,
to_path,
io_action=io_action,
extra={
"md5": md5,
"st_size": st_size,
"rel_path": rel_path
})
except IOError, e:
raise CheckinException('IO Error occurred. %s' % e.__str__())
# check to see that the file exists.
if not os.path.exists(to_path):
if mode in ["inplace", "preallocate"]:
raise CheckinException("File not found in repo at [%s]" %
to_path)
else:
raise CheckinException("Failed move [%s] to [%s]" % \
(files[i], to_path) )
file_object.set_value('type', file_type)
if not os.path.isdir(to_path):
md5_checksum = None
if md5s:
md5_checksum = md5s[i]
if not md5_checksum:
md5_checksum = File.get_md5(to_path)
if md5_checksum:
file_object.set_value("md5", md5_checksum)
file_object.commit(triggers=False)
def handle_dir_or_item(my, item_div, dirname, basename):
spath = "%s/%s" % (dirname, basename)
fspath = "%s/%s" % (dirname, File.get_filesystem_name(basename))
md5 = my.md5s.get(fspath)
changed = False
context = None
error_msg = None
snapshot = None
file_obj = my.checked_in_paths.get(fspath)
if not file_obj:
if fspath.startswith(my.base_dir):
rel = fspath.replace("%s/" % my.base_dir, "")
file_obj = my.checked_in_paths.get(rel)
if file_obj != None:
snapshot_code = file_obj.get_value("snapshot_code")
snapshot = my.snapshots_dict.get(snapshot_code)
if not snapshot:
# last resort
snapshot = file_obj.get_parent()
if snapshot:
context = snapshot.get_value("context")
item_div.add_attr("spt_snapshot_code", snapshot.get_code())
snapshot_md5 = file_obj.get_value("md5")
item_div.add_attr("spt_md5", snapshot_md5)
item_div.add_attr("title", "Checked-in as: %s" % file_obj.get_value("file_name"))
if md5 and md5 != snapshot_md5:
item_div.add_class("spt_changed")
changed = True
else:
error_msg = 'snapshot not found'
status = None
if file_obj != None:
if changed:
check = IconWdg( "Checked-In", IconWdg.ERROR, width=12 )
status = "changed"
else:
check = IconWdg( "Checked-In", IconWdg.CHECK, width=12 )
status = "same"
item_div.add_color("color", "color", [0, 0, 50])
else:
check = None
item_div.add_style("opacity: 0.8")
status = "unversioned"
if check:
item_div.add(check)
check.add_style("float: left")
check.add_style("margin-left: -16px")
check.add_style("margin-top: 4px")
# add the file name
filename_div = DivWdg()
item_div.add(filename_div)
filename_div.add(basename)
file_info_div = None
if snapshot and status != 'unversioned':
file_info_div = SpanWdg()
filename_div.add(file_info_div)
if error_msg:
filename_div.add(' (%s)'%error_msg)
filename_div.add_style("float: left")
filename_div.add_style("overflow: hidden")
filename_div.add_style("width: 65%")
# DEPRECATED
from pyasm.widget import CheckboxWdg, TextWdg, SelectWdg, HiddenWdg
checkbox = CheckboxWdg("check")
checkbox.add_style("display: none")
checkbox.add_class("spt_select")
checkbox.add_style("float: right")
checkbox.add_style("margin-top: 1px")
item_div.add(checkbox)
subcontext_val = ''
cat_input = None
is_select = True
if my.context_options:
context_sel = SelectWdg("context")
context_sel.add_attr('title', 'context')
context_sel.set_option("show_missing", False)
context_sel.set_option("values", my.context_options)
item_div.add_attr("spt_context", my.context_options[0])
cat_input = context_sel
input_cls = 'spt_context'
else:
if my.subcontext_options in [['(main)'], ['(auto)'] , []]:
is_select = False
#subcontext = TextWdg("subcontext")
subcontext = HiddenWdg("subcontext")
subcontext.add_class("spt_subcontext")
elif my.subcontext_options == ['(text)']:
is_select = False
subcontext = TextWdg("subcontext")
subcontext.add_class("spt_subcontext")
else:
is_select = True
subcontext = SelectWdg("subcontext")
subcontext.set_option("show_missing", False)
subcontext.set_option("values", my.subcontext_options)
#subcontext.add_empty_option("----")
cat_input = subcontext
input_cls = 'spt_subcontext'
if my.subcontext_options == ['(main)'] or my.subcontext_options == ['(auto)']:
subcontext_val = my.subcontext_options[0]
subcontext.set_value(subcontext_val)
item_div.add_attr("spt_subcontext", subcontext_val)
elif context:
parts = context.split("/")
if len(parts) > 1:
# get the actual subcontext value
subcontext_val = "/".join(parts[1:])
# identify a previous "auto" check-in and preselect the item in the select
if is_select and subcontext_val not in my.subcontext_options:
subcontext_val = '(auto)'
elif isinstance(cat_input, HiddenWdg):
subcontext_val = ''
# the Text field will adopt the subcontext value of the last check-in
subcontext.set_value(subcontext_val)
item_div.add_attr("spt_subcontext", subcontext_val)
else:
if is_select:
if my.subcontext_options:
subcontext_val = my.subcontext_options[0]
#subcontext_val = '(auto)'
cat_input.set_value(subcontext_val)
else:
subcontext_val = ''
item_div.add_attr("spt_subcontext", subcontext_val)
item_div.add(cat_input)
cat_input.add_behavior( {
'type': 'click_up',
'propagate_evt': False,
'cbjs_action': '''
bvr.src_el.focus();
'''
} )
cat_input.add_style("display: none")
cat_input.add_class("spt_subcontext")
cat_input.add_style("float: right")
cat_input.add_style("width: 50px")
cat_input.add_style("margin-top: -1px")
cat_input.add_style("font-size: 10px")
cat_input.add_style("height: 16px")
# we depend on the attribute cuz sometimes we go by the initialized value
# since they are not drawn
cat_input.add_behavior( {
'type': 'change',
'cbjs_action': '''
var el = bvr.src_el.getParent('.spt_dir_list_item');
el.setAttribute("%s", bvr.src_el.value);
''' %input_cls
} )
if file_info_div:
if subcontext_val in ['(auto)','(main)', '']:
file_info_div.add(" <i style='font-size: 9px; opacity: 0.6'>(v%s)</i>" % snapshot.get_value("version") )
else:
file_info_div.add(" <i style='font-size: 9px; opacity: 0.6'>(v%s - %s)</i>" % (snapshot.get_value("version"), subcontext_val) )
def _test_inplace_checkin(self):
# create a new test.txt file
tmp_dir = Environment.get_tmp_dir()
dir = "%s/temp" % tmp_dir
if not os.path.exists(dir):
os.makedirs(dir)
file_path = "%s/test_inplace.txt" % dir
if os.path.exists(file_path):
os.unlink(file_path)
file = open(file_path, 'w')
file.write("whatever")
file.close()
# inplace checkin: tell tactic that this is the correct path
mode = "inplace"
base_dir = tmp_dir
context = "inplace"
checkin = FileCheckin(self.person,
file_path,
context=context,
mode=mode)
checkin.execute()
snapshot = checkin.get_snapshot()
file_code = snapshot.get_file_code_by_type("main")
file_object = File.get_by_code(file_code)
relative_dir = file_object.get_value("relative_dir")
# The relative dir is empty if the file is outside the repository
self.assertEquals("", relative_dir)
lib_dir = snapshot.get_lib_dir(file_type="main")
file_name = snapshot.get_file_name_by_type("main")
lib_path = "%s/%s" % (lib_dir, file_name)
self.assertEquals(True, os.path.exists(lib_path))
self.assertEquals(file_path, lib_path)
# check in a file alredy in the repository
asset_dir = Config.get_value("checkin",
"asset_base_dir",
sub_key="default")
file_path2 = "%s/unittest/text.txt" % asset_dir
file = open(file_path2, 'w')
file.write("whatever")
file.close()
checkin = FileCheckin(self.person,
file_path2,
context=context,
mode=mode)
checkin.execute()
snapshot = checkin.get_snapshot()
file_code = snapshot.get_file_code_by_type("main")
file_object = File.get_by_code(file_code)
# check that the relative dir is as expected
relative_dir = file_object.get_value("relative_dir")
self.assertEquals(relative_dir, "unittest")
# check that the path returned correctly
lib_path = snapshot.get_path_by_type("main")
self.assertEquals(file_path2, lib_path)
if os.path.exists(file_path):
os.unlink(file_path)
if os.path.exists(file_path2):
os.unlink(file_path2)
