Exemple #1
0
    def execute(my):


        filenames = my.kwargs.get("filenames")

        upload_dir = Environment.get_upload_dir()
        base_dir = upload_dir


        search_type = my.kwargs.get("search_type")
        key = my.kwargs.get("key")
        relative_dir = my.kwargs.get("relative_dir")
        if not relative_dir:
            project_code = Project.get_project_code()
            search_type_obj = SearchType.get(search_type)
            table = search_type_obj.get_table()
            relative_dir = "%s/%s" % (project_code, table)



        server = TacticServerStub.get()

        parent_key = my.kwargs.get("parent_key")
        category = my.kwargs.get("category")
        keywords = my.kwargs.get("keywords")
        extra_data = my.kwargs.get("extra_data")
        if extra_data:
            extra_data = jsonloads(extra_data)
        else:
            extra_data = {}


        # TODO: use this to generate a category
        category_script_path = my.kwargs.get("category_script_path")
        """
        ie:
            from pyasm.checkin import ExifMetadataParser
            parser = ExifMetadataParser(path=file_path)
            tags = parser.get_metadata()

            date = tags.get("EXIF DateTimeOriginal")
            return date.split(" ")[0]
        """
 
    

    
        if not SearchType.column_exists(search_type, "name"):
            raise TacticException('The Ingestion puts the file name into the name column which is the minimal requirement. Please first create a "name" column for this sType.')

        for count, filename in enumerate(filenames):

            # first see if this sobjects still exists
            search = Search(search_type)
            search.add_filter("name", filename)
            if relative_dir and search.column_exists("relative_dir"):
                search.add_filter("relative_dir", relative_dir)
            sobject = search.get_sobject()

            # else create a new one
            if not sobject:
                sobject = SearchType.create(search_type)
                sobject.set_value("name", filename)
                if relative_dir and sobject.column_exists("relative_dir"):
                    sobject.set_value("relative_dir", relative_dir)



            # extract metadata
            file_path = "%s/%s" % (base_dir, File.get_filesystem_name(filename))

            # TEST: convert on upload
            try:
                convert = my.kwargs.get("convert")
                if convert:
                    message_key = "IngestConvert001"
                    cmd = ConvertCbk(**convert)
                    cmd.execute()
            except Exception, e:
                print "WARNING: ", e


            if not os.path.exists(file_path):
                raise Exception("Path [%s] does not exist" % file_path)

            # get the metadata from this image
            if SearchType.column_exists(search_type, "relative_dir"):
                if category and category not in ['none', None]:
                    from pyasm.checkin import ExifMetadataParser
                    parser = ExifMetadataParser(path=file_path)
                    tags = parser.get_metadata()

                    date = tags.get("EXIF DateTimeOriginal")
                    if not date:
                        date_str = "No-Date"
                    else:
                        date_str = str(date)
                        # this can't be parsed correctly by dateutils
                        parts = date_str.split(" ")
                        date_str = parts[0].replace(":", "-")
                        date_str = "%s %s" % (date_str, parts[1])

                        from dateutil import parser 
                        orig_date = parser.parse(date_str)

                        if category == "by_day":
                            date_str = orig_date.strftime("%Y/%Y-%m-%d")
                        elif category == "by_month":
                            date_str = orig_date.strftime("%Y-%m")
                        elif category == "by_week":
                            date_str = orig_date.strftime("%Y/Week-%U")

                    full_relative_dir = "%s/%s" % (relative_dir, date_str)
                    sobject.set_value("relative_dir", full_relative_dir)

            if parent_key:
                parent = Search.get_by_search_key(parent_key)
                if parent:
                    sobject.set_sobject_value(sobject)




            if SearchType.column_exists(search_type, "keywords"):
                if keywords:
                    sobject.set_value("keywords", keywords)

            for key, value in extra_data.items():
                if SearchType.column_exists(search_type, key):
                    sobject.set_value(key, value)


            """
            if category:

                if SearchType.column_exists(search_type, "category"):
                    sobject.set_value("category", category)


                if SearchType.column_exists(search_type, "relative_dir"):
                    full_relative_dir = "%s/%s" % (relative_dir, category)
                    sobject.set_value("relative_dir", category)
            """




            sobject.commit()
            search_key = sobject.get_search_key()

            # use API to check in file

            process = my.kwargs.get("process")
            if not process:
                process = "publish"

            if process == "icon":
                context = "icon"
            else:
                context = "%s/%s" % (process, filename)
            

            server.simple_checkin(search_key, context, filename, mode='uploaded')
            percent = int((float(count)+1) / len(filenames)*100)
            print "checking in: ", filename, percent

            msg = {
                'progress': percent,
                'description': 'Checking in file [%s]' % filename,
            }

            server.log_message(key, msg, status="in progress")
Exemple #2
0
    def execute(my):

        filenames = my.kwargs.get("filenames")

        upload_dir = Environment.get_upload_dir()
        base_dir = upload_dir

        update_mode = my.kwargs.get("update_mode")
        search_type = my.kwargs.get("search_type")
        key = my.kwargs.get("key")
        relative_dir = my.kwargs.get("relative_dir")
        if not relative_dir:
            project_code = Project.get_project_code()
            search_type_obj = SearchType.get(search_type)
            table = search_type_obj.get_table()
            relative_dir = "%s/%s" % (project_code, table)

        server = TacticServerStub.get()

        parent_key = my.kwargs.get("parent_key")
        category = my.kwargs.get("category")
        keywords = my.kwargs.get("keywords")
        update_data = my.kwargs.get("update_data")
        extra_data = my.kwargs.get("extra_data")
        if extra_data:
            extra_data = jsonloads(extra_data)
        else:
            extra_data = {}

        # TODO: use this to generate a category
        category_script_path = my.kwargs.get("category_script_path")
        """
        ie:
            from pyasm.checkin import ExifMetadataParser
            parser = ExifMetadataParser(path=file_path)
            tags = parser.get_metadata()

            date = tags.get("EXIF DateTimeOriginal")
            return date.split(" ")[0]
        """

        if not SearchType.column_exists(search_type, "name"):
            raise TacticException(
                'The Ingestion puts the file name into the name column which is the minimal requirement. Please first create a "name" column for this sType.'
            )

        input_prefix = update_data.get('input_prefix')
        non_seq_filenames = []

        # For sequence mode, take all filenames, and regenerate the filenames based on the function "find_sequences"
        if update_mode == "sequence":

            non_seq_filenames_dict, seq_digit_length = my.find_sequences(
                filenames)
            # non_seq_filenames is a list of filenames that are stored in the None key,
            # which are the filenames that are not part of a sequence, or does not contain
            # a sequence pattern.
            non_seq_filenames = non_seq_filenames_dict[None]

            # delete the None key from list so filenames can be used in the latter for loop
            del non_seq_filenames_dict[None]
            filenames = non_seq_filenames_dict.keys()
            if filenames == []:
                raise TacticException(
                    'No sequences are found in files. Please follow the pattern of [filename] + [digits] + [file extension (optional)]. Examples: [abc_1001.png, abc_1002.png] [abc.1001.mp3, abc.1002.mp3] [abc_100_1001.png, abc_100_1002.png]'
                )

        for count, filename in enumerate(filenames):
            # Check if files should be updated.
            # If so, attempt to find one to update.
            # If more than one is found, do not update.
            if update_mode in ["true", "True"]:
                # first see if this sobjects still exists
                search = Search(search_type)
                search.add_filter("name", filename)
                if relative_dir and search.column_exists("relative_dir"):
                    search.add_filter("relative_dir", relative_dir)
                sobjects = search.get_sobjects()
                if len(sobjects) > 1:
                    sobject = None
                elif len(sobjects) == 1:
                    sobject = sobjects[0]
                else:
                    sobject = None

            elif update_mode == "sequence":
                if not FileGroup.is_sequence(filename):
                    raise TacticException(
                        'Please modify sequence naming to have at least three digits.'
                    )
                search = Search(search_type)
                search.add_filter("name", filename)

                if relative_dir and search.column_exists("relative_dir"):
                    search.add_filter("relative_dir", relative_dir)
                sobjects = search.get_sobjects()
                if sobjects:
                    sobject = sobjects[0]
                else:
                    sobject = None

            else:
                sobject = None

            # Create a new file
            if not sobject:
                sobject = SearchType.create(search_type)
                sobject.set_value("name", filename)
                if relative_dir and sobject.column_exists("relative_dir"):
                    sobject.set_value("relative_dir", relative_dir)

            # extract metadata
            #file_path = "%s/%s" % (base_dir, File.get_filesystem_name(filename))
            if update_mode == "sequence":
                first_filename = non_seq_filenames_dict.get(filename)[0]
                last_filename = non_seq_filenames_dict.get(filename)[-1]
                file_path = "%s/%s" % (base_dir, first_filename)
            else:
                file_path = "%s/%s" % (base_dir, filename)

            # TEST: convert on upload
            try:
                convert = my.kwargs.get("convert")
                if convert:
                    message_key = "IngestConvert001"
                    cmd = ConvertCbk(**convert)
                    cmd.execute()
            except Exception, e:
                print "WARNING: ", e

            if not os.path.exists(file_path):
                raise Exception("Path [%s] does not exist" % file_path)

            # get the metadata from this image
            if SearchType.column_exists(search_type, "relative_dir"):
                if category and category not in ['none', None]:
                    from pyasm.checkin import ExifMetadataParser
                    parser = ExifMetadataParser(path=file_path)
                    tags = parser.get_metadata()

                    date = tags.get("EXIF DateTimeOriginal")
                    if not date:
                        date_str = "No-Date"
                    else:
                        date_str = str(date)
                        # this can't be parsed correctly by dateutils
                        parts = date_str.split(" ")
                        date_str = parts[0].replace(":", "-")
                        date_str = "%s %s" % (date_str, parts[1])

                        from dateutil import parser
                        orig_date = parser.parse(date_str)

                        if category == "by_day":
                            date_str = orig_date.strftime("%Y/%Y-%m-%d")
                        elif category == "by_month":
                            date_str = orig_date.strftime("%Y-%m")
                        elif category == "by_week":
                            date_str = orig_date.strftime("%Y/Week-%U")

                    full_relative_dir = "%s/%s" % (relative_dir, date_str)
                    sobject.set_value("relative_dir", full_relative_dir)

            if parent_key:
                parent = Search.get_by_search_key(parent_key)
                if parent:
                    sobject.set_sobject_value(sobject)

            for key, value in update_data.items():
                if input_prefix:
                    key = key.replace('%s|' % input_prefix, '')
                if SearchType.column_exists(search_type, key):
                    if value:
                        sobject.set_value(key, value)
            """
            if SearchType.column_exists(search_type, "keywords"):
                if keywords:
                    sobject.set_value("keywords", keywords)

            """
            for key, value in extra_data.items():
                if SearchType.column_exists(search_type, key):
                    sobject.set_value(key, value)
            """
            if category:

                if SearchType.column_exists(search_type, "category"):
                    sobject.set_value("category", category)


                if SearchType.column_exists(search_type, "relative_dir"):
                    full_relative_dir = "%s/%s" % (relative_dir, category)
                    sobject.set_value("relative_dir", category)
            """

            sobject.commit()
            search_key = sobject.get_search_key()

            # use API to check in file

            process = my.kwargs.get("process")
            if not process:
                process = "publish"

            if process == "icon":
                context = "icon"
            else:
                context = "%s/%s" % (process, filename.lower())

            if update_mode == "sequence":

                pattern_expr = re.compile('^.*(\d{%d})\..*$' %
                                          seq_digit_length)

                m_first = re.match(pattern_expr, first_filename)
                m_last = re.match(pattern_expr, last_filename)
                # for files without extension
                # abc_1001, abc.1123_1001
                if not m_first:
                    no_ext_expr = re.compile('^.*(\d{%d})$' % seq_digit_length)
                    m_first = re.match(no_ext_expr, first_filename)
                    m_last = re.match(no_ext_expr, last_filename)

                # using second last index , to grab the set right before file type
                groups_first = m_first.groups()
                if groups_first:
                    range_start = int(m_first.groups()[0])

                groups_last = m_last.groups()
                if groups_last:
                    range_end = int(m_last.groups()[0])

                file_range = '%s-%s' % (range_start, range_end)

                file_path = "%s/%s" % (base_dir, filename)
                server.group_checkin(search_key,
                                     context,
                                     file_path,
                                     file_range,
                                     mode='uploaded')
            else:
                server.simple_checkin(search_key,
                                      context,
                                      filename,
                                      mode='uploaded')
            percent = int((float(count) + 1) / len(filenames) * 100)
            print "checking in: ", filename, percent

            msg = {
                'progress': percent,
                'description': 'Checking in file [%s]' % filename,
            }

            server.log_message(key, msg, status="in progress")
    def get_display(my):

        search_key = my.kwargs.get("search_key")
        path = my.kwargs.get("path")
        parser_str = my.kwargs.get("parser")

        from pyasm.checkin import PILMetadataParser, ImageMagickMetadataParser, ExifMetadataParser

        if parser_str == "EXIF":
            parser = ExifMetadataParser(path=path)
        elif parser_str == "ImageMagick":
            parser = ImageMagickMetadataParser(path=path)
        elif parser_str == "PIL":
            parser = PILMetadataParser(path=path)
        else:
            parser = None

        if parser:
            metadata = parser.get_metadata()
        else:
            metadata = {}


        top = my.top
        top.add_color("background", "background")


        table = Table()
        table.set_max_width()
        top.add(table)
        table.set_unique_id()
        table.add_border()

        table.add_smart_styles("spt_cell", {
            'padding': '3px'
        } )



        tr = table.add_row()
        tr.add_gradient("background", "background3")
        th = table.add_header("Property")
        th.add_style("min-width: 200px")
        th.add_style("padding: 5px")
        th = table.add_header("Value")
        th.add_style("min-width: 400px")
        th.add_style("padding: 5px")

        keys = metadata.get("__keys__")
        if not keys:
            keys = metadata.keys()

        empty = False
        if not keys:
            empty = True
            keys = ['','','','','','','']
            table.add_smart_styles("spt_cell", {
                'height': '20px'
            } )


        for i, key in enumerate(keys):
            value = metadata.get(key)

            title = Common.get_display_title(key)

            tr = table.add_row()

            if i % 2:
                tr.add_color("background", "background")
                tr.add_color("color", "color")
            else:
                tr.add_color("background", "background", -8)
                tr.add_color("color", "color")

            td = table.add_cell()
            td.add_class("spt_cell")
            td.add(title)

            td = table.add_cell()
            td.add_class("spt_cell")

            if len(str(value)) > 500:
                inside = DivWdg()
                td.add(inside)
                value = value[:500]
                inside.add(value)
                inside.add_style("max-width: 600px")
            else:
                td.add(value)


        if empty:
            div = DivWdg()
            top.add(div)
            div.add_style("height: 30px")
            div.add_style("width: 150px")
            div.add_style("margin-top: -110px")
            div.center()
            div.add("<b>No Metadata</b>")
            div.add_border()
            div.add_color("background", "background3")
            div.add_color("color", "color3")
            div.add_style("padding: 20px")
            div.add_style("text-align: center")

            top.add_style("min-height: 200px")

        return top
Exemple #4
0
    def execute(my):

        filenames = my.kwargs.get("filenames")

        upload_dir = Environment.get_upload_dir()
        base_dir = upload_dir

        search_type = my.kwargs.get("search_type")
        key = my.kwargs.get("key")
        relative_dir = my.kwargs.get("relative_dir")
        if not relative_dir:
            project_code = Project.get_project_code()
            search_type_obj = SearchType.get(search_type)
            table = search_type_obj.get_table()
            relative_dir = "%s/%s" % (project_code, table)

        server = TacticServerStub.get()

        parent_key = my.kwargs.get("parent_key")
        category = my.kwargs.get("category")
        keywords = my.kwargs.get("keywords")
        update_data = my.kwargs.get("update_data")
        extra_data = my.kwargs.get("extra_data")
        if extra_data:
            extra_data = jsonloads(extra_data)
        else:
            extra_data = {}

        # TODO: use this to generate a category
        category_script_path = my.kwargs.get("category_script_path")
        """
        ie:
            from pyasm.checkin import ExifMetadataParser
            parser = ExifMetadataParser(path=file_path)
            tags = parser.get_metadata()

            date = tags.get("EXIF DateTimeOriginal")
            return date.split(" ")[0]
        """

        if not SearchType.column_exists(search_type, "name"):
            raise TacticException(
                'The Ingestion puts the file name into the name column which is the minimal requirement. Please first create a "name" column for this sType.'
            )

        input_prefix = update_data.get('input_prefix')

        for count, filename in enumerate(filenames):

            # first see if this sobjects still exists
            search = Search(search_type)
            search.add_filter("name", filename)
            if relative_dir and search.column_exists("relative_dir"):
                search.add_filter("relative_dir", relative_dir)
            sobject = search.get_sobject()

            # else create a new one
            if not sobject:
                sobject = SearchType.create(search_type)
                sobject.set_value("name", filename)
                if relative_dir and sobject.column_exists("relative_dir"):
                    sobject.set_value("relative_dir", relative_dir)

            # extract metadata
            #file_path = "%s/%s" % (base_dir, File.get_filesystem_name(filename))
            file_path = "%s/%s" % (base_dir, filename)

            # TEST: convert on upload
            try:
                convert = my.kwargs.get("convert")
                if convert:
                    message_key = "IngestConvert001"
                    cmd = ConvertCbk(**convert)
                    cmd.execute()
            except Exception, e:
                print "WARNING: ", e

            if not os.path.exists(file_path):
                raise Exception("Path [%s] does not exist" % file_path)

            # get the metadata from this image
            if SearchType.column_exists(search_type, "relative_dir"):
                if category and category not in ['none', None]:
                    from pyasm.checkin import ExifMetadataParser
                    parser = ExifMetadataParser(path=file_path)
                    tags = parser.get_metadata()

                    date = tags.get("EXIF DateTimeOriginal")
                    if not date:
                        date_str = "No-Date"
                    else:
                        date_str = str(date)
                        # this can't be parsed correctly by dateutils
                        parts = date_str.split(" ")
                        date_str = parts[0].replace(":", "-")
                        date_str = "%s %s" % (date_str, parts[1])

                        from dateutil import parser
                        orig_date = parser.parse(date_str)

                        if category == "by_day":
                            date_str = orig_date.strftime("%Y/%Y-%m-%d")
                        elif category == "by_month":
                            date_str = orig_date.strftime("%Y-%m")
                        elif category == "by_week":
                            date_str = orig_date.strftime("%Y/Week-%U")

                    full_relative_dir = "%s/%s" % (relative_dir, date_str)
                    sobject.set_value("relative_dir", full_relative_dir)

            if parent_key:
                parent = Search.get_by_search_key(parent_key)
                if parent:
                    sobject.set_sobject_value(sobject)

            for key, value in update_data.items():
                if input_prefix:
                    key = key.replace('%s|' % input_prefix, '')
                if SearchType.column_exists(search_type, key):
                    if value:
                        sobject.set_value(key, value)
            """
            if SearchType.column_exists(search_type, "keywords"):
                if keywords:
                    sobject.set_value("keywords", keywords)

            """
            for key, value in extra_data.items():
                if SearchType.column_exists(search_type, key):
                    sobject.set_value(key, value)
            """
            if category:

                if SearchType.column_exists(search_type, "category"):
                    sobject.set_value("category", category)


                if SearchType.column_exists(search_type, "relative_dir"):
                    full_relative_dir = "%s/%s" % (relative_dir, category)
                    sobject.set_value("relative_dir", category)
            """

            sobject.commit()
            search_key = sobject.get_search_key()

            # use API to check in file

            process = my.kwargs.get("process")
            if not process:
                process = "publish"

            if process == "icon":
                context = "icon"
            else:
                context = "%s/%s" % (process, filename)

            server.simple_checkin(search_key,
                                  context,
                                  filename,
                                  mode='uploaded')
            percent = int((float(count) + 1) / len(filenames) * 100)
            print "checking in: ", filename, percent

            msg = {
                'progress': percent,
                'description': 'Checking in file [%s]' % filename,
            }

            server.log_message(key, msg, status="in progress")
Exemple #5
0
    def execute(my):

        filenames = my.kwargs.get("filenames")

        upload_dir = Environment.get_upload_dir()
        base_dir = upload_dir

        update_mode = my.kwargs.get("update_mode")
        search_type = my.kwargs.get("search_type")
        key = my.kwargs.get("key")
        relative_dir = my.kwargs.get("relative_dir")
        if not relative_dir:
            project_code = Project.get_project_code()
            search_type_obj = SearchType.get(search_type)
            table = search_type_obj.get_table()
            relative_dir = "%s/%s" % (project_code, table)

        server = TacticServerStub.get()

        parent_key = my.kwargs.get("parent_key")
        category = my.kwargs.get("category")
        keywords = my.kwargs.get("keywords")
        update_data = my.kwargs.get("update_data")
        extra_data = my.kwargs.get("extra_data")
        if extra_data:
            extra_data = jsonloads(extra_data)
        else:
            extra_data = {}

        # TODO: use this to generate a category
        category_script_path = my.kwargs.get("category_script_path")
        """
        ie:
            from pyasm.checkin import ExifMetadataParser
            parser = ExifMetadataParser(path=file_path)
            tags = parser.get_metadata()

            date = tags.get("EXIF DateTimeOriginal")
            return date.split(" ")[0]
        """

        if not SearchType.column_exists(search_type, "name"):
            raise TacticException('The Ingestion puts the file name into the name column which is the minimal requirement. Please first create a "name" column for this sType.')

        input_prefix = update_data.get('input_prefix')
        non_seq_filenames = []

        # For sequence mode, take all filenames, and regenerate the filenames based on the function "find_sequences"
        if update_mode == "sequence":
            
            non_seq_filenames_dict, seq_digit_length = my.find_sequences(filenames)
            # non_seq_filenames is a list of filenames that are stored in the None key,
            # which are the filenames that are not part of a sequence, or does not contain
            # a sequence pattern.
            non_seq_filenames = non_seq_filenames_dict[None]
            
            # delete the None key from list so filenames can be used in the latter for loop
            del non_seq_filenames_dict[None]
            filenames = non_seq_filenames_dict.keys()
            if filenames == []:
                raise TacticException('No sequences are found in files. Please follow the pattern of [filename] + [digits] + [file extension (optional)]. Examples: [abc_1001.png, abc_1002.png] [abc.1001.mp3, abc.1002.mp3] [abc_100_1001.png, abc_100_1002.png]')

        for count, filename in enumerate(filenames):
        # Check if files should be updated. 
        # If so, attempt to find one to update.
        # If more than one is found, do not update.
            if update_mode in ["true", "True"]:
                # first see if this sobjects still exists
                search = Search(search_type)
                search.add_filter("name", filename)
                if relative_dir and search.column_exists("relative_dir"):
                    search.add_filter("relative_dir", relative_dir)
                sobjects = search.get_sobjects()
                if len(sobjects) > 1:
                    sobject = None
                elif len(sobjects) == 1:
                    sobject = sobjects[0]
                else:
                    sobject = None

            elif update_mode == "sequence":
                if not FileGroup.is_sequence(filename):
                    raise TacticException('Please modify sequence naming to have at least three digits.')
                search = Search(search_type)
                search.add_filter("name", filename)

                if relative_dir and search.column_exists("relative_dir"):
                    search.add_filter("relative_dir", relative_dir)
                sobjects = search.get_sobjects()
                if sobjects:
                    sobject = sobjects[0]
                else:
                    sobject = None

            else:
                sobject = None 

            # Create a new file
            if not sobject:
                sobject = SearchType.create(search_type)
                sobject.set_value("name", filename)
                if relative_dir and sobject.column_exists("relative_dir"):
                    sobject.set_value("relative_dir", relative_dir)



            # extract metadata
            #file_path = "%s/%s" % (base_dir, File.get_filesystem_name(filename))
            if update_mode == "sequence":
                first_filename = non_seq_filenames_dict.get(filename)[0]
                last_filename = non_seq_filenames_dict.get(filename)[-1]
                file_path = "%s/%s" % (base_dir, first_filename)
            else:
                file_path = "%s/%s" % (base_dir, filename)

            # TEST: convert on upload
            try:
                convert = my.kwargs.get("convert")
                if convert:
                    message_key = "IngestConvert001"
                    cmd = ConvertCbk(**convert)
                    cmd.execute()
            except Exception, e:
                print "WARNING: ", e


            if not os.path.exists(file_path):
                raise Exception("Path [%s] does not exist" % file_path)

            # get the metadata from this image
            if SearchType.column_exists(search_type, "relative_dir"):
                if category and category not in ['none', None]:
                    from pyasm.checkin import ExifMetadataParser
                    parser = ExifMetadataParser(path=file_path)
                    tags = parser.get_metadata()

                    date = tags.get("EXIF DateTimeOriginal")
                    if not date:
                        date_str = "No-Date"
                    else:
                        date_str = str(date)
                        # this can't be parsed correctly by dateutils
                        parts = date_str.split(" ")
                        date_str = parts[0].replace(":", "-")
                        date_str = "%s %s" % (date_str, parts[1])

                        from dateutil import parser 
                        orig_date = parser.parse(date_str)

                        if category == "by_day":
                            date_str = orig_date.strftime("%Y/%Y-%m-%d")
                        elif category == "by_month":
                            date_str = orig_date.strftime("%Y-%m")
                        elif category == "by_week":
                            date_str = orig_date.strftime("%Y/Week-%U")

                    full_relative_dir = "%s/%s" % (relative_dir, date_str)
                    sobject.set_value("relative_dir", full_relative_dir)

            if parent_key:
                parent = Search.get_by_search_key(parent_key)
                if parent:
                    sobject.set_sobject_value(sobject)

            for key, value in update_data.items():
                if input_prefix:
                    key = key.replace('%s|'%input_prefix, '')
                if SearchType.column_exists(search_type, key):
                    if value:
                        sobject.set_value(key, value)
            """
            if SearchType.column_exists(search_type, "keywords"):
                if keywords:
                    sobject.set_value("keywords", keywords)

            """
            for key, value in extra_data.items():
                if SearchType.column_exists(search_type, key):
                    sobject.set_value(key, value)

            """
            if category:

                if SearchType.column_exists(search_type, "category"):
                    sobject.set_value("category", category)


                if SearchType.column_exists(search_type, "relative_dir"):
                    full_relative_dir = "%s/%s" % (relative_dir, category)
                    sobject.set_value("relative_dir", category)
            """

            sobject.commit()
            search_key = sobject.get_search_key()

            # use API to check in file

            process = my.kwargs.get("process")
            if not process:
                process = "publish"

            if process == "icon":
                context = "icon"
            else:
                context = "%s/%s" % (process, filename.lower())
            
            if update_mode == "sequence":

                pattern_expr = re.compile('^.*(\d{%d})\..*$'%seq_digit_length)
                
                m_first = re.match(pattern_expr, first_filename)
                m_last = re.match(pattern_expr, last_filename)
                # for files without extension
                # abc_1001, abc.1123_1001
                if not m_first: 
                    no_ext_expr = re.compile('^.*(\d{%d})$'%seq_digit_length)
                    m_first = re.match(no_ext_expr, first_filename)
                    m_last = re.match(no_ext_expr, last_filename)

                # using second last index , to grab the set right before file type
                groups_first = m_first.groups()
                if groups_first:
                    range_start = int(m_first.groups()[0])
                    
                groups_last = m_last.groups()
                if groups_last:
                    range_end = int(m_last.groups()[0])

                file_range = '%s-%s' % (range_start, range_end)

                file_path = "%s/%s" % (base_dir, filename)
                server.group_checkin(search_key, context, file_path, file_range, mode='uploaded')
            else: 
                server.simple_checkin(search_key, context, filename, mode='uploaded')
            percent = int((float(count)+1) / len(filenames)*100)
            print "checking in: ", filename, percent

            msg = {
                'progress': percent,
                'description': 'Checking in file [%s]' % filename,
            }

            server.log_message(key, msg, status="in progress")