def check_sync(svn_check_in_dir, original_doc_dir):
for root, dirs, files in os.walk(original_doc_dir):
for name in files:
full_path = os.path.join(root, name)
simple_path = get_simple_path(original_doc_dir, full_path)
svn_doc_path = os.path.join(svn_check_in_dir, simple_path)
if not os.path.exists(svn_doc_path):
raise ValidationError, "'%s' didn't exist in svn_doc_path" % svn_doc_path
for name in dirs:
full_path = os.path.join(root, name)
simple_path = get_simple_path(original_doc_dir, full_path)
svn_doc_path = os.path.join(svn_check_in_dir, simple_path)
if not os.path.exists(svn_doc_path):
raise ValidationError, "'%s' didn't exist in svn_doc_path" % svn_doc_path
for root, dirs, files in os.walk(svn_check_in_dir):
if '.svn' in dirs:
dirs.remove('.svn')
for name in files:
full_path = os.path.join(root, name)
simple_path = get_simple_path(svn_check_in_dir, full_path)
original_doc_path = os.path.join(original_doc_dir, simple_path)
if not os.path.exists(original_doc_path):
raise ValidationError, "'%s' didn't exist in original_doc_path" % original_doc_path
for name in dirs:
full_path = os.path.join(root, name)
simple_path = get_simple_path(svn_check_in_dir, full_path)
original_doc_path = os.path.join(original_doc_dir, simple_path)
if not os.path.exists(original_doc_path):
raise ValidationError, "'%s' didn't exist in original_doc_path" % original_doc_path
def check_sync(svn_check_in_dir, original_doc_dir):
for root, dirs, files in os.walk(original_doc_dir):
for name in files:
full_path = os.path.join(root, name)
simple_path = get_simple_path(original_doc_dir, full_path)
svn_doc_path = os.path.join(svn_check_in_dir, simple_path)
if not os.path.exists(svn_doc_path):
raise ValidationError, "'%s' didn't exist in svn_doc_path" % svn_doc_path
for name in dirs:
full_path = os.path.join(root, name)
simple_path = get_simple_path(original_doc_dir, full_path)
svn_doc_path = os.path.join(svn_check_in_dir, simple_path)
if not os.path.exists(svn_doc_path):
raise ValidationError, "'%s' didn't exist in svn_doc_path" % svn_doc_path
for root, dirs, files in os.walk(svn_check_in_dir):
if '.svn' in dirs:
dirs.remove('.svn')
for name in files:
full_path = os.path.join(root, name)
simple_path = get_simple_path(svn_check_in_dir, full_path)
original_doc_path = os.path.join(original_doc_dir, simple_path)
if not os.path.exists(original_doc_path):
raise ValidationError, "'%s' didn't exist in original_doc_path" % original_doc_path
for name in dirs:
full_path = os.path.join(root, name)
simple_path = get_simple_path(svn_check_in_dir, full_path)
original_doc_path = os.path.join(original_doc_dir, simple_path)
if not os.path.exists(original_doc_path):
raise ValidationError, "'%s' didn't exist in original_doc_path" % original_doc_path
def remove_first(client, svn_check_in_dir, original_doc_dir, project):
old_py_revision = client.info(svn_check_in_dir).revision
dump_repos(filename=u'r%s.%s_remove_first' %
(old_py_revision.number, project.slug))
for root, dirs, files in os.walk(svn_check_in_dir):
# print 'root %s' % root
# print 'dirs %s' % dirs
# print 'files %s' % files
if '.svn' in dirs:
dirs.remove('.svn')
for name in files:
full_path = os.path.join(root, name)
simple_path = get_simple_path(svn_check_in_dir, full_path)
original_doc_path = os.path.join(original_doc_dir, simple_path)
if not os.path.exists(original_doc_path):
print '%s remove_first' % simple_path
client.remove(full_path, force=True)
for name in dirs:
full_path = os.path.join(root, name)
simple_path = get_simple_path(svn_check_in_dir, full_path)
original_doc_path = os.path.join(original_doc_dir, simple_path)
if not os.path.exists(original_doc_path):
print '%s remove_first' % simple_path
client.remove(full_path, force=True)
client.checkin(svn_check_in_dir, log_message='checkin for delete')
client.update(svn_check_in_dir)
def remove_first(client, svn_check_in_dir, original_doc_dir, project):
old_py_revision = client.info(svn_check_in_dir).revision
dump_repos(filename = u'r%s.%s_remove_first' % (old_py_revision.number, project.slug))
for root, dirs, files in os.walk(svn_check_in_dir):
# print 'root %s' % root
# print 'dirs %s' % dirs
# print 'files %s' % files
if '.svn' in dirs:
dirs.remove('.svn')
for name in files:
full_path = os.path.join(root, name)
simple_path = get_simple_path(svn_check_in_dir, full_path)
original_doc_path = os.path.join(original_doc_dir, simple_path)
if not os.path.exists(original_doc_path):
print '%s remove_first' % simple_path
client.remove(full_path, force=True)
for name in dirs:
full_path = os.path.join(root, name)
simple_path = get_simple_path(svn_check_in_dir, full_path)
original_doc_path = os.path.join(original_doc_dir, simple_path)
if not os.path.exists(original_doc_path):
print '%s remove_first' % simple_path
client.remove(full_path, force=True)
client.checkin(svn_check_in_dir, log_message = 'checkin for delete')
client.update(svn_check_in_dir)
def copy_files(original_doc_dir, svn_check_in_dir):
for root, dirs, files in os.walk(original_doc_dir):
for name in files:
full_path = os.path.join(root, name)
simple_path = get_simple_path(original_doc_dir, full_path)
svn_doc_path = os.path.join(svn_check_in_dir, simple_path)
if not os.path.exists(os.path.dirname(svn_doc_path)):
os.makedirs(os.path.dirname(svn_doc_path))
shutil.copy2(full_path, svn_doc_path)
def copy_files(original_doc_dir, svn_check_in_dir):
for root, dirs, files in os.walk(original_doc_dir):
for name in files:
full_path = os.path.join(root, name)
simple_path = get_simple_path(original_doc_dir, full_path)
svn_doc_path = os.path.join(svn_check_in_dir, simple_path)
if not os.path.exists(os.path.dirname(svn_doc_path)):
os.makedirs(os.path.dirname(svn_doc_path))
shutil.copy2(full_path, svn_doc_path)
def copy_files(original, dest, page=None):
if page:
original_path = os.path.join(original, page.path)
dest_path = os.path.join(dest, page.path)
print original_path
print dest_path
shutil.copy2(original_path, dest_path)
else:
if os.path.exists(dest):
shutil.rmtree(dest)
for root, dirs, files in os.walk(original):
if '.svn' in dirs:
dirs.remove('.svn')
for name in files:
original_path = os.path.join(root, name)
simple_path = get_simple_path(original, original_path)
dest_path = os.path.join(dest, simple_path)
if not os.path.exists(os.path.dirname(dest_path)):
os.makedirs(os.path.dirname(dest_path))
shutil.copy2(original_path, dest_path)
def copy_files(original, dest, page = None):
if page:
original_path = os.path.join(original, page.path)
dest_path = os.path.join(dest, page.path)
print original_path
print dest_path
shutil.copy2(original_path, dest_path)
else:
if os.path.exists(dest):
shutil.rmtree(dest)
for root, dirs, files in os.walk(original):
if '.svn' in dirs:
dirs.remove('.svn')
for name in files:
original_path = os.path.join(root, name)
simple_path = get_simple_path(original, original_path)
dest_path = os.path.join(dest, simple_path)
if not os.path.exists(os.path.dirname(dest_path)):
os.makedirs(os.path.dirname(dest_path))
shutil.copy2(original_path, dest_path)
def handle_project(project):
try:
client = pysvn.Client()
client.exception_style = 1
doc_dir = project.doc_path
print doc_dir
current_version_in_dir = client.info(doc_dir).revision
print "current_version_in_dir is '%s'" % current_version_in_dir.number
try:
current_version_in_db = Version.objects.filter(
project=project).latest('created')
print "current_version_in_db is '%s'" % current_version_in_db.revision
current_version_in_db_pysvn = pysvn.Revision(
pysvn.opt_revision_kind.number,
int(current_version_in_db.revision))
if int(current_version_in_db.revision) == int(
current_version_in_dir.number):
print "current_version_in_db == current_version_in_dir."
for page in Page.objects.filter(
project=project,
deleted=False,
current_version=current_version_in_db):
print 'checking %s' % page.path
page.check()
return
elif int(current_version_in_db.revision) > int(
current_version_in_dir.number):
raise ValidationError, "current_version_in_db > current_version_in_dir."
else:
os.chdir(doc_dir)
diff_summarize = client.diff_summarize(
doc_dir, current_version_in_db_pysvn, doc_dir,
current_version_in_dir)
added_files, modified_files, deleted_files = [], [], []
for diff in diff_summarize:
diff_path = diff.path.replace('\\', '/')
if diff.summarize_kind == pysvn.diff_summarize_kind.delete and diff.node_kind == pysvn.node_kind.dir:
deleted_pages = Page.objects.filter(
project=project,
deleted=False,
current_version=current_version_in_db,
path__startswith=diff_path).values_list('path',
flat=True)
for deleted_page in deleted_pages:
path = os.path.join(doc_dir, deleted_page).replace(
'\\', '/')
simple_path = deleted_page.replace('\\', '/')
deleted_files.append((path, simple_path))
if diff.node_kind == pysvn.node_kind.file:
path = os.path.join(doc_dir,
diff.path).replace('\\', '/')
simple_path = diff.path.replace('\\', '/')
if project.is_doc_file(simple_path):
if diff.summarize_kind == pysvn.diff_summarize_kind.added:
added_files.append((path, simple_path))
if diff.summarize_kind == pysvn.diff_summarize_kind.modified:
modified_files.append((path, simple_path))
if diff.summarize_kind == pysvn.diff_summarize_kind.delete:
deleted_files.append((path, simple_path))
#TODO 处理exclude_dir_names 变更导致的文件增删。
handling_version_in_db = Version.objects.create(
project=project,
revision=current_version_in_dir.number,
last_version=current_version_in_db)
print "created handling_version_in_db '%s'." % handling_version_in_db.revision
for path, simple_path in added_files:
handle_page_added(project, handling_version_in_db, path,
simple_path)
for path, simple_path in modified_files:
handle_page_modified(project, current_version_in_db,
handling_version_in_db, path,
simple_path)
for path, simple_path in deleted_files:
handle_page_deleted(project, current_version_in_db,
handling_version_in_db, path,
simple_path)
un_modified_pages = Page.objects.filter(
project=project,
deleted=False,
current_version=current_version_in_db)
for page in un_modified_pages:
paragraphs = page.paragraphs.filter(
current_version=current_version_in_db)
for p in paragraphs:
p.current_version = handling_version_in_db
p.save()
page.current_version = handling_version_in_db
page.save()
except Version.DoesNotExist:
print "no current_version_in_db in db."
handling_version_in_db = Version.objects.create(
project=project, revision=current_version_in_dir.number)
print "created current_version_in_db '%s'." % handling_version_in_db.revision
files = [(path, get_simple_path(doc_dir, path).replace('\\', '/'))
for path, info_dict in client.info2(doc_dir)
if os.path.isfile(path)]
print "%s files in doc_dir, " % len(files),
doc_files = [(path, simple_path) for path, simple_path in files
if project.is_doc_file(simple_path)]
print "%s doc files." % len(doc_files)
for path, simple_path in doc_files:
handle_page_added(project, handling_version_in_db, path,
simple_path)
now_pages_in_db = set(
Page.objects.filter(
project=project,
deleted=False,
current_version=handling_version_in_db).values_list('path',
flat=True))
now_files_in_dir = [
get_simple_path(doc_dir, path).replace('\\', '/')
for path, info_dict in client.info2(doc_dir)
if os.path.isfile(path)
]
now_pages_in_dir = set(
[path for path in now_files_in_dir if project.is_doc_file(path)])
if now_pages_in_db != now_pages_in_dir:
raise ValidationError, u"project pages in dir diff from pages in db\n\n%s\n\n" % (
"\n".join(now_pages_in_db ^ now_pages_in_dir))
for page in Page.objects.filter(
project=project,
deleted=False,
current_version=handling_version_in_db):
page.check()
# raise ValidationError, u"for debug"
except BaseException, args:
import traceback
traceback.print_exc()
# print "get exception %s:%s" % (args.__class__.__name__, args)
print "rollbacking database operations"
transaction.rollback()
print "rollbacked database operations"
def handle_project(project):
try:
client = pysvn.Client()
client.exception_style = 1
doc_dir = project.doc_path
print doc_dir
current_version_in_dir = client.info(doc_dir).revision
print "current_version_in_dir is '%s'" % current_version_in_dir.number
try:
current_version_in_db = Version.objects.filter(project = project).latest('created')
print "current_version_in_db is '%s'" % current_version_in_db.revision
current_version_in_db_pysvn = pysvn.Revision(pysvn.opt_revision_kind.number, int(current_version_in_db.revision))
if int(current_version_in_db.revision) == int(current_version_in_dir.number):
print "current_version_in_db == current_version_in_dir."
for page in Page.objects.filter(project = project, deleted = False, current_version = current_version_in_db):
print 'checking %s' % page.path
page.check()
return
elif int(current_version_in_db.revision) > int(current_version_in_dir.number):
raise ValidationError, "current_version_in_db > current_version_in_dir."
else:
os.chdir(doc_dir)
diff_summarize = client.diff_summarize(doc_dir, current_version_in_db_pysvn, doc_dir, current_version_in_dir)
added_files, modified_files, deleted_files = [], [], []
for diff in diff_summarize:
diff_path = diff.path.replace('\\', '/')
if diff.summarize_kind == pysvn.diff_summarize_kind.delete and diff.node_kind == pysvn.node_kind.dir:
deleted_pages = Page.objects.filter(project = project, deleted = False, current_version = current_version_in_db, path__startswith = diff_path).values_list('path', flat=True)
for deleted_page in deleted_pages:
path = os.path.join(doc_dir, deleted_page).replace('\\', '/')
simple_path = deleted_page.replace('\\', '/')
deleted_files.append((path, simple_path))
if diff.node_kind == pysvn.node_kind.file:
path = os.path.join(doc_dir, diff.path).replace('\\', '/')
simple_path = diff.path.replace('\\', '/')
if project.is_doc_file(simple_path):
if diff.summarize_kind == pysvn.diff_summarize_kind.added:
added_files.append((path, simple_path))
if diff.summarize_kind == pysvn.diff_summarize_kind.modified:
modified_files.append((path, simple_path))
if diff.summarize_kind == pysvn.diff_summarize_kind.delete:
deleted_files.append((path, simple_path))
#TODO 处理exclude_dir_names 变更导致的文件增删。
handling_version_in_db = Version.objects.create(project = project, revision = current_version_in_dir.number, last_version = current_version_in_db)
print "created handling_version_in_db '%s'." % handling_version_in_db.revision
for path, simple_path in added_files:
handle_page_added(project, handling_version_in_db, path, simple_path)
for path, simple_path in modified_files:
handle_page_modified(project, current_version_in_db, handling_version_in_db, path, simple_path)
for path, simple_path in deleted_files:
handle_page_deleted(project, current_version_in_db, handling_version_in_db, path, simple_path)
un_modified_pages = Page.objects.filter(project = project, deleted = False, current_version = current_version_in_db)
for page in un_modified_pages:
paragraphs = page.paragraphs.filter(current_version = current_version_in_db)
for p in paragraphs:
p.current_version = handling_version_in_db
p.save()
page.current_version = handling_version_in_db
page.save()
except Version.DoesNotExist:
print "no current_version_in_db in db."
handling_version_in_db = Version.objects.create(project = project, revision = current_version_in_dir.number)
print "created current_version_in_db '%s'." % handling_version_in_db.revision
files = [(path, get_simple_path(doc_dir, path).replace('\\', '/') )for path, info_dict in client.info2(doc_dir) if os.path.isfile(path)]
print "%s files in doc_dir, " % len(files),
doc_files = [(path, simple_path)for path, simple_path in files if project.is_doc_file(simple_path)]
print "%s doc files." % len(doc_files)
for path, simple_path in doc_files:
handle_page_added(project, handling_version_in_db, path, simple_path)
now_pages_in_db = set(Page.objects.filter(project = project, deleted = False, current_version = handling_version_in_db).values_list('path', flat=True))
now_files_in_dir = [get_simple_path(doc_dir, path).replace('\\', '/') for path, info_dict in client.info2(doc_dir) if os.path.isfile(path)]
now_pages_in_dir = set([path for path in now_files_in_dir if project.is_doc_file(path)])
if now_pages_in_db != now_pages_in_dir:
raise ValidationError, u"project pages in dir diff from pages in db\n\n%s\n\n" % ("\n".join(now_pages_in_db ^ now_pages_in_dir))
for page in Page.objects.filter(project = project, deleted = False, current_version = handling_version_in_db):
page.check()
# raise ValidationError, u"for debug"
except BaseException, args:
import traceback
traceback.print_exc()
# print "get exception %s:%s" % (args.__class__.__name__, args)
print "rollbacking database operations"
transaction.rollback()
print "rollbacked database operations"
