def sync(args):
"""
Sync pre-built range files from a git repository
Args:
repo: the git URL to a directory full of files
dir: the directory under the cloned repository where the yaml files live
"""
# Import inside the sync module to avoid circular imports
import seco.range.sync.local as local_sync
git_url = args['repo']
git_dir = args['dir']
tmpdir = None
try:
tmpdir = tempfile.mkdtemp(prefix='range_sync_git-')
git_repo = git.Repo.clone_from(git_url, tmpdir)
local_dir = os.path.join(tmpdir, git_dir)
args = {'dir': local_dir}
# Now read all the data in and return it
return local_sync.sync(args)
finally:
if tmpdir and os.path.exists(tmpdir):
shutil.rmtree(tmpdir)
def sync(args):
"""
Sync pre-built range files from a git repository
Args:
repo: the git URL to a directory full of files
dir: the directory under the cloned repository where the yaml files live
"""
# Import inside the sync module to avoid circular imports
import seco.range.sync.local as local_sync
git_url = args['repo']
git_dir = args['dir']
tmpdir = None
try:
tmpdir = tempfile.mkdtemp(prefix='range_sync_git-')
git_repo = git.Repo.clone_from(git_url, tmpdir)
local_dir = os.path.join(tmpdir, git_dir)
args = { 'dir': local_dir }
# Now read all the data in and return it
return local_sync.sync(args)
finally:
if tmpdir and os.path.exists(tmpdir):
shutil.rmtree(tmpdir)
def sync(args):
"""
Sync pre-built range files from a subversion repository.
Args:
repo: the subversion URL to a directory full of files
"""
# Import inside the sync module to avoid circular imports
import seco.range.sync.local as local_sync
svn_url = args['repo']
tmpdir = None
try:
tmpdir = tempfile.mkdtemp(prefix='range_sync_svn-')
svn_client = pysvn.Client()
head = pysvn.Revision(pysvn.opt_revision_kind.head)
svn_client.export(svn_url,
tmpdir,
force=True,
revision=head,
native_eol=None)
args = {'dir': tmpdir}
# Now read all the data in and return it
return local_sync.sync(args)
finally:
if tmpdir and os.path.exists(tmpdir):
shutil.rmtree(tmpdir)
def sync(args):
"""
Sync pre-built range files from a subversion repository.
Args:
repo: the subversion URL to a directory full of files
"""
# Import inside the sync module to avoid circular imports
import seco.range.sync.local as local_sync
svn_url = args['repo']
tmpdir = None
try:
tmpdir = tempfile.mkdtemp(prefix='range_sync_svn-')
svn_client = pysvn.Client()
head = pysvn.Revision(pysvn.opt_revision_kind.head)
svn_client.export(svn_url, tmpdir, force=True, revision=head, native_eol=None)
args = { 'dir': tmpdir }
# Now read all the data in and return it
return local_sync.sync(args)
finally:
if tmpdir and os.path.exists(tmpdir):
shutil.rmtree(tmpdir)
def sync(args, debug=False):
import seco.range.sync.local as local_sync
url = args["url"]
if debug:
print "URL is %s" % url
file_re = re.compile(args["filter"])
if debug:
print "Filter is %s" % file_re
req = requests.get(url)
if debug:
print "Request made"
content = BeautifulSoup(req.content)
if debug:
print "Request parsed"
content_links = content.findAll("a")
files = set()
for link in content_links:
if debug:
print "Inspecting %s" % link
if file_re.search(link["href"]):
if debug:
print "Found match: %s" % link["href"]
files.add(link["href"])
if debug:
print "Have files %s" % (", ".join(files))
try:
tmpdir = tempfile.mkdtemp(prefix="range_sync_http-")
for f in files:
with open(os.path.join(tmpdir, f), "w") as fh:
req_file = None
if url.endswith("/"):
req_file = requests.get(url + f)
else:
req_file = requests.get(url + "/" + f)
fh.write(req_file.content)
return local_sync.sync({"dir": tmpdir})
finally:
if tmpdir and os.path.exists(tmpdir):
if debug:
print "Look in %s" % tmpdir
else:
shutil.rmtree(tmpdir)
def sync(args, debug=False):
import seco.range.sync.local as local_sync
url = args['url']
if debug: print("URL is %s" % url)
file_re = re.compile(args['filter'])
if debug: print("Filter is %s" % file_re)
req = requests.get(url)
if debug: print("Request made")
content = BeautifulSoup(req.content)
if debug: print("Request parsed")
content_links = content.findAll('a')
files = set()
for link in content_links:
if debug: print("Inspecting %s" % link)
if file_re.search(link['href']):
if debug: print("Found match: %s" % link['href'])
files.add(link['href'])
if debug: print("Have files %s" % (', '.join(files)))
try:
tmpdir = tempfile.mkdtemp(prefix='range_sync_http-')
for f in files:
with open(os.path.join(tmpdir, f), 'w') as fh:
req_file = None
if url.endswith('/'):
req_file = requests.get(url + f)
else:
req_file = requests.get(url + '/' + f)
fh.write(req_file.content)
return local_sync.sync({'dir': tmpdir})
finally:
if tmpdir and os.path.exists(tmpdir):
if debug:
print("Look in %s" % tmpdir)
else:
shutil.rmtree(tmpdir)
