def handle(self, *args, **options):
# first: determine the type of server
obj = Synchronization.objects.get_or_create(source='localhost', destination=options['tag'])[0]
if isinstance(options['serial'], int):
first_serial = options['serial']
print(cyan(_('Dumping from serial %(syn)s') % {'syn': first_serial}))
elif obj.last_serial is None:
first_serial = 0
print(cyan(_('No previous sync... Dumping all database')))
else:
first_serial = obj.last_serial
print(cyan(_('Previous sync: serial %(syn)s') % {'syn': obj.last_serial}))
packages = set()
releases = set()
result = {'data': {}}
base_path = os.path.abspath(options['path'])
last_serial = first_serial
next_last_serial = -1
for log in Log.objects.filter(id__gte=first_serial).order_by('id').select_related():
p_path = os.path.join(base_path, log.package.name)
if log.package.id not in packages:
self.write_info(log.package.data(), p_path, 'package')
result['data'][log.package.name] = {}
packages.add(log.package.id)
r_path = os.path.join(p_path, log.release.version)
if log.release.id not in releases:
self.write_info(log.release.data(), r_path, 'release')
result['data'][log.package.name][log.release.version] = []
releases.add(log.release.id)
dst_path = os.path.join(r_path, log.download.filename)
src_path = log.download.abspath
shutil.copy2(src_path, dst_path)
self.write_info(log.download.data(), r_path, log.download.filename)
print(yellow(_('Adding %(fn)s (%(pk)s %(vn)s)') % {'pk': log.package.name, 'fn': log.download.filename,
'vn': log.release.version, }))
result['data'][log.package.name][log.release.version].append(log.download.filename)
last_serial = log.id
next_last_serial = log.id + 1
result['meta'] = {'first_id': first_serial, 'last_id': last_serial}
self.write_info(result, base_path, 'sync')
Synchronization.objects.filter(id=obj.id).update(last_serial=next_last_serial)
def handle(self, *args, **options):
# first: determine the type of server
obj = Synchronization.objects.get_or_create(source='localhost', destination=options['tag'])[0]
if isinstance(options['serial'], int):
first_serial = options['serial']
print(cyan(_('Dumping from serial %(syn)s') % {'syn': first_serial}))
elif obj.last_serial is None:
first_serial = 0
print(cyan(_('No previous sync... Dumping all database')))
else:
first_serial = obj.last_serial
print(cyan(_('Previous sync: serial %(syn)s') % {'syn': obj.last_serial}))
packages = set()
releases = set()
result = {'data': {}}
base_path = os.path.abspath(options['path'])
last_serial = first_serial
next_last_serial = -1
for log in Log.objects.filter(id__gte=first_serial).order_by('id').select_related():
p_path = os.path.join(base_path, log.package.name)
if log.package.id not in packages:
self.write_info(log.package.data(), p_path, 'package')
result['data'][log.package.name] = {}
packages.add(log.package.id)
r_path = os.path.join(p_path, log.release.version)
if log.release.id not in releases:
self.write_info(log.release.data(), r_path, 'release')
result['data'][log.package.name][log.release.version] = []
releases.add(log.release.id)
dst_path = os.path.join(r_path, log.download.filename)
src_path = log.download.abspath
shutil.copy2(src_path, dst_path)
self.write_info(log.download.data(), r_path, log.download.filename)
print(yellow(_('Adding %(fn)s (%(pk)s %(vn)s)') % {'pk': log.package.name, 'fn': log.download.filename,
'vn': log.release.version, }))
result['data'][log.package.name][log.release.version].append(log.download.filename)
last_serial = log.id
next_last_serial = log.id + 1
result['meta'] = {'first_id': first_serial, 'last_id': last_serial}
self.write_info(result, base_path, 'sync')
Synchronization.objects.filter(id=obj.id).update(last_serial=next_last_serial)
def handle(self, *args, **options):
# first: determine the type of server
obj = Synchronization.objects.get_or_create(destination='localhost', source=options['tag'])[0]
if obj.last_serial is None:
first_expected_serial = 0
print(cyan(_('No previous sync')))
else:
first_expected_serial = obj.last_serial + 1
print(cyan(_('Previous sync: serial %(syn)s') % {'syn': obj.last_serial}))
base_path = os.path.abspath(options['path'])
try:
data = self.read_info(base_path, 'sync')
packages = data['data']
first_serial = data['meta']['first_id']
last_serial = data['meta']['last_id']
except KeyError:
print(red(_('Invalid sync file')))
return
except ValueError:
print(red(_('Invalid md5 data')))
return
except IOError:
print(red(_('Invalid sync file')))
return
if first_serial > first_expected_serial:
print(red(_('Missing synchronization between %(f)d and %(l)d') % {'f': first_expected_serial,
'l': first_serial}))
if not options['force']:
return
stop = False
for package_name, releases in packages.items():
p_path = os.path.join(base_path, package_name)
package_data = self.read_info(p_path, 'package')
package, created = Package.objects.get_or_create(name=package_name)
self.set_attr(('name', 'author', 'author_email', 'maintainer', 'maintainer_email', 'home_page', 'license',
'summary', 'download_url', 'project_url', ),
package_data, package)
package.save()
for version, filenames in releases.items():
r_path = os.path.join(base_path, package_name, version)
release_data = self.read_info(r_path, 'release')
release, created = Release.objects.get_or_create(package=package, version=version)
self.set_attr(('version', 'stable_version', 'description', 'platform', 'keywords', 'docs_url', ),
release_data, release)
release.classifiers.clear()
for value in release_data.get('classifiers', []):
release.classifiers.add(Classifier.get(value))
for attr_name in ('requires', 'requires_dist', 'provides', 'provides_dist', 'requires_external',
'requires_python', 'obsoletes', 'obsoletes_dist', ):
getattr(release, attr_name).clear()
for value in release_data.get(attr_name, []):
getattr(release, attr_name).add(Dependence.get(value))
release.save()
for filename in filenames:
filepath = os.path.join(r_path, filename)
download_data = self.read_info(r_path, filename)
if ReleaseDownload.objects.filter(package=package, release=release, filename=filename).count() > 0:
print(yellow(_('Duplicate file: %(f)s') % {'f': filepath}))
continue
download = ReleaseDownload(package=package, release=release, filename=filename)
self.set_attr(('md5_digest', 'downloads', 'pack', 'has_sig', 'comment_text', 'python_version'),
download_data, download)
download.package_type = PackageType.get(download_data.get('packagetype'))
dirname = os.path.dirname(download.abspath)
if not os.path.isdir(dirname):
os.makedirs(dirname)
shutil.copy2(filepath, download.abspath)
download.file = download.relpath
download.url = settings.MEDIA_URL + download.relpath
download.size = os.path.getsize(filepath)
if download_data.get('upload_time'):
download.upload_time = datetime.datetime.strptime(download_data['upload_time'], DATE_FORMAT)\
.replace(tzinfo=utc)
with open(filepath, 'rb') as file_d:
md5 = hashlib.md5(file_d.read()).hexdigest()
download.md5_digest = md5
if md5 != download_data.get('md5_digest'):
print(red(_('Corrupted file: %(f)s') % {'f': filepath}))
stop = True
break
download.log()
if stop:
break
if stop:
break
if not stop:
Synchronization.objects.filter(id=obj.id).update(last_serial=last_serial)
def handle(self, *args, **options):
# first: determine the type of server
sync_obj = Synchronization.objects.get_or_create(
source=options['url'], destination='localhost')[0]
self.retry = options['retry']
socket.setdefaulttimeout(options['timeout'])
download_limit = 0 if not options['limit'] else options['limit']
if isinstance(options['serial'], int):
first_serial = options['serial']
init = False
self.stdout.write(
cyan(
_('Download from serial %(syn)s') % {'syn': first_serial}))
elif sync_obj.last_serial is None or options['init_all']:
first_serial = 0
init = True
self.stdout.write(
cyan(_('No previous sync... Initializing database')))
else:
self.stdout.write(
cyan(
_('Previous sync: serial %(syn)s') %
{'syn': sync_obj.last_serial}))
first_serial = sync_obj.last_serial
init = False
self.connect(options['url'])
last_serial = None
if options['package']:
# get all releases of the given package
package_name = options['package']
self.stdout.write(
cyan(
_('Downloading all versions of %(pkg)s') %
{'pkg': package_name}))
try:
versions = self.try_download(
self.client.package_releases,
_('Unable to get releases of %(pkg)s') %
{'pkg': package_name}, package_name, True)
except DownloadException:
versions = []
for version in versions:
self.download_release(package_name, version)
elif not options['latest'] and (not init or options['init_all']):
# get all releases from the given serial
last_serial = first_serial
try:
modified_packages = self.try_download(
self.client.changelog_since_serial,
_('Unable to download changelog'), first_serial + 1)
except DownloadException:
modified_packages = []
counter = 0
for (package_name, version, timestamp, action,
serial) in modified_packages:
last_serial = max(serial, last_serial)
if counter >= download_limit > 0:
break
if version is None:
continue
self.stdout.write(
cyan(
_('Found %(pkg)s-%(vsn)s') % {
'pkg': package_name,
'vsn': version
}))
counter += self.download_release(package_name, version)
else:
# init: get the last version of all packages
try:
last_serial = self.try_download(
self.client.changelog_last_serial,
_('Unable to download changelog'))
packages = self.try_download(self.client.list_packages,
_('Unable to list packages'))
except DownloadException:
return
counter = 0
for package_name in packages:
if counter >= download_limit > 0:
break
self.stdout.write(
cyan(_('Found %(pkg)s') % {'pkg': package_name}))
self.stdout.write(
yellow(
_('package releases (%(p)s)') % {'p': package_name}))
try:
versions = self.try_download(
self.client.package_releases,
_('Unable to get releases of %(pkg)s') %
{'pkg': package_name}, package_name)
except DownloadException:
continue
versions = [x for x in versions if x]
if not versions:
continue
version = versions[0]
self.stdout.write(
cyan(
_('Found %(pkg)s-%(vsn)s') % {
'pkg': package_name,
'vsn': version
}))
counter += self.download_release(package_name, version)
if last_serial is not None:
Synchronization.objects.filter(id=sync_obj.id).update(
last_serial=last_serial)
for package_name, version in self.error_list:
self.stderr.write((_('Unable to download %(p)s-%(v)s') % {
'p': package_name,
'v': version
}))
def handle(self, *args, **options):
# first: determine the type of server
obj = Synchronization.objects.get_or_create(source=options['url'], destination='localhost')[0]
self.retry = options['retry']
socket.setdefaulttimeout(options['timeout'])
download_limit = 0 if not options['limit'] else options['limit']
if isinstance(options['serial'], int):
first_serial = options['serial']
init = False
self.stdout.write(cyan(_('Download from serial %(syn)s') % {'syn': first_serial}))
elif obj.last_serial is None or options['init_all']:
first_serial = 0
init = True
self.stdout.write(cyan(_('No previous sync... Initializing database')))
else:
self.stdout.write(cyan(_('Previous sync: serial %(syn)s') % {'syn': obj.last_serial}))
first_serial = obj.last_serial
init = False
self.connect(options['url'])
last_serial = None
if options['package']:
# get all releases of the given package
package_name = options['package']
self.stdout.write(cyan(_('Downloading all versions of %(pkg)s') % {'pkg': package_name}))
try:
versions = self.try_download(self.client.package_releases,
_('Unable to get releases of %(pkg)s') % {'pkg': package_name},
package_name, True)
except DownloadException:
versions = []
for version in versions:
self.download_release(package_name, version)
elif not options['latest'] and (not init or options['init_all']):
# get all releases from the given serial
last_serial = first_serial
try:
modified_packages = self.try_download(self.client.changelog_since_serial,
_('Unable to download changelog'), first_serial + 1)
except DownloadException:
modified_packages = []
counter = 0
for (package_name, version, timestamp, action, serial) in modified_packages:
last_serial = max(serial, last_serial)
if counter >= download_limit > 0:
break
if version is None:
continue
self.stdout.write(cyan(_('Found %(pkg)s-%(vsn)s') % {'pkg': package_name, 'vsn': version}))
counter += self.download_release(package_name, version)
else:
# init: get the last version of all packages
try:
last_serial = self.try_download(self.client.changelog_last_serial, _('Unable to download changelog'))
packages = self.try_download(self.client.list_packages, _('Unable to list packages'))
except DownloadException:
return
counter = 0
for package_name in packages:
if counter >= download_limit > 0:
break
self.stdout.write(cyan(_('Found %(pkg)s') % {'pkg': package_name}))
self.stdout.write(yellow(_('package releases (%(p)s)') % {'p': package_name}))
try:
versions = self.try_download(self.client.package_releases,
_('Unable to get releases of %(pkg)s') % {'pkg': package_name},
package_name)
except DownloadException:
continue
versions = [x for x in versions if x]
if not versions:
continue
version = versions[0]
self.stdout.write(cyan(_('Found %(pkg)s-%(vsn)s') % {'pkg': package_name, 'vsn': version}))
counter += self.download_release(package_name, version)
if last_serial is not None:
Synchronization.objects.filter(id=obj.id).update(last_serial=last_serial)
for package_name, version in self.error_list:
self.stderr.write((_('Unable to download %(p)s-%(v)s') % {'p': package_name, 'v': version}))
def handle(self, *args, **options):
# first: determine the type of server
obj = Synchronization.objects.get_or_create(destination='localhost',
source=options['tag'])[0]
if obj.last_serial is None:
first_expected_serial = 0
print(cyan(_('No previous sync')))
else:
first_expected_serial = obj.last_serial + 1
print(
cyan(
_('Previous sync: serial %(syn)s') %
{'syn': obj.last_serial}))
base_path = os.path.abspath(options['path'])
try:
data = self.read_info(base_path, 'sync')
packages = data['data']
first_serial = data['meta']['first_id']
last_serial = data['meta']['last_id']
except KeyError:
print(red(_('Invalid sync file')))
return
except ValueError:
print(red(_('Invalid md5 data')))
return
except IOError:
print(red(_('Invalid sync file')))
return
if first_serial > first_expected_serial:
print(
red(
_('Missing synchronization between %(f)d and %(l)d') % {
'f': first_expected_serial,
'l': first_serial
}))
if not options['force']:
return
stop = False
for package_name, releases in packages.items():
p_path = os.path.join(base_path, package_name)
package_data = self.read_info(p_path, 'package')
package, created = Package.objects.get_or_create(name=package_name)
self.set_attr((
'name',
'author',
'author_email',
'maintainer',
'maintainer_email',
'home_page',
'license',
'summary',
'download_url',
'project_url',
), package_data, package)
package.save()
for version, filenames in releases.items():
r_path = os.path.join(base_path, package_name, version)
release_data = self.read_info(r_path, 'release')
release, created = Release.objects.get_or_create(
package=package, version=version)
self.set_attr((
'version',
'stable_version',
'description',
'platform',
'keywords',
'docs_url',
), release_data, release)
release.classifiers.clear()
for value in release_data.get('classifiers', []):
release.classifiers.add(Classifier.get(value))
for attr_name in (
'requires',
'requires_dist',
'provides',
'provides_dist',
'requires_external',
'requires_python',
'obsoletes',
'obsoletes_dist',
):
getattr(release, attr_name).clear()
for value in release_data.get(attr_name, []):
getattr(release, attr_name).add(Dependence.get(value))
release.save()
for filename in filenames:
filepath = os.path.join(r_path, filename)
download_data = self.read_info(r_path, filename)
if ReleaseDownload.objects.filter(
package=package, release=release,
filename=filename).count() > 0:
print(
yellow(
_('Duplicate file: %(f)s') % {'f': filepath}))
continue
download = ReleaseDownload(package=package,
release=release,
filename=filename)
self.set_attr(
('md5_digest', 'downloads', 'pack', 'has_sig',
'comment_text', 'python_version'), download_data,
download)
download.package_type = PackageType.get(
download_data.get('packagetype'))
dirname = os.path.dirname(download.abspath)
if not os.path.isdir(dirname):
os.makedirs(dirname)
shutil.copy2(filepath, download.abspath)
download.file = download.relpath
download.url = settings.MEDIA_URL + download.relpath
download.size = os.path.getsize(filepath)
if download_data.get('upload_time'):
download.upload_time = datetime.datetime.strptime(download_data['upload_time'], DATE_FORMAT)\
.replace(tzinfo=utc)
with open(filepath, 'rb') as file_d:
md5 = hashlib.md5(file_d.read()).hexdigest()
download.md5_digest = md5
if md5 != download_data.get('md5_digest'):
print(red(
_('Corrupted file: %(f)s') % {'f': filepath}))
stop = True
break
download.log()
if stop:
break
if stop:
break
if not stop:
Synchronization.objects.filter(id=obj.id).update(
last_serial=last_serial)